func/pkg/functions/function.go

769 lines
25 KiB
Go

package functions
import (
"crypto/sha256"
"errors"
"fmt"
"io/fs"
"os"
"path/filepath"
"reflect"
"regexp"
"strings"
"time"
"gopkg.in/yaml.v2"
fnlabels "knative.dev/func/pkg/k8s/labels"
"knative.dev/pkg/ptr"
)
const (
// FunctionFile is the file used for the serialized form of a function.
FunctionFile = "func.yaml"
// RunDataDir holds transient runtime metadata
// By default it is excluded from source control.
RunDataDir = ".func"
// buildstamp is the name of the file within the run data directory whose
// existence indicates the function has been built, and whose content is
// a fingerprint of the filesystem at the time of the build.
buildstamp = "built"
// DefaultPersistentVolumeClaimSize represents default size of PVC created for a Pipeline
DefaultPersistentVolumeClaimSize string = "256Mi"
)
// Function
type Function struct {
// SpecVersion at which this function is known to be compatible.
// More specifically, it is the highest migration which has been applied.
// For details see the .Migrated() and .Migrate() methods.
SpecVersion string `yaml:"specVersion"` // semver format
// Root on disk at which to find/create source and config files.
Root string `yaml:"-"`
// Name of the function.
Name string `yaml:"name,omitempty" jsonschema:"pattern=^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"`
// Runtime is the language plus context. nodejs|go|quarkus|rust etc.
Runtime string `yaml:"runtime,omitempty"`
// Template for the function.
Template string `yaml:"-"`
// Registry at which to store interstitial containers, in the form
// [registry]/[user].
Registry string `yaml:"registry"`
// Optional full OCI image tag in form:
// [registry]/[namespace]/[name]:[tag]
// example:
// quay.io/alice/my.function.name
// Registry is optional and is defaulted to DefaultRegistry
// example:
// alice/my.function.name
// If Image is provided, it overrides the default of concatenating
// "Registry+Name:latest" to derive the Image.
Image string `yaml:"image"`
// SHA256 hash of the latest image that has been built
ImageDigest string `yaml:"imageDigest,omitempty"`
// Created time is the moment that creation was successfully completed
// according to the client which is in charge of what constitutes being
// fully "Created" (aka initialized)
Created time.Time `yaml:"created"`
// Invoke defines hints for use when invoking this function.
// See Client.Invoke for usage.
Invoke string `yaml:"invoke,omitempty"`
//BuildSpec define the build properties for a function
Build BuildSpec `yaml:"build,omitempty"`
//RunSpec define the runtime properties for a function
Run RunSpec `yaml:"run,omitempty"`
//DeploySpec define the deployment properties for a function
Deploy DeploySpec `yaml:"deploy,omitempty"`
// current (last) build stamp for the function if it can be found.
BuildStamp string `yaml:"-"`
// flags that buildstamp needs to be saved to disk
NeedsWriteBuildStamp bool `yaml:"-"`
}
// BuildSpec
type BuildSpec struct {
// Git stores information about an optionally associated git repository.
Git Git `yaml:"git,omitempty"`
// BuilderImages define optional explicit builder images to use by
// builder implementations in leau of the in-code defaults. They key
// is the builder's short name. For example:
// builderImages:
// pack: example.com/user/my-pack-node-builder
// s2i: example.com/user/my-s2i-node-builder
BuilderImages map[string]string `yaml:"builderImages,omitempty"`
// Optional list of buildpacks to use when building the function
Buildpacks []string `yaml:"buildpacks,omitempty"`
// Builder is the name of the subsystem that will complete the underlying
// build (pack, s2i, etc)
Builder string `yaml:"builder,omitempty" jsonschema:"enum=pack,enum=s2i"`
// Build Env variables to be set
BuildEnvs Envs `yaml:"buildEnvs,omitempty"`
// PVCSize specifies the size of persistent volume claim used to store function
// when using deployment and remote build process (only relevant when Remote is true).
PVCSize string `yaml:"pvcSize,omitempty"`
}
// RunSpec
type RunSpec struct {
// List of volumes to be mounted to the function
Volumes []Volume `yaml:"volumes,omitempty"`
// Env variables to be set
Envs Envs `yaml:"envs,omitempty"`
}
// DeploySpec
type DeploySpec struct {
// Namespace into which the function is deployed on supported platforms.
Namespace string `yaml:"namespace,omitempty"`
// Remote indicates the deployment (and possibly build) process are to
// be triggered in a remote environment rather than run locally.
Remote bool `yaml:"remote,omitempty"`
// Map containing user-supplied annotations
// Example: { "division": "finance" }
Annotations map[string]string `yaml:"annotations,omitempty"`
// Options to be set on deployed function (scaling, etc.)
Options Options `yaml:"options,omitempty"`
// Map of user-supplied labels
Labels []Label `yaml:"labels,omitempty"`
// Health endpoints specified by the language pack
HealthEndpoints HealthEndpoints `yaml:"healthEndpoints,omitempty"`
}
// HealthEndpoints specify the liveness and readiness endpoints for a Runtime
type HealthEndpoints struct {
Liveness string `yaml:"liveness,omitempty"`
Readiness string `yaml:"readiness,omitempty"`
}
// BuildConfig defines builders and buildpacks
type BuildConfig struct {
Buildpacks []string `yaml:"buildpacks,omitempty"`
BuilderImages map[string]string `yaml:"builderImages,omitempty"`
}
// NewFunctionWith defaults as provided.
func NewFunctionWith(defaults Function) Function {
if defaults.SpecVersion == "" {
defaults.SpecVersion = LastSpecVersion()
}
if defaults.Template == "" {
defaults.Template = DefaultTemplate
}
if defaults.Build.BuilderImages == nil {
defaults.Build.BuilderImages = make(map[string]string)
}
if defaults.Build.PVCSize == "" {
defaults.Build.PVCSize = DefaultPersistentVolumeClaimSize
}
if defaults.Deploy.Annotations == nil {
defaults.Deploy.Annotations = make(map[string]string)
}
return defaults
}
// NewFunction from a given path.
// Invalid paths, or no function at path are errors.
// Syntactic errors are returned immediately (yaml unmarshal errors).
// Functions which are syntactically valid are also then logically validated.
// Functions from earlier versions are brought up to current using migrations.
// Migrations are run prior to validators such that validation can omit
// concerning itself with backwards compatibility. Migrators must therefore
// selectively consider the minimal validation necessary to enable migration.
func NewFunction(path string) (f Function, err error) {
f.Build.BuilderImages = make(map[string]string)
f.Deploy.Annotations = make(map[string]string)
// Path defaults to current working directory, and if provided explicitly
// Path must exist and be a directory
if path == "" {
if path, err = os.Getwd(); err != nil {
return
}
}
f.Root = path // path is not persisted, as this is the purview of the FS
// Path must exist and be a directory
fd, err := os.Stat(path)
if err != nil {
return f, err
}
if !fd.IsDir() {
return f, fmt.Errorf("function path must be a directory")
}
// If no func.yaml in directory, return the default function which will
// have f.Initialized() == false
var filename = filepath.Join(path, FunctionFile)
if _, err = os.Stat(filename); err != nil {
if os.IsNotExist(err) {
err = nil
}
return
}
// Path is valid and func.yaml exists: load it
bb, err := os.ReadFile(filename)
if err != nil {
return
}
var functionMarshallingError error
var functionMigrationError error
if marshallingErr := yaml.Unmarshal(bb, &f); marshallingErr != nil {
functionMarshallingError = formatUnmarshalError(marshallingErr) // human-friendly unmarshalling errors
}
if f, err = f.Migrate(); err != nil {
functionMigrationError = err
}
// Only if migration fail return errors to the user. include marshalling error if present
if functionMigrationError != nil {
//returning both migrations and marshalling errors to the user
errorText := "Error: \n"
if functionMarshallingError != nil {
errorText += "Marshalling: " + functionMarshallingError.Error()
}
errorText += "\n" + "Migration: " + functionMigrationError.Error()
return Function{}, errors.New(errorText)
}
f.BuildStamp = f.buildStamp()
return
}
// Validate function is logically correct, returning a bundled, and quite
// verbose, formatted error detailing any issues.
func (f Function) Validate() error {
if f.Root == "" {
return errors.New("function root path is required")
}
var ctr int
errs := [][]string{
validateVolumes(f.Run.Volumes),
ValidateBuildEnvs(f.Build.BuildEnvs),
ValidateEnvs(f.Run.Envs),
validateOptions(f.Deploy.Options),
ValidateLabels(f.Deploy.Labels),
validateGit(f.Build.Git),
}
var b strings.Builder
b.WriteString(fmt.Sprintf("'%v' contains errors:", FunctionFile))
for _, ee := range errs {
if len(ee) > 0 {
b.WriteString("\n") // Precede each group of errors with a linebreak
}
for _, e := range ee {
ctr++
b.WriteString("\t" + e)
}
}
if ctr == 0 {
return nil // Return nil if there were no validation errors.
}
return errors.New(b.String())
}
var envPattern = regexp.MustCompile(`^{{\s*(\w+)\s*:(\w+)\s*}}$`)
// Interpolate Env slice
// Values with no special format are preserved as simple values.
// Values which do include the interpolation format (begin with {{) but are not
// keyed as "env" are also preserved as is.
// Values properly formatted as {{ env:NAME }} are interpolated (substituted)
// with the value of the local environment variable "NAME", and an error is
// returned if that environment variable does not exist.
func Interpolate(ee []Env) (map[string]string, error) {
envs := make(map[string]string, len(ee))
for _, e := range ee {
// Assert non-nil name.
if e.Name == nil {
return envs, errors.New("env name may not be nil")
}
// Nil value indicates the resultant map should not include this env var.
if e.Value == nil {
continue
}
k, v := *e.Name, *e.Value
// Simple Values are preserved.
// If not prefixed by {{, no interpolation required (simple value)
if !strings.HasPrefix(v, "{{") {
envs[k] = v // no interpolation required.
continue
}
// Values not matching the interpolation pattern are preserved.
// If not in the form "{{ env:XYZ }}" then return the value as-is for
// 0 1 2 3
// possible match and interpolation in different ways.
parts := envPattern.FindStringSubmatch(v)
if len(parts) <= 2 || parts[1] != "env" {
envs[k] = v
continue
}
// Properly formatted local env var references are interpolated.
localName := parts[2]
localValue, ok := os.LookupEnv(localName)
if !ok {
return envs, fmt.Errorf("expected environment variable '%v' not found", localName)
}
envs[k] = localValue
}
return envs, nil
}
// nameFromPath returns the default name for a function derived from a path.
// This consists of the last directory in the given path, if derivable (empty
// paths, paths consisting of all slashes, etc. return the zero value "")
func nameFromPath(path string) string {
pathParts := strings.Split(strings.TrimRight(path, string(os.PathSeparator)), string(os.PathSeparator))
return pathParts[len(pathParts)-1]
/* the above may have edge conditions as it assumes the trailing value
* is a directory name. If errors are encountered, we _may_ need to use the
* inbuilt logic in the std lib and either check if the path indicated is a
* directory (appending slash) and then run:
base := filepath.Base(filepath.Dir(path))
if base == string(os.PathSeparator) || base == "." {
return "" // Consider it underivable: string zero value
}
return base
*/
}
// Write aka (save, serialize, marshal) the function to disk at its path.
// Only valid functions can be written.
// In order to retain built status (staleness checks), the file is only
// modified if the structure actually changes.
func (f Function) Write() (err error) {
// Skip writing (and dirtying the work tree) if there were no modifications.
f1, _ := NewFunction(f.Root)
if reflect.DeepEqual(f, f1) {
return
}
if err = f.Validate(); err != nil {
return
}
path := filepath.Join(f.Root, FunctionFile)
var bb []byte
if bb, err = yaml.Marshal(&f); err != nil {
return
}
// TODO: open existing file for writing, such that existing permissions
// are preserved.
if err = os.WriteFile(path, bb, 0644); err != nil {
return
}
if f.NeedsWriteBuildStamp {
err = f.writeBuildStamp()
if err != nil {
return
}
f.NeedsWriteBuildStamp = false
}
return
}
// Initialized returns if the function has been initialized.
// Any errors are considered failure (invalid or inaccessible root, config file, etc).
func (f Function) Initialized() bool {
return !f.Created.IsZero()
}
// ImageWithDigest returns the full reference to the image including SHA256 Digest.
// If Digest is empty, image:tag is returned.
// TODO: Populate this only on a successful deploy, as this results on a dirty
// git tree on every build.
func (f Function) ImageWithDigest() string {
// Return image, if Digest is empty
if f.ImageDigest == "" {
return f.Image
}
lastSlashIdx := strings.LastIndexAny(f.Image, "/")
imageAsBytes := []byte(f.Image)
part1 := string(imageAsBytes[:lastSlashIdx+1])
part2 := string(imageAsBytes[lastSlashIdx+1:])
// Remove tag from the image name and append SHA256 hash instead
return part1 + strings.Split(part2, ":")[0] + "@" + f.ImageDigest
}
// LabelsMap combines default labels with the labels slice provided.
// It will the resulting slice with ValidateLabels and return a key/value map.
// - key: EXAMPLE1 # Label directly from a value
// value: value1
// - key: EXAMPLE2 # Label from the local ENV var
// value: {{ env:MY_ENV }}
func (f Function) LabelsMap() (map[string]string, error) {
defaultLabels := []Label{
{
Key: ptr.String(fnlabels.FunctionKey),
Value: ptr.String(fnlabels.FunctionValue),
},
{
Key: ptr.String(fnlabels.FunctionNameKey),
Value: ptr.String(f.Name),
},
{
Key: ptr.String(fnlabels.FunctionRuntimeKey),
Value: ptr.String(f.Runtime),
},
// --- handle usage of deprecated labels (`boson.dev/function`, `boson.dev/runtime`)
{
Key: ptr.String(fnlabels.DeprecatedFunctionKey),
Value: ptr.String(fnlabels.FunctionValue),
},
{
Key: ptr.String(fnlabels.DeprecatedFunctionRuntimeKey),
Value: ptr.String(f.Runtime),
},
// --- end of handling usage of deprecated runtime labels
}
labels := append(defaultLabels, f.Deploy.Labels...)
if err := ValidateLabels(labels); len(err) != 0 {
return nil, errors.New(strings.Join(err, " "))
}
l := map[string]string{}
for _, label := range labels {
if label.Value == nil {
l[*label.Key] = ""
} else {
if strings.HasPrefix(*label.Value, "{{") {
// env variable format is validated above in ValidateLabels
match := regLocalEnv.FindStringSubmatch(*label.Value)
l[*label.Key] = os.Getenv(match[1])
} else {
l[*label.Key] = *label.Value
}
}
}
return l, nil
}
// assertEmptyRoot ensures that the directory is empty enough to be used for
// initializing a new function.
func assertEmptyRoot(path string) (err error) {
// If there exists contentious files (congig files for instance), this function may have already been initialized.
files, err := contentiousFilesIn(path)
if err != nil {
return
} else if len(files) > 0 {
return fmt.Errorf("the chosen directory '%v' contains contentious files: %v. Has the Service function already been created? Try either using a different directory, deleting the function if it exists, or manually removing the files", path, files)
}
// Ensure there are no non-hidden files, and again none of the aforementioned contentious files.
empty, err := isEffectivelyEmpty(path)
if err != nil {
return
} else if !empty {
err = errors.New("the directory must be empty of visible files and recognized config files before it can be initialized")
return
}
return
}
// ImageName returns a full image name (OCI container tag) for the
// Function based off of the Function's `Registry` member plus `Name`.
// Used to calculate the final value for .Image when none is provided
// explicitly.
//
// form: [registry]/[user]/[function]:latest
// example: quay.io/alice/my.function.name:latest
//
// Also nested namespaces should be supported:
// form: [registry]/[parent]/[user]/[function]:latest
// example: quay.io/project/alice/my.function.name:latest
//
// Registry values which only contain a single token are presumed to
// indicate the namespace at the default registry.
func (f Function) ImageName() (image string, err error) {
if f.Registry == "" {
return "", ErrRegistryRequired
}
if f.Name == "" {
return "", ErrNameRequired
}
f.Registry = strings.Trim(f.Registry, "/") // too defensive?
registryTokens := strings.Split(f.Registry, "/")
if len(registryTokens) == 1 { // only namespace provided: ex. 'alice'
image = DefaultRegistry + "/" + f.Registry + "/" + f.Name
} else if len(registryTokens) == 2 || len(registryTokens) == 3 {
// registry/namespace ('quay.io/alice') or
// registry/parent-namespace/namespace ('quay.io/project/alice') provided
image = f.Registry + "/" + f.Name
} else if len(registryTokens) > 3 { // the name of the image is also provided `quay.io/alice/my.function.name`
return "", fmt.Errorf("registry should be either 'namespace', 'registry/namespace' or 'registry/parent/namespace', the name of the image will be derived from the function name.")
}
// Explicitly append :latest tag. We expect source control to drive
// versioning, rather than rely on image tags with explicitly pinned version
// numbers, as is seen in many serverless solutions. This will be updated
// to branch name when we add source-driven canary/ bluegreen deployments.
// For pinning to an exact container image, see ImageWithDigest
return image + ":latest", nil
}
// contentiousFiles are files which, if extant, preclude the creation of a
// function rooted in the given directory.
var contentiousFiles = []string{
FunctionFile,
".gitignore",
}
// contentiousFilesIn the given directory
func contentiousFilesIn(dir string) (contentious []string, err error) {
files, err := os.ReadDir(dir)
for _, file := range files {
for _, name := range contentiousFiles {
if file.Name() == name {
contentious = append(contentious, name)
}
}
}
return
}
// effectivelyEmpty directories are those which have no visible files
func isEffectivelyEmpty(dir string) (bool, error) {
// Check for any non-hidden files
files, err := os.ReadDir(dir)
if err != nil {
return false, err
}
for _, file := range files {
if !strings.HasPrefix(file.Name(), ".") {
return false, nil
}
}
return true, nil
}
// returns true if the given path contains an initialized function.
func hasInitializedFunction(path string) (bool, error) {
var err error
var filename = filepath.Join(path, FunctionFile)
if _, err = os.Stat(filename); err != nil {
if os.IsNotExist(err) {
return false, nil
}
return false, err // invalid path or access error
}
bb, err := os.ReadFile(filename)
if err != nil {
return false, err
}
f := Function{}
if err = yaml.Unmarshal(bb, &f); err != nil {
return false, err
}
if f, err = f.Migrate(); err != nil {
return false, err
}
return f.Initialized(), nil
}
// Format yaml unmarshall error to be more human friendly.
func formatUnmarshalError(err error) error {
var (
e = err.Error()
rxp = regexp.MustCompile("not found in type .*")
header = fmt.Sprintf("'%v' is not valid:\n", FunctionFile)
)
if strings.HasPrefix(e, "yaml: unmarshal errors:") {
e = rxp.ReplaceAllString(e, "is not valid")
e = strings.Replace(e, "yaml: unmarshal errors:\n", header, 1)
} else if strings.HasPrefix(e, "yaml:") {
e = rxp.ReplaceAllString(e, "is not valid")
e = strings.Replace(e, "yaml: ", header+" ", 1)
}
return errors.New(e)
}
// Regex used during instantiation and validation of various function fields
// by labels, envs, options, etc.
var (
regWholeSecret = regexp.MustCompile(`^{{\s*secret:((?:\w|['-]\w)+)\s*}}$`)
regKeyFromSecret = regexp.MustCompile(`^{{\s*secret:((?:\w|['-]\w)+):([-._a-zA-Z0-9]+)\s*}}$`)
regWholeConfigMap = regexp.MustCompile(`^{{\s*configMap:((?:\w|['-]\w)+)\s*}}$`)
regKeyFromConfigMap = regexp.MustCompile(`^{{\s*configMap:((?:\w|['-]\w)+):([-._a-zA-Z0-9]+)\s*}}$`)
regLocalEnv = regexp.MustCompile(`^{{\s*env:(\w+)\s*}}$`)
)
// ensureRuntimeDir creates a .func directory in the root of the given
// function which is also registered as ignored in .gitignore
// TODO: Mutate extant .gitignore file if it exists rather than failing
// if present (see contentious files in function.go), such that a user
// can `git init` a directory prior to `func init` in the same directory).
func (f Function) ensureRuntimeDir() error {
if err := os.MkdirAll(filepath.Join(f.Root, RunDataDir), os.ModePerm); err != nil {
return err
}
_, err := os.Stat(".gitignore")
if err == nil {
return nil
}
if !os.IsNotExist(err) {
return err
}
gitignore := `
# Functions use the .func directory for local runtime data which should
# generally not be tracked in source control:
/.func
`
return os.WriteFile(filepath.Join(f.Root, ".gitignore"), []byte(gitignore), 0644)
}
// Tag the function in memory as having been built
// This is locally-scoped data, only indicating there presumably exists
// a container image in the cache of the the configured builder, thus this info
// is placed in a .func (non-source controlled) local metadata directory, which
// is not stritly required to exist, so it is created if needed.
func (f Function) updateBuildStamp() (Function, error) {
hash, err := f.fingerprint()
if err != nil {
return f, err
}
f.BuildStamp = hash
f.NeedsWriteBuildStamp = true
return f, err
}
// Tag the function on disk as having been built
// This is locally-scoped data, only indicating there presumably exists
// a container image in the cache of the the configured builder, thus this info
// is placed in a .func (non-source controlled) local metadata directory, which
// is not stritly required to exist, so it is created if needed.
func (f Function) writeBuildStamp() (err error) {
if err = f.ensureRuntimeDir(); err != nil {
return err
}
hash, err := f.fingerprint()
if err != nil {
return err
}
if err = os.WriteFile(filepath.Join(f.Root, RunDataDir, buildstamp), []byte(hash), os.ModePerm); err != nil {
return err
}
return
}
// fingerprint returns a hash of the filenames and modification timestamps of
// the files within a function's root.
func (f Function) fingerprint() (string, error) {
h := sha256.New()
err := filepath.Walk(f.Root, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
// Always ignore .func, .git (TODO: .funcignore)
if info.IsDir() && (info.Name() == RunDataDir || info.Name() == ".git") {
return filepath.SkipDir
}
fmt.Fprintf(h, "%v:%v:", path, info.ModTime().UnixNano())
return nil
})
return fmt.Sprintf("%x", h.Sum(nil)), err
}
// buildStamp returns the current (last) build stamp for the function
// at the given path, if it can be found.
func (f Function) buildStamp() string {
buildstampPath := filepath.Join(f.Root, RunDataDir, buildstamp)
if _, err := os.Stat(buildstampPath); err != nil {
return ""
}
b, err := os.ReadFile(buildstampPath)
if err != nil {
return ""
}
return string(b)
}
// Built returns true if the given path contains a function which has been
// built without any filesystem modifications since (is not stale).
func (f Function) Built() bool {
// If there is no build stamp, it is not built.
// This case should be redundant with the below check for an image, but is
// temporarily necessary (see the long-winded caviat note below).
if f.BuildStamp == "" {
return false
}
// Missing an image name always means !Built (but does not satisfy staleness
// checks).
// NOTE: This will be updated in the future such that a build does not
// automatically update the function's serialized, source-controlled state,
// because merely building does not indicate the function has changed, but
// rather that field should be populated on deploy. I.e. the Image name
// and image stamp should reside as transient data in .func until such time
// as the given image has been deployed.
// An example of how this bug manifests is that every rebuild of a function
// registers the func.yaml as being dirty for source-control purposes, when
// this should only happen on deploy.
if f.Image == "" {
return false
}
// Calculate the function's Filesystem hash and see if it has changed.
hash, err := f.fingerprint()
if err != nil {
fmt.Fprintf(os.Stderr, "error calculating function's fingerprint: %v\n", err)
return false
}
if f.BuildStamp != hash {
return false
}
// Function has a populated image, existing buildstamp, and the calculated
// fingerprint has not changed.
// It's a pretty good chance the thing doesn't need to be rebuilt, though
// of course filesystem racing conditions do exist, including both direct
// source code modifications or changes to the image cache.
return true
}