package functions import ( "errors" "fmt" "os" "path/filepath" "reflect" "regexp" "strings" "time" "github.com/google/go-containerregistry/pkg/name" "gopkg.in/yaml.v2" fnlabels "knative.dev/func/pkg/k8s/labels" "knative.dev/pkg/ptr" ) const ( // FunctionFile is the file used for the serialized form of a function. FunctionFile = "func.yaml" // RunDataDir holds transient runtime metadata // By default it is excluded from source control. RunDataDir = ".func" RunDataLocalFile = "local.yaml" // BuiltHash is a name of a file that holds hash of built Function in runtime // metadata dir (RunDataDir) BuiltHash = "built-hash" // BuiltImage is a name of a file that holds name of built image in runtime // metadata dir (RunDataDir) BuiltImage = "built-image" ) // Local represents the transient runtime metadata which // is only relevant to the local copy of the function type Local struct { // Remote indicates the deployment (and possibly build) process are to // be triggered in a remote environment rather than run locally. Remote bool `yaml:"remote,omitempty"` } // Function type Function struct { // SpecVersion at which this function is known to be compatible. // More specifically, it is the highest migration which has been applied. // For details see the .Migrated() and .Migrate() methods. SpecVersion string `yaml:"specVersion"` // semver format // Root on disk at which to find/create source and config files. Root string `yaml:"-"` // Name of the function. Name string `yaml:"name,omitempty" jsonschema:"pattern=^[a-z0-9]([-a-z0-9]*[a-z0-9])?$"` // Domain of the function optionally specifies the domain to use as the // route of the function. By default the cluster's default will be used. // Note that the value defined here must be one which the cluster is // configured to recognize, or this will have no effect and the cluster // default will be applied. This value shuld therefore ideally be // validated by the client. Domain string `yaml:"domain,omitempty"` // Runtime is the language plus context. nodejs|go|quarkus|rust etc. Runtime string `yaml:"runtime,omitempty"` // Template for the function. Template string `yaml:"-"` // Registry at which to store interstitial containers, in the form // [registry]/[user]. Registry string `yaml:"registry,omitempty"` // Image is the full OCI image tag in form: // [registry]/[namespace]/[name]:[tag] // example: // quay.io/alice/my.function.name // Registry is optional and is defaulted to DefaultRegistry // example: // alice/my.function.name // If Image is provided, it overrides the default of concatenating // "Registry+Name:latest" to derive the Image. Image string `yaml:"image,omitempty"` // Namespace in which to deploy the Function Namespace string `yaml:"namespace,omitempty"` // Created time is the moment that creation was successfully completed // according to the client which is in charge of what constitutes being // fully "Created" (aka initialized) Created time.Time `yaml:"created"` // Invoke defines hints for use when invoking this function. // See Client.Invoke for usage. Invoke string `yaml:"invoke,omitempty" jsonschema:"enum=http,enum=cloudevent"` // Build defines the build properties for a function Build BuildSpec `yaml:"build,omitempty"` // Run defines the runtime properties for a function Run RunSpec `yaml:"run,omitempty"` // Deploy defines the deployment properties for a function Deploy DeploySpec `yaml:"deploy,omitempty"` Local Local `yaml:"-"` } // KnativeSubscription type KnativeSubscription struct { Source string `yaml:"source"` Filters map[string]string `yaml:"filters,omitempty"` } // BuildSpec type BuildSpec struct { // Git stores information about an optionally associated git repository. Git Git `yaml:"git,omitempty"` // BuilderImages define optional explicit builder images to use by // builder implementations in leau of the in-code defaults. They key // is the builder's short name. For example: // builderImages: // pack: example.com/user/my-pack-node-builder // s2i: example.com/user/my-s2i-node-builder BuilderImages map[string]string `yaml:"builderImages,omitempty"` // Optional list of buildpacks to use when building the function Buildpacks []string `yaml:"buildpacks,omitempty"` // Builder is the name of the subsystem that will complete the underlying // build (pack, s2i, etc) Builder string `yaml:"builder,omitempty" jsonschema:"enum=pack,enum=s2i"` // Build Env variables to be set BuildEnvs Envs `yaml:"buildEnvs,omitempty"` // PVCSize specifies the size of persistent volume claim used to store function // when using deployment and remote build process (only relevant when Remote is true). PVCSize string `yaml:"pvcSize,omitempty"` // Image stores last built image name NOT in func.yaml, but instead // in .func/built-image Image string `yaml:"-"` } // RunSpec type RunSpec struct { // List of volumes to be mounted to the function Volumes []Volume `yaml:"volumes,omitempty"` // Env variables to be set Envs Envs `yaml:"envs,omitempty"` // StartTimeout specifies that this function should have a custom timeout // when starting. This setting is currently respected by the host runner, // with containerized docker runner and deployed Knative service integration // in development. StartTimeout time.Duration `yaml:"startTimeout,omitempty"` } // DeploySpec type DeploySpec struct { // Namespace into which the function was deployed on supported platforms. Namespace string `yaml:"namespace,omitempty"` // Image is the deployed image including sha256 Image string `yaml:"image,omitempty"` // Map containing user-supplied annotations // Example: { "division": "finance" } Annotations map[string]string `yaml:"annotations,omitempty"` // Options to be set on deployed function (scaling, etc.) Options Options `yaml:"options,omitempty"` // Map of user-supplied labels Labels []Label `yaml:"labels,omitempty"` // Health endpoints specified by the language pack HealthEndpoints HealthEndpoints `yaml:"healthEndpoints,omitempty"` // ServiceAccountName is the name of the service account used for the // function pod. The service account must exist in the namespace to // succeed. // More info: https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/ ServiceAccountName string `yaml:"serviceAccountName,omitempty"` Subscriptions []KnativeSubscription `yaml:"subscriptions,omitempty"` } // HealthEndpoints specify the liveness and readiness endpoints for a Runtime type HealthEndpoints struct { Liveness string `yaml:"liveness,omitempty"` Readiness string `yaml:"readiness,omitempty"` } // BuildConfig defines builders and buildpacks type BuildConfig struct { Buildpacks []string `yaml:"buildpacks,omitempty"` BuilderImages map[string]string `yaml:"builderImages,omitempty"` } // NewFunctionWith defaults as provided. func NewFunctionWith(defaults Function) Function { // Deprecatded: these defaults should be used directly from their // in-code static defaults, config, etc. A function struct is used to hold // overrides (eg. use PVCSize X instead of the default), and to record the // results of operations (eg. the function was deployed with image Y). if defaults.SpecVersion == "" { defaults.SpecVersion = LastSpecVersion() } if defaults.Template == "" { defaults.Template = DefaultTemplate } if defaults.Build.BuilderImages == nil { defaults.Build.BuilderImages = make(map[string]string) } if defaults.Deploy.Annotations == nil { defaults.Deploy.Annotations = make(map[string]string) } return defaults } // NewFunction from a given path. // Invalid paths, or no function at path are errors. // Syntactic errors are returned immediately (yaml unmarshal errors). // Functions which are syntactically valid are also then logically validated. // Functions from earlier versions are brought up to current using migrations. // Migrations are run prior to validators such that validation can omit // concerning itself with backwards compatibility. Migrators must therefore // selectively consider the minimal validation necessary to enable migration. func NewFunction(root string) (f Function, err error) { f.Build.BuilderImages = make(map[string]string) f.Deploy.Annotations = make(map[string]string) // Path defaults to current working directory, and if provided explicitly // Path must exist and be a directory if root == "" { if root, err = os.Getwd(); err != nil { return } } f.Root = root // path is not persisted, as this is the purview of the FS // Path must exist and be a directory fd, err := os.Stat(root) if err != nil { return f, err } if !fd.IsDir() { return f, fmt.Errorf("function path must be a directory") } // If no func.yaml in directory, return the default function which will // have f.Initialized() == false var filename = filepath.Join(root, FunctionFile) if _, err = os.Stat(filename); err != nil { if os.IsNotExist(err) { err = nil } return } // Path is valid and func.yaml exists: load it bb, err := os.ReadFile(filename) if err != nil { return } var functionMarshallingError error var functionMigrationError error if marshallingErr := yaml.Unmarshal(bb, &f); marshallingErr != nil { functionMarshallingError = formatUnmarshalError(marshallingErr) // human-friendly unmarshalling errors } if f, err = f.Migrate(); err != nil { functionMigrationError = err } // Only if migration fail return errors to the user. include marshalling error if present if functionMigrationError != nil { //returning both migrations and marshalling errors to the user errorText := "Error: \n" if functionMarshallingError != nil { errorText += "Marshalling: " + functionMarshallingError.Error() } errorText += "\n" + "Migration: " + functionMigrationError.Error() return Function{}, errors.New(errorText) } f.Local, err = f.newLocal() if err != nil { return } // ---- LOCAL SETTINGS - STUFF NOT IN FUNC.YAML ---- // f.Build.Image, err = f.getLastBuiltImage() return } // Validate function is logically correct, returning a bundled, and quite // verbose, formatted error detailing any issues. func (f Function) Validate() error { if f.Root == "" { return errors.New("function root path is required") } var ctr int errs := [][]string{ validateVolumes(f.Run.Volumes), ValidateBuildEnvs(f.Build.BuildEnvs), ValidateEnvs(f.Run.Envs), validateOptions(f.Deploy.Options), ValidateLabels(f.Deploy.Labels), validateGit(f.Build.Git), } var b strings.Builder b.WriteString(fmt.Sprintf("'%v' contains errors:", FunctionFile)) for _, ee := range errs { if len(ee) > 0 { b.WriteString("\n") // Precede each group of errors with a linebreak } for _, e := range ee { ctr++ b.WriteString("\t" + e) } } if ctr == 0 { return nil // Return nil if there were no validation errors. } return errors.New(b.String()) } var envPattern = regexp.MustCompile(`^{{\s*(\w+)\s*:(\w+)\s*}}$`) // Interpolate Env slice // Values with no special format are preserved as simple values. // Values which do include the interpolation format (begin with {{) but are not // keyed as "env" are also preserved as is. // Values properly formatted as {{ env:NAME }} are interpolated (substituted) // with the value of the local environment variable "NAME", and an error is // returned if that environment variable does not exist. func Interpolate(ee []Env) (map[string]string, error) { envs := make(map[string]string, len(ee)) for _, e := range ee { // Assert non-nil name. if e.Name == nil { return envs, errors.New("env name may not be nil") } // Nil value indicates the resultant map should not include this env var. if e.Value == nil { continue } k, v := *e.Name, *e.Value // Simple Values are preserved. // If not prefixed by {{, no interpolation required (simple value) if !strings.HasPrefix(v, "{{") { envs[k] = v // no interpolation required. continue } // Values not matching the interpolation pattern are preserved. // If not in the form "{{ env:XYZ }}" then return the value as-is for // 0 1 2 3 // possible match and interpolation in different ways. parts := envPattern.FindStringSubmatch(v) if len(parts) <= 2 || parts[1] != "env" { envs[k] = v continue } // Properly formatted local env var references are interpolated. localName := parts[2] localValue, ok := os.LookupEnv(localName) if !ok { return envs, fmt.Errorf("expected environment variable '%v' not found", localName) } envs[k] = localValue } return envs, nil } // nameFromPath returns the default name for a function derived from a path. // This consists of the last directory in the given path, if derivable (empty // paths, paths consisting of all slashes, etc. return the zero value "") func nameFromPath(path string) string { pathParts := strings.Split(strings.TrimRight(path, string(os.PathSeparator)), string(os.PathSeparator)) return pathParts[len(pathParts)-1] /* the above may have edge conditions as it assumes the trailing value * is a directory name. If errors are encountered, we _may_ need to use the * inbuilt logic in the std lib and either check if the path indicated is a * directory (appending slash) and then run: base := filepath.Base(filepath.Dir(path)) if base == string(os.PathSeparator) || base == "." { return "" // Consider it underivable: string zero value } return base */ } // Write Function struct (metadata) to Disk at f.Root func (f Function) Write() (err error) { // Skip writing (and dirtying the work tree) if there were no modifications. f1, _ := NewFunction(f.Root) if reflect.DeepEqual(f, f1) { return } // Do not write invalid functions if err = f.Validate(); err != nil { return } // Write var bb []byte if bb, err = yaml.Marshal(&f); err != nil { return } // TODO: open existing file for writing, such that existing permissions // are preserved? err = os.WriteFile(filepath.Join(f.Root, FunctionFile), bb, 0644) if err != nil { return } // Write local settings err = ensureRunDataDir(f.Root) if err != nil { return } if bb, err = yaml.Marshal(&f.Local); err != nil { return } localConfigPath := filepath.Join(f.Root, RunDataDir, RunDataLocalFile) if err = os.WriteFile(localConfigPath, bb, 0644); err != nil { return } // Write built image to .func err = f.WriteRuntimeBuiltImage(false) return } type stampOptions struct{ journal bool } type stampOption func(o *stampOptions) // WithStampJournaling is a Stamp option which causes the stamp logfile // to be created with a timestamp prefix. This has the effect of creating // a stamp journal, useful for debugging. The default behavior is to only // retain the most recent log file as "built.log". func WithStampJournal() stampOption { return func(o *stampOptions) { o.journal = true } } // Stamp a function as being built. // // This is a performance optimization used when updates to the // function are known to have no effect on its built container. This // stamp is checked before certain operations, and if it has been updated, // the build can be skuipped. If in doubt, just use .Write only. // // Updates the build stamp at ./func/built (and the log // at .func/built.log) to reflect the current state of the filesystem. // Note that the caller should call .Write first to flush any changes to the // function in-memory to the filesystem prior to calling stamp. // // The runtime data directory .func is created in the function root if // necessary. func (f Function) Stamp(oo ...stampOption) (err error) { options := &stampOptions{} for _, o := range oo { o(options) } if err = ensureRunDataDir(f.Root); err != nil { return } // Cacluate the hash and a logfile of what comprised it var hash, log string if hash, log, err = Fingerprint(f.Root); err != nil { return } // Write out the hash if err = os.WriteFile(filepath.Join(f.Root, RunDataDir, BuiltHash), []byte(hash), os.ModePerm); err != nil { return } // Write out the logfile, optionally timestamped for retention. logfileName := "built.log" if options.journal { logfileName = timestamp(logfileName) } logfile, err := os.Create(filepath.Join(f.Root, RunDataDir, logfileName)) if err != nil { return } defer logfile.Close() _, err = fmt.Fprintln(logfile, log) return } // timestamp returns the given string prefixed with a microsecond-precision // timestamp followed by a dot. // YYYYMMDDHHMMSS.$nanosecond.$s func timestamp(s string) string { t := time.Now() return fmt.Sprintf("%s.%09d.%s", t.Format("20060102150405"), t.Nanosecond(), s) } // Initialized returns if the function has been initialized. // Any errors are considered failure (invalid or inaccessible root, config file, etc). func (f Function) Initialized() bool { return !f.Created.IsZero() } // LabelsMap combines default labels with the labels slice provided. // It will the resulting slice with ValidateLabels and return a key/value map. // - key: EXAMPLE1 # Label directly from a value // value: value1 // - key: EXAMPLE2 # Label from the local ENV var // value: {{ env:MY_ENV }} func (f Function) LabelsMap() (map[string]string, error) { defaultLabels := []Label{ { Key: ptr.String(fnlabels.FunctionKey), Value: ptr.String(fnlabels.FunctionValue), }, { Key: ptr.String(fnlabels.FunctionNameKey), Value: ptr.String(f.Name), }, { Key: ptr.String(fnlabels.FunctionRuntimeKey), Value: ptr.String(f.Runtime), }, // --- handle usage of deprecated labels (`boson.dev/function`, `boson.dev/runtime`) { Key: ptr.String(fnlabels.DeprecatedFunctionKey), Value: ptr.String(fnlabels.FunctionValue), }, { Key: ptr.String(fnlabels.DeprecatedFunctionRuntimeKey), Value: ptr.String(f.Runtime), }, // --- end of handling usage of deprecated runtime labels } labels := append(defaultLabels, f.Deploy.Labels...) if err := ValidateLabels(labels); len(err) != 0 { return nil, errors.New(strings.Join(err, " ")) } l := map[string]string{} for _, label := range labels { if label.Value == nil { l[*label.Key] = "" } else { if strings.HasPrefix(*label.Value, "{{") { // env variable format is validated above in ValidateLabels match := regLocalEnv.FindStringSubmatch(*label.Value) l[*label.Key] = os.Getenv(match[1]) } else { l[*label.Key] = *label.Value } } } return l, nil } // ImageName returns a full image name (OCI container tag) for the // Function based off of the Function's `Registry` member plus `Name`. // Used to calculate the final value for .Deploy.Image when none is provided // explicitly. // // form: [registry]/[user]/[function]:latest // example: quay.io/alice/my.function.name:latest // // Also nested namespaces should be supported: // form: [registry]/[parent]/[user]/[function]:latest // example: quay.io/project/alice/my.function.name:latest // // Registry values which only contain a single token are presumed to // indicate the namespace at the default registry. func (f Function) ImageName() (image string, err error) { if f.Registry == "" { return "", ErrRegistryRequired } if f.Name == "" { return "", ErrNameRequired } f.Registry = strings.Trim(f.Registry, "/") // too defensive? // Explicitly append :latest tag. We expect source control to drive // versioning, rather than rely on image tags with explicitly pinned version // numbers, as is seen in many serverless solutions. This will be updated // to branch name when we add source-driven canary/ bluegreen deployments. // For pinning to an exact container image, see ImageWithDigest refStr := f.Registry + "/" + f.Name + ":latest" ref, err := name.ParseReference(refStr) if err != nil { return "", fmt.Errorf("cannot determine function image: %w", err) } return ref.Name(), nil } // Format yaml unmarshall error to be more human friendly. func formatUnmarshalError(err error) error { var ( e = err.Error() rxp = regexp.MustCompile("not found in type .*") header = fmt.Sprintf("'%v' is not valid:\n", FunctionFile) ) if strings.HasPrefix(e, "yaml: unmarshal errors:") { e = rxp.ReplaceAllString(e, "is not valid") e = strings.Replace(e, "yaml: unmarshal errors:\n", header, 1) } else if strings.HasPrefix(e, "yaml:") { e = rxp.ReplaceAllString(e, "is not valid") e = strings.Replace(e, "yaml: ", header+" ", 1) } return errors.New(e) } // Regex used during instantiation and validation of various function fields // by labels, envs, options, etc. var ( regWholeSecret = regexp.MustCompile(`^{{\s*secret:((?:\w|['-]\w)+)\s*}}$`) regKeyFromSecret = regexp.MustCompile(`^{{\s*secret:((?:\w|['-]\w)+):([-._a-zA-Z0-9]+)\s*}}$`) regWholeConfigMap = regexp.MustCompile(`^{{\s*configMap:((?:\w|['-]\w)+)\s*}}$`) regKeyFromConfigMap = regexp.MustCompile(`^{{\s*configMap:((?:\w|['-]\w)+):([-._a-zA-Z0-9]+)\s*}}$`) regLocalEnv = regexp.MustCompile(`^{{\s*env:(\w+)\s*}}$`) ) // Built returns true if the function is considered built. // Note that this only considers the function as it exists on-disk at // f.Root. func (f Function) Built() bool { // If there is no build stamp, it is not built. stamp := f.BuildStamp() if stamp == "" { return false } // Calculate the current filesystem hash and see if it has changed. // // If this comparison returns true, the Function has a populated image, // existing buildstamp, and the calculated fingerprint has not changed. // // It's a pretty good chance the thing doesn't need to be rebuilt, though // of course filesystem racing conditions do exist, including both direct // source code modifications or changes to the image cache. hash, _, err := Fingerprint(f.Root) if err != nil { fmt.Fprintf(os.Stderr, "error calculating function's fingerprint: %v\n", err) return false } if stamp != hash { return false } // Special case of registry change on a subsequent deploy attempt should // result in unbuilt image, forcing a rebuild if possible // Example: Deploy with image using registry X. Then subsequently deploy with // --registry=Y, changing registry resulting in unmatched Registry and Build.Image. // If f.Image is specified, registry is overridden -- meaning its not taken into // consideration and can be different from actually built image. buildImage := f.Build.Image fRegistry := f.Registry if !strings.Contains(buildImage, fRegistry) && f.Image == "" { fmt.Fprintf(os.Stderr, "Warning: registry '%s' does not match currently built image '%s' and no direct image override was provided via --image\n", f.Registry, f.Build.Image) return false } return true } // BuildStamp accesses the current (last) build stamp for the function. // Unbuilt functions return empty string. func (f Function) BuildStamp() string { path := filepath.Join(f.Root, RunDataDir, BuiltHash) if _, err := os.Stat(path); err != nil { return "" } b, err := os.ReadFile(path) if err != nil { return "" } return string(b) } // localSettings returns the local settings set for the function func (f Function) newLocal() (localConfig Local, err error) { err = ensureRunDataDir(f.Root) if err != nil { return } localSettingsPath := filepath.Join(f.Root, RunDataDir, RunDataLocalFile) if _, err = os.Stat(localSettingsPath); os.IsNotExist(err) { err = nil return } b, err := os.ReadFile(localSettingsPath) if err != nil { return } err = yaml.Unmarshal(b, &localConfig) return } // WriteRuntimeBuiltImage writes built image name into runtime metadata // directory (.func/) from f.Build.Image func (f Function) WriteRuntimeBuiltImage(verbose bool) error { path := filepath.Join(f.Root, RunDataDir, BuiltImage) // dont write if empty (not built) if f.Build.Image == "" { return nil } if verbose { fmt.Printf("Writing built image: '%s' at path: '%s'\n", f.Build.Image, path) } return os.WriteFile(path, []byte(f.Build.Image), os.ModePerm) } // getLastBuiltImage reads .func/built-image and returns its value or empty string // if the file doesnt exist (not built yet). Other errors are returned as usual. func (f Function) getLastBuiltImage() (string, error) { path := filepath.Join(f.Root, RunDataDir, BuiltImage) if _, err := os.Stat(path); err != nil { if os.IsNotExist(err) { return "", nil } return "", err } b, err := os.ReadFile(path) if err != nil { return "", err } return string(b), nil } // ImageNameWithDigest works with f.Build.Image and image digest. if the func // parameter newDigest is empty, just return the image name as is. // TODO: This function is a temporary one for a workaround for a current // solution on how the image digest is fetched (which is during/after Push). // Image digest should be gotten from imageID right after building the Function. // PS: I think that imageID item contains "sha256:[digest]" func (f Function) ImageNameWithDigest(newDigest string) string { if newDigest == "" { return f.Build.Image } image := f.Build.Image // overwrite current digest shaIndex := strings.Index(image, "@sha256:") if shaIndex > 0 { return image[:shaIndex] + "@" + newDigest } // image doesnt have a digest yet == image not pushed yet //parse f.Build.Image to separate its name and tag lastSlashIdx := strings.LastIndexAny(image, "/") imageAsBytes := []byte(image) part1 := string(imageAsBytes[:lastSlashIdx+1]) part2 := string(imageAsBytes[lastSlashIdx+1:]) // Remove tag from the image name and append SHA256 hash instead return part1 + strings.Split(part2, ":")[0] + "@" + newDigest }