package functions import ( "errors" "fmt" "os" "path/filepath" "time" "github.com/coreos/go-semver/semver" "gopkg.in/yaml.v2" ) // Migrate applies any necessary migrations, returning a new migrated // version of the function. It is the caller's responsibility to // .Write() the function to persist to disk. func (f Function) Migrate() (migrated Function, err error) { // Return immediately if the function indicates it has already been // migrated. if f.Migrated() { return f, nil } migrated = f // initially equivalent for _, m := range migrations { // Skip this migration if the current function's specVersion is not less than // the migration's applicable specVersion. if f.SpecVersion != "" && !semver.New(migrated.SpecVersion).LessThan(*semver.New(m.version)) { continue } // Apply this migration when the function's specVersion is less than that which // the migration will impart. migrated, err = m.migrate(migrated, m) if err != nil { return // fail fast on any migration errors } } return } // migration is a migration which should be applied to function's whose version // is below that indicated. type migration struct { version string // version before which this migration may be needed. migrate migrator // Migrator migrates. } // migrator is a function which returns a migrated copy of an inbound function. type migrator func(Function, migration) (Function, error) // Migrated returns whether the function has been migrated to the highest // level the currently executing system is aware of (or beyond). // returns true. func (f Function) Migrated() bool { // If the function has no specVersion, it is pre-migrations and is implicitly // not migrated. if f.SpecVersion == "" { return false } // lastMigration is the last registered migration. lastMigration := semver.New(LastSpecVersion()) // Fail the migration test if the function's version is less than // the latest available. return !semver.New(f.SpecVersion).LessThan(*lastMigration) } // LastSpecVersion returns the string value for the most recent migration func LastSpecVersion() string { return migrations[len(migrations)-1].version } // Migrations registry // ------------------- // migrations are all migrators in ascending order. // No two migrations may have the exact version number (introduce a patch // version for the migration if necessary) var migrations = []migration{ {"0.19.0", migrateToCreationStamp}, {"0.23.0", migrateToBuilderImages}, {"0.25.0", migrateToSpecVersion}, {"0.34.0", migrateToSpecsStructure}, {"0.35.0", migrateFromInvokeStructure}, {"0.36.0", migratePersistentVolumeTypoFixup}, // New Migrations Here. } // Individual Migration implementations // ------------------------------------ // migrateToCreationStamp // The initial migration which brings a function from // some unknown point in the past to the point at which it is versioned, // migrated and includes a creation timestamp. Without this migration, // instantiation of old functions will fail with a "Function at path X not // initialized" in func versions above v0.19.0 // // This migration must be aware of the difference between a function which // was previously created (but with no created stamp), and a function which // exists only in memory and should legitimately fail the .Initialized() check. // The only way to know is to check a side effect of earlier versions: // are the `.Name` and `.Runtime` fields populated. This was the way the // `.Initialized` check was implemented prior to versioning being introduced, so // it is equivalent logically to use this here as well. // In summary: if the creation stamp is zero, but name and runtime fields are // populated, then this is an old function and should be migrated to having a // created stamp. Otherwise, this is an in-memory (new) function that is // currently in the process of being created and as such need not be mutated // to consider this migration having been evaluated. func migrateToCreationStamp(f Function, m migration) (Function, error) { // For functions with no creation timestamp, but appear to have been pre- // existing, populate their created stamp and version. // Yes, it's a little gnarly, but bootstrapping into the loveliness of a // versioned/migrated system takes cleaning up the trash. if f.Created.IsZero() { // If there is no created stamp if f.Name != "" && f.Runtime != "" { // and it appears to be an old function f.Created = time.Now() // Migrate it to having a timestamp. } } f.SpecVersion = m.version // Record this migration was evaluated. return f, nil } // migrateToBuilderImages // Prior to this migration, 'builder' and 'builders' attributes of a function // were specific to buildpack builds. In addition, the separation of the two // fields was to facilitate the use of "named" inbuilt builders, which ended // up not being necessary. With the addition of the S2I builder implementation, // it is necessary to differentiate builders for use when building via Pack vs // builder for use when building with S2I. Furthermore, now that the builder // itself is a user-supplied parameter, the short-hand of calling builder images // simply "builder" is not possible, since that term more correctly refers to // the builder being used (S2I, pack, or some future implementation), while this // field very specifically refers to the image the chosen builder should use // (in leau of the inbuilt default). // // For an example of the situation: the 'builder' member used to instruct the // system to use that builder _image_ in all cases. This of course restricts // the system from being able to build with anything other than the builder // implementation to which that builder image applies (pack or s2i). Further, // always including this value in the serialized func.yaml causes this value to // be pegged/immutable (without manual intervention), which hampers our ability // to change out the underlying builder image with future versions. // // The 'builder' and 'builders' members have therefore been removed in favor // of 'builderImages', which is keyed by the short name of the builder // implementation (currently 'pack' and 's2i'). Its existence is optional, // with the default value being provided in the associated builder's impl. // Should the value exist, this indicates the user has overridden the value, // or is using a fully custom language pack. // // This migration allows pre-builder-image functions to load despite their // inclusion of the now removed 'builder' member. If the user had provided // a customized builder image, that value is preserved as the builder image // for the 'pack' builder in the new version (s2i did not exist prior). // See associated unit tests. func migrateToBuilderImages(f1 Function, m migration) (Function, error) { // Load the function using pertinent parts of the previous version's schema: f0Filename := filepath.Join(f1.Root, FunctionFile) bb, err := os.ReadFile(f0Filename) if err != nil { return f1, errors.New("migration 'migrateToBuilderImages' error: " + err.Error()) } f0 := migrateToBuilderImages_previousFunction{} if err = yaml.Unmarshal(bb, &f0); err != nil { return f1, errors.New("migration 'migrateToBuilderImages' error: " + err.Error()) } // At time of this migration, the default pack builder image for all language // runtimes is: defaultPackBuilderImage := "gcr.io/paketo-buildpacks/builder:base" // If the old function had defined something custom if f0.Builder != "" && f0.Builder != defaultPackBuilderImage { // carry it forward as the new pack builder image if f1.Build.BuilderImages == nil { f1.Build.BuilderImages = make(map[string]string) } f1.Build.BuilderImages["pack"] = f0.Builder } // Flag f1 as having had the migration applied f1.SpecVersion = m.version return f1, nil } // migrateToSpecVersion updates a func.yaml file to use SpecVersion // instead of Version to track the migration numbers func migrateToSpecVersion(f Function, m migration) (Function, error) { // Load the function func.yaml file f0Filename := filepath.Join(f.Root, FunctionFile) bb, err := os.ReadFile(f0Filename) if err != nil { return f, errors.New("migration 'migrateToSpecVersion' error: " + err.Error()) } // Only handle the Version field if it exists f0 := migrateToSpecVersion_previousFunction{} if err = yaml.Unmarshal(bb, &f0); err != nil { return f, errors.New("migration 'migrateToSpecVersion' error: " + err.Error()) } f.SpecVersion = m.version return f, nil } // migrateToSpecsStructure migration makes sure use the sub-specs structs for build, run and deploy phases. // To avoid unmarshalling issues with the old format this migration needs to be executed first. // Further migrations will operate on this new struct with sub-specs func migrateToSpecsStructure(f1 Function, m migration) (Function, error) { // Load the Function using pertinent parts of the previous version's schema: f0Filename := filepath.Join(f1.Root, FunctionFile) bb, err := os.ReadFile(f0Filename) if err != nil { return f1, errors.New("migration 'migrateToSpecsStructure' error: " + err.Error()) } f0 := migrateToSpecs_previousFunction{} if err = yaml.Unmarshal(bb, &f0); err != nil { return f1, errors.New("migration 'migrateToSpecsStructure' error: " + err.Error()) } if f0.Git.URL != "" { f1.Build.Git.URL = f0.Git.URL } if f0.Git.Revision != "" { f1.Build.Git.Revision = f0.Git.Revision } if f0.Git.ContextDir != "" { f1.Build.Git.ContextDir = f0.Git.ContextDir } //Append BuilderImages from old format, without destroying previous migrations if f0.BuilderImages != nil { for k, v := range f0.BuilderImages { f1.Build.BuilderImages[k] = v } } if f0.Buildpacks != nil { f1.Build.Buildpacks = append(f1.Build.Buildpacks, f0.Buildpacks...) } if f0.BuildEnvs != nil { f1.Build.BuildEnvs = append(f1.Build.BuildEnvs, f0.BuildEnvs...) } if f0.Volumes != nil { f1.Run.Volumes = append(f1.Run.Volumes, f0.Volumes...) } if f0.Envs != nil { f1.Run.Envs = append(f1.Run.Envs, f0.Envs...) } if f0.Annotations != nil { for k, v := range f0.Annotations { f1.Deploy.Annotations[k] = v } } if f0.Options.Resources != nil { f1.Deploy.Options.Resources = f0.Options.Resources } if f0.Options.Scale != nil { f1.Deploy.Options.Scale = f0.Options.Scale } if f0.Labels != nil { f1.Deploy.Labels = append(f1.Deploy.Labels, f0.Labels...) } if f0.HealthEndpoints.Readiness != "" { f1.Deploy.HealthEndpoints.Readiness = f0.HealthEndpoints.Readiness } if f0.HealthEndpoints.Liveness != "" { f1.Deploy.HealthEndpoints.Liveness = f0.HealthEndpoints.Liveness } f1.Deploy.Namespace = f0.Namespace f1.Build.Builder = f0.Builder f1.SpecVersion = m.version return f1, nil } // migrateFromInvokeStructure migrates functions prior 0.35.0 via changing // the Invocation.format(string) to new Invoke(string) to minimalize func.yaml // file. When Invoke now holds default value (http) it will not show up in // func.yaml as the default value is implicitly expected. Otherwise if Invoke // is non-default value, it will be written in func.yaml. func migrateFromInvokeStructure(f1 Function, m migration) (Function, error) { // Load the Function using pertinent parts of the previous version's schema: f0Filename := filepath.Join(f1.Root, FunctionFile) bb, err := os.ReadFile(f0Filename) if err != nil { return f1, errors.New("migration 'migrateFromInvokeStructure' error: " + err.Error()) } f0 := migrateFromInvokeStructure_previousFunction{} if err = yaml.Unmarshal(bb, &f0); err != nil { return f1, errors.New("migration 'migrateFromInvokeStructure' error: " + err.Error()) } if f0.Invocation.Format != "" && f0.Invocation.Format != "http" { f1.Invoke = f0.Invocation.Format } // Flag f1 as having had the migration applied f1.SpecVersion = m.version return f1, nil } func migratePersistentVolumeTypoFixup(fn Function, m migration) (Function, error) { f, err := os.Open(filepath.Join(fn.Root, FunctionFile)) if err != nil { return Function{}, fmt.Errorf("cannot open func.yaml: %w", err) } defer f.Close() data := struct { Run struct { Volumes []struct { PersistentVolumeClaim *PersistentVolumeClaim `yaml:"presistentVolumeClaim"` } `yaml:"volumes,omitempty"` } }{} dec := yaml.NewDecoder(f) err = dec.Decode(&data) if err != nil { return Function{}, fmt.Errorf("cannot deserialize old sub-structure: %w", err) } for idx, volume := range data.Run.Volumes { if volume.PersistentVolumeClaim != nil { fn.Run.Volumes[idx].PersistentVolumeClaim = volume.PersistentVolumeClaim } } fn.SpecVersion = m.version return fn, nil } // The pertinent aspects of the Function's schema prior the 1.0.0 version migrations type migrateToSpecs_previousFunction struct { // Namespace into which the function is deployed on supported platforms. Namespace string `yaml:"namespace"` // Git stores information about remote git repository, // in case build type "git" is being used Git Git `yaml:"git"` // BuilderImages define optional explicit builder images to use by // builder implementations in leau of the in-code defaults. They key // is the builder's short name. For example: // builderImages: // pack: example.com/user/my-pack-node-builder // s2i: example.com/user/my-s2i-node-builder BuilderImages map[string]string `yaml:"builderImages,omitempty"` // Optional list of buildpacks to use when building the function Buildpacks []string `yaml:"buildpacks"` // Builder is the name of the subsystem that will complete the underlying // build (pack, s2i, etc) Builder string `yaml:"builder" jsonschema:"enum=pack,enum=s2i"` // List of volumes to be mounted to the function Volumes []Volume `yaml:"volumes"` // Build Env variables to be set BuildEnvs []Env `yaml:"buildEnvs"` // Env variables to be set Envs []Env `yaml:"envs"` // Map containing user-supplied annotations // Example: { "division": "finance" } Annotations map[string]string `yaml:"annotations"` // Options to be set on deployed function (scaling, etc.) Options Options `yaml:"options"` // Map of user-supplied labels Labels []Label `yaml:"labels"` // Health endpoints specified by the language pack HealthEndpoints HealthEndpoints `yaml:"healthEndpoints"` } // Functions prior to 0.25 will have a Version field type migrateToSpecVersion_previousFunction struct { Version string `yaml:"version"` } // The pertinent aspects of the function schema prior to the builder images // migration. type migrateToBuilderImages_previousFunction struct { Builder string `yaml:"builder"` } // (Defined only for previous versions migration) // Invocation defines hints on how to accomplish a function invocation. type migrateFromInvokeStructure_invocation struct { Format string `yaml:"format,omitempty"` } // Functions prior to 0.35.0 will have Invocation.Format instead of Invoke type migrateFromInvokeStructure_previousFunction struct { Invocation migrateFromInvokeStructure_invocation `yaml:"invocation,omitempty"` }