knative
/
func
mirror of https://github.com/knative/func.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: write a build stamp log to .func (#1695) 

* feat: build stamp and log

- Adds an explicit "Stamp" step to client builds
- Building always "Stamps" the function, allowing builds to cache
- Commands which alter function in inconsequential ways update the stamp
  as-needed.
- Tests updated to use the API rather than hard-coding func.yaml

* fix misspellings

* temporarily disable Quarkus tests

* stamping also creates necessary run directory

* reenable Quarkus tests

* comments
This commit is contained in:
Luke Kingland 2023-05-04 06:43:56 +09:00 committed by GitHub
parent 6558f9652d
commit 1285176f60
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 167 additions and 201 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

7
cmd/build.go
View File

@ -200,7 +200,12 @@ func runBuild(cmd *cobra.Command, _ []string, newClient ClientFactory) (err erro
}
}
return f.Write()
if err = f.Write(); err != nil {
return
}
// Stamp is a performance optimization: treat the function as being built
// (cached) unless the fs changes.
return f.Stamp()
}
type buildConfig struct {

12
cmd/deploy.go
View File

@ -280,8 +280,16 @@ func runDeploy(cmd *cobra.Command, newClient ClientFactory) (err error) {
}
}
// mutations persisted on success
return f.Write()
// Write
if err = f.Write(); err != nil {
return
}
// Stamp is a performance optimization: treat the function as being built
// (cached) unless the fs changes.
// Updates the build stamp because building must have been accomplished
// during this process, and a future call to deploy without any appreciable
// changes to the filesystem should not rebuild again unless `--build`
return f.Stamp()
}
// shouldBuild returns true if the value of the build option is a truthy value,

101
cmd/run_test.go
View File

@ -3,7 +3,6 @@ package cmd
import (
"context"
"fmt"
"os"
"testing"
fn "knative.dev/func/pkg/functions"
@ -12,92 +11,68 @@ import (
func TestRun_Run(t *testing.T) {
tests := []struct {
name string // name of the test
desc string // description of the test
funcState string // function state, as described in func.yaml
args []string // args for the test case
buildError error // Set the builder to yield this error
runError error // Set the runner to yield this error
buildInvoked bool // should Builder.Build be invoked?
runInvoked bool // should Runner.Run be invoked?
name string // name of the test
desc string // description of the test
setup func(fn.Function, *testing.T) error // Optionally mutate function
args []string // args for the test case
buildError error // Set the builder to yield this error
runError error // Set the runner to yield this error
buildInvoked bool // should Builder.Build be invoked?
runInvoked bool // should Runner.Run be invoked?
}{
{
name: "run and build by default",
desc: "Should run and build when build flag is not specified",
funcState: `name: test-func
runtime: go
created: 2009-11-10 23:00:00`,
name: "run and build by default",
desc: "Should run and build when build flag is not specified",
args: []string{},
buildInvoked: true,
runInvoked: true,
},
{
name: "run and build flag",
desc: "Should run and build when build is merely provided (defaults to true on presence)",
funcState: `name: test-func
runtime: go
created: 2009-11-10 23:00:00`,
name: "run and build flag",
desc: "Should run and build when build is merely provided (defaults to true on presence)",
args: []string{"--build"},
buildInvoked: true,
runInvoked: true,
},
{
name: "run and build",
desc: "Should run and build when build is specifically requested",
funcState: `name: test-func
runtime: go
created: 2009-11-10 23:00:00`,
name: "run and build",
desc: "Should run and build when build is specifically requested",
args: []string{"--build=true"},
buildInvoked: true,
runInvoked: true,
},
{
name: "run and build with builder pack",
desc: "Should run and build when build is specifically requested with builder pack",
funcState: `name: test-func
runtime: go
created: 2023-03-12 15:00:00`,
name: "run and build with builder pack",
desc: "Should run and build when build is specifically requested with builder pack",
args: []string{"--build=true", "--builder=pack"},
buildInvoked: true,
runInvoked: true,
},
{
name: "run and build with builder s2i",
desc: "Should run and build when build is specifically requested with builder s2i",
funcState: `name: test-func
runtime: go
created: 2023-03-12 15:00:00`,
name: "run and build with builder s2i",
desc: "Should run and build when build is specifically requested with builder s2i",
args: []string{"--build=true", "--builder=s2i"},
buildInvoked: true,
runInvoked: true,
},
{
name: "run and build with builder invalid",
desc: "Should run and build when build is specifically requested with builder invalid",
funcState: `name: test-func
runtime: go
created: 2023-03-12 15:00:00`,
name: "run and build with builder invalid",
desc: "Should run and build when build is specifically requested with builder invalid",
args: []string{"--build=true", "--builder=invalid"},
buildError: fmt.Errorf("\"invalid\" is not a known builder. Available builders are \"pack\" and \"s2i\""),
buildInvoked: true,
runInvoked: true,
},
{
name: "run without build when disabled",
desc: "Should run but not build when build is expressly disabled",
funcState: `name: test-func
runtime: go
created: 2009-11-10 23:00:00`,
name: "run without build when disabled",
desc: "Should run but not build when build is expressly disabled",
args: []string{"--build=false"}, // can be any truthy value: 0, 'false' etc.
buildInvoked: false,
runInvoked: true,
},
{
name: "run and build on auto",
desc: "Should run and buil when build flag set to auto",
funcState: `name: test-func
runtime: go
created: 2009-11-10 23:00:00`,
name: "run and build on auto",
desc: "Should run and buil when build flag set to auto",
args: []string{"--build=auto"}, // can be any truthy value: 0, 'false' etc.
buildInvoked: true,
runInvoked: true,
@ -107,20 +82,17 @@ created: 2009-11-10 23:00:00`,
desc: "Should build when image tag exists",
// The existence of an image tag value does not mean the function
// is built; that is the purvew of the buld stamp staleness check.
funcState: `name: test-func
runtime: go
image: exampleimage
created: 2009-11-10 23:00:00`,
setup: func(f fn.Function, t *testing.T) error {
f.Image = "exampleimage"
return f.Write()
},
args: []string{},
buildInvoked: true,
runInvoked: true,
},
{
name: "Build errors return",
desc: "Errors building cause an immediate return with error",
funcState: `name: test-func
runtime: go
created: 2009-11-10 23:00:00`,
name: "Build errors return",
desc: "Errors building cause an immediate return with error",
args: []string{},
buildError: fmt.Errorf("generic build error"),
buildInvoked: true,
@ -128,9 +100,8 @@ created: 2009-11-10 23:00:00`,
},
}
for _, tt := range tests {
// run as a sub-test
t.Run(tt.name, func(t *testing.T) {
_ = fromTempDirectory(t)
root := fromTempDirectory(t)
runner := mock.NewRunner()
if tt.runError != nil {
@ -152,10 +123,16 @@ created: 2009-11-10 23:00:00`,
))
cmd.SetArgs(tt.args) // Do not use test command args
// set test case's func.yaml
if err := os.WriteFile("func.yaml", []byte(tt.funcState), os.ModePerm); err != nil {
// set test case's function instance
f, err := fn.New().Init(fn.Function{Root: root, Runtime: "go"})
if err != nil {
t.Fatal(err)
}
if tt.setup != nil {
if err := tt.setup(f, t); err != nil {
t.Fatal(err)
}
}
ctx, cancel := context.WithCancel(context.Background())
runErrCh := make(chan error, 1)

27
pkg/functions/client.go
View File

@ -557,14 +557,14 @@ func (c *Client) Init(cfg Function) (Function, error) {
// Write out the new function's Template files.
// Templates contain values which may result in the function being mutated
// (default builders, etc), so a new (potentially mutated) function is
// returned from Templates.Write
// (default builders, etc)
err = c.Templates().Write(&f)
if err != nil {
return f, err
}
// Mark the function as having been created
// Mark the function as having been created, and that it is not to be
// considered built.
f.Created = time.Now()
err = f.Write()
if err != nil {
@ -607,8 +607,7 @@ func (c *Client) Build(ctx context.Context, f Function) (Function, error) {
return f, err
}
f, err = f.updateBuildStamp()
if err != nil {
if err = f.Stamp(); err != nil {
return f, err
}
@ -660,9 +659,10 @@ func WithDeploySkipBuildCheck(skipBuiltCheck bool) DeployOption {
}
}
// Deploy the function at path. Errors if the function has not been built.
// Deploy the function at path.
// Errors if the function has not been built unless explicitly instructed
// to ignore this build check.
func (c *Client) Deploy(ctx context.Context, f Function, opts ...DeployOption) (Function, error) {
deployParams := &DeployParams{skipBuiltCheck: false}
for _, opt := range opts {
opt(deployParams)
@ -703,9 +703,7 @@ func (c *Client) Deploy(ctx context.Context, f Function, opts ...DeployOption) (
c.progressListener.Increment(fmt.Sprintf("✅ Function updated in namespace %q and exposed at URL: \n %v", result.Namespace, result.URL))
}
// Metadata generated from deploying (namespace) should not trigger a rebuild
// through a staleness check, so update the build stamp we checked earlier.
return f.updateBuildStamp()
return f, nil
}
// RunPipeline runs a Pipeline to build and deploy the function.
@ -760,7 +758,10 @@ func (c *Client) ConfigurePAC(ctx context.Context, f Function, metadata any) err
}
}
// saves image name/registry to function's metadata (func.yaml)
// saves image name/registry to function's metadata (func.yaml), and
// does not explicitly update the last created build stamp
// (i.e. changes to the function during ConfigurePAC should not cause the
// next deploy to skip building)
if err = f.Write(); err != nil {
return err
}
@ -956,9 +957,7 @@ func (c *Client) Push(ctx context.Context, f Function) (Function, error) {
return f, err
}
// Metadata generated from pushing (ImageDigest) should not trigger a rebuild
// through a staleness check, so update the build stamp we checked earlier.
return f.updateBuildStamp()
return f, nil
}
// DEFAULTS

221
pkg/functions/function.go
View File

@ -1,6 +1,7 @@
package functions
import (
"bytes"
"crypto/sha256"
"errors"
"fmt"
@ -25,11 +26,6 @@ const (
// By default it is excluded from source control.
RunDataDir = ".func"
// buildstamp is the name of the file within the run data directory whose
// existence indicates the function has been built, and whose content is
// a fingerprint of the filesystem at the time of the build.
buildstamp = "built"
// DefaultPersistentVolumeClaimSize represents default size of PVC created for a Pipeline
DefaultPersistentVolumeClaimSize string = "256Mi"
)
@ -88,12 +84,6 @@ type Function struct {
//DeploySpec define the deployment properties for a function
Deploy DeploySpec `yaml:"deploy,omitempty"`
// current (last) build stamp for the function if it can be found.
BuildStamp string `yaml:"-"`
// flags that buildstamp needs to be saved to disk
NeedsWriteBuildStamp bool `yaml:"-"`
}
// BuildSpec
@ -196,21 +186,21 @@ func NewFunctionWith(defaults Function) Function {
// Migrations are run prior to validators such that validation can omit
// concerning itself with backwards compatibility. Migrators must therefore
// selectively consider the minimal validation necessary to enable migration.
func NewFunction(path string) (f Function, err error) {
func NewFunction(root string) (f Function, err error) {
f.Build.BuilderImages = make(map[string]string)
f.Deploy.Annotations = make(map[string]string)
// Path defaults to current working directory, and if provided explicitly
// Path must exist and be a directory
if path == "" {
if path, err = os.Getwd(); err != nil {
if root == "" {
if root, err = os.Getwd(); err != nil {
return
}
}
f.Root = path // path is not persisted, as this is the purview of the FS
f.Root = root // path is not persisted, as this is the purview of the FS
// Path must exist and be a directory
fd, err := os.Stat(path)
fd, err := os.Stat(root)
if err != nil {
return f, err
}
@ -220,7 +210,7 @@ func NewFunction(path string) (f Function, err error) {
// If no func.yaml in directory, return the default function which will
// have f.Initialized() == false
var filename = filepath.Join(path, FunctionFile)
var filename = filepath.Join(root, FunctionFile)
if _, err = os.Stat(filename); err != nil {
if os.IsNotExist(err) {
err = nil
@ -251,7 +241,6 @@ func NewFunction(path string) (f Function, err error) {
errorText += "\n" + "Migration: " + functionMigrationError.Error()
return Function{}, errors.New(errorText)
}
f.BuildStamp = f.buildStamp()
return
}
@ -360,10 +349,7 @@ func nameFromPath(path string) string {
*/
}
// Write aka (save, serialize, marshal) the function to disk at its path.
// Only valid functions can be written.
// In order to retain built status (staleness checks), the file is only
// modified if the structure actually changes.
// Write Function struct (metadata) to Disk at f.Root
func (f Function) Write() (err error) {
// Skip writing (and dirtying the work tree) if there were no modifications.
f1, _ := NewFunction(f.Root)
@ -371,27 +357,48 @@ func (f Function) Write() (err error) {
return
}
// Do not write invalid functions
if err = f.Validate(); err != nil {
return
}
path := filepath.Join(f.Root, FunctionFile)
// Write
var bb []byte
if bb, err = yaml.Marshal(&f); err != nil {
return
}
// TODO: open existing file for writing, such that existing permissions
// are preserved.
if err = os.WriteFile(path, bb, 0644); err != nil {
// are preserved?
return os.WriteFile(filepath.Join(f.Root, FunctionFile), bb, 0644)
}
// Stamp a function as being built.
//
// This is a performance optimization used when updates to the
// function are known to have no effect on its built container. This
// stamp is checked before certain operations, and if it has been updated,
// the build can be skuipped. If in doubt, just use .Write only.
//
// Updates the build stamp at ./func/built (and the log
// at .func/built.log) to reflect the current state of the filesystem.
// Note that the caller should call .Write first to flush any changes to the
// function in-memory to the filesystem prior to calling stamp.
//
// The runtime data directory .func is created in the function root if
// necessary.
func (f Function) Stamp() (err error) {
if err = f.ensureRuntimeDir(); err != nil {
return
}
if f.NeedsWriteBuildStamp {
err = f.writeBuildStamp()
if err != nil {
return
}
f.NeedsWriteBuildStamp = false
var hash, log string
if hash, log, err = fingerprint(f.Root); err != nil {
return
}
return
if err = os.WriteFile(filepath.Join(f.Root, RunDataDir, "built"), []byte(hash), os.ModePerm); err != nil {
return
}
return os.WriteFile(filepath.Join(f.Root, RunDataDir, "built.log"), []byte(log), os.ModePerm)
}
// Initialized returns if the function has been initialized.
@ -654,87 +661,15 @@ func (f Function) ensureRuntimeDir() error {
/.func
`
return os.WriteFile(filepath.Join(f.Root, ".gitignore"), []byte(gitignore), 0644)
}
// Tag the function in memory as having been built
// This is locally-scoped data, only indicating there presumably exists
// a container image in the cache of the the configured builder, thus this info
// is placed in a .func (non-source controlled) local metadata directory, which
// is not stritly required to exist, so it is created if needed.
func (f Function) updateBuildStamp() (Function, error) {
hash, err := f.fingerprint()
if err != nil {
return f, err
}
f.BuildStamp = hash
f.NeedsWriteBuildStamp = true
return f, err
}
// Tag the function on disk as having been built
// This is locally-scoped data, only indicating there presumably exists
// a container image in the cache of the the configured builder, thus this info
// is placed in a .func (non-source controlled) local metadata directory, which
// is not stritly required to exist, so it is created if needed.
func (f Function) writeBuildStamp() (err error) {
if err = f.ensureRuntimeDir(); err != nil {
return err
}
hash, err := f.fingerprint()
if err != nil {
return err
}
if err = os.WriteFile(filepath.Join(f.Root, RunDataDir, buildstamp), []byte(hash), os.ModePerm); err != nil {
return err
}
return
}
// fingerprint returns a hash of the filenames and modification timestamps of
// the files within a function's root.
func (f Function) fingerprint() (string, error) {
h := sha256.New()
err := filepath.Walk(f.Root, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
// Always ignore .func, .git (TODO: .funcignore)
if info.IsDir() && (info.Name() == RunDataDir || info.Name() == ".git") {
return filepath.SkipDir
}
fmt.Fprintf(h, "%v:%v:", path, info.ModTime().UnixNano())
return nil
})
return fmt.Sprintf("%x", h.Sum(nil)), err
}
// buildStamp returns the current (last) build stamp for the function
// at the given path, if it can be found.
func (f Function) buildStamp() string {
buildstampPath := filepath.Join(f.Root, RunDataDir, buildstamp)
if _, err := os.Stat(buildstampPath); err != nil {
return ""
}
b, err := os.ReadFile(buildstampPath)
if err != nil {
return ""
}
return string(b)
}
// Built returns true if the given path contains a function which has been
// built without any filesystem modifications since (is not stale).
// FIXME: This is very specifically to be used for the logic of determining if
// a function as it exists on disk has been built, which is why it was
// originally a package-static function `fn.Built(path string)`. By moving
// it to the Function struct, it must also be modified to return false if the
// serialization of the in-memory struct differs from the function on disk.
// Built returns true if the function is considered built.
// Note that this only considers the function as it exists on-disk at
// f.Root.
func (f Function) Built() bool {
// If there is no build stamp, it is not built.
// This case should be redundant with the below check for an image, but is
// temporarily necessary (see the long-winded caviat note below).
if f.BuildStamp == "" {
stamp := f.BuildStamp()
if stamp == "" {
return false
}
@ -753,21 +688,63 @@ func (f Function) Built() bool {
return false
}
// Calculate the function's Filesystem hash and see if it has changed.
hash, err := f.fingerprint()
// Calculate the current filesystem hash and see if it has changed.
//
// If this comparison returns true, the Function has a populated image,
// existing buildstamp, and the calculated fingerprint has not changed.
//
// It's a pretty good chance the thing doesn't need to be rebuilt, though
// of course filesystem racing conditions do exist, including both direct
// source code modifications or changes to the image cache.
hash, _, err := fingerprint(f.Root)
if err != nil {
fmt.Fprintf(os.Stderr, "error calculating function's fingerprint: %v\n", err)
return false
}
if f.BuildStamp != hash {
return false
}
// Function has a populated image, existing buildstamp, and the calculated
// fingerprint has not changed.
// It's a pretty good chance the thing doesn't need to be rebuilt, though
// of course filesystem racing conditions do exist, including both direct
// source code modifications or changes to the image cache.
return true
return stamp == hash
}
// BuildStamp accesses the current (last) build stamp for the function.
// Unbuilt functions return empty string.
func (f Function) BuildStamp() string {
path := filepath.Join(f.Root, RunDataDir, "built")
if _, err := os.Stat(path); err != nil {
return ""
}
b, err := os.ReadFile(path)
if err != nil {
return ""
}
return string(b)
}
// fingerprint the files at a given path. Returns a hash calculated from the
// filenames and modification timestamps of the files within the given root.
// Also returns a logfile consiting of the filenames and modification times
// which contributed to the hash.
// Intended to determine if there were appreciable changes to a function's
// source code, certain directories and files are ignored, such as
// .git and .func.
// Future updates will include files explicitly marked as ignored by a
// .funcignore.
func fingerprint(root string) (hash, log string, err error) {
h := sha256.New() // Hash builder
l := bytes.Buffer{} // Log buffer
err = filepath.Walk(root, func(path string, info fs.FileInfo, err error) error {
if err != nil {
return err
}
if path == root {
return nil
}
// Always ignore .func, .git (TODO: .funcignore)
if info.IsDir() && (info.Name() == RunDataDir || info.Name() == ".git") {
return filepath.SkipDir
}
fmt.Fprintf(h, "%v:%v:", path, info.ModTime().UnixNano()) // Write to the Hasher
fmt.Fprintf(&l, "%v:%v\n", path, info.ModTime().UnixNano()) // Write to the Log
return nil
})
return fmt.Sprintf("%x", h.Sum(nil)), l.String(), err
}