diff --git a/client.go b/client.go index d94d96d92..0ebcd817a 100644 --- a/client.go +++ b/client.go @@ -339,7 +339,7 @@ func (c *Client) Create(cfg Function) (err error) { } // Write out a template. - w := templateWriter{templates: c.templates, verbose: c.verbose} + w := templateWriter{repositories: c.templates, verbose: c.verbose} if err = w.Write(f.Runtime, f.Trigger, f.Root); err != nil { return } diff --git a/client_test.go b/client_test.go index 066875e9a..cbe1543f4 100644 --- a/client_test.go +++ b/client_test.go @@ -25,7 +25,7 @@ const TestRegistry = "quay.io/alice" // by the client API for those who prefer manual transmissions. func TestNew(t *testing.T) { root := "testdata/example.com/testCreate" // Root from which to run the test - if err := os.MkdirAll(root, 0700); err != nil { + if err := os.MkdirAll(root, 0744); err != nil { t.Fatal(err) } defer os.RemoveAll(root) @@ -179,11 +179,11 @@ func TestExtensibleTemplates(t *testing.T) { // Create a new client with a path to the extensible templates client := bosonFunc.New( - bosonFunc.WithTemplates("testdata/templates"), + bosonFunc.WithTemplates("testdata/repositories"), bosonFunc.WithRegistry(TestRegistry)) // Create a Function specifying a template, 'json' that only exists in the extensible set - if err := client.New(context.Background(), bosonFunc.Function{Root: root, Trigger: "boson-experimental/json"}); err != nil { + if err := client.New(context.Background(), bosonFunc.Function{Root: root, Trigger: "customProvider/json"}); err != nil { t.Fatal(err) } diff --git a/templates.go b/templates.go index a549c8141..6b64c4aad 100644 --- a/templates.go +++ b/templates.go @@ -1,181 +1,192 @@ package function -// Updating Templates: -// See documentation in ./templates/README.md -// go get github.com/markbates/pkger -//go:generate pkger - import ( + "embed" "errors" "fmt" "io" + "io/fs" "os" "path/filepath" "sort" "strings" - - "github.com/markbates/pkger" ) +// Embed all files in ./templates. +//go:embed templates +var embedded embed.FS + // DefautlTemplate is the default Function signature / environmental context // of the resultant template. All runtimes are expected to have at least // an HTTP Handler ("http") and Cloud Events ("events") const DefaultTemplate = "http" -// fileAccessor encapsulates methods for accessing template files. -type fileAccessor interface { - Stat(name string) (os.FileInfo, error) - Open(p string) (file, error) -} +// DefaultTemplateFileMode for embedded files which have lost their mode due to being +// retained in a read-only filesystem (forced 0444 on embed). +const DefaultTemplateFileMode = 0644 -type file interface { - Readdir(int) ([]os.FileInfo, error) - Read([]byte) (int, error) - Close() error -} - -// When pkger is run, code analysis detects this Include statement, -// triggering the serializaation of the templates directory and all -// its contents into pkged.go, which is then made available via -// a pkger fileAccessor. -// Path is relative to the go module root. -func init() { - _ = pkger.Include("/templates") -} +// DefaultTemplateDirMode for embedded files which have lost their mode due to being +// retained in a read-only filesystem (forced 0444 on embed). +const DefaultTemplateDirMode = 0744 type templateWriter struct { - verbose bool - templates string + // Extensible Template Repositories + // templates on disk (extensible templates) + // Stored on disk at path: + // [customTemplatesPath]/[repository]/[runtime]/[template] + // For example + // ~/.config/func/boson/go/http" + // Specified when writing templates as simply: + // Write([runtime], [repository], [path]) + // For example + // w := templateWriter{templates:"/home/username/.config/func/templates") + // w.Write("go", "boson/http") + // Ie. "Using the custom templates in the func configuration directory, + // write the Boson HTTP template for the Go runtime." + repositories string + defaultModes bool + verbose bool } -func (n templateWriter) Write(runtime, template string, dest string) error { +var ( + ErrRepositoryNotFound = errors.New("repository not found") + ErrRepositoriesNotDefined = errors.New("custom template repositories location not specified") + ErrTemplateMissingRepository = errors.New("template name missing repository prefix") +) + +// Write the template for the given runtime to the destination specified. +// Template may be prefixed with a custom repo name. +func (t *templateWriter) Write(runtime, template, dest string) error { if template == "" { template = DefaultTemplate } - // TODO: Confirm the dest path is empty? This is currently in an earlier - // step of the create process but future calls directly to initialize would - // be better off being made safe. + if isCustom(template) { + return t.writeCustom(t.repositories, runtime, template, dest) + } - if isEmbedded(runtime, template) { - return copyEmbedded(runtime, template, dest) - } - if n.templates != "" { - return copyFilesystem(n.templates, runtime, template, dest) - } - return fmt.Errorf("A template for runtime '%v' template '%v' was not found internally and no custom template path was defined.", runtime, template) + t.defaultModes = true // overwrite read-only mode on write to defaults. + return t.writeEmbedded(runtime, template, dest) } -func copyEmbedded(runtime, template, dest string) error { - // Copy files to the destination - // Example embedded path: - // /templates/go/http - src := filepath.Join("/templates", runtime, template) - return copy(src, dest, embeddedAccessor{}) +func isCustom(template string) bool { + return len(strings.Split(template, "/")) > 1 } -func copyFilesystem(templatesPath, runtime, templateFullName, dest string) error { - // ensure that the templateFullName is of the format "repoName/templateName" - cc := strings.Split(templateFullName, "/") - if len(cc) != 2 { - return errors.New("Template name must be in the format 'REPO/NAME'") +func (t *templateWriter) writeCustom(repositories, runtime, template, dest string) error { + if repositories == "" { + return ErrRepositoriesNotDefined } - repo := cc[0] - template := cc[1] - - // Example FileSystem path: - // /home/alice/.config/func/templates/boson-experimental/go/json - src := filepath.Join(templatesPath, repo, runtime, template) - return copy(src, dest, filesystemAccessor{}) + if !repositoryExists(repositories, template) { + return ErrRepositoryNotFound + } + cc := strings.Split(template, "/") + if len(cc) < 2 { + return ErrTemplateMissingRepository + } + // ex: /home/alice/.config/func/repositories/boson/go/http + src := filepath.Join(cc[0], runtime, cc[1]) + return t.cp(src, dest, os.DirFS(repositories)) } -func isEmbedded(runtime, template string) bool { - _, err := pkger.Stat(filepath.Join("/templates", runtime, template)) +func (t *templateWriter) writeEmbedded(runtime, template, dest string) error { + _, err := fs.Stat(embedded, filepath.Join("templates", runtime, template)) + if err != nil { + return err + } + src := filepath.Join("templates", runtime, template) + return t.cp(src, dest, embedded) +} + +func repositoryExists(repositories, template string) bool { + cc := strings.Split(template, "/") + _, err := fs.Stat(os.DirFS(repositories), cc[0]) return err == nil } -type embeddedAccessor struct{} - -func (a embeddedAccessor) Stat(path string) (os.FileInfo, error) { - return pkger.Stat(path) -} - -func (a embeddedAccessor) Open(path string) (file, error) { - return pkger.Open(path) -} - -type filesystemAccessor struct{} - -func (a filesystemAccessor) Stat(path string) (os.FileInfo, error) { - return os.Stat(path) -} - -func (a filesystemAccessor) Open(path string) (file, error) { - return os.Open(path) -} - -func copy(src, dest string, accessor fileAccessor) (err error) { - node, err := accessor.Stat(src) +func (t *templateWriter) cp(src, dest string, files fs.FS) error { + node, err := fs.Stat(files, src) if err != nil { - return + return err } if node.IsDir() { - return copyNode(src, dest, accessor) + return t.copyNode(src, dest, files) } else { - return copyLeaf(src, dest, accessor) + return t.copyLeaf(src, dest, files) } } -func copyNode(src, dest string, accessor fileAccessor) (err error) { - node, err := accessor.Stat(src) +func (t *templateWriter) copyNode(src, dest string, files fs.FS) error { + node, err := fs.Stat(files, src) if err != nil { - return + return err } - err = os.MkdirAll(dest, node.Mode()) - if err != nil { - return + mode := node.Mode() + if t.defaultModes { + mode = DefaultTemplateDirMode } - children, err := readDir(src, accessor) + err = os.MkdirAll(dest, mode) if err != nil { - return + return err + } + + children, err := readDir(src, files) + if err != nil { + return err } for _, child := range children { - if err = copy(filepath.Join(src, child.Name()), filepath.Join(dest, child.Name()), accessor); err != nil { - return + if err = t.cp(filepath.Join(src, child.Name()), filepath.Join(dest, child.Name()), files); err != nil { + return err } } - return + return nil } -func readDir(src string, accessor fileAccessor) ([]os.FileInfo, error) { - f, err := accessor.Open(src) +func readDir(src string, files fs.FS) ([]fs.DirEntry, error) { + f, err := files.Open(src) if err != nil { return nil, err } - list, err := f.Readdir(-1) - f.Close() + defer f.Close() + + fi, err := f.Stat() if err != nil { return nil, err } + + if !fi.IsDir() { + return nil, errors.New(fmt.Sprintf("%v must be a directory", fi.Name())) + } + list, err := f.(fs.ReadDirFile).ReadDir(-1) + if err != nil { + return nil, err + } + sort.Slice(list, func(i, j int) bool { return list[i].Name() < list[j].Name() }) return list, nil } -func copyLeaf(src, dest string, accessor fileAccessor) (err error) { - srcFile, err := accessor.Open(src) +func (t *templateWriter) copyLeaf(src, dest string, files fs.FS) (err error) { + srcFile, err := files.Open(src) if err != nil { return } defer srcFile.Close() - srcFileInfo, err := accessor.Stat(src) + srcFileInfo, err := fs.Stat(files, src) if err != nil { return } - destFile, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, srcFileInfo.Mode()) + // Use the original's mode unless a nonzero mode was explicitly provided. + mode := srcFileInfo.Mode() + if t.defaultModes { + mode = DefaultTemplateFileMode + } + + destFile, err := os.OpenFile(dest, os.O_RDWR|os.O_CREATE|os.O_TRUNC, mode) if err != nil { return } diff --git a/templates/README.md b/templates/README.md index 2fe89a01e..64970567d 100644 --- a/templates/README.md +++ b/templates/README.md @@ -1,24 +1,4 @@ # Templates -## Packaging - -When updates are made to these templates, they must be packaged (serialized as -a Go struture) by running `make`, and checking in the resultant `pkged.go` file. - -## How it works - -running `make` in turn installs the `pkger` binary, which can be installed via: -`go get github.com/markbates/pkger/cmd/pkger` -Make then invokes `pkger` before `go build`. - -The resulting `pkged.go` file includes the contents of the templates directory, -encoded as a Go strucutres which is then makde available in code using an API -similar to the standard library's `os` package. - -## Rationale - -Until such time as embedding static assets in binaries is included in the -base `go build` functionality (see https://github.com/golang/go/issues/35950) -a third-party tool is required and pkger provides an API very similar -to the `os` package. - +This templates directory is embedded entirely into any resultant binaries +using the go:embed directive introduced in Go 1.16 diff --git a/templates/test/http/rtAtplDefault.txt b/templates/test/http/rtAtplDefault.txt new file mode 100644 index 000000000..095ad2b0e --- /dev/null +++ b/templates/test/http/rtAtplDefault.txt @@ -0,0 +1,2 @@ +Runtime A +Template HTTP (Default) diff --git a/templates/test/tpla/rtAtplA.txt b/templates/test/tpla/rtAtplA.txt new file mode 100644 index 000000000..02afcc199 --- /dev/null +++ b/templates/test/tpla/rtAtplA.txt @@ -0,0 +1,2 @@ +Runtime A +Template A diff --git a/templates/test/tplb/rtAtplB.txt b/templates/test/tplb/rtAtplB.txt new file mode 100644 index 000000000..d3179e872 --- /dev/null +++ b/templates/test/tplb/rtAtplB.txt @@ -0,0 +1,2 @@ +Runtime A +Template B diff --git a/templates_test.go b/templates_test.go index 668cf71cb..6fe1632ad 100644 --- a/templates_test.go +++ b/templates_test.go @@ -9,69 +9,145 @@ import ( "testing" ) -// TestTemplatesEmbeddedFileMode ensures that files from the embedded templates are -// written with the same mode from whence they came -func TestTemplatesEmbeddedFileMode(t *testing.T) { - var path = "testdata/example.com/www" - err := os.MkdirAll(path, 0744) - if err != nil { - panic(err) - } - defer os.RemoveAll(path) +const TestRuntime = "test" - client := New() - function := Function{Root: path, Runtime: "quarkus", Trigger: "events"} - if err := client.Create(function); err != nil { - t.Fatal(err) - } +// TestWriteEmbedded ensures that embedded templates are copied. +func TestWriteEmbedded(t *testing.T) { + // create test directory + root := "testdata/testWriteEmbedded" + defer using(t, root)() - // The file mode of the embedded mvnw should be -rwxr-xr-x - // See source file at: templates/quarkus/events/mvnw - // Assert mode is preserved - sourceMode := os.FileMode(0755) - dest, err := os.Stat(filepath.Join(path, "mvnw")) + // write out a template + w := templateWriter{} + err := w.Write(TestRuntime, "tpla", root) if err != nil { t.Fatal(err) } - if runtime.GOOS != "windows" { - if dest.Mode() != sourceMode { - t.Fatalf("The dest mode should be %v but was %v", sourceMode, dest.Mode()) - } + + // Assert file exists as expected + _, err = os.Stat(filepath.Join(root, "rtAtplA.txt")) + if err != nil { + t.Fatal(err) } } -// TestTemplatesExtensibleFileMode ensures that files from a file-system -// derived template is written with mode retained. -func TestTemplatesExtensibleFileMode(t *testing.T) { - var ( - path = "testdata/example.com/www" - template = "boson-experimental/http" - templates = "testdata/templates" - ) - err := os.MkdirAll(path, 0744) - if err != nil { - panic(err) - } - defer os.RemoveAll(path) +// TestWriteCustom ensures that a template from a filesystem source (ie. custom +// provider on disk) can be specified as the source for a template. +func TestWriteCustom(t *testing.T) { + // Create test directory + root := "testdata/testWriteFilesystem" + defer using(t, root)() - client := New(WithTemplates(templates)) - function := Function{Root: path, Runtime: "quarkus", Trigger: template} - if err := client.Create(function); err != nil { + // Writer which includes custom repositories + w := templateWriter{repositories: "testdata/repositories"} + err := w.Write(TestRuntime, "customProvider/tpla", root) + if err != nil { t.Fatal(err) } - // Assert mode is preserved - source, err := os.Stat(filepath.Join("testdata/templates/boson-experimental/quarkus/http/mvnw")) + // Assert file exists as expected + _, err = os.Stat(filepath.Join(root, "customtpl.txt")) if err != nil { t.Fatal(err) } - dest, err := os.Stat(filepath.Join(path, "mvnw")) - if err != nil { - t.Fatal(err) - } - if runtime.GOOS != "windows" { - if dest.Mode() != source.Mode() { - t.Fatalf("The dest mode should be %v but was %v", source.Mode(), dest.Mode()) - } - } +} + +// TestWriteDefault ensures that the default template is used when not specified. +func TestWriteDefault(t *testing.T) { + // create test directory + root := "testdata/testWriteDefault" + defer using(t, root)() + + // write out a template + w := templateWriter{} + err := w.Write(TestRuntime, "", root) + if err != nil { + t.Fatal(err) + } + + // Assert file exists as expected + _, err = os.Stat(filepath.Join(root, "rtAtplDefault.txt")) + if err != nil { + t.Fatal(err) + } +} + +// TestWriteModeDefault ensires that files written to disk are set to writible by the owner by default. Since the embedded filesystem is expressly read-only, copying files to disk results in likewise read-only files. Templates are expected to be mutable, so by default set mode to 0644. +func TestWriteModeDefault(t *testing.T) { + root := "testdata/testWriteModeDefault" + defer using(t, root)() + + w := templateWriter{} + err := w.Write(TestRuntime, "tpla", root) + if err != nil { + t.Fatal(err) + } + + // Verify file mode was defaulted + f, err := os.Stat(filepath.Join(root, "rtAtplA.txt")) + if err != nil { + t.Fatal(err) + } + if f.Mode() != os.FileMode(0644) { + t.Fatalf("The custom file's mode should be 0644 but was %#o", f.Mode()) + } + +} + +// TestWriteCustomMode ensures that templates written from custom templates +// retain their mode. Note that the embed system is expressly for read-only +// filesystems, so the mode is always read-only +// https://golang.org/src/embed/embed.go?s=5559:6576#L235 +func TestWriteModeCustom(t *testing.T) { + if runtime.GOOS == "windows" { + return // not applicable + } + + // test directories + var err error + root := "testdata/testWriteModeCustom" + defer using(t, root)() + + // Write executable from custom repo + w := templateWriter{repositories: "testdata/repositories"} + err = w.Write(TestRuntime, "customProvider/tplb", root) + if err != nil { + t.Fatal(err) + } + + // Verify custom file mode was preserved. + customFile, err := os.Stat(filepath.Join(root, "custom-executable.sh")) + if err != nil { + t.Fatal(err) + } + if customFile.Mode() != os.FileMode(0755) { + t.Fatalf("The custom file's mode should be 0755 but was %v", customFile.Mode()) + } +} + +// Helpers +// ------- + +// using the given directory (creating it) returns a closure which removes the +// directory, intended to be run in a defer statement. +func using(t *testing.T, root string) func() { + t.Helper() + mkdir(t, root) + return func() { + rm(t, root) + } +} + +func mkdir(t *testing.T, dir string) { + t.Helper() + if err := os.MkdirAll(dir, 0744); err != nil { + t.Fatal(err) + } +} + +func rm(t *testing.T, dir string) { + t.Helper() + if err := os.RemoveAll(dir); err != nil { + t.Fatal(err) + } } diff --git a/testdata/templates/boson-experimental/node/json/index.js b/testdata/repositories/customProvider/node/json/index.js similarity index 100% rename from testdata/templates/boson-experimental/node/json/index.js rename to testdata/repositories/customProvider/node/json/index.js diff --git a/testdata/templates/boson-experimental/node/json/json.js b/testdata/repositories/customProvider/node/json/json.js similarity index 100% rename from testdata/templates/boson-experimental/node/json/json.js rename to testdata/repositories/customProvider/node/json/json.js diff --git a/testdata/templates/boson-experimental/quarkus/http/mvnw b/testdata/repositories/customProvider/quarkus/http/mvnw similarity index 100% rename from testdata/templates/boson-experimental/quarkus/http/mvnw rename to testdata/repositories/customProvider/quarkus/http/mvnw diff --git a/testdata/repositories/customProvider/test/tpla/customtpl.txt b/testdata/repositories/customProvider/test/tpla/customtpl.txt new file mode 100644 index 000000000..fd0805a37 --- /dev/null +++ b/testdata/repositories/customProvider/test/tpla/customtpl.txt @@ -0,0 +1,3 @@ +Custom Provider +Runtime A +Template A diff --git a/testdata/repositories/customProvider/test/tplb/custom-executable.sh b/testdata/repositories/customProvider/test/tplb/custom-executable.sh new file mode 100755 index 000000000..236816ecb --- /dev/null +++ b/testdata/repositories/customProvider/test/tplb/custom-executable.sh @@ -0,0 +1,2 @@ +#!/usr/bin/env bash +echo "example executble"