From 9a790f005f0a8c846dc7e6491d98cb1f114a8fd6 Mon Sep 17 00:00:00 2001 From: Luke Kingland Date: Wed, 17 May 2023 19:53:11 +0900 Subject: [PATCH] feat: host oci builder (#1730) * feat: oci builder for host builds * do not expose host builder until fully baked --- cmd/build.go | 24 +-- pkg/functions/client.go | 7 +- pkg/functions/function.go | 4 +- pkg/oci/builder.go | 276 ++++++++++++++++++++++++++ pkg/oci/builder_test.go | 95 +++++++++ pkg/oci/containerize.go | 388 +++++++++++++++++++++++++++++++++++++ pkg/oci/containerize_go.go | 189 ++++++++++++++++++ 7 files changed, 963 insertions(+), 20 deletions(-) create mode 100644 pkg/oci/builder.go create mode 100644 pkg/oci/builder_test.go create mode 100644 pkg/oci/containerize.go create mode 100644 pkg/oci/containerize_go.go diff --git a/cmd/build.go b/cmd/build.go index f4cd2471..03c2a613 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -10,7 +10,7 @@ import ( "github.com/spf13/cobra" "knative.dev/func/pkg/builders" - "knative.dev/func/pkg/builders/buildpacks" + pack "knative.dev/func/pkg/builders/buildpacks" "knative.dev/func/pkg/builders/s2i" "knative.dev/func/pkg/config" fn "knative.dev/func/pkg/functions" @@ -169,25 +169,21 @@ func runBuild(cmd *cobra.Command, _ []string, newClient ClientFactory) (err erro // Client // Concrete implementations (ex builder) vary based on final effective config - var builder fn.Builder + var client *fn.Client + o := []fn.Option{fn.WithRegistry(cfg.Registry)} if f.Build.Builder == builders.Pack { - builder = buildpacks.NewBuilder( - buildpacks.WithName(builders.Pack), - buildpacks.WithVerbose(cfg.Verbose), - buildpacks.WithTimestamp(cfg.WithTimestamp), - ) + o = append(o, fn.WithBuilder(pack.NewBuilder( + pack.WithName(builders.Pack), + pack.WithTimestamp(cfg.WithTimestamp), + pack.WithVerbose(cfg.Verbose)))) } else if f.Build.Builder == builders.S2I { - builder = s2i.NewBuilder( + o = append(o, fn.WithBuilder(s2i.NewBuilder( s2i.WithName(builders.S2I), s2i.WithPlatform(cfg.Platform), - s2i.WithVerbose(cfg.Verbose)) - } else { - return builders.ErrUnknownBuilder{Name: f.Build.Builder, Known: KnownBuilders()} + s2i.WithVerbose(cfg.Verbose)))) } - client, done := newClient(ClientConfig{Verbose: cfg.Verbose}, - fn.WithRegistry(cfg.Registry), - fn.WithBuilder(builder)) + client, done := newClient(ClientConfig{Verbose: cfg.Verbose}, o...) defer done() // Build and (optionally) push diff --git a/pkg/functions/client.go b/pkg/functions/client.go index 15da55e3..48d97b0a 100644 --- a/pkg/functions/client.go +++ b/pkg/functions/client.go @@ -387,8 +387,7 @@ func (c *Client) Registry() string { func (c *Client) Runtimes() ([]string, error) { runtimes := utils.NewSortedSet() - // Gather all runtimes from all repositories - // into a uniqueness map + // Gather all runtimes from all repositories into a uniqueness map repositories, err := c.Repositories().All() if err != nil { return []string{}, err @@ -1026,7 +1025,7 @@ func ensureRunDataDir(root string) error { return nil } -// fingerprint the files at a given path. Returns a hash calculated from the +// Fingerprint the files at a given path. Returns a hash calculated from the // filenames and modification timestamps of the files within the given root. // Also returns a logfile consiting of the filenames and modification times // which contributed to the hash. @@ -1035,7 +1034,7 @@ func ensureRunDataDir(root string) error { // .git and .func. // Future updates will include files explicitly marked as ignored by a // .funcignore. -func fingerprint(root string) (hash, log string, err error) { +func Fingerprint(root string) (hash, log string, err error) { h := sha256.New() // Hash builder l := bytes.Buffer{} // Log buffer diff --git a/pkg/functions/function.go b/pkg/functions/function.go index b1438504..2ffa3b84 100644 --- a/pkg/functions/function.go +++ b/pkg/functions/function.go @@ -415,7 +415,7 @@ func (f Function) Stamp(oo ...stampOption) (err error) { // Cacluate the hash and a logfile of what comprised it var hash, log string - if hash, log, err = fingerprint(f.Root); err != nil { + if hash, log, err = Fingerprint(f.Root); err != nil { return } @@ -633,7 +633,7 @@ func (f Function) Built() bool { // It's a pretty good chance the thing doesn't need to be rebuilt, though // of course filesystem racing conditions do exist, including both direct // source code modifications or changes to the image cache. - hash, _, err := fingerprint(f.Root) + hash, _, err := Fingerprint(f.Root) if err != nil { fmt.Fprintf(os.Stderr, "error calculating function's fingerprint: %v\n", err) return false diff --git a/pkg/oci/builder.go b/pkg/oci/builder.go new file mode 100644 index 00000000..0a398341 --- /dev/null +++ b/pkg/oci/builder.go @@ -0,0 +1,276 @@ +package oci + +import ( + "context" + "fmt" + "os" + "path/filepath" + "strconv" + "syscall" + "time" + + v1 "github.com/google/go-containerregistry/pkg/v1" + + fn "knative.dev/func/pkg/functions" +) + +var path = filepath.Join + +// TODO: This may no longer be necessary, delete if e2e and acceptance tests +// succeed: +// const DefaultName = builders.Host + +var defaultPlatforms = []v1.Platform{ + {OS: "linux", Architecture: "amd64"}, + {OS: "linux", Architecture: "arm64"}, + // {OS: "linux", Architecture: "arm", Variant: "v6"}, + {OS: "linux", Architecture: "arm", Variant: "v7"}, + {OS: "darwin", Architecture: "amd64"}, + {OS: "darwin", Architecture: "arm64"}, +} + +var defaultIgnored = []string{ // TODO: implement and use .funcignore + ".git", + ".func", + ".funcignore", + ".gitignore", +} + +// BuildErr indicates a build error occurred. +type BuildErr struct { + Err error +} + +func (e BuildErr) Error() string { + return fmt.Sprintf("error performing host build. %v", e.Err) +} + +// Builder which creates an OCI-compliant multi-arch (index) container from +// the function at path. +type Builder struct { + name string + client *fn.Client + verbose bool +} + +// NewBuilder creates a builder instance. +func NewBuilder(name string, client *fn.Client, verbose bool) *Builder { + return &Builder{name, client, verbose} +} + +// Build an OCI-compliant Mult-arch (v1.ImageIndex) container on disk +// in the function's runtime data directory at: +// +// .func/builds/by-hash/$HASH +// +// Updates a symlink to this directory at: +// +// .func/builds/last +func (b *Builder) Build(ctx context.Context, f fn.Function) (err error) { + cfg := &buildConfig{ctx, b.client, f, time.Now(), b.verbose, ""} + + if err = setup(cfg); err != nil { // create directories and links + return + } + defer teardown(cfg) + + //TODO: Use clien't actual Scaffold when ready: + /* + if err = cfg.client.Scaffold(ctx, f, cfg.buildDir()); err != nil { + return + } + */ + // IN the meantime, use an airball mainfile + data := ` +package main + +import "fmt" + +func main () { + fmt.Println("Hello, world!") +} +` + if err = os.WriteFile(path(cfg.buildDir(), "main.go"), []byte(data), 0664); err != nil { + return + } + + if err = containerize(cfg); err != nil { + return + } + return updateLastLink(cfg) + + // TODO: communicating build completeness throgh returning without error + // relies on the implicit availability of the OIC image in this process' + // build directory. Would be better to have a formal build result object + // which includes a general struct which can be used by all builders to + // communicate to the pusher where the image can be found. +} + +// buildConfig contains various settings for a single build +type buildConfig struct { + ctx context.Context // build context + client *fn.Client // backreference to the client for Scaffolding + f fn.Function // Function being built + t time.Time // Timestamp for this build + verbose bool // verbose logging + h string // hash cache (use .hash() accessor) +} + +func (c *buildConfig) hash() string { + if c.h != "" { + return c.h + } + var err error + if c.h, _, err = fn.Fingerprint(c.f.Root); err != nil { + fmt.Fprintf(os.Stderr, "error calculating fingerprint for build. %v", err) + } + return c.h +} + +func (c *buildConfig) lastLink() string { + return path(c.f.Root, fn.RunDataDir, "builds", "last") +} +func (c *buildConfig) pidsDir() string { + return path(c.f.Root, fn.RunDataDir, "builds", "by-pid") +} +func (c *buildConfig) pidLink() string { + return path(c.f.Root, fn.RunDataDir, "builds", "by-pid", strconv.Itoa(os.Getpid())) +} +func (c *buildConfig) buildsDir() string { + return path(c.f.Root, fn.RunDataDir, "builds", "by-hash") +} +func (c *buildConfig) buildDir() string { + return path(c.f.Root, fn.RunDataDir, "builds", "by-hash", c.hash()) +} +func (c *buildConfig) ociDir() string { + return path(c.f.Root, fn.RunDataDir, "builds", "by-hash", c.hash(), "oci") +} +func (c *buildConfig) blobsDir() string { + return path(c.f.Root, fn.RunDataDir, "builds", "by-hash", c.hash(), "oci", "blobs", "sha256") +} + +// setup errors if there already exists a build directory. Otherwise, it +// creates a build directory based on the function's hash, and creates +// a link to this build directory for the current pid to denote the build +// is in progress. +func setup(cfg *buildConfig) (err error) { + // error if already in progress + if isActive(cfg, cfg.buildDir()) { + return BuildErr{fmt.Errorf("Build directory already exists for this version hash and is associated with an active PID. Is a build already in progress? %v", cfg.buildDir())} + } + + // create build files directory + if cfg.verbose { + fmt.Printf("mkdir -p %v\n", cfg.buildDir()) + } + if err = os.MkdirAll(cfg.buildDir(), 0774); err != nil { + return + } + + // create pid links directory + if _, err = os.Stat(cfg.pidsDir()); os.IsNotExist(err) { + if cfg.verbose { + fmt.Printf("mkdir -p %v\n", cfg.pidsDir()) + } + if err = os.MkdirAll(cfg.pidsDir(), 0774); err != nil { + return + } + } + + // create a link named $pid to the current build files directory + target := path("..", "by-hash", cfg.hash()) + if cfg.verbose { + fmt.Printf("ln -s %v %v\n", target, cfg.pidLink()) + } + return os.Symlink(target, cfg.pidLink()) +} + +func teardown(cfg *buildConfig) { + // remove the pid link for the current process indicating the build is + // no longer in progress. + _ = os.RemoveAll(cfg.pidLink()) + + // remove pid links for processes which no longer exist. + dd, _ := os.ReadDir(cfg.pidsDir()) + for _, d := range dd { + if processExists(d.Name()) { + continue + } + dir := path(cfg.pidsDir(), d.Name()) + if cfg.verbose { + fmt.Printf("rm %v\n", dir) + } + _ = os.RemoveAll(dir) + } + + // remove build file directories unless they are either: + // 1. The build files from the last successful build + // 2. Are associated with a pid link (currently in progress) + dd, _ = os.ReadDir(cfg.buildsDir()) + for _, d := range dd { + dir := path(cfg.buildsDir(), d.Name()) + if isLinkTo(cfg.lastLink(), dir) { + continue + } + if isActive(cfg, dir) { + continue + } + if cfg.verbose { + fmt.Printf("rm %v\n", dir) + } + _ = os.RemoveAll(dir) + } +} + +func processExists(pid string) bool { + p, err := strconv.Atoi(pid) + if err != nil { + return false + } + process, err := os.FindProcess(p) + if err != nil { + return false + } + err = process.Signal(syscall.Signal(0)) + return err == nil +} + +func isLinkTo(link, target string) bool { + var err error + if link, err = filepath.EvalSymlinks(link); err != nil { + return false + } + if link, err = filepath.Abs(link); err != nil { + return false + } + + if target, err = filepath.EvalSymlinks(target); err != nil { + return false + } + if target, err = filepath.Abs(target); err != nil { + return false + } + + return link == target +} + +// isActive returns whether or not the given directory path is for a build +// which is currently active or in progress. +func isActive(cfg *buildConfig, dir string) bool { + dd, _ := os.ReadDir(cfg.pidsDir()) + for _, d := range dd { + link := path(cfg.pidsDir(), d.Name()) + if processExists(d.Name()) && isLinkTo(link, dir) { + return true + } + } + return false +} + +func updateLastLink(cfg *buildConfig) error { + if cfg.verbose { + fmt.Printf("ln -s %v %v\n", cfg.buildDir(), cfg.lastLink()) + } + _ = os.RemoveAll(cfg.lastLink()) + return os.Symlink(cfg.buildDir(), cfg.lastLink()) +} diff --git a/pkg/oci/builder_test.go b/pkg/oci/builder_test.go new file mode 100644 index 00000000..ff3a5e29 --- /dev/null +++ b/pkg/oci/builder_test.go @@ -0,0 +1,95 @@ +package oci + +import ( + "context" + "encoding/json" + "os" + "path/filepath" + "testing" + + fn "knative.dev/func/pkg/functions" + . "knative.dev/func/pkg/testing" +) + +// TestBuilder ensures that, when given a Go Function, an OCI-compliant +// directory structure is created on .Build in the expected path. +func TestBuilder(t *testing.T) { + root, done := Mktemp(t) + defer done() + + client := fn.New() + + f, err := client.Init(fn.Function{Root: root, Runtime: "go"}) + if err != nil { + t.Fatal(err) + } + + builder := NewBuilder("", client, true) + + if err := builder.Build(context.Background(), f); err != nil { + t.Fatal(err) + } + + last := path(f.Root, fn.RunDataDir, "builds", "last", "oci") + + validateOCI(last, t) +} + +// ImageIndex represents the structure of an OCI Image Index. +type ImageIndex struct { + SchemaVersion int `json:"schemaVersion"` + Manifests []struct { + MediaType string `json:"mediaType"` + Size int64 `json:"size"` + Digest string `json:"digest"` + Platform struct { + Architecture string `json:"architecture"` + OS string `json:"os"` + } `json:"platform"` + } `json:"manifests"` +} + +// validateOCI performs a cursory check that the given path exists and +// has the basics of an OCI compliant structure. +func validateOCI(path string, t *testing.T) { + if _, err := os.Stat(path); err != nil { + t.Fatalf("unable to stat output path. %v", path) + return + } + + ociLayoutFile := filepath.Join(path, "oci-layout") + indexJSONFile := filepath.Join(path, "index.json") + blobsDir := filepath.Join(path, "blobs") + + // Check if required files and directories exist + if _, err := os.Stat(ociLayoutFile); os.IsNotExist(err) { + t.Fatal("missing oci-layout file") + } + if _, err := os.Stat(indexJSONFile); os.IsNotExist(err) { + t.Fatal("missing index.json file") + } + if _, err := os.Stat(blobsDir); os.IsNotExist(err) { + t.Fatal("missing blobs directory") + } + + // Load and validate index.json + indexJSONData, err := os.ReadFile(indexJSONFile) + if err != nil { + t.Fatalf("failed to read index.json: %v", err) + } + + var imageIndex ImageIndex + err = json.Unmarshal(indexJSONData, &imageIndex) + if err != nil { + t.Fatalf("failed to parse index.json: %v", err) + } + + if imageIndex.SchemaVersion != 2 { + t.Fatalf("invalid schema version, expected 2, got %d", imageIndex.SchemaVersion) + } + + // Additional validation of the Image Index structure can be added here + // extract. for example checking that the path includes the README.md + // and one of the binaries in the exact location expected (the data layer + // blob and exec layer blob, respectively) +} diff --git a/pkg/oci/containerize.go b/pkg/oci/containerize.go new file mode 100644 index 00000000..0306c82f --- /dev/null +++ b/pkg/oci/containerize.go @@ -0,0 +1,388 @@ +package oci + +import ( + "archive/tar" + "compress/gzip" + "encoding/json" + "errors" + "fmt" + "io" + "os" + "path/filepath" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +// languageLayerBuilder builds the layer for the given language whuch may +// be different from one platform to another. For example, this is the +// layer in the image which contains the Go cross-compiled binary. +type languageLayerBuilder interface { + Build(*buildConfig, v1.Platform) (v1.Descriptor, v1.Layer, error) +} + +func newLanguageLayerBuilder(cfg *buildConfig) (l languageLayerBuilder, err error) { + switch cfg.f.Runtime { + case "go": + l = goLayerBuilder{} + case "python": + // Likely the next to be supported after Go + err = errors.New("functions written in Python are not yet supported by the host builder") + case "node": + // Likely the next to be supported after Python + err = errors.New("functions written in Node are not yet supported by the host builder") + case "rust": + // Likely the next to be supprted after Node + err = errors.New("functions written in Rust are not yet supported by the host builder") + default: + // Others are not likely to be supported in the near future without + // increased contributions. + err = fmt.Errorf("the language runtime '%v' is not a recognized language by the host builder", cfg.f.Runtime) + } + return +} + +// containerize the scaffolded project by creating and writing an OCI +// conformant directory structure into the functions .func/builds directory. +// The source code to be containerized is indicated by cfg.dir +func containerize(cfg *buildConfig) (err error) { + // Create the required directories: oci/blobs/sha256 + if err = os.MkdirAll(cfg.blobsDir(), os.ModePerm); err != nil { + return + } + + // Create the static, required oci-layout metadata file + if err = os.WriteFile(path(cfg.ociDir(), "oci-layout"), + []byte(`{ "imageLayoutVersion": "1.0.0" }`), os.ModePerm); err != nil { + return + } + + // Create the data layer and its descriptor + dataDesc, dataLayer, err := newDataLayer(cfg) // shared + if err != nil { + return + } + + // TODO: if the base image is not provided, create a certificates layer + // which includes root certificates such that the resultant container + // can validate SSL (make HTTPS requests) + /* + certsDesc, certsLayer, err := newCerts(cfg) // shared + if err != nil { + return + } + */ + + // Create an image for each platform consisting of the shared data layer + // and an os/platform specific layer. + imageDescs := []v1.Descriptor{} + for _, p := range defaultPlatforms { // TODO: Configurable additions. + imageDesc, err := newImage(cfg, dataDesc, dataLayer, p, cfg.verbose) + if err != nil { + return err + } + imageDescs = append(imageDescs, imageDesc) + } + + // Create the Image Index which enumerates all images contained within + // the container. + _, err = newImageIndex(cfg, imageDescs) + return +} + +// newDataLayer creates the shared data layer in the container file hierarchy and +// returns both its descriptor and layer metadata. +func newDataLayer(cfg *buildConfig) (desc v1.Descriptor, layer v1.Layer, err error) { + + // Create the data tarball + // TODO: try WithCompressedCaching? + source := cfg.f.Root // The source is the function's entire filesystem + target := path(cfg.buildDir(), "datalayer.tar.gz") + + if err = newDataTarball(source, target, defaultIgnored, cfg.verbose); err != nil { + return + } + + // Layer + if layer, err = tarball.LayerFromFile(target); err != nil { + return + } + + // Descriptor + if desc, err = newDescriptor(layer); err != nil { + return + } + + // Blob + blob := path(cfg.blobsDir(), desc.Digest.Hex) + if cfg.verbose { + fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), target), rel(cfg.buildDir(), blob)) + } + err = os.Rename(target, blob) + return +} + +func newDataTarball(source, target string, ignored []string, verbose bool) error { + targetFile, err := os.Create(target) + if err != nil { + return err + } + defer targetFile.Close() + + gw := gzip.NewWriter(targetFile) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + return filepath.Walk(source, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + for _, v := range ignored { + if info.Name() == v { + if info.IsDir() { + return filepath.SkipDir + } + return nil + } + } + + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return err + } + + relPath, err := filepath.Rel(source, path) + if err != nil { + return err + } + + header.Name = filepath.Join("/func", relPath) + // TODO: should we set file timestamps to the build start time of cfg.t? + // header.ModTime = timestampArgument + + if err := tw.WriteHeader(header); err != nil { + return err + } + if verbose { + fmt.Printf("→ %v \n", header.Name) + } + if info.IsDir() { + return nil + } + + file, err := os.Open(path) + if err != nil { + return err + } + defer file.Close() + + _, err = io.Copy(tw, file) + return err + }) +} + +func newDescriptor(layer v1.Layer) (desc v1.Descriptor, err error) { + size, err := layer.Size() + if err != nil { + return + } + digest, err := layer.Digest() + if err != nil { + return + } + return v1.Descriptor{ + MediaType: types.OCILayer, + Size: size, + Digest: digest, + }, nil +} + +// newImage creates an image for the given platform. +// The image consists of the shared data layer which is provided +func newImage(cfg *buildConfig, dataDesc v1.Descriptor, dataLayer v1.Layer, p v1.Platform, verbose bool) (imageDesc v1.Descriptor, err error) { + b, err := newLanguageLayerBuilder(cfg) + if err != nil { + return + } + + // Write Exec Layer as Blob -> Layer + execDesc, execLayer, err := b.Build(cfg, p) + if err != nil { + return + } + + // Write Config Layer as Blob -> Layer + configDesc, _, err := newConfig(cfg, p, dataLayer, execLayer) + if err != nil { + return + } + + // Image Manifest + image := v1.Manifest{ + SchemaVersion: 2, + MediaType: types.OCIManifestSchema1, + Config: configDesc, + Layers: []v1.Descriptor{dataDesc, execDesc}, + } + + // Write image manifest out as json to a tempfile + filePath := fmt.Sprintf("image.%v.%v.json", p.OS, p.Architecture) + file, err := os.Create(filePath) + if err != nil { + return + } + enc := json.NewEncoder(file) + enc.SetIndent("", " ") + if err = enc.Encode(image); err != nil { + return + } + if err = file.Close(); err != nil { + return + } + + // Create a descriptor from hash and size + file, err = os.Open(filePath) + if err != nil { + return + } + hash, size, err := v1.SHA256(file) + if err != nil { + return + } + imageDesc = v1.Descriptor{ + MediaType: types.OCIManifestSchema1, + Digest: hash, + Size: size, + Platform: &p, + } + if err = file.Close(); err != nil { + return + } + + // move image into blobs + blob := path(cfg.blobsDir(), hash.Hex) + if cfg.verbose { + fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), filePath), rel(cfg.buildDir(), blob)) + } + err = os.Rename(filePath, blob) + return +} + +func newConfig(cfg *buildConfig, p v1.Platform, layers ...v1.Layer) (desc v1.Descriptor, config v1.ConfigFile, err error) { + volumes := make(map[string]struct{}) // Volumes are odd, see spec. + for _, v := range cfg.f.Run.Volumes { + if v.Path == nil { + continue // TODO: remove pointers from Volume and Env struct members + } + volumes[*v.Path] = struct{}{} + } + + rootfs := v1.RootFS{ + Type: "layers", + } + var diff v1.Hash + for _, v := range layers { + if diff, err = v.DiffID(); err != nil { + return + } + rootfs.DiffIDs = append(rootfs.DiffIDs, diff) + } + + config = v1.ConfigFile{ + Created: v1.Time{Time: cfg.t}, + Architecture: p.Architecture, + OS: p.OS, + OSVersion: p.OSVersion, + // OSFeatures: p.OSFeatures, // TODO: need to update dep to get this + Variant: p.Variant, + Config: v1.Config{ + ExposedPorts: map[string]struct{}{"8080/tcp": {}}, + Env: cfg.f.Run.Envs.Slice(), + Cmd: []string{"/func/f"}, // NOTE: Using Cmd because Entrypoint can not be overridden + WorkingDir: "/func/", + StopSignal: "SIGKILL", + Volumes: volumes, + // Labels + // History + }, + RootFS: rootfs, + } + + // Write the config out as json to a tempfile + filePath := path(cfg.buildDir(), "config.json") + file, err := os.Create(filePath) + if err != nil { + return + } + enc := json.NewEncoder(file) + enc.SetIndent("", " ") + if err = enc.Encode(config); err != nil { + return + } + if err = file.Close(); err != nil { + return + } + + // Create a descriptor using hash and size + file, err = os.Open(filePath) + if err != nil { + return + } + hash, size, err := v1.SHA256(file) + if err != nil { + return + } + desc = v1.Descriptor{ + MediaType: types.OCIConfigJSON, + Digest: hash, + Size: size, + } + if err = file.Close(); err != nil { + return + } + + // move config into blobs + blobPath := path(cfg.blobsDir(), hash.Hex) + if cfg.verbose { + fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), filePath), rel(cfg.buildDir(), blobPath)) + } + err = os.Rename(filePath, blobPath) + return +} + +func newImageIndex(cfg *buildConfig, imageDescs []v1.Descriptor) (index v1.IndexManifest, err error) { + index = v1.IndexManifest{ + SchemaVersion: 2, + MediaType: types.OCIImageIndex, + Manifests: imageDescs, + } + + filePath := path(cfg.ociDir(), "index.json") + file, err := os.Create(filePath) + if err != nil { + return + } + defer file.Close() + + enc := json.NewEncoder(file) + enc.SetIndent("", " ") + err = enc.Encode(index) + return +} + +// rel is a simple prefix trim used exclusively for verbose debugging +// statements to print paths as relative to the current build directory +// rather than absolute. Returns the path relative to the current working +// build directory. If it is not a subpath, the full path is returned +// unchanged. +func rel(base, path string) string { + if strings.HasPrefix(path, base) { + return "." + strings.TrimPrefix(path, base) + } + return path +} diff --git a/pkg/oci/containerize_go.go b/pkg/oci/containerize_go.go new file mode 100644 index 00000000..57dd345c --- /dev/null +++ b/pkg/oci/containerize_go.go @@ -0,0 +1,189 @@ +package oci + +import ( + "archive/tar" + "compress/gzip" + "fmt" + "io" + "os" + "os/exec" + "path/filepath" + "strings" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" + fn "knative.dev/func/pkg/functions" +) + +type goLayerBuilder struct{} + +// Build the source code as Go, cross compiled for the given platform, placing +// the statically linked binary in a tarred layer and return the Descriptor +// and Layer metadata. +func (c goLayerBuilder) Build(cfg *buildConfig, p v1.Platform) (desc v1.Descriptor, layer v1.Layer, err error) { + + // Executable + exe, err := goBuild(cfg, p) // Compile binary returning its path + if err != nil { + return + } + + // Tarball + target := path(cfg.buildDir(), fmt.Sprintf("execlayer.%v.%v.tar.gz", p.OS, p.Architecture)) + if err = newExecTarball(exe, target, cfg.verbose); err != nil { + return + } + + // Layer + if layer, err = tarball.LayerFromFile(target); err != nil { + return + } + + // Descriptor + if desc, err = newDescriptor(layer); err != nil { + return + } + desc.Platform = &p + + // Blob + blob := path(cfg.blobsDir(), desc.Digest.Hex) + if cfg.verbose { + fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), target), rel(cfg.buildDir(), blob)) + } + err = os.Rename(target, blob) + return +} + +func goBuild(cfg *buildConfig, p v1.Platform) (binPath string, err error) { + gobin, args, outpath, err := goBuildCmd(p, cfg) + if err != nil { + return + } + envs := goBuildEnvs(cfg.f, p) + if cfg.verbose { + fmt.Printf("%v %v\n", gobin, strings.Join(args, " ")) + } else { + fmt.Printf(" %v\n", filepath.Base(outpath)) + } + cmd := exec.CommandContext(cfg.ctx, gobin, args...) + cmd.Env = envs + cmd.Dir = cfg.buildDir() + cmd.Stderr = os.Stderr + cmd.Stdout = os.Stdout + + return outpath, cmd.Run() +} + +func goBuildCmd(p v1.Platform, cfg *buildConfig) (gobin string, args []string, outpath string, err error) { + /* TODO: Use Build Command override from the function if provided + * A future PR will include the ability to specify a + * f.Build.BuildCommand, or BuildArgs for use here to customize + * This will be useful when, for example, the function is written in + * Go and the function developer needs Libc compatibility, in which case + * the default command will need to be replaced with: + * go build -ldflags "-linkmode 'external' -extldflags '-static'" + * Pseudocode: + * if BuildArgs or BuildCommand + * Validate command or args are safe to run + * no other commands injected + * does not contain Go's "toolexec" + * does not specify the output path + * Either replace or append to gobin + */ + + // Use the binary specified FUNC_GO_PATH if defined + gobin = os.Getenv("FUNC_GO_PATH") // TODO: move to main and plumb through + if gobin == "" { + gobin = "go" + } + + // Build as ./func/builds/$PID/result/f.$OS.$Architecture + name := fmt.Sprintf("f.%v.%v", p.OS, p.Architecture) + if p.Variant != "" { + name = name + "." + p.Variant + } + outpath = path(cfg.buildDir(), "result", name) + args = []string{"build", "-o", outpath} + return gobin, args, outpath, nil +} + +func goBuildEnvs(f fn.Function, p v1.Platform) (envs []string) { + pegged := []string{ + "CGO_ENABLED=0", + "GOOS=" + p.OS, + "GOARCH=" + p.Architecture, + } + if p.Variant != "" && p.Architecture == "arm" { + pegged = append(pegged, "GOARM="+strings.TrimPrefix(p.Variant, "v")) + } else if p.Variant != "" && p.Architecture == "amd64" { + pegged = append(pegged, "GOAMD64="+p.Variant) + } + + isPegged := func(env string) bool { + for _, v := range pegged { + name := strings.Split(v, "=")[0] + if strings.HasPrefix(env, name) { + return true + } + } + return false + } + + envs = append(envs, pegged...) + for _, env := range os.Environ() { + if !isPegged(env) { + envs = append(envs, env) + } + } + return envs +} + +func newExecTarball(source, target string, verbose bool) error { + targetFile, err := os.Create(target) + if err != nil { + return err + } + defer targetFile.Close() + + gw := gzip.NewWriter(targetFile) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + info, err := os.Stat(source) + if err != nil { + return err + } + + header, err := tar.FileInfoHeader(info, info.Name()) + if err != nil { + return err + } + + header.Name = path("/func", "f") + // TODO: should we set file timestamps to the build start time of cfg.t? + // header.ModTime = timestampArgument + + if err = tw.WriteHeader(header); err != nil { + return err + } + if verbose { + fmt.Printf("→ %v \n", header.Name) + } + + file, err := os.Open(source) + if err != nil { + return err + } + defer file.Close() + + i, err := io.Copy(tw, file) + if err != nil { + return err + } + if verbose { + fmt.Printf(" wrote %v bytes \n", i) + } + return nil +}