mirror of https://github.com/knative/func.git
feat: host oci builder (#1730)
* feat: oci builder for host builds * do not expose host builder until fully baked
This commit is contained in:
Luke Kingland
GitHub
parent
64337b4df5
commit
9a790f005f
24
cmd/build.go
24
cmd/build.go
|
|
@ -10,7 +10,7 @@ import (
|
|||
"github.com/spf13/cobra"
|
||||
|
||||
"knative.dev/func/pkg/builders"
|
||||
"knative.dev/func/pkg/builders/buildpacks"
|
||||
pack "knative.dev/func/pkg/builders/buildpacks"
|
||||
"knative.dev/func/pkg/builders/s2i"
|
||||
"knative.dev/func/pkg/config"
|
||||
fn "knative.dev/func/pkg/functions"
|
||||
|
|
@ -169,25 +169,21 @@ func runBuild(cmd *cobra.Command, _ []string, newClient ClientFactory) (err erro
|
|||
|
||||
// Client
|
||||
// Concrete implementations (ex builder) vary based on final effective config
|
||||
var builder fn.Builder
|
||||
var client *fn.Client
|
||||
o := []fn.Option{fn.WithRegistry(cfg.Registry)}
|
||||
if f.Build.Builder == builders.Pack {
|
||||
builder = buildpacks.NewBuilder(
|
||||
buildpacks.WithName(builders.Pack),
|
||||
buildpacks.WithVerbose(cfg.Verbose),
|
||||
buildpacks.WithTimestamp(cfg.WithTimestamp),
|
||||
)
|
||||
o = append(o, fn.WithBuilder(pack.NewBuilder(
|
||||
pack.WithName(builders.Pack),
|
||||
pack.WithTimestamp(cfg.WithTimestamp),
|
||||
pack.WithVerbose(cfg.Verbose))))
|
||||
} else if f.Build.Builder == builders.S2I {
|
||||
builder = s2i.NewBuilder(
|
||||
o = append(o, fn.WithBuilder(s2i.NewBuilder(
|
||||
s2i.WithName(builders.S2I),
|
||||
s2i.WithPlatform(cfg.Platform),
|
||||
s2i.WithVerbose(cfg.Verbose))
|
||||
} else {
|
||||
return builders.ErrUnknownBuilder{Name: f.Build.Builder, Known: KnownBuilders()}
|
||||
s2i.WithVerbose(cfg.Verbose))))
|
||||
}
|
||||
|
||||
client, done := newClient(ClientConfig{Verbose: cfg.Verbose},
|
||||
fn.WithRegistry(cfg.Registry),
|
||||
fn.WithBuilder(builder))
|
||||
client, done := newClient(ClientConfig{Verbose: cfg.Verbose}, o...)
|
||||
defer done()
|
||||
|
||||
// Build and (optionally) push
|
||||
|
|
|
|||
|
|
@ -387,8 +387,7 @@ func (c *Client) Registry() string {
|
|||
func (c *Client) Runtimes() ([]string, error) {
|
||||
runtimes := utils.NewSortedSet()
|
||||
|
||||
// Gather all runtimes from all repositories
|
||||
// into a uniqueness map
|
||||
// Gather all runtimes from all repositories into a uniqueness map
|
||||
repositories, err := c.Repositories().All()
|
||||
if err != nil {
|
||||
return []string{}, err
|
||||
|
|
@ -1026,7 +1025,7 @@ func ensureRunDataDir(root string) error {
|
|||
return nil
|
||||
}
|
||||
|
||||
// fingerprint the files at a given path. Returns a hash calculated from the
|
||||
// Fingerprint the files at a given path. Returns a hash calculated from the
|
||||
// filenames and modification timestamps of the files within the given root.
|
||||
// Also returns a logfile consiting of the filenames and modification times
|
||||
// which contributed to the hash.
|
||||
|
|
@ -1035,7 +1034,7 @@ func ensureRunDataDir(root string) error {
|
|||
// .git and .func.
|
||||
// Future updates will include files explicitly marked as ignored by a
|
||||
// .funcignore.
|
||||
func fingerprint(root string) (hash, log string, err error) {
|
||||
func Fingerprint(root string) (hash, log string, err error) {
|
||||
h := sha256.New() // Hash builder
|
||||
l := bytes.Buffer{} // Log buffer
|
||||
|
||||
|
|
|
|||
|
|
@ -415,7 +415,7 @@ func (f Function) Stamp(oo ...stampOption) (err error) {
|
|||
|
||||
// Cacluate the hash and a logfile of what comprised it
|
||||
var hash, log string
|
||||
if hash, log, err = fingerprint(f.Root); err != nil {
|
||||
if hash, log, err = Fingerprint(f.Root); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
|
|
@ -633,7 +633,7 @@ func (f Function) Built() bool {
|
|||
// It's a pretty good chance the thing doesn't need to be rebuilt, though
|
||||
// of course filesystem racing conditions do exist, including both direct
|
||||
// source code modifications or changes to the image cache.
|
||||
hash, _, err := fingerprint(f.Root)
|
||||
hash, _, err := Fingerprint(f.Root)
|
||||
if err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error calculating function's fingerprint: %v\n", err)
|
||||
return false
|
||||
|
|
|
|||
|
|
@ -0,0 +1,276 @@
|
|||
package oci
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strconv"
|
||||
"syscall"
|
||||
"time"
|
||||
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
|
||||
fn "knative.dev/func/pkg/functions"
|
||||
)
|
||||
|
||||
var path = filepath.Join
|
||||
|
||||
// TODO: This may no longer be necessary, delete if e2e and acceptance tests
|
||||
// succeed:
|
||||
// const DefaultName = builders.Host
|
||||
|
||||
var defaultPlatforms = []v1.Platform{
|
||||
{OS: "linux", Architecture: "amd64"},
|
||||
{OS: "linux", Architecture: "arm64"},
|
||||
// {OS: "linux", Architecture: "arm", Variant: "v6"},
|
||||
{OS: "linux", Architecture: "arm", Variant: "v7"},
|
||||
{OS: "darwin", Architecture: "amd64"},
|
||||
{OS: "darwin", Architecture: "arm64"},
|
||||
}
|
||||
|
||||
var defaultIgnored = []string{ // TODO: implement and use .funcignore
|
||||
".git",
|
||||
".func",
|
||||
".funcignore",
|
||||
".gitignore",
|
||||
}
|
||||
|
||||
// BuildErr indicates a build error occurred.
|
||||
type BuildErr struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (e BuildErr) Error() string {
|
||||
return fmt.Sprintf("error performing host build. %v", e.Err)
|
||||
}
|
||||
|
||||
// Builder which creates an OCI-compliant multi-arch (index) container from
|
||||
// the function at path.
|
||||
type Builder struct {
|
||||
name string
|
||||
client *fn.Client
|
||||
verbose bool
|
||||
}
|
||||
|
||||
// NewBuilder creates a builder instance.
|
||||
func NewBuilder(name string, client *fn.Client, verbose bool) *Builder {
|
||||
return &Builder{name, client, verbose}
|
||||
}
|
||||
|
||||
// Build an OCI-compliant Mult-arch (v1.ImageIndex) container on disk
|
||||
// in the function's runtime data directory at:
|
||||
//
|
||||
// .func/builds/by-hash/$HASH
|
||||
//
|
||||
// Updates a symlink to this directory at:
|
||||
//
|
||||
// .func/builds/last
|
||||
func (b *Builder) Build(ctx context.Context, f fn.Function) (err error) {
|
||||
cfg := &buildConfig{ctx, b.client, f, time.Now(), b.verbose, ""}
|
||||
|
||||
if err = setup(cfg); err != nil { // create directories and links
|
||||
return
|
||||
}
|
||||
defer teardown(cfg)
|
||||
|
||||
//TODO: Use clien't actual Scaffold when ready:
|
||||
/*
|
||||
if err = cfg.client.Scaffold(ctx, f, cfg.buildDir()); err != nil {
|
||||
return
|
||||
}
|
||||
*/
|
||||
// IN the meantime, use an airball mainfile
|
||||
data := `
|
||||
package main
|
||||
|
||||
import "fmt"
|
||||
|
||||
func main () {
|
||||
fmt.Println("Hello, world!")
|
||||
}
|
||||
`
|
||||
if err = os.WriteFile(path(cfg.buildDir(), "main.go"), []byte(data), 0664); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
if err = containerize(cfg); err != nil {
|
||||
return
|
||||
}
|
||||
return updateLastLink(cfg)
|
||||
|
||||
// TODO: communicating build completeness throgh returning without error
|
||||
// relies on the implicit availability of the OIC image in this process'
|
||||
// build directory. Would be better to have a formal build result object
|
||||
// which includes a general struct which can be used by all builders to
|
||||
// communicate to the pusher where the image can be found.
|
||||
}
|
||||
|
||||
// buildConfig contains various settings for a single build
|
||||
type buildConfig struct {
|
||||
ctx context.Context // build context
|
||||
client *fn.Client // backreference to the client for Scaffolding
|
||||
f fn.Function // Function being built
|
||||
t time.Time // Timestamp for this build
|
||||
verbose bool // verbose logging
|
||||
h string // hash cache (use .hash() accessor)
|
||||
}
|
||||
|
||||
func (c *buildConfig) hash() string {
|
||||
if c.h != "" {
|
||||
return c.h
|
||||
}
|
||||
var err error
|
||||
if c.h, _, err = fn.Fingerprint(c.f.Root); err != nil {
|
||||
fmt.Fprintf(os.Stderr, "error calculating fingerprint for build. %v", err)
|
||||
}
|
||||
return c.h
|
||||
}
|
||||
|
||||
func (c *buildConfig) lastLink() string {
|
||||
return path(c.f.Root, fn.RunDataDir, "builds", "last")
|
||||
}
|
||||
func (c *buildConfig) pidsDir() string {
|
||||
return path(c.f.Root, fn.RunDataDir, "builds", "by-pid")
|
||||
}
|
||||
func (c *buildConfig) pidLink() string {
|
||||
return path(c.f.Root, fn.RunDataDir, "builds", "by-pid", strconv.Itoa(os.Getpid()))
|
||||
}
|
||||
func (c *buildConfig) buildsDir() string {
|
||||
return path(c.f.Root, fn.RunDataDir, "builds", "by-hash")
|
||||
}
|
||||
func (c *buildConfig) buildDir() string {
|
||||
return path(c.f.Root, fn.RunDataDir, "builds", "by-hash", c.hash())
|
||||
}
|
||||
func (c *buildConfig) ociDir() string {
|
||||
return path(c.f.Root, fn.RunDataDir, "builds", "by-hash", c.hash(), "oci")
|
||||
}
|
||||
func (c *buildConfig) blobsDir() string {
|
||||
return path(c.f.Root, fn.RunDataDir, "builds", "by-hash", c.hash(), "oci", "blobs", "sha256")
|
||||
}
|
||||
|
||||
// setup errors if there already exists a build directory. Otherwise, it
|
||||
// creates a build directory based on the function's hash, and creates
|
||||
// a link to this build directory for the current pid to denote the build
|
||||
// is in progress.
|
||||
func setup(cfg *buildConfig) (err error) {
|
||||
// error if already in progress
|
||||
if isActive(cfg, cfg.buildDir()) {
|
||||
return BuildErr{fmt.Errorf("Build directory already exists for this version hash and is associated with an active PID. Is a build already in progress? %v", cfg.buildDir())}
|
||||
}
|
||||
|
||||
// create build files directory
|
||||
if cfg.verbose {
|
||||
fmt.Printf("mkdir -p %v\n", cfg.buildDir())
|
||||
}
|
||||
if err = os.MkdirAll(cfg.buildDir(), 0774); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// create pid links directory
|
||||
if _, err = os.Stat(cfg.pidsDir()); os.IsNotExist(err) {
|
||||
if cfg.verbose {
|
||||
fmt.Printf("mkdir -p %v\n", cfg.pidsDir())
|
||||
}
|
||||
if err = os.MkdirAll(cfg.pidsDir(), 0774); err != nil {
|
||||
return
|
||||
}
|
||||
}
|
||||
|
||||
// create a link named $pid to the current build files directory
|
||||
target := path("..", "by-hash", cfg.hash())
|
||||
if cfg.verbose {
|
||||
fmt.Printf("ln -s %v %v\n", target, cfg.pidLink())
|
||||
}
|
||||
return os.Symlink(target, cfg.pidLink())
|
||||
}
|
||||
|
||||
func teardown(cfg *buildConfig) {
|
||||
// remove the pid link for the current process indicating the build is
|
||||
// no longer in progress.
|
||||
_ = os.RemoveAll(cfg.pidLink())
|
||||
|
||||
// remove pid links for processes which no longer exist.
|
||||
dd, _ := os.ReadDir(cfg.pidsDir())
|
||||
for _, d := range dd {
|
||||
if processExists(d.Name()) {
|
||||
continue
|
||||
}
|
||||
dir := path(cfg.pidsDir(), d.Name())
|
||||
if cfg.verbose {
|
||||
fmt.Printf("rm %v\n", dir)
|
||||
}
|
||||
_ = os.RemoveAll(dir)
|
||||
}
|
||||
|
||||
// remove build file directories unless they are either:
|
||||
// 1. The build files from the last successful build
|
||||
// 2. Are associated with a pid link (currently in progress)
|
||||
dd, _ = os.ReadDir(cfg.buildsDir())
|
||||
for _, d := range dd {
|
||||
dir := path(cfg.buildsDir(), d.Name())
|
||||
if isLinkTo(cfg.lastLink(), dir) {
|
||||
continue
|
||||
}
|
||||
if isActive(cfg, dir) {
|
||||
continue
|
||||
}
|
||||
if cfg.verbose {
|
||||
fmt.Printf("rm %v\n", dir)
|
||||
}
|
||||
_ = os.RemoveAll(dir)
|
||||
}
|
||||
}
|
||||
|
||||
func processExists(pid string) bool {
|
||||
p, err := strconv.Atoi(pid)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
process, err := os.FindProcess(p)
|
||||
if err != nil {
|
||||
return false
|
||||
}
|
||||
err = process.Signal(syscall.Signal(0))
|
||||
return err == nil
|
||||
}
|
||||
|
||||
func isLinkTo(link, target string) bool {
|
||||
var err error
|
||||
if link, err = filepath.EvalSymlinks(link); err != nil {
|
||||
return false
|
||||
}
|
||||
if link, err = filepath.Abs(link); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
if target, err = filepath.EvalSymlinks(target); err != nil {
|
||||
return false
|
||||
}
|
||||
if target, err = filepath.Abs(target); err != nil {
|
||||
return false
|
||||
}
|
||||
|
||||
return link == target
|
||||
}
|
||||
|
||||
// isActive returns whether or not the given directory path is for a build
|
||||
// which is currently active or in progress.
|
||||
func isActive(cfg *buildConfig, dir string) bool {
|
||||
dd, _ := os.ReadDir(cfg.pidsDir())
|
||||
for _, d := range dd {
|
||||
link := path(cfg.pidsDir(), d.Name())
|
||||
if processExists(d.Name()) && isLinkTo(link, dir) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
func updateLastLink(cfg *buildConfig) error {
|
||||
if cfg.verbose {
|
||||
fmt.Printf("ln -s %v %v\n", cfg.buildDir(), cfg.lastLink())
|
||||
}
|
||||
_ = os.RemoveAll(cfg.lastLink())
|
||||
return os.Symlink(cfg.buildDir(), cfg.lastLink())
|
||||
}
|
||||
|
|
@ -0,0 +1,95 @@
|
|||
package oci
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"testing"
|
||||
|
||||
fn "knative.dev/func/pkg/functions"
|
||||
. "knative.dev/func/pkg/testing"
|
||||
)
|
||||
|
||||
// TestBuilder ensures that, when given a Go Function, an OCI-compliant
|
||||
// directory structure is created on .Build in the expected path.
|
||||
func TestBuilder(t *testing.T) {
|
||||
root, done := Mktemp(t)
|
||||
defer done()
|
||||
|
||||
client := fn.New()
|
||||
|
||||
f, err := client.Init(fn.Function{Root: root, Runtime: "go"})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
builder := NewBuilder("", client, true)
|
||||
|
||||
if err := builder.Build(context.Background(), f); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
last := path(f.Root, fn.RunDataDir, "builds", "last", "oci")
|
||||
|
||||
validateOCI(last, t)
|
||||
}
|
||||
|
||||
// ImageIndex represents the structure of an OCI Image Index.
|
||||
type ImageIndex struct {
|
||||
SchemaVersion int `json:"schemaVersion"`
|
||||
Manifests []struct {
|
||||
MediaType string `json:"mediaType"`
|
||||
Size int64 `json:"size"`
|
||||
Digest string `json:"digest"`
|
||||
Platform struct {
|
||||
Architecture string `json:"architecture"`
|
||||
OS string `json:"os"`
|
||||
} `json:"platform"`
|
||||
} `json:"manifests"`
|
||||
}
|
||||
|
||||
// validateOCI performs a cursory check that the given path exists and
|
||||
// has the basics of an OCI compliant structure.
|
||||
func validateOCI(path string, t *testing.T) {
|
||||
if _, err := os.Stat(path); err != nil {
|
||||
t.Fatalf("unable to stat output path. %v", path)
|
||||
return
|
||||
}
|
||||
|
||||
ociLayoutFile := filepath.Join(path, "oci-layout")
|
||||
indexJSONFile := filepath.Join(path, "index.json")
|
||||
blobsDir := filepath.Join(path, "blobs")
|
||||
|
||||
// Check if required files and directories exist
|
||||
if _, err := os.Stat(ociLayoutFile); os.IsNotExist(err) {
|
||||
t.Fatal("missing oci-layout file")
|
||||
}
|
||||
if _, err := os.Stat(indexJSONFile); os.IsNotExist(err) {
|
||||
t.Fatal("missing index.json file")
|
||||
}
|
||||
if _, err := os.Stat(blobsDir); os.IsNotExist(err) {
|
||||
t.Fatal("missing blobs directory")
|
||||
}
|
||||
|
||||
// Load and validate index.json
|
||||
indexJSONData, err := os.ReadFile(indexJSONFile)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to read index.json: %v", err)
|
||||
}
|
||||
|
||||
var imageIndex ImageIndex
|
||||
err = json.Unmarshal(indexJSONData, &imageIndex)
|
||||
if err != nil {
|
||||
t.Fatalf("failed to parse index.json: %v", err)
|
||||
}
|
||||
|
||||
if imageIndex.SchemaVersion != 2 {
|
||||
t.Fatalf("invalid schema version, expected 2, got %d", imageIndex.SchemaVersion)
|
||||
}
|
||||
|
||||
// Additional validation of the Image Index structure can be added here
|
||||
// extract. for example checking that the path includes the README.md
|
||||
// and one of the binaries in the exact location expected (the data layer
|
||||
// blob and exec layer blob, respectively)
|
||||
}
|
||||
|
|
@ -0,0 +1,388 @@
|
|||
package oci
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
"github.com/google/go-containerregistry/pkg/v1/tarball"
|
||||
"github.com/google/go-containerregistry/pkg/v1/types"
|
||||
)
|
||||
|
||||
// languageLayerBuilder builds the layer for the given language whuch may
|
||||
// be different from one platform to another. For example, this is the
|
||||
// layer in the image which contains the Go cross-compiled binary.
|
||||
type languageLayerBuilder interface {
|
||||
Build(*buildConfig, v1.Platform) (v1.Descriptor, v1.Layer, error)
|
||||
}
|
||||
|
||||
func newLanguageLayerBuilder(cfg *buildConfig) (l languageLayerBuilder, err error) {
|
||||
switch cfg.f.Runtime {
|
||||
case "go":
|
||||
l = goLayerBuilder{}
|
||||
case "python":
|
||||
// Likely the next to be supported after Go
|
||||
err = errors.New("functions written in Python are not yet supported by the host builder")
|
||||
case "node":
|
||||
// Likely the next to be supported after Python
|
||||
err = errors.New("functions written in Node are not yet supported by the host builder")
|
||||
case "rust":
|
||||
// Likely the next to be supprted after Node
|
||||
err = errors.New("functions written in Rust are not yet supported by the host builder")
|
||||
default:
|
||||
// Others are not likely to be supported in the near future without
|
||||
// increased contributions.
|
||||
err = fmt.Errorf("the language runtime '%v' is not a recognized language by the host builder", cfg.f.Runtime)
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// containerize the scaffolded project by creating and writing an OCI
|
||||
// conformant directory structure into the functions .func/builds directory.
|
||||
// The source code to be containerized is indicated by cfg.dir
|
||||
func containerize(cfg *buildConfig) (err error) {
|
||||
// Create the required directories: oci/blobs/sha256
|
||||
if err = os.MkdirAll(cfg.blobsDir(), os.ModePerm); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Create the static, required oci-layout metadata file
|
||||
if err = os.WriteFile(path(cfg.ociDir(), "oci-layout"),
|
||||
[]byte(`{ "imageLayoutVersion": "1.0.0" }`), os.ModePerm); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Create the data layer and its descriptor
|
||||
dataDesc, dataLayer, err := newDataLayer(cfg) // shared
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// TODO: if the base image is not provided, create a certificates layer
|
||||
// which includes root certificates such that the resultant container
|
||||
// can validate SSL (make HTTPS requests)
|
||||
/*
|
||||
certsDesc, certsLayer, err := newCerts(cfg) // shared
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
*/
|
||||
|
||||
// Create an image for each platform consisting of the shared data layer
|
||||
// and an os/platform specific layer.
|
||||
imageDescs := []v1.Descriptor{}
|
||||
for _, p := range defaultPlatforms { // TODO: Configurable additions.
|
||||
imageDesc, err := newImage(cfg, dataDesc, dataLayer, p, cfg.verbose)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
imageDescs = append(imageDescs, imageDesc)
|
||||
}
|
||||
|
||||
// Create the Image Index which enumerates all images contained within
|
||||
// the container.
|
||||
_, err = newImageIndex(cfg, imageDescs)
|
||||
return
|
||||
}
|
||||
|
||||
// newDataLayer creates the shared data layer in the container file hierarchy and
|
||||
// returns both its descriptor and layer metadata.
|
||||
func newDataLayer(cfg *buildConfig) (desc v1.Descriptor, layer v1.Layer, err error) {
|
||||
|
||||
// Create the data tarball
|
||||
// TODO: try WithCompressedCaching?
|
||||
source := cfg.f.Root // The source is the function's entire filesystem
|
||||
target := path(cfg.buildDir(), "datalayer.tar.gz")
|
||||
|
||||
if err = newDataTarball(source, target, defaultIgnored, cfg.verbose); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Layer
|
||||
if layer, err = tarball.LayerFromFile(target); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Descriptor
|
||||
if desc, err = newDescriptor(layer); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Blob
|
||||
blob := path(cfg.blobsDir(), desc.Digest.Hex)
|
||||
if cfg.verbose {
|
||||
fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), target), rel(cfg.buildDir(), blob))
|
||||
}
|
||||
err = os.Rename(target, blob)
|
||||
return
|
||||
}
|
||||
|
||||
func newDataTarball(source, target string, ignored []string, verbose bool) error {
|
||||
targetFile, err := os.Create(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer targetFile.Close()
|
||||
|
||||
gw := gzip.NewWriter(targetFile)
|
||||
defer gw.Close()
|
||||
|
||||
tw := tar.NewWriter(gw)
|
||||
defer tw.Close()
|
||||
|
||||
return filepath.Walk(source, func(path string, info os.FileInfo, err error) error {
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, v := range ignored {
|
||||
if info.Name() == v {
|
||||
if info.IsDir() {
|
||||
return filepath.SkipDir
|
||||
}
|
||||
return nil
|
||||
}
|
||||
}
|
||||
|
||||
header, err := tar.FileInfoHeader(info, info.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
relPath, err := filepath.Rel(source, path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header.Name = filepath.Join("/func", relPath)
|
||||
// TODO: should we set file timestamps to the build start time of cfg.t?
|
||||
// header.ModTime = timestampArgument
|
||||
|
||||
if err := tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
if verbose {
|
||||
fmt.Printf("→ %v \n", header.Name)
|
||||
}
|
||||
if info.IsDir() {
|
||||
return nil
|
||||
}
|
||||
|
||||
file, err := os.Open(path)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
_, err = io.Copy(tw, file)
|
||||
return err
|
||||
})
|
||||
}
|
||||
|
||||
func newDescriptor(layer v1.Layer) (desc v1.Descriptor, err error) {
|
||||
size, err := layer.Size()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
digest, err := layer.Digest()
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
return v1.Descriptor{
|
||||
MediaType: types.OCILayer,
|
||||
Size: size,
|
||||
Digest: digest,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// newImage creates an image for the given platform.
|
||||
// The image consists of the shared data layer which is provided
|
||||
func newImage(cfg *buildConfig, dataDesc v1.Descriptor, dataLayer v1.Layer, p v1.Platform, verbose bool) (imageDesc v1.Descriptor, err error) {
|
||||
b, err := newLanguageLayerBuilder(cfg)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Write Exec Layer as Blob -> Layer
|
||||
execDesc, execLayer, err := b.Build(cfg, p)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Write Config Layer as Blob -> Layer
|
||||
configDesc, _, err := newConfig(cfg, p, dataLayer, execLayer)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Image Manifest
|
||||
image := v1.Manifest{
|
||||
SchemaVersion: 2,
|
||||
MediaType: types.OCIManifestSchema1,
|
||||
Config: configDesc,
|
||||
Layers: []v1.Descriptor{dataDesc, execDesc},
|
||||
}
|
||||
|
||||
// Write image manifest out as json to a tempfile
|
||||
filePath := fmt.Sprintf("image.%v.%v.json", p.OS, p.Architecture)
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
enc := json.NewEncoder(file)
|
||||
enc.SetIndent("", " ")
|
||||
if err = enc.Encode(image); err != nil {
|
||||
return
|
||||
}
|
||||
if err = file.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Create a descriptor from hash and size
|
||||
file, err = os.Open(filePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
hash, size, err := v1.SHA256(file)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
imageDesc = v1.Descriptor{
|
||||
MediaType: types.OCIManifestSchema1,
|
||||
Digest: hash,
|
||||
Size: size,
|
||||
Platform: &p,
|
||||
}
|
||||
if err = file.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// move image into blobs
|
||||
blob := path(cfg.blobsDir(), hash.Hex)
|
||||
if cfg.verbose {
|
||||
fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), filePath), rel(cfg.buildDir(), blob))
|
||||
}
|
||||
err = os.Rename(filePath, blob)
|
||||
return
|
||||
}
|
||||
|
||||
func newConfig(cfg *buildConfig, p v1.Platform, layers ...v1.Layer) (desc v1.Descriptor, config v1.ConfigFile, err error) {
|
||||
volumes := make(map[string]struct{}) // Volumes are odd, see spec.
|
||||
for _, v := range cfg.f.Run.Volumes {
|
||||
if v.Path == nil {
|
||||
continue // TODO: remove pointers from Volume and Env struct members
|
||||
}
|
||||
volumes[*v.Path] = struct{}{}
|
||||
}
|
||||
|
||||
rootfs := v1.RootFS{
|
||||
Type: "layers",
|
||||
}
|
||||
var diff v1.Hash
|
||||
for _, v := range layers {
|
||||
if diff, err = v.DiffID(); err != nil {
|
||||
return
|
||||
}
|
||||
rootfs.DiffIDs = append(rootfs.DiffIDs, diff)
|
||||
}
|
||||
|
||||
config = v1.ConfigFile{
|
||||
Created: v1.Time{Time: cfg.t},
|
||||
Architecture: p.Architecture,
|
||||
OS: p.OS,
|
||||
OSVersion: p.OSVersion,
|
||||
// OSFeatures: p.OSFeatures, // TODO: need to update dep to get this
|
||||
Variant: p.Variant,
|
||||
Config: v1.Config{
|
||||
ExposedPorts: map[string]struct{}{"8080/tcp": {}},
|
||||
Env: cfg.f.Run.Envs.Slice(),
|
||||
Cmd: []string{"/func/f"}, // NOTE: Using Cmd because Entrypoint can not be overridden
|
||||
WorkingDir: "/func/",
|
||||
StopSignal: "SIGKILL",
|
||||
Volumes: volumes,
|
||||
// Labels
|
||||
// History
|
||||
},
|
||||
RootFS: rootfs,
|
||||
}
|
||||
|
||||
// Write the config out as json to a tempfile
|
||||
filePath := path(cfg.buildDir(), "config.json")
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
enc := json.NewEncoder(file)
|
||||
enc.SetIndent("", " ")
|
||||
if err = enc.Encode(config); err != nil {
|
||||
return
|
||||
}
|
||||
if err = file.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Create a descriptor using hash and size
|
||||
file, err = os.Open(filePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
hash, size, err := v1.SHA256(file)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
desc = v1.Descriptor{
|
||||
MediaType: types.OCIConfigJSON,
|
||||
Digest: hash,
|
||||
Size: size,
|
||||
}
|
||||
if err = file.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// move config into blobs
|
||||
blobPath := path(cfg.blobsDir(), hash.Hex)
|
||||
if cfg.verbose {
|
||||
fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), filePath), rel(cfg.buildDir(), blobPath))
|
||||
}
|
||||
err = os.Rename(filePath, blobPath)
|
||||
return
|
||||
}
|
||||
|
||||
func newImageIndex(cfg *buildConfig, imageDescs []v1.Descriptor) (index v1.IndexManifest, err error) {
|
||||
index = v1.IndexManifest{
|
||||
SchemaVersion: 2,
|
||||
MediaType: types.OCIImageIndex,
|
||||
Manifests: imageDescs,
|
||||
}
|
||||
|
||||
filePath := path(cfg.ociDir(), "index.json")
|
||||
file, err := os.Create(filePath)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
enc := json.NewEncoder(file)
|
||||
enc.SetIndent("", " ")
|
||||
err = enc.Encode(index)
|
||||
return
|
||||
}
|
||||
|
||||
// rel is a simple prefix trim used exclusively for verbose debugging
|
||||
// statements to print paths as relative to the current build directory
|
||||
// rather than absolute. Returns the path relative to the current working
|
||||
// build directory. If it is not a subpath, the full path is returned
|
||||
// unchanged.
|
||||
func rel(base, path string) string {
|
||||
if strings.HasPrefix(path, base) {
|
||||
return "." + strings.TrimPrefix(path, base)
|
||||
}
|
||||
return path
|
||||
}
|
||||
|
|
@ -0,0 +1,189 @@
|
|||
package oci
|
||||
|
||||
import (
|
||||
"archive/tar"
|
||||
"compress/gzip"
|
||||
"fmt"
|
||||
"io"
|
||||
"os"
|
||||
"os/exec"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
v1 "github.com/google/go-containerregistry/pkg/v1"
|
||||
"github.com/google/go-containerregistry/pkg/v1/tarball"
|
||||
fn "knative.dev/func/pkg/functions"
|
||||
)
|
||||
|
||||
type goLayerBuilder struct{}
|
||||
|
||||
// Build the source code as Go, cross compiled for the given platform, placing
|
||||
// the statically linked binary in a tarred layer and return the Descriptor
|
||||
// and Layer metadata.
|
||||
func (c goLayerBuilder) Build(cfg *buildConfig, p v1.Platform) (desc v1.Descriptor, layer v1.Layer, err error) {
|
||||
|
||||
// Executable
|
||||
exe, err := goBuild(cfg, p) // Compile binary returning its path
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Tarball
|
||||
target := path(cfg.buildDir(), fmt.Sprintf("execlayer.%v.%v.tar.gz", p.OS, p.Architecture))
|
||||
if err = newExecTarball(exe, target, cfg.verbose); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Layer
|
||||
if layer, err = tarball.LayerFromFile(target); err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
// Descriptor
|
||||
if desc, err = newDescriptor(layer); err != nil {
|
||||
return
|
||||
}
|
||||
desc.Platform = &p
|
||||
|
||||
// Blob
|
||||
blob := path(cfg.blobsDir(), desc.Digest.Hex)
|
||||
if cfg.verbose {
|
||||
fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), target), rel(cfg.buildDir(), blob))
|
||||
}
|
||||
err = os.Rename(target, blob)
|
||||
return
|
||||
}
|
||||
|
||||
func goBuild(cfg *buildConfig, p v1.Platform) (binPath string, err error) {
|
||||
gobin, args, outpath, err := goBuildCmd(p, cfg)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
envs := goBuildEnvs(cfg.f, p)
|
||||
if cfg.verbose {
|
||||
fmt.Printf("%v %v\n", gobin, strings.Join(args, " "))
|
||||
} else {
|
||||
fmt.Printf(" %v\n", filepath.Base(outpath))
|
||||
}
|
||||
cmd := exec.CommandContext(cfg.ctx, gobin, args...)
|
||||
cmd.Env = envs
|
||||
cmd.Dir = cfg.buildDir()
|
||||
cmd.Stderr = os.Stderr
|
||||
cmd.Stdout = os.Stdout
|
||||
|
||||
return outpath, cmd.Run()
|
||||
}
|
||||
|
||||
func goBuildCmd(p v1.Platform, cfg *buildConfig) (gobin string, args []string, outpath string, err error) {
|
||||
/* TODO: Use Build Command override from the function if provided
|
||||
* A future PR will include the ability to specify a
|
||||
* f.Build.BuildCommand, or BuildArgs for use here to customize
|
||||
* This will be useful when, for example, the function is written in
|
||||
* Go and the function developer needs Libc compatibility, in which case
|
||||
* the default command will need to be replaced with:
|
||||
* go build -ldflags "-linkmode 'external' -extldflags '-static'"
|
||||
* Pseudocode:
|
||||
* if BuildArgs or BuildCommand
|
||||
* Validate command or args are safe to run
|
||||
* no other commands injected
|
||||
* does not contain Go's "toolexec"
|
||||
* does not specify the output path
|
||||
* Either replace or append to gobin
|
||||
*/
|
||||
|
||||
// Use the binary specified FUNC_GO_PATH if defined
|
||||
gobin = os.Getenv("FUNC_GO_PATH") // TODO: move to main and plumb through
|
||||
if gobin == "" {
|
||||
gobin = "go"
|
||||
}
|
||||
|
||||
// Build as ./func/builds/$PID/result/f.$OS.$Architecture
|
||||
name := fmt.Sprintf("f.%v.%v", p.OS, p.Architecture)
|
||||
if p.Variant != "" {
|
||||
name = name + "." + p.Variant
|
||||
}
|
||||
outpath = path(cfg.buildDir(), "result", name)
|
||||
args = []string{"build", "-o", outpath}
|
||||
return gobin, args, outpath, nil
|
||||
}
|
||||
|
||||
func goBuildEnvs(f fn.Function, p v1.Platform) (envs []string) {
|
||||
pegged := []string{
|
||||
"CGO_ENABLED=0",
|
||||
"GOOS=" + p.OS,
|
||||
"GOARCH=" + p.Architecture,
|
||||
}
|
||||
if p.Variant != "" && p.Architecture == "arm" {
|
||||
pegged = append(pegged, "GOARM="+strings.TrimPrefix(p.Variant, "v"))
|
||||
} else if p.Variant != "" && p.Architecture == "amd64" {
|
||||
pegged = append(pegged, "GOAMD64="+p.Variant)
|
||||
}
|
||||
|
||||
isPegged := func(env string) bool {
|
||||
for _, v := range pegged {
|
||||
name := strings.Split(v, "=")[0]
|
||||
if strings.HasPrefix(env, name) {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
|
||||
envs = append(envs, pegged...)
|
||||
for _, env := range os.Environ() {
|
||||
if !isPegged(env) {
|
||||
envs = append(envs, env)
|
||||
}
|
||||
}
|
||||
return envs
|
||||
}
|
||||
|
||||
func newExecTarball(source, target string, verbose bool) error {
|
||||
targetFile, err := os.Create(target)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer targetFile.Close()
|
||||
|
||||
gw := gzip.NewWriter(targetFile)
|
||||
defer gw.Close()
|
||||
|
||||
tw := tar.NewWriter(gw)
|
||||
defer tw.Close()
|
||||
|
||||
info, err := os.Stat(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header, err := tar.FileInfoHeader(info, info.Name())
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
header.Name = path("/func", "f")
|
||||
// TODO: should we set file timestamps to the build start time of cfg.t?
|
||||
// header.ModTime = timestampArgument
|
||||
|
||||
if err = tw.WriteHeader(header); err != nil {
|
||||
return err
|
||||
}
|
||||
if verbose {
|
||||
fmt.Printf("→ %v \n", header.Name)
|
||||
}
|
||||
|
||||
file, err := os.Open(source)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer file.Close()
|
||||
|
||||
i, err := io.Copy(tw, file)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if verbose {
|
||||
fmt.Printf(" wrote %v bytes \n", i)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
Loading…
Reference in New Issue