func/pkg/oci/containerize.go

package oci

import (
	"archive/tar"
	"compress/gzip"
	"encoding/json"
	"fmt"
	"io"
	"io/fs"
	"os"
	"os/exec"
	slashpath "path"
	"path/filepath"
	"strings"
	"time"

	v1 "github.com/google/go-containerregistry/pkg/v1"
	"github.com/google/go-containerregistry/pkg/v1/tarball"
	"github.com/google/go-containerregistry/pkg/v1/types"
	"github.com/pkg/errors"
)

// languageLayerBuilder builds the layer for the given language whuch may
// be different from one platform to another.  For example, this is the
// layer in the image which contains the Go cross-compiled binary.
type languageLayerBuilder func(*buildConfig, v1.Platform) (v1.Descriptor, v1.Layer, error)

var languageLayerBuilders = map[string]languageLayerBuilder{
	"go":     buildGoLayer,
	"python": layerBuilderNotImplemented,
	"node":   layerBuilderNotImplemented,
	"rust":   layerBuilderNotImplemented,
}

func layerBuilderNotImplemented(cfg *buildConfig, _ v1.Platform) (d v1.Descriptor, l v1.Layer, err error) {
	err = fmt.Errorf("%v functions are not yet supported by the host builder", cfg.f.Runtime)
	return
}

func getLanguageLayerBuilder(cfg *buildConfig) (l languageLayerBuilder, err error) {
	// use the custom implementation, if provided
	if cfg.buildFn != nil {
		return cfg.buildFn, nil
	}
	// otherwise lookup the build function
	l, ok := languageLayerBuilders[cfg.f.Runtime]
	if !ok {
		err = fmt.Errorf("the language runtime '%v' is not a recognized language by the host builder", cfg.f.Runtime)
		return
	}
	return
}

// containerize the scaffolded project by creating and writing an OCI
// conformant directory structure into the functions .func/builds directory.
// The source code to be containerized is indicated by cfg.dir
func containerize(cfg *buildConfig) (err error) {
	// Create the required directories: oci/blobs/sha256
	if err = os.MkdirAll(cfg.blobsDir(), os.ModePerm); err != nil {
		return
	}

	// Create the static, required oci-layout metadata file
	if err = os.WriteFile(path(cfg.ociDir(), "oci-layout"),
		[]byte(`{ "imageLayoutVersion": "1.0.0" }`), os.ModePerm); err != nil {
		return
	}

	// Create the data layer and its descriptor
	dataDesc, dataLayer, err := newDataLayer(cfg) // shared
	if err != nil {
		return
	}

	// Create the root certificates layer and its decriptor
	certsDesc, certsLayer, err := newCertsLayer(cfg) // shared
	if err != nil {
		return
	}

	// Create an image for each platform consisting of the shared data layer,
	// the shared root certs layer, and an os/platform specific layer.
	imageDescs := []v1.Descriptor{}
	for _, p := range cfg.platforms {
		imageDesc, err := newImage(cfg, dataDesc, dataLayer, certsDesc, certsLayer, p, cfg.verbose)
		if err != nil {
			return err
		}
		imageDescs = append(imageDescs, imageDesc)
	}

	// Create the Image Index which enumerates all images contained within
	// the container.
	_, err = newImageIndex(cfg, imageDescs)
	return
}

// newDataLayer creates the shared data layer in the container file hierarchy and
// returns both its descriptor and layer metadata.
func newDataLayer(cfg *buildConfig) (desc v1.Descriptor, layer v1.Layer, err error) {

	// Create the data tarball
	// TODO: try WithCompressedCaching?
	source := cfg.f.Root // The source is the function's entire filesystem
	target := path(cfg.buildDir(), "datalayer.tar.gz")

	if err = newDataTarball(source, target, defaultIgnored, cfg.verbose); err != nil {
		return
	}

	// Layer
	if layer, err = tarball.LayerFromFile(target); err != nil {
		return
	}

	// Descriptor
	if desc, err = newDescriptor(layer); err != nil {
		return
	}

	// Blob
	blob := path(cfg.blobsDir(), desc.Digest.Hex)
	if cfg.verbose {
		fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), target), rel(cfg.buildDir(), blob))
	}
	err = os.Rename(target, blob)
	return
}

func newDataTarball(root, target string, ignored []string, verbose bool) error {
	targetFile, err := os.Create(target)
	if err != nil {
		return err
	}
	defer targetFile.Close()

	gw := gzip.NewWriter(targetFile)
	defer gw.Close()

	tw := tar.NewWriter(gw)
	defer tw.Close()

	return filepath.Walk(root, func(path string, info os.FileInfo, err error) error {
		if err != nil {
			return err
		}

		// Skip files explicitly ignored
		for _, v := range ignored {
			if info.Name() == v {
				if info.IsDir() {
					return filepath.SkipDir
				}
				return nil
			}
		}

		lnk := "" // if link, this will be used as the target
		if info.Mode()&fs.ModeSymlink != 0 {
			if lnk, err = validatedLinkTarget(root, path); err != nil {
				return err
			}
		}

		header, err := tar.FileInfoHeader(info, lnk)
		if err != nil {
			return err
		}

		relPath, err := filepath.Rel(root, path)
		if err != nil {
			return err
		}
		header.Name = slashpath.Join("/func", filepath.ToSlash(relPath))
		if err := tw.WriteHeader(header); err != nil {
			return err
		}
		if verbose {
			fmt.Printf("→ %v \n", header.Name)
		}
		if !info.Mode().IsRegular() { //nothing more to do for non-regular
			return nil
		}

		file, err := os.Open(path)
		if err != nil {
			return err
		}
		defer file.Close()

		_, err = io.Copy(tw, file)
		return err
	})
}

// validatedLinkTarget returns the target of a given link or an error if
// that target is either absolute or outside the given project root.
func validatedLinkTarget(root, path string) (tgt string, err error) {
	// tgt is the raw target of the link.
	// This path is either absolute or relative to the link's location.
	tgt, err = os.Readlink(path)
	if err != nil {
		return tgt, fmt.Errorf("cannot read link: %w", err)
	}

	// Absolute links will not be correct when copied into the runtime
	// container, because they are placed into path into '/func',
	if filepath.IsAbs(tgt) {
		return tgt, errors.New("project may not contain absolute links")
	}

	// Calculate the actual target of the link
	// (relative to the parent of the symlink)
	lnkTgt := filepath.Join(filepath.Dir(path), tgt)

	// Calculate the relative path from the function's root to
	// this actual target location
	relLnkTgt, err := filepath.Rel(root, lnkTgt)
	if err != nil {
		return
	}

	// Fail if this path is outside the function's root.
	if strings.HasPrefix(relLnkTgt, ".."+string(filepath.Separator)) || relLnkTgt == ".." {
		return tgt, errors.New("links must stay within project root")
	}
	return
}

// newCertLayer creates the shared data layer in the container file hierarchy and
// returns both its descriptor and layer metadata.
func newCertsLayer(cfg *buildConfig) (desc v1.Descriptor, layer v1.Layer, err error) {

	// Create the data tarball
	// TODO: try WithCompressedCaching?
	source := filepath.Join(cfg.buildDir(), "ca-certificates.crt")
	target := path(cfg.buildDir(), "certslayer.tar.gz")

	if err = newCertsTarball(source, target, defaultIgnored, cfg.verbose); err != nil {
		return
	}

	// Layer
	if layer, err = tarball.LayerFromFile(target); err != nil {
		return
	}

	// Descriptor
	if desc, err = newDescriptor(layer); err != nil {
		return
	}

	// Blob
	blob := path(cfg.blobsDir(), desc.Digest.Hex)
	if cfg.verbose {
		fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), target), rel(cfg.buildDir(), blob))
	}
	err = os.Rename(target, blob)
	return
}

func newCertsTarball(source, target string, ignored []string, verbose bool) error {
	targetFile, err := os.Create(target)
	if err != nil {
		return err
	}
	defer targetFile.Close()

	gw := gzip.NewWriter(targetFile)
	defer gw.Close()

	tw := tar.NewWriter(gw)
	defer tw.Close()

	paths := []string{
		"/etc/ssl/certs/ca-certificates.crt",
		"/etc/pki/tls/certs/ca-certificates.crt",
	}

	fi, err := os.Stat(source)
	if err != nil {
		return err
	}

	// For each ssl certs path we want to create
	for _, path := range paths {
		// Create a header for it
		header, err := tar.FileInfoHeader(fi, "")
		if err != nil {
			return err
		}
		header.Name = path

		if err := tw.WriteHeader(header); err != nil {
			return err
		}
		if verbose {
			fmt.Printf("→ %v \n", header.Name)
		}
		file, err := os.Open(source)
		if err != nil {
			return err
		}
		defer file.Close()
		_, err = io.Copy(tw, file)
		if err != nil {
			return err
		}
	}

	return nil
}

func newDescriptor(layer v1.Layer) (desc v1.Descriptor, err error) {
	size, err := layer.Size()
	if err != nil {
		return
	}
	digest, err := layer.Digest()
	if err != nil {
		return
	}
	return v1.Descriptor{
		MediaType: types.OCILayer,
		Size:      size,
		Digest:    digest,
	}, nil
}

// newImage creates an image for the given platform.
// The image consists of the shared data layer which is provided
func newImage(cfg *buildConfig, dataDesc v1.Descriptor, dataLayer v1.Layer, certsDesc v1.Descriptor, certsLayer v1.Layer, p v1.Platform, verbose bool) (imageDesc v1.Descriptor, err error) {
	buildFn, err := getLanguageLayerBuilder(cfg)
	if err != nil {
		return
	}

	// Write Exec Layer as Blob -> Layer
	execDesc, execLayer, err := buildFn(cfg, p)
	if err != nil {
		return
	}

	// Write Config Layer as Blob -> Layer
	configDesc, _, err := newConfig(cfg, p, dataLayer, certsLayer, execLayer)
	if err != nil {
		return
	}

	// Image Manifest
	image := v1.Manifest{
		SchemaVersion: 2,
		MediaType:     types.OCIManifestSchema1,
		Config:        configDesc,
		Layers:        []v1.Descriptor{dataDesc, certsDesc, execDesc},
	}

	// Write image manifest out as json to a tempfile
	filePath := fmt.Sprintf("image.%v.%v.json", p.OS, p.Architecture)
	file, err := os.Create(filePath)
	if err != nil {
		return
	}
	enc := json.NewEncoder(file)
	enc.SetIndent("", "  ")
	if err = enc.Encode(image); err != nil {
		return
	}
	if err = file.Close(); err != nil {
		return
	}

	// Create a descriptor from hash and size
	file, err = os.Open(filePath)
	if err != nil {
		return
	}
	hash, size, err := v1.SHA256(file)
	if err != nil {
		return
	}
	imageDesc = v1.Descriptor{
		MediaType: types.OCIManifestSchema1,
		Digest:    hash,
		Size:      size,
		Platform:  &p,
	}
	if err = file.Close(); err != nil {
		return
	}

	// move image into blobs
	blob := path(cfg.blobsDir(), hash.Hex)
	if cfg.verbose {
		fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), filePath), rel(cfg.buildDir(), blob))
	}
	err = os.Rename(filePath, blob)
	return
}

func newConfig(cfg *buildConfig, p v1.Platform, layers ...v1.Layer) (desc v1.Descriptor, config v1.ConfigFile, err error) {
	volumes := make(map[string]struct{}) // Volumes are odd, see spec.
	for _, v := range cfg.f.Run.Volumes {
		if v.Path == nil {
			continue // TODO: remove pointers from Volume and Env struct members
		}
		volumes[*v.Path] = struct{}{}
	}

	rootfs := v1.RootFS{
		Type: "layers",
	}
	var diff v1.Hash
	for _, v := range layers {
		if v == nil {
			continue
		}
		if diff, err = v.DiffID(); err != nil {
			return
		}
		rootfs.DiffIDs = append(rootfs.DiffIDs, diff)
	}

	config = v1.ConfigFile{
		Created:      v1.Time{Time: cfg.t},
		Architecture: p.Architecture,
		OS:           p.OS,
		OSVersion:    p.OSVersion,
		// OSFeatures:   p.OSFeatures, // TODO: need to update dep to get this
		Variant: p.Variant,
		Config: v1.Config{
			ExposedPorts: map[string]struct{}{"8080/tcp": {}},
			Env:          newConfigEnvs(cfg),
			Cmd:          []string{"/func/f"}, // NOTE: Using Cmd because Entrypoint can not be overridden
			WorkingDir:   "/func/",
			StopSignal:   "SIGKILL",
			User:         "1000",
			Volumes:      volumes,
			// Labels
			// History
		},
		RootFS: rootfs,
	}

	// Write the config out as json to a tempfile
	filePath := path(cfg.buildDir(), "config.json")
	file, err := os.Create(filePath)
	if err != nil {
		return
	}
	enc := json.NewEncoder(file)
	enc.SetIndent("", "  ")
	if err = enc.Encode(config); err != nil {
		return
	}
	if err = file.Close(); err != nil {
		return
	}

	// Create a descriptor using hash and size
	file, err = os.Open(filePath)
	if err != nil {
		return
	}
	hash, size, err := v1.SHA256(file)
	if err != nil {
		return
	}
	desc = v1.Descriptor{
		MediaType: types.OCIConfigJSON,
		Digest:    hash,
		Size:      size,
	}
	if err = file.Close(); err != nil {
		return
	}

	// move config into blobs
	blobPath := path(cfg.blobsDir(), hash.Hex)
	if cfg.verbose {
		fmt.Printf("mv %v %v\n", rel(cfg.buildDir(), filePath), rel(cfg.buildDir(), blobPath))
	}
	err = os.Rename(filePath, blobPath)
	return
}

// newConfigEnvs returns the final set of environment variables to build into
// the container.  This consists of func-provided build metadata envs as well
// as any environment variables provided on the function itself.
func newConfigEnvs(cfg *buildConfig) []string {
	envs := []string{}

	// FUNC_CREATED
	// Formats container timestamp as RFC3339; a stricter version of the ISO 8601
	// format used by the container image manifest's 'Created' attribute.
	envs = append(envs, "FUNC_CREATED="+cfg.t.Format(time.RFC3339))

	// FUNC_VERSION
	// If source controlled, and if being built from a system with git, the
	// environment FUNC_VERSION will be populated.  Otherwise it will exist
	// (to indicate this logic was executed) but have an empty value.
	if cfg.verbose {
		fmt.Printf("cd %v && export FUNC_VERSION=$(git describe --tags)\n", cfg.f.Root)
	}
	cmd := exec.CommandContext(cfg.ctx, "git", "describe", "--tags")
	cmd.Dir = cfg.f.Root
	output, err := cmd.Output()
	if err != nil {
		if cfg.verbose {
			fmt.Fprintf(os.Stderr, "unable to determine function version. %v", err)
		}
		envs = append(envs, "FUNC_VERSION=")
	} else {
		envs = append(envs, "FUNC_VERSION="+strings.TrimSpace(string(output)))
	}

	// TODO: OTHERS?
	// Other metadata that may be useful. Perhaps:
	//   - func client version (func cli) used when building this file?
	//   - user/environment which triggered this build?
	//   - A reflection of the function itself?  Image, registry, etc. etc?

	// ENVs defined on the Function
	return append(envs, cfg.f.Run.Envs.Slice()...)
}

func newImageIndex(cfg *buildConfig, imageDescs []v1.Descriptor) (index v1.IndexManifest, err error) {
	index = v1.IndexManifest{
		SchemaVersion: 2,
		MediaType:     types.OCIImageIndex,
		Manifests:     imageDescs,
	}

	filePath := path(cfg.ociDir(), "index.json")
	file, err := os.Create(filePath)
	if err != nil {
		return
	}
	defer file.Close()

	enc := json.NewEncoder(file)
	enc.SetIndent("", "  ")
	err = enc.Encode(index)
	return
}

// rel is a simple prefix trim used exclusively for verbose debugging
// statements to print paths as relative to the current build directory
// rather than absolute. Returns the path relative to the current working
// build directory.  If it is not a subpath, the full path is returned
// unchanged.
func rel(base, path string) string {
	if strings.HasPrefix(path, base) {
		return "." + strings.TrimPrefix(path, base)
	}
	return path
}