From 9c0c2161933d5387564c6dfcfd4b8eab2f2acd68 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Matej=20Va=C5=A1ek?= Date: Thu, 6 Feb 2025 08:02:13 +0100 Subject: [PATCH] Optimise func-utils image (#2686) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Use command instad of script in some tkn tasks The "script" requires /bin/sh present in the image. Signed-off-by: Matej Vašek * Add s2i-generate command to func-util image The command encompasses some logic previously implemented as shell script defined in tekton task. This allows us to remove sh/shell from the func-util image. Signed-off-by: Matej Vašek * Make func-util image "FROM scratch" Signed-off-by: Matej Vašek * Change func-utils image tag latest->v2 Since there are backward incompatible changes we must not change how 'latest' tag work (at least for some time). For this reason we change tag to v2, so newer versions of func use that and older use 'latest' that is compatible with them. Signed-off-by: Matej Vašek --------- Signed-off-by: Matej Vašek --- .github/workflows/ci.yaml | 7 +- Dockerfile.utils | 36 +------ Makefile | 2 +- cmd/func-util/main.go | 2 + cmd/func-util/s2i_generate.go | 143 +++++++++++++++++++++++++++ func-util-symlinks.tgz | Bin 0 -> 192 bytes hack/fetch-util-img-prerequisites.sh | 60 ----------- hack/setup-testing-images.sh | 6 +- pkg/k8s/dialer.go | 2 +- pkg/k8s/persistent_volumes.go | 2 +- pkg/pipelines/tekton/tasks.go | 54 ++++------ 11 files changed, 178 insertions(+), 136 deletions(-) create mode 100644 cmd/func-util/s2i_generate.go create mode 100644 func-util-symlinks.tgz delete mode 100755 hack/fetch-util-img-prerequisites.sh diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 03a3b8f22..0a7eb7d26 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -180,6 +180,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 + - uses: knative/actions/setup-go@main - uses: docker/setup-qemu-action@v3 - name: Login to GitHub Container Registry uses: docker/login-action@v3 @@ -189,12 +190,14 @@ jobs: password: ${{ secrets.GITHUB_TOKEN }} - name: Build and push run: | - ./hack/fetch-util-img-prerequisites.sh + for a in amd64 arm64 ppc64le s390x; do + CGO_ENABLED=0 go build -o "func-util-$a" -trimpath -ldflags '-w -s' ./cmd/func-util + done docker buildx create --name multiarch --driver docker-container --use docker buildx build . -f Dockerfile.utils \ --platform=linux/ppc64le,linux/s390x,linux/amd64,linux/arm64 \ --push \ - -t "ghcr.io/knative/func-utils:latest" \ + -t "ghcr.io/knative/func-utils:v2" \ --annotation index:org.opencontainers.image.description="Knative Func Utils Image" \ --annotation index:org.opencontainers.image.source="https://github.com/knative/func" \ --annotation index:org.opencontainers.image.vendor="https://github.com/knative/func" \ diff --git a/Dockerfile.utils b/Dockerfile.utils index c7e2b2f96..d0ba642ce 100644 --- a/Dockerfile.utils +++ b/Dockerfile.utils @@ -1,41 +1,13 @@ -FROM --platform=$BUILDPLATFORM scratch AS builder - -ARG BUILDPLATFORM -ARG BUILDARCH -ARG TARGETPLATFORM -ARG TARGETARCH - -ADD .artifacts/alpine-minirootfs-$BUILDARCH.tar.gz / - -COPY .artifacts/go.tar.gz /tmp/go.tar.gz -RUN tar -C /usr/local -xzf /tmp/go.tar.gz -ENV PATH="/usr/local/go/bin:$PATH" - -WORKDIR /workspace - -COPY go.mod go.sum ./ -RUN go mod download -x - -COPY . . - -RUN GOARCH=$TARGETARCH go build -o func-util -trimpath -ldflags '-w -s' ./cmd/func-util - -######################### - FROM scratch ARG TARGETARCH +ARG FUNC_UTIL_BINARY=func-util-$TARGETARCH -ADD .artifacts/alpine-minirootfs-$TARGETARCH.tar.gz / +ENV PATH=/ -RUN apk add --no-cache tar +COPY $FUNC_UTIL_BINARY /func-util -COPY --from=builder /workspace/func-util /usr/local/bin/ -RUN ln -s /usr/local/bin/func-util /usr/local/bin/deploy && \ - ln -s /usr/local/bin/func-util /usr/local/bin/scaffold && \ - ln -s /usr/local/bin/func-util /usr/local/bin/s2i && \ - ln -s /usr/local/bin/func-util /usr/local/bin/sh && \ - ln -s /usr/local/bin/func-util /usr/local/bin/socat +ADD func-util-symlinks.tgz / LABEL \ org.opencontainers.image.description="Knative Func Utils Image" \ diff --git a/Makefile b/Makefile index dd0fc9bab..9e4c72c8f 100644 --- a/Makefile +++ b/Makefile @@ -30,7 +30,7 @@ KVER ?= $(shell git describe --tags --match 'knative-*') LDFLAGS := -X knative.dev/func/pkg/app.vers=$(VERS) -X knative.dev/func/pkg/app.kver=$(KVER) -X knative.dev/func/pkg/app.hash=$(HASH) -FUNC_UTILS_IMG ?= ghcr.io/knative/func-utils:latest +FUNC_UTILS_IMG ?= ghcr.io/knative/func-utils:v2 LDFLAGS += -X knative.dev/func/pkg/k8s.SocatImage=$(FUNC_UTILS_IMG) LDFLAGS += -X knative.dev/func/pkg/k8s.TarImage=$(FUNC_UTILS_IMG) LDFLAGS += -X knative.dev/func/pkg/pipelines/tekton.DeployerImage=$(FUNC_UTILS_IMG) diff --git a/cmd/func-util/main.go b/cmd/func-util/main.go index 26900ece3..0857cb3fd 100644 --- a/cmd/func-util/main.go +++ b/cmd/func-util/main.go @@ -52,6 +52,8 @@ func main() { cmd = socat case "sh": cmd = sh + case "s2i-generate": + cmd = s2iGenerate } err := cmd(ctx) diff --git a/cmd/func-util/s2i_generate.go b/cmd/func-util/s2i_generate.go new file mode 100644 index 000000000..40c79abb3 --- /dev/null +++ b/cmd/func-util/s2i_generate.go @@ -0,0 +1,143 @@ +//go:build exclude_graphdriver_btrfs || !cgo +// +build exclude_graphdriver_btrfs !cgo + +package main + +import ( + "context" + "fmt" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/openshift/source-to-image/pkg/api" + "github.com/openshift/source-to-image/pkg/build" + "github.com/openshift/source-to-image/pkg/build/strategies" + "github.com/openshift/source-to-image/pkg/scm/git" + "github.com/spf13/cobra" + + fn "knative.dev/func/pkg/functions" +) + +func s2iGenerate(ctx context.Context) error { + cmd := newS2IGenerateCmd() + err := cmd.ExecuteContext(ctx) + if err != nil { + return fmt.Errorf("cannot s2i generate: %w", err) + } + return nil +} + +type genConfig struct { + target string + pathContext string + builderImage string + registry string + imageScriptUrl string + logLevel string + envVars []string +} + +func newS2IGenerateCmd() *cobra.Command { + var config genConfig + + genCmd := &cobra.Command{ + RunE: func(cmd *cobra.Command, args []string) error { + config.envVars = args + return runS2IGenerate(cmd.Context(), config) + }, + } + genCmd.Flags().StringVar(&config.target, "target", "/gen-source", "") + genCmd.Flags().StringVar(&config.pathContext, "path-context", ".", "") + genCmd.Flags().StringVar(&config.builderImage, "builder-image", "", "") + genCmd.Flags().StringVar(&config.registry, "registry", "", "") + genCmd.Flags().StringVar(&config.imageScriptUrl, "image-script-url", "image:///usr/libexec/s2i", "") + genCmd.Flags().StringVar(&config.logLevel, "log-level", "0", "") + + return genCmd +} + +func runS2IGenerate(ctx context.Context, c genConfig) error { + + wd, err := os.Getwd() + if err != nil { + return fmt.Errorf("cannot get working directory: %w", err) + } + + funcRoot := filepath.Join(wd, c.pathContext) + + // replace registry in func.yaml + f, err := fn.NewFunction(funcRoot) + if err != nil { + return fmt.Errorf("cannot load function: %w", err) + } + f.Registry = c.registry + err = f.Write() + if err != nil { + return fmt.Errorf("cannot write function: %w", err) + } + + // append node_modules into .s2iignore + s2iIgnorePath := filepath.Join(funcRoot, ".s2iignore") + if fi, _ := os.Stat(s2iIgnorePath); fi != nil { + var file *os.File + + file, err = os.OpenFile(s2iIgnorePath, os.O_APPEND|os.O_WRONLY, 0644) + if err != nil { + return fmt.Errorf("cannot open s2i ignore file for append: %w", err) + } + defer func(file *os.File) { + _ = file.Close() + + }(file) + + _, err = file.Write([]byte("\nnode_modules")) + if err != nil { + return fmt.Errorf("cannot append node_modules directory to s2i ignore file: %w", err) + } + } + + // prepare envvars + var envs = make([]api.EnvironmentSpec, 0, len(c.envVars)) + for _, e := range c.envVars { + var es api.EnvironmentSpec + part := strings.SplitN(e, "=", 2) + switch len(part) { + case 1: + es.Name = part[0] + case 2: + es.Name = part[0] + es.Value = part[1] + default: + continue + } + if es.Name != "" { + envs = append(envs, es) + } + } + + s2iConfig := api.Config{ + Source: &git.URL{ + URL: url.URL{Path: funcRoot}, + Type: git.URLTypeLocal, + }, + BuilderImage: c.builderImage, + ImageScriptsURL: c.imageScriptUrl, + KeepSymlinks: true, + Environment: envs, + AsDockerfile: filepath.Join(c.target, "Dockerfile.gen"), + } + + builder, _, err := strategies.Strategy(nil, &s2iConfig, build.Overrides{}) + if err != nil { + return fmt.Errorf("cannot create builder: %w", err) + } + + _, err = builder.Build(&s2iConfig) + if err != nil { + return fmt.Errorf("cannot build: %w", err) + } + + return nil +} diff --git a/func-util-symlinks.tgz b/func-util-symlinks.tgz new file mode 100644 index 0000000000000000000000000000000000000000..63939b94bcdf043ae9b0d08120dd96093dfedeae GIT binary patch literal 192 zcmb2|=3oE==C@ZIxehA`v_6zk71xlMZS{QOku8k^-X`7mcG)^gZ+fw)dm4)f*Z=L3 zj)_x_-`Mr~LB4BHmZDO2Y|XXOzctl&LRGhhxUSq?nZsB7roCnQZ09Y(kz$j-f4sdj z{&&6B3%knxw?CzoXZ`UAOEkf?)=ZP q#rv0B^}l-6C-a>@^!ol?C;PcR \ - "${tarball}" - curl -sSL "${alpine_rel_url}/${arch}/alpine-minirootfs-${alpine_version}-${arch}.tar.gz.asc" > \ - "${signature}" - gpg --assert-signer="$alpine_release_key" --verify "${signature}" "${tarball}" - done -} - -function fetch_golang() { - - local arch - arch="$(uname -m)" - local arch_alt="${arch_map[$arch]:-$arch}" - local tarball="${artifacts_dir}/go.tar.gz" - local signature="${artifacts_dir}/go.tar.gz.asc" - curl -sSL "https://go.dev/dl/go${go_version}.linux-${arch_alt}.tar.gz" > "${tarball}" - curl -sSL "https://go.dev/dl/go${go_version}.linux-${arch_alt}.tar.gz.asc" > "${signature}" - gpg --assert-signer="$google_release_key" --verify "${signature}" "${tarball}" -} - -function main() { - fetch_alpine_minirootfs - fetch_golang -} - -main "$@" diff --git a/hack/setup-testing-images.sh b/hack/setup-testing-images.sh index 3cc031829..53137c771 100755 --- a/hack/setup-testing-images.sh +++ b/hack/setup-testing-images.sh @@ -4,11 +4,11 @@ set -o errexit set -o nounset set -o pipefail -FUNC_UTILS_IMG="localhost:50000/knative/func-utils:latest" +FUNC_UTILS_IMG="localhost:50000/knative/func-utils:v2" -"$(dirname "$(realpath "$0")")/fetch-util-img-prerequisites.sh" +CGO_ENABLED=0 go build -o "func-util" -trimpath -ldflags '-w -s' ./cmd/func-util -docker build . -f Dockerfile.utils -t "${FUNC_UTILS_IMG}" +docker build . -f Dockerfile.utils -t "${FUNC_UTILS_IMG}" --build-arg FUNC_UTIL_BINARY=func-util docker push "${FUNC_UTILS_IMG}" # Build custom buildah image for tests. diff --git a/pkg/k8s/dialer.go b/pkg/k8s/dialer.go index 80f8067d4..dd1b6740d 100644 --- a/pkg/k8s/dialer.go +++ b/pkg/k8s/dialer.go @@ -28,7 +28,7 @@ import ( "k8s.io/client-go/tools/remotecommand" ) -var SocatImage = "ghcr.io/knative/func-utils:latest" +var SocatImage = "ghcr.io/knative/func-utils:v2" // NewInClusterDialer creates context dialer that will dial TCP connections via POD running in k8s cluster. // This is useful when accessing k8s services that are not exposed outside cluster (e.g. openshift image registry). diff --git a/pkg/k8s/persistent_volumes.go b/pkg/k8s/persistent_volumes.go index 6c0694f54..b46f4a216 100644 --- a/pkg/k8s/persistent_volumes.go +++ b/pkg/k8s/persistent_volumes.go @@ -67,7 +67,7 @@ func DeletePersistentVolumeClaims(ctx context.Context, namespaceOverride string, return client.CoreV1().PersistentVolumeClaims(namespace).DeleteCollection(ctx, metav1.DeleteOptions{}, listOptions) } -var TarImage = "ghcr.io/knative/func-utils:latest" +var TarImage = "ghcr.io/knative/func-utils:v2" // UploadToVolume uploads files (passed in form of tar stream) into volume. func UploadToVolume(ctx context.Context, content io.Reader, claimName, namespace string) error { diff --git a/pkg/pipelines/tekton/tasks.go b/pkg/pipelines/tekton/tasks.go index ed1d8349a..5770b2433 100644 --- a/pkg/pipelines/tekton/tasks.go +++ b/pkg/pipelines/tekton/tasks.go @@ -5,7 +5,7 @@ import ( "strings" ) -var DeployerImage = "ghcr.io/knative/func-utils:latest" +var DeployerImage = "ghcr.io/knative/func-utils:v2" func getBuildpackTask() string { return `apiVersion: tekton.dev/v1 @@ -306,37 +306,21 @@ spec: - name: generate image: %s workingDir: $(workspaces.source.path) - args: ["$(params.ENV_VARS[*])"] - script: | - echo "Processing Build Environment Variables" - echo "" > /env-vars/env-file - for var in "$@" - do - if [[ "$var" != "=" ]]; then - echo "$var" >> /env-vars/env-file - fi - done - - echo "Generated Build Env Var file" - echo "------------------------------" - cat /env-vars/env-file - echo "------------------------------" - - /usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build --keep-symlinks $(params.PATH_CONTEXT) $(params.BUILDER_IMAGE) \ - --image-scripts-url $(params.S2I_IMAGE_SCRIPTS_URL) \ - --as-dockerfile /gen-source/Dockerfile.gen --environment-file /env-vars/env-file - - echo "Preparing func.yaml for later deployment" - func_file="$(workspaces.source.path)/func.yaml" - if [ "$(params.PATH_CONTEXT)" != "" ]; then - func_file="$(workspaces.source.path)/$(params.PATH_CONTEXT)/func.yaml" - fi - sed -i "s|^registry:.*$|registry: $(params.REGISTRY)|" "$func_file" - echo "Function image registry: $(params.REGISTRY)" - - s2iignore_file="$(dirname "$func_file")/.s2iignore" - [ -f "$s2iignore_file" ] || echo "node_modules" >> "$s2iignore_file" - + command: + - s2i-generate + - "--target" + - /gen-source + - "--path-context" + - $(params.PATH_CONTEXT) + - "--builder-image" + - $(params.BUILDER_IMAGE) + - "--registry" + - $(params.REGISTRY) + - "--image-script-url" + - $(params.S2I_IMAGE_SCRIPTS_URL) + - "--log-level" + - $(params.LOGLEVEL) + - $(params.ENV_VARS[*]) volumeMounts: - mountPath: /gen-source name: gen-source @@ -418,8 +402,7 @@ spec: steps: - name: func-deploy image: "%s" - script: | - deploy $(params.path) "$(params.image)" + command: ["deploy", "$(params.path)", "$(params.image)"] `, DeployerImage) } @@ -446,8 +429,7 @@ spec: steps: - name: func-scaffold image: %s - script: | - scaffold $(params.path) + command: ["scaffold", "$(params.path)"] `, DeployerImage) }