feat: embed tkn tasks in func binary (#2396)

* feat: embed tkn tasks in func binary

Embed Tekton tasks for direct on cluster build.

Signed-off-by: Matej Vašek <mvasek@redhat.com>

* feat: embed tkn tasks in func binary

Embed Tekton tasks for PaC build.

Signed-off-by: Matej Vašek <mvasek@redhat.com>

* feat: added tkn-tasks sub-command

This new command prints tektons tasks in form of multi-document yaml,
these tekton tasks may requird to be installed for some advanced
functionality.

Signed-off-by: Matej Vašek <mvasek@redhat.com>

* chore: remove unused files

Signed-off-by: Matej Vašek <mvasek@redhat.com>

* feat: make deployer image settable via linker

Signed-off-by: Matej Vašek <mvasek@redhat.com>

---------

Signed-off-by: Matej Vašek <mvasek@redhat.com>

cmd/environment.go

@@ -16,7 +16,6 @@ import (
 	"knative.dev/func/pkg/config"
 	"knative.dev/func/pkg/functions"
 	"knative.dev/func/pkg/k8s"
-	"knative.dev/func/pkg/pipelines/tekton"
 )
 
 var format string = "json"
@@ -67,7 +66,6 @@ type Environment struct {
 	Templates            map[string][]string
 	Environment          []string
 	Cluster              string
-	TektonTasks          map[string]string
 	Defaults             config.Global
 	Function             *functions.Function `json:",omitempty" yaml:",omitempty"`
 	Instance             *functions.Instance `json:",omitempty" yaml:",omitempty"`
@@ -137,11 +135,6 @@ func runEnvironment(cmd *cobra.Command, newClient ClientFactory, v *Version) (er
 		Environment:          envs,
 		Cluster:              host,
 		Defaults:             defaults,
-		TektonTasks: map[string]string{
-			"func-buildpack": tekton.BuildpackTaskURL,
-			"func-s2i":       tekton.S2ITaskURL,
-			"func-deploy":    tekton.DeployTaskURL,
-		},
 	}
 
 	function, instance := describeFuncInformation(cmd.Context(), newClient, cfg)

cmd/root.go

@@ -107,6 +107,7 @@ Learn more about Knative at: https://knative.dev`, cfg.Name),
 			Commands: []*cobra.Command{
 				NewCompletionCmd(),
 				NewVersionCmd(cfg.Version),
+				NewTektonClusterTasksCmd(),
 			},
 		},
 	}

cmd/tkn_tasks.go

@@ -0,0 +1,27 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+
+	"knative.dev/func/pkg/pipelines/tekton"
+)
+
+func NewTektonClusterTasksCmd() *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "tkn-tasks",
+		Short: "List tekton cluster tasks as multi-document yaml",
+		Long: `This command prints tekton tekton task embed in the func binary.
+Some advanced functionality like OpenShift's Web Console build my require installation of these tasks.
+Installation: func tkn-tasks | kubectl apply -f -
+`,
+		Hidden: true,
+		RunE: func(cmd *cobra.Command, args []string) error {
+			_, err := fmt.Fprintln(cmd.OutOrStdout(), tekton.GetClusterTasks())
+			return err
+		},
+	}
+
+	return cmd
+}

docs/generator/main.go

@@ -96,6 +96,9 @@ func ignoreConfigEnv() (done func()) {
 // string to the file name, and recursively calls itself for each subcommand.
 func processSubCommands(c *cobra.Command, parent string, opts TemplateOptions) error {
 	for _, cc := range c.Commands() {
+		if cc.Hidden {
+			continue
+		}
 		name := cc.Name()
 		if name == "help" {
 			continue

pkg/pipelines/resources/tekton/task/func-deploy/0.1/func-deploy.yaml

@@ -1,29 +0,0 @@
-apiVersion: tekton.dev/v1
-kind: Task
-metadata:
-  name: func-deploy
-  labels:
-    app.kubernetes.io/version: "0.1"
-  annotations:
-    tekton.dev/pipelines.minVersion: "0.12.1"
-    tekton.dev/categories: CLI
-    tekton.dev/tags: cli
-    tekton.dev/platforms: "linux/amd64"
-spec:
-  description: >-
-    This Task performs a deploy operation using the Knative `func` CLI
-  params:
-    - name: path
-      description: Path to the function project
-      default: ""
-    - name: image
-      description: Container image to be deployed
-      default: ""
-  workspaces:
-    - name: source
-      description: The workspace containing the function project
-  steps:
-    - name: func-deploy
-      image: "ghcr.io/knative/func-utils:latest"
-      script: |
-        deploy $(params.path) "$(params.image)"

pkg/pipelines/resources/tekton/task/func-s2i/0.1/func-s2i.yaml

@@ -1,136 +0,0 @@
-apiVersion: tekton.dev/v1
-kind: Task
-metadata:
-  name: func-s2i
-  labels:
-    app.kubernetes.io/version: "0.1"
-  annotations:
-    tekton.dev/pipelines.minVersion: "0.17.0"
-    tekton.dev/categories: Image Build
-    tekton.dev/tags: image-build
-    tekton.dev/platforms: "linux/amd64"
-spec:
-  description: >-
-    Knative Functions Source-to-Image (S2I) is a toolkit and workflow for building reproducible
-    container images from source code
-
-    S2I produces images by injecting source code into a base S2I container image
-    and letting the container prepare that source code for execution. The base
-    S2I container images contains the language runtime and build tools needed for
-    building and running the source code.
-
-  params:
-    - name: BUILDER_IMAGE
-      description: The location of the s2i builder image.
-    - name: IMAGE
-      description: Reference of the image S2I will produce.
-    - name: REGISTRY
-      description: The registry associated with the function image.
-    - name: PATH_CONTEXT
-      description: The location of the path to run s2i from.
-      default: .
-    - name: TLSVERIFY
-      description: Verify the TLS on the registry endpoint (for push/pull to a non-TLS registry)
-      default: "true"
-    - name: LOGLEVEL
-      description: Log level when running the S2I binary
-      default: "0"
-    - name: ENV_VARS
-      type: array
-      description: Environment variables to set during _build-time_.
-      default: []
-    - name: S2I_IMAGE_SCRIPTS_URL
-      description: The URL containing the default assemble and run scripts for the builder image.
-      default: "image:///usr/libexec/s2i"
-  workspaces:
-    - name: source
-    - name: cache
-      description: Directory where cache is stored (e.g. local mvn repo).
-      optional: true
-    - name: sslcertdir
-      optional: true
-    - name: dockerconfig
-      description: >-
-        An optional workspace that allows providing a .docker/config.json file
-        for Buildah to access the container registry.
-        The file should be placed at the root of the Workspace with name config.json.
-      optional: true
-  results:
-    - name: IMAGE_DIGEST
-      description: Digest of the image just built.
-  steps:
-    - name: generate
-      image: quay.io/boson/s2i:latest
-      workingDir: $(workspaces.source.path)
-      args: ["$(params.ENV_VARS[*])"]
-      script: |
-        echo "Processing Build Environment Variables"
-        echo "" > /env-vars/env-file
-        for var in "$@"
-        do
-            if [[ "$var" != "=" ]]; then
-                echo "$var" >> /env-vars/env-file
-            fi
-        done
-
-        echo "Generated Build Env Var file"
-        echo "------------------------------"
-        cat /env-vars/env-file
-        echo "------------------------------"
-
-        /usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build $(params.PATH_CONTEXT) $(params.BUILDER_IMAGE) \
-        --image-scripts-url $(params.S2I_IMAGE_SCRIPTS_URL) \
-        --as-dockerfile /gen-source/Dockerfile.gen --environment-file /env-vars/env-file
-
-        echo "Preparing func.yaml for later deployment"
-        func_file="$(workspaces.source.path)/func.yaml"
-        if [ "$(params.PATH_CONTEXT)" != "" ]; then
-          func_file="$(workspaces.source.path)/$(params.PATH_CONTEXT)/func.yaml"
-        fi
-        sed -i "s|^registry:.*$|registry: $(params.REGISTRY)|" "$func_file"
-        echo "Function image registry: $(params.REGISTRY)"
-
-        s2iignore_file="$(dirname "$func_file")/.s2iignore"
-        [ -f "$s2iignore_file" ] || echo "node_modules" >> "$s2iignore_file"
-
-      volumeMounts:
-        - mountPath: /gen-source
-          name: gen-source
-        - mountPath: /env-vars
-          name: env-vars
-    - name: build
-      image: quay.io/buildah/stable:v1.31.0
-      workingDir: /gen-source
-      script: |
-        TLS_VERIFY_FLAG=""
-        if [ "$(params.TLSVERIFY)" = "false" ] || [ "$(params.TLSVERIFY)" = "0" ]; then
-          TLS_VERIFY_FLAG="--tls-verify=false"
-        fi
-
-        [[ "$(workspaces.sslcertdir.bound)" == "true" ]] && CERT_DIR_FLAG="--cert-dir $(workspaces.sslcertdir.path)"
-        ARTIFACTS_CACHE_PATH="$(workspaces.cache.path)/mvn-artifacts"
-        [ -d "${ARTIFACTS_CACHE_PATH}" ] || mkdir "${ARTIFACTS_CACHE_PATH}"
-        buildah ${CERT_DIR_FLAG} bud --storage-driver=vfs ${TLS_VERIFY_FLAG} --layers \
-          -v "${ARTIFACTS_CACHE_PATH}:/tmp/artifacts/:rw,z,U" \
-          -f /gen-source/Dockerfile.gen -t $(params.IMAGE) .
-
-        [[ "$(workspaces.dockerconfig.bound)" == "true" ]] && export DOCKER_CONFIG="$(workspaces.dockerconfig.path)"
-        buildah ${CERT_DIR_FLAG} push --storage-driver=vfs ${TLS_VERIFY_FLAG} --digestfile $(workspaces.source.path)/image-digest \
-          $(params.IMAGE) docker://$(params.IMAGE)
-
-        cat $(workspaces.source.path)/image-digest | tee /tekton/results/IMAGE_DIGEST
-      volumeMounts:
-      - name: varlibcontainers
-        mountPath: /var/lib/containers
-      - mountPath: /gen-source
-        name: gen-source
-      securityContext:
-        capabilities:
-          add: ["SETFCAP"]
-  volumes:
-    - emptyDir: {}
-      name: varlibcontainers
-    - emptyDir: {}
-      name: gen-source
-    - emptyDir: {}
-      name: env-vars

pkg/pipelines/tekton/tasks.go

@@ -1,5 +1,14 @@
----
+package tekton
-apiVersion: tekton.dev/v1
+
+import (
+	"fmt"
+	"strings"
+)
+
+var DeployerImage = "ghcr.io/knative/func-utils:latest"
+
+func getBuildpackTask() string {
+	return `apiVersion: tekton.dev/v1
 kind: Task
 metadata:
   name: func-buildpacks
@@ -37,7 +46,7 @@ spec:
     - name: BUILDER_IMAGE
       description: The image on which builds will run (must include lifecycle and compatible buildpacks).
     - name: SOURCE_SUBPATH
-      description: A subpath within the `source` input where the source to build is located.
+      description: A subpath within the "source" input where the source to build is located.
       default: ""
     - name: ENV_VARS
       type: array
@@ -66,7 +75,7 @@ spec:
 
   results:
     - name: IMAGE_DIGEST
-      description: The digest of the built `APP_IMAGE`.
+      description: The digest of the built "APP_IMAGE".
 
   stepTemplate:
     env:
@@ -228,3 +237,186 @@ spec:
       emptyDir: {}
     - name: layers-dir
       emptyDir: {}
+`
+}
+
+func getS2ITask() string {
+	return `apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  name: func-s2i
+  labels:
+    app.kubernetes.io/version: "0.1"
+  annotations:
+    tekton.dev/pipelines.minVersion: "0.17.0"
+    tekton.dev/categories: Image Build
+    tekton.dev/tags: image-build
+    tekton.dev/platforms: "linux/amd64"
+spec:
+  description: >-
+    Knative Functions Source-to-Image (S2I) is a toolkit and workflow for building reproducible
+    container images from source code
+
+    S2I produces images by injecting source code into a base S2I container image
+    and letting the container prepare that source code for execution. The base
+    S2I container images contains the language runtime and build tools needed for
+    building and running the source code.
+
+  params:
+    - name: BUILDER_IMAGE
+      description: The location of the s2i builder image.
+    - name: IMAGE
+      description: Reference of the image S2I will produce.
+    - name: REGISTRY
+      description: The registry associated with the function image.
+    - name: PATH_CONTEXT
+      description: The location of the path to run s2i from.
+      default: .
+    - name: TLSVERIFY
+      description: Verify the TLS on the registry endpoint (for push/pull to a non-TLS registry)
+      default: "true"
+    - name: LOGLEVEL
+      description: Log level when running the S2I binary
+      default: "0"
+    - name: ENV_VARS
+      type: array
+      description: Environment variables to set during _build-time_.
+      default: []
+    - name: S2I_IMAGE_SCRIPTS_URL
+      description: The URL containing the default assemble and run scripts for the builder image.
+      default: "image:///usr/libexec/s2i"
+  workspaces:
+    - name: source
+    - name: cache
+      description: Directory where cache is stored (e.g. local mvn repo).
+      optional: true
+    - name: sslcertdir
+      optional: true
+    - name: dockerconfig
+      description: >-
+        An optional workspace that allows providing a .docker/config.json file
+        for Buildah to access the container registry.
+        The file should be placed at the root of the Workspace with name config.json.
+      optional: true
+  results:
+    - name: IMAGE_DIGEST
+      description: Digest of the image just built.
+  steps:
+    - name: generate
+      image: quay.io/boson/s2i:latest
+      workingDir: $(workspaces.source.path)
+      args: ["$(params.ENV_VARS[*])"]
+      script: |
+        echo "Processing Build Environment Variables"
+        echo "" > /env-vars/env-file
+        for var in "$@"
+        do
+            if [[ "$var" != "=" ]]; then
+                echo "$var" >> /env-vars/env-file
+            fi
+        done
+
+        echo "Generated Build Env Var file"
+        echo "------------------------------"
+        cat /env-vars/env-file
+        echo "------------------------------"
+
+        /usr/local/bin/s2i --loglevel=$(params.LOGLEVEL) build $(params.PATH_CONTEXT) $(params.BUILDER_IMAGE) \
+        --image-scripts-url $(params.S2I_IMAGE_SCRIPTS_URL) \
+        --as-dockerfile /gen-source/Dockerfile.gen --environment-file /env-vars/env-file
+
+        echo "Preparing func.yaml for later deployment"
+        func_file="$(workspaces.source.path)/func.yaml"
+        if [ "$(params.PATH_CONTEXT)" != "" ]; then
+          func_file="$(workspaces.source.path)/$(params.PATH_CONTEXT)/func.yaml"
+        fi
+        sed -i "s|^registry:.*$|registry: $(params.REGISTRY)|" "$func_file"
+        echo "Function image registry: $(params.REGISTRY)"
+
+        s2iignore_file="$(dirname "$func_file")/.s2iignore"
+        [ -f "$s2iignore_file" ] || echo "node_modules" >> "$s2iignore_file"
+
+      volumeMounts:
+        - mountPath: /gen-source
+          name: gen-source
+        - mountPath: /env-vars
+          name: env-vars
+    - name: build
+      image: quay.io/buildah/stable:v1.31.0
+      workingDir: /gen-source
+      script: |
+        TLS_VERIFY_FLAG=""
+        if [ "$(params.TLSVERIFY)" = "false" ] || [ "$(params.TLSVERIFY)" = "0" ]; then
+          TLS_VERIFY_FLAG="--tls-verify=false"
+        fi
+
+        [[ "$(workspaces.sslcertdir.bound)" == "true" ]] && CERT_DIR_FLAG="--cert-dir $(workspaces.sslcertdir.path)"
+        ARTIFACTS_CACHE_PATH="$(workspaces.cache.path)/mvn-artifacts"
+        [ -d "${ARTIFACTS_CACHE_PATH}" ] || mkdir "${ARTIFACTS_CACHE_PATH}"
+        buildah ${CERT_DIR_FLAG} bud --storage-driver=vfs ${TLS_VERIFY_FLAG} --layers \
+          -v "${ARTIFACTS_CACHE_PATH}:/tmp/artifacts/:rw,z,U" \
+          -f /gen-source/Dockerfile.gen -t $(params.IMAGE) .
+
+        [[ "$(workspaces.dockerconfig.bound)" == "true" ]] && export DOCKER_CONFIG="$(workspaces.dockerconfig.path)"
+        buildah ${CERT_DIR_FLAG} push --storage-driver=vfs ${TLS_VERIFY_FLAG} --digestfile $(workspaces.source.path)/image-digest \
+          $(params.IMAGE) docker://$(params.IMAGE)
+
+        cat $(workspaces.source.path)/image-digest | tee /tekton/results/IMAGE_DIGEST
+      volumeMounts:
+      - name: varlibcontainers
+        mountPath: /var/lib/containers
+      - mountPath: /gen-source
+        name: gen-source
+      securityContext:
+        capabilities:
+          add: ["SETFCAP"]
+  volumes:
+    - emptyDir: {}
+      name: varlibcontainers
+    - emptyDir: {}
+      name: gen-source
+    - emptyDir: {}
+      name: env-vars
+`
+}
+
+func getDeployTask() string {
+	return fmt.Sprintf(`apiVersion: tekton.dev/v1
+kind: Task
+metadata:
+  name: func-deploy
+  labels:
+    app.kubernetes.io/version: "0.1"
+  annotations:
+    tekton.dev/pipelines.minVersion: "0.12.1"
+    tekton.dev/categories: CLI
+    tekton.dev/tags: cli
+    tekton.dev/platforms: "linux/amd64"
+spec:
+  description: >-
+    This Task performs a deploy operation using the Knative "func"" CLI
+  params:
+    - name: path
+      description: Path to the function project
+      default: ""
+    - name: image
+      description: Container image to be deployed
+      default: ""
+  workspaces:
+    - name: source
+      description: The workspace containing the function project
+  steps:
+    - name: func-deploy
+      image: "%s"
+      script: |
+        deploy $(params.path) "$(params.image)"
+`, DeployerImage)
+}
+
+// GetClusterTasks returns multi-document yaml containing tekton tasks used by func.
+func GetClusterTasks() string {
+	tasks := getBuildpackTask() + "\n---\n" + getS2ITask() + "\n---\n" + getDeployTask()
+	tasks = strings.Replace(tasks, "kind: Task", "kind: ClusterTask", -1)
+	tasks = strings.ReplaceAll(tasks, "apiVersion: tekton.dev/v1", "apiVersion: tekton.dev/v1beta1")
+	return tasks
+}

pkg/pipelines/tekton/templates.go

@@ -3,7 +3,6 @@ package tekton
 import (
 	"bytes"
 	"fmt"
-	"net/http"
 	"os"
 	"path"
 	"strings"
@@ -87,30 +86,6 @@ const (
 	defaultPipelinesTargetBranch = "main"
 )
 
-var (
-	FuncRepoRef       = "knative/func"
-	FuncRepoBranchRef = "main"
-
-	taskBasePath = "https://raw.githubusercontent.com/" +
-		FuncRepoRef + "/" + FuncRepoBranchRef + "/pkg/pipelines/resources/tekton/task/"
-	BuildpackTaskURL string
-	S2ITaskURL       string
-	DeployTaskURL    string
-)
-
-// sets the values only if they have not been set via the linker "-X" flag
-func init() {
-	if BuildpackTaskURL == "" {
-		BuildpackTaskURL = taskBasePath + "func-buildpacks/0.2/func-buildpacks.yaml"
-	}
-	if S2ITaskURL == "" {
-		S2ITaskURL = taskBasePath + "func-s2i/0.1/func-s2i.yaml"
-	}
-	if DeployTaskURL == "" {
-		DeployTaskURL = taskBasePath + "func-deploy/0.1/func-deploy.yaml"
-	}
-}
-
 type templateData struct {
 	FunctionName  string
 	Annotations   map[string]string
@@ -152,15 +127,27 @@ type templateData struct {
 // it creates the resource in the project directory
 func createPipelineTemplatePAC(f fn.Function, labels map[string]string) error {
 	data := templateData{
-		FunctionName:          f.Name,
+		FunctionName:         f.Name,
-		Annotations:           f.Deploy.Annotations,
+		Annotations:          f.Deploy.Annotations,
-		Labels:                labels,
+		Labels:               labels,
-		PipelineName:          getPipelineName(f),
+		PipelineName:         getPipelineName(f),
-		RunAfterFetchSources:  runAfterFetchSourcesRef,
+		RunAfterFetchSources: runAfterFetchSourcesRef,
-		GitCloneTaskRef:       taskGitClonePACTaskRef,
+		GitCloneTaskRef:      taskGitClonePACTaskRef,
-		FuncBuildpacksTaskRef: taskFuncBuildpacksPACTaskRef,
+	}
-		FuncS2iTaskRef:        taskFuncS2iPACTaskRef,
+
-		FuncDeployTaskRef:     taskFuncDeployPACTaskRef,
+	for _, val := range []struct {
+		ref   string
+		field *string
+	}{
+		{getBuildpackTask(), &data.FuncBuildpacksTaskRef},
+		{getS2ITask(), &data.FuncS2iTaskRef},
+		{getDeployTask(), &data.FuncDeployTaskRef},
+	} {
+		ts, err := getTaskSpec(val.ref)
+		if err != nil {
+			return err
+		}
+		*val.field = ts
 	}
 
 	var template string
@@ -226,10 +213,7 @@ func createPipelineRunTemplatePAC(f fn.Function, labels map[string]string) error
 
 		PipelinesTargetBranch: pipelinesTargetBranch,
 
-		GitCloneTaskRef:       taskGitCloneRef,
+		GitCloneTaskRef: taskGitCloneRef,
-		FuncBuildpacksTaskRef: BuildpackTaskURL,
-		FuncS2iTaskRef:        S2ITaskURL,
-		FuncDeployTaskRef:     DeployTaskURL,
 
 		PipelineYamlURL: fmt.Sprintf("%s/%s", resourcesDirectory, pipelineFileNamePAC),
 
@@ -304,17 +288,10 @@ func deleteAllPipelineTemplates(f fn.Function) string {
 	return ""
 }
 
-func getTaskSpec(taskUrlTemplate string) (string, error) {
+func getTaskSpec(taskYaml string) (string, error) {
-	resp, err := http.Get(taskUrlTemplate)
+	var err error
-	if err != nil {
-		return "", err
-	}
-	if resp.StatusCode != http.StatusOK {
-		return "", fmt.Errorf("cannot get task: %q bad http code: %d", taskUrlTemplate, resp.StatusCode)
-	}
-	defer resp.Body.Close()
 	var data map[string]any
-	dec := yaml.NewDecoder(resp.Body)
+	dec := yaml.NewDecoder(strings.NewReader(taskYaml))
 	err = dec.Decode(&data)
 	if err != nil {
 		return "", err
@@ -361,9 +338,9 @@ func createAndApplyPipelineTemplate(f fn.Function, namespace string, labels map[
 		ref   string
 		field *string
 	}{
-		{BuildpackTaskURL, &data.FuncBuildpacksTaskRef},
+		{getBuildpackTask(), &data.FuncBuildpacksTaskRef},
-		{S2ITaskURL, &data.FuncS2iTaskRef},
+		{getS2ITask(), &data.FuncS2iTaskRef},
-		{DeployTaskURL, &data.FuncDeployTaskRef},
+		{getDeployTask(), &data.FuncDeployTaskRef},
 	} {
 		ts, err := getTaskSpec(val.ref)
 		if err != nil {

pkg/pipelines/tekton/templates_pack.go

@@ -157,15 +157,6 @@ metadata:
     # Fetch the git-clone task from hub
     pipelinesascode.tekton.dev/task: {{.GitCloneTaskRef}}
 
-    # Fetch the func-buildpacks task
-    pipelinesascode.tekton.dev/task-1: {{.FuncBuildpacksTaskRef}}
-
-    # Fetch the func-deploy task
-    pipelinesascode.tekton.dev/task-2: {{.FuncDeployTaskRef}}
-
-    # Fetch the pipelie definition from the .tekton directory
-    pipelinesascode.tekton.dev/pipeline: {{.PipelineYamlURL}}
-
     # How many runs we want to keep attached to this event
     pipelinesascode.tekton.dev/max-keep-runs: "5"