Generate templates zip unconditionally (#879)

* Improve formant performance in template generation

Signed-off-by: Matej Vasek <mvasek@redhat.com>

* Improve memory footprint of template generation

Use streamlined generation which is limiting need for in memory buffers.
Might be imprtant if we had big files in templates.

Signed-off-by: Matej Vasek <mvasek@redhat.com>

* Generate templates unconditionally

Rationale:
The way we create prerequisites (the find command) we cannot detect deleted files.
The generation is fast (around 500ms) so we can afford that.

Signed-off-by: Matej Vasek <mvasek@redhat.com>

* Test embbeded templates content on push to main

This has to run prior to `make test` because the `test` recipe
enforces generation of the templates zip.
This would prevent test from being run against git commited templates
zip.

Signed-off-by: Matej Vasek <mvasek@redhat.com>

* Optimization don't use unnecessary io.Pipe

Signed-off-by: Matej Vasek <mvasek@redhat.com>

.github/workflows/ci.yaml

@@ -15,6 +15,8 @@ jobs:
       - uses: actions/setup-java@v1
         with:
           java-version: '11'
+      - name: Check embedded templates content
+        run: go test -run "^\QTestFileSystems\E$/^\Qembedded\E$"
       - name: Unit Test
         run:  make test
       - name: Template Unit Tests

Makefile

@@ -60,7 +60,9 @@ check: bin/golangci-lint ## Check code quality (lint)
 bin/golangci-lint:
 	curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b ./bin v1.43.0
 
-zz_filesystem_generated.go: $(TEMPLATES)
+.PHONY: zz_filesystem_generated.go
+
+zz_filesystem_generated.go:
 	# Removing temporary template files
 	@rm -rf templates/node/cloudevents/node_modules
 	@rm -rf templates/node/http/node_modules

generate/templates/main.go

@@ -3,8 +3,6 @@ package main
 import (
 	"archive/zip"
 	"bufio"
-	"bytes"
-	"errors"
 	"fmt"
 	"io"
 	"io/fs"
@@ -16,9 +14,23 @@ import (
 // This program generates zz_filesystem_generated.go file containing byte array variable named templatesZip.
 // The variable contains zip of "./templates" directory.
 func main() {
-	var zipBuff bytes.Buffer
-	zipWriter := zip.NewWriter(&zipBuff)
-	err := filepath.Walk("templates", func(path string, info fs.FileInfo, err error) error {
+	f, err := os.OpenFile("zz_filesystem_generated.go", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		log.Fatal(err)
+	}
+	defer f.Close()
+
+	srcOut := bufio.NewWriter(f)
+	defer srcOut.Flush()
+
+	_, err = fmt.Fprintf(srcOut, "// Code generated by go generate; DO NOT EDIT.\npackage function\n\nvar templatesZip = []byte{\n")
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	zipWriter := zip.NewWriter(newGoByteArrayWriter(srcOut))
+	buff := make([]byte, 4*1024)
+	err = filepath.Walk("templates", func(path string, info fs.FileInfo, err error) error {
 		if err != nil {
 			return err
 		}
@@ -48,12 +60,12 @@ func main() {
 		}
 
 		if !info.IsDir() {
-			b, err := os.ReadFile(path)
+			f, err := os.Open(path)
 			if err != nil {
 				return err
 			}
 
-			_, err = w.Write(b)
+			_, err = io.CopyBuffer(w, f, buff)
 			if err != nil {
 				return err
 			}
@@ -66,59 +78,72 @@ func main() {
 		log.Fatal(err)
 	}
 
-	f, err := os.OpenFile("zz_filesystem_generated.go", os.O_TRUNC|os.O_CREATE|os.O_WRONLY, 0644)
-	if err != nil {
-		log.Fatal(err)
-	}
-	defer f.Close()
-
-	srcOut := bufio.NewWriter(f)
-	defer srcOut.Flush()
-
-	_, err = fmt.Fprintf(srcOut, "// Code generated by go generate; DO NOT EDIT.\npackage function\n\nvar templatesZip = []byte{\n")
-	if err != nil {
-		log.Fatal(err)
-	}
-
-	zipBytesReader := bytes.NewReader(zipBuff.Bytes())
-	buff := make([]byte, 32)
-	for {
-		n, err := zipBytesReader.Read(buff)
-		if err != nil {
-			if errors.Is(err, io.EOF) {
-				break
-			} else {
-				log.Fatal(err)
-			}
-		}
-
-		_, err = fmt.Fprintf(srcOut, "\t")
-		if err != nil {
-			log.Fatal(err)
-		}
-
-		for i, b := range buff[:n] {
-			_, err = fmt.Fprintf(srcOut, "0x%02x,", b)
-			if err != nil {
-				log.Fatal(err)
-			}
-			if i < n-1 {
-				_, err = fmt.Fprintf(srcOut, " ")
-				if err != nil {
-					log.Fatal(err)
-				}
-			}
-		}
-
-		_, err = fmt.Fprintf(srcOut, "\n")
-		if err != nil {
-			log.Fatal(err)
-		}
-
-	}
-
-	_, err = fmt.Fprintf(srcOut, "}\n")
+	_, err = fmt.Fprint(srcOut, "}\n")
 	if err != nil {
 		log.Fatal(err)
 	}
 }
+
+// goByteArrayWriter dumps bytes as a Go integer hex literals separated by commas into underlying Writer.
+// Each line of the output will be indented by a tab and each line will contain at most 32 integer literals.
+// This is useful when generating Go array literals.
+type goByteArrayWriter struct {
+	i                 uint32
+	w                 io.Writer
+	hexDigitWithComma []byte
+}
+
+func newGoByteArrayWriter(w io.Writer) *goByteArrayWriter {
+	return &goByteArrayWriter{
+		i:                 0,
+		w:                 w,
+		hexDigitWithComma: []byte("0x00,"),
+	}
+}
+
+var hexs = []byte("0123456789abcdef")
+var space = []byte(" ")
+var newLine = []byte("\n")
+var tab = []byte("\t")
+
+const bytesInLine = 32
+
+func (g *goByteArrayWriter) Write(bs []byte) (written int, err error) {
+	var n int
+
+	for _, b := range bs {
+		if g.i == 0 {
+			n, err = g.w.Write(tab)
+			written += n
+			if err != nil {
+				return
+			}
+		}
+
+		g.hexDigitWithComma[2] = hexs[b>>4]
+		g.hexDigitWithComma[3] = hexs[b&0x0f]
+		n, err = g.w.Write(g.hexDigitWithComma)
+		written += n
+		if err != nil {
+			return
+		}
+
+		if g.i == bytesInLine-1 {
+			n, err = g.w.Write(newLine)
+			written += n
+			if err != nil {
+				return
+			}
+			g.i = 0
+		} else {
+			n, err = g.w.Write(space)
+			written += n
+			if err != nil {
+				return
+			}
+			g.i++
+		}
+	}
+
+	return
+}