istio.io/pkg/test/istioio/framework.go

// Copyright 2020 Istio Authors
//
// Package istioio includes the framework for doc testing. This package will
// first scan through all the docs to collect their information, and then
// setup istio as appropriate to run each test.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
//	http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package istioio

import (
	"fmt"
	"os"
	"path/filepath"
	"regexp"
	"strings"
	"time"

	"istio.io/istio/pkg/test/framework"
	"istio.io/istio/pkg/test/scopes"
)

// TestCase is a description of a test extracted from a file
type TestCase struct {
	valid         bool   // whether it is a valid test case that can be run
	path          string // path of the test file
	config        string // setup config of the test
	testScript    string // test script to be run
	cleanupScript string // cleanup script to be run
}

var (
	testsToRun  = os.Getenv("TEST")
	runAllTests = testsToRun == ""

	// folder location to be traversed to look for test files
	defaultPath   = "content/en/docs"
	contentFolder = fmt.Sprintf("%v/%v/", os.Getenv("REPO_ROOT"), defaultPath)

	// scripts that are sourced for all tests
	scriptPrefixTemplate = `
### BEGIN INJECTED SCRIPT ###
cd ${REPO_ROOT}
source "%v/%v" # snips.sh
source "tests/util/verify.sh"
source "tests/util/debug.sh"
source "tests/util/helpers.sh"
### END INJECTED SCRIPT ###
`

	// command injected at start of cleanup script
	cleanupScriptPrefix = `
set +e # ignore cleanup errors
`
	snipsFileSuffix = "snips.sh"
	testFileSuffix  = "test.sh"

	setupSpec      = "# @setup"
	testCleanupSep = "# @cleanup"

	// constructed test cases for the specified tests
	testCases []TestCase
)

// init constructs test cases from all specified tests
func init() {
	startTime := time.Now()
	defer func() {
		scopes.Framework.Infof("Finished initializing test doc(s). Elapsed time: %v",
			time.Since(startTime))
	}()

	if runAllTests {
		scopes.Framework.Infof("Initializing test doc(s): all docs")
	} else {
		scopes.Framework.Infof("Initializing test doc(s): %v", testsToRun)
	}

	// scan for the test script files
	for _, testFolder := range split(testsToRun) {
		err := filepath.Walk(
			contentFolder+testFolder,
			func(path string, info os.FileInfo, walkError error) error {
				if walkError != nil {
					return walkError
				}
				// check if ends with test.sh
				if strings.HasSuffix(path, testFileSuffix) {
					if testCase, err := checkFile(path); testCase.valid {
						testCases = append(testCases, *testCase)
					} else if err != nil {
						scopes.Framework.Fatalf("Error occurred while processing %v: %v", testCase.path, err)
					}
				}
				return nil
			},
		)
		if err != nil {
			scopes.Framework.Fatalf("Error occurred while traversing content: %v", err)
		}
	}

	// in case no matched script files were found
	if len(testCases) == 0 {
		scopes.Framework.Infof("Warning: no test scripts are found that match '%v'", testsToRun)
	}
}

// checkFile takes a file path as the input and returns a TestCase object
// that is constructed out of the file as a file description
func checkFile(path string) (*TestCase, error) {
	shortPath := path[len(contentFolder):]
	testCase := &TestCase{path: shortPath}

	// read the script file
	script, err := os.ReadFile(path)
	if err != nil {
		return testCase, err
	}

	// parse the script into test and cleanup
	splitScript := strings.Split(string(script), testCleanupSep)
	if numParts := len(splitScript); numParts != 2 {
		err := fmt.Errorf(
			"script error: expected two-part script separated by '%v', got %v part(s)",
			testCleanupSep, numParts,
		)
		return testCase, err
	}
	testScript := splitScript[0]
	cleanupScript := splitScript[1]

	// copy the files sourced by test to cleanup
	sources := getNonTestSources(testScript, shortPath)
	cleanupScript = strings.Join(sources, "\n") + cleanupScriptPrefix + cleanupScript

	// find setup configuration
	re := regexp.MustCompile(fmt.Sprintf("(?m)^%v (.*)$", setupSpec))
	setups := re.FindAllStringSubmatch(testScript, -1)

	if numSetups := len(setups); numSetups != 1 {
		err := fmt.Errorf(
			"script error: expected one line that starts with '%v', got %v line(s)",
			setupSpec, numSetups,
		)
		return testCase, err
	}
	config := setups[0][1]

	// Check for proper test cleanup
	scriptPrefix := getTemplateScript(scriptPrefixTemplate, shortPath)
	testScript = addScriptPrefix(scriptPrefix, testScript)
	cleanupScript = addScriptPrefix(scriptPrefix, cleanupScript)

	testCase = &TestCase{
		valid:         true,
		path:          shortPath,
		config:        config,
		testScript:    testScript,
		cleanupScript: cleanupScript,
	}
	return testCase, nil
}

func addScriptPrefix(prefix, script string) string {
	out := ""
	needToAdd := true

	// Add the prefix before the first uncommented line.
	for _, line := range strings.Split(script, "\n") {
		if needToAdd && !strings.HasPrefix(line, "#") {
			out += prefix + "\n"
			needToAdd = false
		}

		out += line + "\n"
	}

	return out
}

// NewTestDocsFunc returns a test function that traverses through all test
// cases and runs those that need the setup config specified by the input.
func NewTestDocsFunc(config string) func(framework.TestContext) {
	return func(ctx framework.TestContext) {
		testsToRun := testsForConfig(config)
		if len(testsToRun) == 0 {
			ctx.Skipf("No tests need to be run with %v", config)
		}

		scopes.Framework.Infof("Setting up istio with %v", config)

		for _, testCase := range testsToRun {
			path := testCase.path
			testScriptName := filepath.Base(path)
			cleanupScriptName := "cleanup.sh"

			// Create the mesh snapshotters
			kubeConfig := getKubeConfig(ctx)
			beforeSnapshotter := &Snapshotter{
				StepName:   "before snapshot",
				KubeConfig: kubeConfig,
			}
			afterSnapshotter := &Snapshotter{
				StepName:   "after snapshot",
				KubeConfig: kubeConfig,
			}

			ctx.NewSubTest(path).
				Run(NewBuilder().
					Add(beforeSnapshotter).
					Add(Script{
						Input: Inline{
							FileName: testScriptName,
							Value:    testCase.testScript,
						},
					}).
					Defer(Script{
						Input: Inline{
							FileName: cleanupScriptName,
							Value:    testCase.cleanupScript,
						},
					}).
					Defer(SnapshotValidator{
						Before: beforeSnapshotter,
						After:  afterSnapshotter,
					}).
					Build())
		}
	}
}

// Helper functions

// testsForConfig returns the tests that match the given configuration.
func testsForConfig(config string) []TestCase {
	out := make([]TestCase, 0, len(testCases))
	for _, testCase := range testCases {
		if testCase.config == config {
			out = append(out, testCase)
		}
	}
	return out
}

// split breaks down the test names specified into a slice.
// It receives a comma-separated string of test names that the user has
// specified from command line, and returns a slice of separated strings.
func split(testsAsString string) []string {
	return strings.Split(testsAsString, ",")
}

// getTemplateScript returns a script that automatically sources the
// snippets and some test utilities. It receives `testPath`, which is the
// path of the test script file to be run.
func getTemplateScript(template, testPath string) string {
	splitPath := strings.Split(testPath, "/")
	splitPath[len(splitPath)-1] = snipsFileSuffix
	snipsPath := strings.Join(splitPath, "/")
	return fmt.Sprintf(template, defaultPath, snipsPath)
}

// getNonTestSources returns test source commands that are not test files.
func getNonTestSources(testScript string, testPath string) []string {
	re := regexp.MustCompile("(?m)^source \".*\\.sh\"$")
	sources := re.FindAllString(testScript, -1)
	testDir := testPath[:strings.LastIndex(testPath, "/")]
	re = regexp.MustCompile(fmt.Sprintf("source \"%s/%s/.*test\\.sh\"", defaultPath, testDir))
	for i := 0; i < len(sources); i++ {
		if re.MatchString(sources[i]) {
			sources = append(sources[:i], sources[i+1:]...)
			i--
		}
	}
	return sources
}