istio.io/pkg/test/istioio/script.go

748 lines
24 KiB
Go

// Copyright Istio Authors
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package istioio
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"os/exec"
"path"
"path/filepath"
"regexp"
"strconv"
"strings"
"istio.io/istio/pkg/test/framework/resource/environment"
"istio.io/istio/pkg/test/scopes"
)
// outputStream enumerates the selectable output streams.
type outputStream string
const (
outputStreamStdout outputStream = "stdout"
outputStreamStderr outputStream = "stderr"
outputStreamAll outputStream = "all"
)
// Map of supported output streams.
var outputStreams = map[outputStream]struct{}{
outputStreamStdout: {},
outputStreamStderr: {},
outputStreamAll: {},
}
const (
snippetStartToken = "# $snippet"
verifyToken = "# $verify"
snippetOutputToken = "# $snippetoutput"
snippetEndToken = "# $endsnippet"
syntaxKey = "syntax"
outputIsKey = "outputis"
outputSnippetKey = "outputsnippet"
outputStreamKey = "outputstream"
sourceKey = "source"
verifierKey = "verifier"
defaultCommandSyntax = "bash"
defaultOutputSyntax = "text"
commandLinePrefix = "$ "
testOutputDirEnvVar = "TEST_OUTPUT_DIR"
kubeConfigEnvVar = "KUBECONFIG"
outputSnippetExtension = "_output"
stdoutExtension = ".stdout.txt"
stderrExtension = ".stderr.txt"
)
var (
// Matcher for links to the Istio github repository.
githubLinkMatch = regexp.MustCompile("@.*@")
)
var _ Step = Script{}
// Script is a test Step that parses an input script which may contain a mix of raw shell commands
// and snippets.
//
// Snippets must be surrounded by the tokens "# $snippet" and "# $endsnippet", each of which must be
// on placed their own line and must be at the start of that line. For example:
//
// # $snippet dostuff syntax="bash"
// $ kubectl apply -f @some/path/relative/to/istio/src/dir.yaml@
// # $endsnippet
//
// This will run the command
// `kubectl apply -f some/path/relative/to/istio/src/dir.yaml` and also generate
// the snippet.
//
// Snippets that reference files in `https://github.com/istio/istio`, as shown above, should
// surround those links with `@`. The reference will be converted into an actual link
// when rendering the page on `istio.io`.
//
// The `$snippet` line supports a number of fields separated by whitespace. The first field is the
// name of the snippet and is required. After the name, a number of arguments in the form of
// <key>="<value>" may be provided. The following arguments are supported:
//
// - syntax:
// Sets the syntax for the command text. This is used to properly highlight the output
// on istio.io. Defaults to "bash".
// - outputis:
// Sets the syntax for the output of the last command in the snippet. Snippet output
// will only be generated if this is set. By default, the commands and output will be merged
// into a single snippet. This can be overridden with outputsnippet.
// - outputsnippet:
// A boolean value, which if "true" indicates that the output for the last command of
// the snippet should appear in a separate snippet. The name of the generated snippet
// will append "_output" to the name of the current snippet. Defaults to "false".
// - outputstream:
// Indicates which command output stream should be used as the snippet output.
// Can be one of "stdout", "stderr", or "all". Defaults to "all". The command output
// stream is ignored if using a custom $snippetoutput (see below).
//
// ==== Verifying Command Output ====
//
// You can add verification logic to the output of the last command run within the snippet block
// by adding one or more `# $verify` block. Each `verify` block specified will have to succeed
// in order for the command to be considered successful. For example:
//
// # $snippet mysnippet syntax="bash"
// $ kubectl apply -f somefile.yaml
// # $verify verifier="contains" source="stdout"
// stdout must contain this string!
// # $verify verifier="contains" source="stderr"
// and stderr must contain this string!
// # $endsnippet
//
// The following arguments are supported by `$verify`:
//
// - verifier:
// Sets the verification algorithm to be used. Defaults to "token". The following\
// values are supported:
// - "token":
// Performs a token-based comparison of expected and actual output. The syntax
// supports the wildcard `?` character to skip comparison for a given token.
// - "contains":
// verifies that the output contains the given string.
// - "notContains":
// verifies that the output does not contain the given string.
// - "lineRegex":
// matches each line of the output against a regex for that line. The number of
// lines of the output must match the number of regexes provided. Each regex
// must be on a separate line.
// - source:
// Indicates which command output stream should be used as the input for the verifier.
// Can be one of "stdout", "stderr", or "all". Defaults to "all".
//
// ==== Generating Snippet Output ====
//
// You can indicate that a snippet should display output with the argument `outputis`:
//
// # $snippet mysnippet syntax="bash" outputis="text"
// $ kubectl apply -f somefile.yaml
// # $endsnippet
//
// This will run the command `kubectl apply -f somefile.yaml` and add the output of the command to the
// generated snippet:
//
// # $snippet mysnippet syntax="bash" outputis="text"
// $ kubectl apply -f somefile.yaml
// this is the actual output of the command
// # $endsnippet
//
// You can create a separate snippet for the output with the `outputsnippet` argument. When specifying
// `outputsnippet="true"`, a second snippet with the suffix "_output" will be generated:
//
// # $snippet mysnippet syntax="bash" outputis="text" outputsnippet="true"
// $ kubectl apply -f somefile.yaml
// # $endsnippet
//
// Generates:
//
// # $snippet mysnippet syntax="bash"
// $ kubectl apply -f somefile.yaml
// # $endsnippet
//
// # $snippet mysnippetoutput syntax="text"
// this is the actual output of the command
// # $endsnippet
//
// You can specify which output stream of the command to use for the output with the `outputstream`
// argument (see table above for possible values). This can be useful in cases where a command writes
// to both stdout and stderr (which can cause unreliable ordering between the two streams). For example:
//
// # $snippet mysnippet syntax="bash" outputis="text" outputstream="stdout"
// $ kubectl apply -f somefile.yaml
// # $endsnippet
//
// Generates:
//
// # $snippet mysnippet syntax="bash" outputis="text"
// $ kubectl apply -f somefile.yaml
// this is the command's stdout
// # $endsnippet
//
// You can configure the snippet to ignore the actual output of the command and to use a custom output
// value instead by specifying a `$snippetoutput block. This is useful for omitting parts of the
// expected output. For example:
//
// # $snippet mysnippet syntax="bash" outputis="text"
// $ kubectl apply -f somefile.yaml
// $ $snippetoutput
// ... // Omitting parts of the output
// hello world
// # $endsnippet
//
// Generates:
//
// # $snippet mysnippet syntax="bash" outputis="text"
// $ kubectl apply -f somefile.yaml
// ... // Omitting parts of the output
// hello world
// # $endsnippet
//
// === Customizing Snippet Commands ===
//
// You can execute different commands than are used in the generated snippets. This is
// useful when you need to add additional logic for handling things like retries, which
// wouldn't be desirable in the online documentation. This is done by evaluating the
// input as a golang template and applying different parameters for the command and
// snippet. For example:
//
// {{- $curlOptions := "--retry 10 --retry-connrefused --retry-delay 5 " -}}
// {{- if .isSnippet -}}
// {{- $curlOptions = "" -}}
// {{- end -}}
//
// # $snippet mysnippet syntax="bash"
// $ curl {{ $curlOptions }} http://www.google.com
// # $endsnippet
//
// When the Script is created, you can evaluate the input differently:
//
// istioio.Script{
// Input: istioio.Evaluate(istioio.Path("scripts/myfile.txt"), map[string]interface{}{
// "isSnippet": false,
// }),
// SnippetInput: istioio.Evaluate(istioio.Path("scripts/myfile.txt"), map[string]interface{}{
// "isSnippet": true,
// }),
// }
//
// This will run the command:
//
// curl --retry 10 --retry-connrefused --retry-delay 5 http://www.google.com
//
// And will generate the snippet:
//
// # $snippet mysnippet syntax="bash"
// $ curl http://www.google.com
// # $endsnippet
//
// === Environment ===
//
// To simplify common tasks, the following environment variables are set when the command is executed:
//
// - TEST_OUTPUT_DIR:
// Set to the working directory of the current test. By default, scripts are run from this
// directory. This variable is useful for cases where the execution `WorkDir` has been set,
// but the script needs to access files in the test working directory.
// - KUBECONFIG:
// Set to the value from the test framework. This is necessary to make kubectl commands execute
// with the configuration specified on the command line.
//
type Script struct {
// Input for the parser.
Input InputSelector
// SnippetInput allows for using a separate Input for generation of snippet files.
// This allows, for example, applying different template parameters when generating
// the commands shown in the snippet file.
SnippetInput InputSelector
// Shell to use when running the command. By default "bash" will be used.
Shell string
// WorkDir specifies the working directory when executing the script.
WorkDir string
// Env user-provided environment variables for the generated Command.
Env map[string]string
}
func (s Script) run(ctx Context) {
if s.SnippetInput == nil {
// No snippet input was defined, just use the input for snippet generation.
s.SnippetInput = s.Input
}
s.runCommand(ctx)
s.createSnippets(ctx)
}
func (s Script) runCommand(ctx Context) {
input := s.Input.SelectInput(ctx)
content, err := input.ReadAll()
if err != nil {
ctx.Fatalf("failed reading command input %s: %v", input.Name(), err)
}
// Generate the body of the command.
commandLines := make([]string, 0)
lines := strings.Split(content, "\n")
for index := 0; index < len(lines); index++ {
line := lines[index]
if isStartOfSnippet(line) {
// Parse the snippet and advance the index past it.
sinfo := parseSnippet(ctx, &index, lines)
// Gather all the command lines from the snippet.
snippetCommands := sinfo.getCommandLines()
if sinfo.needsOutput() {
// Copy stderr and stdout to output files so that we can validate later.
snippetCommands[len(snippetCommands)-1] += fmt.Sprintf(" 1> >(tee %s) 2> >(tee %s >&2)",
sinfo.getStdoutFile(),
sinfo.getStderrFile())
}
// Copy the commands from the snippet.
for _, snippetCommand := range snippetCommands {
if sinfo.name != "" {
snippetCommand = filterCommandLine(snippetCommand)
}
commandLines = append(commandLines, snippetCommand)
}
} else {
// Not a snippet, just copy the line directly to the command.
// TODO commandLines = append(commandLines, line) // Commands outside of snippets should be proper bash
// TODO Need to fix some tests that are incorrectly annotating commands outside of snippets.
commandLines = append(commandLines, filterCommandLine(line))
}
}
// Merge the command lines together.
command := strings.TrimSpace(strings.Join(commandLines, "\n"))
// Now run the command...
scopes.CI.Infof("Running command script %s", input.Name())
// Copy the command to workDir.
_, fileName := filepath.Split(input.Name())
if err := ioutil.WriteFile(path.Join(ctx.WorkDir(), fileName), []byte(command), 0644); err != nil {
ctx.Fatalf("failed copying command %s to workDir: %v", input.Name(), err)
}
// Get the shell.
shell := s.Shell
if shell == "" {
shell = "bash"
}
// Create the command.
cmd := exec.Command(shell)
cmd.Dir = s.getWorkDir(ctx)
cmd.Env = s.getEnv(ctx)
cmd.Stdin = strings.NewReader(command)
// Run the command and get the output.
output, err := cmd.CombinedOutput()
// Copy the command output from the script to workDir
outputFileName := fileName + "_output.txt"
if err := ioutil.WriteFile(filepath.Join(ctx.WorkDir(), outputFileName), bytes.TrimSpace(output), 0644); err != nil {
ctx.Fatalf("failed copying output for command %s: %v", input.Name(), err)
}
if err != nil {
ctx.Fatalf("script %s returned an error: %v. Output:\n%s", input.Name(), err, string(output))
}
}
func (s Script) createSnippets(ctx Context) {
input := s.SnippetInput.SelectInput(ctx)
content, err := input.ReadAll()
if err != nil {
ctx.Fatalf("failed reading snippet input %s: %v", input.Name(), err)
}
scopes.CI.Infof("Creating snippets for input: %s", input.Name())
// Process the input line-by-line
lines := strings.Split(content, "\n")
for index := 0; index < len(lines); index++ {
line := lines[index]
if !isStartOfSnippet(line) {
// Not the start of a snippet, skip this line.
continue
}
// Parse the snippet and advance the index past it.
sinfo := parseSnippet(ctx, &index, lines)
// Verify the output for this snippet.
sinfo.verify()
if strings.HasPrefix(sinfo.name, "_NOGEN_") {
// No snippet to generate, just verifying output.
continue
}
// Verify the output, if configured to do so.
snippetOutput := ""
if sinfo.outputIs != "" {
if len(sinfo.customOutputLines) > 0 {
// Use the custom output defined in the snippet.
snippetOutput = sinfo.getCustomOutput()
} else {
// Read the output for the snippet.
snippetOutput = sinfo.readOutput()
}
}
// Join the command lines for the snippet into a single string.
commands := strings.Join(filterCommentLines(sinfo.getCommandLines()), "\n")
// Check to see if the snippet specifies that output should be in a separate snippet.
if sinfo.outputSnippet {
// Create the snippet for the command.
Snippet{
Name: sinfo.name,
Syntax: sinfo.syntax,
Input: Inline{
FileName: sinfo.name,
Value: commands,
},
}.run(ctx)
// Create a separate snippet for the output.
outputSnippetName := sinfo.name + outputSnippetExtension
Snippet{
Name: outputSnippetName,
Syntax: sinfo.outputIs,
Input: Inline{
FileName: outputSnippetName,
Value: snippetOutput,
},
}.run(ctx)
} else {
// Combine the command and output in the same snippet.
Snippet{
Name: sinfo.name,
Syntax: sinfo.syntax,
OutputIs: sinfo.outputIs,
Input: Inline{
FileName: sinfo.name,
Value: strings.TrimSpace(commands + "\n" + snippetOutput),
},
}.run(ctx)
}
}
}
func (s Script) getWorkDir(ctx Context) string {
if s.WorkDir != "" {
// User-specified work dir for the script.
return s.WorkDir
}
return ctx.WorkDir()
}
func (s Script) getEnv(ctx Context) []string {
// Start with the environment for the current process.
e := os.Environ()
// Copy the user-specified environment (if set) and add the k8s config.
customVars := map[string]string{
// Set the output dir for the test.
testOutputDirEnvVar: ctx.WorkDir(),
}
ctx.Environment().Case(environment.Kube, func() {
customVars[kubeConfigEnvVar] = ctx.KubeEnv().Settings().KubeConfig[0]
})
for k, v := range s.Env {
customVars[k] = v
}
// Append the custom vars to the list.
for name, value := range customVars {
e = append(e, fmt.Sprintf("%s=%s", name, value))
}
return e
}
func isStartOfSnippet(line string) bool {
return strings.HasPrefix(line, snippetStartToken)
}
func parseSnippet(ctx Context, lineIndex *int, lines []string) snippetInfo {
ctx.Helper()
// Remove the start token
trimmedLine := strings.TrimPrefix(lines[*lineIndex], snippetStartToken)
// Parse the start line.
info := snippetInfo{
ctx: ctx,
outputSource: outputStreamAll,
syntax: defaultCommandSyntax,
}
for _, fields := range strings.Fields(trimmedLine) {
arg := strings.TrimSpace(fields)
if arg == "" {
continue
}
// Assume the first non-empty argument to be the snippet name.
if info.name == "" {
info.name = arg
continue
}
key, value, err := parseArg(arg)
if err != nil {
ctx.Fatalf("snippet %s: failed parsing snippet start line %s: %v",
info.name, trimmedLine, err)
}
switch key {
case syntaxKey:
info.syntax = value
case outputIsKey:
info.outputIs = value
case outputSnippetKey:
outputSnippet, err := strconv.ParseBool(value)
if err != nil {
ctx.Fatalf("failed parsing arg %s for snippet %s: %v", arg, info.name, err)
}
info.outputSnippet = outputSnippet
case outputStreamKey:
info.outputSource = outputStream(value)
if _, ok := outputStreams[info.outputSource]; !ok {
ctx.Fatalf("snippet %s: unsupported %s: %s. %s must be in %v",
info.name, key, value, info.outputSource, outputStreams)
}
default:
ctx.Fatalf("unsupported snippet attribute: %s", key)
}
}
if info.name == "" {
// If no snippet name is set, the framework will run the commands/verifiers without generating snippets.
info.name = fmt.Sprintf("_NOGEN_%d", *lineIndex)
}
if info.outputIs == "" && info.outputSnippet {
info.outputIs = defaultOutputSyntax
}
// Read the body lines for the snippet.
foundEnd := false
processingCustomOutput := false
var currentVerifier *verifierInfo
finishCurrentVerifier := func() {
if currentVerifier != nil {
info.verifiers = append(info.verifiers, *currentVerifier)
currentVerifier = nil
}
}
for *lineIndex++; !foundEnd && *lineIndex < len(lines); *lineIndex++ {
line := lines[*lineIndex]
if strings.HasPrefix(line, snippetEndToken) {
// Found the end of the snippet.
foundEnd = true
} else if strings.HasPrefix(line, snippetOutputToken) {
processingCustomOutput = true
finishCurrentVerifier()
} else if strings.HasPrefix(line, verifyToken) {
finishCurrentVerifier()
currentVerifier = &verifierInfo{
name: tokenVerifierKey,
verifier: verifyTokens,
outputSource: outputStreamAll,
}
// Parse the start line of the output.
for _, arg := range strings.Fields(line[len(verifyToken):]) {
arg = strings.TrimSpace(arg)
if arg != "" {
key, value, err := parseArg(arg)
if err != nil {
ctx.Fatalf("snippet %s: failed parsing snippet output line %s: %v",
info.name, trimmedLine, err)
}
switch key {
case verifierKey:
currentVerifier.name = value
currentVerifier.verifier = verifiers[value]
if currentVerifier.verifier == nil {
ctx.Fatalf("snippet %s: contains invalid snippet output verifier: %s. Must be in %v",
info.name, value, verifiers)
}
case sourceKey:
currentVerifier.outputSource = outputStream(value)
if _, ok := outputStreams[currentVerifier.outputSource]; !ok {
ctx.Fatalf("snippet %s: unsupported %s: %s. %s must be in %v",
info.name, sourceKey, value, currentVerifier.outputSource, outputStreams)
}
default:
ctx.Fatalf("unsupported snippet output attribute: %s", key)
}
}
}
} else if currentVerifier != nil {
currentVerifier.expectedOutput = append(currentVerifier.expectedOutput, line)
} else if processingCustomOutput {
info.customOutputLines = append(info.customOutputLines, line)
} else {
info.commandLines = append(info.commandLines, line)
}
}
// Finish the current verifier, if one exists.
finishCurrentVerifier()
if !foundEnd {
ctx.Fatalf("snippet %s missing end token", info.name)
}
// Back up the index to point to the endSnippet line, so that the outer loop will
// increment properly.
*lineIndex--
return info
}
type verifierInfo struct {
name string
verifier verifier
expectedOutput []string
outputSource outputStream
}
func (i verifierInfo) verify(sinfo *snippetInfo) {
expectedOutput := strings.TrimSpace(strings.Join(i.expectedOutput, "\n"))
// Read the output file for the last command in the snippet.
actualOutput := sinfo.readFrom(i.outputSource)
// If the snippet provided expected output, validate that the actual output
// from the command matches.
if expectedOutput != "" {
scopes.CI.Infof("Verifying results for snippet %s with verifier %s", sinfo.name, i.name)
i.verifier(sinfo.ctx, sinfo.name, expectedOutput, actualOutput)
}
}
type snippetInfo struct {
ctx Context
name string
syntax string
outputIs string
outputSnippet bool
customOutputLines []string
outputSource outputStream
commandLines []string
verifiers []verifierInfo
}
func (i snippetInfo) verify() {
for _, v := range i.verifiers {
v.verify(&i)
}
}
func (i snippetInfo) readOutput() string {
return i.readFrom(i.outputSource)
}
func (i snippetInfo) readFrom(source outputStream) string {
switch source {
case outputStreamStdout:
return i.readFile(i.getStdoutFile())
case outputStreamStderr:
return i.readFile(i.getStderrFile())
case outputStreamAll:
// Concatenate the stdout and stderr.
return i.readFile(i.getStdoutFile()) + i.readFile(i.getStderrFile())
default:
i.ctx.Fatalf("snippet %s: attempting to read from invalid output source: %s", i.name, source)
return ""
}
}
func (i snippetInfo) readFile(path string) string {
// Read the output file for the last command in the snippet.
output, err := ioutil.ReadFile(path)
if err != nil {
i.ctx.Fatalf("snippet %s: failed reading output file %s: %v", i.name, path, err)
}
return string(output)
}
func (i snippetInfo) getStdoutFile() string {
return filepath.Join(i.ctx.WorkDir(), i.name+stdoutExtension)
}
func (i snippetInfo) getStderrFile() string {
return filepath.Join(i.ctx.WorkDir(), i.name+stderrExtension)
}
func (i snippetInfo) getCommandLines() []string {
return append([]string{}, i.commandLines...)
}
func (i snippetInfo) getCustomOutput() string {
return strings.Join(i.customOutputLines, "\n")
}
func (i snippetInfo) needsOutput() bool {
return len(i.commandLines) > 0 && (i.outputIs != "" || len(i.verifiers) > 0)
}
func filterCommentLines(lines []string) []string {
out := make([]string, 0, len(lines))
for _, line := range lines {
if !strings.HasPrefix(line, "#") {
out = append(out, line)
}
}
return out
}
// filterCommandLine scrubs the given command line so that it is ready for execution.
func filterCommandLine(line string) string {
// Remove surrounding @'s for github links.
line = githubLinkMatch.ReplaceAllStringFunc(line, func(input string) string {
return input[1 : len(input)-1]
})
// Remove any command line prefixes.
return strings.TrimPrefix(line, commandLinePrefix)
}
func parseArg(arg string) (string, string, error) {
// All subsequent arguments must be of the form key=value
keyAndValue := strings.Split(arg, "=")
if len(keyAndValue) != 2 {
return "", "", fmt.Errorf("invalid argument %s", arg)
}
key := keyAndValue[0]
value := strings.ReplaceAll(keyAndValue[1], "\"", "")
return key, value, nil
}