support JVM attack (#33)

Signed-off-by: xiang <xiang13225080@163.com>

cmd/attack/attack.go

@@ -27,6 +27,7 @@ func NewAttackCommand() *cobra.Command {
 		NewStressAttackCommand(),
 		NewDiskAttackCommand(),
 		NewHostAttackCommand(),
+		NewJVMAttackCommand(),
 	)
 
 	return cmd

cmd/attack/jvm.go

@@ -0,0 +1,179 @@
+// Copyright 2021 Chaos Mesh Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package attack
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+	"go.uber.org/fx"
+
+	"github.com/chaos-mesh/chaosd/cmd/server"
+	"github.com/chaos-mesh/chaosd/pkg/core"
+	"github.com/chaos-mesh/chaosd/pkg/server/chaosd"
+	"github.com/chaos-mesh/chaosd/pkg/utils"
+)
+
+func NewJVMAttackCommand() *cobra.Command {
+	options := core.NewJVMCommand()
+	dep := fx.Options(
+		server.Module,
+		fx.Provide(func() *core.JVMCommand {
+			return options
+		}),
+	)
+
+	cmd := &cobra.Command{
+		Use:   "jvm <subcommand>",
+		Short: "JVM attack related commands",
+	}
+
+	cmd.AddCommand(
+		NewJVMInstallCommand(dep, options),
+		NewJVMSubmitCommand(dep, options),
+	)
+
+	return cmd
+}
+
+func NewJVMInstallCommand(dep fx.Option, options *core.JVMCommand) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "install [options]",
+		Short: "install agent to Java process",
+		Run: func(*cobra.Command, []string) {
+			options.Type = core.JVMInstallType
+			utils.FxNewAppWithoutLog(dep, fx.Invoke(jvmCommandFunc)).Run()
+		},
+	}
+
+	cmd.Flags().IntVarP(&options.Port, "port", "", 9288, "the port of agent server")
+	cmd.Flags().IntVarP(&options.Pid, "pid", "", 0, "the pid of Java process which need to attach")
+
+	return cmd
+}
+
+func NewJVMSubmitCommand(dep fx.Option, options *core.JVMCommand) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "submit [options]",
+		Short: "submit rules for byteman agent",
+	}
+	options.Type = core.JVMSubmitType
+
+	cmd.PersistentFlags().IntVarP(&options.Port, "port", "", 9288, "the port of agent server")
+
+	cmd.AddCommand(
+		NewJVMLatencyCommand(dep, options),
+		NewJVMReturnCommand(dep, options),
+		NewJVMExceptionCommand(dep, options),
+		NewJVMStressCommand(dep, options),
+		NewJVMGCCommand(dep, options),
+	)
+
+	return cmd
+}
+
+func NewJVMLatencyCommand(dep fx.Option, options *core.JVMCommand) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "latency [options]",
+		Short: "inject latency to specified method",
+		Run: func(*cobra.Command, []string) {
+			options.Action = core.JVMLatencyAction
+			utils.FxNewAppWithoutLog(dep, fx.Invoke(jvmCommandFunc)).Run()
+		},
+	}
+
+	cmd.Flags().StringVarP(&options.Class, "class", "c", "", "Java class name")
+	cmd.Flags().StringVarP(&options.Method, "method", "m", "", "the method name in Java class")
+	cmd.Flags().StringVarP(&options.LatencyDuration, "latency", "", "", "the latency duration")
+
+	return cmd
+}
+
+func NewJVMReturnCommand(dep fx.Option, options *core.JVMCommand) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "return [options]",
+		Short: "return specified value for specified method",
+		Run: func(*cobra.Command, []string) {
+			options.Action = core.JVMReturnAction
+			utils.FxNewAppWithoutLog(dep, fx.Invoke(jvmCommandFunc)).Run()
+		},
+	}
+
+	cmd.Flags().StringVarP(&options.Class, "class", "c", "", "Java class name")
+	cmd.Flags().StringVarP(&options.Method, "method", "m", "", "the method name in Java class")
+	cmd.Flags().StringVarP(&options.ReturnValue, "value", "", "", "the return value for action 'return', only support number and string type now")
+
+	return cmd
+}
+
+func NewJVMExceptionCommand(dep fx.Option, options *core.JVMCommand) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "exception [options]",
+		Short: "throw specified exception for specified method",
+		Run: func(*cobra.Command, []string) {
+			options.Action = core.JVMExceptionAction
+			utils.FxNewAppWithoutLog(dep, fx.Invoke(jvmCommandFunc)).Run()
+		},
+	}
+
+	cmd.Flags().StringVarP(&options.Class, "class", "c", "", "Java class name")
+	cmd.Flags().StringVarP(&options.Method, "method", "m", "", "the method name in Java class")
+	cmd.Flags().StringVarP(&options.ThrowException, "exception", "", "", "the exception which needs to throw for action 'exception'")
+
+	return cmd
+}
+
+func NewJVMStressCommand(dep fx.Option, options *core.JVMCommand) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "stress [options]",
+		Short: "inject stress to JVM",
+		Run: func(*cobra.Command, []string) {
+			options.Action = core.JVMStressAction
+			utils.FxNewAppWithoutLog(dep, fx.Invoke(jvmCommandFunc)).Run()
+		},
+	}
+
+	cmd.Flags().IntVarP(&options.CPUCount, "cpu-count", "", 0, "the CPU core number need to use")
+	cmd.Flags().IntVarP(&options.MemorySize, "mem-size", "", 0, "the memory size need to locate, the unit is MB")
+
+	return cmd
+}
+
+func NewJVMGCCommand(dep fx.Option, options *core.JVMCommand) *cobra.Command {
+	cmd := &cobra.Command{
+		Use:   "gc",
+		Short: "trigger GC for JVM",
+		Run: func(*cobra.Command, []string) {
+			options.Action = core.JVMGCAction
+			utils.FxNewAppWithoutLog(dep, fx.Invoke(jvmCommandFunc)).Run()
+		},
+	}
+
+	return cmd
+}
+
+func jvmCommandFunc(options *core.JVMCommand, chaos *chaosd.Server) {
+	options.CompleteDefaults()
+
+	if err := options.Validate(); err != nil {
+		utils.ExitWithError(utils.ExitBadArgs, err)
+	}
+
+	uid, err := chaos.ExecuteAttack(chaosd.JVMAttack, options)
+	if err != nil {
+		utils.ExitWithError(utils.ExitError, err)
+	}
+
+	utils.NormalExit(fmt.Sprintf("Attack jvm successfully, uid: %s", uid))
+}

pkg/core/experiment.go

@@ -33,6 +33,7 @@ const (
 	StressAttack  = "stress"
 	DiskAttack    = "disk"
 	HostAttack    = "host"
+	JVMAttack     = "jvm"
 )
 
 // ExperimentStore defines operations for working with experiments

pkg/core/jvm.go

@@ -0,0 +1,146 @@
+// Copyright 2021 Chaos Mesh Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package core
+
+import (
+	"encoding/json"
+	"fmt"
+
+	"github.com/pingcap/errors"
+
+	"github.com/chaos-mesh/chaosd/pkg/utils"
+)
+
+const (
+	JVMInstallType = "install"
+	JVMSubmitType  = "submit"
+
+	JVMLatencyAction   = "latency"
+	JVMExceptionAction = "exception"
+	JVMReturnAction    = "return"
+	JVMStressAction    = "stress"
+	JVMGCAction        = "gc"
+)
+
+type JVMCommand struct {
+	CommonAttackConfig
+
+	// rule name, should be unique, and will generate by chaosd automatically
+	Name string
+
+	// Java class
+	Class string
+
+	// the method in Java class
+	Method string
+
+	// fault action, values can be latency, exception, return, stress
+	Action string
+
+	// the return value for action 'return'
+	ReturnValue string
+
+	// the exception which needs to throw dor action `exception`
+	ThrowException string
+
+	// the latency duration for action 'latency'
+	LatencyDuration string
+
+	// the CPU core number need to use, only set it when action is stress
+	CPUCount int
+
+	// the memory size need to locate, only set it when action is stress
+	MemorySize int
+
+	// attach or agent
+	Type string
+
+	// the port of agent server
+	Port int
+
+	// the pid of Java process which need to attach
+	Pid int
+
+	// below is only used for template
+	Do string
+
+	StressType string
+
+	StressValueName string
+
+	StressValue int
+}
+
+func (j *JVMCommand) Validate() error {
+	switch j.Type {
+	case JVMInstallType:
+		if j.Pid == 0 {
+			return errors.New("pid can't be 0")
+		}
+	case JVMSubmitType:
+		switch j.Action {
+		case JVMStressAction:
+			if j.CPUCount == 0 && j.MemorySize == 0 {
+				return errors.New("must set one of cpu-count and mem-size when action is 'stress'")
+			}
+
+			if j.CPUCount > 0 && j.MemorySize > 0 {
+				return errors.New("inject stress on both CPU and memory is not support now")
+			}
+		case JVMGCAction:
+			// do nothing
+		case JVMExceptionAction, JVMReturnAction, JVMLatencyAction:
+			if len(j.Class) == 0 {
+				return errors.New("class not provided")
+			}
+
+			if len(j.Method) == 0 {
+				return errors.New("method not provided")
+			}
+		case "":
+			return errors.New("action not provided, action can be 'latency', 'exception', 'return', 'stress' or 'gc'")
+		default:
+			return errors.New(fmt.Sprintf("action %s not supported, action can be 'latency', 'exception', 'return', 'stress' or 'gc'", j.Action))
+		}
+
+	case "":
+		return errors.New("type not provided, type can be 'install' or 'submit'")
+	default:
+		return errors.New(fmt.Sprintf("type %s not supported, type can be 'install' or 'submit'", j.Type))
+	}
+
+	return nil
+}
+
+func (j *JVMCommand) RecoverData() string {
+	data, _ := json.Marshal(j)
+
+	return string(data)
+}
+
+func (j *JVMCommand) CompleteDefaults() {
+	if j.Type == JVMSubmitType {
+		if len(j.Name) == 0 {
+			j.Name = fmt.Sprintf("%s-%s-%s-%s-%s", j.Class, j.Method, j.Action, j.Type, utils.RandomStringWithCharset(5))
+		}
+	}
+}
+
+func NewJVMCommand() *JVMCommand {
+	return &JVMCommand{
+		CommonAttackConfig: CommonAttackConfig{
+			Kind: JVMAttack,
+		},
+	}
+}

pkg/core/jvm_test.go

@@ -0,0 +1,117 @@
+// Copyright 2021 Chaos Mesh Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package core
+
+import (
+	"testing"
+
+	. "github.com/onsi/gomega"
+)
+
+func TestJVMCommand(t *testing.T) {
+	g := NewGomegaWithT(t)
+
+	testCases := []struct {
+		cmd    *JVMCommand
+		errMsg string
+	}{
+		{
+			&JVMCommand{},
+			"type not provided",
+		},
+		{
+			&JVMCommand{
+				Type: JVMInstallType,
+			},
+			"pid can't be 0",
+		},
+		{
+			&JVMCommand{
+				Type: JVMInstallType,
+				Pid:  123,
+			},
+			"",
+		},
+		{
+			&JVMCommand{
+				Type: JVMSubmitType,
+			},
+			"action not provided",
+		},
+		{
+			&JVMCommand{
+				Type:   JVMSubmitType,
+				Action: "test",
+			},
+			"action test not supported",
+		},
+		{
+			&JVMCommand{
+				Type:   JVMSubmitType,
+				Action: JVMLatencyAction,
+			},
+			"class not provided",
+		},
+		{
+			&JVMCommand{
+				Type:   JVMSubmitType,
+				Action: JVMExceptionAction,
+				Class:  "test",
+			},
+			"method not provided",
+		},
+		{
+			&JVMCommand{
+				Type:   JVMSubmitType,
+				Action: JVMExceptionAction,
+				Class:  "test",
+				Method: "test",
+			},
+			"",
+		},
+		{
+			&JVMCommand{
+				Type:   JVMSubmitType,
+				Action: JVMStressAction,
+			},
+			"must set one of cpu-count and mem-size",
+		},
+		{
+			&JVMCommand{
+				Type:       JVMSubmitType,
+				Action:     JVMStressAction,
+				CPUCount:   1,
+				MemorySize: 1,
+			},
+			"inject stress on both CPU and memory is not support now",
+		},
+		{
+			&JVMCommand{
+				Type:     JVMSubmitType,
+				Action:   JVMStressAction,
+				CPUCount: 1,
+			},
+			"",
+		},
+	}
+
+	for _, testCase := range testCases {
+		err := testCase.cmd.Validate()
+		if len(testCase.errMsg) == 0 {
+			g.Expect(err).ShouldNot(HaveOccurred())
+		} else {
+			g.Expect(err.Error()).Should(ContainSubstring(testCase.errMsg))
+		}
+	}
+}

pkg/server/chaosd/jvm.go

@@ -0,0 +1,214 @@
+// Copyright 2020 Chaos Mesh Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package chaosd
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"text/template"
+
+	"github.com/pingcap/errors"
+	"github.com/pingcap/log"
+	"go.uber.org/zap"
+
+	"github.com/chaos-mesh/chaosd/pkg/core"
+)
+
+const ruleTemplate = `
+RULE {{.Name}}
+CLASS {{.Class}}
+METHOD {{.Method}}
+AT ENTRY
+IF true
+DO 
+	{{.Do}};
+ENDRULE
+`
+
+const stressRuleTemplate = `
+RULE {{.Name}}
+STRESS {{.StressType}}
+{{.StressValueName}} {{.StressValue}}
+ENDRULE
+`
+
+const gcRuleTemplate = `
+RULE {{.Name}}
+GC
+ENDRULE
+`
+
+type jvmAttack struct{}
+
+var JVMAttack AttackType = jvmAttack{}
+
+const bmInstallCommand = "bminstall.sh -b -Dorg.jboss.byteman.transform.all -Dorg.jboss.byteman.verbose -p %d %d"
+const bmSubmitCommand = "bmsubmit.sh -p %d -%s %s"
+
+func (j jvmAttack) Attack(options core.AttackConfig, env Environment) (err error) {
+	attack := options.(*core.JVMCommand)
+
+	if attack.Type == core.JVMInstallType {
+		return j.install(attack)
+	} else if attack.Type == core.JVMSubmitType {
+		return j.submit(attack)
+	}
+
+	return errors.Errorf("attack type %s not supported", attack.Type)
+}
+
+func (j jvmAttack) install(attack *core.JVMCommand) error {
+	var err error
+
+	bmInstallCmd := fmt.Sprintf(bmInstallCommand, attack.Port, attack.Pid)
+	cmd := exec.Command("bash", "-c", bmInstallCmd)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Error(string(output), zap.Error(err))
+		return err
+	}
+
+	log.Info(string(output))
+	return err
+}
+
+func (j jvmAttack) submit(attack *core.JVMCommand) error {
+	var err error
+
+	if len(attack.Do) == 0 {
+		switch attack.Action {
+		case core.JVMLatencyAction:
+			attack.Do = fmt.Sprintf("Thread.sleep(%s)", attack.LatencyDuration)
+		case core.JVMExceptionAction:
+			attack.Do = fmt.Sprintf("throw new %s", attack.ThrowException)
+		case core.JVMReturnAction:
+			attack.Do = fmt.Sprintf("return %s", attack.ReturnValue)
+		case core.JVMStressAction:
+			if attack.CPUCount > 0 {
+				attack.StressType = "CPU"
+				attack.StressValueName = "CPUCOUNT"
+				attack.StressValue = attack.CPUCount
+			} else {
+				attack.StressType = "MEMORY"
+				attack.StressValueName = "MEMORYSIZE"
+				attack.StressValue = attack.MemorySize
+			}
+
+		}
+	}
+
+	buf := new(bytes.Buffer)
+
+	var t *template.Template
+	switch attack.Action {
+	case core.JVMStressAction:
+		t = template.Must(template.New("byteman rule").Parse(stressRuleTemplate))
+	case core.JVMExceptionAction, core.JVMLatencyAction, core.JVMReturnAction:
+		t = template.Must(template.New("byteman rule").Parse(ruleTemplate))
+	case core.JVMGCAction:
+		t = template.Must(template.New("byteman rule").Parse(gcRuleTemplate))
+	default:
+		return errors.Errorf("jvm action %s not supported", attack.Action)
+	}
+
+	if t == nil {
+		return errors.Errorf("parse byeman rule template failed")
+	}
+
+	err = t.Execute(buf, attack)
+	if err != nil {
+		log.Error("executing template", zap.Error(err))
+		return err
+	}
+
+	log.Info("byteman rule", zap.String("rule", buf.String()))
+
+	tmpfile, err := ioutil.TempFile("", "rule.btm")
+	if err != nil {
+		return err
+	}
+
+	log.Info("create btm file", zap.String("file", tmpfile.Name()))
+
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	if _, err := tmpfile.Write(buf.Bytes()); err != nil {
+		return err
+	}
+
+	if err := tmpfile.Close(); err != nil {
+		return err
+	}
+
+	bmSubmitCmd := fmt.Sprintf(bmSubmitCommand, attack.Port, "l", tmpfile.Name())
+	cmd := exec.Command("bash", "-c", bmSubmitCmd)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Error(string(output), zap.Error(err))
+		return err
+	}
+
+	log.Info(string(output))
+	return nil
+}
+
+func (j jvmAttack) Recover(exp core.Experiment, env Environment) error {
+	attack := &core.JVMCommand{}
+	if err := json.Unmarshal([]byte(exp.RecoverCommand), attack); err != nil {
+		return err
+	}
+
+	// Create a new template and parse the letter into it.
+	t := template.Must(template.New("byteman rule").Parse(ruleTemplate))
+
+	buf := new(bytes.Buffer)
+	err := t.Execute(buf, attack)
+	if err != nil {
+		log.Error("executing template", zap.Error(err))
+		return err
+	}
+
+	tmpfile, err := ioutil.TempFile("", "rule.btm")
+	if err != nil {
+		return err
+	}
+
+	defer os.Remove(tmpfile.Name()) // clean up
+
+	if _, err := tmpfile.Write(buf.Bytes()); err != nil {
+		return err
+	}
+
+	if err := tmpfile.Close(); err != nil {
+		return err
+	}
+
+	log.Info("create btm file", zap.String("file", tmpfile.Name()))
+
+	bmSubmitCmd := fmt.Sprintf(bmSubmitCommand, attack.Port, "u", tmpfile.Name())
+	cmd := exec.Command("bash", "-c", bmSubmitCmd)
+	output, err := cmd.CombinedOutput()
+	if err != nil {
+		log.Error(string(output), zap.Error(err))
+		return err
+	}
+
+	log.Info(string(output))
+
+	return nil
+}

pkg/server/chaosd/recover.go

@@ -56,6 +56,8 @@ func (s *Server) RecoverAttack(uid string) error {
 		attackType = StressAttack
 	case core.DiskAttack:
 		attackType = DiskAttack
+	case core.JVMAttack:
+		attackType = JVMAttack
 	default:
 		return perr.Errorf("chaos experiment kind %s not found", exp.Kind)
 	}

pkg/utils/util.go

@@ -0,0 +1,30 @@
+// Copyright 2021 Chaos Mesh Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"math/rand"
+	"time"
+)
+
+const charset = "abcdefghijklmnopqrstuvwxyz0123456789"
+
+func RandomStringWithCharset(length int) string {
+	var seededRand = rand.New(rand.NewSource(time.Now().UnixNano()))
+	b := make([]byte, length)
+	for i := range b {
+		b[i] = charset[seededRand.Intn(len(charset))]
+	}
+	return string(b)
+}

test/integration_test/jvm/run.sh

@@ -0,0 +1,54 @@
+#!/usr/bin/env bash
+
+# Copyright 2021 Chaos Mesh Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+set -u
+
+cur=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )
+cd $cur
+
+bin_path=../../../bin
+
+echo "download && build && run Java example program"
+git clone https://github.com/WangXiangUSTC/byteman-example.git
+cd byteman-example/example.helloworld
+javac HelloWorld/Main.java
+jar cfme HelloWorld.jar Manifest.txt HelloWorld.Main HelloWorld/Main.class
+cd -
+java -jar byteman-example/example.helloworld/HelloWorld.jar > helloworld.log &
+# make sure it works
+sleep 3
+cat helloworld.log
+# TODO: get the PID more accurately
+pid=`pgrep -n java`
+
+echo "download byteman && set environment variable"
+curl -fsSL -o chaosd-byteman-download.tar.gz https://mirrors.chaos-mesh.org/jvm/chaosd-byteman-download.tar.gz
+tar zxvf chaosd-byteman-download.tar.gz
+export BYTEMAN_HOME=$cur/chaosd-byteman-download
+export PATH=$PATH:${BYTEMAN_HOME}/bin
+
+echo "run chaosd to inject failure into JVM, and check"
+$bin_path/chaosd attack jvm install --port 9288 --pid $pid
+
+$bin_path/chaosd attack jvm submit return --class Main --method getnum --port 9288  --value 99999
+check_contains "99999" helloworld.log
+
+$bin_path/chaosd attack jvm submit exception  --class Main --method sayhello --port 9288 --exception 'java.io.IOException("BOOM")'
+check_contains "BOOM" helloworld.log
+
+# TODO: add test for latency, stress and gc
+
+echo "clean"
+kill $pid

test/integration_test/run.sh

@@ -15,12 +15,14 @@
 
 set -eu
 
+pwd=`pwd`
 test_dir=test/integration_test
 
 function run() {
     script=$1
     echo "Running test $script..."
     TEST_NAME="$(basename "$(dirname "$script")")" \
+    PATH="$pwd/$test_dir/../utilities:$PATH" \
     bash +x "$script"
 }
 

test/utilities/check_contains

@@ -0,0 +1,11 @@
+#!/bin/sh
+
+set -eu
+
+if ! grep -Fq "$1" "$2"; then
+    echo "TEST FAILED: $2 DOES NOT CONTAIN '$1'"
+    echo "____________________________________"
+    cat $2
+    echo "^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^"
+    exit 1
+fi