test: adapt tests new crun error messages

Needed-by: https://github.com/containers/crun/pull/1672

Signed-off-by: Giuseppe Scrivano <gscrivan@redhat.com>

libpod/define/exec_codes.go

@@ -46,6 +46,7 @@ func ExitCode(err error) int {
 	e := strings.ToLower(err.Error())
 	logrus.Debugf("ExitCode msg: %q", e)
 	if strings.Contains(e, "not found") ||
+		strings.Contains(e, "executable path is empty") ||
 		strings.Contains(e, "no such file") {
 		return ExecErrorCodeNotFound
 	}

test/e2e/exec_test.go

@@ -421,19 +421,20 @@ var _ = Describe("Podman exec", func() {
 		session := podmanTest.Podman([]string{"exec", "test1", "/etc"})
 		session.WaitWithDefaultTimeout()
 
-		// crun (and, we hope, any other future runtimes)
-		expectedStatus := 126
-		expectedMessage := "open executable: Operation not permitted: OCI permission denied"
-
 		// ...but it's much more complicated under runc (#19552)
 		if podmanTest.OCIRuntime == "runc" {
-			expectedMessage = `exec failed: unable to start container process: exec: "/etc": is a directory`
+			expectedMessage := `exec failed: unable to start container process: exec: "/etc": is a directory`
-			expectedStatus = 255
+			expectedStatus := 255
 			if IsRemote() {
 				expectedStatus = 125
 			}
-		}
 			Expect(session).Should(ExitWithError(expectedStatus, expectedMessage))
+		} else {
+			// crun (and, we hope, any other future runtimes)
+			expectedStatus := 126
+			expectedMessage := ".*(open executable|the path `/etc` is not a regular file): Operation not permitted: OCI permission denied.*"
+			Expect(session).Should(ExitWithErrorRegex(expectedStatus, expectedMessage))
+		}
 	})
 
 	It("podman exec command not found", func() {

test/e2e/run_entrypoint_test.go

@@ -18,7 +18,7 @@ CMD []
 		podmanTest.BuildImage(dockerfile, "foobar.com/entrypoint:latest", "false")
 		session := podmanTest.Podman([]string{"run", "foobar.com/entrypoint:latest"})
 		session.WaitWithDefaultTimeout()
-		Expect(session).Should(ExitWithError(126, "open executable: Operation not permitted: OCI permission denied"))
+		Expect(session).Should(ExitWithErrorRegex(126, ".*(open executable|executable path is empty): Operation not permitted: OCI permission denied.*"))
 	})
 
 	It("podman run entrypoint == [\"\"]", func() {

test/e2e/run_exit_test.go

@@ -22,7 +22,7 @@ var _ = Describe("Podman run exit", func() {
 	It("podman run exit ExecErrorCodeCannotInvoke", func() {
 		result := podmanTest.Podman([]string{"run", ALPINE, "/etc"})
 		result.WaitWithDefaultTimeout()
-		Expect(result).Should(ExitWithError(define.ExecErrorCodeCannotInvoke, "open executable: Operation not permitted: OCI permission denied"))
+		Expect(result).Should(ExitWithErrorRegex(define.ExecErrorCodeCannotInvoke, ".*(open executable|the path `/etc` is not a regular file): Operation not permitted: OCI permission denied.*"))
 	})
 
 	It("podman run exit ExecErrorCodeNotFound", func() {

test/system/030-run.bats

@@ -10,7 +10,7 @@ load helpers.network
     err_no_such_cmd="Error:.*/no/such/command.*[Nn]o such file or directory"
     # runc: RHEL8 on 2023-07-17: "is a directory".
     # Everything else (crun; runc on debian): "permission denied"
-    err_no_exec_dir="Error:.*exec.*\\\(permission denied\\\|is a directory\\\)"
+    err_no_exec_dir="Error:.*\\\(exec.*\\\(permission denied\\\|is a directory\\\)\\\|is not a regular file\\\)"
 
     tests="
 true              |   0 |
@@ -1657,14 +1657,14 @@ search               | $IMAGE           |
     # runc and crun emit different diagnostics
     runtime=$(podman_runtime)
     case "$runtime" in
-        crun) expect='crun: executable file `` not found in $PATH: No such file or directory: OCI runtime attempted to invoke a command that was not found' ;;
+        crun) expect='\(executable file `` not found in $PATH: No such file or directory: OCI runtime attempted to invoke a command that was not found\|executable path is empty\)' ;;
         runc) expect='runc: runc create failed: unable to start container process: exec: "": executable file not found in $PATH: OCI runtime attempted to invoke a command that was not found' ;;
         *)    skip "Unknown runtime '$runtime'" ;;
     esac
 
     # The '.*' in the error below is for dealing with podman-remote, which
     # includes "error preparing container <sha> for attach" in output.
-    is "$output" "Error.*: $expect" "podman emits useful diagnostic when no entrypoint is set"
+    is "$output" "Error.* $expect" "podman emits useful diagnostic when no entrypoint is set"
 }
 
 # bats test_tags=ci:parallel

test/utils/matchers.go

@@ -3,6 +3,7 @@ package utils
 import (
 	"encoding/json"
 	"fmt"
+	"regexp"
 	"strings"
 
 	"github.com/onsi/gomega/format"
@@ -20,6 +21,7 @@ type ExitMatcher struct {
 	ExpectedExitCode    int
 	ExitCode            int
 	ExpectedStderr      string
+	ExpectedStderrRegex string
 	msg                 string
 }
 
@@ -29,6 +31,12 @@ func ExitWithError(expectExitCode int, expectStderr string) *ExitMatcher {
 	return &ExitMatcher{ExpectedExitCode: expectExitCode, ExpectedStderr: expectStderr}
 }
 
+// ExitWithErrorRegex checks both exit code and the stderr regex, fails if either does not match
+// Modeled after the gomega Exit() matcher and also operates on sessions.
+func ExitWithErrorRegex(expectExitCode int, expectStderrRegex string) *ExitMatcher {
+	return &ExitMatcher{ExpectedExitCode: expectExitCode, ExpectedStderrRegex: expectStderrRegex}
+}
+
 // Match follows gexec.Matcher interface.
 func (matcher *ExitMatcher) Match(actual interface{}) (success bool, err error) {
 	session, ok := actual.(podmanSession)
@@ -49,12 +57,23 @@ func (matcher *ExitMatcher) Match(actual interface{}) (success bool, err error)
 		return false, nil
 	}
 
-	if matcher.ExpectedStderr != "" {
+	switch {
+	case matcher.ExpectedStderrRegex != "":
+		matched, err := regexp.MatchString(matcher.ExpectedStderrRegex, session.ErrorToString())
+		if err != nil {
+			matcher.msg = fmt.Sprintf("Invalid regex pattern: %s", err)
+			return false, err
+		}
+		if !matched {
+			matcher.msg = fmt.Sprintf("Command exited %d as expected, but stderr did not match regex '%s'", matcher.ExitCode, matcher.ExpectedStderrRegex)
+			return false, nil
+		}
+	case matcher.ExpectedStderr != "":
 		if !strings.Contains(session.ErrorToString(), matcher.ExpectedStderr) {
 			matcher.msg = fmt.Sprintf("Command exited %d as expected, but did not emit '%s'", matcher.ExitCode, matcher.ExpectedStderr)
 			return false, nil
 		}
-	} else {
+	default:
 		if session.ErrorToString() != "" {
 			matcher.msg = "Command exited with expected exit status, but emitted unwanted stderr"
 			return false, nil