system tests: instrument, to try to catch unlinkat-ebusy

Several tweaks to see if we can track down #17216, the unlinkat-ebusy
flake:

 - teardown(): if a cleanup command fails, display it and its
   output to the debug channel. This should never happen, but
   it can and does (see #18180, dependent containers). We
   need to know about it.

 - selinux tests: use unique pod names. This should help when
   scanning journal logs.

 - many tests: add "-f -t0" to "pod rm"

And, several unrelated changes caught by accident:
 - images-commit-with-comment test: was leaving a stray image
   behind. Clean it up, and make a few more readability tweaks

 - podman-remote-group-add test: add an explicit skip()
   when not remote. (Otherwise, test passes cleanly on
   podman local, which is misleading)

 - lots of container cleanup and/or adding "--rm" to run commands,
   to avoid leaving stray containers

Signed-off-by: Ed Santiago <santiago@redhat.com>

test/system/010-images.bats

@@ -310,15 +310,23 @@ Deleted: $pauseID"
 }
 
 @test "podman images - commit docker with comment" {
-    run_podman run --name my-container -itd $IMAGE sleep 1d
+    run_podman run --name my-container -d $IMAGE top
     run_podman 125 commit -m comment my-container my-test-image
     assert "$output" == "Error: messages are only compatible with the docker image format (-f docker)" "podman should fail unless docker format"
-    run_podman commit my-container --format docker -m comment my-test-image
-    run_podman commit -q my-container --format docker -m comment my-test-image
-    assert "$output" =~ "^[0-9a-f]{64}\$" \
-           "Output is a commit ID, no warnings or other output"
 
-    run_podman rmi my-test-image
+    # Without -q: verbose output, but only on podman-local, not remote
+    run_podman commit my-container --format docker -m comment my-test-image1
+    if ! is_remote; then
+        assert "$output" =~ "Getting image.*Writing manif.*Storing signatu" \
+               "Without -q, verbose output"
+    fi
+
+    # With -q, both local and remote: only an image ID
+    run_podman commit -q my-container --format docker -m comment my-test-image2
+    assert "$output" =~ "^[0-9a-f]{64}\$" \
+           "With -q, output is a commit ID, no warnings or other output"
+
+    run_podman rmi my-test-image1 my-test-image2
     run_podman rm my-container --force -t 0
 }
 

test/system/170-run-userns.bats

@@ -53,10 +53,12 @@ function _require_crun() {
 }
 
 @test "podman --remote --group-add keep-groups " {
-    if is_remote; then
+    if ! is_remote; then
-        run_podman 125 run --rm --group-add keep-groups $IMAGE id
+        skip "this test only meaningful under podman-remote"
-        is "$output" ".*not supported in remote mode" "Remote check --group-add keep-groups"
     fi
+
+    run_podman 125 run --rm --group-add keep-groups $IMAGE id
+    is "$output" ".*not supported in remote mode" "Remote check --group-add keep-groups"
 }
 
 @test "podman --group-add without keep-groups " {

test/system/410-selinux.bats

@@ -89,6 +89,8 @@ function check_label() {
     run_podman create --runtime=${KATA} --name myc $IMAGE
     run_podman inspect --format='{{ .ProcessLabel }}' myc
     is "$output" ".*container_kvm_t"
+
+    run_podman rm myc
 }
 
 # pr #6752
@@ -154,43 +156,49 @@ function check_label() {
 @test "podman selinux: containers in pods share full context" {
     skip_if_no_selinux
 
+    # unique pod name helps when tracking down failure in journal
+    local podname=myselinuxpod_do_share
+
     # We don't need a fullblown pause container; avoid pulling the k8s one
-    run_podman pod create --name myselinuxpod \
+    run_podman pod create --name $podname \
                --infra-image $IMAGE \
                --infra-command /home/podman/pause
 
     # Get baseline
-    run_podman run --rm --pod myselinuxpod $IMAGE cat -v /proc/self/attr/current
+    run_podman run --rm --pod $podname $IMAGE cat -v /proc/self/attr/current
     context_c1="$output"
 
     # Prior to #7902, the labels (':c123,c456') would be different
-    run_podman run --rm --pod myselinuxpod $IMAGE cat -v /proc/self/attr/current
+    run_podman run --rm --pod $podname $IMAGE cat -v /proc/self/attr/current
     is "$output" "$context_c1" "SELinux context of 2nd container matches 1st"
 
     # What the heck. Try a third time just for extra confidence
-    run_podman run --rm --pod myselinuxpod $IMAGE cat -v /proc/self/attr/current
+    run_podman run --rm --pod $podname $IMAGE cat -v /proc/self/attr/current
     is "$output" "$context_c1" "SELinux context of 3rd container matches 1st"
 
-    run_podman pod rm myselinuxpod
+    run_podman pod rm -f -t0 $podname
 }
 
 # more pr #7902
 @test "podman selinux: containers in --no-infra pods do not share context" {
     skip_if_no_selinux
 
+    # unique pod name helps when tracking down failure in journal
+    local podname=myselinuxpod_dont_share
+
     # We don't need a fullblown pause container; avoid pulling the k8s one
-    run_podman pod create --name myselinuxpod --infra=false
+    run_podman pod create --name $podname --infra=false
 
     # Get baseline
-    run_podman run --rm --pod myselinuxpod $IMAGE cat -v /proc/self/attr/current
+    run_podman run --rm --pod $podname $IMAGE cat -v /proc/self/attr/current
     context_c1="$output"
 
     # Even after #7902, labels (':c123,c456') should be different
-    run_podman run --rm --pod myselinuxpod $IMAGE cat -v /proc/self/attr/current
+    run_podman run --rm --pod $podname $IMAGE cat -v /proc/self/attr/current
     assert "$output" != "$context_c1" \
            "context of two separate containers should be different"
 
-    run_podman pod rm myselinuxpod
+    run_podman pod rm -f -t0 $podname
 }
 
 # #8946 - better diagnostics for nonexistent attributes
@@ -214,7 +222,7 @@ function check_label() {
 
     # The '.*' in the error below is for dealing with podman-remote, which
     # includes "error preparing container <sha> for attach" in output.
-    run_podman 126 run --security-opt label=type:foo.bar $IMAGE true
+    run_podman 126 run --rm --security-opt label=type:foo.bar $IMAGE true
     is "$output" "Error.*: $expect" "podman emits useful diagnostic on failure"
 }
 
@@ -228,15 +236,15 @@ function check_label() {
     chcon -vR ${LABEL} $tmpdir
     ls -Z $tmpdir
 
-    run_podman run -v $tmpdir:/test $IMAGE cat /proc/self/attr/current
+    run_podman run --rm -v $tmpdir:/test $IMAGE cat /proc/self/attr/current
     run ls -dZ ${tmpdir}
     is "$output" "${LABEL} ${tmpdir}" "No Relabel Correctly"
 
-    run_podman run -v $tmpdir:/test:z --security-opt label=disable $IMAGE cat /proc/self/attr/current
+    run_podman run --rm -v $tmpdir:/test:z --security-opt label=disable $IMAGE cat /proc/self/attr/current
     run ls -dZ $tmpdir
     is "$output" "${RELABEL} $tmpdir" "Privileged Relabel Correctly"
 
-    run_podman run -v $tmpdir:/test:z --privileged $IMAGE cat /proc/self/attr/current
+    run_podman run --rm -v $tmpdir:/test:z --privileged $IMAGE cat /proc/self/attr/current
     run ls -dZ $tmpdir
     is "$output" "${RELABEL} $tmpdir" "Privileged Relabel Correctly"
 
@@ -271,7 +279,9 @@ function check_label() {
         is "$output" "system_u:object_r:usr_t:s0 $tmpdir/test1" \
            "Start did not Relabel"
     fi
-    run_podman run -v $tmpdir:/test:z $IMAGE cat /proc/self/attr/current
+    run_podman rm label
+
+    run_podman run --rm -v $tmpdir:/test:z $IMAGE cat /proc/self/attr/current
     run ls -dZ $tmpdir
     is "$output" "${RELABEL} $tmpdir" "Shared Relabel Correctly"
 }

test/system/500-networking.bats

@@ -168,7 +168,7 @@ load helpers.network
     is "${lines[0]}" "$pod_name" "hostname is the pod hostname"
     is "${lines[1]}" "$pod_name" "/etc/hostname contains correct pod hostname"
 
-    run_podman pod rm $pod_name
+    run_podman pod rm -f -t0 $pod_name
     is "$output" "$pid" "Only ID in output (no extra errors)"
 
     # Clean up

test/system/helpers.bash

@@ -102,10 +102,26 @@ function basic_setup() {
 # Basic teardown: remove all pods and containers
 function basic_teardown() {
     echo "# [teardown]" >&2
-    run_podman '?' pod rm -t 0 --all --force --ignore
+    local actions=(
-    run_podman '?'     rm -t 0 --all --force --ignore
+        "pod rm -t 0 --all --force --ignore"
-    run_podman '?' network prune --force
+            "rm -t 0 --all --force --ignore"
-    run_podman '?' volume rm -a -f
+        "network prune --force"
+        "volume rm -a -f"
+    )
+    for action in "${actions[@]}"; do
+        run_podman '?' $action
+
+        # The -f commands should never exit nonzero, but if they do we want
+        # to know about it.
+        #   FIXME: someday: also test for [[ -n "$output" ]] - can't do this
+        #   yet because too many tests don't clean up their containers
+        if [[ $status -ne 0 ]]; then
+            echo "# [teardown] $_LOG_PROMPT podman $action" >&3
+            for line in "${lines[*]}"; do
+                echo "# $line" >&3
+            done
+        fi
+    done
 
     command rm -rf $PODMAN_TMPDIR
 }