CI: systests: instrument flaky tests

Three infrequent flakes. Add debug code to help track
down if/when they happen again.

And, one of them, fix a logic bug that will save us 8-10s
on system tests runs.

Signed-off-by: Ed Santiago <santiago@redhat.com>

test/system/250-systemd.bats

@@ -455,8 +455,11 @@ $name stderr" "logs work with passthrough"
     # Kill the pod and make sure the service is not running.
     run_podman pod kill test_pod
     for i in {0..20}; do
+        # echos are for debugging test flakes
+        echo "$_LOG_PROMPT systemctl is-active $service_name"
         run systemctl is-active $service_name
-        if [[ $output == "failed" ]]; then
+        echo "$output"
+        if [[ "$output" == "inactive" ]]; then
             break
         fi
         sleep 0.5

test/system/700-play.bats

@@ -734,8 +734,18 @@ spec:
 
     run_podman kube play --configmap=$configmap_file $pod_file
     run_podman wait test_pod-server
-    run_podman logs test_pod-server
+
-    is "$output" "foo:bar"
+    # systemd logs are unreliable; we may need to retry a few times
+    # https://github.com/systemd/systemd/issues/28650
+    local retries=10
+    while [[ $retries -gt 0 ]]; do
+        run_podman logs test_pod-server
+        test -n "$output" && break
+        sleep 0.1
+        retries=$((retries - 1))
+    done
+    assert "$retries" -gt 0 "Timed out waiting for podman logs"
+    assert "$output" = "foo:bar" "output from podman logs"
 
     run_podman kube down $pod_file
 }

test/system/helpers.registry.bash

@@ -112,6 +112,13 @@ function stop_registry() {
 
     # Make sure socket is closed
     if tcp_port_probe $PODMAN_LOGIN_REGISTRY_PORT; then
+        # for debugging flakes
+        echo ""
+        echo "ps auxww --forest"
+        ps auxww --forest
+        echo ""
+        echo "lsof -i -P"
+        lsof -i -P
         die "Socket still seems open"
     fi
 }