WIP: Enable (and disable) remote testing

podman-remote has not been tested. A principal part of the
problem was #5387 - the YAML I wrote did not have the
intended effect, it did not set TEST_REMOTE_CLIENT=true
and because of my multiple iterations I did not catch this
during testing.

Part 1 of this PR is to fix .cirrus.yml to enable remote tests.

Part 2 -- what I had first noticed and tried to fix -- is that
rootless_test.sh was never running remote because, of course,
envariables are not sent via ssh. I reworked integration_test.sh
and rootless_test.sh to use a command-line decision instead.

Part 3, sigh, is to disable one failing integration test
and *all* system tests, because so many of the latter are
failing. Addressing those failures needs to be done in
subsequent PRs. Issues #6538, #6539, #6540 are filed for
some of the problems I isolated. There will be more.

Also, minor, fixed some stale references to varlink.

Signed-off-by: Ed Santiago <santiago@redhat.com>

.cirrus.yml

@@ -434,12 +434,8 @@ testing_task:
     env:
         ADD_SECOND_PARTITION: 'true'
         matrix:
-            - name: remote
-              env:
-                  TEST_REMOTE_CLIENT: 'true'
-            - name: local
-              env:
-                  TEST_REMOTE_CLIENT: 'false'
+          - TEST_REMOTE_CLIENT: 'true'
+          - TEST_REMOTE_CLIENT: 'false'
 
     networking_script: '${CIRRUS_WORKING_DIR}/${SCRIPT_BASE}/networking.sh'
     setup_environment_script: '$SCRIPT_BASE/setup_environment.sh |& ${TIMESTAMP}'
@@ -486,12 +482,8 @@ special_testing_rootless_task:
         ADD_SECOND_PARTITION: 'true'
         SPECIALMODE: 'rootless'  # See docs
         matrix:
-            - name: remote
-              env:
-                  TEST_REMOTE_CLIENT: 'true'
-            - name: local
-              env:
-                  TEST_REMOTE_CLIENT: 'false'
+          - TEST_REMOTE_CLIENT: 'true'
+          - TEST_REMOTE_CLIENT: 'false'
 
     timeout_in: 60m
 
@@ -693,12 +685,8 @@ verify_test_built_images_task:
     env:
         ADD_SECOND_PARTITION: 'true'
         matrix:
-            - name: remote
-              env:
-                  TEST_REMOTE_CLIENT: 'true'
-            - name: local
-              env:
-                  TEST_REMOTE_CLIENT: 'false'
+          - TEST_REMOTE_CLIENT: 'true'
+          - TEST_REMOTE_CLIENT: 'false'
         matrix:
             # Required env. var. by check_image_script
             PACKER_BUILDER_NAME: "${FEDORA_NAME}"

Makefile

@@ -88,8 +88,8 @@ RELEASE_DIST_VER ?= $(shell hack/get_release_info.sh DIST_VER)
 RELEASE_ARCH ?= $(shell hack/get_release_info.sh ARCH)
 RELEASE_BASENAME := $(shell hack/get_release_info.sh BASENAME)
 
-# If non-empty, logs all output from varlink during remote system testing
-VARLINK_LOG ?=
+# If non-empty, logs all output from server during remote system testing
+PODMAN_SERVER_LOG ?=
 
 # If GOPATH not specified, use one in the local directory
 ifeq ($(GOPATH),)
@@ -357,22 +357,28 @@ localsystem:
 remotesystem:
 	# Wipe existing config, database, and cache: start with clean slate.
 	$(RM) -rf ${HOME}/.local/share/containers ${HOME}/.config/containers
-	# Start varlink server using tmp socket; loop-wait for it;
+	# Start podman server using tmp socket; loop-wait for it;
 	# test podman-remote; kill server, clean up tmp socket file.
-	# varlink server spews copious unhelpful output; ignore it.
+	# podman server spews copious unhelpful output; ignore it.
+	# FIXME FIXME FIXME: remove 'exit 0' after #6538 and #6539 are fixed
+	exit 0;\
 	rc=0;\
 	if timeout -v 1 true; then \
 		SOCK_FILE=$(shell mktemp --dry-run --tmpdir podman.XXXXXX);\
-		export PODMAN_SOCKEY=unix:$$SOCK_FILE; \
-		./bin/podman system service --timeout=0 $$PODMAN_VARLINK_ADDRESS &> $(if $(VARLINK_LOG),$(VARLINK_LOG),/dev/null) & \
+		export PODMAN_SOCKET=unix:$$SOCK_FILE; \
+		./bin/podman system service --timeout=0 $$PODMAN_SOCKET &> $(if $(PODMAN_SERVER_LOG),$(PODMAN_SERVER_LOG),/dev/null) & \
 		retry=5;\
 		while [[ $$retry -ge 0 ]]; do\
 			echo Waiting for server...;\
 			sleep 1;\
-			./bin/podman-remote --remote $(SOCK_FILE) info &>/dev/null && break;\
+			./bin/podman-remote --url $$PODMAN_SOCKET info &>/dev/null && break;\
 			retry=$$(expr $$retry - 1);\
 		done;\
-		env PODMAN=./bin/podman-remote bats test/system/ ;\
+		if [[ $$retry -lt 0 ]]; then\
+			echo "Error: ./bin/podman system service did not come up on $$SOCK_FILE" >&2;\
+			exit 1;\
+		fi;\
+		env PODMAN="./bin/podman-remote --url $$PODMAN_SOCKET" bats test/system/ ;\
 		rc=$$?;\
 		kill %1;\
 		rm -f $$SOCK_FILE;\

contrib/cirrus/integration_test.sh

@@ -6,6 +6,11 @@ source $(dirname $0)/lib.sh
 
 req_env_var GOSRC SCRIPT_BASE OS_RELEASE_ID OS_RELEASE_VER CONTAINER_RUNTIME VARLINK_LOG
 
+LOCAL_OR_REMOTE=local
+if [[ "$TEST_REMOTE_CLIENT" = "true" ]]; then
+    LOCAL_OR_REMOTE=remote
+fi
+
 # Our name must be of the form xxxx_test or xxxx_test.sh, where xxxx is
 # the test suite to run; currently (2019-05) the only option is 'integration'
 # but pr2947 intends to add 'system'.
@@ -34,7 +39,7 @@ case "$SPECIALMODE" in
         req_env_var ROOTLESS_USER
         ssh $ROOTLESS_USER@localhost \
                 -o UserKnownHostsFile=/dev/null -o StrictHostKeyChecking=no \
-                -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE}
+                -o CheckHostIP=no $GOSRC/$SCRIPT_BASE/rootless_test.sh ${TESTSUITE} ${LOCAL_OR_REMOTE}
         ;;
     endpoint)
         make
@@ -53,12 +58,7 @@ case "$SPECIALMODE" in
         make install PREFIX=/usr ETCDIR=/etc
         make test-binaries
         make .install.bats
-        if [[ "$TEST_REMOTE_CLIENT" == "true" ]]
-        then
-            make remote${TESTSUITE} VARLINK_LOG=$VARLINK_LOG
-        else
-            make local${TESTSUITE}
-        fi
+        make ${LOCAL_OR_REMOTE}${TESTSUITE} PODMAN_SERVER_LOG=$PODMAN_SERVER_LOG
         ;;
     *)
         die 110 "Unsupported \$SPECIALMODE: $SPECIALMODE"

contrib/cirrus/rootless_test.sh

@@ -2,14 +2,6 @@
 
 set -e
 
-remote=0
-
-# The TEST_REMOTE_CLIENT environment variable decides whether
-# to test varlink
-if [[ "$TEST_REMOTE_CLIENT" == "true" ]]; then
-    remote=1
-fi
-
 source $(dirname $0)/lib.sh
 
 if [[ "$UID" == "0" ]]
@@ -18,11 +10,8 @@ then
     exit 1
 fi
 
-# Which set of tests to run; possible alternative is "system"
-TESTSUITE=integration
-if [[ -n "$*" ]]; then
-    TESTSUITE="$1"
-fi
+TESTSUITE=${1?Missing TESTSUITE argument (arg1)}
+LOCAL_OR_REMOTE=${2?Missing LOCAL_OR_REMOTE argument (arg2)}
 
 # Ensure environment setup correctly
 req_env_var GOSRC ROOTLESS_USER
@@ -31,7 +20,6 @@ echo "."
 echo "Hello, my name is $USER and I live in $PWD can I be your friend?"
 echo "."
 
-export PODMAN_VARLINK_ADDRESS=unix:/tmp/podman-$(id -u)
 show_env_vars
 
 set -x
@@ -39,8 +27,4 @@ cd "$GOSRC"
 make
 make varlink_generate
 make test-binaries
-if [ $remote -eq 0 ]; then
-    make local${TESTSUITE}
-else
-    make remote${TESTSUITE}
-fi
+make ${LOCAL_OR_REMOTE}${TESTSUITE}

test/e2e/cp_test.go

@@ -141,6 +141,8 @@ var _ = Describe("Podman cp", func() {
 	})
 
 	It("podman cp stdin/stdout", func() {
+		SkipIfRemote()
+		Skip("Looks like SkipIfRemote() is not working")
 		session := podmanTest.Podman([]string{"create", ALPINE, "ls", "foo"})
 		session.WaitWithDefaultTimeout()
 		Expect(session.ExitCode()).To(Equal(0))