Merge pull request #14569 from cevich/podman_machine_poc

Cirrus: Add podman-machine integration test

.cirrus.yml

@@ -30,8 +30,10 @@ env:
     PRIOR_FEDORA_NAME: "fedora-35"
     UBUNTU_NAME: "ubuntu-2110"
 
-    # Google-cloud VM Images
+    # Image identifiers
     IMAGE_SUFFIX: "c6211193021923328"
+    FEDORA_AMI_ID: "ami-06a41d8a81ab56afa"
+    # Complete image names
     FEDORA_CACHE_IMAGE_NAME: "fedora-${IMAGE_SUFFIX}"
     PRIOR_FEDORA_CACHE_IMAGE_NAME: "prior-fedora-${IMAGE_SUFFIX}"
     UBUNTU_CACHE_IMAGE_NAME: "ubuntu-${IMAGE_SUFFIX}"
@@ -66,6 +68,8 @@ timeout_in: 60m
 
 gcp_credentials: ENCRYPTED[a28959877b2c9c36f151781b0a05407218cda646c7d047fc556e42f55e097e897ab63ee78369dae141dcf0b46a9d0cdd]
 
+aws_credentials: ENCRYPTED[4ca070bffe28eb9b27d63c568b52970dd46f119c3a83b8e443241e895dbf1737580b4d84eed27a311a2b74287ef9f79f]
+
 
 # Attempt to prevent flakes by confirming all required external/3rd-party
 # services are available and functional.
@@ -577,6 +581,35 @@ rootless_integration_test_task:
     always: *int_logs_artifacts
 
 
+podman_machine_task:
+    name: *std_name_fmt
+    alias: podman_machine
+    only_if: *not_tag_branch_build_docs
+    # Manually-triggered task: This is "expensive" to run.
+    # DO NOT ADD THIS TASK AS DEPENDENCY FOR `success_task`
+    # it will cause 'success' to block.
+    trigger_type: manual
+    depends_on:
+        - build
+        - local_integration_test
+        - remote_integration_test
+        - container_integration_test
+        - rootless_integration_test
+    ec2_instance:
+        image: "${VM_IMAGE_NAME}"
+        type: m5zn.metal  # Bare-metal instance is required
+        region: us-east-1
+    env:
+      TEST_FLAVOR: "machine"
+      PRIV_NAME: "rootless"  # intended use-case
+      DISTRO_NV: "${FEDORA_NAME}"
+      VM_IMAGE_NAME: "${FEDORA_AMI_ID}"
+    clone_script: *get_gosrc
+    setup_script: *setup
+    main_script: *main
+    always: *int_logs_artifacts
+
+
 # Always run subsequent to integration tests.  While parallelism is lost
 # with runtime, debugging system-test failures can be more challenging
 # for some golang developers.  Otherwise the following tasks run across
@@ -842,6 +875,9 @@ success_task:
         - remote_integration_test
         - container_integration_test
         - rootless_integration_test
+        # Manually triggered task. If made automatic, remove bypass
+        # in contrib/cirrus/cirrus_yaml_test.py for this task.
+        # - podman_machine
         - local_system_test
         - remote_system_test
         - rootless_system_test

Makefile

@@ -112,6 +112,10 @@ LIBSECCOMP_COMMIT := v2.3.3
 # Rarely if ever should integration tests take more than 50min,
 # caller may override in special circumstances if needed.
 GINKGOTIMEOUT ?= -timeout=90m
+# By default, run test/e2e
+GINKGOWHAT ?= test/e2e/.
+# By default, run tests in parallel across 3 nodes.
+GINKGONODES ?= 3
 
 # Conditional required to produce empty-output if binary not built yet.
 RELEASE_VERSION = $(shell if test -x test/version/version; then test/version/version; fi)
@@ -524,7 +528,7 @@ test: localunit localintegration remoteintegration localsystem remotesystem  ##
 .PHONY: ginkgo-run
 ginkgo-run:
 	ACK_GINKGO_RC=true ginkgo version
-	ACK_GINKGO_RC=true ginkgo -v $(TESTFLAGS) -tags "$(TAGS)" $(GINKGOTIMEOUT) -cover -flakeAttempts 3 -progress -trace -noColor -nodes 3 -debug test/e2e/. $(HACK)
+	ACK_GINKGO_RC=true ginkgo -v $(TESTFLAGS) -tags "$(TAGS)" $(GINKGOTIMEOUT) -cover -flakeAttempts 3 -progress -trace -noColor -nodes $(GINKGONODES) -debug $(GINKGOWHAT) $(HACK)
 
 .PHONY: ginkgo
 ginkgo:
@@ -540,6 +544,10 @@ localintegration: test-binaries ginkgo
 .PHONY: remoteintegration
 remoteintegration: test-binaries ginkgo-remote
 
+.PHONY: localmachine
+localmachine: test-binaries
+	$(MAKE) ginkgo-run GINKGONODES=1 GINKGOWHAT=pkg/machine/e2e/. HACK=
+
 .PHONY: localbenchmarks
 localbenchmarks: test-binaries
 	PATH=$(PATH):$(shell pwd)/hack ACK_GINKGO_RC=true ginkgo \

contrib/cirrus/cirrus_yaml_test.py

@@ -26,7 +26,7 @@ class TestCaseBase(unittest.TestCase):
 class TestDependsOn(TestCaseBase):
 
     ALL_TASK_NAMES = None
-    SUCCESS_DEPS_EXCLUDE = set(['success', 'artifacts',
+    SUCCESS_DEPS_EXCLUDE = set(['success', 'artifacts', 'podman_machine',
         'test_image_build', 'release', 'release_test'])
 
     def setUp(self):

contrib/cirrus/runner.sh

@@ -379,6 +379,13 @@ dotest() {
         |& logformatter
 }
 
+_run_machine() {
+    # TODO: This is a manually-triggered task, if that ever changes need to
+    # add something like:
+    # _bail_if_test_can_be_skipped docs test/e2e test/system test/python
+    make localmachine |& logformatter
+}
+
 # Optimization: will exit if the only PR diffs are under docs/ or tests/
 # with the exception of any given arguments. E.g., don't run e2e or upgrade
 # or bud tests if the only PR changes are in test/system.

contrib/cirrus/setup_environment.sh

@@ -99,11 +99,28 @@ esac
 if ((CONTAINER==0)); then  # Not yet running inside a container
     # Discovered reemergence of BFQ scheduler bug in kernel 5.8.12-200
     # which causes a kernel panic when system is under heavy I/O load.
-    # Previously discovered in F32beta and confirmed fixed. It's been
-    # observed in F31 kernels as well.  Deploy workaround for all VMs
-    # to ensure a more stable I/O scheduler (elevator).
-    echo "mq-deadline" > /sys/block/sda/queue/scheduler
-    warn "I/O scheduler: $(cat /sys/block/sda/queue/scheduler)"
+    # Disable the I/O scheduler (a.k.a. elevator) for all environments,
+    # leaving optimization up to underlying storage infrastructure.
+    testfs="/"  # mountpoint that experiences the most I/O during testing
+    msg "Querying block device owning partition hosting the '$testfs' filesystem"
+    # Need --nofsroot b/c btrfs appends subvolume label to `source` name
+    testdev=$(findmnt --canonicalize --noheadings --nofsroot \
+              --output source --mountpoint $testfs)
+    msg "    found partition: '$testdev'"
+    testdisk=$(lsblk --noheadings --output pkname --paths $testdev)
+    msg "    found block dev: '$testdisk'"
+    testsched="/sys/block/$(basename $testdisk)/queue/scheduler"
+    if [[ -n "$testdev" ]] && [[ -n "$testdisk" ]] && [[ -e "$testsched" ]]; then
+        msg "    Found active I/O scheduler: $(cat $testsched)"
+        if [[ ! "$(<$testsched)" =~ \[none\]  ]]; then
+            msg "    Disabling elevator for '$testsched'"
+            echo "none" > "$testsched"
+        else
+            msg "    Elevator already disabled"
+        fi
+    else
+        warn "Sys node for elevator doesn't exist: '$testsched'"
+    fi
 fi
 
 # Which distribution are we testing on.
@@ -294,6 +311,13 @@ case "$TEST_FLAVOR" in
             die "Invalid value for \$TEST_ENVIRON=$TEST_ENVIRON"
         fi
 
+        install_test_configs
+        ;;
+    machine)
+        rpm -ivh $PACKAGE_DOWNLOAD_DIR/podman-gvproxy*
+        remove_packaged_podman_files
+        make install.tools
+        make install PREFIX=/usr ETCDIR=/etc
         install_test_configs
         ;;
     gitlab)