Fixing e2e tests, podman and scheduler fail with mariner images (#1450)

* Fix standalone e2e tests
Fix podman e2e install
Fix scheduler start failure on standalone mariner image variant

Signed-off-by: Anton Troshin <anton@diagrid.io>

* fix test

Signed-off-by: Anton Troshin <anton@diagrid.io>

* revert

Signed-off-by: Anton Troshin <anton@diagrid.io>

* Fix podman machine settings
Add cleanups
Remove parallel tests
Fix mariner volume mount location
Remove old build tags

Signed-off-by: Anton Troshin <anton@diagrid.io>

---------

Signed-off-by: Anton Troshin <anton@diagrid.io>

.github/workflows/kind_e2e.yaml

@@ -50,11 +50,10 @@ jobs:
     name: E2E tests for K8s (KinD)
     runs-on: ubuntu-latest
     env:
-      DAPR_RUNTIME_PINNED_VERSION: 1.13.0-rc.2
+      DAPR_RUNTIME_PINNED_VERSION: 1.13.5
       DAPR_DASHBOARD_PINNED_VERSION: 0.14.0
       DAPR_RUNTIME_LATEST_STABLE_VERSION:
       DAPR_DASHBOARD_LATEST_STABLE_VERSION:
-      DAPR_TGZ: dapr-1.13.0-rc.2.tgz
     strategy:
       fail-fast: false # Keep running if one leg fails.
       matrix:

.github/workflows/self_hosted_e2e.yaml

@@ -38,12 +38,12 @@ jobs:
       GOARCH: ${{ matrix.target_arch }}
       GOPROXY: https://proxy.golang.org
       ARCHIVE_OUTDIR: dist/archives
-      DAPR_RUNTIME_PINNED_VERSION: "1.13.0-rc.2"
+      DAPR_RUNTIME_PINNED_VERSION: "1.13.5"
       DAPR_DASHBOARD_PINNED_VERSION: 0.14.0
       DAPR_RUNTIME_LATEST_STABLE_VERSION: ""
       DAPR_DASHBOARD_LATEST_STABLE_VERSION: ""
       GOLANG_PROTOBUF_REGISTRATION_CONFLICT: warn
-      PODMAN_VERSION: 4.4.4
+      PODMAN_VERSION: 4.9.3
     strategy:
       # TODO: Remove this when our E2E tests are stable for podman on MacOS.
       fail-fast: false # Keep running if one leg fails.
@@ -122,6 +122,7 @@ jobs:
           sudo podman-mac-helper install
           podman machine init
           podman machine start --log-level debug
+          podman machine ssh sudo sysctl -w kernel.keys.maxkeys=20000
           echo "CONTAINER_RUNTIME=podman" >> $GITHUB_ENV
       - name: Determine latest Dapr Runtime version including Pre-releases
         if: github.base_ref == 'master'

pkg/standalone/standalone.go

@@ -652,7 +652,11 @@ func runSchedulerService(wg *sync.WaitGroup, errorChan chan<- error, info initIn
 		// /var/lock and can therefore mount the Docker volume here.
 		// TODO: update the Dapr scheduler dockerfile to create a scheduler user id writeable
 		// directory at /var/lib/dapr/scheduler, then update the path here.
-		args = append(args, "--volume", *info.schedulerVolume+":/var/lock")
+		if strings.EqualFold(info.imageVariant, "mariner") {
+			args = append(args, "--volume", *info.schedulerVolume+":/var/tmp")
+		} else {
+			args = append(args, "--volume", *info.schedulerVolume+":/var/lock")
+		}
 	}
 
 	if info.dockerNetwork != "" {
@@ -673,7 +677,11 @@ func runSchedulerService(wg *sync.WaitGroup, errorChan chan<- error, info initIn
 		)
 	}
 
-	args = append(args, image, "--etcd-data-dir=/var/lock/dapr/scheduler")
+	if strings.EqualFold(info.imageVariant, "mariner") {
+		args = append(args, image, "--etcd-data-dir=/var/tmp/dapr/scheduler")
+	} else {
+		args = append(args, image, "--etcd-data-dir=/var/lock/dapr/scheduler")
+	}
 
 	_, err = utils.RunCmdAndWait(runtimeCmd, args...)
 	if err != nil {

tests/e2e/standalone/commands.go

@@ -1,5 +1,4 @@
 //go:build e2e || template
-// +build e2e template
 
 /*
 Copyright 2022 The Dapr Authors

tests/e2e/standalone/init_negative_test.go

@@ -1,5 +1,4 @@
 //go:build e2e && !template
-// +build e2e,!template
 
 /*
 Copyright 2022 The Dapr Authors
@@ -33,7 +32,6 @@ func TestStandaloneInitNegatives(t *testing.T) {
 	require.NoError(t, err, "expected no error on querying for os home dir")
 
 	t.Run("run without install", func(t *testing.T) {
-		t.Parallel()
 		output, err := cmdRun("")
 		require.Error(t, err, "expected error status on run without install")
 		path := filepath.Join(homeDir, ".dapr", "components")
@@ -45,21 +43,18 @@ func TestStandaloneInitNegatives(t *testing.T) {
 	})
 
 	t.Run("list without install", func(t *testing.T) {
-		t.Parallel()
 		output, err := cmdList("")
 		require.NoError(t, err, "expected no error status on list without install")
 		require.Equal(t, "No Dapr instances found.\n", output)
 	})
 
 	t.Run("stop without install", func(t *testing.T) {
-		t.Parallel()
 		output, err := cmdStopWithAppID("test")
 		require.NoError(t, err, "expected no error on stop without install")
 		require.Contains(t, output, "failed to stop app id test: couldn't find app id test", "expected output to match")
 	})
 
 	t.Run("uninstall without install", func(t *testing.T) {
-		t.Parallel()
 		output, err := cmdUninstall()
 		require.NoError(t, err, "expected no error on uninstall without install")
 		require.Contains(t, output, "Removing Dapr from your machine...", "expected output to contain message")

tests/e2e/standalone/init_run_custom_path_test.go

@@ -1,5 +1,4 @@
 //go:build e2e && !template
-// +build e2e,!template
 
 /*
 Copyright 2022 The Dapr Authors

tests/e2e/standalone/init_test.go

@@ -1,5 +1,4 @@
 //go:build e2e && !template
-// +build e2e,!template
 
 /*
 Copyright 2022 The Dapr Authors
@@ -167,15 +166,17 @@ func TestStandaloneInit(t *testing.T) {
 			placementPort = 6050
 		}
 
-		verifyTCPLocalhost(t,  placementPort)
+		verifyTCPLocalhost(t, placementPort)
 	})
 
 	t.Run("init version with scheduler", func(t *testing.T) {
 		// Ensure a clean environment
 		must(t, cmdUninstall, "failed to uninstall Dapr")
 
+		latestDaprRuntimeVersion, latestDaprDashboardVersion := common.GetVersionsFromEnv(t, true)
+
 		args := []string{
-			"--runtime-version", "1.14.0-rc.3",
+			"--runtime-version", latestDaprRuntimeVersion,
 			"--dev",
 		}
 		output, err := cmdInit(args...)
@@ -189,9 +190,8 @@ func TestStandaloneInit(t *testing.T) {
 		daprPath := filepath.Join(homeDir, ".dapr")
 		require.DirExists(t, daprPath, "Directory %s does not exist", daprPath)
 
-		_, latestDaprDashboardVersion := common.GetVersionsFromEnv(t, true)
+		verifyContainers(t, latestDaprRuntimeVersion)
-		verifyContainers(t, "1.14.0-rc.3")
+		verifyBinaries(t, daprPath, latestDaprRuntimeVersion, latestDaprDashboardVersion)
-		verifyBinaries(t, daprPath, "1.14.0-rc.3", latestDaprDashboardVersion)
 		verifyConfigs(t, daprPath)
 
 		placementPort := 50005
@@ -201,8 +201,8 @@ func TestStandaloneInit(t *testing.T) {
 			schedulerPort = 6060
 		}
 
-		verifyTCPLocalhost(t,  placementPort)
+		verifyTCPLocalhost(t, placementPort)
-		verifyTCPLocalhost(t,  schedulerPort)
+		verifyTCPLocalhost(t, schedulerPort)
 	})
 
 	t.Run("init without runtime-version flag with mariner images", func(t *testing.T) {

tests/e2e/standalone/invoke_test.go

@@ -1,5 +1,4 @@
 //go:build e2e && !template
-// +build e2e,!template
 
 /*
 Copyright 2022 The Dapr Authors

tests/e2e/standalone/list_test.go

@@ -1,5 +1,4 @@
 //go:build e2e && !template
-// +build e2e,!template
 
 /*
 Copyright 2022 The Dapr Authors
@@ -137,6 +136,13 @@ func TestStandaloneList(t *testing.T) {
 		t.Log(output)
 		require.NoError(t, err, "expected no error status on list")
 		require.Equal(t, "No Dapr instances found.\n", output)
+
+		// This test is skipped on Windows, but just in case, try to terminate dashboard process only on non-Windows OSs
+		// stopProcess uses kill command, won't work on windows
+		if runtime.GOOS != "windows" {
+			err = stopProcess("dashboard", "--port", "5555")
+			require.NoError(t, err, "failed to stop dashboard process")
+		}
 	})
 }
 

tests/e2e/standalone/publish_test.go

@@ -1,5 +1,4 @@
 //go:build e2e && !template
-// +build e2e,!template
 
 /*
 Copyright 2022 The Dapr Authors

tests/e2e/standalone/run_template_test.go

@@ -1,6 +1,4 @@
 //go:build !windows && (e2e || template)
-// +build !windows
-// +build e2e template
 
 /*
 Copyright 2023 The Dapr Authors
@@ -23,7 +21,6 @@ import (
 	"context"
 	"fmt"
 	"io/ioutil"
-	"os"
 	"path/filepath"
 	"strings"
 	"testing"
@@ -43,6 +40,7 @@ type AppTestOutput struct {
 }
 
 func TestRunWithTemplateFile(t *testing.T) {
+	cleanUpLogs()
 	ensureDaprInstallation(t)
 	t.Cleanup(func() {
 		// remove dapr installation after all tests in this function.
@@ -54,8 +52,7 @@ func TestRunWithTemplateFile(t *testing.T) {
 		runFilePath := "../testdata/run-template-files/wrong_emit_metrics_app_dapr.yaml"
 		t.Cleanup(func() {
 			// assumption in the test is that there is only one set of app and daprd logs in the logs directory.
-			os.RemoveAll("../../apps/emit-metrics/.dapr/logs")
+			cleanUpLogs()
-			os.RemoveAll("../../apps/processor/.dapr/logs")
 			stopAllApps(t, runFilePath)
 		})
 		args := []string{
@@ -104,8 +101,7 @@ func TestRunWithTemplateFile(t *testing.T) {
 		runFilePath := "../testdata/run-template-files/dapr.yaml"
 		t.Cleanup(func() {
 			// assumption in the test is that there is only one set of app and daprd logs in the logs directory.
-			os.RemoveAll("../../apps/emit-metrics/.dapr/logs")
+			cleanUpLogs()
-			os.RemoveAll("../../apps/processor/.dapr/logs")
 			stopAllApps(t, runFilePath)
 		})
 		args := []string{
@@ -161,8 +157,7 @@ func TestRunWithTemplateFile(t *testing.T) {
 		runFilePath := "../testdata/run-template-files/env_var_not_set_dapr.yaml"
 		t.Cleanup(func() {
 			// assumption in the test is that there is only one set of app and daprd logs in the logs directory.
-			os.RemoveAll("../../apps/emit-metrics/.dapr/logs")
+			cleanUpLogs()
-			os.RemoveAll("../../apps/processor/.dapr/logs")
 			stopAllApps(t, runFilePath)
 		})
 		args := []string{
@@ -212,8 +207,7 @@ func TestRunWithTemplateFile(t *testing.T) {
 		runFilePath := "../testdata/run-template-files/no_app_command.yaml"
 		t.Cleanup(func() {
 			// assumption in the test is that there is only one set of app and daprd logs in the logs directory.
-			os.RemoveAll("../../apps/emit-metrics/.dapr/logs")
+			cleanUpLogs()
-			os.RemoveAll("../../apps/processor/.dapr/logs")
 			stopAllApps(t, runFilePath)
 		})
 		args := []string{
@@ -264,8 +258,7 @@ func TestRunWithTemplateFile(t *testing.T) {
 		runFilePath := "../testdata/run-template-files/empty_app_command.yaml"
 		t.Cleanup(func() {
 			// assumption in the test is that there is only one set of app and daprd logs in the logs directory.
-			os.RemoveAll("../../apps/emit-metrics/.dapr/logs")
+			cleanUpLogs()
-			os.RemoveAll("../../apps/processor/.dapr/logs")
 			stopAllApps(t, runFilePath)
 		})
 		args := []string{
@@ -313,8 +306,7 @@ func TestRunWithTemplateFile(t *testing.T) {
 		runFilePath := "../testdata/run-template-files/app_output_to_file_and_console.yaml"
 		t.Cleanup(func() {
 			// assumption in the test is that there is only one set of app and daprd logs in the logs directory.
-			os.RemoveAll("../../apps/emit-metrics/.dapr/logs")
+			cleanUpLogs()
-			os.RemoveAll("../../apps/processor/.dapr/logs")
 			stopAllApps(t, runFilePath)
 		})
 		args := []string{
@@ -372,6 +364,10 @@ func TestRunTemplateFileWithoutDaprInit(t *testing.T) {
 	// remove any dapr installation before this test.
 	must(t, cmdUninstall, "failed to uninstall Dapr")
 	t.Run("valid template file without dapr init", func(t *testing.T) {
+		t.Cleanup(func() {
+			// assumption in the test is that there is only one set of app and daprd logs in the logs directory.
+			cleanUpLogs()
+		})
 		args := []string{
 			"-f", "../testdata/run-template-files/no_app_command.yaml",
 		}

tests/e2e/standalone/run_test.go

@@ -1,5 +1,4 @@
 //go:build e2e && !template
-// +build e2e,!template
 
 /*
 Copyright 2022 The Dapr Authors

tests/e2e/standalone/stop_test.go

@@ -1,5 +1,4 @@
 //go:build e2e && !template
-// +build e2e,!template
 
 /*
 Copyright 2022 The Dapr Authors
@@ -18,6 +17,7 @@ package standalone_test
 
 import (
 	"testing"
+	"time"
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -25,6 +25,14 @@ import (
 
 func TestStandaloneStop(t *testing.T) {
 	ensureDaprInstallation(t)
+
+	time.Sleep(5 * time.Second)
+
+	t.Cleanup(func() {
+		// remove dapr installation after all tests in this function.
+		must(t, cmdUninstall, "failed to uninstall Dapr")
+	})
+
 	executeAgainstRunningDapr(t, func() {
 		t.Run("stop", func(t *testing.T) {
 			output, err := cmdStopWithAppID("dapr_e2e_stop")

tests/e2e/standalone/stop_with_run_template_test.go

@@ -1,6 +1,4 @@
 //go:build !windows && (e2e || template)
-// +build !windows
-// +build e2e template
 
 /*
 Copyright 2023 The Dapr Authors
@@ -22,7 +20,6 @@ package standalone_test
 import (
 	"encoding/json"
 	"fmt"
-	"os"
 	"testing"
 	"time"
 
@@ -31,6 +28,9 @@ import (
 )
 
 func TestStopAppsStartedWithRunTemplate(t *testing.T) {
+	// clean up logs before starting the tests
+	cleanUpLogs()
+
 	ensureDaprInstallation(t)
 	t.Cleanup(func() {
 		// remove dapr installation after all tests in this function.
@@ -38,6 +38,9 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) {
 	})
 
 	t.Run("stop apps by passing run template file", func(t *testing.T) {
+		t.Cleanup(func() {
+			cleanUpLogs()
+		})
 		go ensureAllAppsStartedWithRunTemplate(t)
 		time.Sleep(10 * time.Second)
 		cliPID := getCLIPID(t)
@@ -50,6 +53,9 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) {
 	})
 
 	t.Run("stop apps by passing a directory containing dapr.yaml", func(t *testing.T) {
+		t.Cleanup(func() {
+			cleanUpLogs()
+		})
 		go ensureAllAppsStartedWithRunTemplate(t)
 		time.Sleep(10 * time.Second)
 		cliPID := getCLIPID(t)
@@ -60,6 +66,9 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) {
 	})
 
 	t.Run("stop apps by passing an invalid directory", func(t *testing.T) {
+		t.Cleanup(func() {
+			cleanUpLogs()
+		})
 		go ensureAllAppsStartedWithRunTemplate(t)
 		time.Sleep(10 * time.Second)
 		output, err := cmdStopWithRunTemplate("../testdata/invalid-dir")
@@ -72,6 +81,9 @@ func TestStopAppsStartedWithRunTemplate(t *testing.T) {
 	})
 
 	t.Run("stop apps started with run template", func(t *testing.T) {
+		t.Cleanup(func() {
+			cleanUpLogs()
+		})
 		go ensureAllAppsStartedWithRunTemplate(t)
 		time.Sleep(10 * time.Second)
 		cliPID := getCLIPID(t)
@@ -95,8 +107,7 @@ func ensureAllAppsStartedWithRunTemplate(t *testing.T) {
 func tearDownTestSetup(t *testing.T) {
 	// remove dapr installation after all tests in this function.
 	must(t, cmdUninstall, "failed to uninstall Dapr")
-	os.RemoveAll("../../apps/emit-metrics/.dapr/logs")
+	cleanUpLogs()
-	os.RemoveAll("../../apps/processor/.dapr/logs")
 }
 
 func getCLIPID(t *testing.T) string {

tests/e2e/standalone/uninstall_test.go

@@ -1,5 +1,4 @@
 //go:build e2e && !template
-// +build e2e,!template
 
 /*
 Copyright 2022 The Dapr Authors

tests/e2e/standalone/utils.go

@@ -1,5 +1,4 @@
 //go:build e2e || template
-// +build e2e template
 
 /*
 Copyright 2022 The Dapr Authors
@@ -157,3 +156,40 @@ func containerRuntime() string {
 	}
 	return ""
 }
+
+func getRunningProcesses() []string {
+	cmd := exec.Command("ps", "-o", "pid,command")
+	output, err := cmd.Output()
+	if err != nil {
+		return nil
+	}
+
+	processes := strings.Split(string(output), "\n")
+
+	// clean the process output whitespace
+	for i, process := range processes {
+		processes[i] = strings.TrimSpace(process)
+	}
+	return processes
+}
+
+func stopProcess(args ...string) error {
+	processCommand := strings.Join(args, " ")
+	processes := getRunningProcesses()
+	for _, process := range processes {
+		if strings.Contains(process, processCommand) {
+			processSplit := strings.SplitN(process, " ", 2)
+			cmd := exec.Command("kill", "-9", processSplit[0])
+			err := cmd.Run()
+			if err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func cleanUpLogs() {
+	os.RemoveAll("../../apps/emit-metrics/.dapr/logs")
+	os.RemoveAll("../../apps/processor/.dapr/logs")
+}

tests/e2e/standalone/version_test.go

@@ -1,5 +1,4 @@
 //go:build e2e && !template
-// +build e2e,!template
 
 /*
 Copyright 2022 The Dapr Authors

tests/e2e/standalone/windows_run_template_test.go

@@ -1,6 +1,4 @@
 //go:build windows && (e2e || template)
-// +build windows
-// +build e2e template
 
 /*
 Copyright 2023 The Dapr Authors