Merge pull request #87 from deepakpunia-suse/k3s-pipeline-creation

Added pipeline to run observability and backup tests on K3s and Rancher
2025-08-21 15:05:55 +05:30 · 2025-08-21 15:05:55 +05:30 · 6cff2ced7b
parent 9564d6b546 0e7c61015c
commit 6cff2ced7b
4 changed files with 143 additions and 24 deletions
--- a/.github/workflows/docker_run.yaml
+++ b/.github/workflows/docker_run.yaml
@ -1,4 +1,4 @@
-name: Rancher Setup, Chart Installation & E2E Tests
+name: Docker Rancher Setup, Chart Installation & E2E Tests

 on:
  push:
@ -15,7 +15,7 @@ jobs:
    runs-on: ubuntu-latest
    strategy:
      matrix:
-        tag: ['v2.8-head', 'v2.9-head', 'v2.10-head', 'v2.11-head', 'head'] # Rancher version tags
+        tag: ['v2.10-head', 'v2.11-head', 'v2.12-head', 'head'] # Rancher version tags
      fail-fast: false

    steps:
--- a/.github/workflows/k3s_run.yaml
+++ b/.github/workflows/k3s_run.yaml
@ -0,0 +1,108 @@
+name: K3S Installation, Rancher Setup, Chart Installation & E2E Tests
+
+on:
+  push:
+  pull_request:
+    branches:
+      - 'main'
+  workflow_dispatch:
+  schedule:
+    - cron: '0 4 * * *' # Runs daily at 3:00 AM UTC (9:30 AM IST)
+
+jobs:
+  setup_rancher:
+    name: Setup Rancher - Tag - ${{ matrix.rancher_version }}
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        include:
+          - k3s_version: v1.30.14+k3s2
+            rancher_version: latest/devel/2.10
+          - k3s_version: v1.31.11+k3s1
+            rancher_version: latest/devel/2.11
+          - k3s_version: v1.32.7+k3s1
+            rancher_version: latest/devel/2.12
+      fail-fast: false
+
+    steps:
+      - name: Set environment variables
+        run: |
+          echo "HOSTNAME_NAME=localhost" >> $GITHUB_ENV
+          echo "RANCHER_VERSION=${{ matrix.rancher_version }}" >> $GITHUB_ENV
+          echo "K3S_VERSION=${{ matrix.k3s_version }}" >> $GITHUB_ENV
+
+      - name: Install Dependencies
+        run: |
+          sudo apt-get update
+          sudo apt-get install -y curl jq
+      - name: Set up Permissions
+        run: |
+          sudo mkdir -p /etc/rancher/k3s
+          sudo chown -R runner:runner /etc/rancher/k3s
+          sudo mkdir -p $HOME/.kube
+          sudo chown -R runner:runner $HOME/.kube
+
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: './go.mod'
+
+      - name: Install K3S Version ${{ matrix.k3s_version }} & Rancher Version ${{ matrix.rancher_version }}
+        id: install_k3s_and_rancher
+        run: |
+          set -e
+          go test -timeout 30m -run ^TestE2E$ github.com/rancher/observability-e2e/installations/k3s -v -count=1 -ginkgo.v
+
+      - name: Create artifacts directory
+        run: mkdir -p ~/artifacts
+
+      - name: Export CATTLE_TEST_CONFIG environment variable
+        run: echo "CATTLE_TEST_CONFIG=$HOME/cattle-config.yaml" >> $GITHUB_ENV
+
+      - name: Run Observability Charts Tests
+        id: run_observability_tests
+        run: |
+          set -e
+          TEST_LABEL_FILTER=installation go test -timeout 20m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-installation-${{ matrix.k3s_version }}.txt
+          TEST_LABEL_FILTER=E2E go test -timeout 30m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-e2e-${{ matrix.k3s_version }}.txt
+
+      - name: Run Installation Charts Tests For Backup and Restore
+        id: run_installation_tests_backup_restore
+        run: |
+          set -e
+          mv ./tests/helper/yamls/inputBackupRestoreConfig.yaml.example ./tests/helper/yamls/inputBackupRestoreConfig.yaml
+          TEST_LABEL_FILTER=installation go test -timeout 20m github.com/rancher/observability-e2e/tests/backuprestore/functional -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-installation-${{ matrix.k3s_version }}.txt
+
+      - name: Run Observability Upgrade tests
+        id: run_observability_upgrade_tests
+        run: |
+          set -e
+          TEST_LABEL_FILTER=beforeUpgrade go test -timeout 20m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-upgrade-${{ matrix.k3s_version }}.txt
+          TEST_LABEL_FILTER=E2E go test -timeout 30m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-e2e-${{ matrix.k3s_version }}.txt
+          TEST_LABEL_FILTER=afterUpgrade go test -timeout 20m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-upgrade-${{ matrix.k3s_version }}.txt
+          TEST_LABEL_FILTER=E2E go test -timeout 30m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-e2e-${{ matrix.k3s_version }}.txt
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-artifacts-${{ matrix.k3s_version }}
+          path: ~/artifacts
+
+      - name: Cleanup all containers
+        if: always()
+        run: |
+          echo "Cleaning up all containers..."
+          docker ps -q | xargs -r docker rm -f
+
+      - name: Check Test Results and Mark Pipeline
+        if: always()
+        run: |
+          for log_file in ~/artifacts/test-output-installation-${{ matrix.k3s_version }}.txt ~/artifacts/test-output-e2e-${{ matrix.k3s_version }}.txt ~/artifacts/test-output-upgrade-${{ matrix.k3s_version }}.txt; do
+            if [[ -f "$log_file" ]] && grep -q "FAIL" "$log_file"; then
+              echo "$(basename "$log_file") contains failures!"
+              exit 1
+            fi
+          done
--- a/installations/k3s/k3s_install_test.go
+++ b/installations/k3s/k3s_install_test.go
@ -22,6 +22,7 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"

@ -43,19 +44,19 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), Ordered, fun

 	It("Installs Helm CLI", func() {
 		By("Downloading the Helm install script", func() {
-			cmd := exec.Command("curl", "-fsSL", "-o", "/root/get_helm.sh", "https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3")
+			cmd := exec.Command("curl", "-fsSL", "-o", "/tmp/get_helm.sh", "https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3")
 			_, err := cmd.CombinedOutput()
 			Expect(err).ToNot(HaveOccurred())
 		})

 		By("Making the Helm install script executable", func() {
-			cmd := exec.Command("chmod", "+x", "/root/get_helm.sh")
+			cmd := exec.Command("chmod", "+x", "/tmp/get_helm.sh")
 			_, err := cmd.CombinedOutput()
 			Expect(err).ToNot(HaveOccurred())
 		})

 		By("Running the Helm install script", func() {
-			cmd := exec.Command("/root/get_helm.sh")
+			cmd := exec.Command("/tmp/get_helm.sh")
 			cmd.Env = os.Environ()
 			out, err := cmd.CombinedOutput()
 			GinkgoWriter.Printf("helm install script output: %s\n", string(out))
@ -82,7 +83,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), Ordered, fun
 			// Set command and arguments
 			installCmd := exec.Command("sh", fileName)
 			// Add the INSTALL_K3S_VERSION to the command's environment
-			installCmd.Env = append(os.Environ(), "INSTALL_K3S_VERSION="+k3sVersion)
+			installCmd.Env = append(os.Environ(), "INSTALL_K3S_VERSION="+k3sVersion, "K3S_KUBECONFIG_MODE=644")

 			// Retry in case of (sporadic) failure...
 			count := 1
@ -96,24 +97,12 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), Ordered, fun
 		})

 		By("Starting K3s", func() {
+			// Start K3s service with sudo
 			err := exec.Command("sudo", "systemctl", "start", "k3s").Run()
 			Expect(err).To(Not(HaveOccurred()))

-			// Delay few seconds before checking
-			time.Sleep(tools.SetTimeout(20 * time.Second))
-		})
-
-		By("Waiting for K3s to be started", func() {
-			// Wait for all pods to be started
-			checkList := [][]string{
-				{"kube-system", "app=local-path-provisioner"},
-				{"kube-system", "k8s-app=kube-dns"},
-				{"kube-system", "app.kubernetes.io/name=traefik"},
-				{"kube-system", "svccontroller.k3s.cattle.io/svcname=traefik"},
-			}
-			Eventually(func() error {
-				return rancher.CheckPod(k, checkList)
-			}, tools.SetTimeout(4*time.Minute), 30*time.Second).Should(BeNil())
+			// Wait longer for the service to stabilize
+			time.Sleep(tools.SetTimeout(30 * time.Second))
 		})

 		By("Configuring Kubeconfig file", func() {
@ -128,6 +117,24 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), Ordered, fun
 			Expect(err).To(Not(HaveOccurred()))
 		})

+		By("Waiting for K3s to be started", func() {
+			// Configure kubeconfig to use the correct K3s API server[](https://localhost:6443)
+			kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config")
+			err := os.Setenv("KUBECONFIG", kubeconfig)
+			Expect(err).To(Not(HaveOccurred()))
+
+			// Wait for all pods to be started
+			checkList := [][]string{
+				{"kube-system", "app=local-path-provisioner"},
+				{"kube-system", "k8s-app=kube-dns"},
+				{"kube-system", "app.kubernetes.io/name=traefik"},
+				{"kube-system", "svccontroller.k3s.cattle.io/svcname=traefik"},
+			}
+			Eventually(func() error {
+				return rancher.CheckPod(k, checkList)
+			}, tools.SetTimeout(5*time.Minute), 30*time.Second).Should(BeNil())
+		})
+
 		By("Installing CertManager", func() {
 			RunHelmCmdWithRetry("repo", "add", "jetstack", "https://charts.jetstack.io")
 			RunHelmCmdWithRetry("repo", "update")
@ -177,8 +184,10 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), Ordered, fun
 			var err error
 			url := "https://localhost/v3-public/localProviders/local?action=login"
 			username := "admin"
-			password := os.Getenv("PASSWORD")
-			Expect(password).ToNot(BeEmpty(), "PASSWORD env var must be set")
+			var password = "rancherpassword"
+			if envPW := os.Getenv("RANCHER_PASSWORD"); envPW != "" {
+				password = envPW
+			}

 			for i := 0; i < 3; i++ {
 				payload := fmt.Sprintf(`{"username":"%s","password":"%s"}`, username, password)
--- a/installations/k3s/suite_test.go
+++ b/installations/k3s/suite_test.go
@ -62,8 +62,10 @@ func TestE2E(t *testing.T) {
 }

 var _ = BeforeSuite(func() {
-	// export PASSWORD=rancherpassword
 	// export HOSTNAME_NAME=localhost
+	// export RANCHER_VERSION="latest/devel/2.12"
+	// export K3S_VERSION=v1.32.6+k3s1
+	// export RANCHER_PASSWORD="rancherpassword123"
 	hostname = os.Getenv("HOSTNAME_NAME")
 	if hostname == "" {
 		hostname = os.Getenv("localhost")