From 0e7c61015c7cd006555773048e6a0ab52520e863 Mon Sep 17 00:00:00 2001 From: Deepak Punia Date: Mon, 11 Aug 2025 17:41:09 +0530 Subject: [PATCH] Added pipeline to run observability and backup tests on K3s and Rancher --- .github/workflows/docker_run.yaml | 4 +- .github/workflows/k3s_run.yaml | 108 ++++++++++++++++++++++++++ installations/k3s/k3s_install_test.go | 51 +++++++----- installations/k3s/suite_test.go | 4 +- 4 files changed, 143 insertions(+), 24 deletions(-) create mode 100644 .github/workflows/k3s_run.yaml diff --git a/.github/workflows/docker_run.yaml b/.github/workflows/docker_run.yaml index af15014..fded12b 100644 --- a/.github/workflows/docker_run.yaml +++ b/.github/workflows/docker_run.yaml @@ -1,4 +1,4 @@ -name: Rancher Setup, Chart Installation & E2E Tests +name: Docker Rancher Setup, Chart Installation & E2E Tests on: push: @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - tag: ['v2.8-head', 'v2.9-head', 'v2.10-head', 'v2.11-head', 'head'] # Rancher version tags + tag: ['v2.10-head', 'v2.11-head', 'v2.12-head', 'head'] # Rancher version tags fail-fast: false steps: diff --git a/.github/workflows/k3s_run.yaml b/.github/workflows/k3s_run.yaml new file mode 100644 index 0000000..6dd2ec1 --- /dev/null +++ b/.github/workflows/k3s_run.yaml @@ -0,0 +1,108 @@ +name: K3S Installation, Rancher Setup, Chart Installation & E2E Tests + +on: + push: + pull_request: + branches: + - 'main' + workflow_dispatch: + schedule: + - cron: '0 4 * * *' # Runs daily at 3:00 AM UTC (9:30 AM IST) + +jobs: + setup_rancher: + name: Setup Rancher - Tag - ${{ matrix.rancher_version }} + runs-on: ubuntu-latest + strategy: + matrix: + include: + - k3s_version: v1.30.14+k3s2 + rancher_version: latest/devel/2.10 + - k3s_version: v1.31.11+k3s1 + rancher_version: latest/devel/2.11 + - k3s_version: v1.32.7+k3s1 + rancher_version: latest/devel/2.12 + fail-fast: false + + steps: + - name: Set environment variables + run: | + echo "HOSTNAME_NAME=localhost" >> $GITHUB_ENV + echo "RANCHER_VERSION=${{ matrix.rancher_version }}" >> $GITHUB_ENV + echo "K3S_VERSION=${{ matrix.k3s_version }}" >> $GITHUB_ENV + + - name: Install Dependencies + run: | + sudo apt-get update + sudo apt-get install -y curl jq + - name: Set up Permissions + run: | + sudo mkdir -p /etc/rancher/k3s + sudo chown -R runner:runner /etc/rancher/k3s + sudo mkdir -p $HOME/.kube + sudo chown -R runner:runner $HOME/.kube + + - name: Checkout code + uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v4 + with: + go-version-file: './go.mod' + + - name: Install K3S Version ${{ matrix.k3s_version }} & Rancher Version ${{ matrix.rancher_version }} + id: install_k3s_and_rancher + run: | + set -e + go test -timeout 30m -run ^TestE2E$ github.com/rancher/observability-e2e/installations/k3s -v -count=1 -ginkgo.v + + - name: Create artifacts directory + run: mkdir -p ~/artifacts + + - name: Export CATTLE_TEST_CONFIG environment variable + run: echo "CATTLE_TEST_CONFIG=$HOME/cattle-config.yaml" >> $GITHUB_ENV + + - name: Run Observability Charts Tests + id: run_observability_tests + run: | + set -e + TEST_LABEL_FILTER=installation go test -timeout 20m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-installation-${{ matrix.k3s_version }}.txt + TEST_LABEL_FILTER=E2E go test -timeout 30m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-e2e-${{ matrix.k3s_version }}.txt + + - name: Run Installation Charts Tests For Backup and Restore + id: run_installation_tests_backup_restore + run: | + set -e + mv ./tests/helper/yamls/inputBackupRestoreConfig.yaml.example ./tests/helper/yamls/inputBackupRestoreConfig.yaml + TEST_LABEL_FILTER=installation go test -timeout 20m github.com/rancher/observability-e2e/tests/backuprestore/functional -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-installation-${{ matrix.k3s_version }}.txt + + - name: Run Observability Upgrade tests + id: run_observability_upgrade_tests + run: | + set -e + TEST_LABEL_FILTER=beforeUpgrade go test -timeout 20m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-upgrade-${{ matrix.k3s_version }}.txt + TEST_LABEL_FILTER=E2E go test -timeout 30m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-e2e-${{ matrix.k3s_version }}.txt + TEST_LABEL_FILTER=afterUpgrade go test -timeout 20m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-upgrade-${{ matrix.k3s_version }}.txt + TEST_LABEL_FILTER=E2E go test -timeout 30m github.com/rancher/observability-e2e/tests/e2e -v -count=1 -ginkgo.v | tee ~/artifacts/test-output-e2e-${{ matrix.k3s_version }}.txt + + - name: Upload artifacts + uses: actions/upload-artifact@v4 + with: + name: test-artifacts-${{ matrix.k3s_version }} + path: ~/artifacts + + - name: Cleanup all containers + if: always() + run: | + echo "Cleaning up all containers..." + docker ps -q | xargs -r docker rm -f + + - name: Check Test Results and Mark Pipeline + if: always() + run: | + for log_file in ~/artifacts/test-output-installation-${{ matrix.k3s_version }}.txt ~/artifacts/test-output-e2e-${{ matrix.k3s_version }}.txt ~/artifacts/test-output-upgrade-${{ matrix.k3s_version }}.txt; do + if [[ -f "$log_file" ]] && grep -q "FAIL" "$log_file"; then + echo "$(basename "$log_file") contains failures!" + exit 1 + fi + done diff --git a/installations/k3s/k3s_install_test.go b/installations/k3s/k3s_install_test.go index 2a64aaf..5f0f482 100644 --- a/installations/k3s/k3s_install_test.go +++ b/installations/k3s/k3s_install_test.go @@ -22,6 +22,7 @@ import ( "net/http" "os" "os/exec" + "path/filepath" "strings" "time" @@ -43,19 +44,19 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), Ordered, fun It("Installs Helm CLI", func() { By("Downloading the Helm install script", func() { - cmd := exec.Command("curl", "-fsSL", "-o", "/root/get_helm.sh", "https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3") + cmd := exec.Command("curl", "-fsSL", "-o", "/tmp/get_helm.sh", "https://raw.githubusercontent.com/helm/helm/main/scripts/get-helm-3") _, err := cmd.CombinedOutput() Expect(err).ToNot(HaveOccurred()) }) By("Making the Helm install script executable", func() { - cmd := exec.Command("chmod", "+x", "/root/get_helm.sh") + cmd := exec.Command("chmod", "+x", "/tmp/get_helm.sh") _, err := cmd.CombinedOutput() Expect(err).ToNot(HaveOccurred()) }) By("Running the Helm install script", func() { - cmd := exec.Command("/root/get_helm.sh") + cmd := exec.Command("/tmp/get_helm.sh") cmd.Env = os.Environ() out, err := cmd.CombinedOutput() GinkgoWriter.Printf("helm install script output: %s\n", string(out)) @@ -82,7 +83,7 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), Ordered, fun // Set command and arguments installCmd := exec.Command("sh", fileName) // Add the INSTALL_K3S_VERSION to the command's environment - installCmd.Env = append(os.Environ(), "INSTALL_K3S_VERSION="+k3sVersion) + installCmd.Env = append(os.Environ(), "INSTALL_K3S_VERSION="+k3sVersion, "K3S_KUBECONFIG_MODE=644") // Retry in case of (sporadic) failure... count := 1 @@ -96,24 +97,12 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), Ordered, fun }) By("Starting K3s", func() { + // Start K3s service with sudo err := exec.Command("sudo", "systemctl", "start", "k3s").Run() Expect(err).To(Not(HaveOccurred())) - // Delay few seconds before checking - time.Sleep(tools.SetTimeout(20 * time.Second)) - }) - - By("Waiting for K3s to be started", func() { - // Wait for all pods to be started - checkList := [][]string{ - {"kube-system", "app=local-path-provisioner"}, - {"kube-system", "k8s-app=kube-dns"}, - {"kube-system", "app.kubernetes.io/name=traefik"}, - {"kube-system", "svccontroller.k3s.cattle.io/svcname=traefik"}, - } - Eventually(func() error { - return rancher.CheckPod(k, checkList) - }, tools.SetTimeout(4*time.Minute), 30*time.Second).Should(BeNil()) + // Wait longer for the service to stabilize + time.Sleep(tools.SetTimeout(30 * time.Second)) }) By("Configuring Kubeconfig file", func() { @@ -128,6 +117,24 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), Ordered, fun Expect(err).To(Not(HaveOccurred())) }) + By("Waiting for K3s to be started", func() { + // Configure kubeconfig to use the correct K3s API server[](https://localhost:6443) + kubeconfig := filepath.Join(os.Getenv("HOME"), ".kube", "config") + err := os.Setenv("KUBECONFIG", kubeconfig) + Expect(err).To(Not(HaveOccurred())) + + // Wait for all pods to be started + checkList := [][]string{ + {"kube-system", "app=local-path-provisioner"}, + {"kube-system", "k8s-app=kube-dns"}, + {"kube-system", "app.kubernetes.io/name=traefik"}, + {"kube-system", "svccontroller.k3s.cattle.io/svcname=traefik"}, + } + Eventually(func() error { + return rancher.CheckPod(k, checkList) + }, tools.SetTimeout(5*time.Minute), 30*time.Second).Should(BeNil()) + }) + By("Installing CertManager", func() { RunHelmCmdWithRetry("repo", "add", "jetstack", "https://charts.jetstack.io") RunHelmCmdWithRetry("repo", "update") @@ -177,8 +184,10 @@ var _ = Describe("E2E - Install Rancher Manager", Label("install"), Ordered, fun var err error url := "https://localhost/v3-public/localProviders/local?action=login" username := "admin" - password := os.Getenv("PASSWORD") - Expect(password).ToNot(BeEmpty(), "PASSWORD env var must be set") + var password = "rancherpassword" + if envPW := os.Getenv("RANCHER_PASSWORD"); envPW != "" { + password = envPW + } for i := 0; i < 3; i++ { payload := fmt.Sprintf(`{"username":"%s","password":"%s"}`, username, password) diff --git a/installations/k3s/suite_test.go b/installations/k3s/suite_test.go index 71fe8c6..79a001c 100644 --- a/installations/k3s/suite_test.go +++ b/installations/k3s/suite_test.go @@ -62,8 +62,10 @@ func TestE2E(t *testing.T) { } var _ = BeforeSuite(func() { - // export PASSWORD=rancherpassword // export HOSTNAME_NAME=localhost + // export RANCHER_VERSION="latest/devel/2.12" + // export K3S_VERSION=v1.32.6+k3s1 + // export RANCHER_PASSWORD="rancherpassword123" hostname = os.Getenv("HOSTNAME_NAME") if hostname == "" { hostname = os.Getenv("localhost")