From 2c5e96dfd062b937d0b8b0a3cae0472a09b292e9 Mon Sep 17 00:00:00 2001 From: Yehudit Kerido Date: Mon, 15 Sep 2025 09:11:39 +0300 Subject: [PATCH] feat: Migrate JWA test-related workflows from kubeflow/kubeflow to notebooks-v1 branch #587 Signed-off-by: Yehudit Kerido --- .github/workflows/jwa_backend_unittests.yaml | 38 +++++++ .github/workflows/jwa_frontend_tests.yaml | 98 +++++++++++++++++++ .github/workflows/jwa_integration_test.yaml | 68 +++++++++++++ .github/workflows/jwa_multi_arch_test.yaml | 75 ++++++++++++++ components/crud-web-apps/jupyter/Makefile | 6 +- .../jupyter/manifests/base/deployment.yaml | 2 +- .../jupyter/manifests/base/kustomization.yaml | 4 +- .../testing/gh-actions/install_istio.sh | 15 +++ components/testing/gh-actions/install_kind.sh | 17 ++++ .../testing/gh-actions/install_kustomize.sh | 12 +++ components/testing/gh-actions/kind-1-25.yaml | 24 +++++ releasing/version/VERSION | 1 + 12 files changed, 356 insertions(+), 4 deletions(-) create mode 100644 .github/workflows/jwa_backend_unittests.yaml create mode 100644 .github/workflows/jwa_frontend_tests.yaml create mode 100644 .github/workflows/jwa_integration_test.yaml create mode 100644 .github/workflows/jwa_multi_arch_test.yaml create mode 100755 components/testing/gh-actions/install_istio.sh create mode 100755 components/testing/gh-actions/install_kind.sh create mode 100755 components/testing/gh-actions/install_kustomize.sh create mode 100644 components/testing/gh-actions/kind-1-25.yaml create mode 100644 releasing/version/VERSION diff --git a/.github/workflows/jwa_backend_unittests.yaml b/.github/workflows/jwa_backend_unittests.yaml new file mode 100644 index 00000000..974b01b5 --- /dev/null +++ b/.github/workflows/jwa_backend_unittests.yaml @@ -0,0 +1,38 @@ +name: JWA Backend Tests +on: + pull_request: + paths: + - components/crud-web-apps/jupyter/backend/** + - releasing/version/VERSION + branches: + - master + - notebooks-v1 + - 'v*-branch' + +jobs: + run-backend-unittests: + name: Unittests + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Install KinD + run: ./components/testing/gh-actions/install_kind.sh + + - name: Create KinD Cluster + run: kind create cluster --config components/testing/gh-actions/kind-1-25.yaml + + - uses: actions/setup-python@v5 + with: + python-version: "3.7" + + - name: Setup Python environment + run: | + cd components/crud-web-apps/jupyter/backend + make install-deps + + - name: Run unittests + run: | + cd components/crud-web-apps/jupyter/backend + make unittest \ No newline at end of file diff --git a/.github/workflows/jwa_frontend_tests.yaml b/.github/workflows/jwa_frontend_tests.yaml new file mode 100644 index 00000000..f3cb597d --- /dev/null +++ b/.github/workflows/jwa_frontend_tests.yaml @@ -0,0 +1,98 @@ +name: JWA Frontend Tests +on: + pull_request: + paths: + - components/crud-web-apps/jupyter/frontend/** + - releasing/version/VERSION + branches: + - master + - notebooks-v1 + - 'v*-branch' + +jobs: + frontend-format-linting-check: + name: Check code format and lint + runs-on: ubuntu-22.04 + + steps: + - name: Check out code + uses: actions/checkout@v4 + + - name: Setup Node + uses: actions/setup-node@v4 + with: + node-version: 16 + + - name: Check frontend formatting + run: | + cd components/crud-web-apps/jupyter/frontend + npm i + npm run format:check + + - name: Check frontend code linting + run: | + cd components/crud-web-apps/jupyter/frontend + npm i + npm run lint-check + + frontend-unit-tests: + runs-on: ubuntu-22.04 + name: Unit tests + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup node version to 16 + uses: actions/setup-node@v4 + with: + node-version: 16 + + - name: Install Kubeflow common library dependecies + run: | + cd components/crud-web-apps/common/frontend/kubeflow-common-lib + npm i + npm run build + npm link ./dist/kubeflow + + - name: Install JWA dependencies + run: | + cd components/crud-web-apps/jupyter/frontend + npm i + npm link kubeflow + + - name: Run unit tests + run: | + cd components/crud-web-apps/jupyter/frontend + npm run test:prod + + run-ui-tests: + name: UI tests with Playwright + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup node version to 16 + uses: actions/setup-node@v4 + with: + node-version: 16 + + - name: Install Kubeflow common library dependecies + run: | + cd components/crud-web-apps/common/frontend/kubeflow-common-lib + npm i + npm run build + npm link ./dist/kubeflow + + - name: Install JWA dependencies + run: | + cd components/crud-web-apps/jupyter/frontend + npm i + npm link kubeflow + + - name: Serve UI & run Playwright tests in Chrome and Firefox + run: | + cd components/crud-web-apps/jupyter/frontend + npm run serve > serve.log 2>&1 & npx wait-on http://localhost:4200 + npx playwright install + npm run ui-test-ci-all \ No newline at end of file diff --git a/.github/workflows/jwa_integration_test.yaml b/.github/workflows/jwa_integration_test.yaml new file mode 100644 index 00000000..eb0181ca --- /dev/null +++ b/.github/workflows/jwa_integration_test.yaml @@ -0,0 +1,68 @@ +name: JWA Integration Test +on: + pull_request: + paths: + - components/crud-web-apps/jupyter/** + - components/crud-web-apps/common/** + - releasing/version/VERSION + branches: + - master + - notebooks-v1 + - 'v*-branch' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.number }} + cancel-in-progress: true + +env: + IMG: ghcr.io/kubeflow/notebooks/jupyter-web-app + TAG: integration-test + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build JWA Image + run: | + cd components/crud-web-apps/jupyter + make docker-build-multi-arch + + - name: Install KinD + run: ./components/testing/gh-actions/install_kind.sh + + - name: Create KinD Cluster + run: kind create cluster --config components/testing/gh-actions/kind-1-25.yaml + + - name: Load Image into KinD Cluster + run: | + kind load docker-image "${IMG}:${TAG}" + + - name: Install kustomize + run: ./components/testing/gh-actions/install_kustomize.sh + + - name: Install Istio + run: ./components/testing/gh-actions/install_istio.sh + + - name: Build & Apply manifests + run: | + cd components/crud-web-apps/jupyter/manifests + kubectl create ns kubeflow + + export CURRENT_IMAGE="${IMG}" + export PR_IMAGE="${IMG}:${TAG}" + + # escape "." in the image names, as it is a special characters in sed + export CURRENT_IMAGE=$(echo "$CURRENT_IMAGE" | sed 's|\.|\\.|g') + export PR_IMAGE=$(echo "$PR_IMAGE" | sed 's|\.|\\.|g') + + kustomize build overlays/istio \ + | sed "s|${CURRENT_IMAGE}:[a-zA-Z0-9_.-]*|${PR_IMAGE}|g" \ + | kubectl apply -f - + + kubectl wait pods -n kubeflow -l app=jupyter-web-app --for=condition=Ready --timeout=300s \ No newline at end of file diff --git a/.github/workflows/jwa_multi_arch_test.yaml b/.github/workflows/jwa_multi_arch_test.yaml new file mode 100644 index 00000000..f8dbbfa7 --- /dev/null +++ b/.github/workflows/jwa_multi_arch_test.yaml @@ -0,0 +1,75 @@ +name: JWA Multi-Arch Build Test +on: + pull_request: + paths: + - components/crud-web-apps/jupyter/** + - components/crud-web-apps/common/** + - releasing/version/VERSION + branches: + - master + - notebooks-v1 + - 'v*-branch' + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ github.event.number }} + cancel-in-progress: true + +env: + IMG: ghcr.io/kubeflow/notebooks/jupyter-web-app + +jobs: + build: + runs-on: ubuntu-22.04 + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Setup QEMU + uses: docker/setup-qemu-action@v3 + + - name: Setup Docker Buildx + uses: docker/setup-buildx-action@v3 + + - name: Build multi-arch Images (AMD64 & PPC64LE) + run: | + cd components/crud-web-apps/jupyter + ARCH=linux/amd64 make docker-build-multi-arch-verify + ARCH=linux/ppc64le make docker-build-multi-arch-verify + + - name: Build ARM64 Image (Optional) + timeout-minutes: 60 + continue-on-error: true + run: | + cd components/crud-web-apps/jupyter + ARCH=linux/arm64/v8 make docker-build-multi-arch-verify + echo "Completed linux/arm64/v8 build!" + + - name: Verify multi-arch builds + run: | + echo "Verifying multi-arch build results:" + echo "Listing all jupyter-web-app images:" + docker images --format "table {{.Repository}}\t{{.Tag}}\t{{.Size}}\t{{.CreatedAt}}" | grep jupyter-web-app || echo "No jupyter-web-app images found" + + echo "" + echo "Checking individual architectures:" + TAG=$(git describe --tags --always --dirty) + echo "Base TAG: $TAG" + + # Check each architecture + if docker images | grep -q "jupyter-web-app.*${TAG}-linux-amd64"; then + echo "✅ AMD64: SUCCESS" + else + echo "❌ AMD64: FAILED" + fi + + if docker images | grep -q "jupyter-web-app.*${TAG}-linux-ppc64le"; then + echo "✅ PPC64LE: SUCCESS" + else + echo "❌ PPC64LE: FAILED" + fi + + if docker images | grep -q "jupyter-web-app.*${TAG}-linux-arm64-v8"; then + echo "✅ ARM64: SUCCESS" + else + echo "ℹ️ ARM64: Not built (expected if timeout occurred)" + fi \ No newline at end of file diff --git a/components/crud-web-apps/jupyter/Makefile b/components/crud-web-apps/jupyter/Makefile index b8dbe53d..2a7a07bc 100644 --- a/components/crud-web-apps/jupyter/Makefile +++ b/components/crud-web-apps/jupyter/Makefile @@ -1,4 +1,4 @@ -IMG ?= ghcr.io/kubeflow/kubeflow/jupyter-web-app +IMG ?= ghcr.io/kubeflow/notebooks/jupyter-web-app TAG ?= $(shell git describe --tags --always --dirty) DOCKERFILE ?= jupyter/Dockerfile ARCH ?= linux/amd64 @@ -14,6 +14,10 @@ docker-push: docker-build-multi-arch: ## Build multi-arch docker images with docker buildx cd ../ && docker buildx build --load --platform ${ARCH} --tag ${IMG}:${TAG} -f ${DOCKERFILE} . +.PHONY: docker-build-multi-arch-verify +docker-build-multi-arch-verify: ## Build multi-arch docker images with arch-specific tags for verification + $(eval ARCH_SAFE := $(shell echo ${ARCH} | sed 's|/|-|g')) + cd ../ && docker buildx build --load --platform ${ARCH} --tag ${IMG}:${TAG}-${ARCH_SAFE} -f ${DOCKERFILE} . .PHONY: docker-build-push-multi-arch docker-build-push-multi-arch: ## Build multi-arch docker images with docker buildx and push to docker registry diff --git a/components/crud-web-apps/jupyter/manifests/base/deployment.yaml b/components/crud-web-apps/jupyter/manifests/base/deployment.yaml index f9d34b3a..9292c05b 100644 --- a/components/crud-web-apps/jupyter/manifests/base/deployment.yaml +++ b/components/crud-web-apps/jupyter/manifests/base/deployment.yaml @@ -8,7 +8,7 @@ spec: spec: containers: - name: jupyter-web-app - image: ghcr.io/kubeflow/kubeflow/jupyter-web-app + image: ghcr.io/kubeflow/notebooks/jupyter-web-app ports: - containerPort: 5000 volumeMounts: diff --git a/components/crud-web-apps/jupyter/manifests/base/kustomization.yaml b/components/crud-web-apps/jupyter/manifests/base/kustomization.yaml index fa87185a..6dae0630 100644 --- a/components/crud-web-apps/jupyter/manifests/base/kustomization.yaml +++ b/components/crud-web-apps/jupyter/manifests/base/kustomization.yaml @@ -21,8 +21,8 @@ commonLabels: app: jupyter-web-app kustomize.component: jupyter-web-app images: -- name: ghcr.io/kubeflow/kubeflow/jupyter-web-app - newName: ghcr.io/kubeflow/kubeflow/jupyter-web-app +- name: ghcr.io/kubeflow/notebooks/jupyter-web-app + newName: ghcr.io/kubeflow/notebooks/jupyter-web-app newTag: latest # We need the name to be unique without the suffix because the original name is what # gets used with patches diff --git a/components/testing/gh-actions/install_istio.sh b/components/testing/gh-actions/install_istio.sh new file mode 100755 index 00000000..a53943ac --- /dev/null +++ b/components/testing/gh-actions/install_istio.sh @@ -0,0 +1,15 @@ +#!/bin/bash + +set -euo pipefail + +ISTIO_VERSION="1.17.8" +ISTIO_URL="https://istio.io/downloadIstio" + +echo "Installing Istio ${ISTIO_VERSION} ..." +mkdir istio_tmp +pushd istio_tmp >/dev/null + curl -sL "$ISTIO_URL" | ISTIO_VERSION=${ISTIO_VERSION} sh - + cd istio-${ISTIO_VERSION} + export PATH=$PWD/bin:$PATH + istioctl install -y +popd \ No newline at end of file diff --git a/components/testing/gh-actions/install_kind.sh b/components/testing/gh-actions/install_kind.sh new file mode 100755 index 00000000..4dab0d6e --- /dev/null +++ b/components/testing/gh-actions/install_kind.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +set -euo pipefail + +KIND_VERSION="0.22.0" +KIND_URL="https://kind.sigs.k8s.io/dl/v${KIND_VERSION}/kind-linux-amd64" + +echo "Setting up kind environment..." +sudo swapoff -a +sudo rm -f /swapfile +sudo mkdir -p /tmp/etcd +sudo mount -t tmpfs tmpfs /tmp/etcd + +echo "Installing kind ${KIND_VERSION} ..." +curl -sL -o kind "$KIND_URL" +chmod +x ./kind +sudo mv kind /usr/local/bin \ No newline at end of file diff --git a/components/testing/gh-actions/install_kustomize.sh b/components/testing/gh-actions/install_kustomize.sh new file mode 100755 index 00000000..29f1cdf5 --- /dev/null +++ b/components/testing/gh-actions/install_kustomize.sh @@ -0,0 +1,12 @@ +#!/bin/bash + +set -euo pipefail + +KUSTOMIZE_VERSION="5.4.1" +KUSTOMIZE_URL="https://github.com/kubernetes-sigs/kustomize/releases/download/kustomize/v${KUSTOMIZE_VERSION}/kustomize_v${KUSTOMIZE_VERSION}_linux_amd64.tar.gz" + +echo "Installing kustomize ${KUSTOMIZE_VERSION} ..." +curl -sL -o kustomize.tar.gz "$KUSTOMIZE_URL" +tar -xzf kustomize.tar.gz +chmod +x kustomize +sudo mv kustomize /usr/local/bin \ No newline at end of file diff --git a/components/testing/gh-actions/kind-1-25.yaml b/components/testing/gh-actions/kind-1-25.yaml new file mode 100644 index 00000000..5ba34518 --- /dev/null +++ b/components/testing/gh-actions/kind-1-25.yaml @@ -0,0 +1,24 @@ +apiVersion: kind.x-k8s.io/v1alpha4 +kind: Cluster +# Configure registry for KinD. +containerdConfigPatches: +- |- + [plugins."io.containerd.grpc.v1.cri".registry.mirrors."$REGISTRY_NAME:$REGISTRY_PORT"] + endpoint = ["http://$REGISTRY_NAME:$REGISTRY_PORT"] +# This is needed in order to support projected volumes with service account tokens. +# See: https://kubernetes.slack.com/archives/CEKK1KTN2/p1600268272383600 +kubeadmConfigPatches: + - | + apiVersion: kubeadm.k8s.io/v1beta2 + kind: ClusterConfiguration + metadata: + name: config + apiServer: + extraArgs: + "service-account-issuer": "kubernetes.default.svc" + "service-account-signing-key-file": "/etc/kubernetes/pki/sa.key" +nodes: +- role: control-plane + image: kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1 +- role: worker + image: kindest/node:v1.25.3@sha256:f52781bc0d7a19fb6c405c2af83abfeb311f130707a0e219175677e366cc45d1 \ No newline at end of file diff --git a/releasing/version/VERSION b/releasing/version/VERSION new file mode 100644 index 00000000..b9bc2fdc --- /dev/null +++ b/releasing/version/VERSION @@ -0,0 +1 @@ +latest \ No newline at end of file