name: Test End-to-End Integration
on:
  workflow_dispatch:
  push:
    branches:
    - master
  pull_request:
    branches:
    - master

permissions:
  contents: read
  actions: read

env:
  KF_PROFILE: kubeflow-user-example-com

jobs:
  kubeflow-integration:
    name: Kubeflow Installation and Testing
    if: ${{ github.repository == 'kubeflow/manifests' }}
    runs-on:
      labels: oracle-vm-16cpu-64gb-x86-64
      # labels: ubuntu-latest-16-cores
    timeout-minutes: 45
    steps:
    - name: Checkout
      uses: actions/checkout@v4

    - name: Install KinD, Create KinD cluster and Install kustomize
      run: ./tests/install_KinD_create_KinD_cluster_install_kustomize.sh

    - name: Install kubectl
      run: ./tests/kubectl_install.sh

    - name: Create Kubeflow Namespace
      run: kustomize build common/kubeflow-namespace/base | kubectl apply -f -

    - name: Install Certificate Manager
      run: ./tests/cert_manager_install.sh

    - name: Install Istio CNI
      run: ./tests/istio-cni_install.sh

    - name: Install OAuth2 Proxy
      run: ./tests/oauth2-proxy_install.sh

    - name: Install Kubeflow Istio Resources
      run: kustomize build common/istio/kubeflow-istio-resources/base | kubectl apply -f -

    - name: Install Multi-Tenancy
      run: ./tests/multi_tenancy_install.sh

    - name: Install Dex
      run: ./tests/dex_install.sh

    - name: Install Central Dashboard
      run: ./tests/central_dashboard_install.sh

    - name: Install Knative
      run: ./tests/knative-cni_install.sh

    - name: Install KServe
      run: ./tests/kserve_install.sh

    #- name: Install Pipelines
    #  run: ./tests/pipelines_install.sh

    - name: Install Pipelines with SeaweedFS
      run: ./tests/pipelines_swfs_install.sh

    - name: Create KF Profile
      run: ./tests/kubeflow_profile_install.sh

    - name: Install Jupyter Web Application
      run: kustomize build applications/jupyter/jupyter-web-app/upstream/overlays/istio/ | kubectl apply -f -

    - name: Install Notebook Controller
      run: kustomize build applications/jupyter/notebook-controller/upstream/overlays/kubeflow/ | kubectl apply -f -

    - name: Install Admission Webhook
      run: kustomize build applications/admission-webhook/upstream/overlays/cert-manager | kubectl apply -f -

    - name: Install PodDefaults CRD
      run: kubectl get crd poddefaults.kubeflow.org || kubectl apply -f https://raw.githubusercontent.com/kubeflow/kubeflow/master/components/admission-webhook/manifests/base/crd.yaml

    - name: Install Volumes Web Application
      run: ./tests/volumes_web_application_install.sh

    - name: Install Katib
      run: ./tests/katib_install.sh

    - name: Install Training Operator
      run: ./tests/training_operator_install.sh

    - name: Install Model Registry
      run: |
        kustomize build applications/model-registry/upstream/overlays/db | kubectl apply -n kubeflow -f -
        kustomize build applications/model-registry/upstream/options/istio | kubectl apply -n kubeflow -f -

    - name: Install Model Registry UI
      run: |
        kustomize build applications/model-registry/upstream/options/ui/overlays/istio | kubectl apply -n kubeflow -f -

    - name: Install Spark
      run: chmod u+x tests/*.sh && ./tests/spark_install.sh

    - name: Wait for All Pods to be Ready
      run: kubectl wait --for=condition=Ready pods --all --all-namespaces --timeout 90s --field-selector=status.phase!=Succeeded

    - name: Port-forward the istio-ingress gateway
      run: ./tests/port_forward_gateway.sh

    #  name: Setup OAuth2 and Dex Credentials
    #  run: chmod +x tests/oauth2_dex_credentials.sh && ./tests/oauth2_dex_credentials.sh

    - name: Test Dex Login
      run: |
        pip3 install -q requests
        python3 tests/dex_login_test.py
        echo "Dex login test completed successfully."

    - name: Verify Pipeline Integration
      run: |
        KF_PROFILE=kubeflow-user-example-com
        if ! kubectl get secret mlpipeline-minio-artifact -n $KF_PROFILE > /dev/null 2>&1; then
          echo "Error: Secret mlpipeline-minio-artifact not found in namespace $KF_PROFILE"
          exit 1
        fi
        kubectl get secret mlpipeline-minio-artifact -n "$KF_PROFILE" -o json | jq -r '.data | keys[] as $k | "\($k): \(. | .[$k] | @base64d)"' | tr '\n' ' '

    - name: V1 Pipeline Test
      run: |
        pip3 install "kfp>=1.8.22,<2.0.0"
        TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)"
        python3 tests/pipeline_v1_test.py "${TOKEN}" "${KF_PROFILE}"

    - name: V2 Pipeline Test
      run: |
        pip3 install -U "kfp>=2.13.0"
        TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)"
        python3 tests/pipeline_v2_test.py run_pipeline "${TOKEN}" "${KF_PROFILE}"

    - name: Test Pipeline Access with Unauthorized Token
      run: |
        kubectl create namespace test-unauthorized
        kubectl create serviceaccount test-unauthorized -n test-unauthorized
        UNAUTHORIZED_TOKEN=$(kubectl -n test-unauthorized create token test-unauthorized)
        python3 tests/pipeline_v2_test.py test_unauthorized_access "$UNAUTHORIZED_TOKEN" "${KF_PROFILE}"

    - name: Test SeaweedFS Namespace Isolation
      run: ./tests/swfs_namespace_isolation_test.sh

    - name: Test Volumes Web Application API
      run: ./tests/volumes_web_application_test.sh "${KF_PROFILE}"

    - name: Apply PodDefault for Pipeline Access Token
      run: sed "s/kubeflow-user-example-com/$KF_PROFILE/g" tests/poddefaults.access-ml-pipeline.kubeflow-user-example-com.yaml | kubectl apply -f -

    - name: Create Test Notebook
      run: |
        sed "s/kubeflow-user-example-com/$KF_PROFILE/g" tests/notebook.test.kubeflow-user-example.com.yaml | kubectl apply -f -
        kubectl wait --for=condition=Ready pod -l app=test -n $KF_PROFILE --timeout=300s

    - name: Copy and execute the pipeline run script in KF Notebook
      run: |
        cp tests/pipeline_run_and_wait_kubeflow.py /tmp/run_pipeline_temp.py
        sed -i "s/experiment_namespace = \"kubeflow-user-example-com\"/experiment_namespace = \"$KF_PROFILE\"/g" /tmp/run_pipeline_temp.py
        sed -i 's/except Exception:/except Exception as e:/g' /tmp/run_pipeline_temp.py
        sed -i 's/logger.info("Experiment not found, trying to create experiment.")/logger.info("Experiment not found, trying to create experiment. Error: " + str(e))/g' /tmp/run_pipeline_temp.py

        kubectl -n $KF_PROFILE cp /tmp/run_pipeline_temp.py test-0:/home/jovyan/pipeline_run_and_wait_kubeflow.py

        kubectl -n $KF_PROFILE exec test-0 -- python /home/jovyan/pipeline_run_and_wait_kubeflow.py

    - name: Run Katib Test
      run: |
        kubectl apply -f tests/katib_test.yaml
        kubectl wait --for=condition=Running experiments.kubeflow.org -n $KF_PROFILE --all --timeout=300s
        echo "Waiting for all Trials to be Completed..."
        kubectl wait --for=condition=Created trials.kubeflow.org -n $KF_PROFILE --all --timeout=60s
        kubectl get trials.kubeflow.org -n $KF_PROFILE
        kubectl wait --for=condition=Succeeded trials.kubeflow.org -n $KF_PROFILE --all --timeout 600s
        kubectl get trials.kubeflow.org -n $KF_PROFILE

    - name: Run Training Operator Test
      run: ./tests/training_operator_test.sh "${KF_PROFILE}"

    - name: Run KServe Test
      run: |
        ./tests/kserve_test.sh ${KF_PROFILE}

    - name: Run Spark Test
      run: chmod u+x tests/*.sh && ./tests/spark_test.sh "${KF_PROFILE}"

    - name: Test Model Registry Deployment
      run: |
        kubectl wait --for=condition=available -n kubeflow deployment/model-registry-db --timeout=600s
        kubectl wait --for=condition=available -n kubeflow deployment/model-registry-deployment --timeout=600s

    - name: Test Model Registry UI Deployment
      run: kubectl wait --for=condition=available -n kubeflow deployment/model-registry-ui --timeout=600s

    - name: Test Model Registry API
      run: |
        export KF_TOKEN="$(kubectl -n $KF_PROFILE create token default-editor)"

        nohup kubectl port-forward svc/model-registry-service -n kubeflow 8082:8080 &
        sleep 5
        curl -s -X 'GET' \
          'http://localhost:8082/api/model_registry/v1alpha3/registered_models?pageSize=100&orderBy=ID&sortOrder=DESC' \
          -H 'accept: application/json'

        curl -s --fail \
          "localhost:8080/model-registry/api/v1/model_registry?namespace=${KF_PROFILE}" \
          -H "Authorization: Bearer ${KF_TOKEN}"

    - name: Test Model Registry API with Unauthorized Token
      run: |
        UNAUTHORIZED_TOKEN=$(kubectl -n test-unauthorized create token test-unauthorized || kubectl -n test-unauthorized create token default)

        STATUS_CODE=$(curl -s \
          --output /dev/stderr --write-out "%{http_code}" \
          "localhost:8080/model-registry/api/v1/model_registry?namespace=${KF_PROFILE}" \
          -H "Authorization: Bearer ${UNAUTHORIZED_TOKEN}")

        if test $STATUS_CODE -ne 403; then
          echo "Error: Unauthorized access was not correctly rejected. Got status code: ${STATUS_CODE}"
          exit 1
        fi

    - name: Apply Pod Security Standards Baseline
      run: ./tests/PSS_baseline_enable.sh

    - name: Remove Pod Security Labels
      run: |
        NAMESPACES=("istio-system" "auth" "cert-manager" "oauth2-proxy" "kubeflow" "knative-serving")
        for namespace in "${NAMESPACES[@]}"; do
            kubectl label namespace $namespace pod-security.kubernetes.io/enforce-
        done

    - name: Apply Pod Security Standards Restricted
      run: ./tests/PSS_restricted_enable.sh

    - name: Verify Components
      run: kubectl get pods --all-namespaces | grep -E '(Error|CrashLoopBackOff)' && exit 1 || true

    - name: Install Metrics Server
      run: ./tests/metrics-server_install.sh

    - name: Check Pod Resource Usage
      run: |
        echo "==== Resource Usage Table ===="
        pip3 install -q PyYAML
        python3 tests/metrics-server_resource_table.py

    - name: Collect Logs on Failure
      if: failure()
      run: |
        mkdir -p logs
        kubectl get all --all-namespaces > logs/resources.txt
        kubectl get events --all-namespaces --sort-by=.metadata.creationTimestamp > logs/events.txt
        for namespace in kubeflow istio-system cert-manager auth kubeflow-user-example-com; do
            kubectl describe pods -n $namespace > logs/$namespace-pods.txt
            for pod in $(kubectl get pods -n $namespace -o jsonpath='{.items[*].metadata.name}'); do
                kubectl logs -n $namespace $pod --tail=100 > logs/$namespace-$pod.txt 2>&1 || true
            done
        done

    - name: Upload Diagnostic Logs
      if: always()
      uses: actions/upload-artifact@v4
      with:
        name: kubeflow-test-logs
        path: logs/
