Write xgboost_synthetic test output to html (#735)

* use nbconvert to write output as html

* write local file

* change dir

* write to gcs

* add kubeflow/testing

* update to env and checkout_repos

* format gcs path

* fix syntax

* fix

* add option notebook_artifacts_dir

* download to artifacts

* fix

* shorten name

* fix

* fix

* mkdirs

* fix

* fix

* log error

* use notebook_artifacts_path

py/kubeflow/examples/create_e2e_workflow.py

@@ -251,6 +251,7 @@ class Builder:
                # Test timeout in seconds.
                "--timeout=1800",
                "--junitxml=" + self.artifacts_dir + "/junit_xgboost-synthetic-test.xml",
+               "--notebook_artifacts_dir=" + self.artifacts_dir + "/xgboost-synthetic-test-notebooks",
                ]
 
     dependencies = []

xgboost_synthetic/testing/conftest.py

@@ -16,6 +16,9 @@ def pytest_addoption(parser):
   parser.addoption(
     "--repos", help="The repos to checkout; leave blank to use defaults",
     type=str, default="")
+  parser.addoption(
+    "--notebook_artifacts_dir", help="Directory to store notebook artifacts",
+    type=str, default="")
 
 @pytest.fixture
 def name(request):
@@ -32,3 +35,7 @@ def image(request):
 @pytest.fixture
 def repos(request):
   return request.config.getoption("--repos")
+
+@pytest.fixture
+def notebook_artifacts_dir(request):
+  return request.config.getoption("--notebook_artifacts_dir")

xgboost_synthetic/testing/execute_notebook.py

@@ -3,11 +3,14 @@ import logging
 import os
 import subprocess
 
+from kubeflow.testing import util as kf_util
 
 logger = logging.getLogger(__name__)
 
 def prepare_env():
   subprocess.check_call(["pip3", "install", "-U", "papermill"])
+  subprocess.check_call(["pip3", "install", "-U", "nbconvert"])
+  subprocess.check_call(["pip3", "install", "-U", "nbformat"])
   subprocess.check_call(["pip3", "install", "-r", "../requirements.txt"])
 
 
@@ -22,7 +25,18 @@ def execute_notebook(notebook_path, parameters=None):
 
 def run_notebook_test(notebook_path, expected_messages, parameters=None):
   output_path = execute_notebook(notebook_path, parameters=parameters)
+
+  import nbformat #pylint: disable=import-error
+  import nbconvert #pylint: disable=import-error
+
   actual_output = open(output_path, 'r').read()
+
+  nb = nbformat.reads(actual_output, as_version=4)
+  html_exporter = nbconvert.HTMLExporter()
+  (html_output, _) = html_exporter.from_notebook_node(nb)
+  gcs_path = os.getenv("OUTPUT_GCS")
+  kf_util.upload_to_gcs(html_output, gcs_path)
+
   for expected_message in expected_messages:
     if not expected_message in actual_output:
       logger.error(actual_output)

xgboost_synthetic/testing/xgboost_test.py

@@ -6,6 +6,7 @@ import yaml
 
 import pytest
 
+from google.cloud import storage
 from kubernetes import client as k8s_client
 from kubeflow.testing import argo_build_util
 from kubeflow.testing import util
@@ -17,7 +18,7 @@ from kubeflow.testing import util
 # and we want signal in postsubmits and periodics
 @pytest.mark.xfail(os.getenv("JOB_TYPE") == "presubmit", reason="Flaky")
 def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disable=too-many-branches,too-many-statements
-                           repos, image):
+                           repos, image, notebook_artifacts_dir):
   '''Generate Job and summit.'''
   util.set_pytest_junit(record_xml_attribute, "test_xgboost_synthetic")
 
@@ -35,6 +36,7 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa
   if not repos:
     repos = argo_build_util.get_repo_from_prow_env()
 
+  repos += ",kubeflow/testing@HEAD"
   logging.info("Repos set to %s", repos)
   job["spec"]["template"]["spec"]["initContainers"][0]["command"] = [
     "/usr/local/bin/checkout_repos.sh",
@@ -42,6 +44,20 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa
     "--src_dir=/src",
     "--depth=all",
   ]
+
+  nb_bucket = "kubeflow-ci-deployment"
+  nb_path = os.path.join(
+    "xgboost_synthetic_testing",
+    os.getenv("JOB_TYPE"),
+    os.getenv("HOSTNAME"),
+    "notebook.html"
+  )
+  output_gcs = util.to_gcs_uri(nb_bucket, nb_path)
+  logging.info("Tested notebook will be outputed to: %s", output_gcs)
+  job["spec"]["template"]["spec"]["containers"][0]["env"] = [
+    {"name": "PYTHONPATH", "value": "/src/kubeflow/testing/py"},
+    {"name": "OUTPUT_GCS", "value": output_gcs},
+  ]
   job["spec"]["template"]["spec"]["containers"][0]["image"] = image
   util.load_kube_config(persist_config=False)
 
@@ -75,6 +91,15 @@ def test_xgboost_synthetic(record_xml_attribute, name, namespace, # pylint: disa
 
   last_condition = final_job.status.conditions[-1]
 
+  # Download notebook html to artifacts
+  notebook_artifacts_path = os.path.join(notebook_artifacts_dir, "notebook.html")
+  logging.info("Writing notebook artifact to: %s", notebook_artifacts_path)
+  os.makedirs(notebook_artifacts_dir, exist_ok=True)
+  storage_client = storage.Client()
+  bucket = storage_client.get_bucket(nb_bucket)
+  blob = bucket.get_blob(nb_path)
+  blob.download_to_filename(notebook_artifacts_path)
+
   if last_condition.type not in ["Complete"]:
     logging.error("Job didn't complete successfully")
     raise RuntimeError("Job {0}.{1} failed".format(namespace, name))