From 8aeeb3d0b98579d262ba48321a718f8c436ee4db Mon Sep 17 00:00:00 2001
From: Adriano Cunha <35786489+adrcunha@users.noreply.github.com>
Date: Mon, 1 Oct 2018 12:17:23 -0700
Subject: [PATCH] Update knative/test-infra (#13)

Major changes:
* fixes and updates to E2E test framework
* better parsing of E2E test results
---
 Gopkg.lock                                    |   4 +-
 .../knative/test-infra/scripts/e2e-tests.sh   |  14 +-
 .../knative/test-infra/scripts/library.sh     | 144 ++++++++++++++----
 .../test-infra/scripts/presubmit-tests.sh     |   4 +
 4 files changed, 129 insertions(+), 37 deletions(-)

diff --git a/Gopkg.lock b/Gopkg.lock
index d21ff825..8124a5bf 100644
--- a/Gopkg.lock
+++ b/Gopkg.lock
@@ -133,11 +133,11 @@
 
 [[projects]]
   branch = "master"
-  digest = "1:893ebf54301776b1a306f8c10eafea7c3e9d51b946e59a5b05bfbb1fca60c4ce"
+  digest = "1:00f434f614520a4b48ef3f6d3ed5c7a1a4f89be1bed98c250711413f6da2eb51"
   name = "github.com/knative/test-infra"
   packages = ["."]
   pruneopts = "T"
-  revision = "84856ef3511c4eb653aaba3ca6f55f656551ad58"
+  revision = "c21d3a832727d5d4334e734c0878eac3a5a07ae7"
 
 [[projects]]
   digest = "1:2f42fa12d6911c7b7659738758631bec870b7e9b4c6be5444f963cdcfccc191f"
diff --git a/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh b/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh
index 1262f982..7c921048 100755
--- a/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh
+++ b/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh
@@ -103,7 +103,7 @@ function download_k8s() {
     local gke_versions=(`echo -n ${versions//;/ /}`)
     # Get first (latest) version, excluding the "-gke.#" suffix
     version="${gke_versions[0]%-*}"
-    echo "Latest GKE is ${version}, from [${versions//;/, /}]"
+    echo "Latest GKE is ${version}, from [${versions//;/, }]"
   elif [[ "${version}" == "default" ]]; then
     echo "ERROR: `default` GKE version is not supported yet"
     return 1
@@ -152,10 +152,14 @@ function dump_cluster_state() {
 
 # Create a test cluster with kubetest and call the current script again.
 function create_test_cluster() {
+  # Fail fast during setup.
+  set -o errexit
+  set -o pipefail
+
   header "Creating test cluster"
   # Smallest cluster required to run the end-to-end-tests
   local CLUSTER_CREATION_ARGS=(
-    --gke-create-args="--enable-autoscaling --min-nodes=1 --max-nodes=${E2E_CLUSTER_NODES} --scopes=cloud-platform"
+    --gke-create-args="--enable-autoscaling --min-nodes=1 --max-nodes=${E2E_CLUSTER_NODES} --scopes=cloud-platform --enable-basic-auth --no-issue-client-certificate"
     --gke-shape={\"default\":{\"Nodes\":${E2E_CLUSTER_NODES}\,\"MachineType\":\"${E2E_CLUSTER_MACHINE}\"}}
     --provider=gke
     --deployment=gke
@@ -171,6 +175,7 @@ function create_test_cluster() {
   fi
   # SSH keys are not used, but kubetest checks for their existence.
   # Touch them so if they don't exist, empty files are create to satisfy the check.
+  mkdir -p $HOME/.ssh
   touch $HOME/.ssh/google_compute_engine.pub
   touch $HOME/.ssh/google_compute_engine
   # Clear user and cluster variables, so they'll be set to the test cluster.
@@ -187,6 +192,9 @@ function create_test_cluster() {
   (( EMIT_METRICS )) && test_cmd_args+=" --emit-metrics"
   echo "Test script is ${E2E_SCRIPT}"
   download_k8s || return 1
+  # Don't fail test for kubetest, as it might incorrectly report test failure
+  # if teardown fails (for details, see success() below)
+  set +o errexit
   kubetest "${CLUSTER_CREATION_ARGS[@]}" \
     --up \
     --down \
@@ -195,6 +203,8 @@ function create_test_cluster() {
     --test-cmd "${E2E_SCRIPT}" \
     --test-cmd-args "${test_cmd_args}"
   echo "Test subprocess exited with code $?"
+  # Ignore any errors below, this is a best-effort cleanup and shouldn't affect the test result.
+  set +o errexit
   # Delete target pools and health checks that might have leaked.
   # See https://github.com/knative/serving/issues/959 for details.
   # TODO(adrcunha): Remove once the leak issue is resolved.
diff --git a/vendor/github.com/knative/test-infra/scripts/library.sh b/vendor/github.com/knative/test-infra/scripts/library.sh
index d9a2e0bd..a330f4db 100755
--- a/vendor/github.com/knative/test-infra/scripts/library.sh
+++ b/vendor/github.com/knative/test-infra/scripts/library.sh
@@ -28,8 +28,16 @@ readonly KNATIVE_SERVING_RELEASE=https://storage.googleapis.com/knative-releases
 readonly KNATIVE_BUILD_RELEASE=https://storage.googleapis.com/knative-releases/build/latest/release.yaml
 readonly KNATIVE_EVENTING_RELEASE=https://storage.googleapis.com/knative-releases/eventing/latest/release.yaml
 
+# Conveniently set GOPATH if unset
+if [[ -z "${GOPATH:-}" ]]; then
+  export GOPATH="$(go env GOPATH)"
+  if [[ -z "${GOPATH}" ]]; then
+    echo "WARNING: GOPATH not set and go binary unable to provide it"
+  fi
+fi
+
 # Useful environment variables
-[[ -n "${PROW_JOB_ID}" ]] && IS_PROW=1 || IS_PROW=0
+[[ -n "${PROW_JOB_ID:-}" ]] && IS_PROW=1 || IS_PROW=0
 readonly IS_PROW
 readonly REPO_ROOT_DIR="$(git rev-parse --show-toplevel)"
 
@@ -146,6 +154,24 @@ function wait_until_service_has_external_ip() {
   return 1
 }
 
+# Waits for the endpoint to be routable.
+# Parameters: $1 - External ingress IP address.
+#             $2 - cluster hostname.
+function wait_until_routable() {
+  echo -n "Waiting until cluster $2 at $1 has a routable endpoint"
+  for i in {1..150}; do  # timeout after 5 minutes
+    local val=$(curl -H "Host: $2" "http://$1" 2>/dev/null)
+    if [[ -n "$val" ]]; then
+      echo "\nEndpoint is now routable"
+      return 0
+    fi
+    echo -n "."
+    sleep 2
+  done
+  echo -e "\n\nERROR: Timed out waiting for endpoint to be routable"
+  return 1
+}
+
 # Returns the name of the pod of the given app.
 # Parameters: $1 - app name.
 #             $2 - namespace (optional).
@@ -164,8 +190,21 @@ function acquire_cluster_admin_role() {
   # might not have the necessary permission.
   local password=$(gcloud --format="value(masterAuth.password)" \
       container clusters describe $2 --zone=$3)
-  kubectl config set-credentials cluster-admin \
-      --username=admin --password=${password}
+  if [[ -n "${password}" ]]; then
+    # Cluster created with basic authentication
+    kubectl config set-credentials cluster-admin \
+        --username=admin --password=${password}
+  else
+    local cert=$(mktemp)
+    local key=$(mktemp)
+    echo "Certificate in ${cert}, key in ${key}"
+    gcloud --format="value(masterAuth.clientCertificate)" \
+      container clusters describe $2 --zone=$3 | base64 -d > ${cert}
+    gcloud --format="value(masterAuth.clientKey)" \
+      container clusters describe $2 --zone=$3 | base64 -d > ${key}
+    kubectl config set-credentials cluster-admin \
+      --client-certificate=${cert} --client-key=${key}
+  fi
   kubectl config set-context $(kubectl config current-context) \
       --user=cluster-admin
   kubectl create clusterrolebinding cluster-admin-binding \
@@ -179,62 +218,81 @@ function acquire_cluster_admin_role() {
 # Runs a go test and generate a junit summary through bazel.
 # Parameters: $1... - parameters to go test
 function report_go_test() {
-  # Just run regular go tests if not on Prow.
-  if (( ! IS_PROW )); then
-    go test $@
-    return
-  fi
-  local report=$(mktemp)
-  local summary=$(mktemp)
-  local failed=0
   # Run tests in verbose mode to capture details.
   # go doesn't like repeating -v, so remove if passed.
   local args=("${@/-v}")
-  go test -race -v ${args[@]} > ${report} || failed=$?
+  local go_test="go test -race -v ${args[@]}"
+  # Just run regular go tests if not on Prow.
+  if (( ! IS_PROW )); then
+    ${go_test}
+    return
+  fi
+  echo "Running tests with '${go_test}'"
+  local report=$(mktemp)
+  local failed=0
+  local test_count=0
+  ${go_test} > ${report} || failed=$?
+  echo "Finished run, return code is ${failed}"
   # Tests didn't run.
   [[ ! -s ${report} ]] && return 1
   # Create WORKSPACE file, required to use bazel
   touch WORKSPACE
   local targets=""
   # Parse the report and generate fake tests for each passing/failing test.
+  echo "Start parsing results, summary:"
   while read line ; do
     local fields=(`echo -n ${line}`)
     local field0="${fields[0]}"
     local field1="${fields[1]}"
-    local name=${fields[2]}
+    local name="${fields[2]}"
     # Ignore subtests (those containing slashes)
     if [[ -n "${name##*/*}" ]]; then
-      if [[ ${field1} == PASS: || ${field1} == FAIL: ]]; then
-        # Populate BUILD.bazel
+      local error=""
+      # Deal with a fatal log entry, which has a different format:
+      # fatal   TestFoo   foo_test.go:275 Expected "foo" but got "bar"
+      if [[ "${field0}" == "fatal" ]]; then
+        name="${fields[1]}"
+        field1="FAIL:"
+        error="${fields[@]:3}"
+      fi
+      # Handle regular go test pass/fail entry for a test.
+      if [[ "${field1}" == "PASS:" || "${field1}" == "FAIL:" ]]; then
+        echo "- ${name} :${field1}"
+        test_count=$(( test_count + 1 ))
         local src="${name}.sh"
         echo "exit 0" > ${src}
-        if [[ ${field1} == "FAIL:" ]]; then
-          read error
+        if [[ "${field1}" == "FAIL:" ]]; then
+          [[ -z "${error}" ]] && read error
           echo "cat <<ERROR-EOF" > ${src}
           echo "${error}" >> ${src}
           echo "ERROR-EOF" >> ${src}
           echo "exit 1" >> ${src}
         fi
         chmod +x ${src}
+        # Populate BUILD.bazel
         echo "sh_test(name=\"${name}\", srcs=[\"${src}\"])" >> BUILD.bazel
-      elif [[ ${field0} == FAIL || ${field0} == ok ]]; then
-        # Update the summary with the result for the package
-        echo "${line}" >> ${summary}
+      elif [[ "${field0}" == "FAIL" || "${field0}" == "ok" ]] && [[ -n "${field1}" ]]; then
+        echo "- ${field0} ${field1}"
         # Create the package structure, move tests and BUILD file
         local package=${field1/github.com\//}
-        mkdir -p ${package}
-        targets="${targets} //${package}/..."
-        mv *.sh BUILD.bazel ${package}
+        local bazel_files="$(ls -1 *.sh BUILD.bazel 2> /dev/null)"
+        if [[ -n "${bazel_files}" ]]; then
+          mkdir -p ${package}
+          targets="${targets} //${package}/..."
+          mv ${bazel_files} ${package}
+        else
+          echo "*** INTERNAL ERROR: missing tests for ${package}, got [${bazel_files/$'\n'/, }]"
+        fi
       fi
     fi
   done < ${report}
+  echo "Done parsing ${test_count} tests"
   # If any test failed, show the detailed report.
-  # Otherwise, just show the summary.
+  # Otherwise, we already shown the summary.
   # Exception: when emitting metrics, dump the full report.
   if (( failed )) || [[ "$@" == *" -emitmetrics"* ]]; then
+    echo "At least one test failed, full log:"
     cat ${report}
-  else
-    cat ${summary}
   fi
   # Always generate the junit summary.
   bazel test ${targets} > /dev/null 2>&1
@@ -294,16 +352,36 @@ function check_licenses() {
   run_dep_collector -check $@
 }
 
-# Check links in all .md files in the repo.
-function check_links_in_markdown() {
-  local checker="markdown-link-check"
+# Run the given linter on the given files, checking it exists first.
+# Parameters: $1 - tool
+#             $2 - tool purpose (for error message if tool not installed)
+#             $3 - tool parameters (quote if multiple parameters used)
+#             $4..$n - files to run linter on
+function run_lint_tool() {
+  local checker=$1
+  local params=$3
   if ! hash ${checker} 2>/dev/null; then
-    warning "${checker} not installed, not checking links in .md files"
-    return 0
+    warning "${checker} not installed, not $2"
+    return 127
   fi
+  shift 3
   local failed=0
-  for md_file in $(find ${REPO_ROOT_DIR} -name \*.md); do
-    ${checker} -q ${md_file} || failed=1
+  for file in $@; do
+    ${checker} ${params} ${file} || failed=1
   done
   return ${failed}
 }
+
+# Check links in the given markdown files.
+# Parameters: $1...$n - files to inspect
+function check_links_in_markdown() {
+  # https://github.com/tcort/markdown-link-check
+  run_lint_tool markdown-link-check "checking links in markdown files" -q $@
+}
+
+# Check format of the given markdown files.
+# Parameters: $1...$n - files to inspect
+function lint_markdown() {
+  # https://github.com/markdownlint/markdownlint
+  run_lint_tool mdl "linting markdown files" "-r ~MD013" $@
+}
diff --git a/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh b/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh
index 37c00f43..b6116817 100755
--- a/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh
+++ b/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh
@@ -46,6 +46,10 @@ function exit_if_presubmit_not_required() {
   if [[ -n "${PULL_PULL_SHA}" ]]; then
     # On a presubmit job
     local changes="$(/workspace/githubhelper -list-changed-files)"
+    if [[ -z "${changes}" ]]; then
+      header "NO CHANGED FILES REPORTED, ASSUMING IT'S AN ERROR AND RUNNING TESTS ANYWAY"
+      return
+    fi
     local no_presubmit_pattern="${NO_PRESUBMIT_FILES[*]}"
     local no_presubmit_pattern="\(${no_presubmit_pattern// /\\|}\)$"
     echo -e "Changed files in commit ${PULL_PULL_SHA}:\n${changes}"