diff --git a/Gopkg.lock b/Gopkg.lock index 9eb3b1d3..144bbb82 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -147,14 +147,14 @@ [[projects]] branch = "master" - digest = "1:921c8888ac7bf7240bc5473d029201b3eb19f9592c7647c4c29f96b6c901b1bb" + digest = "1:f27efa11fb1aec3e502e2ed95d9a1211a93f8a791b7da0a06fdade0d98de5008" name = "github.com/knative/test-infra" packages = [ "scripts", "tools/dep-collector", ] pruneopts = "UT" - revision = "582b6fbee8ae00260baf916b0f9db2aa5f3483d9" + revision = "75f6ca1c4dc3b3ae5dc1a1a433753957a9340e83" [[projects]] digest = "1:2f42fa12d6911c7b7659738758631bec870b7e9b4c6be5444f963cdcfccc191f" diff --git a/vendor/github.com/knative/test-infra/scripts/README.md b/vendor/github.com/knative/test-infra/scripts/README.md index 07398fb7..f7841871 100644 --- a/vendor/github.com/knative/test-infra/scripts/README.md +++ b/vendor/github.com/knative/test-infra/scripts/README.md @@ -108,8 +108,10 @@ This is a helper script for Knative E2E test scripts. To use it: if the default values don't fit your needs: - `E2E_CLUSTER_REGION`: Cluster region, defaults to `us-central1`. + - `E2E_CLUSTER_BACKUP_REGIONS`: Space-separated list of regions to retry test cluster creation in case of stockout. Defaults to `us-west1 us-east1`. - `E2E_CLUSTER_ZONE`: Cluster zone (e.g., `a`), defaults to none (i.e. use a regional cluster). + - `E2E_CLUSTER_BACKUP_ZONES`: Space-separated list of zones to retry test cluster creation in case of stockout. If defined, `E2E_CLUSTER_BACKUP_REGIONS` will be ignored thus it defaults to none. - `E2E_CLUSTER_MACHINE`: Cluster node machine type, defaults to `n1-standard-4}`. - `E2E_MIN_CLUSTER_NODES`: Minimum number of nodes in the cluster when autoscaling, defaults to 1. @@ -153,8 +155,9 @@ This is a helper script for Knative E2E test scripts. To use it: 1. Write logic for the end-to-end tests. Run all go tests using `go_test_e2e()` (or `report_go_test()` if you need a more fine-grained control) and call `fail_test()` or `success()` if any of them failed. The environment variable - `KO_DOCKER_REPO` will be set according to the test cluster. You can also use - the following boolean (0 is false, 1 is true) environment variables for the logic: + `KO_DOCKER_REPO` and `E2E_PROJECT_ID` will be set according to the test cluster. + You can also use the following boolean (0 is false, 1 is true) environment + variables for the logic: - `EMIT_METRICS`: true if `--emit-metrics` was passed. @@ -169,6 +172,9 @@ This is a helper script for Knative E2E test scripts. To use it: will immediately start the tests against the cluster currently configured for `kubectl`. +1. By default Istio is installed on the cluster via Addon, using `--skip-istio` if + you choose not to have it preinstalled. + 1. You can force running the tests against a specific GKE cluster version by using the `--cluster-version` flag and passing a full version as the flag value. diff --git a/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh b/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh index c2328e7b..5c515594 100755 --- a/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh +++ b/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh @@ -1,6 +1,6 @@ #!/bin/bash -# Copyright 2018 The Knative Authors +# Copyright 2019 The Knative Authors # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -38,9 +38,15 @@ function build_resource_name() { # Test cluster parameters # Configurable parameters -readonly E2E_CLUSTER_REGION=${E2E_CLUSTER_REGION:-us-central1} +# export E2E_CLUSTER_REGION and E2E_CLUSTER_ZONE as they're used in the cluster setup subprocess +export E2E_CLUSTER_REGION=${E2E_CLUSTER_REGION:-us-central1} # By default we use regional clusters. -readonly E2E_CLUSTER_ZONE=${E2E_CLUSTER_ZONE:-} +export E2E_CLUSTER_ZONE=${E2E_CLUSTER_ZONE:-} + +# Default backup regions in case of stockouts; by default we don't fall back to a different zone in the same region +readonly E2E_CLUSTER_BACKUP_REGIONS=${E2E_CLUSTER_BACKUP_REGIONS:-us-west1 us-east1} +readonly E2E_CLUSTER_BACKUP_ZONES=${E2E_CLUSTER_BACKUP_ZONES:-} + readonly E2E_CLUSTER_MACHINE=${E2E_CLUSTER_MACHINE:-n1-standard-4} readonly E2E_GKE_ENVIRONMENT=${E2E_GKE_ENVIRONMENT:-prod} readonly E2E_GKE_COMMAND_GROUP=${E2E_GKE_COMMAND_GROUP:-beta} @@ -61,6 +67,8 @@ IS_BOSKOS=0 # Tear down the test resources. function teardown_test_resources() { + # On boskos, save time and don't teardown as the cluster will be destroyed anyway. + (( IS_BOSKOS )) && return header "Tearing down test environment" function_exists test_teardown && test_teardown (( ! SKIP_KNATIVE_SETUP )) && function_exists knative_teardown && knative_teardown @@ -107,8 +115,7 @@ function save_metadata() { geo_key="Zone" geo_value="${E2E_CLUSTER_REGION}-${E2E_CLUSTER_ZONE}" fi - local gcloud_project="$(gcloud config get-value project)" - local cluster_version="$(gcloud container clusters list --project=${gcloud_project} --format='value(currentMasterVersion)')" + local cluster_version="$(gcloud container clusters list --project=${E2E_PROJECT_ID} --format='value(currentMasterVersion)')" cat << EOF > ${ARTIFACTS}/metadata.json { "E2E:${geo_key}": "${geo_value}", @@ -124,6 +131,7 @@ EOF # See https://github.com/knative/serving/issues/959 for details. # TODO(adrcunha): Remove once the leak issue is resolved. function delete_leaked_network_resources() { + # On boskos, don't bother with leaks as the janitor will delete everything in the project. (( IS_BOSKOS )) && return # Ensure we're using the GCP project used by kubetest local gcloud_project="$(gcloud config get-value project)" @@ -154,20 +162,15 @@ function create_test_cluster() { cluster_setup || fail_test "cluster setup failed" fi - header "Creating test cluster" - echo "Cluster will have a minimum of ${E2E_MIN_CLUSTER_NODES} and a maximum of ${E2E_MAX_CLUSTER_NODES} nodes." # Smallest cluster required to run the end-to-end-tests - local geoflag="--gcp-region=${E2E_CLUSTER_REGION}" - [[ -n "${E2E_CLUSTER_ZONE}" ]] && geoflag="--gcp-zone=${E2E_CLUSTER_REGION}-${E2E_CLUSTER_ZONE}" local CLUSTER_CREATION_ARGS=( - --gke-create-command="container clusters create --quiet --enable-autoscaling --min-nodes=${E2E_MIN_CLUSTER_NODES} --max-nodes=${E2E_MAX_CLUSTER_NODES} --scopes=cloud-platform --enable-basic-auth --no-issue-client-certificate ${EXTRA_CLUSTER_CREATION_FLAGS[@]}" + --gke-create-command="container clusters create --quiet --enable-autoscaling --min-nodes=${E2E_MIN_CLUSTER_NODES} --max-nodes=${E2E_MAX_CLUSTER_NODES} --scopes=cloud-platform --enable-basic-auth --no-issue-client-certificate ${GKE_ADDONS} ${EXTRA_CLUSTER_CREATION_FLAGS[@]}" --gke-shape={\"default\":{\"Nodes\":${E2E_MIN_CLUSTER_NODES}\,\"MachineType\":\"${E2E_CLUSTER_MACHINE}\"}} --provider=gke --deployment=gke --cluster="${E2E_CLUSTER_NAME}" - ${geoflag} --gcp-network="${E2E_NETWORK_NAME}" --gke-environment="${E2E_GKE_ENVIRONMENT}" --gke-command-group="${E2E_GKE_COMMAND_GROUP}" @@ -198,11 +201,7 @@ function create_test_cluster() { local extra_flags=() # If using boskos, save time and let it tear down the cluster (( ! IS_BOSKOS )) && extra_flags+=(--down) - # Don't fail test for kubetest, as it might incorrectly report test failure - # if teardown fails (for details, see success() below) - set +o errexit - run_go_tool k8s.io/test-infra/kubetest \ - kubetest "${CLUSTER_CREATION_ARGS[@]}" \ + create_test_cluster_with_retries "${CLUSTER_CREATION_ARGS[@]}" \ --up \ --extract "${E2E_CLUSTER_VERSION}" \ --gcp-node-image "${SERVING_GKE_IMAGE}" \ @@ -215,12 +214,55 @@ function create_test_cluster() { set +o errexit function_exists cluster_teardown && cluster_teardown delete_leaked_network_resources - local result="$(cat ${TEST_RESULT_FILE})" + local result=$(get_test_return_code) echo "Artifacts were written to ${ARTIFACTS}" echo "Test result code is ${result}" exit ${result} } +# Retry backup regions/zones if cluster creations failed due to stockout. +# Parameters: $1..$n - any kubetest flags other than geo flag. +function create_test_cluster_with_retries() { + local cluster_creation_log=/tmp/${E2E_BASE_NAME}-cluster_creation-log + # zone_not_provided is a placeholder for e2e_cluster_zone to make for loop below work + local zone_not_provided="zone_not_provided" + + local e2e_cluster_regions=(${E2E_CLUSTER_REGION}) + local e2e_cluster_zones=(${E2E_CLUSTER_ZONE}) + + if [[ -n "${E2E_CLUSTER_BACKUP_ZONES}" ]]; then + e2e_cluster_zones+=(${E2E_CLUSTER_BACKUP_ZONES}) + elif [[ -n "${E2E_CLUSTER_BACKUP_REGIONS}" ]]; then + e2e_cluster_regions+=(${E2E_CLUSTER_BACKUP_REGIONS}) + e2e_cluster_zones=(${zone_not_provided}) + else + echo "No backup region/zone set, cluster creation will fail in case of stockout" + fi + + for e2e_cluster_region in "${e2e_cluster_regions[@]}"; do + for e2e_cluster_zone in "${e2e_cluster_zones[@]}"; do + E2E_CLUSTER_REGION=${e2e_cluster_region} + E2E_CLUSTER_ZONE=${e2e_cluster_zone} + [[ "${E2E_CLUSTER_ZONE}" == "${zone_not_provided}" ]] && E2E_CLUSTER_ZONE="" + + local geoflag="--gcp-region=${E2E_CLUSTER_REGION}" + [[ -n "${E2E_CLUSTER_ZONE}" ]] && geoflag="--gcp-zone=${E2E_CLUSTER_REGION}-${E2E_CLUSTER_ZONE}" + + header "Creating test cluster in $E2E_CLUSTER_REGION $E2E_CLUSTER_ZONE" + # Don't fail test for kubetest, as it might incorrectly report test failure + # if teardown fails (for details, see success() below) + set +o errexit + { run_go_tool k8s.io/test-infra/kubetest \ + kubetest "$@" ${geoflag}; } 2>&1 | tee ${cluster_creation_log} + + # Exit if test succeeded + [[ "$(get_test_return_code)" == "0" ]] && return + # If test failed not because of cluster creation stockout, return + [[ -z "$(grep -Eio 'does not have enough resources available to fulfill the request' ${cluster_creation_log})" ]] && return + done + done +} + # Setup the test cluster for running the tests. function setup_test_cluster() { # Fail fast during setup. @@ -229,6 +271,11 @@ function setup_test_cluster() { header "Setting up test cluster" + # Set the actual project the test cluster resides in + # It will be a project assigned by Boskos if test is running on Prow, + # otherwise will be ${GCP_PROJECT} set up by user. + readonly export E2E_PROJECT_ID="$(gcloud config get-value project)" + # Save some metadata about cluster creation for using in prow and testgrid save_metadata @@ -240,9 +287,10 @@ function setup_test_cluster() { if [[ -z "$(kubectl get clusterrolebinding cluster-admin-binding 2> /dev/null)" ]]; then acquire_cluster_admin_role ${k8s_user} ${E2E_CLUSTER_NAME} ${E2E_CLUSTER_REGION} ${E2E_CLUSTER_ZONE} kubectl config set-context ${k8s_cluster} --namespace=default - export KO_DOCKER_REPO=gcr.io/$(gcloud config get-value project)/${E2E_BASE_NAME}-e2e-img + export KO_DOCKER_REPO=gcr.io/${E2E_PROJECT_ID}/${E2E_BASE_NAME}-e2e-img fi + echo "- Project is ${E2E_PROJECT_ID}" echo "- Cluster is ${k8s_cluster}" echo "- User is ${k8s_user}" echo "- Docker is ${KO_DOCKER_REPO}" @@ -263,6 +311,12 @@ function setup_test_cluster() { fi } +# Gets the exit of the test script. +# For more details, see set_test_return_code(). +function get_test_return_code() { + echo $(cat ${TEST_RESULT_FILE}) +} + # Set the return code that the test script will return. # Parameters: $1 - return code (0-255) function set_test_return_code() { @@ -295,9 +349,11 @@ function fail_test() { RUN_TESTS=0 EMIT_METRICS=0 SKIP_KNATIVE_SETUP=0 +SKIP_ISTIO=0 GCP_PROJECT="" E2E_SCRIPT="" E2E_CLUSTER_VERSION="" +GKE_ADDONS="" EXTRA_CLUSTER_CREATION_FLAGS=() EXTRA_KUBETEST_FLAGS=() E2E_SCRIPT_CUSTOM_FLAGS=() @@ -329,6 +385,7 @@ function initialize() { --run-tests) RUN_TESTS=1 ;; --emit-metrics) EMIT_METRICS=1 ;; --skip-knative-setup) SKIP_KNATIVE_SETUP=1 ;; + --skip-istio) SKIP_ISTIO=1 ;; *) [[ $# -ge 2 ]] || abort "missing parameter after $1" shift @@ -358,6 +415,8 @@ function initialize() { is_protected_gcr ${KO_DOCKER_REPO} && \ abort "\$KO_DOCKER_REPO set to ${KO_DOCKER_REPO}, which is forbidden" + (( SKIP_ISTIO )) || GKE_ADDONS="--addons=Istio" + readonly RUN_TESTS readonly EMIT_METRICS readonly GCP_PROJECT @@ -365,6 +424,7 @@ function initialize() { readonly EXTRA_CLUSTER_CREATION_FLAGS readonly EXTRA_KUBETEST_FLAGS readonly SKIP_KNATIVE_SETUP + readonly GKE_ADDONS if (( ! RUN_TESTS )); then create_test_cluster diff --git a/vendor/github.com/knative/test-infra/scripts/library.sh b/vendor/github.com/knative/test-infra/scripts/library.sh index 89a93c91..b3216758 100755 --- a/vendor/github.com/knative/test-infra/scripts/library.sh +++ b/vendor/github.com/knative/test-infra/scripts/library.sh @@ -146,22 +146,23 @@ function wait_until_pods_running() { return 1 } -# Waits until all batch job pods are running in the given namespace. +# Waits until all batch jobs complete in the given namespace. # Parameters: $1 - namespace. function wait_until_batch_job_complete() { - echo -n "Waiting until all batch job pods in namespace $1 run to completion." + echo -n "Waiting until all batch jobs in namespace $1 run to completion." for i in {1..150}; do # timeout after 5 minutes - local pods="$(kubectl get pods --selector=job-name --no-headers -n $1 2>/dev/null | grep -v '^[[:space:]]*$')" - # All pods must be complete - local not_complete=$(echo "${pods}" | grep -v Completed | wc -l) + local jobs=$(kubectl get jobs -n $1 --no-headers \ + -ocustom-columns='n:{.metadata.name},c:{.spec.completions},s:{.status.succeeded}') + # All jobs must be complete + local not_complete=$(echo "${jobs}" | awk '{if ($2!=$3) print $0}' | wc -l) if [[ ${not_complete} -eq 0 ]]; then - echo -e "\nAll pods are complete:\n${pods}" + echo -e "\nAll jobs are complete:\n${jobs}" return 0 fi echo -n "." sleep 2 done - echo -e "\n\nERROR: timeout waiting for pods to complete\n${pods}" + echo -e "\n\nERROR: timeout waiting for jobs to complete\n${jobs}" return 1 } @@ -320,14 +321,6 @@ function report_go_test() { # Install the latest stable Knative/serving in the current cluster. function start_latest_knative_serving() { header "Starting Knative Serving" - subheader "Installing Istio" - echo "Running Istio CRD from ${KNATIVE_ISTIO_CRD_YAML}" - kubectl apply -f ${KNATIVE_ISTIO_CRD_YAML} || return 1 - wait_until_batch_job_complete istio-system || return 1 - echo "Installing Istio from ${KNATIVE_ISTIO_YAML}" - kubectl apply -f ${KNATIVE_ISTIO_YAML} || return 1 - wait_until_pods_running istio-system || return 1 - kubectl label namespace default istio-injection=enabled || return 1 subheader "Installing Knative Serving" echo "Installing Serving from ${KNATIVE_SERVING_RELEASE}" kubectl apply -f ${KNATIVE_SERVING_RELEASE} || return 1 diff --git a/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh b/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh index b10a8515..3f15cf3a 100755 --- a/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh +++ b/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh @@ -43,7 +43,7 @@ IS_DOCUMENTATION_PR=0 # Returns true if PR only contains the given file regexes. # Parameters: $1 - file regexes, space separated. function pr_only_contains() { - [[ -z "$(echo "${CHANGED_FILES}" | grep -v \(${1// /\\|}\)$))" ]] + [[ -z "$(echo "${CHANGED_FILES}" | grep -v "\(${1// /\\|}\)$")" ]] } # List changed files in the current PR.