diff --git a/Gopkg.lock b/Gopkg.lock index 80f6d2089..6e48a09b6 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -70,11 +70,11 @@ [[projects]] branch = "master" - digest = "1:00f434f614520a4b48ef3f6d3ed5c7a1a4f89be1bed98c250711413f6da2eb51" + digest = "1:09521a823a008f7df66962ac2637c22e1cdc842ebfcc7a083c444d35258986f7" name = "github.com/knative/test-infra" packages = ["."] pruneopts = "T" - revision = "c21d3a832727d5d4334e734c0878eac3a5a07ae7" + revision = "7ed32409fa2c447a44a4281f0022ab25ce955f51" [[projects]] digest = "1:5985ef4caf91ece5d54817c11ea25f182697534f8ae6521eadcd628c142ac4b6" diff --git a/vendor/github.com/knative/test-infra/scripts/README.md b/vendor/github.com/knative/test-infra/scripts/README.md index 5ff9ccbce..0e7ab5726 100644 --- a/vendor/github.com/knative/test-infra/scripts/README.md +++ b/vendor/github.com/knative/test-infra/scripts/README.md @@ -1,3 +1,200 @@ # Helper scripts -This directory contains helper scripts used by Prow test jobs, as well and local development scripts. +This directory contains helper scripts used by Prow test jobs, as well and +local development scripts. + +## Using the `presubmit-tests.sh` helper script + +This is a helper script to run the presubmit tests. To use it: + +1. Source this script. + +1. Define the functions `build_tests()` and `unit_tests()`. They should run all +tests (i.e., not fail fast), and return 0 if all passed, 1 if a failure +occurred. The environment variables `RUN_BUILD_TESTS`, `RUN_UNIT_TESTS` and +`RUN_INTEGRATION_TESTS` are set to 0 (false) or 1 (true) accordingly. If +`--emit-metrics` is passed, `EMIT_METRICS` will be set to 1. + +1. [optional] Define the function `integration_tests()`, just like the previous +ones. If you don't define this function, the default action for running the +integration tests is to call the `./test/e2e-tests.sh` script (passing the +`--emit-metrics` flag if necessary). + +1. [optional] Define the functions `pre_integration_tests()` or +`post_integration_tests()`. These functions will be called before or after the +integration tests (either your custom one or the default action) and will cause +the test to fail if they don't return success. + +1. Call the `main()` function passing `$@` (without quotes). + +Running the script without parameters, or with the `--all-tests` flag causes +all tests to be executed, in the right order (i.e., build, then unit, then +integration tests). + +Use the flags `--build-tests`, `--unit-tests` and `--integration-tests` to run +a specific set of tests. The flag `--emit-metrics` is used to emit metrics when +running the tests, and is automatically handled by the default action (see +above). + +### Sample presubmit test script + +```bash +source vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh + +function build_tests() { + go build . +} + +function unit_tests() { + report_go_test . +} + +function pre_integration_tests() { + echo "Cleaning up before integration tests" + rm -fr ./staging-area +} + +# We use the default integration test runner. + +main $@ +``` + +## Using the `e2e-tests.sh` helper script + +This is a helper script for Knative E2E test scripts. To use it: + +1. Source the script. + +1. [optional] Write the `teardown()` function, which will tear down your test +resources. + +1. [optional] Write the `dump_extra_cluster_state()` function. It will be +called when a test fails, and can dump extra information about the current state +of the cluster (tipically using `kubectl`). + +1. [optional] Write the `parse_flags()` function. It will be called whenever an +unrecognized flag is passed to the script, allowing you to define your own flags. +The function must return 0 if the flag is unrecognized, or the number of items +to skip in the command line if the flag was parsed successfully. For example, +return 1 for a simple flag, and 2 for a flag with a parameter. + +1. Call the `initialize()` function passing `$@` (without quotes). + +1. Write logic for the end-to-end tests. Run all go tests using `go_test_e2e()` +(or `report_go_test()` if you need a more fine-grained control) and call +`fail_test()` or `success()` if any of them failed. The environment variables +`DOCKER_REPO_OVERRIDE`, `K8S_CLUSTER_OVERRIDE` and `K8S_USER_OVERRIDE` will be set +according to the test cluster. You can also use the following boolean (0 is false, +1 is true) environment variables for the logic: + * `EMIT_METRICS`: true if `--emit-metrics` was passed. + * `USING_EXISTING_CLUSTER`: true if the test cluster is an already existing one, +and not a temporary cluster created by `kubetest`. + + All environment variables above are marked read-only. + +**Notes:** + +1. Calling your script without arguments will create a new cluster in the GCP +project `$PROJECT_ID` and run the tests against it. + +1. Calling your script with `--run-tests` and the variables `K8S_CLUSTER_OVERRIDE`, +`K8S_USER_OVERRIDE` and `DOCKER_REPO_OVERRIDE` set will immediately start the +tests against the cluster. + +1. You can force running the tests against a specific GKE cluster version by using +the `--cluster-version` flag and passing a X.Y.Z version as the flag value. + +### Sample end-to-end test script + +This script will test that the latest Knative Serving nightly release works. It +defines a special flag (`--no-knative-wait`) that causes the script not to +wait for Knative Serving to be up before running the tests. + +```bash +source vendor/github.com/knative/test-infra/scripts/e2e-tests.sh + +function teardown() { + echo "TODO: tear down test resources" +} + +function parse_flags() { + if [[ "$1" == "--no-knative-wait" ]]; then + WAIT_FOR_KNATIVE=0 + return 1 + fi + return 0 +} + +WAIT_FOR_KNATIVE=1 + +initialize $@ + +start_latest_knative_serving + +if (( WAIT_FOR_KNATIVE )); then + wait_until_pods_running knative-serving || fail_test "Knative Serving is not up" +fi + +# TODO: use go_test_e2e to run the tests. +kubectl get pods || fail_test + +success +``` + +## Using the `release.sh` helper script + +This is a helper script for Knative release scripts. To use it: + +1. Source the script. + +1. Call the `initialize()` function passing `$@` (without quotes). + +1. Call the `run_validation_tests()` function passing the script or executable that +runs the release validation tests. It will call the script to run the tests unless +`--skip_tests` was passed. + +1. Write logic for the release process. Call `publish_yaml()` to publish the manifest(s), +`tag_releases_in_yaml()` to tag the generated images, `branch_release()` to branch +named releases. Use the following boolean (0 is false, 1 is true) and string environment +variables for the logic: + * `RELEASE_VERSION`: contains the release version if `--version` was passed. This +also overrides the value of the `TAG` variable as `v`. + * `RELEASE_BRANCH`: contains the release branch if `--branch` was passed. Otherwise +it's empty and `master` HEAD will be considered the release branch. + * `RELEASE_NOTES`: contains the filename with the release notes if `--release-notes` +was passed. The release notes is a simple markdown file. + * `SKIP_TESTS`: true if `--skip-tests` was passed. This is handled automatically +by the run_validation_tests() function. + * `TAG_RELEASE`: true if `--tag-release` was passed. In this case, the environment +variable `TAG` will contain the release tag in the form `vYYYYMMDD-`. + * `PUBLISH_RELEASE`: true if `--publish` was passed. In this case, the environment +variable `KO_FLAGS` will be updated with the `-L` option. + * `BRANCH_RELEASE`: true if both `--version` and `--publish-release` were passed. + + All boolean environment variables default to false for safety. + + All environment variables above, except `KO_FLAGS`, are marked read-only once +`initialize()` is called. + +### Sample release script + +```bash +source vendor/github.com/knative/test-infra/scripts/release.sh + +initialize $@ + +run_validation_tests ./test/presubmit-tests.sh + +# config/ contains the manifests +KO_DOCKER_REPO=gcr.io/knative-foo +ko resolve ${KO_FLAGS} -f config/ > release.yaml + +tag_images_in_yaml release.yaml $KO_DOCKER_REPO $TAG + +if (( PUBLISH_RELEASE )); then + # gs://knative-foo hosts the manifest + publish_yaml release.yaml knative-foo $TAG +fi + +branch_release "Knative Foo" release.yaml +``` diff --git a/vendor/github.com/knative/test-infra/scripts/dummy.go b/vendor/github.com/knative/test-infra/scripts/dummy.go new file mode 100644 index 000000000..e6cc380fd --- /dev/null +++ b/vendor/github.com/knative/test-infra/scripts/dummy.go @@ -0,0 +1,26 @@ +/* +Copyright 2018 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + https://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package scripts + +import ( + "fmt" +) + +func main() { + fmt.Println("This is a dummy go file so `go dep` can be used with knative/test-infra/scripts") + fmt.Println("This file can be safely removed if one day this directory contains real, useful go code") +} diff --git a/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh b/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh index 7c9210482..b1dd6644a 100755 --- a/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh +++ b/vendor/github.com/knative/test-infra/scripts/e2e-tests.sh @@ -14,29 +14,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This is a helper script for Knative E2E test scripts. To use it: -# 1. Source this script. -# 2. [optional] Write the teardown() function, which will tear down your test -# resources. -# 3. [optional] Write the dump_extra_cluster_state() function. It will be called -# when a test fails, and can dump extra information about the current state of -# the cluster (tipically using kubectl). -# 4. Call the initialize() function passing $@ (without quotes). -# 5. Write logic for the end-to-end tests. Run all go tests using report_go_test() -# and call fail_test() or success() if any of them failed. The envitronment -# variables DOCKER_REPO_OVERRIDE, K8S_CLUSTER_OVERRIDE and K8S_USER_OVERRIDE -# will be set accordingly to the test cluster. You can also use the following -# boolean (0 is false, 1 is true) environment variables for the logic: -# EMIT_METRICS: true if --emit-metrics is passed. -# USING_EXISTING_CLUSTER: true if the test cluster is an already existing one, -# and not a temporary cluster created by kubetest. -# All environment variables above are marked read-only. -# Notes: -# 1. Calling your script without arguments will create a new cluster in the GCP -# project $PROJECT_ID and run the tests against it. -# 2. Calling your script with --run-tests and the variables K8S_CLUSTER_OVERRIDE, -# K8S_USER_OVERRIDE and DOCKER_REPO_OVERRIDE set will immediately start the -# tests against the cluster. +# This is a helper script for Knative E2E test scripts. +# See README.md for instructions on how to use it. source $(dirname ${BASH_SOURCE})/library.sh @@ -51,7 +30,9 @@ function build_resource_name() { if [[ -n "${suffix}" ]]; then suffix=${suffix:${#suffix}<20?0:-20} fi - echo "${prefix:0:20}${suffix}" + local name="${prefix:0:20}${suffix}" + # Ensure name doesn't end with "-" + echo "${name%-}" } # Test cluster parameters @@ -64,23 +45,19 @@ readonly E2E_CLUSTER_NODES=3 readonly E2E_CLUSTER_MACHINE=n1-standard-4 readonly TEST_RESULT_FILE=/tmp/${E2E_BASE_NAME}-e2e-result +# Flag whether test is using a boskos GCP project +IS_BOSKOS=0 + # Tear down the test resources. function teardown_test_resources() { header "Tearing down test environment" # Free resources in GCP project. - if (( ! USING_EXISTING_CLUSTER )) && [[ "$(type -t teardown)" == "function" ]]; then + if (( ! USING_EXISTING_CLUSTER )) && function_exists teardown; then teardown fi - # Delete Knative Serving images when using prow. - if (( IS_PROW )); then - echo "Images in ${DOCKER_REPO_OVERRIDE}:" - gcloud container images list --repository=${DOCKER_REPO_OVERRIDE} - delete_gcr_images ${DOCKER_REPO_OVERRIDE} - else - # Delete the kubernetes source downloaded by kubetest - rm -fr kubernetes kubernetes.tar.gz - fi + # Delete the kubernetes source downloaded by kubetest + rm -fr kubernetes kubernetes.tar.gz } # Exit test, dumping current state info. @@ -91,22 +68,36 @@ function fail_test() { exit 1 } +# Run the given E2E tests. Assume tests are tagged e2e, unless `-tags=XXX` is passed. +# Parameters: $1..$n - any go test flags, then directories containing the tests to run. +function go_test_e2e() { + local test_options="" + local go_options="" + (( EMIT_METRICS )) && test_options="-emitmetrics" + [[ ! " $@" == *" -tags="* ]] && go_options="-tags=e2e" + report_go_test -v -count=1 ${go_options} $@ ${test_options} +} + # Download the k8s binaries required by kubetest. +# Parameters: $1 - GCP project that will host the test cluster. function download_k8s() { - local version=${SERVING_GKE_VERSION} + local version=${E2E_CLUSTER_VERSION} + # Fetch valid versions + local versions="$(gcloud container get-server-config \ + --project=$1 \ + --format='value(validMasterVersions)' \ + --region=${E2E_CLUSTER_REGION})" + local gke_versions=(`echo -n ${versions//;/ /}`) + echo "Valid GKE versions are [${versions//;/, }]" if [[ "${version}" == "latest" ]]; then - # Fetch latest valid version - local versions="$(gcloud container get-server-config \ - --project=${GCP_PROJECT} \ - --format='value(validMasterVersions)' \ - --region=${E2E_CLUSTER_REGION})" - local gke_versions=(`echo -n ${versions//;/ /}`) # Get first (latest) version, excluding the "-gke.#" suffix version="${gke_versions[0]%-*}" - echo "Latest GKE is ${version}, from [${versions//;/, }]" + echo "Using latest version, ${version}" elif [[ "${version}" == "default" ]]; then echo "ERROR: `default` GKE version is not supported yet" return 1 + else + echo "Using command-line supplied version ${version}" fi # Download k8s to staging dir version=v${version} @@ -124,6 +115,7 @@ function download_k8s() { mv kubernetes/client/kubernetes-client-*.tar.gz . rm -fr kubernetes # Create an empty kubernetes test tarball; we don't use it but kubetest will fetch it + # As of August 21 2018 this means avoiding a useless 1.2GB download tar -czf kubernetes-test.tar.gz -T /dev/null fi popd @@ -134,7 +126,7 @@ function download_k8s() { # This is intended to be called when a test fails to provide debugging information. function dump_cluster_state() { echo "***************************************" - echo "*** TEST FAILED ***" + echo "*** E2E TEST FAILED ***" echo "*** Start of information dump ***" echo "***************************************" echo ">>> All resources:" @@ -143,9 +135,9 @@ function dump_cluster_state() { kubectl get services --all-namespaces echo ">>> Events:" kubectl get events --all-namespaces - [[ "$(type -t dump_extra_cluster_state)" == "function" ]] && dump_extra_cluster_state + function_exists dump_extra_cluster_state && dump_extra_cluster_state echo "***************************************" - echo "*** TEST FAILED ***" + echo "*** E2E TEST FAILED ***" echo "*** End of information dump ***" echo "***************************************" } @@ -168,10 +160,10 @@ function create_test_cluster() { --gcp-network="${E2E_NETWORK_NAME}" --gke-environment=prod ) - if (( ! IS_PROW )); then - CLUSTER_CREATION_ARGS+=(--gcp-project=${PROJECT_ID:?"PROJECT_ID must be set to the GCP project where the tests are run."}) - else + if (( IS_BOSKOS )); then CLUSTER_CREATION_ARGS+=(--gcp-service-account=/etc/service-account/service-account.json) + else + CLUSTER_CREATION_ARGS+=(--gcp-project=${GCP_PROJECT}) fi # SSH keys are not used, but kubetest checks for their existence. # Touch them so if they don't exist, empty files are create to satisfy the check. @@ -183,45 +175,52 @@ function create_test_cluster() { # be a writeable docker repo. export K8S_USER_OVERRIDE= export K8S_CLUSTER_OVERRIDE= - # Get the current GCP project - export GCP_PROJECT=${PROJECT_ID} - [[ -z ${GCP_PROJECT} ]] && export GCP_PROJECT=$(gcloud config get-value project) # Assume test failed (see more details at the end of this script). echo -n "1"> ${TEST_RESULT_FILE} local test_cmd_args="--run-tests" (( EMIT_METRICS )) && test_cmd_args+=" --emit-metrics" + [[ -n "${GCP_PROJECT}" ]] && test_cmd_args+=" --gcp-project ${GCP_PROJECT}" + # Get the current GCP project for downloading kubernetes + local gcloud_project="${GCP_PROJECT}" + [[ -z "${gcloud_project}" ]] && gcloud_project="$(gcloud config get-value project)" + echo "gcloud project is ${gcloud_project}" + (( IS_BOSKOS )) && echo "Using boskos for the test cluster" + [[ -n "${GCP_PROJECT}" ]] && echo "GCP project for test cluster is ${GCP_PROJECT}" echo "Test script is ${E2E_SCRIPT}" - download_k8s || return 1 + download_k8s ${gcloud_project} || return 1 # Don't fail test for kubetest, as it might incorrectly report test failure # if teardown fails (for details, see success() below) set +o errexit - kubetest "${CLUSTER_CREATION_ARGS[@]}" \ + run_go_tool k8s.io/test-infra/kubetest \ + kubetest "${CLUSTER_CREATION_ARGS[@]}" \ --up \ --down \ --extract local \ - --gcp-node-image ${SERVING_GKE_IMAGE} \ + --gcp-node-image "${SERVING_GKE_IMAGE}" \ --test-cmd "${E2E_SCRIPT}" \ --test-cmd-args "${test_cmd_args}" echo "Test subprocess exited with code $?" # Ignore any errors below, this is a best-effort cleanup and shouldn't affect the test result. set +o errexit + # Ensure we're using the GCP project used by kubetest + gcloud_project="$(gcloud config get-value project)" # Delete target pools and health checks that might have leaked. # See https://github.com/knative/serving/issues/959 for details. # TODO(adrcunha): Remove once the leak issue is resolved. local http_health_checks="$(gcloud compute target-pools list \ - --project=${GCP_PROJECT} --format='value(healthChecks)' --filter="instances~-${E2E_CLUSTER_NAME}-" | \ + --project=${gcloud_project} --format='value(healthChecks)' --filter="instances~-${E2E_CLUSTER_NAME}-" | \ grep httpHealthChecks | tr '\n' ' ')" local target_pools="$(gcloud compute target-pools list \ - --project=${GCP_PROJECT} --format='value(name)' --filter="instances~-${E2E_CLUSTER_NAME}-" | \ + --project=${gcloud_project} --format='value(name)' --filter="instances~-${E2E_CLUSTER_NAME}-" | \ tr '\n' ' ')" if [[ -n "${target_pools}" ]]; then echo "Found leaked target pools, deleting" - gcloud compute forwarding-rules delete -q --project=${GCP_PROJECT} --region=${E2E_CLUSTER_REGION} ${target_pools} - gcloud compute target-pools delete -q --project=${GCP_PROJECT} --region=${E2E_CLUSTER_REGION} ${target_pools} + gcloud compute forwarding-rules delete -q --project=${gcloud_project} --region=${E2E_CLUSTER_REGION} ${target_pools} + gcloud compute target-pools delete -q --project=${gcloud_project} --region=${E2E_CLUSTER_REGION} ${target_pools} fi if [[ -n "${http_health_checks}" ]]; then echo "Found leaked health checks, deleting" - gcloud compute http-health-checks delete -q --project=${GCP_PROJECT} ${http_health_checks} + gcloud compute http-health-checks delete -q --project=${gcloud_project} ${http_health_checks} fi local result="$(cat ${TEST_RESULT_FILE})" echo "Test result code is $result" @@ -257,9 +256,11 @@ function setup_test_cluster() { echo "- User is ${K8S_USER_OVERRIDE}" echo "- Docker is ${DOCKER_REPO_OVERRIDE}" + export KO_DOCKER_REPO="${DOCKER_REPO_OVERRIDE}" + trap teardown_test_resources EXIT - if (( USING_EXISTING_CLUSTER )) && [[ "$(type -t teardown)" == "function" ]]; then + if (( USING_EXISTING_CLUSTER )) && function_exists teardown; then echo "Deleting any previous SUT instance" teardown fi @@ -281,7 +282,7 @@ function success() { # TODO(adrcunha): Get rid of this workaround. echo -n "0"> ${TEST_RESULT_FILE} echo "**************************************" - echo "*** ALL TESTS PASSED ***" + echo "*** E2E TESTS PASSED ***" echo "**************************************" exit 0 } @@ -289,7 +290,14 @@ function success() { RUN_TESTS=0 EMIT_METRICS=0 USING_EXISTING_CLUSTER=1 +GCP_PROJECT="" E2E_SCRIPT="" +E2E_CLUSTER_VERSION="" + +function abort() { + echo "error: $@" + exit 1 +} # Parse flags and initialize the test cluster. function initialize() { @@ -299,21 +307,66 @@ function initialize() { E2E_SCRIPT="$(cd ${E2E_SCRIPT%/*} && echo $PWD/${E2E_SCRIPT##*/})" readonly E2E_SCRIPT + E2E_CLUSTER_VERSION="${SERVING_GKE_VERSION}" + cd ${REPO_ROOT_DIR} - for parameter in $@; do + while [[ $# -ne 0 ]]; do + local parameter=$1 + # Try parsing flag as a custom one. + if function_exists parse_flags; then + parse_flags $@ + local skip=$? + if [[ ${skip} -ne 0 ]]; then + # Skip parsed flag (and possibly argument) and continue + shift ${skip} + continue + fi + fi + # Try parsing flag as a standard one. case $parameter in --run-tests) RUN_TESTS=1 ;; --emit-metrics) EMIT_METRICS=1 ;; + --gcp-project) + shift + [[ $# -ge 1 ]] || abort "missing project name after --gcp-project" + GCP_PROJECT=$1 + ;; + --cluster-version) + shift + [[ $# -ge 1 ]] || abort "missing version after --cluster-version" + [[ $1 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || abort "kubernetes version must be 'X.Y.Z'" + E2E_CLUSTER_VERSION=$1 + ;; *) - echo "error: unknown option ${parameter}" - echo "usage: $0 [--run-tests][--emit-metrics]" - exit 1 + echo "usage: $0 [--run-tests][--emit-metrics][--cluster-version X.Y.Z][--gcp-project name]" + abort "unknown option ${parameter}" ;; esac shift done + + # Use PROJECT_ID if set, unless --gcp-project was used. + if [[ -n "${PROJECT_ID:-}" && -z "${GCP_PROJECT}" ]]; then + echo "\$PROJECT_ID is set to '${PROJECT_ID}', using it to run the tests" + GCP_PROJECT="${PROJECT_ID}" + fi + if (( ! IS_PROW )) && [[ -z "${GCP_PROJECT}" ]]; then + abort "set \$PROJECT_ID or use --gcp-project to select the GCP project where the tests are run" + fi + + (( IS_PROW )) && [[ -z "${GCP_PROJECT}" ]] && IS_BOSKOS=1 + + # Safety checks + + if [[ "${DOCKER_REPO_OVERRIDE}" =~ ^gcr.io/knative-(releases|nightly)/?$ ]]; then + abort "\$DOCKER_REPO_OVERRIDE is set to ${DOCKER_REPO_OVERRIDE}, which is forbidden" + fi + readonly RUN_TESTS readonly EMIT_METRICS + readonly E2E_CLUSTER_VERSION + readonly GCP_PROJECT + readonly IS_BOSKOS if (( ! RUN_TESTS )); then create_test_cluster diff --git a/vendor/github.com/knative/test-infra/scripts/library.sh b/vendor/github.com/knative/test-infra/scripts/library.sh index a330f4dbc..ba267d97c 100755 --- a/vendor/github.com/knative/test-infra/scripts/library.sh +++ b/vendor/github.com/knative/test-infra/scripts/library.sh @@ -22,11 +22,12 @@ readonly SERVING_GKE_VERSION=latest readonly SERVING_GKE_IMAGE=cos -# Public images and yaml files. -readonly KNATIVE_ISTIO_YAML=https://storage.googleapis.com/knative-releases/serving/latest/istio.yaml -readonly KNATIVE_SERVING_RELEASE=https://storage.googleapis.com/knative-releases/serving/latest/release.yaml -readonly KNATIVE_BUILD_RELEASE=https://storage.googleapis.com/knative-releases/build/latest/release.yaml -readonly KNATIVE_EVENTING_RELEASE=https://storage.googleapis.com/knative-releases/eventing/latest/release.yaml +# Public latest stable nightly images and yaml files. +readonly KNATIVE_ISTIO_CRD_YAML=https://storage.googleapis.com/knative-nightly/serving/latest/istio-crds.yaml +readonly KNATIVE_ISTIO_YAML=https://storage.googleapis.com/knative-nightly/serving/latest/istio.yaml +readonly KNATIVE_SERVING_RELEASE=https://storage.googleapis.com/knative-nightly/serving/latest/release.yaml +readonly KNATIVE_BUILD_RELEASE=https://storage.googleapis.com/knative-nightly/build/latest/release.yaml +readonly KNATIVE_EVENTING_RELEASE=https://storage.googleapis.com/knative-nightly/eventing/latest/release.yaml # Conveniently set GOPATH if unset if [[ -z "${GOPATH:-}" ]]; then @@ -46,7 +47,7 @@ readonly REPO_ROOT_DIR="$(git rev-parse --show-toplevel)" # $2 - banner message. function make_banner() { local msg="$1$1$1$1 $2 $1$1$1$1" - local border="${msg//[-0-9A-Za-z _.,]/$1}" + local border="${msg//[-0-9A-Za-z _.,\/]/$1}" echo -e "${border}\n${msg}\n${border}" } @@ -66,6 +67,11 @@ function warning() { make_banner "!" "$1" } +# Checks whether the given function exists. +function function_exists() { + [[ "$(type -t $1)" == "function" ]] +} + # Remove ALL images in the given GCR repository. # Parameters: $1 - GCR repository. function delete_gcr_images() { @@ -94,7 +100,10 @@ function wait_until_object_does_not_exist() { fi echo -n "Waiting until ${DESCRIPTION} does not exist" for i in {1..150}; do # timeout after 5 minutes - kubectl ${KUBECTL_ARGS} 2>&1 > /dev/null || return 0 + if kubectl ${KUBECTL_ARGS} > /dev/null 2>&1; then + echo "\n${DESCRIPTION} does not exist" + return 0 + fi echo -n "." sleep 2 done @@ -172,13 +181,21 @@ function wait_until_routable() { return 1 } -# Returns the name of the pod of the given app. +# Returns the name of the first pod of the given app. # Parameters: $1 - app name. # $2 - namespace (optional). function get_app_pod() { + local pods=($(get_app_pods $1 $2)) + echo "${pods[0]}" +} + +# Returns the name of all pods of the given app. +# Parameters: $1 - app name. +# $2 - namespace (optional). +function get_app_pods() { local namespace="" [[ -n $2 ]] && namespace="-n $2" - kubectl get pods ${namespace} --selector=app=$1 --output=jsonpath="{.items[0].metadata.name}" + kubectl get pods ${namespace} --selector=app=$1 --output=jsonpath="{.items[*].metadata.name}" } # Sets the given user as cluster admin. @@ -220,8 +237,8 @@ function acquire_cluster_admin_role() { function report_go_test() { # Run tests in verbose mode to capture details. # go doesn't like repeating -v, so remove if passed. - local args=("${@/-v}") - local go_test="go test -race -v ${args[@]}" + local args=" $@ " + local go_test="go test -race -v ${args/ -v / }" # Just run regular go tests if not on Prow. if (( ! IS_PROW )); then ${go_test} @@ -231,13 +248,16 @@ function report_go_test() { local report=$(mktemp) local failed=0 local test_count=0 + local tests_failed=0 ${go_test} > ${report} || failed=$? echo "Finished run, return code is ${failed}" # Tests didn't run. [[ ! -s ${report} ]] && return 1 - # Create WORKSPACE file, required to use bazel + # Create WORKSPACE file, required to use bazel, if necessary. touch WORKSPACE local targets="" + local last_run="" + local test_files="" # Parse the report and generate fake tests for each passing/failing test. echo "Start parsing results, summary:" while read line ; do @@ -245,16 +265,35 @@ function report_go_test() { local field0="${fields[0]}" local field1="${fields[1]}" local name="${fields[2]}" + # Deal with a SIGQUIT log entry (usually a test timeout). + # This is a fallback in case there's no kill signal log entry. + # SIGQUIT: quit + if [[ "${field0}" == "SIGQUIT:" ]]; then + name="${last_run}" + field1="FAIL:" + error="${fields[@]}" + fi # Ignore subtests (those containing slashes) if [[ -n "${name##*/*}" ]]; then local error="" + # Deal with a kill signal log entry (usually a test timeout). + # *** Test killed with quit: ran too long (10m0s). + if [[ "${field0}" == "***" ]]; then + name="${last_run}" + field1="FAIL:" + error="${fields[@]:1}" + fi # Deal with a fatal log entry, which has a different format: # fatal TestFoo foo_test.go:275 Expected "foo" but got "bar" if [[ "${field0}" == "fatal" ]]; then - name="${fields[1]}" + name="${field1}" field1="FAIL:" error="${fields[@]:3}" fi + # Keep track of the test currently running. + if [[ "${field1}" == "RUN" ]]; then + last_run="${name}" + fi # Handle regular go test pass/fail entry for a test. if [[ "${field1}" == "PASS:" || "${field1}" == "FAIL:" ]]; then echo "- ${name} :${field1}" @@ -262,6 +301,7 @@ function report_go_test() { local src="${name}.sh" echo "exit 0" > ${src} if [[ "${field1}" == "FAIL:" ]]; then + tests_failed=$(( tests_failed + 1 )) [[ -z "${error}" ]] && read error echo "cat < ${src} echo "${error}" >> ${src} @@ -269,13 +309,14 @@ function report_go_test() { echo "exit 1" >> ${src} fi chmod +x ${src} + test_files="${test_files} ${src}" # Populate BUILD.bazel echo "sh_test(name=\"${name}\", srcs=[\"${src}\"])" >> BUILD.bazel elif [[ "${field0}" == "FAIL" || "${field0}" == "ok" ]] && [[ -n "${field1}" ]]; then echo "- ${field0} ${field1}" # Create the package structure, move tests and BUILD file local package=${field1/github.com\//} - local bazel_files="$(ls -1 *.sh BUILD.bazel 2> /dev/null)" + local bazel_files="$(ls -1 ${test_files} BUILD.bazel 2> /dev/null)" if [[ -n "${bazel_files}" ]]; then mkdir -p ${package} targets="${targets} //${package}/..." @@ -283,19 +324,24 @@ function report_go_test() { else echo "*** INTERNAL ERROR: missing tests for ${package}, got [${bazel_files/$'\n'/, }]" fi + test_files="" fi fi done < ${report} - echo "Done parsing ${test_count} tests" + echo "Done parsing ${test_count} tests, ${tests_failed} tests failed" # If any test failed, show the detailed report. # Otherwise, we already shown the summary. # Exception: when emitting metrics, dump the full report. if (( failed )) || [[ "$@" == *" -emitmetrics"* ]]; then - echo "At least one test failed, full log:" + if (( failed )); then + echo "There were ${tests_failed} test failures, full log:" + else + echo "Dumping full log as metrics were requested:" + fi cat ${report} fi # Always generate the junit summary. - bazel test ${targets} > /dev/null 2>&1 + bazel test ${targets} > /dev/null 2>&1 || true return ${failed} } @@ -303,6 +349,7 @@ function report_go_test() { function start_latest_knative_serving() { header "Starting Knative Serving" subheader "Installing Istio" + kubectl apply -f ${KNATIVE_ISTIO_CRD_YAML} || return 1 kubectl apply -f ${KNATIVE_ISTIO_YAML} || return 1 wait_until_pods_running istio-system || return 1 kubectl label namespace default istio-injection=enabled || return 1 @@ -323,14 +370,19 @@ function start_latest_knative_build() { wait_until_pods_running knative-build || return 1 } -# Run dep-collector, installing it first if necessary. -# Parameters: $1..$n - parameters passed to dep-collector. -function run_dep_collector() { - local local_dep_collector="$(which dep-collector)" - if [[ -z ${local_dep_collector} ]]; then - go get -u github.com/mattmoor/dep-collector +# Run a go tool, installing it first if necessary. +# Parameters: $1 - tool package/dir for go get/install. +# $2 - tool to run. +# $3..$n - parameters passed to the tool. +function run_go_tool() { + local tool=$2 + if [[ -z "$(which ${tool})" ]]; then + local action=get + [[ $1 =~ ^[\./].* ]] && action=install + go ${action} $1 fi - dep-collector $@ + shift 2 + ${tool} "$@" } # Run dep-collector to update licenses. @@ -340,7 +392,7 @@ function update_licenses() { cd ${REPO_ROOT_DIR} || return 1 local dst=$1 shift - run_dep_collector $@ > ./${dst} + run_go_tool ./vendor/github.com/knative/test-infra/tools/dep-collector dep-collector $@ > ./${dst} } # Run dep-collector to check for forbidden liceses. @@ -349,7 +401,7 @@ function check_licenses() { # Fetch the google/licenseclassifier for its license db go get -u github.com/google/licenseclassifier # Check that we don't have any forbidden licenses in our images. - run_dep_collector -check $@ + run_go_tool ./vendor/github.com/knative/test-infra/tools/dep-collector dep-collector -check $@ } # Run the given linter on the given files, checking it exists first. diff --git a/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh b/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh index b6116817c..5818fab98 100755 --- a/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh +++ b/vendor/github.com/knative/test-infra/scripts/presubmit-tests.sh @@ -14,26 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This is a helper script to run the presubmit tests. To use it: -# 1. Source this script. -# 2. Define the functions build_tests(), unit_tests() and -# integration_tests(). They should run all tests (i.e., not fail -# fast), and return 0 if all passed, 1 if a failure occurred. -# The environment variables RUN_BUILD_TESTS, RUN_UNIT_TESTS and -# RUN_INTEGRATION_TESTS are set to 0 (false) or 1 (true) accordingly. -# If --emit-metrics is passed, EMIT_METRICS will be set to 1. -# 3. Call the main() function passing $@ (without quotes). -# -# Running the script without parameters, or with the --all-tests -# flag, causes all tests to be executed, in the right order. -# Use the flags --build-tests, --unit-tests and --integration-tests -# to run a specific set of tests. The flag --emit-metrics is used -# to emit metrics when running the tests. +# This is a helper script for Knative presubmit test scripts. +# See README.md for instructions on how to use it. source $(dirname ${BASH_SOURCE})/library.sh # Extensions or file patterns that don't require presubmit tests. -readonly NO_PRESUBMIT_FILES=(\.md \.png ^OWNERS) +readonly NO_PRESUBMIT_FILES=(\.md \.png ^OWNERS ^OWNERS_ALIASES) # Options set by command-line flags. RUN_BUILD_TESTS=0 @@ -61,62 +48,114 @@ function exit_if_presubmit_not_required() { fi } +function abort() { + echo "error: $@" + exit 1 +} + # Process flags and run tests accordingly. function main() { exit_if_presubmit_not_required - local all_parameters=$@ - [[ -z $1 ]] && all_parameters="--all-tests" + # Show the version of the tools we're using + if (( IS_PROW )); then + # Disable gcloud update notifications + gcloud config set component_manager/disable_update_check true + header "Current test setup" + echo ">> gcloud SDK version" + gcloud version + echo ">> kubectl version" + kubectl version + echo ">> go version" + go version + echo ">> git version" + git version + fi - for parameter in ${all_parameters}; do + [[ -z $1 ]] && set -- "--all-tests" + + local TEST_TO_RUN="" + + while [[ $# -ne 0 ]]; do + local parameter=$1 case ${parameter} in + --build-tests) RUN_BUILD_TESTS=1 ;; + --unit-tests) RUN_UNIT_TESTS=1 ;; + --integration-tests) RUN_INTEGRATION_TESTS=1 ;; + --emit-metrics) EMIT_METRICS=1 ;; --all-tests) RUN_BUILD_TESTS=1 RUN_UNIT_TESTS=1 RUN_INTEGRATION_TESTS=1 + ;; + --run-test) shift + [[ $# -ge 1 ]] || abort "missing executable after --run-test" + TEST_TO_RUN=$1 ;; - --build-tests) - RUN_BUILD_TESTS=1 - shift - ;; - --unit-tests) - RUN_UNIT_TESTS=1 - shift - ;; - --integration-tests) - RUN_INTEGRATION_TESTS=1 - shift - ;; - --emit-metrics) - EMIT_METRICS=1 - shift - ;; - *) - echo "error: unknown option ${parameter}" - exit 1 - ;; + *) abort "error: unknown option ${parameter}" ;; esac + shift done readonly RUN_BUILD_TESTS readonly RUN_UNIT_TESTS readonly RUN_INTEGRATION_TESTS readonly EMIT_METRICS + readonly TEST_TO_RUN cd ${REPO_ROOT_DIR} # Tests to be performed, in the right order if --all-tests is passed. - local result=0 + local failed=0 + + if [[ -n "${TEST_TO_RUN}" ]]; then + if (( RUN_BUILD_TESTS || RUN_UNIT_TESTS || RUN_INTEGRATION_TESTS )); then + abort "--run-test must be used alone" + fi + ${TEST_TO_RUN} || failed=1 + fi + if (( RUN_BUILD_TESTS )); then - build_tests || result=1 + build_tests || failed=1 fi if (( RUN_UNIT_TESTS )); then - unit_tests || result=1 + unit_tests || failed=1 fi if (( RUN_INTEGRATION_TESTS )); then - integration_tests || result=1 + local e2e_failed=0 + # Run pre-integration tests, if any + if function_exists pre_integration_tests; then + if ! pre_integration_tests; then + failed=1 + e2e_failed=1 + fi + fi + # Don't run integration tests if pre-integration tests failed + if (( ! e2e_failed )); then + if function_exists integration_tests; then + if ! integration_tests; then + failed=1 + e2e_failed=1 + fi + else + local options="" + (( EMIT_METRICS )) && options="--emit-metrics" + for e2e_test in ./test/e2e-*tests.sh; do + echo "Running integration test ${e2e_test}" + if ! ${e2e_test} ${options}; then + failed=1 + e2e_failed=1 + fi + done + fi + fi + # Don't run post-integration + if (( ! e2e_failed )) && function_exists post_integration_tests; then + post_integration_tests || failed=1 + fi fi - exit ${result} + + exit ${failed} } diff --git a/vendor/github.com/knative/test-infra/scripts/release.sh b/vendor/github.com/knative/test-infra/scripts/release.sh index abf89ddf4..cbde3c642 100755 --- a/vendor/github.com/knative/test-infra/scripts/release.sh +++ b/vendor/github.com/knative/test-infra/scripts/release.sh @@ -14,27 +14,13 @@ # See the License for the specific language governing permissions and # limitations under the License. -# This is a helper script for Knative release scripts. To use it: -# 1. Source this script. -# 2. Call the parse_flags() function passing $@ (without quotes). -# 3. Call the run_validation_tests() passing the script or executable that -# runs the release validation tests. -# 4. Write logic for the release process. Use the following boolean (0 is -# false, 1 is true) environment variables for the logic: -# SKIP_TESTS: true if --skip-tests is passed. This is handled automatically -# by the run_validation_tests() function. -# TAG_RELEASE: true if --tag-release is passed. In this case, the -# environment variable TAG will contain the release tag in the -# form vYYYYMMDD-. -# PUBLISH_RELEASE: true if --publish is passed. In this case, the environment -# variable KO_FLAGS will be updated with the -L option. -# SKIP_TESTS, TAG_RELEASE and PUBLISH_RELEASE default to false for safety. -# All environment variables above, except KO_FLAGS, are marked read-only once -# parse_flags() is called. +# This is a helper script for Knative release scripts. +# See README.md for instructions on how to use it. source $(dirname ${BASH_SOURCE})/library.sh # Simple banner for logging purposes. +# Parameters: $1 - message to display. function banner() { make_banner "@" "$1" } @@ -45,8 +31,10 @@ function banner() { # $3 - tag to apply (optional). function tag_images_in_yaml() { [[ -z $3 ]] && return 0 - echo "Tagging images with $3" - for image in $(grep -o "$2/[a-z\./-]\+@sha256:[0-9a-f]\+" $1); do + local src_dir="${GOPATH}/src/" + local BASE_PATH="${REPO_ROOT_DIR/$src_dir}" + echo "Tagging images under '${BASE_PATH}' with $3" + for image in $(grep -o "$2/${BASE_PATH}/[a-z\./-]\+@sha256:[0-9a-f]\+" $1); do gcloud -q container images add-tag ${image} ${image%%@*}:$3 done } @@ -57,54 +45,92 @@ function tag_images_in_yaml() { # $3 - tag to apply (optional). function publish_yaml() { gsutil cp $1 gs://$2/latest/ - [[ -n $3 ]] && gsutil cp $1 gs://$2/previous/$3/ + [[ -n $3 ]] && gsutil cp $1 gs://$2/previous/$3/ || true } +# These are global environment variables. SKIP_TESTS=0 TAG_RELEASE=0 PUBLISH_RELEASE=0 +BRANCH_RELEASE=0 TAG="" -KO_FLAGS="-P -L" +RELEASE_VERSION="" +RELEASE_NOTES="" +RELEASE_BRANCH="" +KO_FLAGS="" + +function abort() { + echo "error: $@" + exit 1 +} # Parses flags and sets environment variables accordingly. function parse_flags() { + TAG="" + RELEASE_VERSION="" + RELEASE_NOTES="" + RELEASE_BRANCH="" + KO_FLAGS="-P" cd ${REPO_ROOT_DIR} - for parameter in $@; do + while [[ $# -ne 0 ]]; do + local parameter=$1 case $parameter in --skip-tests) SKIP_TESTS=1 ;; --tag-release) TAG_RELEASE=1 ;; --notag-release) TAG_RELEASE=0 ;; - --publish) - PUBLISH_RELEASE=1 - # Remove -L from ko flags - KO_FLAGS="${KO_FLAGS/-L}" - ;; - --nopublish) - PUBLISH_RELEASE=0 - # Add -L to ko flags - KO_FLAGS="-L ${KO_FLAGS}" + --publish) PUBLISH_RELEASE=1 ;; + --nopublish) PUBLISH_RELEASE=0 ;; + --version) shift + [[ $# -ge 1 ]] || abort "missing version after --version" + [[ $1 =~ ^[0-9]+\.[0-9]+\.[0-9]+$ ]] || abort "version format must be '[0-9].[0-9].[0-9]'" + RELEASE_VERSION=$1 ;; - *) - echo "error: unknown option ${parameter}" - exit 1 + --branch) + shift + [[ $# -ge 1 ]] || abort "missing branch after --commit" + [[ $1 =~ ^release-[0-9]+\.[0-9]+$ ]] || abort "branch name must be 'release-[0-9].[0-9]'" + RELEASE_BRANCH=$1 ;; + --release-notes) + shift + [[ $# -ge 1 ]] || abort "missing release notes file after --release-notes" + [[ ! -f "$1" ]] && abort "file $1 doesn't exist" + RELEASE_NOTES=$1 + ;; + *) abort "unknown option ${parameter}" ;; esac shift done - TAG="" + # Update KO_DOCKER_REPO and KO_FLAGS if we're not publishing. + if (( ! PUBLISH_RELEASE )); then + KO_DOCKER_REPO="ko.local" + KO_FLAGS="-L ${KO_FLAGS}" + fi + if (( TAG_RELEASE )); then - # Currently we're not considering the tags in refs/tags namespace. - commit=$(git describe --always --dirty) + # Get the commit, excluding any tags but keeping the "dirty" flag + local commit="$(git describe --always --dirty --match '^$')" + [[ -n "${commit}" ]] || abort "Error getting the current commit" # Like kubernetes, image tag is vYYYYMMDD-commit TAG="v$(date +%Y%m%d)-${commit}" fi + if [[ -n "${RELEASE_VERSION}" ]]; then + TAG="v${RELEASE_VERSION}" + fi + + [[ -n "${RELEASE_VERSION}" ]] && (( PUBLISH_RELEASE )) && BRANCH_RELEASE=1 + readonly SKIP_TESTS readonly TAG_RELEASE readonly PUBLISH_RELEASE + readonly BRANCH_RELEASE readonly TAG + readonly RELEASE_VERSION + readonly RELEASE_NOTES + readonly RELEASE_BRANCH } # Run tests (unless --skip-tests was passed). Conveniently displays a banner indicating so. @@ -113,6 +139,46 @@ function run_validation_tests() { if (( ! SKIP_TESTS )); then banner "Running release validation tests" # Run tests. - $1 + if ! $1; then + banner "Release validation tests failed, aborting" + exit 1 + fi fi } + +# Initialize everything (flags, workspace, etc) for a release. +function initialize() { + parse_flags $@ + # Checkout specific branch, if necessary + if (( BRANCH_RELEASE )); then + git checkout upstream/${RELEASE_BRANCH} || abort "cannot checkout branch ${RELEASE_BRANCH}" + fi +} + +# Create a new release on GitHub, also git tagging it (unless this is not a versioned release). +# Parameters: $1 - Module name (e.g., "Knative Serving"). +# $2 - YAML files to add to the release, space separated. +function branch_release() { + (( BRANCH_RELEASE )) || return 0 + local title="$1 release ${TAG}" + local attachments=() + local description="$(mktemp)" + local attachments_dir="$(mktemp -d)" + # Copy each YAML to a separate dir + for yaml in $2; do + cp ${yaml} ${attachments_dir}/ + attachments+=("--attach=${yaml}#$(basename ${yaml})") + done + echo -e "${title}\n" > ${description} + if [[ -n "${RELEASE_NOTES}" ]]; then + cat ${RELEASE_NOTES} >> ${description} + fi + git tag -a ${TAG} -m "${title}" + git push $(git remote get-url upstream) tag ${TAG} + run_go_tool github.com/github/hub hub release create \ + --prerelease \ + ${attachments[@]} \ + --file=${description} \ + --commitish=${RELEASE_BRANCH} \ + ${TAG} +}