mirror of https://github.com/knative/caching.git
Update test-infra: some fixes (#29)
* Update test-infra: some fixes * install istio from addon
This commit is contained in:
chaodaiG
Knative Prow Robot
parent
afa9ee174c
commit
d4b89dbc0b
|
|
@ -147,14 +147,14 @@
|
|||
|
||||
[[projects]]
|
||||
branch = "master"
|
||||
digest = "1:921c8888ac7bf7240bc5473d029201b3eb19f9592c7647c4c29f96b6c901b1bb"
|
||||
digest = "1:f27efa11fb1aec3e502e2ed95d9a1211a93f8a791b7da0a06fdade0d98de5008"
|
||||
name = "github.com/knative/test-infra"
|
||||
packages = [
|
||||
"scripts",
|
||||
"tools/dep-collector",
|
||||
]
|
||||
pruneopts = "UT"
|
||||
revision = "582b6fbee8ae00260baf916b0f9db2aa5f3483d9"
|
||||
revision = "75f6ca1c4dc3b3ae5dc1a1a433753957a9340e83"
|
||||
|
||||
[[projects]]
|
||||
digest = "1:2f42fa12d6911c7b7659738758631bec870b7e9b4c6be5444f963cdcfccc191f"
|
||||
|
|
|
|||
|
|
@ -108,8 +108,10 @@ This is a helper script for Knative E2E test scripts. To use it:
|
|||
if the default values don't fit your needs:
|
||||
|
||||
- `E2E_CLUSTER_REGION`: Cluster region, defaults to `us-central1`.
|
||||
- `E2E_CLUSTER_BACKUP_REGIONS`: Space-separated list of regions to retry test cluster creation in case of stockout. Defaults to `us-west1 us-east1`.
|
||||
- `E2E_CLUSTER_ZONE`: Cluster zone (e.g., `a`), defaults to none (i.e. use a regional
|
||||
cluster).
|
||||
- `E2E_CLUSTER_BACKUP_ZONES`: Space-separated list of zones to retry test cluster creation in case of stockout. If defined, `E2E_CLUSTER_BACKUP_REGIONS` will be ignored thus it defaults to none.
|
||||
- `E2E_CLUSTER_MACHINE`: Cluster node machine type, defaults to `n1-standard-4}`.
|
||||
- `E2E_MIN_CLUSTER_NODES`: Minimum number of nodes in the cluster when autoscaling,
|
||||
defaults to 1.
|
||||
|
|
@ -153,8 +155,9 @@ This is a helper script for Knative E2E test scripts. To use it:
|
|||
1. Write logic for the end-to-end tests. Run all go tests using `go_test_e2e()`
|
||||
(or `report_go_test()` if you need a more fine-grained control) and call
|
||||
`fail_test()` or `success()` if any of them failed. The environment variable
|
||||
`KO_DOCKER_REPO` will be set according to the test cluster. You can also use
|
||||
the following boolean (0 is false, 1 is true) environment variables for the logic:
|
||||
`KO_DOCKER_REPO` and `E2E_PROJECT_ID` will be set according to the test cluster.
|
||||
You can also use the following boolean (0 is false, 1 is true) environment
|
||||
variables for the logic:
|
||||
|
||||
- `EMIT_METRICS`: true if `--emit-metrics` was passed.
|
||||
|
||||
|
|
@ -169,6 +172,9 @@ This is a helper script for Knative E2E test scripts. To use it:
|
|||
will immediately start the tests against the cluster currently configured for
|
||||
`kubectl`.
|
||||
|
||||
1. By default Istio is installed on the cluster via Addon, using `--skip-istio` if
|
||||
you choose not to have it preinstalled.
|
||||
|
||||
1. You can force running the tests against a specific GKE cluster version by using
|
||||
the `--cluster-version` flag and passing a full version as the flag value.
|
||||
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
#!/bin/bash
|
||||
|
||||
# Copyright 2018 The Knative Authors
|
||||
# Copyright 2019 The Knative Authors
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
|
|
@ -38,9 +38,15 @@ function build_resource_name() {
|
|||
# Test cluster parameters
|
||||
|
||||
# Configurable parameters
|
||||
readonly E2E_CLUSTER_REGION=${E2E_CLUSTER_REGION:-us-central1}
|
||||
# export E2E_CLUSTER_REGION and E2E_CLUSTER_ZONE as they're used in the cluster setup subprocess
|
||||
export E2E_CLUSTER_REGION=${E2E_CLUSTER_REGION:-us-central1}
|
||||
# By default we use regional clusters.
|
||||
readonly E2E_CLUSTER_ZONE=${E2E_CLUSTER_ZONE:-}
|
||||
export E2E_CLUSTER_ZONE=${E2E_CLUSTER_ZONE:-}
|
||||
|
||||
# Default backup regions in case of stockouts; by default we don't fall back to a different zone in the same region
|
||||
readonly E2E_CLUSTER_BACKUP_REGIONS=${E2E_CLUSTER_BACKUP_REGIONS:-us-west1 us-east1}
|
||||
readonly E2E_CLUSTER_BACKUP_ZONES=${E2E_CLUSTER_BACKUP_ZONES:-}
|
||||
|
||||
readonly E2E_CLUSTER_MACHINE=${E2E_CLUSTER_MACHINE:-n1-standard-4}
|
||||
readonly E2E_GKE_ENVIRONMENT=${E2E_GKE_ENVIRONMENT:-prod}
|
||||
readonly E2E_GKE_COMMAND_GROUP=${E2E_GKE_COMMAND_GROUP:-beta}
|
||||
|
|
@ -61,6 +67,8 @@ IS_BOSKOS=0
|
|||
|
||||
# Tear down the test resources.
|
||||
function teardown_test_resources() {
|
||||
# On boskos, save time and don't teardown as the cluster will be destroyed anyway.
|
||||
(( IS_BOSKOS )) && return
|
||||
header "Tearing down test environment"
|
||||
function_exists test_teardown && test_teardown
|
||||
(( ! SKIP_KNATIVE_SETUP )) && function_exists knative_teardown && knative_teardown
|
||||
|
|
@ -107,8 +115,7 @@ function save_metadata() {
|
|||
geo_key="Zone"
|
||||
geo_value="${E2E_CLUSTER_REGION}-${E2E_CLUSTER_ZONE}"
|
||||
fi
|
||||
local gcloud_project="$(gcloud config get-value project)"
|
||||
local cluster_version="$(gcloud container clusters list --project=${gcloud_project} --format='value(currentMasterVersion)')"
|
||||
local cluster_version="$(gcloud container clusters list --project=${E2E_PROJECT_ID} --format='value(currentMasterVersion)')"
|
||||
cat << EOF > ${ARTIFACTS}/metadata.json
|
||||
{
|
||||
"E2E:${geo_key}": "${geo_value}",
|
||||
|
|
@ -124,6 +131,7 @@ EOF
|
|||
# See https://github.com/knative/serving/issues/959 for details.
|
||||
# TODO(adrcunha): Remove once the leak issue is resolved.
|
||||
function delete_leaked_network_resources() {
|
||||
# On boskos, don't bother with leaks as the janitor will delete everything in the project.
|
||||
(( IS_BOSKOS )) && return
|
||||
# Ensure we're using the GCP project used by kubetest
|
||||
local gcloud_project="$(gcloud config get-value project)"
|
||||
|
|
@ -154,20 +162,15 @@ function create_test_cluster() {
|
|||
cluster_setup || fail_test "cluster setup failed"
|
||||
fi
|
||||
|
||||
header "Creating test cluster"
|
||||
|
||||
echo "Cluster will have a minimum of ${E2E_MIN_CLUSTER_NODES} and a maximum of ${E2E_MAX_CLUSTER_NODES} nodes."
|
||||
|
||||
# Smallest cluster required to run the end-to-end-tests
|
||||
local geoflag="--gcp-region=${E2E_CLUSTER_REGION}"
|
||||
[[ -n "${E2E_CLUSTER_ZONE}" ]] && geoflag="--gcp-zone=${E2E_CLUSTER_REGION}-${E2E_CLUSTER_ZONE}"
|
||||
local CLUSTER_CREATION_ARGS=(
|
||||
--gke-create-command="container clusters create --quiet --enable-autoscaling --min-nodes=${E2E_MIN_CLUSTER_NODES} --max-nodes=${E2E_MAX_CLUSTER_NODES} --scopes=cloud-platform --enable-basic-auth --no-issue-client-certificate ${EXTRA_CLUSTER_CREATION_FLAGS[@]}"
|
||||
--gke-create-command="container clusters create --quiet --enable-autoscaling --min-nodes=${E2E_MIN_CLUSTER_NODES} --max-nodes=${E2E_MAX_CLUSTER_NODES} --scopes=cloud-platform --enable-basic-auth --no-issue-client-certificate ${GKE_ADDONS} ${EXTRA_CLUSTER_CREATION_FLAGS[@]}"
|
||||
--gke-shape={\"default\":{\"Nodes\":${E2E_MIN_CLUSTER_NODES}\,\"MachineType\":\"${E2E_CLUSTER_MACHINE}\"}}
|
||||
--provider=gke
|
||||
--deployment=gke
|
||||
--cluster="${E2E_CLUSTER_NAME}"
|
||||
${geoflag}
|
||||
--gcp-network="${E2E_NETWORK_NAME}"
|
||||
--gke-environment="${E2E_GKE_ENVIRONMENT}"
|
||||
--gke-command-group="${E2E_GKE_COMMAND_GROUP}"
|
||||
|
|
@ -198,11 +201,7 @@ function create_test_cluster() {
|
|||
local extra_flags=()
|
||||
# If using boskos, save time and let it tear down the cluster
|
||||
(( ! IS_BOSKOS )) && extra_flags+=(--down)
|
||||
# Don't fail test for kubetest, as it might incorrectly report test failure
|
||||
# if teardown fails (for details, see success() below)
|
||||
set +o errexit
|
||||
run_go_tool k8s.io/test-infra/kubetest \
|
||||
kubetest "${CLUSTER_CREATION_ARGS[@]}" \
|
||||
create_test_cluster_with_retries "${CLUSTER_CREATION_ARGS[@]}" \
|
||||
--up \
|
||||
--extract "${E2E_CLUSTER_VERSION}" \
|
||||
--gcp-node-image "${SERVING_GKE_IMAGE}" \
|
||||
|
|
@ -215,12 +214,55 @@ function create_test_cluster() {
|
|||
set +o errexit
|
||||
function_exists cluster_teardown && cluster_teardown
|
||||
delete_leaked_network_resources
|
||||
local result="$(cat ${TEST_RESULT_FILE})"
|
||||
local result=$(get_test_return_code)
|
||||
echo "Artifacts were written to ${ARTIFACTS}"
|
||||
echo "Test result code is ${result}"
|
||||
exit ${result}
|
||||
}
|
||||
|
||||
# Retry backup regions/zones if cluster creations failed due to stockout.
|
||||
# Parameters: $1..$n - any kubetest flags other than geo flag.
|
||||
function create_test_cluster_with_retries() {
|
||||
local cluster_creation_log=/tmp/${E2E_BASE_NAME}-cluster_creation-log
|
||||
# zone_not_provided is a placeholder for e2e_cluster_zone to make for loop below work
|
||||
local zone_not_provided="zone_not_provided"
|
||||
|
||||
local e2e_cluster_regions=(${E2E_CLUSTER_REGION})
|
||||
local e2e_cluster_zones=(${E2E_CLUSTER_ZONE})
|
||||
|
||||
if [[ -n "${E2E_CLUSTER_BACKUP_ZONES}" ]]; then
|
||||
e2e_cluster_zones+=(${E2E_CLUSTER_BACKUP_ZONES})
|
||||
elif [[ -n "${E2E_CLUSTER_BACKUP_REGIONS}" ]]; then
|
||||
e2e_cluster_regions+=(${E2E_CLUSTER_BACKUP_REGIONS})
|
||||
e2e_cluster_zones=(${zone_not_provided})
|
||||
else
|
||||
echo "No backup region/zone set, cluster creation will fail in case of stockout"
|
||||
fi
|
||||
|
||||
for e2e_cluster_region in "${e2e_cluster_regions[@]}"; do
|
||||
for e2e_cluster_zone in "${e2e_cluster_zones[@]}"; do
|
||||
E2E_CLUSTER_REGION=${e2e_cluster_region}
|
||||
E2E_CLUSTER_ZONE=${e2e_cluster_zone}
|
||||
[[ "${E2E_CLUSTER_ZONE}" == "${zone_not_provided}" ]] && E2E_CLUSTER_ZONE=""
|
||||
|
||||
local geoflag="--gcp-region=${E2E_CLUSTER_REGION}"
|
||||
[[ -n "${E2E_CLUSTER_ZONE}" ]] && geoflag="--gcp-zone=${E2E_CLUSTER_REGION}-${E2E_CLUSTER_ZONE}"
|
||||
|
||||
header "Creating test cluster in $E2E_CLUSTER_REGION $E2E_CLUSTER_ZONE"
|
||||
# Don't fail test for kubetest, as it might incorrectly report test failure
|
||||
# if teardown fails (for details, see success() below)
|
||||
set +o errexit
|
||||
{ run_go_tool k8s.io/test-infra/kubetest \
|
||||
kubetest "$@" ${geoflag}; } 2>&1 | tee ${cluster_creation_log}
|
||||
|
||||
# Exit if test succeeded
|
||||
[[ "$(get_test_return_code)" == "0" ]] && return
|
||||
# If test failed not because of cluster creation stockout, return
|
||||
[[ -z "$(grep -Eio 'does not have enough resources available to fulfill the request' ${cluster_creation_log})" ]] && return
|
||||
done
|
||||
done
|
||||
}
|
||||
|
||||
# Setup the test cluster for running the tests.
|
||||
function setup_test_cluster() {
|
||||
# Fail fast during setup.
|
||||
|
|
@ -229,6 +271,11 @@ function setup_test_cluster() {
|
|||
|
||||
header "Setting up test cluster"
|
||||
|
||||
# Set the actual project the test cluster resides in
|
||||
# It will be a project assigned by Boskos if test is running on Prow,
|
||||
# otherwise will be ${GCP_PROJECT} set up by user.
|
||||
readonly export E2E_PROJECT_ID="$(gcloud config get-value project)"
|
||||
|
||||
# Save some metadata about cluster creation for using in prow and testgrid
|
||||
save_metadata
|
||||
|
||||
|
|
@ -240,9 +287,10 @@ function setup_test_cluster() {
|
|||
if [[ -z "$(kubectl get clusterrolebinding cluster-admin-binding 2> /dev/null)" ]]; then
|
||||
acquire_cluster_admin_role ${k8s_user} ${E2E_CLUSTER_NAME} ${E2E_CLUSTER_REGION} ${E2E_CLUSTER_ZONE}
|
||||
kubectl config set-context ${k8s_cluster} --namespace=default
|
||||
export KO_DOCKER_REPO=gcr.io/$(gcloud config get-value project)/${E2E_BASE_NAME}-e2e-img
|
||||
export KO_DOCKER_REPO=gcr.io/${E2E_PROJECT_ID}/${E2E_BASE_NAME}-e2e-img
|
||||
fi
|
||||
|
||||
echo "- Project is ${E2E_PROJECT_ID}"
|
||||
echo "- Cluster is ${k8s_cluster}"
|
||||
echo "- User is ${k8s_user}"
|
||||
echo "- Docker is ${KO_DOCKER_REPO}"
|
||||
|
|
@ -263,6 +311,12 @@ function setup_test_cluster() {
|
|||
fi
|
||||
}
|
||||
|
||||
# Gets the exit of the test script.
|
||||
# For more details, see set_test_return_code().
|
||||
function get_test_return_code() {
|
||||
echo $(cat ${TEST_RESULT_FILE})
|
||||
}
|
||||
|
||||
# Set the return code that the test script will return.
|
||||
# Parameters: $1 - return code (0-255)
|
||||
function set_test_return_code() {
|
||||
|
|
@ -295,9 +349,11 @@ function fail_test() {
|
|||
RUN_TESTS=0
|
||||
EMIT_METRICS=0
|
||||
SKIP_KNATIVE_SETUP=0
|
||||
SKIP_ISTIO=0
|
||||
GCP_PROJECT=""
|
||||
E2E_SCRIPT=""
|
||||
E2E_CLUSTER_VERSION=""
|
||||
GKE_ADDONS=""
|
||||
EXTRA_CLUSTER_CREATION_FLAGS=()
|
||||
EXTRA_KUBETEST_FLAGS=()
|
||||
E2E_SCRIPT_CUSTOM_FLAGS=()
|
||||
|
|
@ -329,6 +385,7 @@ function initialize() {
|
|||
--run-tests) RUN_TESTS=1 ;;
|
||||
--emit-metrics) EMIT_METRICS=1 ;;
|
||||
--skip-knative-setup) SKIP_KNATIVE_SETUP=1 ;;
|
||||
--skip-istio) SKIP_ISTIO=1 ;;
|
||||
*)
|
||||
[[ $# -ge 2 ]] || abort "missing parameter after $1"
|
||||
shift
|
||||
|
|
@ -358,6 +415,8 @@ function initialize() {
|
|||
is_protected_gcr ${KO_DOCKER_REPO} && \
|
||||
abort "\$KO_DOCKER_REPO set to ${KO_DOCKER_REPO}, which is forbidden"
|
||||
|
||||
(( SKIP_ISTIO )) || GKE_ADDONS="--addons=Istio"
|
||||
|
||||
readonly RUN_TESTS
|
||||
readonly EMIT_METRICS
|
||||
readonly GCP_PROJECT
|
||||
|
|
@ -365,6 +424,7 @@ function initialize() {
|
|||
readonly EXTRA_CLUSTER_CREATION_FLAGS
|
||||
readonly EXTRA_KUBETEST_FLAGS
|
||||
readonly SKIP_KNATIVE_SETUP
|
||||
readonly GKE_ADDONS
|
||||
|
||||
if (( ! RUN_TESTS )); then
|
||||
create_test_cluster
|
||||
|
|
|
|||
|
|
@ -146,22 +146,23 @@ function wait_until_pods_running() {
|
|||
return 1
|
||||
}
|
||||
|
||||
# Waits until all batch job pods are running in the given namespace.
|
||||
# Waits until all batch jobs complete in the given namespace.
|
||||
# Parameters: $1 - namespace.
|
||||
function wait_until_batch_job_complete() {
|
||||
echo -n "Waiting until all batch job pods in namespace $1 run to completion."
|
||||
echo -n "Waiting until all batch jobs in namespace $1 run to completion."
|
||||
for i in {1..150}; do # timeout after 5 minutes
|
||||
local pods="$(kubectl get pods --selector=job-name --no-headers -n $1 2>/dev/null | grep -v '^[[:space:]]*$')"
|
||||
# All pods must be complete
|
||||
local not_complete=$(echo "${pods}" | grep -v Completed | wc -l)
|
||||
local jobs=$(kubectl get jobs -n $1 --no-headers \
|
||||
-ocustom-columns='n:{.metadata.name},c:{.spec.completions},s:{.status.succeeded}')
|
||||
# All jobs must be complete
|
||||
local not_complete=$(echo "${jobs}" | awk '{if ($2!=$3) print $0}' | wc -l)
|
||||
if [[ ${not_complete} -eq 0 ]]; then
|
||||
echo -e "\nAll pods are complete:\n${pods}"
|
||||
echo -e "\nAll jobs are complete:\n${jobs}"
|
||||
return 0
|
||||
fi
|
||||
echo -n "."
|
||||
sleep 2
|
||||
done
|
||||
echo -e "\n\nERROR: timeout waiting for pods to complete\n${pods}"
|
||||
echo -e "\n\nERROR: timeout waiting for jobs to complete\n${jobs}"
|
||||
return 1
|
||||
}
|
||||
|
||||
|
|
@ -320,14 +321,6 @@ function report_go_test() {
|
|||
# Install the latest stable Knative/serving in the current cluster.
|
||||
function start_latest_knative_serving() {
|
||||
header "Starting Knative Serving"
|
||||
subheader "Installing Istio"
|
||||
echo "Running Istio CRD from ${KNATIVE_ISTIO_CRD_YAML}"
|
||||
kubectl apply -f ${KNATIVE_ISTIO_CRD_YAML} || return 1
|
||||
wait_until_batch_job_complete istio-system || return 1
|
||||
echo "Installing Istio from ${KNATIVE_ISTIO_YAML}"
|
||||
kubectl apply -f ${KNATIVE_ISTIO_YAML} || return 1
|
||||
wait_until_pods_running istio-system || return 1
|
||||
kubectl label namespace default istio-injection=enabled || return 1
|
||||
subheader "Installing Knative Serving"
|
||||
echo "Installing Serving from ${KNATIVE_SERVING_RELEASE}"
|
||||
kubectl apply -f ${KNATIVE_SERVING_RELEASE} || return 1
|
||||
|
|
|
|||
|
|
@ -43,7 +43,7 @@ IS_DOCUMENTATION_PR=0
|
|||
# Returns true if PR only contains the given file regexes.
|
||||
# Parameters: $1 - file regexes, space separated.
|
||||
function pr_only_contains() {
|
||||
[[ -z "$(echo "${CHANGED_FILES}" | grep -v \(${1// /\\|}\)$))" ]]
|
||||
[[ -z "$(echo "${CHANGED_FILES}" | grep -v "\(${1// /\\|}\)$")" ]]
|
||||
}
|
||||
|
||||
# List changed files in the current PR.
|
||||
|
|
|
|||
Loading…
Reference in New Issue