[master] Auto-update dependencies (#327)

Produced via:
  `./hack/update-deps.sh --upgrade && ./hack/update-codegen.sh`
/assign n3wscott vagababov
/cc n3wscott vagababov
This commit is contained in:
Matt Moore 2020-09-10 07:26:51 -07:00 committed by GitHub
parent 1c094a1d6e
commit bb94d39d0c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 68 additions and 216 deletions

4
go.mod
View File

@ -16,8 +16,8 @@ require (
k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible
k8s.io/code-generator v0.18.6
k8s.io/kube-openapi v0.0.0-20200410145947-bcb3869e6f29
knative.dev/pkg v0.0.0-20200908235250-56fba14ba7df
knative.dev/test-infra v0.0.0-20200908182932-5a8105609141
knative.dev/pkg v0.0.0-20200910010051-a79a813ce123
knative.dev/test-infra v0.0.0-20200909211651-72eb6ae3c773
)
replace (

8
go.sum
View File

@ -1868,8 +1868,8 @@ knative.dev/pkg v0.0.0-20200505191044-3da93ebb24c2/go.mod h1:Q6sL35DdGs8hIQZKdaC
knative.dev/pkg v0.0.0-20200515002500-16d7b963416f/go.mod h1:tMOHGbxtRz8zYFGEGpV/bpoTEM1o89MwYFC4YJXl3GY=
knative.dev/pkg v0.0.0-20200528142800-1c6815d7e4c9/go.mod h1:QgNZTxnwpB/oSpNcfnLVlw+WpEwwyKAvJlvR3hgeltA=
knative.dev/pkg v0.0.0-20200711004937-22502028e31a/go.mod h1:AqAJV6rYi8IGikDjJ/9ZQd9qKdkXVlesVnVjwx62YB8=
knative.dev/pkg v0.0.0-20200908235250-56fba14ba7df h1:5J1YYhiU5g8a2pdnPFiRy+vtPsUHbdoLflJe1RqZnHk=
knative.dev/pkg v0.0.0-20200908235250-56fba14ba7df/go.mod h1:q+4+Cm768P6vsAvsD9J+cZ1hoy4aHyHSfRTvaFyPd3g=
knative.dev/pkg v0.0.0-20200910010051-a79a813ce123 h1:vmsCAlZi/R7FMkgDdb5pMK27wDHFCIwqVR9XYbDOS80=
knative.dev/pkg v0.0.0-20200910010051-a79a813ce123/go.mod h1:q+4+Cm768P6vsAvsD9J+cZ1hoy4aHyHSfRTvaFyPd3g=
knative.dev/test-infra v0.0.0-20200407185800-1b88cb3b45a5/go.mod h1:xcdUkMJrLlBswIZqL5zCuBFOC22WIPMQoVX1L35i0vQ=
knative.dev/test-infra v0.0.0-20200505052144-5ea2f705bb55/go.mod h1:WqF1Azka+FxPZ20keR2zCNtiQA1MP9ZB4BH4HuI+SIU=
knative.dev/test-infra v0.0.0-20200513011557-d03429a76034/go.mod h1:aMif0KXL4g19YCYwsy4Ocjjz5xgPlseYV+B95Oo4JGE=
@ -1877,8 +1877,8 @@ knative.dev/test-infra v0.0.0-20200519015156-82551620b0a9/go.mod h1:A5b2OAXTOeHT
knative.dev/test-infra v0.0.0-20200707183444-aed09e56ddc7/go.mod h1:RjYAhXnZqeHw9+B0zsbqSPlae0lCvjekO/nw5ZMpLCs=
knative.dev/test-infra v0.0.0-20200828211307-9d4372c9b1c7 h1:yIzao6i9Hu51SCdpyqBEnhpt7G12FqFrxHK6ZnQnu8o=
knative.dev/test-infra v0.0.0-20200828211307-9d4372c9b1c7/go.mod h1:Pmg2c7Z7q7BGFUV/GOpU5BlrD3ePJft4MPqx8AYBplc=
knative.dev/test-infra v0.0.0-20200908182932-5a8105609141 h1:bgtLzFLtOYETW03Dm4i9TLZ7m9ycYNTAGZixsZdcn7Q=
knative.dev/test-infra v0.0.0-20200908182932-5a8105609141/go.mod h1:Pmg2c7Z7q7BGFUV/GOpU5BlrD3ePJft4MPqx8AYBplc=
knative.dev/test-infra v0.0.0-20200909211651-72eb6ae3c773 h1:R/kYdvSoein2a6BwY1FSZH8wk0aHQ9gN3GQf94AGqcs=
knative.dev/test-infra v0.0.0-20200909211651-72eb6ae3c773/go.mod h1:Pmg2c7Z7q7BGFUV/GOpU5BlrD3ePJft4MPqx8AYBplc=
modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw=
modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk=
modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k=

View File

@ -16,16 +16,51 @@ limitations under the License.
package reconciler
import (
"strings"
apierrs "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/client-go/util/retry"
)
// RetryUpdateConflicts retries the inner function if it returns conflict errors.
// This can be used to retry status updates without constantly reenqueuing keys.
func RetryUpdateConflicts(updater func(int) error) error {
return RetryErrors(updater, apierrs.IsConflict)
}
// RetryErrors retries the inner function if it returns matching errors.
func RetryErrors(updater func(int) error, fns ...func(error) bool) error {
attempts := 0
return retry.RetryOnConflict(retry.DefaultRetry, func() error {
return retry.OnError(retry.DefaultRetry, func(err error) bool {
for _, fn := range fns {
if fn(err) {
return true
}
}
return false
}, func() error {
err := updater(attempts)
attempts++
return err
})
}
// RetryTestErrors retries the inner function if it hits an error type that is
// common in our test environments.
func RetryTestErrors(updater func(int) error) error {
return RetryErrors(updater,
// Example: conflicts updating `gke-resource-quotas` (implicit on Service/Pod/Ingress creations)
apierrs.IsConflict,
// Example: https://github.com/knative/test-infra/issues/2346#issuecomment-687220045
func(err error) bool {
return strings.Contains(err.Error(), "gke-resource-quotas")
},
// Example: `etcdserver: request timed out`
// TODO(mattmoor): Does apierrs.IsServerTimeout catch the above?
func(err error) bool {
return strings.Contains(err.Error(), "etcdserver")
},
)
}

View File

@ -110,28 +110,8 @@ main "$@"
This is a helper script for Knative E2E test scripts. To use it:
1. [optional] Customize the test cluster. Set the following environment
variables if the default values don't fit your needs:
- `E2E_GCP_PROJECT_ID`: GCP project ID for creating the clusters, defaults to
none.
- `E2E_GKE_CLUSTER_REGION`: Cluster region, defaults to `us-central1`.
- `E2E_GKE_CLUSTER_BACKUP_REGIONS`: Space-separated list of regions to retry
test cluster creation in case of stockout. Defaults to `us-west1 us-east1`.
- `E2E_GKE_CLUSTER_ZONE`: Cluster zone (e.g., `a`), defaults to none (i.e.
use a regional cluster).
- `E2E_GKE_CLUSTER_BACKUP_ZONES`: Space-separated list of zones to retry test
cluster creation in case of stockout. If defined,
`E2E_GKE_CLUSTER_BACKUP_REGIONS` will be ignored thus it defaults to none.
- `E2E_GKE_CLUSTER_MACHINE`: Cluster node machine type, defaults to
`e2-standard-4}`.
- `E2E_MIN_CLUSTER_NODES`: Minimum number of nodes in the cluster when
autoscaling, defaults to 1.
- `E2E_MAX_CLUSTER_NODES`: Maximum number of nodes in the cluster when
autoscaling, defaults to 3.
- `E2E_GKE_SCOPES`: Scopes for the GKE node instances, defaults to
`cloud-platform`.
- `E2E_CLUSTER_VERSION`: Version for the cluster, defaults to `latest`.
1. [optional] Customize the test cluster. Pass the flags as described [here](../kntest/pkg/kubetest2/gke/README.md)
to the `initialize` function call if the default values don't fit your needs.
1. Source the script.
@ -163,7 +143,7 @@ This is a helper script for Knative E2E test scripts. To use it:
of items to skip in the command line if the flag was parsed successfully. For
example, return 1 for a simple flag, and 2 for a flag with a parameter.
1. Call the `initialize()` function passing `$@` (without quotes).
1. Call the `initialize()` function passing `"$@"`.
1. Write logic for the end-to-end tests. Run all go tests using `go_test_e2e()`
(or `report_go_test()` if you need a more fine-grained control) and call
@ -173,8 +153,8 @@ This is a helper script for Knative E2E test scripts. To use it:
**Notes:**
1. Calling your script without arguments will create a new cluster in the GCP
project `$E2E_GCP_PROJECT_ID` and run the tests against it.
1. Calling your script without arguments will create a new cluster in your current
GCP project and run the tests against it.
1. Calling your script with `--run-tests` and the variable `KO_DOCKER_REPO` set
will immediately start the tests against the cluster currently configured for
@ -195,9 +175,6 @@ test cluster is created in a specific region, `us-west2`.
```bash
# This test requires a cluster in LA
E2E_CLUSTER_REGION=us-west2
source vendor/knative.dev/test-infra/scripts/e2e-tests.sh
function knative_setup() {
@ -217,7 +194,8 @@ function parse_flags() {
WAIT_FOR_KNATIVE=1
initialize $@
# This test requires a cluster in LA
initialize $@ --region=us-west2
# TODO: use go_test_e2e to run the tests.
kubectl get pods || fail_test

View File

@ -128,14 +128,14 @@ CLOUD_PROVIDER="gke"
# Parse flags and initialize the test cluster.
function initialize() {
local run_tests=0
local extra_kubetest2_flags=()
local extra_cluster_creation_flags=()
local custom_flags=()
E2E_SCRIPT="$(get_canonical_path "$0")"
local e2e_script_command=( "${E2E_SCRIPT}" "--run-tests" )
cd "${REPO_ROOT_DIR}"
while [[ $# -ne 0 ]]; do
local parameter=$1
# TODO(chizhg): remove parse_flags logic if no repos are using it.
# Try parsing flag as a custom one.
if function_exists parse_flags; then
parse_flags "$@"
@ -157,13 +157,9 @@ function initialize() {
# TODO(chizhg): remove this flag once the addons is defined as an env var.
--skip-istio-addon) SKIP_ISTIO_ADDON=1 ;;
*)
[[ $# -ge 2 ]] || abort "missing parameter after $1"
shift
case ${parameter} in
--cloud-provider) CLOUD_PROVIDER="$1" ;;
--kubetest2-flag) extra_kubetest2_flags+=("$1") ;;
--cluster-creation-flag) extra_cluster_creation_flags+=("$1") ;;
*) abort "unknown option ${parameter}" ;;
--cloud-provider) shift; CLOUD_PROVIDER="$1" ;;
*) custom_flags+=("$parameter") ;;
esac
esac
shift
@ -172,16 +168,16 @@ function initialize() {
(( IS_PROW )) && [[ -z "${GCP_PROJECT_ID:-}" ]] && IS_BOSKOS=1
if (( SKIP_ISTIO_ADDON )); then
extra_cluster_creation_flags+=("--addons=NodeLocalDNS")
custom_flags+=("--addons=NodeLocalDNS")
else
extra_cluster_creation_flags+=("--addons=Istio,NodeLocalDNS")
custom_flags+=("--addons=Istio,NodeLocalDNS")
fi
readonly IS_BOSKOS
readonly SKIP_TEARDOWNS
if (( ! run_tests )); then
create_test_cluster "${CLOUD_PROVIDER}" extra_kubetest2_flags extra_cluster_creation_flags e2e_script_command
create_test_cluster "${CLOUD_PROVIDER}" custom_flags e2e_script_command
else
setup_test_cluster
fi

View File

@ -19,34 +19,6 @@
source $(dirname "${BASH_SOURCE[0]}")/library.sh
# Test cluster parameters
# Configurable parameters
export E2E_GCP_PROJECT_ID=${E2E_GCP_PROJECT_ID:-}
# export E2E_GKE_CLUSTER_REGION and E2E_GKE_CLUSTER_ZONE as they're used in the cluster setup subprocess
export E2E_GKE_CLUSTER_REGION=${E2E_GKE_CLUSTER_REGION:-us-central1}
# By default we use regional clusters.
export E2E_GKE_CLUSTER_ZONE=${E2E_GKE_CLUSTER_ZONE:-}
# Default backup regions in case of stockouts; by default we don't fall back to a different zone in the same region
readonly E2E_GKE_CLUSTER_BACKUP_REGIONS=${E2E_GKE_CLUSTER_BACKUP_REGIONS:-us-west1 us-east1}
readonly E2E_GKE_CLUSTER_BACKUP_ZONES=${E2E_GKE_CLUSTER_BACKUP_ZONES:-}
readonly E2E_GKE_ENVIRONMENT=${E2E_GKE_ENVIRONMENT:-prod}
readonly E2E_GKE_COMMAND_GROUP=${E2E_GKE_COMMAND_GROUP:-beta}
readonly E2E_GKE_CLUSTER_MACHINE=${E2E_GKE_CLUSTER_MACHINE:-e2-standard-4}
readonly E2E_GKE_SCOPES=${E2E_GKE_SCOPES:-cloud-platform}
# Each knative repository may have a different cluster size requirement here,
# so we allow calling code to set these parameters. If they are not set we
# use some sane defaults.
readonly E2E_MIN_CLUSTER_NODES=${E2E_MIN_CLUSTER_NODES:-1}
readonly E2E_MAX_CLUSTER_NODES=${E2E_MAX_CLUSTER_NODES:-3}
readonly E2E_CLUSTER_VERSION=${E2E_CLUSTER_VERSION:-latest}
readonly E2E_CLUSTER_NAME=$(build_resource_name e2e-cls)
readonly E2E_GKE_NETWORK_NAME=$(build_resource_name e2e-net)
# Dumps the k8s api server metrics. Spins up a proxy, waits a little bit and
# dumps the metrics to ${ARTIFACTS}/k8s.metrics.txt
function dump_metrics() {
@ -105,26 +77,6 @@ function dump_cluster_state() {
echo "***************************************"
}
# On a Prow job, save some metadata about the test for Testgrid.
# Parameters: $1 - cluster provider name
function save_metadata() {
(( ! IS_PROW )) && return
local geo_key="Region"
local geo_value="${E2E_GKE_CLUSTER_REGION}"
if [[ -n "${E2E_GKE_CLUSTER_ZONE}" ]]; then
geo_key="Zone"
geo_value="${E2E_GKE_CLUSTER_REGION}-${E2E_GKE_CLUSTER_ZONE}"
fi
local cluster_version
cluster_version="$(kubectl version --short=true)"
run_kntest metadata set --key="E2E:Provider" --value="$1"
run_kntest metadata set --key="E2E:${geo_key}" --value="${geo_value}"
run_kntest metadata set --key="E2E:Machine" --value="${E2E_GKE_CLUSTER_MACHINE}"
run_kntest metadata set --key="E2E:Version" --value="${cluster_version}"
run_kntest metadata set --key="E2E:MinNodes" --value="${E2E_MIN_CLUSTER_NODES}"
run_kntest metadata set --key="E2E:MaxNodes" --value="${E2E_MAX_CLUSTER_NODES}"
}
# Sets the current user as cluster admin for the given cluster.
# Parameters: $1 - cluster context name
function acquire_cluster_admin_role() {
@ -141,9 +93,8 @@ function acquire_cluster_admin_role() {
# Create a test cluster and run the tests if provided.
# Parameters: $1 - cluster provider name, e.g. gke
# $2 - extra kubetest2 flags
# $3 - extra cluster creation flags
# $4 - test command to run by the kubetest2 tester
# $2 - custom flags supported by kntest
# $3 - test command to run after cluster is created
function create_test_cluster() {
# Fail fast during setup.
set -o errexit
@ -170,111 +121,19 @@ function create_test_cluster() {
}
# Create a KIND test cluster with kubetest2 and run the test command.
# Parameters: $1 - extra kubetest2 flags
# $2 - extra cluster creation flags
# $3 - test command to run by the kubetest2 tester
# Parameters: $1 - extra cluster creation flags
# $2 - test command to run by the kubetest2 tester
function create_kind_test_cluster() {
# TODO(chizhg): implement with kubetest2
return 0
}
# Create a GKE test cluster with kubetest2 and run the test command.
# Parameters: $1 - extra kubetest2 flags
# $2 - extra cluster creation flags
# $3 - test command to run by the kubetest2 tester after the cluster is created (optional)
# Parameters: $1 - custom flags defined in kntest
# $2 - test command to run after the cluster is created (optional)
function create_gke_test_cluster() {
local -n _extra_kubetest2_flags=$1
local -n _extra_cluster_creation_flags=$2
local -n _custom_flags=$1
local -n _test_command=$2
echo "Cluster will have a minimum of ${E2E_MIN_CLUSTER_NODES} and a maximum of ${E2E_MAX_CLUSTER_NODES} nodes."
local _kubetest2_flags=(
"gke"
"--create-command=${E2E_GKE_COMMAND_GROUP} container clusters create --quiet --enable-autoscaling
--min-nodes=${E2E_MIN_CLUSTER_NODES} --max-nodes=${E2E_MAX_CLUSTER_NODES}
--cluster-version=${E2E_CLUSTER_VERSION}
--scopes=${E2E_GKE_SCOPES} --no-enable-autoupgrade
${_extra_cluster_creation_flags[@]}"
"--environment=${E2E_GKE_ENVIRONMENT}"
"--cluster-name=${E2E_CLUSTER_NAME}"
"--num-nodes=${E2E_MIN_CLUSTER_NODES}"
"--machine-type=${E2E_GKE_CLUSTER_MACHINE}"
"--network=${E2E_GKE_NETWORK_NAME}"
"--ignore-gcp-ssh-key=true"
--up
)
_kubetest2_flags+=( "${_extra_kubetest2_flags[@]}" )
if (( ! IS_BOSKOS )); then
local gcloud_project="${E2E_GCP_PROJECT_ID}"
[[ -z "${gcloud_project}" ]] && gcloud_project="$(gcloud config get-value project)"
_kubetest2_flags+=("--project=${gcloud_project}")
echo "gcloud project is ${gcloud_project}"
else
echo "Using boskos for provisioning the GCP project to create the test cluster"
fi
if (( IS_BOSKOS )); then
# Add arbitrary duration, wait for Boskos projects acquisition before error out
_kubetest2_flags+=("--boskos-acquire-timeout-seconds=1200")
elif (( ! SKIP_TEARDOWNS )); then
# Only let kubetest2 tear down the cluster if not using Boskos and teardowns are not expected to be skipped,
# it's done by Janitor if using Boskos
_kubetest2_flags+=("--down")
fi
# Create cluster and run the tests
create_gke_test_cluster_with_retries _kubetest2_flags "$3"
}
# TODO(chizhg): move this to kubetest2 gke deployer.
# Retry backup regions/zones if cluster creations failed due to stockout.
# Parameters: $1 - kubetest2 flags other than geo flag
# $2 - test command to run by the kubetest2 tester after the cluster is created (optional)
function create_gke_test_cluster_with_retries() {
local -n kubetest2_flags=$1
local -n command=$2
if (( ${#command[@]} )); then
tester_command=("--test=exec" "--" "${command[@]}")
fi
local cluster_creation_log=/tmp/${REPO_NAME}-cluster_creation-log
# zone_not_provided is a placeholder for e2e_cluster_zone to make for loop below work
local zone_not_provided="zone_not_provided"
local e2e_cluster_regions=("${E2E_GKE_CLUSTER_REGION}")
local e2e_cluster_zones=("${E2E_GKE_CLUSTER_ZONE}")
if [[ -n "${E2E_GKE_CLUSTER_BACKUP_ZONES}" ]]; then
e2e_cluster_zones+=("${E2E_GKE_CLUSTER_BACKUP_ZONES}")
elif [[ -n "${E2E_GKE_CLUSTER_BACKUP_REGIONS}" ]]; then
e2e_cluster_regions+=("${E2E_GKE_CLUSTER_BACKUP_REGIONS}")
e2e_cluster_zones=("${zone_not_provided}")
else
echo "No backup region/zone set, cluster creation will fail in case of stockout"
fi
for e2e_cluster_region in "${e2e_cluster_regions[@]}"; do
for e2e_cluster_zone in "${e2e_cluster_zones[@]}"; do
E2E_GKE_CLUSTER_REGION=${e2e_cluster_region}
E2E_GKE_CLUSTER_ZONE=${e2e_cluster_zone}
[[ "${E2E_GKE_CLUSTER_ZONE}" == "${zone_not_provided}" ]] && E2E_GKE_CLUSTER_ZONE=""
local cluster_creation_zone="${E2E_GKE_CLUSTER_REGION}"
[[ -n "${E2E_GKE_CLUSTER_ZONE}" ]] && cluster_creation_zone="${E2E_GKE_CLUSTER_REGION}-${E2E_GKE_CLUSTER_ZONE}"
header "Creating test cluster ${E2E_CLUSTER_VERSION} in ${cluster_creation_zone}"
if run_go_tool k8s-sigs.io/kubetest2 \
kubetest2 "${kubetest2_flags[@]}" --region="${cluster_creation_zone}" "${tester_command[@]}" 2>&1 \
| tee "${cluster_creation_log}"; then
# Save some metadata about cluster creation for using in prow and testgrid
save_metadata "gke"
return 0
fi
# Retry if cluster creation failed because of stockout (https://github.com/knative/test-infra/issues/592)
# or Nodes failing to start correctly.
# shellcheck disable=SC2143
[[ -z "$(grep -Fo 'does not have enough resources available to fulfill' "${cluster_creation_log}")" \
&& -z "$(grep -Po 'only \d+ nodes out of \d+ have registered; this is likely due to Nodes failing to start correctly' "${cluster_creation_log}")" ]] \
&& return 1
done
done
echo "No more region/zones to try, quitting"
return 1
run_kntest kubetest2 gke "${_custom_flags[@]}" --test-command="${_test_command[*]}"
}

View File

@ -65,22 +65,6 @@ function abort() {
exit 1
}
# Build a resource name based on $REPO_NAME, a suffix and $BUILD_NUMBER.
# Restricts the name length to 40 chars (the limit for resource names in GCP).
# Name will have the form $REPO_NAME-<PREFIX>$BUILD_NUMBER.
# Parameters: $1 - name suffix
function build_resource_name() {
local prefix=${REPO_NAME}-$1
local suffix=${BUILD_NUMBER}
# Restrict suffix length to 20 chars
if [[ -n "${suffix}" ]]; then
suffix=${suffix:${#suffix}<20?0:-20}
fi
local name="${prefix:0:20}${suffix}"
# Ensure name doesn't end with "-"
echo "${name%-}"
}
# Display a box banner.
# Parameters: $1 - character to use for the box.
# $2 - banner message.

View File

@ -272,7 +272,7 @@ function run_integration_tests() {
# Default integration test runner that runs all `test/e2e-*tests.sh`.
function default_integration_test_runner() {
local failed=0
find test/ ! -name "$(printf "*\n*")" -name "e2e-*tests.sh" > tmp
find test/ ! -name "$(printf "*\n*")" -name "e2e-*tests.sh" -maxdepth 1 > tmp
while IFS= read -r e2e_test
do
echo "Running integration test ${e2e_test}"

4
vendor/modules.txt vendored
View File

@ -752,7 +752,7 @@ k8s.io/kube-openapi/pkg/util/sets
k8s.io/utils/buffer
k8s.io/utils/integer
k8s.io/utils/trace
# knative.dev/pkg v0.0.0-20200908235250-56fba14ba7df
# knative.dev/pkg v0.0.0-20200910010051-a79a813ce123
## explicit
knative.dev/pkg/apis
knative.dev/pkg/apis/duck/ducktypes
@ -776,7 +776,7 @@ knative.dev/pkg/metrics/metricskey
knative.dev/pkg/network
knative.dev/pkg/reconciler
knative.dev/pkg/system
# knative.dev/test-infra v0.0.0-20200908182932-5a8105609141
# knative.dev/test-infra v0.0.0-20200909211651-72eb6ae3c773
## explicit
knative.dev/test-infra/scripts
knative.dev/test-infra/tools/dep-collector