From b733357b68cdf269d9e45e3ffdc87f83c785d843 Mon Sep 17 00:00:00 2001 From: knative-automation Date: Wed, 7 Sep 2022 10:30:19 -0400 Subject: [PATCH] upgrade to latest dependencies (#5211) bumping knative.dev/hack 8d1e4cc...6c30196: > 6c30196 :broom: Rewrite unit tests in Golang (# 215) > f9dc722 Format go code (# 218) > 664eac5 Fixing go.mod foreach (# 217) > 97fd663 :gift: Support multiple Golang modules in a single project (# 205) > f472a96 fix go run invocation of github hub cli (# 212) > 7d3bed1 go run... (# 210) > e81d63a Update community files (# 209) Signed-off-by: Knative Automation Signed-off-by: Knative Automation --- go.mod | 2 +- go.sum | 4 +- vendor/knative.dev/hack/e2e-tests.sh | 14 ++- vendor/knative.dev/hack/go.mod | 2 +- vendor/knative.dev/hack/go.work | 7 ++ vendor/knative.dev/hack/infra-library.sh | 8 +- vendor/knative.dev/hack/library.sh | 125 +++++++++++++++++---- vendor/knative.dev/hack/presubmit-tests.sh | 51 ++++++--- vendor/modules.txt | 2 +- 9 files changed, 160 insertions(+), 55 deletions(-) create mode 100644 vendor/knative.dev/hack/go.work diff --git a/go.mod b/go.mod index c1ac350c7..44adc514a 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( gopkg.in/go-playground/webhooks.v3 v3.13.0 gopkg.in/yaml.v2 v2.3.0 honnef.co/go/tools v0.0.1-2020.1.5 // indirect - knative.dev/hack v0.0.0-20220823140917-8d1e4ccf9dc3 + knative.dev/hack v0.0.0-20220907111718-6c301965af4c ) replace go.opencensus.io => go.opencensus.io v0.20.2 diff --git a/go.sum b/go.sum index 49fc91340..f22502c64 100644 --- a/go.sum +++ b/go.sum @@ -538,8 +538,8 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.5 h1:nI5egYTGJakVyOryqLs1cQO5dO0ksin5XXs2pspk75k= honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= -knative.dev/hack v0.0.0-20220823140917-8d1e4ccf9dc3 h1:umaeMRecA0g5g48L9tnEAkTBIitr9eKWMyJYo9YttAA= -knative.dev/hack v0.0.0-20220823140917-8d1e4ccf9dc3/go.mod h1:t/azP8I/Cygaw+87O7rkAPrNRjCelmtfSzWzu/9TM7I= +knative.dev/hack v0.0.0-20220907111718-6c301965af4c h1:TTVBodIukkW7CUXSep9k/nc+nynXbjkBqXyYZxIS9Jw= +knative.dev/hack v0.0.0-20220907111718-6c301965af4c/go.mod h1:yk2OjGDsbEnQjfxdm0/HJKS2WqTLEFg/N6nUs6Rqx3Q= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/vendor/knative.dev/hack/e2e-tests.sh b/vendor/knative.dev/hack/e2e-tests.sh index 57222f32c..cb648d7e0 100644 --- a/vendor/knative.dev/hack/e2e-tests.sh +++ b/vendor/knative.dev/hack/e2e-tests.sh @@ -76,7 +76,7 @@ function setup_test_cluster() { # Setup KO_DOCKER_REPO if it is a GKE cluster. Incorporate an element of # randomness to ensure that each run properly publishes images. Don't # owerwrite KO_DOCKER_REPO if already set. - [ -z "${KO_DOCKER_REPO}" ] && \ + [ -z "${KO_DOCKER_REPO:-}" ] && \ [[ "${k8s_cluster}" =~ ^gke_.* ]] && \ export KO_DOCKER_REPO=gcr.io/${E2E_PROJECT_ID}/${REPO_NAME}-e2e-img/${RANDOM} @@ -120,12 +120,14 @@ function success() { } # Exit test, dumping current state info. -# Parameters: $1 - error message (optional). +# Parameters: $* - error message (optional). function fail_test() { - [[ -n $1 ]] && echo "ERROR: $1" - dump_cluster_state - dump_metrics - exit 1 + local message="$*" + if [[ -n ${message:-} ]]; then + message='test failed' + fi + add_trap "dump_cluster_state;dump_metrics" EXIT + abort "${message}" } SKIP_TEARDOWNS=0 diff --git a/vendor/knative.dev/hack/go.mod b/vendor/knative.dev/hack/go.mod index 7fece868a..c3481abab 100644 --- a/vendor/knative.dev/hack/go.mod +++ b/vendor/knative.dev/hack/go.mod @@ -1,3 +1,3 @@ module knative.dev/hack -go 1.17 +go 1.18 diff --git a/vendor/knative.dev/hack/go.work b/vendor/knative.dev/hack/go.work new file mode 100644 index 000000000..b20aabbbc --- /dev/null +++ b/vendor/knative.dev/hack/go.work @@ -0,0 +1,7 @@ +go 1.18 + +use ( + . + schema + test/e2e +) diff --git a/vendor/knative.dev/hack/infra-library.sh b/vendor/knative.dev/hack/infra-library.sh index 0d0e002f4..f2100fe24 100644 --- a/vendor/knative.dev/hack/infra-library.sh +++ b/vendor/knative.dev/hack/infra-library.sh @@ -91,8 +91,8 @@ function create_test_cluster() { fi case "$1" in - gke) create_gke_test_cluster "$2" "$3" "$4" ;; - kind) create_kind_test_cluster "$2" "$3" "$4" ;; + gke) create_gke_test_cluster "$2" "$3" "${4:-}" ;; + kind) create_kind_test_cluster "$2" "$3" "${4:-}" ;; *) echo "unsupported provider: $1"; exit 1 ;; esac @@ -126,11 +126,11 @@ function create_gke_test_cluster() { # We are disabling logs and metrics on Boskos Clusters by default as they are not used. Manually set ENABLE_GKE_TELEMETRY to true to enable telemetry # and ENABLE_PREEMPTIBLE_NODES to true to create preemptible/spot VMs. VM Preemption is a rare event and shouldn't be distruptive given the fault tolerant nature of our tests. local extra_gcloud_flags="" - if [[ "$ENABLE_GKE_TELEMETRY" != "true" ]]; then + if [[ "${ENABLE_GKE_TELEMETRY:-}" != "true" ]]; then extra_gcloud_flags="${extra_gcloud_flags} --logging=NONE --monitoring=NONE" fi - if [[ "$ENABLE_PREEMPTIBLE_NODES" == "true" ]]; then + if [[ "${ENABLE_PREEMPTIBLE_NODES:-}" == "true" ]]; then extra_gcloud_flags="${extra_gcloud_flags} --preemptible" fi run_kntest kubetest2 gke "${_custom_flags[@]}" --test-command="${_test_command[*]}" --extra-gcloud-flags="${extra_gcloud_flags}" diff --git a/vendor/knative.dev/hack/library.sh b/vendor/knative.dev/hack/library.sh index 26cd00cb8..1016d1cc9 100644 --- a/vendor/knative.dev/hack/library.sh +++ b/vendor/knative.dev/hack/library.sh @@ -40,7 +40,19 @@ fi readonly IS_PROW [[ ! -v REPO_ROOT_DIR ]] && REPO_ROOT_DIR="$(git rev-parse --show-toplevel)" readonly REPO_ROOT_DIR -readonly REPO_NAME="${REPO_NAME:-$(basename "${REPO_ROOT_DIR}")}" + +# Resolves the repository name given a root directory. +# Parameters: $1 - repository root directory. +function __resolveRepoName() { + local repoName + repoName="$(basename "${1:-$(git rev-parse --show-toplevel)}")" + repoName="${repoName#knative-sandbox-}" # Remove knative-sandbox- prefix if any + repoName="${repoName#knative-}" # Remove knative- prefix if any + echo "${repoName}" +} +default_repo_name="$(__resolveRepoName "${REPO_ROOT_DIR}")" +readonly REPO_NAME="${REPO_NAME:-$default_repo_name}" +unset default_repo_name # Useful flags about the current OS IS_LINUX=0 @@ -65,10 +77,6 @@ if [[ -z "${ARTIFACTS:-}" ]]; then fi mkdir -p "$ARTIFACTS" - -# On a Prow job, redirect stderr to stdout so it's synchronously added to log -(( IS_PROW )) && exec 2>&1 - # Return the major version of a release. # For example, "v0.2.1" returns "0" # Parameters: $1 - release version label. @@ -94,11 +102,49 @@ function patch_version() { echo "${tokens[2]}" } -# Print error message and exit 1 +# Calculates the hashcode for a given string. +# Parameters: $* - string to be hashed. +# See: https://stackoverflow.com/a/48863502/844449 +function hashCode() { + local input="$1" + local -i h=0 + for ((i = 0; i < ${#input}; i++)); do + # val is ASCII val + printf -v val "%d" "'${input:$i:1}" + hval=$((31 * h + val)) + # hash scheme + if ((hval > 2147483647)); then + h=$(( (hval - 2147483648) % 2147483648 )) + elif ((hval < -2147483648)); then + h=$(( (hval + 2147483648) % 2147483648 )) + else + h=$(( hval )) + fi + done + # final hashCode in decimal + printf "%d" $h +} + +# Calculates the retcode for a given string. Makes sure the return code is +# non-zero. +# Parameters: $* - string to be hashed. +function calcRetcode() { + local rc=1 + local rcc + rcc="$(hashCode "$*")" + if [[ $rcc != 0 ]]; then + rc=$(( rcc % 255 )) + fi + echo "$rc" +} + +# Print error message and call exit(n) where n calculated from the error message. # Parameters: $1..$n - error message to be displayed +# Globals: abort_retcode will change the default retcode to be returned function abort() { - echo "error: $*" - exit 1 + make_banner '*' "ERROR: $*" >&2 + readonly abort_retcode="${abort_retcode:-$(calcRetcode "$*")}" + exit "$abort_retcode" } # Display a box banner. @@ -106,11 +152,13 @@ function abort() { # $2 - banner message. function make_banner() { local msg="$1$1$1$1 $2 $1$1$1$1" - local border="${msg//[-0-9A-Za-z _.,\/()\']/$1}" + local border="${msg//[^$1]/$1}" echo -e "${border}\n${msg}\n${border}" # TODO(adrcunha): Remove once logs have timestamps on Prow # For details, see https://github.com/kubernetes/test-infra/issues/10100 - echo -e "$1$1$1$1 $(TZ='UTC' date)\n${border}" + if (( IS_PROW )); then + echo -e "$1$1$1$1 $(TZ='UTC' date --rfc-3339=ns)\n${border}" + fi } # Simple header for logging purposes. @@ -126,7 +174,7 @@ function subheader() { # Simple warning banner for logging purposes. function warning() { - make_banner "!" "$1" + make_banner '!' "WARN: $*" >&2 } # Checks whether the given function exists. @@ -448,14 +496,14 @@ function report_go_test() { logfile="${xml/junit_/go_test_}" logfile="${logfile/.xml/.jsonl}" echo "Running go test with args: ${go_test_args[*]}" + local gotest_retcode=0 go_run gotest.tools/gotestsum@v1.8.0 \ --format "${GO_TEST_VERBOSITY:-testname}" \ --junitfile "${xml}" \ --junitfile-testsuite-name relative \ --junitfile-testcase-classname relative \ --jsonfile "${logfile}" \ - -- "${go_test_args[@]}" - local gotest_retcode=$? + -- "${go_test_args[@]}" || gotest_retcode=$? echo "Finished run, return code is ${gotest_retcode}" echo "XML report written to ${xml}" @@ -558,6 +606,9 @@ function go_run() { if [[ "$package" != *@* ]]; then abort 'Package for "go_run" needs to have @version' fi + if [[ "$package" == *@latest ]] && [[ "$package" != knative.dev* ]]; then + warning 'Using @latest version for external dependencies is unsafe. Use numbered version!' + fi shift 1 GORUN_PATH="${GORUN_PATH:-$(go env GOPATH)}" # Some CI environments may have non-writable GOPATH @@ -567,7 +618,6 @@ function go_run() { export GORUN_PATH GOPATH="${GORUN_PATH}" \ GOFLAGS='' \ - GO111MODULE='' \ go run "$package" "$@" } @@ -577,6 +627,7 @@ function go_run() { # $3..$n - parameters passed to the tool. # Deprecated: use go_run instead function run_go_tool() { + warning 'The "run_go_tool" function is deprecated. Use "go_run" instead.' local package=$1 # If no `@version` is provided, default to adding `@latest` if [[ "$package" != *@* ]]; then @@ -601,6 +652,26 @@ function add_trap { done } +# Run a command, described by $1, for every go module in the project. +# Parameters: $1 - Description of the command being run, +# $2 - $n - Arguments to pass to the command. +function foreach_go_module() { + local failed=0 + local -r cmd="$1" + shift + local gomod_filepath gomod_dir + while read -r gomod_filepath; do + gomod_dir="$(dirname "$gomod_filepath")" + pushd "$gomod_dir" > /dev/null + "$cmd" "$@" || failed=$? + popd > /dev/null + if (( failed )); then + echo "Command '${cmd}' failed in module $gomod_dir: $failed" >&2 + return $failed + fi + done < <(find . -name go.mod -type f ! -path "*/vendor/*" ! -path "*/third_party/*") +} + # Update go deps. # Parameters (parsed as flags): # "--upgrade", bool, do upgrade. @@ -613,14 +684,18 @@ function add_trap { # global env var: FLOATING_DEPS # --upgrade will set GOPROXY to direct unless it is already set. function go_update_deps() { - cd "${REPO_ROOT_DIR}" || return 1 + foreach_go_module __go_update_deps_for_module "$@" +} + +function __go_update_deps_for_module() { + ( # do not modify the environment + set -Eeuo pipefail - export GO111MODULE=on export GOFLAGS="" export GONOSUMDB="${GONOSUMDB:-},knative.dev/*" export GONOPROXY="${GONOPROXY:-},knative.dev/*" - echo "=== Update Deps for Golang" + echo "=== Update Deps for Golang module: $(go_mod_module_name)" local UPGRADE=0 local RELEASE="v9000.1" # release v9000 is so far in the future, it will always pick the default branch. @@ -646,7 +721,7 @@ function go_update_deps() { else group "Upgrading to release ${RELEASE}" fi - FLOATING_DEPS+=( $(go_run knative.dev/test-infra/buoy@latest float ${REPO_ROOT_DIR}/go.mod "${buoyArgs[@]}") ) + FLOATING_DEPS+=( $(go_run knative.dev/test-infra/buoy@latest float ./go.mod "${buoyArgs[@]}") ) if [[ ${#FLOATING_DEPS[@]} > 0 ]]; then echo "Floating deps to ${FLOATING_DEPS[@]}" go get -d ${FLOATING_DEPS[@]} @@ -664,6 +739,9 @@ function go_update_deps() { go mod vendor 2>&1 | grep -v "ignoring symlink" || true eval "$orig_pipefail_opt" + if ! [ -d vendor ]; then + return 0 + fi group "Removing unwanted vendor files" # Remove unwanted vendor files @@ -680,13 +758,15 @@ function go_update_deps() { group "Removing broken symlinks" remove_broken_symlinks ./vendor + ) } + # Return the go module name of the current module. # Intended to be used like: # export MODULE_NAME=$(go_mod_module_name) function go_mod_module_name() { - go mod graph | cut -d' ' -f 1 | grep -v '@' | head -1 + grep -E '^module ' go.mod | cut -d' ' -f2 } # Return a GOPATH to a temp directory. Works around the out-of-GOPATH issues @@ -717,11 +797,10 @@ function run_kntest() { # Parameters: $1 - output file, relative to repo root dir. # $2 - directory to inspect. function update_licenses() { - cd "${REPO_ROOT_DIR}" || return 1 local dst=$1 local dir=$2 shift - go_run github.com/google/go-licenses@v1.2.0 \ + go_run github.com/google/go-licenses@v1.2.1 \ save "${dir}" --save_path="${dst}" --force || \ { echo "--- FAIL: go-licenses failed to update licenses"; return 1; } } @@ -729,7 +808,7 @@ function update_licenses() { # Run go-licenses to check for forbidden licenses. function check_licenses() { # Check that we don't have any forbidden licenses. - go_run github.com/google/go-licenses@v1.2.0 \ + go_run github.com/google/go-licenses@v1.2.1 \ check "${REPO_ROOT_DIR}/..." || \ { echo "--- FAIL: go-licenses failed the license check"; return 1; } } @@ -762,7 +841,7 @@ function is_protected_project() { # Remove symlinks in a path that are broken or lead outside the repo. # Parameters: $1 - path name, e.g. vendor function remove_broken_symlinks() { - for link in $(find $1 -type l); do + for link in $(find "$1" -type l); do # Remove broken symlinks if [[ ! -e ${link} ]]; then unlink ${link} diff --git a/vendor/knative.dev/hack/presubmit-tests.sh b/vendor/knative.dev/hack/presubmit-tests.sh index d9549f51b..d99a923a6 100644 --- a/vendor/knative.dev/hack/presubmit-tests.sh +++ b/vendor/knative.dev/hack/presubmit-tests.sh @@ -122,7 +122,12 @@ function report_build_test() { # * `go build` on the entire repo # * check licenses in all go packages function default_build_test_runner() { + foreach_go_module __build_test_runner_for_module +} + +function __build_test_runner_for_module() { local failed=0 + subheader "Build tests for $(go_mod_module_name)" # Run verify-codegen check if [[ -f ./hack/verify-codegen.sh ]]; then subheader "Checking autogenerated code is up-to-date" @@ -132,18 +137,26 @@ function default_build_test_runner() { # verify-codegen (as md files can be auto-generated in some repos). (( IS_DOCUMENTATION_PR )) && return ${failed} # Don't merge these two lines, or return code will always be 0. - local go_pkg_dirs - go_pkg_dirs="$(go list ./...)" || return 1 - # Skip build test if there is no go code - [[ -z "${go_pkg_dirs}" ]] && return ${failed} - # Ensure all the code builds - subheader "Checking that go code builds" # Get all build tags in go code (ignore /vendor, /hack and /third_party) local tags - tags="$(find . \ - -path './vendor' -prune -o -path './hack' -prune -o -path './third_party' -prune \ - -o -type f -name '*.go' -exec grep '// +build' {} + \ - | cut -f3 -d' ' | tr ',' '\n' | uniq | sort | tr '\n' ' ')" + tags="$(grep -I -r '// +build' . | \ + grep -v '/vendor/' | \ + grep -v '/hack/' | \ + grep -v '/third_party' | \ + cut -f3 -d' ' | \ + tr ',' '\n' | \ + sort | uniq | \ + grep -v '^!' | \ + tr '\n' ' ')" + local go_pkg_dirs + go_pkg_dirs="$(go list -tags "${tags}" ./...)" || return $? + if [[ -z "${go_pkg_dirs}" ]]; then + subheader "No golang code found, skipping build tests" + return 0 + fi + # Ensure all the code builds + subheader "Checking that go code builds" + report_build_test Build_Go \ go test -vet=off -tags "${tags}" -exec echo ./... || failed=2 @@ -185,6 +198,11 @@ function run_unit_tests() { # Default unit test runner that runs all go tests in the repo. function default_unit_test_runner() { + foreach_go_module __unit_test_runner_for_module +} + +function __unit_test_runner_for_module() { + subheader "Unit tests for $(go_mod_module_name)" report_go_test -short -race -count 1 ./... } @@ -222,15 +240,14 @@ function run_integration_tests() { # Default integration test runner that runs all `test/e2e-*tests.sh`. function default_integration_test_runner() { local failed=0 - find test/ ! -name "$(printf "*\n*")" -name "e2e-*tests.sh" -maxdepth 1 > tmp - while IFS= read -r e2e_test - do + + while IFS= read -r e2e_test; do echo "Running integration test ${e2e_test}" if ! ${e2e_test}; then failed=1 step_failed "${e2e_test}" fi - done < tmp + done < <(find test/ ! -name "$(printf "*\n*")" -name "e2e-*tests.sh" -maxdepth 1) return ${failed} } @@ -266,16 +283,16 @@ function main() { git version echo ">> ko version" [[ -f /ko_version ]] && cat /ko_version || echo "unknown" - if [[ "${DOCKER_IN_DOCKER_ENABLED}" == "true" ]]; then + if [[ "${DOCKER_IN_DOCKER_ENABLED:-}" == "true" ]]; then echo ">> docker version" docker version fi if type java > /dev/null; then echo ">> java version" java -version - echo "JAVA_HOME: $JAVA_HOME" + echo "JAVA_HOME: ${JAVA_HOME:-}" fi - if type mvn > /dev/null; then + if command -v mvn > /dev/null; then echo ">> maven version" mvn --version fi diff --git a/vendor/modules.txt b/vendor/modules.txt index ce9cd2c8d..7dea6842e 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -286,7 +286,7 @@ gopkg.in/go-playground/webhooks.v3/github gopkg.in/yaml.v2 # honnef.co/go/tools v0.0.1-2020.1.5 ## explicit -# knative.dev/hack v0.0.0-20220823140917-8d1e4ccf9dc3 +# knative.dev/hack v0.0.0-20220907111718-6c301965af4c ## explicit knative.dev/hack # go.opencensus.io => go.opencensus.io v0.20.2