Introduce integration tests into all ci runs (#3293)

The integration tests under `/test` were run separately via l5d-bot, lacking the feedback and job management provided by ci. Enable integration tests in ci, via a docker build and kind clusters executed on a remote DOCKER_HOST. CI runs are now broken into two stages, run serially. Each stage is composed of jobs run in parallel: - Setup stage - Validate go deps - Remote docker build - Kind cluster setup (deep) - Kind cluster setup (upgrade) - Kind cluster setup (helm) - Test stage - Go unit tests - Node.js unit tests - Kind integration tests (deep) - Kind integration tests (upgrade) - Kind integration tests (helm) This PR also modifies `bin/test-run.sh` to always set `--failfast` for Go tests. Also introduce `bin/docker` and `bin/kubectl` scripts, to ensure cacheable, pinned executables in ci. The existing integration tests for master merges and docker pushes, running against GKE, remain in place. Signed-off-by: Andrew Seigner <siggy@buoyant.io>
2019-08-26 11:41:17 -07:00 · 2019-08-26 11:41:17 -07:00 · ea27e0ca0e
parent 02efb46e45
commit ea27e0ca0e
6 changed files with 219 additions and 17 deletions
--- a/.travis.rsa.enc
+++ b/.travis.rsa.enc
--- a/.travis.yml
+++ b/.travis.yml
@ -1,5 +1,5 @@
 ---
-dist: trusty
+dist: xenial
 sudo: false

 # We do not test pushes to branches, since they are redundant with the pull_request build
@ -10,6 +10,7 @@ branches:
    - /^(edge|stable)-\d+\.\d+\.[\w-]+$/ # build version tags

 stages:
+  - name: setup
  - name: test
  - name: docker-deploy
    if: type != pull_request
@ -18,8 +19,81 @@ stages:

 jobs:
  include:
+    # TODO: If any of the jobs in the setup stage fail, clean up all the kind clusters.
+    - stage: setup
+      name: "Validate go deps"
+      language: generic
+      script:
+        - |
+          (
+            . bin/_tag.sh
+            for f in $( grep -lR --include=Dockerfile\* go-deps: . ) ; do
+              validate_go_deps_tag $f
+            done
+          )

-    - language: go
+    - name: "Remote docker build"
+      if: fork = false
+      language: generic
+      cache:
+        directories:
+          - target/bin
+      addons: &addons_ssh_known_hosts
+        ssh_known_hosts:
+          # Same value as $REMOTE_DOCKER.
+          secure: "H44Np1YIvtnGc1PGE72AVS65inamSEimxNGZej3l0au8FbM/z6v1nkEVRQ2N72MXK9cdkjvNdjyYw21KA8vR74uDUpDiTh1duDNZq3sQCwcf4/v6sDDSjA9ZgN7YfrydGbBagCEdL9p1Y22X4lFggXHSi2yYeuK3m2HE+vtaHCcNQb+z1liccD5KkScaMAd8zorwf/dRqSjuJmbnBkNyw6IhSGVYluybPbH0meB8a6CPGRkluNGSu0S4N8QDkJZqhXk3ELYr1u9gfmsJxZ2omB2RBkhhIPEmwZroAcGTfmnqiak6IO7oONJQwAxY3WYmNHZmka8tlFORjhLtm6AGe+k41x3g/sEz8gDN/16AGJqi3MzlLNUrJQuPxIkXrPXEn08nyMDrhz6LuDVikBKeFunflRp07d7hwCoRz7RiFqiJP946XdrV/ozP1054RrZU5RhlcWK17sFUmY9HYgNAYuMpCjfTOWxqKX09aKU5IM9XDV4xXPZIFVBGi9Egk8XRCCf+WHr9eNDBpBHL5eILPH5xrTw9OP86HdkVIKn//AJZ+JxU7hb0bujQbnDJVTXTzG4XkxNfUTba3CivA7gRl0ThtbpVz6/FilR2/IEoTmpjhmRjVW0XGkXsuRw04U53AIL8DIW6P9ramY/+yMEWWm7ppHA5F4lJIofPEMAEDpA="
+      env: &env_remote_docker
+        # REMOTE_DOCKER=x.x.x.x (not strictly necessary for this "Remote docker build job", but useful to share with subsequent jobs)
+        - secure: YAFyiPcsdmM/GhkDiNTmj6pHAWxsqOfHC7NCQixRN+936FSuIbmnmgFy9AWsVva8u7QGcqHNzxoiHZt7T9bokUSRFBWPGWqsA1vEWiihX2vcl/d2nuoZ/fPYfcudobJasjCaQeKCefW/mt6SCRX/2K8FahYHjhK5iyynVKkLFDrmKW4ax3TggQmsOBIKXNB/gUjYugz06o9b5Xgm4zdBa37j7wcXCUVkWTyWvhhPBKOjLMqXKumzCSLDVLoPKbVgFPVKtfC+ftin3W9wFpU4dI0tng0i/6Mysy8K3eU238+WSNizTf+fXwjeunDFJwWl5X8xRpBk/vvvt0CMERQrs5Y+cEq8ME7F265EfoKKkTexRgdI3Id7WrgV7bMJZBwfmMyMF71XAx3nxkbj2JXXwjj3HneZihSf1OmQyQ/nuvlm2jdqKiTOzSruO/2hp3/bJCHZhmD98I1GMXKcuw9dA/pccoNVQvXhDZRgqI35nk5dnviF4UYSqV/wSWhEyCEhR7/G3HZGoOy3WBrL01Em3KAC7CcI54dYsZXO0XvaGqu7HMfgi9uch2n6wbFP9KufKtnHvCWyspPHrenfZMy1c2pAdguyP/fugXZR5JGQl+Tu2y3iOMxlDFJcrXMcEx9N57yVnvlBotUCyL3Jjbx9BXwb4ypLOEtMhdJaOPvKWcg=
+        # DOCKER_HOST=ssh://x.x.x.x
+        - secure: AP6owqkStO9n0+ZNj4hoIkAmY8tYoG96IrsNjsDAbq5tbh0QaVknLmNXVei0Jk8bRdulXR2DFsHNhV+pkh6BT2/77t2fvvBcBX0IYYB34hlHqafusffXfQRvQ4DIh68Sk0XqMd7OKnb9rNVYQQPdwDO5lHt4borLsOAN20J/pv06V5y+IHiS9QRhRYxNsN7KKgqQKooY5EZbsEjCJaFDsrvn4KwBJEEpxFSivSL8aehrNk0olQygsmbZ7WBwWnuhqvD6PntkOL1kqzgxFcwh7rcZCB50DD5/dItQXI7jomT82MyFGb8diIkxiggHrJVSy7xWWyup51xWOHv+W4CX+m0u0m+sgd7smtOpyl+mlhkdAY6alkBzpoVN8gFubpiZltS4yBnZmubmTo4erQSa8M7d2Cagi6AqxUqYrwBQ5R7jarr6hneVWC5Sy7sPt2EcqrEW8Qy5LwSm1dnprKKHVpxn8naZLTVvQN8l1EBWqX8lhzdkqpqKvdIS37/ql0ZBAema1RluS4AcpnSA+uyobAojScdK3s+LWmn+MEwts2QM7iikOIP8p0DiWPY3aQ4zu3iR7BueWGuuts/zcqkLzQJY4/+RtjKlGLR3BdTVMk9m5RfP/+MdrqVyFz2X6AJeCyO+57kCBJd0T1deXrL4ZrJXVAso9WiRxATpQPMJNb4=
+      before_install: &remote_docker_ssh
+        - |
+          # Set up remote docker host ssh.
+          openssl aes-256-cbc -K $encrypted_833c0dff2c1b_key -iv $encrypted_833c0dff2c1b_iv -in .travis.rsa.enc -out /tmp/.travis.rsa -d
+          eval "$(ssh-agent -s)"
+          chmod 600 /tmp/.travis.rsa
+          ssh-add /tmp/.travis.rsa
+      script:
+        - PATH="`pwd`/bin:$PATH" DOCKER_TRACE=1 bin/docker-build
+
+    - &kind-setup
+      if: fork = false
+      name: "Kind cluster setup (deep)"
+      language: generic
+      cache:
+        directories:
+          - target/bin
+      addons: *addons_ssh_known_hosts
+      env: *env_remote_docker
+      before_install: *remote_docker_ssh
+      before_script:
+        - export INTEGRATION_TEST=deep
+      script:
+        - |
+          # kind shells out to docker so we need bin/docker in PATH.
+          export PATH="`pwd`/bin:$PATH"
+        - |
+          # create kind cluster
+          export KIND_CLUSTER=travis-$TRAVIS_BUILD_ID-$INTEGRATION_TEST
+          bin/kind create cluster --name=$KIND_CLUSTER
+        - |
+          # Save kubeconfig to remote docker host for kind integration test stages.
+          scp $(bin/kind get kubeconfig-path --name=$KIND_CLUSTER) $USER@$REMOTE_DOCKER:/tmp
+      after_failure:
+        - bin/kind delete cluster --name=$KIND_CLUSTER
+    - <<: *kind-setup
+      name: "Kind cluster setup (upgrade)"
+      before_script:
+        - export INTEGRATION_TEST=upgrade
+    - <<: *kind-setup
+      name: "Kind cluster setup (helm)"
+      before_script:
+        - export INTEGRATION_TEST=helm
+
+    - stage: test
+      name: "Go unit tests"
+      language: go
      # Quote the version number to avoid parsing issues like
      # https://github.com/travis-ci/gimme/issues/132.
      go: "1.12.9"
@ -40,7 +114,8 @@ jobs:
        - go test -cover -race -v -mod=readonly ./...
        - ./bin/lint --verbose

-    - language: node_js
+    - name: "Node.js unit tests"
+      language: node_js
      node_js:
        - "10"
      cache: yarn
@ -53,15 +128,78 @@ jobs:
      script:
        - ./bin/web test --reporters=jest-dot-reporter

-    - language: generic
+    - &kind-test
+      if: fork = false
+      name: "Kind integration tests (deep)"
+      language: generic
+      cache:
+        directories:
+          - target/bin
+          - "$GOPATH/pkg/mod"
+          - "$HOME/.cache"
+      addons: *addons_ssh_known_hosts
+      env: *env_remote_docker
+      before_install: *remote_docker_ssh
+      before_script:
+        - export INTEGRATION_TEST=deep
      script:
        - |
-          (
-            . bin/_tag.sh
-            for f in $( grep -lR --include=Dockerfile\* go-deps: . ) ; do
-              validate_go_deps_tag $f
+          # Install Go.
+          curl -sL -o /tmp/go.tar.gz https://dl.google.com/go/go1.12.9.linux-amd64.tar.gz
+          rm -rf $GOROOT
+          sudo rm -rf /usr/local/go
+          sudo tar -C /usr/local -xzf /tmp/go.tar.gz
+          export GO111MODULE=on
+          export GOROOT=/usr/local/go
+        - |
+          # Misc env setup.
+          export PATH=/usr/local/go/bin:$PATH
+          export PATH="`pwd`/bin:$PATH"
+          export KIND_CLUSTER=travis-$TRAVIS_BUILD_ID-$INTEGRATION_TEST
+          echo $GITCOOKIE_SH | base64 -d | bash
+        - |
+          # Restore kubeconfig from remote docker host.
+          mkdir -p $HOME/.kube
+          scp $USER@$REMOTE_DOCKER:/tmp/kind-config-$KIND_CLUSTER $HOME/.kube
+          export KUBECONFIG=$(bin/kind get kubeconfig-path --name=$KIND_CLUSTER)
+        - |
+          # Start ssh tunnel to allow kubectl to connect via localhost.
+          export KIND_PORT=$(bin/kubectl config view -o jsonpath="{.clusters[?(@.name=='$KIND_CLUSTER')].cluster.server}" | cut -d':' -f3)
+          ssh -4 -N -L $KIND_PORT:localhost:$KIND_PORT $USER@$REMOTE_DOCKER &
+          sleep 2 # Wait for ssh tunnel to come up.
+          bin/kubectl version --short # Test connection to kind cluster.
+        - |
+          # Install linkerd cli.
+          version="$(CI_FORCE_CLEAN=1 bin/root-tag)"
+          image="gcr.io/linkerd-io/cli-bin:$version"
+          id=$(bin/docker create $image)
+          mkdir -p ./target/cli/linux
+          bin/docker cp "$id:/out/linkerd-linux" "./target/cli/linux/linkerd"
+        - |
+          # Load Linkerd docker images into kind cluster.
+          ssh -T $USER@$REMOTE_DOCKER &> /dev/null << EOF
+            for IMG in controller grafana proxy web ; do
+              # TODO: This is using the kind binary on the remote host.
+              kind load docker-image gcr.io/linkerd-io/\$IMG:$version --name=$KIND_CLUSTER
            done
+          EOF
+        - |
+          # Run the integration tests.
+          (
+            . bin/_test-run.sh
+            init_test_run `pwd`/bin/linkerd
+            "$INTEGRATION_TEST"_integration_tests
          )
+      after_script:
+        - bin/kind delete cluster --name=$KIND_CLUSTER
+    - <<: *kind-test
+      name: "Kind integration tests (upgrade)"
+      before_script:
+        - export INTEGRATION_TEST=upgrade
+    - <<: *kind-test
+      name: "Kind integration tests (helm)"
+      before_script:
+        - export INTEGRATION_TEST=helm

    # Push container images to Google Container Registry.
    - stage: docker-deploy
--- a/bin/_test-run.sh
+++ b/bin/_test-run.sh
@ -57,12 +57,11 @@ function helm_integration_tests() {
 }

 function deep_integration_tests() {
-    run_test "$test_directory/install_test.go" -failfast --linkerd-namespace=$linkerd_namespace
+    run_test "$test_directory/install_test.go" --linkerd-namespace=$linkerd_namespace
    exit_on_err "error during install"

-    for test in $(find "$test_directory" -mindepth 2 -name '*_test.go'); do
-        run_test "$test" --linkerd-namespace=$linkerd_namespace || exit_code=$?
-    done
+    run_test "$(go list $test_directory/.../...)" --linkerd-namespace=$linkerd_namespace || exit_code=$?
+    exit_on_err "error during deep tests"
 }

 #
@ -114,8 +113,8 @@ function run_test(){
    filename="$1"
    shift

-    printf "Test script: [%s] Params: [%s]\n" "$(basename $filename)" "$*"
-    GO111MODULE=on go test --mod=readonly "$filename" --linkerd="$linkerd_path" --k8s-context="$k8s_context" --integration-tests "$@"
+    printf "Test script: [%s] Params: [%s]\n" "$(basename $filename 2>/dev/null || echo $filename )" "$*"
+    GO111MODULE=on go test --failfast --mod=readonly $filename --linkerd="$linkerd_path" --k8s-context="$k8s_context" --integration-tests "$@"
 }

 # Install the latest stable release.
@ -140,7 +139,7 @@ function run_upgrade_test() {
    local stable_version=$(curl -s https://versioncheck.linkerd.io/version.json | grep -o "stable-[0-9]*.[0-9]*.[0-9]*")

    install_stable $stable_namespace
-    run_test "$test_directory/install_test.go" -failfast --upgrade-from-version=$stable_version --linkerd-namespace=$stable_namespace
+    run_test "$test_directory/install_test.go" --upgrade-from-version=$stable_version --linkerd-namespace=$stable_namespace
 }

 function run_helm_test() {
@ -154,7 +153,7 @@ function run_helm_test() {
        $helm_path --kube-context=$k8s_context --tiller-namespace=$tiller_namespace dependency update $helm_chart
    )
    exit_on_err "error setting up Helm"
-    run_test "$test_directory/install_test.go" -failfast --linkerd-namespace=$linkerd_namespace-helm \
+    run_test "$test_directory/install_test.go" --linkerd-namespace=$linkerd_namespace-helm \
        --helm-path=$helm_path --helm-chart=$helm_chart --helm-release=$helm_release_name --tiller-ns=$tiller_namespace
 }

--- a/bin/docker
+++ b/bin/docker
@ -0,0 +1,32 @@
+#!/bin/bash
+
+set -eu
+
+dockerversion=19.03.1
+
+bindir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+targetbin="$( cd $bindir/.. && pwd )"/target/bin
+dockerbin="${targetbin}/.docker-${dockerversion}"
+
+if [ ! -f "$dockerbin" ]; then
+  if [ "$(uname -s)" = "Darwin" ]; then
+    os=mac
+    filename="docker-$dockerversion.tgz"
+  else
+    os=linux
+    filename="docker-$dockerversion.tgz"
+  fi
+
+  url="https://download.docker.com/${os}/static/stable/x86_64/${filename}"
+  tmp=$(mktemp -d -t docker.XXX)
+  mkdir -p "$targetbin"
+  (
+      cd "$tmp"
+      curl -Lsf -o "./docker.tar.gz" "$url"
+      tar zf "./docker.tar.gz" -x docker/docker
+  )
+  mv "$tmp/docker/docker" "$dockerbin"
+  rm -rf "$tmp"
+fi
+
+$dockerbin "$@"
--- a/bin/kind
+++ b/bin/kind
@ -2,7 +2,7 @@

 set -eu

-kindversion=v0.4.0
+kindversion=v0.5.1

 bindir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
 targetbin="$( cd $bindir/.. && pwd )"/target/bin
--- a/bin/kubectl
+++ b/bin/kubectl
@ -0,0 +1,33 @@
+#!/bin/bash
+
+set -eu
+
+kubectlversion=v1.15.3
+
+bindir="$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )"
+targetbin="$( cd $bindir/.. && pwd )"/target/bin
+kubectlbin="${targetbin}/.kubectl-${kubectlversion}"
+
+if [ ! -f "$kubectlbin" ]; then
+  exe=
+  if [ "$(uname -s)" = "Darwin" ]; then
+    os=darwin
+    arch=amd64
+  elif [ "$(uname -o)" = "Msys" ]; then
+    os=windows
+    arch=amd64
+    exe=.exe
+  else
+    os=linux
+    case $(uname -m) in
+      x86_64) arch="amd64" ;;
+      arm) dpkg --print-architecture | grep -q "arm64" && arch="arm64" || arch="arm" ;;
+    esac
+  fi
+
+  mkdir -p "$targetbin"
+  curl -sfL -o "$kubectlbin" https://storage.googleapis.com/kubernetes-release/release/$kubectlversion/bin/$os/$arch/kubectl${exe}
+  chmod +x "$kubectlbin"
+fi
+
+$kubectlbin "$@"