linkerd2/.github/workflows/workflow.yml

name: CI

on:
  pull_request: {}
  push:
    branches:
    - master
    tags:
    - "*"

# Jobs by event type and dependencies:

# Unit tests for every master/tag push and PR
#
# go_dependencies
# go_unit_tests
# go_lint
# js_unit_tests

# All master/tag pushes and PRs
#
# docker_build
# -> kind_integration_tests

# Docker push and cloud integration tests for every master/tag push
#
#     -> docker_push
#        -> cloud_integration_tests

jobs:

  #
  # Unit tests run for:
  # - every master push
  # - every tag push
  # - every PR
  #

  go_dependencies:
    name: Go dependencies
    runs-on: ubuntu-18.04
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Dump env
      run: env | sort
    - name: Dump GitHub context
      env:
        GITHUB_CONTEXT: ${{ toJson(github) }}
      run: echo "$GITHUB_CONTEXT"
    - name: Dump job context
      env:
        JOB_CONTEXT: ${{ toJson(job) }}
      run: echo "$JOB_CONTEXT"
    - name: Validate go deps
      run: |
        . bin/_tag.sh
        for f in $( grep -lR --include=Dockerfile\* go-deps: . ) ; do
          validate_go_deps_tag $f
        done        

  go_unit_tests:
    name: Go unit tests
    runs-on: ubuntu-18.04
    container:
      image: golang:1.13.4
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Go unit tests
      env:
        GITCOOKIE_SH: ${{ secrets.GITCOOKIE_SH }}
      run: |
        echo "$GITCOOKIE_SH" | bash
        # TODO: validate bin/protoc-go.sh does not dirty the repo
        go test -cover -race -v -mod=readonly ./...        

  go_lint:
    name: Go lint
    runs-on: ubuntu-18.04
    container:
      image: golang:1.13.4
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Go lint
      env:
        GITCOOKIE_SH: ${{ secrets.GITCOOKIE_SH }}
      run: |
        echo "$GITCOOKIE_SH" | bash
        bin/lint --verbose        

  go_format:
    name: Go format
    runs-on: ubuntu-18.04
    container:
      image: golang:1.13.4
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Format
      env:
        GITCOOKIE_SH: ${{ secrets.GITCOOKIE_SH }}
      run: |
        echo "$GITCOOKIE_SH" | bash
        bin/fmt        

  js_unit_tests:
    name: JS unit tests
    runs-on: ubuntu-18.04
    container:
      image: node:10.16.0-stretch
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Yarn setup
      run: curl -o- -L https://yarnpkg.com/install.sh | bash -s -- --version 1.21.1 --network-concurrency 1
    - name: JS unit tests
      run: |
        export PATH="$HOME/.yarn/bin:$PATH"
        export NODE_ENV=test
        bin/web
        bin/web test --reporters=jest-dot-reporter        

  #
  # Docker build and kind integration tests run for:
  # - every master push
  # - every tag push
  # - every PR
  #

  docker_build:
    name: Docker build
    runs-on: ubuntu-18.04
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Set environment variables from scripts
      run: |
        . bin/_tag.sh
        echo ::set-env name=TAG::$(CI_FORCE_CLEAN=1 bin/root-tag)

        . bin/_docker.sh
        echo ::set-env name=DOCKER_REGISTRY::$DOCKER_REGISTRY        
    - name: Setup SSH config for Packet
      if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
      run: |
        mkdir -p ~/.ssh/
        touch ~/.ssh/id && chmod 600 ~/.ssh/id
        echo "${{ secrets.DOCKER_SSH_CONFIG }}"  > ~/.ssh/config
        echo "${{ secrets.DOCKER_PRIVATE_KEY }}" > ~/.ssh/id
        echo "${{ secrets.DOCKER_KNOWN_HOSTS }}" > ~/.ssh/known_hosts
        ssh linkerd-docker docker version
        echo ::set-env name=DOCKER_HOST::ssh://linkerd-docker        
    - name: Build docker images
      env:
        DOCKER_TRACE: 1
      run: |
        export PATH="`pwd`/bin:$PATH"
        bin/docker-build        
    - name: Create artifact with CLI and image archives (Forked repositories)
      if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork
      env:
        ARCHIVES: /home/runner/archives
      run: |
        mkdir -p $ARCHIVES

        for image in proxy controller web cni-plugin debug cli-bin grafana; do
          docker save "$DOCKER_REGISTRY/$image:$TAG" > $ARCHIVES/$image.tar || tee save_fail &
        done

        # Wait for `docker save` background processes to complete. Exit early
        # if any job failed.
        wait < <(jobs -p)

        test -f save_fail && exit 1 || true        
    # `with.path` values do not support environment variables yet, so an
    # absolute path is used here.
    #
    # https://github.com/actions/upload-artifact/issues/8
    - name: Upload artifact (Forked repositories)
      if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork
      uses: actions/upload-artifact@v1
      with:
        name: image-archives
        path: /home/runner/archives

  kind_integration_tests:
    strategy:
      matrix:
        integration_test: [deep, upgrade, helm, helm_upgrade, custom_domain, external_issuer]
    needs: [docker_build]
    name: Integration tests (${{ matrix.integration_test }})
    runs-on: ubuntu-18.04
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Try to load cached Go modules
      uses: actions/cache@v1
      with:
        path: ~/go/pkg/mod
        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
        restore-keys: |
          ${{ runner.os }}-go-          
    - name: Set environment variables from scripts
      run: |
        . bin/_tag.sh
        echo ::set-env name=TAG::$(CI_FORCE_CLEAN=1 bin/root-tag)

        . bin/_docker.sh
        echo ::set-env name=DOCKER_REGISTRY::$DOCKER_REGISTRY        
    - name: Setup SSH config for Packet
      if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
      run: |
        mkdir -p ~/.ssh/
        touch ~/.ssh/id && chmod 600 ~/.ssh/id
        echo "${{ secrets.DOCKER_SSH_CONFIG }}"  > ~/.ssh/config
        echo "${{ secrets.DOCKER_PRIVATE_KEY }}" > ~/.ssh/id
        echo "${{ secrets.DOCKER_KNOWN_HOSTS }}" > ~/.ssh/known_hosts        
    - name: Download image archives (Forked repositories)
      if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork
      uses: actions/download-artifact@v1
      with:
        name: image-archives
    - name: Load cli-bin image into local docker images
      if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
      run: |
        # `docker load` only accepts input from STDIN, so pipe the image
        # archive into the command.
        #
        # In order to pipe the image archive, set `DOCKER_HOST` for a single
        # command and `docker save` the CLI image from the Packet host.
        DOCKER_HOST=ssh://linkerd-docker docker save "$DOCKER_REGISTRY/cli-bin:$TAG" | docker load        
    - name: Load cli-bin image into local docker images (Forked repositories)
      if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork
      run: docker load < image-archives/cli-bin.tar
    - name: Install CLI
      run: |
        # Copy the CLI out of the local cli-bin container.
        container_id=$(docker create "$DOCKER_REGISTRY/cli-bin:$TAG")
        docker cp $container_id:/out/linkerd-linux $HOME/.linkerd

        # Validate the CLI version matches the current build tag.
        [[ "$TAG" == "$($HOME/.linkerd version --short --client)" ]]        
    - name: Setup default KinD cluster
      if: matrix.integration_test != 'custom_domain'
      uses: engineerd/setup-kind@v0.3.0
      with:
        version: "v0.6.1"
    - name: Setup custom_domain KinD cluster
      if: matrix.integration_test == 'custom_domain'
      uses: engineerd/setup-kind@v0.3.0
      with:
        config: test/testdata/custom_cluster_domain_config.yaml
        version: "v0.6.1"
    - name: Load image archives into the local KinD cluster
      if: github.event_name != 'pull_request' || !github.event.pull_request.head.repo.fork
      env:
        PROXY_INIT_IMAGE_NAME: gcr.io/linkerd-io/proxy-init:v1.3.1
        PROMETHEUS_IMAGE_NAME: prom/prometheus:v2.15.2
      run: |
        # For each container, load the image archive into the KinD cluster.
        #
        # `kind load` cannot take input from STDIN, so `<(command)` syntax is
        # used to load the output into the KinD cluster. Set `DOCKER_HOST` for
        # a single command, and `docker save` the container from the Packet
        # host.
        for image in proxy controller web cni-plugin debug grafana; do
          kind load image-archive <(DOCKER_HOST=ssh://linkerd-docker docker save "$DOCKER_REGISTRY/$image:$TAG") || tee load_fail &
        done

        # Wait for `kind load` background processes to complete. Exit early if
        # any job failed.
        wait < <(jobs -p)
        test -f load_fail && exit 1 || true

        # Load proxy-init and prometheus images into KinD while it is
        # available. Allow these commands to fail since they will be cached
        # for the next run.
        kind load image-archive <(DOCKER_HOST=ssh://linkerd-docker docker save $PROXY_INIT_IMAGE_NAME) 2>&1 || true
        kind load image-archive <(DOCKER_HOST=ssh://linkerd-docker docker save $PROMETHEUS_IMAGE_NAME) 2>&1 || true        
    - name: Load image archives into the local KinD cluster (Forked repositories)
      if: github.event_name == 'pull_request' && github.event.pull_request.head.repo.fork
      run: |
        for image in proxy controller web cni-plugin debug grafana; do
          kind load image-archive image-archives/$image.tar || tee load_fail &
        done

        # Wait for `kind load` background processes to complete. Exit early if
        # any job failed.
        wait < <(jobs -p)
        test -f load_fail && exit 1 || true        
    - name: Run integration tests
      run: |
        # Export `init_test_run` and `*_integration_tests` into the
        # environment.
        . bin/_test-run.sh

        init_test_run $HOME/.linkerd
        ${{ matrix.integration_test }}_integration_tests        

  #
  # Docker push and cloud integration tests run for:
  # - every master push
  # - every tag push
  #

  docker_push:
    name: Docker push
    if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags')
    runs-on: ubuntu-18.04
    needs: [go_dependencies, go_unit_tests, go_lint, js_unit_tests, kind_integration_tests]
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Set environment variables from scripts
      run: |
        . bin/_tag.sh
        echo ::set-env name=TAG::$(CI_FORCE_CLEAN=1 bin/root-tag)        
    - name: Configure gcloud
      uses: linkerd/linkerd2-action-gcloud@v1.0.0
      with:
        cloud_sdk_service_account_key: ${{ secrets.CLOUD_SDK_SERVICE_ACCOUNT_KEY }}
        gcp_project: ${{ secrets.GCP_PROJECT }}
        gcp_zone: ${{ secrets.GCP_ZONE }}
    - name: Docker SSH setup
      run: |
        mkdir -p ~/.ssh/
        touch ~/.ssh/id && chmod 600 ~/.ssh/id
        echo "${{ secrets.DOCKER_SSH_CONFIG }}"  > ~/.ssh/config
        echo "${{ secrets.DOCKER_PRIVATE_KEY }}" > ~/.ssh/id
        echo "${{ secrets.DOCKER_KNOWN_HOSTS }}" > ~/.ssh/known_hosts
        ssh linkerd-docker docker version        
    - name: Push docker images to registry
      env:
        DOCKER_HOST: ssh://linkerd-docker
      run: |
        export PATH="`pwd`/bin:$PATH"
        bin/docker-push-deps
        bin/docker-push $TAG
        bin/docker-retag-all $TAG master
        bin/docker-push master        

  cloud_integration_tests:
    name: Cloud integration tests
    if: github.ref == 'refs/heads/master' || startsWith(github.ref, 'refs/tags')
    runs-on: ubuntu-18.04
    needs: [docker_push]
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Install linkerd CLI
      id: install_cli
      run: |
        TAG="$(CI_FORCE_CLEAN=1 bin/root-tag)"
        image="gcr.io/linkerd-io/cli-bin:$TAG"
        id=$(bin/docker create $image)
        bin/docker cp "$id:/out/linkerd-linux" "$HOME/.linkerd"
        $HOME/.linkerd version --client
        # validate CLI version matches the repo
        [[ "$TAG" == "$($HOME/.linkerd version --short --client)" ]]
        echo "Installed Linkerd CLI version: $TAG"
        echo "::set-output name=tag::$TAG"        
    - name: Create GKE cluster
      uses: linkerd/linkerd2-action-gcloud@v1.0.0
      with:
        cloud_sdk_service_account_key: ${{ secrets.CLOUD_SDK_SERVICE_ACCOUNT_KEY }}
        gcp_project: ${{ secrets.GCP_PROJECT }}
        gcp_zone: ${{ secrets.GCP_ZONE }}
        create: true
        name: testing-${{ steps.install_cli.outputs.tag }}-${{ github.run_id }}
    - name: Run integration tests
      env:
        GITCOOKIE_SH: ${{ secrets.GITCOOKIE_SH }}
      run: |
        export PATH="`pwd`/bin:$PATH"
        echo "$GITCOOKIE_SH" | bash
        version="$($HOME/.linkerd version --client --short | tr -cd '[:alnum:]-')"
        bin/test-run $HOME/.linkerd linkerd-$version        
    - name: CNI tests
      run: |
        export TAG="$($HOME/.linkerd version --client --short)"
        go test -cover -race -v -mod=readonly ./cni-plugin/test -integration-tests        

  #
  # Helm chart artifact deploy run for:
  # - every tag push
  #

  chart_deploy:
    name: Helm chart deploy
    if: startsWith(github.ref, 'refs/tags')
    runs-on: ubuntu-18.04
    needs: [cloud_integration_tests]
    steps:
    - name: Checkout code
      uses: actions/checkout@v2
    - name: Configure gsutils
      uses: linkerd/linkerd2-action-gcloud@v1.0.0
      with:
        cloud_sdk_service_account_key: ${{ secrets.CLOUD_SDK_SERVICE_ACCOUNT_KEY }}
        gcp_project: ${{ secrets.GCP_PROJECT }}
        gcp_zone: ${{ secrets.GCP_ZONE }}
    - name: Edge Helm chart creation and upload
      if: startsWith(github.ref, 'refs/tags/edge')
      run: |
        mkdir -p target/helm
        gsutil cp gs://helm.linkerd.io/edge/index.yaml target/helm/index-pre.yaml
        bin/helm-build package
        gsutil rsync target/helm gs://helm.linkerd.io/edge        
    - name: Stable Helm chart creation and upload
      if: startsWith(github.ref, 'refs/tags/stable')
      run: |
        mkdir -p target/helm
        gsutil cp gs://helm.linkerd.io/stable/index.yaml target/helm/index-pre.yaml
        bin/helm-build package
        gsutil rsync target/helm gs://helm.linkerd.io/stable