#!/bin/bash
#
# Copyright 2018 The Kubeflow Authors
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#      http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.

set -ex

# Env inputs:
# * $GCR_IMAGE_BASE_DIR
# * $GCR_IMAGE_TAG
# * $KFP_DEPLOY_RELEASE
# * $ENABLE_WORKLOAD_IDENTITY
GCR_IMAGE_TAG=${GCR_IMAGE_TAG:-latest}
ENABLE_WORKLOAD_IDENTITY=${ENABLE_WORKLOAD_IDENTITY:-false}

KFP_MANIFEST_DIR="${DIR}/manifests"
pushd ${KFP_MANIFEST_DIR}

if ! which kustomize; then
  # Download kustomize cli tool
  TOOL_DIR=${DIR}/bin
  mkdir -p ${TOOL_DIR}
  # Use 2.0.3 because we want it to be compatible with kubectl apply -k.
  # The change in https://github.com/kubernetes-sigs/kustomize/blob/master/docs/v2.1.0.md#envs-field broke backward compatibility.
  wget --no-verbose https://github.com/kubernetes-sigs/kustomize/releases/download/v2.0.3/kustomize_2.0.3_linux_amd64 \
    -O ${TOOL_DIR}/kustomize --no-verbose
  chmod +x ${TOOL_DIR}/kustomize
  PATH=${PATH}:${TOOL_DIR}
fi

if [ -z "$KFP_DEPLOY_RELEASE" ]; then
  echo "Deploying KFP in working directory..."
  KFP_MANIFEST_DIR=${DIR}/manifests

  pushd ${KFP_MANIFEST_DIR}/cluster-scoped-resources
  kubectl apply -k .
  kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io
  popd

  pushd ${KFP_MANIFEST_DIR}/dev
  # This is the recommended approach to do this.
  # reference: https://github.com/kubernetes-sigs/kustomize/blob/master/docs/eschewedFeatures.md#build-time-side-effects-from-cli-args-or-env-variables
  kustomize edit set image gcr.io/ml-pipeline/api-server=${GCR_IMAGE_BASE_DIR}/api-server:${GCR_IMAGE_TAG}
  kustomize edit set image gcr.io/ml-pipeline/persistenceagent=${GCR_IMAGE_BASE_DIR}/persistenceagent:${GCR_IMAGE_TAG}
  kustomize edit set image gcr.io/ml-pipeline/scheduledworkflow=${GCR_IMAGE_BASE_DIR}/scheduledworkflow:${GCR_IMAGE_TAG}
  kustomize edit set image gcr.io/ml-pipeline/frontend=${GCR_IMAGE_BASE_DIR}/frontend:${GCR_IMAGE_TAG}
  kustomize edit set image gcr.io/ml-pipeline/viewer-crd-controller=${GCR_IMAGE_BASE_DIR}/viewer-crd-controller:${GCR_IMAGE_TAG}
  kustomize edit set image gcr.io/ml-pipeline/visualization-server=${GCR_IMAGE_BASE_DIR}/visualization-server:${GCR_IMAGE_TAG}
  kustomize edit set image gcr.io/ml-pipeline/inverse-proxy-agent=${GCR_IMAGE_BASE_DIR}/inverse-proxy-agent:${GCR_IMAGE_TAG}
  kustomize edit set image gcr.io/ml-pipeline/metadata-writer=${GCR_IMAGE_BASE_DIR}/metadata-writer:${GCR_IMAGE_TAG}
  kustomize edit set image gcr.io/ml-pipeline/cache-server=${GCR_IMAGE_BASE_DIR}/cache-server:${GCR_IMAGE_TAG}
  kustomize edit set image gcr.io/ml-pipeline/cache-deployer=${GCR_IMAGE_BASE_DIR}/cache-deployer:${GCR_IMAGE_TAG}
  kustomize edit set image gcr.io/ml-pipeline/metadata-envoy=${GCR_IMAGE_BASE_DIR}/metadata-envoy:${GCR_IMAGE_TAG}
  cat kustomization.yaml

  kubectl apply -k .
  popd
else
  # exclude SDK release tags
  KFP_LATEST_RELEASE=$(git tag --sort=v:refname | grep -v "sdk-" | tail -1)
  echo "Deploying KFP release $KFP_LATEST_RELEASE"

  # temporarily checkout last release tag
  git checkout $KFP_LATEST_RELEASE

  pushd ${KFP_MANIFEST_DIR}/cluster-scoped-resources
  kubectl apply -k .

  kubectl wait --for condition=established --timeout=60s crd/applications.app.k8s.io
  popd

  pushd ${KFP_MANIFEST_DIR}/dev
  kubectl apply -k .
  popd

  # go back to previous commit
  git checkout -
fi

# show current info
echo "Status of pods after kubectl apply"
kubectl get pods -n ${NAMESPACE}

# wait for all deployments to be successful
# note, after we introduce statefulset and daemonsets, we need to wait their rollout status here too
for deployment in $(kubectl get deployments -n ${NAMESPACE} -o name)
do
  kubectl rollout status $deployment -n ${NAMESPACE}
done

echo "Status of pods after rollouts are successful"
kubectl get pods -n ${NAMESPACE}

if [ "$ENABLE_WORKLOAD_IDENTITY" = true ]; then
  # Use static GSAs for testing, so we don't need to GC them.
  export SYSTEM_GSA="test-kfp-system"
  export USER_GSA="test-kfp-user"
  source "${DIR}/scripts/retry.sh"

  function setup_workload_identity {
    # Workaround for flakiness from gcp-workload-identity-setup.sh:
    # When two tests add iam policy bindings at the same time, one will fail because
    # there could be two concurrent changes.
    # Wait here randomly to reduce chance both scripts are run at the same time
    # between tests. Unless for testing scenario like this, it won't
    # meet the concurrent change issue.
    sleep $((RANDOM%30))
    yes | PROJECT_ID=$PROJECT RESOURCE_PREFIX=$TEST_CLUSTER NAMESPACE=$NAMESPACE \
      ${DIR}/../manifests/kustomize/gcp-workload-identity-setup.sh
  }
  retry setup_workload_identity

  retry gcloud projects add-iam-policy-binding $PROJECT \
    --member="serviceAccount:$SYSTEM_GSA@$PROJECT.iam.gserviceaccount.com" \
    --role="roles/editor"
  retry gcloud projects add-iam-policy-binding $PROJECT \
    --member="serviceAccount:$USER_GSA@$PROJECT.iam.gserviceaccount.com" \
    --role="roles/editor"

  source "$DIR/../manifests/kustomize/wi-utils.sh"
  # TODO(Bobgy): re-enable this after temporary flakiness is resolved.
  # verify_workload_identity_binding "pipeline-runner" $NAMESPACE
  sleep 30
fi

popd