chore: make api-proto to support go module style (#4402)

tweak on make api-proto to support go module style

Signed-off-by: Rafael Brito <rafa@stormforge.io>

.argo-ci/ci.yaml

@@ -1,87 +0,0 @@
-apiVersion: argoproj.io/v1alpha1
-kind: Workflow
-metadata:
-  generateName: argo-rollouts-ci-
-spec:
-  entrypoint: argo-rollouts-ci
-  arguments:
-    parameters:
-    - name: revision
-      value: master
-    - name: repo
-      value: https://github.com/argoproj/argo-rollouts.git
-
-  templates:
-  - name: argo-rollouts-ci
-    steps:
-    - - name: build
-        template: ci-dind
-        arguments:
-          parameters:
-          - name: cmd
-            value: make image
-      - name: test
-        template: ci-builder
-        arguments:
-          parameters:
-          - name: cmd
-            value: "dep ensure && make lint test && bash <(curl -s https://codecov.io/bash) -f coverage.out"
-
-  - name: ci-builder
-    inputs:
-      parameters:
-      - name: cmd
-      artifacts:
-      - name: code
-        path: /go/src/github.com/argoproj/argo-rollouts
-        git:
-          repo: "{{workflow.parameters.repo}}"
-          revision: "{{workflow.parameters.revision}}"
-    container:
-      image: argoproj/argo-rollouts-ci-builder:latest
-      command: [bash, -c]
-      args: ["{{inputs.parameters.cmd}}"]
-      workingDir: /go/src/github.com/argoproj/argo-rollouts
-      env:
-      - name: CODECOV_TOKEN
-        valueFrom:
-          secretKeyRef:
-            name: argo-rollouts-codecov-token
-            key: codecov-token
-      resources:
-        requests:
-          memory: 1024Mi
-          cpu: 200m
-    archiveLocation:
-      archiveLogs: true
-
-  - name: ci-dind
-    inputs:
-      parameters:
-      - name: cmd
-      artifacts:
-      - name: code
-        path: /go/src/github.com/argoproj/argo-rollouts
-        git:
-          repo: "{{workflow.parameters.repo}}"
-          revision: "{{workflow.parameters.revision}}"
-    container:
-      image: argoproj/argo-rollouts-ci-builder:latest
-      command: [sh, -c]
-      args: ["until docker ps; do sleep 3; done && {{inputs.parameters.cmd}}"]
-      workingDir: /go/src/github.com/argoproj/argo-rollouts
-      env:
-      - name: DOCKER_HOST
-        value: 127.0.0.1
-      resources:
-        requests:
-          memory: 1024Mi
-          cpu: 200m
-    sidecars:
-    - name: dind
-      image: docker:18.09-dind
-      securityContext:
-        privileged: true
-      mirrorVolumeMounts: true
-    archiveLocation:
-      archiveLogs: true

.chglog/CHANGELOG.tpl.md

@@ -0,0 +1,30 @@
+{{ range .Versions }}
+<a name="{{ .Tag.Name }}"></a>
+## {{ if .Tag.Previous }}[{{ .Tag.Name }}]({{ $.Info.RepositoryURL }}/compare/{{ .Tag.Previous.Name }}...{{ .Tag.Name }}){{ else }}{{ .Tag.Name }}{{ end }} ({{ datetime "2006-01-02" .Tag.Date }})
+
+{{ range .CommitGroups -}}
+### {{ .Title }}
+
+{{ range .Commits -}}
+* {{ if .Scope }}**{{ .Scope }}:** {{ end }}{{ .Subject }}
+{{ end }}
+{{ end -}}
+
+{{- if .RevertCommits -}}
+### Reverts
+
+{{ range .RevertCommits -}}
+* {{ .Revert.Header }}
+{{ end }}
+{{ end -}}
+
+{{- if .NoteGroups -}}
+{{ range .NoteGroups -}}
+### {{ .Title }}
+
+{{ range .Notes }}
+{{ .Body }}
+{{ end }}
+{{ end -}}
+{{ end -}}
+{{ end -}}

.chglog/config.yml

@@ -0,0 +1,28 @@
+style: github
+template: CHANGELOG.tpl.md
+info:
+  title: CHANGELOG
+  repository_url: https://github.com/argoproj/argo-rollouts
+options:
+  commits:
+    # filters:
+    #   Type:
+    #     - feat
+    #     - fix
+    #     - perf
+    #     - refactor
+  commit_groups:
+    # title_maps:
+    #   feat: Features
+    #   fix: Bug Fixes
+    #   perf: Performance Improvements
+    #   refactor: Code Refactoring
+  header:
+    pattern: "^(\\w*)(?:\\(([\\w\\$\\.\\-\\*\\s]*)\\))?\\:\\s(.*)$"
+    pattern_maps:
+      - Type
+      - Scope
+      - Subject
+  notes:
+    keywords:
+      - BREAKING CHANGE

.circleci/config.yml

@@ -1,48 +0,0 @@
-version: 2.1
-jobs:
-  build:
-    working_directory: /go/src/github.com/argoproj/argo-rollouts
-    docker:
-      # CircleCI Go images available at: https://hub.docker.com/r/circleci/golang/
-      - image: circleci/golang:1.13.1
-    resource_class: xlarge
-    environment:
-      TEST_RESULTS: /tmp/test-results
-    steps:
-      - checkout
-      - run: mkdir -p $TEST_RESULTS
-      - restore_cache:
-          keys:
-            - v3-pkg-cache
-      - run: go get github.com/jstemmer/go-junit-report
-      - run: go get github.com/golangci/golangci-lint/cmd/golangci-lint
-      - run:
-          command: dep ensure
-      - run:
-          name: Run tests
-          command: |
-            set -x
-            trap "go-junit-report <${TEST_RESULTS}/go-test.out > ${TEST_RESULTS}/go-test-report.xml" EXIT
-            make test | tee ${TEST_RESULTS}/go-test.out
-      - store_test_results:
-          path: /tmp/test-results
-      - run:
-          name: Uploading code coverage
-          command: bash <(curl -s https://codecov.io/bash) -f coverage.out
-          # This occasionally takes 2m
-          background: true
-      - run:
-          command: make lint
-      - run:
-          name: Check nothing has changed after lint
-          command: git diff --exit-code
-      - save_cache:
-          key: v3-pkg-cache
-          paths:
-            - /go/pkg
-            - ~/.cache/go-build
-workflows:
-  version: 2
-  workflow:
-    jobs:
-      - build

.codecov.yml

@@ -6,4 +6,18 @@ coverage:
         threshold: 0.1
     project:
       default:
-        threshold: 0.1
+        threshold: 0.1
+ignore:
+  - 'pkg/apis/rollouts/v1alpha1'
+  - 'test'
+  - '**/*.pb.go'
+  - '**/*.pb.gw.go'
+  - '**/*generated.go'
+  - '**/*generated.deepcopy.go'
+  - '**/*_test.go'
+  - 'pkg/apis/client/.*'
+  - 'pkg/client/.*'
+  - 'vendor/.*'
+  - '**/mocks/*'
+  - 'hack/gen-crd-spec/main.go'
+  - 'hack/gen-docs/main.go'

.dockerignore

@@ -1,12 +1,11 @@
-# Prevent vendor directory from being copied to ensure we are not not pulling unexpected cruft from 
-# a user's workspace, and are only building off of what is locked by dep.
 .vscode/
 .idea/
 .DS_Store
-vendor/
 dist/
+ui/dist/
 *.iml
 # delve debug binaries
 cmd/**/debug
 debug.test
 coverage.*
+ui/node_modules

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,52 @@
+---
+name: Bug report
+about: Create a report to help us improve
+title: ''
+labels: 'bug'
+assignees: ''
+---
+
+<!-- If you are trying to resolve an environment-specific issue or have a one-off question about the edge case that does not require a feature then please consider asking a question in argo rollouts slack [channel](https://argoproj.github.io/community/join-slack). -->
+
+Checklist:
+
+* [ ] I've included steps to reproduce the bug.
+* [ ] I've included the version of argo rollouts.
+
+**Describe the bug**
+
+<!-- A clear and concise description of what the bug is. -->
+
+**To Reproduce**
+
+<!-- A list of the steps required to reproduce the issue. Best of all, give us the URL to a repository that exhibits this issue. -->
+
+**Expected behavior**
+
+<!-- A clear and concise description of what you expected to happen. -->
+
+**Screenshots**
+
+<!-- If applicable, add screenshots to help explain your problem. -->
+
+**Version**
+
+<!-- What version of argo rollouts controller are you running? -->
+
+**Logs**
+
+```
+# Paste the logs from the rollout controller
+
+# Logs for the entire controller:
+kubectl logs -n argo-rollouts deployment/argo-rollouts
+
+# Logs for a specific rollout:
+kubectl logs -n argo-rollouts deployment/argo-rollouts | grep rollout=<ROLLOUTNAME
+```
+
+---
+<!-- Issue Author: Don't delete this message to encourage other users to support your issue! -->
+**Message from the maintainers**:
+
+Impacted by this bug? Give it a 👍. We prioritize the issues with the most 👍.

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,12 @@
+blank_issues_enabled: false
+
+contact_links:
+  - name: Have you read the docs?
+    url: https://argo-rollouts.readthedocs.io/
+    about: Much help can be found in the docs
+  - name: Ask a question
+    url: https://github.com/argoproj/argo-rollouts/discussions/new
+    about: Ask a question or start a discussion about Argo Rollouts
+  - name: Chat on Slack
+    url: https://argoproj.github.io/community/join-slack
+    about: Maybe chatting with the community can help

.github/ISSUE_TEMPLATE/enhancement_proposal.md

@@ -0,0 +1,18 @@
+---
+name: Enhancement proposal
+about: Propose an enhancement for this project
+labels: 'enhancement'
+---
+# Summary
+
+What change needs making?
+
+# Use Cases
+
+When would you use this?
+
+---
+<!-- Issue Author: Don't delete this message to encourage other users to support your issue! -->
+**Message from the maintainers**:
+
+Impacted by this bug? Give it a 👍. We prioritize the issues with the most 👍.

.github/dependabot.yml

@@ -0,0 +1,13 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "daily"
+    ignore:
+    - dependency-name: k8s.io/*
+
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/no-response.yml

@@ -0,0 +1,13 @@
+# Configuration for probot-no-response - https://github.com/probot/no-response
+
+# Number of days of inactivity before an Issue is closed for lack of response
+daysUntilClose: 14
+# Label requiring a response
+responseRequiredLabel: more-information-needed
+# Comment to post when closing an Issue for lack of response. Set to `false` to disable
+closeComment: >
+  This issue has been automatically closed because there has been no response
+  to our request for more information from the original author. With only the
+  information that is currently in the issue, we don't have enough information
+  to take action. Please reach out if you have or find the answers we need so
+  that we can investigate further.

.github/pull_request_template.md

@@ -0,0 +1,8 @@
+Checklist:
+
+* [ ] Either (a) I've created an [enhancement proposal](https://github.com/argoproj/argo-rollouts/issues/new/choose) and discussed it with the community, (b) this is a bug fix, or (c) this is a chore.
+* [ ] The title of the PR is (a) [conventional](https://www.conventionalcommits.org/en/v1.0.0/) with a list of types and scopes found [here](https://github.com/argoproj/argo-rollouts/blob/master/.github/workflows/pr-title-check.yml), (b) states what changed, and (c) suffixes the related issues number. E.g. `"fix(controller): Updates such and such. Fixes #1234"`.  
+* [ ] I've signed my commits with [DCO](https://github.com/argoproj/argoproj/blob/main/community/CONTRIBUTING.md#legal)
+* [ ] I have written unit and/or e2e tests for my change. PRs without these are unlikely to be merged.
+* [ ] My builds are green. Try syncing with master if they are not. 
+* [ ] My organization is added to [USERS.md](https://github.com/argoproj/argo-rollouts/blob/master/USERS.md).

.github/workflows/README.md

@@ -0,0 +1,39 @@
+# Workflows
+
+| Workflow            | Description                                                     |
+|---------------------|-----------------------------------------------------------------|
+| changelog.yml       | Updates changelog when a release is published                   |
+| codeql.yaml         | CodeQL analysis                                                 |
+| docker-publish.yaml | Build container image for PR's & publish for push events        |
+| image-reuse.yaml    | Build, push, and Sign container images                          |
+| go.yaml             | lint, build, codegen                                            |
+| pr-title-check.yaml | Lint PR for semantic information                                |
+| init-release.yaml   | Build manifests and version then create a PR for release branch |
+| release.yaml        | Build images, cli-binaries, provenances, and post actions       |
+
+
+# Reusable workflows
+
+## image-reuse.yaml
+
+- The resuable workflow can be used to publish or build images with multiple container registries(Quay,GHCR, dockerhub), and then sign them with cosign when an image is published.
+- A GO version `must` be specified e.g. 1.19
+- The image name for each registry *must* contain the tag. Note: multiple tags are allowed for each registry using a CSV type.
+- Multiple platforms can be specified e.g. linux/amd64,linux/arm64
+- Images are not published by default. A boolean value must be set to `true` to push images.
+- An optional target can be specified.
+
+| Inputs            | Description                         | Type        | Required | Defaults        |
+|-------------------|-------------------------------------|-------------|----------|-----------------|
+| go-version        | Version of Go to be used            | string      | true     | none            |
+| quay_image_name   | Full image name and tag             | CSV, string | false    | none            |
+| ghcr_image_name   | Full image name and tag             | CSV, string | false    | none            |
+| docker_image_name | Full image name and tag             | CSV, string | false    | none            |
+| platforms         | Platforms to build (linux/amd64)    | CSV, string | false    | linux/amd64     |
+| push              | Whether to push image/s to registry | boolean     | false    | false           |
+| target            | Target build stage                  | string      | false    | none            |
+
+| Outputs     | Description                              | Type  |
+|-------------|------------------------------------------|-------|
+|image-digest | Image digest of image container created  | string|
+

.github/workflows/changelog.yml

@@ -0,0 +1,34 @@
+name: Update Changelog
+on:
+  release:
+    types: [published]
+permissions:
+  contents: read
+
+jobs:
+  updateChangelog:
+    permissions:
+      contents: write  # for peter-evans/create-pull-request to create branch
+      pull-requests: write  # for peter-evans/create-pull-request to create a PR
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Update Changelog
+        run: |
+          curl -o git-chglog.tar.gz -L https://github.com/git-chglog/git-chglog/releases/download/v0.15.1/git-chglog_0.15.1_linux_amd64.tar.gz
+          tar -zxvf git-chglog.tar.gz
+          chmod u+x git-chglog
+          ./git-chglog --sort semver -o CHANGELOG.md v1.3.1..
+          rm git-chglog
+      - name: Create Pull Request
+        uses: peter-evans/create-pull-request@v7
+        with:
+          commit-message: update changelog
+          title: "docs: Update Changelog"
+          body: Update changelog to reflect release changes
+          branch: update-changelog
+          base: master
+          add-paths: |
+            CHANGELOG.md

.github/workflows/codeql.yml

@@ -0,0 +1,56 @@
+name: "Code scanning - action"
+
+on:
+  push:
+    branches: [master]
+  pull_request:
+    branches: [master]
+    paths-ignore:
+      - '**/*.md'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:  # added using https://github.com/step-security/secure-workflows
+  contents: read
+
+jobs:
+  CodeQL-Build:
+
+    # CodeQL runs on ubuntu-latest and windows-latest
+    permissions:
+      actions: read  # for github/codeql-action/init to get workflow details
+      contents: read  # for actions/checkout to fetch code
+      security-events: write  # for github/codeql-action/autobuild to send a status report
+    runs-on: ubuntu-latest
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        # Override language selection by uncommenting this and choosing your languages
+        # with:
+        #   languages: go, javascript, csharp, python, cpp, java
+
+      # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+      # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+
+      # ℹ️ Command-line programs to run using the OS shell.
+      # 📚 https://git.io/JvXDl
+
+      # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+      #    and modify them (or add more) to build your code if your project
+      #    uses a compiled language
+
+      #- run: |
+      #   make bootstrap
+      #   make release
+
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

.github/workflows/docker-publish.yml

@@ -0,0 +1,93 @@
+name: Docker
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+
+  # Run tests for any PRs.
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions: {}
+
+jobs:
+  set-vars:
+    permissions:
+      contents: read
+    runs-on: ubuntu-latest
+    outputs:
+      controller-meta-tags: ${{ steps.controller-meta.outputs.tags }}
+      plugin-meta-tags: ${{ steps.plugin-meta.outputs.tags }}
+      platforms: ${{ steps.platform-matrix.outputs.platform-matrix }}
+
+    steps:
+      - name: Docker meta (controller)
+        id: controller-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            quay.io/argoproj/argo-rollouts
+          tags: |
+            type=ref,event=branch,enable=${{ github.ref != 'refs/heads/master'}}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
+
+      - name: Docker meta (plugin)
+        id: plugin-meta
+        uses: docker/metadata-action@v5
+        with:
+          images: |
+            quay.io/argoproj/kubectl-argo-rollouts
+          tags: |
+            type=ref,event=branch,enable=${{ github.ref != 'refs/heads/master'}}
+            type=raw,value=latest,enable=${{ github.ref == 'refs/heads/master' }}
+
+      # avoid building linux/arm64 for PRs since it takes so long
+      - name: Set Platform Matrix
+        id: platform-matrix
+        run: |
+          PLATFORM_MATRIX=linux/amd64
+          if [[ "${{ github.event_name }}" == "push" || "${{ contains(github.event.pull_request.labels.*.name, 'test-arm-image') }}" == "true" ]]
+          then
+            PLATFORM_MATRIX=$PLATFORM_MATRIX,linux/arm64
+          fi
+          echo "platform-matrix=$PLATFORM_MATRIX" >> $GITHUB_OUTPUT
+
+  build-and-push-controller-image:
+    needs: [set-vars]
+    permissions:
+      contents: read
+      packages: write # for pushing packages to GHCR, which is used by cd.apps.argoproj.io to avoid polluting Quay with tags
+      id-token: write # for creating OIDC tokens for signing.
+    uses: ./.github/workflows/image-reuse.yaml
+    with:
+      quay_image_name: ${{ needs.set-vars.outputs.controller-meta-tags }}
+      # Note: cannot use env variables to set go-version (https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations)
+      go-version: '1.23'
+      platforms: ${{ needs.set-vars.outputs.platforms }}
+      push: ${{ github.event_name != 'pull_request' }}
+    secrets:
+      quay_username: ${{ secrets.QUAY_USERNAME }}
+      quay_password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+  build-and-push-plugin-image:
+    needs: [set-vars]
+    permissions:
+      contents: read
+      packages: write # for pushing packages to GHCR, which is used by cd.apps.argoproj.io to avoid polluting Quay with tags
+      id-token: write # for creating OIDC tokens for signing.
+    uses: ./.github/workflows/image-reuse.yaml
+    with:
+      quay_image_name: ${{ needs.set-vars.outputs.plugin-meta-tags }}
+      # Note: cannot use env variables to set go-version (https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations)
+      go-version: '1.23'
+      platforms: ${{ needs.set-vars.outputs.platforms }}
+      push: ${{ github.event_name != 'pull_request' }}
+      target: kubectl-argo-rollouts
+    secrets:
+      quay_username: ${{ secrets.QUAY_USERNAME }}
+      quay_password: ${{ secrets.QUAY_ROBOT_TOKEN }}

.github/workflows/gh-pages.yaml

@@ -0,0 +1,39 @@
+name: Deploy-Docs
+
+on:
+  push:
+    branches:
+      - master
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  deploy:
+    permissions:
+      contents: write # for peaceiris/actions-gh-pages to push pages branch
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - name: Setup Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: 3.x
+      - name: Set up Go
+        uses: actions/setup-go@v5.4.0
+        with:
+          go-version: '1.23'
+      - name: build
+        run: |
+          pip install mkdocs mkdocs_material
+          make docs
+          mkdocs build
+      - name: Deploy
+        uses: peaceiris/actions-gh-pages@v4
+        with:
+          github_token: ${{ secrets.GITHUB_TOKEN }}
+          publish_dir: ./site

.github/workflows/go.yml

@@ -1,28 +1,112 @@
 name: Go
-on: [push]
-jobs:
+on:
+  push:
+    branches:
+      - 'master'
+      - 'release-*'
+  pull_request:
+    branches:
+      - 'master'
+env:
+  # Golang version to use across CI steps
+  GOLANG_VERSION: '1.23'
 
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  lint-go:
+    permissions:
+      contents: read # for actions/checkout to fetch code
+      pull-requests: read # for golangci/golangci-lint-action to fetch pull requests
+    name: Lint Go code
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v5.4.0
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Run golangci-lint
+        uses: golangci/golangci-lint-action@v8
+        with:
+          version: v2.1.6
+          args: --timeout 6m
   build:
     name: Build
     runs-on: ubuntu-latest
     steps:
+      - name: Set up Go
+        uses: actions/setup-go@v5.4.0
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+        id: go
 
-    - name: Set up Go 1.13
-      uses: actions/setup-go@v1
-      with:
-        go-version: 1.13
-      id: go
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v1
+      - name: Restore go build cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/go-build
+          key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
 
-    - name: Get dependencies
-      run: |
-        go get -v -t -d ./...
-        if [ -f Gopkg.toml ]; then
-            curl https://raw.githubusercontent.com/golang/dep/master/install.sh | sh
-            dep ensure
-        fi
+      - name: Download all Go modules
+        run: |
+          go mod download
 
-    - name: Test
-      run: go test -failfast -covermode=count -coverprofile=coverage.out
+      - name: Compile all packages
+        run: make controller plugin
+
+  codegen:
+    name: Verify Codegen
+    runs-on: ubuntu-latest
+    env:
+      GOPATH: /home/runner/go
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+      - name: Setup Golang
+        uses: actions/setup-go@v5.4.0
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+      # k8s codegen generates files into GOPATH location instead of the GitHub git checkout location
+      # This symlink is necessary to ensure that `git diff` detects changes
+      - name: Create symlink in GOPATH
+        run: |
+          mkdir -p ~/go/src/github.com/argoproj
+          ln -s $(pwd) ~/go/src/github.com/argoproj/argo-rollouts
+      - uses: actions/cache@v4
+        with:
+          path: /home/runner/.cache/go-build
+          key: GOCACHE-${{ hashFiles('**/go.mod') }}
+      - uses: actions/cache@v4
+        with:
+          path: /home/runner/go/pkg/mod
+          key: GOMODCACHE-${{ hashFiles('**/go.mod') }}
+      - uses: actions/cache@v4
+        with:
+          path: /home/runner/go/bin
+          key: go-bin-v1-${{ hashFiles('**/go.mod') }}
+      - name: Install protoc
+        run: |
+          make install-toolchain
+      - name: Add ~/go/bin to PATH
+        run: |
+          echo "/home/runner/go/bin" >> $GITHUB_PATH
+      - name: Add /usr/local/bin to PATH
+        run: |
+          echo "/usr/local/bin" >> $GITHUB_PATH
+
+      - name: Run codegen
+        run: |
+          make go-mod-vendor
+          make codegen
+
+      - name: Ensure nothing changed
+        run: git diff --exit-code

.github/workflows/image-reuse.yaml

@@ -0,0 +1,153 @@
+name: Publish and Sign Container Image
+on:
+  workflow_call:
+    inputs:
+      go-version:
+        required: true
+        type: string
+      quay_image_name:
+        required: false
+        type: string
+      ghcr_image_name:
+        required: false
+        type: string
+      docker_image_name:
+        required: false
+        type: string
+      platforms:
+        required: true
+        type: string
+        default: linux/amd64
+      push:
+        required: true
+        type: boolean
+        default: false
+      target:
+        required: false
+        type: string
+
+    secrets:
+      quay_username:
+        required: false
+      quay_password:
+        required: false
+      ghcr_username:
+        required: false
+      ghcr_password:
+        required: false
+      docker_username:
+        required: false
+      docker_password:
+        required: false
+
+    outputs:
+      image-digest:
+        description: "sha256 digest of container image"
+        value: ${{ jobs.publish.outputs.image-digest }}
+
+permissions: {}
+
+jobs:
+  publish:
+    permissions:
+      contents: read
+      packages: write # Used to push images to `ghcr.io` if used.
+      id-token: write # Needed to create an OIDC token for keyless signing
+    runs-on: ubuntu-22.04
+    outputs:
+      image-digest: ${{ steps.image.outputs.digest }}
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4 # v3.3.0
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+        if: ${{ github.ref_type == 'tag'}}
+
+      - name: Checkout code
+        uses: actions/checkout@v4 # v3.3.0
+        if: ${{ github.ref_type != 'tag'}}
+
+      - name: Setup Golang
+        uses: actions/setup-go@v5.4.0 # v3.5.0
+        with:
+          go-version: ${{ inputs.go-version }}
+
+      - name: Install cosign
+        uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
+        with:
+          cosign-release: 'v2.2.0'
+
+      - uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
+      - uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Setup tags for container image as a CSV type
+        run: |
+          IMAGE_TAGS=$(for str in \
+            ${{ inputs.quay_image_name }} \
+            ${{ inputs.ghcr_image_name }} \
+            ${{ inputs.docker_image_name}}; do
+            echo -n "${str}",;done | sed 's/,$//')
+
+          echo $IMAGE_TAGS
+          echo "TAGS=$IMAGE_TAGS" >> $GITHUB_ENV
+
+      - name: Setup image namespace for signing, strip off the tag
+        run: |
+          TAGS=$(for tag in \
+            ${{ inputs.quay_image_name }} \
+            ${{ inputs.ghcr_image_name }} \
+            ${{ inputs.docker_image_name}}; do
+            echo -n "${tag}" | awk -F ":" '{print $1}' -;done)
+          
+            echo $TAGS
+            echo 'SIGNING_TAGS<<EOF' >> $GITHUB_ENV
+            echo $TAGS >> $GITHUB_ENV
+            echo 'EOF' >> $GITHUB_ENV
+
+      - name: Login to Quay.io
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: quay.io
+          username: ${{ secrets.quay_username }}
+          password: ${{ secrets.quay_password }}
+        if: ${{ inputs.quay_image_name && inputs.push }}
+
+      - name: Login to GitHub Container Registry
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          registry: ghcr.io
+          username: ${{ secrets.ghcr_username }}
+          password: ${{ secrets.ghcr_password }}
+        if: ${{ inputs.ghcr_image_name && inputs.push }}
+
+      - name: Login to dockerhub Container Registry
+        uses: docker/login-action@74a5d142397b4f367a81961eba4e8cd7edddf772 # v3.4.0
+        with:
+          username: ${{ secrets.docker_username }}
+          password: ${{ secrets.docker_password }}
+        if: ${{ inputs.docker_image_name && inputs.push }}
+
+      - name: Build and push container image
+        id: image
+        uses: docker/build-push-action@263435318d21b8e681c14492fe198d362a7d2c83 #v6.18.0
+        with:
+          context: .
+          platforms: ${{ inputs.platforms }}
+          push: ${{ inputs.push }}
+          tags: ${{ env.TAGS }}
+          target: ${{ inputs.target }}
+          provenance: false
+          sbom: false
+ 
+      - name: Sign container images
+        run: |
+          for signing_tag in $SIGNING_TAGS; do
+            cosign sign \
+            -a "repo=${{ github.repository }}" \
+            -a "workflow=${{ github.workflow }}" \
+            -a "sha=${{ github.sha }}" \
+            --yes \
+            "$signing_tag"@${{ steps.image.outputs.digest }}
+          done
+        if: ${{ inputs.push }}

.github/workflows/pr-title-check.yml

@@ -0,0 +1,53 @@
+name: "Lint PR"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+permissions:
+  contents: read
+
+jobs:
+  main:
+    permissions:
+      pull-requests: read  # for amannn/action-semantic-pull-request to analyze PRs
+      statuses: write  # for amannn/action-semantic-pull-request to mark status of analyzed PR
+    name: Validate PR title
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@v5
+        with:
+          # Configure which types are allowed (newline delimited).
+          # Default: https://github.com/commitizen/conventional-commit-types
+          types: |
+            feat
+            fix
+            docs
+            style
+            refactor
+            perf
+            test
+            build
+            ci
+            chore
+            revert
+
+          # Configure which scopes are allowed (newline delimited).
+          scopes: |
+            controller
+            dashboard
+            trafficrouting
+            analysis
+            metricprovider
+            experiments
+            deps
+            example
+            cli
+
+          # Configure that a scope must always be provided.
+          requireScope: false
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/release.yaml

@@ -0,0 +1,269 @@
+name: Release
+on:
+  push:
+    tags:
+      - 'v*'
+
+permissions: {}
+
+env:
+  GOLANG_VERSION: '1.23' # Note: go-version must also be set in job controller-image.with.go-version & plugin-image.with.go-version.
+
+jobs:
+  controller-image:
+    permissions:
+      contents: read
+      packages: write # Required and used to push images to `ghcr.io` if used.
+      id-token: write # For creating OIDC tokens for signing.
+    uses: ./.github/workflows/image-reuse.yaml
+    with:
+      quay_image_name: quay.io/argoproj/argo-rollouts:${{ github.ref_name }}
+      # Note: cannot use env variables to set go-version (https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations)
+      go-version: '1.23'
+      platforms: linux/amd64,linux/arm64
+      push: true
+    secrets:
+      quay_username: ${{ secrets.QUAY_USERNAME }}
+      quay_password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+  plugin-image:
+    permissions:
+      contents: read
+      packages: write # Required and used to push images to `ghcr.io` if used.
+      id-token: write # For creating OIDC tokens for signing.
+    uses: ./.github/workflows/image-reuse.yaml
+    with:
+      quay_image_name: quay.io/argoproj/kubectl-argo-rollouts:${{ github.ref_name }}
+      # Note: cannot use env variables to set go-version (https://docs.github.com/en/actions/using-workflows/reusing-workflows#limitations)
+      go-version: '1.23'
+      platforms: linux/amd64,linux/arm64
+      push: true
+      target: kubectl-argo-rollouts
+    secrets:
+      quay_username: ${{ secrets.QUAY_USERNAME }}
+      quay_password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+  controller-image-provenance:
+    needs:
+      - controller-image
+    permissions:
+      actions: read # for detecting the Github Actions environment.
+      id-token: write # for creating OIDC tokens for signing.
+      packages: write # for uploading attestations. (https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#known-issues)
+    # Must be refernced by a tag. https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#referencing-the-slsa-generator
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.1.0
+    with:
+      image: quay.io/argoproj/argo-rollouts
+      digest: ${{ needs.controller-image.outputs.image-digest }}
+    secrets:
+      registry-username: ${{ secrets.QUAY_USERNAME }}
+      registry-password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+  plugin-image-provenance:
+    needs:
+      - plugin-image
+    permissions:
+      actions: read # for detecting the Github Actions environment.
+      id-token: write # for creating OIDC tokens for signing.
+      packages: write # for uploading attestations. (https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#known-issues)
+    # Must be refernced by a tag. https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#referencing-the-slsa-generator
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_container_slsa3.yml@v2.1.0
+    with:
+      image: quay.io/argoproj/kubectl-argo-rollouts
+      digest: ${{ needs.plugin-image.outputs.image-digest }}
+    secrets:
+      registry-username: ${{ secrets.QUAY_USERNAME }}
+      registry-password: ${{ secrets.QUAY_ROBOT_TOKEN }}
+
+  release-artifacts:
+    permissions:
+      contents: write # for softprops/action-gh-release to create GitHub release
+    runs-on: ubuntu-latest
+    outputs:
+      hashes: ${{ steps.hash.outputs.hashes }}
+
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v4 # v3.5.2
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup Golang
+        uses: actions/setup-go@v5.4.0 # v4.0.1
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@29109295f81e9208d7d86ff1c6c12d2833863392 # v3.6.0
+
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@e468171a9de216ec08956ac3ada2f0791b6bd435 # v3.11.1
+
+      - name: Generate release artifacts
+        run: |
+          make release-plugins
+          make checksums
+          make manifests IMAGE_TAG=${{ github.ref_name }}
+
+      - name: Draft release
+        uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v0.1.15
+        with:
+          tag_name: ${{ github.event.inputs.tag }}
+          draft: true
+          files: |
+            dist/kubectl-argo-rollouts-linux-amd64
+            dist/kubectl-argo-rollouts-linux-arm64
+            dist/kubectl-argo-rollouts-darwin-amd64
+            dist/kubectl-argo-rollouts-darwin-arm64
+            dist/kubectl-argo-rollouts-windows-amd64
+            dist/argo-rollouts-checksums.txt
+            manifests/dashboard-install.yaml
+            manifests/install.yaml
+            manifests/namespace-install.yaml
+            manifests/notifications-install.yaml
+            docs/features/kustomize/rollout_cr_schema.json
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Generate hashes for provenance
+        id: hash
+        run: |
+          echo "hashes=$(sha256sum ./dist/kubectl-argo-rollouts-* ./manifests/*.yaml | base64 -w0)" >> "$GITHUB_OUTPUT"
+
+  release-artifacts-provenance:
+    needs:
+      - release-artifacts
+    permissions:
+      actions: read # for detecting the Github Actions environment
+      id-token: write # Needed for provenance signing and ID
+      contents: write #  Needed for release uploads
+    # Must be refernced by a tag. https://github.com/slsa-framework/slsa-github-generator/blob/main/internal/builders/container/README.md#referencing-the-slsa-generator
+    uses: slsa-framework/slsa-github-generator/.github/workflows/generator_generic_slsa3.yml@v2.1.0
+    with:
+      base64-subjects: '${{ needs.release-artifacts.outputs.hashes }}'
+      provenance-name: 'argo-rollouts.intoto.jsonl'
+      upload-assets: true
+      draft-release: true
+
+  generate-sbom:
+    name: Create Sbom and sign assets
+    needs:
+      - release-artifacts
+      - release-artifacts-provenance
+    permissions:
+      contents: write # Needed for release uploads
+      id-token: write # Needed for signing Sbom
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4 # v3.3.0
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup Golang
+        uses: actions/setup-go@v5.4.0 # v4.0.0
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+
+      - name: Install cosign
+        uses: sigstore/cosign-installer@3454372f43399081ed03b604cb2d021dabca52bb # v3.8.2
+        with:
+          cosign-release: 'v2.2.0'
+
+      - name: Generate SBOM (spdx)
+        id: spdx-builder
+        env:
+          # defines the spdx/spdx-sbom-generator version to use.
+          SPDX_GEN_VERSION: v0.0.13
+          # defines the sigs.k8s.io/bom version to use.
+          SIGS_BOM_VERSION: v0.2.1
+          # comma delimited list of project relative folders to inspect for package
+          # managers (gomod, yarn, npm).
+          PROJECT_FOLDERS: '.,./ui'
+          # full qualified name of the container image to be inspected
+          CONTAINER_IMAGE: quay.io/argoproj/argo-rollouts:${{ github.event.inputs.tag }}
+
+        run: |
+          yarn install --cwd ./ui
+          go install github.com/spdx/spdx-sbom-generator/cmd/generator@$SPDX_GEN_VERSION
+          go install sigs.k8s.io/bom/cmd/bom@$SIGS_BOM_VERSION
+
+          # Generate SPDX for project dependencies analyzing package managers
+          for folder in $(echo $PROJECT_FOLDERS | sed "s/,/ /g")
+          do
+            generator -p $folder -o /tmp
+          done
+
+          # Generate SPDX for binaries analyzing the container image
+          if [[ ! -z CONTAINER_IMAGE ]]; then
+            bom generate -o /tmp/bom-docker-image.spdx -i $CONTAINER_IMAGE
+          fi
+
+          cd /tmp && tar -zcf sbom.tar.gz *.spdx
+
+      - name: Sign SBOM
+        run: |
+          cosign sign-blob \
+            --output-certificate=/tmp/sbom.tar.gz.pem \
+            --output-signature=/tmp/sbom.tar.gz.sig \
+            --yes \
+            /tmp/sbom.tar.gz
+
+      - name: Upload SBOM and signature assets
+        uses: softprops/action-gh-release@da05d552573ad5aba039eaac05058a918a7bf631 # v0.1.15
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        with:
+          tag_name: ${{ github.ref_name }}
+          draft: true
+          files: |
+            /tmp/sbom.tar.*
+
+  post-release:
+    needs:
+      - release-artifacts
+      - generate-sbom
+    permissions:
+      contents: write # Needed to push commit to update stable tag
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4 # v3.3.0
+        with:
+          fetch-depth: 0
+          token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Setup Git author information
+        run: |
+          set -ue
+          git config --global user.email 'ci@argoproj.com'
+          git config --global user.name 'CI'
+
+      - name: Check if tag is the latest version and not a pre-release
+        run: |
+          set -xue
+          # Fetch all tag information
+          git fetch --prune --tags --force
+
+          LATEST_TAG=$(git -c 'versionsort.suffix=-rc' tag --list --sort=version:refname | tail -n1)
+
+          PRE_RELEASE=false
+          # Check if latest tag is a pre-release
+          if echo $LATEST_TAG | grep -E -- '-rc[0-9]+$';then
+            PRE_RELEASE=true
+          fi
+
+          # Ensure latest tag matches github.ref_name & not a pre-release
+          if [[ $LATEST_TAG == ${{ github.ref_name }} ]] && [[ $PRE_RELEASE != 'true' ]];then
+            echo "TAG_STABLE=true" >> $GITHUB_ENV
+          else
+            echo "TAG_STABLE=false" >> $GITHUB_ENV
+          fi
+
+      - name: Update stable tag to latest version
+        run: |
+          git tag -f stable ${{ github.ref_name }}
+          git push -f origin stable
+        if: ${{ env.TAG_STABLE == 'true' }}

.github/workflows/stale-issues-pr.yml

@@ -0,0 +1,31 @@
+name: 'Close stale issues and PRs'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9
+        with:
+          operations-per-run: 250
+          stale-issue-message: 'This issue is stale because it has been open 60 days with no activity.'
+          stale-pr-message: 'This PR is stale because it has been open 90 days with no activity.'
+          close-issue-message: 'This issue was closed because it has been stalled for 7 days with no activity.'
+          close-pr-message: 'This PR was closed because it has been stalled for 14 days with no activity.'
+          days-before-issue-stale: 60
+          days-before-pr-stale: 90
+          days-before-issue-close: -1
+          days-before-pr-close: -1
+          stale-issue-label: 'no-issue-activity'
+          exempt-issue-labels: 'awaiting-approval,work-in-progress'
+          stale-pr-label: 'no-pr-activity'
+          exempt-pr-labels: 'awaiting-approval,work-in-progress'
+          exempt-all-milestones: true

.github/workflows/testing-results.yml

@@ -0,0 +1,49 @@
+# use separate workflow to support fork repositories and dependabot branches when publishing test results: see https://github.com/EnricoMi/publish-unit-test-result-action#support-fork-repositories-and-dependabot-branches
+name: Testing Results
+
+on:
+  workflow_run:
+    workflows: ["Testing"]
+    types:
+      - completed
+permissions: {}
+
+jobs:
+  test-results:
+    name: "${{ github.event.workflow.name }} Test Results"
+    runs-on: ubuntu-latest
+    if: github.event.workflow_run.conclusion != 'skipped'
+    permissions:
+      checks: write
+      pull-requests: write
+      actions: read
+    steps:
+      - name: Download and Extract Artifacts
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          run-id: ${{ github.event.workflow_run.id }}
+          path: artifacts
+          github-token: ${{ secrets.GITHUB_TOKEN }}
+
+      - name: Publish E2E Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: "Published E2E Test Results"
+          compare_to_earlier_commit: false
+          test_changes_limit: 0
+          fail_on: "errors"
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: artifacts/Event File/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: "artifacts/**/junit-e2e-test.xml"
+      - name: Publish Unit Test Results
+        uses: EnricoMi/publish-unit-test-result-action@v2
+        with:
+          check_name: "Published Unit Test Results"
+          compare_to_earlier_commit: false
+          test_changes_limit: 0
+          fail_on: "errors"
+          commit: ${{ github.event.workflow_run.head_sha }}
+          event_file: artifacts/Event File/event.json
+          event_name: ${{ github.event.workflow_run.event }}
+          files: "artifacts/**/junit-unit-test.xml"

.github/workflows/testing.yaml

@@ -0,0 +1,195 @@
+name: Testing
+
+on:
+  push:
+    branches:
+      - 'master'
+      - 'release-*'
+  pull_request:
+    branches:
+      - 'master'
+      - 'release-*'
+  workflow_dispatch:
+    inputs:
+      debug_enabled:
+        description: 'Run the build with tmate debugging enabled (https://github.com/marketplace/actions/debugging-with-tmate)'
+        required: false
+        default: false
+env:
+  # Golang version to use across CI steps
+  GOLANG_VERSION: '1.23'
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.ref }}
+  cancel-in-progress: true
+
+permissions:
+  contents: read
+
+jobs:
+  event_file:
+    name: 'Event File'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Upload
+        uses: actions/upload-artifact@v4
+        with:
+          name: Event File
+          path: ${{ github.event_path }}
+  test-unit:
+    name: Run unit tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v5.4.0
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+        id: go
+
+      - name: Check out code into the Go module directory
+        uses: actions/checkout@v4
+
+      - name: Restore go build cache
+        uses: actions/cache@v4
+        with:
+          path: ~/.cache/go-build
+          key: ${{ runner.os }}-go-build-v1-${{ github.run_id }}
+
+      - name: Download all Go modules
+        run: |
+          go mod download
+      - name: Test
+        run: make test-unit
+
+      - name: Upload Unit Test Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: Unit Test Results
+          path: |
+            junit-unit-test.xml
+      - name: Generate code coverage artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-output-unit
+          path: coverage-output-unit
+
+  test-e2e:
+    strategy:
+      fail-fast: false
+      matrix:
+        kubernetes:
+          - version: '1.28'
+            latest: false
+          - version: '1.29'
+            latest: false
+          - version: '1.30'
+            latest: false
+          - version: '1.31'
+            latest: true
+    name: Run end-to-end tests
+    runs-on: ubuntu-latest
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v5.4.0
+        with:
+          go-version: '1.23'
+      - uses: actions/checkout@v4
+      - name: Setup k3s
+        env:
+          INSTALL_K3S_CHANNEL: v${{ matrix.kubernetes.version }}
+        run: |
+          curl -sfL https://get.k3s.io | sh -
+          sudo mkdir ~/.kube
+          sudo cp /etc/rancher/k3s/k3s.yaml ~/.kube/config
+          sudo chmod 755 ~/.kube/config
+          kubectl version
+          kubectl create ns argo-rollouts
+      - uses: actions/cache@v4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-go-
+      - name: Download cache
+        run: go mod download
+      - name: Install CRDs
+        run: |
+          kubectl apply -k  manifests/crds
+          kubectl apply -f  test/e2e/crds
+      - name: Start controller
+        run: make start-e2e 2>&1 | sed -r "s/[[:cntrl:]]\[[0-9]{1,3}m//g" > /tmp/e2e-controller.log &
+      - name: Setup tmate session
+        uses: mxschmitt/action-tmate@v3
+        if: ${{ github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true'}}
+      - name: Run e2e tests
+        run: |
+          make test-e2e
+        if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true') }}
+      - name: Stop e2e tests controller
+        run: |
+          pgrep -f go-build -a
+          pkill -f go-build
+          sleep 5
+          echo "done stopping process"
+          ls -lah coverage-output-e2e/
+        if: ${{ !(github.event_name == 'workflow_dispatch' && github.event.inputs.debug_enabled == 'true' && matrix.kubernetes.latest)}}
+      - name: Output Rerun Overview
+        run: |
+          [[ -f rerunreport.txt ]] && cat rerunreport.txt || echo "No rerun report found"
+      - name: Upload E2E Test Results
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: E2E Test Results (k8s ${{ matrix.kubernetes.version }})
+          path: |
+            junit-e2e-test.xml
+      - name: Upload e2e-controller logs
+        uses: actions/upload-artifact@v4
+        with:
+          name: e2e-controller-k8s-${{ matrix.kubernetes.version }}.log
+          path: /tmp/e2e-controller.log
+        if: ${{ always() }}
+      - name: Upload code coverage artifacts
+        uses: actions/upload-artifact@v4
+        with:
+          name: coverage-output-e2e
+          path: coverage-output-e2e
+        if: ${{ matrix.kubernetes.latest }}
+
+  coverage-process:
+    name: Process Coverage Files
+    runs-on: ubuntu-latest
+    needs:
+      - test-unit
+      - test-e2e
+    steps:
+      - name: Set up Go
+        uses: actions/setup-go@v5.4.0
+        with:
+          go-version: ${{ env.GOLANG_VERSION }}
+        id: go
+      - uses: actions/checkout@v4
+      - name: Get e2e code coverage
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: coverage-output-e2e
+          path: coverage-output-e2e
+      - name: Get unit test code coverage
+        uses: actions/download-artifact@d3f86a106a0bac45b974a628896c90dbdf5c8093 # v4.3.0
+        with:
+          name: coverage-output-unit
+          path: coverage-output-unit
+      - name: combine-go-coverage
+        run: |
+          go tool covdata textfmt -i=coverage-output-unit/,coverage-output-e2e/ -o full-coverage.out
+      - name: Upload code coverage information to codecov.io
+        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
+        with:
+          file: full-coverage.out
+          fail_ci_if_error: false
+          codecov_yml_path: .codecov.yml
+          disable_search: true
+          verbose: true
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}

.github/workflows/waiting-issues.yml

@@ -0,0 +1,32 @@
+name: 'Close stale issues and PRs waiting for response'
+on:
+  schedule:
+    - cron: '30 1 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  stale:
+    runs-on: ubuntu-latest
+    permissions:
+      issues: write
+      pull-requests: write
+    steps:
+      - uses: actions/stale@v9
+        with:
+          operations-per-run: 250
+          stale-issue-message: 'This issue is stale because it has awaiting-response label for 5 days with no activity. Remove stale label or comment or this will be closed in 5 days.'
+          stale-pr-message: 'This PR is stale because it has awaiting-response label for 10 days with no activity. Remove stale label or comment or this will be closed in 10 days.'
+          close-issue-message: 'This issue was closed because it has been stalled for 5 days with no activity and has the awaiting-response label.'
+          close-pr-message: 'This PR was closed because it has been stalled for 10 days with no activity and has the awaiting-response label.'
+          days-before-issue-stale: 5
+          days-before-pr-stale: 10
+          days-before-issue-close: 5
+          days-before-pr-close: 10
+          stale-issue-label: 'no-issue-activity'
+          exempt-issue-labels: 'awaiting-approval,work-in-progress,enhancement'
+          stale-pr-label: 'no-pr-activity'
+          exempt-pr-labels: 'awaiting-approval,work-in-progress'
+          exempt-all-milestones: true
+          any-of-labels: 'awaiting-response'

.gitignore

@@ -1,11 +1,25 @@
 .vscode/
 .idea/
 .DS_Store
-vendor/
 dist/
+github.com/
+k8s.io/
 *.iml
 # delve debug binaries
+__debug_bin*
 cmd/**/debug
 debug.test
 coverage.out
 coverage.html
+junit.xml
+rerunreport.txt
+site/
+vendor/
+plugin-bin/
+# static
+server/static/*
+!server/static/.gitkeep
+coverage-output-e2e/
+coverage-output-unit/
+junit*
+

.golangci.yml

@@ -1,20 +1,41 @@
+version: "2"
 run:
-  deadline: 1m
-  skip-dirs:
-    - pkg/client
-    - vendor
-linter-settings:
-  goimports:
-    local-prefixes: github.com/argoproj/argo-rollouts
+  modules-download-mode: readonly
+  timeout: 10m
 linters:
+  default: none
+  enable:
+    - govet
+    - ineffassign
+    - misspell
+    - unconvert
+    - unused
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    paths:
+      - .*\.pb\.go
+      - pkg/client
+      - third_party$
+      - builtin$
+      - examples$
+formatters:
   enable:
-    - vet
     - gofmt
     - goimports
-    - deadcode
-    - varcheck
-    - structcheck
-    - ineffassign
-    - unconvert
-    - misspell
-  disable-all: true
+  settings:
+    goimports:
+      local-prefixes:
+        - github.com/argoproj/argo-rollouts
+  exclusions:
+    generated: lax
+    paths:
+      - .*\.pb\.go
+      - pkg/client
+      - third_party$
+      - builtin$
+      - examples$

.readthedocs.yaml

@@ -0,0 +1,12 @@
+version: 2
+formats: all
+mkdocs:
+  configuration: mkdocs.yml
+  fail_on_warning: false
+python:
+  install:
+    - requirements: docs/requirements.txt
+build:
+  os: "ubuntu-22.04"
+  tools:
+    python: "3.7"

CHANGELOG-v0.1.0_v1.1.0.md

@@ -0,0 +1,839 @@
+# Changelog
+
+# v1.1.0
+
+## Notable Features
+* Rollout Notifications
+* Dynamic scaling of stable ReplicaSet (dynamicStableScale)
+* Automated rollbacks without analysis (progressDeadlineAbort)
+* Kustomize Open API Schema
+* Rollout Dashboard as a Service
+* Controlling Scaledown Behavior During Aborts (abortScaleDownDelaySeconds)
+* Analysis: AWS CloudWatch Metric Provider
+* AWS TargetGroup IP Verification
+* Weighted Experiment Canary Steps
+* Istio: Multicluster Support
+* Istio: TLS Route Support
+* Istio: Multiple VirtualServices
+* AnalysisRun GC
+* Analysis: Graphite Metric Provider
+
+## Changes since v1.0
+
+### Controller
+* feat: support dynamic scaling of stable ReplicaSet as inverse of canary weight (#1430)
+* fix: promote nil pointer error when there are no steps (#1510)
+* feat: support management of multiple Istio VirtualService objects (#1381)
+* feat: verify AWS TargetGroup after updating active/stable services (#1348)
+* feat: ALB TrafficRouting with experiment step
+* feat: TrafficRouting SMI with Experiment Step in Canary (#1351)
+* feat: ability to abort an update when exceeding progressDeadlineSeconds (#1397)
+* feat: add support for Istio VirtualService spec.tls[] (#1380)
+* feat: configurable and more aggressive cleanup of old AnalysisRuns and Experiments (#1342)
+* feat: ability to auto-create Services for each template in an Experiment (#1158)
+* feat: introduce abortScaleDownDelaySeconds to control scale down of preview/canary upon abort (#1160)
+* feat: argo rollout compatibility with emissary and edge stack v2.0 (#1330)
+* feat: Add support for Istio multicluster (#1274)
+* feat: add workload-ref/generation to rollout (#1198)
+* feat: support notifications on rollout events using notifications-engine (#1175)
+* chore: add liveness and readiness probe to the install manifests (#1324)
+* fix: Nginx ingressClassName passed to canary ingress (#1448)
+* fix: canary scaledown event could violate maxUnavailable (#1429)
+* fix: analysis runs to wait for all metrics to complete (#1407)
+* fix: Promote full did not work against BlueGreen with previewReplicaCount (#1384)
+* fix: retarget blue-green previewService before scaling up preview ReplicaSet (#1368)
+* fix: zero-value abortScaleDownDelay was not honored with setCanaryScale (#1375)
+* fix: abort scaledown stable RS for canary with traffic routing (#1331)
+
+### Analysis
+* feat: add support for Graphite metrics provider (#1406)
+* feat: Support CloudWatch as a metric provider (#1338)
+* fix: Analysis argument validation (#1412)
+
+### Plugin
+* feat: create windows version for CLI (#1517)
+* feat: provide shell completion. Closes #619 (#1478)
+* fix: create analysisrun cmd using template generated name (#1471)
+* fix: nil pointer in create analysisrun cmd (#1399)
+* fix: lint subcommand for workload ref rollout (#1328)
+* fix: undo referenced object for workloadRef rollout (#1275)
+
+### Dashboard
+* feat: allow selection of namespace in rollout dashboard (#1291)
+* fix(ui): UI crashes on rollout view due to undefined status (#1287)
+
+### Misc
+* feat: kustomize rollout: add openapi to doc and examples (#1371)
+* feat: add rollout stat row to grafana dashboard (#1343)
+
+## Upgrade Notes
+### Difference in scale down behavior during aborts
+
+The v1.1 `abortScaleDownDelaySeconds` feature now allows users full control over the scaling
+behavior of the canary/preview ReplicaSet during an abort. Previously in v1.0, it was not possible
+to affect this behavior. As part of this feature, v1.1 also fixes some inconsistencies in behavior
+with respect to abort scale down.
+
+The most notable change is that upon an abort, the blue-green preview ReplicaSet in v1.1 will now
+scale down 30 seconds after the abort, whereas in v1.0 the preview ReplicaSet was left running
+indefinitely (without any option to scale it down). If you prefer the v1.0 behavior, you can set
+`abortScaleDownDelaySeconds: 0`, which will leave the preview ReplicaSet running indefinitely
+on abort:
+
+```yaml
+spec:
+  strategy:
+     blueGreen:
+       abortScaleDownDelaySeconds: 0
+```
+
+Please read the full
+[documentation](https://argoproj.github.io/argo-rollouts/features/scaledown-aborted-rs/) to understand
+the differences in canary/preview scaling behavior for aborted Rollouts from v1.0 to v1.1.
+
+# v1.0.6
+## Changes since v1.0.4
+
+### Bug Fixes
+
+- fix: replica count for new deployment (#1449)
+- fix: Nginx ingressClassName passed to canary ingress (#1448)
+- fix: Analysis argument validation (#1412)
+- fix: retarget blue-green previewService before scaling up preview ReplicaSet (#1368)
+- fix: analysis runs to wait for all metrics to complete (#1407)
+- fix: canary scaledown event could violate maxUnavailable (#1429)
+- chore: release workflow docker build context should use local path and not git context (#1388)
+- chore: github release action was using incorect docker cache (#1387)
+
+# v1.0.4
+## Changes since v1.0.3
+
+### Controller
+* fix: Promote full did not work against BlueGreen with previewReplicaCount
+
+# v1.0.3
+
+## Changes since v1.0.2
+
+### Controller
+
+* fix: nil pointer dereference when reconciling paused blue-green rollout (#1378)
+* fix: Abort rollout doesn't remove all canary pods for setCanaryScale (#1352)
+* fix: unsolicited rollout after upgrade from v0.10->v1.0 when pod was using service account (#1367)
+* fix: default replica before resolving workloadRef (#1304)
+
+# v1.0.2
+
+## Changes since v1.0.1
+
+### Controller
+
+* feat: allow VirtualService HTTPRoute to be inferred if there is single route (#1273)
+* fix: rollout paused longer than progressDeadlineSeconds would briefly degrade (#1268)
+* fix: controller would drop fields when updating DestinationRules (#1253)
+* fix: the wrong panel title on the sample dashboard (#1260)
+* fix: analysis with multiple metrics (#1261)
+* fix: Mitigate the bug where items are re-added constantly to the workqueue. #1193 (#1243)
+* fix: workload rollout spec is invalid template is not empty (#1224)
+* fix: Fix error check in validation for AnalysisTemplates not found (#1249)
+* fix: make function call consistent with otherRSs definition (#1171)
+
+### Plugin
+
+* fix: avoid using root user in plugin container (#1256)
+
+
+# v1.0.1
+
+## Changes since v1.0.0
+
+### Controller
+
+* feat: WebMetric to support string body responses (#1212)
+* fix: Modify validation to check Analysis args passed through RO spec (#1215)
+* fix: AnalysisRun args could not be resolved from secret (#1213)
+
+
+# v1.0.0
+
+## Notable Features
+* New Argo Rollouts UI available in `kubectl argo rollouts dashboard`
+* Ability to reference existing Deployment workloads instead of inlining a PodTemplate at spec.template
+* Richer prometheus stats and Kubernetes events
+* Support for Ambassador as a canary traffic router
+* Support canarying using Istio DestinationRule subsets
+
+## Upgrade Notes
+
+### Installation Manifests
+
+Installation manifests are now attached as GitHub Release artifacts (as opposed to raw files checked into git)
+and can be installed with the release download URL. e.g.:
+
+```
+kubectl apply -f https://github.com/argoproj/argo-rollouts/releases/download/v1.0.0/install.yaml
+```
+
+### Argo CD OutOfSync status on Rollout v1.0.0 CRDs:
+
+Argo Rollouts v1.0 now vends apiextensions.k8s.io/v1 CustomResourceDefinitions (previously apiextensions.k8s.io/v1beta1).
+Kubernetes v1 CRDs no longer supports the preservation of unknown fields in objects, and rejects
+attempts to set `spec.preserveUnknownFields: true` (the previous default). In order to support a
+smooth upgrade from Argo Rollouts v0.10 to v1.0, `spec.preserveUnknownFields` is explicitly set to
+`false` in the manifests, despite `false` being the default, and only option in v1 CRDs. However 
+this causes diffing tools (such as Argo CD) to report the manifest as OutOfSync (since K8s drops the
+false field).
+
+More information:
+* https://github.com/kubernetes-sigs/controller-tools/issues/476
+* https://github.com/argoproj/argo-rollouts/pull/1069
+
+To avoid the Argo CD OutOfSync conditions, you can remove `spec.preserveUnknownFields` from the manifests
+entirely *after upgrading to v1.0*.
+
+Alternatively, you can instruct Argo CD to ignore differences using ignoreDifferences in the Application spec:
+
+```yaml
+spec:
+  ignoreDifferences:
+  - group: apiextensions.k8s.io
+    kind: CustomResourceDefinition
+    jsonPointers:
+    - /spec/preserveUnknownFields
+```
+
+### Deprcation of `kubectl argo rollouts promote --skip-current-step` flag
+
+The promote flag `--skip-current-step` which skips the current running canary step has been
+deprecated and will be removed in a future release. Its logic to skipping the current step has
+been merged with the existing command:
+
+```shell
+kubectl argo rollouts promote ROLLOUT
+```
+
+The `promote ROLLOUT` command can now be used to handle both the case where the rollout needs to be
+unpaused, as well as to skip the currently running canary step (e.g. an analysis/experirment/pause
+step).
+
+## Changes since v0.10
+
+### Controller
+* feat: support reference model for workloads (#676) (#1072)
+* feat: Implement Ambassador to be used as traffic router for canary deployments (#1025)
+* feat: support canarying using Istio DestinationRule subsets (#985)
+* feat: istio virtualservice and rollout in different namespaces
+* feat: add ability to verify canary weights before advancing steps (#957)
+* feat: support scaleDownDelaySeconds in canary w/ traffic routing (#1056)
+* feat: Add ability to restart maxUnavailable pods to BlueGreen strategy (#937)
+* feat(controller): Add support for ephemeral metadata on BlueGreen rollouts. Fixes #973 (#974)
+* feat: Allow user to handle NaN result in Analysis (#977)
+* feat: Wait for canary RS to have ready replicas before shifting labels (#1022)
+* feat: Create RolloutPaused condition (#1054)
+* feat: Add RolloutCompleted condition (#1074)
+* feat: add print version flag to rollouts-controller
+* feat: calculate rollout phase & message controller side
+* fix: Fixes the regression of dropping resources from argo-rollouts crds. Fixes #1043 (#1044)
+* fix: Set Canary Strategy default maxUnavailable to 25% (#981)
+* fix: blue-green rollouts could pause prematurely during prePromotionAnalysis (#1007)
+* fix: Clear ProgressDeadlineExceeded Condition in paused BlueGreen Rollout (#1002)
+* fix: analysis template arguments validate (#1038)
+* fix: calculate scale down count. (#1047)
+* fix: verify analysis arguments name with those in the rollout (#1071)
+* fix: rollout status always in progressing if analysis fails (#1099)
+* fix: Add edge case handling to traffic routing (#1190)
+* fix: unhandled error patchVirtualService (#1168)
+* fix: handling error on f.close (#1167)
+* fix: rollouts in middle of restart should be considered Progressing
+
+### Analysis
+* feat: metric fields can be parameterized by analysis arguments (#901)
+* feat: support a custom base URL for the new relic provider (#1053)
+* feat: Allow Datadog API and APP keys to be consumed from env vars (#1073)
+* fix: Improve validation for AnalysisTemplates referenced by RO (#1094)
+* fix: wavefront queries would return no datapoints. surface evaluate errors
+* fix: metrics which errored did not retry at error interval
+* fix: Improve and refactor validation for AnalysisTemplates
+
+### Plugin
+* feat: Argo Rollouts api-server and UI (#1015)
+* feat: Implement rollout status command. Fixes #596 (#1001)
+* feat: lint supporting rollout in multiple doc
+* fix: get rollout always return not found except default namespace (#961)
+* fix: create command not support namespace in yaml file (#962)
+* fix: kubectl argo create panic: runtime error: invalid memory address or nil pointer dereference
+
+### Misc
+* chore: publish plugin image automatically. migrate to quay.io (#1102)
+* feat: support ARM builds, remove unused components in Dockerfile (#889)
+* chore: update k8s dependencies to v1.20. improve logging (#994) 
+* fix: add informational exposed ports to deployment (#1066)
+* chore: Outsource reusable UI components to argo-ux npm package
+* fix: use fixed size int32
+
+# v0.10.2
+
+## Changes since v0.10.1
+
+### Controller
+* fix: switch pod restart to use evict API to honor PDBs
+* fix: ephemeral metadata injection was dropping metadata injected by mutating webhooks
+* fix: requiredForCompletion did not work for an experiment started by a rollout
+* fix: Add missing RoleBinding file to namespace installation
+
+# v0.10.1
+
+## Changes since v0.10.0
+
+### Controller
+* fix: Correct Istio VirtualService immediately (#874)
+* fix: restart was restarting too many pods when available > spec.replicas (#856)
+
+### Plugin
+* fix: plugin incorrectly treated v0.9 rollout as v0.10 when it had numeric observedGeneration (#875)
+
+# v0.10.0
+
+## Notable Features
+* Ability to set canary vs. stable ephemeral metadata on rollout Pods during an update
+* Support new metric providers: New Relic, Datadog
+* Ability to control canary scale during an update
+* Ability to restart up to maxUnavailable pods at a time for a canary rollout
+* Ability to self reference rollout metadata as arguments to analysis
+* Ability to fully promote blue-green and canary rollouts (skipping steps, analysis, pauses)
+* kubectl-argo-rollouts plugin command to lint rollout
+* kubectl-argo-rollouts plugin command to undo a rollout (same as kubectl rollout undo)
+
+## Upgrade Notes
+
+Rollouts v0.10 has switched to using Kubernetes [CRD Status Subresources](https://kubernetes.io/docs/tasks/extend-kubernetes/custom-resources/custom-resource-definitions/#status-subresource) ([PR #789](https://github.com/argoproj/argo-rollouts/pull/789)). This feature allows the rollout controller to record the numeric `metadata.generation` into `status.observedGeneration` which provides a reliable indicator of a Rollout who's spec has (or has not yet) been observed by the controller (for example if the argo-rollouts controller was down or delayed).
+
+A consequence of this change, is that the v0.10 rollout controller should be used with the v0.10 kubectl-argo-rollouts plugin in order to perform actions such as abort, pause, promote. Similarly, Argo CD v1.8 should be with the v0.10 rollout controller when performing those same actions. Both kubectl-argo-rollouts plugin v0.10 and Argo CD v1.8 are backwards compatible with v0.9 rollouts controller.
+
+## Changes since v0.9
+
+### Controller
+
+* feat: set canary/stable ephemeral metadata to pods during updates (#770)
+* feat: add support for valueFrom in analysis arguments. (#797)
+* feat: Adding rollout_info_replicas_desired metric. Fixes #748 (#749)
+* feat: restart pods up to maxUnavailable at a time
+* feat: add full rollout promotion (skip analysis, pause, steps) 
+* feat: use CRD status subresource (#789)
+* feat: Allow setting canary weight without side-effects. Fixes #556 (#677)
+* fix: namespaced scoped controller support (#818)
+* fix: fetch secrets on-demand to fix controller boot for large clusters (#829)
+
+### Analysis
+* feat: Add New Relic metricprovider (#772)
+* feat: Add Datadog metric provider. Fixes #702 (#705)
+
+### Plugin
+* feat: Implement kubectl argo rollouts lint
+* feat: Add undo command in kubectl plugin. Fixes #575 (#812)
+* fix: kubectl plugin should use dynamic client
+
+### Misc
+* fix: rollout kustomize transform analysis ref should use templateName instead of name (#809)
+* fix: add missing Service kustomize name reference in trafficRouting/alb/rootService (#699)
+
+
+# v0.9.3
+
+## Changes since v0.9.2
+
+### Controller
+* fix: scaleDownDelayRevisionLimit was off by one (#816)
+* fix: background analysis refs were not verified. requeue InvalidSpec rollouts (#814)
+* fix(controller): fix unhandled panic from malformed rollout (#801)
+* fix(controller): validation should not consider privileged security context (#802)
+
+# v0.9.2
+
+## Changes since v0.9.1
+
+### Controller
+* fix(controller): controller did not honor maxUnavailable during rollback (#786)
+* fix(controller): blue-green with analysis was broken (#780)
+* fix(controller): blue-green fast-tracked rollbacks would still start analysis templates
+* fix(controller): prePromotionAnalysis with previewReplicaCount would pause indefinitely w/o running analysis
+* fix(controller): calculate available replicas from active ReplicaSet (#757)
+
+### Plugin
+* feat(plugin): indicate the stable ReplicaSet for blue-green rollouts in plugin
+* feat(plugin): plugin now surfaces InvalidSpec errors and failed analysisrun messages (#729)
+* fix(plugin): bluegreen scaleDownDelay was delaying Healthy status. Present errors in message field (#768)
+
+# v0.9.1
+
+## Changes since v0.9.0
+### General
+* feat: writeback rollout updates to informer to prevent stale data (#726)
+* fix: unavailable stable RS was not scaled down to make room for canary (#739)
+* fix: make controllers tolerant to spec marshalling errors (#666)
+* perf: Create IstioVirtualServiceLister (#656)
+* fix: add missing log message when a controller's syncHandler returns error (#658)
+* fix: support azure auth (#664)
+
+### Analysis
+* feat: web metrics preserve data types, allow insecure tls, and make jsonPath optional (#731)
+* fix: analysis controller could get into a hotloop with terminated run (#724)
+* fix: do not create analysisruns with initial deploy (#722)
+* fix: add Failed AnalysisRun phase status to analysis_run_metric_phase and analysis_run_phase metrics. (#618)
+
+# v0.9.0
+
+## Changes since v0.8
+### General
+* fix: Fix various panics #603
+* feat: add security context to run as non-root #498
+  
+### Rollouts
+* feat: Controller Validation #549
+* feat: Controller Validation for objects referenced by Rollout #600
+* feat: Add Rollout replicas metrics (#507) #581
+* feat: Add support for rootService within ALB traffic routing #634
+
+* fix: Populate .spec.template with default values before Rollout Validation #580
+* fix: Add Rollout/scale to aggregate roles #637
+* Fix: remove hash selector after switching from bg to canary #515
+* fix: Set the currentStepIndex to max after bg to canary #558
+
+### Traffic Routing
+* feat: SMI TrafficSplit Support for Canary #520
+
+### Kubectl Plugin
+* feat: add shortened option -A for --all-namespaces #615
+
+### Analysis
+* feat: ClusterAnalysisTemplates (Cluster scoped AnalysisTemplates) #560
+* feat: Uplevel AnalysisRun status to Rollout status #578
+
+* fix: Modify arg verification to check ValueFrom #500
+* fix: Fix analysis validation to include Kayenta #545
+
+# v0.8.3
+
+## Changes since v0.8.2
+### General
+* fix: Modify arg verification to check ValueFrom (#500)
+* fix: remove hash selector after switching from bg to canary (#515)
+
+# v0.8.2
+
+## Changes since v0.8.1
+### Rollouts
+* fix: Ensure ALB action with weight 0 marshalls correctly (#493)
+* fix: Add missing clusterrole for deleting pods (#490)
+
+# v0.8.1
+
+## Changes since v0.8.0
+### General
+* fix: Remove validation for limits and requests (#480)
+
+### Rollouts
+* fix: Duplicate StableRS to canary.StableRS (#483)
+
+### Kubectl plugin
+* fix: Make kubectl plugin backwards compat with canary.stableRS (#482)
+
+# v0.8.0
+
+## Breaking Changes
+* The metric `rollout_created_time` is being removed.
+* The `.status.canary.stableRS` is being deprecated for `.status.stableRS`. This release has the code to handle the migration, and the Rollout spec will updated to remove `.status.stableRS` in a future release.
+
+## Contributors
+Thank you to the following contributors for their work in this release!
+* cronik
+* dthomson25
+* duboisf
+* jessesuen
+* khhirani
+* moensch
+* nghialv
+  
+## Changes since v0.7.2
+### General
+* feat: Improve Prometheus metrics (#461)
+* feat: Add metrics on queues and go client http calls (#416)
+* feat: Add patchMergeKey and patchStrategy struct tags and comments (#386)
+* feat: Improve removing k8s 1.18 fields (#436)
+* fix: Reduce log from error to warning (#394)
+* chore: Download go deps explicitly in Dockerfile (#464)
+* chore: Standardize controller-gen to v0.2.5 (#431)
+* chore: Migrate from dep to go modules (#331)
+* chore: Add auto generated sites/ to gitignore (#398)
+* docs: Add remote name to 'make release-docs' (#435)
+* docs: Documentation cleanup (#437)
+* docs: Add Go mod download command to contributor docs (#425)
+* docs: Corrected HPA doc (#396)
+* docs: Remove extra comma in docs
+* docs: Update README.md (#411)
+
+### Rollouts
+* feat: Introduce Anti-Affinity option to rollout strategies (#445)
+* feat: Add ability to restart Pods (#453)
+* feat: Add ALB Ingress controller support (#444)
+* feat: Add Nginx canary traffic management (#426)
+* feat: Add BlueGreen Pre Promotion Analysis (#415)
+* feat: Add BlueGreen Post Promotion Analysis (#442)
+* feat: Allow Rollout to specify multiples templates (#409)
+* feat: Make pause duration as string with time unit (#423)
+* feat: Use managed-by annotation (#448)
+* refactor: Refactor BlueGreen Strategy (#388)
+* fix: Update Role/ClusterRole for Ingress access (#439)
+* fix: rollout transformer for pod affinity. add new v0.7 name references and testing (#399)
+* chore: Add StableRS to rollout status (#441)
+* chore: Fix wrong comment about the formula of calculating the replica number (#447)
+
+### Analysis
+* feat: Improve wavefront provider (#465)
+* feat: Allow AnalysisTemplates to reference secrets (#420)
+* improvement: Surface failure reasons for Rollouts/AnalysisRuns (#434)
+* refactor: Perform arg substitution in Analysis controller (#407)
+* docs: Use correct podTemplateHashValue attribute for valueFrom (#417)
+* docs: Update web metrics section (#381)
+* docs: Use correct magic value in Analysis docs (#378)
+
+### Experiments
+* feat: Experiments passed duration succeed with running analysis (#392)
+* feat: Allow ex to use availableAt and finishedAt as args (#400)
+* refactor: Refactor Experiment handling of pod hashes (#385)
+
+### Kubectl plugin
+* feat: Show scale down time for Blue Green ReplicaSets (#370) (#382)
+* feat: Add more command aliases in kubectl plugin (#414)
+* chore: Set kubectl flags on root command (#456)
+* docs: Generate kubectl plugin docs (#422)
+* docs: Plugin command enhancements (#454)
+
+# v0.7.2
+
+## Changes since v0.7.1
+### Rollouts
+* Update RS if RS's annotations need to be changed #413
+
+# v0.7.1
+
+## Changes since v0.7.0
+
+### General
+* Adding ca-certificates to docker image (#393)
+* Add patchMergeKey and patchStrategy struct tags and comments (#386)
+* Reduce log from error to warning (#394)
+
+### Experiments
+* Allow ex to use availableAt and finishedAt as args (#400)
+* Experiments passed duration succeed with running analysis (#392)
+* Refactor Experiment handling of pod hashes (#385)
+
+# v0.7.0
+
+## Important Notices
+- Please upgrade to v0.6.x before upgrading to v0.7. Pre v0.6.0 has a different pausing logic, and v0.7.0 removes the depreciated PauseStartTime field. The v0.6.x versions have a migration script that is removed in v0.7.0. 
+- This release introduces an alpha implementation of Rollouts leveraging Istio resources for traffic shaping. Check out [traffic management](https://argoproj.github.io/argo-rollouts/features/traffic-management/) for more info.
+
+## Changes since v0.6.3
+
+### General
+* Support instance ids for rollout controller segregation #342
+* Remove PauseStartTime #349
+* Vendor mockery utility #347
+* Remove loud log message #333
+
+### Rollouts
+#### General
+* Add stableService field #337
+
+#### Traffic Routing
+* Initial Istio implementation #341
+* Implement watch for Istio resources #354
+* Add validation to istio virtual services #355
+
+### Kubectl Plugin
+* Introduce 'kubectl argo rollouts terminate' command #297
+
+### Analysis
+
+#### General
+* Allow controller to delay analysis #350
+* Create one background analysis per revision #309
+* Allow AnalysisRun to complete an experiment #345
+
+#### Providers
+* Wavefront metric provider #338
+* Web metric provider #318
+* Refactor common logic in providers to library #368
+* Allow web provider to be parameterized #368
+
+# v0.6.3
+
+## Changes since v0.6.2
+### Bug Fixes
+
+* Fix premature scaledown (#365)
+* Add namespace restriction to job informer (#362)
+* Fix honoring autoPromotionSeconds (#360)
+* Ensure podHash stays on stable-svc selector (#340)
+
+# v0.6.2
+
+## Changes since v0.6.1
+### Bug Fixes
+
+* omitted revisionHistoryLimit was not defaulting to 10 (#330)
+* Fix panic if rollout cannot create a new RS (#328)
+* Enable controller to handle panics with crashing (#328)
+
+# v0.6.1
+
+## Changes since v0.6.0
+### Bug Fixes
+
+- Create one background analysis per revision (#309)
+- Fix Infinite loop with PreviewReplicaCount set (#308)
+- Fix a delete by zero in get command (#310)
+- Set StableRS hash to current if replicaset does not actually exist (#320) 
+- Bluegreen: allow preview service/replica sets to be replaced and fix sg fault in syncReplicasOnly (#314)
+
+# v0.6.0
+
+## Important Notices
+- The pause functionality was reworked in the v0.6 release. Previously, the `.spec.paused` field was used by the controller to pause rollouts. However, this was an issue for users who wanted to manually pause the rollout since the controller assumed it was the only entity that set the field. In v0.6, the controller will add a pause condition to the `.status.pauseCondition` to pause a controller instead of setting `spec.paused`. The pause condition has a start time and a reason explaining why it paused. This allows users to set the `spec.paused` field manually and let the controller respect that pause. The v0.6 controller has a migration function to convert pre v0.6 rollouts to the new pause condition. The migration function will be removed in a future release.
+- In pre-v0.6 versions, the BlueGreen strategy would have the preview service point at no ReplicaSets if the new ReplicaSet was receiving traffic from the active service. V0.6 changes that behavior to make the preview service always point at the latest ReplicaSet
+
+## Changes since v0.5.0
+#### General
+- Update k8s library dependencies to v1.16 (##192)
+
+#### Rollouts
+
+###### Enhancements
+- Add Rollout Context to reconciliation loop (##205)
+- Refactor pausing (##211)
+- Allow User pause (##216)
+- Stop progress while paused (##193)
+- Add pause condition migration (##229)
+- Add abort functionality (##224)
+- Rollout analysis plumbing (##183)
+- Add AnalysisStep for Rollouts (##188)
+- Add background analysis runs for rollouts (##196)
+- Clean up old Background AnalysisRuns (##246)
+- Clean up Experiments and AnalysisRuns (##197)
+- Add initial Experiment Step (##165)
+- Make specifying replicas/duration optional in the experiment step (##241)
+- Terminate experiments from previous steps (##280)
+- Add Analysis to RolloutExperimentStep (##238)
+- Fix TimeOut check to consider experiment/analysis steps (##278)
+- Pause a rollout upon inconclusive experiment (##256)
+- Abort a rollout upon a failed experiment (##256)
+- Add create AnalysisRun action in clusterrole (##231)
+
+###### Bug Fixes
+- Fix nil ptr for newRS (##233)
+- Fix Rollout transformer config (##247)
+- Always point preview service at the newRS (##217)
+- Make active service required (##235)
+- Reset ProgressDeadline on retry (##282)
+- Ignore old running rs for RolloutCompleted status (##218)
+- Remove scale down annot after scaling down (##187)
+- Renames golang field names for blueGreen/canary to eliminate two API violations (##206)
+
+#### Experiments
+Check out the [Experiment Docs](https://github.com/argoproj/argo-rollouts/blob/release-v0.6/docs/features/experiments.md) for more information.
+
+###### Enhancements
+- Refactor experiments to use a context object (##208)
+- Allow selectors to be overwritten when starting experiments (##249)
+- Simplify experiment replicaset names (##274)
+- Integrate Experiments with Analysis (##210)
+
+#### Bug Fixes
+- Fix experiment enqueue logic (##239)
+- Annotate instead of label experiment names in replicasets (##262)
+- Fix issue where a replicaset name collision could cause hot loop (##236)
+
+#### Analysis
+Check out the [Analysis Docs](https://github.com/argoproj/argo-rollouts/blob/release-v0.6/docs/features/analysis.md) for more information.
+###### Enhancements
+- AnalysisRun AnalysisTemplate Spec (##166)
+- Initial analysis controller implementation (##168)
+- Integrate analysis controller with provider interfaces (##171)
+- Add metric knob for maxInconclusive (##181)
+- Simplify provider interfaces to set error messages (##189)
+- Implement ResumeAt logic (##232)
+- Define explicit args in AnalysisTemplates and simplify AnalysisRun spec (##283)
+- Use a duration string instead of int to represent duration (##286)
+- Truncate measurements when greater than default (10) (##191)
+- Add counter for consecutiveError (##191)
+
+###### Prometheus Provider
+-  Add initial provider and Prometheus implementation (##170)
+- Rename prometheus.server to address to better reflect API client interface (##207)
+- Treat NaN as inconclusive on Prometheus provider (##275)
+
+###### Job Provider
+- Implement job-based metric provider (##186)
+- Job metric argument substitution. Simplify metric provider interface (##268)
+
+###### Kayenta Provider
+- Initialize check in for kayenta metric provider ##284
+
+
+#### Kubectl Plugin
+Check out the [kubectl plugin docs](https://github.com/argoproj/argo-rollouts/blob/release-v0.6/docs/features/kubectl-plugin.md) for more information.
+
+###### Enhancements
+- Implement argo rollouts kubectl plugin (##195)
+- Introduce `kubectl argo rollout list rollouts` command (##204)
+- Introduce `kubectl argo rollout list experiments` command (##267)
+- Introduce `kubectl argo rollout set image` command (##251)
+- Introduce `kubectl argo rollout get` command (##230)
+- Introduce `kubectl argo rollout promote` command (##277)
+- Add ability to `kubectl argo rollouts set image *=myrepo/myimage` (##290)
+- Add `get/retry experiment` commands. Support experiment retries (##263)
+- Show running jobs as part of analysis runs (##278)
+- Surface experiment images to CLI (##274)
+
+# v0.5.0
+
+## Changes since v0.4.2
+#### Bug Fixes
+- Rollout deletionTimestamp are not honored (##109)
+- status.availableReplicas should not count old stacks (##143)
+- Fix Infinitely loop on controller loop (##146)
+
+#### Enhancements
+- Fast rollback in BlueGreen during scale down period ##127
+- Attach independent scaleDownDelays to older ReplicaSets ##145
+- Add scaleDownDelayRevisionLimit to limit the number of old ReplicaSets scaled up ##129
+
+#### Experiment CRD
+This release of Argo Rollouts introduces the experiment CRD. The experiment CRD allows users to define multiple PodSpec's to run for a specific duration of time. This will help enable the Kayenta use-case where a user will need to start two versions of their application at the same time. Otherwise, the users cannot have an apples-to-apples comparison of these two versions as one will skew as a result of running for a longer period.
+
+# v0.4.2
+
+## Changes since v0.4.1
+#### Bug Fixes
+- Honor MaxSurge and MaxUnavailable after last step (##141)
+- Fix maxSurge maxUnavailable zero check (##135)
+- Add .Spec.Replicas if not set in rollouts (##125)
+
+# v0.4.1
+
+## Changes since v0.4.0
+#### Bug Fixes
+- Workaround K8s inability to properly validate 'items' subfields ##114
+
+# v0.4.0
+
+## Important Notes Before Upgrading
+- For the BlueGreen strategy, Argo Rollouts will only pause rollouts that have the field `spec.strategy.blueGreen.autoPromotionEnabled` set to false. The default value of `autoPromotionEnabled` is true and causes the rollout to immediately promote the new version once it is available. This change was implemented to make the pausing behavior of rollouts more straight-forward and you can read more about it at ##80. Argo Rollouts v0.3.2 introduces the `autoPromotionEnabled` flag without making any behavior changes, and those behavior changes are enforced starting at v0.4.0. __In order to upgrade without any issues__, the operator should first upgrade to v0.3.2 and add the `autoPromotionEnabled` flag with the appropriate value.  Afterward, they will be safe to upgrade to v0.4.
+
+- For the Canary Strategy, the Argo Rollouts controller stores a hash of the canary steps in the rollout status to be able to detect changes in steps. If the canary steps change during a progressing canary update, the controller will change the hash and restart the steps.  If the rollout is in a completed state, the controller will only update the hash. In v0.4.0, the controller changed how the hash of the steps was calculated, and you can read more about that at this issues: ##103.  As a result, __the operator should only upgrade Argo Rollouts to v0.4.0 when all the canary rollouts have executed all steps and have completed__. Otherwise, the controller will restart the steps it has executed.
+
+## Changes since v0.3.2
+#### Enhancements
+- Add Ability to specify canaryService Service object to reach only canary pods ##91
+- Simplify unintuitive automatic pause behavior for blue green strategy ##80 
+- Add back service informer to handle Service recreations quicker ##71 
+- Use lister instead of kubernetes api call to load service ##98
+- Switch to controller-gen to generate crd with complete openapi validation spec ##84
+
+#### Bug Fixes
+- Change step hashing function to derive hash from json representation ##103
+- CRD validation needs to be removed for resource requests/limits ##101
+- Possible to exceed revisionHistoryLimit with canary strategy ##93
+- Change in pod template hash after upgrading k8s dependencies ##88
+- Controller is missing patch event privileges bug ##86
+- Rollouts unprotected from invalid specs ##84 
+- Fix logging fields ##97
+
+# v0.3.2
+
+## Important BlueGreen Strategy Change
+In v0.4.0, Argo Rollouts will have a breaking change where we will only pause BlueGreen rollouts if they have a new field called `autoPromotionEnabled` under the `spec.strategy.blueGreen` set to false. If the field is not listed, the default value will be true, and the rollout will immediately promote the new Rollout. This change was introduced to address https://github.com/argoproj/argo-rollouts/issues/80. 
+
+To prepare for v0.4.0, v0.3.2 will introduce the `autoPromotionEnabled` field, but the controller will not act on the field. As a result, you can add the `autoPromotionEnabled` field without breaking your existing rollouts.
+
+## Enhancements
+- Add autoPromotionEnabled with no behavior change 
+## Fixes
+- Fix controller crash caused by glog attempting to write to /tmp (##94)
+
+# v0.3.1
+
+## Breaking Changes
+Rename autoPromoteActiveServiceDelaySeconds to autoPromotionSeconds ##77
+
+## Changes since v0.3.0
+
+#### Enhancements
+* Switch to Scratch final image ##67
+
+#### Fixes
+* Enable fast Rollback in BlueGreen ##78 
+* Respect ScaleDownDelay during non-happy path ##79
+* Scale down older RS on non-happy path ##76
+* Fix issue where pod template hash could be computed inconsistently ##75
+* Cleanup replicasets in canary deployment ##73
+* Don't requeue 404 errors ##72
+
+# v0.3.0
+
+## New Features
+- HPA Support ##37
+- Prometheus Metrics ##29 ##47 
+- Introduce ProgressDeadlineSeconds ##54
+- Improved Scalability ##45 
+
+## Breaking Changes
+
+The `status.verifyingPreview` field was depreciated and move to `spec.pause`.
+
+#### BlueGreen Specific
+- Add previewReplicaCount ##64
+- Add ability to auto-promote active service ##59
+- Add ScaleDownDelaySeconds ##57
+
+## Changes since v0.2.2
+#### Enchantments 
+- HPA Support ##37
+- Prometheus Metrics ##29
+- Add previewReplicaCount ##64
+- Add ProgressDeadlineSeconds ##54
+- Add Invalid spec checks with regards to ProgressDeadlineSeconds ##62
+- Improve eventing and metrics ##61
+- Improve Available Condition ##60
+- Convert Kustomize V1.0 to Kustomize v2.0 ##56
+- Make Metrics port customizable ##55
+- Replace gometalinter with golangci ##46
+- Add support for gotestsum ##52
+- Remove service informer ##45 
+- Replace verifying preview with Paused ##43
+
+
+#### Bug fixes
+- Prevent early pause before svc change in BG ##51
+- Fix aggregate roles naming collision with Argo Workflows  ##44
+- Use recreate strategy for controller  ##44
+
+# v0.2.2
+Add missing events permissions to the clusterrole
+
+# v0.2.1
+Changes the following clusterroles to prevent name collision with Argo Workflows
+- `argo-aggregate-to-admin` to `argo-rollouts-aggregate-to-admin`
+- `argo-aggregate-to-edit` to `argo-rollouts-aggregate-to-edit`
+- `argo-aggregate-to-view` to `argo-rollouts-aggregate-to-view`
+
+# v0.2.0
+- Implements the initial ReplicaSet-based Canary Strategy
+- Cleans up Status fields
+- Implicit understanding of rollback based on steps completion and pod hash for Blue Green and Canary
+
+# v0.1.0
+* Creates a controller that manages a rollout object that mimics a deployment object
+* Declaratively offers a Blue Green Strategy by creating the replicaset from the spec and managing an active and preview service to point to the new replicaset

CONTRIBUTING.md

@@ -0,0 +1 @@
+See [docs/CONTRIBUTING.md](docs/CONTRIBUTING.md).

Dockerfile

@@ -3,33 +3,16 @@
 # Initial stage which pulls prepares build dependencies and CLI tooling we need for our final image
 # Also used as the image in CI jobs so needs all dependencies
 ####################################################################################################
-FROM golang:1.13.1 as builder
+FROM --platform=$BUILDPLATFORM golang:1.23 AS builder
 
 RUN apt-get update && apt-get install -y \
-    git \
-    make \
     wget \
-    gcc \
-    zip && \
+    ca-certificates && \
     apt-get clean && \
     rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
 
-# Install docker
-ENV DOCKER_VERSION=18.06.0
-RUN curl -O https://download.docker.com/linux/static/stable/x86_64/docker-${DOCKER_VERSION}-ce.tgz && \
-  tar -xzf docker-${DOCKER_VERSION}-ce.tgz && \
-  mv docker/docker /usr/local/bin/docker && \
-  rm -rf ./docker
-
-# Install dep
-ENV DEP_VERSION=0.5.0
-RUN wget https://github.com/golang/dep/releases/download/v${DEP_VERSION}/dep-linux-amd64 -O /usr/local/bin/dep && \
-    chmod +x /usr/local/bin/dep
-
 # Install golangci-lint
-RUN wget https://install.goreleaser.com/github.com/golangci/golangci-lint.sh  && \
-    chmod +x ./golangci-lint.sh && \
-    ./golangci-lint.sh -b $GOPATH/bin && \
+RUN curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin v2.1.6 && \
     golangci-lint linters
 
 COPY .golangci.yml ${GOPATH}/src/dummy/.golangci.yml
@@ -38,48 +21,77 @@ RUN cd ${GOPATH}/src/dummy && \
     touch dummy.go \
     golangci-lint run
 
+####################################################################################################
+# UI build stage
+####################################################################################################
+FROM --platform=$BUILDPLATFORM docker.io/library/node:18 AS argo-rollouts-ui
+
+WORKDIR /src
+ADD ["ui/package.json", "ui/yarn.lock", "./"]
+
+RUN yarn install --network-timeout 300000
+
+ADD ["ui/", "."]
+
+ARG ARGO_VERSION=latest
+ENV ARGO_VERSION=$ARGO_VERSION
+RUN NODE_ENV='production' yarn build
+
 ####################################################################################################
 # Rollout Controller Build stage which performs the actual build of argo-rollouts binaries
 ####################################################################################################
-FROM golang:1.12.6 as argo-rollouts-build
+FROM --platform=$BUILDPLATFORM golang:1.23 AS argo-rollouts-build
 
-COPY --from=builder /usr/local/bin/dep /usr/local/bin/dep
+WORKDIR /go/src/github.com/argoproj/argo-rollouts
 
+# Copy only go.mod and go.sum files. This way on subsequent docker builds if the
+# dependencies didn't change it won't re-download the dependencies for nothing.
+COPY go.mod go.sum ./
+RUN go mod download
 
-# A dummy directory is created under $GOPATH/src/dummy so we are able to use dep
-# to install all the packages of our dep lock file
-COPY Gopkg.toml ${GOPATH}/src/dummy/Gopkg.toml
-COPY Gopkg.lock ${GOPATH}/src/dummy/Gopkg.lock
-
-RUN cd ${GOPATH}/src/dummy && \
-    dep ensure -vendor-only && \
-    mv vendor/* ${GOPATH}/src/ && \
-    rmdir vendor
+# Copy UI files for plugin build
+COPY --from=argo-rollouts-ui /src/dist/app ./ui/dist/app
 
 # Perform the build
-WORKDIR /go/src/github.com/argoproj/argo-rollouts
 COPY . .
-ARG MAKE_TARGET="controller plugin-linux plugin-darwin"
-RUN make ${MAKE_TARGET}
 
+# stop make from trying to re-build this without yarn installed
+RUN touch ui/dist/node_modules.marker && \
+    mkdir -p ui/dist/app && \
+    touch ui/dist/app/index.html && \
+    find ui/dist
 
-RUN groupadd -g 999 argo-rollouts && \
-    useradd -r -u 999 -g argo-rollouts argo-rollouts && \
-    mkdir -p /home/argo-rollouts && \
-    chown argo-rollouts:argo-rollouts /home/argo-rollouts
+ARG TARGETOS
+ARG TARGETARCH
+ARG MAKE_TARGET="controller plugin"
+RUN GOOS=$TARGETOS GOARCH=$TARGETARCH make ${MAKE_TARGET}
 
+####################################################################################################
+# Kubectl plugin image
+####################################################################################################
+FROM gcr.io/distroless/static-debian11 AS kubectl-argo-rollouts
+
+COPY --from=argo-rollouts-build /go/src/github.com/argoproj/argo-rollouts/dist/kubectl-argo-rollouts /bin/kubectl-argo-rollouts
+
+USER 999
+
+WORKDIR /home/argo-rollouts
+
+ENTRYPOINT ["/bin/kubectl-argo-rollouts"]
+
+CMD ["dashboard"]
 
 ####################################################################################################
 # Final image
 ####################################################################################################
-FROM scratch
+FROM gcr.io/distroless/static-debian11
 
 COPY --from=argo-rollouts-build /go/src/github.com/argoproj/argo-rollouts/dist/rollouts-controller /bin/
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
 
-# Import the user and group files from the builder.
-COPY --from=argo-rollouts-build /etc/passwd /etc/passwd
-
-USER argo-rollouts
+# Use numeric user, allows kubernetes to identify this user as being
+# non-root when we use a security context with runAsNonRoot: true
+USER 999
 
 WORKDIR /home/argo-rollouts
 

Dockerfile.dev

@@ -1,6 +1,23 @@
 ####################################################################################################
 # argo-rollouts-dev
 ####################################################################################################
-FROM scratch
-COPY dist/rollouts-controller-linux-amd64 /bin/rollouts-controller
+FROM golang:1.20 AS builder
+
+RUN apt-get update && apt-get install -y \
+    ca-certificates && \
+    apt-get clean && \
+    rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/*
+
+FROM gcr.io/distroless/static-debian11
+
+COPY --from=builder /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/
+COPY rollouts-controller-linux-amd64 /bin/rollouts-controller
+COPY step-plugin-e2e-linux-amd64 /bin/plugin-files/step-plugin-e2e
+
+# Use numeric user, allows kubernetes to identify this user as being
+# non-root when we use a security context with runAsNonRoot: true
+USER 999
+
+WORKDIR /home/argo-rollouts
+
 ENTRYPOINT [ "/bin/rollouts-controller" ]

Gopkg.toml

@@ -1,65 +0,0 @@
-required = [
-    "k8s.io/code-generator/cmd/client-gen",
-    "github.com/emicklei/go-restful",
-    "github.com/go-openapi/spec",
-    "k8s.io/kube-openapi/cmd/openapi-gen",
-    "github.com/vektra/mockery",
-]
-
-[[constraint]]
-  name = "k8s.io/client-go"
-  branch = "release-13.0"
-
-[[constraint]]
-  name = "k8s.io/code-generator"
-  branch = "release-1.16"
-
-[[constraint]]
-  branch = "release-1.16"
-  name = "k8s.io/api"
-
-[[constraint]]
-  name = "k8s.io/apiserver"
-  branch = "release-1.16"
-
-[[constraint]]
-  name = "k8s.io/apimachinery"
-  branch = "release-1.16"
-
-[[constraint]]
-  name = "k8s.io/kubernetes"
-  branch = "release-1.16"
-
-[[constraint]]
-  name = "k8s.io/apiextensions-apiserver"
-  branch = "release-1.16"
-
-[[constraint]]
-  name = "k8s.io/cli-runtime"
-  branch = "release-1.16"
-
-[[constraint]]
-  name = "k8s.io/kubectl"
-  branch = "release-1.16"
-
-[[constraint]]
-  name = "github.com/bouk/monkey"
-  version = "1.0.0"
-
-# vendor/k8s.io/kubectl/pkg/util/term/resize.go:30:23: undefined: term.GetFdInfo
-[[override]]
-  name = "github.com/docker/docker"
-  revision = "be7ac8be2ae072032a4005e8f232be3fc57e4127"
-
-# vendor/k8s.io/kubectl/pkg/util/templates/markdown.go:66:11: undefined: blackfriday.LIST_ITEM_BEGINNING_OF_LIST
-[[override]]
-  name = "github.com/russross/blackfriday"
-  version = "v1.5.2"
-
-[[override]]
-  name = "github.com/ghodss/yaml"
-  branch = "master"
-
-[[constraint]]
-  name = "github.com/vektra/mockery"
-  revision = "e78b021dcbb558a8e7ac1fc5bc757ad7c277bb81"

Makefile

@@ -1,17 +1,36 @@
 PACKAGE=github.com/argoproj/argo-rollouts
 CURRENT_DIR=$(shell pwd)
 DIST_DIR=${CURRENT_DIR}/dist
+PATH := $(DIST_DIR):$(PATH)
 PLUGIN_CLI_NAME?=kubectl-argo-rollouts
 TEST_TARGET ?= ./...
 
-VERSION=$(shell cat ${CURRENT_DIR}/VERSION)
 BUILD_DATE=$(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
 GIT_COMMIT=$(shell git rev-parse HEAD)
 GIT_TAG=$(shell if [ -z "`git status --porcelain`" ]; then git describe --exact-match --tags HEAD 2>/dev/null; fi)
 GIT_TREE_STATE=$(shell if [ -z "`git status --porcelain`" ]; then echo "clean" ; else echo "dirty"; fi)
+GIT_REMOTE_REPO=upstream
+VERSION=$(shell if [ ! -z "${GIT_TAG}" ] ; then echo "${GIT_TAG}" | sed -e "s/^v//"  ; else cat VERSION ; fi)
 
+
+TARGET_ARCH?=linux/amd64
+
+# docker image publishing options
+DOCKER_PUSH ?= false
+IMAGE_TAG ?= latest
 # build development images
-DEV_IMAGE=false
+DEV_IMAGE ?= false
+
+# E2E variables
+E2E_K8S_CONTEXT ?= rancher-desktop
+E2E_INSTANCE_ID ?= argo-rollouts-e2e
+E2E_TEST_OPTIONS ?=
+E2E_PARALLEL ?= 1
+E2E_WAIT_TIMEOUT ?= 90
+GOPATH ?= $(shell go env GOPATH)
+
+# Global toolchain configuration
+NODE_OPTIONS=""
 
 override LDFLAGS += \
   -X ${PACKAGE}/utils/version.version=${VERSION} \
@@ -19,23 +38,14 @@ override LDFLAGS += \
   -X ${PACKAGE}/utils/version.gitCommit=${GIT_COMMIT} \
   -X ${PACKAGE}/utils/version.gitTreeState=${GIT_TREE_STATE}
 
-# docker image publishing options
-DOCKER_PUSH=false
-IMAGE_TAG=latest
 ifneq (${GIT_TAG},)
 IMAGE_TAG=${GIT_TAG}
-LDFLAGS += -X ${PACKAGE}.gitTag=${GIT_TAG}
-endif
-ifneq (${IMAGE_NAMESPACE},)
-override LDFLAGS += -X ${PACKAGE}/install.imageNamespace=${IMAGE_NAMESPACE}
-endif
-ifneq (${IMAGE_TAG},)
-override LDFLAGS += -X ${PACKAGE}/install.imageTag=${IMAGE_TAG}
+override LDFLAGS += -X ${PACKAGE}.gitTag=${GIT_TAG}
 endif
 
 ifeq (${DOCKER_PUSH},true)
 ifndef IMAGE_NAMESPACE
-$(error IMAGE_NAMESPACE must be set to push images (e.g. IMAGE_NAMESPACE=argoproj))
+$(error IMAGE_NAMESPACE must be set to push images (e.g. IMAGE_NAMESPACE=quay.io/argoproj))
 endif
 endif
 
@@ -43,88 +53,293 @@ ifdef IMAGE_NAMESPACE
 IMAGE_PREFIX=${IMAGE_NAMESPACE}/
 endif
 
+## protoc,my.proto
+define protoc
+	# protoc $(1)
+    PATH=${DIST_DIR}:$$PATH protoc \
+      -I /usr/local/include \
+      -I ${DIST_DIR}/protoc-include \
+      -I . \
+      -I ./vendor \
+      -I ${GOPATH}/src \
+      -I ${GOPATH}/pkg/mod/github.com/gogo/protobuf@v1.3.2/gogoproto \
+      -I ${GOPATH}/pkg/mod/github.com/grpc-ecosystem/grpc-gateway@v1.16.0/third_party/googleapis \
+      --gogofast_out=plugins=grpc:${CURDIR} \
+      --grpc-gateway_out=logtostderr=true:${CURDIR} \
+      --swagger_out=logtostderr=true,fqn_for_swagger_name=true:. \
+      $(1)
+endef
+
 .PHONY: all
 all: controller image
 
-.PHONY: codegen
-codegen: mocks
-	./hack/update-codegen.sh
-	./hack/update-openapigen.sh
-	go run ./hack/gen-crd-spec/main.go
+##@ Development
+.PHONY: go-mod-vendor
+go-mod-vendor: ## downloads vendor files needed by tools.go (i.e. go_install)
+	go mod tidy
+	go mod vendor
 
-.PHONY: controller
-controller: clean-debug
-	CGO_ENABLED=0 go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/rollouts-controller ./cmd/rollouts-controller
+.PHONY: lint
+lint: go-mod-vendor ## run all linters
+	golangci-lint run --fix
+
+.PHONY: install-go-tools-local
+install-go-tools-local: go-mod-vendor ## install all go tools
+	./hack/installers/install-codegen-go-tools.sh
+
+.PHONY: install-protoc-local
+install-protoc-local: ## install protoc tool
+	./hack/installers/install-protoc.sh
+
+.PHONY: install-devtools-local
+install-devtools-local: ## install dev tools
+	./hack/installers/install-dev-tools.sh
+
+# Installs all tools required to build and test locally
+.PHONY: install-tools-local
+install-tools-local: install-go-tools-local install-protoc-local install-devtools-local
+
+TYPES := $(shell find pkg/apis/rollouts/v1alpha1 -type f -name '*.go' -not -name openapi_generated.go -not -name '*generated*' -not -name '*test.go')
+APIMACHINERY_PKGS=k8s.io/apimachinery/pkg/util/intstr,+k8s.io/apimachinery/pkg/api/resource,+k8s.io/apimachinery/pkg/runtime/schema,+k8s.io/apimachinery/pkg/runtime,k8s.io/apimachinery/pkg/apis/meta/v1,k8s.io/api/core/v1,k8s.io/api/batch/v1
+PKG := $(shell go list ./pkg/apis/rollouts/v1alpha1)
+
+.PHONY: install-toolchain
+install-toolchain: install-go-tools-local install-protoc-local
+
+##@ Code Generation
+
+# generates all auto-generated code
+.PHONY: codegen
+codegen: go-mod-vendor gen-proto gen-k8scodegen gen-openapi gen-mocks gen-crd manifests docs
+
+# generates all files related to proto files
+.PHONY: gen-proto
+gen-proto: k8s-proto api-proto ui-proto
+
+# generates the .proto files affected by changes to types.go
+.PHONY: k8s-proto
+k8s-proto: go-mod-vendor install-protoc-local install-go-tools-local $(TYPES) ## generate kubernetes protobuf files
+	mkdir -p ${PKG}
+	cp -f $(CURDIR)/pkg/apis/rollouts/v1alpha1/*.* ${PKG}/
+	PATH=${DIST_DIR}:$$PATH GOPATH=${GOPATH} go-to-protobuf \
+		--go-header-file=./hack/custom-boilerplate.go.txt \
+		--packages=${PKG} \
+		--apimachinery-packages=${APIMACHINERY_PKGS} \
+		--proto-import=${CURDIR}/vendor \
+		--proto-import=${GOPATH}/src \
+		--proto-import=${DIST_DIR}/protoc-include 
+	touch pkg/apis/rollouts/v1alpha1/generated.proto
+	cp -Rf $(CURDIR)/github.com/argoproj/argo-rollouts/pkg . | true
+	# cleaning up
+	rm -Rf $(CURDIR)/github.com/
+	rm -Rf $(CURDIR)/k8s.io/
+
+# generates *.pb.go, *.pb.gw.go, swagger from .proto files
+.PHONY: api-proto
+api-proto: go-mod-vendor k8s-proto ## generate api protobuf files
+	mkdir -p ${PKG}
+	cp -f $(CURDIR)/pkg/apis/rollouts/v1alpha1/generated.proto ${PKG}
+	$(call protoc,pkg/apiclient/rollout/rollout.proto)
+	cp -Rf $(CURDIR)/github.com/argoproj/argo-rollouts/pkg . | true
+	# cleaning up
+	rm -Rf $(CURDIR)/github.com/
+
+# generates ui related proto files
+.PHONY: ui-proto
+ui-proto: ## generate ui protobuf files
+	yarn --cwd ui run protogen
+
+# generates k8s client, informer, lister, deepcopy from types.go
+.PHONY: gen-k8scodegen
+gen-k8scodegen: go-mod-vendor ## generate kubernetes codegen files
+	./hack/update-codegen.sh
+
+# generates ./manifests/crds/
+.PHONY: gen-crd
+gen-crd: go-mod-vendor install-go-tools-local ## generate crd manifests
+	go run -mod=mod ./hack/gen-crd-spec/main.go
+
+# generates mock files from interfaces
+.PHONY: gen-mocks
+gen-mocks: install-go-tools-local ## generate mock files
+	./hack/update-mocks.sh
+
+gen-mocks-fast:
+	./hack/update-mocks.sh
+
+# generates openapi_generated.go
+.PHONY: gen-openapi
+gen-openapi: install-go-tools-local $(DIST_DIR)/openapi-gen ## generate openapi files
+	PATH=${DIST_DIR}:$$PATH GOPATH=${GOPATH} openapi-gen ${CURRENT_DIR}/pkg/apis/rollouts/v1alpha1 \
+		--go-header-file ${CURRENT_DIR}/hack/custom-boilerplate.go.txt \
+		--output-pkg github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1 \
+		--output-dir ${CURRENT_DIR}/pkg/apis/rollouts/v1alpha1 \
+		--output-file openapi_generated.go \
+		--report-filename ${CURRENT_DIR}/pkg/apis/api-rules/violation_exceptions.list
+
+##@ Plugins
 
 .PHONY: plugin
-plugin:
-	CGO_ENABLED=0 go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME} ./cmd/kubectl-argo-rollouts
+plugin: ui/dist ## build plugin
+	cp -r ui/dist/app/* server/static
+	CGO_ENABLED=0 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME} ./cmd/kubectl-argo-rollouts
+
+ui/dist:
+	yarn --cwd ui install
+	yarn --cwd ui build
 
 .PHONY: plugin-linux
-plugin-linux:
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME}-linux-amd64 ./cmd/kubectl-argo-rollouts
+plugin-linux: ui/dist ## build plugin for linux
+	cp -r ui/dist/app/* server/static
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME}-linux-amd64 ./cmd/kubectl-argo-rollouts
+	CGO_ENABLED=0 GOOS=linux GOARCH=arm64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME}-linux-arm64 ./cmd/kubectl-argo-rollouts
 
 .PHONY: plugin-darwin
-plugin-darwin:
-	CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME}-darwin-amd64 ./cmd/kubectl-argo-rollouts
+plugin-darwin: ui/dist ## build plugin for darwin
+	cp -r ui/dist/app/* server/static
+	CGO_ENABLED=0 GOOS=darwin GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME}-darwin-amd64 ./cmd/kubectl-argo-rollouts
+	CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME}-darwin-arm64 ./cmd/kubectl-argo-rollouts
+
+.PHONY: plugin-windows
+plugin-windows: ui/dist  ## build plugin for windows
+	cp -r ui/dist/app/* server/static
+	CGO_ENABLED=0 GOOS=windows GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/${PLUGIN_CLI_NAME}-windows-amd64 ./cmd/kubectl-argo-rollouts
+
+##@ Build
+
+.PHONY: controller
+controller: ## build controller binary
+	CGO_ENABLED=0 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/rollouts-controller ./cmd/rollouts-controller
 
 .PHONY: builder-image
-builder-image:
-	docker build  -t $(IMAGE_PREFIX)argo-rollouts-ci-builder:$(IMAGE_TAG) --target builder .
+builder-image: ## build builder image
+	DOCKER_BUILDKIT=1 docker build  -t $(IMAGE_PREFIX)argo-rollouts-ci-builder:$(IMAGE_TAG) --target builder .
 		@if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG) ; fi
 
 .PHONY: image
 image:
 ifeq ($(DEV_IMAGE), true)
-	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -i -ldflags '${LDFLAGS}' -o ${DIST_DIR}/rollouts-controller-linux-amd64 ./cmd/rollouts-controller
-	docker build -t $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG) -f Dockerfile.dev .
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/step-plugin-e2e-linux-amd64 ./test/cmd/step-plugin-e2e
+	CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -v -ldflags '${LDFLAGS}' -o ${DIST_DIR}/rollouts-controller-linux-amd64 ./cmd/rollouts-controller
+	DOCKER_BUILDKIT=1 docker build --platform=$(TARGET_ARCH) -t $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG) -f Dockerfile.dev ${DIST_DIR}
 else
-	docker build -t $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG)  .
+	DOCKER_BUILDKIT=1 docker build --platform=$(TARGET_ARCH) -t $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG)  .
 endif
 	@if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)argo-rollouts:$(IMAGE_TAG) ; fi
 
-.PHONY: lint
-lint:
-	golangci-lint run --fix
+
+# Build sample plugin with debug info
+# https://www.jetbrains.com/help/go/attach-to-running-go-processes-with-debugger.html
+.PHONY: build-sample-metric-plugin-debug
+build-sample-metric-plugin-debug: ## build sample metric plugin with debug info
+	go build -gcflags="all=-N -l" -o plugin-bin/metric-plugin test/cmd/metrics-plugin-sample/main.go
+
+.PHONY: build-sample-traffic-plugin-debug
+build-sample-traffic-plugin-debug: ## build sample traffic plugin with debug info
+	go build -gcflags="all=-N -l" -o plugin-bin/traffic-plugin test/cmd/trafficrouter-plugin-sample/main.go
+
+.PHONY: build-sample-step-plugin-debug
+build-sample-step-plugin-debug: ## build sample traffic plugin with debug info
+	go build -gcflags="all=-N -l" -o plugin-bin/step-plugin test/cmd/step-plugin-sample/main.go
+
+.PHONY: plugin-image
+plugin-image: ## build plugin image
+	DOCKER_BUILDKIT=1 docker build --platform=$(TARGET_ARCH) --target kubectl-argo-rollouts -t $(IMAGE_PREFIX)kubectl-argo-rollouts:$(IMAGE_TAG) .
+	if [ "$(DOCKER_PUSH)" = "true" ] ; then docker push $(IMAGE_PREFIX)kubectl-argo-rollouts:$(IMAGE_TAG) ; fi
+
+##@ Test
 
 .PHONY: test
-test:
-	go test -failfast -covermode=count -coverprofile=coverage.out ${TEST_TARGET}
+test: test-kustomize ## run all tests
+	@make test-unit
+
+.PHONY: test-kustomize
+test-kustomize: ## run kustomize tests
+	./test/kustomize/test.sh
+
+setup-e2e:
+	@kubectl apply --context='${E2E_K8S_CONTEXT}' -f manifests/crds/rollout-crd.yaml
+	@kubectl apply --context='${E2E_K8S_CONTEXT}' -n argo-rollouts -f test/e2e/step-plugin/argo-rollouts-config.yaml
+	@rm -rf plugin-bin
+	@go build -gcflags="all=-N -l" -o plugin-bin/e2e-step-plugin test/cmd/step-plugin-e2e/main.go
+
+.PHONY: start-e2e
+start-e2e: ## start e2e test environment
+	mkdir -p coverage-output-e2e
+	GOCOVERDIR=coverage-output-e2e go run -cover ./cmd/rollouts-controller/main.go --instance-id ${E2E_INSTANCE_ID} --loglevel debug --kloglevel 6
+
+.PHONY: test-e2e
+test-e2e: install-devtools-local
+	${DIST_DIR}/gotestsum --rerun-fails-report=rerunreport.txt --junitfile=junit-e2e-test.xml --format=testname --packages="./test/e2e" --rerun-fails=5 -- -timeout 60m -count 1 --tags e2e -p ${E2E_PARALLEL} -parallel ${E2E_PARALLEL} -v --short ./test/e2e ${E2E_TEST_OPTIONS}
+
+.PHONY: test-unit
+ test-unit: install-devtools-local ## run unit tests
+	mkdir -p coverage-output-unit
+	${DIST_DIR}/gotestsum --junitfile=junit-unit-test.xml --format=testname -- `go list ./... | grep -v ./test/cmd/metrics-plugin-sample` -cover -test.gocoverdir=$(CURDIR)/coverage-output-unit
+
 
 .PHONY: coverage
-coverage: test
+coverage: test ## run coverage tests
 	go tool cover -html=coverage.out -o coverage.html
 	open coverage.html
 
-.PHONY: mocks
-mocks:
-	go run ./vendor/github.com/vektra/mockery/cmd/mockery/mockery.go -dir ./metricproviders -name Provider -output ./metricproviders/mocks
-
 .PHONY: manifests
-manifests:
+manifests: ## generate manifests e.g. CRD, RBAC etc.
 	./hack/update-manifests.sh
 
-# Cleans VSCode debug.test files from sub-dirs to prevent them from being included in packr boxes
-.PHONY: clean-debug
-clean-debug:
-	-find ${CURRENT_DIR} -name debug.test | xargs rm -f
-
 .PHONY: clean
-clean: clean-debug
+clean: ## clean up build artifacts
 	-rm -rf ${CURRENT_DIR}/dist
+	-rm -rf ${CURRENT_DIR}/ui/dist
+	-rm -Rf ${CURRENT_DIR}/github.com/
+	-rm -Rf ${CURRENT_DIR}/k8s.io/
 
 .PHONY: precheckin
 precheckin: test lint
 
+##@ Docs
+
+# convenience target to run `mkdocs serve` using a docker container
+.PHONY: serve-docs
+serve-docs: docs ## serve docs locally
+	docker run --rm -it -p 8000:8000 -v ${CURRENT_DIR}:/docs squidfunk/mkdocs-material serve -a 0.0.0.0:8000
+
+.PHONY: docs
+docs: go-mod-vendor install-go-tools-local ## build docs
+	go run -mod=mod ./hack/gen-docs/main.go
+
+##@ Release
+
+.PHONY: release-docs
+release-docs: docs ## build and deploy docs
+	docker run --rm -it \
+		-v ~/.ssh:/root/.ssh \
+		-v ${CURRENT_DIR}:/docs \
+		-v ~/.gitconfig:/root/.gitconfig \
+		squidfunk/mkdocs-material gh-deploy -r ${GIT_REMOTE_REPO}
+
 .PHONY: release-precheck
-release-precheck: manifests
+release-precheck: manifests ## precheck release
 	@if [ "$(GIT_TREE_STATE)" != "clean" ]; then echo 'git tree state is $(GIT_TREE_STATE)' ; exit 1; fi
 	@if [ -z "$(GIT_TAG)" ]; then echo 'commit must be tagged to perform release' ; exit 1; fi
 	@if [ "$(GIT_TAG)" != "v`cat VERSION`" ]; then echo 'VERSION does not match git tag'; exit 1; fi
 
 .PHONY: release-plugins
-release-plugins:
+release-plugins: ## build and push plugins
 	./hack/build-release-plugins.sh
 
 .PHONY: release
-release: release-precheck precheckin image release-plugins
+release: release-precheck precheckin image plugin-image release-plugins
+
+.PHONY: trivy
+trivy: ## run trivy scan
+	@trivy fs --clear-cache
+	@trivy fs .
+
+.PHONY: checksums
+checksums:
+	shasum -a 256 ./dist/kubectl-argo-rollouts-* | awk -F './dist/' '{print $$1 $$2}' > ./dist/argo-rollouts-checksums.txt
+
+
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

OWNERS

@@ -1,9 +1,18 @@
 owners:
 - alexmt
-- dthomson25
 - jessesuen
+- zachaller
 
 approvers:
 - alexmt
-- dthomson25
+- huikang
 - jessesuen
+- khhirani
+- leoluz
+
+reviewers:
+- agrawroh
+- dthomson25
+- harikrongali
+- kostis-codefresh
+- perenesenko

README.md

@@ -1,22 +1,106 @@
 
-# Argo Rollouts - Advanced Kubernetes Deployment Controller
+# Argo Rollouts - Progressive Delivery for Kubernetes
+
 [![codecov](https://codecov.io/gh/argoproj/argo-rollouts/branch/master/graph/badge.svg)](https://codecov.io/gh/argoproj/argo-rollouts)
 [![slack](https://img.shields.io/badge/slack-argoproj-brightgreen.svg?logo=slack)](https://argoproj.github.io/community/join-slack)
+[![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/3834/badge)](https://bestpractices.coreinfrastructure.org/projects/3834)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/argoproj/argo-rollouts/badge)](https://api.securityscorecards.dev/projects/github.com/argoproj/argo-rollouts)
+[![Artifact HUB](https://img.shields.io/endpoint?url=https://artifacthub.io/badge/repository/argo-rollouts)](https://artifacthub.io/packages/helm/argo/argo-rollouts)
 
 ## What is Argo Rollouts?
-Argo Rollouts controller, uses the Rollout custom resource to provide additional deployment strategies such as Blue Green and Canary to Kubernetes.  The Rollout custom resource provides feature parity with the deployment resource but with additional deployment strategies.
 
-## Why use Argo Rollouts?
-Deployments resources offer two strategies to deploy changes: `RollingUpdate` and `Recreate`. While these strategies can solve a wide number of use cases, large scale production deployments use additional strategies, such as blue-green or canary, that are missing from the Deployment controller.  In order to use these strategies in Kubernetes, users are forced to build scripts on top of their deployments. The Argo Rollouts controller provides these strategies as simple declarative, configurable options.
+Argo Rollouts is a Kubernetes controller and set of CRDs which provide advanced deployment capabilities such as blue-green, canary, canary analysis, experimentation, and progressive delivery features to Kubernetes.
+
+Argo Rollouts (optionally) integrates with ingress controllers and service meshes, leveraging their traffic shaping abilities to gradually shift traffic to the new version during an update. Additionally, Rollouts can query and interpret metrics from various providers to verify key KPIs and drive automated promotion or rollback during an update.
+
+[![Argo Rollotus Demo](https://img.youtube.com/vi/hIL0E2gLkf8/0.jpg)](https://youtu.be/hIL0E2gLkf8)
+
+## Quick Start
+
+```bash
+kubectl create namespace argo-rollouts
+kubectl apply -n argo-rollouts -f https://github.com/argoproj/argo-rollouts/releases/latest/download/install.yaml
+```
+
+Follow the full [getting started guide](docs/getting-started.md) to walk through creating and then updating a rollout object.
+
+## Why Argo Rollouts?
+
+Kubernetes Deployments provides the `RollingUpdate` strategy which provide a basic set of safety guarantees (readiness probes) during an update. However the rolling update strategy faces many limitations:
+
+* Few controls over the speed of the rollout
+* Inability to control traffic flow to the new version
+* Readiness probes are unsuitable for deeper, stress, or one-time checks
+* No ability to query external metrics to verify an update
+* Can halt the progression, but unable to automatically abort and rollback the update
+
+For these reasons, in large scale high-volume production environments, a rolling update is often considered too risky of an update procedure since it provides no control over the blast radius, may rollout too aggressively, and provides no automated rollback upon failures.
+
+## Features
+
+* Blue-Green update strategy
+* Canary update strategy
+* Fine-grained, weighted traffic shifting
+* Automated rollbacks and promotions
+* Manual judgement
+* Customizable metric queries and analysis of business KPIs
+* Ingress controller integration: NGINX, ALB, Apache APISIX
+* Service Mesh integration: Istio, Linkerd, SMI
+* Metric provider integration: Prometheus, Wavefront, Kayenta, Web, Kubernetes Jobs, Datadog, New Relic, InfluxDB
+
+## Supported Traffic Shaping Integrations
+| Traffic Shaping Integration       | SetWeight                    | SetWeightExperiments        | SetMirror                  | SetHeader                  | Implemented As Plugin       |
+|-----------------------------------|------------------------------|-----------------------------|----------------------------|----------------------------|-----------------------------|
+| ALB Ingress Controller            | :white_check_mark: (stable)  | :white_check_mark: (stable) | :x:                        | :white_check_mark: (alpha) |                             |
+| Ambassador                        | :white_check_mark: (stable)  | :x:                         | :x:                        | :x:                        |                             |
+| Apache APISIX Ingress Controller  | :white_check_mark: (alpha)   | :x:                         | :x:                        | :white_check_mark: (alpha) |                             |
+| Istio                             | :white_check_mark: (stable)  | :white_check_mark: (stable) | :white_check_mark: (alpha) | :white_check_mark: (alpha) |                             |
+| Nginx Ingress Controller          | :white_check_mark: (stable)  | :x:                         | :x:                        | :x:                        |                             |
+| SMI                               | :white_check_mark: (stable)  | :white_check_mark: (stable) | :x:                        | :x:                        |                             |
+| Traefik                           | :white_check_mark: (beta)    | :x:                         | :x:                        | :x:                        |                             |
+| Contour                           | :white_check_mark: (beta)    | :x:                         | :x:                        | :x:                        | :heavy_check_mark:          |
+| Gateway API                       | :white_check_mark: (alpha)   | :x:                         | :x:                        | :x:                        | :heavy_check_mark:          |
+
+:white_check_mark: = Supported
+
+:x: = Not Supported
+
+:heavy_check_mark: = Yes
 
 ## Documentation
-To learn more about Argo Rollouts go to the [complete documentation](https://argoproj.github.io/argo-rollouts/).
+
+To learn more about Argo Rollouts go to the [complete documentation](https://argo-rollouts.readthedocs.io/en/stable/).
+
+## Community
+
+You can reach the Argo Rollouts community and developers via the following channels:
+
+* Q & A: [Github Discussions](https://github.com/argoproj/argo-rollouts/discussions)
+* Chat: [The #argo-rollouts Slack channel](https://argoproj.github.io/community/join-slack)
+* Contributors Office Hours: [Every Thursday](https://calendar.google.com/calendar/u/0/embed?src=argoproj@gmail.com) | [Agenda](https://docs.google.com/document/d/1xkoFkVviB70YBzSEa4bDnu-rUZ1sIFtwKKG1Uw8XsY8)
+* User Community meeting: [First Wednesday of each month](https://calendar.google.com/calendar/u/0/embed?src=argoproj@gmail.com) | [Agenda](https://docs.google.com/document/d/1ttgw98MO45Dq7ZUHpIiOIEfbyeitKHNfMjbY5dLLMKQ)
 
 ## Who uses Argo Rollouts?
-Organizations below are **officially** using Argo Rollouts. Please send a PR with your organization name if you are using Argo Rollouts.
 
-1. [Intuit](https://www.intuit.com/)
+[Official Argo Rollouts User List](https://github.com/argoproj/argo-rollouts/blob/master/USERS.md)
 
 ## Community Blogs and Presentations
+
+* [Awesome-Argo: A Curated List of Awesome Projects and Resources Related to Argo](https://github.com/terrytangyuan/awesome-argo)
+* [Automation of Everything - How To Combine Argo Events, Workflows & Pipelines, CD, and Rollouts](https://youtu.be/XNXJtxkUKeY)
+* [Argo Rollouts - Canary Deployments Made Easy In Kubernetes](https://youtu.be/84Ky0aPbHvY)
 * [How Intuit Does Canary and Blue Green Deployments](https://www.youtube.com/watch?v=yeVkTTO9nOA)
 * [Leveling Up Your CD: Unlocking Progressive Delivery on Kubernetes](https://www.youtube.com/watch?v=Nv0PPwbIEkY)
+* [Minimize failed deployments with Argo Rollouts and Smoke tests](https://codefresh.io/continuous-deployment/minimize-failed-deployments-argo-rollouts-smoke-tests/)
+* [Recover automatically from failed deployments with Argo Rollouts and Prometheus metrics](https://codefresh.io/continuous-deployment/recover-automatically-from-failed-deployments/)
+* [Kubernetes Blue-Green deployments with Argo Rollouts](https://www.youtube.com/watch?v=krDxDz4V4Tg)
+* [Kubernetes canary deployments with Argo Rollouts](https://www.youtube.com/watch?v=fviYWA2mcF8)
+* [GitOps with Argo CD and an Argo Rollouts canary release](https://www.youtube.com/watch?v=35Qimb_AZ8U)
+* [Multi-Stage Delivery with Keptn and Argo Rollouts](https://www.youtube.com/watch?v=w-E8FzTbN3g&t=1s)
+* [Gradual Code Releases Using an In-House Kubernetes Canary Controller on top of Argo Rollouts](https://doordash.engineering/2021/04/14/gradual-code-releases-using-an-in-house-kubernetes-canary-controller/)
+* [How Scalable is Argo-Rollouts: A Cloud Operator’s Perspective](https://www.youtube.com/watch?v=rCEhxJ2NSTI)
+* [Minimize Impact in Kubernetes Using Argo Rollouts](https://medium.com/@arielsimhon/minimize-impact-in-kubernetes-using-argo-rollouts-992fb9519969)
+* [Progressive Application Delivery with GitOps on Red Hat OpenShift](https://www.youtube.com/watch?v=DfeL7cdTx4c)
+* [Progressive delivery for Kubernetes Config Maps using Argo Rollouts](https://codefresh.io/blog/progressive-delivery-for-kubernetes-config-maps-using-argo-rollouts/)
+* [Multi-Service Progressive Delivery with Argo Rollouts](https://codefresh.io/blog/multi-service-progressive-delivery-with-argo-rollouts/)
+* [Progressive Delivery for Stateful Services Using Argo Rollouts](https://codefresh.io/blog/progressive-delivery-for-stateful-services-using-argo-rollouts/)

USERS.md

@@ -0,0 +1,78 @@
+## Who uses Argo Rollouts?
+
+Organizations below are **officially** using Argo Rollouts. Please send a PR with your organization name if you are using Argo Rollouts.
+
+1. [7shifts](https://www.7shifts.com)
+1. [Ada](https://www.ada.cx)
+1. [ADP](https://www.adp.com)
+1. [Akuity](https://akuity.io/)
+1. [Alibaba Group](https://www.alibabagroup.com/)
+1. [Amadeus IT Group](https://amadeus.com/)
+1. [Ambassador Labs](https://www.getambassador.io)
+1. [Ant Group](https://www.antgroup.com/)
+1. [AppsFlyer](https://www.appsflyer.com/)
+1. [Batumbu](https://batumbu.id/)
+1. [Bucketplace](https://www.bucketplace.co.kr/)
+1. [BukuKas](https://bukukas.co.id/)
+1. [Calm](https://www.calm.com/)
+1. [CarGurus](https://www.cargurus.com/)
+1. [CircleCI](https://circleci.com/)
+1. [Cloudflare](https://cloudflare.com/)
+1. [Codefresh](https://codefresh.io/)
+1. [Credit Karma](https://creditkarma.com/)
+1. [DaoCloud](https://daocloud.io)
+1. [Databricks](https://github.com/databricks)
+1. [Devtron Labs](https://github.com/devtron-labs/devtron)
+1. [Doubble](https://www.doubble.app)
+1. [Factorial](https://factorialhr.com)
+1. [Farfetch](https://www.farfetch.com/)
+1. [Flipkart](https://flipkart.com)
+1. [GetYourGuide](https://www.getyourguide.com)
+1. [Gllue](https://gllue.com)
+1. [Groww](https://groww.in/)
+1. [HashiCorp](https://www.hashicorp.com/)
+1. [Hepsiburada](https://hepsiburada.com/)
+1. [Ibotta](https://home.ibotta.com/)
+1. [Intuit](https://www.intuit.com/)
+1. [Microsoft](https://microsoft.com/)
+1. [New Relic](https://newrelic.com/)
+1. [Nextdoor](https://nextdoor.com)
+1. [Nitro](https://www.gonitro.com)
+1. [Nozzle](https://nozzle.io)
+1. [Opensurvey Inc.](https://opensurvey.co.kr)
+1. [OpsMx](https://opsmx.io)
+1. [OpsVerse](https://opsverse.io)
+1. [Optum](https://www.optum.com)
+1. [Outreach](https://www.outreach.io)
+1. [PagerDuty](https://www.pagerduty.com/)
+1. [PayPal](https://www.paypal.com/)
+1. [PayPay](https://paypay.ne.jp/)
+1. [Plaid](https://plaid.com/)
+1. [Priceline](https://priceline.com)
+1. [Productboard](https://www.productboard.com)
+1. [Quipper](https://www.quipper.com/)
+1. [Quizlet](https://quizlet.com)
+1. [Red Hat](https://www.redhat.com/)
+1. [Salesforce](https://www.salesforce.com/)
+1. [SAP Concur](https://www.concur.com/)
+1. [Schneider Electric](https://www.se.com)
+1. [Shipt](https://www.shipt.com/)
+1. [Skillz](https://www.skillz.com)
+1. [Spotify](https://www.spotify.com/)
+1. [suXess-it](https://www.suxess-it.com/)
+1. [Synamedia](https://www.synamedia.com)
+1. [StormForge](https://www.stormforge.io)
+1. [Taboola](https://www.taboola.com)
+1. [TBC Bank](https://tbcbank.ge/)
+1. [Trustly](https://www.trustly.com/)
+1. [Tuhu](https://www.tuhu.cn/)
+1. [Twilio SendGrid](https://sendgrid.com)
+1. [Ubie](https://ubie.life/)
+1. [Verkada](https://verkada.com)
+1. [VISITS Technologies](https://visits.world/en)
+1. [WeLab Bank](https://www.welab.bank/)
+1. [Wolt](https://wolt.com/)
+1. [Yotpo](https://www.yotpo.com/)
+1. [Yuno](https://y.uno/)
+1. [VGS](https://www.vgs.io)
+1. [X3M ads](https://x3mads.com)

VERSION

@@ -1 +1 @@
-0.7.0
+1.2.0

analysis/analysis.go

@@ -1,35 +1,36 @@
 package analysis
 
 import (
+	"context"
 	"fmt"
+	"strings"
 	"sync"
 	"time"
 
 	log "github.com/sirupsen/logrus"
 	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 
 	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
 	analysisutil "github.com/argoproj/argo-rollouts/utils/analysis"
+	"github.com/argoproj/argo-rollouts/utils/defaults"
 	logutil "github.com/argoproj/argo-rollouts/utils/log"
+	"github.com/argoproj/argo-rollouts/utils/record"
+	timeutil "github.com/argoproj/argo-rollouts/utils/time"
 )
 
 const (
 	// DefaultMeasurementHistoryLimit is the default maximum number of measurements to retain per metric,
 	// before trimming the list.
 	DefaultMeasurementHistoryLimit = 10
-	// DefaultConsecutiveErrorLimit is the default number times a metric can error in sequence before
-	// erroring the entire metric.
-	DefaultConsecutiveErrorLimit int32 = 4
 	// DefaultErrorRetryInterval is the default interval to retry a measurement upon error, in the
 	// event an interval was not specified
-	DefaultErrorRetryInterval time.Duration = 10 * time.Second
-)
-
-// Event reasons for analysis events
-const (
-	EventReasonStatusFailed    = "Failed"
-	EventReasonStatusCompleted = "Complete"
+	DefaultErrorRetryInterval = 10 * time.Second
+	// SuccessfulAssessmentRunTerminatedResult is used for logging purposes when the metrics evaluation
+	// is successful and the run is terminated.
+	SuccessfulAssessmentRunTerminatedResult = "Metric Assessment Result - Successful: Run Terminated"
 )
 
 // metricTask holds the metric which need to be measured during this reconciliation along with
@@ -39,91 +40,165 @@ type metricTask struct {
 	incompleteMeasurement *v1alpha1.Measurement
 }
 
-func (c *AnalysisController) reconcileAnalysisRun(origRun *v1alpha1.AnalysisRun) *v1alpha1.AnalysisRun {
+func (c *Controller) reconcileAnalysisRun(origRun *v1alpha1.AnalysisRun) *v1alpha1.AnalysisRun {
+	logger := logutil.WithAnalysisRun(origRun)
 	if origRun.Status.Phase.Completed() {
+		err := c.maybeGarbageCollectAnalysisRun(origRun, logger)
+		if err != nil {
+			// TODO(jessesuen): surface errors to controller so they can be retried
+			logger.Warnf("Failed to garbage collect analysis run: %v", err)
+		}
 		return origRun
 	}
-	log := logutil.WithAnalysisRun(origRun)
 	run := origRun.DeepCopy()
 
 	if run.Status.MetricResults == nil {
 		run.Status.MetricResults = make([]v1alpha1.MetricResult, 0)
-		err := analysisutil.ValidateMetrics(run.Spec.Metrics)
-		if err != nil {
-			message := fmt.Sprintf("analysis spec invalid: %v", err)
-			log.Warn(message)
-			run.Status.Phase = v1alpha1.AnalysisPhaseError
-			run.Status.Message = message
-			c.recorder.Eventf(run, corev1.EventTypeWarning, EventReasonStatusFailed, "analysis completed %s", run.Status.Phase)
-			return run
-		}
 	}
-	tasks := generateMetricTasks(run)
-	log.Infof("taking %d measurements", len(tasks))
-	c.runMeasurements(run, tasks)
 
-	newStatus := c.asssessRunStatus(run)
+	resolvedMetrics, err := getResolvedMetricsWithoutSecrets(run.Spec.Metrics, run.Spec.Args)
+	if err != nil {
+		message := fmt.Sprintf("Unable to resolve metric arguments: %v", err)
+		logger.Warn(message)
+		run.Status.Phase = v1alpha1.AnalysisPhaseError
+		run.Status.Message = message
+		c.recordAnalysisRunCompletionEvent(run)
+		return run
+	}
+
+	err = analysisutil.ValidateMetrics(resolvedMetrics)
+	if err != nil {
+		message := fmt.Sprintf("Analysis spec invalid: %v", err)
+		logger.Warn(message)
+		run.Status.Phase = v1alpha1.AnalysisPhaseError
+		run.Status.Message = message
+		c.recordAnalysisRunCompletionEvent(run)
+		return run
+	}
+
+	dryRunMetricsMap, err := analysisutil.GetDryRunMetrics(run.Spec.DryRun, resolvedMetrics)
+	if err != nil {
+		message := fmt.Sprintf("Analysis spec invalid: %v", err)
+		logger.Warn(message)
+		run.Status.Phase = v1alpha1.AnalysisPhaseError
+		run.Status.Message = message
+		c.recordAnalysisRunCompletionEvent(run)
+		return run
+	}
+
+	measurementRetentionMetricsMap, err := analysisutil.GetMeasurementRetentionMetrics(run.Spec.MeasurementRetention, resolvedMetrics)
+	if err != nil {
+		message := fmt.Sprintf("Analysis spec invalid: %v", err)
+		logger.Warn(message)
+		run.Status.Phase = v1alpha1.AnalysisPhaseError
+		run.Status.Message = message
+		c.recordAnalysisRunCompletionEvent(run)
+		return run
+	}
+
+	tasks := generateMetricTasks(run, resolvedMetrics)
+	logger.Infof("Taking %d Measurement(s)...", len(tasks))
+	err = c.runMeasurements(run, tasks, dryRunMetricsMap)
+	if err != nil {
+		message := fmt.Sprintf("Unable to resolve metric arguments: %v", err)
+		logger.Warn(message)
+		run.Status.Phase = v1alpha1.AnalysisPhaseError
+		run.Status.Message = message
+		c.recordAnalysisRunCompletionEvent(run)
+		return run
+	}
+
+	newStatus, newMessage := c.assessRunStatus(run, resolvedMetrics, dryRunMetricsMap)
 	if newStatus != run.Status.Phase {
-		message := fmt.Sprintf("analysis transitioned from %s -> %s", run.Status.Phase, newStatus)
+		run.Status.Phase = newStatus
+		run.Status.Message = newMessage
 		if newStatus.Completed() {
-			switch newStatus {
-			case v1alpha1.AnalysisPhaseError, v1alpha1.AnalysisPhaseFailed:
-				c.recorder.Eventf(run, corev1.EventTypeWarning, EventReasonStatusFailed, "analysis completed %s", newStatus)
-			default:
-				c.recorder.Eventf(run, corev1.EventTypeNormal, EventReasonStatusCompleted, "analysis completed %s", newStatus)
+			c.recordAnalysisRunCompletionEvent(run)
+			if run.Status.CompletedAt == nil {
+				now := timeutil.MetaNow()
+				run.Status.CompletedAt = &now
 			}
 		}
-		log.Info(message)
-		run.Status.Phase = newStatus
 	}
 
-	err := c.garbageCollectMeasurements(run, DefaultMeasurementHistoryLimit)
+	err = c.garbageCollectMeasurements(run, measurementRetentionMetricsMap, DefaultMeasurementHistoryLimit)
 	if err != nil {
 		// TODO(jessesuen): surface errors to controller so they can be retried
-		log.Warnf("Failed to garbage collect measurements: %v", err)
+		logger.Warnf("Failed to garbage collect measurements: %v", err)
 	}
 
-	nextReconcileTime := calculateNextReconcileTime(run)
+	nextReconcileTime := calculateNextReconcileTime(run, resolvedMetrics)
 	if nextReconcileTime != nil {
-		enqueueSeconds := nextReconcileTime.Sub(time.Now())
+		enqueueSeconds := nextReconcileTime.Sub(timeutil.Now())
 		if enqueueSeconds < 0 {
 			enqueueSeconds = 0
 		}
-		log.Infof("enqueuing analysis after %v", enqueueSeconds)
+		logger.Infof("Enqueueing analysis after %v", enqueueSeconds)
 		c.enqueueAnalysisAfter(run, enqueueSeconds)
 	}
 	return run
 }
 
+func getResolvedMetricsWithoutSecrets(metrics []v1alpha1.Metric, args []v1alpha1.Argument) ([]v1alpha1.Metric, error) {
+	newArgs := make([]v1alpha1.Argument, 0)
+	for _, arg := range args {
+		newArg := arg.DeepCopy()
+		if newArg.ValueFrom != nil && newArg.ValueFrom.SecretKeyRef != nil {
+			newArg.ValueFrom = nil
+			newArg.Value = ptr.To[string]("temp-for-secret")
+		}
+		newArgs = append(newArgs, *newArg)
+	}
+	resolvedMetrics := make([]v1alpha1.Metric, 0)
+	for _, metric := range metrics {
+		resolvedMetric, err := analysisutil.ResolveMetricArgs(metric, newArgs)
+		if err != nil {
+			return nil, err
+		}
+		resolvedMetrics = append(resolvedMetrics, *resolvedMetric)
+	}
+	return resolvedMetrics, nil
+}
+
+func (c *Controller) recordAnalysisRunCompletionEvent(run *v1alpha1.AnalysisRun) {
+	eventType := corev1.EventTypeNormal
+	switch run.Status.Phase {
+	case v1alpha1.AnalysisPhaseError, v1alpha1.AnalysisPhaseFailed:
+		eventType = corev1.EventTypeWarning
+	}
+	c.recorder.Eventf(run, record.EventOptions{EventType: eventType, EventReason: "AnalysisRun" + string(run.Status.Phase)}, "Analysis Completed. Result: %s", run.Status.Phase)
+}
+
 // generateMetricTasks generates a list of metrics tasks needed to be measured as part of this
 // sync, based on the last completion times that metric was measured (if ever). If the run is
 // terminating (e.g. due to manual termination or failing metric), will not schedule further
 // measurements other than to resume any in-flight measurements.
-func generateMetricTasks(run *v1alpha1.AnalysisRun) []metricTask {
-	log := logutil.WithAnalysisRun(run)
+func generateMetricTasks(run *v1alpha1.AnalysisRun, metrics []v1alpha1.Metric) []metricTask {
+	logger := logutil.WithAnalysisRun(run)
 	var tasks []metricTask
 	terminating := analysisutil.IsTerminating(run)
-	for _, metric := range run.Spec.Metrics {
+
+	for i, metric := range metrics {
 		if analysisutil.MetricCompleted(run, metric.Name) {
 			continue
 		}
-		logCtx := log.WithField("metric", metric.Name)
+		logCtx := logger.WithField("metric", metric.Name)
 		lastMeasurement := analysisutil.LastMeasurement(run, metric.Name)
 		if lastMeasurement != nil && lastMeasurement.FinishedAt == nil {
-			now := metav1.Now()
+			now := timeutil.MetaNow()
 			if lastMeasurement.ResumeAt != nil && lastMeasurement.ResumeAt.After(now.Time) {
 				continue
 			}
 			// last measurement is still in-progress. need to complete it
-			logCtx.Infof("resuming in-progress measurement")
+			logCtx.Infof("Resuming in-progress measurement")
 			tasks = append(tasks, metricTask{
-				metric:                metric,
+				metric:                run.Spec.Metrics[i],
 				incompleteMeasurement: lastMeasurement,
 			})
 			continue
 		}
 		if terminating {
-			logCtx.Infof("skipping measurement: run is terminating")
+			logCtx.Infof("Skipping measurement: run is terminating")
 			continue
 		}
 		if lastMeasurement == nil {
@@ -136,19 +211,19 @@ func generateMetricTasks(run *v1alpha1.AnalysisRun) []metricTask {
 					logCtx.Warnf("failed to parse duration: %v", err)
 					continue
 				}
-				if run.Status.StartedAt.Add(duration).After(time.Now()) {
-					logCtx.Infof("waiting until start delay duration passes")
+				if run.Status.StartedAt.Add(duration).After(timeutil.Now()) {
+					logCtx.Infof("Waiting until start delay duration passes")
 					continue
 				}
 			}
 			// measurement never taken
-			tasks = append(tasks, metricTask{metric: metric})
-			logCtx.Infof("running initial measurement")
+			tasks = append(tasks, metricTask{metric: run.Spec.Metrics[i]})
+			logCtx.Infof("Running initial measurement")
 			continue
 		}
 		metricResult := analysisutil.GetResult(run, metric.Name)
 		effectiveCount := metric.EffectiveCount()
-		if effectiveCount != nil && metricResult.Count >= *effectiveCount {
+		if effectiveCount != nil && metricResult.Count >= int32(effectiveCount.IntValue()) {
 			// we have reached desired count
 			continue
 		}
@@ -156,70 +231,128 @@ func generateMetricTasks(run *v1alpha1.AnalysisRun) []metricTask {
 		// to decide if it should be taken now. metric.Interval can be null because we may be
 		// retrying a metric due to error.
 		interval := DefaultErrorRetryInterval
-		if metric.Interval != "" {
-			metricInterval, err := metric.Interval.Duration()
+		if lastMeasurement.Phase == v1alpha1.AnalysisPhaseError {
+			interval = DefaultErrorRetryInterval
+		} else if metric.Interval != "" {
+			parsedInterval, err := parseMetricInterval(*logCtx, metric.Interval)
 			if err != nil {
-				logCtx.Warnf("failed to parse interval: %v", err)
 				continue
 			}
-			interval = metricInterval
+			interval = parsedInterval
 		}
-		if time.Now().After(lastMeasurement.FinishedAt.Add(interval)) {
-			tasks = append(tasks, metricTask{metric: metric})
-			logCtx.Infof("running overdue measurement")
+		if timeutil.Now().After(lastMeasurement.FinishedAt.Add(interval)) {
+			tasks = append(tasks, metricTask{metric: run.Spec.Metrics[i]})
+			logCtx.Infof("Running overdue measurement")
 			continue
 		}
 	}
 	return tasks
 }
 
+// parseMetricInterval is a helper method to parse the given metric interval and return the
+// parsed duration or error (if any)
+func parseMetricInterval(logCtx log.Entry, metricDurationString v1alpha1.DurationString) (time.Duration, error) {
+	metricInterval, err := metricDurationString.Duration()
+	if err != nil {
+		logCtx.Warnf("Failed to parse interval: %v", err)
+		return -1, err
+	}
+	return metricInterval, nil
+}
+
+// resolveArgs resolves args for metricTasks, including secret references
+// returns resolved metricTasks and secrets for log redaction
+func (c *Controller) resolveArgs(tasks []metricTask, args []v1alpha1.Argument, namespace string) ([]metricTask, []string, error) {
+	//create set of secret values for redaction
+	secretSet := map[string]bool{}
+	for i, arg := range args {
+		//if secret specified in valueFrom, replace value with secret value
+		//error if arg has both value and valueFrom
+		if arg.ValueFrom != nil && arg.ValueFrom.SecretKeyRef != nil {
+			name := arg.ValueFrom.SecretKeyRef.Name
+			secret, err := c.kubeclientset.CoreV1().Secrets(namespace).Get(context.TODO(), name, metav1.GetOptions{})
+			if err != nil {
+				return nil, nil, err
+			}
+
+			secretContentBytes, ok := secret.Data[arg.ValueFrom.SecretKeyRef.Key]
+			if !ok {
+				err := fmt.Errorf("key '%s' does not exist in secret '%s'", arg.ValueFrom.SecretKeyRef.Key, arg.ValueFrom.SecretKeyRef.Name)
+				return nil, nil, err
+			}
+			secretContent := string(secretContentBytes)
+			secretSet[secretContent] = true
+			resolvedArg := arg.DeepCopy()
+			resolvedArg.Value = &secretContent
+			args[i] = *resolvedArg
+		} else {
+			args[i] = arg
+		}
+	}
+
+	// creates list of secret values from secretSet for RedactorFormatter
+	secrets := make([]string, 0, len(secretSet))
+	for k := range secretSet {
+		secrets = append(secrets, k)
+	}
+
+	// resolves arguments in each metric task
+	for i, task := range tasks {
+		resolvedMetric, err := analysisutil.ResolveMetricArgs(task.metric, args)
+		if err != nil {
+			return nil, nil, err
+		}
+		tasks[i].metric = *resolvedMetric
+	}
+
+	return tasks, secrets, nil
+}
+
 // runMeasurements iterates a list of metric tasks, and runs, resumes, or terminates measurements
-func (c *AnalysisController) runMeasurements(run *v1alpha1.AnalysisRun, tasks []metricTask) {
+func (c *Controller) runMeasurements(run *v1alpha1.AnalysisRun, tasks []metricTask, dryRunMetricsMap map[string]bool) error {
 	var wg sync.WaitGroup
 	// resultsLock should be held whenever we are accessing or setting status.metricResults since
 	// we are performing queries in parallel
 	var resultsLock sync.Mutex
 	terminating := analysisutil.IsTerminating(run)
 
+	// resolve args for metric tasks
+	// get list of secret values for log redaction
+	tasks, secrets, err := c.resolveArgs(tasks, run.Spec.Args, run.Namespace)
+	if err != nil {
+		return err
+	}
+
 	for _, task := range tasks {
 		wg.Add(1)
 
-		go func(t metricTask) {
+		go func(t metricTask) error {
 			defer wg.Done()
-			log := logutil.WithAnalysisRun(run).WithField("metric", t.metric.Name)
-
-			resultsLock.Lock()
-			metricResult := analysisutil.GetResult(run, t.metric.Name)
-			resultsLock.Unlock()
-
-			if metricResult == nil {
-				metricResult = &v1alpha1.MetricResult{
-					Name:  t.metric.Name,
-					Phase: v1alpha1.AnalysisPhaseRunning,
-				}
-			}
+			//redact secret values from logs
+			logger := logutil.WithRedactor(*logutil.WithAnalysisRun(run).WithField("metric", t.metric.Name), secrets)
 
 			var newMeasurement v1alpha1.Measurement
-			provider, err := c.newProvider(*log, t.metric)
-			if err != nil {
+			provider, providerErr := c.newProvider(*logger, run.Namespace, t.metric)
+			if providerErr != nil {
+				log.Errorf("Error in getting metric provider :%v", providerErr)
 				if t.incompleteMeasurement != nil {
 					newMeasurement = *t.incompleteMeasurement
 				} else {
-					startedAt := metav1.Now()
+					startedAt := timeutil.MetaNow()
 					newMeasurement.StartedAt = &startedAt
 				}
 				newMeasurement.Phase = v1alpha1.AnalysisPhaseError
-				newMeasurement.Message = err.Error()
+				newMeasurement.Message = providerErr.Error()
 			} else {
 				if t.incompleteMeasurement == nil {
 					newMeasurement = provider.Run(run, t.metric)
 				} else {
 					// metric is incomplete. either terminate or resume it
 					if terminating {
-						log.Infof("terminating in-progress measurement")
+						logger.Infof("Terminating in-progress measurement")
 						newMeasurement = provider.Terminate(run, t.metric, *t.incompleteMeasurement)
 						if newMeasurement.Phase == v1alpha1.AnalysisPhaseSuccessful {
-							newMeasurement.Message = "metric terminated"
+							newMeasurement.Message = "Metric Terminated"
 						}
 					} else {
 						newMeasurement = provider.Resume(run, t.metric, *t.incompleteMeasurement)
@@ -227,31 +360,59 @@ func (c *AnalysisController) runMeasurements(run *v1alpha1.AnalysisRun, tasks []
 				}
 			}
 
+			resultsLock.Lock()
+			metricResult := analysisutil.GetResult(run, t.metric.Name)
+			resultsLock.Unlock()
+			if metricResult == nil {
+				metricResult = &v1alpha1.MetricResult{
+					Name:   t.metric.Name,
+					Phase:  v1alpha1.AnalysisPhaseRunning,
+					DryRun: dryRunMetricsMap[t.metric.Name],
+				}
+
+				if provider != nil && providerErr == nil {
+					metricResult.Metadata = provider.GetMetadata(t.metric)
+				}
+			}
+
 			if newMeasurement.Phase.Completed() {
-				log.Infof("measurement completed %s", newMeasurement.Phase)
+				logger.Infof("Measurement Completed. Result: %s", newMeasurement.Phase)
 				if newMeasurement.FinishedAt == nil {
-					finishedAt := metav1.Now()
+					finishedAt := timeutil.MetaNow()
 					newMeasurement.FinishedAt = &finishedAt
 				}
+
 				switch newMeasurement.Phase {
 				case v1alpha1.AnalysisPhaseSuccessful:
 					metricResult.Successful++
 					metricResult.Count++
 					metricResult.ConsecutiveError = 0
+					metricResult.ConsecutiveSuccess++
 				case v1alpha1.AnalysisPhaseFailed:
 					metricResult.Failed++
 					metricResult.Count++
 					metricResult.ConsecutiveError = 0
+					metricResult.ConsecutiveSuccess = 0
 				case v1alpha1.AnalysisPhaseInconclusive:
 					metricResult.Inconclusive++
 					metricResult.Count++
 					metricResult.ConsecutiveError = 0
+					metricResult.ConsecutiveSuccess = 0
 				case v1alpha1.AnalysisPhaseError:
 					metricResult.Error++
 					metricResult.ConsecutiveError++
-					log.Warnf("measurement had error: %s", newMeasurement.Message)
+					metricResult.ConsecutiveSuccess = 0
+					logger.Warnf("Measurement had error: %s", newMeasurement.Message)
 				}
 			}
+
+			//redact secret values from measurement message
+			for _, secret := range secrets {
+				if secret != "" {
+					newMeasurement.Message = strings.ReplaceAll(newMeasurement.Message, secret, "*****")
+				}
+			}
+
 			if t.incompleteMeasurement == nil {
 				metricResult.Measurements = append(metricResult.Measurements, newMeasurement)
 			} else {
@@ -261,39 +422,67 @@ func (c *AnalysisController) runMeasurements(run *v1alpha1.AnalysisRun, tasks []
 			resultsLock.Lock()
 			analysisutil.SetResult(run, *metricResult)
 			resultsLock.Unlock()
-
+			return nil
 		}(task)
 	}
 	wg.Wait()
+
+	return nil
 }
 
-// asssessRunStatus assesses the overall status of this AnalysisRun
+// assessRunStatus assesses the overall status of this AnalysisRun
 // If any metric is not yet completed, the AnalysisRun is still considered Running
 // Once all metrics are complete, the worst status is used as the overall AnalysisRun status
-func (c *AnalysisController) asssessRunStatus(run *v1alpha1.AnalysisRun) v1alpha1.AnalysisPhase {
+func (c *Controller) assessRunStatus(run *v1alpha1.AnalysisRun, metrics []v1alpha1.Metric, dryRunMetricsMap map[string]bool) (v1alpha1.AnalysisPhase, string) {
 	var worstStatus v1alpha1.AnalysisPhase
+	var worstMessage string
 	terminating := analysisutil.IsTerminating(run)
 	everythingCompleted := true
 
 	if run.Status.StartedAt == nil {
-		now := metav1.Now()
+		now := timeutil.MetaNow()
 		run.Status.StartedAt = &now
 	}
+	if run.Spec.Terminate {
+		worstMessage = "Run Terminated"
+	}
 
-	// Iterate all metrics and update MetricResult.Phase fields based on lastest measurement(s)
-	for _, metric := range run.Spec.Metrics {
+	// Initialize Run & Dry-Run summary object
+	runSummary := v1alpha1.RunSummary{
+		Count:        0,
+		Successful:   0,
+		Failed:       0,
+		Inconclusive: 0,
+		Error:        0,
+	}
+	dryRunSummary := v1alpha1.RunSummary{
+		Count:        0,
+		Successful:   0,
+		Failed:       0,
+		Inconclusive: 0,
+		Error:        0,
+	}
+
+	// Iterate all metrics and update `MetricResult.Phase` fields based on latest measurement(s)
+	for _, metric := range metrics {
+		if dryRunMetricsMap[metric.Name] {
+			log.Infof("Metric '%s' is running in the Dry-Run mode.", metric.Name)
+			dryRunSummary.Count++
+		} else {
+			runSummary.Count++
+		}
 		if result := analysisutil.GetResult(run, metric.Name); result != nil {
-			log := logutil.WithAnalysisRun(run).WithField("metric", metric.Name)
+			logger := logutil.WithAnalysisRun(run).WithField("metric", metric.Name)
 			metricStatus := assessMetricStatus(metric, *result, terminating)
 			if result.Phase != metricStatus {
-				log.Infof("metric transitioned from %s -> %s", result.Phase, metricStatus)
+				logger.Infof("Metric '%s' transitioned from %s -> %s", metric.Name, result.Phase, metricStatus)
 				if metricStatus.Completed() {
+					eventType := corev1.EventTypeNormal
 					switch metricStatus {
 					case v1alpha1.AnalysisPhaseError, v1alpha1.AnalysisPhaseFailed:
-						c.recorder.Eventf(run, corev1.EventTypeWarning, EventReasonStatusFailed, "metric '%s' completed %s", metric.Name, metricStatus)
-					default:
-						c.recorder.Eventf(run, corev1.EventTypeNormal, EventReasonStatusCompleted, "metric '%s' completed %s", metric.Name, metricStatus)
+						eventType = corev1.EventTypeWarning
 					}
+					c.recorder.Eventf(run, record.EventOptions{EventType: eventType, EventReason: "Metric" + string(metricStatus)}, "Metric '%s' Completed. Result: %s", metric.Name, metricStatus)
 				}
 				if lastMeasurement := analysisutil.LastMeasurement(run, metric.Name); lastMeasurement != nil {
 					result.Message = lastMeasurement.Message
@@ -305,82 +494,203 @@ func (c *AnalysisController) asssessRunStatus(run *v1alpha1.AnalysisRun) v1alpha
 				// if any metric is in-progress, then entire analysis run will be considered running
 				everythingCompleted = false
 			} else {
+				phase, message := assessMetricFailureInconclusiveOrError(metric, *result)
+				// NOTE: We don't care about the status if the metric is marked as a Dry-Run
 				// otherwise, remember the worst status of all completed metric results
-				if worstStatus == "" {
-					worstStatus = metricStatus
-				} else {
-					if analysisutil.IsWorse(worstStatus, metricStatus) {
+				if !dryRunMetricsMap[metric.Name] {
+					if worstStatus == "" || analysisutil.IsWorse(worstStatus, metricStatus) {
 						worstStatus = metricStatus
+						if message != "" {
+							worstMessage = fmt.Sprintf("Metric \"%s\" assessed %s due to %s", metric.Name, metricStatus, message)
+							if result.Message != "" {
+								worstMessage += fmt.Sprintf(": \"Error Message: %s\"", result.Message)
+							}
+						}
+					}
+					// Update Run Summary
+					switch phase {
+					case v1alpha1.AnalysisPhaseError:
+						runSummary.Error++
+					case v1alpha1.AnalysisPhaseFailed:
+						runSummary.Failed++
+					case v1alpha1.AnalysisPhaseInconclusive:
+						runSummary.Inconclusive++
+					case v1alpha1.AnalysisPhaseSuccessful:
+						runSummary.Successful++
+					default:
+						// We'll mark the status as success by default if it doesn't match anything.
+						runSummary.Successful++
+					}
+				} else {
+					// We don't really care about the failures from dry-runs and hence, if there is no current status
+					// found then we just set it to `AnalysisPhaseSuccessful`
+					if worstStatus == "" {
+						worstStatus = v1alpha1.AnalysisPhaseSuccessful
+					}
+					// Update metric result message
+					if message != "" {
+						result.Message = fmt.Sprintf("Metric assessed %s due to %s", metricStatus, message)
+						analysisutil.SetResult(run, *result)
+					}
+					// Update DryRun Summary
+					switch phase {
+					case v1alpha1.AnalysisPhaseError:
+						dryRunSummary.Error++
+					case v1alpha1.AnalysisPhaseFailed:
+						dryRunSummary.Failed++
+					case v1alpha1.AnalysisPhaseInconclusive:
+						dryRunSummary.Inconclusive++
+					case v1alpha1.AnalysisPhaseSuccessful:
+						dryRunSummary.Successful++
+					default:
+						// We'll mark the status as success by default if it doesn't match anything.
+						dryRunSummary.Successful++
 					}
 				}
 			}
+		} else {
+			// metric hasn't started running. possible cases where some metrics starts with delay
+			everythingCompleted = false
 		}
 	}
-	if !everythingCompleted || worstStatus == "" {
-		return v1alpha1.AnalysisPhaseRunning
+	// Append Dry-Run metrics results if any.
+	worstMessage = strings.TrimSpace(worstMessage)
+	run.Status.RunSummary = runSummary
+	run.Status.DryRunSummary = &dryRunSummary
+	if terminating {
+		if worstStatus == "" {
+			// we have yet to take a single measurement, but have already been instructed to stop
+			log.Infof(SuccessfulAssessmentRunTerminatedResult)
+			return v1alpha1.AnalysisPhaseSuccessful, worstMessage
+		}
+		log.Infof("Metric Assessment Result - %s: Run Terminated", worstStatus)
+		return worstStatus, worstMessage
 	}
-	return worstStatus
+	if !everythingCompleted || worstStatus == "" {
+		return v1alpha1.AnalysisPhaseRunning, ""
+	}
+	return worstStatus, worstMessage
 }
 
 // assessMetricStatus assesses the status of a single metric based on:
-// * current/latest measurement status
+// * current or latest measurement status
 // * parameters given by the metric (failureLimit, count, etc...)
-// * whether or not we are terminating (e.g. due to failing run, or termination request)
+// * whether we are terminating (e.g. due to failing run, or termination request)
 func assessMetricStatus(metric v1alpha1.Metric, result v1alpha1.MetricResult, terminating bool) v1alpha1.AnalysisPhase {
 	if result.Phase.Completed() {
 		return result.Phase
 	}
-	log := log.WithField("metric", metric.Name)
+	logger := log.WithField("metric", metric.Name)
 	if len(result.Measurements) == 0 {
 		if terminating {
 			// we have yet to take a single measurement, but have already been instructed to stop
-			log.Infof("metric assessed %s: run terminated", v1alpha1.AnalysisPhaseSuccessful)
+			logger.Infof(SuccessfulAssessmentRunTerminatedResult)
 			return v1alpha1.AnalysisPhaseSuccessful
 		}
 		return v1alpha1.AnalysisPhasePending
 	}
 	lastMeasurement := result.Measurements[len(result.Measurements)-1]
 	if !lastMeasurement.Phase.Completed() {
-		// we still have a in-flight measurement
+		// we still have an in-flight measurement
 		return v1alpha1.AnalysisPhaseRunning
 	}
-	if result.Failed > metric.FailureLimit {
-		log.Infof("metric assessed %s: failed (%d) > failureLimit (%d)", v1alpha1.AnalysisPhaseFailed, result.Failed, metric.FailureLimit)
-		return v1alpha1.AnalysisPhaseFailed
+
+	// Check if metric was considered Failed, Inconclusive, or Error
+	// If true, then return AnalysisRunPhase as Failed, Inconclusive, or Error respectively
+	phaseFailureInconclusiveOrError, message := assessMetricFailureInconclusiveOrError(metric, result)
+	if phaseFailureInconclusiveOrError != "" {
+		logger.Infof("Metric Assessment Result - %s: %s", phaseFailureInconclusiveOrError, message)
+		return phaseFailureInconclusiveOrError
 	}
-	if result.Inconclusive > metric.InconclusiveLimit {
-		log.Infof("metric assessed %s: inconclusive (%d) > inconclusiveLimit (%d)", v1alpha1.AnalysisPhaseInconclusive, result.Inconclusive, metric.InconclusiveLimit)
-		return v1alpha1.AnalysisPhaseInconclusive
-	}
-	consecutiveErrorLimit := DefaultConsecutiveErrorLimit
-	if metric.ConsecutiveErrorLimit != nil {
-		consecutiveErrorLimit = *metric.ConsecutiveErrorLimit
-	}
-	if result.ConsecutiveError > consecutiveErrorLimit {
-		log.Infof("metric assessed %s: consecutiveErrors (%d) > consecutiveErrorLimit (%d)", v1alpha1.AnalysisPhaseError, result.ConsecutiveError, consecutiveErrorLimit)
-		return v1alpha1.AnalysisPhaseError
+
+	// Check if consecutiveSuccessLimit is applicable and was reached.
+	if metric.ConsecutiveSuccessLimit != nil && metric.ConsecutiveSuccessLimit.IntValue() > 0 && result.ConsecutiveSuccess >= int32(metric.ConsecutiveSuccessLimit.IntValue()) {
+		logger.Infof("Metric Assessment Result - %s: ConsecutiveSuccessLimit (%s) Reached", v1alpha1.AnalysisPhaseSuccessful, metric.ConsecutiveSuccessLimit.String())
+		return v1alpha1.AnalysisPhaseSuccessful
 	}
+
 	// If a count was specified, and we reached that count, then metric is considered Successful.
 	// The Error, Failed, Inconclusive counters are ignored because those checks have already been
 	// taken into consideration above, and we do not want to fail if failures < failureLimit.
 	effectiveCount := metric.EffectiveCount()
-	if effectiveCount != nil && result.Count >= *effectiveCount {
-		log.Infof("metric assessed %s: count (%d) reached", v1alpha1.AnalysisPhaseSuccessful, *effectiveCount)
+	if effectiveCount != nil && result.Count >= int32(effectiveCount.IntValue()) {
+
+		failureApplicable := (metric.FailureLimit != nil && metric.FailureLimit.IntValue() >= 0) || metric.FailureLimit == nil
+		successApplicable := metric.ConsecutiveSuccessLimit != nil && metric.ConsecutiveSuccessLimit.IntValue() > 0
+
+		if failureApplicable && successApplicable {
+
+			// failureLimit was checked above and not reached.
+			// consecutiveSuccessLimit was checked above and not reached.
+
+			failureLimit := "0"
+			if metric.FailureLimit != nil {
+				failureLimit = metric.FailureLimit.String()
+			}
+
+			logger.Infof("Metric Assessment Result - %s: ConsecutiveSuccessLimit (%s) Not Reached and FailureLimit (%s) Not Violated", v1alpha1.AnalysisPhaseInconclusive, metric.ConsecutiveSuccessLimit.String(), failureLimit)
+			return v1alpha1.AnalysisPhaseInconclusive
+
+		} else if successApplicable {
+
+			logger.Infof("Metric Assessment Result - %s: ConsecutiveSuccessLimit (%s) Not Reached", v1alpha1.AnalysisPhaseFailed, metric.ConsecutiveSuccessLimit.String())
+			return v1alpha1.AnalysisPhaseFailed
+
+		} else if failureApplicable {
+			// failureLimit was not reached in assessMetricFailureInconclusiveOrError above.
+			// AnalysisPhaseSuccessful below.
+		} else {
+			// This cannot happen, since one of failureLimit or consecutiveSuccessLimit will be applicable
+			// We validate that failureLimit >= 0 when consecutiveSuccessLimit == 0
+		}
+
+		logger.Infof("Metric Assessment Result - %s: Count (%s) Reached", v1alpha1.AnalysisPhaseSuccessful, effectiveCount.String())
 		return v1alpha1.AnalysisPhaseSuccessful
 	}
 	// if we get here, this metric runs indefinitely
 	if terminating {
-		log.Infof("metric assessed %s: run terminated", v1alpha1.AnalysisPhaseSuccessful)
+		logger.Infof(SuccessfulAssessmentRunTerminatedResult)
 		return v1alpha1.AnalysisPhaseSuccessful
 	}
 	return v1alpha1.AnalysisPhaseRunning
 }
 
+func assessMetricFailureInconclusiveOrError(metric v1alpha1.Metric, result v1alpha1.MetricResult) (v1alpha1.AnalysisPhase, string) {
+	var message string
+	var phase v1alpha1.AnalysisPhase
+
+	failureLimit := int32(0)
+	if metric.FailureLimit != nil {
+		failureLimit = int32(metric.FailureLimit.IntValue())
+	}
+	// If failureLimit is negative, that means it isn't applicable.
+	if failureLimit >= 0 && result.Failed > failureLimit {
+		phase = v1alpha1.AnalysisPhaseFailed
+		message = fmt.Sprintf("failed (%d) > failureLimit (%d)", result.Failed, failureLimit)
+	}
+
+	inconclusiveLimit := int32(0)
+	if metric.InconclusiveLimit != nil {
+		inconclusiveLimit = int32(metric.InconclusiveLimit.IntValue())
+	}
+	if result.Inconclusive > inconclusiveLimit {
+		phase = v1alpha1.AnalysisPhaseInconclusive
+		message = fmt.Sprintf("inconclusive (%d) > inconclusiveLimit (%d)", result.Inconclusive, inconclusiveLimit)
+	}
+
+	consecutiveErrorLimit := defaults.GetConsecutiveErrorLimitOrDefault(&metric)
+	if result.ConsecutiveError > consecutiveErrorLimit {
+		phase = v1alpha1.AnalysisPhaseError
+		message = fmt.Sprintf("consecutiveErrors (%d) > consecutiveErrorLimit (%d)", result.ConsecutiveError, consecutiveErrorLimit)
+	}
+	return phase, message
+}
+
 // calculateNextReconcileTime calculates the next time that this AnalysisRun should be reconciled,
 // based on the earliest time of all metrics intervals, counts, and their finishedAt timestamps
-func calculateNextReconcileTime(run *v1alpha1.AnalysisRun) *time.Time {
+func calculateNextReconcileTime(run *v1alpha1.AnalysisRun, metrics []v1alpha1.Metric) *time.Time {
 	var reconcileTime *time.Time
-	for _, metric := range run.Spec.Metrics {
+	for _, metric := range metrics {
 		if analysisutil.MetricCompleted(run, metric.Name) {
 			// NOTE: this also covers the case where metric.Count is reached
 			continue
@@ -389,23 +699,22 @@ func calculateNextReconcileTime(run *v1alpha1.AnalysisRun) *time.Time {
 		lastMeasurement := analysisutil.LastMeasurement(run, metric.Name)
 		if lastMeasurement == nil {
 			if metric.InitialDelay != "" {
-				startTime := metav1.Now()
+				startTime := timeutil.MetaNow()
 				if run.Status.StartedAt != nil {
 					startTime = *run.Status.StartedAt
 				}
-				duration, err := metric.InitialDelay.Duration()
+				parsedInterval, err := parseMetricInterval(*logCtx, metric.InitialDelay)
 				if err != nil {
-					logCtx.Warnf("failed to parse interval: %v", err)
 					continue
 				}
-				endInitialDelay := startTime.Add(duration)
+				endInitialDelay := startTime.Add(parsedInterval)
 				if reconcileTime == nil || reconcileTime.After(endInitialDelay) {
 					reconcileTime = &endInitialDelay
 				}
 				continue
 			}
 			// no measurement was started . we should never get here
-			logCtx.Warnf("metric never started. not factored into enqueue time")
+			logCtx.Warnf("Metric never started. Not factored into enqueue time.")
 			continue
 		}
 		if lastMeasurement.FinishedAt == nil {
@@ -419,26 +728,25 @@ func calculateNextReconcileTime(run *v1alpha1.AnalysisRun) *time.Time {
 		}
 		metricResult := analysisutil.GetResult(run, metric.Name)
 		effectiveCount := metric.EffectiveCount()
-		if effectiveCount != nil && metricResult.Count >= *effectiveCount {
+		if effectiveCount != nil && metricResult.Count >= int32(effectiveCount.IntValue()) {
 			// we have reached desired count
 			continue
 		}
 		var interval time.Duration
-		if metric.Interval != "" {
-			metricInterval, err := metric.Interval.Duration()
+		if lastMeasurement.Phase == v1alpha1.AnalysisPhaseError {
+			interval = DefaultErrorRetryInterval
+		} else if metric.Interval != "" {
+			parsedInterval, err := parseMetricInterval(*logCtx, metric.Interval)
 			if err != nil {
-				logCtx.Warnf("failed to parse interval: %v", err)
 				continue
 			}
-			interval = metricInterval
-		} else if lastMeasurement.Phase == v1alpha1.AnalysisPhaseError {
-			interval = DefaultErrorRetryInterval
+			interval = parsedInterval
 		} else {
 			// if we get here, an interval was not set (meaning reoccurrence was not desired), and
 			// there was no error (meaning we don't need to retry). no need to requeue this metric.
 			// NOTE: we shouldn't ever get here since it means we are not doing proper bookkeeping
 			// of count.
-			logCtx.Warnf("skipping requeue. no interval or error (count: %d, effectiveCount: %d)", metricResult.Count, metric.EffectiveCount())
+			logCtx.Warnf("Skipping requeue. No interval or error (count: %d, effectiveCount: %s)", metricResult.Count, metric.EffectiveCount().String())
 			continue
 		}
 		// Take the earliest time of all metrics
@@ -451,32 +759,42 @@ func calculateNextReconcileTime(run *v1alpha1.AnalysisRun) *time.Time {
 }
 
 // garbageCollectMeasurements trims the measurement history to the specified limit and GCs old measurements
-func (c *AnalysisController) garbageCollectMeasurements(run *v1alpha1.AnalysisRun, limit int) error {
+func (c *Controller) garbageCollectMeasurements(run *v1alpha1.AnalysisRun, measurementRetentionMetricNamesMap map[string]*v1alpha1.MeasurementRetention, limit int) error {
 	var errors []error
 
+	resolvedArgsMetric, err := getResolvedMetricsWithoutSecrets(run.Spec.Metrics, run.Spec.Args)
+	if err != nil {
+		return fmt.Errorf("failed to resolve args on metrics during garbage collection: %w", err)
+	}
+
 	metricsByName := make(map[string]v1alpha1.Metric)
-	for _, metric := range run.Spec.Metrics {
+	for _, metric := range resolvedArgsMetric {
 		metricsByName[metric.Name] = metric
 	}
 
 	for i, result := range run.Status.MetricResults {
 		length := len(result.Measurements)
-		if length > limit {
+		measurementRetentionObject := measurementRetentionMetricNamesMap[result.Name]
+		measurementsLimit := limit
+		if measurementRetentionObject != nil && measurementRetentionObject.Limit > 0 {
+			measurementsLimit = int(measurementRetentionObject.Limit)
+		}
+		if length > measurementsLimit {
 			metric, ok := metricsByName[result.Name]
 			if !ok {
 				continue
 			}
-			log := logutil.WithAnalysisRun(run).WithField("metric", metric.Name)
-			provider, err := c.newProvider(*log, metric)
+			logger := logutil.WithAnalysisRun(run).WithField("metric", metric.Name)
+			provider, err := c.newProvider(*logger, run.Namespace, metric)
 			if err != nil {
 				errors = append(errors, err)
 				continue
 			}
-			err = provider.GarbageCollect(run, metric, limit)
+			err = provider.GarbageCollect(run, metric, measurementsLimit)
 			if err != nil {
 				return err
 			}
-			result.Measurements = result.Measurements[length-limit : length]
+			result.Measurements = result.Measurements[length-measurementsLimit : length]
 		}
 		run.Status.MetricResults[i] = result
 	}
@@ -485,3 +803,40 @@ func (c *AnalysisController) garbageCollectMeasurements(run *v1alpha1.AnalysisRu
 	}
 	return nil
 }
+
+func (c *Controller) maybeGarbageCollectAnalysisRun(run *v1alpha1.AnalysisRun, logger *log.Entry) error {
+	ctx := context.TODO()
+	if run.DeletionTimestamp != nil || !isAnalysisRunTtlExceeded(run) {
+		return nil
+	}
+	logger.Infof("Trying to cleanup TTL exceeded analysis run")
+	err := c.argoProjClientset.ArgoprojV1alpha1().AnalysisRuns(run.Namespace).Delete(ctx, run.Name, metav1.DeleteOptions{})
+	if err != nil && !k8serrors.IsNotFound(err) {
+		return err
+	}
+	return nil
+}
+
+func isAnalysisRunTtlExceeded(run *v1alpha1.AnalysisRun) bool {
+	// TTL only counted for completed runs with TTL strategy.
+	if !run.Status.Phase.Completed() || run.Spec.TTLStrategy == nil {
+		return false
+	}
+	// Cannot determine TTL if run has no completion time.
+	if run.Status.CompletedAt == nil {
+		return false
+	}
+	secondsCompleted := timeutil.MetaNow().Sub(run.Status.CompletedAt.Time).Seconds()
+	var ttlSeconds *int32
+	if run.Status.Phase == v1alpha1.AnalysisPhaseSuccessful && run.Spec.TTLStrategy.SecondsAfterSuccess != nil {
+		ttlSeconds = run.Spec.TTLStrategy.SecondsAfterSuccess
+	} else if run.Status.Phase == v1alpha1.AnalysisPhaseFailed && run.Spec.TTLStrategy.SecondsAfterFailure != nil {
+		ttlSeconds = run.Spec.TTLStrategy.SecondsAfterFailure
+	} else if run.Spec.TTLStrategy.SecondsAfterCompletion != nil {
+		ttlSeconds = run.Spec.TTLStrategy.SecondsAfterCompletion
+	}
+	if ttlSeconds == nil {
+		return false
+	}
+	return int32(secondsCompleted) > *ttlSeconds
+}

analysis/controller.go

@@ -1,8 +1,19 @@
 package analysis
 
 import (
+	"context"
+	"sync"
 	"time"
 
+	"github.com/aws/smithy-go/ptr"
+	v1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+
+	"github.com/argoproj/argo-rollouts/metric"
+	jobProvider "github.com/argoproj/argo-rollouts/metricproviders/job"
+
+	unstructuredutil "github.com/argoproj/argo-rollouts/utils/unstructured"
+
 	log "github.com/sirupsen/logrus"
 	batchv1 "k8s.io/api/batch/v1"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
@@ -10,7 +21,6 @@ import (
 	batchinformers "k8s.io/client-go/informers/batch/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 
 	"github.com/argoproj/argo-rollouts/controller/metrics"
@@ -22,10 +32,16 @@ import (
 	listers "github.com/argoproj/argo-rollouts/pkg/client/listers/rollouts/v1alpha1"
 	controllerutil "github.com/argoproj/argo-rollouts/utils/controller"
 	logutil "github.com/argoproj/argo-rollouts/utils/log"
+	"github.com/argoproj/argo-rollouts/utils/record"
+	timeutil "github.com/argoproj/argo-rollouts/utils/time"
 )
 
-// AnalysisController is the controller implementation for Analysis resources
-type AnalysisController struct {
+var (
+	analysisRunGVK = v1alpha1.SchemeGroupVersion.WithKind("AnalysisRun")
+)
+
+// Controller is the controller implementation for Analysis resources
+type Controller struct {
 	// kubeclientset is a standard kubernetes clientset
 	kubeclientset kubernetes.Interface
 	// analysisclientset is a clientset for our own API group
@@ -39,11 +55,11 @@ type AnalysisController struct {
 
 	metricsServer *metrics.MetricsServer
 
-	newProvider func(logCtx log.Entry, metric v1alpha1.Metric) (metricproviders.Provider, error)
+	newProvider func(logCtx log.Entry, namespace string, metric v1alpha1.Metric) (metric.Provider, error)
 
 	// used for unit testing
-	enqueueAnalysis      func(obj interface{})
-	enqueueAnalysisAfter func(obj interface{}, duration time.Duration)
+	enqueueAnalysis      func(obj any)
+	enqueueAnalysisAfter func(obj any, duration time.Duration)
 
 	// workqueue is a rate limited work queue. This is used to queue work to be
 	// processed instead of performing it as soon as a change happens. This
@@ -57,82 +73,98 @@ type AnalysisController struct {
 	resyncPeriod time.Duration
 }
 
-// NewAnalysisController returns a new analysis controller
-func NewAnalysisController(
-	kubeclientset kubernetes.Interface,
-	argoProjClientset clientset.Interface,
-	analysisRunInformer informers.AnalysisRunInformer,
-	jobInformer batchinformers.JobInformer,
-	resyncPeriod time.Duration,
-	analysisRunWorkQueue workqueue.RateLimitingInterface,
-	metricsServer *metrics.MetricsServer,
-	recorder record.EventRecorder) *AnalysisController {
+// ControllerConfig describes the data required to instantiate a new analysis controller
+type ControllerConfig struct {
+	KubeClientSet        kubernetes.Interface
+	ArgoProjClientset    clientset.Interface
+	AnalysisRunInformer  informers.AnalysisRunInformer
+	JobInformer          batchinformers.JobInformer
+	ResyncPeriod         time.Duration
+	AnalysisRunWorkQueue workqueue.RateLimitingInterface
+	MetricsServer        *metrics.MetricsServer
+	Recorder             record.EventRecorder
+}
 
-	controller := &AnalysisController{
-		kubeclientset:        kubeclientset,
-		argoProjClientset:    argoProjClientset,
-		analysisRunLister:    analysisRunInformer.Lister(),
-		metricsServer:        metricsServer,
-		analysisRunWorkQueue: analysisRunWorkQueue,
-		jobInformer:          jobInformer,
-		analysisRunSynced:    analysisRunInformer.Informer().HasSynced,
-		recorder:             recorder,
-		resyncPeriod:         resyncPeriod,
+// NewController returns a new analysis controller
+func NewController(cfg ControllerConfig) *Controller {
+
+	controller := &Controller{
+		kubeclientset:        cfg.KubeClientSet,
+		argoProjClientset:    cfg.ArgoProjClientset,
+		analysisRunLister:    cfg.AnalysisRunInformer.Lister(),
+		metricsServer:        cfg.MetricsServer,
+		analysisRunWorkQueue: cfg.AnalysisRunWorkQueue,
+		jobInformer:          cfg.JobInformer,
+		analysisRunSynced:    cfg.AnalysisRunInformer.Informer().HasSynced,
+		recorder:             cfg.Recorder,
+		resyncPeriod:         cfg.ResyncPeriod,
 	}
 
-	controller.enqueueAnalysis = func(obj interface{}) {
-		controllerutil.Enqueue(obj, analysisRunWorkQueue)
+	controller.enqueueAnalysis = func(obj any) {
+		controllerutil.Enqueue(obj, cfg.AnalysisRunWorkQueue)
 	}
-	controller.enqueueAnalysisAfter = func(obj interface{}, duration time.Duration) {
-		controllerutil.EnqueueAfter(obj, duration, analysisRunWorkQueue)
+	controller.enqueueAnalysisAfter = func(obj any, duration time.Duration) {
+		controllerutil.EnqueueAfter(obj, duration, cfg.AnalysisRunWorkQueue)
 	}
 
 	providerFactory := metricproviders.ProviderFactory{
 		KubeClient: controller.kubeclientset,
-		JobLister:  jobInformer.Lister(),
+		JobLister:  cfg.JobInformer.Lister(),
 	}
 	controller.newProvider = providerFactory.NewProvider
 
-	jobInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
-		AddFunc: func(obj interface{}) {
-			controller.enqueueIfCompleted(obj)
+	cfg.JobInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+		AddFunc: func(obj any) {
+			controller.enqueueJobIfCompleted(obj)
 		},
-		UpdateFunc: func(oldObj, newObj interface{}) {
-			controller.enqueueIfCompleted(newObj)
+		UpdateFunc: func(oldObj, newObj any) {
+			controller.enqueueJobIfCompleted(newObj)
 		},
-		DeleteFunc: func(obj interface{}) {
-			controller.enqueueIfCompleted(obj)
+		DeleteFunc: func(obj any) {
+			controller.enqueueJobIfCompleted(obj)
 		},
 	})
 
 	log.Info("Setting up analysis event handlers")
 	// Set up an event handler for when analysis resources change
-	analysisRunInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
+	cfg.AnalysisRunInformer.Informer().AddEventHandler(cache.ResourceEventHandlerFuncs{
 		AddFunc: controller.enqueueAnalysis,
-		UpdateFunc: func(old, new interface{}) {
+		UpdateFunc: func(old, new any) {
 			controller.enqueueAnalysis(new)
 		},
-		DeleteFunc: controller.enqueueAnalysis,
+		DeleteFunc: func(obj any) {
+			controller.enqueueAnalysis(obj)
+			if ar := unstructuredutil.ObjectToAnalysisRun(obj); ar != nil {
+				logCtx := logutil.WithAnalysisRun(ar)
+				logCtx.Info("analysis run deleted")
+				controller.metricsServer.Remove(ar.Namespace, ar.Name, logutil.AnalysisRunKey)
+			}
+		},
 	})
 	return controller
 }
 
-func (c *AnalysisController) Run(threadiness int, stopCh <-chan struct{}) error {
+func (c *Controller) Run(ctx context.Context, threadiness int) error {
 	log.Info("Starting analysis workers")
+	wg := sync.WaitGroup{}
 	for i := 0; i < threadiness; i++ {
+		wg.Add(1)
 		go wait.Until(func() {
-			controllerutil.RunWorker(c.analysisRunWorkQueue, logutil.AnalysisRunKey, c.syncHandler, c.metricsServer)
-		}, time.Second, stopCh)
+			controllerutil.RunWorker(ctx, c.analysisRunWorkQueue, logutil.AnalysisRunKey, c.syncHandler, c.metricsServer)
+			log.Debug("Analysis worker has stopped")
+			wg.Done()
+		}, time.Second, ctx.Done())
 	}
 	log.Infof("Started %d analysis workers", threadiness)
-	<-stopCh
-	log.Info("Shutting down analysis workers")
+	<-ctx.Done()
+	wg.Wait()
+	log.Info("All analysis workers have stopped")
 
 	return nil
 }
 
-func (c *AnalysisController) syncHandler(key string) error {
-	startTime := time.Now()
+func (c *Controller) syncHandler(ctx context.Context, key string) error {
+	startTime := timeutil.Now()
 	namespace, name, err := cache.SplitMetaNamespaceKey(key)
 	if err != nil {
 		return err
@@ -149,8 +181,7 @@ func (c *AnalysisController) syncHandler(key string) error {
 
 	defer func() {
 		duration := time.Since(startTime)
-		//TODO(jessesuen) Add metrics for analysis
-		//arc.metricsServer.IncReconcile(r, duration)
+		c.metricsServer.IncAnalysisRunReconcile(run, duration)
 		logCtx := logutil.WithAnalysisRun(run).WithField("time_ms", duration.Seconds()*1e3)
 		logCtx.Info("Reconciliation completed")
 	}()
@@ -164,7 +195,35 @@ func (c *AnalysisController) syncHandler(key string) error {
 	return c.persistAnalysisRunStatus(run, newRun.Status)
 }
 
-func (c *AnalysisController) enqueueIfCompleted(obj interface{}) {
+func (c *Controller) jobParentReference(obj any) (*v1.OwnerReference, string) {
+	job, ok := obj.(*batchv1.Job)
+	if !ok {
+		return nil, ""
+	}
+	// if it has owner reference, return it as is
+	ownerRef := v1.GetControllerOf(job)
+	// else if it's missing owner reference check if analysis run uid is set and
+	// if it is there use labels/annotations to create owner reference
+	if ownerRef == nil && job.Labels[jobProvider.AnalysisRunUIDLabelKey] != "" {
+		ownerRef = &v1.OwnerReference{
+			APIVersion:         analysisRunGVK.GroupVersion().String(),
+			Kind:               analysisRunGVK.Kind,
+			Name:               job.Annotations[jobProvider.AnalysisRunNameAnnotationKey],
+			UID:                types.UID(job.Labels[jobProvider.AnalysisRunUIDLabelKey]),
+			BlockOwnerDeletion: ptr.Bool(true),
+			Controller:         ptr.Bool(true),
+		}
+	}
+	ns := job.GetNamespace()
+	if job.Annotations != nil {
+		if job.Annotations[jobProvider.AnalysisRunNamespaceAnnotationKey] != "" {
+			ns = job.Annotations[jobProvider.AnalysisRunNamespaceAnnotationKey]
+		}
+	}
+	return ownerRef, ns
+}
+
+func (c *Controller) enqueueJobIfCompleted(obj any) {
 	job, ok := obj.(*batchv1.Job)
 	if !ok {
 		return
@@ -172,7 +231,7 @@ func (c *AnalysisController) enqueueIfCompleted(obj interface{}) {
 	for _, condition := range job.Status.Conditions {
 		switch condition.Type {
 		case batchv1.JobFailed, batchv1.JobComplete:
-			controllerutil.EnqueueParentObject(job, register.AnalysisRunKind, c.enqueueAnalysis)
+			controllerutil.EnqueueParentObject(job, register.AnalysisRunKind, c.enqueueAnalysis, c.jobParentReference)
 			return
 		}
 	}

analysis/controller_test.go

@@ -1,12 +1,20 @@
 package analysis
 
 import (
+	"context"
 	"encoding/json"
+	"fmt"
 	"reflect"
+	"sync"
 	"testing"
 	"time"
 
-	"github.com/bouk/monkey"
+	"github.com/argoproj/argo-rollouts/metric"
+
+	timeutil "github.com/argoproj/argo-rollouts/utils/time"
+
+	"github.com/argoproj/argo-rollouts/utils/queue"
+
 	log "github.com/sirupsen/logrus"
 	"github.com/stretchr/testify/assert"
 	"k8s.io/apimachinery/pkg/api/equality"
@@ -18,15 +26,14 @@ import (
 	k8sfake "k8s.io/client-go/kubernetes/fake"
 	core "k8s.io/client-go/testing"
 	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/tools/record"
 	"k8s.io/client-go/util/workqueue"
 
 	"github.com/argoproj/argo-rollouts/controller/metrics"
-	"github.com/argoproj/argo-rollouts/metricproviders"
 	"github.com/argoproj/argo-rollouts/metricproviders/mocks"
 	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
 	"github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake"
 	informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions"
+	"github.com/argoproj/argo-rollouts/utils/record"
 )
 
 var (
@@ -44,11 +51,18 @@ type fixture struct {
 	// Actions expected to happen on the client.
 	actions []core.Action
 	// Objects from here preloaded into NewSimpleFake.
-	objects         []runtime.Object
-	enqueuedObjects map[string]int
-	unfreezeTime    func()
+	objects []runtime.Object
+
+	// Acquire 'enqueuedObjectMutex' before accessing enqueuedObjects
+	enqueuedObjects     map[string]int
+	enqueuedObjectMutex sync.Mutex
+
+	unfreezeTime func() error
 	// fake provider
 	provider *mocks.Provider
+
+	// Reference to frozen now
+	now time.Time
 }
 
 func newFixture(t *testing.T) *fixture {
@@ -56,11 +70,14 @@ func newFixture(t *testing.T) *fixture {
 	f.t = t
 	f.objects = []runtime.Object{}
 	f.enqueuedObjects = make(map[string]int)
-	now := time.Now()
-	patch := monkey.Patch(time.Now, func() time.Time {
-		return now
+	f.now = time.Now()
+	timeutil.SetNowTimeFunc(func() time.Time {
+		return f.now
 	})
-	f.unfreezeTime = patch.Unpatch
+	f.unfreezeTime = func() error {
+		timeutil.SetNowTimeFunc(time.Now)
+		return nil
+	}
 	return f
 }
 
@@ -79,31 +96,41 @@ func getKey(analysisRun *v1alpha1.AnalysisRun, t *testing.T) string {
 
 type resyncFunc func() time.Duration
 
-func (f *fixture) newController(resync resyncFunc) (*AnalysisController, informers.SharedInformerFactory, kubeinformers.SharedInformerFactory) {
+func (f *fixture) newController(resync resyncFunc) (*Controller, informers.SharedInformerFactory, kubeinformers.SharedInformerFactory) {
 	f.client = fake.NewSimpleClientset(f.objects...)
+	f.kubeclient = k8sfake.NewSimpleClientset()
 
 	i := informers.NewSharedInformerFactory(f.client, resync())
 	k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, resync())
-	jobI := kubeinformers.NewSharedInformerFactory(f.kubeclient, resync())
 
-	analysisRunWorkqueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AnalysisRuns")
+	analysisRunWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "AnalysisRuns")
 
-	c := NewAnalysisController(
-		f.kubeclient,
-		f.client,
-		i.Argoproj().V1alpha1().AnalysisRuns(),
-		jobI.Batch().V1().Jobs(),
-		resync(),
-		analysisRunWorkqueue,
-		metrics.NewMetricsServer("localhost:8080", i.Argoproj().V1alpha1().Rollouts().Lister()),
-		&record.FakeRecorder{})
+	metricsServer := metrics.NewMetricsServer(metrics.ServerConfig{
+		Addr:               "localhost:8080",
+		K8SRequestProvider: &metrics.K8sRequestsCountProvider{},
+	})
 
-	c.enqueueAnalysis = func(obj interface{}) {
+	c := NewController(ControllerConfig{
+		KubeClientSet:        f.kubeclient,
+		ArgoProjClientset:    f.client,
+		AnalysisRunInformer:  i.Argoproj().V1alpha1().AnalysisRuns(),
+		JobInformer:          k8sI.Batch().V1().Jobs(),
+		ResyncPeriod:         resync(),
+		AnalysisRunWorkQueue: analysisRunWorkqueue,
+		MetricsServer:        metricsServer,
+		Recorder:             record.NewFakeEventRecorder(),
+	})
+
+	c.enqueueAnalysis = func(obj any) {
 		var key string
 		var err error
 		if key, err = cache.MetaNamespaceKeyFunc(obj); err != nil {
 			panic(err)
 		}
+
+		f.enqueuedObjectMutex.Lock()
+		defer f.enqueuedObjectMutex.Unlock()
+
 		count, ok := f.enqueuedObjects[key]
 		if !ok {
 			count = 0
@@ -112,11 +139,11 @@ func (f *fixture) newController(resync resyncFunc) (*AnalysisController, informe
 		f.enqueuedObjects[key] = count
 		c.analysisRunWorkQueue.Add(obj)
 	}
-	c.enqueueAnalysisAfter = func(obj interface{}, duration time.Duration) {
+	c.enqueueAnalysisAfter = func(obj any, duration time.Duration) {
 		c.enqueueAnalysis(obj)
 	}
 	f.provider = &mocks.Provider{}
-	c.newProvider = func(logCtx log.Entry, metric v1alpha1.Metric) (metricproviders.Provider, error) {
+	c.newProvider = func(logCtx log.Entry, namespace string, metric v1alpha1.Metric) (metric.Provider, error) {
 		return f.provider, nil
 	}
 
@@ -132,12 +159,12 @@ func (f *fixture) run(analysisRunName string) {
 	f.runController(analysisRunName, true, false, c, i, k8sI)
 }
 
-func (f *fixture) runExpectError(analysisRunName string, startInformers bool) {
+func (f *fixture) runExpectError(analysisRunName string, startInformers bool) { //nolint:unused
 	c, i, k8sI := f.newController(noResyncPeriodFunc)
 	f.runController(analysisRunName, startInformers, true, c, i, k8sI)
 }
 
-func (f *fixture) runController(analysisRunName string, startInformers bool, expectError bool, c *AnalysisController, i informers.SharedInformerFactory, k8sI kubeinformers.SharedInformerFactory) *AnalysisController {
+func (f *fixture) runController(analysisRunName string, startInformers bool, expectError bool, c *Controller, i informers.SharedInformerFactory, k8sI kubeinformers.SharedInformerFactory) *Controller {
 	if startInformers {
 		stopCh := make(chan struct{})
 		defer close(stopCh)
@@ -147,7 +174,7 @@ func (f *fixture) runController(analysisRunName string, startInformers bool, exp
 		assert.True(f.t, cache.WaitForCacheSync(stopCh, c.analysisRunSynced))
 	}
 
-	err := c.syncHandler(analysisRunName)
+	err := c.syncHandler(context.Background(), analysisRunName)
 	if !expectError && err != nil {
 		f.t.Errorf("error syncing experiment: %v", err)
 	} else if expectError && err == nil {
@@ -167,7 +194,7 @@ func (f *fixture) runController(analysisRunName string, startInformers bool, exp
 	}
 
 	if len(f.actions) > len(actions) {
-		f.t.Errorf("%d additional expected actions:%+v", len(f.actions)-len(actions), f.actions[len(actions):])
+		f.t.Errorf("%d expected actions did not occur:%+v", len(f.actions)-len(actions), f.actions[len(actions):])
 	}
 
 	// k8sActions := filterInformerActions(f.kubeclient.Actions())
@@ -182,7 +209,7 @@ func (f *fixture) runController(analysisRunName string, startInformers bool, exp
 	// }
 
 	// if len(f.kubeactions) > len(k8sActions) {
-	// 	f.t.Errorf("%d additional expected actions:%+v", len(f.kubeactions)-len(k8sActions), f.kubeactions[len(k8sActions):])
+	// 	f.t.Errorf("%d expected actions did not occur:%+v", len(f.kubeactions)-len(k8sActions), f.kubeactions[len(k8sActions):])
 	// }
 	return c
 }
@@ -226,14 +253,14 @@ func filterInformerActions(actions []core.Action) []core.Action {
 	return ret
 }
 
-func (f *fixture) expectUpdateAnalysisRunAction(analysisRun *v1alpha1.AnalysisRun) int {
+func (f *fixture) expectUpdateAnalysisRunAction(analysisRun *v1alpha1.AnalysisRun) int { //nolint:unused
 	action := core.NewUpdateAction(schema.GroupVersionResource{Resource: "analysisrun"}, analysisRun.Namespace, analysisRun)
 	len := len(f.actions)
 	f.actions = append(f.actions, action)
 	return len
 }
 
-func (f *fixture) getUpdatedAnalysisRun(index int) *v1alpha1.AnalysisRun {
+func (f *fixture) getUpdatedAnalysisRun(index int) *v1alpha1.AnalysisRun { //nolint:unused
 	action := filterInformerActions(f.client.Actions())[index]
 	updateAction, ok := action.(core.UpdateAction)
 	if !ok {
@@ -247,7 +274,7 @@ func (f *fixture) getUpdatedAnalysisRun(index int) *v1alpha1.AnalysisRun {
 	return ar
 }
 
-func (f *fixture) expectPatchAnalysisRunAction(analysisRun *v1alpha1.AnalysisRun) int {
+func (f *fixture) expectPatchAnalysisRunAction(analysisRun *v1alpha1.AnalysisRun) int { //nolint:unused
 	analysisRunSchema := schema.GroupVersionResource{
 		Resource: "analysisruns",
 		Version:  "v1alpha1",
@@ -257,7 +284,7 @@ func (f *fixture) expectPatchAnalysisRunAction(analysisRun *v1alpha1.AnalysisRun
 	return len
 }
 
-func (f *fixture) getPatchedAnalysisRun(index int) v1alpha1.AnalysisRun {
+func (f *fixture) getPatchedAnalysisRun(index int) v1alpha1.AnalysisRun { //nolint:unused
 	action := filterInformerActions(f.client.Actions())[index]
 	patchAction, ok := action.(core.PatchAction)
 	if !ok {
@@ -271,6 +298,22 @@ func (f *fixture) getPatchedAnalysisRun(index int) v1alpha1.AnalysisRun {
 	return ar
 }
 
+func (f *fixture) expectDeleteAnalysisRunAction(analysisRun *v1alpha1.AnalysisRun) int { //nolint:unused
+	action := core.NewDeleteAction(schema.GroupVersionResource{Resource: "analysisrun"}, analysisRun.Namespace, analysisRun.Name)
+	len := len(f.actions)
+	f.actions = append(f.actions, action)
+	return len
+}
+
+func (f *fixture) getDeletedAnalysisRunNamespaceAndName(index int) string { //nolint:unused
+	action := filterInformerActions(f.client.Actions())[index]
+	deleteAction, ok := action.(core.DeleteAction)
+	if !ok {
+		f.t.Fatalf("Expected Patch action, not %s", action.GetVerb())
+	}
+	return fmt.Sprintf("%s/%s", deleteAction.GetNamespace(), deleteAction.GetName())
+}
+
 func TestNoReconcileForNotFoundAnalysisRun(t *testing.T) {
 	f := newFixture(t)
 	defer f.Close()
@@ -302,3 +345,57 @@ func TestNoReconcileForAnalysisRunWithDeletionTimestamp(t *testing.T) {
 
 	f.run(getKey(ar, t))
 }
+
+func TestFailedToCreateProviderError(t *testing.T) {
+	f := newFixture(t)
+	defer f.Close()
+
+	ar := &v1alpha1.AnalysisRun{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "foo",
+			Namespace: metav1.NamespaceDefault,
+		},
+		Spec: v1alpha1.AnalysisRunSpec{
+			Metrics: []v1alpha1.Metric{
+				{
+					Name: "metric1",
+					Provider: v1alpha1.MetricProvider{
+						Plugin: map[string]json.RawMessage{"mypluginns/myplugin": []byte(`{"invalid": "json"}`)},
+					},
+				},
+			},
+		},
+	}
+	f.analysisRunLister = append(f.analysisRunLister, ar)
+	f.objects = append(f.objects, ar)
+
+	c, i, k8sI := f.newController(noResyncPeriodFunc)
+	c.newProvider = func(logCtx log.Entry, namespace string, metric v1alpha1.Metric) (metric.Provider, error) {
+		return nil, fmt.Errorf("failed to create provider")
+	}
+
+	pi := f.expectPatchAnalysisRunAction(ar)
+
+	f.runController(getKey(ar, t), true, false, c, i, k8sI)
+
+	updatedAr := f.getPatchedAnalysisRun(pi)
+
+	assert.Equal(t, v1alpha1.AnalysisPhaseError, updatedAr.Status.MetricResults[0].Measurements[0].Phase)
+	assert.Equal(t, "failed to create provider", updatedAr.Status.MetricResults[0].Measurements[0].Message)
+}
+
+func TestRun(t *testing.T) {
+	f := newFixture(t)
+	defer f.Close()
+
+	// make sure we can start and top the controller
+	c, _, _ := f.newController(noResyncPeriodFunc)
+	ctx, cancel := context.WithCancel(context.TODO())
+	defer cancel()
+	go func() {
+		time.Sleep(1000 * time.Millisecond)
+		c.analysisRunWorkQueue.ShutDownWithDrain()
+		cancel()
+	}()
+	c.Run(ctx, 1)
+}

analysis/sync.go

@@ -1,6 +1,9 @@
 package analysis
 
 import (
+	"context"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	patchtypes "k8s.io/apimachinery/pkg/types"
 
 	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
@@ -8,7 +11,8 @@ import (
 	logutil "github.com/argoproj/argo-rollouts/utils/log"
 )
 
-func (c *AnalysisController) persistAnalysisRunStatus(orig *v1alpha1.AnalysisRun, newStatus v1alpha1.AnalysisRunStatus) error {
+func (c *Controller) persistAnalysisRunStatus(orig *v1alpha1.AnalysisRun, newStatus v1alpha1.AnalysisRunStatus) error {
+	ctx := context.TODO()
 	logCtx := logutil.WithAnalysisRun(orig)
 	patch, modified, err := diff.CreateTwoWayMergePatch(
 		&v1alpha1.AnalysisRun{
@@ -26,7 +30,7 @@ func (c *AnalysisController) persistAnalysisRunStatus(orig *v1alpha1.AnalysisRun
 		return nil
 	}
 	logCtx.Debugf("AnalysisRun Patch: %s", patch)
-	_, err = c.argoProjClientset.ArgoprojV1alpha1().AnalysisRuns(orig.Namespace).Patch(orig.Name, patchtypes.MergePatchType, patch)
+	_, err = c.argoProjClientset.ArgoprojV1alpha1().AnalysisRuns(orig.Namespace).Patch(ctx, orig.Name, patchtypes.MergePatchType, patch, metav1.PatchOptions{})
 	if err != nil {
 		logCtx.Warningf("Error updating analysisRun: %v", err)
 		return err

cmd/kubectl-argo-rollouts/main.go

@@ -3,10 +3,14 @@ package main
 import (
 	"os"
 
+	log "github.com/sirupsen/logrus"
 	"k8s.io/cli-runtime/pkg/genericclioptions"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
-	"k8s.io/klog"
+	"k8s.io/klog/v2"
+
+	logutil "github.com/argoproj/argo-rollouts/utils/log"
 
 	"github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/cmd"
 	"github.com/argoproj/argo-rollouts/pkg/kubectl-argo-rollouts/options"
@@ -14,6 +18,7 @@ import (
 
 func main() {
 	klog.InitFlags(nil)
+	logutil.SetKLogLogger(log.New())
 	streams := genericclioptions.IOStreams{In: os.Stdin, Out: os.Stdout, ErrOut: os.Stderr}
 	o := options.NewArgoRolloutsOptions(streams)
 	root := cmd.NewCmdArgoRollouts(o)

cmd/rollouts-controller/main.go

@@ -1,143 +1,340 @@
 package main
 
 import (
-	"flag"
 	"fmt"
+	"net/http"
 	"os"
-	"strconv"
+	"strings"
 	"time"
 
+	"github.com/argoproj/pkg/kubeclientmetrics"
+	smiclientset "github.com/servicemeshinterface/smi-sdk-go/pkg/gen/client/split/clientset/versioned"
 	log "github.com/sirupsen/logrus"
 	"github.com/spf13/cobra"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
+	"k8s.io/client-go/dynamic/dynamicinformer"
 	kubeinformers "k8s.io/client-go/informers"
 	"k8s.io/client-go/kubernetes"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/azure"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/oidc"
 	"k8s.io/client-go/tools/clientcmd"
-	"k8s.io/klog"
+
+	"github.com/argoproj/argo-rollouts/metricproviders"
+	"github.com/argoproj/argo-rollouts/rollout"
+	"github.com/argoproj/argo-rollouts/utils/errors"
+	"github.com/argoproj/argo-rollouts/utils/record"
 
 	"github.com/argoproj/argo-rollouts/controller"
+	"github.com/argoproj/argo-rollouts/controller/metrics"
 	jobprovider "github.com/argoproj/argo-rollouts/metricproviders/job"
 	clientset "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned"
-	informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions"
 	"github.com/argoproj/argo-rollouts/pkg/signals"
 	controllerutil "github.com/argoproj/argo-rollouts/utils/controller"
+	"github.com/argoproj/argo-rollouts/utils/defaults"
+	ingressutil "github.com/argoproj/argo-rollouts/utils/ingress"
+	istioutil "github.com/argoproj/argo-rollouts/utils/istio"
+	logutil "github.com/argoproj/argo-rollouts/utils/log"
+	"github.com/argoproj/argo-rollouts/utils/tolerantinformer"
+	"github.com/argoproj/argo-rollouts/utils/version"
 )
 
 const (
 	// CLIName is the name of the CLI
-	cliName = "argo-rollouts"
+	cliName    = "argo-rollouts"
+	jsonFormat = "json"
+	textFormat = "text"
 
-	defaultIstioVersion = "v1alpha3"
+	controllerAnalysis = "analysis"
 )
 
+var supportedControllers = map[string]bool{controllerAnalysis: true}
+
 func newCommand() *cobra.Command {
 	var (
-		clientConfig        clientcmd.ClientConfig
-		rolloutResyncPeriod int64
-		logLevel            string
-		glogLevel           int
-		metricsPort         int
-		instanceID          string
-		rolloutThreads      int
-		experimentThreads   int
-		analysisThreads     int
-		serviceThreads      int
-		istioVersion        string
+		clientConfig                   clientcmd.ClientConfig
+		rolloutResyncPeriod            int64
+		logLevel                       string
+		logFormat                      string
+		klogLevel                      int
+		metricsPort                    int
+		healthzPort                    int
+		instanceID                     string
+		qps                            float32
+		burst                          int
+		rolloutThreads                 int
+		experimentThreads              int
+		analysisThreads                int
+		serviceThreads                 int
+		ingressThreads                 int
+		ephemeralMetadataThreads       int
+		targetGroupBindingVersion      string
+		albTagKeyResourceID            string
+		istioVersion                   string
+		trafficSplitVersion            string
+		traefikAPIGroup                string
+		traefikVersion                 string
+		ambassadorVersion              string
+		ingressVersion                 string
+		appmeshCRDVersion              string
+		albIngressClasses              []string
+		nginxIngressClasses            []string
+		awsVerifyTargetGroup           bool
+		namespaced                     bool
+		printVersion                   bool
+		selfServiceNotificationEnabled bool
+		controllersEnabled             []string
+		pprofAddress                   string
 	)
+	electOpts := controller.NewLeaderElectionOptions()
 	var command = cobra.Command{
 		Use:   cliName,
 		Short: "argo-rollouts is a controller to operate on rollout CRD",
 		RunE: func(c *cobra.Command, args []string) error {
-			setLogLevel(logLevel)
-			formatter := &log.TextFormatter{
-				FullTimestamp: true,
+			if printVersion {
+				fmt.Println(version.GetVersion())
+				return nil
 			}
-			log.SetFormatter(formatter)
-			setGLogLevel(glogLevel)
+			logger := log.New()
+			setLogLevel(logLevel)
+			if logFormat != "" {
+				log.SetFormatter(createFormatter(logFormat))
+				logger.SetFormatter(createFormatter(logFormat))
+			}
+			logutil.SetKLogLogger(logger)
+			logutil.SetKLogLevel(klogLevel)
 
 			// set up signals so we handle the first shutdown signal gracefully
-			stopCh := signals.SetupSignalHandler()
+			ctx := signals.SetupSignalHandlerContext()
+
+			defaults.SetVerifyTargetGroup(awsVerifyTargetGroup)
+			defaults.SetTargetGroupBindingAPIVersion(targetGroupBindingVersion)
+			defaults.SetalbTagKeyResourceID(albTagKeyResourceID)
+			defaults.SetIstioAPIVersion(istioVersion)
+			defaults.SetAmbassadorAPIVersion(ambassadorVersion)
+			defaults.SetSMIAPIVersion(trafficSplitVersion)
+			defaults.SetAppMeshCRDVersion(appmeshCRDVersion)
+			defaults.SetTraefikAPIGroup(traefikAPIGroup)
+			defaults.SetTraefikVersion(traefikVersion)
 
-			// cfg, err := clientcmd.BuildConfigFromFlags(masterURL, kubeconfig)
 			config, err := clientConfig.ClientConfig()
-			checkError(err)
+			errors.CheckError(err)
+			config.QPS = qps
+			config.Burst = burst
 			namespace := metav1.NamespaceAll
-			configNS, modified, err := clientConfig.Namespace()
-			checkError(err)
-			if modified {
+			configNS, _, err := clientConfig.Namespace()
+			errors.CheckError(err)
+			if namespaced {
 				namespace = configNS
-				log.Infof("Using namespace %s", namespace)
 			}
+			log.WithFields(log.Fields{
+				"version":     version.GetVersion(),
+				"namespace":   namespace,
+				"instanceID":  instanceID,
+				"metricsPort": metricsPort,
+				"healthzPort": healthzPort,
+			}).Info("Argo Rollouts controller starting")
+
+			k8sRequestProvider := &metrics.K8sRequestsCountProvider{}
+			kubeclientmetrics.AddMetricsTransportWrapper(config, k8sRequestProvider.IncKubernetesRequest)
 
 			kubeClient, err := kubernetes.NewForConfig(config)
-			checkError(err)
-			rolloutClient, err := clientset.NewForConfig(config)
-			checkError(err)
+			errors.CheckError(err)
+			argoprojClient, err := clientset.NewForConfig(config)
+			errors.CheckError(err)
 			dynamicClient, err := dynamic.NewForConfig(config)
-			checkError(err)
+			errors.CheckError(err)
+			discoveryClient, err := discovery.NewDiscoveryClientForConfig(config)
+			errors.CheckError(err)
+			smiClient, err := smiclientset.NewForConfig(config)
+			errors.CheckError(err)
 			resyncDuration := time.Duration(rolloutResyncPeriod) * time.Second
 			kubeInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
 				kubeClient,
 				resyncDuration,
 				kubeinformers.WithNamespace(namespace))
 			instanceIDSelector := controllerutil.InstanceIDRequirement(instanceID)
-			argoRolloutsInformerFactory := informers.NewSharedInformerFactoryWithOptions(
-				rolloutClient,
-				resyncDuration,
-				informers.WithNamespace(namespace),
-				informers.WithTweakListOptions(func(options *metav1.ListOptions) {
-					options.LabelSelector = instanceIDSelector.String()
-				}))
+			instanceIDTweakListFunc := func(options *metav1.ListOptions) {
+				options.LabelSelector = instanceIDSelector.String()
+			}
+			jobKubeClient, _, err := metricproviders.GetAnalysisJobClientset(kubeClient)
+			errors.CheckError(err)
+			jobNs := metricproviders.GetAnalysisJobNamespace()
+			if jobNs == "" {
+				// if not set explicitly use the configured ns
+				jobNs = namespace
+			}
 			jobInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
-				kubeClient,
+				jobKubeClient,
 				resyncDuration,
-				kubeinformers.WithNamespace(namespace),
+				kubeinformers.WithNamespace(jobNs),
 				kubeinformers.WithTweakListOptions(func(options *metav1.ListOptions) {
 					options.LabelSelector = jobprovider.AnalysisRunUIDLabelKey
 				}))
-			cm := controller.NewManager(
-				namespace,
+			// We need three dynamic informer factories:
+			// 1. The first is the dynamic informer for rollouts, analysisruns, analysistemplates, experiments
+			dynamicInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, resyncDuration, namespace, instanceIDTweakListFunc)
+			// 2. The second is for the clusteranalysistemplate. Notice we must instantiate this with
+			// metav1.NamespaceAll. The reason why we need a cluster specific dynamic informer factory
+			// is to support the mode when the rollout controller is started and only operating against
+			// a single namespace (i.e. rollouts-controller --namespace foo).
+			clusterDynamicInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(dynamicClient, resyncDuration, metav1.NamespaceAll, instanceIDTweakListFunc)
+			// 3. We finally need an istio dynamic informer factory which does not use a tweakListFunc.
+			_, istioPrimaryDynamicClient := istioutil.GetPrimaryClusterDynamicClient(kubeClient, namespace)
+			if istioPrimaryDynamicClient == nil {
+				istioPrimaryDynamicClient = dynamicClient
+			}
+			istioDynamicInformerFactory := dynamicinformer.NewFilteredDynamicSharedInformerFactory(istioPrimaryDynamicClient, resyncDuration, namespace, nil)
+
+			var notificationConfigNamespace string
+			if selfServiceNotificationEnabled {
+				notificationConfigNamespace = metav1.NamespaceAll
+			} else {
+				notificationConfigNamespace = defaults.Namespace()
+			}
+			notificationSecretInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
 				kubeClient,
-				rolloutClient,
-				dynamicClient,
-				kubeInformerFactory.Apps().V1().ReplicaSets(),
-				kubeInformerFactory.Core().V1().Services(),
-				jobInformerFactory.Batch().V1().Jobs(),
-				argoRolloutsInformerFactory.Argoproj().V1alpha1().Rollouts(),
-				argoRolloutsInformerFactory.Argoproj().V1alpha1().Experiments(),
-				argoRolloutsInformerFactory.Argoproj().V1alpha1().AnalysisRuns(),
-				argoRolloutsInformerFactory.Argoproj().V1alpha1().AnalysisTemplates(),
 				resyncDuration,
-				instanceID,
-				metricsPort,
-				defaultIstioVersion)
+				kubeinformers.WithNamespace(notificationConfigNamespace),
+				kubeinformers.WithTweakListOptions(func(options *metav1.ListOptions) {
+					options.Kind = "Secret"
+					options.FieldSelector = fmt.Sprintf("metadata.name=%s", record.NotificationSecret)
+				}),
+			)
 
-			// notice that there is no need to run Start methods in a separate goroutine. (i.e. go kubeInformerFactory.Start(stopCh)
-			// Start method is non-blocking and runs all registered informers in a dedicated goroutine.
-			kubeInformerFactory.Start(stopCh)
-			argoRolloutsInformerFactory.Start(stopCh)
-			jobInformerFactory.Start(stopCh)
+			notificationConfigMapInformerFactory := kubeinformers.NewSharedInformerFactoryWithOptions(
+				kubeClient,
+				resyncDuration,
+				kubeinformers.WithNamespace(notificationConfigNamespace),
+				kubeinformers.WithTweakListOptions(func(options *metav1.ListOptions) {
+					options.Kind = "ConfigMap"
+					options.FieldSelector = fmt.Sprintf("metadata.name=%s", record.NotificationConfigMap)
+				}),
+			)
 
-			if err = cm.Run(rolloutThreads, serviceThreads, experimentThreads, analysisThreads, stopCh); err != nil {
+			mode, err := ingressutil.DetermineIngressMode(ingressVersion, kubeClient.DiscoveryClient)
+			errors.CheckError(err)
+			ingressWrapper, err := ingressutil.NewIngressWrapper(mode, kubeClient, kubeInformerFactory)
+			errors.CheckError(err)
+
+			if pprofAddress != "" {
+				mux := controller.NewPProfServer()
+				go func() { log.Println(http.ListenAndServe(pprofAddress, mux)) }()
+			}
+
+			var cm *controller.Manager
+
+			enabledControllers, err := getEnabledControllers(controllersEnabled)
+			errors.CheckError(err)
+
+			// currently only supports running analysis controller independently
+			if enabledControllers[controllerAnalysis] {
+				log.Info("Running only analysis controller")
+				cm = controller.NewAnalysisManager(
+					namespace,
+					kubeClient,
+					argoprojClient,
+					jobInformerFactory.Batch().V1().Jobs(),
+					tolerantinformer.NewTolerantAnalysisRunInformer(dynamicInformerFactory),
+					tolerantinformer.NewTolerantAnalysisTemplateInformer(dynamicInformerFactory),
+					tolerantinformer.NewTolerantClusterAnalysisTemplateInformer(clusterDynamicInformerFactory),
+					resyncDuration,
+					metricsPort,
+					healthzPort,
+					k8sRequestProvider,
+					dynamicInformerFactory,
+					clusterDynamicInformerFactory,
+					namespaced,
+					kubeInformerFactory,
+					jobInformerFactory)
+			} else {
+				cm = controller.NewManager(
+					namespace,
+					kubeClient,
+					argoprojClient,
+					dynamicClient,
+					smiClient,
+					discoveryClient,
+					kubeInformerFactory.Apps().V1().ReplicaSets(),
+					kubeInformerFactory.Core().V1().Services(),
+					ingressWrapper,
+					jobInformerFactory.Batch().V1().Jobs(),
+					tolerantinformer.NewTolerantRolloutInformer(dynamicInformerFactory),
+					tolerantinformer.NewTolerantExperimentInformer(dynamicInformerFactory),
+					tolerantinformer.NewTolerantAnalysisRunInformer(dynamicInformerFactory),
+					tolerantinformer.NewTolerantAnalysisTemplateInformer(dynamicInformerFactory),
+					tolerantinformer.NewTolerantClusterAnalysisTemplateInformer(clusterDynamicInformerFactory),
+					istioPrimaryDynamicClient,
+					istioDynamicInformerFactory.ForResource(istioutil.GetIstioVirtualServiceGVR()).Informer(),
+					istioDynamicInformerFactory.ForResource(istioutil.GetIstioDestinationRuleGVR()).Informer(),
+					notificationConfigMapInformerFactory,
+					notificationSecretInformerFactory,
+					resyncDuration,
+					instanceID,
+					metricsPort,
+					healthzPort,
+					k8sRequestProvider,
+					nginxIngressClasses,
+					albIngressClasses,
+					dynamicInformerFactory,
+					clusterDynamicInformerFactory,
+					istioDynamicInformerFactory,
+					namespaced,
+					kubeInformerFactory,
+					jobInformerFactory,
+					ephemeralMetadataThreads)
+			}
+			if err = cm.Run(ctx, rolloutThreads, serviceThreads, ingressThreads, experimentThreads, analysisThreads, electOpts); err != nil {
 				log.Fatalf("Error running controller: %s", err.Error())
 			}
 			return nil
 		},
 	}
+
+	defaultALBIngressClass := []string{"alb"}
+	defaultNGINXIngressClass := []string{"nginx"}
+
 	clientConfig = addKubectlFlagsToCmd(&command)
 	command.Flags().Int64Var(&rolloutResyncPeriod, "rollout-resync", controller.DefaultRolloutResyncPeriod, "Time period in seconds for rollouts resync.")
+	command.Flags().BoolVar(&namespaced, "namespaced", false, "runs controller in namespaced mode (does not require cluster RBAC)")
 	command.Flags().StringVar(&logLevel, "loglevel", "info", "Set the logging level. One of: debug|info|warn|error")
-	command.Flags().IntVar(&glogLevel, "gloglevel", 0, "Set the glog logging level")
-	command.Flags().IntVar(&metricsPort, "metricsport", controller.DefaultMetricsPort, "Set the port the metrics endpoint should be exposed over")
+	command.Flags().StringVar(&logFormat, "logformat", "", "Set the logging format. One of: text|json")
+	command.Flags().IntVar(&klogLevel, "kloglevel", 0, "Set the klog logging level")
+	command.Flags().IntVar(&metricsPort, "metricsPort", controller.DefaultMetricsPort, "Set the port the metrics endpoint should be exposed over")
+	command.Flags().IntVar(&healthzPort, "healthzPort", controller.DefaultHealthzPort, "Set the port the healthz endpoint should be exposed over")
 	command.Flags().StringVar(&instanceID, "instance-id", "", "Indicates which argo rollout objects the controller should operate on")
+	command.Flags().Float32Var(&qps, "qps", defaults.DefaultQPS, "Maximum QPS (queries per second) to the K8s API server")
+	command.Flags().IntVar(&burst, "burst", defaults.DefaultBurst, "Maximum burst for throttle.")
 	command.Flags().IntVar(&rolloutThreads, "rollout-threads", controller.DefaultRolloutThreads, "Set the number of worker threads for the Rollout controller")
 	command.Flags().IntVar(&experimentThreads, "experiment-threads", controller.DefaultExperimentThreads, "Set the number of worker threads for the Experiment controller")
 	command.Flags().IntVar(&analysisThreads, "analysis-threads", controller.DefaultAnalysisThreads, "Set the number of worker threads for the Experiment controller")
 	command.Flags().IntVar(&serviceThreads, "service-threads", controller.DefaultServiceThreads, "Set the number of worker threads for the Service controller")
-	command.Flags().StringVar(&istioVersion, "istio-api-version", defaultIstioVersion, "Set the default Istio apiVersion that controller should look when manipulating VirtualServices.")
+	command.Flags().IntVar(&ingressThreads, "ingress-threads", controller.DefaultIngressThreads, "Set the number of worker threads for the Ingress controller")
+	command.Flags().IntVar(&ephemeralMetadataThreads, "ephemeral-metadata-threads", rollout.DefaultEphemeralMetadataThreads, "Set the number of worker threads for the Ephemeral Metadata reconciler")
+	command.Flags().StringVar(&targetGroupBindingVersion, "aws-target-group-binding-api-version", defaults.DefaultTargetGroupBindingAPIVersion, "Set the default AWS TargetGroupBinding apiVersion that controller uses when verifying target group weights.")
+	command.Flags().StringVar(&albTagKeyResourceID, "alb-tag-key-resource-id", defaults.DefaultAlbTagKeyResourceID, "Set the default AWS LoadBalancer tag key for resource ID that controller uses when verifying target group weights.")
+	command.Flags().StringVar(&istioVersion, "istio-api-version", defaults.DefaultIstioVersion, "Set the default Istio apiVersion that controller should look when manipulating VirtualServices.")
+	command.Flags().StringVar(&ambassadorVersion, "ambassador-api-version", defaults.DefaultAmbassadorVersion, "Set the Ambassador apiVersion that controller should look when manipulating Ambassador Mappings.")
+	command.Flags().StringVar(&trafficSplitVersion, "traffic-split-api-version", defaults.DefaultSMITrafficSplitVersion, "Set the default TrafficSplit apiVersion that controller uses when creating TrafficSplits.")
+	command.Flags().StringVar(&traefikAPIGroup, "traefik-api-group", defaults.DefaultTraefikAPIGroup, "Set the default Traefik apiGroup that controller uses.")
+	command.Flags().StringVar(&traefikVersion, "traefik-api-version", defaults.DefaultTraefikVersion, "Set the default Traefik apiVersion that controller uses.")
+	command.Flags().StringVar(&ingressVersion, "ingress-api-version", "", "Set the Ingress apiVersion that the controller should use.")
+	command.Flags().StringVar(&appmeshCRDVersion, "appmesh-crd-version", defaults.DefaultAppMeshCRDVersion, "Set the default AppMesh CRD Version that controller uses when manipulating resources.")
+	command.Flags().StringArrayVar(&albIngressClasses, "alb-ingress-classes", defaultALBIngressClass, "Defines all the ingress class annotations that the alb ingress controller operates on. Defaults to alb")
+	command.Flags().StringArrayVar(&nginxIngressClasses, "nginx-ingress-classes", defaultNGINXIngressClass, "Defines all the ingress class annotations that the nginx ingress controller operates on. Defaults to nginx")
+	command.Flags().BoolVar(&awsVerifyTargetGroup, "alb-verify-weight", false, "Verify ALB target group weights before progressing through steps (requires AWS privileges)")
+	command.Flags().MarkDeprecated("alb-verify-weight", "Use --aws-verify-target-group instead")
+	command.Flags().BoolVar(&awsVerifyTargetGroup, "aws-verify-target-group", false, "Verify ALB target group before progressing through steps (requires AWS privileges)")
+	command.Flags().BoolVar(&printVersion, "version", false, "Print version")
+	command.Flags().BoolVar(&electOpts.LeaderElect, "leader-elect", controller.DefaultLeaderElect, "If true, controller will perform leader election between instances to ensure no more than one instance of controller operates at a time")
+	command.Flags().DurationVar(&electOpts.LeaderElectionLeaseDuration, "leader-election-lease-duration", controller.DefaultLeaderElectionLeaseDuration, "The duration that non-leader candidates will wait after observing a leadership renewal until attempting to acquire leadership of a led but unrenewed leader slot. This is effectively the maximum duration that a leader can be stopped before it is replaced by another candidate. This is only applicable if leader election is enabled.")
+	command.Flags().DurationVar(&electOpts.LeaderElectionRenewDeadline, "leader-election-renew-deadline", controller.DefaultLeaderElectionRenewDeadline, "The interval between attempts by the acting master to renew a leadership slot before it stops leading. This must be less than or equal to the lease duration. This is only applicable if leader election is enabled.")
+	command.Flags().DurationVar(&electOpts.LeaderElectionRetryPeriod, "leader-election-retry-period", controller.DefaultLeaderElectionRetryPeriod, "The duration the clients should wait between attempting acquisition and renewal of a leadership. This is only applicable if leader election is enabled.")
+	command.Flags().BoolVar(&selfServiceNotificationEnabled, "self-service-notification-enabled", false, "Allows rollouts controller to pull notification config from the namespace that the rollout resource is in. This is useful for self-service notification.")
+	command.Flags().StringSliceVar(&controllersEnabled, "controllers", nil, "Explicitly specify the list of controllers to run, currently only supports 'analysis', eg. --controller=analysis. Default: all controllers are enabled")
+	command.Flags().StringVar(&pprofAddress, "enable-pprof-address", "", "Enable pprof profiling on controller by providing a server address.")
 	return &command
 }
 
@@ -167,15 +364,33 @@ func setLogLevel(logLevel string) {
 	log.SetLevel(level)
 }
 
-// setGLogLevel set the glog level for the k8s go-client
-func setGLogLevel(glogLevel int) {
-	klog.InitFlags(nil)
-	_ = flag.Set("logtostderr", "true")
-	_ = flag.Set("v", strconv.Itoa(glogLevel))
+func createFormatter(logFormat string) log.Formatter {
+	var formatType log.Formatter
+	switch strings.ToLower(logFormat) {
+	case jsonFormat:
+		formatType = &log.JSONFormatter{}
+	case textFormat:
+		formatType = &log.TextFormatter{
+			FullTimestamp: true,
+		}
+	default:
+		log.Infof("Unknown format: %s. Using text logformat", logFormat)
+		formatType = &log.TextFormatter{
+			FullTimestamp: true,
+		}
+	}
+
+	return formatType
 }
 
-func checkError(err error) {
-	if err != nil {
-		log.Fatal(err)
+func getEnabledControllers(controllersEnabled []string) (map[string]bool, error) {
+	enabledControllers := make(map[string]bool)
+	for _, controller := range controllersEnabled {
+		if supportedControllers[controller] {
+			enabledControllers[controller] = true
+		} else {
+			return nil, fmt.Errorf("unsupported controller: %s", controller)
+		}
 	}
+	return enabledControllers, nil
 }

controller/controller.go

@@ -1,80 +1,250 @@
 package controller
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
+	"net/http"
+	"os"
+	"sync"
 	"time"
 
-	"github.com/pkg/errors"
-	log "github.com/sirupsen/logrus"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/util/runtime"
-	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"github.com/argoproj/argo-rollouts/utils/plugin"
+
+	istioutil "github.com/argoproj/argo-rollouts/utils/istio"
+
+	goPlugin "github.com/hashicorp/go-plugin"
+
+	rolloutsConfig "github.com/argoproj/argo-rollouts/utils/config"
+
 	"k8s.io/apimachinery/pkg/util/wait"
+
+	"k8s.io/client-go/dynamic/dynamicinformer"
+	kubeinformers "k8s.io/client-go/informers"
+
+	notificationapi "github.com/argoproj/notifications-engine/pkg/api"
+	notificationcontroller "github.com/argoproj/notifications-engine/pkg/controller"
+
+	"github.com/pkg/errors"
+	smiclientset "github.com/servicemeshinterface/smi-sdk-go/pkg/gen/client/split/clientset/versioned"
+	log "github.com/sirupsen/logrus"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/util/runtime"
+	"k8s.io/apimachinery/pkg/util/uuid"
+	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/dynamic"
 	appsinformers "k8s.io/client-go/informers/apps/v1"
 	batchinformers "k8s.io/client-go/informers/batch/v1"
 	coreinformers "k8s.io/client-go/informers/core/v1"
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/kubernetes/scheme"
-	typedcorev1 "k8s.io/client-go/kubernetes/typed/core/v1"
 	"k8s.io/client-go/tools/cache"
-	"k8s.io/client-go/tools/record"
+	"k8s.io/client-go/tools/leaderelection"
+	"k8s.io/client-go/tools/leaderelection/resourcelock"
 	"k8s.io/client-go/util/workqueue"
 
 	"github.com/argoproj/argo-rollouts/analysis"
 	"github.com/argoproj/argo-rollouts/controller/metrics"
 	"github.com/argoproj/argo-rollouts/experiments"
+	"github.com/argoproj/argo-rollouts/ingress"
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
 	clientset "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned"
 	rolloutscheme "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/scheme"
 	informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions/rollouts/v1alpha1"
 	"github.com/argoproj/argo-rollouts/rollout"
 	"github.com/argoproj/argo-rollouts/service"
+	"github.com/argoproj/argo-rollouts/utils/defaults"
+	ingressutil "github.com/argoproj/argo-rollouts/utils/ingress"
+	"github.com/argoproj/argo-rollouts/utils/queue"
+	"github.com/argoproj/argo-rollouts/utils/record"
 )
 
-const controllerAgentName = "rollouts-controller"
-
 const (
-	// DefaultRolloutResyncPeriod Default time in seconds for rollout resync period
+	// DefaultRolloutResyncPeriod is the default time in seconds for rollout resync period
 	DefaultRolloutResyncPeriod = 15 * 60
 
-	// DefaultMetricsPort Default port to expose the metrics endpoint
+	// DefaultHealthzPort is the default port to check controller's health
+	DefaultHealthzPort = 8080
+
+	// DefaultMetricsPort is the default port to expose the metrics endpoint
 	DefaultMetricsPort = 8090
 
-	// DefaultRolloutThreads Default number of rollout worker threads to start with the controller
+	// DefaultRolloutThreads is the default number of rollout worker threads to start with the controller
 	DefaultRolloutThreads = 10
 
-	// DefaultExperimentThreads Default number of experiment worker threads to start with the controller
+	// DefaultExperimentThreads is the default number of experiment worker threads to start with the controller
 	DefaultExperimentThreads = 10
 
-	// DefaultAnalysisThreads Default number of analysis worker threads to start with the controller
+	// DefaultAnalysisThreads is the default number of analysis worker threads to start with the controller
 	DefaultAnalysisThreads = 30
 
-	// DefaultServiceThreads Default number of service worker threads to start with the controller
+	// DefaultServiceThreads is the default number of service worker threads to start with the controller
 	DefaultServiceThreads = 10
+
+	// DefaultIngressThreads is the default number of ingress worker threads to start with the controller
+	DefaultIngressThreads = 10
+
+	// DefaultLeaderElect is the default true leader election should be enabled
+	DefaultLeaderElect = true
+
+	// DefaultLeaderElectionLeaseDuration is the default time in seconds that non-leader candidates will wait to force acquire leadership
+	DefaultLeaderElectionLeaseDuration = 15 * time.Second
+
+	// DefaultLeaderElectionRenewDeadline is the default time in seconds that the acting master will retry refreshing leadership before giving up
+	DefaultLeaderElectionRenewDeadline = 10 * time.Second
+
+	// DefaultLeaderElectionRetryPeriod is the default time in seconds that the leader election clients should wait between tries of actions
+	DefaultLeaderElectionRetryPeriod = 2 * time.Second
+
+	defaultLeaderElectionLeaseLockName = "argo-rollouts-controller-lock"
+	listenAddr                         = "0.0.0.0:%d"
 )
 
+type LeaderElectionOptions struct {
+	LeaderElect                 bool
+	LeaderElectionNamespace     string
+	LeaderElectionLeaseDuration time.Duration
+	LeaderElectionRenewDeadline time.Duration
+	LeaderElectionRetryPeriod   time.Duration
+}
+
+func NewLeaderElectionOptions() *LeaderElectionOptions {
+	return &LeaderElectionOptions{
+		LeaderElect:                 DefaultLeaderElect,
+		LeaderElectionNamespace:     defaults.Namespace(),
+		LeaderElectionLeaseDuration: DefaultLeaderElectionLeaseDuration,
+		LeaderElectionRenewDeadline: DefaultLeaderElectionRenewDeadline,
+		LeaderElectionRetryPeriod:   DefaultLeaderElectionRetryPeriod,
+	}
+}
+
 // Manager is the controller implementation for Argo-Rollout resources
 type Manager struct {
-	metricsServer        *metrics.MetricsServer
-	rolloutController    *rollout.RolloutController
-	experimentController *experiments.ExperimentController
-	analysisController   *analysis.AnalysisController
-	serviceController    *service.ServiceController
+	wg                      *sync.WaitGroup
+	metricsServer           *metrics.MetricsServer
+	healthzServer           *http.Server
+	rolloutController       *rollout.Controller
+	experimentController    *experiments.Controller
+	analysisController      *analysis.Controller
+	serviceController       *service.Controller
+	ingressController       *ingress.Controller
+	notificationsController notificationcontroller.NotificationController
 
-	rolloutSynced          cache.InformerSynced
-	experimentSynced       cache.InformerSynced
-	analysisRunSynced      cache.InformerSynced
-	analysisTemplateSynced cache.InformerSynced
-	serviceSynced          cache.InformerSynced
-	jobSynced              cache.InformerSynced
-	replicasSetSynced      cache.InformerSynced
+	rolloutSynced                 cache.InformerSynced
+	experimentSynced              cache.InformerSynced
+	analysisRunSynced             cache.InformerSynced
+	analysisTemplateSynced        cache.InformerSynced
+	clusterAnalysisTemplateSynced cache.InformerSynced
+	serviceSynced                 cache.InformerSynced
+	ingressSynced                 cache.InformerSynced
+	jobSynced                     cache.InformerSynced
+	replicasSetSynced             cache.InformerSynced
+	configMapSynced               cache.InformerSynced
+	secretSynced                  cache.InformerSynced
 
 	rolloutWorkqueue     workqueue.RateLimitingInterface
 	serviceWorkqueue     workqueue.RateLimitingInterface
+	ingressWorkqueue     workqueue.RateLimitingInterface
 	experimentWorkqueue  workqueue.RateLimitingInterface
 	analysisRunWorkqueue workqueue.RateLimitingInterface
 
-	defaultIstioVersion string
+	refResolver rollout.TemplateRefResolver
+
+	kubeClientSet kubernetes.Interface
+
+	namespace string
+
+	dynamicInformerFactory               dynamicinformer.DynamicSharedInformerFactory
+	clusterDynamicInformerFactory        dynamicinformer.DynamicSharedInformerFactory
+	istioDynamicInformerFactory          dynamicinformer.DynamicSharedInformerFactory
+	namespaced                           bool
+	kubeInformerFactory                  kubeinformers.SharedInformerFactory
+	notificationConfigMapInformerFactory kubeinformers.SharedInformerFactory
+	notificationSecretInformerFactory    kubeinformers.SharedInformerFactory
+	jobInformerFactory                   kubeinformers.SharedInformerFactory
+	istioPrimaryDynamicClient            dynamic.Interface
+
+	onlyAnalysisMode bool
+}
+
+func NewAnalysisManager(
+	namespace string,
+	kubeclientset kubernetes.Interface,
+	argoprojclientset clientset.Interface,
+	jobInformer batchinformers.JobInformer,
+	analysisRunInformer informers.AnalysisRunInformer,
+	analysisTemplateInformer informers.AnalysisTemplateInformer,
+	clusterAnalysisTemplateInformer informers.ClusterAnalysisTemplateInformer,
+	resyncPeriod time.Duration,
+	metricsPort int,
+	healthzPort int,
+	k8sRequestProvider *metrics.K8sRequestsCountProvider,
+	dynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory,
+	clusterDynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory,
+	namespaced bool,
+	kubeInformerFactory kubeinformers.SharedInformerFactory,
+	jobInformerFactory kubeinformers.SharedInformerFactory,
+) *Manager {
+	runtime.Must(rolloutscheme.AddToScheme(scheme.Scheme))
+	log.Info("Creating event broadcaster")
+
+	metricsAddr := fmt.Sprintf(listenAddr, metricsPort)
+	metricsServer := metrics.NewMetricsServer(metrics.ServerConfig{
+		Addr:                          metricsAddr,
+		RolloutLister:                 nil,
+		AnalysisRunLister:             analysisRunInformer.Lister(),
+		AnalysisTemplateLister:        analysisTemplateInformer.Lister(),
+		ClusterAnalysisTemplateLister: clusterAnalysisTemplateInformer.Lister(),
+		ExperimentLister:              nil,
+		K8SRequestProvider:            k8sRequestProvider,
+	})
+
+	healthzServer := NewHealthzServer(fmt.Sprintf(listenAddr, healthzPort))
+	analysisRunWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "AnalysisRuns")
+	recorder := record.NewEventRecorder(kubeclientset, metrics.MetricRolloutEventsTotal, metrics.MetricNotificationFailedTotal, metrics.MetricNotificationSuccessTotal, metrics.MetricNotificationSend, nil)
+	analysisController := analysis.NewController(analysis.ControllerConfig{
+		KubeClientSet:        kubeclientset,
+		ArgoProjClientset:    argoprojclientset,
+		AnalysisRunInformer:  analysisRunInformer,
+		JobInformer:          jobInformer,
+		ResyncPeriod:         resyncPeriod,
+		AnalysisRunWorkQueue: analysisRunWorkqueue,
+		MetricsServer:        metricsServer,
+		Recorder:             recorder,
+	})
+
+	cm := &Manager{
+		wg:                            &sync.WaitGroup{},
+		metricsServer:                 metricsServer,
+		healthzServer:                 healthzServer,
+		jobSynced:                     jobInformer.Informer().HasSynced,
+		analysisRunSynced:             analysisRunInformer.Informer().HasSynced,
+		analysisTemplateSynced:        analysisTemplateInformer.Informer().HasSynced,
+		clusterAnalysisTemplateSynced: clusterAnalysisTemplateInformer.Informer().HasSynced,
+		analysisRunWorkqueue:          analysisRunWorkqueue,
+		analysisController:            analysisController,
+		namespace:                     namespace,
+		kubeClientSet:                 kubeclientset,
+		dynamicInformerFactory:        dynamicInformerFactory,
+		clusterDynamicInformerFactory: clusterDynamicInformerFactory,
+		namespaced:                    namespaced,
+		kubeInformerFactory:           kubeInformerFactory,
+		jobInformerFactory:            jobInformerFactory,
+		onlyAnalysisMode:              true,
+	}
+
+	_, err := rolloutsConfig.InitializeConfig(kubeclientset, defaults.DefaultRolloutsConfigMapName)
+	if err != nil {
+		log.Fatalf("Failed to init config: %v", err)
+	}
+
+	err = plugin.DownloadPlugins(plugin.FileDownloaderImpl{}, kubeclientset)
+	if err != nil {
+		log.Fatalf("Failed to download plugins: %v", err)
+	}
+
+	return cm
 }
 
 // NewManager returns a new manager to manage all the controllers
@@ -83,146 +253,356 @@ func NewManager(
 	kubeclientset kubernetes.Interface,
 	argoprojclientset clientset.Interface,
 	dynamicclientset dynamic.Interface,
+	smiclientset smiclientset.Interface,
+	discoveryClient discovery.DiscoveryInterface,
 	replicaSetInformer appsinformers.ReplicaSetInformer,
 	servicesInformer coreinformers.ServiceInformer,
+	ingressWrap *ingressutil.IngressWrap,
 	jobInformer batchinformers.JobInformer,
 	rolloutsInformer informers.RolloutInformer,
 	experimentsInformer informers.ExperimentInformer,
 	analysisRunInformer informers.AnalysisRunInformer,
 	analysisTemplateInformer informers.AnalysisTemplateInformer,
+	clusterAnalysisTemplateInformer informers.ClusterAnalysisTemplateInformer,
+	istioPrimaryDynamicClient dynamic.Interface,
+	istioVirtualServiceInformer cache.SharedIndexInformer,
+	istioDestinationRuleInformer cache.SharedIndexInformer,
+	notificationConfigMapInformerFactory kubeinformers.SharedInformerFactory,
+	notificationSecretInformerFactory kubeinformers.SharedInformerFactory,
 	resyncPeriod time.Duration,
 	instanceID string,
 	metricsPort int,
-	defaultIstioVersion string,
+	healthzPort int,
+	k8sRequestProvider *metrics.K8sRequestsCountProvider,
+	nginxIngressClasses []string,
+	albIngressClasses []string,
+	dynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory,
+	clusterDynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory,
+	istioDynamicInformerFactory dynamicinformer.DynamicSharedInformerFactory,
+	namespaced bool,
+	kubeInformerFactory kubeinformers.SharedInformerFactory,
+	jobInformerFactory kubeinformers.SharedInformerFactory,
+	ephemeralMetadataThreads int,
 ) *Manager {
-
-	utilruntime.Must(rolloutscheme.AddToScheme(scheme.Scheme))
+	runtime.Must(rolloutscheme.AddToScheme(scheme.Scheme))
 	log.Info("Creating event broadcaster")
 
-	// Create event broadcaster
-	// Add argo-rollouts custom resources to the default Kubernetes Scheme so Events can be
-	// logged for argo-rollouts types.
-	eventBroadcaster := record.NewBroadcaster()
-	eventBroadcaster.StartLogging(log.Infof)
-	eventBroadcaster.StartRecordingToSink(&typedcorev1.EventSinkImpl{Interface: kubeclientset.CoreV1().Events("")})
-	recorder := eventBroadcaster.NewRecorder(scheme.Scheme, corev1.EventSource{Component: controllerAgentName})
-	metricsAddr := fmt.Sprintf("0.0.0.0:%d", metricsPort)
+	metricsAddr := fmt.Sprintf(listenAddr, metricsPort)
+	metricsServer := metrics.NewMetricsServer(metrics.ServerConfig{
+		Addr:                          metricsAddr,
+		RolloutLister:                 rolloutsInformer.Lister(),
+		AnalysisRunLister:             analysisRunInformer.Lister(),
+		AnalysisTemplateLister:        analysisTemplateInformer.Lister(),
+		ClusterAnalysisTemplateLister: clusterAnalysisTemplateInformer.Lister(),
+		ExperimentLister:              experimentsInformer.Lister(),
+		K8SRequestProvider:            k8sRequestProvider,
+	})
 
-	rolloutWorkqueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Rollouts")
-	experimentWorkqueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Experiments")
-	analysisRunWorkqueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "AnalysisRuns")
-	serviceWorkqueue := workqueue.NewNamedRateLimitingQueue(workqueue.DefaultControllerRateLimiter(), "Services")
+	healthzServer := NewHealthzServer(fmt.Sprintf(listenAddr, healthzPort))
+	rolloutWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "Rollouts")
+	experimentWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "Experiments")
+	analysisRunWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "AnalysisRuns")
+	serviceWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "Services")
+	ingressWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "Ingresses")
 
-	metricsServer := metrics.NewMetricsServer(metricsAddr, rolloutsInformer.Lister())
+	refResolver := rollout.NewInformerBasedWorkloadRefResolver(namespace, dynamicclientset, discoveryClient, argoprojclientset, rolloutsInformer.Informer())
+	apiFactory := notificationapi.NewFactory(record.NewAPIFactorySettings(analysisRunInformer), defaults.Namespace(), notificationSecretInformerFactory.Core().V1().Secrets().Informer(), notificationConfigMapInformerFactory.Core().V1().ConfigMaps().Informer())
+	recorder := record.NewEventRecorder(kubeclientset, metrics.MetricRolloutEventsTotal, metrics.MetricNotificationFailedTotal, metrics.MetricNotificationSuccessTotal, metrics.MetricNotificationSend, apiFactory)
+	notificationsController := notificationcontroller.NewControllerWithNamespaceSupport(dynamicclientset.Resource(v1alpha1.RolloutGVR), rolloutsInformer.Informer(), apiFactory,
+		notificationcontroller.WithToUnstructured(func(obj metav1.Object) (*unstructured.Unstructured, error) {
+			data, err := json.Marshal(obj)
+			if err != nil {
+				return nil, err
+			}
+			res := &unstructured.Unstructured{}
+			err = json.Unmarshal(data, res)
+			if err != nil {
+				return nil, err
+			}
+			return res, nil
+		}),
+	)
 
-	rolloutController := rollout.NewRolloutController(
-		namespace,
-		kubeclientset,
-		argoprojclientset,
-		dynamicclientset,
-		experimentsInformer,
-		analysisRunInformer,
-		analysisTemplateInformer,
-		replicaSetInformer,
-		servicesInformer,
-		rolloutsInformer,
-		resyncPeriod,
-		rolloutWorkqueue,
-		serviceWorkqueue,
-		metricsServer,
-		recorder,
-		defaultIstioVersion)
+	rolloutController := rollout.NewController(rollout.ControllerConfig{
+		Namespace:                       namespace,
+		KubeClientSet:                   kubeclientset,
+		ArgoProjClientset:               argoprojclientset,
+		DynamicClientSet:                dynamicclientset,
+		RefResolver:                     refResolver,
+		SmiClientSet:                    smiclientset,
+		ExperimentInformer:              experimentsInformer,
+		AnalysisRunInformer:             analysisRunInformer,
+		AnalysisTemplateInformer:        analysisTemplateInformer,
+		ClusterAnalysisTemplateInformer: clusterAnalysisTemplateInformer,
+		IstioPrimaryDynamicClient:       istioPrimaryDynamicClient,
+		IstioVirtualServiceInformer:     istioVirtualServiceInformer,
+		IstioDestinationRuleInformer:    istioDestinationRuleInformer,
+		ReplicaSetInformer:              replicaSetInformer,
+		ServicesInformer:                servicesInformer,
+		IngressWrapper:                  ingressWrap,
+		RolloutsInformer:                rolloutsInformer,
+		ResyncPeriod:                    resyncPeriod,
+		RolloutWorkQueue:                rolloutWorkqueue,
+		ServiceWorkQueue:                serviceWorkqueue,
+		IngressWorkQueue:                ingressWorkqueue,
+		MetricsServer:                   metricsServer,
+		Recorder:                        recorder,
+		EphemeralMetadataThreads:        ephemeralMetadataThreads,
+	})
 
-	experimentController := experiments.NewExperimentController(
-		kubeclientset,
-		argoprojclientset,
-		replicaSetInformer,
-		experimentsInformer,
-		analysisRunInformer,
-		analysisTemplateInformer,
-		resyncPeriod,
-		rolloutWorkqueue,
-		experimentWorkqueue,
-		metricsServer,
-		recorder)
+	experimentController := experiments.NewController(experiments.ControllerConfig{
+		KubeClientSet:                   kubeclientset,
+		ArgoProjClientset:               argoprojclientset,
+		ReplicaSetInformer:              replicaSetInformer,
+		ExperimentsInformer:             experimentsInformer,
+		AnalysisRunInformer:             analysisRunInformer,
+		AnalysisTemplateInformer:        analysisTemplateInformer,
+		ClusterAnalysisTemplateInformer: clusterAnalysisTemplateInformer,
+		ServiceInformer:                 servicesInformer,
+		ResyncPeriod:                    resyncPeriod,
+		RolloutWorkQueue:                rolloutWorkqueue,
+		ExperimentWorkQueue:             experimentWorkqueue,
+		MetricsServer:                   metricsServer,
+		Recorder:                        recorder,
+	})
 
-	analysisController := analysis.NewAnalysisController(
-		kubeclientset,
-		argoprojclientset,
-		analysisRunInformer,
-		jobInformer,
-		resyncPeriod,
-		analysisRunWorkqueue,
-		metricsServer,
-		recorder)
+	analysisController := analysis.NewController(analysis.ControllerConfig{
+		KubeClientSet:        kubeclientset,
+		ArgoProjClientset:    argoprojclientset,
+		AnalysisRunInformer:  analysisRunInformer,
+		JobInformer:          jobInformer,
+		ResyncPeriod:         resyncPeriod,
+		AnalysisRunWorkQueue: analysisRunWorkqueue,
+		MetricsServer:        metricsServer,
+		Recorder:             recorder,
+	})
 
-	serviceController := service.NewServiceController(
-		kubeclientset,
-		servicesInformer,
-		rolloutsInformer,
-		resyncPeriod,
-		rolloutWorkqueue,
-		serviceWorkqueue,
-		metricsServer)
+	serviceController := service.NewController(service.ControllerConfig{
+		Kubeclientset:     kubeclientset,
+		Argoprojclientset: argoprojclientset,
+		RolloutsInformer:  rolloutsInformer,
+		ServicesInformer:  servicesInformer,
+		RolloutWorkqueue:  rolloutWorkqueue,
+		ServiceWorkqueue:  serviceWorkqueue,
+		ResyncPeriod:      resyncPeriod,
+		MetricsServer:     metricsServer,
+	})
+
+	ingressController := ingress.NewController(ingress.ControllerConfig{
+		Client:           kubeclientset,
+		IngressWrap:      ingressWrap,
+		IngressWorkQueue: ingressWorkqueue,
+
+		RolloutsInformer: rolloutsInformer,
+		RolloutWorkQueue: rolloutWorkqueue,
+
+		MetricsServer: metricsServer,
+
+		ALBClasses:   albIngressClasses,
+		NGINXClasses: nginxIngressClasses,
+	})
 
 	cm := &Manager{
-		metricsServer:          metricsServer,
-		rolloutSynced:          rolloutsInformer.Informer().HasSynced,
-		serviceSynced:          servicesInformer.Informer().HasSynced,
-		jobSynced:              jobInformer.Informer().HasSynced,
-		experimentSynced:       experimentsInformer.Informer().HasSynced,
-		analysisRunSynced:      analysisRunInformer.Informer().HasSynced,
-		analysisTemplateSynced: analysisTemplateInformer.Informer().HasSynced,
-		replicasSetSynced:      replicaSetInformer.Informer().HasSynced,
-		rolloutWorkqueue:       rolloutWorkqueue,
-		experimentWorkqueue:    experimentWorkqueue,
-		analysisRunWorkqueue:   analysisRunWorkqueue,
-		serviceWorkqueue:       serviceWorkqueue,
-		rolloutController:      rolloutController,
-		serviceController:      serviceController,
-		experimentController:   experimentController,
-		analysisController:     analysisController,
-		defaultIstioVersion:    defaultIstioVersion,
+		wg:                                   &sync.WaitGroup{},
+		metricsServer:                        metricsServer,
+		healthzServer:                        healthzServer,
+		rolloutSynced:                        rolloutsInformer.Informer().HasSynced,
+		serviceSynced:                        servicesInformer.Informer().HasSynced,
+		ingressSynced:                        ingressWrap.HasSynced,
+		jobSynced:                            jobInformer.Informer().HasSynced,
+		experimentSynced:                     experimentsInformer.Informer().HasSynced,
+		analysisRunSynced:                    analysisRunInformer.Informer().HasSynced,
+		analysisTemplateSynced:               analysisTemplateInformer.Informer().HasSynced,
+		clusterAnalysisTemplateSynced:        clusterAnalysisTemplateInformer.Informer().HasSynced,
+		replicasSetSynced:                    replicaSetInformer.Informer().HasSynced,
+		configMapSynced:                      notificationConfigMapInformerFactory.Core().V1().ConfigMaps().Informer().HasSynced,
+		secretSynced:                         notificationSecretInformerFactory.Core().V1().Secrets().Informer().HasSynced,
+		rolloutWorkqueue:                     rolloutWorkqueue,
+		experimentWorkqueue:                  experimentWorkqueue,
+		analysisRunWorkqueue:                 analysisRunWorkqueue,
+		serviceWorkqueue:                     serviceWorkqueue,
+		ingressWorkqueue:                     ingressWorkqueue,
+		rolloutController:                    rolloutController,
+		serviceController:                    serviceController,
+		ingressController:                    ingressController,
+		experimentController:                 experimentController,
+		analysisController:                   analysisController,
+		notificationsController:              notificationsController,
+		refResolver:                          refResolver,
+		namespace:                            namespace,
+		kubeClientSet:                        kubeclientset,
+		dynamicInformerFactory:               dynamicInformerFactory,
+		clusterDynamicInformerFactory:        clusterDynamicInformerFactory,
+		istioDynamicInformerFactory:          istioDynamicInformerFactory,
+		namespaced:                           namespaced,
+		kubeInformerFactory:                  kubeInformerFactory,
+		jobInformerFactory:                   jobInformerFactory,
+		istioPrimaryDynamicClient:            istioPrimaryDynamicClient,
+		notificationConfigMapInformerFactory: notificationConfigMapInformerFactory,
+		notificationSecretInformerFactory:    notificationSecretInformerFactory,
+	}
+
+	_, err := rolloutsConfig.InitializeConfig(kubeclientset, defaults.DefaultRolloutsConfigMapName)
+	if err != nil {
+		log.Fatalf("Failed to init config: %v", err)
+	}
+
+	err = plugin.DownloadPlugins(plugin.FileDownloaderImpl{}, kubeclientset)
+	if err != nil {
+		log.Fatalf("Failed to download plugins: %v", err)
 	}
 
 	return cm
 }
 
-// Run will set up the event handlers for types we are interested in, as well
-// as syncing informer caches and starting workers. It will block until stopCh
+// Run will sync informer caches and start controllers. It will block until stopCh
 // is closed, at which point it will shutdown the workqueue and wait for
-// workers to finish processing their current work items.
-func (c *Manager) Run(rolloutThreadiness, serviceThreadiness, experimentThreadiness, analysisThreadiness int, stopCh <-chan struct{}) error {
-
+// controllers to finish processing their current work items.
+func (c *Manager) Run(ctx context.Context, rolloutThreadiness, serviceThreadiness, ingressThreadiness, experimentThreadiness, analysisThreadiness int, electOpts *LeaderElectionOptions) error {
 	defer runtime.HandleCrash()
-	defer c.serviceWorkqueue.ShutDown()
-	defer c.rolloutWorkqueue.ShutDown()
-	defer c.experimentWorkqueue.ShutDown()
-	defer c.analysisRunWorkqueue.ShutDown()
-	// Wait for the caches to be synced before starting workers
-	log.Info("Waiting for controller's informer caches to sync")
-	if ok := cache.WaitForCacheSync(stopCh, c.serviceSynced, c.jobSynced, c.rolloutSynced, c.experimentSynced, c.analysisRunSynced, c.analysisTemplateSynced, c.replicasSetSynced); !ok {
-		return fmt.Errorf("failed to wait for caches to sync")
-	}
+	defer func() {
+		log.Infof("Exiting Main Run function")
+	}()
 
-	// Start the informer factories to begin populating the informer caches
-	log.Info("Starting Controllers")
-	go wait.Until(func() { c.rolloutController.Run(rolloutThreadiness, stopCh) }, time.Second, stopCh)
-	go wait.Until(func() { c.serviceController.Run(serviceThreadiness, stopCh) }, time.Second, stopCh)
-	go wait.Until(func() { c.experimentController.Run(experimentThreadiness, stopCh) }, time.Second, stopCh)
-	go wait.Until(func() { c.analysisController.Run(analysisThreadiness, stopCh) }, time.Second, stopCh)
-	log.Info("Started controller")
+	go func() {
+		log.Infof("Starting Healthz Server at %s", c.healthzServer.Addr)
+		err := c.healthzServer.ListenAndServe()
+		if err != nil {
+			err = errors.Wrap(err, "Healthz Server Error")
+			log.Error(err)
+		}
+	}()
 
 	go func() {
 		log.Infof("Starting Metric Server at %s", c.metricsServer.Addr)
-		err := c.metricsServer.ListenAndServe()
-		if err != nil {
-			err = errors.Wrap(err, "Starting Metric Server")
-			log.Fatal(err)
+		if err := c.metricsServer.ListenAndServe(); err != nil {
+			log.Error(errors.Wrap(err, "Metric Server Error"))
 		}
 	}()
-	<-stopCh
+
+	if !electOpts.LeaderElect {
+		log.Info("Leader election is turned off. Running in single-instance mode")
+		go c.startLeading(ctx, rolloutThreadiness, serviceThreadiness, ingressThreadiness, experimentThreadiness, analysisThreadiness)
+		<-ctx.Done()
+	} else {
+		// id used to distinguish between multiple controller manager instances
+		id, err := os.Hostname()
+		if err != nil {
+			log.Fatalf("Error getting hostname for leader election %v", err)
+		}
+
+		if electOpts.LeaderElectionNamespace == "" {
+			log.Fatalf("Error LeaderElectionNamespace is empty")
+		}
+
+		// add a uniquifier so that two processes on the same host don't accidentally both become active
+		id = id + "_" + string(uuid.NewUUID())
+		log.Infof("Leaderelection get id %s", id)
+		leaderelection.RunOrDie(ctx, leaderelection.LeaderElectionConfig{
+			Lock: &resourcelock.LeaseLock{
+				LeaseMeta: metav1.ObjectMeta{Name: defaultLeaderElectionLeaseLockName, Namespace: electOpts.LeaderElectionNamespace}, Client: c.kubeClientSet.CoordinationV1(),
+				LockConfig: resourcelock.ResourceLockConfig{Identity: id},
+			},
+			ReleaseOnCancel: false, // We can not set this to true because our context is sent on sig which means our code
+			// is still running prior to calling cancel. We would need to shut down and then call cancel in order to set this to true.
+			LeaseDuration: electOpts.LeaderElectionLeaseDuration,
+			RenewDeadline: electOpts.LeaderElectionRenewDeadline,
+			RetryPeriod:   electOpts.LeaderElectionRetryPeriod,
+			Callbacks: leaderelection.LeaderCallbacks{
+				OnStartedLeading: func(ctx context.Context) {
+					log.Infof("I am the new leader: %s", id)
+					c.startLeading(ctx, rolloutThreadiness, serviceThreadiness, ingressThreadiness, experimentThreadiness, analysisThreadiness)
+				},
+				OnStoppedLeading: func() {
+					log.Infof("OnStoppedLeading called, shutting down: %s, context err: %s", id, ctx.Err())
+				},
+				OnNewLeader: func(identity string) {
+					log.Infof("New leader elected: %s", identity)
+				},
+			},
+		})
+	}
 	log.Info("Shutting down workers")
+	goPlugin.CleanupClients()
+
+	if !c.onlyAnalysisMode {
+		c.serviceWorkqueue.ShutDownWithDrain()
+		c.ingressWorkqueue.ShutDownWithDrain()
+		c.rolloutWorkqueue.ShutDownWithDrain()
+		c.experimentWorkqueue.ShutDownWithDrain()
+	}
+
+	c.analysisRunWorkqueue.ShutDownWithDrain()
+
+	ctxWithTimeout, cancel := context.WithTimeout(ctx, 5*time.Second) // give max of 10 seconds for http servers to shut down
+	defer cancel()
+	c.healthzServer.Shutdown(ctxWithTimeout)
+	c.metricsServer.Shutdown(ctxWithTimeout)
+
+	c.wg.Wait()
 
 	return nil
 }
+
+func (c *Manager) startLeading(ctx context.Context, rolloutThreadiness, serviceThreadiness, ingressThreadiness, experimentThreadiness, analysisThreadiness int) {
+	defer runtime.HandleCrash()
+	// Start the informer factories to begin populating the informer caches
+	log.Info("Starting Controllers")
+
+	// notice that there is no need to run Start methods in a separate goroutine. (i.e. go kubeInformerFactory.Start(stopCh)
+	// Start method is non-blocking and runs all registered informers in a dedicated goroutine.
+	c.dynamicInformerFactory.Start(ctx.Done())
+	if !c.namespaced {
+		c.clusterDynamicInformerFactory.Start(ctx.Done())
+	}
+	c.kubeInformerFactory.Start(ctx.Done())
+
+	c.jobInformerFactory.Start(ctx.Done())
+
+	if c.onlyAnalysisMode {
+		log.Info("Waiting for controller's informer caches to sync")
+		if ok := cache.WaitForCacheSync(ctx.Done(), c.analysisRunSynced, c.analysisTemplateSynced, c.jobSynced); !ok {
+			log.Fatalf("failed to wait for caches to sync, exiting")
+		}
+		// only wait for cluster scoped informers to sync if we are running in cluster-wide mode
+		if c.namespace == metav1.NamespaceAll {
+			if ok := cache.WaitForCacheSync(ctx.Done(), c.clusterAnalysisTemplateSynced); !ok {
+				log.Fatalf("failed to wait for cluster-scoped caches to sync, exiting")
+			}
+		}
+		go wait.Until(func() { c.wg.Add(1); c.analysisController.Run(ctx, analysisThreadiness); c.wg.Done() }, time.Second, ctx.Done())
+	} else {
+
+		c.notificationConfigMapInformerFactory.Start(ctx.Done())
+		c.notificationSecretInformerFactory.Start(ctx.Done())
+		if ok := cache.WaitForCacheSync(ctx.Done(), c.configMapSynced, c.secretSynced); !ok {
+			log.Fatalf("failed to wait for configmap/secret caches to sync, exiting")
+		}
+
+		// Check if Istio installed on cluster before starting dynamicInformerFactory
+		if istioutil.DoesIstioExist(c.istioPrimaryDynamicClient, c.namespace) {
+			c.istioDynamicInformerFactory.Start(ctx.Done())
+		}
+
+		// Wait for the caches to be synced before starting workers
+		log.Info("Waiting for controller's informer caches to sync")
+		if ok := cache.WaitForCacheSync(ctx.Done(), c.serviceSynced, c.ingressSynced, c.jobSynced, c.rolloutSynced, c.experimentSynced, c.analysisRunSynced, c.analysisTemplateSynced, c.replicasSetSynced, c.configMapSynced, c.secretSynced); !ok {
+			log.Fatalf("failed to wait for caches to sync, exiting")
+		}
+		// only wait for cluster scoped informers to sync if we are running in cluster-wide mode
+		if c.namespace == metav1.NamespaceAll {
+			if ok := cache.WaitForCacheSync(ctx.Done(), c.clusterAnalysisTemplateSynced); !ok {
+				log.Fatalf("failed to wait for cluster-scoped caches to sync, exiting")
+			}
+		}
+
+		go wait.Until(func() { c.wg.Add(1); c.rolloutController.Run(ctx, rolloutThreadiness); c.wg.Done() }, time.Second, ctx.Done())
+		go wait.Until(func() { c.wg.Add(1); c.serviceController.Run(ctx, serviceThreadiness); c.wg.Done() }, time.Second, ctx.Done())
+		go wait.Until(func() { c.wg.Add(1); c.ingressController.Run(ctx, ingressThreadiness); c.wg.Done() }, time.Second, ctx.Done())
+		go wait.Until(func() { c.wg.Add(1); c.experimentController.Run(ctx, experimentThreadiness); c.wg.Done() }, time.Second, ctx.Done())
+		go wait.Until(func() { c.wg.Add(1); c.analysisController.Run(ctx, analysisThreadiness); c.wg.Done() }, time.Second, ctx.Done())
+		go wait.Until(func() { c.wg.Add(1); c.notificationsController.Run(rolloutThreadiness, ctx.Done()); c.wg.Done() }, time.Second, ctx.Done())
+
+	}
+	log.Info("Started controller")
+}

controller/controller_test.go

@@ -0,0 +1,337 @@
+package controller
+
+import (
+	"context"
+	"fmt"
+	"sync"
+	"testing"
+	"time"
+
+	notificationapi "github.com/argoproj/notifications-engine/pkg/api"
+	notificationcontroller "github.com/argoproj/notifications-engine/pkg/controller"
+	smifake "github.com/servicemeshinterface/smi-sdk-go/pkg/gen/client/split/clientset/versioned/fake"
+	"github.com/stretchr/testify/assert"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	discoveryfake "k8s.io/client-go/discovery/fake"
+	"k8s.io/client-go/dynamic/dynamicinformer"
+	dynamicfake "k8s.io/client-go/dynamic/fake"
+	kubeinformers "k8s.io/client-go/informers"
+	k8sfake "k8s.io/client-go/kubernetes/fake"
+
+	"k8s.io/client-go/util/workqueue"
+
+	"github.com/argoproj/argo-rollouts/analysis"
+	"github.com/argoproj/argo-rollouts/controller/metrics"
+	experimentsController "github.com/argoproj/argo-rollouts/experiments"
+	"github.com/argoproj/argo-rollouts/ingress"
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
+	"github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake"
+	informers "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions"
+	rolloutController "github.com/argoproj/argo-rollouts/rollout"
+	"github.com/argoproj/argo-rollouts/service"
+	ingressutil "github.com/argoproj/argo-rollouts/utils/ingress"
+	istioutil "github.com/argoproj/argo-rollouts/utils/istio"
+	"github.com/argoproj/argo-rollouts/utils/queue"
+	"github.com/argoproj/argo-rollouts/utils/record"
+)
+
+var (
+	alwaysReady        = func() bool { return true }
+	noResyncPeriodFunc = func() time.Duration { return 0 }
+)
+
+type fixture struct {
+	t *testing.T
+
+	client     *fake.Clientset
+	kubeclient *k8sfake.Clientset
+}
+
+func newFixture(t *testing.T) *fixture {
+	f := &fixture{}
+	f.t = t
+
+	f.client = fake.NewSimpleClientset()
+	f.kubeclient = k8sfake.NewSimpleClientset()
+	return f
+}
+
+func (f *fixture) newManager(t *testing.T) *Manager {
+	rolloutWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "Rollouts")
+	serviceWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "Services")
+	ingressWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "Ingresses")
+	experimentWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "Experiments")
+	analysisRunWorkqueue := workqueue.NewNamedRateLimitingQueue(queue.DefaultArgoRolloutsRateLimiter(), "AnalysisRuns")
+
+	cm := &Manager{
+		wg:                                   &sync.WaitGroup{},
+		healthzServer:                        NewHealthzServer(fmt.Sprintf(listenAddr, 8080)),
+		rolloutSynced:                        alwaysReady,
+		experimentSynced:                     alwaysReady,
+		analysisRunSynced:                    alwaysReady,
+		analysisTemplateSynced:               alwaysReady,
+		clusterAnalysisTemplateSynced:        alwaysReady,
+		serviceSynced:                        alwaysReady,
+		ingressSynced:                        alwaysReady,
+		jobSynced:                            alwaysReady,
+		replicasSetSynced:                    alwaysReady,
+		configMapSynced:                      alwaysReady,
+		secretSynced:                         alwaysReady,
+		rolloutWorkqueue:                     rolloutWorkqueue,
+		serviceWorkqueue:                     serviceWorkqueue,
+		ingressWorkqueue:                     ingressWorkqueue,
+		experimentWorkqueue:                  experimentWorkqueue,
+		analysisRunWorkqueue:                 analysisRunWorkqueue,
+		kubeClientSet:                        f.kubeclient,
+		namespace:                            "",
+		namespaced:                           false,
+		notificationSecretInformerFactory:    kubeinformers.NewSharedInformerFactoryWithOptions(f.kubeclient, noResyncPeriodFunc()),
+		notificationConfigMapInformerFactory: kubeinformers.NewSharedInformerFactoryWithOptions(f.kubeclient, noResyncPeriodFunc()),
+	}
+
+	metricsAddr := fmt.Sprintf(listenAddr, 8090)
+	cm.metricsServer = metrics.NewMetricsServer(metrics.ServerConfig{
+		Addr:               metricsAddr,
+		K8SRequestProvider: &metrics.K8sRequestsCountProvider{},
+	})
+
+	i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc())
+	k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc())
+	tgbGVR := schema.GroupVersionResource{
+		Group:    "elbv2.k8s.aws",
+		Version:  "v1beta1",
+		Resource: "targetgroupbindings",
+	}
+	vsvcGVR := istioutil.GetIstioVirtualServiceGVR()
+	destGVR := istioutil.GetIstioDestinationRuleGVR()
+	scheme := runtime.NewScheme()
+	listMapping := map[schema.GroupVersionResource]string{
+		tgbGVR:  "TargetGroupBindingList",
+		vsvcGVR: vsvcGVR.Resource + "List",
+		destGVR: destGVR.Resource + "List",
+	}
+
+	dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, listMapping)
+	dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0)
+	istioVirtualServiceInformer := dynamicInformerFactory.ForResource(istioutil.GetIstioVirtualServiceGVR()).Informer()
+	istioDestinationRuleInformer := dynamicInformerFactory.ForResource(istioutil.GetIstioDestinationRuleGVR()).Informer()
+
+	cm.dynamicInformerFactory = dynamicInformerFactory
+	cm.clusterDynamicInformerFactory = dynamicInformerFactory
+	cm.kubeInformerFactory = k8sI
+	cm.jobInformerFactory = k8sI
+	cm.istioPrimaryDynamicClient = dynamicClient
+	cm.istioDynamicInformerFactory = dynamicInformerFactory
+
+	mode, err := ingressutil.DetermineIngressMode("extensions/v1beta1", &discoveryfake.FakeDiscovery{})
+	assert.NoError(t, err)
+	ingressWrapper, err := ingressutil.NewIngressWrapper(mode, f.kubeclient, k8sI)
+	assert.NoError(t, err)
+
+	cm.rolloutController = rolloutController.NewController(rolloutController.ControllerConfig{
+		Namespace:                       metav1.NamespaceAll,
+		KubeClientSet:                   f.kubeclient,
+		ArgoProjClientset:               f.client,
+		DynamicClientSet:                dynamicClient,
+		ExperimentInformer:              i.Argoproj().V1alpha1().Experiments(),
+		AnalysisRunInformer:             i.Argoproj().V1alpha1().AnalysisRuns(),
+		AnalysisTemplateInformer:        i.Argoproj().V1alpha1().AnalysisTemplates(),
+		ClusterAnalysisTemplateInformer: i.Argoproj().V1alpha1().ClusterAnalysisTemplates(),
+		ReplicaSetInformer:              k8sI.Apps().V1().ReplicaSets(),
+		ServicesInformer:                k8sI.Core().V1().Services(),
+		IngressWrapper:                  ingressWrapper,
+		RolloutsInformer:                i.Argoproj().V1alpha1().Rollouts(),
+		IstioPrimaryDynamicClient:       dynamicClient,
+		IstioVirtualServiceInformer:     istioVirtualServiceInformer,
+		IstioDestinationRuleInformer:    istioDestinationRuleInformer,
+		ResyncPeriod:                    noResyncPeriodFunc(),
+		RolloutWorkQueue:                rolloutWorkqueue,
+		ServiceWorkQueue:                serviceWorkqueue,
+		IngressWorkQueue:                ingressWorkqueue,
+		MetricsServer:                   cm.metricsServer,
+		Recorder:                        record.NewFakeEventRecorder(),
+	})
+
+	cm.analysisController = analysis.NewController(analysis.ControllerConfig{
+		KubeClientSet:        f.kubeclient,
+		ArgoProjClientset:    f.client,
+		AnalysisRunInformer:  i.Argoproj().V1alpha1().AnalysisRuns(),
+		JobInformer:          k8sI.Batch().V1().Jobs(),
+		ResyncPeriod:         noResyncPeriodFunc(),
+		AnalysisRunWorkQueue: analysisRunWorkqueue,
+		MetricsServer:        cm.metricsServer,
+		Recorder:             record.NewFakeEventRecorder(),
+	})
+
+	cm.ingressController = ingress.NewController(ingress.ControllerConfig{
+		Client:           f.kubeclient,
+		IngressWrap:      ingressWrapper,
+		IngressWorkQueue: ingressWorkqueue,
+
+		RolloutsInformer: i.Argoproj().V1alpha1().Rollouts(),
+		RolloutWorkQueue: rolloutWorkqueue,
+		ALBClasses:       []string{"alb"},
+		NGINXClasses:     []string{"nginx"},
+		MetricsServer:    cm.metricsServer,
+	})
+
+	cm.serviceController = service.NewController(service.ControllerConfig{
+		Kubeclientset:     f.kubeclient,
+		Argoprojclientset: f.client,
+		RolloutsInformer:  i.Argoproj().V1alpha1().Rollouts(),
+		ServicesInformer:  k8sI.Core().V1().Services(),
+		RolloutWorkqueue:  rolloutWorkqueue,
+		ServiceWorkqueue:  serviceWorkqueue,
+		ResyncPeriod:      0,
+		MetricsServer:     cm.metricsServer,
+	})
+
+	cm.experimentController = experimentsController.NewController(experimentsController.ControllerConfig{
+		KubeClientSet:                   f.kubeclient,
+		ArgoProjClientset:               f.client,
+		ReplicaSetInformer:              k8sI.Apps().V1().ReplicaSets(),
+		ExperimentsInformer:             i.Argoproj().V1alpha1().Experiments(),
+		AnalysisRunInformer:             i.Argoproj().V1alpha1().AnalysisRuns(),
+		AnalysisTemplateInformer:        i.Argoproj().V1alpha1().AnalysisTemplates(),
+		ClusterAnalysisTemplateInformer: i.Argoproj().V1alpha1().ClusterAnalysisTemplates(),
+		ServiceInformer:                 k8sI.Core().V1().Services(),
+		ResyncPeriod:                    noResyncPeriodFunc(),
+		RolloutWorkQueue:                rolloutWorkqueue,
+		ExperimentWorkQueue:             experimentWorkqueue,
+		MetricsServer:                   cm.metricsServer,
+		Recorder:                        record.NewFakeEventRecorder(),
+	})
+
+	apiFactory := notificationapi.NewFactory(record.NewAPIFactorySettings(i.Argoproj().V1alpha1().AnalysisRuns()), "default", k8sI.Core().V1().Secrets().Informer(), k8sI.Core().V1().ConfigMaps().Informer())
+	// rolloutsInformer := rolloutinformers.NewRolloutInformer(f.client, "", time.Minute, cache.Indexers{})
+	cm.notificationsController = notificationcontroller.NewController(dynamicClient.Resource(v1alpha1.RolloutGVR), i.Argoproj().V1alpha1().Rollouts().Informer(), apiFactory,
+		notificationcontroller.WithToUnstructured(func(obj metav1.Object) (*unstructured.Unstructured, error) {
+			return nil, nil
+		}),
+	)
+
+	return cm
+}
+
+func TestNewManager(t *testing.T) {
+	f := newFixture(t)
+
+	i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc())
+	k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc())
+
+	scheme := runtime.NewScheme()
+	listMapping := map[schema.GroupVersionResource]string{}
+	dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, listMapping)
+	dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0)
+	istioVirtualServiceInformer := dynamicInformerFactory.ForResource(istioutil.GetIstioVirtualServiceGVR()).Informer()
+	istioDestinationRuleInformer := dynamicInformerFactory.ForResource(istioutil.GetIstioDestinationRuleGVR()).Informer()
+
+	mode, err := ingressutil.DetermineIngressMode("extensions/v1beta1", &discoveryfake.FakeDiscovery{})
+	assert.NoError(t, err)
+	ingressWrapper, err := ingressutil.NewIngressWrapper(mode, f.kubeclient, k8sI)
+	assert.NoError(t, err)
+
+	k8sRequestProvider := &metrics.K8sRequestsCountProvider{}
+	cm := NewManager(
+		"default",
+		f.kubeclient,
+		f.client,
+		dynamicClient,
+		smifake.NewSimpleClientset(),
+		&discoveryfake.FakeDiscovery{},
+		k8sI.Apps().V1().ReplicaSets(),
+		k8sI.Core().V1().Services(),
+		ingressWrapper,
+		k8sI.Batch().V1().Jobs(),
+		i.Argoproj().V1alpha1().Rollouts(),
+		i.Argoproj().V1alpha1().Experiments(),
+		i.Argoproj().V1alpha1().AnalysisRuns(),
+		i.Argoproj().V1alpha1().AnalysisTemplates(),
+		i.Argoproj().V1alpha1().ClusterAnalysisTemplates(),
+		dynamicClient,
+		istioVirtualServiceInformer,
+		istioDestinationRuleInformer,
+		k8sI,
+		k8sI,
+		noResyncPeriodFunc(),
+		"test",
+		8090,
+		8080,
+		k8sRequestProvider,
+		nil,
+		nil,
+		dynamicInformerFactory,
+		nil,
+		nil,
+		false,
+		nil,
+		nil,
+		rolloutController.DefaultEphemeralMetadataThreads,
+	)
+
+	assert.NotNil(t, cm)
+}
+
+func TestNewAnalysisManager(t *testing.T) {
+	f := newFixture(t)
+
+	i := informers.NewSharedInformerFactory(f.client, noResyncPeriodFunc())
+	k8sI := kubeinformers.NewSharedInformerFactory(f.kubeclient, noResyncPeriodFunc())
+
+	scheme := runtime.NewScheme()
+	listMapping := map[schema.GroupVersionResource]string{}
+	dynamicClient := dynamicfake.NewSimpleDynamicClientWithCustomListKinds(scheme, listMapping)
+	dynamicInformerFactory := dynamicinformer.NewDynamicSharedInformerFactory(dynamicClient, 0)
+
+	k8sRequestProvider := &metrics.K8sRequestsCountProvider{}
+	cm := NewAnalysisManager(
+		"default",
+		f.kubeclient,
+		f.client,
+		k8sI.Batch().V1().Jobs(),
+		i.Argoproj().V1alpha1().AnalysisRuns(),
+		i.Argoproj().V1alpha1().AnalysisTemplates(),
+		i.Argoproj().V1alpha1().ClusterAnalysisTemplates(),
+		noResyncPeriodFunc(),
+		8090,
+		8080,
+		k8sRequestProvider,
+		nil,
+		dynamicInformerFactory,
+		false,
+		nil,
+		nil,
+	)
+
+	assert.NotNil(t, cm)
+}
+
+func TestPrimaryController(t *testing.T) {
+	f := newFixture(t)
+
+	cm := f.newManager(t)
+	electOpts := NewLeaderElectionOptions()
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(5 * time.Second)
+		cancel()
+	}()
+	cm.Run(ctx, 1, 1, 1, 1, 1, electOpts)
+}
+
+func TestPrimaryControllerSingleInstanceWithShutdown(t *testing.T) {
+	f := newFixture(t)
+
+	cm := f.newManager(t)
+	electOpts := NewLeaderElectionOptions()
+	electOpts.LeaderElect = false
+	ctx, cancel := context.WithCancel(context.Background())
+	go func() {
+		time.Sleep(5 * time.Second)
+		cancel()
+	}()
+	cm.Run(ctx, 1, 1, 1, 1, 1, electOpts)
+}

controller/healthz.go

@@ -0,0 +1,31 @@
+package controller
+
+import (
+	"fmt"
+	"net/http"
+)
+
+const (
+	// HealthzPath is the endpoint to probe if controller is running
+	HealthzPath = "/healthz"
+)
+
+type healthzHandler struct{}
+
+func (h *healthzHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
+	w.Header().Set("Content-Type", "text/plain; charset=utf-8")
+	w.Header().Set("X-Content-Type-Options", "nosniff")
+	w.WriteHeader(http.StatusOK)
+
+	fmt.Fprintf(w, "ok")
+}
+
+func NewHealthzServer(addr string) *http.Server {
+	mux := http.NewServeMux()
+	mux.Handle(HealthzPath, &healthzHandler{})
+
+	return &http.Server{
+		Addr:    addr,
+		Handler: mux,
+	}
+}

controller/healthz_test.go

@@ -0,0 +1,37 @@
+package controller
+
+import (
+	"fmt"
+	"log"
+	"net/http"
+	"net/http/httptest"
+	"strings"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func TestHealthzServer(t *testing.T) {
+	expectedResponse := `ok`
+
+	addr := fmt.Sprintf("0.0.0.0:%d", DefaultHealthzPort)
+	healthzServ := NewHealthzServer(addr)
+
+	t.Helper()
+	req, err := http.NewRequest("GET", "/healthz", nil)
+	assert.NoError(t, err)
+	rr := httptest.NewRecorder()
+	healthzServ.Handler.ServeHTTP(rr, req)
+	assert.Equal(t, rr.Code, http.StatusOK)
+	body := rr.Body.String()
+	log.Println(body)
+	for _, line := range strings.Split(expectedResponse, "\n") {
+		assert.Contains(t, body, line)
+	}
+
+	req, err = http.NewRequest("GET", "/extraneousPath", nil)
+	assert.NoError(t, err)
+	rr = httptest.NewRecorder()
+	healthzServ.Handler.ServeHTTP(rr, req)
+	assert.Equal(t, rr.Code, http.StatusNotFound)
+}

controller/metrics/analysis.go

@@ -0,0 +1,114 @@
+package metrics
+
+import (
+	"fmt"
+
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/labels"
+
+	"github.com/argoproj/argo-rollouts/metricproviders"
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
+	rolloutlister "github.com/argoproj/argo-rollouts/pkg/client/listers/rollouts/v1alpha1"
+	analysisutil "github.com/argoproj/argo-rollouts/utils/analysis"
+)
+
+type analysisRunCollector struct {
+	runs             rolloutlister.AnalysisRunLister
+	templates        rolloutlister.AnalysisTemplateLister
+	clusterTemplates rolloutlister.ClusterAnalysisTemplateLister
+}
+
+// NewAnalysisRunCollector returns a prometheus collector for AnalysisRun metrics
+func NewAnalysisRunCollector(
+	analysisRunLister rolloutlister.AnalysisRunLister,
+	analysisTemplateLister rolloutlister.AnalysisTemplateLister,
+	clusterAnalysisTemplateLister rolloutlister.ClusterAnalysisTemplateLister,
+) prometheus.Collector {
+	return &analysisRunCollector{
+		runs:             analysisRunLister,
+		templates:        analysisTemplateLister,
+		clusterTemplates: clusterAnalysisTemplateLister,
+	}
+}
+
+// Describe implements the prometheus.Collector interface
+func (c *analysisRunCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- MetricAnalysisRunInfo
+}
+
+// Collect implements the prometheus.Collector interface
+func (c *analysisRunCollector) Collect(ch chan<- prometheus.Metric) {
+	analysisRuns, err := c.runs.List(labels.NewSelector())
+	if err != nil {
+		log.Warnf("Failed to collect analysis runs: %v", err)
+	} else {
+		for _, ar := range analysisRuns {
+			collectAnalysisRuns(ch, ar)
+		}
+	}
+	analysisTemplates, err := c.templates.List(labels.NewSelector())
+	if err != nil {
+		log.Warnf("Failed to collect analysis templates: %v", err)
+	} else {
+		for _, at := range analysisTemplates {
+			collectAnalysisTemplate(ch, at.Namespace, at.Name, &at.Spec)
+		}
+	}
+	clusterAnalysisTemplates, err := c.clusterTemplates.List(labels.NewSelector())
+	if err != nil {
+		log.Warnf("Failed to collect cluster analysis templates: %v", err)
+	} else {
+		for _, at := range clusterAnalysisTemplates {
+			collectAnalysisTemplate(ch, at.Namespace, at.Name, &at.Spec)
+		}
+	}
+}
+
+func collectAnalysisRuns(ch chan<- prometheus.Metric, ar *v1alpha1.AnalysisRun) {
+	addGauge := func(desc *prometheus.Desc, v float64, lv ...string) {
+		lv = append([]string{ar.Namespace, ar.Name}, lv...)
+		ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, v, lv...)
+	}
+	calculatedPhase := ar.Status.Phase
+
+	addGauge(MetricAnalysisRunInfo, 1, string(calculatedPhase))
+
+	// DEPRECATED
+	addGauge(MetricAnalysisRunPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhasePending || calculatedPhase == ""), string(v1alpha1.AnalysisPhasePending))
+	addGauge(MetricAnalysisRunPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseError), string(v1alpha1.AnalysisPhaseError))
+	addGauge(MetricAnalysisRunPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseFailed), string(v1alpha1.AnalysisPhaseFailed))
+	addGauge(MetricAnalysisRunPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseSuccessful), string(v1alpha1.AnalysisPhaseSuccessful))
+	addGauge(MetricAnalysisRunPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseRunning), string(v1alpha1.AnalysisPhaseRunning))
+	addGauge(MetricAnalysisRunPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseInconclusive), string(v1alpha1.AnalysisPhaseInconclusive))
+
+	dryRunMetricsMap, _ := analysisutil.GetDryRunMetrics(ar.Spec.DryRun, ar.Spec.Metrics)
+	for _, metric := range ar.Spec.Metrics {
+		metricType := metricproviders.Type(metric)
+		metricResult := analysisutil.GetResult(ar, metric.Name)
+		addGauge(MetricAnalysisRunMetricType, 1, metric.Name, metricType)
+		calculatedPhase := v1alpha1.AnalysisPhase("")
+		if metricResult != nil {
+			calculatedPhase = metricResult.Phase
+		}
+		addGauge(MetricAnalysisRunMetricPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhasePending || calculatedPhase == ""), metric.Name, metricType, fmt.Sprint(dryRunMetricsMap[metric.Name]), string(v1alpha1.AnalysisPhasePending))
+		addGauge(MetricAnalysisRunMetricPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseError), metric.Name, metricType, fmt.Sprint(dryRunMetricsMap[metric.Name]), string(v1alpha1.AnalysisPhaseError))
+		addGauge(MetricAnalysisRunMetricPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseFailed), metric.Name, metricType, fmt.Sprint(dryRunMetricsMap[metric.Name]), string(v1alpha1.AnalysisPhaseFailed))
+		addGauge(MetricAnalysisRunMetricPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseSuccessful), metric.Name, metricType, fmt.Sprint(dryRunMetricsMap[metric.Name]), string(v1alpha1.AnalysisPhaseSuccessful))
+		addGauge(MetricAnalysisRunMetricPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseRunning), metric.Name, metricType, fmt.Sprint(dryRunMetricsMap[metric.Name]), string(v1alpha1.AnalysisPhaseRunning))
+		addGauge(MetricAnalysisRunMetricPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseInconclusive), metric.Name, metricType, fmt.Sprint(dryRunMetricsMap[metric.Name]), string(v1alpha1.AnalysisPhaseInconclusive))
+	}
+}
+
+func collectAnalysisTemplate(ch chan<- prometheus.Metric, namespace, name string, at *v1alpha1.AnalysisTemplateSpec) {
+	addGauge := func(desc *prometheus.Desc, v float64, lv ...string) {
+		lv = append([]string{namespace, name}, lv...)
+		ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, v, lv...)
+	}
+	addGauge(MetricAnalysisTemplateInfo, 1)
+
+	for _, metric := range at.Metrics {
+		metricType := metricproviders.Type(metric)
+		addGauge(MetricAnalysisTemplateMetricInfo, 1, metricType, metric.Name)
+	}
+}

controller/metrics/analysis_test.go

@@ -0,0 +1,212 @@
+package metrics
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
+)
+
+const (
+	fakeAnalysisRun = `
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisRun
+metadata:
+  creationTimestamp: "2020-03-16T20:01:13Z"
+  name: http-benchmark-test-tr8rn
+  namespace: jesse-test
+spec:
+  metrics:
+  - name: webmetric
+    provider:
+      web:
+        jsonPath: .
+        url: https://www.google.com
+    successCondition: "true"
+status:
+  metricResults:
+  - consecutiveError: 5
+    error: 5
+    measurements:
+    - finishedAt: "2020-03-16T20:02:15Z"
+      message: 'Could not parse JSON body: invalid character ''<'' looking for beginning
+        of value'
+      phase: Error
+      startedAt: "2020-03-16T20:02:14Z"
+    name: webmetric
+    phase: Error
+  phase: Error
+  startedAt: "2020-03-16T20:02:15Z"
+`
+
+	fakeAnalysisTemplate = `
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  creationTimestamp: "2020-03-16T20:01:13Z"
+  name: http-benchmark-test
+  namespace: jesse-test
+spec:
+  metrics:
+  - name: web-metric-1
+    provider:
+      web:
+        jsonPath: .
+        url: https://www.google.com
+    successCondition: "true"
+  - name: web-metric-2
+    dryRun: true
+    provider:
+      web:
+        jsonPath: .
+        url: https://www.msn.com
+    successCondition: "false"
+`
+
+	fakeClusterAnalysisTemplate = `
+apiVersion: argoproj.io/v1alpha1
+kind: ClusterAnalysisTemplate
+metadata:
+  creationTimestamp: "2020-03-16T20:01:13Z"
+  name: http-benchmark-cluster-test
+spec:
+  metrics:
+  - name: web-metric-1
+    provider:
+      web:
+        jsonPath: .
+        url: https://www.google.com
+    successCondition: "true"
+  - name: web-metric-2
+    dryRun: true
+    provider:
+      web:
+        jsonPath: .
+        url: https://www.msn.com
+    successCondition: "false"
+`
+)
+const expectedAnalysisRunResponse = `# HELP analysis_run_info Information about analysis run.
+# TYPE analysis_run_info gauge
+analysis_run_info{name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Error"} 1
+# HELP analysis_run_metric_phase Information on the duration of a specific metric in the Analysis Run
+# TYPE analysis_run_metric_phase gauge
+analysis_run_metric_phase{dry_run="false",metric="webmetric",name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Error",type="Web"} 1
+analysis_run_metric_phase{dry_run="false",metric="webmetric",name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Failed",type="Web"} 0
+analysis_run_metric_phase{dry_run="false",metric="webmetric",name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Inconclusive",type="Web"} 0
+analysis_run_metric_phase{dry_run="false",metric="webmetric",name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Pending",type="Web"} 0
+analysis_run_metric_phase{dry_run="false",metric="webmetric",name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Running",type="Web"} 0
+analysis_run_metric_phase{dry_run="false",metric="webmetric",name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Successful",type="Web"} 0
+# HELP analysis_run_metric_type Information on the type of a specific metric in the Analysis Runs
+# TYPE analysis_run_metric_type gauge
+analysis_run_metric_type{metric="webmetric",name="http-benchmark-test-tr8rn",namespace="jesse-test",type="Web"} 1
+# HELP analysis_run_phase Information on the state of the Analysis Run (DEPRECATED - use analysis_run_info)
+# TYPE analysis_run_phase gauge
+analysis_run_phase{name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Error"} 1
+analysis_run_phase{name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Failed"} 0
+analysis_run_phase{name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Inconclusive"} 0
+analysis_run_phase{name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Pending"} 0
+analysis_run_phase{name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Running"} 0
+analysis_run_phase{name="http-benchmark-test-tr8rn",namespace="jesse-test",phase="Successful"} 0
+`
+
+func newFakeAnalysisRun(fakeAnalysisRun string) *v1alpha1.AnalysisRun {
+	var ar v1alpha1.AnalysisRun
+	err := yaml.Unmarshal([]byte(fakeAnalysisRun), &ar)
+	if err != nil {
+		panic(err)
+	}
+	return &ar
+}
+
+func newFakeAnalysisTemplate(yamlStr string) *v1alpha1.AnalysisTemplate {
+	var at v1alpha1.AnalysisTemplate
+	err := yaml.Unmarshal([]byte(yamlStr), &at)
+	if err != nil {
+		panic(err)
+	}
+	return &at
+}
+
+func newFakeClusterAnalysisTemplate(yamlStr string) *v1alpha1.ClusterAnalysisTemplate {
+	var at v1alpha1.ClusterAnalysisTemplate
+	err := yaml.Unmarshal([]byte(yamlStr), &at)
+	if err != nil {
+		panic(err)
+	}
+	return &at
+}
+
+func TestCollectAnalysisRuns(t *testing.T) {
+	combinations := []testCombination{
+		{
+			resource:         fakeAnalysisRun,
+			expectedResponse: expectedAnalysisRunResponse,
+		},
+	}
+
+	for _, combination := range combinations {
+		testAnalysisRunDescribe(t, combination.resource, combination.expectedResponse)
+	}
+}
+
+func testAnalysisRunDescribe(t *testing.T, fakeAnalysisRun string, expectedResponse string) {
+	registry := prometheus.NewRegistry()
+	serverCfg := newFakeServerConfig(newFakeAnalysisRun(fakeAnalysisRun))
+	registry.MustRegister(NewAnalysisRunCollector(serverCfg.AnalysisRunLister, serverCfg.AnalysisTemplateLister, serverCfg.ClusterAnalysisTemplateLister))
+	mux := http.NewServeMux()
+	mux.Handle(MetricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+	testHttpResponse(t, mux, expectedResponse, assert.Contains)
+}
+
+func TestIncAnalysisRunReconcile(t *testing.T) {
+	expectedResponse := `# HELP analysis_run_reconcile Analysis Run reconciliation performance.
+# TYPE analysis_run_reconcile histogram
+analysis_run_reconcile_bucket{name="ar-test",namespace="ar-namespace",le="0.01"} 1
+analysis_run_reconcile_bucket{name="ar-test",namespace="ar-namespace",le="0.15"} 1
+analysis_run_reconcile_bucket{name="ar-test",namespace="ar-namespace",le="0.25"} 1
+analysis_run_reconcile_bucket{name="ar-test",namespace="ar-namespace",le="0.5"} 1
+analysis_run_reconcile_bucket{name="ar-test",namespace="ar-namespace",le="1"} 1
+analysis_run_reconcile_bucket{name="ar-test",namespace="ar-namespace",le="+Inf"} 1
+analysis_run_reconcile_sum{name="ar-test",namespace="ar-namespace"} 0.001
+analysis_run_reconcile_count{name="ar-test",namespace="ar-namespace"} 1`
+	metricsServ := NewMetricsServer(newFakeServerConfig())
+	ar := &v1alpha1.AnalysisRun{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ar-test",
+			Namespace: "ar-namespace",
+		},
+	}
+	metricsServ.IncAnalysisRunReconcile(ar, time.Millisecond)
+	testHttpResponse(t, metricsServ.Handler, expectedResponse, assert.Contains)
+}
+
+func TestAnalysisTemplateDescribe(t *testing.T) {
+	expectedResponse := `# HELP analysis_template_info Information about analysis templates.
+# TYPE analysis_template_info gauge
+analysis_template_info{name="http-benchmark-cluster-test",namespace=""} 1
+analysis_template_info{name="http-benchmark-test",namespace="jesse-test"} 1
+# HELP analysis_template_metric_info Information on metrics in analysis templates.
+# TYPE analysis_template_metric_info gauge
+analysis_template_metric_info{metric="web-metric-1",name="http-benchmark-cluster-test",namespace="",type="Web"} 1
+analysis_template_metric_info{metric="web-metric-1",name="http-benchmark-test",namespace="jesse-test",type="Web"} 1
+analysis_template_metric_info{metric="web-metric-2",name="http-benchmark-cluster-test",namespace="",type="Web"} 1
+analysis_template_metric_info{metric="web-metric-2",name="http-benchmark-test",namespace="jesse-test",type="Web"} 1
+`
+	registry := prometheus.NewRegistry()
+	at := newFakeAnalysisTemplate(fakeAnalysisTemplate)
+	cat := newFakeClusterAnalysisTemplate(fakeClusterAnalysisTemplate)
+	serverCfg := newFakeServerConfig(at, cat)
+	registry.MustRegister(NewAnalysisRunCollector(serverCfg.AnalysisRunLister, serverCfg.AnalysisTemplateLister, serverCfg.ClusterAnalysisTemplateLister))
+	mux := http.NewServeMux()
+	mux.Handle(MetricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+	testHttpResponse(t, mux, expectedResponse, assert.Contains)
+}

controller/metrics/client.go

@@ -0,0 +1,42 @@
+package metrics
+
+import (
+	"strconv"
+
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/argoproj/pkg/kubeclientmetrics"
+)
+
+const (
+	clientsetMetricsNamespace = "controller_clientset"
+)
+
+type K8sRequestsCountProvider struct {
+	k8sRequestsCount *prometheus.CounterVec
+}
+
+func (f *K8sRequestsCountProvider) MustRegister(registerer prometheus.Registerer) {
+	f.k8sRequestsCount = MetricK8sRequestTotal
+	registerer.MustRegister(f.k8sRequestsCount)
+}
+
+// IncKubernetesRequest increments the kubernetes client counter
+func (m *K8sRequestsCountProvider) IncKubernetesRequest(resourceInfo kubeclientmetrics.ResourceInfo) error {
+	name := resourceInfo.Name
+	namespace := resourceInfo.Namespace
+	kind := resourceInfo.Kind
+	statusCode := strconv.Itoa(resourceInfo.StatusCode)
+	if resourceInfo.Verb == kubeclientmetrics.List || kind == "events" || kind == "replicasets" {
+		name = "N/A"
+	}
+	if resourceInfo.Verb == kubeclientmetrics.Unknown {
+		namespace = "Unknown"
+		name = "Unknown"
+		kind = "Unknown"
+	}
+	if m.k8sRequestsCount != nil {
+		m.k8sRequestsCount.WithLabelValues(kind, namespace, name, string(resourceInfo.Verb), statusCode).Inc()
+	}
+	return nil
+}

controller/metrics/client_test.go

@@ -0,0 +1,30 @@
+package metrics
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+
+	"github.com/argoproj/pkg/kubeclientmetrics"
+)
+
+const expectedKubernetesRequest = `# TYPE controller_clientset_k8s_request_total counter
+controller_clientset_k8s_request_total{kind="Unknown",name="Unknown",namespace="Unknown",status_code="200",verb="Unknown"} 1
+controller_clientset_k8s_request_total{kind="replicasets",name="N/A",namespace="default",status_code="200",verb="List"} 1`
+
+func TestIncKubernetesRequest(t *testing.T) {
+	config := newFakeServerConfig()
+	metricsServ := NewMetricsServer(config)
+	config.K8SRequestProvider.IncKubernetesRequest(kubeclientmetrics.ResourceInfo{
+		Kind:       "replicasets",
+		Namespace:  "default",
+		Name:       "test",
+		Verb:       kubeclientmetrics.List,
+		StatusCode: 200,
+	})
+	config.K8SRequestProvider.IncKubernetesRequest(kubeclientmetrics.ResourceInfo{
+		Verb:       kubeclientmetrics.Unknown,
+		StatusCode: 200,
+	})
+	testHttpResponse(t, metricsServ.Handler, expectedKubernetesRequest, assert.Contains)
+}

controller/metrics/experiment_test.go

@@ -0,0 +1,163 @@
+package metrics
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
+)
+
+const (
+	fakeExperiment = `
+apiVersion: argoproj.io/v1alpha1
+kind: Experiment
+metadata:
+  annotations:
+    kubectl.kubernetes.io/last-applied-configuration: |
+      {"apiVersion":"argoproj.io/v1alpha1","kind":"Experiment","metadata":{"annotations":{},"name":"experiment-with-analysis","namespace":"argo-rollouts"},"spec":{"analyses":[{"name":"pass","templateName":"pass"}],"duration":"30s","templates":[{"name":"purple","selector":{"matchLabels":{"app":"rollouts-demo"}},"template":{"metadata":{"labels":{"app":"rollouts-demo"}},"spec":{"containers":[{"image":"argoproj/rollouts-demo:purple","imagePullPolicy":"Always","name":"rollouts-demo"}]}}},{"name":"orange","selector":{"matchLabels":{"app":"rollouts-demo"}},"template":{"metadata":{"labels":{"app":"rollouts-demo"}},"spec":{"containers":[{"image":"argoproj/rollouts-demo:orange","imagePullPolicy":"Always","name":"rollouts-demo"}]}}}]}}
+  creationTimestamp: "2020-02-04T19:12:03Z"
+  generation: 16
+  name: experiment-with-analysis
+  namespace: argo-rollouts
+  resourceVersion: "81876006"
+  selfLink: /apis/argoproj.io/v1alpha1/namespaces/argo-rollouts/experiments/experiment-with-analysis
+  uid: 3a64d561-4782-11ea-b316-42010aa80065
+spec:
+  analyses:
+  - name: pass
+    templateName: pass
+  duration: 30s
+  templates:
+  - name: purple
+    selector:
+      matchLabels:
+        app: rollouts-demo
+    template:
+      metadata:
+        labels:
+          app: rollouts-demo
+      spec:
+        containers:
+        - image: argoproj/rollouts-demo:purple
+          imagePullPolicy: Always
+          name: rollouts-demo
+  - name: orange
+    selector:
+      matchLabels:
+        app: rollouts-demo
+    template:
+      metadata:
+        labels:
+          app: rollouts-demo
+      spec:
+        containers:
+        - image: argoproj/rollouts-demo:orange
+          imagePullPolicy: Always
+          name: rollouts-demo
+status:
+  analysisRuns:
+  - analysisRun: experiment-with-analysis-pass
+    name: pass
+    phase: Running
+  availableAt: "2020-02-04T19:12:07Z"
+  conditions:
+  - lastTransitionTime: "2020-02-04T19:12:37Z"
+    lastUpdateTime: "2020-02-04T19:12:37Z"
+    message: Experiment "experiment-with-analysis" has successfully ran and completed.
+    reason: ExperimentCompleted
+    status: "False"
+    type: Progressing
+  phase: Successful
+  templateStatuses:
+  - availableReplicas: 0
+    lastTransitionTime: "2020-02-04T19:12:37Z"
+    name: purple
+    readyReplicas: 0
+    replicas: 0
+    status: Successful
+    updatedReplicas: 0
+  - availableReplicas: 0
+    lastTransitionTime: "2020-02-04T19:12:37Z"
+    name: orange
+    readyReplicas: 0
+    replicas: 0
+    status: Successful
+    updatedReplicas: 0
+`
+)
+
+const expectedExperimentResponse = `
+# HELP experiment_info Information about Experiment.
+# TYPE experiment_info gauge
+experiment_info{name="experiment-with-analysis",namespace="argo-rollouts",phase="Successful"} 1
+# HELP experiment_phase Information on the state of the experiment (DEPRECATED - use experiment_info)
+# TYPE experiment_phase gauge
+experiment_phase{name="experiment-with-analysis",namespace="argo-rollouts",phase="Error"} 0
+experiment_phase{name="experiment-with-analysis",namespace="argo-rollouts",phase="Inconclusive"} 0
+experiment_phase{name="experiment-with-analysis",namespace="argo-rollouts",phase="Pending"} 0
+experiment_phase{name="experiment-with-analysis",namespace="argo-rollouts",phase="Running"} 0
+experiment_phase{name="experiment-with-analysis",namespace="argo-rollouts",phase="Successful"} 1
+`
+
+func newFakeExperiment(fakeExperiment string) *v1alpha1.Experiment {
+	var experiment v1alpha1.Experiment
+	err := yaml.Unmarshal([]byte(fakeExperiment), &experiment)
+	if err != nil {
+		panic(err)
+	}
+	return &experiment
+}
+
+func TestCollectExperiments(t *testing.T) {
+	combinations := []testCombination{
+		{
+			resource:         fakeExperiment,
+			expectedResponse: expectedExperimentResponse,
+		},
+	}
+
+	for _, combination := range combinations {
+		testExperimentDescribe(t, combination.resource, combination.expectedResponse)
+	}
+}
+
+func testExperimentDescribe(t *testing.T, fakeExperiment string, expectedResponse string) {
+	registry := prometheus.NewRegistry()
+	config := newFakeServerConfig(newFakeExperiment(fakeExperiment))
+	registry.MustRegister(NewExperimentCollector(config.ExperimentLister))
+	mux := http.NewServeMux()
+	mux.Handle(MetricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+	testHttpResponse(t, mux, expectedResponse, assert.Contains)
+}
+
+func TestIncExperimentReconcile(t *testing.T) {
+	expectedResponse := `# HELP experiment_reconcile Experiments reconciliation performance.
+# TYPE experiment_reconcile histogram
+experiment_reconcile_bucket{name="ex-test",namespace="ex-namespace",le="0.01"} 1
+experiment_reconcile_bucket{name="ex-test",namespace="ex-namespace",le="0.15"} 1
+experiment_reconcile_bucket{name="ex-test",namespace="ex-namespace",le="0.25"} 1
+experiment_reconcile_bucket{name="ex-test",namespace="ex-namespace",le="0.5"} 1
+experiment_reconcile_bucket{name="ex-test",namespace="ex-namespace",le="1"} 1
+experiment_reconcile_bucket{name="ex-test",namespace="ex-namespace",le="+Inf"} 1
+experiment_reconcile_sum{name="ex-test",namespace="ex-namespace"} 0.001
+experiment_reconcile_count{name="ex-test",namespace="ex-namespace"} 1`
+
+	metricsServ := NewMetricsServer(newFakeServerConfig())
+	ex := &v1alpha1.Experiment{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ex-test",
+			Namespace: "ex-namespace",
+		},
+	}
+	metricsServ.IncExperimentReconcile(ex, time.Millisecond)
+	testHttpResponse(t, metricsServ.Handler, expectedResponse, assert.Contains)
+}

controller/metrics/experiments.go

@@ -0,0 +1,55 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/labels"
+
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
+	rolloutlister "github.com/argoproj/argo-rollouts/pkg/client/listers/rollouts/v1alpha1"
+)
+
+type experimentCollector struct {
+	store rolloutlister.ExperimentLister
+}
+
+// NewExperimentCollector returns a prometheus collector for experiment metrics
+func NewExperimentCollector(experimentLister rolloutlister.ExperimentLister) prometheus.Collector {
+	return &experimentCollector{
+		store: experimentLister,
+	}
+}
+
+// Describe implements the prometheus.Collector interface
+func (c *experimentCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- MetricExperimentInfo
+}
+
+// Collect implements the prometheus.Collector interface
+func (c *experimentCollector) Collect(ch chan<- prometheus.Metric) {
+	experiments, err := c.store.List(labels.NewSelector())
+	if err != nil {
+		log.Warnf("Failed to collect experiments: %v", err)
+		return
+	}
+	for _, experiment := range experiments {
+		collectExperiments(ch, experiment)
+	}
+}
+
+func collectExperiments(ch chan<- prometheus.Metric, ex *v1alpha1.Experiment) {
+	addGauge := func(desc *prometheus.Desc, v float64, lv ...string) {
+		lv = append([]string{ex.Namespace, ex.Name}, lv...)
+		ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, v, lv...)
+	}
+	calculatedPhase := ex.Status.Phase
+
+	addGauge(MetricExperimentInfo, 1, string(calculatedPhase))
+
+	// DEPRECATED
+	addGauge(MetricExperimentPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhasePending || calculatedPhase == ""), string("Pending"))
+	addGauge(MetricExperimentPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseError), string("Error"))
+	addGauge(MetricExperimentPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseSuccessful), string("Successful"))
+	addGauge(MetricExperimentPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseRunning), string("Running"))
+	addGauge(MetricExperimentPhase, boolFloat64(calculatedPhase == v1alpha1.AnalysisPhaseInconclusive), string("Inconclusive"))
+}

controller/metrics/metrics.go

@@ -2,24 +2,38 @@ package metrics
 
 import (
 	"net/http"
+	"runtime"
+	"time"
+
+	"github.com/argoproj/argo-rollouts/utils/defaults"
 
 	"github.com/prometheus/client_golang/prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
-	log "github.com/sirupsen/logrus"
-	"k8s.io/apimachinery/pkg/labels"
+	registry "k8s.io/component-base/metrics/legacyregistry"
 
-	"time"
+	// make sure to register workqueue prometheus metrics
+	_ "k8s.io/component-base/metrics/prometheus/workqueue"
 
-	v1alpha1 "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
 	rolloutlister "github.com/argoproj/argo-rollouts/pkg/client/listers/rollouts/v1alpha1"
-	"github.com/argoproj/argo-rollouts/utils/conditions"
-	"github.com/argoproj/argo-rollouts/utils/defaults"
+	"github.com/argoproj/argo-rollouts/utils/log"
+	"github.com/argoproj/argo-rollouts/utils/version"
 )
 
 type MetricsServer struct {
 	*http.Server
-	reconcileHistogram *prometheus.HistogramVec
-	errorCounter       *prometheus.CounterVec
+	reconcileRolloutHistogram *prometheus.HistogramVec
+	errorRolloutCounter       *prometheus.CounterVec
+
+	reconcileExperimentHistogram *prometheus.HistogramVec
+	errorExperimentCounter       *prometheus.CounterVec
+
+	reconcileAnalysisRunHistogram *prometheus.HistogramVec
+	errorAnalysisRunCounter       *prometheus.CounterVec
+	successNotificationCounter    *prometheus.CounterVec
+	errorNotificationCounter      *prometheus.CounterVec
+	sendNotificationRunHistogram  *prometheus.HistogramVec
+	k8sRequestsCounter            *K8sRequestsCountProvider
 }
 
 const (
@@ -27,164 +41,158 @@ const (
 	MetricsPath = "/metrics"
 )
 
-// Follow Prometheus naming practices
-// https://prometheus.io/docs/practices/naming/
 var (
-	descRolloutDefaultLabels = []string{"namespace", "name"}
-
-	descRolloutWithStrategyLabels = append(descRolloutDefaultLabels, "strategy")
-
-	descRolloutReconcilePhaseLabels = append(descRolloutWithStrategyLabels, "phase")
-
-	descRolloutInfo = prometheus.NewDesc(
-		"rollout_info",
-		"Information about rollout.",
-		descRolloutWithStrategyLabels,
-		nil,
-	)
-
-	descRolloutCreated = prometheus.NewDesc(
-		"rollout_created_time",
-		"Creation time in unix timestamp for an rollout.",
-		descRolloutWithStrategyLabels,
-		nil,
-	)
-
-	descRolloutPhaseLabels = prometheus.NewDesc(
-		"rollout_phase",
-		"Information on the state of the rollout",
-		descRolloutReconcilePhaseLabels,
-		nil,
+	buildInfo = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "build_info",
+			Help: "A metric with a constant '1' value labeled by version from which Argo-Rollouts was built.",
+		},
+		[]string{"version", "goversion", "goarch", "commit"},
 	)
 )
 
-// RolloutPhase the phases of a reconcile can have
-type RolloutPhase string
-
-const (
-
-	// InvalidSpec means the rollout had an InvalidSpec during reconciliation
-	InvalidSpec RolloutPhase = "InvalidSpec"
-	// Completed means the rollout finished the reconciliation with no remaining work
-	Completed RolloutPhase = "Completed"
-	// Progressing means the rollout finished the reconciliation with remaining work
-	Progressing RolloutPhase = "Progressing"
-	// Paused means the rollout finished the reconciliation with a paused status
-	Paused RolloutPhase = "Paused"
-	// Timeout means the rollout finished the reconciliation with an timeout message
-	Timeout RolloutPhase = "Timeout"
-	// Error means the rollout finished the reconciliation with an error
-	Error RolloutPhase = "Error"
-)
+type ServerConfig struct {
+	Addr                          string
+	RolloutLister                 rolloutlister.RolloutLister
+	AnalysisRunLister             rolloutlister.AnalysisRunLister
+	AnalysisTemplateLister        rolloutlister.AnalysisTemplateLister
+	ClusterAnalysisTemplateLister rolloutlister.ClusterAnalysisTemplateLister
+	ExperimentLister              rolloutlister.ExperimentLister
+	K8SRequestProvider            *K8sRequestsCountProvider
+}
 
 // NewMetricsServer returns a new prometheus server which collects rollout metrics
-func NewMetricsServer(addr string, rolloutLister rolloutlister.RolloutLister) *MetricsServer {
+func NewMetricsServer(cfg ServerConfig) *MetricsServer {
 	mux := http.NewServeMux()
-	rolloutRegistry := NewRolloutRegistry(rolloutLister)
-	mux.Handle(MetricsPath, promhttp.HandlerFor(rolloutRegistry, promhttp.HandlerOpts{}))
 
-	reconcileHistogram := prometheus.NewHistogramVec(
-		prometheus.HistogramOpts{
-			Name:    "rollout_reconcile",
-			Help:    "Rollout reconciliation performance.",
-			Buckets: []float64{0.01, 0.15, .25, .5, 1},
-		},
-		append(descRolloutWithStrategyLabels),
-	)
+	reg := prometheus.NewRegistry()
 
-	rolloutRegistry.MustRegister(reconcileHistogram)
+	if cfg.RolloutLister != nil {
+		reg.MustRegister(NewRolloutCollector(cfg.RolloutLister))
+	}
+	if cfg.ExperimentLister != nil {
+		reg.MustRegister(NewExperimentCollector(cfg.ExperimentLister))
+	}
+	reg.MustRegister(NewAnalysisRunCollector(cfg.AnalysisRunLister, cfg.AnalysisTemplateLister, cfg.ClusterAnalysisTemplateLister))
+	cfg.K8SRequestProvider.MustRegister(reg)
+	reg.MustRegister(MetricRolloutReconcile)
+	reg.MustRegister(MetricRolloutReconcileError)
+	reg.MustRegister(MetricRolloutEventsTotal)
+	reg.MustRegister(MetricExperimentReconcile)
+	reg.MustRegister(MetricExperimentReconcileError)
+	reg.MustRegister(MetricAnalysisRunReconcile)
+	reg.MustRegister(MetricAnalysisRunReconcileError)
+	reg.MustRegister(MetricNotificationSuccessTotal)
+	reg.MustRegister(MetricNotificationFailedTotal)
+	reg.MustRegister(MetricNotificationSend)
+	reg.MustRegister(MetricVersionGauge)
+	reg.MustRegister(buildInfo)
 
-	errorCounter := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "rollout_reconcile_error",
-			Help: "Error occurring during the rollout",
-		},
-		append(descRolloutDefaultLabels),
-	)
-
-	rolloutRegistry.MustRegister(errorCounter)
+	recordBuildInfo()
 
+	mux.Handle(MetricsPath, promhttp.HandlerFor(prometheus.Gatherers{
+		// contains app controller specific metrics
+		reg,
+		// contains process, golang and controller workqueues metrics
+		registry.DefaultGatherer,
+	}, promhttp.HandlerOpts{}))
 	return &MetricsServer{
 		Server: &http.Server{
-			Addr:    addr,
+			Addr:    cfg.Addr,
 			Handler: mux,
 		},
-		reconcileHistogram: reconcileHistogram,
-		errorCounter:       errorCounter,
+		reconcileRolloutHistogram: MetricRolloutReconcile,
+		errorRolloutCounter:       MetricRolloutReconcileError,
+
+		reconcileExperimentHistogram: MetricExperimentReconcile,
+		errorExperimentCounter:       MetricExperimentReconcileError,
+
+		reconcileAnalysisRunHistogram: MetricAnalysisRunReconcile,
+		errorAnalysisRunCounter:       MetricAnalysisRunReconcileError,
+		successNotificationCounter:    MetricNotificationSuccessTotal,
+		errorNotificationCounter:      MetricNotificationFailedTotal,
+		sendNotificationRunHistogram:  MetricNotificationSend,
+
+		k8sRequestsCounter: cfg.K8SRequestProvider,
 	}
 }
 
-// IncReconcile increments the reconcile counter for an rollout
-func (m *MetricsServer) IncReconcile(rollout *v1alpha1.Rollout, duration time.Duration) {
-	m.reconcileHistogram.WithLabelValues(rollout.Namespace, rollout.Name, defaults.GetStrategyType(rollout)).Observe(duration.Seconds())
+// IncRolloutReconcile increments the reconcile counter for a Rollout
+func (m *MetricsServer) IncRolloutReconcile(rollout *v1alpha1.Rollout, duration time.Duration) {
+	m.reconcileRolloutHistogram.WithLabelValues(rollout.Namespace, rollout.Name).Observe(duration.Seconds())
+}
+
+// IncExperimentReconcile increments the reconcile counter for an Experiment
+func (m *MetricsServer) IncExperimentReconcile(ex *v1alpha1.Experiment, duration time.Duration) {
+	m.reconcileExperimentHistogram.WithLabelValues(ex.Namespace, ex.Name).Observe(duration.Seconds())
+}
+
+// IncAnalysisRunReconcile increments the reconcile counter for an AnalysisRun
+func (m *MetricsServer) IncAnalysisRunReconcile(ar *v1alpha1.AnalysisRun, duration time.Duration) {
+	m.reconcileAnalysisRunHistogram.WithLabelValues(ar.Namespace, ar.Name).Observe(duration.Seconds())
 }
 
 // IncError increments the reconcile counter for an rollout
-func (m *MetricsServer) IncError(namespace, name string) {
-	m.errorCounter.WithLabelValues(namespace, name).Inc()
+func (m *MetricsServer) IncError(namespace, name string, kind string) {
+	switch kind {
+	case log.RolloutKey:
+		m.errorRolloutCounter.WithLabelValues(namespace, name).Inc()
+	case log.AnalysisRunKey:
+		m.errorAnalysisRunCounter.WithLabelValues(namespace, name).Inc()
+	case log.ExperimentKey:
+		m.errorExperimentCounter.WithLabelValues(namespace, name).Inc()
+	}
 }
 
-// calculatePhase calculates where a Rollout is in a Completed, Paused, Error, Timeout, or InvalidSpec phase
-func calculatePhase(rollout *v1alpha1.Rollout) RolloutPhase {
-	phase := Progressing
-	progressing := conditions.GetRolloutCondition(rollout.Status, v1alpha1.RolloutProgressing)
-	if progressing != nil {
-		if progressing.Reason == conditions.NewRSAvailableReason {
-			phase = Completed
+// Remove removes the metrics server from the registry
+func (m *MetricsServer) Remove(namespace string, name string, kind string) {
+	go func(namespace string, name string, kind string) {
+		// wait for the metrics to be collected, prometheus scrape interval is 60 seconds by default
+		time.Sleep(defaults.GetMetricCleanupDelaySeconds())
+		switch kind {
+		case log.RolloutKey:
+			m.reconcileRolloutHistogram.Delete(map[string]string{"namespace": namespace, "name": name})
+			m.errorRolloutCounter.Delete(map[string]string{"namespace": namespace, "name": name})
+
+			m.successNotificationCounter.DeletePartialMatch(map[string]string{"namespace": namespace, "name": name})
+			m.errorNotificationCounter.DeletePartialMatch(map[string]string{"namespace": namespace, "name": name})
+			m.sendNotificationRunHistogram.DeletePartialMatch(map[string]string{"namespace": namespace, "name": name})
+
+			MetricRolloutReconcile.Delete(map[string]string{"namespace": namespace, "name": name})
+
+			MetricRolloutReconcileError.Delete(map[string]string{"namespace": namespace, "name": name})
+
+			MetricRolloutEventsTotal.DeletePartialMatch(map[string]string{"namespace": namespace, "name": name})
+		case log.AnalysisRunKey:
+			m.reconcileAnalysisRunHistogram.Delete(map[string]string{"namespace": namespace, "name": name})
+			m.errorAnalysisRunCounter.Delete(map[string]string{"namespace": namespace, "name": name})
+
+			m.successNotificationCounter.DeletePartialMatch(map[string]string{"namespace": namespace, "name": name})
+			m.errorNotificationCounter.DeletePartialMatch(map[string]string{"namespace": namespace, "name": name})
+			m.sendNotificationRunHistogram.DeletePartialMatch(map[string]string{"namespace": namespace, "name": name})
+
+			MetricAnalysisRunReconcile.Delete(map[string]string{"namespace": namespace, "name": name})
+			MetricAnalysisRunReconcileError.Delete(map[string]string{"namespace": namespace, "name": name})
+
+		case log.ExperimentKey:
+			m.reconcileExperimentHistogram.Delete(map[string]string{"namespace": namespace, "name": name})
+			m.errorExperimentCounter.Delete(map[string]string{"namespace": namespace, "name": name})
+
+			m.successNotificationCounter.DeletePartialMatch(map[string]string{"namespace": namespace, "name": name})
+			m.errorNotificationCounter.DeletePartialMatch(map[string]string{"namespace": namespace, "name": name})
+			m.sendNotificationRunHistogram.DeletePartialMatch(map[string]string{"namespace": namespace, "name": name})
+
+			MetricExperimentReconcile.Delete(map[string]string{"namespace": namespace, "name": name})
+			MetricExperimentReconcileError.Delete(map[string]string{"namespace": namespace, "name": name})
 		}
-		if progressing.Reason == conditions.PausedRolloutReason {
-			phase = Paused
-		}
-		if progressing.Reason == conditions.ServiceNotFoundReason || progressing.Reason == conditions.FailedRSCreateReason {
-			phase = Error
-		}
-		if progressing.Reason == conditions.TimedOutReason {
-			phase = Timeout
-		}
-	}
-	invalidSpec := conditions.GetRolloutCondition(rollout.Status, v1alpha1.InvalidSpec)
-	if invalidSpec != nil {
-		phase = InvalidSpec
-	}
-	return phase
+	}(namespace, name, kind)
+
 }
 
-type rolloutCollector struct {
-	store rolloutlister.RolloutLister
-}
-
-// NewRolloutCollector returns a prometheus collector for rollout metrics
-func NewRolloutCollector(rolloutLister rolloutlister.RolloutLister) prometheus.Collector {
-	return &rolloutCollector{
-		store: rolloutLister,
-	}
-}
-
-// NewRolloutRegistry creates a new prometheus registry that collects rollouts
-func NewRolloutRegistry(rolloutLister rolloutlister.RolloutLister) *prometheus.Registry {
-	registry := prometheus.NewRegistry()
-	registry.MustRegister(NewRolloutCollector(rolloutLister))
-	registry.MustRegister(prometheus.NewProcessCollector(prometheus.ProcessCollectorOpts{}))
-	registry.MustRegister(prometheus.NewGoCollector())
-	return registry
-}
-
-// Describe implements the prometheus.Collector interface
-func (c *rolloutCollector) Describe(ch chan<- *prometheus.Desc) {
-	ch <- descRolloutInfo
-	ch <- descRolloutCreated
-}
-
-// Collect implements the prometheus.Collector interface
-func (c *rolloutCollector) Collect(ch chan<- prometheus.Metric) {
-	rollouts, err := c.store.List(labels.NewSelector())
-	if err != nil {
-		log.Warnf("Failed to collect rollouts: %v", err)
-		return
-	}
-	for _, rollout := range rollouts {
-		collectRollouts(ch, rollout)
-	}
+// recordBuildInfo publishes information about Argo-Rollouts version and runtime info through an info metric (gauge).
+func recordBuildInfo() {
+	vers := version.GetVersion()
+	buildInfo.WithLabelValues(vers.Version, runtime.Version(), runtime.GOARCH, vers.GitCommit).Set(1)
 }
 
 func boolFloat64(b bool) float64 {
@@ -193,26 +201,3 @@ func boolFloat64(b bool) float64 {
 	}
 	return 0
 }
-
-func collectRollouts(ch chan<- prometheus.Metric, rollout *v1alpha1.Rollout) {
-
-	addConstMetric := func(desc *prometheus.Desc, t prometheus.ValueType, v float64, lv ...string) {
-		lv = append([]string{rollout.Namespace, rollout.Name, defaults.GetStrategyType(rollout)}, lv...)
-		ch <- prometheus.MustNewConstMetric(desc, t, v, lv...)
-	}
-	addGauge := func(desc *prometheus.Desc, v float64, lv ...string) {
-		addConstMetric(desc, prometheus.GaugeValue, v, lv...)
-	}
-
-	addGauge(descRolloutInfo, 1)
-
-	addGauge(descRolloutCreated, float64(rollout.CreationTimestamp.Unix()))
-
-	calculatedPhase := calculatePhase(rollout)
-	addGauge(descRolloutPhaseLabels, boolFloat64(calculatedPhase == Completed), string(Completed))
-	addGauge(descRolloutPhaseLabels, boolFloat64(calculatedPhase == Progressing), string(Progressing))
-	addGauge(descRolloutPhaseLabels, boolFloat64(calculatedPhase == Paused), string(Paused))
-	addGauge(descRolloutPhaseLabels, boolFloat64(calculatedPhase == Timeout), string(Timeout))
-	addGauge(descRolloutPhaseLabels, boolFloat64(calculatedPhase == Error), string(Error))
-	addGauge(descRolloutPhaseLabels, boolFloat64(calculatedPhase == InvalidSpec), string(InvalidSpec))
-}

controller/metrics/metrics_test.go

@@ -7,114 +7,118 @@ import (
 	"net/http/httptest"
 	"strings"
 	"testing"
+	"time"
 
-	"github.com/ghodss/yaml"
 	"github.com/stretchr/testify/assert"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/tools/cache"
 
-	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
-	clientset "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake"
-	informer "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions"
-	lister "github.com/argoproj/argo-rollouts/pkg/client/listers/rollouts/v1alpha1"
+	"github.com/argoproj/argo-rollouts/utils/defaults"
+
+	"github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake"
+	informerfactory "github.com/argoproj/argo-rollouts/pkg/client/informers/externalversions"
+	logutil "github.com/argoproj/argo-rollouts/utils/log"
 )
 
-// assertMetricsPrinted asserts every line in the expected lines appears in the body
-func assertMetricsPrinted(t *testing.T, expectedLines, body string) {
-	for _, line := range strings.Split(expectedLines, "\n") {
-		assert.Contains(t, body, line)
+func newFakeServerConfig(objs ...runtime.Object) ServerConfig {
+	fakeClient := fake.NewSimpleClientset(objs...)
+	factory := informerfactory.NewSharedInformerFactory(fakeClient, 0)
+	roInformer := factory.Argoproj().V1alpha1().Rollouts()
+	arInformer := factory.Argoproj().V1alpha1().AnalysisRuns()
+	atInformer := factory.Argoproj().V1alpha1().AnalysisTemplates()
+	catInformer := factory.Argoproj().V1alpha1().ClusterAnalysisTemplates()
+	exInformer := factory.Argoproj().V1alpha1().Experiments()
+	ctx, cancel := context.WithCancel(context.TODO())
+
+	var hasSyncedFuncs = make([]cache.InformerSynced, 0)
+	for _, inf := range []cache.SharedIndexInformer{
+		roInformer.Informer(),
+		arInformer.Informer(),
+		atInformer.Informer(),
+		catInformer.Informer(),
+		exInformer.Informer(),
+	} {
+		go inf.Run(ctx.Done())
+		hasSyncedFuncs = append(hasSyncedFuncs, inf.HasSynced)
+
+	}
+	cache.WaitForCacheSync(ctx.Done(), hasSyncedFuncs...)
+	cancel()
+
+	return ServerConfig{
+		RolloutLister:                 roInformer.Lister(),
+		AnalysisRunLister:             arInformer.Lister(),
+		AnalysisTemplateLister:        atInformer.Lister(),
+		ClusterAnalysisTemplateLister: catInformer.Lister(),
+		ExperimentLister:              exInformer.Lister(),
+		K8SRequestProvider:            &K8sRequestsCountProvider{},
 	}
 }
 
-func newFakeRollout(fakeRollout string) *v1alpha1.Rollout {
-	var rollout v1alpha1.Rollout
-	err := yaml.Unmarshal([]byte(fakeRollout), &rollout)
-	if err != nil {
-		panic(err)
-	}
-	return &rollout
-}
-
-func newFakeLister(fakeRollout ...string) (context.CancelFunc, lister.RolloutLister) {
-	ctx, cancel := context.WithCancel(context.Background())
-	defer cancel()
-	var fakeRollouts []runtime.Object
-	for _, name := range fakeRollout {
-		fakeRollouts = append(fakeRollouts, newFakeRollout(name))
-	}
-	appClientset := clientset.NewSimpleClientset(fakeRollouts...)
-	factory := informer.NewSharedInformerFactoryWithOptions(appClientset, 0)
-	rolloutInformer := factory.Argoproj().V1alpha1().Rollouts().Informer()
-	go rolloutInformer.Run(ctx.Done())
-	if !cache.WaitForCacheSync(ctx.Done(), rolloutInformer.HasSynced) {
-		log.Fatal("Timed out waiting for caches to sync")
-	}
-	return cancel, factory.Argoproj().V1alpha1().Rollouts().Lister()
-}
-
-func testRolloutDescribe(t *testing.T, fakeRollout string, expectedResponse string) {
-	cancel, rolloutLister := newFakeLister(fakeRollout)
-	defer cancel()
-	metricsServ := NewMetricsServer("localhost:8080", rolloutLister)
+func testHttpResponse(t *testing.T, handler http.Handler, expectedResponse string, testFunc func(t assert.TestingT, s any, contains any, msgAndArgs ...any) bool) {
+	t.Helper()
 	req, err := http.NewRequest("GET", "/metrics", nil)
 	assert.NoError(t, err)
 	rr := httptest.NewRecorder()
-	metricsServ.Handler.ServeHTTP(rr, req)
+	handler.ServeHTTP(rr, req)
 	assert.Equal(t, rr.Code, http.StatusOK)
 	body := rr.Body.String()
 	log.Println(body)
-	assertMetricsPrinted(t, expectedResponse, body)
+	for _, line := range strings.Split(expectedResponse, "\n") {
+		testFunc(t, body, line)
+	}
 }
 
 type testCombination struct {
-	rollout          string
+	resource         string
 	expectedResponse string
 }
 
-const fakeRollout = `
-apiVersion: argoproj.io/v1alpha1
-kind: Rollout
-metadata:
-  name: guestbook-bluegreen
-  namespace: default
-spec:
-  replicas: 1
-  selector:
-    matchLabels:
-      app: guestbook
-  template:
-    metadata:
-      labels:
-        app: guestbook
-    spec:
-      containers:
-      - name: guestbook
-        # The image below can be flip from 0.1 to 0.2
-        image: gcr.io/heptio-images/ks-guestbook-demo:0.1
-        ports:
-        - containerPort: 80
-  minReadySeconds: 30
-  revisionHistoryLimit: 3
-  strategy:
-    blueGreen:
-      activeService: active-service
-      previewService: preview-service
-`
+func TestIncError(t *testing.T) {
+	expectedResponse := `# HELP analysis_run_reconcile_error Error occurring during the analysis run
+# TYPE analysis_run_reconcile_error counter
+analysis_run_reconcile_error{name="name",namespace="ns"} 1
+# HELP experiment_reconcile_error Error occurring during the experiment
+# TYPE experiment_reconcile_error counter
+experiment_reconcile_error{name="name",namespace="ns"} 1
+# HELP rollout_reconcile_error Error occurring during the rollout
+# TYPE rollout_reconcile_error counter
+rollout_reconcile_error{name="name",namespace="ns"} 1`
 
-const expectedResponse = `# HELP rollout_created_time Creation time in unix timestamp for an rollout.
-# TYPE rollout_created_time gauge
-rollout_created_time{name="guestbook-bluegreen",namespace="default",strategy="blueGreen"} -6.21355968e+10
-`
+	metricsServ := NewMetricsServer(newFakeServerConfig())
 
-func TestMetrics(t *testing.T) {
-	combinations := []testCombination{
-		{
-			rollout:          fakeRollout,
-			expectedResponse: expectedResponse,
-		},
-	}
-
-	for _, combination := range combinations {
-		testRolloutDescribe(t, combination.rollout, combination.expectedResponse)
-	}
+	metricsServ.IncError("ns", "name", logutil.AnalysisRunKey)
+	metricsServ.IncError("ns", "name", logutil.ExperimentKey)
+	metricsServ.IncError("ns", "name", logutil.RolloutKey)
+	testHttpResponse(t, metricsServ.Handler, expectedResponse, assert.Contains)
+}
+
+func TestVersionInfo(t *testing.T) {
+	expectedResponse := `# HELP argo_rollouts_controller_info Running Argo-rollouts version
+# TYPE argo_rollouts_controller_info gauge`
+	metricsServ := NewMetricsServer(newFakeServerConfig())
+	testHttpResponse(t, metricsServ.Handler, expectedResponse, assert.Contains)
+}
+
+func TestRemove(t *testing.T) {
+	defaults.SetMetricCleanupDelaySeconds(1)
+
+	expectedResponse := `analysis_run_reconcile_error{name="name1",namespace="ns"} 1
+experiment_reconcile_error{name="name1",namespace="ns"} 1
+rollout_reconcile_error{name="name1",namespace="ns"} 1`
+
+	metricsServ := NewMetricsServer(newFakeServerConfig())
+
+	metricsServ.IncError("ns", "name1", logutil.RolloutKey)
+	metricsServ.IncError("ns", "name1", logutil.AnalysisRunKey)
+	metricsServ.IncError("ns", "name1", logutil.ExperimentKey)
+	testHttpResponse(t, metricsServ.Handler, expectedResponse, assert.Contains)
+
+	metricsServ.Remove("ns", "name1", logutil.AnalysisRunKey)
+	metricsServ.Remove("ns", "name1", logutil.ExperimentKey)
+	metricsServ.Remove("ns", "name1", logutil.RolloutKey)
+
+	//Sleep for 2x the cleanup delay to allow metrics to be removed
+	time.Sleep(defaults.GetMetricCleanupDelaySeconds() * 2)
+	testHttpResponse(t, metricsServ.Handler, expectedResponse, assert.NotContains)
 }

controller/metrics/prommetrics.go

@@ -0,0 +1,239 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+
+	"github.com/argoproj/argo-rollouts/utils/version"
+)
+
+// Follow Prometheus naming practices
+// https://prometheus.io/docs/practices/naming/
+var (
+	namespaceNameLabels = []string{"namespace", "name"}
+)
+
+// Rollout metrics
+var (
+	MetricRolloutReconcile = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "rollout_reconcile",
+			Help:    "Rollout reconciliation performance.",
+			Buckets: []float64{0.01, 0.15, .25, .5, 1},
+		},
+		namespaceNameLabels,
+	)
+
+	MetricRolloutReconcileError = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "rollout_reconcile_error",
+			Help: "Error occurring during the rollout",
+		},
+		namespaceNameLabels,
+	)
+
+	MetricRolloutInfo = prometheus.NewDesc(
+		"rollout_info",
+		"Information about rollout.",
+		append(namespaceNameLabels, "strategy", "traffic_router", "phase"),
+		nil,
+	)
+
+	MetricRolloutInfoReplicasAvailable = prometheus.NewDesc(
+		"rollout_info_replicas_available",
+		"The number of available replicas per rollout.",
+		namespaceNameLabels,
+		nil,
+	)
+
+	MetricRolloutInfoReplicasUnavailable = prometheus.NewDesc(
+		"rollout_info_replicas_unavailable",
+		"The number of unavailable replicas per rollout.",
+		namespaceNameLabels,
+		nil,
+	)
+
+	MetricRolloutInfoReplicasDesired = prometheus.NewDesc(
+		"rollout_info_replicas_desired",
+		"The number of desired replicas per rollout.",
+		namespaceNameLabels,
+		nil,
+	)
+
+	MetricRolloutInfoReplicasUpdated = prometheus.NewDesc(
+		"rollout_info_replicas_updated",
+		"The number of updated replicas per rollout.",
+		namespaceNameLabels,
+		nil,
+	)
+
+	MetricRolloutEventsTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "rollout_events_total",
+			Help: "Count of rollout events",
+		},
+		append(namespaceNameLabels, "type", "reason"),
+	)
+
+	// DEPRECATED in favor of rollout_info
+	MetricRolloutPhase = prometheus.NewDesc(
+		"rollout_phase",
+		"Information on the state of the rollout (DEPRECATED - use rollout_info)",
+		append(namespaceNameLabels, "strategy", "phase"),
+		nil,
+	)
+)
+
+// AnalysisRun metrics
+var (
+	MetricAnalysisRunReconcile = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "analysis_run_reconcile",
+			Help:    "Analysis Run reconciliation performance.",
+			Buckets: []float64{0.01, 0.15, .25, .5, 1},
+		},
+		namespaceNameLabels,
+	)
+
+	MetricAnalysisRunReconcileError = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "analysis_run_reconcile_error",
+			Help: "Error occurring during the analysis run",
+		},
+		namespaceNameLabels,
+	)
+	MetricAnalysisRunInfo = prometheus.NewDesc(
+		"analysis_run_info",
+		"Information about analysis run.",
+		append(namespaceNameLabels, "phase"),
+		nil,
+	)
+
+	// DEPRECATED in favor of analysis_run_info
+	MetricAnalysisRunPhase = prometheus.NewDesc(
+		"analysis_run_phase",
+		"Information on the state of the Analysis Run (DEPRECATED - use analysis_run_info)",
+		append(namespaceNameLabels, "phase"),
+		nil,
+	)
+
+	MetricAnalysisRunMetricType = prometheus.NewDesc(
+		"analysis_run_metric_type",
+		"Information on the type of a specific metric in the Analysis Runs",
+		append(namespaceNameLabels, "metric", "type"),
+		nil,
+	)
+
+	MetricAnalysisRunMetricPhase = prometheus.NewDesc(
+		"analysis_run_metric_phase",
+		"Information on the duration of a specific metric in the Analysis Run",
+		append(namespaceNameLabels, "metric", "type", "dry_run", "phase"),
+		nil,
+	)
+)
+
+// AnalysisTemplate metrics
+var (
+	MetricAnalysisTemplateInfo = prometheus.NewDesc(
+		"analysis_template_info",
+		"Information about analysis templates.",
+		namespaceNameLabels,
+		nil,
+	)
+
+	MetricAnalysisTemplateMetricInfo = prometheus.NewDesc(
+		"analysis_template_metric_info",
+		"Information on metrics in analysis templates.",
+		append(namespaceNameLabels, "type", "metric"),
+		nil,
+	)
+)
+
+// Experiment metrics
+var (
+	MetricExperimentReconcile = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "experiment_reconcile",
+			Help:    "Experiments reconciliation performance.",
+			Buckets: []float64{0.01, 0.15, .25, .5, 1},
+		},
+		namespaceNameLabels,
+	)
+
+	MetricExperimentReconcileError = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "experiment_reconcile_error",
+			Help: "Error occurring during the experiment",
+		},
+		namespaceNameLabels,
+	)
+
+	MetricExperimentInfo = prometheus.NewDesc(
+		"experiment_info",
+		"Information about Experiment.",
+		append(namespaceNameLabels, "phase"),
+		nil,
+	)
+
+	// DEPRECATED in favor of experiment_info
+	MetricExperimentPhase = prometheus.NewDesc(
+		"experiment_phase",
+		"Information on the state of the experiment (DEPRECATED - use experiment_info)",
+		append(namespaceNameLabels, "phase"),
+		nil,
+	)
+)
+
+// Notification metrics
+var (
+	MetricNotificationSuccessTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "notification_send_success",
+			Help: "Notification send success.",
+		},
+		append(namespaceNameLabels, "type", "reason"),
+	)
+
+	MetricNotificationFailedTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "notification_send_error",
+			Help: "Error sending the notification",
+		},
+		append(namespaceNameLabels, "type", "reason"),
+	)
+
+	MetricNotificationSend = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name:    "notification_send",
+			Help:    "Notification send performance.",
+			Buckets: []float64{0.01, 0.15, .25, .5, 1},
+		},
+		namespaceNameLabels,
+	)
+)
+
+// K8s Client metrics
+var (
+	// Custom events metric
+	MetricK8sRequestTotal = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Namespace: clientsetMetricsNamespace,
+			Name:      "k8s_request_total",
+			Help:      "Number of kubernetes requests executed during application reconciliation.",
+		},
+		[]string{"kind", "namespace", "name", "verb", "status_code"},
+	)
+)
+
+// MetricVersionGauge version info
+var (
+	MetricVersionGauge = prometheus.NewGaugeFunc(
+		prometheus.GaugeOpts{
+			Name:        "argo_rollouts_controller_info",
+			Help:        "Running Argo-rollouts version",
+			ConstLabels: prometheus.Labels{"version": version.GetVersion().Version},
+		},
+		func() float64 {
+			return float64(1)
+		},
+	)
+)

controller/metrics/rollout_test.go

@@ -0,0 +1,333 @@
+package metrics
+
+import (
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"sigs.k8s.io/yaml"
+
+	"github.com/prometheus/client_golang/prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
+
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
+	"github.com/argoproj/argo-rollouts/utils/conditions"
+)
+
+const (
+	fakeRollout = `
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  name: guestbook-bluegreen
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: guestbook
+  template:
+    metadata:
+      labels:
+        app: guestbook
+    spec:
+      containers:
+      - name: guestbook
+        image: argoproj/rollouts-demo:blue
+        ports:
+        - containerPort: 80
+  minReadySeconds: 30
+  revisionHistoryLimit: 3
+  strategy:
+    blueGreen:
+      activeService: active-service
+      previewService: preview-service
+status:
+  replicas: 1
+  availableReplicas: 1
+`
+
+	fakeCanaryRollout = `
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  name: guestbook-canary
+  namespace: default
+spec:
+  replicas: 1
+  selector:
+    matchLabels:
+      app: guestbook
+  template:
+    metadata:
+      labels:
+        app: guestbook
+    spec:
+      containers:
+      - name: guestbook
+        image: argoproj/rollouts-demo:blue
+        ports:
+        - containerPort: 80
+  minReadySeconds: 30
+  revisionHistoryLimit: 3
+  strategy:
+    canary:
+      trafficRouting:
+        smi: {}
+status:
+  replicas: 1
+  availableReplicas: 1
+`
+)
+
+func newFakeRollout(fakeRollout string, cond *v1alpha1.RolloutCondition) *v1alpha1.Rollout {
+	var rollout v1alpha1.Rollout
+	err := yaml.Unmarshal([]byte(fakeRollout), &rollout)
+	if err != nil {
+		panic(err)
+	}
+	rollout.Status.Conditions = append(rollout.Status.Conditions, *cond)
+	return &rollout
+}
+
+func TestCollectRollouts(t *testing.T) {
+	combinations := []struct {
+		fakeRollout      string
+		fakeCondition    *v1alpha1.RolloutCondition
+		expectedResponse string
+	}{
+		{
+			fakeRollout,
+			conditions.NewRolloutCondition(v1alpha1.RolloutProgressing, corev1.ConditionFalse, "Progressing", ""),
+			`
+# HELP rollout_info Information about rollout.
+# TYPE rollout_info gauge
+rollout_info{name="guestbook-bluegreen",namespace="default",phase="Progressing",strategy="blueGreen",traffic_router=""} 1
+# HELP rollout_info_replicas_available The number of available replicas per rollout.
+# TYPE rollout_info_replicas_available gauge
+rollout_info_replicas_available{name="guestbook-bluegreen",namespace="default"} 1
+# HELP rollout_info_replicas_desired The number of desired replicas per rollout.
+# TYPE rollout_info_replicas_desired gauge
+rollout_info_replicas_desired{name="guestbook-bluegreen",namespace="default"} 1
+# HELP rollout_info_replicas_unavailable The number of unavailable replicas per rollout.
+# TYPE rollout_info_replicas_unavailable gauge
+rollout_info_replicas_unavailable{name="guestbook-bluegreen",namespace="default"} 0
+# HELP rollout_info_replicas_unavailable The number of unavailable replicas per rollout.
+# TYPE rollout_info_replicas_unavailable gauge
+rollout_info_replicas_unavailable{name="guestbook-bluegreen",namespace="default"} 0
+# HELP rollout_info_replicas_updated The number of updated replicas per rollout.
+# TYPE rollout_info_replicas_updated gauge
+rollout_info_replicas_updated{name="guestbook-bluegreen",namespace="default"} 0`,
+		},
+
+		{
+			fakeRollout,
+			conditions.NewRolloutCondition(v1alpha1.RolloutProgressing, corev1.ConditionFalse, conditions.FailedRSCreateReason, "test"),
+			`
+# HELP rollout_info Information about rollout.
+# TYPE rollout_info gauge
+rollout_info{name="guestbook-bluegreen",namespace="default",phase="Error",strategy="blueGreen",traffic_router=""} 1
+# HELP rollout_info_replicas_available The number of available replicas per rollout.
+# TYPE rollout_info_replicas_available gauge
+rollout_info_replicas_available{name="guestbook-bluegreen",namespace="default"} 1
+# HELP rollout_info_replicas_desired The number of desired replicas per rollout.
+# TYPE rollout_info_replicas_desired gauge
+rollout_info_replicas_desired{name="guestbook-bluegreen",namespace="default"} 1
+# HELP rollout_info_replicas_unavailable The number of unavailable replicas per rollout.
+# TYPE rollout_info_replicas_unavailable gauge
+rollout_info_replicas_unavailable{name="guestbook-bluegreen",namespace="default"} 0
+# HELP rollout_info_replicas_unavailable The number of unavailable replicas per rollout.
+# TYPE rollout_info_replicas_unavailable gauge
+rollout_info_replicas_unavailable{name="guestbook-bluegreen",namespace="default"} 0
+# HELP rollout_info_replicas_updated The number of updated replicas per rollout.
+# TYPE rollout_info_replicas_updated gauge
+rollout_info_replicas_updated{name="guestbook-bluegreen",namespace="default"} 0`,
+		},
+		{
+			fakeCanaryRollout,
+			conditions.NewRolloutCondition(v1alpha1.RolloutProgressing, corev1.ConditionFalse, "Progressing", "test"),
+			`
+# HELP rollout_info Information about rollout.
+# TYPE rollout_info gauge
+rollout_info{name="guestbook-canary",namespace="default",phase="Progressing",strategy="canary",traffic_router="SMI"} 1
+# HELP rollout_info_replicas_available The number of available replicas per rollout.
+# TYPE rollout_info_replicas_available gauge
+rollout_info_replicas_available{name="guestbook-canary",namespace="default"} 1
+# HELP rollout_info_replicas_desired The number of desired replicas per rollout.
+# TYPE rollout_info_replicas_desired gauge
+rollout_info_replicas_desired{name="guestbook-canary",namespace="default"} 1
+# HELP rollout_info_replicas_unavailable The number of unavailable replicas per rollout.
+# TYPE rollout_info_replicas_unavailable gauge
+rollout_info_replicas_unavailable{name="guestbook-canary",namespace="default"} 0
+# HELP rollout_info_replicas_unavailable The number of unavailable replicas per rollout.
+# TYPE rollout_info_replicas_unavailable gauge
+rollout_info_replicas_unavailable{name="guestbook-canary",namespace="default"} 0
+# HELP rollout_info_replicas_updated The number of updated replicas per rollout.
+# TYPE rollout_info_replicas_updated gauge
+rollout_info_replicas_updated{name="guestbook-canary",namespace="default"} 0`,
+		},
+	}
+
+	for _, combination := range combinations {
+		testRolloutDescribe(t, combination.fakeRollout, combination.fakeCondition, combination.expectedResponse)
+	}
+}
+
+func testRolloutDescribe(t *testing.T, fakeRollout string, cond *v1alpha1.RolloutCondition, expectedResponse string) {
+	registry := prometheus.NewRegistry()
+	config := newFakeServerConfig(newFakeRollout(fakeRollout, cond))
+	registry.MustRegister(NewRolloutCollector(config.RolloutLister))
+	mux := http.NewServeMux()
+	mux.Handle(MetricsPath, promhttp.HandlerFor(registry, promhttp.HandlerOpts{}))
+	testHttpResponse(t, mux, expectedResponse, assert.Contains)
+}
+
+func TestIncRolloutReconcile(t *testing.T) {
+	expectedResponse := `
+# HELP rollout_reconcile Rollout reconciliation performance.
+# TYPE rollout_reconcile histogram
+rollout_reconcile_bucket{name="ro-test",namespace="ro-namespace",le="0.01"} 1
+rollout_reconcile_bucket{name="ro-test",namespace="ro-namespace",le="0.15"} 1
+rollout_reconcile_bucket{name="ro-test",namespace="ro-namespace",le="0.25"} 1
+rollout_reconcile_bucket{name="ro-test",namespace="ro-namespace",le="0.5"} 1
+rollout_reconcile_bucket{name="ro-test",namespace="ro-namespace",le="1"} 1
+rollout_reconcile_bucket{name="ro-test",namespace="ro-namespace",le="+Inf"} 1
+rollout_reconcile_sum{name="ro-test",namespace="ro-namespace"} 0.001
+rollout_reconcile_count{name="ro-test",namespace="ro-namespace"} 1
+`
+
+	metricsServ := NewMetricsServer(newFakeServerConfig())
+	ro := &v1alpha1.Rollout{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      "ro-test",
+			Namespace: "ro-namespace",
+		},
+	}
+	metricsServ.IncRolloutReconcile(ro, time.Millisecond)
+	testHttpResponse(t, metricsServ.Handler, expectedResponse, assert.Contains)
+}
+
+func TestGetStrategyAndTrafficRouter(t *testing.T) {
+	var tests = []struct {
+		strategy              v1alpha1.RolloutStrategy
+		expectedStrategy      string
+		expectedTrafficRouter string
+	}{
+		{
+			strategy: v1alpha1.RolloutStrategy{
+				BlueGreen: &v1alpha1.BlueGreenStrategy{},
+			},
+			expectedStrategy:      "blueGreen",
+			expectedTrafficRouter: "",
+		},
+		{
+			strategy: v1alpha1.RolloutStrategy{
+				Canary: &v1alpha1.CanaryStrategy{},
+			},
+			expectedStrategy:      "canary",
+			expectedTrafficRouter: "",
+		},
+		{
+			strategy:              v1alpha1.RolloutStrategy{},
+			expectedStrategy:      "none",
+			expectedTrafficRouter: "",
+		},
+		{
+			strategy: v1alpha1.RolloutStrategy{
+				Canary: &v1alpha1.CanaryStrategy{
+					TrafficRouting: &v1alpha1.RolloutTrafficRouting{
+						SMI: &v1alpha1.SMITrafficRouting{},
+					},
+				},
+			},
+			expectedStrategy:      "canary",
+			expectedTrafficRouter: "SMI",
+		},
+		{
+			strategy: v1alpha1.RolloutStrategy{
+				Canary: &v1alpha1.CanaryStrategy{
+					TrafficRouting: &v1alpha1.RolloutTrafficRouting{
+						Istio: &v1alpha1.IstioTrafficRouting{},
+					},
+				},
+			},
+			expectedStrategy:      "canary",
+			expectedTrafficRouter: "Istio",
+		},
+		{
+			strategy: v1alpha1.RolloutStrategy{
+				Canary: &v1alpha1.CanaryStrategy{
+					TrafficRouting: &v1alpha1.RolloutTrafficRouting{
+						ALB: &v1alpha1.ALBTrafficRouting{},
+					},
+				},
+			},
+			expectedStrategy:      "canary",
+			expectedTrafficRouter: "ALB",
+		},
+		{
+			strategy: v1alpha1.RolloutStrategy{
+				Canary: &v1alpha1.CanaryStrategy{
+					TrafficRouting: &v1alpha1.RolloutTrafficRouting{
+						Ambassador: &v1alpha1.AmbassadorTrafficRouting{},
+					},
+				},
+			},
+			expectedStrategy:      "canary",
+			expectedTrafficRouter: "Ambassador",
+		},
+		{
+			strategy: v1alpha1.RolloutStrategy{
+				Canary: &v1alpha1.CanaryStrategy{
+					TrafficRouting: &v1alpha1.RolloutTrafficRouting{
+						Nginx: &v1alpha1.NginxTrafficRouting{},
+					},
+				},
+			},
+			expectedStrategy:      "canary",
+			expectedTrafficRouter: "Nginx",
+		},
+		{
+			strategy: v1alpha1.RolloutStrategy{
+				Canary: &v1alpha1.CanaryStrategy{
+					TrafficRouting: &v1alpha1.RolloutTrafficRouting{
+						AppMesh: &v1alpha1.AppMeshTrafficRouting{},
+					},
+				},
+			},
+			expectedStrategy:      "canary",
+			expectedTrafficRouter: "AppMesh",
+		},
+	}
+
+	for _, test := range tests {
+		ro := &v1alpha1.Rollout{
+			Spec: v1alpha1.RolloutSpec{
+				Strategy: test.strategy,
+			},
+		}
+		strategy, trafficRouter := getStrategyAndTrafficRouter(ro)
+		assert.Equal(t, test.expectedStrategy, strategy)
+		assert.Equal(t, test.expectedTrafficRouter, trafficRouter)
+	}
+}
+
+func TestIncRolloutEvents(t *testing.T) {
+	expectedResponse := `
+# HELP rollout_events_total Count of rollout events
+# TYPE rollout_events_total counter
+rollout_events_total{name="ro-test-1",namespace="ro-namespace",reason="BarEvent",type="Normal"} 1
+rollout_events_total{name="ro-test-1",namespace="ro-namespace",reason="FooEvent",type="Normal"} 1
+rollout_events_total{name="ro-test-2",namespace="ro-namespace",reason="BazEvent",type="Warning"} 2
+`
+
+	metricsServ := NewMetricsServer(newFakeServerConfig())
+	MetricRolloutEventsTotal.WithLabelValues("ro-namespace", "ro-test-1", corev1.EventTypeNormal, "FooEvent").Inc()
+	MetricRolloutEventsTotal.WithLabelValues("ro-namespace", "ro-test-1", corev1.EventTypeNormal, "BarEvent").Inc()
+	MetricRolloutEventsTotal.WithLabelValues("ro-namespace", "ro-test-2", corev1.EventTypeWarning, "BazEvent").Inc()
+	MetricRolloutEventsTotal.WithLabelValues("ro-namespace", "ro-test-2", corev1.EventTypeWarning, "BazEvent").Inc()
+	testHttpResponse(t, metricsServ.Handler, expectedResponse, assert.Contains)
+}

controller/metrics/rollouts.go

@@ -0,0 +1,143 @@
+package metrics
+
+import (
+	"github.com/prometheus/client_golang/prometheus"
+	log "github.com/sirupsen/logrus"
+	"k8s.io/apimachinery/pkg/labels"
+
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
+	rolloutlister "github.com/argoproj/argo-rollouts/pkg/client/listers/rollouts/v1alpha1"
+	"github.com/argoproj/argo-rollouts/utils/conditions"
+	"github.com/argoproj/argo-rollouts/utils/defaults"
+)
+
+// RolloutPhase the phases of a reconcile can have
+type RolloutPhase string
+
+const (
+
+	// RolloutInvalidSpec means the rollout had an invalid spec during reconciliation
+	RolloutInvalidSpec RolloutPhase = "InvalidSpec"
+	// RolloutCompleted means the rollout finished the reconciliation with no remaining work
+	RolloutCompleted RolloutPhase = "Completed"
+	// RolloutProgressing means the rollout finished the reconciliation with remaining work
+	RolloutProgressing RolloutPhase = "Progressing"
+	// RolloutPaused means the rollout finished the reconciliation with a paused status
+	RolloutPaused RolloutPhase = "Paused"
+	// RolloutTimeout means the rollout finished the reconciliation with an timeout message
+	RolloutTimeout RolloutPhase = "Timeout"
+	// RolloutError means the rollout finished the reconciliation with an error
+	RolloutError RolloutPhase = "Error"
+	// RolloutAbort means the rollout finished the reconciliation in an aborted state
+	RolloutAbort RolloutPhase = "Abort"
+)
+
+type rolloutCollector struct {
+	store rolloutlister.RolloutLister
+}
+
+// NewRolloutCollector returns a prometheus collector for rollout metrics
+func NewRolloutCollector(rolloutLister rolloutlister.RolloutLister) prometheus.Collector {
+	return &rolloutCollector{
+		store: rolloutLister,
+	}
+}
+
+// Describe implements the prometheus.Collector interface
+func (c *rolloutCollector) Describe(ch chan<- *prometheus.Desc) {
+	ch <- MetricRolloutInfo
+}
+
+// Collect implements the prometheus.Collector interface
+func (c *rolloutCollector) Collect(ch chan<- prometheus.Metric) {
+	rollouts, err := c.store.List(labels.NewSelector())
+	if err != nil {
+		log.Warnf("Failed to collect rollouts: %v", err)
+		return
+	}
+	for _, rollout := range rollouts {
+		collectRollouts(ch, rollout)
+	}
+}
+
+// calculatePhase calculates where a Rollout is in a RolloutCompleted, Paused, Error, Timeout, InvalidSpec, or ABorted phase
+func calculatePhase(rollout *v1alpha1.Rollout) RolloutPhase {
+	phase := RolloutProgressing
+	progressing := conditions.GetRolloutCondition(rollout.Status, v1alpha1.RolloutProgressing)
+	if progressing != nil {
+		if progressing.Reason == conditions.NewRSAvailableReason {
+			phase = RolloutCompleted
+		}
+		if progressing.Reason == conditions.RolloutPausedReason {
+			phase = RolloutPaused
+		}
+		if progressing.Reason == conditions.FailedRSCreateReason {
+			phase = RolloutError
+		}
+		if progressing.Reason == conditions.TimedOutReason {
+			phase = RolloutTimeout
+		}
+		if progressing.Reason == conditions.RolloutAbortedReason {
+			phase = RolloutAbort
+		}
+	}
+	invalidSpec := conditions.GetRolloutCondition(rollout.Status, v1alpha1.InvalidSpec)
+	if invalidSpec != nil {
+		phase = RolloutInvalidSpec
+	}
+	return phase
+}
+
+func getStrategyAndTrafficRouter(rollout *v1alpha1.Rollout) (string, string) {
+	strategy := "none"
+	trafficRouter := ""
+	if rollout.Spec.Strategy.BlueGreen != nil {
+		strategy = "blueGreen"
+	} else if rollout.Spec.Strategy.Canary != nil {
+		strategy = "canary"
+		if rollout.Spec.Strategy.Canary.TrafficRouting != nil {
+			if rollout.Spec.Strategy.Canary.TrafficRouting.ALB != nil {
+				trafficRouter = "ALB"
+			}
+			if rollout.Spec.Strategy.Canary.TrafficRouting.Ambassador != nil {
+				trafficRouter = "Ambassador"
+			}
+			if rollout.Spec.Strategy.Canary.TrafficRouting.Istio != nil {
+				trafficRouter = "Istio"
+			}
+			if rollout.Spec.Strategy.Canary.TrafficRouting.Nginx != nil {
+				trafficRouter = "Nginx"
+			}
+			if rollout.Spec.Strategy.Canary.TrafficRouting.SMI != nil {
+				trafficRouter = "SMI"
+			}
+			if rollout.Spec.Strategy.Canary.TrafficRouting.AppMesh != nil {
+				trafficRouter = "AppMesh"
+			}
+		}
+	}
+	return strategy, trafficRouter
+}
+
+func collectRollouts(ch chan<- prometheus.Metric, rollout *v1alpha1.Rollout) {
+	strategyType, trafficRouter := getStrategyAndTrafficRouter(rollout)
+	calculatedPhase := calculatePhase(rollout)
+
+	addGauge := func(desc *prometheus.Desc, v float64, lv ...string) {
+		lv = append([]string{rollout.Namespace, rollout.Name}, lv...)
+		ch <- prometheus.MustNewConstMetric(desc, prometheus.GaugeValue, v, lv...)
+	}
+	addGauge(MetricRolloutInfo, 1, strategyType, trafficRouter, string(calculatedPhase))
+	addGauge(MetricRolloutInfoReplicasAvailable, float64(rollout.Status.AvailableReplicas))
+	addGauge(MetricRolloutInfoReplicasUnavailable, float64(rollout.Status.Replicas-rollout.Status.AvailableReplicas))
+	addGauge(MetricRolloutInfoReplicasDesired, float64(defaults.GetReplicasOrDefault(rollout.Spec.Replicas)))
+	addGauge(MetricRolloutInfoReplicasUpdated, float64(rollout.Status.UpdatedReplicas))
+
+	// DEPRECATED
+	addGauge(MetricRolloutPhase, boolFloat64(calculatedPhase == RolloutCompleted), strategyType, string(RolloutCompleted))
+	addGauge(MetricRolloutPhase, boolFloat64(calculatedPhase == RolloutProgressing), strategyType, string(RolloutProgressing))
+	addGauge(MetricRolloutPhase, boolFloat64(calculatedPhase == RolloutPaused), strategyType, string(RolloutPaused))
+	addGauge(MetricRolloutPhase, boolFloat64(calculatedPhase == RolloutTimeout), strategyType, string(RolloutTimeout))
+	addGauge(MetricRolloutPhase, boolFloat64(calculatedPhase == RolloutError), strategyType, string(RolloutError))
+	addGauge(MetricRolloutPhase, boolFloat64(calculatedPhase == RolloutAbort), strategyType, string(RolloutAbort))
+}

controller/profiling.go

@@ -0,0 +1,21 @@
+package controller
+
+import (
+	"net/http"
+	"net/http/pprof"
+)
+
+// NewPProfServer returns a new pprof server to gather runtime profiling data
+func NewPProfServer() *http.ServeMux {
+	mux := http.NewServeMux()
+
+	// TODO: Remove enumerating all pprof endpoints if/when a more ergonomic
+	// attachment solution is introduced. See: https://github.com/golang/go/issues/71213.
+	mux.HandleFunc("/debug/pprof/", pprof.Index)
+	mux.HandleFunc("/debug/pprof/cmdline/", pprof.Cmdline)
+	mux.HandleFunc("/debug/pprof/profile/", pprof.Profile)
+	mux.HandleFunc("/debug/pprof/symbol/", pprof.Symbol)
+	mux.HandleFunc("/debug/pprof/trace/", pprof.Trace)
+
+	return mux
+}

docs/CONTRIBUTING.md

@@ -1,32 +1,42 @@
 # Contributing
+
 ## Before You Start
+
 Argo Rollouts is written in Golang. If you do not have a good grounding in Go, try out [the tutorial](https://tour.golang.org/).
 
 ## Pre-requisites
+
 Install:
 
-* [docker](https://docs.docker.com/install/#supported-platforms)
-* [golang](https://golang.org/)
-* [dep](https://github.com/golang/dep)
-* [kubectl](https://kubernetes.io/docs/tasks/tools/install-kubectl/)
-* [kustomize](https://github.com/kubernetes-sigs/kustomize/releases)
-* [minikube](https://kubernetes.io/docs/setup/minikube/) or Docker for Desktop
+- [docker](https://docs.docker.com/install/#supported-platforms)
+- [golang](https://golang.org/)
+- [kubectl](https://kubernetes.io/docs/tasks/tools/#kubectl)
+- [kustomize](https://github.com/kubernetes-sigs/kustomize/releases) >= 4.5.5
+- [k3d](https://k3d.io/) recommended
 
-Argo Rollout additionally uses
-* `controller-gen` binary in order to auto-generate the crd manifest
-* `golangci-lint` to lint the project.
+Kustomize is required for unit tests (`make test` is using it), so you [must install it](https://kubectl.docs.kubernetes.io/installation/kustomize/)
+locally if you wish to make code contributions to Argo Rollouts.
+
+Argo Rollout additionally uses the following tools
+
+- `golangci-lint` to lint the project.
+- `protoc` and `swagger-codegen` to generate proto related files
+- `yarn` to build the UI
 
 Run the following commands to install them:
+
 ```bash
-go get -u github.com/kubernetes-sigs/controller-tools/cmd/controller-gen
+# macOS
+brew install golangci-lint
+
+# linux
 go get -u github.com/golangci/golangci-lint/cmd/golangci-lint
 ```
 
-
 Brew users can quickly install the lot:
-    
+
 ```bash
-brew install go dep kubectl kustomize
+brew install go kubectl kustomize golangci-lint protobuf swagger-codegen k3d
 ```
 
 Set up environment variables (e.g. is `~/.bashrc`):
@@ -45,18 +55,26 @@ cd ~/go/src/github.com/argoproj/argo-rollouts
 
 ## Building
 
-Ensure dependencies are up to date first:
-
-```shell
-dep ensure -v
-```
+`go.mod` is used, so the `go build/test` commands automatically install the needed dependencies
 
 The `make controller` command will build the controller.
 
-* `make codegen` - Runs the code generator that creates the informers, client, lister, and deepcopies from the types.go and modifies the open-api spec.
+- `make install-tools-local` - Runs scripts to install codegen utility CLIs necessary for codegen.
 
+- `make codegen` - Runs the code generator that creates the informers, client, lister, and deepcopies from the types.go and modifies the open-api spec.
 
-## Running Tests
+## Running Controller Locally
+
+It is much easier to run and debug if you run Argo Rollout in your local machine than in the Kubernetes cluster.
+
+```bash
+cd ~/go/src/github.com/argoproj/argo-rollouts
+go run ./cmd/rollouts-controller/main.go
+```
+
+When running locally it will connect to whatever kubernetes cluster you have configured in your kubeconfig. You will need to make sure to install the Argo Rollout CRDs into your local cluster, and have the `argo-rollouts` namespace.
+
+## Running Unit Tests
 
 To run unit tests:
 
@@ -64,17 +82,170 @@ To run unit tests:
 make test
 ```
 
+## Running E2E tests
 
-## Running Locally
+The end-to-end tests need to run against a kubernetes cluster with the Argo Rollouts controller
+running.
 
-It is much easier to run and debug if you run Argo Rollout in your local machine than in the Kubernetes cluster.
+Start and prepare your cluster for e2e tests:
+
+```
+k3d cluster create
+kubectl create ns argo-rollouts
+kubectl apply -k manifests/crds
+kubectl apply -f test/e2e/crds
+```
+
+The rollout controller can be started with the command:
+
+```
+make start-e2e
+```
+
+Then run the e2e tests:
+
+```
+make test-e2e
+```
+
+To run a subset of e2e tests, you need to specify the suite with `-run`, and the specific test regex with `-testify.m`.
+
+```
+E2E_TEST_OPTIONS="-run 'TestCanarySuite' -testify.m 'TestCanaryScaleDownOnAbortNoTrafficRouting'" make test-e2e
+```
+
+Available test suites [are the following](https://github.com/argoproj/argo-rollouts/tree/master/test/e2e):
+
+* `TestBlueGreenSuite`
+* `TestCanarySuite`
+* `TestAnalysisSuite`
+* `TestExperimentSuite`
+* `TestFunctionalSuite`
+* `TestRollbackSuite`
+* `TestIstioSuite`
+* `TestHeaderRoutingSuite` (Istio only)
+* `TestMirrorRouteSuite` (Istio only)
+* `TestAPISIXSuite`
+* `TestAppMeshSuite`
+* `TestAWSSuite`
+* `StepPluginSuite`
+* `SMISuite` (SMI is [deprecated](https://www.cncf.io/blog/2023/10/03/cncf-archives-the-service-mesh-interface-smi-project/))
+* `SMIIngressSuite` (SMI is [deprecated](https://www.cncf.io/blog/2023/10/03/cncf-archives-the-service-mesh-interface-smi-project/))
+
+## Running the UI
+
+If you'd like to run the UI locally, you first need a running Rollouts controller. This can be a locally running controller with a k3d cluster, as described above, or a controller running in a remote Kubernetes cluster.
+
+In order for the local React app to communicate with the controller and Kubernetes API, run the following to open a port forward to the dashboard:
 
 ```bash
-cd ~/go/src/github.com/argoproj/argo-rollouts
-make controller
-./dist/rollouts-controller
+kubectl argo rollouts dashboard
 ```
 
+Note that you can also build the API server and run this instead,
+
+```
+make plugin
+./dist/kubectl-argo-rollouts dashboard
+```
+
+In another terminal, run the following to start the UI:
+
+```bash
+cd ui
+yarn install
+yarn start
+```
+
+## Getting your feature accepted 
+
+To be eligible for inclusion in a minor release, a new feature must meet the following criteria before the release’s RC
+date.
+
+If it is a large feature that involves significant design decisions, that feature must be described in a Proposal.
+
+The feature PR must include:
+
+* Tests (passing)
+* Documentation
+* If necessary, a note in the Upgrading docs for the planned minor release
+* The PR must be reviewed, approved, and merged by an Approver.
+
+If these criteria are not met by the RC date, the feature will be ineligible for inclusion in the RC series or GA for
+that minor release. It will have to wait for the next minor release.
+
+## Controller architecture
+
+Argo Rollouts is actually a collection of individual controllers
+that handle a specific aspect of Progressive Delivery.
+
+[![Internal Architecture](architecture-assets/internal-architecture.png)](architecture-assets/internal-architecture.png)
+
+The controllers are:
+
+- [Rollout Controller](https://github.com/argoproj/argo-rollouts/blob/master/rollout/controller.go)
+- [Service Controller](https://github.com/argoproj/argo-rollouts/blob/master/service/service.go)
+- [Ingress Controller](https://github.com/argoproj/argo-rollouts/blob/master/ingress/ingress.go)
+- [Experiment Controller](https://github.com/argoproj/argo-rollouts/blob/master/experiments/controller.go)
+- [AnalysisRun Controller](https://github.com/argoproj/argo-rollouts/blob/master/analysis/controller.go)
+
+### Tips
+
+1. You can run the tests using a different kubeconfig by setting the `KUBECONFIG` environment variable:
+
+```shell
+KUBECONFIG=~/.kube/minikube make start-e2e
+KUBECONFIG=~/.kube/minikube make test-e2e
+```
+
+2. To run a specific e2e test, set the `E2E_TEST_OPTIONS` environment variable to specify the test
+   (or test regex):
+
+```shell
+make test-e2e E2E_TEST_OPTIONS="-testify.m ^TestRolloutRestart$"
+```
+
+3. The e2e tests are designed to run as quickly as possible, eliminating readiness and termination
+   delays. However, it is often desired to artificially slow down the tests for debugging purposes,
+   as well as to understand what the test is doing. To delay startup and termination of pods, set the
+   `E2E_POD_DELAY` to an integer value in seconds. This environment variable is often coupled with
+   `E2E_TEST_OPTIONS` to debug and slow down a specific test.
+
+```shell
+make test-e2e E2E_POD_DELAY=10
+```
+
+4. Increasing the timeout. The E2E tests time out waiting on conditions to be met within 60 seconds.
+   If debugging the rollout controller, it may be useful to increase this timeout while say sitting
+   at a debugger breakpoint:
+
+```shell
+make test-e2e E2E_WAIT_TIMEOUT=999999
+```
+
+5. The e2e tests leverage a feature of the controller allowing the controller to be sharded with
+   a user-specific "instance id" label. This allows the tests to operate only on rollouts with the
+   specified label, and prevents any other controllers (including the system rollout controller),
+   from also operating on the same set of rollouts. This value can be changed (from the default of
+   `argo-rollouts-e2e`), using the `E2E_INSTANCE_ID` environment variable:
+
+```shell
+make start-e2e E2E_INSTANCE_ID=foo
+make test-e2e E2E_INSTANCE_ID=foo
+```
+
+Alternatively, the e2e tests can be run against the system controller (i.e. without an instance id):
+
+```shell
+make start-e2e E2E_INSTANCE_ID=''
+```
+
+6. Working on CRDs? While editing them directly works when you are finding the shape of things you want, the final CRDs are autogenerated. Make sure to regenerate them by running `make gen-crd` before submitting PRs. They are controlled by the relevant annotations in the types file:
+
+eg: Analysis Templates are controlled by annotations in `pkg/apis/rollouts/v1alpha1/analysis_types.go`.
+
+Refer to https://book-v1.book.kubebuilder.io/beyond_basics/generating_crd.html and https://book.kubebuilder.io/reference/markers/crd-validation.html for more info on annotations you can use.
+
 ## Running Local Containers
 
 You may need to run containers locally, so here's how:
@@ -109,10 +280,43 @@ Install the manifests:
 kubectl -n argo-rollouts apply -f manifests/install.yaml
 ```
 
+## Upgrading Kubernetes Libraries
+
+Argo Rollouts has a dependency on the kubernetes/kubernetes repo for some of the functionality that has not been
+pushed into the other kubernetes repositories yet. In order to import the kubernetes/kubernetes repo, all of the
+associated repos have to pinned to the correct version specified by the kubernetes/kubernetes release. The
+`./hack/update-k8s-dependencies.sh` updates all the dependencies to the those correct versions.
+
+## Upgrading Notifications Engine
+
+Argo Rollouts has a dependency on the [argoproj/notifications-engines](https://github.com/argoproj/notifications-engine) repo
+for the notifications functionality and related documentation.
+
+This is updated by upgrading the Go library in `go.mod` by running the commands:
+
+```shell
+go get github.com/argoproj/notifications-engine@LATEST_COMMIT_HASH
+go mod tidy
+```
+
+Next the latest notifications documentation can be imported by running:
+
+```shell
+make docs
+```
+
 ## Documentation Changes
-If you need to run the mkdocs server, you will need to do the following:
 
-* Follow the instruction guide to install [mkDocs](https://www.mkdocs.org/#installation)
-* Install the `material` theme with the [following guide](https://squidfunk.github.io/mkdocs-material/#quick-start)
+Modify contents in `docs/` directory.
 
-Afterwards, you can run `mkdocs serve` and access your documentation at [http://127.0.0.1:8000/](http://127.0.0.1:8000/)
+Preview changes in your browser by visiting http://localhost:8000 after running:
+
+```shell
+make serve-docs
+```
+
+To publish changes, run:
+
+```shell
+make release-docs
+```

docs/FAQ.md

@@ -0,0 +1,121 @@
+# FAQ
+
+Be sure to read the [Best practices page](../best-practices) as well.
+
+## General
+
+### Does Argo Rollouts depend on Argo CD or any other Argo project?
+
+Argo Rollouts is a standalone project. Even though it works great with Argo CD and other Argo projects, it can be used
+on its own for Progressive Delivery scenarios. More specifically, Argo Rollouts does **NOT** require that you also have installed Argo CD on the same cluster.
+
+### How does Argo Rollouts integrate with Argo CD?
+Argo CD understands the health of Argo Rollouts resources via Argo CD’s [Lua health check](https://github.com/argoproj/argo-cd/blob/master/docs/operator-manual/health.md). These Health checks understand when the Argo Rollout objects are Progressing, Suspended, Degraded, or Healthy.  Additionally, Argo CD has Lua based Resource Actions that can mutate an Argo Rollouts resource (i.e. unpause a Rollout).
+
+As a result, an operator can build automation to react to the states of the Argo Rollouts resources. For example, if a Rollout created by Argo CD is paused, Argo CD detects that and marks the Application as suspended. Once the new version is verified to be good, the operator can use Argo CD’s resume resource action to unpause the Rollout so it can continue to make progress. 
+
+### Can we run the Argo Rollouts kubectl plugin commands via Argo CD?
+Argo CD supports running Lua scripts to modify resource kinds (i.e. suspending a CronJob by setting the `.spec.suspend` to true). These Lua Scripts can be configured in the argocd-cm ConfigMap or upstreamed to the Argo CD's [resource_customizations](https://github.com/argoproj/argo-cd/tree/master/resource_customizations) directory. These custom actions have two Lua scripts: one to modify the said resource and another to detect if the action can be executed (i.e. A user should not be able to resuming an unpaused Rollout). Argo CD allows users to execute these actions via the UI or CLI.
+
+In the CLI, a user (or a CI system) can run
+```bash
+argocd app actions run <APP_NAME> <ACTION> 
+```
+This command executes the action listed on the application listed.
+
+In the UI, a user can click the hamburger button of a resource and the available actions will appear in a couple of seconds. The user can click and confirm that action to execute it.
+
+Currently, the Rollout action has two available custom actions in Argo CD: resume and restart.
+
+* Resume unpauses a Rollout with a PauseCondition
+* Restart: Sets the RestartAt and causes all the pods to be restarted.
+
+### Does Argo Rollout require a Service Mesh like Istio?
+Argo Rollouts does not require a service mesh or ingress controller to be used. In the absence of a traffic routing provider, Argo Rollouts manages the replica counts of the canary/stable ReplicaSets to achieve the desired canary weights. Normal Kubernetes Service routing (via kube-proxy) is used to split traffic between the ReplicaSets. 
+
+### Does Argo Rollout require we follow GitOps in my organization?
+
+Argo Rollouts is a Kubernetes controller that will react to any manifest change regardless of how the manifest was changed. The manifest can be changed
+by a Git commit, an API call, another controller or even a manual `kubectl` command. You can use Argo Rollouts with any traditional CI/CD
+solution that does not follow the GitOps approach.
+
+### Can we run the Argo Rollouts controller in HA mode?
+
+Yes. A k8s cluster can run multiple replicas of Argo-rollouts controllers to achieve HA. To enable this feature, run the controller with `--leader-elect` flag and increase the number of replicas in the controller's deployment manifest. The implementation is based on the [k8s client-go's leaderelection package](https://pkg.go.dev/k8s.io/client-go/tools/leaderelection#section-documentation). This implementation is tolerant to *arbitrary clock skew* among replicas. The level of tolerance to skew rate can be configured by setting `--leader-election-lease-duration` and `--leader-election-renew-deadline` appropriately. Please refer to the [package documentation](https://pkg.go.dev/k8s.io/client-go/tools/leaderelection#pkg-overview) for details.
+
+### Can we install Argo Rollouts centrally in a cluster and manage Rollout resources in external clusters? 
+
+No you cannot do that (even though Argo CD can work that way). This is by design because the Rollout is a custom resource unknown to vanilla Kubernetes. You need the Rollout CRD as well as the controller in the deployment cluster (every cluster that will use workloads with Rollouts).
+
+### What is the version skew policy between the controller and the kubectl plugin?
+
+The Argo Rollout CLI/Kubectl plugin just patches the Rollout object or reads fields from it. [There is no separate "Argo Rollouts API"](../best-practices#there-is-no-argo-rollouts-api). Old versions of the plugin might not understand new fields that are added in the [Rollout specification](../features/specification/). We have never made a breaking change intentionally (removed something from the Rollout Spec). So old clients should work even with newer Rollout versions (excluding new features).
+
+## Rollouts
+
+### Which deployment strategies does Argo Rollouts support?
+Argo Rollouts supports BlueGreen, Canary, and Rolling Update. Additionally, Progressive Delivery features can be enabled on top of the blue-green/canary update, which further provides advanced deployment such as automated analysis and rollback.
+
+### Does the Rollout object follow the provided strategy when it is first created?
+As with Deployments, Rollouts does not follow the strategy parameters on the initial deploy. The controller tries to get the Rollout into a steady state as fast as possible by creating a fully scaled up ReplicaSet from the provided `.spec.template`. Once the Rollout has a stable ReplicaSet to transition from, the controller starts using the provided strategy to transition the previous ReplicaSet to the desired ReplicaSet.
+
+### How does BlueGreen rollback work?
+A BlueGreen Rollout keeps the old ReplicaSet up and running for 30 seconds or the value of the scaleDownDelaySeconds. The controller tracks the remaining time before scaling down by adding an annotation called `argo-rollouts.argoproj.io/scale-down-deadline` to the old ReplicaSet. If the user applies the old Rollout manifest before the old ReplicaSet scales down, the controller does something called a fast rollback. The controller immediately switches the active service’s selector back to the old ReplicaSet’s rollout-pod-template-hash and removes the scaled down annotation from that ReplicaSet. The controller does not do any of the normal operations when trying to introduce a new version since it is trying to revert as fast as possible. A non-fast-track rollback occurs when the scale down annotation has past and the old ReplicaSet has been scaled down. In this case, the Rollout treats the ReplicaSet like any other new ReplicaSet and follows the usual procedure for deploying a new ReplicaSet.
+
+### What is the `argo-rollouts.argoproj.io/managed-by-rollouts` annotation?
+Argo Rollouts adds an `argo-rollouts.argoproj.io/managed-by-rollouts` annotation to Services and Ingresses that the controller modifies. They are used when the Rollout managing these resources is deleted and the controller tries to revert them back into their previous state.
+
+## Rollbacks
+
+### Does Argo Rollouts write back in Git when a rollback takes place?
+No. Argo Rollouts doesn't read/write anything to Git. Actually Argo Rollouts knows nothing about Git repositories (only Argo CD has this information if it manages the Rollout). When a rollback takes place, Argo Rollouts marks the application as "degraded" and changes the version on the cluster back to the known stable one.
+
+### If I use both Argo Rollouts and Argo CD wouldn't I have an endless loop in the case of a Rollback?
+No there is no endless loop. As explained already in the previous question, Argo Rollouts doesn't tamper with Git in any way. If you use both Argo projects together, the sequence of events for a rollback is the following:
+
+1. Version N runs on the cluster as a Rollout (managed by Argo CD). The Git repository is updated with version N+1 in the Rollout/Deployment manifest
+1. Argo CD sees the changes in Git and updates the live state in the cluster with the new Rollout object
+1. Argo Rollouts takes over as it watches for all changes in Rollout Objects. Argo Rollouts is completely oblivious to what is happening in Git. It only cares about what is happening with Rollout objects that are live in the cluster.
+1. Argo Rollouts tries to apply version N+1 with the selected strategy (e.g. blue/green)
+1. Version N+1 fails to deploy for some reason
+1. Argo Rollouts scales back again (or switches traffic back) to version N in the cluster. **No change in Git takes place from Argo Rollouts**
+1. Cluster is running version N and is completely healthy
+1. The Rollout is marked as "Degraded" both in ArgoCD and Argo Rollouts.
+1. Argo CD syncs take no further action as the Rollout object in Git is exactly the same as in the cluster. They both mention version N+1
+
+
+### So how can I make Argo Rollouts write back in Git when a rollback takes place?
+You don't need to do that if you simply want to go back to the previous version using Argo CD. When a deployment fails, Argo Rollouts automatically sets the cluster back to the stable/previous version as explained in the previous question. You don't need to write anything in Git to achieve this. The cluster is still healthy and you have avoided downtime. You are then expected to fix the issue and roll-forward (i.e. deploy the next version) if you want to follow GitOps in a pedantic manner. If you want Argo Rollouts to write back in Git after a failed deployment then you need to orchestrate this with an external system or write custom glue code. But this is normally not needed.
+
+### What is the relationship between Rollbacks with Argo Rollouts and Rollbacks with Argo CD?
+They are completely unrelated. Argo Rollouts "rollbacks" switch the cluster back to the previous version as explained in the previous question. They don't touch or affect Git in any way. Argo CD [rollbacks](https://argo-cd.readthedocs.io/en/stable/user-guide/commands/argocd_app_rollback/) simply point the cluster back a previous Git hash. Normally if you have Argo Rollouts, you don't need to use the Argo CD rollback command.
+
+
+### How can I deploy multiple services in a single step and roll them back according to their dependencies?
+
+The Rollout specification focuses on a single application/deployment. Argo Rollouts knows nothing about application dependencies. If you want to deploy multiple applications together in a smart way (e.g. automatically rollback a frontend if backend deployment fails) you need to write your own solution
+on top of Argo Rollouts. In most cases, you would need one Rollout resource for each application that you
+are deploying. Ideally you should also make your services backwards and forwards compatible (i.e. frontend should be able to work with both backend-preview and backend-active).
+
+### How can I run my own custom tests (e.g. smoke tests) to decide if a Rollback should take place or not?
+Use a custom [Job](https://argo-rollouts.readthedocs.io/en/stable/analysis/job/) or [Web](https://argo-rollouts.readthedocs.io/en/stable/analysis/web/) Analysis. You can pack all your smoke tests in a single container and run them as a Job analysis. Argo Rollouts will use the results of the analysis to automatically rollback if the tests fail.
+
+
+## Experiments
+
+### Why doesn't my Experiment end?
+An Experiment’s duration is controlled by the `.spec.duration` field and the analyses created for the Experiment. The `.spec.duration` indicates how long the ReplicaSets created by the Experiment should run. Once the duration passes, the experiment scales down the ReplicaSets it created and marks the AnalysisRuns successful unless the `requiredForCompletion` field is used in the Experiment. If enabled, the ReplicaSets are still scaled-down, but the Experiment does not finish until the Analysis Run finishes.
+
+Additionally, the `.spec.duration` is an optional field. If it’s left unset, and the Experiment creates no AnalysisRuns, the ReplicaSets run indefinitely. The Experiment creates AnalysisRuns without the `requiredForCompletion` field, the Experiment fails only when the AnalysisRun created fails or errors out. If the `requiredForCompletion` field is set, the Experiment only marks itself as Successful and scales down the created ReplicaSets when the AnalysisRun finishes Successfully.
+
+Additionally, an Experiment ends if the `.spec.terminate` field is set to true regardless of the state of the Experiment.
+
+## Analysis
+### Why doesn't my AnalysisRun end?
+The AnalysisRun’s duration is controlled by the metrics specified. Each Metric can specify an interval, count, and various limits (ConsecutiveErrorLimit, InconclusiveLimit, FailureLimit). If the interval is omitted, the AnalysisRun takes a single measurement. The count indicates how many measurements should be taken and causes the AnalysisRun to run indefinitely if omitted. The ConsecutiveErrorLimit, InconclusiveLimit, and FailureLimit define the thresholds allowed before putting the rollout into a completed state.
+
+Additionally, an AnalysisRun ends if the `.spec.terminate` field is set to true regardless of the state of the AnalysisRun.
+
+### What is the difference between failures and errors?
+Failures are when the failure condition evaluates to true or an AnalysisRun without a failure condition evaluates the success condition to false. Errors are when the controller has any kind of issue with taking a measurement (i.e. invalid Prometheus URL).
+

docs/analysis/cloudwatch.md

@@ -0,0 +1,140 @@
+# CloudWatch Metrics
+
+!!! important
+    Available since v1.1.0
+
+A [CloudWatch](https://aws.amazon.com/cloudwatch/) using [GetMetricData](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_GetMetricData.html) can be used to obtain measurements for analysis.
+
+## Setup
+
+You can use CloudWatch Metrics if you have used to EKS or not. This analysis is required IAM permission for `cloudwatch:GetMetricData` and you need to define `AWS_REGION` in Deployment for `argo-rollouts`.
+
+### EKS
+
+If you create new cluster on EKS, you can attach [cluster IAM role](https://docs.aws.amazon.com/eks/latest/userguide/service_IAM_role.html) or attach [IAM roles for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html).
+If you have already cluster on EKS, you can attach [IAM roles for service accounts](https://docs.aws.amazon.com/eks/latest/userguide/iam-roles-for-service-accounts.html).
+
+### not EKS
+
+You need to define access key and secret key.
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: cloudwatch-secret
+type: Opaque
+stringData:
+  AWS_ACCESS_KEY_ID: <aws-access-key-id>
+  AWS_SECRET_ACCESS_KEY: <aws-secret-access-key>
+  AWS_REGION: <aws-region>
+```
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: argo-rollouts
+spec:
+  template:
+    spec:
+      containers:
+      - name: argo-rollouts
+        env:
+        - name: AWS_ACCESS_KEY_ID
+          valueFrom:
+            secretKeyRef:
+              name: cloudwatch-secret
+              key: AWS_ACCESS_KEY_ID
+        - name: AWS_SECRET_ACCESS_KEY
+          valueFrom:
+            secretKeyRef:
+              name: cloudwatch-secret
+              key: AWS_SECRET_ACCESS_KEY
+        - name: AWS_REGION
+          valueFrom:
+            secretKeyRef:
+              name: cloudwatch-secret
+              key: AWS_REGION
+```
+
+## Configuration
+
+- `metricDataQueries` - GetMetricData query: [MetricDataQuery](https://docs.aws.amazon.com/AmazonCloudWatch/latest/APIReference/API_MetricDataQuery.html)
+- `interval` - optional interval, e.g. 30m, default: 5m
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: success-rate
+spec:
+  metrics:
+  - name: success-rate
+    interval: 1m
+    successCondition: "len(result[0].Values) >= 5 and all(result[0].Values, {# <= 0.01})"
+    failureLimit: 3
+    provider:
+      cloudWatch:
+        interval: 30m
+        metricDataQueries:
+        - {
+            "id": "rate",
+            "expression": "errors / requests"
+          }
+        - {
+            "id": "errors",
+            "metricStat": {
+              "metric": {
+                "namespace": "app",
+                "metricName": "errors"
+              },
+              "period": 300,
+              "stat": "Sum",
+              "unit": "Count"
+            },
+            "returnData": false
+          }
+        - {
+            "id": "requests",
+            "metricStat": {
+              "metric": {
+                "namespace": "app",
+                "metricName": "requests"
+              },
+              "period": 300,
+              "stat": "Sum",
+              "unit": "Count"
+            },
+            "returnData": false
+          }
+```
+
+## debug
+
+You can confirm the results value in `AnalysisRun`.
+
+```bash
+$ kubectl get analysisrun/rollouts-name-xxxxxxxxxx-xx -o yaml
+(snip)
+status:
+  metricResults:
+  - count: 2
+    failed: 1
+    measurements:
+    - finishedAt: "2021-09-08T17:29:14Z"
+      phase: Failed
+      startedAt: "2021-09-08T17:29:13Z"
+      value: '[[0.0029476787030213707 0.006100422336931018 0.01020408163265306 0.007932573128408527
+        0.00589622641509434 0.006339144215530904]]'
+    - finishedAt: "2021-09-08T17:30:14Z"
+      phase: Successful
+      startedAt: "2021-09-08T17:30:14Z"
+      value: '[[0.004484304932735426 0.0058374494836102376 0.006736068585425597 0.008444444444444444
+        0.006859756097560976 0.0045385779122541605]]'
+    name: success-rate
+    phase: Running
+    successful: 1
+  phase: Running
+  startedAt: "2021-09-08T17:29:14Z"
+```

docs/analysis/datadog.md

@@ -0,0 +1,243 @@
+# Datadog Metrics
+
+A [Datadog](https://www.datadoghq.com/) query can be used to obtain measurements for analysis.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: loq-error-rate
+spec:
+  args:
+  - name: service-name
+  metrics:
+  - name: error-rate
+    interval: 5m
+    successCondition: result <= 0.01
+    failureLimit: 3
+    provider:
+      datadog:
+        apiVersion: v2
+        interval: 5m
+        query: |
+          sum:requests.error.rate{service:{{args.service-name}}}
+```
+
+The field `apiVersion` refers to the API version of Datadog (v1 or v2). Default value is `v1` if this is omitted. See "Working with Datadog API v2" below for more information.
+
+Datadog api and app tokens can be configured in a kubernetes secret in argo-rollouts namespace.
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: datadog
+type: Opaque
+stringData:
+  address: https://api.datadoghq.com
+  api-key: <datadog-api-key>
+  app-key: <datadog-app-key>
+```
+
+`apiVersion` here is different from the `apiVersion` from the Datadog configuration above.
+
+!!! important
+    ###### Namespaced secret
+    Datadog integration supports referring to secrets inside the same namespace as argo-rollouts (by default)
+    or referring to a secret in the same namespace as the `AnalysisTemplate`.
+
+    To use a secret from the `AnalysisTemplate` namespace, include a `secretRef` section in the template, specifying the `name` of the secret and setting the `namespaced` property to `true`.
+
+    The process for retrieving Datadog credentials is as follows:
+    1. **If a `secretRef` is defined in the `AnalysisTemplate`:** Argo Rollouts will search for the secret with the specified name in the namespace where the template resides.
+    2. **If the secret is not found in the specified namespace:** Argo Rollouts will then check the environment variables.
+    3. **If the credentials are not found in environment variables:** Argo Rollouts will look for a secret named "Datadog" in the namespace where Argo Rollouts itself is deployed.
+
+--- 
+
+Let me know if there's anything else you'd like to adjust!
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: loq-error-rate
+spec:
+  args:
+  - name: service-name
+  metrics:
+  - name: error-rate
+    interval: 5m
+    successCondition: result <= 0.01
+    failureLimit: 3
+    provider:
+      datadog:
+        apiVersion: v2
+        interval: 5m
+        secretRef:
+          name: "mysecret"
+          namespaced: true
+        query: |
+          sum:requests.error.rate{service:{{args.service-name}}}
+  ```
+
+
+
+### Working with Datadog API v2
+
+!!! important
+    While some basic v2 functionality is working in earlier versions, the new properties of `formula` and `queries` are only available as of v1.7
+
+#### Moving to v2
+
+If your old v1 was just a simple metric query - no formula as part of the query - then you can just move to v2 by updating the `apiVersion` in your existing Analysis Template, and everything should work.
+
+If you have a formula, you will need to update how you configure your metric. Here is a before/after example of what your Analysis Template should look like:
+
+Before:
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: log-error-rate
+spec:
+  args:
+  - name: service-name
+  metrics:
+  - name: error-rate
+    interval: 30s
+    successCondition: default(result, 0) < 10
+    failureLimit: 3
+    provider:
+      datadog:
+        apiVersion: v1
+        interval: 5m
+        query: "moving_rollup(sum:requests.errors{service:{{args.service-name}}}.as_count(), 60, 'sum') / sum:requests{service:{{args.service-name}}}.as_count()"
+```
+
+After:
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: loq-error-rate
+spec:
+  args:
+  - name: service-name
+  metrics:
+  - name: error-rate
+    # Polling rate against the Datadog API
+    interval: 30s
+    successCondition: default(result, 0) < 10
+    failureLimit: 3
+    provider:
+      datadog:
+        apiVersion: v2
+        # The window of time we are looking at in DD. Basically we will fetch data from (now-5m) to now.
+        interval: 5m
+        queries:
+          a: sum:requests.errors{service:{{args.service-name}}}.as_count()
+          b: sum:requests{service:{{args.service-name}}}.as_count()
+        formula: "moving_rollup(a, 60, 'sum') / b"
+```
+
+#### Examples
+
+Simple v2 query with no formula
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: canary-container-restarts
+spec:
+  args:
+    # This is set in rollout using the valueFrom: podTemplateHashValue functionality
+    - name: canary-hash
+    - name: service-name
+    - name: restarts.initial-delay
+      value: "60s"
+    - name: restarts.max-restarts
+      value: "4"
+  metrics:
+    - name: kubernetes.containers.restarts
+      initialDelay: "{{ args.restarts.initial-delay }}"
+      interval: 15s
+      failureCondition: default(result, 0) > {{ args.restarts.max-restarts }}
+      failureLimit: 0
+      provider:
+        datadog:
+          apiVersion: v2
+          interval: 5m
+          queries:
+            # The key is arbitrary - you will use this key to refer to the query if you use a formula.
+            q: "max:kubernetes.containers.restarts{service-name:{{args.service-name}},rollouts_pod_template_hash:{{args.canary-hash}}}"
+```
+
+### Tips
+
+#### Datadog Results
+
+Datadog queries can return empty results if the query takes place during a time interval with no metrics. The Datadog provider will return a `nil` value yielding an error during the evaluation phase like:
+
+```
+invalid operation: < (mismatched types <nil> and float64)
+```
+
+However, empty query results yielding a `nil` value can be handled using the `default()` function. Here is a succeeding example using the `default()` function:
+
+```yaml
+successCondition: default(result, 0) < 0.05
+```
+
+#### Metric aggregation (v2 only)
+
+By default, Datadog analysis run is configured to use `last` metric aggregator when querying Datadog v2 API. This value can be overridden by specifying a new `aggregator` value from a list of supported aggregators (`avg,min,max,sum,last,percentile,mean,l2norm,area`) for the V2 API ([docs](https://docs.datadoghq.com/api/latest/metrics/#query-scalar-data-across-multiple-products)).
+
+For example, using count-based distribution metric (`count:metric{*}.as_count()`) with values `1,9,3,7,5` in a given `interval` will make `last` aggregator return `5`. To return a sum of all values (`25`), set `aggregator: sum` in Datadog provider block and use `moving_rollup()` function to aggregate values in the specified rollup interval. These functions can be combined in a `formula` to perform additional calculations:
+
+```yaml
+...<snip>
+  metrics:
+  - name: error-percentage
+    interval: 30s
+    successCondition: default(result, 0) < 5
+    failureLimit: 3
+    provider:
+      datadog:
+        apiVersion: v2
+        interval: 5m
+        aggregator: sum # override default aggregator
+        queries:
+          a: count:requests.errors{service:my-service}.as_count()
+          b: count:requests{service:my-service}.as_count()
+        formula: "moving_rollup(a, 300, 'sum') / moving_rollup(b, 300, 'sum') * 100" # percentage of requests with errors
+```
+
+#### Templates and Helm
+
+Helm and Argo Rollouts both try to parse things between `{{ ... }}` when rendering templates. If you use Helm to deliver your manifests, you will need to escape `{{ args.whatever }}`. Using the example above, here it is set up for Helm:
+
+```yaml
+...<snip>
+metrics:
+  - name: kubernetes.containers.restarts
+      initialDelay: "{{ `{{ args.restarts.initial-delay }}` }}"
+    interval: 15s
+      failureCondition: default(result, 0) > {{ `{{ args.restarts.max-restarts }}` }}
+    failureLimit: 0
+    provider:
+      datadog:
+        apiVersion: v2
+        interval: 5m
+        queries:
+          q: "{{ `max:kubernetes.containers.restarts{kube_app_name:{{args.kube_app_name}},rollouts_pod_template_hash:{{args.canary-hash}}}` }}"
+```
+
+#### Rate Limits
+
+For the `v1` API, you ask for an increase on the `api/v1/query` route.
+
+For the `v2` API, the Ratelimit-Name you ask for an increase in is the `query_scalar_public`.

docs/analysis/graphite.md

@@ -0,0 +1,35 @@
+# Graphite Metrics
+
+A [Graphite](https://graphiteapp.org/) query can be used to obtain measurements for analysis.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: success-rate
+spec:
+  args:
+  - name: service-name
+  metrics:
+  - name: success-rate
+    interval: 5m
+    # Note that the Argo Rollouts Graphite metrics provider returns results as an array of float64s with 6 decimal places.
+    successCondition: results[0] >= 90.000000
+    failureLimit: 3
+    provider:
+      graphite:
+        address: http://graphite.example.com:9090
+        query: |
+          target=summarize(
+            asPercent(
+              sumSeries(
+                stats.timers.httpServerRequests.app.{{args.service-name}}.exception.*.method.*.outcome.{CLIENT_ERROR,INFORMATIONAL,REDIRECTION,SUCCESS}.status.*.uri.*.count
+              ),
+              sumSeries(
+                stats.timers.httpServerRequests.app.{{args.service-name}}.exception.*.method.*.outcome.*.status.*.uri.*.count
+              )
+            ),
+            '5min',
+            'avg'
+          )
+```

docs/analysis/influxdb.md

@@ -0,0 +1,41 @@
+# InfluxDB Metrics
+
+An [InfluxDB](https://www.influxdata.com/) query using [Flux](https://docs.influxdata.com/influxdb/cloud/query-data/get-started/query-influxdb/) can be used to obtain measurements for analysis.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: error-rate
+spec:
+  args:
+  - name: application-name
+  metrics:
+  - name: error-rate
+    # NOTE: To be consistent with the prometheus metrics provider InfluxDB query results are returned as an array.
+    # In the example we're looking at index 0 of the returned array to obtain the value we're using for the success condition
+    successCondition: result[0] <= 0.01
+    provider:
+      influxdb:
+        profile: my-influxdb-secret  # optional, defaults to 'influxdb'
+        query: |
+          from(bucket: "app_istio")
+            |> range(start: -15m)
+            |> filter(fn: (r) => r["destination_workload"] == "{{ args.application-name }}")
+            |> filter(fn: (r) => r["_measurement"] == "istio:istio_requests_errors_percentage:rate1m:5xx")
+
+```
+
+An InfluxDB access profile can be configured using a Kubernetes secret in the `argo-rollouts` namespace. Alternate accounts can be used by creating more secrets of the same format and specifying which secret to use in the metric provider configuration using the `profile` field.
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: influxdb
+type: Opaque
+stringData:
+  address: <infuxdb-url>
+  authToken: <influxdb-auth-token>
+  org: <influxdb-org>
+```

docs/analysis/job.md

@@ -0,0 +1,36 @@
+# Job Metrics
+
+A Kubernetes Job can be used to run analysis. When a Job is used, the metric is considered
+successful if the Job completes and had an exit code of zero, otherwise it is failed.
+
+```yaml
+metrics:
+  - name: test
+    provider:
+      job:
+        metadata:
+          annotations:
+            foo: bar # annotations defined here will be copied to the Job object
+          labels:
+            foo: bar # labels defined here will be copied to the Job object
+        spec:
+          backoffLimit: 1
+          template:
+            spec:
+              containers:
+                - name: test
+                  image: my-image:latest
+                  command:
+                    [my-test-script, my-service.default.svc.cluster.local]
+              restartPolicy: Never
+```
+
+## Control where the jobs run
+
+Argo Rollouts allows you some control over where your metric job runs.
+
+The following env vars can be set on the Rollouts controller:
+
+`ARGO_ROLLOUTS_ANALYSIS_JOB_NAMESPACE` will allow you to run your metric jobs in a namespace other than the default (which can vary depending on if you are running Rollouts in cluster mode or not).
+
+`ARGO_ROLLOUTS_ANALYSIS_JOB_KUBECONFIG` will allow running metric jobs in a different cluster entirely. This should be a path to the kubeconfig you want to use.

docs/analysis/kayenta.md

@@ -0,0 +1,155 @@
+## Kayenta (e.g. Mann-Whitney Analysis)
+
+Analysis can also be done as part of an [Experiment](../features/experiment.md).
+
+This example starts both a canary and baseline ReplicaSet. The ReplicaSets run for 1 hour, then
+scale down to zero. Call out to Kayenta to perform Mann-Whintney analysis against the two pods. Demonstrates ability to start a
+short-lived experiment and an asynchronous analysis.
+
+This example demonstrates:
+
+* The ability to start an [Experiment](../features/experiment.md) as part of rollout steps, which launches multiple ReplicaSets (e.g. baseline & canary)
+* The ability to reference and supply pod-template-hash to an AnalysisRun
+* Kayenta metrics
+
+=== "Rollout"
+
+    ```yaml
+    apiVersion: argoproj.io/v1alpha1
+    kind: Rollout
+    metadata:
+      name: guestbook
+      labels:
+        app: guestbook
+    spec:
+      strategy:
+        canary:
+          steps:
+          - experiment:
+              duration: 1h
+              templates:
+              - name: baseline
+                specRef: stable
+              - name: canary
+                specRef: canary
+              analyses:
+              - templateName: mann-whitney
+                args:
+                - name: stable-hash
+                  valueFrom:
+                    podTemplateHashValue: Stable
+                - name: canary-hash
+                  valueFrom:
+                    podTemplateHashValue: Latest
+    ```
+
+=== "AnalysisTemplate"
+
+    ```yaml
+    apiVersion: argoproj.io/v1alpha1
+    kind: AnalysisTemplate
+    metadata:
+      name: mann-whitney
+    spec:
+      args:
+      - name: start-time
+      - name: end-time
+      - name: stable-hash
+      - name: canary-hash
+      metrics:
+      - name: mann-whitney
+        provider:
+          kayenta:
+            address: https://kayenta.example.com
+            application: guestbook
+            canaryConfigName: my-test
+            thresholds:
+              pass: 90
+              marginal: 75
+            scopes:
+            - name: default
+              controlScope:
+                scope: app=guestbook and rollouts-pod-template-hash={{args.stable-hash}}
+                step: 60
+                start: "{{args.start-time}}"
+                end: "{{args.end-time}}"
+              experimentScope:
+                scope: app=guestbook and rollouts-pod-template-hash={{args.canary-hash}}
+                step: 60
+                start: "{{args.start-time}}"
+                end: "{{args.end-time}}"
+    ```
+
+=== "Experiment"
+
+    ```yaml
+    # This is the resulting experiment that is produced by the step
+    apiVersion: argoproj.io/v1alpha1
+    kind: Experiment
+    name:
+      name: guestbook-6c54544bf9-0
+    spec:
+      duration: 1h
+      templates:
+      - name: baseline
+        replicas: 1
+        spec:
+          containers:
+          - name: guestbook
+            image: guestbook:v1
+      - name: canary
+        replicas: 1
+        spec:
+          containers:
+          - name: guestbook
+            image: guestbook:v2
+      analysis:
+        templateName: mann-whitney
+        args:
+        - name: start-time
+          value: "{{experiment.availableAt}}"
+        - name: end-time
+          value: "{{experiment.finishedAt}}"
+    ```
+
+
+In order to perform multiple kayenta runs over some time duration, the `interval` and `count` fields
+can be supplied. When the `start` and `end` fields are omitted from the kayenta scopes, the values
+will be implicitly decided as:
+
+* `start` = if `lookback: true` start of analysis, otherwise current time - interval
+* `end` = current time
+
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: mann-whitney
+spec:
+  args:
+  - name: stable-hash
+  - name: canary-hash
+  metrics:
+  - name: mann-whitney
+    provider:
+      kayenta:
+        address: https://kayenta.intuit.com
+        application: guestbook
+        canaryConfigName: my-test
+        interval: 3600
+        count: 3
+        # loopback will cause start time value to be equal to start of analysis
+        # lookback: true
+        thresholds:
+          pass: 90
+          marginal: 75
+        scopes:
+        - name: default
+          controlScope:
+            scope: app=guestbook and rollouts-pod-template-hash={{args.stable-hash}}
+            step: 60
+          experimentScope:
+            scope: app=guestbook and rollouts-pod-template-hash={{args.canary-hash}}
+            step: 60
+```

docs/analysis/newrelic.md

@@ -0,0 +1,65 @@
+# NewRelic Metrics
+
+!!! important
+    Available since v0.10.0
+
+A [New Relic](https://newrelic.com/) query using [NRQL](https://docs.newrelic.com/docs/query-your-data/nrql-new-relic-query-language/get-started/introduction-nrql-new-relics-query-language) can be used to obtain measurements for analysis.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: success-rate
+spec:
+  args:
+  - name: application-name
+  metrics:
+  - name: success-rate
+    successCondition: result.successRate >= 0.95
+    provider:
+      newRelic:
+        profile: my-newrelic-secret  # optional, defaults to 'newrelic'
+        query: |
+          FROM Transaction SELECT percentage(count(*), WHERE httpResponseCode != 500) as successRate where appName = '{{ args.application-name }}'
+        timeout: 10 # NRQL query timeout in seconds. Optional, defaults to 5
+```
+
+The `result` evaluated for the condition will always be map or list of maps. The name will follow the pattern of either `function` or `function.field`, e.g. `SELECT average(duration) from Transaction` will yield `average.duration`. In this case the field result cannot be accessed with dot notation and instead should be accessed like `result['average.duration']`. Query results can be renamed using the [NRQL clause `AS`](https://docs.newrelic.com/docs/query-your-data/nrql-new-relic-query-language/get-started/nrql-syntax-clauses-functions#sel-as) as seen above.
+
+A New Relic access profile can be configured using a Kubernetes secret in the `argo-rollouts` namespace. Alternate accounts can be used by creating more secrets of the same format and specifying which secret to use in the metric provider configuration using the `profile` field.
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: newrelic
+type: Opaque
+stringData:
+  personal-api-key: <newrelic-personal-api-key>
+  account-id: <newrelic-account-id>
+  region: "us" # optional, defaults to "us" if not set. Only set to "eu" if you use EU New Relic
+```
+
+To use the New Relic metric provider from behind a proxy, provide a `base-url-rest` key pointing to the base URL of the New Relic REST API for your proxy, and a `base-url-nerdgraph` key pointing to the base URL for NerdGraph for your proxy:
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: newrelic
+type: Opaque
+stringData:
+  personal-api-key: <newrelic-personal-api-key>
+  account-id: <newrelic-account-id>
+  region: "us" # optional, defaults to "us" if not set. Only set to "eu" if you use EU New Relic
+  base-url-rest: <your-base-url>
+  base-url-nerdgraph: <your-base-url>
+```
+
+## Additional Metadata
+
+The New Relic provider returns the below metadata under the `Metadata` map in the `MetricsResult` object of `AnalysisRun`.
+
+| KEY                   | Description |
+|-----------------------|-------------|
+| ResolvedNewRelicQuery | Resolved query after substituting the template's arguments |

docs/analysis/plugins.md

@@ -0,0 +1,107 @@
+# Metric Plugins
+
+!!! warning "Alpha Feature (Since 1.5.0)"
+
+    This is an experimental, [alpha-quality](https://github.com/argoproj/argoproj/blob/main/community/feature-status.md#alpha)
+    feature that allows you to support metric providers that are not natively supported.
+
+Argo Rollouts supports getting analysis metrics via 3rd party [plugin system](../plugins.md). This allows users to extend the capabilities of Rollouts
+to support metric providers that are not natively supported. Rollout's uses a plugin library called
+[go-plugin](https://github.com/hashicorp/go-plugin) to do this. You can find a sample plugin
+here: [rollouts-plugin-metric-sample-prometheus](https://github.com/argoproj-labs/rollouts-plugin-metric-sample-prometheus)
+
+## Installing
+
+There are two methods of installing and using an argo rollouts plugin. The first method is to mount up the plugin executable
+into the rollouts controller container. The second method is to use an HTTP(S) server to host the plugin executable.
+
+### Mounting the plugin executable into the rollouts controller container
+
+There are a few ways to mount the plugin executable into the rollouts controller container. Some of these will depend on your
+particular infrastructure. Here are a few methods:
+
+- Using an init container to download the plugin executable
+- Using a Kubernetes volume mount with a shared volume such as NFS, EBS, etc.
+- Building the plugin into the rollouts controller container
+
+Then you can use the configmap to point to the plugin executable file location. Example:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argo-rollouts-config
+data:
+  metricProviderPlugins: |-
+    - name: "argoproj-labs/sample-prometheus" # name of the plugin, it must match the name required by the plugin so it can find its configuration
+      location: "file://./my-custom-plugin" # supports http(s):// urls and file://
+```
+
+### Using an HTTP(S) server to host the plugin executable
+
+!!! warning "Installing a plugin with http(s)"
+
+    Depending on which method you use to install and the plugin, there are some things to be aware of.
+    The rollouts controller will not start if it can not download or find the plugin executable. This means that if you are using
+    a method of installation that requires a download of the plugin and the server hosting the plugin for some reason is not available and the rollouts
+    controllers pod got deleted while the server was down or is coming up for the first time, it will not be able to start until
+    the server hosting the plugin is available again.
+
+    Argo Rollouts will download the plugin at startup only once but if the pod is deleted it will need to download the plugin again on next startup. Running
+    Argo Rollouts in HA mode can help a little with this situation because each pod will download the plugin at startup. So if a single pod gets
+    deleted during a server outage, the other pods will still be able to take over because there will already be a plugin executable available to it. It is the
+    responsibility of the Argo Rollouts administrator to define the plugin installation method considering the risks of each approach.
+
+Argo Rollouts supports downloading the plugin executable from an HTTP(S) server. To use this method, you will need to
+configure the controller via the `argo-rollouts-config` configmap and set `pluginLocation` to a http(s) url. Example:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: argo-rollouts-config
+data:
+  metricProviderPlugins: |-
+    - name: "argoproj-labs/sample-prometheus" # name of the plugin, it must match the name required by the plugin so it can find its configuration
+      location: "https://github.com/argoproj-labs/rollouts-plugin-metric-sample-prometheus/releases/download/v0.0.4/metric-plugin-linux-amd64" # supports http(s):// urls and file://
+      sha256: "dac10cbf57633c9832a17f8c27d2ca34aa97dd3d" #optional sha256 checksum of the plugin executable
+      headersFrom: #optional headers for the download via http request 
+        - secretRef:
+            name: secret-name
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: secret-name
+stringData:
+  Authorization: Basic <Base 64 TOKEN>
+  My-Header: value
+```
+
+
+
+## Some words of caution
+
+Depending on which method you use to install and the plugin, there are some things to be aware of.
+The rollouts controller will not start if it can not download or find the plugin executable. This means that if you are using
+a method of installation that requires a download of the plugin and the server hosting the plugin for some reason is not available and the rollouts
+controllers pod got deleted while the server was down or is coming up for the first time, it will not be able to start until
+the server hosting the plugin is available again.
+
+Argo Rollouts will download the plugin at startup only once but if the pod is deleted it will need to download the plugin again on next startup. Running
+Argo Rollouts in HA mode can help a little with this situation because each pod will download the plugin at startup. So if a single pod gets
+deleted during a server outage, the other pods will still be able to take over because there will already be a plugin executable available to it. It is the
+responsibility of the Argo Rollouts administrator to define the plugin installation method considering the risks of each approach.
+
+## List of Available Plugins (alphabetical order)
+
+If you have created a plugin, please submit a PR to add it to this list.
+
+### [rollouts-plugin-metric-opensearch](https://github.com/argoproj-labs/rollouts-plugin-metric-opensearch)
+
+- The application is an OpenSearch plugin designed for use with the Argo Rollouts plugin system. This plugin enables the integration of OpenSearch metrics into Argo Rollouts, allowing for advanced metric analysis and monitoring during application rollouts.
+
+### [rollouts-plugin-metric-sample-prometheus](https://github.com/argoproj-labs/rollouts-plugin-metric-sample-prometheus)
+
+- This is just a sample plugin that can be used as a starting point for creating your own plugin.
+  It is not meant to be used in production. It is based on the built-in prometheus provider.

docs/analysis/prometheus.md

@@ -0,0 +1,179 @@
+# Prometheus Metrics
+
+A [Prometheus](https://prometheus.io/) query can be used to obtain measurements for analysis.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: success-rate
+spec:
+  args:
+  - name: service-name
+  metrics:
+  - name: success-rate
+    interval: 5m
+    # NOTE: prometheus queries return results in the form of a vector.
+    # So it is common to access the index 0 of the returned array to obtain the value
+    successCondition: result[0] >= 0.95
+    failureLimit: 3
+    provider:
+      prometheus:
+        address: http://prometheus.example.com:9090
+        # timeout is expressed in seconds
+        timeout: 40
+        headers:
+        - key: X-Scope-OrgID
+          value: tenant_a
+        query: |
+          sum(irate(
+            istio_requests_total{reporter="source",destination_service=~"{{args.service-name}}",response_code!~"5.*"}[5m]
+          )) /
+          sum(irate(
+            istio_requests_total{reporter="source",destination_service=~"{{args.service-name}}"}[5m]
+          ))
+```
+
+The example shows Istio metrics, but you can use any kind of metric available to your prometheus instance. We suggest
+you validate your [PromQL expression](https://prometheus.io/docs/prometheus/latest/querying/basics/) using the [Prometheus GUI first](https://prometheus.io/docs/introduction/first_steps/#using-the-expression-browser).
+
+See the [Analysis Overview page](../../features/analysis) for more details on the available options.
+
+## Range queries
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: range-query-example
+spec:
+  args:
+  - name: service-name
+  - name: lookback-duration
+    value: 5m
+  metrics:
+  - name: success-rate
+    # checks that all returned values are under 1000ms
+    successCondition: "all(result, # < 1000)"
+    failureLimit: 3
+    provider:
+      prometheus:
+        rangeQuery:
+          # See https://expr-lang.org/docs/language-definition#date-functions
+          # for value date functions
+          # The start point to query from
+          start: 'now() - duration("{{args.lookback-duration}}")'
+          # The end point to query to
+          end: 'now()'
+          # Query resolution width 
+          step: 1m
+        address: http://prometheus.example.com:9090
+        query: http_latency_ms{service="{{args.service-name}}"}
+```
+
+### Range query and successCondition/failureCondition
+
+Since range queries will usually return multiple values from prometheus. It is important to assert on every value returned. See the following examples:
+
+* ❌ `result[0] < 1000` - this will only check the first value returned
+* ✅ `all(result, # < 1000)` - checks every value returns from prometheus
+
+See [expr](https://github.com/expr-lang/expr) for more expression options.
+
+## Authorization
+
+### Utilizing Amazon Managed Prometheus
+
+Amazon Managed Prometheus can be used as the prometheus data source for analysis. In order to do this the namespace where your analysis is running will have to have the appropriate [IRSA attached](https://docs.aws.amazon.com/prometheus/latest/userguide/AMP-onboard-ingest-metrics-new-Prometheus.html#AMP-onboard-new-Prometheus-IRSA) to allow for prometheus queries. Once you ensure the proper permissions are in place to access AMP, you can use an AMP workspace url in your ```provider``` block and add a SigV4 config for Sigv4 signing:
+
+```yaml
+provider:
+  prometheus:
+    address: https://aps-workspaces.$REGION.amazonaws.com/workspaces/$WORKSPACEID
+    query: |
+      sum(irate(
+        istio_requests_total{reporter="source",destination_service=~"{{args.service-name}}",response_code!~"5.*"}[5m]
+      )) /
+      sum(irate(
+        istio_requests_total{reporter="source",destination_service=~"{{args.service-name}}"}[5m]
+      ))
+    authentication:
+      sigv4:
+        region: $REGION
+        profile: $PROFILE
+        roleArn: $ROLEARN
+```
+
+### With OAuth2
+
+You can setup an [OAuth2 client credential](https://datatracker.ietf.org/doc/html/rfc6749#section-4.4) flow using the following values:
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: success-rate
+spec:
+  args:
+  - name: service-name
+  # from secret
+  - name: oauthSecret  # This is the OAuth2 shared secret
+    valueFrom:
+      secretKeyRef:
+        name: oauth-secret
+        key: secret
+  metrics:
+  - name: success-rate
+    interval: 5m
+    # NOTE: prometheus queries return results in the form of a vector.
+    # So it is common to access the index 0 of the returned array to obtain the value
+    successCondition: result[0] >= 0.95
+    failureLimit: 3
+    provider:
+      prometheus:
+        address: http://prometheus.example.com:9090
+        # timeout is expressed in seconds
+        timeout: 40
+        authentication:
+          oauth2:
+            tokenUrl: https://my-oauth2-provider/token
+            clientId: my-client-id
+            clientSecret: "{{ args.oauthSecret }}"
+            scopes: [
+              "my-oauth2-scope"
+            ]
+        query: |
+          sum(irate(
+            istio_requests_total{reporter="source",destination_service=~"{{args.service-name}}",response_code!~"5.*"}[5m]
+          )) /
+          sum(irate(
+            istio_requests_total{reporter="source",destination_service=~"{{args.service-name}}"}[5m]
+          ))
+```
+
+The AnalysisRun will first get an access token using that information, and provide it as an `Authorization: Bearer` header for the metric provider call.
+
+## Additional Metadata
+
+Any additional metadata from the Prometheus controller, like the resolved queries after substituting the template's
+arguments, etc. will appear under the `Metadata` map in the `MetricsResult` object of `AnalysisRun`.
+
+
+
+## Skip TLS verification
+
+You can skip the TLS verification of the prometheus host provided by setting the options `insecure: true`.
+
+```yaml
+provider:
+  prometheus:
+    address: https://prometheus.example.com
+    insecure: true
+    query: |
+      sum(irate(
+        istio_requests_total{reporter="source",destination_service=~"{{args.service-name}}",response_code!~"5.*"}[5m]
+      )) /
+      sum(irate(
+        istio_requests_total{reporter="source",destination_service=~"{{args.service-name}}"}[5m]
+      ))
+```

docs/analysis/skywalking.md

@@ -0,0 +1,35 @@
+# Apache SkyWalking Metrics
+
+!!! important
+    Available since v1.5.0
+
+A [SkyWalking](https://skywalking.apache.org/) query using GraphQL can be used to obtain measurements for analysis.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: apdex
+spec:
+  args:
+  - name: service-name
+  metrics:
+  - name: apdex
+    interval: 5m
+    successCondition: "all(result.service_apdex.values.values, {asFloat(.value) >= 9900})"
+    failureLimit: 3
+    provider:
+      skywalking:
+        interval: 3m
+        address: http://skywalking-oap.istio-system:12800
+        query: |
+          query queryData($duration: Duration!) {
+            service_apdex: readMetricsValues(
+              condition: { name: "service_apdex", entity: { scope: Service, serviceName: "{{ args.service-name }}", normal: true } },
+              duration: $duration) {
+                label values { values { value } }
+              }
+          }
+```
+
+The `result` evaluated for the query depends on the specific GraphQL you give, you can try to run the GraphQL query first and inspect the output format, then compose the condition.

docs/analysis/wavefront.md

@@ -0,0 +1,41 @@
+# Wavefront Metrics
+
+A [Wavefront](https://www.wavefront.com/) query can be used to obtain measurements for analysis.
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: success-rate
+spec:
+  args:
+  - name: service-name
+  metrics:
+  - name: success-rate
+    interval: 5m
+    successCondition: result >= 0.95
+    failureLimit: 3
+    provider:
+      wavefront:
+        address: example.wavefront.com
+        query: |
+          sum(rate(
+            5m, ts("istio.requestcount.count", response_code!=500 and destination_service="{{args.service-name}}"
+          ))) /
+          sum(rate(
+            5m, ts("istio.requestcount.count", reporter=client and destination_service="{{args.service-name}}"
+          )))
+```
+
+Wavefront api tokens can be configured in a kubernetes secret in argo-rollouts namespace.
+
+```yaml
+apiVersion: v1
+kind: Secret
+metadata:
+  name: wavefront-api-tokens
+type: Opaque
+stringData:
+  example1.wavefront.com: <token1>
+  example2.wavefront.com: <token2>
+```

docs/analysis/web.md

@@ -0,0 +1,158 @@
+# Web Metrics
+
+An HTTP request can be performed against some external service to obtain the measurement. This example
+makes an HTTP GET request to some URL. The webhook response must return JSON content. The result of
+the optional `jsonPath` expression will be assigned to the `result` variable that can be referenced
+in the `successCondition` and `failureCondition` expressions. If omitted, will use the entire body
+of the as the result variable.
+
+```yaml
+  metrics:
+  - name: webmetric
+    successCondition: result == true
+    provider:
+      web:
+        url: "http://my-server.com/api/v1/measurement?service={{ args.service-name }}"
+        timeoutSeconds: 20 # defaults to 10 seconds
+        headers:
+          - key: Authorization
+            value: "Bearer {{ args.api-token }}"
+        jsonPath: "{$.data.ok}"
+```
+
+In the following example, given the payload, the measurement will be Successful if the `data.ok` field was `true`, and the `data.successPercent`
+was greater than `0.90`
+
+```json
+{
+  "data": {
+    "ok": true,
+    "successPercent": 0.95
+  }
+}
+```
+
+```yaml
+  metrics:
+  - name: webmetric
+    successCondition: "result.ok && result.successPercent >= 0.90"
+    provider:
+      web:
+        url: "http://my-server.com/api/v1/measurement?service={{ args.service-name }}"
+        headers:
+          - key: Authorization
+            value: "Bearer {{ args.api-token }}"
+        jsonPath: "{$.data}"
+```
+
+NOTE: if the result is a string, two convenience functions `asInt` and `asFloat` are provided
+to convert a result value to a numeric type so that mathematical comparison operators can be used
+(e.g. >, <, >=, <=).
+
+## Optional web methods
+It is possible to use a POST or PUT requests, by specifying the `method` and either `body` or `jsonBody` fields
+
+```yaml
+  metrics:
+  - name: webmetric
+    successCondition: result == true
+    provider:
+      web:
+        method: POST # valid values are GET|POST|PUT, defaults to GET
+        url: "http://my-server.com/api/v1/measurement?service={{ args.service-name }}"
+        timeoutSeconds: 20 # defaults to 10 seconds
+        headers:
+          - key: Authorization
+            value: "Bearer {{ args.api-token }}"
+          - key: Content-Type # if body is a json, it is recommended to set the Content-Type
+            value: "application/json"
+        body: "string value"
+        jsonPath: "{$.data.ok}"
+```
+  !!! tip
+      In order to send in JSON, you can use jsonBody and Content-Type will be automatically set as json.
+      Setting a `body` or `jsonBody` field for a `GET` request will result in an error.
+      Set either `body` or `jsonBody` and setting both will result in an error.
+
+```yaml
+  metrics:
+  - name: webmetric
+    successCondition: result == true
+    provider:
+      web:
+        method: POST # valid values are GET|POST|PUT, defaults to GET
+        url: "http://my-server.com/api/v1/measurement?service={{ args.service-name }}"
+        timeoutSeconds: 20 # defaults to 10 seconds
+        headers:
+          - key: Authorization
+            value: "Bearer {{ args.api-token }}"
+          - key: Content-Type # if body is a json, it is recommended to set the Content-Type
+            value: "application/json"
+        jsonBody: # If using jsonBody Content-Type header will be automatically set to json
+          key1: value_1
+          key2:
+            nestedObj: nested value
+            key3: "{{ args.service-name }}"
+        jsonPath: "{$.data.ok}"
+```
+
+## Skip TLS verification
+
+You can skip the TLS verification of the web host provided by setting the options `insecure: true`.
+
+```yaml
+  metrics:
+  - name: webmetric
+    successCondition: "result.ok && result.successPercent >= 0.90"
+    provider:
+      web:
+        url: "https://my-server.com/api/v1/measurement?service={{ args.service-name }}"
+        insecure: true
+        headers:
+          - key: Authorization
+            value: "Bearer {{ args.api-token }}"
+        jsonPath: "{$.data}"
+```
+## Authorization
+
+### With OAuth2
+
+You can setup an [OAuth2 client credential](https://datatracker.ietf.org/doc/html/rfc6749#section-4.4) flow using the following values:
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: AnalysisTemplate
+metadata:
+  name: success-rate
+spec:
+  args:
+  - name: service-name
+  # from secret
+  - name: oauthSecret  # This is the OAuth2 shared secret
+    valueFrom:
+      secretKeyRef:
+        name: oauth-secret
+        key: secret
+  metrics:
+  - name: webmetric
+    successCondition: result == true
+    provider:
+      web:
+        url: "http://my-server.com/api/v1/measurement?service={{ args.service-name }}"
+        timeoutSeconds: 20 # defaults to 10 seconds
+        authentication:
+          oauth2:
+            tokenUrl: https://my-oauth2-provider/token
+            clientId: my-client-id
+            clientSecret: "{{ args.oauthSecret }}"
+            scopes: [
+              "my-oauth2-scope"
+            ]
+        headers:
+          - key: Content-Type # if body is a json, it is recommended to set the Content-Type
+            value: "application/json"
+        jsonPath: "{$.data.ok}"
+```
+
+In that case, no need to provide specifically the `Authentication` header.
+The AnalysisRun will first get an access token using that information, and provide it as an `Authorization: Bearer` header for the metric provider call.

docs/architecture.md

@@ -0,0 +1,59 @@
+# Architecture
+
+Here is an overview of all the components that take part in a deployment managed by Argo Rollouts.
+
+[![Argo Rollouts Architecture](architecture-assets/argo-rollout-architecture.png)](architecture-assets/argo-rollout-architecture.png)
+
+## Argo Rollouts controller
+
+This is the main controller that monitors the cluster for events and reacts whenever a resource of type `Rollout` is changed. The controller
+will read all the details of the rollout and bring the cluster to the same state as described in the rollout definition.
+
+Note that Argo Rollouts will not tamper with or respond to any changes that happen on normal [Deployment Resources](https://kubernetes.io/docs/concepts/workloads/controllers/deployment/). This means
+that you can install Argo Rollouts in a cluster that is also deploying applications with alternative methods.
+
+To install the controller in your cluster and get started with Progressive Delivery, see the [Installation page](../installation/).
+
+## Rollout resource
+
+The Rollout resource is a custom Kubernetes resource introduced and managed by Argo Rollouts. It is mostly compatible with the native Kubernetes Deployment resource but with extra
+fields that control the stages, thresholds and methods of advanced deployment methods such as canaries and blue/green deployments.
+
+Note that the Argo Rollouts controller will only respond to those changes that happen in Rollout sources. It will do nothing for normal deployment resources. This means that you need to [migrate your Deployments to Rollouts](../migrating/) if you want to manage them with Argo Rollouts.
+
+You can see all possible options of a Rollout in the [full specification page](../features/specification/).
+
+## Replica sets for old and new version
+
+These are instances of the [standard Kubernetes ReplicaSet resources](https://kubernetes.io/docs/concepts/workloads/controllers/replicaset/). Argo Rollouts puts some extra metadata on them in order to keep track of the different versions that are part of an application.
+
+Note also that the replica sets that take part in a Rollout are fully managed by the controller in an automatic way. You should not tamper with them with external tools.
+
+## Ingress/Service
+
+This is the mechanism that traffic from live users enters your cluster and is redirected to the appropriate version. Argo Rollouts use the [standard Kubernetes service resource](https://kubernetes.io/docs/concepts/services-networking/service/), but with some extra metadata needed for management.
+
+Argo Rollouts is very flexible on networking options. First of all you can have different services during a Rollout, that go only to the new version, only to the old version or both.
+Specifically for Canary deployments, Argo Rollouts supports several [service mesh and ingress solutions](../features/traffic-management/) for splitting traffic with specific percentages instead of simple balancing based on pod counts and it is possible to use multiple routing providers simultaneously.
+
+## AnalysisTemplate and AnalysisRun
+
+Analysis is the capability to connect a Rollout to your metrics provider and define specific thresholds for certain metrics that will decide if an update is successful or not. For each analysis you can define one or more metric queries along with their expected results. A Rollout will progress on its own if metric queries are good, rollback automatically if metrics show failure and pause the rollout if metrics cannot provide a success/failure answer.
+
+For performing an analysis, Argo Rollouts includes two custom Kubernetes resources: `AnalysisTemplate` and `AnalysisRun`.
+
+`AnalysisTemplate` contains instructions on what metrics to query. The actual result that is attached to a Rollout is the `AnalysisRun` custom resource. You can define an `AnalysisTemplate` on a specific Rollout or globally on the cluster to be shared by multiple rollouts as a `ClusterAnalysisTemplate`. The `AnalysisRun` resource is scoped on a specific rollout.
+
+Note that using an analysis and metrics in a Rollout is completely optional. You can manually pause and promote a rollout or use other external methods (e.g. smoke tests) via the API or the CLI. You don't need a metric solution just to use Argo Rollouts. You can also mix both automated (i.e. analysis based) and manual steps in a Rollout.
+
+Apart from metrics, you can also decide the success of a rollout by running a [Kubernetes job](../analysis/job/) or running [a webhook](../analysis/web/).
+
+
+## Metric providers
+
+Argo Rollouts includes [native integration for several popular metrics providers](../features/analysis/) that you can use in the Analysis resources to automatically promote or rollback  a rollout. See the documentation of each provider for specific setup options.
+
+## CLI and UI (Not shown in the diagram)
+
+You can view and manage Rollouts with the [Argo Rollouts CLI](../features/kubectl-plugin/) or the [integrated UI](../dashboard/). Both are optional.
+

docs/assets/versions.css

@@ -0,0 +1,175 @@
+.md-header__title {
+  display: flex;
+}
+
+.dropdown-caret {
+  display: inline-block !important;
+  position: absolute;
+  right: 4px;
+}
+
+.fa .fa-caret-down {
+  display: none !important;
+}
+
+.rst-other-versions {
+  text-align: right;
+}
+
+.rst-other-versions > dl, .rst-other-versions dt, .rst-other-versions small {
+  display: none;
+}
+
+.rst-other-versions > dl:first-child {
+  display: flex !important;
+  flex-direction: column;
+  line-height: 0px !important;
+}
+
+.rst-versions.shift-up .rst-other-versions {
+  display: flex !important;
+}
+
+.rst-versions .rst-other-versions {
+  display: none;
+}
+
+/* Version Warning */
+div[data-md-component=announce] {
+  background-color: rgb(248, 243, 236);
+  position: sticky;
+  top: 0;
+  z-index: 2;
+}
+div[data-md-component=announce]>div#announce-msg{
+  color: var(--md-code-hl-number-color);
+  font-size: .8rem;
+  text-align: center;
+  margin: 15px;
+}
+div[data-md-component=announce]>div#announce-msg>a{
+  color: var(--md-typeset-a-color);
+  text-decoration: underline;
+}
+
+/* from https://assets.readthedocs.org/static/css/badge_only.css,
+most styles have to be overridden here */
+.rst-versions{
+  position: relative !important;
+  bottom: 0;
+  left: 0;
+  width: 100px !important;
+  background: hsla(173, 100%, 24%, 1) !important;
+  font-family: inherit !important;
+  z-index: 0 !important;
+}
+.rst-versions a{
+  color:#2980B9;
+  text-decoration:none
+}
+.rst-versions .rst-badge-small{
+  display:none
+}
+.rst-versions .rst-current-version{
+  padding:12px;
+  background: hsla(173, 100%, 24%, 1) !important;
+  display:block;
+  text-align:right;
+  font-size:90%;
+  cursor:pointer;
+  color: white !important;
+  *zoom:1
+}
+.rst-versions .rst-current-version:before,.rst-versions .rst-current-version:after{
+  display:table;content:""
+}
+.rst-versions .rst-current-version:after{
+  clear:both
+}
+.rst-versions .rst-current-version .fa{
+  color:#fcfcfc
+}
+.rst-versions .rst-current-version .fa-caret-down{
+  display: none;
+}
+.rst-versions.shift-up .rst-other-versions{
+  display:block
+}
+.rst-versions .rst-other-versions{
+  font-size:90%;
+  padding:12px;
+  color:gray;
+  display:none
+}
+.rst-versions .rst-other-versions hr{
+  display: none !important;
+  height: 0px !important;
+  border: 0px;
+  margin: 0px !important;
+  padding: 0px;
+  border-top: none !important;
+}
+.rst-versions .rst-other-versions dd{
+  display:inline-block;
+  margin:0
+}
+.rst-versions .rst-other-versions dd a{
+  display:inline-block;
+  padding: 1em 0em !important;
+  color:#fcfcfc;
+  font-size: .6rem !important;
+  white-space: nowrap;
+  text-overflow: ellipsis;
+  overflow: hidden;
+  width: 80px;
+}
+.rst-versions .rst-other-versions dd a:hover{
+  font-size: .7rem !important;
+  font-weight: bold;
+}
+.rst-versions.rst-badge{
+  display: block !important;
+  width: 100px !important;
+  bottom: 0px !important;
+  right: 0px !important;
+  left:auto;
+  border:none;
+  text-align: center !important;
+  line-height: 0;
+}
+.rst-versions.rst-badge .icon-book{
+  display: none;
+}
+.rst-versions.rst-badge .fa-book{
+  display: none !important;
+}
+.rst-versions.rst-badge.shift-up .rst-current-version{
+  text-align: left !important;
+}
+.rst-versions.rst-badge.shift-up .rst-current-version .fa-book{
+  display: none !important;
+}
+.rst-versions.rst-badge.shift-up .rst-current-version .icon-book{
+  display: none !important;
+}
+.rst-versions.rst-badge .rst-current-version{
+  width: 70px !important;
+  height: 2.4rem !important;
+  line-height:2.4rem !important;
+  padding: 0px 5px !important;
+  display: inline-block !important;
+  font-size: .6rem !important;
+  overflow: hidden !important;
+  text-overflow: ellipsis !important;
+  white-space: nowrap !important;
+  text-align: left !important;
+}
+@media screen and (max-width: 768px){
+  .rst-versions{
+      width:85%;
+      display:none
+  }
+  .rst-versions.shift{
+      display:block
+  }
+}

docs/assets/versions.js

@@ -0,0 +1,75 @@
+const targetNode = document.querySelector('.md-header__inner');
+const observerOptions = {
+  childList: true,
+  subtree: true
+};
+
+const observerCallback = function(mutationsList, observer) {
+  for (let mutation of mutationsList) {
+    if (mutation.type === 'childList') {
+      const titleElement = document.querySelector('.md-header__inner > .md-header__title');
+      if (titleElement) {
+        initializeVersionDropdown();
+        observer.disconnect();
+      }
+    }
+  }
+};
+
+const observer = new MutationObserver(observerCallback);
+observer.observe(targetNode, observerOptions);
+
+function initializeVersionDropdown() {
+  const callbackName = 'callback_' + new Date().getTime();
+  window[callbackName] = function(response) {
+    const div = document.createElement('div');
+    div.innerHTML = response.html;
+    document.querySelector(".md-header__inner > .md-header__title").appendChild(div);
+    const container = div.querySelector('.rst-versions');
+    var caret = document.createElement('div');
+    caret.innerHTML = "<i class='fa fa-caret-down dropdown-caret'></i>";
+    caret.classList.add('dropdown-caret');
+    div.querySelector('.rst-current-version').appendChild(caret);
+
+    div.querySelector('.rst-current-version').addEventListener('click', function() {
+      container.classList.toggle('shift-up');
+    });
+  };
+
+  var CSSLink = document.createElement('link');
+  CSSLink.rel='stylesheet';
+  CSSLink.href = '/assets/versions.css';
+  document.getElementsByTagName('head')[0].appendChild(CSSLink);
+
+  var script = document.createElement('script');
+  script.src = 'https://argo-rollouts.readthedocs.io/_/api/v2/footer_html/?'+
+      'callback=' + callbackName + '&project=argo-rollouts&page=&theme=mkdocs&format=jsonp&docroot=docs&source_suffix=.md&version=' + (window['READTHEDOCS_DATA'] || { version: 'latest' }).version;
+  document.getElementsByTagName('head')[0].appendChild(script);
+}
+
+// VERSION WARNINGS
+window.addEventListener("DOMContentLoaded", function() {
+  var currentVersion = window.location.href.match(/\/en\/(release-(?:v\d+|\w+)|latest|stable)\//);
+  var margin = 30;
+  var headerHeight = document.getElementsByClassName("md-header")[0].offsetHeight;
+  if (currentVersion && currentVersion.length > 1) {
+    currentVersion = currentVersion[1];
+    if (currentVersion === "latest") {
+      document.querySelector("div[data-md-component=announce]").innerHTML = "<div id='announce-msg'>You are viewing the docs for an unreleased version of Argo Rollouts, <a href='https://argoproj.github.io/argo-rollouts/'>click here to go to the latest stable version.</a></div>";
+      var bannerHeight = document.getElementById('announce-msg').offsetHeight + margin;
+      document.querySelector("header.md-header").style.top = bannerHeight + "px";
+      document.querySelector('style').textContent +=
+          "@media screen and (min-width: 76.25em){ .md-sidebar { height: 0;  top:" + (bannerHeight + headerHeight) + "px !important; }}";
+      document.querySelector('style').textContent +=
+          "@media screen and (min-width: 60em){ .md-sidebar--secondary { height: 0;  top:" + (bannerHeight + headerHeight) + "px !important; }}";
+    } else if (currentVersion !== "stable") {
+      document.querySelector("div[data-md-component=announce]").innerHTML = "<div id='announce-msg'>You are viewing the docs for a previous version of Argo Rollouts, <a href='https://argoproj.github.io/argo-rollouts/'>click here to go to the latest stable version.</a></div>";
+      var bannerHeight = document.getElementById('announce-msg').offsetHeight + margin;
+      document.querySelector("header.md-header").style.top = bannerHeight + "px";
+      document.querySelector('style').textContent +=
+          "@media screen and (min-width: 76.25em){ .md-sidebar { height: 0;  top:" + (bannerHeight + headerHeight) + "px !important; }}";
+      document.querySelector('style').textContent +=
+          "@media screen and (min-width: 60em){ .md-sidebar--secondary { height: 0;  top:" + (bannerHeight + headerHeight) + "px !important; }}";
+    }
+  }
+});

docs/best-practices.md

@@ -0,0 +1,190 @@
+# Best Practices
+
+This document describes some best practices, tips and tricks when using Argo Rollouts. Be sure to read the [FAQ page](../FAQ) as well.
+
+
+## Check application compatibility
+
+Argo Rollouts is a great solution for applications that your team is deploying in a continuous manner (and you have access to the source code). Before using Argo Rollouts you need to contact the developers of the application and verify that you can indeed run multiple versions of the same application at the same time. 
+
+Not all applications can work with Argo Rollouts. Applications that use shared resources (e.g. writing to a shared file) will have issues, and "worker" type applications (that load data from queues) will rarely work ok without source code modifications.
+
+Note that using Argo Rollouts for "infrastructure" applications such as cert-manager, nginx, coredns, sealed-secrets etc is **NOT** recommended.
+
+## Understand the scope of Argo Rollouts
+
+Currently, Argo Rollouts works with a single Kubernetes deployment/application and within a single cluster only. You also need to have the controller deployed on *every* cluster where a Rollout is running if have more than one clusters using Rollout workloads.
+
+If you want to look at multiple-services on multiple clusters
+see discussion at issues [2737](https://github.com/argoproj/argo-rollouts/issues/2737), [451](https://github.com/argoproj/argo-rollouts/issues/451) and [2088](https://github.com/argoproj/argo-rollouts/issues/2088).
+
+
+Note also that Argo Rollouts is a self-contained solution. It doesn't need Argo CD or any other Argo project to work.
+
+## Understand your use case
+
+Argo Rollouts is perfect for all progressive delivery scenarios as explained in [the concepts page](../concepts).
+
+You should *NOT* use Argo Rollouts for preview/ephemeral environments. For that use case check the [Argo CD Pull Request generator](https://argo-cd.readthedocs.io/en/stable/operator-manual/applicationset/Generators-Pull-Request/).
+
+The recommended way to use Argo Rollouts is for brief deployments that take 15-20 minutes or maximum 1-2 hours. If you want to run new versions for days or weeks before deciding to promote, then Argo Rollouts is probably not the best solution for you.
+
+Keeping parallel releases for long times, complicates the deployment process a lot and opens several questions where different people have different views on how Argo Rollouts should work.
+
+For example let's say that you are testing for a week version 1.3 as stable and 1.4 as preview.
+Then somebody deploys 1.5
+
+1. Some people believe that the new state should be 1.3 stable and 1.5 as preview
+1. Some people believe that the new state should be 1.4 stable and 1.5 as preview
+
+Currently, Argo Rollouts follows the first approach, under the assumption that something was really wrong with 1.4 and 1.5 is the hotfix. 
+
+And then let's say that 1.5 has an issue. Some people believe that Argo rollouts should "rollback" to 1.3 while other people think it should rollback to 1.4
+
+Currently, Argo Rollouts assumes that the version to rollback is always 1.3 regardless of how many "hotfixes" have been previewed in-between.
+
+All these problems are not present if you make the assumption that each release stays active only for a minimal time and you always create one new version when the previous one has finished.
+
+Also, if you want to run a wave of multiple versions at the same time (i.e. have 1.1 and 1.2 and 1.3 running at the same time), know that Argo Rollouts was not designed for this scenario. Argo Rollouts always works with the assumption that there is one stable/previous version and one preview/next version.
+
+A version that has just been promoted is assumed to be ready for production and has already passed all your tests (either manual or automated).
+
+## Prepare your metrics
+
+The end-goal for using Argo Rollouts is to have **fully automated** deployments that also include rollbacks when needed.
+
+While Argo Rollouts supports manual promotions and other manual pauses, these are best used for experimentation and test reasons.
+
+Ideally you should have proper metrics that tell you in 5-15 minutes if a deployment is successful or not. If you don't have those metrics, then you will miss a lot of value from Argo Rollouts.
+
+If you are doing a deployment right now and then have an actual human looking at logs/metrics/traces for the next 2 hours, adopting Argo Rollouts is not going to help you a lot with automated deployments.
+
+Get your [metrics](../features/analysis) in place first and test them with dry-runs before applying them to production deployments.
+
+
+## There is no "Argo Rollouts API"
+
+A lot of people want to find an official API for managing Rollouts. There isn't any separate Argo Rollouts API. You can always use the Kubernetes API and patching of resources if you want to control a rollout.
+
+But as explained in the previous point the end goal should be fully automated deployments without you having to tell Argo Rollouts to promote or abort.
+
+## Integrating with other systems and processes
+
+There are two main ways to integrate other systems with Argo Rollouts.
+
+The easiest way is to use [Notifications](../features/notifications). This means that when a rollout is finished/aborted you send a notification to another system that does other tasks that you want to happen.
+
+Alternatively you can control Rollouts with the CLI or by patching manually the Kubernetes resources.
+
+
+## Use the Kubernetes Downward API
+
+If you want your applications to know if they are part of a canary or not, you can use [Ephemeral labels](../features/ephemeral-metadata) along with the [Kubernetes downward api](https://kubernetes.io/docs/concepts/workloads/pods/downward-api/).
+
+This means that your application will read from files its configuration in a dynamic manner and adapt according to the situation.
+
+
+
+## Ingress desired/stable host routes
+
+For various reasons, it is often desired that external services are able to reach the
+desired pods (aka canary/preview) or stable pods specifically, without the possibility of traffic
+arbitrarily being split between the two versions. Some use cases include:
+
+* The new version of the service is able to be reach internally/privately (e.g. for manual verification),
+  before exposing it externally.
+* An external CI/CD pipeline runs tests against the blue-green preview stack before it is
+  promoted to production.
+* Running tests which compare the behavior of old version against the new version.
+
+If you are using an Ingress to route traffic to the service, additional host rules can be added
+to the ingress rules so that it is possible to specifically reach to the desired (canary/preview)
+pods or stable pods.
+
+```yaml
+apiVersion: networking.k8s.io/v1
+kind: Ingress
+metadata:
+  name: guestbook
+spec:
+  rules:
+  # host rule to only reach the desired pods (aka canary/preview)
+  - host: guestbook-desired.argoproj.io
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: guestbook-desired
+            port:
+              number: 443
+
+  # host rule to only reach the stable pods
+  - host: guestbook-stable.argoproj.io
+    http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: guestbook-stable
+            port:
+              number: 443
+
+  # default rule which omits host, and will split traffic between desired vs. stable
+  - http:
+      paths:
+      - path: /
+        pathType: Prefix
+        backend:
+          service:
+            name: guestbook-root
+            port:
+            number: 443
+```
+
+The above technique has a benefit in that it would not incur additional cost of allocating
+additional load balancers.
+
+## Reducing operator memory usage
+
+On clusters with thousands of rollouts memory usage for the argo-rollouts
+controller can be reduced significantly by changing the `RevisionHistoryLimit` property from the
+default of 10 to a lower number. 
+
+One user of Argo Rollouts saw a 27% reduction
+in memory usage for a cluster with 1290 rollouts by changing
+`RevisionHistoryLimit` from 10 to 0.
+
+
+## Rollout a ConfigMap change
+
+Argo Rollouts is meant to work on a Kubernetes Deployment. When a ConfigMap is mounted inside one the Deployment container and a change occurs inside the ConfigMap, it won't trigger a new Rollout by default.
+
+One technique to trigger the Rollout it to name dynamically the ConfigMap.
+For example, adding a hash of its content at the end of the name:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: my-config-7270e14e6
+```
+
+Each time a change occurs in the ConfigMap, its name will change in the Deployment reference as well, triggering a Rollout.
+
+However, it's not enough to perform correctly progressive rollouts, as the old ConfigMap might get deleted as soon as the new one is created. This would prevent Experiments and rollbacks in case of rollout failure to work correctly.
+
+While no magical solution exist to work aroud that, you can tweak your deployment tool to remove the ConfigMap only when the Rollout is completed successfully.
+
+Example with Argo CD:
+
+```yaml
+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: my-config-7270e14e6
+  annotations:
+    argocd.argoproj.io/sync-options: PruneLast=true
+```

docs/concepts.md

@@ -0,0 +1,78 @@
+# Concepts
+
+## Rollout
+
+A Rollout is Kubernetes workload resource which is equivalent to a Kubernetes Deployment object.
+It is intended to replace a Deployment object in scenarios when more advanced deployment or
+progressive delivery functionality is needed. A Rollout provides the following features which
+a Kubernetes Deployment cannot:
+
+* blue-green deployments
+* canary deployments
+* integration with ingress controllers and service meshes for advanced traffic routing
+* integration with metric providers for blue-green & canary analysis
+* automated promotion or rollback based on successful or failed metrics
+
+## Progressive Delivery
+
+Progressive delivery is the process of releasing updates of a product in a controlled and gradual
+manner, thereby reducing the risk of the release, typically coupling automation and metric analysis
+to drive the automated promotion or rollback of the update.
+
+Progressive delivery is often described as an evolution of continuous delivery, extending the
+speed benefits made in CI/CD to the deployment process. This is accomplished by limiting the
+exposure of the new version to a subset of users, observing and analyzing for correct behavior,
+then progressively increasing the exposure to a broader and wider audience while continuously
+verifying correctness.
+
+## Deployment Strategies
+
+While the industry has used a consistent terminology to describe various deployment strategies, the implementations of these strategies tend to differ across tooling. To make it clear how the Argo Rollouts will behave, here are the descriptions of the various deployment strategy implementations. Argo Rollouts only supports Blue-Green and Canary.
+
+### Rolling Update
+A `RollingUpdate` slowly replaces the old version with the new version. As the new version comes up, the old version is scaled down in order to maintain the overall count of the application. This is the default strategy of the Deployment object.
+
+### Recreate
+A Recreate deployment deletes the old version of the application before bring up the new version. As a result, this ensures that two versions of the application never run at the same time, but there is downtime during the deployment.
+
+### Blue-Green
+A Blue-Green deployment (sometimes referred to as a Red-Black) has both the new and old version of the application deployed at the same time.  During this time, only the old version of the application will receive production traffic. This allows the developers to run tests against the new version before switching the live traffic to the new version.
+
+[![How Blue Green deployments work](concepts-assets/blue-green-deployments.png)](concepts-assets/blue-green-deployments.png)
+
+### Canary
+A Canary deployment exposes a subset of users to the new version of the application while serving the rest of the traffic to the old version. Once the new version is verified to be correct, the new version can gradually replace the old version. Ingress controllers and service meshes such as NGINX and Istio, enable more sophisticated traffic shaping patterns for canarying than what is natively available (e.g. achieving very fine-grained traffic splitting, or splitting based on HTTP headers).
+
+[![How Canary deployments work](concepts-assets/canary-deployments.png)](concepts-assets/canary-deployments.png)
+
+The picture above shows a canary with two stages (10% and 33% of traffic goes to new version) but this is just an example. With Argo Rollouts you can define the exact number of stages
+and percentages of traffic according to your use case.
+
+## Which strategy to choose
+
+In general Blue/Green is the easier strategy to start with, but also the more limited. We recommend you start with Blue/Green deployments first and as you gain confidence for your metrics and applications switch to Canaries.
+
+You also need to examine if your application can handle canaries or not.
+
+* Blue/Green always works because only one application is active at a time. Not all applications can have different versions running in parallel at the same time (which is what canaries are doing). This can be a showstopper for adopting canary deployments especially for legacy applications.
+* Blue/Green is simpler because you can get their full value WITHOUT a traffic manager. While canaries can also work without a traffic manager, most of their advanced features assume a fine-grained way to control traffic. If you don't have a traffic manager, then you can easily get the full value
+of blue/green deployments but only the basic capabilities of canaries.
+* Blue/Green also works with services that use queues and databases (workers that fetch tasks). Argo Rollouts doesn't control traffic flow for
+connections it doesn't understand (i.e. binary/queue channels).
+
+Here is a summary table for the possible approaches.
+
+|                           |         Blue/Green        |       Basic Canary         | Canary with Traffic manager    |
+|--------------------------:|:-------------------------:|:--------------------------:| :-----------------------------:|
+|       Adoption Complexity |           Low             |       Medium               |        High                    |
+|        Flexibility        |           Low             |        High                |        Maximum                 |
+|    Needs traffic provider |                 No        |         No                 |           Yes                  |
+|  Works with queue workers |                Yes        |         No                 |            No                  |
+|  Works with shared/locked resources | Yes             |         No                 |            No                  |
+|            Traffic switch |     All or nothing        |  Gradual percentage        |      Gradual percentage        |
+|           Traffic control |     0% or 100%            |  coarse grained            |     fine grained               |
+|       Traffic depends on  |     deployment state      |  number of canary pods     |     Any split option is possible                |
+| Advanced routing scenarios |         No               |       No                   |         Yes                    |
+|      Failure Blast Radius | Massive impact            |  Low impact                       | Low impact              |
+
+Note that the traffic manager can be any compatible Service Mesh or Ingress Controller or Gateway API implementation (via a plugin).

docs/dashboard.md

@@ -0,0 +1,14 @@
+# UI Dashboard
+
+The Argo Rollouts Kubectl plugin can serve a local UI Dashboard to visualize your Rollouts.
+
+To start it, run `kubectl argo rollouts dashboard` in the namespace that contains your Rollouts.
+Then visit `localhost:3100` to view the user interface.
+
+## List view
+
+![Rollouts List](dashboard/rollouts-list.png)
+
+## Individual Rollout view
+
+![Rollouts List](dashboard/rollout-ui.png)

docs/deployment-concepts.md

@@ -1,14 +0,0 @@
-# Deployment Concepts
-While the industry has agreed upon high-level definitions of various deployment strategies, the implementations of these strategies tend to differ across tooling.  To make it clear how the Argo Rollouts will behave, here are the descriptions of the various deployment strategies implementations offered by the Argo Rollouts.
-
-## Rolling Update
-A `RollingUpdate` slowly replaces the old version with the new version. As the new version comes up, the old version is scaled down in order to maintain the overall count of the application. This is the default strategy of the deployment object.
-
-## Recreate
-A Recreate deployment deletes the old version of the application before bring up the new version. As a result, this ensures that two versions of the application never run at the same time, but there is downtime during the deployment.
-
-## Blue Green
-A Blue Green deployment (which is sometimes referred to as a Red-Black) has both the new and old version of the application deployed at the same time.  During this time, only the old version of the application will receive production traffic. This allows the developers to run tests against the new version before switching the live traffic to the new version.
-
-## Canary
-A Canary deployment exposes a subset of users (usually 5%) to the new version of the application while serving the rest of the traffic to the old version.  Once the new version is verified to be correct that the new version can gradually replace the old version.  Service Meshes like Istio make canary deployments easily to rollout as the service mesh can filter the traffic for you.

docs/features/anti-affinity/anti-affinity.md

@@ -0,0 +1,67 @@
+# Anti Affinity
+
+## Background
+
+Depending on a cluster's configuration, a Blue Green Rollout (or a Canary rollout that uses traffic management) can cause newly created pods to restart after deploying a new version. This can be problematic, especially for applications that cannot startup quickly or do not gracefully exit.
+
+
+This behavior occurs because cluster auto-scaler wants to scale down the extra capacity which was created to support a Rollout
+running in double capacity. When a node is scaled down, the pods it owns are deleted and recreated. This usually happens if a Rollout has its own dedicated instance group since a Rollout has a greater effect on
+cluster auto-scaling. Therefore, clusters with a large pool of shared nodes experience the behavior less often.
+
+For example, here is a Rollout is running with 8 pods spread across 2 nodes. Each node can hold 6 pods:
+![ Original Rollout is running, spread across two nodes](images/step-0.png)
+
+When the `spec.template` of the Rollout changes, the controller creates a new ReplicaSet with the spec update and the total number of pods doubles. In this case, the number of pods increases
+to 16.
+
+Since each node can only hold 6 pods, the cluster autoscaler must increase the node count to 3 to accommodate
+all 16 pods. The resulting distribution of pods across nodes is shown here:
+![ Rollout running at double capacity](images/step-1.png)
+
+Once the Rollout finishes progressing, the old version is scaled down. This leaves the cluster with more nodes than necessary, thus wasting resources (as shown below).
+![ Original Rollout is running, spread across two nodes](images/step-2.png)
+
+The cluster auto-scaler terminates the extra node and the pods are rescheduled on the remaining 2 nodes.
+![ Original Rollout is running, spread across two nodes](images/step-3.png)
+
+To reduce the chance of this behavior, a rollout can inject anti-affinity into the ReplicaSet. This prevents new pods from running on nodes which have the previous version's pods.
+
+You can learn more about anti-affinity [here](https://kubernetes.io/docs/concepts/configuration/assign-pod-node/#inter-pod-affinity-and-anti-affinity).
+
+Repeating the above example with anti-affinity enabled, here is what happens when the `.spec.template` of the Rollout changes. Due to anti-affinity, the new pods cannot be scheduled on nodes which run the old ReplicaSet's pods.
+As a result, the cluster auto-scaler must create 2 nodes to host the new ReplicaSet's pods. In this case, pods won't be started since the scaled-down nodes are guaranteed to not have the new pods.
+
+![ Original Rollout is running, spread across two nodes](images/solution.png)
+
+## Enabling Anti-Affinity in Rollouts
+
+Anti-affinity is enabled by adding the anti-affinity struct to the Blue-Green or Canary strategy. When the anti-affinity struct is set, controller injects a PodAntiAffinity struct into the ReplicaSet's Affinity.
+This feature will not modify any of the ReplicaSet's pre-existing affinity rules. 
+
+Users have a choice between these scheduling rules: `RequiredDuringSchedulingIgnoredDuringExecution` and `PreferredDuringSchedulingIgnoredDuringExecution`.
+
+`RequiredDuringSchedulingIgnoredDuringExecution` requires a new version's pods to be on a separate node than the previous versions. If this
+is not possible, the new version's pods will not be scheduled.
+
+```yaml
+strategy:
+    bluegreen:
+      antiAffinity:
+          requiredDuringSchedulingIgnoredDuringExecution: {}
+```
+
+Unlike the Required strategy, `PreferredDuringSchedulingIgnoredDuringExecution` does not force a new version's pods to be on a separate node than the previous versions.
+The scheduler attempts to place the new version's pods on separate node(s). If that's not possible, the new version's pods will still be scheduled. The `Weight` is used to create a priority order for preferred anti-affinity rules. 
+
+```yaml
+strategy:
+    canary:
+      antiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+            weight: 1 # Between 1 - 100
+```
+
+!!! important
+    The main downside to this approach is that deployments can take longer because new nodes are more likely to be created in order to schedule pods with respect to anti-affinity rules. This delay most frequently occurs when a rollout has its own dedicated instance group,
+    since new nodes are more likely to be created to honor anti-affinity rules.

docs/features/bluegreen.md

@@ -1,5 +1,6 @@
 # BlueGreen Deployment Strategy
-A Blue Green Deployment allows users to reduce the amount of time multiple versions running at the same time.
+
+A Blue Green Deployment allows users to reduce the amount of time multiple versions are running at the same time.
 
 ## Overview
 
@@ -7,8 +8,54 @@ In addition to managing ReplicaSets, the rollout controller will modify a Servic
 
 When there is a change to the `.spec.template` field of a rollout, the controller will create the new ReplicaSet.  If the active service is not sending traffic to a ReplicaSet, the controller will immediately start sending traffic to the ReplicaSet. Otherwise, the active service will point at the old ReplicaSet while the ReplicaSet becomes available. Once the new ReplicaSet becomes available, the controller will modify the active service to point at the new ReplicaSet. After waiting some time configured by the `.spec.strategy.blueGreen.scaleDownDelaySeconds`, the controller will scale down the old ReplicaSet.
 
-__Important note__
-When the rollout changes the selector on a service, there is a propagation delay before all the nodes update their IP tables to send traffic to the new pods instead of the old. During this delay, traffic will be directed to the old pods if the nodes have not been updated yet. In order to prevent the packets from being sent to a node that killed the old pod, the rollout uses the scaleDownDelaySeconds field to give nodes enough time to broadcast the IP table changes.
+!!! important
+    When the rollout changes the selector on a service, there is a propagation delay before all the nodes update their IP tables to send traffic to the new pods instead of the old. During this delay, traffic will be directed to the old pods if the nodes have not been updated yet. In order to prevent the packets from being sent to a node that killed the old pod, the rollout uses the scaleDownDelaySeconds field to give nodes enough time to broadcast the IP table changes.
+
+!!! important
+    ALB Ingress with Rollouts blue-green strategy is not supported without a chance of downtime.
+
+    When using an AWS ALB to route traffic to a service, the ALB Ingress Controller does not update the target groups in an atomic or safe manner. This can result in a situation where, during a deployment, the stable target group temporarily has no pods registered. This occurs because the ALB Controller removes all current pods from the target group before registering pods from the desired ReplicaSet.
+    The desired pods must pass their initial configured health check on the stable target group to be considered healthy by the ALB. This creates a risk where the ALB may temporarily have no healthy pods registered to the target group, depending on the timing of deregistration and registration of new pods. This can lead to application downtime that the rollouts controller cannot prevent.
+
+## Example
+
+```yaml
+apiVersion: argoproj.io/v1alpha1
+kind: Rollout
+metadata:
+  name: rollout-bluegreen
+spec:
+  replicas: 2
+  revisionHistoryLimit: 2
+  selector:
+    matchLabels:
+      app: rollout-bluegreen
+  template:
+    metadata:
+      labels:
+        app: rollout-bluegreen
+    spec:
+      containers:
+      - name: rollouts-demo
+        image: argoproj/rollouts-demo:blue
+        imagePullPolicy: Always
+        ports:
+        - containerPort: 8080
+  strategy:
+    blueGreen: 
+      # activeService specifies the service to update with the new template hash at time of promotion.
+      # This field is mandatory for the blueGreen update strategy.
+      activeService: rollout-bluegreen-active
+      # previewService specifies the service to update with the new template hash before promotion.
+      # This allows the preview stack to be reachable without serving production traffic.
+      # This field is optional.
+      previewService: rollout-bluegreen-preview
+      # autoPromotionEnabled disables automated promotion of the new stack by pausing the rollout
+      # immediately before the promotion. If omitted, the default behavior is to promote the new
+      # stack as soon as the ReplicaSet are completely ready/available.
+      # Rollouts can be resumed using: `kubectl argo rollouts promote ROLLOUT`
+      autoPromotionEnabled: false
+```
 
 ## Configurable Features
 Here are the optional fields that will change the behavior of BlueGreen deployment:
@@ -16,45 +63,93 @@ Here are the optional fields that will change the behavior of BlueGreen deployme
 spec:
   strategy:
     blueGreen:
-      previewService: string
-      previewReplicaCount: *int32
-      autoPromotionEnabled: true
+      autoPromotionEnabled: boolean
       autoPromotionSeconds: *int32
+      antiAffinity: object
+      previewService: string
+      prePromotionAnalysis: object
+      postPromotionAnalysis: object
+      previewReplicaCount: *int32
       scaleDownDelaySeconds: *int32
       scaleDownDelayRevisionLimit: *int32
 ```
 
-### PreviewService
-The PreviewService field references a Service that will be modified to send traffic to the new replicaset before the new one is promoted to receiving traffic from the active service. Once the new replicaset start receives traffic from the active service, the preview service will be modified to send traffic to no ReplicaSets. The Rollout always makes sure that the preview service is sending traffic to the new ReplicaSet.  As a result, if a new version is introduced before the old version is promoted to the active service, the controller will immediately switch over to the new version.
+## Sequence of Events
+
+The following describes the sequence of events that happen during a blue-green update.
+
+1. Beginning at a fully promoted, steady-state, a revision 1 ReplicaSet is pointed to by both the `activeService` and `previewService`.
+1. A user initiates an update by modifying the pod template (`spec.template.spec`).
+1. The revision 2 ReplicaSet is created with size 0.
+1. The `previewService` is modified to point to the revision 2 ReplicaSet. The `activeService` remains pointing to revision 1.
+1. The revision 2 ReplicaSet is scaled to either `spec.replicas` or `previewReplicaCount` if set.
+1. Once revision 2 ReplicaSet Pods are fully available, `prePromotionAnalysis` begins.
+1. Upon success of `prePromotionAnalysis`, the blue/green pauses if `autoPromotionEnabled` is false, or `autoPromotionSeconds` is non-zero.
+1. The rollout is resumed either manually by a user, or automatically by surpassing `autoPromotionSeconds`.
+1. The revision 2 ReplicaSet is scaled to the `spec.replicas`, if the `previewReplicaCount` feature was used.
+1. The rollout "promotes" the revision 2 ReplicaSet by updating the `activeService` to point to it. At this point, there are no services pointing to revision 1
+1. `postPromotionAnalysis` analysis begins
+1. Once `postPromotionAnalysis` completes successfully, the update is successful and the revision 2 ReplicaSet is marked as stable. The rollout is considered fully-promoted.
+1. After waiting `scaleDownDelaySeconds` (default 30 seconds), the revision 1 ReplicaSet is scaled down 
+
+
+### autoPromotionEnabled
+The AutoPromotionEnabled will make the rollout automatically promote the new ReplicaSet to the active service once the new ReplicaSet is healthy. This field is defaulted to true if it is not specified.
+
+Defaults to true
+
+### autoPromotionSeconds
+Setting a positive non-zero value here would make the rollout automatically promote the new `ReplicaSet` to active Service after this much time has been elapsed since the rollout has entered a paused state. If the `AutoPromotionEnabled` field is set to **false**, this field would be ignored.
+
+Defaults to nil
+
+### antiAffinity
+Check out the [Anti Affinity document](anti-affinity/anti-affinity.md) for more information.
+
+Defaults to nil
+
+### maxUnavailable
+The maximum number of pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). This can not be 0 if MaxSurge is 0.
+
+Defaults to 0
+
+### prePromotionAnalysis
+Configures the [Analysis](analysis.md#bluegreen-pre-promotion-analysis) before it switches traffic to the new version. The
+AnalysisRun can be used to block the Service selector switch until the AnalysisRun finishes successful. The success or
+failure of the analysis run decides if the Rollout will switch traffic, or abort the Rollout completely.
+
+Defaults to nil
+
+### postPromotionAnalysis
+Configures the [Analysis](analysis.md#bluegreen-pre-promotion-analysis) after the traffic switch to new version. If the analysis
+run fails or errors out, the Rollout enters an aborted state and switch traffic back to the previous stable Replicaset.
+If `scaleDownDelaySeconds` is specified, the controller will cancel any AnalysisRuns at time of `scaleDownDelay` to 
+scale down the ReplicaSet. If it is omitted, and post analysis is specified, it will scale down the ReplicaSet only 
+after the AnalysisRun completes (with a minimum of 30 seconds).
+
+Defaults to nil
+
+### previewService
+The PreviewService field references a Service that will be modified to send traffic to the new ReplicaSet before the new one is promoted to receiving traffic from the active service. Once the new ReplicaSet starts receiving traffic from the active service, the preview service will also be modified to send traffic to the new ReplicaSet as well. The Rollout always makes sure that the preview service is sending traffic to the newest ReplicaSet.  As a result, if a new version is introduced before the old version is promoted to the active service, the controller will immediately switch over to that brand new version.
 
 This feature is used to provide an endpoint that can be used to test a new version of an application.
 
 Defaults to an empty string
 
-### PreviewReplicaCount
-The PreviewReplicaCount will indicate the number of replicas that the new version of an application should run.  Once the application is ready to promote to the active service, the controller will scale the new ReplicaSet to the value of the `spec.replicas`. The rollout will not switch over the active service to the new ReplicaSet until it matches the `spec.replicas` count.
+### previewReplicaCount
+The PreviewReplicaCount field will indicate the number of replicas that the new version of an application should run.  Once the application is ready to promote to the active service, the controller will scale the new ReplicaSet to the value of the `spec.replicas`. The rollout will not switch over the active service to the new ReplicaSet until it matches the `spec.replicas` count.
 
 This feature is mainly used to save resources during the testing phase. If the application does not need a fully scaled up application for the tests, this feature can help save some resources.
 
-Defaults to nil
+If omitted, the preview ReplicaSet stack will be scaled to 100% of the replicas.
 
-### AutoPromotionEnabled
-The AutoPromotionEnabled will make the rollout automatically promote the new ReplicaSet to the active service once the new ReplicaSet is healthy. This field is defaulted to true if it is not specified.
-
-Defaults to true
-
-### AutoPromotionSeconds
-The AutoPromotionSeconds will make the rollout automatically promote the new ReplicaSet to active Service after the AutoPromotionSeconds time has passed since the rollout has entered a paused state. If the `AutoPromotionEnabled` field is set to true, this field will be ignored
-
-Defaults to nil
-
-
-### ScaleDownDelaySeconds
+### scaleDownDelaySeconds
 The ScaleDownDelaySeconds is used to delay scaling down the old ReplicaSet after the active Service is switched to the new ReplicaSet.
 
 Defaults to 30
 
-### ScaleDownDelayRevisionLimit
+### scaleDownDelayRevisionLimit
 The ScaleDownDelayRevisionLimit limits the number of old active ReplicaSets to keep scaled up while they wait for the scaleDownDelay to pass after being removed from the active service. 
 
-Default to nil
+If omitted, all ReplicaSets will be retained for the specified scaleDownDelay
+

docs/features/canary.md

@@ -1,69 +0,0 @@
-# Canary Deployment Strategy
-A canary rollout is a deployment strategy where the operator releases a new version of their application to a small percentage of the production traffic. 
-
-## Overview
-Since there is no agreed upon standard for a canary deployment, the rollouts controller allows users to outline how they want to run their canary deployment. Users can define a list of steps the controller uses to manipulate the RepliaSets where there is a change to the `.spec.template`. Each step will be evaluated before the new ReplicaSet is promoted to the stable version, and the old version is completely scaled down.
-
-Each step can have one of two fields. The `setWeight` field dictates the percentage of traffic that should be sent to the canary, and the `pause` struct instructs the rollout to pause.  When the controller reaches a `pause` step for a rollout, it will set the `.spec.paused` field to `true`. If the `duration` field within the `pause` struct is set, the rollout will not progress to the next step until it has waited for the value of the `duration` field. Otherwise, the rollout will wait indefinitely until the `.spec.paused` field is set to `false`. By using the `setWeight` and the `pause` fields, a user can declarative describe how they want to progress to the new version. Below is an example of a canary strategy.
-
-## Example
-```yaml
-apiVersion: argoproj.io/v1alpha1
-kind: Rollout
-metadata:
-  name: example-rollout
-spec:
-  replicas: 10
-  selector:
-    matchLabels:
-      app: nginx
-  template:
-    metadata:
-      labels:
-        app: nginx
-    spec:
-      containers:
-      - name: nginx
-        image: nginx:1.15.4
-        ports:
-        - containerPort: 80
-  minReadySeconds: 30
-  revisionHistoryLimit: 3
-  strategy:
-    canary: #Indicates that the rollout should use the Canary strategy
-      maxSurge: "25%"
-      maxUnavailable: 0,
-      steps:
-      - setWeight: 10
-      - pause:
-          duration: 3600 # 1 hour
-      - setWeight: 20
-      - pause: {}
-```
-
-## Mimicking Rolling Update
-If the steps field is omitted, the canary strategy will mimic the rolling update behavior. Similar to the deployment, the canary strategy has the `maxSurge` and `maxUnavailable` fields to configure how the Rollout should progress to the new version.
-
-## Other Configurable Features
-Here are the optional fields that will modify the behavior of canary strategy:
-```yaml
-spec:
-  strategy:
-    canary:
-      maxSurge: stringOrInt
-      maxUnavailable: stringOrInt
-      canaryService: string
-```
-
-### maxSurge
-`maxSurge` defines the maximum number of replicas the rollout can create to move to the correct ratio set by the last setWeight. Max Surge can either be an integer or percentage as a string (i.e. "20%")
-
-Defaults to "25%".
-### maxUnavailable
-The maximum number of pods that can be unavailable during the update. Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%). This can not be 0 if MaxSurge is 0.
-    // Defaults to 0
-
-### CanaryService
-`canaryService` references a Service that will be modified to send traffic to only the canary ReplicaSet. This allows users to only hit the canary ReplicaSet.
-
-Defaults to an empty string