Merge pull request #2719 from Rishab87/reason-label

feat: include reason label to `kube_deployment_status_condition`

.github/ISSUE_TEMPLATE/bug_report.md

@@ -9,7 +9,7 @@ assignees: ''
 
 <!-- Please use this template while reporting a bug and provide as much info as possible. Not doing so may result in your bug not being addressed in a timely manner. Thanks!
 
-If the matter is security related, please disclose it privately see https://github.com/kubernetes/kube-state-metrics/blob/master/SECURITY.md
+If the matter is security related, please disclose it privately see https://github.com/kubernetes/kube-state-metrics/blob/main/SECURITY.md
 -->
 
 **What happened**:
@@ -18,10 +18,18 @@ If the matter is security related, please disclose it privately see https://gith
 
 **How to reproduce it (as minimally and precisely as possible)**:
 
+```bash
+# An example: https://github.com/kubernetes/kube-state-metrics/issues/2223#issuecomment-1792850276
+minikube start
+...
+go run main.go --custom-resource-state-only --custom-resource-state-config-file ksm-2223/custom-resource-config-file.yaml --kubeconfig ~/.kube/config
+```
+
 **Anything else we need to know?**:
 
 **Environment**:
-- kube-state-metrics version: 
-- Kubernetes version (use `kubectl version`):
-- Cloud provider or hardware configuration:
-- Other info:
+
+* kube-state-metrics version:
+* Kubernetes version (use `kubectl version`):
+* Cloud provider or hardware configuration:
+* Other info:

.github/ISSUE_TEMPLATE/feature_request.md

@@ -9,10 +9,10 @@ assignees: ''
 
 <!-- Please only use this template for submitting feature requests -->
 
-**What would you like to be added**:
+**What would you like to be added:**
 
-**Why is this needed**:
+**Why is this needed:**
 
-**Describe the solution you'd like**
+**Describe the solution you'd like:**
 
-**Additional context**
+**Additional context:**

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,8 +5,10 @@
 4. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
 -->
 
-**What this PR does / why we need it**:
+<!-- markdownlint-disable-next-line MD041 -->
+**What this PR does / why we need it:**
 
-**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
+**How does this change affect the cardinality of KSM:** *(increases, decreases or does not change cardinality)*
+
+**Which issue(s) this PR fixes:** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*
 Fixes #
-

.github/dependabot.yml

@@ -0,0 +1,14 @@
+version: 2
+updates:
+  - package-ecosystem: "gomod"
+    directory: "/"
+    groups:
+      k8s-dependencies:
+        patterns:
+          - "k8s.io*"
+    schedule:
+      interval: "weekly"
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

.github/workflows/ci.yml

@@ -3,62 +3,57 @@ name: continuous-integration
 on:
   push:
     branches:
-    - master
+    - main
     - release*
     tags:
     - v1.*
     - v2.*
   pull_request:
     branches:
-    - master
+    - main
     - release*
 
+permissions:
+  contents: read
+
 env:
   E2E_SETUP_KIND: yes
   E2E_SETUP_KUBECTL: yes
   SUDO: sudo
+  GO_VERSION: "^1.24"
+  GOLANGCI_LINT_VERSION: "v2.0.2"
 
 jobs:
   ci-go-lint:
     name: ci-go-lint
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.16
-      id: go
-
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
-    - name: Setup environment
-      run: |
-        go get -v -t -d ./...
-        make install-tools
+    - name: Set up Go 1.x
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+      with:
+        go-version: ${{ env.GO_VERSION }}
+      id: go
 
     - name: Lint
       run: |
-        curl -sfL https://install.goreleaser.com/github.com/golangci/golangci-lint.sh | sh -s -- -b $(go env GOPATH)/bin v1.30.0
+        curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin ${{ env.GOLANGCI_LINT_VERSION }}
         make lint
 
   ci-validate-manifests:
     name: ci-validate-manifests
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.16
-      id: go
-
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
-    - name: Setup environment
-      run: |
-        go get -v -t -d ./...
-        make install-tools
+    - name: Set up Go 1.x
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+      with:
+        go-version: ${{ env.GO_VERSION }}
+      id: go
 
     - name: Validate generated manifests
       run: |
@@ -68,19 +63,14 @@ jobs:
     name: ci-validate-go-modules
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.16
-      id: go
-
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
-    - name: Setup environment
-      run: |
-        go get -v -t -d ./...
-        make install-tools
+    - name: Set up Go 1.x
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+      with:
+        go-version: ${{ env.GO_VERSION }}
+      id: go
 
     - name: Validate go modules
       run: |
@@ -90,21 +80,16 @@ jobs:
     name: ci-validate-docs
     runs-on: ubuntu-latest
     steps:
+    - name: Check out code into the Go module directory
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
     - name: Set up Go 1.x
-      uses: actions/setup-go@v2
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
       with:
-        go-version: ^1.16
+        go-version: ${{ env.GO_VERSION }}
       id: go
 
-    - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
-
-    - name: Setup environment
-      run: |
-        go get -v -t -d ./...
-        make install-tools
-
-    - name: Check that all metrics are documented
+    - name: Check that all metrics are documented and templates have no delta
       run: |
         make doccheck
 
@@ -112,19 +97,14 @@ jobs:
     name: ci-unit-tests
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.16
-      id: go
-
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
-    - name: Setup environment
-      run: |
-        go get -v -t -d ./...
-        make install-tools
+    - name: Set up Go 1.x
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+      with:
+        go-version: ${{ env.GO_VERSION }}
+      id: go
 
     - name: Unit tests
       run: |
@@ -135,7 +115,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@v2
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Setup promtool
         run: |
@@ -149,41 +129,41 @@ jobs:
     name: ci-benchmark-tests
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.16
-      id: go
-
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
-    - name: Setup environment
-      run: |
-        go get -v -t -d ./...
-        make install-tools
+    - name: Set up Go 1.x
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+      with:
+        go-version: ${{ env.GO_VERSION }}
+      id: go
 
     - name: Benchmark tests
       run: |
-        make test-benchmark-compare
+        BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare
+    - run: |
+        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+        cat result.txt >> "$GITHUB_STEP_SUMMARY"
+        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+        cat <<EOL >> "$GITHUB_STEP_SUMMARY"
+        <hr />
+        The table shows the median and 95% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10).
+        The last row has the Geometric Mean (geomean) for the given rows in the table.
+        Refer to <a href="https://pkg.go.dev/golang.org/x/perf/cmd/benchstat">benchstat's documentation</a> for more help.
+        EOL
 
   ci-build-kube-state-metrics:
     name: ci-build-kube-state-metrics
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.16
-      id: go
-
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
-    - name: Setup environment
-      run: |
-        go get -v -t -d ./...
-        make install-tools
+    - name: Set up Go 1.x
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+      with:
+        go-version: ${{ env.GO_VERSION }}
+      id: go
 
     - name: Build
       run: |
@@ -193,20 +173,15 @@ jobs:
     name: ci-e2e-tests
     runs-on: ubuntu-latest
     steps:
-    - name: Set up Go 1.x
-      uses: actions/setup-go@v2
-      with:
-        go-version: ^1.16
-      id: go
-
     - name: Check out code into the Go module directory
-      uses: actions/checkout@v2
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
-    - name: Setup environment
-      run: |
-        go get -v -t -d ./...
-        make install-tools
+    - name: Set up Go 1.x
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+      with:
+        go-version: ${{ env.GO_VERSION }}
+      id: go
 
     - name: End-to-end tests
       run: |
-        REGISTRY="quay.io/coreos" make e2e
+        make e2e

.github/workflows/govulncheck.yml

@@ -0,0 +1,29 @@
+name: govulncheck
+
+on:
+  schedule:
+    # Run every Monday
+    - cron: '0 0 * * 1'
+
+env:
+  GO_VERSION: "^1.24"
+
+permissions:
+  contents: read
+
+jobs:
+  ci-security-checks:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+        name: Checkout code
+      - name: Set up Go 1.x
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Install govulncheck binary
+        run: |
+          go install golang.org/x/vuln/cmd/govulncheck@latest
+      - name: Run security checks
+        run: |
+          govulncheck ./...

.github/workflows/helm-cd.yaml

@@ -1,30 +0,0 @@
-name: Release Charts
-
-on:
-  push:
-    branches:
-      - master
-    paths:
-      - 'charts/**'
-
-jobs:
-  release:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Configure Git
-        run: |
-          git config user.name "$GITHUB_ACTOR"
-          git config user.email "$GITHUB_ACTOR@users.noreply.github.com"
-
-      - name: Run chart-releaser
-        uses: helm/chart-releaser-action@v1.2.0
-        with:
-          charts_dir: charts
-        env:
-          CR_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
-          CR_RELEASE_NAME_TEMPLATE: "kube-state-metrics-helm-chart-{{ .Version }}"

.github/workflows/helm-ci.yaml

@@ -1,50 +0,0 @@
-name: Helm CI
-
-on:
-  pull_request:
-    paths:
-      - 'charts/**'
-
-jobs:
-  lint-and-test:
-    runs-on: ubuntu-latest
-    steps:
-      - name: Checkout
-        uses: actions/checkout@v2
-        with:
-          fetch-depth: 0
-
-      - name: Set up Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: v3.5.2
-
-      # Python is required because `ct lint` runs Yamale (https://github.com/23andMe/Yamale) and
-      # yamllint (https://github.com/adrienverge/yamllint) which require Python
-      - name: Set up Python
-        uses: actions/setup-python@v2
-        with:
-          python-version: 3.7
-
-      - name: Set up chart-testing
-        uses: helm/chart-testing-action@v2.0.1
-        with:
-          version: v3.3.1
-
-      - name: Run chart-testing (list-changed)
-        id: list-changed
-        run: |
-          changed=$(ct list-changed --config tests/ct.yaml)
-          if [[ -n "$changed" ]]; then
-            echo "::set-output name=changed::true"
-          fi
-
-      - name: Run chart-testing (lint)
-        run: ct lint --config tests/ct.yaml
-
-      - name: Create kind cluster
-        uses: helm/kind-action@v1.1.0
-        if: steps.list-changed.outputs.changed == 'true'
-
-      - name: Run chart-testing (install)
-        run: ct install --config tests/ct.yaml

.github/workflows/openvex.yml

@@ -0,0 +1,37 @@
+name: openvex
+
+on:
+  workflow_dispatch:
+  release:
+    types:
+      - released
+
+permissions:
+  contents: read
+
+jobs:
+  vexctl:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Set environment variables
+        run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV
+
+      - uses: openvex/generate-vex@c59881b41451d7ccba5c3b74cd195382b8971fcd
+        # Refer: https://github.com/openvex/vexctl#operational-model
+        name: Run vexctl
+        with:
+            product: pkg:golang/k8s.io/kube-state-metrics/v2@${{ env.RELEASE_VERSION }}
+            file: kube-state-metrics.openvex.json
+
+      - name: Upload OpenVEX document to GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload ${{ env.RELEASE_VERSION }} kube-state-metrics.openvex.json

.github/workflows/sbom.yaml

@@ -0,0 +1,41 @@
+name: Generate SBOM with Kubernetes BOM
+
+on:
+  release:
+    types:
+      - released
+
+permissions:
+  contents: read
+
+jobs:
+  sbom:
+    runs-on: ubuntu-latest
+
+    permissions:
+      contents: write
+
+    env:
+      OUTPUT: sbom.spdx
+      TAG: ${{ github.event.release.tag_name }}
+
+    steps:
+      - name: Fetch source code into GITHUB_WORKSPACE
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
+
+      - name: Install Kubernetes BOM
+        uses: kubernetes-sigs/release-actions/setup-bom@a30d93cf2aa029e1e4c8a6c79f766aebf429fddb # v0.3.1
+
+      - name: Generate SBOM
+        run: |
+          bom generate \
+            --dirs=. \
+            --image=registry.k8s.io/kube-state-metrics/kube-state-metrics:$TAG \
+            --namespace=https://github.com/kubernetes/kube-state-metrics/releases/download/$TAG/$OUTPUT \
+            --output=$OUTPUT
+
+      - name: Upload SBOM to GitHub Release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          gh release upload $TAG $OUTPUT

.github/workflows/semantic.yml

@@ -0,0 +1,23 @@
+name: "Lint pull request title"
+
+on:
+  pull_request_target:
+    types:
+      - opened
+      - edited
+      - synchronize
+
+permissions:
+  contents: read
+
+jobs:
+  main:
+    permissions:
+      pull-requests: read  # for amannn/action-semantic-pull-request to analyze PRs
+      statuses: write  # for amannn/action-semantic-pull-request to mark status of analyzed PR
+    name: Validate PR title for semantic commit message
+    runs-on: ubuntu-latest
+    steps:
+      - uses: amannn/action-semantic-pull-request@0723387faaf9b38adef4775cd42cfd5155ed6017 # v5.5.3
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

.golangci.yml

@@ -1,23 +1,60 @@
-run:
-  deadline: 2m
-
+version: "2"
 linters:
-  disable-all: true
+  default: none
+  enable:
+    - gocritic
+    - gocyclo
+    - gosec
+    - govet
+    - ineffassign
+    - misspell
+    - promlinter
+    - revive
+    - staticcheck
+    - unconvert
+    - unused
+  exclusions:
+    generated: lax
+    rules:
+      - linters:
+          - promlinter
+        path: _test\.go
+      - linters:
+          - gosec
+        text: 'G104:'
+      - linters:
+          - revive
+        text: 'package-comments:|var-naming:'
+      # This needs to stay as long as we support exposing v1.endpoints metrics
+      - linters:
+          - staticcheck
+        text: 'SA1019: v1.Endpoint'
+        path: 'internal/store/endpoint.*.go|internal/store/builder.go'
+      # TODO: Use functions with context https://github.com/kubernetes/kube-state-metrics/issues/2721
+      - linters:
+          - staticcheck
+        text: 'SA1019: .*List|Watch'
+        path: 'pkg/sharding/listwatch.go|pkg/watch/watch.go'
+
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+issues:
+  max-issues-per-linter: 0
+  max-same-issues: 0
+formatters:
   enable:
     - gofmt
     - goimports
-    - golint
-    - staticcheck
-    - gocyclo
-    - ineffassign
-    - misspell
-    - gocritic
-    - govet
-    - unconvert
-
-linters-settings:
-  goimports:
-    local-prefixes: k8s.io/kube-state-metrics
-
-issues:
-  exclude-use-default: false
+  settings:
+    goimports:
+      local-prefixes:
+        - k8s.io/kube-state-metrics
+        - k8s.io/kube-state-metrics/v2
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

.markdownlint-cli2.jsonc

@@ -0,0 +1,19 @@
+{
+  "globs": [
+    "**/*.md"
+  ],
+  // ToDo: Following rules can't be fixed automatically. They should be enabled when fixed.
+  "config": {
+    "MD004": {
+      "style": "asterisk"
+    },
+    "MD013": false, // https://github.com/markdownlint/markdownlint/blob/main/docs/RULES.md#md013---line-length
+    "MD033": {
+      "allowed_elements": [
+        "details",
+        "summary",
+        "br"
+      ]
+    }
+  }
+}

.openvex/templates/README.md

@@ -0,0 +1,28 @@
+# OpenVEX Templates Directory
+
+This directory contains the OpenVEX data for this repository.
+The files stored in this directory are used as templates by
+`vexctl generate` when generating VEX data for a release or
+a specific artifact.
+
+To add new statements to publish data about a vulnerability,
+download [vexctl](https://github.com/openvex/vexctl)
+and append new statements using `vexctl add`. For example:
+
+```bash
+vexctl add --in-place main.openvex.json pkg:oci/test CVE-2014-1234567 fixed
+```
+
+That will add a new VEX statement expressing that the impact of
+CVE-2014-1234567 is under investigation in the test image. When
+cutting a new release, for `pkg:oci/test` the new file will be
+incorporated to the relase's VEX data.
+
+## Read more about OpenVEX
+
+To know more about generating, publishing and using VEX data
+in your project, please check out the [vexctl repository and
+documentation](https://github.com/openvex/vexctl).
+
+OpenVEX also has an [examples repository](https://github.com/openvex/examples)
+with samples and docs.

.openvex/templates/main.openvex.json

@@ -0,0 +1,8 @@
+{
+  "@context": "https://openvex.dev/ns/v0.2.0",
+  "@id": "https://openvex.dev/docs/public/vex-2912204db7d51d98234a931d600f7cc2dd0bf24a5b5b326de138d64b30c22911",
+  "author": "vexctl (automated template)",
+  "timestamp": "2023-12-15T22:55:18.754525+05:30",
+  "version": 1,
+  "statements": []
+}

CHANGELOG.md

@@ -1,3 +1,351 @@
+# Changelog
+
+## v2.16.0 / 2025-06-23
+
+**Note:**
+
+* This release builds with Golang `v1.24.4`
+* This release builds with `k8s.io/client-go`: `v0.32.6`
+
+* [FEATURE] Add a `reclaim_policy` label to the `kube_persistentvolume_info` metric by @SuperQ in <https://github.com/kubernetes/kube-state-metrics/pull/2615>
+* [FEATURE] Use dlclark/regexp2 over standard library's package to support lookarounds by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2616>
+* [BUGFIX] Report correct values in `kube_pod_status_reason` metric by @carlosmorenokm1 in <https://github.com/kubernetes/kube-state-metrics/pull/2644>
+* [FEATURE] Add add `pathType` to `kube_ingress_path` by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2670>
+* [FEATURE] Introduce object limits by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2626>
+* [BUGFIX] Close reflectors once their corresponding CRDs are dropped by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2672>
+* [FEATURE] Incorporate `WithAuthenticationAndAuthorization` to support endpoint authn/z by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2686>
+
+## v2.15.0 / 2025-02-03
+
+**Note:**
+
+* This release builds with Golang `v1.23.5`
+* This release builds with `k8s.io/client-go`: `v0.32.1`
+
+* [BUGFIX] Fix CR cache for GVK all specified case by @chelseychen in <https://github.com/kubernetes/kube-state-metrics/pull/2567>
+* [BUGFIX] Deduplicate tolerations before creating metric by @RiRa12621 in <https://github.com/kubernetes/kube-state-metrics/pull/2559>
+* [BUGFIX] Make `$VERSION` 3rd-party independant by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2572>
+* [BUGFIX] Fix NoNodePod naming in jsonnet by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2598>
+* [BUGFIX] Panic in `util.GVRFromType` for structured types by @L3n41c in <https://github.com/kubernetes/kube-state-metrics/pull/2553>
+* [FEATURE] Add external traffic policy to kube_service_info by @jahantech in <https://github.com/kubernetes/kube-state-metrics/pull/2584>
+* [FEATURE] Promote kube_statefulset_ordinals_start from ALPHA -> STABLE by @pwschuurman in <https://github.com/kubernetes/kube-state-metrics/pull/2415>
+* [FEATURE] Add timezone to kube_cronjob_info / Make kube_cronjob_next_schedule_time timezone-aware by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2376>
+
+## v2.14.0 / 2024-11-08
+
+**Note:**
+
+* This release builds with Golang `v1.23.3`
+* This release builds with `k8s.io/client-go`: `v0.31.2`
+* This release removes `kube_endpoint_address_not_ready` and `kube_endpoint_address_available` which have been deprecated in 2022. Please use `kube_endpoint_address`as a replacement.
+
+* [BUGFIX] Use --track-unscheduled-pods to select unscheduled pods in Daemonset sharding by @CatherineF-dev in <https://github.com/kubernetes/kube-state-metrics/pull/2388>
+* [BUGFIX] Install tools so VERSION gets set by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2456>
+* [BUGFIX] Syntax errors in kube-state-metrics.libsonnet by @jeffmccune in <https://github.com/kubernetes/kube-state-metrics/pull/2454>
+* [BUGFIX] Set kube_job_status_failed metric even when there are no job.Status.Conditions present by @richabanker in <https://github.com/kubernetes/kube-state-metrics/pull/2485>
+* [BUGFIX] de-duplication of custom resource metrics by @bartlettc22 in <https://github.com/kubernetes/kube-state-metrics/pull/2502>
+* [BUGFIX] Configure sharding every time MetricsHandler.Run runs by @wallee94 in <https://github.com/kubernetes/kube-state-metrics/pull/2478>
+* [BUGFIX] Panic in `util.GVRFromType` for core objects by @L3n41c in <https://github.com/kubernetes/kube-state-metrics/pull/2535>
+* [BUGFIX] Big memory value overflow by @leiwingqueen in <https://github.com/kubernetes/kube-state-metrics/pull/2540>
+* [BUGFIX] Expose empty labels by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2539>
+* [BUGFIX] CustomResourceMetrics: Convert status condition Unknown to a valid value by @Haleygo in <https://github.com/kubernetes/kube-state-metrics/pull/2536>
+* [CHANGE] Remove deprecated endpoint address metric by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2527>
+* [FEATURE] Add new metric kube_job_status_suspended by @Indresh2410 in <https://github.com/kubernetes/kube-state-metrics/pull/2542>
+* [FEATURE] Move endpoint ports into address metric by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2503>
+* [ENHANCEMENT] Use concurrent map when storing metrics by @rarruda in <https://github.com/kubernetes/kube-state-metrics/pull/2510>
+
+## v2.13.0 / 2024-07-18
+
+**Note:**
+
+* This release builds with Golang `v1.22.5`.
+* This release builds with `k8s.io/client-go`: `v0.30.3`.
+* This release adds read and write timeouts for requests. The defaults might have an impact on scrapes that take a long time.
+
+* [BUGFIX]  Pod autosharding: transition from labelselector to fieldselector by @pkoutsovasilis in <https://github.com/kubernetes/kube-state-metrics/pull/2347>
+* [ENHANCEMENT] Add automatic detection of memory limits by @SuperQ in <https://github.com/kubernetes/kube-state-metrics/pull/2447>
+* [FEATURE] Add `readyz` endpoint by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2442>
+* [FEATURE] Add `livez` endpoint by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2418>
+* [FEATURE] Add kube_persistentvolume_volume_mode metric by @ricardoapl in <https://github.com/kubernetes/kube-state-metrics/pull/2370>
+* [FEATURE] Add read and write timeouts by @Pokom in <https://github.com/kubernetes/kube-state-metrics/pull/2412>
+
+## v2.12.0 / 2024-04-02
+
+**Note:**
+
+* This release addresses a critical issue where scraping the exposition data for certain types caused metrics-backends to crash: <https://github.com/kubernetes/kube-state-metrics/issues/2248>.
+* This release builds with `k8s.io/client-go`: `v0.29.3`.
+
+* [BUGFIX] Fallback to `gauge` for `protobuf`-based negotiations by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2270>
+* [ENHANCEMENT] Add `kube_pod_container_status_last_terminated_timestamp`by @tetianakravchenko in <https://github.com/kubernetes/kube-state-metrics/pull/2291>
+* [FEATURE] Allow prefixing expandable paths by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2052>
+
+## v2.11.0 / 2024-03-04
+
+**Note:**
+
+This release builds with Golang `v1.21.8`.
+
+* [ENHANCEMENT] Add OpenSSF Scorecard to README by @dalehenries in <https://github.com/kubernetes/kube-state-metrics/pull/2277>
+* [ENHANCEMENT] Generate OpenVX data for every release by @shafeeqes in <https://github.com/kubernetes/kube-state-metrics/pull/2276>
+* [ENHANCEMENT] Add restartPolicy to `kube_pod_init_container_info` metric by @changhyuni in <https://github.com/kubernetes/kube-state-metrics/pull/2240>
+* [FEATURE] Add `kube_node_status_addresses` metric by @stonith in <https://github.com/kubernetes/kube-state-metrics/pull/2252>
+* [FEATURE] Add namespace label to `endpointslice` metrics by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2266>
+* [FEATURE] Add opt-in `kube_persistentvolume_csi_attributes` metric by @machadovilaca in <https://github.com/kubernetes/kube-state-metrics/pull/2133>
+* [FEATURE] Add new metric `kube_pod_scheduler` metric by @adinhodovic in <https://github.com/kubernetes/kube-state-metrics/pull/2222>
+* [FEATURE] Support filtering annotations allow-list by `*` by @xonvanetta in <https://github.com/kubernetes/kube-state-metrics/pull/2234>
+* [ENHANCEMENT] Support scraping pod metrics that are still in scheduling status and have no assigned nodes by @mickeyzzc in <https://github.com/kubernetes/kube-state-metrics/pull/2217>
+* [FEATURE] Add backend resource info for `kube_ingress_path` by @rohitphatak in <https://github.com/kubernetes/kube-state-metrics/pull/2109>
+
+## v2.10.1 / 2023-10-09
+
+**Note:**
+
+* This release addresses a regression introduced in [#2105](https://github.com/kubernetes/kube-state-metrics/pull/2105).
+
+* [BUGFIX] Remove FieldSelector from non-namespaced resources by @mrueg and @dgrisonnet in [#2190](https://github.com/kubernetes/kube-state-metrics/pull/2190)
+* [ENHANCEMENT] Bump Go to v1.20.8
+
+## v2.10.0 / 2023-08-31
+
+**Note:**
+
+* Label and annotation metrics aren't exposed by default anymore to reduce the memory usage of the default configuration of kube-state-metrics. Before this change, they used to only include the name and namespace of the objects which is not relevant to users not opting in these metrics.
+
+* [BUGFIX] Log no _info suffix in name only once per reading the configuration for custommetrics by @chrischdi in <https://github.com/kubernetes/kube-state-metrics/pull/2157>
+* [BUGFIX] Don't crash on non-existent valueFrom path values by @chihshenghuang in <https://github.com/kubernetes/kube-state-metrics/pull/2140>
+* [BUGFIX] Index out of range in metrics_store.SanitizeHeaders by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2166>
+* [BUGFIX] Always extract the headers but only write it when we have custommetrics by @chrischdi in <https://github.com/kubernetes/kube-state-metrics/pull/2154>
+* [BUGFIX] Add filtering for Lease metrics by @ntoofu in <https://github.com/kubernetes/kube-state-metrics/pull/2122>
+* [FEATURE] Implement kube_pod_status_initialized_time by @opeco17 in <https://github.com/kubernetes/kube-state-metrics/pull/2148>
+* [FEATURE] Disable labels and annotations metrics when metric-annotations-… by @opeco17 in <https://github.com/kubernetes/kube-state-metrics/pull/2145>
+* [FEATURE] Add webhooks client config service metrics by @dgrisonnet in <https://github.com/kubernetes/kube-state-metrics/pull/2114>
+* [FEATURE] Support kube_persistentvolumeclaim_deletion_timestamp by @maxime1907 in <https://github.com/kubernetes/kube-state-metrics/pull/2074>
+* [FEATURE] Support kube_persistentvolume_deletion_timestamp by @maxime1907 in <https://github.com/kubernetes/kube-state-metrics/pull/2075>
+* [FEATURE] Adds new metric `kube_pod_service_account` by @swarup-stripe in <https://github.com/kubernetes/kube-state-metrics/pull/2096>
+* [FEATURE] Add volumemode to PVC info metric by @machadovilaca in <https://github.com/kubernetes/kube-state-metrics/pull/2134>
+* [FEATURE] Enable metric-annotations-allowlist and metric-labels-allowlist for ResourceQuota by @opeco17 in <https://github.com/kubernetes/kube-state-metrics/pull/2175>
+* [FEATURE] Allow field KV general matching by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2067>
+* [FEATURE] Support hot reload for kubeconfig by @opeco17 in <https://github.com/kubernetes/kube-state-metrics/pull/2144>
+* [FEATURE] Add support for endpoint topology routing hints by @MarkSRobinson in <https://github.com/kubernetes/kube-state-metrics/pull/2090>
+
+## v2.9.2 / 2023-05-30
+
+This release does not incorporate any user-facing changes. Re-running release procedures as the process for the previous release failed. Changes are listed in v2.9.0.
+
+## v2.9.1 / 2023-05-29
+
+This release does not incorporate any user-facing changes. Re-running release procedures as the process for the previous release failed. Changes are listed in v2.9.0.
+
+## v2.9.0 / 2023-05-23
+
+The changes mentioned below are only the user-facing ones. For a list of the complete set of changes, refer the changelog associated with the release tag.
+
+**Note:**
+
+* The deprecated experimental VerticalPodAutoscaler metrics are no longer supported, and have been removed. We recommend to use CustomResourceState metrics to gather metrics from custom resources like the Vertical Pod Autoscaler.
+* #2004 regulated label names to adhere with [OTel-Prometheus standards](https://github.com/open-telemetry/opentelemetry-specification/blob/8946dfc6a2302f78b0224fcc3f4dfb816a7bb1f4/specification/compatibility/prometheus_and_openmetrics.md?plain=1#L224-L229), so existing label names that do not follow the same may be replaced by the ones that do. Please refer to the PR for more details.
+
+* [BUGFIX] Adhere to OTel-Prometheus standard for labels #2004 @rexagod
+* [BUGFIX] Respect relative paths for label resolutions #2007 @rexagod
+* [BUGFIX] Support LabelsFromPath functionality for Info-typed metrics #2048 @murphd40
+* [CHANGE] Remove VerticalPodAutoscaler #2017 @mrueg
+* [ENHANCEMENT] Add StatefulSet Start Ordinal metrics for KEP-3335 #1959 @pwschuurman
+* [ENHANCEMENT] Add namespace label to `kube_lease_renew_time` #2073 @A-Hilaly
+* [ENHANCEMENT] Add parameters for PodSecurity #2042 @jcpunk
+* [ENHANCEMENT] Add support for multi cluster prometheus alerts #2058 @jkroepke
+* [ENHANCEMENT] Don't crash on non-existent path values #1998 @rexagod
+* [ENHANCEMENT] Only use OpenMetrics and Text in contentType #2024 @CatherineF-dev
+* [ENHANCEMENT] Update go version to 1.20.4 #2056 @RamakrishnanArun
+* [FEATURE] Add support for variable VKs in CRS config #1851 @rexagod
+* [FEATURE] Support quantities and percentages #1989 @mrueg
+* [FEATURE] Use prometheus/exporter-toolkit landing page #2034 @mrueg
+
+## v2.8.2 / 2023-03-17
+
+* [BUGFIX]      Only use OpenMetrics and Text in contentType #2024 @CatherineF-dev
+
+## v2.8.1 / 2023-02-22
+
+* [BUGFIX]      Don't crash on non-existent paths @rexagod #1998
+* [BUGFIX]      Fix public Builder compatibility with BuilderInterface @clamoriniere #1994
+
+## v2.8.0 / 2023-02-10
+
+Note: The `--version` flag was removed as `kube-state-metrics version` also provides the same information. See #1956
+
+Note: Experimental CustomResourceState changed their labels for better usability. See #1942
+kube_crd_uptime{group="myteam.io", kind="Foo", version="v1"} -> kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"}
+
+* [CHANGE]       Prefix Group, Version and Kind labels for CustomResourceState Metrics #1942 @bavarianbidi
+* [CHANGE]       Fix empty string for "owner_\*" dimensions #1923 @pawcykca
+* [CHANGE]       Remove broken --version flag, replace by version command #1956 @bjorand
+* [FEATURE]      Add metrics for EndpointSlices #1910 @mrueg
+* [FEATURE]      Add metrics for config file changes #1916 @mrueg
+* [FEATURE]      Add metrics for CustomResource State config file change #1928 @mrueg
+* [FEATURE]      Reload Kube-State-Metrics on CustomResourceState config file change #1930 @mrueg
+* [FEATURE]      Make CustomResourceState metrics type dynamic #1930 @rexagod
+* [FEATURE]      Add kube_pod_status_qos_class to pod metrics #1932 @frezes
+* [FEATURE]      Add kube_pod_status_ready_time and kube_pod_status_containers_ready_time metrics #1938 @ryanrolds
+* [FEATURE]      Enrich UserAgent with more information about kube-state-metrics #1960 @mrueg
+* [FEATURE]      Convert True/False to 1.0/0.0 values in CustomResourceState metrics #1963 @jabdoa2
+* [FEATURE]      Expose metrics in OpenMetrics format #1974 @mrueg
+* [BUGFIX]       Handle unit length `valueFrom` values #1958 @rexagod
+* [ENHANCEMENT]  Build with kubernetes 1.26 #1933 @mrueg
+
+## v2.7.0 / 2022-11-25
+
+Note: Experimental VerticalPodAutoscaler metrics are considered deprecated in this release and will be removed in v2.9.0.
+Please use CustomResourceStateMetrics to gather metrics from VPA resources. See: #1718
+
+Note: Experimental CustomResourceState changed their naming convention for better usability.
+The name of the CRD used to be interpolated into the name of the metric which made it impossible to aggregate a CRD across different versions.
+This was changed to have the GVK information represented as labels:
+kube_myteam_io_v1_Foo_uptime -> kube_crd_uptime{group="myteam.io", kind="Foo", version="v1"}
+See: #1847
+
+* [CHANGE]      Deprecate VerticalPodAutoscaler metrics #1835 @rexagod
+* [CHANGE]      Recommend kube-scheduler metrics for container resource limits and requests #1849 @rexagod
+* [FEATURE]     Add experimental kube_pod_container_status_last_terminated_exitcode metric #1752 @ssabo
+* [FEATURE]     Introduce custom-resources-only flag #1813 @bavarianbidi
+* [FEATURE]     Allow allowlist filtering by wildcard key #1823 @rexagod
+* [FEATURE]     Add ContainerResourceSourceType to hpa spec and target metrics #1831 @whitebear009
+* [FEATURE]     Represent group, version and kind of a resource as labels #1850 @rexagod
+* [FEATURE]     Sharding metrics per node via fieldSelector #1864 @CatherineF-dev
+* [FEATURE]     Add experimental StatefulSet retention policy metrics #1876 @mattcary
+* [FEATURE]     Allow labelFromKey field for all applicable metric types #1880 @rexagod
+* [FEATURE]     Introduce Viper, allow hot-reload on config change #1827 @rexagod
+* [FEATURE]     Introduce Cobra, allow configuration via environment variables #1834 @rexagod
+* [FEATURE]     Add experimental kube_node_deletion_timestamp metric #1890 @rexagod
+* [FEATURE]     Support autoscaling/v2 resources for HorizontalPodAutoscaler #1906 @JoaoBraveCoding
+* [FEATURE]     Add IngressClass metrics #1905 @kaitoii11
+* [ENHANCEMENT] Import Kubernetes metrics stability framework #1844 @CatherineF-dev
+* [ENHANCEMENT] Promote kube_pod_container_status_waiting_reason and kube_deployment_status_replicas_ready to stable #1821 @CatherineF-dev
+* [ENHANCEMENT] Build with Kubernetes 1.25 and go 1.19 #1819 @mrueg
+* [BUGFIX]      Handle singular labels in allowlist #1826 @rexagod
+* [BUGFIX]      Do not expose ingress path metric when service is nil #1841 @evir35
+* [BUGFIX]      Allow lease metrics to be exported across all namespaces #1845 @lantingchiang
+
+## v2.6.0 / 2022-08-26
+
+* [FEATURE]     Add local storage labels to kube_persistentvolume_info #1814 @nabokihms
+* [FEATURE]     Add support for StateSet and Info metrics for Custom-Resource State #1777 @chrischdi
+* [FEATURE]     Add support for Rolebinding resource metrics #1799 @kaitoii11
+* [FEATURE]     Add new kube_horizontalpodautoscaler_status_target_metric #1725 @tanguyfalconnet
+* [FEATURE]     Add support for Role and Clusterrole resource metrics #1759 @kaitoii11
+* [FEATURE]     Add support for the ServiceAccount resource metrics #1773 @Serializator
+* [FEATURE]     Add metric for pod tolerations #1769 @Serializator
+* [FEATURE]     Add new kube_endpoint_address metric #1761 @bavarianbidi
+* [FEATURE]     Support reading the timestamp fields of custom resources #1766 @dontan001
+* [ENHANCEMENT] Migrate to structured logging #1807 @dmpe
+* [ENHANCEMENT] Switch registry vanity domain from k8s.gcr.io to registry.k8s.io #1750 @mrueg
+* [ENHANCEMENT] Graduate new endpoint metrics to stable #1812 @bavarianbidi
+* [BUGFIX]      Fix label name for kube_pod_nodeselector #1756 @yosshi825
+* [BUGFIX]      Fix Custom-Resource State Metrics not to contain underscores #1754 @chrischdi
+* [BUGFIX]      Fix kube_node_status_allocatable unit doc #1760 @jumbosushi
+
+## v2.5.0 / 2022-06-03
+
+* [FEATURE]     Add experimental Custom-Resource State Metrics #1710 @iamnoah
+* [FEATURE]     Add kube_pod_ips metric #1740 @bjschafer
+* [FEATURE]     Add kube_pod_nodeselector metric #1675 @geojaz
+* [FEATURE]     Add CSIPersistentVolumeSource to persistent volume metric #1727 @luke-sprig
+* [FEATURE]     Add kube_cronjob_status_last_successful_time metric #1732 @splitice
+* [FEATURE]     Add kube_persistentvolumeclaim_created metric #1741 @aidan-ogorman-dev
+* [ENHANCEMENT] Build with Go 1.18 #1726, #1745 @mrueg
+* [ENHANCEMENT] Bump kubernetes 1.24 and other go dependencies #1726, #1743 @mrueg
+* [ENHANCEMENT] Update x/crypto to mitigate CVE-2022-27191 #1721 @pgvishnuram
+* [BUGFIX]      Assert that newlines in comma separated arguments are ignored #1706 @sthaha
+* [BUGFIX]      Fix potential panic in pod store #1723 @mfojtik
+* [BUGFIX]      Fix potential panic in internal store #1731 @jan--f
+* [BUGFIX]      Properly initialize KSM Server test #1699 @fpetkovski
+
+## v2.4.2 / 2022-02-10
+
+* [BUGFIX]  Publish images with with the correct tag
+
+## v2.4.1 / 2022-02-10
+
+* [FEATURE]     Add `ingressclass` label to `kube_ingress_info` metric #1652 @adammw
+* [FEATURE]     Extend KSM library to support custom resource metrics #1644 @Garrybest
+* [ENHANCEMENT] Use apiVersion `v1` for `PodDisruptionBudget` and `CronJob` resources #1491 @bison
+* [ENHANCEMENT] Optimize slice allocations #1676 @sherifabdlnaby
+* [BUGFIX]      Use plural form of resource name in `allowlist` for PodDisruptionBudget #1653 @arajkumar
+
+## v2.3.0 / 2021-12-09
+
+* [FEATURE]     Add a `--namespace-denylist` command line flag to exclude metrics from certain namespaces #1596 #1595 @mallow111
+* [FEATURE]     Add `kube_*_labels` and `kube_*_annotations` metrics for Pod Disruption Budgets #1623 @arajkumar
+* [FEATURE]     Add a Kustomization file for deploying KSM in authosharding mode #1603 @karancode
+* [FEATURE]     Expose a metric for the number of ports in endpoint objects #1571 @bavarianbidi
+* [FEATURE]     Add a command line flag for opt-in metrics #1643 @Serializator
+* [FEATURE]     Add `kube_horizontalpodautoscaler_info` metric #1648 @Serializator
+* [ENHANCEMENT] Update Go to 1.17.4 #1649 @fpetkovski
+* [ENHANCEMENT] Update Kubernetes to 1.23 #1649 @fpetkovski
+* [BUGFIX]      Report the correct architecture for arm64 images #1629 @Serializator
+
+## v2.2.4 / 2021-11-08
+
+* [BUGFIX] Fix BuilderInterface and BuildStoresFunc to allow using KSM as a library #1618 @ahmed-mez
+
+## v2.2.3 / 2021-10-13
+
+* [BUGFIX] Fix the image build job. Reverts #1602
+
+## v2.2.2 / 2021-10-13
+
+* [BUGFIX]  Downgrade latest allowed go version to 1.16.9 #1601 @mrueg
+* [BUGFIX]  Fix CI variable names used for building KSM images @mrueg
+
+## v2.2.1 / 2021-09-24
+
+* [FEATURE] Add the kube_persistentvolumeclaim_annotations metric which exposes annotations on PVCs #1566 @arajkumar
+* [BUGFIX]  Revert the accidentally removed kube_persistentvolumeclaim_labels metric #1566 @arajkumar
+* [BUGFIX]  Filter annotations in metrics based on `--metric-annotations-allowlist` instead of `--metric-labels-allowlist` for
+CronJob, Daemonset, HPA and Ingress resources #1580 @midnw
+* [BUGFIX]  Avoid panicking when VPA objects have no targetRef #1584 @nabokihms
+
+## v2.2.0 / 2021-08-24
+
+* [FEATURE] Add --use-apiserver-cache flag to set resourceVersion=0 for ListWatch requests #1548
+* [FEATURE] Introduce metrics for Kubernetes object annotations #1468
+* [FEATURE] Introduce start time metric for containers in terminated state #1519
+* [FEATURE] Introduce metrics for cronjob job history limits #1535
+* [FEATURE] Add system_uuid dimension to kube_node_info metric #1535
+* [FEATURE] Add available replica metric for statefulsets #1532
+* [FEATURE] Add ready replica metric for deployments #1534
+* [CHANGE]  Update go clients for Kubernetes to support 1.22 #1545
+* [CHANGE]  Use new promlint package and update prometheus cli to 2.28.1 #1531
+
+## v2.1.1 / 2021-07-28
+
+* [CHANGE] go.mod: Update and minimize dependencies #1529
+* [CHANGE] Use BuilderInterface instead of internal/store.Builder in metricshandler #1537
+* [CHANGE] Add WithAllowLabels to public BuilderInterface #1514
+* [BUGFIX] Fixes a bug where KSM did not export any metrics when it had no permissions for resources in at least one namespace #1499
+
+## v2.1.0 / 2021-06-04
+
+* [CHANGE] Update go version and dependencies #1493
+
+## v2.1.0-rc.0 / 2021-05-20
+
+* [FEATURE] Add support for native TLS #1354
+* [FEATURE] Add wildcard option to metric-labels-allowlist #1403
+* [FEATURE] Add build info metric #1332
+* [CHANGE] Add "uid" label to every pod metric #1304
+* [CHANGE] Add resourceVersion to CronJob metrics #1447
+* [CHANGE] Update go version and dependencies #1474
+* [CHANGE] Bump client-go and friends to v0.21 (Kubernetes v1.21) #1463
+* [CHANGE] Replace deprecated use of ioutil #1458
+* [BUGFIX] Fix builder.Builder WithMetrics signature #1455
+* [BUGFIX] Fix pod-metric missing reasons #1287
+* [BUGFIX] Fix multiListWatch resourceVersion mismatch if watch reconnected #1377
+
+## v2.0.0 / 2021-04-13
+
+* [CHANGE] Update go version and dependencies #1440
+
 ## v2.0.0-rc.1 / 2021-03-26
 
 * [CHANGE] Rename --labels-metric-allow-list to --metric-labels-allowlist #1424
@@ -19,6 +367,7 @@ Location on quay.io will not be updated anymore. Previously pushed images will b
 * [BUGFIX] Propagate resource version when sharded #1402
 
 ## v2.0.0-beta / 2020-12-04
+
 Promotion to beta release after a period of no bugs.
 
 ## v2.0.0-alpha.3 / 2020-11-19
@@ -41,69 +390,68 @@ kube_pod_container_resource_requests and kube_pod_container_resource_limits
 
 * [CHANGE] Update go module path to k8s.io/kube-state-metrics/v2 #1238
 * [CHANGE] Bump klog to v2 and client-go to 1.19 #1250
-* [FEATURE] Add iscsi initiator name to persistentvolume_info #1235 
-* [BUGFIX] Added Namespace to Rolebinding Jsonnet #1233 
-* [BUGFIX] Reference closure scoped family generator #1240 
+* [FEATURE] Add iscsi initiator name to persistentvolume_info #1235
+* [BUGFIX] Added Namespace to Rolebinding Jsonnet #1233
+* [BUGFIX] Reference closure scoped family generator #1240
 
 ## v2.0.0-alpha / 2020-09-16
 
 NOTE: This is a major new alpha 2.0 release with breaking changes and removed metrics. See details below!
 
-* [CHANGE] Apply boundaries to metrics and allow via flag (--labels-allow-list) what labels to include #1125 
+* [CHANGE] Apply boundaries to metrics and allow via flag (--labels-allow-list) what labels to include #1125
 * [CHANGE] Update DaemonSet updated_number_scheduled metric name to be consistent #1181
-Metric was changed from kube_daemonset_updated_number_scheduled to kube_daemonset_status_updated_number_scheduled 
+Metric was changed from kube_daemonset_updated_number_scheduled to kube_daemonset_status_updated_number_scheduled
 * [CHANGE] Rework resource metrics #1168
 Metrics kube_pod_container_resource_requests, kube_pod_container_resource_limits, kube_pod_overhead, kube_pod_init_container_resource_limits, kube_pod_init_container_resource_requests changed
-* [CHANGE] Convert k8s labels to snake case #1165 
-* [CHANGE] Mutatingwebhookconfiguration.go: Switch to v1 #1144 
+* [CHANGE] Convert k8s labels to snake case #1165
+* [CHANGE] Mutatingwebhookconfiguration.go: Switch to v1 #1144
 * [CHANGE] v2: Rename storage class labels reclaimPolicy to reclaim_policy and volumeBindingMode to volume_binding_mode #1107
 * [CHANGE] v2: Renamed --namespace flag to --namespaces #1098
 * [CHANGE] Rename kube_pod_deleted to kube_pod_deletion_timestamp #1079
 * [CHANGE] v2: Rename collector flag to resource flag #1006
 --resources is the new flag
-* [CHANGE] Remove non-identifying labels from pod metrics #1009 
+* [CHANGE] Remove non-identifying labels from pod metrics #1009
 * [CHANGE] v2: Remove deprecated stable metrics #1004
 Note that some of these were replaced with EXPERIMENTAL resource metrics. See #1168 for more details.
   `kube_pod_container_resource_requests` and `kube_pod_container_resource_limits` are the replacements with `resource` labels
   representing the resource name and `unit` labels representing the resource unit.
-  - kube_pod_container_resource_requests_cpu_cores
-  - kube_pod_container_resource_limits_cpu_cores
-  - kube_pod_container_resource_requests_memory_bytes
-  - kube_pod_container_resource_limits_memory_bytes
-  - `kube_node_status_capacity` and `kube_node_status_allocatable` are the replacements with `resource` labels
+  * kube_pod_container_resource_requests_cpu_cores
+  * kube_pod_container_resource_limits_cpu_cores
+  * kube_pod_container_resource_requests_memory_bytes
+  * kube_pod_container_resource_limits_memory_bytes
+  * `kube_node_status_capacity` and `kube_node_status_allocatable` are the replacements with `resource` labels
   representing the resource name and `unit` labels representing the resource unit.
-  - kube_node_status_capacity_pods
-  - kube_node_status_capacity_cpu_cores
-  - kube_node_status_capacity_memory_bytes
-  - kube_node_status_allocatable_pods
-  - kube_node_status_allocatable_cpu_cores
-  - kube_node_status_allocatable_memory_bytes
+  * kube_node_status_capacity_pods
+  * kube_node_status_capacity_cpu_cores
+  * kube_node_status_capacity_memory_bytes
+  * kube_node_status_allocatable_pods
+  * kube_node_status_allocatable_cpu_cores
+  * kube_node_status_allocatable_memory_bytes
 * [CHANGE] Rename black-/whitelist to allow/deny-list #1045
 New flags are --metric-allowlist and --metric-denylist
-* [CHANGE] Update telemetry port to 8081 #1049 
-* [CHANGE] v2: Rename hpa metrics to use full horizontalpodautoscaler #1003 
+* [CHANGE] Update telemetry port to 8081 #1049
+* [CHANGE] v2: Rename hpa metrics to use full horizontalpodautoscaler #1003
 All metrics with prefix of kube_hpa_ were renamed to kube_horizontalpodautoscaler_
-* [CHANGE] v2: change metrics port to 8080, telemetry port to 8081 #1005 
+* [CHANGE] v2: change metrics port to 8080, telemetry port to 8081 #1005
 * [FEATURE] Add http request metric for kube-state-metrics main /metrics #1218  
-* [FEATURE] Add fc/iscsi/nfs identifier tags to persistentvolume_info metric #1208 
-* [FEATURE] Adds new pod metric kube_pod_container_state_started #1183 
+* [FEATURE] Add fc/iscsi/nfs identifier tags to persistentvolume_info metric #1208
+* [FEATURE] Adds new pod metric kube_pod_container_state_started #1183
 * [FEATURE] Add observedGeneration metric (kube_daemonset_status_observed_generation) for DaemonSets #1178
-* [FEATURE] Add internal_ip to node_info #1172 
-* [FEATURE] Avoid conflicts when mapping Kubernetes labels to Prometheus labels #1156 
+* [FEATURE] Add internal_ip to node_info #1172
+* [FEATURE] Avoid conflicts when mapping Kubernetes labels to Prometheus labels #1156
 * [FEATURE] Add aws/gce volume id to kube_persistentvolume_info #1146
 * [FEATURE] Add UnexpectedAdmissionError to kube_pod_status_reason metric #1145
 * [FEATURE] Add init container requests (kube_pod_init_container_resource_requests) #1123
-* [FEATURE] Add host_network to kube_pod_info #1100 
-* [FEATURE] Add kube_replicationcontroller_owner #1058 
-* [FEATURE] Pod: add gauges for pod overhead (kube_pod_overhead) #1053 
-* [FEATURE] Add "Terminating" status in kube_pod_status_phase metrics #1013 
-* [FEATURE] Add lease collector metrics #1038 
+* [FEATURE] Add host_network to kube_pod_info #1100
+* [FEATURE] Add kube_replicationcontroller_owner #1058
+* [FEATURE] Pod: add gauges for pod overhead (kube_pod_overhead) #1053
+* [FEATURE] Add "Terminating" status in kube_pod_status_phase metrics #1013
+* [FEATURE] Add lease collector metrics #1038
 * [ENHANCEMENT] Add DeprecatedVersion to struct FamilyGenerator and func NewFamilyGenerator #1160
-* [ENHANCEMENT] Add security context to deployment and statefulset #1034 
+* [ENHANCEMENT] Add security context to deployment and statefulset #1034
 * [BUGFIX] Fix VolumeAttachment API version mismatch: expected v1 but watching v1beta1 #1136
-* [BUGFIX] Fix various CI issues (kube_volumeattachment_info nodeName -> node label rename) #1117 
-* [BUGFIX] Fix maxUnavailable to round down instead up #1076 
-
+* [BUGFIX] Fix various CI issues (kube_volumeattachment_info nodeName -> node label rename) #1117
+* [BUGFIX] Fix maxUnavailable to round down instead up #1076
 
 ## v1.9.7 / 2020-05-24
 

CONTRIBUTING.md

@@ -6,21 +6,49 @@ _As contributors and maintainers of this project, and in the interest of fosteri
 
 ## Getting Started
 
-We have full documentation on how to get started contributing here: 
+We have full documentation on how to get started contributing here:
 
-<!---
-If your repo has certain guidelines for contribution, put them here ahead of the general k8s resources
--->
+### Semantic Commit Messages
 
-- [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests
-- [Kubernetes Contributor Guide](http://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](http://git.k8s.io/community/contributors/guide#contributing)
-- [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet/README.md) - Common resources for existing developers
+We use [semantic commit messages](https://www.conventionalcommits.org/en/v1.0.0/) in this repository.
+
+They follow this format: `<type>[optional scope]: <description>`
+
+Examples for commit messages following this are:
+
+`feat: allow provided config object to extend other configs`
+
+You can also include a scope within parenthesis:
+
+`fix(scope): Prevent wrong calculation of storage`
+
+Here's a list of types that we use:
+
+| Type | Explanation |
+|---|---|
+| feat | A new feature |
+| fix | A bug fix |
+| docs | Documentation only changes |
+| style | Changes that do not affect the meaning of the code (white-space, formatting, missing semi-colons, etc) |
+| refactor | A code change that neither fixes a bug nor adds a feature |
+| perf |  A code change that improves performance |
+| test | Adding missing tests or correcting existing tests |
+| build |Changes that affect the build system or external dependencies (example scopes: gulp, broccoli, npm) |
+| ci | Changes to our CI configuration files and scripts (example scopes: Travis, Circle, BrowserStack, SauceLabs) |
+| chore | Other changes that don't modify src or test files |
+| revert | Reverts a previous commit |
+
+### Further Information
+
+* [Contributor License Agreement](https://git.k8s.io/community/CLA.md) Kubernetes projects require that you sign a Contributor License Agreement (CLA) before we can accept your pull requests
+* [Kubernetes Contributor Guide](http://git.k8s.io/community/contributors/guide) - Main contributor documentation, or you can just jump directly to the [contributing section](http://git.k8s.io/community/contributors/guide#contributing)
+* [Contributor Cheat Sheet](https://git.k8s.io/community/contributors/guide/contributor-cheatsheet/README.md) - Common resources for existing developers
 
 ## Mentorship
 
-- [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers!
+* [Mentoring Initiatives](https://git.k8s.io/community/mentoring) - We have a diverse set of mentorship programs available that are always looking for volunteers!
 
 ## Contact Information
 
-- [Join Slack](http://slack.k8s.io) to sign up and join the Kubernetes Slack. Please make sure to read our [Slack Guidelines](https://github.com/kubernetes/community/blob/master/communication/slack-guidelines.md) before participating.
--  The [kube-state-metrics slack channel](https://kubernetes.slack.com/messages/CJJ529RUY) provides an effective communication platform to reach out to members and other users of the project. It offers an alternative to submitting a GitHub issue for when you have questions and issues.
+* [Join Slack](http://slack.k8s.io) to sign up and join the Kubernetes Slack. Please make sure to read our [Slack Guidelines](https://github.com/kubernetes/community/blob/master/communication/slack-guidelines.md) before participating.
+* The [kube-state-metrics slack channel](https://kubernetes.slack.com/messages/CJJ529RUY) provides an effective communication platform to reach out to members and other users of the project. It offers an alternative to submitting a GitHub issue for when you have questions and issues.

Dockerfile

@@ -1,5 +1,6 @@
-ARG GOVERSION=1.16
-FROM golang:${GOVERSION} as builder
+ARG GOVERSION=1.24
+ARG GOARCH
+FROM golang:${GOVERSION} AS builder
 ARG GOARCH
 ENV GOARCH=${GOARCH}
 WORKDIR /go/src/k8s.io/kube-state-metrics/
@@ -7,7 +8,7 @@ COPY . /go/src/k8s.io/kube-state-metrics/
 
 RUN make build-local
 
-FROM gcr.io/distroless/static:latest
+FROM gcr.io/distroless/static-debian12:latest-${GOARCH}
 COPY --from=builder /go/src/k8s.io/kube-state-metrics/kube-state-metrics /
 
 USER nobody

MAINTAINER.md

@@ -0,0 +1,19 @@
+# Maintaining kube-state-metrics
+
+kube-state-metrics is welcoming contributions from the community. If you are interested in intensifying your contributions and becoming a maintainer, this doc describes the necessary steps.
+
+As part of the Kubernetes project, we rely on the [community membership process](https://github.com/kubernetes/community/blob/master/community-membership.md). We do not adhere strictly to the numbers of contributions and reviews. Still as becoming a maintainer is a trust-based process and we desire positive outcomes for the project, we look for a long-term interest and engagement.
+
+## Adding a new reviewer
+
+* Ensure the new reviewer is a member of the [kubernetes organization](https://github.com/kubernetes/org/blob/main/config/kubernetes/org.yaml).
+* Add the new reviewer to the [OWNERS](OWNERS) file to be able to review pull requests.
+* Add the new reviewer to the [kube-state-metrics-maintainers group](https://github.com/kubernetes/org/blob/main/config/kubernetes/sig-instrumentation/teams.yaml), to gain write access to the kube-state-metrics repository (e.g. for creating new releases).
+
+## Adding a new approver
+
+* Ensure the new approver is already a reviewer in the [OWNERS](OWNERS) file.
+* Add the new approver to the [OWNERS](OWNERS) file to be able to approve pull requests.
+* Add the new approver to the [SECURITY_CONTACTS](SECURITY_CONTACTS) file to be able to get notified on security related incidents.
+* Add the new approver to the [kube-state-metrics-admin group](https://github.com/kubernetes/org/blob/main/config/kubernetes/sig-instrumentation/teams.yaml), to get admin access to the kube-state-metrics repository.
+* Add the new approver to the k8s.io [OWNERS](https://github.com/kubernetes/k8s.io/blob/main/k8s.gcr.io/images/k8s-staging-kube-state-metrics/OWNERS) file to be able to approve image promotion from the staging registry.

Makefile

@@ -2,12 +2,10 @@ FLAGS =
 TESTENVVAR =
 REGISTRY ?= gcr.io/k8s-staging-kube-state-metrics
 TAG_PREFIX = v
-VERSION = $(shell cat VERSION)
+VERSION = $(shell grep '^version:' data.yaml | grep -oE "[0-9]+.[0-9]+.[0-9]+")
 TAG ?= $(TAG_PREFIX)$(VERSION)
-LATEST_RELEASE_BRANCH := release-$(shell grep -ohE "[0-9]+.[0-9]+" VERSION)
+LATEST_RELEASE_BRANCH := release-$(shell echo $(VERSION) | grep -ohE "[0-9]+.[0-9]+")
 BRANCH = $(strip $(shell git rev-parse --abbrev-ref HEAD))
-DOCKER_CLI ?= docker
-PROMTOOL_CLI ?= promtool
 PKGS = $(shell go list ./... | grep -v /vendor/ | grep -v /tests/e2e)
 ARCH ?= $(shell go env GOARCH)
 BUILD_DATE = $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
@@ -15,12 +13,21 @@ GIT_COMMIT ?= $(shell git rev-parse --short HEAD)
 OS ?= $(shell uname -s | tr A-Z a-z)
 ALL_ARCH = amd64 arm arm64 ppc64le s390x
 PKG = github.com/prometheus/common
-GO_VERSION = 1.16.2
-PROMETHEUS_VERSION = 2.25.2
+PROMETHEUS_VERSION = 3.4.1
+GO_VERSION = 1.24.4
 IMAGE = $(REGISTRY)/kube-state-metrics
 MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 USER ?= $(shell id -u -n)
 HOST ?= $(shell hostname)
+MARKDOWNLINT_CLI2_VERSION = 0.18.1
+
+DOCKER_CLI ?= docker
+PROMTOOL_CLI ?= promtool
+GOMPLATE_CLI ?= go tool github.com/hairyhenderson/gomplate/v4/cmd/gomplate 
+GOJSONTOYAML_CLI ?= go tool github.com/brancz/gojsontoyaml
+EMBEDMD_CLI ?= go tool github.com/campoy/embedmd
+JSONNET_CLI ?= go tool github.com/google/go-jsonnet/cmd/jsonnet
+JB_CLI ?= go tool github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb
 
 export DOCKER_CLI_EXPERIMENTAL=enabled
 
@@ -41,20 +48,24 @@ licensecheck:
                exit 1; \
        fi
 
-lint: shellcheck licensecheck
+lint: shellcheck licensecheck lint-markdown-format
 	golangci-lint run
 
-doccheck: generate
+lint-fix: fix-markdown-format
+	golangci-lint run --fix -v
+
+
+doccheck: generate validate-template
 	@echo "- Checking if the generated documentation is up to date..."
 	@git diff --exit-code
 	@echo "- Checking if the documentation is in sync with the code..."
-	@grep -hoE -d skip '\| kube_[^ |]+' docs/* --exclude=README.md | sed -E 's/\| //g' | sort -u > documented_metrics
+	@grep -rhoE '\| kube_[^ |]+' docs/metrics/* --exclude=README.md | sed -E 's/\| //g' | sort -u > documented_metrics
 	@find internal/store -type f -not -name '*_test.go' -exec sed -nE 's/.*"(kube_[^"]+)".*/\1/p' {} \; | sort -u > code_metrics
 	@diff -u0 code_metrics documented_metrics || (echo "ERROR: Metrics with - are present in code but missing in documentation, metrics with + are documented but not found in code."; exit 1)
 	@echo OK
 	@rm -f code_metrics documented_metrics
 	@echo "- Checking for orphan documentation files"
-	@cd docs; for doc in *.md; do if [ "$$doc" != "README.md" ] && ! grep -q "$$doc" *.md; then echo "ERROR: No link to documentation file $${doc} detected"; exit 1; fi; done
+	@cd docs; for doc in $$(find metrics/* -name '*.md' | sed 's/.*\///'); do if [ "$$doc" != "README.md" ] && ! grep -q "$$doc" *.md; then echo "ERROR: No link to documentation file $${doc} detected"; exit 1; fi; done
 	@echo OK
 
 build-local:
@@ -63,7 +74,8 @@ build-local:
 build: kube-state-metrics
 
 kube-state-metrics:
-	${DOCKER_CLI} run --rm -v "${PWD}:/go/src/k8s.io/kube-state-metrics" -w /go/src/k8s.io/kube-state-metrics golang:${GO_VERSION} make build-local
+	# Need to update git setting to prevent failing builds due to https://github.com/docker-library/golang/issues/452
+	${DOCKER_CLI} run --rm -v "${PWD}:/go/src/k8s.io/kube-state-metrics" -w /go/src/k8s.io/kube-state-metrics -e GOOS=$(OS) -e GOARCH=$(ARCH) golang:${GO_VERSION} git config --global --add safe.directory "*" && make build-local
 
 test-unit:
 	GOOS=$(shell uname -s | tr A-Z a-z) GOARCH=$(ARCH) $(TESTENVVAR) go test --race $(FLAGS) $(PKGS)
@@ -74,12 +86,24 @@ test-rules:
 shellcheck:
 	${DOCKER_CLI} run -v "${PWD}:/mnt" koalaman/shellcheck:stable $(shell find . -type f -name "*.sh" -not -path "*vendor*")
 
+lint-markdown-format:
+	${DOCKER_CLI} run -v "${PWD}:/workdir" davidanson/markdownlint-cli2:v${MARKDOWNLINT_CLI2_VERSION} --config .markdownlint-cli2.jsonc
+
+fix-markdown-format:
+	${DOCKER_CLI} run -v "${PWD}:/workdir" davidanson/markdownlint-cli2:v${MARKDOWNLINT_CLI2_VERSION} --fix --config .markdownlint-cli2.jsonc
+
+generate-template:
+	${GOMPLATE_CLI} -d config=./data.yaml --file README.md.tpl > README.md
+
+validate-template: generate-template
+	git diff --no-ext-diff --quiet --exit-code README.md
+
 # Runs benchmark tests on the current git ref and the last release and compares
 # the two.
 test-benchmark-compare:
 	@git fetch
-	./tests/compare_benchmarks.sh master
-	./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH}
+	./tests/compare_benchmarks.sh main 2
+	./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH} 2
 
 all: all-container
 
@@ -109,15 +133,6 @@ push-multi-arch:
 	@for arch in $(ALL_ARCH); do ${DOCKER_CLI} manifest annotate --arch $${arch} $(IMAGE):$(TAG) $(IMAGE)-$${arch}:$(TAG); done
 	${DOCKER_CLI} manifest push --purge $(IMAGE):$(TAG)
 
-quay-push: .quay-push-$(ARCH)
-.quay-push-$(ARCH): container-$(ARCH)
-	${DOCKER_CLI} push $(MULTI_ARCH_IMG):$(TAG)
-	${DOCKER_CLI} push $(MULTI_ARCH_IMG):latest
-ifeq ($(ARCH), amd64)
-	${DOCKER_CLI} push $(IMAGE):$(TAG)
-	${DOCKER_CLI} push $(IMAGE):latest
-endif
-
 clean:
 	rm -f kube-state-metrics
 	git clean -Xfd .
@@ -125,10 +140,10 @@ clean:
 e2e:
 	./tests/e2e.sh
 
-generate: build-local
+generate: build-local generate-template
 	@echo ">> generating docs"
 	@./scripts/generate-help-text.sh
-	embedmd -w `find . -path ./vendor -prune -o -name "*.md" -print`
+	${EMBEDMD_CLI} -w `find . -path ./vendor -prune -o -name "*.md" -print`
 
 validate-manifests: examples
 	@git diff --exit-code
@@ -137,29 +152,31 @@ mixin: examples/prometheus-alerting-rules/alerts.yaml
 
 examples/prometheus-alerting-rules/alerts.yaml: jsonnet $(shell find jsonnet | grep ".libsonnet") scripts/mixin.jsonnet scripts/vendor
 	mkdir -p examples/prometheus-alerting-rules
-	jsonnet -J scripts/vendor scripts/mixin.jsonnet | gojsontoyaml > examples/prometheus-alerting-rules/alerts.yaml
+	${JSONNET_CLI}  -J scripts/vendor scripts/mixin.jsonnet | ${GOJSONTOYAML_CLI} > examples/prometheus-alerting-rules/alerts.yaml
 
-examples: examples/standard examples/autosharding mixin
+examples: examples/standard examples/autosharding examples/daemonsetsharding mixin
 
-examples/standard: jsonnet $(shell find jsonnet | grep ".libsonnet") scripts/standard.jsonnet scripts/vendor VERSION
+examples/standard: jsonnet $(shell find jsonnet | grep ".libsonnet") scripts/standard.jsonnet scripts/vendor
 	mkdir -p examples/standard
-	jsonnet -J scripts/vendor -m examples/standard --ext-str version="$(VERSION)" scripts/standard.jsonnet | xargs -I{} sh -c 'cat {} | gojsontoyaml > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
+	${JSONNET_CLI} -J scripts/vendor -m examples/standard --ext-str version="$(VERSION)" scripts/standard.jsonnet | xargs -I{} sh -c 'cat {} | ${GOJSONTOYAML_CLI} > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
 	find examples -type f ! -name '*.yaml' -delete
 
-examples/autosharding: jsonnet $(shell find jsonnet | grep ".libsonnet") scripts/autosharding.jsonnet scripts/vendor VERSION
+examples/autosharding: jsonnet $(shell find jsonnet | grep ".libsonnet") scripts/autosharding.jsonnet scripts/vendor
 	mkdir -p examples/autosharding
-	jsonnet -J scripts/vendor -m examples/autosharding --ext-str version="$(VERSION)" scripts/autosharding.jsonnet | xargs -I{} sh -c 'cat {} | gojsontoyaml > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
+	${JSONNET_CLI} -J scripts/vendor -m examples/autosharding --ext-str version="$(VERSION)" scripts/autosharding.jsonnet | xargs -I{} sh -c 'cat {} | ${GOJSONTOYAML_CLI} > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
+	find examples -type f ! -name '*.yaml' -delete
+
+examples/daemonsetsharding: jsonnet $(shell find jsonnet | grep ".libsonnet") scripts/daemonsetsharding.jsonnet scripts/vendor
+	mkdir -p examples/daemonsetsharding
+	${JSONNET_CLI} -J scripts/vendor -m examples/daemonsetsharding --ext-str version="$(VERSION)" scripts/daemonsetsharding.jsonnet | xargs -I{} sh -c 'cat {} | ${GOJSONTOYAML_CLI} > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
 	find examples -type f ! -name '*.yaml' -delete
 
 scripts/vendor: scripts/jsonnetfile.json scripts/jsonnetfile.lock.json
-	cd scripts && jb install
-
-install-tools:
-	@echo Installing tools from tools.go
-	@cat tools/tools.go | grep _ | awk -F'"' '{print $$2}' | xargs -tI % go install %
+	cd scripts && ${JB_CLI} install
 
 install-promtool:
 	@echo Installing promtool
-	@wget -qO- "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.${OS}-${ARCH}.tar.gz" | tar xvz --strip-components=1
+	@wget -qO- "https://github.com/prometheus/prometheus/releases/download/v${PROMETHEUS_VERSION}/prometheus-${PROMETHEUS_VERSION}.${OS}-${ARCH}.tar.gz" |\
+	tar xvz --strip-components=1 prometheus-${PROMETHEUS_VERSION}.${OS}-${ARCH}/promtool
 
-.PHONY: all build build-local all-push all-container container container-* do-push-* sub-push-* push push-multi-arch quay-push test-unit test-rules test-benchmark-compare clean e2e validate-modules shellcheck licensecheck lint generate embedmd
+.PHONY: all build build-local all-push all-container container container-* do-push-* sub-push-* push push-multi-arch test-unit test-rules test-benchmark-compare clean e2e validate-modules shellcheck licensecheck lint lint-fix generate generate-template validate-template embedmd

OWNERS

@@ -1,12 +1,18 @@
 reviewers:
-  - brancz
-  - tariq1890
-  - LiliC
+  - CatherineF-dev
+  - dgrisonnet
+  - logicalhan
   - mrueg
+  - rexagod
 approvers:
+  - CatherineF-dev
+  - dgrisonnet
+  - mrueg
+  - rexagod
+emeritus_approvers:
+  - LiliC
+  - andyxning
   - brancz
   - tariq1890
-  - LiliC
-emeritus-approvers:
-  - andyxning
   - zouyee
+  - fpetkovski

README.md

@@ -1,9 +1,13 @@
 # Overview
 
 [![Build Status](https://github.com/kubernetes/kube-state-metrics/workflows/continuous-integration/badge.svg)](https://github.com/kubernetes/kube-state-metrics/actions)
-[![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes/kube-state-metrics)](https://goreportcard.com/report/github.com/kubernetes/kube-state-metrics) [![GoDoc](https://godoc.org/github.com/kubernetes/kube-state-metrics?status.svg)](https://godoc.org/github.com/kubernetes/kube-state-metrics)
+[![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes/kube-state-metrics)](https://goreportcard.com/report/github.com/kubernetes/kube-state-metrics)
+[![Go Reference](https://pkg.go.dev/badge/github.com/kubernetes/kube-state-metrics.svg)](https://pkg.go.dev/github.com/kubernetes/kube-state-metrics)
+[![govulncheck](https://github.com/kubernetes/kube-state-metrics/actions/workflows/govulncheck.yml/badge.svg)](https://github.com/kubernetes/kube-state-metrics/actions/workflows/govulncheck.yml)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8696/badge)](https://www.bestpractices.dev/projects/8696)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/kubernetes/kube-state-metrics/badge)](https://api.securityscorecards.dev/projects/github.com/kubernetes/kube-state-metrics)
 
-kube-state-metrics is a simple service that listens to the Kubernetes API
+kube-state-metrics (KSM) is a simple service that listens to the Kubernetes API
 server and generates metrics about the state of the objects. (See examples in
 the Metrics section below.) It is not focused on the health of the individual
 Kubernetes components, but rather on the health of the various objects inside,
@@ -26,56 +30,61 @@ the raw metrics. Note that the metrics exposed on the `/metrics` endpoint
 reflect the current state of the Kubernetes cluster. When Kubernetes objects
 are deleted they are no longer visible on the `/metrics` endpoint.
 
-Note that any new features will be merged into master but released with v2.1.0 release, as currently v2.0.0 is in post feature freeze and only accepting bug fixes.
+> [!NOTE]
+> This README is generated from a [template](./README.md.tpl). Please make your changes there and run `make generate-template`.
 
 ## Table of Contents
 
-- [Versioning](#versioning)
-  - [Kubernetes Version](#kubernetes-version)
-  - [Compatibility matrix](#compatibility-matrix)
-  - [Resource group version compatibility](#resource-group-version-compatibility)
-  - [Container Image](#container-image)
-- [Metrics Documentation](#metrics-documentation)
-  - [Conflict resolution in label names](#conflict-resolution-in-label-names)
-  - [Enabling VerticalPodAutoscalers](#enabling-verticalpodautoscalers)
-- [Kube-state-metrics self metrics](#kube-state-metrics-self-metrics)
-- [Resource recommendation](#resource-recommendation)
-- [A note on costing](#a-note-on-costing)
-- [kube-state-metrics vs. metrics-server](#kube-state-metrics-vs-metrics-server)
-- [Scaling kube-state-metrics](#scaling-kube-state-metrics)
-  - [Resource recommendation](#resource-recommendation)
-  - [Horizontal scaling (sharding)](#horizontal-scaling-sharding)
-    - [Automated sharding](#automated-sharding)
-- [Setup](#setup)
-  - [Building the Docker container](#building-the-docker-container)
-- [Usage](#usage)
-  - [Kubernetes Deployment](#kubernetes-deployment)
-  - [Limited privileges environment](#limited-privileges-environment)
-  - [Development](#development)
-  - [Developer Contributions](#developer-contributions)
+* [Versioning](#versioning)
+  * [Kubernetes Version](#kubernetes-version)
+  * [Compatibility matrix](#compatibility-matrix)
+  * [Resource group version compatibility](#resource-group-version-compatibility)
+  * [Container Image](#container-image)
+* [Metrics Documentation](#metrics-documentation)
+  * [ECMAScript regular expression support for allow and deny lists](#ecmascript-regular-expression-support-for-allow-and-deny-lists)
+  * [Conflict resolution in label names](#conflict-resolution-in-label-names)
+* [Kube-state-metrics self metrics](#kube-state-metrics-self-metrics)
+* [Resource recommendation](#resource-recommendation)
+* [Latency](#latency)
+* [A note on costing](#a-note-on-costing)
+* [kube-state-metrics vs. metrics-server](#kube-state-metrics-vs-metrics-server)
+* [Scaling kube-state-metrics](#scaling-kube-state-metrics)
+  * [Resource recommendation](#resource-recommendation)
+  * [Horizontal sharding](#horizontal-sharding)
+    * [Automated sharding](#automated-sharding)
+  * [Daemonset sharding for pod metrics](#daemonset-sharding-for-pod-metrics)
+* [Setup](#setup)
+  * [Building the Docker container](#building-the-docker-container)
+* [Usage](#usage)
+  * [Kubernetes Deployment](#kubernetes-deployment)
+  * [Limited privileges environment](#limited-privileges-environment)
+  * [Helm Chart](#helm-chart)
+  * [Development](#development)
+  * [Developer Contributions](#developer-contributions)
+  * [Community](#community)
+
+### Versioning
 
 #### Kubernetes Version
 
 kube-state-metrics uses [`client-go`](https://github.com/kubernetes/client-go) to talk with
-Kubernetes clusters. The supported Kubernetes cluster version is determined by `client-go`.
-The compatibility matrix for client-go and Kubernetes cluster can be found
-[here](https://github.com/kubernetes/client-go#compatibility-matrix).
+Kubernetes clusters. The supported Kubernetes cluster version is determined by
+[`client-go`](https://github.com/kubernetes/client-go#compatibility-matrix).
 All additional compatibility is only best effort, or happens to still/already be supported.
 
 #### Compatibility matrix
 
 At most, 5 kube-state-metrics and 5 [kubernetes releases](https://github.com/kubernetes/kubernetes/releases) will be recorded below.
+Generally, it is recommended to use the latest release of kube-state-metrics. If you run a very recent version of Kubernetes, you might want to use an unreleased version to have the full range of supported resources. If you run an older version of Kubernetes, you might need to run an older version in order to have full support for all resources. Be aware, that the maintainers will only support the latest release. Older versions might be supported by interested users of the community.
 
-| kube-state-metrics | **Kubernetes 1.16** |  **Kubernetes 1.17** |  **Kubernetes 1.18** |  **Kubernetes 1.19** |  **Kubernetes 1.20** |
-|--------------------|---------------------|---------------------|----------------------|----------------------|-----------------------|
-| **v1.8.0**         |         -           |          -           |          -           |          -           |          -           |
-| **v1.9.8**         |         ✓           |          -           |          -           |          -           |          -           |
-| **v2.0.0-rc.1**    |         -           |          -/✓         |         -/✓          |          ✓           |          ✓           |
-| **master**         |         -           |          -/✓         |         -/✓          |          ✓           |          ✓           |
-- `✓` Fully supported version range.
-- `-` The Kubernetes cluster has features the client-go library can't use (additional API objects, deprecated APIs, etc).
-
-**Note:** The `v2.0.0-alpha.2+` and `master` releases of kube-state-metrics work on Kubernetes v1.17 and v1.18 excluding Ingress or CertificateSigningRequest resource metrics. If you require those metrics and are on an older Kubernetes version, use v2.0.0-alpha.1 or v1.9.8 kube-state-metrics release.
+| kube-state-metrics | Kubernetes client-go Version |
+|--------------------|:----------------------------:|
+| **v2.12.0**        | v1.29                        |
+| **v2.13.0**        | v1.30                        |
+| **v2.14.0**        | v1.31                        |
+| **v2.15.0**        | v1.32                        |
+| **v2.16.0**        | v1.32                        |
+| **main**           | v1.33                        |
 
 #### Resource group version compatibility
 
@@ -86,7 +95,9 @@ release.
 #### Container Image
 
 The latest container image can be found at:
-* `k8s.gcr.io/kube-state-metrics/kube-state-metrics:v2.0.0-rc.1` (arch: `amd64`, `arm`, `arm64`, `ppc64le` and `s390x`)
+
+* `registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.16.0` (arch: `amd64`, `arm`, `arm64`, `ppc64le` and `s390x`)
+* [Multi-architecture images](https://explore.ggcr.dev/?image=registry.k8s.io%2Fkube-state-metrics%2Fkube-state-metrics:v2.16.0)
 
 ### Metrics Documentation
 
@@ -118,12 +129,9 @@ e.g. by standardizing Kubernetes labels using an
 [Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/)
 that ensures that there are no possible conflicts.
 
-#### Enabling VerticalPodAutoscalers
+#### ECMAScript regular expression support for allow and deny lists
 
-Please note that the collector for `verticalpodautoscalers` is **disabled** by default; Vertical Pod Autoscaler metrics will not be collected until the collector is enabled. This is because Vertical Pod Autoscalers are managed as custom resources.
-
-If you want to enable this collector,
-the [instructions](./docs/verticalpodautoscaler-metrics.md#Configuration) are located in the [Vertical Pod Autoscaler Metrics](./docs/verticalpodautoscaler-metrics.md) documentation.
+Starting from [#2616](https://github.com/kubernetes/kube-state-metrics/pull/2616/files), kube-state-metrics supports ECMAScript's `regexp` for allow and deny lists. This was incorporated as a workaround for the limitations of the `regexp` package in Go, which does not support lookarounds due to their non-linear time complexity. Please note that while lookarounds are now supported for allow and deny lists, regular expressions' evaluation time is capped at a minute to prevent performance issues.
 
 ### Kube-state-metrics self metrics
 
@@ -134,14 +142,16 @@ If you encounter those errors in the metrics, it is most likely a configuration
 at the logs of kube-state-metrics.
 
 Example of the above mentioned metrics:
-```
+
+```prometheus
 kube_state_metrics_list_total{resource="*v1.Node",result="success"} 1
 kube_state_metrics_list_total{resource="*v1.Node",result="error"} 52
 kube_state_metrics_watch_total{resource="*v1beta1.Ingress",result="success"} 1
 ```
 
 kube-state-metrics also exposes some http request metrics, examples of those are:
-```
+
+```prometheus
 http_request_duration_seconds_bucket{handler="metrics",method="get",le="2.5"} 30
 http_request_duration_seconds_bucket{handler="metrics",method="get",le="5"} 30
 http_request_duration_seconds_bucket{handler="metrics",method="get",le="10"} 30
@@ -151,43 +161,56 @@ http_request_duration_seconds_count{handler="metrics",method="get"} 30
 ```
 
 kube-state-metrics also exposes build and configuration metrics:
-```
-kube_state_metrics_build_info{branch="master",goversion="go1.15.3",revision="6c9d775d",version="v2.0.0-beta"} 1
+
+```prometheus
+kube_state_metrics_build_info{branch="main",goversion="go1.15.3",revision="6c9d775d",version="v2.0.0-beta"} 1
 kube_state_metrics_shard_ordinal{shard_ordinal="0"} 0
 kube_state_metrics_total_shards 1
 ```
 
 `kube_state_metrics_build_info` is used to expose version and other build information. For more usage about the info pattern,
-please check the blog post [here](https://www.robustperception.io/exposing-the-software-version-to-prometheus).
+please check this [blog post](https://www.robustperception.io/exposing-the-software-version-to-prometheus).
 Sharding metrics expose `--shard` and `--total-shards` flags and can be used to validate
 run-time configuration, see [`/examples/prometheus-alerting-rules`](./examples/prometheus-alerting-rules).
 
+kube-state-metrics also exposes metrics about it config file and the Custom Resource State config file:
+
+```prometheus
+kube_state_metrics_config_hash{filename="crs.yml",type="customresourceconfig"} 2.38272279311849e+14
+kube_state_metrics_config_hash{filename="config.yml",type="config"} 2.65285922340846e+14
+kube_state_metrics_last_config_reload_success_timestamp_seconds{filename="crs.yml",type="customresourceconfig"} 1.6704882592037103e+09
+kube_state_metrics_last_config_reload_success_timestamp_seconds{filename="config.yml",type="config"} 1.6704882592035313e+09
+kube_state_metrics_last_config_reload_successful{filename="crs.yml",type="customresourceconfig"} 1
+kube_state_metrics_last_config_reload_successful{filename="config.yml",type="config"} 1
+```
+
 ### Scaling kube-state-metrics
 
 #### Resource recommendation
 
-> Note: These recommendations are based on scalability tests done over a year ago. They may differ significantly today.
-
 Resource usage for kube-state-metrics changes with the Kubernetes objects (Pods/Nodes/Deployments/Secrets etc.) size of the cluster.
 To some extent, the Kubernetes objects in a cluster are in direct proportion to the node number of the cluster.
 
-As a general rule, you should allocate
+As a general rule, you should allocate:
 
-* 200MiB memory
+* 250MiB memory
 * 0.1 cores
 
-For clusters of more than 100 nodes, allocate at least
+Note that if CPU limits are set too low, kube-state-metrics' internal queues will not be able to be worked off quickly enough, resulting in increased memory consumption as the queue length grows. If you experience problems resulting from high memory allocation or CPU throttling, try increasing the CPU limits.
 
-* 2MiB memory per node
-* 0.001 cores per node
+### Latency
 
-These numbers are based on [scalability tests](https://github.com/kubernetes/kube-state-metrics/issues/124#issuecomment-318394185) at 30 pods per node.
+In a 100 node cluster scaling test the latency numbers were as follows:
 
-Note that if CPU limits are set too low, kube-state-metrics' internal queues will not be able to be worked off quickly enough, resulting in increased memory consumption as the queue length grows. If you experience problems resulting from high memory allocation, try increasing the CPU limits.
+```text
+"Perc50": 259615384 ns,
+"Perc90": 475000000 ns,
+"Perc99": 906666666 ns.
+```
 
 ### A note on costing
 
-By default, kube-state-metrics exposes several metrics for events across your cluster. If you have a large number of frequently-updating resources on your cluster, you may find that a lot of data is ingested into these metrics. This can incur high costs on some cloud providers. Please take a moment to [configure what metrics you'd like to expose](docs/cli-arguments.md), as well as consult the documentation for your Kubernetes environment in order to avoid unexpectedly high costs.
+By default, kube-state-metrics exposes several metrics for events across your cluster. If you have a large number of frequently-updating resources on your cluster, you may find that a lot of data is ingested into these metrics. This can incur high costs on some cloud providers. Please take a moment to [configure what metrics you'd like to expose](docs/developer/cli-arguments.md), as well as consult the documentation for your Kubernetes environment in order to avoid unexpectedly high costs.
 
 ### kube-state-metrics vs. metrics-server
 
@@ -195,9 +218,9 @@ The [metrics-server](https://github.com/kubernetes-incubator/metrics-server)
 is a project that has been inspired by
 [Heapster](https://github.com/kubernetes-retired/heapster) and is implemented
 to serve the goals of core metrics pipelines in [Kubernetes monitoring
-architecture](https://github.com/kubernetes/community/blob/master/contributors/design-proposals/instrumentation/monitoring_architecture.md).
+architecture](https://github.com/kubernetes/design-proposals-archive/blob/main/instrumentation/monitoring_architecture.md).
 It is a cluster level component which periodically scrapes metrics from all
-Kubernetes nodes served by Kubelet through Summary API. The metrics are
+Kubernetes nodes served by Kubelet through Metrics API. The metrics are
 aggregated, stored in memory and served in [Metrics API
 format](https://git.k8s.io/metrics/pkg/apis/metrics/v1alpha1/types.go). The
 metrics-server stores the latest values only and is not responsible for
@@ -212,38 +235,89 @@ metrics-server it too is not responsible for exporting its metrics anywhere.
 Having kube-state-metrics as a separate project also enables access to these
 metrics from monitoring systems such as Prometheus.
 
-### Horizontal scaling (sharding)
+### Horizontal sharding
 
-In order to scale kube-state-metrics horizontally, some automated sharding capabilities have been implemented. It is configured with the following flags:
+In order to shard kube-state-metrics horizontally, some automated sharding capabilities have been implemented. It is configured with the following flags:
 
 * `--shard` (zero indexed)
 * `--total-shards`
 
-Sharding is done by taking an md5 sum of the Kubernetes Object's UID and performing a modulo operation on it, with the total number of shards. The configured shard decides whether the object is handled by the respective instance of kube-state-metrics or not. Note that this means all instances of kube-state-metrics even if sharded will have the network traffic and the resource consumption for unmarshaling objects for all objects, not just the ones it is responsible for. To optimize this further, the Kubernetes API would need to support sharded list/watch capabilities. Overall memory consumption should be 1/n th of each shard compared to an unsharded setup. Typically, kube-state-metrics needs to be memory and latency optimized in order for it to return its metrics rather quickly to Prometheus.
+Sharding is done by taking an md5 sum of the Kubernetes Object's UID and performing a modulo operation on it with the total number of shards. Each shard decides whether the object is handled by the respective instance of kube-state-metrics or not. Note that this means all instances of kube-state-metrics, even if sharded, will have the network traffic and the resource consumption for unmarshaling objects for all objects, not just the ones they are responsible for. To optimize this further, the Kubernetes API would need to support sharded list/watch capabilities. In the optimal case, memory consumption for each shard will be 1/n compared to an unsharded setup. Typically, kube-state-metrics needs to be memory and latency optimized in order for it to return its metrics rather quickly to Prometheus. One way to reduce the latency between kube-state-metrics and the kube-apiserver is to run KSM with the `--use-apiserver-cache` flag. In addition to reducing the latency, this option will also lead to a reduction in the load on etcd.
 
-Sharding should be used carefully, and additional monitoring should be set up in order to ensure that sharding is set up and functioning as expected (eg. instances for each shard out of the total shards are configured).
+Sharding should be used carefully and additional monitoring should be set up in order to ensure that sharding is set up and functioning as expected (eg. instances for each shard out of the total shards are configured).
 
-##### Automated sharding
+#### Automated sharding
 
-There is also an experimental feature, that allows kube-state-metrics to auto discover its nominal position if it is deployed in a StatefulSet, in order to automatically configure sharding. This is an experimental feature and may be broken or removed without notice.
+Automatic sharding allows each shard to discover its nominal position when deployed in a StatefulSet which is useful for automatically configuring sharding. This is an experimental feature and may be broken or removed without notice.
 
-To enable automated sharding kube-state-metrics must be run by a `StatefulSet` and the pod names and namespace must be handed to the kube-state-metrics process via the `--pod` and `--pod-namespace` flags.
+To enable automated sharding, kube-state-metrics must be run by a `StatefulSet` and the pod name and namespace must be handed to the kube-state-metrics process via the `--pod` and `--pod-namespace` flags. Example manifests demonstrating the autosharding functionality can be found in [`/examples/autosharding`](./examples/autosharding).
 
-There are example manifests demonstrating the autosharding functionality in [`/examples/autosharding`](./examples/autosharding).
+This way of deploying shards is useful when you want to manage KSM shards through a single Kubernetes resource (a single `StatefulSet` in this case) instead of having one `Deployment` per shard. The advantage can be especially significant when deploying a high number of shards.
+
+The downside of using an auto-sharded setup comes from the rollout strategy supported by `StatefulSet`s. When managed by a `StatefulSet`, pods are replaced one at a time with each pod first getting terminated and then recreated. Besides such rollouts being slower, they will also lead to short downtime for each shard. If a Prometheus scrape happens during a rollout, it can miss some of the metrics exported by kube-state-metrics.
+
+### Daemonset sharding for pod metrics
+
+For pod metrics, they can be sharded per node with the following flag:
+
+* `--node=$(NODE_NAME)`
+
+Each kube-state-metrics pod uses FieldSelector (spec.nodeName) to watch/list pod metrics only on the same node.
+
+A daemonset kube-state-metrics example:
+
+```yaml
+apiVersion: apps/v1
+kind: DaemonSet
+spec:
+  template:
+    spec:
+      containers:
+      - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:IMAGE_TAG
+        name: kube-state-metrics
+        args:
+        - --resource=pods
+        - --node=$(NODE_NAME)
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+```
+
+To track metrics for unassigned pods, you need to add an additional deployment and set `--track-unscheduled-pods`, as shown in the following example:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+spec:
+  template:
+    spec:
+      containers:
+      - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:IMAGE_TAG
+        name: kube-state-metrics
+        args:
+        - --resources=pods
+        - --track-unscheduled-pods
+```
+
+Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding).
 
 ### Setup
 
 Install this project to your `$GOPATH` using `go get`:
 
-```
-go get k8s.io/kube-state-metrics
+```bash
+go get k8s.io/kube-state-metrics/v2
 ```
 
 #### Building the Docker container
 
 Simply run the following command in this root folder, which will create a
 self-contained, statically-linked binary and build a Docker image:
-```
+
+```bash
 make container
 ```
 
@@ -252,34 +326,44 @@ make container
 Simply build and run kube-state-metrics inside a Kubernetes pod which has a
 service account token that has read-only access to the Kubernetes cluster.
 
-### For users of prometheus-operator/kube-prometheus stack
+#### For users of prometheus-operator/kube-prometheus stack
 
 The ([`kube-prometheus`](https://github.com/prometheus-operator/kube-prometheus/)) stack installs kube-state-metrics as one of its [components](https://github.com/prometheus-operator/kube-prometheus#kube-prometheus); you do not need to install kube-state-metrics if you're using the kube-prometheus stack.
 
-If you want to revise the default configuration for kube-prometheus, for example to enable non-default metrics, have a look at [Customizing Kube-Prometheus](https://github.com/prometheus-operator/kube-prometheus#customizing-kube-prometheus).
+If you want to revise the default configuration for kube-prometheus, for example to enable non-default metrics, have a look at [Customizing Kube-Prometheus](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizing.md).
 
 #### Kubernetes Deployment
 
-To deploy this project, you can simply run `kubectl apply -f examples/standard` and a
-Kubernetes service and deployment will be created. (Note: Adjust the apiVersion of some resource if your kubernetes cluster's version is not 1.8+, check the yaml file for more information).
+To deploy this project, you can simply run `kubectl apply -f examples/standard` and a Kubernetes service and deployment will be created. (Note: Adjust the apiVersion of some resource if your kubernetes cluster's version is not 1.8+, check the yaml file for more information).
 
 To have Prometheus discover kube-state-metrics instances it is advised to create a specific Prometheus scrape config for kube-state-metrics that picks up both metrics endpoints. Annotation based discovery is discouraged as only one of the endpoints would be able to be selected, plus kube-state-metrics in most cases has special authentication and authorization requirements as it essentially grants read access through the metrics endpoint to most information available to it.
 
 **Note:** Google Kubernetes Engine (GKE) Users - GKE has strict role permissions that will prevent the kube-state-metrics roles and role bindings from being created. To work around this, you can give your GCP identity the cluster-admin role by running the following one-liner:
 
-```
+```bash
 kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=$(gcloud info --format='value(config.account)')
 ```
 
-Note that your GCP identity is case sensitive but `gcloud info` as of Google Cloud SDK 221.0.0 is not. This means that if your IAM member contains capital letters, the above one-liner may not work for you. If you have 403 forbidden responses after running the above command and `kubectl apply -f examples/standard`, check the IAM member associated with your account at https://console.cloud.google.com/iam-admin/iam?project=PROJECT_ID. If it contains capital letters, you may need to set the --user flag in the command above to the case-sensitive role listed at https://console.cloud.google.com/iam-admin/iam?project=PROJECT_ID.
+Note that your GCP identity is case sensitive but `gcloud info` as of Google Cloud SDK 221.0.0 is not. This means that if your IAM member contains capital letters, the above one-liner may not work for you. If you have 403 forbidden responses after running the above command and `kubectl apply -f examples/standard`, check the IAM member associated with your account at <https://console.cloud.google.com/iam-admin/iam?project=PROJECT_ID>. If it contains capital letters, you may need to set the --user flag in the command above to the case-sensitive role listed at <https://console.cloud.google.com/iam-admin/iam?project=PROJECT_ID>.
 
 After running the above, if you see `Clusterrolebinding "cluster-admin-binding" created`, then you are able to continue with the setup of this service.
 
+#### Healthcheck Endpoints
+
+The following healthcheck endpoints are available (`self` refers to the telemetry port, while `main` refers to the exposition port):
+
+* `/healthz` (exposed on `main`): Returns a 200 status code if the application is running. We recommend to use this for the startup probe.
+* `/livez` (exposed on `main`): Returns a 200 status code if the application is not affected by an outage of the Kubernetes API Server. We recommend to using this for the liveness probe.
+* `/readyz` (exposed on `self`): Returns a 200 status code if the application is ready to accept requests and expose metrics. We recommend using this for the readiness probe.
+
+Note that it is discouraged to use the telemetry metrics endpoint for any probe when proxying the exposition data.
+
 #### Limited privileges environment
 
 If you want to run kube-state-metrics in an environment where you don't have cluster-reader role, you can:
 
-- create a serviceaccount
+* create a serviceaccount
+
 ```yaml
 apiVersion: v1
 kind: ServiceAccount
@@ -288,7 +372,8 @@ metadata:
   namespace: your-namespace-where-kube-state-metrics-will-deployed
 ```
 
-- give it `view` privileges on specific namespaces (using roleBinding) (*note: you can add this roleBinding to all the NS you want your serviceaccount to access*)
+* give it `view` privileges on specific namespaces (using roleBinding) (*note: you can add this roleBinding to all the NS you want your serviceaccount to access*)
+
 ```yaml
 apiVersion: rbac.authorization.k8s.io/v1
 kind: RoleBinding
@@ -305,7 +390,7 @@ subjects:
     namespace: your-namespace-where-kube-state-metrics-will-deployed
 ```
 
-- then specify a set of namespaces (using the `--namespaces` option) and a set of kubernetes objects (using the `--resources`) that your serviceaccount has access to in the `kube-state-metrics` deployment configuration
+* then specify a set of namespaces (using the `--namespaces` option) and a set of kubernetes objects (using the `--resources`) that your serviceaccount has access to in the `kube-state-metrics` deployment configuration
 
 ```yaml
 spec:
@@ -318,24 +403,43 @@ spec:
           - '--namespaces=project1'
 ```
 
-For the full list of arguments available, see the documentation in [docs/cli-arguments.md](./docs/cli-arguments.md)
+For the full list of arguments available, see the documentation in [docs/developer/cli-arguments.md](./docs/developer/cli-arguments.md)
+
+#### Helm Chart
+
+Starting from the kube-state-metrics chart `v2.13.3` (kube-state-metrics image `v1.9.8`), the official [Helm chart](https://artifacthub.io/packages/helm/prometheus-community/kube-state-metrics/) is maintained in [prometheus-community/helm-charts](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics). Starting from kube-state-metrics chart `v3.0.0` only kube-state-metrics images of `v2.0.0 +` are supported.
 
 #### Development
 
-When developing, test a metric dump against your local Kubernetes cluster by
-running:
+When developing, test a metric dump against your local Kubernetes cluster by running:
 
 > Users can override the apiserver address in KUBE-CONFIG file with `--apiserver` command line.
 
-	go install
-	kube-state-metrics --port=8080 --telemetry-port=8081 --kubeconfig=<KUBE-CONFIG> --apiserver=<APISERVER>
+```bash
+go install
+kube-state-metrics --port=8080 --telemetry-port=8081 --kubeconfig=<KUBE-CONFIG> --apiserver=<APISERVER>
+```
 
 Then curl the metrics endpoint
 
-	curl localhost:8080/metrics
+```bash
+curl localhost:8080/metrics
+```
 
 To run the e2e tests locally see the documentation in [tests/README.md](./tests/README.md).
 
 #### Developer Contributions
 
 When developing, there are certain code patterns to follow to better your contributing experience and likelihood of e2e and other ci tests to pass. To learn more about them, see the documentation in [docs/developer/guide.md](./docs/developer/guide.md).
+
+#### Community
+
+This project is sponsored by [SIG Instrumentation](https://github.com/kubernetes/community/tree/master/sig-instrumentation).
+
+There is also a channel for [#kube-state-metrics](https://kubernetes.slack.com/archives/CJJ529RUY) on Kubernetes' Slack.
+
+You can also join the SIG Instrumentation [mailing list](https://groups.google.com/forum/#!forum/kubernetes-sig-instrumentation).
+This will typically add invites for the following meetings to your calendar, in which topics around kube-state-metrics can be discussed.
+
+* Regular SIG Meeting: [Thursdays at 9:30 PT (Pacific Time)](https://zoom.us/j/5342565819?pwd=RlVsK21NVnR1dmE3SWZQSXhveHZPdz09) (biweekly). [Convert to your timezone](http://www.thetimezoneconverter.com/?t=9:30&tz=PT%20%28Pacific%20Time%29).
+* Regular Triage Meeting: [Thursdays at 9:30 PT (Pacific Time)](https://zoom.us/j/5342565819?pwd=RlVsK21NVnR1dmE3SWZQSXhveHZPdz09) (biweekly - alternating with regular meeting). [Convert to your timezone](http://www.thetimezoneconverter.com/?t=9:30&tz=PT%20%28Pacific%20Time%29).

README.md.tpl

@@ -0,0 +1,446 @@
+# Overview
+
+[![Build Status](https://github.com/kubernetes/kube-state-metrics/workflows/continuous-integration/badge.svg)](https://github.com/kubernetes/kube-state-metrics/actions)
+[![Go Report Card](https://goreportcard.com/badge/github.com/kubernetes/kube-state-metrics)](https://goreportcard.com/report/github.com/kubernetes/kube-state-metrics)
+[![Go Reference](https://pkg.go.dev/badge/github.com/kubernetes/kube-state-metrics.svg)](https://pkg.go.dev/github.com/kubernetes/kube-state-metrics)
+[![govulncheck](https://github.com/kubernetes/kube-state-metrics/actions/workflows/govulncheck.yml/badge.svg)](https://github.com/kubernetes/kube-state-metrics/actions/workflows/govulncheck.yml)
+[![OpenSSF Best Practices](https://www.bestpractices.dev/projects/8696/badge)](https://www.bestpractices.dev/projects/8696)
+[![OpenSSF Scorecard](https://api.securityscorecards.dev/projects/github.com/kubernetes/kube-state-metrics/badge)](https://api.securityscorecards.dev/projects/github.com/kubernetes/kube-state-metrics)
+
+kube-state-metrics (KSM) is a simple service that listens to the Kubernetes API
+server and generates metrics about the state of the objects. (See examples in
+the Metrics section below.) It is not focused on the health of the individual
+Kubernetes components, but rather on the health of the various objects inside,
+such as deployments, nodes and pods.
+
+kube-state-metrics is about generating metrics from Kubernetes API objects
+without modification. This ensures that features provided by kube-state-metrics
+have the same grade of stability as the Kubernetes API objects themselves. In
+turn, this means that kube-state-metrics in certain situations may not show the
+exact same values as kubectl, as kubectl applies certain heuristics to display
+comprehensible messages. kube-state-metrics exposes raw data unmodified from the
+Kubernetes API, this way users have all the data they require and perform
+heuristics as they see fit.
+
+The metrics are exported on the HTTP endpoint `/metrics` on the listening port
+(default 8080). They are served as plaintext. They are designed to be consumed
+either by Prometheus itself or by a scraper that is compatible with scraping a
+Prometheus client endpoint. You can also open `/metrics` in a browser to see
+the raw metrics. Note that the metrics exposed on the `/metrics` endpoint
+reflect the current state of the Kubernetes cluster. When Kubernetes objects
+are deleted they are no longer visible on the `/metrics` endpoint.
+
+> [!NOTE]
+> This README is generated from a [template](./README.md.tpl). Please make your changes there and run `make generate-template`.
+
+## Table of Contents
+
+* [Versioning](#versioning)
+  * [Kubernetes Version](#kubernetes-version)
+  * [Compatibility matrix](#compatibility-matrix)
+  * [Resource group version compatibility](#resource-group-version-compatibility)
+  * [Container Image](#container-image)
+* [Metrics Documentation](#metrics-documentation)
+  * [ECMAScript regular expression support for allow and deny lists](#ecmascript-regular-expression-support-for-allow-and-deny-lists)
+  * [Conflict resolution in label names](#conflict-resolution-in-label-names)
+* [Kube-state-metrics self metrics](#kube-state-metrics-self-metrics)
+* [Resource recommendation](#resource-recommendation)
+* [Latency](#latency)
+* [A note on costing](#a-note-on-costing)
+* [kube-state-metrics vs. metrics-server](#kube-state-metrics-vs-metrics-server)
+* [Scaling kube-state-metrics](#scaling-kube-state-metrics)
+  * [Resource recommendation](#resource-recommendation)
+  * [Horizontal sharding](#horizontal-sharding)
+    * [Automated sharding](#automated-sharding)
+  * [Daemonset sharding for pod metrics](#daemonset-sharding-for-pod-metrics)
+* [Setup](#setup)
+  * [Building the Docker container](#building-the-docker-container)
+* [Usage](#usage)
+  * [Kubernetes Deployment](#kubernetes-deployment)
+  * [Limited privileges environment](#limited-privileges-environment)
+  * [Helm Chart](#helm-chart)
+  * [Development](#development)
+  * [Developer Contributions](#developer-contributions)
+  * [Community](#community)
+
+### Versioning
+
+#### Kubernetes Version
+
+kube-state-metrics uses [`client-go`](https://github.com/kubernetes/client-go) to talk with
+Kubernetes clusters. The supported Kubernetes cluster version is determined by
+[`client-go`](https://github.com/kubernetes/client-go#compatibility-matrix).
+All additional compatibility is only best effort, or happens to still/already be supported.
+
+#### Compatibility matrix
+
+At most, 5 kube-state-metrics and 5 [kubernetes releases](https://github.com/kubernetes/kubernetes/releases) will be recorded below.
+Generally, it is recommended to use the latest release of kube-state-metrics. If you run a very recent version of Kubernetes, you might want to use an unreleased version to have the full range of supported resources. If you run an older version of Kubernetes, you might need to run an older version in order to have full support for all resources. Be aware, that the maintainers will only support the latest release. Older versions might be supported by interested users of the community.
+
+| kube-state-metrics | Kubernetes client-go Version |
+|--------------------|:----------------------------:|
+{{ define "compat-matrix" -}}
+{{- range . -}}
+| **{{ .version }}**{{ strings.Repeat (conv.ToInt (math.Sub 15 (len .version))) " " }}| v{{ .kubernetes }}                        |
+{{ end -}}
+{{- end -}}
+{{ template "compat-matrix" (datasource "config").compat }}
+#### Resource group version compatibility
+
+Resources in Kubernetes can evolve, i.e., the group version for a resource may change from alpha to beta and finally GA
+in different Kubernetes versions. For now, kube-state-metrics will only use the oldest API available in the latest
+release.
+
+#### Container Image
+
+The latest container image can be found at:
+{{ define "get-latest-release" -}}
+{{ (index . (math.Sub (len .) 2)).version -}}
+{{ end }}
+* `registry.k8s.io/kube-state-metrics/kube-state-metrics:{{ template "get-latest-release" (datasource "config").compat }}` (arch: `amd64`, `arm`, `arm64`, `ppc64le` and `s390x`)
+* [Multi-architecture images](https://explore.ggcr.dev/?image=registry.k8s.io%2Fkube-state-metrics%2Fkube-state-metrics:{{ template "get-latest-release" (datasource "config").compat -}})
+
+### Metrics Documentation
+
+Any resources and metrics based on alpha Kubernetes APIs are excluded from any stability guarantee,
+which may be changed at any given release.
+
+See the [`docs`](docs) directory for more information on the exposed metrics.
+
+#### Conflict resolution in label names
+
+The `*_labels` family of metrics exposes Kubernetes labels as Prometheus labels.
+As [Kubernetes](https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#syntax-and-character-set)
+is more liberal than
+[Prometheus](https://prometheus.io/docs/concepts/data_model/#metric-names-and-labels)
+in terms of allowed characters in label names,
+we automatically convert unsupported characters to underscores.
+For example, `app.kubernetes.io/name` becomes `label_app_kubernetes_io_name`.
+
+This conversion can create conflicts when multiple Kubernetes labels like
+`foo-bar` and `foo_bar` would be converted to the same Prometheus label `label_foo_bar`.
+
+Kube-state-metrics automatically adds a suffix `_conflictN` to resolve this conflict,
+so it converts the above labels to
+`label_foo_bar_conflict1` and `label_foo_bar_conflict2`.
+
+If you'd like to have more control over how this conflict is resolved,
+you might want to consider addressing this issue on a different level of the stack,
+e.g. by standardizing Kubernetes labels using an
+[Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/)
+that ensures that there are no possible conflicts.
+
+#### ECMAScript regular expression support for allow and deny lists
+
+Starting from [#2616](https://github.com/kubernetes/kube-state-metrics/pull/2616/files), kube-state-metrics supports ECMAScript's `regexp` for allow and deny lists. This was incorporated as a workaround for the limitations of the `regexp` package in Go, which does not support lookarounds due to their non-linear time complexity. Please note that while lookarounds are now supported for allow and deny lists, regular expressions' evaluation time is capped at a minute to prevent performance issues.
+
+### Kube-state-metrics self metrics
+
+kube-state-metrics exposes its own general process metrics under `--telemetry-host` and `--telemetry-port` (default 8081).
+
+kube-state-metrics also exposes list and watch success and error metrics. These can be used to calculate the error rate of list or watch resources.
+If you encounter those errors in the metrics, it is most likely a configuration or permission issue, and the next thing to investigate would be looking
+at the logs of kube-state-metrics.
+
+Example of the above mentioned metrics:
+
+```prometheus
+kube_state_metrics_list_total{resource="*v1.Node",result="success"} 1
+kube_state_metrics_list_total{resource="*v1.Node",result="error"} 52
+kube_state_metrics_watch_total{resource="*v1beta1.Ingress",result="success"} 1
+```
+
+kube-state-metrics also exposes some http request metrics, examples of those are:
+
+```prometheus
+http_request_duration_seconds_bucket{handler="metrics",method="get",le="2.5"} 30
+http_request_duration_seconds_bucket{handler="metrics",method="get",le="5"} 30
+http_request_duration_seconds_bucket{handler="metrics",method="get",le="10"} 30
+http_request_duration_seconds_bucket{handler="metrics",method="get",le="+Inf"} 30
+http_request_duration_seconds_sum{handler="metrics",method="get"} 0.021113919999999998
+http_request_duration_seconds_count{handler="metrics",method="get"} 30
+```
+
+kube-state-metrics also exposes build and configuration metrics:
+
+```prometheus
+kube_state_metrics_build_info{branch="main",goversion="go1.15.3",revision="6c9d775d",version="v2.0.0-beta"} 1
+kube_state_metrics_shard_ordinal{shard_ordinal="0"} 0
+kube_state_metrics_total_shards 1
+```
+
+`kube_state_metrics_build_info` is used to expose version and other build information. For more usage about the info pattern,
+please check this [blog post](https://www.robustperception.io/exposing-the-software-version-to-prometheus).
+Sharding metrics expose `--shard` and `--total-shards` flags and can be used to validate
+run-time configuration, see [`/examples/prometheus-alerting-rules`](./examples/prometheus-alerting-rules).
+
+kube-state-metrics also exposes metrics about it config file and the Custom Resource State config file:
+
+```prometheus
+kube_state_metrics_config_hash{filename="crs.yml",type="customresourceconfig"} 2.38272279311849e+14
+kube_state_metrics_config_hash{filename="config.yml",type="config"} 2.65285922340846e+14
+kube_state_metrics_last_config_reload_success_timestamp_seconds{filename="crs.yml",type="customresourceconfig"} 1.6704882592037103e+09
+kube_state_metrics_last_config_reload_success_timestamp_seconds{filename="config.yml",type="config"} 1.6704882592035313e+09
+kube_state_metrics_last_config_reload_successful{filename="crs.yml",type="customresourceconfig"} 1
+kube_state_metrics_last_config_reload_successful{filename="config.yml",type="config"} 1
+```
+
+### Scaling kube-state-metrics
+
+#### Resource recommendation
+
+Resource usage for kube-state-metrics changes with the Kubernetes objects (Pods/Nodes/Deployments/Secrets etc.) size of the cluster.
+To some extent, the Kubernetes objects in a cluster are in direct proportion to the node number of the cluster.
+
+As a general rule, you should allocate:
+
+* 250MiB memory
+* 0.1 cores
+
+Note that if CPU limits are set too low, kube-state-metrics' internal queues will not be able to be worked off quickly enough, resulting in increased memory consumption as the queue length grows. If you experience problems resulting from high memory allocation or CPU throttling, try increasing the CPU limits.
+
+### Latency
+
+In a 100 node cluster scaling test the latency numbers were as follows:
+
+```text
+"Perc50": 259615384 ns,
+"Perc90": 475000000 ns,
+"Perc99": 906666666 ns.
+```
+
+### A note on costing
+
+By default, kube-state-metrics exposes several metrics for events across your cluster. If you have a large number of frequently-updating resources on your cluster, you may find that a lot of data is ingested into these metrics. This can incur high costs on some cloud providers. Please take a moment to [configure what metrics you'd like to expose](docs/developer/cli-arguments.md), as well as consult the documentation for your Kubernetes environment in order to avoid unexpectedly high costs.
+
+### kube-state-metrics vs. metrics-server
+
+The [metrics-server](https://github.com/kubernetes-incubator/metrics-server)
+is a project that has been inspired by
+[Heapster](https://github.com/kubernetes-retired/heapster) and is implemented
+to serve the goals of core metrics pipelines in [Kubernetes monitoring
+architecture](https://github.com/kubernetes/design-proposals-archive/blob/main/instrumentation/monitoring_architecture.md).
+It is a cluster level component which periodically scrapes metrics from all
+Kubernetes nodes served by Kubelet through Metrics API. The metrics are
+aggregated, stored in memory and served in [Metrics API
+format](https://git.k8s.io/metrics/pkg/apis/metrics/v1alpha1/types.go). The
+metrics-server stores the latest values only and is not responsible for
+forwarding metrics to third-party destinations.
+
+kube-state-metrics is focused on generating completely new metrics from
+Kubernetes' object state (e.g. metrics based on deployments, replica sets,
+etc.). It holds an entire snapshot of Kubernetes state in memory and
+continuously generates new metrics based off of it. And just like the
+metrics-server it too is not responsible for exporting its metrics anywhere.
+
+Having kube-state-metrics as a separate project also enables access to these
+metrics from monitoring systems such as Prometheus.
+
+### Horizontal sharding
+
+In order to shard kube-state-metrics horizontally, some automated sharding capabilities have been implemented. It is configured with the following flags:
+
+* `--shard` (zero indexed)
+* `--total-shards`
+
+Sharding is done by taking an md5 sum of the Kubernetes Object's UID and performing a modulo operation on it with the total number of shards. Each shard decides whether the object is handled by the respective instance of kube-state-metrics or not. Note that this means all instances of kube-state-metrics, even if sharded, will have the network traffic and the resource consumption for unmarshaling objects for all objects, not just the ones they are responsible for. To optimize this further, the Kubernetes API would need to support sharded list/watch capabilities. In the optimal case, memory consumption for each shard will be 1/n compared to an unsharded setup. Typically, kube-state-metrics needs to be memory and latency optimized in order for it to return its metrics rather quickly to Prometheus. One way to reduce the latency between kube-state-metrics and the kube-apiserver is to run KSM with the `--use-apiserver-cache` flag. In addition to reducing the latency, this option will also lead to a reduction in the load on etcd.
+
+Sharding should be used carefully and additional monitoring should be set up in order to ensure that sharding is set up and functioning as expected (eg. instances for each shard out of the total shards are configured).
+
+#### Automated sharding
+
+Automatic sharding allows each shard to discover its nominal position when deployed in a StatefulSet which is useful for automatically configuring sharding. This is an experimental feature and may be broken or removed without notice.
+
+To enable automated sharding, kube-state-metrics must be run by a `StatefulSet` and the pod name and namespace must be handed to the kube-state-metrics process via the `--pod` and `--pod-namespace` flags. Example manifests demonstrating the autosharding functionality can be found in [`/examples/autosharding`](./examples/autosharding).
+
+This way of deploying shards is useful when you want to manage KSM shards through a single Kubernetes resource (a single `StatefulSet` in this case) instead of having one `Deployment` per shard. The advantage can be especially significant when deploying a high number of shards.
+
+The downside of using an auto-sharded setup comes from the rollout strategy supported by `StatefulSet`s. When managed by a `StatefulSet`, pods are replaced one at a time with each pod first getting terminated and then recreated. Besides such rollouts being slower, they will also lead to short downtime for each shard. If a Prometheus scrape happens during a rollout, it can miss some of the metrics exported by kube-state-metrics.
+
+### Daemonset sharding for pod metrics
+
+For pod metrics, they can be sharded per node with the following flag:
+
+* `--node=$(NODE_NAME)`
+
+Each kube-state-metrics pod uses FieldSelector (spec.nodeName) to watch/list pod metrics only on the same node.
+
+A daemonset kube-state-metrics example:
+
+```yaml
+apiVersion: apps/v1
+kind: DaemonSet
+spec:
+  template:
+    spec:
+      containers:
+      - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:IMAGE_TAG
+        name: kube-state-metrics
+        args:
+        - --resource=pods
+        - --node=$(NODE_NAME)
+        env:
+        - name: NODE_NAME
+          valueFrom:
+            fieldRef:
+              apiVersion: v1
+              fieldPath: spec.nodeName
+```
+
+To track metrics for unassigned pods, you need to add an additional deployment and set `--track-unscheduled-pods`, as shown in the following example:
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+spec:
+  template:
+    spec:
+      containers:
+      - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:IMAGE_TAG
+        name: kube-state-metrics
+        args:
+        - --resources=pods
+        - --track-unscheduled-pods
+```
+
+Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding).
+
+### Setup
+
+Install this project to your `$GOPATH` using `go get`:
+
+```bash
+go get k8s.io/kube-state-metrics/v2
+```
+
+#### Building the Docker container
+
+Simply run the following command in this root folder, which will create a
+self-contained, statically-linked binary and build a Docker image:
+
+```bash
+make container
+```
+
+### Usage
+
+Simply build and run kube-state-metrics inside a Kubernetes pod which has a
+service account token that has read-only access to the Kubernetes cluster.
+
+#### For users of prometheus-operator/kube-prometheus stack
+
+The ([`kube-prometheus`](https://github.com/prometheus-operator/kube-prometheus/)) stack installs kube-state-metrics as one of its [components](https://github.com/prometheus-operator/kube-prometheus#kube-prometheus); you do not need to install kube-state-metrics if you're using the kube-prometheus stack.
+
+If you want to revise the default configuration for kube-prometheus, for example to enable non-default metrics, have a look at [Customizing Kube-Prometheus](https://github.com/prometheus-operator/kube-prometheus/blob/main/docs/customizing.md).
+
+#### Kubernetes Deployment
+
+To deploy this project, you can simply run `kubectl apply -f examples/standard` and a Kubernetes service and deployment will be created. (Note: Adjust the apiVersion of some resource if your kubernetes cluster's version is not 1.8+, check the yaml file for more information).
+
+To have Prometheus discover kube-state-metrics instances it is advised to create a specific Prometheus scrape config for kube-state-metrics that picks up both metrics endpoints. Annotation based discovery is discouraged as only one of the endpoints would be able to be selected, plus kube-state-metrics in most cases has special authentication and authorization requirements as it essentially grants read access through the metrics endpoint to most information available to it.
+
+**Note:** Google Kubernetes Engine (GKE) Users - GKE has strict role permissions that will prevent the kube-state-metrics roles and role bindings from being created. To work around this, you can give your GCP identity the cluster-admin role by running the following one-liner:
+
+```bash
+kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=$(gcloud info --format='value(config.account)')
+```
+
+Note that your GCP identity is case sensitive but `gcloud info` as of Google Cloud SDK 221.0.0 is not. This means that if your IAM member contains capital letters, the above one-liner may not work for you. If you have 403 forbidden responses after running the above command and `kubectl apply -f examples/standard`, check the IAM member associated with your account at <https://console.cloud.google.com/iam-admin/iam?project=PROJECT_ID>. If it contains capital letters, you may need to set the --user flag in the command above to the case-sensitive role listed at <https://console.cloud.google.com/iam-admin/iam?project=PROJECT_ID>.
+
+After running the above, if you see `Clusterrolebinding "cluster-admin-binding" created`, then you are able to continue with the setup of this service.
+
+#### Healthcheck Endpoints
+
+The following healthcheck endpoints are available (`self` refers to the telemetry port, while `main` refers to the exposition port):
+
+* `/healthz` (exposed on `main`): Returns a 200 status code if the application is running. We recommend to use this for the startup probe.
+* `/livez` (exposed on `main`): Returns a 200 status code if the application is not affected by an outage of the Kubernetes API Server. We recommend to using this for the liveness probe.
+* `/readyz` (exposed on `self`): Returns a 200 status code if the application is ready to accept requests and expose metrics. We recommend using this for the readiness probe.
+
+Note that it is discouraged to use the telemetry metrics endpoint for any probe when proxying the exposition data.
+
+#### Limited privileges environment
+
+If you want to run kube-state-metrics in an environment where you don't have cluster-reader role, you can:
+
+* create a serviceaccount
+
+```yaml
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: kube-state-metrics
+  namespace: your-namespace-where-kube-state-metrics-will-deployed
+```
+
+* give it `view` privileges on specific namespaces (using roleBinding) (*note: you can add this roleBinding to all the NS you want your serviceaccount to access*)
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kube-state-metrics
+  namespace: project1
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: view
+subjects:
+  - kind: ServiceAccount
+    name: kube-state-metrics
+    namespace: your-namespace-where-kube-state-metrics-will-deployed
+```
+
+* then specify a set of namespaces (using the `--namespaces` option) and a set of kubernetes objects (using the `--resources`) that your serviceaccount has access to in the `kube-state-metrics` deployment configuration
+
+```yaml
+spec:
+  template:
+    spec:
+      containers:
+      - name: kube-state-metrics
+        args:
+          - '--resources=pods'
+          - '--namespaces=project1'
+```
+
+For the full list of arguments available, see the documentation in [docs/developer/cli-arguments.md](./docs/developer/cli-arguments.md)
+
+#### Helm Chart
+
+Starting from the kube-state-metrics chart `v2.13.3` (kube-state-metrics image `v1.9.8`), the official [Helm chart](https://artifacthub.io/packages/helm/prometheus-community/kube-state-metrics/) is maintained in [prometheus-community/helm-charts](https://github.com/prometheus-community/helm-charts/tree/main/charts/kube-state-metrics). Starting from kube-state-metrics chart `v3.0.0` only kube-state-metrics images of `v2.0.0 +` are supported.
+
+#### Development
+
+When developing, test a metric dump against your local Kubernetes cluster by running:
+
+> Users can override the apiserver address in KUBE-CONFIG file with `--apiserver` command line.
+
+```bash
+go install
+kube-state-metrics --port=8080 --telemetry-port=8081 --kubeconfig=<KUBE-CONFIG> --apiserver=<APISERVER>
+```
+
+Then curl the metrics endpoint
+
+```bash
+curl localhost:8080/metrics
+```
+
+To run the e2e tests locally see the documentation in [tests/README.md](./tests/README.md).
+
+#### Developer Contributions
+
+When developing, there are certain code patterns to follow to better your contributing experience and likelihood of e2e and other ci tests to pass. To learn more about them, see the documentation in [docs/developer/guide.md](./docs/developer/guide.md).
+
+#### Community
+
+This project is sponsored by [SIG Instrumentation](https://github.com/kubernetes/community/tree/master/sig-instrumentation).
+
+There is also a channel for [#kube-state-metrics](https://kubernetes.slack.com/archives/CJJ529RUY) on Kubernetes' Slack.
+
+You can also join the SIG Instrumentation [mailing list](https://groups.google.com/forum/#!forum/kubernetes-sig-instrumentation).
+This will typically add invites for the following meetings to your calendar, in which topics around kube-state-metrics can be discussed.
+
+* Regular SIG Meeting: [Thursdays at 9:30 PT (Pacific Time)](https://zoom.us/j/5342565819?pwd=RlVsK21NVnR1dmE3SWZQSXhveHZPdz09) (biweekly). [Convert to your timezone](http://www.thetimezoneconverter.com/?t=9:30&tz=PT%20%28Pacific%20Time%29).
+* Regular Triage Meeting: [Thursdays at 9:30 PT (Pacific Time)](https://zoom.us/j/5342565819?pwd=RlVsK21NVnR1dmE3SWZQSXhveHZPdz09) (biweekly - alternating with regular meeting). [Convert to your timezone](http://www.thetimezoneconverter.com/?t=9:30&tz=PT%20%28Pacific%20Time%29).

RELEASE.md

@@ -6,33 +6,40 @@ We use [Semantic Versioning](http://semver.org/).
 
 We maintain a separate branch for each minor release, named `release-<major>.<minor>`, e.g. `release-1.1`, `release-2.0`.
 
-The usual flow is to merge new features and changes into the master branch and to merge bug fixes into the latest release branch. Bug fixes are then merged into master from the latest release branch. The master branch should always contain all commits from the latest release branch.
+The usual flow is to merge new features and changes into the main branch and to merge bug fixes into the latest release branch. Bug fixes are then merged into main from the latest release branch. The main branch should always contain all commits from the latest release branch.
 
-If a bug fix got accidentally merged into master, cherry-pick commits have to be created in the latest release branch, which then have to be merged back into master. Try to avoid that situation.
+If a bug fix got accidentally merged into main, cherry-pick commits have to be created in the latest release branch, which then have to be merged back into main. Try to avoid that situation.
 
 Maintaining the release branches for older minor releases happens on a best effort basis.
 
 ## Prepare your release
 
-* Bump the version in the `VERSION` file in the root of the repository.
+* Update the [data.yaml](data.yaml)
+  * Update the compat list
+  * Update the version key to refer to your new release
 * Run `make examples`, which will re-generate all example manifests to use the new version.
 * Make a PR to update:
-  * [Compatibility matrix](README.md#compatibility-matrix)
+  * Run `make generate`, which will update the compatibility matrix in README.md
   * Changelog entry
     * Only include user relevant changes
     * Entries in the [`CHANGELOG.md`](CHANGELOG.md) are meant to be in this order:
-    ```
+
+    ```text
     [CHANGE]
     [FEATURE]
     [ENHANCEMENT]
     [BUGFIX]
     ```
+
     * All lines should be full sentences
   * kube-state-metrics image tag used in Kubernetes deployment yaml config.
-* Cut the new release branch, i.e., `release-1.2`, or merge/cherry-pick changes onto the minor release branch you intend to tag the release on
-* Cut the new release tag, i.e., `v1.2.0-rc.0`
+* Cut the new release branch, e.g. `release-1.2`, or merge/cherry-pick changes onto the minor release branch you intend to tag the release on
+* Cut the new release tag, e.g. `v1.2.0-rc.0`
+* Create a new **pre-release** on github
 * New images are automatically built and pushed to `gcr.io/k8s-staging-kube-state-metrics/kube-state-metrics`
-* Promote image by sending a PR to [kubernetes/k8s.io](https://github.com/kubernetes/k8s.io) repo. Follow the [example PR](https://github.com/kubernetes/k8s.io/pull/1260).
+* Promote image by sending a PR to [kubernetes/k8s.io](https://github.com/kubernetes/k8s.io) repository. Follow the [example PR](https://github.com/kubernetes/k8s.io/pull/3798). Use [kpromo pr](https://github.com/kubernetes-sigs/promo-tools/blob/main/docs/promotion-pull-requests.md) to update the manifest files in this repository, e.g. `kpromo pr --fork=$YOURNAME -i --project=kube-state-metrics -t=v2.5.0`
+* Create a PR to merge the changes of this release back into the main branch.
+* Once the PR to promote the image is merged, mark the pre-release as a regular release.
 
 ## Stable release
 

SECURITY-INSIGHTS.yml

@@ -0,0 +1,49 @@
+# Refer: https://github.com/ossf/security-insights-spec/blob/main/specification.md#specification
+header:
+  schema-version: "1.0.0"
+  expiration-date: "2024-12-15T19:10:00.000Z"
+  project-url: https://github.com/kubernetes/kube-state-metrics
+  changelog: https://github.com/kubernetes/kube-state-metrics/blob/main/CHANGELOG.md
+  license: https://github.com/kubernetes/kube-state-metrics/blob/main/LICENSE
+project-lifecycle:
+  status: active
+  bug-fixes-only: false
+  core-maintainers:
+    - github:dgrisonnet
+    - github:mrueg
+    - github:rexagod
+  release-process: https://github.com/kubernetes/kube-state-metrics/blob/main/RELEASE.md
+contribution-policy:
+  accepts-pull-requests: true
+  accepts-automated-pull-requests: true
+  contributing-policy: https://github.com/kubernetes/kube-state-metrics/blob/main/CONTRIBUTING.md
+  code-of-conduct: https://github.com/kubernetes/kube-state-metrics/blob/main/code-of-conduct.md
+distribution-points:
+  - https://github.com/kubernetes/kube-state-metrics/releases
+  - https://github.com/kubernetes/k8s.io/blob/main/registry.k8s.io/images/k8s-staging-kube-state-metrics/images.yaml
+security-contacts:
+  - type: website
+    value: https://github.com/kubernetes/kube-state-metrics/blob/main/SECURITY_CONTACTS
+vulnerability-reporting:
+  accepts-vulnerability-reports: true
+  security-policy: https://github.com/kubernetes/kube-state-metrics/blob/main/SECURITY.md
+dependencies:
+  third-party-packages: true
+  dependencies-lists:
+    - https://github.com/kubernetes/kube-state-metrics/blob/main/go.mod
+    - https://github.com/kubernetes/kube-state-metrics/blob/main/Dockerfile
+  env-dependencies-policy:
+    policy-url: https://github.com/kubernetes/kube-state-metrics/blob/main/docs/dependencies-policy.md
+documentation:
+    - https://github.com/kubernetes/kube-state-metrics/tree/main/docs
+security-testing:
+- tool-type: dast
+  tool-name: govulncheck
+  tool-version: latest
+  tool-url: https://go.googlesource.com/vuln
+  tool-rulesets:
+  - built-in
+  integration:
+    ci: true
+  comment: |
+    Detects vulnerabilities as a result of the affected call-paths being invoked directly in the repository, while reducing false positives by ignoring dormant call-paths for package dependencies.

SECURITY_CONTACTS

@@ -10,6 +10,7 @@
 # DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, FOLLOW THE
 # INSTRUCTIONS AT https://kubernetes.io/security/
 
-brancz
-tariq1890
-LiliC
+dgrisonnet
+mrueg
+rexagod
+CatherineF-dev

VERSION

@@ -1 +0,0 @@
-2.0.0-rc.1

charts/kube-state-metrics/.helmignore

@@ -1,21 +0,0 @@
-# Patterns to ignore when building packages.
-# This supports shell glob matching, relative path matching, and
-# negation (prefixed with !). Only one pattern per line.
-.DS_Store
-# Common VCS dirs
-.git/
-.gitignore
-.bzr/
-.bzrignore
-.hg/
-.hgignore
-.svn/
-# Common backup files
-*.swp
-*.bak
-*.tmp
-*~
-# Various IDEs
-.project
-.idea/
-*.tmproj

charts/kube-state-metrics/Chart.yaml

@@ -1,18 +0,0 @@
-apiVersion: v1
-name: kube-state-metrics
-description: Install kube-state-metrics to generate and expose cluster-level metrics
-keywords:
-- metric
-- monitoring
-- prometheus
-- kubernetes
-version: 2.13.1
-appVersion: 1.9.8
-home: https://github.com/kubernetes/kube-state-metrics/
-sources:
-- https://github.com/kubernetes/kube-state-metrics/
-maintainers:
-- name: tariq1890
-  email: tariq.ibrahim@mulesoft.com
-- name: mrueg
-  email: manuel@rueg.eu

charts/kube-state-metrics/LICENSE

@@ -1,202 +0,0 @@
-
-                                 Apache License
-                           Version 2.0, January 2004
-                        http://www.apache.org/licenses/
-
-   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
-
-   1. Definitions.
-
-      "License" shall mean the terms and conditions for use, reproduction,
-      and distribution as defined by Sections 1 through 9 of this document.
-
-      "Licensor" shall mean the copyright owner or entity authorized by
-      the copyright owner that is granting the License.
-
-      "Legal Entity" shall mean the union of the acting entity and all
-      other entities that control, are controlled by, or are under common
-      control with that entity. For the purposes of this definition,
-      "control" means (i) the power, direct or indirect, to cause the
-      direction or management of such entity, whether by contract or
-      otherwise, or (ii) ownership of fifty percent (50%) or more of the
-      outstanding shares, or (iii) beneficial ownership of such entity.
-
-      "You" (or "Your") shall mean an individual or Legal Entity
-      exercising permissions granted by this License.
-
-      "Source" form shall mean the preferred form for making modifications,
-      including but not limited to software source code, documentation
-      source, and configuration files.
-
-      "Object" form shall mean any form resulting from mechanical
-      transformation or translation of a Source form, including but
-      not limited to compiled object code, generated documentation,
-      and conversions to other media types.
-
-      "Work" shall mean the work of authorship, whether in Source or
-      Object form, made available under the License, as indicated by a
-      copyright notice that is included in or attached to the work
-      (an example is provided in the Appendix below).
-
-      "Derivative Works" shall mean any work, whether in Source or Object
-      form, that is based on (or derived from) the Work and for which the
-      editorial revisions, annotations, elaborations, or other modifications
-      represent, as a whole, an original work of authorship. For the purposes
-      of this License, Derivative Works shall not include works that remain
-      separable from, or merely link (or bind by name) to the interfaces of,
-      the Work and Derivative Works thereof.
-
-      "Contribution" shall mean any work of authorship, including
-      the original version of the Work and any modifications or additions
-      to that Work or Derivative Works thereof, that is intentionally
-      submitted to Licensor for inclusion in the Work by the copyright owner
-      or by an individual or Legal Entity authorized to submit on behalf of
-      the copyright owner. For the purposes of this definition, "submitted"
-      means any form of electronic, verbal, or written communication sent
-      to the Licensor or its representatives, including but not limited to
-      communication on electronic mailing lists, source code control systems,
-      and issue tracking systems that are managed by, or on behalf of, the
-      Licensor for the purpose of discussing and improving the Work, but
-      excluding communication that is conspicuously marked or otherwise
-      designated in writing by the copyright owner as "Not a Contribution."
-
-      "Contributor" shall mean Licensor and any individual or Legal Entity
-      on behalf of whom a Contribution has been received by Licensor and
-      subsequently incorporated within the Work.
-
-   2. Grant of Copyright License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      copyright license to reproduce, prepare Derivative Works of,
-      publicly display, publicly perform, sublicense, and distribute the
-      Work and such Derivative Works in Source or Object form.
-
-   3. Grant of Patent License. Subject to the terms and conditions of
-      this License, each Contributor hereby grants to You a perpetual,
-      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
-      (except as stated in this section) patent license to make, have made,
-      use, offer to sell, sell, import, and otherwise transfer the Work,
-      where such license applies only to those patent claims licensable
-      by such Contributor that are necessarily infringed by their
-      Contribution(s) alone or by combination of their Contribution(s)
-      with the Work to which such Contribution(s) was submitted. If You
-      institute patent litigation against any entity (including a
-      cross-claim or counterclaim in a lawsuit) alleging that the Work
-      or a Contribution incorporated within the Work constitutes direct
-      or contributory patent infringement, then any patent licenses
-      granted to You under this License for that Work shall terminate
-      as of the date such litigation is filed.
-
-   4. Redistribution. You may reproduce and distribute copies of the
-      Work or Derivative Works thereof in any medium, with or without
-      modifications, and in Source or Object form, provided that You
-      meet the following conditions:
-
-      (a) You must give any other recipients of the Work or
-          Derivative Works a copy of this License; and
-
-      (b) You must cause any modified files to carry prominent notices
-          stating that You changed the files; and
-
-      (c) You must retain, in the Source form of any Derivative Works
-          that You distribute, all copyright, patent, trademark, and
-          attribution notices from the Source form of the Work,
-          excluding those notices that do not pertain to any part of
-          the Derivative Works; and
-
-      (d) If the Work includes a "NOTICE" text file as part of its
-          distribution, then any Derivative Works that You distribute must
-          include a readable copy of the attribution notices contained
-          within such NOTICE file, excluding those notices that do not
-          pertain to any part of the Derivative Works, in at least one
-          of the following places: within a NOTICE text file distributed
-          as part of the Derivative Works; within the Source form or
-          documentation, if provided along with the Derivative Works; or,
-          within a display generated by the Derivative Works, if and
-          wherever such third-party notices normally appear. The contents
-          of the NOTICE file are for informational purposes only and
-          do not modify the License. You may add Your own attribution
-          notices within Derivative Works that You distribute, alongside
-          or as an addendum to the NOTICE text from the Work, provided
-          that such additional attribution notices cannot be construed
-          as modifying the License.
-
-      You may add Your own copyright statement to Your modifications and
-      may provide additional or different license terms and conditions
-      for use, reproduction, or distribution of Your modifications, or
-      for any such Derivative Works as a whole, provided Your use,
-      reproduction, and distribution of the Work otherwise complies with
-      the conditions stated in this License.
-
-   5. Submission of Contributions. Unless You explicitly state otherwise,
-      any Contribution intentionally submitted for inclusion in the Work
-      by You to the Licensor shall be under the terms and conditions of
-      this License, without any additional terms or conditions.
-      Notwithstanding the above, nothing herein shall supersede or modify
-      the terms of any separate license agreement you may have executed
-      with Licensor regarding such Contributions.
-
-   6. Trademarks. This License does not grant permission to use the trade
-      names, trademarks, service marks, or product names of the Licensor,
-      except as required for reasonable and customary use in describing the
-      origin of the Work and reproducing the content of the NOTICE file.
-
-   7. Disclaimer of Warranty. Unless required by applicable law or
-      agreed to in writing, Licensor provides the Work (and each
-      Contributor provides its Contributions) on an "AS IS" BASIS,
-      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
-      implied, including, without limitation, any warranties or conditions
-      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
-      PARTICULAR PURPOSE. You are solely responsible for determining the
-      appropriateness of using or redistributing the Work and assume any
-      risks associated with Your exercise of permissions under this License.
-
-   8. Limitation of Liability. In no event and under no legal theory,
-      whether in tort (including negligence), contract, or otherwise,
-      unless required by applicable law (such as deliberate and grossly
-      negligent acts) or agreed to in writing, shall any Contributor be
-      liable to You for damages, including any direct, indirect, special,
-      incidental, or consequential damages of any character arising as a
-      result of this License or out of the use or inability to use the
-      Work (including but not limited to damages for loss of goodwill,
-      work stoppage, computer failure or malfunction, or any and all
-      other commercial damages or losses), even if such Contributor
-      has been advised of the possibility of such damages.
-
-   9. Accepting Warranty or Additional Liability. While redistributing
-      the Work or Derivative Works thereof, You may choose to offer,
-      and charge a fee for, acceptance of support, warranty, indemnity,
-      or other liability obligations and/or rights consistent with this
-      License. However, in accepting such obligations, You may act only
-      on Your own behalf and on Your sole responsibility, not on behalf
-      of any other Contributor, and only if You agree to indemnify,
-      defend, and hold each Contributor harmless for any liability
-      incurred by, or claims asserted against, such Contributor by reason
-      of your accepting any such warranty or additional liability.
-
-   END OF TERMS AND CONDITIONS
-
-   APPENDIX: How to apply the Apache License to your work.
-
-      To apply the Apache License to your work, attach the following
-      boilerplate notice, with the fields enclosed by brackets "[]"
-      replaced with your own identifying information. (Don't include
-      the brackets!)  The text should be enclosed in the appropriate
-      comment syntax for the file format. We also recommend that a
-      file or class name and description of purpose be included on the
-      same "printed page" as the copyright notice for easier
-      identification within third-party archives.
-
-   Copyright The Helm Authors.
-
-   Licensed under the Apache License, Version 2.0 (the "License");
-   you may not use this file except in compliance with the License.
-   You may obtain a copy of the License at
-
-       http://www.apache.org/licenses/LICENSE-2.0
-
-   Unless required by applicable law or agreed to in writing, software
-   distributed under the License is distributed on an "AS IS" BASIS,
-   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-   See the License for the specific language governing permissions and
-   limitations under the License.

charts/kube-state-metrics/OWNERS

@@ -1,6 +0,0 @@
-approvers:
-- tariq1890
-- mrueg
-reviewers:
-- tariq1890
-- mrueg

charts/kube-state-metrics/README.md

@@ -1,66 +0,0 @@
-# kube-state-metrics Helm Chart
-
-Installs the [kube-state-metrics agent](https://github.com/kubernetes/kube-state-metrics).
-
-## Get Repo Info
-
-```console
-helm repo add kube-state-metrics https://kubernetes.github.io/kube-state-metrics
-helm repo update
-```
-
-_See [helm repo](https://helm.sh/docs/helm/helm_repo/) for command documentation._
-
-## Install Chart
-
-```console
-# Helm 3
-$ helm install [RELEASE_NAME] kube-state-metrics/kube-state-metrics [flags]
-
-# Helm 2
-$ helm install --name [RELEASE_NAME] kube-state-metrics/kube-state-metrics [flags]
-```
-
-_See [configuration](#configuration) below._
-
-_See [helm install](https://helm.sh/docs/helm/helm_install/) for command documentation._
-
-## Uninstall Chart
-
-```console
-# Helm 3
-$ helm uninstall [RELEASE_NAME]
-
-# Helm 2
-# helm delete --purge [RELEASE_NAME]
-```
-
-This removes all the Kubernetes components associated with the chart and deletes the release.
-
-_See [helm uninstall](https://helm.sh/docs/helm/helm_uninstall/) for command documentation._
-
-## Upgrading Chart
-
-```console
-# Helm 3 or 2
-$ helm upgrade [RELEASE_NAME] kube-state-metrics/kube-state-metrics [flags]
-```
-
-_See [helm upgrade](https://helm.sh/docs/helm/helm_upgrade/) for command documentation._
-
-### From stable/kube-state-metrics
-
-You can upgrade in-place:
-
-1. [get repo info](#get-repo-info)
-1. [upgrade](#upgrading-chart) your existing release name using the new chart repo
-
-## Configuration
-
-See [Customizing the Chart Before Installing](https://helm.sh/docs/intro/using_helm/#customizing-the-chart-before-installing). To see all configurable options with detailed comments:
-
-```console
-helm show values kube-state-metrics/kube-state-metrics
-```
-
-You may also `helm show values` on this chart's [dependencies](#dependencies) for additional options.

charts/kube-state-metrics/templates/NOTES.txt

@@ -1,10 +0,0 @@
-kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects.
-The exposed metrics can be found here:
-https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics
-
-The metrics are exported on the HTTP endpoint /metrics on the listening port.
-In your case, {{ template "kube-state-metrics.fullname" . }}.{{ template "kube-state-metrics.namespace" . }}.svc.cluster.local:{{ .Values.service.port }}/metrics
-
-They are served either as plaintext or protobuf depending on the Accept header.
-They are designed to be consumed either by Prometheus itself or by a scraper that is compatible with scraping a Prometheus client endpoint.
-

charts/kube-state-metrics/templates/_helpers.tpl

@@ -1,47 +0,0 @@
-{{/* vim: set filetype=mustache: */}}
-{{/*
-Expand the name of the chart.
-*/}}
-{{- define "kube-state-metrics.name" -}}
-{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-
-{{/*
-Create a default fully qualified app name.
-We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
-If release name contains chart name it will be used as a full name.
-*/}}
-{{- define "kube-state-metrics.fullname" -}}
-{{- if .Values.fullnameOverride -}}
-{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- $name := default .Chart.Name .Values.nameOverride -}}
-{{- if contains $name .Release.Name -}}
-{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
-{{- else -}}
-{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
-{{- end -}}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Create the name of the service account to use
-*/}}
-{{- define "kube-state-metrics.serviceAccountName" -}}
-{{- if .Values.serviceAccount.create -}}
-    {{ default (include "kube-state-metrics.fullname" .) .Values.serviceAccount.name }}
-{{- else -}}
-    {{ default "default" .Values.serviceAccount.name }}
-{{- end -}}
-{{- end -}}
-
-{{/*
-Allow the release namespace to be overridden for multi-namespace deployments in combined charts
-*/}}
-{{- define "kube-state-metrics.namespace" -}}
-  {{- if .Values.namespaceOverride -}}
-    {{- .Values.namespaceOverride -}}
-  {{- else -}}
-    {{- .Release.Namespace -}}
-  {{- end -}}
-{{- end -}}

charts/kube-state-metrics/templates/clusterrolebinding.yaml

@@ -1,23 +0,0 @@
-{{- if and .Values.rbac.create .Values.rbac.useClusterRole -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-  name: {{ template "kube-state-metrics.fullname" . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-{{- if .Values.rbac.useExistingRole }}
-  name: {{ .Values.rbac.useExistingRole }}
-{{- else }}
-  name: {{ template "kube-state-metrics.fullname" . }}
-{{- end }}
-subjects:
-- kind: ServiceAccount
-  name: {{ template "kube-state-metrics.fullname" . }}
-  namespace: {{ template "kube-state-metrics.namespace" . }}
-{{- end -}}

charts/kube-state-metrics/templates/deployment.yaml

@@ -1,216 +0,0 @@
-apiVersion: apps/v1
-{{- if .Values.autosharding.enabled }}
-kind: StatefulSet
-{{- else }}
-kind: Deployment
-{{- end }}
-metadata:
-  name: {{ template "kube-state-metrics.fullname" . }}
-  namespace: {{ template "kube-state-metrics.namespace" . }}
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
-    app.kubernetes.io/instance: "{{ .Release.Name }}"
-    app.kubernetes.io/managed-by: "{{ .Release.Service }}"
-    app.kubernetes.io/version: "{{ .Chart.AppVersion }}"
-{{- if .Values.customLabels }}
-{{ toYaml .Values.customLabels | indent 4 }}
-{{- end }}
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-  replicas: {{ .Values.replicas }}
-{{- if .Values.autosharding.enabled }}
-  serviceName: {{ template "kube-state-metrics.fullname" . }}
-  volumeClaimTemplates: []
-{{- end }}
-  template:
-    metadata:
-      labels:
-        app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-        app.kubernetes.io/instance: "{{ .Release.Name }}"
-{{- if .Values.customLabels }}
-{{ toYaml .Values.customLabels | indent 8 }}
-{{- end }}
-{{- if .Values.podAnnotations }}
-      annotations:
-{{ toYaml .Values.podAnnotations | indent 8 }}
-{{- end }}
-    spec:
-      hostNetwork: {{ .Values.hostNetwork }}
-      serviceAccountName: {{ template "kube-state-metrics.serviceAccountName" . }}
-      {{- if .Values.securityContext.enabled }}
-      securityContext:
-        fsGroup: {{ .Values.securityContext.fsGroup }}
-        runAsGroup: {{ .Values.securityContext.runAsGroup }}
-        runAsUser: {{ .Values.securityContext.runAsUser }}
-      {{- end }}
-    {{- if .Values.priorityClassName }}
-      priorityClassName: {{ .Values.priorityClassName }}
-    {{- end }}
-      containers:
-      - name: {{ .Chart.Name }}
-{{- if .Values.autosharding.enabled }}
-        env:
-        - name: POD_NAME
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.name
-        - name: POD_NAMESPACE
-          valueFrom:
-            fieldRef:
-              fieldPath: metadata.namespace
-{{- end }}
-        args:
-{{  if .Values.extraArgs  }}
-        {{- range .Values.extraArgs  }}
-        - {{ . }}
-        {{- end  }}
-{{  end  }}
-{{  if .Values.collectors.certificatesigningrequests  }}
-        - --collectors=certificatesigningrequests
-{{  end  }}
-{{  if .Values.collectors.configmaps  }}
-        - --collectors=configmaps
-{{  end  }}
-{{  if .Values.collectors.cronjobs  }}
-        - --collectors=cronjobs
-{{  end  }}
-{{ if .Values.collectors.daemonsets  }}
-        - --collectors=daemonsets
-{{  end  }}
-{{  if .Values.collectors.deployments  }}
-        - --collectors=deployments
-{{  end  }}
-{{  if .Values.collectors.endpoints  }}
-        - --collectors=endpoints
-{{  end  }}
-{{  if .Values.collectors.horizontalpodautoscalers  }}
-        - --collectors=horizontalpodautoscalers
-{{  end  }}
-{{  if .Values.collectors.ingresses  }}
-        - --collectors=ingresses
-{{  end  }}
-{{  if .Values.collectors.jobs  }}
-        - --collectors=jobs
-{{  end  }}
-{{  if .Values.collectors.limitranges  }}
-        - --collectors=limitranges
-{{  end  }}
-{{  if .Values.collectors.mutatingwebhookconfigurations  }}
-        - --collectors=mutatingwebhookconfigurations
-{{  end  }}
-{{  if .Values.collectors.namespaces  }}
-        - --collectors=namespaces
-{{  end  }}
-{{  if .Values.collectors.networkpolicies  }}
-        - --collectors=networkpolicies
-{{  end  }}
-{{  if .Values.collectors.nodes  }}
-        - --collectors=nodes
-{{  end  }}
-{{  if .Values.collectors.persistentvolumeclaims  }}
-        - --collectors=persistentvolumeclaims
-{{  end  }}
-{{  if .Values.collectors.persistentvolumes  }}
-        - --collectors=persistentvolumes
-{{  end  }}
-{{  if .Values.collectors.poddisruptionbudgets  }}
-        - --collectors=poddisruptionbudgets
-{{  end  }}
-{{  if .Values.collectors.pods  }}
-        - --collectors=pods
-{{  end  }}
-{{  if .Values.collectors.replicasets  }}
-        - --collectors=replicasets
-{{  end  }}
-{{  if .Values.collectors.replicationcontrollers  }}
-        - --collectors=replicationcontrollers
-{{  end  }}
-{{  if .Values.collectors.resourcequotas  }}
-        - --collectors=resourcequotas
-{{  end  }}
-{{  if .Values.collectors.secrets  }}
-        - --collectors=secrets
-{{  end  }}
-{{  if .Values.collectors.services  }}
-        - --collectors=services
-{{  end  }}
-{{  if .Values.collectors.statefulsets  }}
-        - --collectors=statefulsets
-{{  end  }}
-{{  if .Values.collectors.storageclasses  }}
-        - --collectors=storageclasses
-{{  end  }}
-{{  if .Values.collectors.validatingwebhookconfigurations  }}
-        - --collectors=validatingwebhookconfigurations
-{{  end  }}
-{{  if .Values.collectors.verticalpodautoscalers  }}
-        - --collectors=verticalpodautoscalers
-{{  end  }}
-{{  if .Values.collectors.volumeattachments  }}
-        - --collectors=volumeattachments
-{{  end  }}
-{{ if .Values.namespace }}
-        - --namespace={{ .Values.namespace | join "," }}
-{{ end }}
-{{ if .Values.autosharding.enabled }}
-        - --pod=$(POD_NAME)
-        - --pod-namespace=$(POD_NAMESPACE)
-{{ end }}
-{{ if .Values.kubeconfig.enabled }}
-        - --kubeconfig=/opt/k8s/.kube/config
-{{ end }}
-{{ if .Values.selfMonitor.telemetryHost }}
-        - --telemetry-host={{ .Values.selfMonitor.telemetryHost }}
-{{ end }}
-        - --telemetry-port=8081
-{{- if .Values.kubeconfig.enabled }}
-        volumeMounts:
-        - name: kubeconfig
-          mountPath: /opt/k8s/.kube/
-          readOnly: true
-{{- end }}
-        imagePullPolicy: {{ .Values.image.pullPolicy }}
-        image: "{{ .Values.image.repository }}:{{ .Values.image.tag }}"
-        ports:
-        - containerPort: 8080
-        livenessProbe:
-          httpGet:
-            path: /healthz
-            port: 8080
-          initialDelaySeconds: 5
-          timeoutSeconds: 5
-        readinessProbe:
-          httpGet:
-            path: /
-            port: 8080
-          initialDelaySeconds: 5
-          timeoutSeconds: 5
-{{- if .Values.resources }}
-        resources:
-{{ toYaml .Values.resources | indent 10 }}
-{{- end }}
-{{- if .Values.imagePullSecrets }}
-      imagePullSecrets:
-{{ toYaml .Values.imagePullSecrets | indent 8 }}
-{{- end }}
-{{- if .Values.affinity }}
-      affinity:
-{{ toYaml .Values.affinity | indent 8 }}
-{{- end }}
-{{- if .Values.nodeSelector }}
-      nodeSelector:
-{{ toYaml .Values.nodeSelector | indent 8 }}
-{{- end }}
-{{- if .Values.tolerations }}
-      tolerations:
-{{ toYaml .Values.tolerations | indent 8 }}
-{{- end }}
-{{- if .Values.kubeconfig.enabled}}
-      volumes:
-        - name: kubeconfig
-          secret:
-            secretName: {{ template "kube-state-metrics.fullname" . }}-kubeconfig
-{{- end }}

charts/kube-state-metrics/templates/kubeconfig-secret.yaml

@@ -1,15 +0,0 @@
-{{- if .Values.kubeconfig.enabled -}}
-apiVersion: v1
-kind: Secret
-metadata:
-  name: {{ template "kube-state-metrics.fullname" . }}-kubeconfig
-  namespace: {{ template "kube-state-metrics.namespace" . }}
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
-    app.kubernetes.io/instance: "{{ .Release.Name }}"
-    app.kubernetes.io/managed-by: "{{ .Release.Service }}"
-type: Opaque
-data:
-  config: '{{ .Values.kubeconfig.secret }}'
-{{- end -}}

charts/kube-state-metrics/templates/pdb.yaml

@@ -1,20 +0,0 @@
-{{- if .Values.podDisruptionBudget -}}
-apiVersion: policy/v1beta1
-kind: PodDisruptionBudget
-metadata:
-  name: {{ template "kube-state-metrics.fullname" . }}
-  namespace: {{ template "kube-state-metrics.namespace" . }}
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
-    app.kubernetes.io/instance: "{{ .Release.Name }}"
-    app.kubernetes.io/managed-by: "{{ .Release.Service }}"
-{{- if .Values.customLabels }}
-{{ toYaml .Values.customLabels | indent 4 }}
-{{- end }}
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-{{ toYaml .Values.podDisruptionBudget | indent 2 }}
-{{- end -}}

charts/kube-state-metrics/templates/podsecuritypolicy.yaml

@@ -1,42 +0,0 @@
-{{- if .Values.podSecurityPolicy.enabled }}
-apiVersion: policy/v1beta1
-kind: PodSecurityPolicy
-metadata:
-  name: {{ template "kube-state-metrics.fullname" . }}
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-{{- if .Values.podSecurityPolicy.annotations }}
-  annotations:
-{{ toYaml .Values.podSecurityPolicy.annotations | indent 4 }}
-{{- end }}
-spec:
-  privileged: false
-  volumes:
-    - 'secret'
-{{- if .Values.podSecurityPolicy.additionalVolumes }}
-{{ toYaml .Values.podSecurityPolicy.additionalVolumes | indent 4 }}
-{{- end }}
-  hostNetwork: false
-  hostIPC: false
-  hostPID: false
-  runAsUser:
-    rule: 'MustRunAsNonRoot'
-  seLinux:
-    rule: 'RunAsAny'
-  supplementalGroups:
-    rule: 'MustRunAs'
-    ranges:
-      # Forbid adding the root group.
-      - min: 1
-        max: 65535
-  fsGroup:
-    rule: 'MustRunAs'
-    ranges:
-      # Forbid adding the root group.
-      - min: 1
-        max: 65535
-  readOnlyRootFilesystem: false
-{{- end }}

charts/kube-state-metrics/templates/psp-clusterrole.yaml

@@ -1,22 +0,0 @@
-{{- if and .Values.podSecurityPolicy.enabled .Values.rbac.create -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRole
-metadata:
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-  name: psp-{{ template "kube-state-metrics.fullname" . }}
-rules:
-{{- $kubeTargetVersion := default .Capabilities.KubeVersion.GitVersion .Values.kubeTargetVersionOverride }}
-{{- if semverCompare "> 1.15.0-0" $kubeTargetVersion }}
-- apiGroups: ['policy']
-{{- else }}
-- apiGroups: ['extensions']
-{{- end }}
-  resources: ['podsecuritypolicies']
-  verbs:     ['use']
-  resourceNames:
-  - {{ template "kube-state-metrics.fullname" . }}
-{{- end }}

charts/kube-state-metrics/templates/psp-clusterrolebinding.yaml

@@ -1,19 +0,0 @@
-{{- if and .Values.podSecurityPolicy.enabled .Values.rbac.create -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: ClusterRoleBinding
-metadata:
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-  name: psp-{{ template "kube-state-metrics.fullname" . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: ClusterRole
-  name: psp-{{ template "kube-state-metrics.fullname" . }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ template "kube-state-metrics.fullname" . }}
-    namespace: {{ template "kube-state-metrics.namespace" . }}
-{{- end }}

charts/kube-state-metrics/templates/role.yaml

@@ -1,192 +0,0 @@
-{{- if and (eq $.Values.rbac.create true) (not .Values.rbac.useExistingRole)  -}}
-{{- if eq .Values.rbac.useClusterRole false }}
-{{-   range (split "," $.Values.namespace) }}
-{{- end }}
-{{- end -}}
----
-apiVersion: rbac.authorization.k8s.io/v1
-{{- if eq .Values.rbac.useClusterRole false }}
-kind: Role
-{{- else }}
-kind: ClusterRole
-{{- end }}
-metadata:
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" $ }}
-    helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }}
-    app.kubernetes.io/managed-by: {{ $.Release.Service }}
-    app.kubernetes.io/instance: {{ $.Release.Name }}
-  name: {{ template "kube-state-metrics.fullname" $ }}
-{{- if eq .Values.rbac.useClusterRole false }}
-  namespace: {{ . }}
-{{- end }}
-rules:
-{{ if $.Values.collectors.certificatesigningrequests }}
-- apiGroups: ["certificates.k8s.io"]
-  resources:
-  - certificatesigningrequests
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.configmaps }}
-- apiGroups: [""]
-  resources:
-  - configmaps
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.cronjobs }}
-- apiGroups: ["batch"]
-  resources:
-  - cronjobs
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.daemonsets }}
-- apiGroups: ["extensions", "apps"]
-  resources:
-  - daemonsets
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.deployments }}
-- apiGroups: ["extensions", "apps"]
-  resources:
-  - deployments
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.endpoints }}
-- apiGroups: [""]
-  resources:
-  - endpoints
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.horizontalpodautoscalers }}
-- apiGroups: ["autoscaling"]
-  resources:
-  - horizontalpodautoscalers
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.ingresses }}
-- apiGroups: ["extensions", "networking.k8s.io"]
-  resources:
-  - ingresses
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.jobs }}
-- apiGroups: ["batch"]
-  resources:
-  - jobs
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.limitranges }}
-- apiGroups: [""]
-  resources:
-  - limitranges
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.mutatingwebhookconfigurations }}
-- apiGroups: ["admissionregistration.k8s.io"]
-  resources:
-    - mutatingwebhookconfigurations
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.namespaces }}
-- apiGroups: [""]
-  resources:
-  - namespaces
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.networkpolicies }}
-- apiGroups: ["networking.k8s.io"]
-  resources:
-  - networkpolicies
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.nodes }}
-- apiGroups: [""]
-  resources:
-  - nodes
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.persistentvolumeclaims }}
-- apiGroups: [""]
-  resources:
-  - persistentvolumeclaims
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.persistentvolumes }}
-- apiGroups: [""]
-  resources:
-  - persistentvolumes
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.poddisruptionbudgets }}
-- apiGroups: ["policy"]
-  resources:
-    - poddisruptionbudgets
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.pods }}
-- apiGroups: [""]
-  resources:
-  - pods
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.replicasets }}
-- apiGroups: ["extensions", "apps"]
-  resources:
-  - replicasets
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.replicationcontrollers }}
-- apiGroups: [""]
-  resources:
-  - replicationcontrollers
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.resourcequotas }}
-- apiGroups: [""]
-  resources:
-  - resourcequotas
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.secrets }}
-- apiGroups: [""]
-  resources:
-  - secrets
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.services }}
-- apiGroups: [""]
-  resources:
-  - services
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.statefulsets }}
-- apiGroups: ["apps"]
-  resources:
-  - statefulsets
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.storageclasses }}
-- apiGroups: ["storage.k8s.io"]
-  resources:
-    - storageclasses
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.validatingwebhookconfigurations }}
-- apiGroups: ["admissionregistration.k8s.io"]
-  resources:
-    - validatingwebhookconfigurations
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.volumeattachments }}
-- apiGroups: ["storage.k8s.io"]
-  resources:
-    - volumeattachments
-  verbs: ["list", "watch"]
-{{ end -}}
-{{ if $.Values.collectors.verticalpodautoscalers }}
-- apiGroups: ["autoscaling.k8s.io"]
-  resources:
-    - verticalpodautoscalers
-  verbs: ["list", "watch"]
-{{ end -}}
-{{- end -}}

charts/kube-state-metrics/templates/rolebinding.yaml

@@ -1,27 +0,0 @@
-{{- if and (eq  .Values.rbac.create true) (eq .Values.rbac.useClusterRole false) -}}
-{{- range (split "," $.Values.namespace) }}
----
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" $ }}
-    helm.sh/chart: {{ $.Chart.Name }}-{{ $.Chart.Version }}
-    app.kubernetes.io/managed-by: {{ $.Release.Service }}
-    app.kubernetes.io/instance: {{ $.Release.Name }}
-  name: {{ template "kube-state-metrics.fullname" $ }}
-  namespace: {{ . }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-{{- if (not $.Values.rbac.useExistingRole) }}
-  name: {{ template "kube-state-metrics.fullname" $ }}
-{{- else }}
-  name: {{ $.Values.rbac.useExistingRole }}
-{{- end }}
-subjects:
-- kind: ServiceAccount
-  name: {{ template "kube-state-metrics.fullname" $ }}
-  namespace: {{ template "kube-state-metrics.namespace" $ }}
-{{- end -}}
-{{- end -}}

charts/kube-state-metrics/templates/service.yaml

@@ -1,42 +0,0 @@
-apiVersion: v1
-kind: Service
-metadata:
-  name: {{ template "kube-state-metrics.fullname" . }}
-  namespace: {{ template "kube-state-metrics.namespace" . }}
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
-    app.kubernetes.io/instance: "{{ .Release.Name }}"
-    app.kubernetes.io/managed-by: "{{ .Release.Service }}"
-{{- if .Values.customLabels }}
-{{ toYaml .Values.customLabels | indent 4 }}
-{{- end }}
-  annotations:
-    {{- if .Values.prometheusScrape }}
-    prometheus.io/scrape: '{{ .Values.prometheusScrape }}'
-    {{- end }}
-    {{- if .Values.service.annotations }}
-    {{- toYaml .Values.service.annotations | nindent 4 }}
-    {{- end }}
-spec:
-  type: "{{ .Values.service.type }}"
-  ports:
-  - name: "http"
-    protocol: TCP
-    port: {{ .Values.service.port }}
-  {{- if .Values.service.nodePort }}
-    nodePort: {{ .Values.service.nodePort }}
-  {{- end }}
-    targetPort: 8080
-  {{ if .Values.selfMonitor.enabled }}
-  - name: "metrics"
-    protocol: TCP
-    port: {{ .Values.selfMonitor.telemetryPort | default 8081 }}
-    targetPort: 8081
-  {{ end }}
-{{- if .Values.service.loadBalancerIP }}
-  loadBalancerIP: "{{ .Values.service.loadBalancerIP }}"
-{{- end }}
-  selector:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    app.kubernetes.io/instance: {{ .Release.Name }}

charts/kube-state-metrics/templates/serviceaccount.yaml

@@ -1,18 +0,0 @@
-{{- if .Values.serviceAccount.create -}}
-apiVersion: v1
-kind: ServiceAccount
-metadata:
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-  name: {{ template "kube-state-metrics.fullname" . }}
-  namespace: {{ template "kube-state-metrics.namespace" . }}
-{{- if .Values.serviceAccount.annotations }}
-  annotations:
-{{ toYaml .Values.serviceAccount.annotations | indent 4 }}
-{{- end }}
-imagePullSecrets:
-{{ toYaml .Values.serviceAccount.imagePullSecrets | indent 2 }}
-{{- end -}}

charts/kube-state-metrics/templates/servicemonitor.yaml

@@ -1,34 +0,0 @@
-{{- if .Values.prometheus.monitor.enabled }}
-apiVersion: monitoring.coreos.com/v1
-kind: ServiceMonitor
-metadata:
-  name: {{ template "kube-state-metrics.fullname" . }}
-  namespace: {{ template "kube-state-metrics.namespace" . }}
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: "{{ .Chart.Name }}-{{ .Chart.Version }}"
-    app.kubernetes.io/instance: "{{ .Release.Name }}"
-    app.kubernetes.io/managed-by: "{{ .Release.Service }}"
-    {{- if .Values.prometheus.monitor.additionalLabels }}
-{{ toYaml .Values.prometheus.monitor.additionalLabels | indent 4 }}
-    {{- end }}
-{{- if .Values.customLabels }}
-{{ toYaml .Values.customLabels | indent 4 }}
-{{- end }}
-spec:
-  selector:
-    matchLabels:
-      app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-      app.kubernetes.io/instance: {{ .Release.Name }}
-  endpoints:
-    - port: http
-      {{- if .Values.prometheus.monitor.honorLabels }}
-      honorLabels: true
-      {{- end }}
-    {{ if .Values.selfMonitor.enabled }}
-    - port: metrics
-      {{- if .Values.prometheus.monitor.honorLabels }}
-      honorLabels: true
-      {{- end }}
-    {{ end }}
-{{- end }}

charts/kube-state-metrics/templates/stsdiscovery-role.yaml

@@ -1,29 +0,0 @@
-{{- if and .Values.autosharding.enabled .Values.rbac.create -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: Role
-metadata:
-  name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }}
-  namespace: {{ template "kube-state-metrics.namespace" . }}
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-rules:
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - get
-- apiGroups:
-  - apps
-  resourceNames:
-  - {{ template "kube-state-metrics.fullname" . }}
-  resources:
-  - statefulsets
-  verbs:
-  - get
-  - list
-  - watch
-{{- end }}

charts/kube-state-metrics/templates/stsdiscovery-rolebinding.yaml

@@ -1,20 +0,0 @@
-{{- if and .Values.autosharding.enabled .Values.rbac.create -}}
-apiVersion: rbac.authorization.k8s.io/v1
-kind: RoleBinding
-metadata:
-  name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }}
-  namespace: {{ template "kube-state-metrics.namespace" . }}
-  labels:
-    app.kubernetes.io/name: {{ template "kube-state-metrics.name" . }}
-    helm.sh/chart: {{ .Chart.Name }}-{{ .Chart.Version }}
-    app.kubernetes.io/managed-by: {{ .Release.Service }}
-    app.kubernetes.io/instance: {{ .Release.Name }}
-roleRef:
-  apiGroup: rbac.authorization.k8s.io
-  kind: Role
-  name: stsdiscovery-{{ template "kube-state-metrics.fullname" . }}
-subjects:
-  - kind: ServiceAccount
-    name: {{ template "kube-state-metrics.fullname" . }}
-    namespace: {{ template "kube-state-metrics.namespace" . }}
-{{- end }}

charts/kube-state-metrics/values.yaml

@@ -1,179 +0,0 @@
-# Default values for kube-state-metrics.
-prometheusScrape: true
-image:
-  repository: k8s.gcr.io/kube-state-metrics/kube-state-metrics
-  tag: v1.9.8
-  pullPolicy: IfNotPresent
-
-imagePullSecrets: []
-# - name: "image-pull-secret"
-
-# If set to true, this will deploy kube-state-metrics as a StatefulSet and the data
-# will be automatically sharded across <.Values.replicas> pods using the built-in
-# autodiscovery feature: https://github.com/kubernetes/kube-state-metrics#automated-sharding
-# This is an experimental feature and there are no stability guarantees.
-autosharding:
-  enabled: false
-
-replicas: 1
-
-# List of additional cli arguments to configure kube-state-metrics
-# for example: --enable-gzip-encoding, --log-file, etc.
-# all the possible args can be found here: https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md
-extraArgs: []
-
-service:
-  port: 8080
-  # Default to clusterIP for backward compatibility
-  type: ClusterIP
-  nodePort: 0
-  loadBalancerIP: ""
-  annotations: {}
-
-customLabels: {}
-
-hostNetwork: false
-
-rbac:
-  # If true, create & use RBAC resources
-  create: true
-
-  # Set to a rolename to use existing role - skipping role creating - but still doing serviceaccount and rolebinding to it, rolename set here.
-  # useExistingRole: your-existing-role
-
-  # If set to false - Run without Cluteradmin privs needed - ONLY works if namespace is also set (if useExistingRole is set this name is used as ClusterRole or Role to bind to)
-  useClusterRole: true
-
-serviceAccount:
-  # Specifies whether a ServiceAccount should be created, require rbac true
-  create: true
-  # The name of the ServiceAccount to use.
-  # If not set and create is true, a name is generated using the fullname template
-  name:
-  # Reference to one or more secrets to be used when pulling images
-  # ref: https://kubernetes.io/docs/tasks/configure-pod-container/pull-image-private-registry/
-  imagePullSecrets: []
-  # ServiceAccount annotations.
-  # Use case: AWS EKS IAM roles for service accounts
-  # ref: https://docs.aws.amazon.com/eks/latest/userguide/specify-service-account-role.html
-  annotations: {}
-
-prometheus:
-  monitor:
-    enabled: false
-    additionalLabels: {}
-    namespace: ""
-    honorLabels: false
-
-## Specify if a Pod Security Policy for kube-state-metrics must be created
-## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/
-##
-podSecurityPolicy:
-  enabled: false
-  annotations: {}
-    ## Specify pod annotations
-    ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#apparmor
-    ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#seccomp
-    ## Ref: https://kubernetes.io/docs/concepts/policy/pod-security-policy/#sysctl
-    ##
-    # seccomp.security.alpha.kubernetes.io/allowedProfileNames: '*'
-    # seccomp.security.alpha.kubernetes.io/defaultProfileName: 'docker/default'
-    # apparmor.security.beta.kubernetes.io/defaultProfileName: 'runtime/default'
-
-  additionalVolumes: []
-
-securityContext:
-  enabled: true
-  runAsGroup: 65534
-  runAsUser: 65534
-  fsGroup: 65534
-
-## Node labels for pod assignment
-## Ref: https://kubernetes.io/docs/user-guide/node-selection/
-nodeSelector: {}
-
-## Affinity settings for pod assignment
-## Ref: https://kubernetes.io/docs/concepts/configuration/assign-pod-node/
-affinity: {}
-
-## Tolerations for pod assignment
-## Ref: https://kubernetes.io/docs/concepts/configuration/taint-and-toleration/
-tolerations: []
-
-# Annotations to be added to the pod
-podAnnotations: {}
-
-## Assign a PriorityClassName to pods if set
-# priorityClassName: ""
-
-# Ref: https://kubernetes.io/docs/tasks/run-application/configure-pdb/
-podDisruptionBudget: {}
-
-# Available collectors for kube-state-metrics. By default all available
-# collectors are enabled.
-collectors:
-  certificatesigningrequests: true
-  configmaps: true
-  cronjobs: true
-  daemonsets: true
-  deployments: true
-  endpoints: true
-  horizontalpodautoscalers: true
-  ingresses: true
-  jobs: true
-  limitranges: true
-  mutatingwebhookconfigurations: true
-  namespaces: true
-  networkpolicies: true
-  nodes: true
-  persistentvolumeclaims: true
-  persistentvolumes: true
-  poddisruptionbudgets: true
-  pods: true
-  replicasets: true
-  replicationcontrollers: true
-  resourcequotas: true
-  secrets: true
-  services: true
-  statefulsets: true
-  storageclasses: true
-  validatingwebhookconfigurations: true
-  verticalpodautoscalers: false
-  volumeattachments: true
-
-# Enabling kubeconfig will pass the --kubeconfig argument to the container
-kubeconfig:
-  enabled: false
-  # base64 encoded kube-config file
-  secret:
-
-# Namespace to be enabled for collecting resources. By default all namespaces are collected.
-# namespace: ""
-
-## Override the deployment namespace
-##
-namespaceOverride: ""
-
-resources: {}
-  # We usually recommend not to specify default resources and to leave this as a conscious
-  # choice for the user. This also increases chances charts run on environments with little
-  # resources, such as Minikube. If you do want to specify resources, uncomment the following
-  # lines, adjust them as necessary, and remove the curly braces after 'resources:'.
-  # limits:
-  #  cpu: 100m
-  #  memory: 64Mi
-  # requests:
-  #  cpu: 10m
-  #  memory: 32Mi
-
-## Provide a k8s version to define apiGroups for podSecurityPolicy Cluster Role.
-## For example: kubeTargetVersionOverride: 1.14.9
-##
-kubeTargetVersionOverride: ""
-
-# Enable self metrics configuration for service and Service Monitor
-# Default values for telemetry configuration can be overriden
-selfMonitor:
-  enabled: false
-  # telemetryHost: 0.0.0.0
-  # telemetryPort: 8081

cloudbuild.yaml

@@ -1,9 +1,9 @@
 # See https://cloud.google.com/cloud-build/docs/build-config
-timeout: 1200s
+timeout: 2700s
 options:
   substitution_option: ALLOW_LOOSE
 steps:
-  - name: 'gcr.io/k8s-testimages/gcb-docker-gcloud:v20190906-745fed4'
+  - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20241111-71c32dbdcc'
     entrypoint: make
     env:
     - TAG=$_PULL_BASE_REF

data.yaml

@@ -0,0 +1,19 @@
+# The purpose of this config is to keep all versions in a single file and make them machine accessible
+
+# Marks the latest release
+version: "2.16.0"
+
+# List at max 5 releases here + the main branch
+compat:
+  - version: "v2.12.0"
+    kubernetes: "1.29"
+  - version: "v2.13.0"
+    kubernetes: "1.30"
+  - version: "v2.14.0"
+    kubernetes: "1.31"
+  - version: "v2.15.0"
+    kubernetes: "1.32"
+  - version: "v2.16.0"
+    kubernetes: "1.32"
+  - version: "main"
+    kubernetes: "1.33"

docs/README.md

@@ -6,10 +6,10 @@ Any contribution to improving this documentation or adding sample usages will be
 
 ## Table of Contents
 
-- [Metrics Stages](#metrics-stages)
-- [Exposed Metrics](#exposed-metrics)
-- [Join Metrics](#join-metrics)
-- [CLI arguments](#cli-arguments)
+* [Metrics Stages](#metrics-stages)
+* [Exposed Metrics](#exposed-metrics)
+* [Join Metrics](#join-metrics)
+* [CLI arguments](#cli-arguments)
 
 ## Metrics Stages
 
@@ -21,39 +21,55 @@ Stages about metrics are grouped into three categories：
 | STABLE       | Metrics which should have very few backwards-incompatible changes outside of major version updates.                        |
 | DEPRECATED   | Metrics which will be removed once the deprecation timeline is met.                                                        |
 
+## Opt-in Metrics
+
+As of v2.3.0, kube-state-metrics supports additional opt-in metrics via the CLI flag `--metric-opt-in-list`. See the metric documentation to identify which metrics need to be specified.
+
 ## Exposed Metrics
 
-Per group of metrics there is one file for each metrics. See each file for specific documentation about the exposed metrics:
+Per group of metrics there is one file for each metrics.
+See each file for specific documentation about the exposed metrics:
 
-- [CertificateSigningRequest Metrics](certificatessigningrequest-metrics.md)
-- [ConfigMap Metrics](configmap-metrics.md)
-- [CronJob Metrics](cronjob-metrics.md)
-- [DaemonSet Metrics](daemonset-metrics.md)
-- [Deployment Metrics](deployment-metrics.md)
-- [Endpoint Metrics](endpoint-metrics.md)
-- [Horizontal Pod Autoscaler Metrics](horizontalpodautoscaler-metrics.md)
-- [Ingress Metrics](ingress-metrics.md)
-- [Job Metrics](job-metrics.md)
-- [Lease Metrics](lease-metrics.md)
-- [LimitRange Metrics](limitrange-metrics.md)
-- [MutatingWebhookConfiguration Metrics](mutatingwebhookconfiguration-metrics.md)
-- [Namespace Metrics](namespace-metrics.md)
-- [NetworkPolicy Metrics](networkpolicy-metrics.md)
-- [Node Metrics](node-metrics.md)
-- [PersistentVolume Metrics](persistentvolume-metrics.md)
-- [PersistentVolumeClaim Metrics](persistentvolumeclaim-metrics.md)
-- [Pod Disruption Budget Metrics](poddisruptionbudget-metrics.md)
-- [Pod Metrics](pod-metrics.md)
-- [ReplicaSet Metrics](replicaset-metrics.md)
-- [ReplicationController Metrics](replicationcontroller-metrics.md)
-- [ResourceQuota Metrics](resourcequota-metrics.md)
-- [Secret Metrics](secret-metrics.md)
-- [Service Metrics](service-metrics.md)
-- [StatefulSet Metrics](statefulset-metrics.md)
-- [StorageClass Metrics](storageclass-metrics.md)
-- [ValidatingWebhookConfiguration Metrics](validatingwebhookconfiguration-metrics.md)
-- [VerticalPodAutoscaler Metrics](verticalpodautoscaler-metrics.md)
-- [VolumeAttachment Metrics](volumeattachment-metrics.md)
+### Default Resources
+
+* [CertificateSigningRequest Metrics](metrics/auth/certificatesigningrequest-metrics.md)
+* [ConfigMap Metrics](metrics/storage/configmap-metrics.md)
+* [CronJob Metrics](metrics/workload/cronjob-metrics.md)
+* [DaemonSet Metrics](metrics/workload/daemonset-metrics.md)
+* [Deployment Metrics](metrics/workload/deployment-metrics.md)
+* [Endpoint Metrics](metrics/service/endpoint-metrics.md)
+* [Horizontal Pod Autoscaler Metrics](metrics/workload/horizontalpodautoscaler-metrics.md)
+* [Ingress Metrics](metrics/service/ingress-metrics.md)
+* [Job Metrics](metrics/workload/job-metrics.md)
+* [Lease Metrics](metrics/cluster/lease-metrics.md)
+* [LimitRange Metrics](metrics/policy/limitrange-metrics.md)
+* [MutatingWebhookConfiguration Metrics](metrics/extend/mutatingwebhookconfiguration-metrics.md)
+* [Namespace Metrics](metrics/cluster/namespace-metrics.md)
+* [NetworkPolicy Metrics](metrics/policy/networkpolicy-metrics.md)
+* [Node Metrics](metrics/cluster/node-metrics.md)
+* [PersistentVolume Metrics](metrics/storage/persistentvolume-metrics.md)
+* [PersistentVolumeClaim Metrics](metrics/storage/persistentvolumeclaim-metrics.md)
+* [Pod Disruption Budget Metrics](metrics/policy/poddisruptionbudget-metrics.md)
+* [Pod Metrics](metrics/workload/pod-metrics.md)
+* [ReplicaSet Metrics](metrics/workload/replicaset-metrics.md)
+* [ReplicationController Metrics](metrics/workload/replicationcontroller-metrics.md)
+* [ResourceQuota Metrics](metrics/policy/resourcequota-metrics.md)
+* [Secret Metrics](metrics/storage/secret-metrics.md)
+* [Service Metrics](metrics/service/service-metrics.md)
+* [StatefulSet Metrics](metrics/workload/statefulset-metrics.md)
+* [StorageClass Metrics](metrics/storage/storageclass-metrics.md)
+* [ValidatingWebhookConfiguration Metrics](metrics/extend/validatingwebhookconfiguration-metrics.md)
+* [VolumeAttachment Metrics](metrics/storage/volumeattachment-metrics.md)
+
+### Optional Resources
+
+* [ClusterRole Metrics](metrics/cluster/clusterrole-metrics.md)
+* [ClusterRoleBinding Metrics](metrics/cluster/clusterrolebinding-metrics.md)
+* [EndpointSlice Metrics](metrics/service/endpointslice-metrics.md)
+* [IngressClass Metrics](metrics/service/ingressclass-metrics.md)
+* [Role Metrics](metrics/auth/role-metrics.md)
+* [RoleBinding Metrics](metrics/auth/rolebinding-metrics.md)
+* [ServiceAccount Metrics](metrics/auth/serviceaccount-metrics.md)
 
 ## Join Metrics
 
@@ -63,17 +79,67 @@ can be used to extend single metrics output.
 This example adds `label_release` to the set of default labels of the `kube_pod_status_ready` metric
 and allows you select or group the metrics by Helm release label:
 
-```
+```promql
 kube_pod_status_ready * on (namespace, pod) group_left(label_release) kube_pod_labels
 ```
 
 Another useful example would be to query the memory usage of pods by its `phase`, such as `Running`:
 
-```
+```promql
 sum(kube_pod_container_resource_requests{resource="memory"}) by (namespace, pod, node)
   * on (namespace, pod) group_left() (sum(kube_pod_status_phase{phase="Running"}) by (pod, namespace) == 1)
 ```
 
+## Metrics from Custom Resources
+
+See [Custom Resource State Metrics](metrics/extend/customresourcestate-metrics.md) for experimental support for custom resources.
+
 ## CLI Arguments
 
-Additionally, options for `kube-state-metrics` can be passed when executing as a CLI, or in a kubernetes / openshift environment. More information can be found here: [CLI Arguments](cli-arguments.md)
+Additionally, options for `kube-state-metrics` can be passed when executing as a CLI, or in a kubernetes / openshift environment. More information can be found here: [CLI Arguments](developer/cli-arguments.md)
+
+## Protecting /metrics endpoints
+
+Kube-State-Metrics' metrics can contain sensitive information about the state of the cluster, which you as an operator might want to additionally protect from unauthorized access.
+In order to achieve this, you need to enable the `--auth-filter` flag on kube-state-metrics.
+With this, kube-state-metrics will only accept authenticated and authorized requests to the /metrics endpoints.
+Kube-state-metrics uses Kubernetes' RBAC mechanisms for this, so this means that every scrape will trigger a request against the API Server for TokenReview and SubjectAccessReview.
+The clients scraping the endpoint, need to use a token which can be provided by a ServiceAccount that can be set up the following way:
+
+A ClusterRole providing access like this:
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-reader
+rules:
+- nonResourceURLs:
+  - "/metrics"
+  verbs:
+  - get
+```
+
+and a matching ClusterRoleBinding
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-reader-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: YOUR_SERVICE_ACCOUNT
+  namespace: NAMESPACE_OF_THE_SERVICE_ACCOUNT
+```
+
+Your client can then use either this ServiceAccount to gather metrics or you can create a token, that can be used to fetch data like this:
+
+```bash
+TOKEN=$(kubectl create token YOUR_SERVICE_ACCOUNT -n NAMESPACE_OF_THE_SERVICE_ACCOUNT)
+curl -H "Authorization: Bearer $TOKEN" KUBE_STATE_METRICS_URL:8080/metrics
+```

docs/certificatessigningrequest-metrics.md

@@ -1,8 +0,0 @@
-# CertificateSigningRequest Metrics
-
-| Metric name| Metric type | Labels/tags | Status |
-| ---------- | ----------- | ----------- | ----------- |
-| kube_certificatesigningrequest_created| Gauge | `certificatesigningrequest`=<certificatesigningrequest-name>| STABLE |
-| kube_certificatesigningrequest_condition | Gauge | `certificatesigningrequest`=&lt;certificatesigningrequest-name&gt; <br> `condition`=&lt;approved\|denied&gt; | STABLE |
-| kube_certificatesigningrequest_labels | Gauge | `certificatesigningrequest`=&lt;certificatesigningrequest-name&gt;| STABLE |
-| kube_certificatesigningrequest_cert_length | Gauge | `certificatesigningrequest`=&lt;certificatesigningrequest-name&gt;| STABLE |

docs/cli-arguments.md

@@ -1,60 +0,0 @@
-# Command line arguments
-
-kube-state-metrics can be configured through command line arguments.
-
-Those arguments can be passed during startup when running locally:
-
-`kube-state-metrics --telemetry-port=8081 --kubeconfig=<KUBE-CONFIG> --apiserver=<APISERVER> ...`
-
-Or configured in the `args` section of your deployment configuration in a Kubernetes / Openshift context:
-
-```yaml
-spec:
-  template:
-    spec:
-      containers:
-        - args:
-          - '--telemetry-port=8081'
-          - '--kubeconfig=<KUBE-CONFIG>'
-          - '--apiserver=<APISERVER>'
-```
-
-## Available options:
-
-[embedmd]:# (../help.txt)
-```txt
-$ kube-state-metrics -h
-Usage of ./kube-state-metrics:
-      --add_dir_header                   If true, adds the file directory to the header of the log messages
-      --alsologtostderr                  log to standard error as well as files
-      --apiserver string                 The URL of the apiserver to use as a master
-      --enable-gzip-encoding             Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.
-  -h, --help                             Print Help text
-      --host string                      Host to expose metrics on. (default "::")
-      --kubeconfig string                Absolute path to the kubeconfig file
-      --log_backtrace_at traceLocation   when logging hits line file:N, emit a stack trace (default :0)
-      --log_dir string                   If non-empty, write log files in this directory
-      --log_file string                  If non-empty, use this log file
-      --log_file_max_size uint           Defines the maximum size a log file can grow to. Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
-      --logtostderr                      log to standard error instead of files (default true)
-      --metric-allowlist string          Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.
-      --metric-denylist string           Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.
-      --metric-labels-allowlist string   Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the metric contains only name and namespace labels. To include additional labels provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'
-      --namespaces string                Comma-separated list of namespaces to be enabled. Defaults to ""
-      --one_output                       If true, only write logs to their native severity level (vs also writing to each lower severity level)
-      --pod string                       Name of the pod that contains the kube-state-metrics container. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
-      --pod-namespace string             Name of the namespace of the pod specified by --pod. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
-      --port int                         Port to expose metrics on. (default 8080)
-      --resources string                 Comma-separated list of Resources to be enabled. Defaults to "certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments"
-      --shard int32                      The instances shard nominal (zero indexed) within the total number of shards. (default 0)
-      --skip_headers                     If true, avoid header prefixes in the log messages
-      --skip_log_headers                 If true, avoid headers when opening log files
-      --stderrthreshold severity         logs at or above this threshold go to stderr (default 2)
-      --telemetry-host string            Host to expose kube-state-metrics self metrics on. (default "::")
-      --telemetry-port int               Port to expose kube-state-metrics self metrics on. (default 8081)
-      --tls-config string                Path to the TLS configuration file
-      --total-shards int                 The total number of shards. Sharding is disabled when total shards is set to 1. (default 1)
-  -v, --v Level                          number for the log level verbosity
-      --version                          kube-state-metrics build version information
-      --vmodule moduleSpec               comma-separated list of pattern=N settings for file-filtered logging
-```

docs/configmap-metrics.md

@@ -1,7 +0,0 @@
-# ConfigMap Metrics
-
-| Metric name| Metric type | Labels/tags | Status |
-| ---------- | ----------- | ----------- | ----------- |
-| kube_configmap_info | Gauge | `configmap`=&lt;configmap-name&gt; <br> `namespace`=&lt;configmap-namespace&gt; | STABLE |
-| kube_configmap_created  | Gauge | `configmap`=&lt;configmap-name&gt; <br> `namespace`=&lt;configmap-namespace&gt; | STABLE |
-| kube_configmap_metadata_resource_version | Gauge | `configmap`=&lt;configmap-name&gt; <br> `namespace`=&lt;configmap-namespace&gt; | EXPERIMENTAL |

docs/cronjob-metrics.md

@@ -1,12 +0,0 @@
-# CronJob Metrics
-
-| Metric name| Metric type | Labels/tags | Status |
-| ---------- | ----------- | ----------- | ----------- |
-| kube_cronjob_info | Gauge | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `schedule`=&lt;schedule&gt; <br> `concurrency_policy`=&lt;concurrency-policy&gt; | STABLE
-| kube_cronjob_labels | Gauge | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `label_CRONJOB_LABEL`=&lt;CRONJOB_LABEL&gt;  | STABLE
-| kube_cronjob_created  | Gauge | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; | STABLE
-| kube_cronjob_next_schedule_time  | Gauge | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; | STABLE
-| kube_cronjob_status_active | Gauge | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; | STABLE
-| kube_cronjob_status_last_schedule_time | Gauge | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; | STABLE
-| kube_cronjob_spec_suspend | Gauge | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; | STABLE
-| kube_cronjob_spec_starting_deadline_seconds | Gauge | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; | STABLE

docs/daemonset-metrics.md

@@ -1,15 +0,0 @@
-# DaemonSet Metrics
-
-| Metric name| Metric type | Labels/tags | Status |
-| ---------- | ----------- | ----------- | ----------- |
-| kube_daemonset_created | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; | STABLE |
-| kube_daemonset_status_current_number_scheduled | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; | STABLE |
-| kube_daemonset_status_desired_number_scheduled | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; | STABLE |
-| kube_daemonset_status_number_available | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; | STABLE |
-| kube_daemonset_status_number_misscheduled | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; | STABLE |
-| kube_daemonset_status_number_ready | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; | STABLE |
-| kube_daemonset_status_number_unavailable | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; | STABLE |
-| kube_daemonset_status_observed_generation | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; | STABLE |
-| kube_daemonset_status_updated_number_scheduled | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; | STABLE |
-| kube_daemonset_metadata_generation | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; | STABLE |
-| kube_daemonset_labels | Gauge | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; <br> `label_DAEMONSET_LABEL`=&lt;DAEMONSET_LABEL&gt; | STABLE |

docs/dependencies-policy.md

@@ -0,0 +1,43 @@
+# Dependencies Policy
+
+## Purpose
+
+This policy describes how kube-state-metrics maintainers consume third-party packages.
+
+## Scope
+
+This policy applies to all kube-state-metrics maintainers and all third-party packages used in the kube-state-metrics project.
+
+## Policy
+
+kube-state-metrics maintainers must follow these guidelines when consuming third-party packages:
+
+* Only use third-party packages that are necessary for the functionality of kube-state-metrics.
+* Use the latest version of all third-party packages whenever possible.
+* Avoid using third-party packages that are known to have security vulnerabilities.
+* Pin all third-party packages to specific versions in the kube-state-metrics codebase.
+* Use a dependency management tool, such as Go modules, to manage third-party dependencies.
+
+## Procedure
+
+When adding a new third-party package to kube-state-metrics, maintainers must follow these steps:
+
+1. Evaluate the need for the package. Is it necessary for the functionality of kube-state-metrics?
+2. Research the package. Is it actively maintained? Does it have a good reputation?
+3. Choose a version of the package. Use the latest version whenever possible.
+4. Pin the package to the specific version in the kube-state-metrics codebase.
+5. Update the kube-state-metrics documentation to reflect the new dependency.
+
+## Enforcement
+
+This policy is enforced by the kube-state-metrics maintainers.
+
+Maintainers are expected to review each other's code changes to ensure that they comply with this policy.
+
+## Exceptions
+
+Exceptions to this policy may be granted by the kube-state-metrics project owners on a case-by-case basis.
+
+## Credits
+
+This policy was adapted from Kubescape's [Environment Dependencies Policy](https://github.com/kubescape/kubescape/blob/master/docs/environment-dependencies-policy.md).

docs/deployment-metrics.md

@@ -1,17 +0,0 @@
-# Deployment Metrics
-
-| Metric name| Metric type | Labels/tags | Status |
-| ---------- | ----------- | ----------- | ----------- |
-| kube_deployment_status_replicas | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |
-| kube_deployment_status_replicas_available | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |
-| kube_deployment_status_replicas_unavailable | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |
-| kube_deployment_status_replicas_updated | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |
-| kube_deployment_status_observed_generation | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |
-| kube_deployment_status_condition | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; <br> `condition`=&lt;deployment-condition&gt; <br> `status`=&lt;true\|false\|unknown&gt; | STABLE |
-| kube_deployment_spec_replicas | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |
-| kube_deployment_spec_paused | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |
-| kube_deployment_spec_strategy_rollingupdate_max_unavailable | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |
-| kube_deployment_spec_strategy_rollingupdate_max_surge | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |
-| kube_deployment_metadata_generation | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |
-| kube_deployment_labels | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; <br> `label_DEPLOYMENT_LABEL`=&lt;DEPLOYMENT_LABEL&gt; | STABLE |
-| kube_deployment_created | Gauge | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; | STABLE |

docs/design/metrics-best-practices.md

@@ -0,0 +1,72 @@
+# Kube-State-Metrics - Timeseries best practices
+
+---
+
+Author: Manuel Rüger (<manuel@rueg.eu>)
+
+Date: October 17th 2024
+
+---
+
+## Introduction
+
+Kube-State-Metrics' goal is to provide insights into the state of Kubernetes objects by exposing them as metrics.
+This document provides guidelines with the goal to create a good user experience when using these metrics.
+
+Please be aware that this document is introduced in a later stage of the project and there might be metrics that do not follow these best practices.
+Feel encouraged to report these metrics and provide a pull request to improve them.
+
+## General best practices
+
+We follow [Prometheus](https://prometheus.io/docs/practices/naming/) best practices in terms of naming and labeling.
+
+## Best practices for kube-state-metrics
+
+### Avoid pre-computation
+
+kube-state-metrics should expose metrics on an individual object level and avoid any sort of pre-computation unless it is required due to for example high cardinality on objects.
+We prefer not to add metrics that can be derived from existing raw metrics. For example, we would not want to expose a metric called `kube_pod_total` as it can be computed with `count(kube_pod_info)`.
+This way kube-state-metrics allows the user to have full control on how they want to use the metrics and gives them flexibility to do specific computation.
+
+### Static object properties
+
+An object usually has a stable set of properties that do not change during its lifecycle in Kubernetes.
+This includes properties like name, namespace, uid etc. that have a 1:1 relationship with the object.
+It is a good practice to group those together into an `_info` metric.
+If there is a 1:n relationship (e.g. a list of ports), it should be in a separate metric to avoid generating too many metrics.
+
+### Dynamic object properties
+
+An object can also have a dynamic set of properties, which are usually part of the status field.
+These change during the lifecycle of the object.
+For example a pod can be in different states like "Pending", "Running" etc.
+These should be part of a "State Set" that includes labels that identify the object as well as the dynamic property.
+
+### Linked properties
+
+If an object contains a substructure that links multiple properties together (e.g. endpoint address and port), those should be reported in the same metric.
+
+### Optional properties
+
+Some Kubernetes objects have optional fields. In case there is an optional value, the label should still be exposed, ideally as an empty string.
+
+### Timestamps
+
+Timestamps like creation time or modification time should be exposed as a value. The metric should end with `_timestamp_seconds`. The date value is represented in [UNIX epoch seconds](https://en.wikipedia.org/wiki/Unix_time).
+
+### Cardinality
+
+Some object properties can cause cardinality issues if they can contain a lot of different values or are linked together with multiple properties that also can change a lot.
+In this case it is better to limit the number of values that can be exposed within kube-state-metrics by allowing only a few of them and have a default for others.
+If for example the Kubernetes object contains a status field that contains an error message that can change a lot, it would be better to have a boolean `error="true"` label in case there is an error.
+If there are some error messages that are worth exposing, those could be exposed and for any other message, a default value could be provided.
+
+## Stability
+
+We follow the stability framework derived from Kubernetes, in which we expose experimental and stable metrics.
+Experimental metrics are recently introduced or expose alpha/beta resources in the Kubernetes API.
+They can change anytime and should be used with caution.
+They can be promoted to a stable metric once the object stabilized in the Kubernetes API or they were part of two consecutive releases and haven't observed any changes in them.
+
+Stable metrics are considered frozen with the exception of new labels being added.
+A stable metric or a label on a stable metric can be deprecated in release Major.Minor and the earliest point it will be removed is the release Major.Minor+2.

docs/design/metrics-store-performance-optimization.md

@@ -1,9 +1,8 @@
 # Kube-State-Metrics - Performance Optimization Proposal
 
-
 ---
 
-Author: Max Inden (IndenML@gmail.com)
+Author: Max Inden (<IndenML@gmail.com>)
 
 Date: 23. July 2018
 
@@ -11,16 +10,14 @@ Target release: v1.5.0
 
 ---
 
-
 ## Glossary
 
-- kube-state-metrics: “Simple service that listens to the Kubernetes API server
+* kube-state-metrics: “Simple service that listens to the Kubernetes API server
   and generates metrics about the state of the objects”
 
-- Time series: A single line in a /metrics response e.g.
+* Time series: A single line in a /metrics response e.g.
   “metric_name{label="value"} 1”
 
-
 ## Problem Statement
 
 There has been repeated reports of two issues running kube-state-metrics on
@@ -30,7 +27,6 @@ instances dropping the scrape interval request and marking the given time series
 as stale. Second kube-state-metrics uses a lot of memory and thereby being
 out-of-memory killed due to low set Kubernetes resource limits.
 
-
 ## Goal
 
 The goal of this proposal can be split into the following sub-goals ordered by
@@ -40,7 +36,6 @@ their priority:
 
 2. Decrease overall runtime memory usage
 
-
 ## Status Quo
 
 Instead of requesting the needed information from the Kubernetes API-Server on
@@ -56,7 +51,6 @@ On a scrape request by e.g. Prometheus on the /metrics endpoint
 kube-state-metrics calculates the configured time series on demand based on the
 objects in its cache and converts them to the Prometheus string representation.
 
-
 ## Proposal
 
 Instead of a full representation of all Kubernetes objects with all its
@@ -64,7 +58,7 @@ properties in memory via the Kubernetes client-go cache, use a map, addressable
 by the Kubernetes object uuid, containing all time series of that object as a
 single multi-line string.
 
-```
+```go
 var cache = map[uuid][]byte{}
 ```
 
@@ -82,7 +76,7 @@ On a scrape request on the /metrics endpoint, kube-state-metrics iterates over
 the cache map and concatenates all time series string blobs into a single
 string, which is finally passed on as a response.
 
-```
+```text
        +---------------+ +-----------+                 +---------------+         +-------------------+
        | pod_reflector | | pod_store |                 | pod_collector |         | metrics_endpoint  |
        +---------------+ +-----------+                 +---------------+         +-------------------+
@@ -122,7 +116,7 @@ string, which is finally passed on as a response.
 
 Build via [text-diagram](http://weidagang.github.io/text-diagram/)
 
-```
+```text
 object pod_reflector pod_store pod_collector metrics_endpoint
 
 note left of pod_reflector: new pod p1
@@ -139,10 +133,9 @@ pod_collector -> metrics_endpoint: concat(metrics)
 
 </details>
 
-
 ## FAQ / Follow up improvements
 
-- If kube-state-metrics only listens on add, update and delete events, how is it
+* If kube-state-metrics only listens on add, update and delete events, how is it
   aware of already existing Kubernetes objects created before kube-state-metrics
   was started? Leveraging Kubernetes client-go, reflectors can initialize all
   existing objects before any add, update or delete events. To ensure no events
@@ -151,27 +144,25 @@ pod_collector -> metrics_endpoint: concat(metrics)
   costs, as Kubernetes client-go already gives decent guarantees on event
   delivery.
 
-- What about metadata (HELP and description) in the /metrics output? As a first
+* What about metadata (HELP and description) in the /metrics output? As a first
   iteration they would be skipped until we have a better idea on the design.
 
-- How can the cache map be concurrently accessed? The core golang map
+* How can the cache map be concurrently accessed? The core golang map
   implementation is not thread-safe. As a first iteration a simple mutex should
   be sufficient. Golang's sync.Map might be considered.
 
-- To solve the problem of out of order events send by the Kubernetes API-Server
+* To solve the problem of out of order events send by the Kubernetes API-Server
   to kube-state-metrics, to each blob of time series inside the cache map it can
   keep the Kubernetes resource version. On add and update events, first compare
   the resource version of the event with than the resource version in the cache.
   Only move forward if the former is higher than the latter.
 
-- In case the memory consumption of the time series string blobs is a problem
+* In case the memory consumption of the time series string blobs is a problem
   the following optimization can be considered: Among the time series strings,
   multiple sub-strings will be heavily duplicated like the metric name. Instead
   of saving unstructured strings inside the cache map, one can structure them,
   using pointers to deduplicate e.g. metric names.
 
-- ...
-
-- Kube-state-metrics does not make use of all properties of all Kubernetes
+* Kube-state-metrics does not make use of all properties of all Kubernetes
   objects. Instead of unmarshalling unused properties, their json struct tags or
   their Protobuf representation could be removed.

docs/developer/cli-arguments.md

@@ -0,0 +1,94 @@
+# Command line arguments
+
+kube-state-metrics can be configured through command line arguments.
+
+Those arguments can be passed during startup when running locally:
+
+`kube-state-metrics --telemetry-port=8081 --kubeconfig=<KUBE-CONFIG> --apiserver=<APISERVER> ...`
+
+Or configured in the `args` section of your deployment configuration in a Kubernetes / Openshift context:
+
+```yaml
+spec:
+  template:
+    spec:
+      containers:
+        - args:
+          - '--telemetry-port=8081'
+          - '--kubeconfig=<KUBE-CONFIG>'
+          - '--apiserver=<APISERVER>'
+```
+
+## Available options
+
+<!-- markdownlint-disable blanks-around-fences -->
+<!-- markdownlint-disable link-image-reference-definitions -->
+[embedmd]:# (../../help.txt)
+```txt
+$ kube-state-metrics -h
+kube-state-metrics is a simple service that listens to the Kubernetes API server and generates metrics about the state of the objects.
+
+Usage:
+  kube-state-metrics [flags]
+  kube-state-metrics [command]
+
+Available Commands:
+  completion  Generate completion script for kube-state-metrics.
+  help        Help about any command
+  version     Print version information.
+
+Flags:
+      --add_dir_header                             If true, adds the file directory to the header of the log messages
+      --alsologtostderr                            log to standard error as well as files (no effect when -logtostderr=true)
+      --apiserver string                           The URL of the apiserver to use as a master
+      --auth-filter                                If true, requires authentication and authorization through Kubernetes API to access metrics endpoints
+      --auto-gomemlimit                            Automatically set GOMEMLIMIT to match container or system memory limit. (experimental)
+      --auto-gomemlimit-ratio float                The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory. (experimental) (default 0.9)
+      --config string                              Path to the kube-state-metrics options config YAML file. If this flag is set, the flags defined in the file override the command line flags.
+      --custom-resource-state-config string        Inline Custom Resource State Metrics config YAML (experimental)
+      --custom-resource-state-config-file string   Path to a Custom Resource State Metrics config file (experimental)
+      --custom-resource-state-only                 Only provide Custom Resource State metrics (experimental)
+      --enable-gzip-encoding                       Gzip responses when requested by clients via 'Accept-Encoding: gzip' header.
+  -h, --help                                       Print Help text
+      --host string                                Host to expose metrics on. (default "::")
+      --kubeconfig string                          Absolute path to the kubeconfig file
+      --log_backtrace_at traceLocation             when logging hits line file:N, emit a stack trace (default :0)
+      --log_dir string                             If non-empty, write log files in this directory (no effect when -logtostderr=true)
+      --log_file string                            If non-empty, use this log file (no effect when -logtostderr=true)
+      --log_file_max_size uint                     Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
+      --logtostderr                                log to standard error instead of files (default true)
+      --metric-allowlist string                    Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.
+      --metric-annotations-allowlist string        Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').
+      --metric-denylist string                     Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.
+      --metric-labels-allowlist string             Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.
+      --metric-opt-in-list string                  Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists
+      --namespaces string                          Comma-separated list of namespaces to be enabled. Defaults to ""
+      --namespaces-denylist string                 Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.
+      --node string                                Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.
+      --object-limit int                           The total number of objects to list per resource from the API Server. (experimental)
+      --one_output                                 If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true)
+      --pod string                                 Name of the pod that contains the kube-state-metrics container. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
+      --pod-namespace string                       Name of the namespace of the pod specified by --pod. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
+      --port int                                   Port to expose metrics on. (default 8080)
+      --resources string                           Comma-separated list of Resources to be enabled. Defaults to "certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,pods,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments"
+      --server-idle-timeout duration               The maximum amount of time to wait for the next request when keep-alives are enabled. Align with the idletimeout of your scrape clients. (default 5m0s)
+      --server-read-header-timeout duration        The maximum duration for reading the header of requests. (default 5s)
+      --server-read-timeout duration               The maximum duration for reading the entire request, including the body. Align with the scrape interval or timeout of scraping clients.  (default 1m0s)
+      --server-write-timeout duration              The maximum duration before timing out writes of the response. Align with the scrape interval or timeout of scraping clients.. (default 1m0s)
+      --shard int32                                The instances shard nominal (zero indexed) within the total number of shards. (default 0)
+      --skip_headers                               If true, avoid header prefixes in the log messages
+      --skip_log_headers                           If true, avoid headers when opening log files (no effect when -logtostderr=true)
+      --stderrthreshold severity                   logs at or above this threshold go to stderr when writing to files and stderr (no effect when -logtostderr=true or -alsologtostderr=true) (default 2)
+      --telemetry-host string                      Host to expose kube-state-metrics self metrics on. (default "::")
+      --telemetry-port int                         Port to expose kube-state-metrics self metrics on. (default 8081)
+      --tls-config string                          Path to the TLS configuration file
+      --total-shards int                           The total number of shards. Sharding is disabled when total shards is set to 1. (default 1)
+      --track-unscheduled-pods                     This configuration is used in conjunction with node configuration. When this configuration is true, node configuration is empty and the metric of unscheduled pods is fetched from the Kubernetes API Server. This is experimental.
+      --use-apiserver-cache                        Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read.
+  -v, --v Level                                    number for the log level verbosity
+      --vmodule moduleSpec                         comma-separated list of pattern=N settings for file-filtered logging
+
+Use "kube-state-metrics [command] --help" for more information about a command.
+```
+<!-- markdownlint-enable link-image-reference-definitions -->
+<!-- markdownlint-enable blanks-around-fences -->

docs/developer/guide.md

@@ -1,21 +1,32 @@
-# Developer Guide 
+# Developer Guide
 
 This developer guide documentation is intended to assist all contributors in various code contributions.
 Any contribution to improving this documentation will be appreciated.
 
 ## Table of Contents
 
-- [Add New Kubernetes Resource Metric Collector](#add-new-kubernetes-resource-metric-collector)
+* [Add New Kubernetes Resource Metric Collector](#add-new-kubernetes-resource-metric-collector)
+* [Add New Metrics](#add-new-metrics)
 
 ### Add New Kubernetes Resource Metric Collector
 
 The following steps are needed to introduce a new resource and its respective resource metrics.
 
-- Reference your new resource(s) to the [docs/README.md](https://github.com/kubernetes/kube-state-metrics/blob/master/docs/README.md#exposed-metrics).
-- Reference your new resource(s) in the [docs/cli-arguments.md](https://github.com/kubernetes/kube-state-metrics/blob/master/docs/cli-arguments.md#available-options) as part of the `--resources` flag.
-- Create a new `<name-of-resource>.md` in the [docs](https://github.com/kubernetes/kube-state-metrics/tree/master/docs) directory to provide documentation on the resource(s) and metrics you implemented. Follow the formatting of all other resources.
-- Add the resource(s) you are representing to the [examples/standard/cluster-role.yaml](https://github.com/kubernetes/kube-state-metrics/blob/master/examples/standard/cluster-role.yaml) under the appropriate `apiGroup` using the `verbs`: `list` and `watch`.
-- Reference and add build functions for the new resource(s) in [internal/store/builder.go](https://github.com/kubernetes/kube-state-metrics/blob/master/internal/store/builder.go).
-- Reference the new resource in [pkg/options/resource.go](https://github.com/kubernetes/kube-state-metrics/blob/master/pkg/options/resource.go).
-- Add a sample Kubernetes manifest to be used by tests in the [tests/manifests/](https://github.com/kubernetes/kube-state-metrics/tree/master/tests/manifests) directory.
-- Lastly, and most importantly, actually implement your new resource(s) and its test binary in [internal/store](https://github.com/kubernetes/kube-state-metrics/tree/master/internal/store). Follow the formatting and structure of other resources.
+* Reference your new resource(s) to the [docs/README.md](./../README.md#exposed-metrics).
+* Reference your new resource(s) in the [docs/developer/cli-arguments.md](./cli-arguments.md#available-options) as part of the `--resources` flag.
+* Create a new `<name-of-resource>.md` in the [docs](./../docs) directory to provide documentation on the resource(s) and metrics you implemented. Follow the formatting of all other resources.
+* Add the resource(s) you are representing to the [jsonnet/kube-state-metrics/kube-state-metrics.libsonnet](./../../jsonnet/kube-state-metrics/kube-state-metrics.libsonnet) under the appropriate `apiGroup` using the `verbs`: `list` and `watch`.
+* Run `make examples/standard`, this should re-generate [examples/standard/cluster-role.yaml](./../../examples/standard/cluster-role.yaml) with the resource(s) added to [jsonnet/kube-state-metrics/kube-state-metrics.libsonnet](./../../jsonnet/kube-state-metrics/kube-state-metrics.libsonnet).
+* Reference and add build functions for the new resource(s) in [internal/store/builder.go](./../../internal/store/builder.go).
+* Reference the new resource in [pkg/options/resource.go](./../../pkg/options/resource.go).
+* Add a sample Kubernetes manifest to be used by tests in the [tests/manifests/](./../../tests/manifests) directory.
+* Lastly, and most importantly, actually implement your new resource(s) and its test binary in [internal/store](./../../internal/store). Follow the formatting and structure of other resources.
+
+### Add New Metrics
+
+* Make metrics experimental first when introducing them, refer [#1910](https://github.com/kubernetes/kube-state-metrics/pull/1910) for more information.
+
+| Metric stability level |                    |
+|------------------------|--------------------|
+| EXPERIMENTAL           | basemetrics.ALPHA  |
+| STABLE                 | basemetrics.STABLE |

docs/endpoint-metrics.md

@@ -1,9 +0,0 @@
-# Endpoint Metrics
-
-| Metric name| Metric type | Labels/tags | Status |
-| ---------- | ----------- | ----------- | ----------- |
-| kube_endpoint_address_not_ready | Gauge | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; | STABLE |
-| kube_endpoint_address_available | Gauge | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; | STABLE |
-| kube_endpoint_info | Gauge | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt;  | STABLE |
-| kube_endpoint_labels | Gauge | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `label_ENDPOINT_LABEL`=&lt;ENDPOINT_LABEL&gt;  | STABLE |
-| kube_endpoint_created | Gauge | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; | STABLE |

docs/horizontalpodautoscaler-metrics.md

@@ -1,12 +0,0 @@
-# Horizontal Pod Autoscaler Metrics
-
-| Metric name                       | Metric type | Labels/tags                                                   | Status |
-| --------------------------------  | ----------- | ------------------------------------------------------------- | ------ |
-| kube_horizontalpodautoscaler_labels                   | Gauge       | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
-| kube_horizontalpodautoscaler_metadata_generation      | Gauge       | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
-| kube_horizontalpodautoscaler_spec_max_replicas        | Gauge       | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
-| kube_horizontalpodautoscaler_spec_min_replicas        | Gauge       | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
-| kube_horizontalpodautoscaler_spec_target_metric       | Gauge       | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; <br> `metric_name`=&lt;metric-name&gt; <br> `metric_target_type`=&lt;value\|utilization\|average&gt; | EXPERIMENTAL |
-| kube_horizontalpodautoscaler_status_condition         | Gauge       | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; <br> `condition`=&lt;hpa-condition&gt; <br> `status`=&lt;true\|false\|unknown&gt; | STABLE |
-| kube_horizontalpodautoscaler_status_current_replicas  | Gauge       | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |
-| kube_horizontalpodautoscaler_status_desired_replicas  | Gauge       | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; | STABLE |

docs/ingress-metrics.md

@@ -1,10 +0,0 @@
-# Ingress Metrics
-
-| Metric name| Metric type | Labels/tags | Status |
-| ---------- | ----------- | ----------- | ----------- |
-| kube_ingress_info | Gauge | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; | STABLE |
-| kube_ingress_labels | Gauge | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `label_INGRESS_LABEL`=&lt;INGRESS_LABEL&gt; | STABLE |
-| kube_ingress_created  | Gauge | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; | STABLE |
-| kube_ingress_metadata_resource_version  | Gauge | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; | EXPERIMENTAL |
-| kube_ingress_path | Gauge | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `host`=&lt;ingress-host&gt; <br> `path`=&lt;ingress-path&gt; <br> `service_name`=&lt;service name for the path&gt; <br> `service_port`=&lt;service port for the path&gt; | STABLE |
-| kube_ingress_tls | Gauge | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `tls_host`=&lt;tls hostname&gt; <br> `secret`=&lt;tls secret name&gt;| STABLE |

docs/job-metrics.md

@@ -1,18 +0,0 @@
-# Job Metrics
-
-| Metric name| Metric type | Labels/tags | Status |
-| ---------- | ----------- | ----------- | ----------- |
-| kube_job_info | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; | STABLE |
-| kube_job_labels | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; <br> `label_JOB_LABEL`=&lt;JOB_LABEL&gt;  | STABLE |
-| kube_job_owner | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; <br> `owner_kind`=&lt;owner kind&gt; <br> `owner_name`=&lt;owner name&gt; <br> `owner_is_controller`=&lt;whether owner is controller&gt;  | STABLE |
-| kube_job_spec_parallelism | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; | STABLE |
-| kube_job_spec_completions | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; | STABLE |
-| kube_job_spec_active_deadline_seconds | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; | STABLE |
-| kube_job_status_active | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; | STABLE |
-| kube_job_status_succeeded | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; | STABLE |
-| kube_job_status_failed | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; <br> `reason`=&lt;failure reason&gt; | STABLE |
-| kube_job_status_start_time | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; | STABLE |
-| kube_job_status_completion_time | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; | STABLE |
-| kube_job_complete | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; <br> `condition`=&lt;true\|false\|unknown&gt; | STABLE |
-| kube_job_failed | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; <br> `condition`=&lt;true\|false\|unknown&gt; | STABLE |
-| kube_job_created | Gauge | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; | STABLE |

docs/lease-metrics.md

@@ -1,6 +0,0 @@
-# Lease Metrics
-
-| Metric name| Metric type | Labels/tags | Status |
-| ---------- | ----------- | ----------- | ----------- |
-| kube_lease_owner | Gauge | `lease`=&lt;lease-name&gt; <br> `owner_kind`=&lt;onwer kind&gt; <br> `owner_name`=&lt;owner name&gt; | EXPERIMENTAL |
-| kube_lease_renew_time | Gauge | `lease`=&lt;lease-name&gt; | EXPERIMENTAL |

docs/limitrange-metrics.md

@@ -1,6 +0,0 @@
-# LimitRange Metrics
-
-| Metric name| Metric type | Labels/tags | Status |
-| ---------- | ----------- | ----------- | ----------- |
-| kube_limitrange | Gauge | `limitrange`=&lt;limitrange-name&gt; <br> `namespace`=&lt;namespace&gt; <br> `resource`=&lt;ResourceName&gt; <br> `type`=&lt;Pod\|Container\|PersistentVolumeClaim&gt; <br> `constraint`=&lt;constraint&gt;| STABLE |
-| kube_limitrange_created | Gauge | `limitrange`=&lt;limitrange-name&gt; <br> `namespace`=&lt;namespace&gt; | STABLE |

docs/metrics/auth/certificatesigningrequest-metrics.md

@@ -0,0 +1,9 @@
+# CertificateSigningRequest Metrics
+
+| Metric name                                | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                   | Status       |
+| ------------------------------------------ | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_certificatesigningrequest_annotations | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `certificatesigningrequest`=&lt;certificatesigningrequest-name&gt; <br> `signer_name`=&lt;certificatesigningrequest-signer-name&gt;                                           | EXPERIMENTAL |
+| kube_certificatesigningrequest_created     | Gauge       |                                                                                                                           | `certificatesigningrequest`=&lt;certificatesigningrequest-name&gt; <br> `signer_name`=&lt;certificatesigningrequest-signer-name&gt;                                           | STABLE       |
+| kube_certificatesigningrequest_condition   | Gauge       |                                                                                                                           | `certificatesigningrequest`=&lt;certificatesigningrequest-name&gt; <br> `signer_name`=&lt;certificatesigningrequest-signer-name&gt; <br> `condition`=&lt;approved\|denied&gt; | STABLE       |
+| kube_certificatesigningrequest_labels      | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `certificatesigningrequest`=&lt;certificatesigningrequest-name&gt; <br> `signer_name`=&lt;certificatesigningrequest-signer-name&gt;                                           | STABLE       |
+| kube_certificatesigningrequest_cert_length | Gauge       |                                                                                                                           | `certificatesigningrequest`=&lt;certificatesigningrequest-name&gt; <br> `signer_name`=&lt;certificatesigningrequest-signer-name&gt;                                           | STABLE       |

docs/metrics/auth/role-metrics.md

@@ -0,0 +1,9 @@
+# Role Metrics
+
+| Metric name                         | Metric type | Description                                                                                                               | Labels/tags                                                      | Status       |
+| ----------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------- | ------------ |
+| kube_role_annotations               | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `role`=&lt;role-name&gt; <br> `namespace`=&lt;role-namespace&gt; | EXPERIMENTAL |
+| kube_role_labels                    | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `role`=&lt;role-name&gt; <br> `namespace`=&lt;role-namespace&gt; | EXPERIMENTAL |
+| kube_role_info                      | Gauge       |                                                                                                                           | `role`=&lt;role-name&gt; <br> `namespace`=&lt;role-namespace&gt; | EXPERIMENTAL |
+| kube_role_created                   | Gauge       |                                                                                                                           | `role`=&lt;role-name&gt; <br> `namespace`=&lt;role-namespace&gt; | EXPERIMENTAL |
+| kube_role_metadata_resource_version | Gauge       |                                                                                                                           | `role`=&lt;role-name&gt; <br> `namespace`=&lt;role-namespace&gt; | EXPERIMENTAL |

docs/metrics/auth/rolebinding-metrics.md

@@ -0,0 +1,9 @@
+# RoleBinding Metrics
+
+| Metric name                                | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                       | Status       |
+| ------------------------------------------ | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_rolebinding_annotations               | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `rolebinding`=&lt;rolebinding-name&gt; <br> `namespace`=&lt;rolebinding-namespace&gt;                                                                             | EXPERIMENTAL |
+| kube_rolebinding_labels                    | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `rolebinding`=&lt;rolebinding-name&gt; <br> `namespace`=&lt;rolebinding-namespace&gt;                                                                             | EXPERIMENTAL |
+| kube_rolebinding_info                      | Gauge       |                                                                                                                           | `rolebinding`=&lt;rolebinding-name&gt; <br> `namespace`=&lt;rolebinding-namespace&gt; <br> `roleref_kind`=&lt;role-kind&gt; <br> `roleref_name`=&lt;role-name&gt; | EXPERIMENTAL |
+| kube_rolebinding_created                   | Gauge       |                                                                                                                           | `rolebinding`=&lt;rolebinding-name&gt; <br> `namespace`=&lt;rolebinding-namespace&gt;                                                                             | EXPERIMENTAL |
+| kube_rolebinding_metadata_resource_version | Gauge       |                                                                                                                           | `rolebinding`=&lt;rolebinding-name&gt; <br> `namespace`=&lt;rolebinding-namespace&gt;                                                                             | EXPERIMENTAL |

docs/metrics/auth/serviceaccount-metrics.md

@@ -0,0 +1,11 @@
+# ServiceAccount Metrics
+
+| Metric name                           | Metric type | Description                                                                                                               | Unit (where applicable) | Labels/tags                                                                                                                                                                                                          | Status       |
+| ------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_serviceaccount_info              | Gauge       | Information about a service account                                                                                       |                         | `namespace`=&lt;serviceaccount-namespace&gt; <br> `serviceaccount`=&lt;serviceaccount-name&gt; <br> `uid`=&lt;serviceaccount-uid&gt; <br> `automount_token`=&lt;serviceaccount-automount-token&gt;                   | EXPERIMENTAL |
+| kube_serviceaccount_created           | Gauge       | Unix creation timestamp                                                                                                   |                         | `namespace`=&lt;serviceaccount-namespace&gt; <br> `serviceaccount`=&lt;serviceaccount-name&gt; <br> `uid`=&lt;serviceaccount-uid&gt;                                                                                 | EXPERIMENTAL |
+| kube_serviceaccount_deleted           | Gauge       | Unix deletion timestamp                                                                                                   |                         | `namespace`=&lt;serviceaccount-namespace&gt; <br> `serviceaccount`=&lt;serviceaccount-name&gt; <br> `uid`=&lt;serviceaccount-uid&gt;                                                                                 | EXPERIMENTAL |
+| kube_serviceaccount_secret            | Gauge       | Secret being referenced by a service account                                                                              |                         | `namespace`=&lt;serviceaccount-namespace&gt; <br> `serviceaccount`=&lt;serviceaccount-name&gt; <br> `uid`=&lt;serviceaccount-uid&gt; <br> `name`=&lt;secret-name&gt;                                                 | EXPERIMENTAL |
+| kube_serviceaccount_image_pull_secret | Gauge       | Secret being referenced by a service account for the purpose of pulling images                                            |                         | `namespace`=&lt;serviceaccount-namespace&gt; <br> `serviceaccount`=&lt;serviceaccount-name&gt; <br> `uid`=&lt;serviceaccount-uid&gt; <br> `name`=&lt;secret-name&gt;                                                 | EXPERIMENTAL |
+| kube_serviceaccount_annotations       | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) |                         | `namespace`=&lt;serviceaccount-namespace&gt; <br> `serviceaccount`=&lt;serviceaccount-name&gt; <br> `uid`=&lt;serviceaccount-uid&gt; <br> `annotation_SERVICE_ACCOUNT_ANNOTATION`=&lt;SERVICE_ACCOUNT_ANNOTATION&gt; | EXPERIMENTAL |
+| kube_serviceaccount_labels            | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           |                         | `namespace`=&lt;serviceaccount-namespace&gt; <br> `serviceaccount`=&lt;serviceaccount-name&gt; <br> `uid`=&lt;serviceaccount-uid&gt; <br> `label_SERVICE_ACCOUNT_LABEL`=&lt;SERVICE_ACCOUNT_LABEL&gt;                | EXPERIMENTAL |

docs/metrics/cluster/clusterrole-metrics.md

@@ -0,0 +1,9 @@
+# ClusterRole Metrics
+
+| Metric name                                | Metric type | Description                                                                                                               | Labels/tags                            | Status       |
+| ------------------------------------------ | ----------- | ------------------------------------------------------------------------------------------------------------------------- | -------------------------------------- | ------------ |
+| kube_clusterrole_annotations               | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `clusterrole`=<clusterrole-name> | EXPERIMENTAL |
+| kube_clusterrole_labels                    | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `clusterrole`=<clusterrole-name> | EXPERIMENTAL |
+| kube_clusterrole_info                      | Gauge       |                                                                                                                           | `clusterrole`=<clusterrole-name> | EXPERIMENTAL |
+| kube_clusterrole_created                   | Gauge       |                                                                                                                           | `clusterrole`=<clusterrole-name> | EXPERIMENTAL |
+| kube_clusterrole_metadata_resource_version | Gauge       |                                                                                                                           | `clusterrole`=<clusterrole-name> | EXPERIMENTAL |

docs/metrics/cluster/clusterrolebinding-metrics.md

@@ -0,0 +1,9 @@
+# ClusterRoleBinding Metrics
+
+| Metric name                                       | Metric type | Description                                                                                                               | Labels/tags                                                                                                                      | Status       |
+| ------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_clusterrolebinding_annotations               | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `clusterrolebinding`=<clusterrolebinding-name>                                                                             | EXPERIMENTAL |
+| kube_clusterrolebinding_labels                    | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `clusterrolebinding`=<clusterrolebinding-name>                                                                             | EXPERIMENTAL |
+| kube_clusterrolebinding_info                      | Gauge       |                                                                                                                           | `clusterrolebinding`=&lt;clusterrolebinding-name&gt; <br> `roleref_kind`=&lt;role-kind&gt; <br> `roleref_name`=&lt;role-name&gt; | EXPERIMENTAL |
+| kube_clusterrolebinding_created                   | Gauge       |                                                                                                                           | `clusterrolebinding`=&lt;clusterrolebinding-name&gt;                                                                             | EXPERIMENTAL |
+| kube_clusterrolebinding_metadata_resource_version | Gauge       |                                                                                                                           | `clusterrolebinding`=&lt;clusterrolebinding-name&gt;                                                                             | EXPERIMENTAL |

docs/metrics/cluster/lease-metrics.md

@@ -0,0 +1,6 @@
+# Lease Metrics
+
+| Metric name           | Metric type | Description | Labels/tags                                                                                                                                                                             | Status       |
+| --------------------- | ----------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ----------- |
+| kube_lease_owner      | Gauge       |             | `lease`=&lt;lease-name&gt; <br> `owner_kind`=&lt;onwer kind&gt; <br> `owner_name`=&lt;owner name&gt; <br> `namespace` = &lt;namespace&gt; <br> `lease_holder`=&lt;lease holder name&gt; | EXPERIMENTAL |
+| kube_lease_renew_time | Gauge       |             | `lease`=&lt;lease-name&gt;  <br> `namespace` = &lt;namespace&gt;                                                                                                                        | EXPERIMENTAL |

docs/metrics/cluster/namespace-metrics.md

@@ -0,0 +1,9 @@
+# Namespace Metrics
+
+| Metric name                     | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                                                             | Status       |
+| ------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_namespace_annotations      | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `namespace`=&lt;namespace-name&gt; <br> `label_NS_ANNOTATION`=&lt;NS_ANNOTATION&gt;                                                                                                                                     | EXPERIMENTAL |
+| kube_namespace_created          | Gauge       |                                                                                                                           | `namespace`=&lt;namespace-name&gt;                                                                                                                                                                                      | STABLE       |
+| kube_namespace_labels           | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `namespace`=&lt;namespace-name&gt; <br> `label_NS_LABEL`=&lt;NS_LABEL&gt;                                                                                                                                               | STABLE       |
+| kube_namespace_status_condition | Gauge       |                                                                                                                           | `namespace`=&lt;namespace-name&gt; <br> `condition`=&lt;NamespaceDeletionDiscoveryFailure\|NamespaceDeletionContentFailure\|NamespaceDeletionGroupVersionParsingFailure&gt;  <br> `status`=&lt;true\|false\|unknown&gt; | EXPERIMENTAL |
+| kube_namespace_status_phase     | Gauge       |                                                                                                                           | `namespace`=&lt;namespace-name&gt; <br> `phase`=&lt;Active\|Terminating&gt;                                                                                                                                             | STABLE       |

docs/metrics/cluster/node-metrics.md

@@ -0,0 +1,16 @@
+# Node Metrics
+
+| Metric name                  | Metric type | Description                                                                                                               | Unit (where applicable)                                                                                                                                                                  | Labels/tags                                                                                                                                                                                                                                                                                                                                                                                                                                               | Status       |
+| ---------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_node_annotations        | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `annotation_NODE_ANNOTATION`=&lt;NODE_ANNOTATION&gt;                                                                                                                                                                                                                                                                                                                                                                     | EXPERIMENTAL |
+| kube_node_info               | Gauge       | Information about a cluster node                                                                                          |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `kernel_version`=&lt;kernel-version&gt; <br> `os_image`=&lt;os-image-name&gt; <br> `container_runtime_version`=&lt;container-runtime-and-version-combination&gt; <br> `kubelet_version`=&lt;kubelet-version&gt; <br> `kubeproxy_version`=&lt;deprecated&gt; <br> `pod_cidr`=&lt;pod-cidr&gt; <br> `provider_id`=&lt;provider-id&gt; <br> `system_uuid`=&lt;system-uuid&gt; <br> `internal_ip`=&lt;internal-ip&gt; | STABLE       |
+| kube_node_labels             | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `label_NODE_LABEL`=&lt;NODE_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                                    | STABLE       |
+| kube_node_role               | Gauge       | The role of a cluster node                                                                                                |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `role`=&lt;NODE_ROLE&gt;                                                                                                                                                                                                                                                                                                                                                                                                 | EXPERIMENTAL |
+| kube_node_spec_unschedulable | Gauge       | Whether a node can schedule new pods                                                                                      |                                                                                                                                                                                          | `node`=&lt;node-address&gt;                                                                                                                                                                                                                                                                                                                                                                                                                               | STABLE       |
+| kube_node_spec_taint         | Gauge       | The taint of a cluster node.                                                                                              |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `key`=&lt;taint-key&gt; <br> `value=`&lt;taint-value&gt; <br> `effect=`&lt;taint-effect&gt;                                                                                                                                                                                                                                                                                                                              | STABLE       |
+| kube_node_status_capacity    | Gauge       | The total amount of resources available for a node                                                                        | `cpu`=&lt;core&gt; <br> `ephemeral_storage`=&lt;byte&gt; <br> `pods`=&lt;integer&gt; <br> `attachable_volumes_*`=&lt;byte&gt; <br> `hugepages_*`=&lt;byte&gt; <br> `memory`=&lt;byte&gt; | `node`=&lt;node-address&gt; <br> `resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt;                                                                                                                                                                                                                                                                                                                                                       | STABLE       |
+| kube_node_status_addresses         | Gauge       | The addresses of a node                                                                                              |                                                                                                                                                                                          |  `node`=&lt;node-address&gt; <br> `type`=&lt;address-type&gt; <br> `address`=&lt;address-value&gt;                                                                                                                                                                                                                                           | EXPERIMENTAL       |
+| kube_node_status_allocatable | Gauge       | The amount of resources allocatable for pods (after reserving some for system daemons)                                    | `cpu`=&lt;core&gt; <br> `ephemeral_storage`=&lt;byte&gt; <br> `pods`=&lt;integer&gt; <br> `attachable_volumes_*`=&lt;byte&gt; <br> `hugepages_*`=&lt;byte&gt; <br> `memory`=&lt;byte&gt; | `node`=&lt;node-address&gt; <br> `resource`=&lt;resource-name&gt; <br> `unit`=&lt;resource-unit&gt;                                                                                                                                                                                                                                                                                                                                                       | STABLE       |
+| kube_node_status_condition   | Gauge       | The condition of a cluster node                                                                                           |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `condition`=&lt;node-condition&gt; <br> `status`=&lt;true\|false\|unknown&gt;                                                                                                                                                                                                                                                                                                                                            | STABLE       |
+| kube_node_created            | Gauge       | Unix creation timestamp                                                                                                   | seconds                                                                                                                                                                                  | `node`=&lt;node-address&gt;                                                                                                                                                                                                                                                                                                                                                                                                                               | STABLE       |
+| kube_node_deletion_timestamp | Gauge       | Unix deletion timestamp                                                                                                   | seconds                                                                                                                                                                                  | `node`=&lt;node-address&gt;                                                                                                                                                                                                                                                                                                                                                                                                                               | EXPERIMENTAL |

docs/metrics/extend/customresourcestate-metrics.md

@@ -0,0 +1,852 @@
+# Custom Resource State Metrics
+
+This section describes how to add metrics based on the state of a custom resource without writing a custom resource
+registry and running your own build of KSM.
+
+## Configuration
+
+A YAML configuration file described below is required to define your custom resources and the fields to turn into metrics.
+
+Two flags can be used:
+
+* `--custom-resource-state-config "inline yaml (see example)"` or
+* `--custom-resource-state-config-file /path/to/config.yaml`
+
+If both flags are provided, the inline configuration will take precedence.
+When multiple entries for the same resource exist, kube-state-metrics will exit with an error.
+This includes configuration which refers to a different API version.
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+  namespace: kube-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: kube-state-metrics
+        args:
+          - --custom-resource-state-config
+          # in YAML files, | allows a multi-line string to be passed as a flag value
+          # see https://yaml-multiline.info
+          -  |
+              kind: CustomResourceStateMetrics
+              spec:
+                resources:
+                  - groupVersionKind:
+                      group: myteam.io
+                      version: "v1"
+                      kind: Foo
+                    metrics:
+                      - name: active_count
+                        help: "Count of active Foo"
+                        each:
+                          type: Gauge
+                          ...
+```
+
+It's also possible to configure kube-state-metrics to run in a `custom-resource-mode` only. In addition to specifying one of `--custom-resource-state-config*` flags, you could set `--custom-resource-state-only` to `true`.
+With this configuration only the known custom resources configured in `--custom-resource-state-config*` will be taken into account by kube-state-metrics.
+
+```yaml
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: kube-state-metrics
+  namespace: kube-system
+spec:
+  template:
+    spec:
+      containers:
+      - name: kube-state-metrics
+        args:
+          - --custom-resource-state-config
+          # in YAML files, | allows a multi-line string to be passed as a flag value
+          # see https://yaml-multiline.info
+          -  |
+              kind: CustomResourceStateMetrics
+              spec:
+                resources:
+                  - groupVersionKind:
+                      group: myteam.io
+                      version: "v1"
+                      kind: Foo
+                    metrics:
+                      - name: active_count
+                        help: "Count of active Foo"
+                        each:
+                          type: Gauge
+                          ...
+          - --custom-resource-state-only=true
+```
+
+NOTE: The `customresource_group`, `customresource_version`, and `customresource_kind` common labels are reserved, and will be overwritten by the values from the `groupVersionKind` field.
+
+### RBAC-enabled Clusters
+
+Please be aware that kube-state-metrics needs list and watch permissions granted to `customresourcedefinitions.apiextensions.k8s.io` as well as to the resources you want to gather metrics from.
+
+### Examples
+
+The examples in this section will use the following custom resource:
+
+```yaml
+kind: Foo
+apiVersion: myteam.io/vl
+metadata:
+    annotations:
+        bar: baz
+        qux: quxx
+    labels:
+        foo: bar
+    name: foo
+spec:
+    version: v1.2.3
+    order:
+        - id: 1
+          value: true
+        - id: 3
+          value: false
+    replicas: 1
+    refs:
+        - my_other_foo
+        - foo_2
+        - foo_with_extensions
+status:
+    phase: Pending
+    active:
+        type-a: 1
+        type-b: 3
+    conditions:
+        - name: a
+          value: 45
+        - name: b
+          value: 66
+    sub:
+        type-a:
+            active: 1
+            ready: 2
+        type-b:
+            active: 3
+            ready: 4
+    uptime: 43.21
+```
+
+#### Single Values
+
+The config:
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: myteam.io
+        kind: "Foo"
+        version: "v1"
+      metrics:
+        - name: "uptime"
+          help: "Foo uptime"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, uptime]
+```
+
+Produces the metric:
+
+```prometheus
+kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
+```
+
+#### Multiple Metrics/Kitchen Sink
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: myteam.io
+        kind: "Foo"
+        version: "v1"
+      # labels can be added to all metrics from a resource
+      commonLabels:
+        crd_type: "foo"
+      labelsFromPath:
+        name: [metadata, name]
+      metrics:
+        - name: "ready_count"
+          help: "Number Foo Bars ready"
+          each:
+            type: Gauge
+            gauge:
+              # targeting an object or array will produce a metric for each element
+              # labelsFromPath and value are relative to this path
+              path: [status, sub]
+
+              # if path targets an object, the object key will be used as label value
+              # This is not supported for StateSet type as all values will be truthy, which is redundant.
+              labelFromKey: type
+              # label values can be resolved specific to this path 
+              labelsFromPath:
+                active: [active]
+              # The actual field to use as metric value. Should be a number, boolean or RFC3339 timestamp string.
+              valueFrom: [ready]
+          commonLabels:
+            custom_metric: "yes"
+          labelsFromPath:
+            # whole objects may be copied into labels by prefixing with "*"
+            # *anything will be copied into labels, with the highest sorted * strings first
+            "*": [metadata, labels]
+            # a prefix before the asterisk will be used as a label prefix
+            "lorem_*": [metadata, annotations]
+            "**": [metadata, annotations]
+            
+            # or specific fields may be copied. these fields will always override values from *s
+            name: [metadata, name]
+            foo: [metadata, labels, foo]
+```
+
+Produces the following metrics:
+
+```prometheus
+kube_customresource_ready_count{customresource_group="myteam.io", customresource_kind="Foo", 
+customresource_version="v1", active="1",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-a",
+lorem_bar="baz",lorem_qux="quxx",} 2
+kube_customresource_ready_count{customresource_group="myteam.io", customresource_kind="Foo", 
+customresource_version="v1", active="3",custom_metric="yes",foo="bar",name="foo",bar="baz",qux="quxx",type="type-b",
+lorem_bar="baz",lorem_qux="quxx",} 4
+```
+
+#### Non-map Arrays
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: myteam.io
+        kind: "Foo"
+        version: "v1"
+      labelsFromPath:
+        name: [metadata, name]
+      metrics:
+        - name: "ref_info"
+          help: "Reference to other Foo"
+          each:
+            type: Info
+            info:
+              # targeting an array will produce a metric for each element
+              # labelsFromPath and value are relative to this path
+              path: [spec, refs]
+
+              # if path targets a list of values (e.g. strings or numbers, not objects or maps), individual values can
+              # referenced by a label using this syntax
+              labelsFromPath:
+                ref: []
+```
+
+Produces the following metrics:
+
+```prometheus
+kube_customresource_ref_info{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", name="foo",ref="my_other_foo"} 1
+kube_customresource_ref_info{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", name="foo",ref="foo_2"} 1
+kube_customresource_ref_info{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", name="foo",ref="foo_with_extensions"} 1
+```
+
+#### Same Metrics with Different Labels
+
+```yaml
+  recommendation:
+    containerRecommendations:
+    - containerName: consumer
+      lowerBound:
+        cpu: 100m
+        memory: 262144k
+```
+
+For example in VPA we have above attributes and we want to have a same metrics for both CPU and Memory, you can use below config:
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: autoscaling.k8s.io
+        kind: "VerticalPodAutoscaler"
+        version: "v1"
+      labelsFromPath:
+        verticalpodautoscaler: [metadata, name]
+        namespace: [metadata, namespace]
+        target_api_version: [apiVersion]
+        target_kind: [spec, targetRef, kind]
+        target_name: [spec, targetRef, name]
+      metrics:
+        # for memory
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound"
+          help: "Minimum memory resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [lowerBound, memory]
+        # for CPU
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound"
+          help: "Minimum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [lowerBound, cpu]
+```
+
+Produces the following metrics:
+
+```prometheus
+# HELP kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound Minimum memory resources the container can use before the VerticalPodAutoscaler updater evicts it.
+# TYPE kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound gauge
+kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="consumer",customresource_group="autoscaling.k8s.io",customresource_kind="VerticalPodAutoscaler",customresource_version="v1",namespace="namespace-example",resource="memory",target_api_version="apps/v1",target_kind="Deployment",target_name="target-name-example",unit="byte",verticalpodautoscaler="vpa-example"} 123456
+# HELP kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound Minimum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it.
+# TYPE kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound gauge
+kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="consumer",customresource_group="autoscaling.k8s.io",customresource_kind="VerticalPodAutoscaler",customresource_version="v1",namespace="namespace-example",resource="cpu",target_api_version="apps/v1",target_kind="Deployment",target_name="target-name-example",unit="core",verticalpodautoscaler="vpa-example"} 0.1
+```
+
+#### VerticalPodAutoscaler
+
+In v2.9.0 the `vericalpodautoscalers` resource was removed from the list of default resources. In order to generate metrics for `verticalpodautoscalers`, you can use the following Custom Resource State config:
+
+```yaml
+# Using --resource=verticalpodautoscalers, we get the following output:
+# HELP kube_verticalpodautoscaler_annotations Kubernetes annotations converted to Prometheus labels.
+# TYPE kube_verticalpodautoscaler_annotations gauge
+# kube_verticalpodautoscaler_annotations{namespace="default",verticalpodautoscaler="hamster-vpa",target_api_version="apps/v1",target_kind="Deployment",target_name="hamster"} 1
+# A similar result can be achieved by specifying the following in --custom-resource-state-config:
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: autoscaling.k8s.io
+        kind: "VerticalPodAutoscaler"
+        version: "v1"
+      labelsFromPath:
+        verticalpodautoscaler: [metadata, name]
+        namespace: [metadata, namespace]
+        target_api_version: [apiVersion]
+        target_kind: [spec, targetRef, kind]
+        target_name: [spec, targetRef, name]
+      metrics:
+        - name: "annotations"
+          help: "Kubernetes annotations converted to Prometheus labels."
+          each:
+            type: Gauge
+            gauge:
+              path: [metadata, annotations]
+# This will output the following metric:
+# HELP kube_customresource_autoscaling_annotations Kubernetes annotations converted to Prometheus labels.
+# TYPE kube_customresource_autoscaling_annotations gauge
+# kube_customresource_autoscaling_annotations{customresource_group="autoscaling.k8s.io", customresource_kind="VerticalPodAutoscaler", customresource_version="v1", namespace="default",target_api_version="autoscaling.k8s.io/v1",target_kind="Deployment",target_name="hamster",verticalpodautoscaler="hamster-vpa"} 123
+```
+
+The above configuration was tested on [this](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/examples/hamster.yaml) VPA configuration, with an added annotation (`foo: 123`).
+
+#### All VerticalPodAutoscaler Metrics
+
+As an addition for the above configuration, here's the complete `CustomResourceStateMetrics` spec to re-enable all of the VPA metrics which are removed from the list of the default resources:
+
+<details>
+
+ <summary>VPA CustomResourceStateMetrics</summary>
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: autoscaling.k8s.io
+        kind: "VerticalPodAutoscaler"
+        version: "v1"
+      labelsFromPath:
+        namespace: [metadata, namespace]
+        target_api_version: [spec, targetRef, apiVersion]
+        target_kind: [spec, targetRef, kind]
+        target_name: [spec, targetRef, name]
+        verticalpodautoscaler: [metadata, name]
+      metricNamePrefix: "kube"
+      metrics:
+        # kube_verticalpodautoscaler_annotations
+        - name: "verticalpodautoscaler_annotations"
+          help: "Kubernetes annotations converted to Prometheus labels."
+          each:
+            type: Info
+            info:
+              labelsFromPath:
+                annotation_*: [metadata, annotations]
+                name: [metadata, name]
+        # kube_verticalpodautoscaler_labels
+        - name: "verticalpodautoscaler_labels"
+          help: "Kubernetes labels converted to Prometheus labels."
+          each:
+            type: Info
+            info:
+              labelsFromPath:
+                label_*: [metadata, labels]
+                name: [metadata, name]
+        # kube_verticalpodautoscaler_spec_updatepolicy_updatemode
+        - name: "verticalpodautoscaler_spec_updatepolicy_updatemode"
+          help: "Update mode of the VerticalPodAutoscaler."
+          each:
+            type: StateSet
+            stateSet:
+              labelName: "update_mode"
+              path: [spec, updatePolicy, updateMode]
+              list: ["Auto", "Initial", "Off", "Recreate"]
+        # Memory kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_memory
+        - name: "verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_memory"
+          help: "Minimum memory resources the VerticalPodAutoscaler can set for containers matching the name."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [spec, resourcePolicy, containerPolicies]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [minAllowed, memory]
+        # CPU kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_cpu
+        - name: "verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_cpu"
+          help: "Minimum cpu resources the VerticalPodAutoscaler can set for containers matching the name."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [spec, resourcePolicy, containerPolicies]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [minAllowed, cpu]
+        # Memory kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_memory
+        - name: "verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_memory"
+          help: "Maximum memory resources the VerticalPodAutoscaler can set for containers matching the name."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [spec, resourcePolicy, containerPolicies]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [maxAllowed, memory]
+        # CPU kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_cpu
+        - name: "verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_cpu"
+          help: "Maximum cpu resources the VerticalPodAutoscaler can set for containers matching the name."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [spec, resourcePolicy, containerPolicies]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [maxAllowed, cpu]
+        # Memory kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_memory
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_memory"
+          help: "Minimum memory resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [lowerBound, memory]
+        # CPU kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_cpu
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_cpu"
+          help: "Minimum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [lowerBound, cpu]
+        # Memory kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_memory
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_memory"
+          help: "Maximum memory resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [upperBound, memory]
+        # CPU kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_cpu
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_cpu"
+          help: "Maximum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [upperBound, cpu]
+        # Memory kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target_memory
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_target_memory"
+          help: "Target memory resources the VerticalPodAutoscaler recommends for the container."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [target, memory]
+        # CPU kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target_cpu
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_target_cpu"
+          help: "Target cpu resources the VerticalPodAutoscaler recommends for the container."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [target, cpu]
+        # Memory kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_memory
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_memory"
+          help: "Target memory resources the VerticalPodAutoscaler recommends for the container ignoring bounds."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [uncappedTarget, memory]
+        # CPU kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_cpu
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_cpu"
+          help: "Target memory resources the VerticalPodAutoscaler recommends for the container ignoring bounds."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [uncappedTarget, cpu]
+```
+
+</details>
+
+### Metric types
+
+The configuration supports three kind of metrics from the [OpenMetrics specification](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md).
+
+The metric type is specified by the `type` field and its specific configuration at the types specific struct.
+
+#### Gauge
+
+> Gauges are current measurements, such as bytes of memory currently used or the number of items in a queue. For gauges the absolute value is what is of interest to a user. [[0]](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#gauge)
+
+Example:
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: myteam.io
+        kind: "Foo"
+        version: "v1"
+      metrics:
+        - name: "uptime"
+          help: "Foo uptime"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, uptime]
+```
+
+Produces the metric:
+
+```prometheus
+kube_customresource_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
+```
+
+##### Type conversion and special handling
+
+Gauges produce values of type float64 but custom resources can be of all kinds of types.
+Kube-state-metrics performs implicit type conversions for a lot of type.
+Supported types are:
+
+* (u)int32/64, int, float32 and byte are cast to float64
+* `nil` is generally mapped to `0.0` if NilIsZero is `true`, otherwise it will throw an error
+* for bool `true` is mapped to `1.0` and `false` is mapped to `0.0`
+* for string the following logic applies
+  * `"true"` and `"yes"` are mapped to `1.0`, `"false"`, `"no"` and `"unknown"` are mapped to `0.0` (all case-insensitive)
+  * RFC3339 times are parsed to float timestamp  
+  * Quantities like "250m" or "512Gi" are parsed to float using <https://github.com/kubernetes/apimachinery/blob/master/pkg/api/resource/quantity.go>
+  * Percentages ending with a "%" are parsed to float
+  * finally the string is parsed to float using <https://pkg.go.dev/strconv#ParseFloat> which should support all common number formats. If that fails an error is yielded
+
+##### Example for status conditions on Kubernetes Controllers
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+  - groupVersionKind:
+      group: myteam.io
+      kind: "Foo"
+      version: "v1"
+    labelsFromPath:
+      name:
+      - metadata
+      - name
+      namespace:
+      - metadata
+      - namespace
+    metrics:
+    - name: "foo_status"
+      help: "status condition "
+      each:
+        type: Gauge
+        gauge:
+          path: [status, conditions]
+          labelsFromPath:
+            type: ["type"]
+          valueFrom: ["status"]
+```
+
+This will work for kubernetes controller CRs which expose status conditions according to the kubernetes api (<https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Condition>):
+
+```yaml
+status:
+  conditions:
+    - lastTransitionTime: "2019-10-22T16:29:31Z"
+      status: "True"
+      type: Ready
+```
+
+kube_customresource_foo_status{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", type="Ready"} 1.0
+
+#### StateSet
+
+> StateSets represent a series of related boolean values, also called a bitset. If ENUMs need to be encoded this MAY be done via StateSet. [[1]](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#stateset)
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: myteam.io
+        kind: "Foo"
+        version: "v1"
+      metrics:
+        - name: "status_phase"
+          help: "Foo status_phase"
+          each:
+            type: StateSet
+            stateSet:
+              labelName: phase
+              path: [status, phase]
+              list: [Pending, Bar, Baz]
+```
+
+Metrics of type `StateSet` will generate a metric for each value defined in `list` for each resource.
+The value will be 1, if the value matches the one in list.
+
+Produces the metric:
+
+```prometheus
+kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Pending"} 1
+kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Bar"} 0
+kube_customresource_status_phase{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", phase="Baz"} 0
+```
+
+#### Info
+
+> Info metrics are used to expose textual information which SHOULD NOT change during process lifetime. Common examples are an application's version, revision control commit, and the version of a compiler. [[2]](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#info)
+
+Metrics of type `Info` will always have a value of 1.
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: myteam.io
+        kind: "Foo"
+        version: "v1"
+      metrics:
+        - name: "version"
+          help: "Foo version"
+          each:
+            type: Info
+            info:
+              labelsFromPath:
+                version: [spec, version]
+```
+
+Produces the metric:
+
+```prometheus
+kube_customresource_version{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", version="v1.2.3"} 1
+```
+
+### Naming
+
+The default metric names are prefixed to avoid collisions with other metrics.
+By default, a metric prefix of `kube_` concatenated with your custom resource's group+version+kind is used.
+You can override this behavior with the `metricNamePrefix` field.
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind: ...
+      metricNamePrefix: myteam_foos
+      metrics:
+        - name: uptime
+          # ...
+```
+
+Produces:
+
+```prometheus
+myteam_foos_uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
+```
+
+To omit namespace and/or subsystem altogether, set them to the empty string:
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind: ...
+      metricNamePrefix: ""
+      metrics:
+        - name: uptime
+          # ...
+```
+
+Produces:
+
+```prometheus
+uptime{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1"} 43.21
+```
+
+### Logging
+
+If a metric path is registered but not found on a custom resource, an error will be logged. For some resources,
+this may produce a lot of noise. The error log [verbosity][vlog] for a metric or resource can be set with `errorLogV` on
+the resource or metric:
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind: ...
+      errorLogV: 0  # 0 = default for errors
+      metrics:
+        - name: uptime
+          errorLogV: 10  # only log at high verbosity
+```
+
+[vlog]: https://github.com/go-logr/logr#why-v-levels
+
+### Path Syntax
+
+Paths are specified as a list of strings. Each string is a path segment, resolved dynamically against the data of the custom resource.
+If any part of a path is missing, the result is nil.
+
+Examples:
+
+```yaml
+# simple path lookup
+[spec, replicas]                         # spec.replicas == 1
+
+# indexing an array
+[spec, order, "0", value]                # spec.order[0].value = true
+
+# finding an element in a list by key=value  
+[status, conditions, "[name=a]", value]  # status.conditions[0].value = 45
+
+# if the value to be matched is a number or boolean, the value is compared as a number or boolean  
+[status, conditions, "[value=66]", name]  # status.conditions[1].name = "b"
+
+# For generally matching against a field in an object schema, use the following syntax:
+[metadata, "name=foo"] # if v, ok := metadata[name]; ok && v == "foo" { return v; } else { /* ignore */ }
+```
+
+### Wildcard matching of version and kind fields
+
+The Custom Resource State (CRS hereon) configuration also allows you to monitor all versions and/or kinds that come under a group. It watches
+the installed CRDs for this purpose. Taking the aforementioned `Foo` object as reference, the configuration below allows
+you to monitor all objects under all versions *and* all kinds that come under the `myteam.io` group.
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: "myteam.io"
+        version: "*" # Set to `v1 to monitor all kinds under `myteam.io/v1`. Wildcard matches all installed versions that come under this group.
+        kind: "*" # Set to `Foo` to monitor all `Foo` objects under the `myteam.io` group (under all versions). Wildcard matches all installed kinds that come under this group (and version, if specified).
+      metrics:
+        - name: "myobject_info"
+          help: "Foo Bar Baz"
+          each:
+            type: Info
+            info:
+              path: [metadata]
+              labelsFromPath:
+                object: [name]
+                namespace: [namespace]
+```
+
+The configuration above produces these metrics.
+
+```yaml
+kube_customresource_myobject_info{customresource_group="myteam.io",customresource_kind="Foo",customresource_version="v1",namespace="ns",object="foo"} 1
+kube_customresource_myobject_info{customresource_group="myteam.io",customresource_kind="Bar",customresource_version="v1",namespace="ns",object="bar"} 1
+```
+
+#### Note
+
+* For cases where the GVKs defined in a CRD have multiple versions under a single group for the same kind, as expected, the wildcard value will resolve to *all* versions, but a query for any specific version will return all resources under all versions, in that versions' representation. This basically means that for two such versions `A` and `B`,  if a resource exists under `B`, it will reflect in the metrics generated for `A` as well, in addition to any resources of itself, and vice-versa. This logic is based on the [current `list`ing behavior](https://github.com/kubernetes/client-go/issues/1251#issuecomment-1544083071) of the client-go library.
+* The introduction of this feature further discourages (and discontinues) the use of native objects in the CRS featureset, since these do not have an explicit CRD associated with them, and conflict with internal stores defined specifically for such native resources. Please consider opening an issue or raising a PR if you'd like to expand on the current metric labelsets for them. Also, any such configuration will be ignored, and no metrics will be generated for the same.

docs/metrics/extend/mutatingwebhookconfiguration-metrics.md

@@ -0,0 +1,8 @@
+# MutatingWebhookConfiguration Metrics
+
+| Metric name                                                    | Metric type | Description | Labels/tags                                                                                                                                                                                                                                                                                   | Status       |
+| -------------------------------------------------------------- | ----------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_mutatingwebhookconfiguration_info                         | Gauge       |             | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt;                                                                                                                                                      | EXPERIMENTAL |
+| kube_mutatingwebhookconfiguration_created                      | Gauge       |             | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt;                                                                                                                                                      | EXPERIMENTAL |
+| kube_mutatingwebhookconfiguration_metadata_resource_version    | Gauge       |             | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt;                                                                                                                                                      | EXPERIMENTAL |
+| kube_mutatingwebhookconfiguration_webhook_clientconfig_service | Gauge       |             | `mutatingwebhookconfiguration`=&lt;mutatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;mutatingwebhookconfiguration-namespace&gt; <br> `webhook_name`=&lt;webhook-name&gt; <br> `service_name`=&lt;webhook-service-name&gt; <br> `service_namespace`=&lt;webhook-service-namespace&gt; | EXPERIMENTAL |

docs/metrics/extend/validatingwebhookconfiguration-metrics.md

@@ -0,0 +1,8 @@
+# ValidatingWebhookConfiguration Metrics
+
+| Metric name                                                      | Metric type | Description | Labels/tags                                                                                                                                                                                                                                                                                         | Status       |
+| ---------------------------------------------------------------- | ----------- | ----------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_validatingwebhookconfiguration_info                         | Gauge       |             | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt;                                                                                                                                                      | EXPERIMENTAL |
+| kube_validatingwebhookconfiguration_created                      | Gauge       |             | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt;                                                                                                                                                      | EXPERIMENTAL |
+| kube_validatingwebhookconfiguration_metadata_resource_version    | Gauge       |             | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt;                                                                                                                                                      | EXPERIMENTAL |
+| kube_validatingwebhookconfiguration_webhook_clientconfig_service | Gauge       |             | `validatingwebhookconfiguration`=&lt;validatingwebhookconfiguration-name&gt; <br> `namespace`=&lt;validatingwebhookconfiguration-namespace&gt; <br> `webhook_name`=&lt;webhook-name&gt; <br> `service_name`=&lt;webhook-service-name&gt; <br> `service_namespace`=&lt;webhook-service-namespace&gt; | EXPERIMENTAL |

docs/metrics/policy/limitrange-metrics.md

@@ -0,0 +1,6 @@
+# LimitRange Metrics
+
+| Metric name             | Metric type | Description | Labels/tags                                                                                                                                                                                                 | Status |
+| ----------------------- | ----------- | ----------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------ |
+| kube_limitrange         | Gauge       |             | `limitrange`=&lt;limitrange-name&gt; <br> `namespace`=&lt;namespace&gt; <br> `resource`=&lt;ResourceName&gt; <br> `type`=&lt;Pod\|Container\|PersistentVolumeClaim&gt; <br> `constraint`=&lt;constraint&gt; | STABLE |
+| kube_limitrange_created | Gauge       |             | `limitrange`=&lt;limitrange-name&gt; <br> `namespace`=&lt;namespace&gt;                                                                                                                                     | STABLE |

docs/metrics/policy/networkpolicy-metrics.md

@@ -0,0 +1,9 @@
+# Network Policy Metrics
+
+| Metric name                           | Metric type | Description                                                                                                               | Labels/tags                                                                   | Status       |
+| ------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------- | ------------ |
+| kube_networkpolicy_annotations        | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `namespace`=<namespace name> `networkpolicy`=<networkpolicy name> | EXPERIMENTAL |
+| kube_networkpolicy_created            | Gauge       |                                                                                                                           | `namespace`=<namespace name> `networkpolicy`=<networkpolicy name> | EXPERIMENTAL |
+| kube_networkpolicy_labels             | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `namespace`=<namespace name> `networkpolicy`=<networkpolicy name> | EXPERIMENTAL |
+| kube_networkpolicy_spec_egress_rules  | Gauge       |                                                                                                                           | `namespace`=<namespace name> `networkpolicy`=<networkpolicy name> | EXPERIMENTAL |
+| kube_networkpolicy_spec_ingress_rules | Gauge       |                                                                                                                           | `namespace`=<namespace name> `networkpolicy`=<networkpolicy name> | EXPERIMENTAL |

docs/metrics/policy/poddisruptionbudget-metrics.md

@@ -0,0 +1,12 @@
+# PodDisruptionBudget Metrics
+
+| Metric name                                             | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                                          | Status       |
+| ------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_poddisruptionbudget_annotations                    | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `poddisruptionbudget`=&lt;poddisruptionbudget-name&gt; <br> `namespace`=&lt;poddisruptionbudget-namespace&gt; <br> `annotation_PODDISRUPTIONBUDGET_ANNOTATION`=&lt;PODDISRUPTIONBUDGET_ANNOATION&gt; | EXPERIMENTAL |
+| kube_poddisruptionbudget_labels                         | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `poddisruptionbudget`=&lt;poddisruptionbudget-name&gt; <br> `namespace`=&lt;poddisruptionbudget-namespace&gt; <br> `label_PODDISRUPTIONBUDGET_LABEL`=&lt;PODDISRUPTIONBUDGET_ANNOATION&gt;           | EXPERIMENTAL |
+| kube_poddisruptionbudget_created                        | Gauge       |                                                                                                                           | `poddisruptionbudget`=&lt;pdb-name&gt; <br> `namespace`=&lt;pdb-namespace&gt;                                                                                                                        | STABLE       |
+| kube_poddisruptionbudget_status_current_healthy         | Gauge       |                                                                                                                           | `poddisruptionbudget`=&lt;pdb-name&gt; <br> `namespace`=&lt;pdb-namespace&gt;                                                                                                                        | STABLE       |
+| kube_poddisruptionbudget_status_desired_healthy         | Gauge       |                                                                                                                           | `poddisruptionbudget`=&lt;pdb-name&gt; <br> `namespace`=&lt;pdb-namespace&gt;                                                                                                                        | STABLE       |
+| kube_poddisruptionbudget_status_pod_disruptions_allowed | Gauge       |                                                                                                                           | `poddisruptionbudget`=&lt;pdb-name&gt; <br> `namespace`=&lt;pdb-namespace&gt;                                                                                                                        | STABLE       |
+| kube_poddisruptionbudget_status_expected_pods           | Gauge       |                                                                                                                           | `poddisruptionbudget`=&lt;pdb-name&gt; <br> `namespace`=&lt;pdb-namespace&gt;                                                                                                                        | STABLE       |
+| kube_poddisruptionbudget_status_observed_generation     | Gauge       |                                                                                                                           | `poddisruptionbudget`=&lt;pdb-name&gt; <br> `namespace`=&lt;pdb-namespace&gt;                                                                                                                        | STABLE       |

docs/metrics/policy/resourcequota-metrics.md

@@ -0,0 +1,8 @@
+# ResourceQuota Metrics
+
+| Metric name                    | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                         | Status       |
+| ------------------------------ | ----------- | ------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_resourcequota             | Gauge       |                                                                                                                           | `resourcequota`=&lt;quota-name&gt; <br> `namespace`=&lt;namespace&gt; <br> `resource`=&lt;ResourceName&gt; <br> `type`=&lt;quota-type&gt;           | STABLE       |
+| kube_resourcequota_created     | Gauge       |                                                                                                                           | `resourcequota`=&lt;quota-name&gt; <br> `namespace`=&lt;namespace&gt;                                                                               | STABLE       |
+| kube_resourcequota_annotations | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `resourcequota`=&lt;quota-name&gt; <br> `namespace`=&lt;namespace&gt; <br> `annotation_RESOURCE_QUOTA_ANNOTATION`=&lt;RESOURCE_QUOTA_ANNOTATION&gt; | EXPERIMENTAL |
+| kube_resourcequota_labels      | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `resourcequota`=&lt;quota-name&gt; <br> `namespace`=&lt;namespace&gt; <br> `label_RESOURCE_QUOTA_LABEL`=&lt;RESOURCE_QUOTA_LABEL&gt;                | EXPERIMENTAL |

docs/metrics/service/endpoint-metrics.md

@@ -0,0 +1,10 @@
+# Endpoint Metrics
+
+| Metric name                     | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                                                                  | Status       |
+| ------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_endpoint_annotations       | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `annotation_ENDPOINT_ANNOTATION`=&lt;ENDPOINT_ANNOTATION&gt;                                                                               | EXPERIMENTAL |
+| kube_endpoint_info              | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt;                                                                                                                                                 | STABLE       |
+| kube_endpoint_labels            | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `label_ENDPOINT_LABEL`=&lt;ENDPOINT_LABEL&gt;                                                                                              | STABLE       |
+| kube_endpoint_created           | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt;                                                                                                                                                 | STABLE       |
+| kube_endpoint_ports             | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `port_name`=&lt;endpoint-port-name&gt; <br> `port_protocol`=&lt;endpoint-port-protocol&gt; <br> `port_number`=&lt;endpoint-port-number&gt; | STABLE (Deprecated from 2.14.0) |
+| kube_endpoint_address           | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `ip`=&lt;endpoint-ip&gt; <br> `port_name`=&lt;endpoint-port-name&gt; <br> `port_protocol`=&lt;endpoint-port-protocol&gt; <br> `port_number`=&lt;endpoint-port-number&gt;`ready`=&lt;true if available, false if unavailalbe&gt;                                                      | STABLE       |

docs/metrics/service/endpointslice-metrics.md

@@ -0,0 +1,11 @@
+# Endpoint Metrics
+
+| Metric name                    | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | Status       |
+| ------------------------------ | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_endpointslice_annotations | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `endpointslice`=&lt;endpointslice-name&gt; <br> `namespace`=&lt;endpointslice-namespace&gt; <br> `annotation_ENDPOINTSLICE_ANNOTATION`=&lt;ENDPOINTSLICE_ANNOTATION&gt;                                                                                                                                                                                                                                                                                                                                                                                                         | EXPERIMENTAL |
+| kube_endpointslice_info        | Gauge       |                                                                                                                           | `endpointslice`=&lt;endpointslice-name&gt; <br> `namespace`=&lt;endpointslice-namespace&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | EXPERIMENTAL |
+| kube_endpointslice_ports       | Gauge       |                                                                                                                           | `endpointslice`=&lt;endpointslice-name&gt; <br> `namespace`=&lt;endpointslice-namespace&gt; <br> `port_name`=&lt;endpointslice-port-name&gt; <br> `port_protocol`=&lt;endpointslice-port-protocol&gt; <br> `port_number`=&lt;endpointslice-port-number&gt;                                                                                                                                                                                                                                                                                                                      | EXPERIMENTAL |
+| kube_endpointslice_endpoints   | Gauge       |                                                                                                                           | `endpointslice`=&lt;endpointslice-name&gt; <br> `namespace`=&lt;endpointslice-namespace&gt; <br> `ready`=&lt;endpointslice-ready&gt; <br> `serving`=&lt;endpointslice-serving&gt; <br> `terminating`=&lt;endpointslice-terminating&gt; <br> `hostname`=&lt;endpointslice-hostname&gt; <br> `targetref_kind`=&lt;endpointslice-targetref-kind&gt; <br> `targetref_name`=&lt;endpointslice-targetref-name&gt; <br> `targetref_namespace`=&lt;endpointslice-targetref-namespace&gt; <br> `nodename`=&lt;endpointslice-nodename&gt; <br> `endpoint_zone`=&lt;endpointslice-zone&gt; | EXPERIMENTAL |
+| kube_endpointslice_endpoints_hints   | Gauge       |  Each line is a hint applied to an endpoint-slice                                                                   | `endpointslice`=&lt;endpointslice-name&gt; <br> `namespace`=&lt;endpointslice-namespace&gt; <br> `address`=&lt;endpointslice-address[0]&gt;  <br> `for_zone`=&lt;endpointslice-hint&gt; | EXPERIMENTAL |
+| kube_endpointslice_labels      | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `endpointslice`=&lt;endpointslice-name&gt; <br> `namespace`=&lt;endpointslice-namespace&gt; <br> `label_ENDPOINTSLICE_LABEL`=&lt;ENDPOINTSLICE_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                                                                        | EXPERIMENTAL |
+| kube_endpointslice_created     | Gauge       |                                                                                                                           | `endpointslice`=&lt;endpointslice-name&gt; <br> `namespace`=&lt;endpointslice-namespace&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     | EXPERIMENTAL |

docs/metrics/service/ingress-metrics.md

@@ -0,0 +1,11 @@
+# Ingress Metrics
+
+| Metric name                            | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | Status       |
+| -------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ------------ |
+| kube_ingress_annotations               | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `annotation_INGRESS_ANNOTATION`=&lt;ANNOTATION_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                                                                          | EXPERIMENTAL |
+| kube_ingress_info                      | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `ingressclass`=&lt;ingress-class&gt; or `_default` if not set                                                                                                                                                                                                                                                                                                                                                                                                                     | STABLE       |
+| kube_ingress_labels                    | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `label_INGRESS_LABEL`=&lt;INGRESS_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                       | STABLE       |
+| kube_ingress_created                   | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | STABLE       |
+| kube_ingress_metadata_resource_version | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | EXPERIMENTAL |
+| kube_ingress_path                      | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `host`=&lt;ingress-host&gt; <br> `path`=&lt;ingress-path&gt; <br> `path_type`=&lt;ingress-path type&gt; <br> If path served by Service Backend <br> `service_name`=&lt;service name for the path&gt; <br> `service_port`=&lt;service port for the path&gt; <br> If path served by Resource Backend <br> `resource_api_group`=&lt;resource backend api group&gt; <br> `resource_kind`=&lt;resource backend kind&gt; <br> `resource_name`=&lt;resource backend name&gt; | STABLE       |
+| kube_ingress_tls                       | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `tls_host`=&lt;tls hostname&gt; <br> `secret`=&lt;tls secret name&gt;                                                                                                                                                                                                                                                                                                                                                                                                             | STABLE       |

docs/metrics/service/ingressclass-metrics.md

@@ -0,0 +1,8 @@
+# IngressClass Metrics
+
+| Metric name                   | Metric type | Description                                                                                                               | Labels/tags                                                                                                        | Status       |
+| ----------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------ | ------------ |
+| kube_ingressclass_annotations | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `ingressclass`=&lt;ingressclass-name&gt; <br> `annotation_INGRESSCLASS_ANNOTATION`=&lt;INGRESSCLASS_ANNOTATION&gt; | EXPERIMENTAL |
+| kube_ingressclass_info        | Gauge       |                                                                                                                           | `ingressclass`=&lt;ingressclass-name&gt; <br> `controller`=&lt;ingress-controller-name&gt; <br>                    | EXPERIMENTAL |
+| kube_ingressclass_labels      | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `ingressclass`=&lt;ingressclass-name&gt; <br> `label_INGRESSCLASS_LABEL`=&lt;INGRESSCLASS_LABEL&gt;                | EXPERIMENTAL |
+| kube_ingressclass_created     | Gauge       |                                                                                                                           | `ingressclass`=&lt;ingressclass-name&gt;                                                                           | EXPERIMENTAL |

docs/metrics/service/service-metrics.md

@@ -0,0 +1,11 @@
+# Service Metrics
+
+| Metric name                               | Metric type | Description                                                                                                               | Unit (where applicable) | Labels/tags                                                                                                                                                                                                                                                       | Status       |
+| ----------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_service_annotations                  | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `annotation_SERVICE_ANNOTATION`=&lt;SERVICE_ANNOTATION&gt;                                                                                          | EXPERIMENTAL |
+| kube_service_info                         | Gauge       | Information about service                                                                                                 |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `cluster_ip`=&lt;service cluster ip&gt; <br> `external_name`=&lt;service external name&gt; <br> `external_traffic_policy`=&lt;service external traffic policy&gt; <br> `load_balancer_ip`=&lt;service load balancer ip&gt; | STABLE       |
+| kube_service_labels                       | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `label_SERVICE_LABEL`=&lt;SERVICE_LABEL&gt;                                                                                                         | STABLE       |
+| kube_service_created                      | Gauge       | Unix creation timestamp                                                                                                   | seconds                 | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt;                                                                                                                                                          | STABLE       |
+| kube_service_spec_type                    | Gauge       | Type about service                                                                                                        |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `type`=&lt;ClusterIP\|NodePort\|LoadBalancer\|ExternalName&gt;                                                                                      | STABLE       |
+| kube_service_spec_external_ip             | Gauge       | Service external ips. One series for each ip                                                                              |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `external_ip`=&lt;external-ip&gt;                                                                                                                   | STABLE       |
+| kube_service_status_load_balancer_ingress | Gauge       | Service load balancer ingress status                                                                                      |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `ip`=&lt;load-balancer-ingress-ip&gt; <br> `hostname`=&lt;load-balancer-ingress-hostname&gt;                                                        | STABLE       |

docs/metrics/storage/configmap-metrics.md

@@ -0,0 +1,9 @@
+# ConfigMap Metrics
+
+| Metric name                              | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                         | Status       |
+| ---------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_configmap_annotations               | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `configmap`=&lt;configmap-name&gt; <br> `namespace`=&lt;configmap-namespace&gt; <br> `annotation_CONFIGMAP_ANNOTATION`=&lt;CONFIGMAP_ANNOTATION&gt; | EXPERIMENTAL |
+| kube_configmap_labels                    | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `configmap`=&lt;configmap-name&gt; <br> `namespace`=&lt;configmap-namespace&gt; <br> `label_CONFIGMAP_LABEL`=&lt;CONFIGMAP_LABEL&gt;                | STABLE       |
+| kube_configmap_info                      | Gauge       |                                                                                                                           | `configmap`=&lt;configmap-name&gt; <br> `namespace`=&lt;configmap-namespace&gt;                                                                     | STABLE       |
+| kube_configmap_created                   | Gauge       |                                                                                                                           | `configmap`=&lt;configmap-name&gt; <br> `namespace`=&lt;configmap-namespace&gt;                                                                     | STABLE       |
+| kube_configmap_metadata_resource_version | Gauge       |                                                                                                                           | `configmap`=&lt;configmap-name&gt; <br> `namespace`=&lt;configmap-namespace&gt;                                                                     | EXPERIMENTAL |

docs/metrics/storage/persistentvolume-metrics.md

@@ -0,0 +1,37 @@
+# PersistentVolume Metrics
+
+| Metric name                              | Metric type | Description                                                                                                               | Unit (where applicable) | Labels/tags                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                          | Status       |
+|------------------------------------------|-------------|---------------------------------------------------------------------------------------------------------------------------|-------------------------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|--------------|
+| kube_persistentvolume_annotations        | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) |                         | `persistentvolume`=&lt;persistentvolume-name&gt; <br> `annotation_PERSISTENTVOLUME_ANNOTATION`=&lt;PERSISTENTVOLUME_ANNOTATION&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | EXPERIMENTAL |
+| kube_persistentvolume_capacity_bytes     | Gauge       |                                                                                                                           |                         | `persistentvolume`=&lt;pv-name&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | STABLE       |
+| kube_persistentvolume_status_phase       | Gauge       |                                                                                                                           |                         | `persistentvolume`=&lt;pv-name&gt; <br>`phase`=&lt;Bound\|Failed\|Pending\|Available\|Released&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | STABLE       |
+| kube_persistentvolume_claim_ref          | Gauge       |                                                                                                                           |                         | `persistentvolume`=&lt;pv-name&gt; <br>`claim_namespace`=`<namespace>`; <br>`name`=`<name>`;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | STABLE       |
+| kube_persistentvolume_labels             | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           |                         | `persistentvolume`=&lt;persistentvolume-name&gt; <br> `label_PERSISTENTVOLUME_LABEL`=&lt;PERSISTENTVOLUME_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  | STABLE       |
+| kube_persistentvolume_info               | Gauge       | Information about Persistent Volumes                                                                                      |                         | `persistentvolume`=&lt;pv-name&gt; <br> `storageclass`=&lt;storageclass-name&gt; <br> `gce_persistent_disk_name`=&lt;pd-name&gt; <br> `host_path`=&lt;path-of-a-host-volume&gt; <br> `host_path_type`=&lt;host-mount-type&gt; <br> `ebs_volume_id`=&lt;ebs-volume-id&gt; <br> `azure_disk_name`=&lt;azure-disk-name&gt; <br> `fc_wwids`=&lt;fc-wwids-comma-separated&gt; <br> `fc_lun`=&lt;fc-lun&gt; <br> `fc_target_wwns`=&lt;fc-target-wwns-comma-separated&gt; <br> `iscsi_target_portal`=&lt;iscsi-target-portal&gt; <br> `iscsi_iqn`=&lt;iscsi-iqn&gt; <br> `iscsi_lun`=&lt;iscsi-lun&gt; <br> `iscsi_initiator_name`=&lt;iscsi-initiator-name&gt; <br> `local_path`=&lt;path-of-a-local-volume&gt; <br> `local_fs`=&lt;local-volume-fs-type&gt; <br> `nfs_server`=&lt;nfs-server&gt; <br> `nfs_path`=&lt;nfs-path&gt; <br> `csi_driver`=&lt;csi-driver&gt; <br> `csi_volume_handle`=&lt;csi-volume-handle&gt; <br> `reclaim_policy`=&lt;reclaim-policy&gt; | STABLE       |
+| kube_persistentvolume_created            | Gauge       | Unix creation timestamp                                                                                                   | seconds                 | `persistentvolume`=&lt;persistentvolume-name&gt; <br>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | EXPERIMENTAL |
+| kube_persistentvolume_deletion_timestamp | Gauge       | Unix deletion timestamp                                                                                                   | seconds                 | `persistentvolume`=&lt;persistentvolume-name&gt; <br>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | EXPERIMENTAL |
+| kube_persistentvolume_csi_attributes     | Gauge       | CSI attributes of the Persistent Volume, disabled by default, manage with [--metric-opt-in-list](../../developer/cli-arguments.md))     |                         | `persistentvolume`=&lt;persistentvolume-name&gt; <br> `csi_mounter`=&lt;csi-mounter&gt; <br> `csi_map_options`=&lt;csi-map-options&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | EXPERIMENTAL |
+| kube_persistentvolume_volume_mode       | Gauge       | Volume Mode information for the PersistentVolume.                                                                          |                         | `persistentvolume`=&lt;persistentvolume-name&gt; <br>`volumemode`=&lt;volumemode&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | EXPERIMENTAL       |
+
+## Useful metrics queries
+
+### How to retrieve non-standard PV state
+
+It is not straightforward to get the PV states for certain cases like "Terminating" since it is not stored behind a field in the `PersistentVolume.Status`.
+
+So to mimic the [logic](https://github.com/kubernetes/kubernetes/blob/v1.27.2/pkg/printers/internalversion/printers.go#L1838) used by the `kubectl` command line, you will need to compose multiple metrics.
+
+Here is an example of a Prometheus rule that can be used to alert on a PV that has been in the `Terminating` state for more than `5m`.
+
+```yaml
+groups:
+- name: PV state
+  rules:
+  - alert: PVBlockedInTerminatingState
+    expr: kube_persistentvolume_deletion_timestamp * on(persistentvolume) group_left() (kube_persistentvolume_status_phase{phase="Bound"} == 1) > 0
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: PV {{$labels.persistentvolume}} blocked in Terminating state.
+```

docs/metrics/storage/persistentvolumeclaim-metrics.md

@@ -0,0 +1,40 @@
+# PersistentVolumeClaim Metrics
+
+| Metric name                                                | Metric type | Description                                                                                                               | Unit (where applicable) | Labels/tags                                                                                                                                                                                                                                                  | Status       |
+| ---------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------- | ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------ | ------------ |
+| kube_persistentvolumeclaim_annotations                     | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) |                         | `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; <br> `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `annotation_PERSISTENTVOLUMECLAIM_ANNOTATION`=&lt;PERSISTENTVOLUMECLAIM_ANNOATION&gt;                                               | EXPERIMENTAL |
+| kube_persistentvolumeclaim_access_mode                     | Gauge       |                                                                                                                           |                         | `access_mode`=&lt;persistentvolumeclaim-access-mode&gt; <br>`namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt;                                                                              | STABLE       |
+| kube_persistentvolumeclaim_info                            | Gauge       |                                                                                                                           |                         | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; <br> `storageclass`=&lt;persistentvolumeclaim-storageclassname&gt;<br>`volumename`=&lt;volumename&gt;<br>`volumemode`=&lt;volumemode&gt; | STABLE       |
+| kube_persistentvolumeclaim_labels                          | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           |                         | `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; <br> `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `label_PERSISTENTVOLUMECLAIM_LABEL`=&lt;PERSISTENTVOLUMECLAIM_LABEL&gt;                                                             | STABLE       |
+| kube_persistentvolumeclaim_resource_requests_storage_bytes | Gauge       |                                                                                                                           |                         | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt;                                                                                                                                          | STABLE       |
+| kube_persistentvolumeclaim_status_condition                | Gauge       |                                                                                                                           |                         | `namespace` =&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; <br> `type`=&lt;persistentvolumeclaim-condition-type&gt; <br> `status`=&lt;true\false\unknown&gt;                                       | EXPERIMENTAL |
+| kube_persistentvolumeclaim_status_phase                    | Gauge       |                                                                                                                           |                         | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt; <br> `phase`=&lt;Pending\Bound\Lost&gt;                                                                                                  | STABLE       |
+| kube_persistentvolumeclaim_created                         | Gauge       | Unix creation timestamp                                                                                                   | seconds                 | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt;                                                                                                                                          | EXPERIMENTAL |
+| kube_persistentvolumeclaim_deletion_timestamp              | Gauge       | Unix deletion timestamp                                                                                                   | seconds                 | `namespace`=&lt;persistentvolumeclaim-namespace&gt; <br> `persistentvolumeclaim`=&lt;persistentvolumeclaim-name&gt;                                                                                                                                          | EXPERIMENTAL |
+
+Note:
+
+* An empty string will be used if PVC has no storage class.
+
+## Useful metrics queries
+
+### How to retrieve non-standard PVC state
+
+It is not straightforward to get the PVC states for certain cases like "Terminating" since it is not stored behind a field in the `PersistentVolumeClaim.Status`.
+
+So to mimic the [logic](https://github.com/kubernetes/kubernetes/blob/v1.27.2/pkg/printers/internalversion/printers.go#L1883) used by the `kubectl` command line, you will need to compose multiple metrics.
+
+Here is an example of a Prometheus rule that can be used to alert on a PVC that has been in the `Terminating` state for more than `5m`.
+
+```yaml
+groups:
+- name: PVC state
+  rules:
+  - alert: PVCBlockedInTerminatingState
+    expr: kube_persistentvolumeclaim_deletion_timestamp * on(namespace, persistentvolumeclaim) group_left() (kube_persistentvolumeclaim_status_phase{phase="Bound"} == 1) > 0
+    for: 5m
+    labels:
+      severity: warning
+    annotations:
+      summary: PVC {{$labels.namespace}}/{{$labels.persistentvolumeclaim}} blocked in Terminating state.
+```

docs/metrics/storage/secret-metrics.md

@@ -0,0 +1,11 @@
+# Secret Metrics
+
+| Metric name                           | Metric type | Description                                                                                                               | Labels/tags                                                                                                                           | Status       |
+| ------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_secret_annotations               | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `secret`=&lt;secret-name&gt; <br> `namespace`=&lt;secret-namespace&gt; <br> `annotations_SECRET_ANNOTATION`=&lt;SECRET_ANNOTATION&gt; | EXPERIMENTAL |
+| kube_secret_info                      | Gauge       |                                                                                                                           | `secret`=&lt;secret-name&gt; <br> `namespace`=&lt;secret-namespace&gt;                                                                | STABLE       |
+| kube_secret_type                      | Gauge       |                                                                                                                           | `secret`=&lt;secret-name&gt; <br> `namespace`=&lt;secret-namespace&gt; <br> `type`=&lt;secret-type&gt;                                | STABLE       |
+| kube_secret_labels                    | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `secret`=&lt;secret-name&gt; <br> `namespace`=&lt;secret-namespace&gt; <br> `label_SECRET_LABEL`=&lt;SECRET_LABEL&gt;                 | STABLE       |
+| kube_secret_created                   | Gauge       |                                                                                                                           | `secret`=&lt;secret-name&gt; <br> `namespace`=&lt;secret-namespace&gt;                                                                | STABLE       |
+| kube_secret_metadata_resource_version | Gauge       |                                                                                                                           | `secret`=&lt;secret-name&gt; <br> `namespace`=&lt;secret-namespace&gt;                                                                | EXPERIMENTAL |
+| kube_secret_owner                         | Gauge       |                                                                                                                           | `secret`=&lt;secret-name&gt; <br> `namespace`=&lt;secret-namespace&gt; <br> `owner_kind`=&lt;owner kind&gt; <br> `owner_name`=&lt;owner name&gt; <br> `owner_is_controller`=&lt;whether owner is controller&gt; | EXPERIMENTAL |

docs/metrics/storage/storageclass-metrics.md

@@ -0,0 +1,8 @@
+# StorageClass Metrics
+
+| Metric name                   | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                                                             | Status       |
+| ----------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_storageclass_annotations | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `storageclass`=&lt;storageclass-name&gt; <br> `annotation_STORAGECLASS_ANNOTATION`=&lt;STORAGECLASS_ANNOTATION&gt;                                                                                                      | EXPERIMENTAL |
+| kube_storageclass_info        | Gauge       |                                                                                                                           | `storageclass`=&lt;storageclass-name&gt; <br> `provisioner`=&lt;storageclass-provisioner&gt; <br> `reclaim_policy`=&lt;storageclass-reclaimPolicy&gt; <br> `volume_binding_mode`=&lt;storageclass-volumeBindingMode&gt; | STABLE       |
+| kube_storageclass_labels      | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `storageclass`=&lt;storageclass-name&gt; <br> `label_STORAGECLASS_LABEL`=&lt;STORAGECLASS_LABEL&gt;                                                                                                                     | STABLE       |
+| kube_storageclass_created     | Gauge       |                                                                                                                           | `storageclass`=&lt;storageclass-name&gt;                                                                                                                                                                                | STABLE       |

docs/metrics/storage/volumeattachment-metrics.md

@@ -0,0 +1,10 @@
+# VolumeAttachment Metrics
+
+| Metric name                                        | Metric type | Description                                                                                                     | Labels/tags                                                                                                          | Status       |
+| -------------------------------------------------- | ----------- | --------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_volumeattachment_info                         | Gauge       |                                                                                                                 | `volumeattachment`=&lt;volumeattachment-name&gt; <br> `attacher`=&lt;attacher-name&gt; <br> `node`=&lt;node-name&gt; | EXPERIMENTAL |
+| kube_volumeattachment_created                      | Gauge       |                                                                                                                 | `volumeattachment`=&lt;volumeattachment-name&gt;                                                                     | EXPERIMENTAL |
+| kube_volumeattachment_labels                       | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md) | `volumeattachment`=&lt;volumeattachment-name&gt; <br> `label_VOLUMEATTACHMENT_LABEL`=&lt;VOLUMEATTACHMENT_LABEL&gt;  | EXPERIMENTAL |
+| kube_volumeattachment_spec_source_persistentvolume | Gauge       |                                                                                                                 | `volumeattachment`=&lt;volumeattachment-name&gt; <br> `volumename`=&lt;persistentvolume-name&gt;                     | EXPERIMENTAL |
+| kube_volumeattachment_status_attached              | Gauge       |                                                                                                                 | `volumeattachment`=&lt;volumeattachment-name&gt;                                                                     | EXPERIMENTAL |
+| kube_volumeattachment_status_attachment_metadata   | Gauge       |                                                                                                                 | `volumeattachment`=&lt;volumeattachment-name&gt; <br> `metadata_METADATA_KEY`=&lt;METADATA_VALUE&gt;                 | EXPERIMENTAL |

docs/metrics/workload/cronjob-metrics.md

@@ -0,0 +1,17 @@
+# CronJob Metrics
+
+| Metric name                                    | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                     | Status       |
+| ---------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_cronjob_annotations                       | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `annotation_CRONJOB_ANNOTATION`=&lt;CRONJOB_ANNOTATION&gt;                       | EXPERIMENTAL |
+| kube_cronjob_info                              | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `schedule`=&lt;schedule&gt; <br> `concurrency_policy`=&lt;concurrency-policy&gt;  <br> `timezone`=&lt;timezone&gt; | STABLE       |
+| kube_cronjob_labels                            | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `label_CRONJOB_LABEL`=&lt;CRONJOB_LABEL&gt;                                      | STABLE       |
+| kube_cronjob_created                           | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | STABLE       |
+| kube_cronjob_next_schedule_time                | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | STABLE       |
+| kube_cronjob_status_active                     | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | STABLE       |
+| kube_cronjob_status_last_schedule_time         | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | STABLE       |
+| kube_cronjob_status_last_successful_time       | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | EXPERIMENTAL |
+| kube_cronjob_spec_suspend                      | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | STABLE       |
+| kube_cronjob_spec_starting_deadline_seconds    | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | STABLE       |
+| kube_cronjob_metadata_resource_version         | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | STABLE       |
+| kube_cronjob_spec_successful_job_history_limit | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | EXPERIMENTAL |
+| kube_cronjob_spec_failed_job_history_limit     | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | EXPERIMENTAL |

docs/metrics/workload/daemonset-metrics.md

@@ -0,0 +1,16 @@
+# DaemonSet Metrics
+
+| Metric name                                    | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                         | Status       |
+| ---------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_daemonset_annotations                     | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; <br> `annotation_DAEMONSET_ANNOTATION`=&lt;DAEMONSET_ANNOTATION&gt; | EXPERIMENTAL |
+| kube_daemonset_created                         | Gauge       |                                                                                                                           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt;                                                                     | STABLE       |
+| kube_daemonset_status_current_number_scheduled | Gauge       |                                                                                                                           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt;                                                                     | STABLE       |
+| kube_daemonset_status_desired_number_scheduled | Gauge       |                                                                                                                           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt;                                                                     | STABLE       |
+| kube_daemonset_status_number_available         | Gauge       |                                                                                                                           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt;                                                                     | STABLE       |
+| kube_daemonset_status_number_misscheduled      | Gauge       |                                                                                                                           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt;                                                                     | STABLE       |
+| kube_daemonset_status_number_ready             | Gauge       |                                                                                                                           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt;                                                                     | STABLE       |
+| kube_daemonset_status_number_unavailable       | Gauge       |                                                                                                                           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt;                                                                     | STABLE       |
+| kube_daemonset_status_observed_generation      | Gauge       |                                                                                                                           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt;                                                                     | STABLE       |
+| kube_daemonset_status_updated_number_scheduled | Gauge       |                                                                                                                           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt;                                                                     | STABLE       |
+| kube_daemonset_metadata_generation             | Gauge       |                                                                                                                           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt;                                                                     | STABLE       |
+| kube_daemonset_labels                          | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `daemonset`=&lt;daemonset-name&gt; <br> `namespace`=&lt;daemonset-namespace&gt; <br> `label_DAEMONSET_LABEL`=&lt;DAEMONSET_LABEL&gt;                | STABLE       |

docs/metrics/workload/deployment-metrics.md

@@ -0,0 +1,19 @@
+# Deployment Metrics
+
+| Metric name                                                 | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                 | Status       |
+| ----------------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_deployment_annotations                                 | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; <br> `annotation_DEPLOYMENT_ANNOTATION`=&lt;DEPLOYMENT_ANNOTATION&gt;                    | EXPERIMENTAL |
+| kube_deployment_status_replicas                             | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_status_replicas_ready                       | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_status_replicas_available                   | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_status_replicas_unavailable                 | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_status_replicas_updated                     | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_status_observed_generation                  | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_status_condition                            |                                                     Gauge                                                                                                                            |       |  `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; <br> `reason`=&lt;deployment-transition-reason&gt; <br> `condition`=&lt;deployment-condition&gt; <br> `status`=&lt;true\|false\|unknown&gt;                                                                                                               | STABLE       |
+| kube_deployment_spec_replicas                               | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_spec_paused                                 | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_spec_strategy_rollingupdate_max_unavailable | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_spec_strategy_rollingupdate_max_surge       | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_metadata_generation                         | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
+| kube_deployment_labels                                      | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; <br> `label_DEPLOYMENT_LABEL`=&lt;DEPLOYMENT_LABEL&gt;                                   | STABLE       |
+| kube_deployment_created                                     | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |

docs/metrics/workload/horizontalpodautoscaler-metrics.md

@@ -0,0 +1,15 @@
+# Horizontal Pod Autoscaler Metrics
+
+| Metric name                                          | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                                                                                              | Status       |
+| ---------------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
+| kube_horizontalpodautoscaler_info                    | Gauge       |                                                                                                                           | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; <br> `scaletargetref_api_version`=&lt;hpa-target-api-version&gt; <br> `scaletargetref_kind`=&lt;hpa-target-kind&gt; <br> `scaletargetref_name`=&lt;hpa-target-name&gt; | EXPERIMENTAL |
+| kube_horizontalpodautoscaler_annotations             | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt;                                                                                                                                                                        | EXPERIMENTAL |
+| kube_horizontalpodautoscaler_labels                  | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt;                                                                                                                                                                        | STABLE       |
+| kube_horizontalpodautoscaler_metadata_generation     | Gauge       |                                                                                                                           | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt;                                                                                                                                                                        | STABLE       |
+| kube_horizontalpodautoscaler_spec_max_replicas       | Gauge       |                                                                                                                           | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt;                                                                                                                                                                        | STABLE       |
+| kube_horizontalpodautoscaler_spec_min_replicas       | Gauge       |                                                                                                                           | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt;                                                                                                                                                                        | STABLE       |
+| kube_horizontalpodautoscaler_spec_target_metric      | Gauge       |                                                                                                                           | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; <br> `metric_name`=&lt;metric-name&gt; <br> `metric_target_type`=&lt;value\|utilization\|average&gt;                                                                   | EXPERIMENTAL |
+| kube_horizontalpodautoscaler_status_target_metric    | Gauge       |                                                                                                                           | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; <br> `metric_name`=&lt;metric-name&gt; <br> `metric_target_type`=&lt;value\|utilization\|average&gt;                                                                   | EXPERIMENTAL |
+| kube_horizontalpodautoscaler_status_condition        | Gauge       |                                                                                                                           | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt; <br> `condition`=&lt;hpa-condition&gt; <br> `status`=&lt;true\|false\|unknown&gt;                                                                                      | STABLE       |
+| kube_horizontalpodautoscaler_status_current_replicas | Gauge       |                                                                                                                           | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt;                                                                                                                                                                        | STABLE       |
+| kube_horizontalpodautoscaler_status_desired_replicas | Gauge       |                                                                                                                           | `horizontalpodautoscaler`=&lt;hpa-name&gt; <br> `namespace`=&lt;hpa-namespace&gt;                                                                                                                                                                        | STABLE       |