Merge pull request #2719 from Rishab87/reason-label

feat: include reason label to `kube_deployment_status_condition`

.github/ISSUE_TEMPLATE/feature_request.md

@@ -9,10 +9,10 @@ assignees: ''
 
 <!-- Please only use this template for submitting feature requests -->
 
-**What would you like to be added**:
+**What would you like to be added:**
 
-**Why is this needed**:
+**Why is this needed:**
 
-**Describe the solution you'd like**
+**Describe the solution you'd like:**
 
-**Additional context**
+**Additional context:**

.github/PULL_REQUEST_TEMPLATE.md

@@ -5,9 +5,10 @@
 4. If the PR is unfinished, see how to mark it: https://git.k8s.io/community/contributors/guide/pull-requests.md#marking-unfinished-pull-requests
 -->
 
-**What this PR does / why we need it**:
+<!-- markdownlint-disable-next-line MD041 -->
+**What this PR does / why we need it:**
 
-**How does this change affect the cardinality of KSM**: *(increases, decreases or does not change cardinality)*
+**How does this change affect the cardinality of KSM:** *(increases, decreases or does not change cardinality)*
 
-**Which issue(s) this PR fixes** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*:
+**Which issue(s) this PR fixes:** *(optional, in `fixes #<issue number>(, fixes #<issue_number>, ...)` format, will close the issue(s) when PR gets merged)*
 Fixes #

.github/dependabot.yml

@@ -8,10 +8,6 @@ updates:
           - "k8s.io*"
     schedule:
       interval: "weekly"
-  - package-ecosystem: "gomod"
-    directory: "tools"
-    schedule:
-      interval: "weekly"
   - package-ecosystem: "github-actions"
     directory: "/"
     schedule:

.github/workflows/ci.yml

@@ -20,8 +20,8 @@ env:
   E2E_SETUP_KIND: yes
   E2E_SETUP_KUBECTL: yes
   SUDO: sudo
-  GO_VERSION: "^1.22"
-  GOLANGCI_LINT_VERSION: "v1.56.2"
+  GO_VERSION: "^1.24"
+  GOLANGCI_LINT_VERSION: "v2.0.2"
 
 jobs:
   ci-go-lint:
@@ -29,18 +29,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code into the Go module directory
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
       with:
         go-version: ${{ env.GO_VERSION }}
       id: go
 
-    - name: Setup environment
-      run: |
-        make install-tools
-
     - name: Lint
       run: |
         curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sh -s -- -b $(go env GOPATH)/bin ${{ env.GOLANGCI_LINT_VERSION }}
@@ -51,18 +47,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code into the Go module directory
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
       with:
         go-version: ${{ env.GO_VERSION }}
       id: go
 
-    - name: Setup environment
-      run: |
-        make install-tools
-
     - name: Validate generated manifests
       run: |
         make validate-manifests
@@ -72,18 +64,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code into the Go module directory
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
       with:
         go-version: ${{ env.GO_VERSION }}
       id: go
 
-    - name: Setup environment
-      run: |
-        make install-tools
-
     - name: Validate go modules
       run: |
         make validate-modules
@@ -93,18 +81,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code into the Go module directory
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
       with:
         go-version: ${{ env.GO_VERSION }}
       id: go
 
-    - name: Setup environment
-      run: |
-        make install-tools
-
     - name: Check that all metrics are documented and templates have no delta
       run: |
         make doccheck
@@ -114,18 +98,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code into the Go module directory
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
       with:
         go-version: ${{ env.GO_VERSION }}
       id: go
 
-    - name: Setup environment
-      run: |
-        make install-tools
-
     - name: Unit tests
       run: |
         make test-unit
@@ -135,7 +115,7 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - name: Check out code into the Go module directory
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Setup promtool
         run: |
@@ -150,39 +130,41 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code into the Go module directory
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
       with:
         go-version: ${{ env.GO_VERSION }}
       id: go
 
-    - name: Setup environment
-      run: |
-        make install-tools
-
     - name: Benchmark tests
       run: |
-        make test-benchmark-compare
+        BENCHSTAT_OUTPUT_FILE=result.txt make test-benchmark-compare
+    - run: |
+        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+        cat result.txt >> "$GITHUB_STEP_SUMMARY"
+        echo "\`\`\`" >> "$GITHUB_STEP_SUMMARY"
+        cat <<EOL >> "$GITHUB_STEP_SUMMARY"
+        <hr />
+        The table shows the median and 95% confidence interval (CI) summaries for each benchmark comparing the HEAD and the BASE, and an A/B comparison under "vs base". The last column shows the statistical p-value with ten runs (n=10).
+        The last row has the Geometric Mean (geomean) for the given rows in the table.
+        Refer to <a href="https://pkg.go.dev/golang.org/x/perf/cmd/benchstat">benchstat's documentation</a> for more help.
+        EOL
 
   ci-build-kube-state-metrics:
     name: ci-build-kube-state-metrics
     runs-on: ubuntu-latest
     steps:
     - name: Check out code into the Go module directory
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
       with:
         go-version: ${{ env.GO_VERSION }}
       id: go
 
-    - name: Setup environment
-      run: |
-        make install-tools
-
     - name: Build
       run: |
         make build
@@ -192,18 +174,14 @@ jobs:
     runs-on: ubuntu-latest
     steps:
     - name: Check out code into the Go module directory
-      uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
     - name: Set up Go 1.x
-      uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+      uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
       with:
         go-version: ${{ env.GO_VERSION }}
       id: go
 
-    - name: Setup environment
-      run: |
-        make install-tools
-
     - name: End-to-end tests
       run: |
         make e2e

.github/workflows/govulncheck.yml

@@ -6,7 +6,7 @@ on:
     - cron: '0 0 * * 1'
 
 env:
-  GO_VERSION: "^1.22"
+  GO_VERSION: "^1.24"
 
 permissions:
   contents: read
@@ -15,10 +15,10 @@ jobs:
   ci-security-checks:
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+      - uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
         name: Checkout code
       - name: Set up Go 1.x
-        uses: actions/setup-go@0a12ed9d6a96ab950c8f026ed9f722fe0da7ef32 # v5.0.2
+        uses: actions/setup-go@d35c59abb061a4a6fb18e82ac0862c26744d6ab5 # v5.5.0
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Install govulncheck binary

.github/workflows/openvex.yml

@@ -18,7 +18,7 @@ jobs:
 
     steps:
       - name: Checkout code
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Set environment variables
         run: echo "RELEASE_VERSION=${GITHUB_REF#refs/*/}" >> $GITHUB_ENV

.github/workflows/sbom.yaml

@@ -21,17 +21,17 @@ jobs:
 
     steps:
       - name: Fetch source code into GITHUB_WORKSPACE
-        uses: actions/checkout@692973e3d937129bcbf40652eb9f2f61becf3332 # v4.1.7
+        uses: actions/checkout@08c6903cd8c0fde910a37f88322edcfb5dd907a8 # v5.0.0
 
       - name: Install Kubernetes BOM
-        uses: kubernetes-sigs/release-actions/setup-bom@2f8b9ec22aedc9ce15039b6c7716aa6c2907df1c # v0.2.0
+        uses: kubernetes-sigs/release-actions/setup-bom@a30d93cf2aa029e1e4c8a6c79f766aebf429fddb # v0.3.1
 
       - name: Generate SBOM
         run: |
           bom generate \
             --dirs=. \
             --image=registry.k8s.io/kube-state-metrics/kube-state-metrics:$TAG \
-            --namespace=https://github.com/kubernetes/kube-state-metrics/releases/download/$TAG/$OUTPUT
+            --namespace=https://github.com/kubernetes/kube-state-metrics/releases/download/$TAG/$OUTPUT \
             --output=$OUTPUT
 
       - name: Upload SBOM to GitHub Release

.golangci.yml

@@ -1,15 +1,10 @@
-run:
-  deadline: 10m
-
+version: "2"
 linters:
-  disable-all: true
+  default: none
   enable:
     - gocritic
     - gocyclo
-    - gofmt
-    - goimports
     - gosec
-    - gosimple
     - govet
     - ineffassign
     - misspell
@@ -17,22 +12,49 @@ linters:
     - revive
     - staticcheck
     - unconvert
+    - unused
+  exclusions:
+    generated: lax
+    rules:
+      - linters:
+          - promlinter
+        path: _test\.go
+      - linters:
+          - gosec
+        text: 'G104:'
+      - linters:
+          - revive
+        text: 'package-comments:|var-naming:'
+      # This needs to stay as long as we support exposing v1.endpoints metrics
+      - linters:
+          - staticcheck
+        text: 'SA1019: v1.Endpoint'
+        path: 'internal/store/endpoint.*.go|internal/store/builder.go'
+      # TODO: Use functions with context https://github.com/kubernetes/kube-state-metrics/issues/2721
+      - linters:
+          - staticcheck
+        text: 'SA1019: .*List|Watch'
+        path: 'pkg/sharding/listwatch.go|pkg/watch/watch.go'
 
-linters-settings:
-  goimports:
-    local-prefixes: k8s.io/kube-state-metrics,k8s.io/kube-state-metrics/v2
-
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
 issues:
-  exclude-use-default: false
-  exclude-rules:
-    # We don't check metrics naming in the tests.
-    - path: _test\.go
-      linters:
-        - promlinter
-    # TODO(mrueg) Improve error handling
-    - text: "G104:"
-      linters:
-        - gosec
-    - text: "package-comments:"
-      linters:
-        - revive
+  max-issues-per-linter: 0
+  max-same-issues: 0
+formatters:
+  enable:
+    - gofmt
+    - goimports
+  settings:
+    goimports:
+      local-prefixes:
+        - k8s.io/kube-state-metrics
+        - k8s.io/kube-state-metrics/v2
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$

.markdownlint-cli2.jsonc

@@ -8,10 +8,12 @@
       "style": "asterisk"
     },
     "MD013": false, // https://github.com/markdownlint/markdownlint/blob/main/docs/RULES.md#md013---line-length
-    "MD024": false, // https://github.com/markdownlint/markdownlint/blob/main/docs/RULES.md#md024---multiple-headers-with-the-same-content
-    "MD033": false, // https://github.com/markdownlint/markdownlint/blob/main/docs/RULES.md#md033---inline-html
-    "MD036": false, // https://github.com/markdownlint/markdownlint/blob/main/docs/RULES.md#md036---emphasis-used-instead-of-a-header
-    "MD040": false, // https://github.com/markdownlint/markdownlint/blob/main/docs/RULES.md#md040---fenced-code-blocks-should-have-a-language-specified
-    "MD041": false // https://github.com/markdownlint/markdownlint/blob/main/docs/RULES.md#md041---first-line-in-file-should-be-a-top-level-header 
+    "MD033": {
+      "allowed_elements": [
+        "details",
+        "summary",
+        "br"
+      ]
+    }
   }
 }

.openvex/templates/README.md

@@ -9,7 +9,7 @@ To add new statements to publish data about a vulnerability,
 download [vexctl](https://github.com/openvex/vexctl)
 and append new statements using `vexctl add`. For example:
 
-```
+```bash
 vexctl add --in-place main.openvex.json pkg:oci/test CVE-2014-1234567 fixed
 ```
 

CHANGELOG.md

@@ -1,6 +1,62 @@
+# Changelog
+
+## v2.16.0 / 2025-06-23
+
+**Note:**
+
+* This release builds with Golang `v1.24.4`
+* This release builds with `k8s.io/client-go`: `v0.32.6`
+
+* [FEATURE] Add a `reclaim_policy` label to the `kube_persistentvolume_info` metric by @SuperQ in <https://github.com/kubernetes/kube-state-metrics/pull/2615>
+* [FEATURE] Use dlclark/regexp2 over standard library's package to support lookarounds by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2616>
+* [BUGFIX] Report correct values in `kube_pod_status_reason` metric by @carlosmorenokm1 in <https://github.com/kubernetes/kube-state-metrics/pull/2644>
+* [FEATURE] Add add `pathType` to `kube_ingress_path` by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2670>
+* [FEATURE] Introduce object limits by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2626>
+* [BUGFIX] Close reflectors once their corresponding CRDs are dropped by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2672>
+* [FEATURE] Incorporate `WithAuthenticationAndAuthorization` to support endpoint authn/z by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2686>
+
+## v2.15.0 / 2025-02-03
+
+**Note:**
+
+* This release builds with Golang `v1.23.5`
+* This release builds with `k8s.io/client-go`: `v0.32.1`
+
+* [BUGFIX] Fix CR cache for GVK all specified case by @chelseychen in <https://github.com/kubernetes/kube-state-metrics/pull/2567>
+* [BUGFIX] Deduplicate tolerations before creating metric by @RiRa12621 in <https://github.com/kubernetes/kube-state-metrics/pull/2559>
+* [BUGFIX] Make `$VERSION` 3rd-party independant by @rexagod in <https://github.com/kubernetes/kube-state-metrics/pull/2572>
+* [BUGFIX] Fix NoNodePod naming in jsonnet by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2598>
+* [BUGFIX] Panic in `util.GVRFromType` for structured types by @L3n41c in <https://github.com/kubernetes/kube-state-metrics/pull/2553>
+* [FEATURE] Add external traffic policy to kube_service_info by @jahantech in <https://github.com/kubernetes/kube-state-metrics/pull/2584>
+* [FEATURE] Promote kube_statefulset_ordinals_start from ALPHA -> STABLE by @pwschuurman in <https://github.com/kubernetes/kube-state-metrics/pull/2415>
+* [FEATURE] Add timezone to kube_cronjob_info / Make kube_cronjob_next_schedule_time timezone-aware by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2376>
+
+## v2.14.0 / 2024-11-08
+
+**Note:**
+
+* This release builds with Golang `v1.23.3`
+* This release builds with `k8s.io/client-go`: `v0.31.2`
+* This release removes `kube_endpoint_address_not_ready` and `kube_endpoint_address_available` which have been deprecated in 2022. Please use `kube_endpoint_address`as a replacement.
+
+* [BUGFIX] Use --track-unscheduled-pods to select unscheduled pods in Daemonset sharding by @CatherineF-dev in <https://github.com/kubernetes/kube-state-metrics/pull/2388>
+* [BUGFIX] Install tools so VERSION gets set by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2456>
+* [BUGFIX] Syntax errors in kube-state-metrics.libsonnet by @jeffmccune in <https://github.com/kubernetes/kube-state-metrics/pull/2454>
+* [BUGFIX] Set kube_job_status_failed metric even when there are no job.Status.Conditions present by @richabanker in <https://github.com/kubernetes/kube-state-metrics/pull/2485>
+* [BUGFIX] de-duplication of custom resource metrics by @bartlettc22 in <https://github.com/kubernetes/kube-state-metrics/pull/2502>
+* [BUGFIX] Configure sharding every time MetricsHandler.Run runs by @wallee94 in <https://github.com/kubernetes/kube-state-metrics/pull/2478>
+* [BUGFIX] Panic in `util.GVRFromType` for core objects by @L3n41c in <https://github.com/kubernetes/kube-state-metrics/pull/2535>
+* [BUGFIX] Big memory value overflow by @leiwingqueen in <https://github.com/kubernetes/kube-state-metrics/pull/2540>
+* [BUGFIX] Expose empty labels by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2539>
+* [BUGFIX] CustomResourceMetrics: Convert status condition Unknown to a valid value by @Haleygo in <https://github.com/kubernetes/kube-state-metrics/pull/2536>
+* [CHANGE] Remove deprecated endpoint address metric by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2527>
+* [FEATURE] Add new metric kube_job_status_suspended by @Indresh2410 in <https://github.com/kubernetes/kube-state-metrics/pull/2542>
+* [FEATURE] Move endpoint ports into address metric by @mrueg in <https://github.com/kubernetes/kube-state-metrics/pull/2503>
+* [ENHANCEMENT] Use concurrent map when storing metrics by @rarruda in <https://github.com/kubernetes/kube-state-metrics/pull/2510>
+
 ## v2.13.0 / 2024-07-18
 
-### Note
+**Note:**
 
 * This release builds with Golang `v1.22.5`.
 * This release builds with `k8s.io/client-go`: `v0.30.3`.
@@ -15,7 +71,7 @@
 
 ## v2.12.0 / 2024-04-02
 
-### Note
+**Note:**
 
 * This release addresses a critical issue where scraping the exposition data for certain types caused metrics-backends to crash: <https://github.com/kubernetes/kube-state-metrics/issues/2248>.
 * This release builds with `k8s.io/client-go`: `v0.29.3`.
@@ -26,7 +82,7 @@
 
 ## v2.11.0 / 2024-03-04
 
-### Note
+**Note:**
 
 This release builds with Golang `v1.21.8`.
 
@@ -43,7 +99,7 @@ This release builds with Golang `v1.21.8`.
 
 ## v2.10.1 / 2023-10-09
 
-### Note
+**Note:**
 
 * This release addresses a regression introduced in [#2105](https://github.com/kubernetes/kube-state-metrics/pull/2105).
 
@@ -52,7 +108,7 @@ This release builds with Golang `v1.21.8`.
 
 ## v2.10.0 / 2023-08-31
 
-### Note
+**Note:**
 
 * Label and annotation metrics aren't exposed by default anymore to reduce the memory usage of the default configuration of kube-state-metrics. Before this change, they used to only include the name and namespace of the objects which is not relevant to users not opting in these metrics.
 
@@ -85,7 +141,7 @@ This release does not incorporate any user-facing changes. Re-running release pr
 
 The changes mentioned below are only the user-facing ones. For a list of the complete set of changes, refer the changelog associated with the release tag.
 
-### Note
+**Note:**
 
 * The deprecated experimental VerticalPodAutoscaler metrics are no longer supported, and have been removed. We recommend to use CustomResourceState metrics to gather metrics from custom resources like the Vertical Pod Autoscaler.
 * #2004 regulated label names to adhere with [OTel-Prometheus standards](https://github.com/open-telemetry/opentelemetry-specification/blob/8946dfc6a2302f78b0224fcc3f4dfb816a7bb1f4/specification/compatibility/prometheus_and_openmetrics.md?plain=1#L224-L229), so existing label names that do not follow the same may be replaced by the ones that do. Please refer to the PR for more details.

Dockerfile

@@ -1,6 +1,6 @@
-ARG GOVERSION=1.22
+ARG GOVERSION=1.24
 ARG GOARCH
-FROM golang:${GOVERSION} as builder
+FROM golang:${GOVERSION} AS builder
 ARG GOARCH
 ENV GOARCH=${GOARCH}
 WORKDIR /go/src/k8s.io/kube-state-metrics/
@@ -8,7 +8,7 @@ COPY . /go/src/k8s.io/kube-state-metrics/
 
 RUN make build-local
 
-FROM gcr.io/distroless/static:latest-${GOARCH}
+FROM gcr.io/distroless/static-debian12:latest-${GOARCH}
 COPY --from=builder /go/src/k8s.io/kube-state-metrics/kube-state-metrics /
 
 USER nobody

MAINTAINER.md

@@ -2,7 +2,7 @@
 
 kube-state-metrics is welcoming contributions from the community. If you are interested in intensifying your contributions and becoming a maintainer, this doc describes the necessary steps.
 
-As part of the Kubernetes project, we use the community membership process as described [here](https://github.com/kubernetes/community/blob/master/community-membership.md). We do not adhere strictly to the numbers of contributions and reviews. Still as becoming a maintainer is a trust-based process and we desire positive outcomes for the project, we look for a long-term interest and engagement.
+As part of the Kubernetes project, we rely on the [community membership process](https://github.com/kubernetes/community/blob/master/community-membership.md). We do not adhere strictly to the numbers of contributions and reviews. Still as becoming a maintainer is a trust-based process and we desire positive outcomes for the project, we look for a long-term interest and engagement.
 
 ## Adding a new reviewer
 

Makefile

@@ -2,12 +2,10 @@ FLAGS =
 TESTENVVAR =
 REGISTRY ?= gcr.io/k8s-staging-kube-state-metrics
 TAG_PREFIX = v
-VERSION = $(shell gomplate -d config=./data.yaml --in '{{ (datasource "config").version }}')
+VERSION = $(shell grep '^version:' data.yaml | grep -oE "[0-9]+.[0-9]+.[0-9]+")
 TAG ?= $(TAG_PREFIX)$(VERSION)
 LATEST_RELEASE_BRANCH := release-$(shell echo $(VERSION) | grep -ohE "[0-9]+.[0-9]+")
 BRANCH = $(strip $(shell git rev-parse --abbrev-ref HEAD))
-DOCKER_CLI ?= docker
-PROMTOOL_CLI ?= promtool
 PKGS = $(shell go list ./... | grep -v /vendor/ | grep -v /tests/e2e)
 ARCH ?= $(shell go env GOARCH)
 BUILD_DATE = $(shell date -u +'%Y-%m-%dT%H:%M:%SZ')
@@ -15,14 +13,21 @@ GIT_COMMIT ?= $(shell git rev-parse --short HEAD)
 OS ?= $(shell uname -s | tr A-Z a-z)
 ALL_ARCH = amd64 arm arm64 ppc64le s390x
 PKG = github.com/prometheus/common
-PROMETHEUS_VERSION = 2.53.1
-GO_VERSION = 1.22.5
+PROMETHEUS_VERSION = 3.4.1
+GO_VERSION = 1.24.4
 IMAGE = $(REGISTRY)/kube-state-metrics
 MULTI_ARCH_IMG = $(IMAGE)-$(ARCH)
 USER ?= $(shell id -u -n)
 HOST ?= $(shell hostname)
-MARKDOWNLINT_CLI2_VERSION = 0.13.0
+MARKDOWNLINT_CLI2_VERSION = 0.18.1
 
+DOCKER_CLI ?= docker
+PROMTOOL_CLI ?= promtool
+GOMPLATE_CLI ?= go tool github.com/hairyhenderson/gomplate/v4/cmd/gomplate 
+GOJSONTOYAML_CLI ?= go tool github.com/brancz/gojsontoyaml
+EMBEDMD_CLI ?= go tool github.com/campoy/embedmd
+JSONNET_CLI ?= go tool github.com/google/go-jsonnet/cmd/jsonnet
+JB_CLI ?= go tool github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb
 
 export DOCKER_CLI_EXPERIMENTAL=enabled
 
@@ -88,7 +93,7 @@ fix-markdown-format:
 	${DOCKER_CLI} run -v "${PWD}:/workdir" davidanson/markdownlint-cli2:v${MARKDOWNLINT_CLI2_VERSION} --fix --config .markdownlint-cli2.jsonc
 
 generate-template:
-	gomplate -d config=./data.yaml --file README.md.tpl > README.md
+	${GOMPLATE_CLI} -d config=./data.yaml --file README.md.tpl > README.md
 
 validate-template: generate-template
 	git diff --no-ext-diff --quiet --exit-code README.md
@@ -97,8 +102,8 @@ validate-template: generate-template
 # the two.
 test-benchmark-compare:
 	@git fetch
-	./tests/compare_benchmarks.sh main
-	./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH}
+	./tests/compare_benchmarks.sh main 2
+	./tests/compare_benchmarks.sh ${LATEST_RELEASE_BRANCH} 2
 
 all: all-container
 
@@ -138,7 +143,7 @@ e2e:
 generate: build-local generate-template
 	@echo ">> generating docs"
 	@./scripts/generate-help-text.sh
-	embedmd -w `find . -path ./vendor -prune -o -name "*.md" -print`
+	${EMBEDMD_CLI} -w `find . -path ./vendor -prune -o -name "*.md" -print`
 
 validate-manifests: examples
 	@git diff --exit-code
@@ -147,31 +152,27 @@ mixin: examples/prometheus-alerting-rules/alerts.yaml
 
 examples/prometheus-alerting-rules/alerts.yaml: jsonnet $(shell find jsonnet | grep ".libsonnet") scripts/mixin.jsonnet scripts/vendor
 	mkdir -p examples/prometheus-alerting-rules
-	jsonnet -J scripts/vendor scripts/mixin.jsonnet | gojsontoyaml > examples/prometheus-alerting-rules/alerts.yaml
+	${JSONNET_CLI}  -J scripts/vendor scripts/mixin.jsonnet | ${GOJSONTOYAML_CLI} > examples/prometheus-alerting-rules/alerts.yaml
 
 examples: examples/standard examples/autosharding examples/daemonsetsharding mixin
 
 examples/standard: jsonnet $(shell find jsonnet | grep ".libsonnet") scripts/standard.jsonnet scripts/vendor
 	mkdir -p examples/standard
-	jsonnet -J scripts/vendor -m examples/standard --ext-str version="$(VERSION)" scripts/standard.jsonnet | xargs -I{} sh -c 'cat {} | gojsontoyaml > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
+	${JSONNET_CLI} -J scripts/vendor -m examples/standard --ext-str version="$(VERSION)" scripts/standard.jsonnet | xargs -I{} sh -c 'cat {} | ${GOJSONTOYAML_CLI} > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
 	find examples -type f ! -name '*.yaml' -delete
 
 examples/autosharding: jsonnet $(shell find jsonnet | grep ".libsonnet") scripts/autosharding.jsonnet scripts/vendor
 	mkdir -p examples/autosharding
-	jsonnet -J scripts/vendor -m examples/autosharding --ext-str version="$(VERSION)" scripts/autosharding.jsonnet | xargs -I{} sh -c 'cat {} | gojsontoyaml > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
+	${JSONNET_CLI} -J scripts/vendor -m examples/autosharding --ext-str version="$(VERSION)" scripts/autosharding.jsonnet | xargs -I{} sh -c 'cat {} | ${GOJSONTOYAML_CLI} > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
 	find examples -type f ! -name '*.yaml' -delete
 
 examples/daemonsetsharding: jsonnet $(shell find jsonnet | grep ".libsonnet") scripts/daemonsetsharding.jsonnet scripts/vendor
 	mkdir -p examples/daemonsetsharding
-	jsonnet -J scripts/vendor -m examples/daemonsetsharding --ext-str version="$(VERSION)" scripts/daemonsetsharding.jsonnet | xargs -I{} sh -c 'cat {} | gojsontoyaml > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
+	${JSONNET_CLI} -J scripts/vendor -m examples/daemonsetsharding --ext-str version="$(VERSION)" scripts/daemonsetsharding.jsonnet | xargs -I{} sh -c 'cat {} | ${GOJSONTOYAML_CLI} > `echo {} | sed "s/\(.\)\([A-Z]\)/\1-\2/g" | tr "[:upper:]" "[:lower:]"`.yaml' -- {}
 	find examples -type f ! -name '*.yaml' -delete
 
 scripts/vendor: scripts/jsonnetfile.json scripts/jsonnetfile.lock.json
-	cd scripts && jb install
-
-install-tools:
-	@echo Installing tools from tools.go
-	grep '^\s*_' tools/tools.go | awk '{print $$2}' | xargs -tI % go install -mod=readonly -modfile=tools/go.mod %
+	cd scripts && ${JB_CLI} install
 
 install-promtool:
 	@echo Installing promtool

OWNERS

@@ -5,6 +5,7 @@ reviewers:
   - mrueg
   - rexagod
 approvers:
+  - CatherineF-dev
   - dgrisonnet
   - mrueg
   - rexagod

README.md

@@ -41,6 +41,7 @@ are deleted they are no longer visible on the `/metrics` endpoint.
   * [Resource group version compatibility](#resource-group-version-compatibility)
   * [Container Image](#container-image)
 * [Metrics Documentation](#metrics-documentation)
+  * [ECMAScript regular expression support for allow and deny lists](#ecmascript-regular-expression-support-for-allow-and-deny-lists)
   * [Conflict resolution in label names](#conflict-resolution-in-label-names)
 * [Kube-state-metrics self metrics](#kube-state-metrics-self-metrics)
 * [Resource recommendation](#resource-recommendation)
@@ -67,9 +68,8 @@ are deleted they are no longer visible on the `/metrics` endpoint.
 #### Kubernetes Version
 
 kube-state-metrics uses [`client-go`](https://github.com/kubernetes/client-go) to talk with
-Kubernetes clusters. The supported Kubernetes cluster version is determined by `client-go`.
-The compatibility matrix for client-go and Kubernetes cluster can be found
-[here](https://github.com/kubernetes/client-go#compatibility-matrix).
+Kubernetes clusters. The supported Kubernetes cluster version is determined by
+[`client-go`](https://github.com/kubernetes/client-go#compatibility-matrix).
 All additional compatibility is only best effort, or happens to still/already be supported.
 
 #### Compatibility matrix
@@ -79,12 +79,12 @@ Generally, it is recommended to use the latest release of kube-state-metrics. If
 
 | kube-state-metrics | Kubernetes client-go Version |
 |--------------------|:----------------------------:|
-| **v2.9.2**         | v1.26                        |
-| **v2.10.1**        | v1.27                        |
-| **v2.11.0**        | v1.28                        |
 | **v2.12.0**        | v1.29                        |
 | **v2.13.0**        | v1.30                        |
-| **main**           | v1.30                        |
+| **v2.14.0**        | v1.31                        |
+| **v2.15.0**        | v1.32                        |
+| **v2.16.0**        | v1.32                        |
+| **main**           | v1.33                        |
 
 #### Resource group version compatibility
 
@@ -96,8 +96,8 @@ release.
 
 The latest container image can be found at:
 
-* `registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0` (arch: `amd64`, `arm`, `arm64`, `ppc64le` and `s390x`)
-* View all multi-architecture images at [here](https://explore.ggcr.dev/?image=registry.k8s.io%2Fkube-state-metrics%2Fkube-state-metrics:v2.13.0)
+* `registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.16.0` (arch: `amd64`, `arm`, `arm64`, `ppc64le` and `s390x`)
+* [Multi-architecture images](https://explore.ggcr.dev/?image=registry.k8s.io%2Fkube-state-metrics%2Fkube-state-metrics:v2.16.0)
 
 ### Metrics Documentation
 
@@ -129,6 +129,10 @@ e.g. by standardizing Kubernetes labels using an
 [Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/)
 that ensures that there are no possible conflicts.
 
+#### ECMAScript regular expression support for allow and deny lists
+
+Starting from [#2616](https://github.com/kubernetes/kube-state-metrics/pull/2616/files), kube-state-metrics supports ECMAScript's `regexp` for allow and deny lists. This was incorporated as a workaround for the limitations of the `regexp` package in Go, which does not support lookarounds due to their non-linear time complexity. Please note that while lookarounds are now supported for allow and deny lists, regular expressions' evaluation time is capped at a minute to prevent performance issues.
+
 ### Kube-state-metrics self metrics
 
 kube-state-metrics exposes its own general process metrics under `--telemetry-host` and `--telemetry-port` (default 8081).
@@ -139,7 +143,7 @@ at the logs of kube-state-metrics.
 
 Example of the above mentioned metrics:
 
-```
+```prometheus
 kube_state_metrics_list_total{resource="*v1.Node",result="success"} 1
 kube_state_metrics_list_total{resource="*v1.Node",result="error"} 52
 kube_state_metrics_watch_total{resource="*v1beta1.Ingress",result="success"} 1
@@ -147,7 +151,7 @@ kube_state_metrics_watch_total{resource="*v1beta1.Ingress",result="success"} 1
 
 kube-state-metrics also exposes some http request metrics, examples of those are:
 
-```
+```prometheus
 http_request_duration_seconds_bucket{handler="metrics",method="get",le="2.5"} 30
 http_request_duration_seconds_bucket{handler="metrics",method="get",le="5"} 30
 http_request_duration_seconds_bucket{handler="metrics",method="get",le="10"} 30
@@ -158,20 +162,20 @@ http_request_duration_seconds_count{handler="metrics",method="get"} 30
 
 kube-state-metrics also exposes build and configuration metrics:
 
-```
+```prometheus
 kube_state_metrics_build_info{branch="main",goversion="go1.15.3",revision="6c9d775d",version="v2.0.0-beta"} 1
 kube_state_metrics_shard_ordinal{shard_ordinal="0"} 0
 kube_state_metrics_total_shards 1
 ```
 
 `kube_state_metrics_build_info` is used to expose version and other build information. For more usage about the info pattern,
-please check the blog post [here](https://www.robustperception.io/exposing-the-software-version-to-prometheus).
+please check this [blog post](https://www.robustperception.io/exposing-the-software-version-to-prometheus).
 Sharding metrics expose `--shard` and `--total-shards` flags and can be used to validate
 run-time configuration, see [`/examples/prometheus-alerting-rules`](./examples/prometheus-alerting-rules).
 
 kube-state-metrics also exposes metrics about it config file and the Custom Resource State config file:
 
-```
+```prometheus
 kube_state_metrics_config_hash{filename="crs.yml",type="customresourceconfig"} 2.38272279311849e+14
 kube_state_metrics_config_hash{filename="config.yml",type="config"} 2.65285922340846e+14
 kube_state_metrics_last_config_reload_success_timestamp_seconds{filename="crs.yml",type="customresourceconfig"} 1.6704882592037103e+09
@@ -198,7 +202,7 @@ Note that if CPU limits are set too low, kube-state-metrics' internal queues wil
 
 In a 100 node cluster scaling test the latency numbers were as follows:
 
-```
+```text
 "Perc50": 259615384 ns,
 "Perc90": 475000000 ns,
 "Perc99": 906666666 ns.
@@ -262,7 +266,7 @@ Each kube-state-metrics pod uses FieldSelector (spec.nodeName) to watch/list pod
 
 A daemonset kube-state-metrics example:
 
-```
+```yaml
 apiVersion: apps/v1
 kind: DaemonSet
 spec:
@@ -282,9 +286,9 @@ spec:
               fieldPath: spec.nodeName
 ```
 
-To track metrics for unassigned pods, you need to add an additional deployment and set `--node=""`, as shown in the following example:
+To track metrics for unassigned pods, you need to add an additional deployment and set `--track-unscheduled-pods`, as shown in the following example:
 
-```
+```yaml
 apiVersion: apps/v1
 kind: Deployment
 spec:
@@ -295,7 +299,7 @@ spec:
         name: kube-state-metrics
         args:
         - --resources=pods
-        - --node=""
+        - --track-unscheduled-pods
 ```
 
 Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding).
@@ -304,8 +308,8 @@ Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding).
 
 Install this project to your `$GOPATH` using `go get`:
 
-```
-go get k8s.io/kube-state-metrics
+```bash
+go get k8s.io/kube-state-metrics/v2
 ```
 
 #### Building the Docker container
@@ -313,7 +317,7 @@ go get k8s.io/kube-state-metrics
 Simply run the following command in this root folder, which will create a
 self-contained, statically-linked binary and build a Docker image:
 
-```
+```bash
 make container
 ```
 
@@ -336,7 +340,7 @@ To have Prometheus discover kube-state-metrics instances it is advised to create
 
 **Note:** Google Kubernetes Engine (GKE) Users - GKE has strict role permissions that will prevent the kube-state-metrics roles and role bindings from being created. To work around this, you can give your GCP identity the cluster-admin role by running the following one-liner:
 
-```
+```bash
 kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=$(gcloud info --format='value(config.account)')
 ```
 
@@ -407,17 +411,20 @@ Starting from the kube-state-metrics chart `v2.13.3` (kube-state-metrics image `
 
 #### Development
 
-When developing, test a metric dump against your local Kubernetes cluster by
-running:
+When developing, test a metric dump against your local Kubernetes cluster by running:
 
 > Users can override the apiserver address in KUBE-CONFIG file with `--apiserver` command line.
 
- go install
- kube-state-metrics --port=8080 --telemetry-port=8081 --kubeconfig=<KUBE-CONFIG> --apiserver=<APISERVER>
+```bash
+go install
+kube-state-metrics --port=8080 --telemetry-port=8081 --kubeconfig=<KUBE-CONFIG> --apiserver=<APISERVER>
+```
 
 Then curl the metrics endpoint
 
- curl localhost:8080/metrics
+```bash
+curl localhost:8080/metrics
+```
 
 To run the e2e tests locally see the documentation in [tests/README.md](./tests/README.md).
 

README.md.tpl

@@ -41,6 +41,7 @@ are deleted they are no longer visible on the `/metrics` endpoint.
   * [Resource group version compatibility](#resource-group-version-compatibility)
   * [Container Image](#container-image)
 * [Metrics Documentation](#metrics-documentation)
+  * [ECMAScript regular expression support for allow and deny lists](#ecmascript-regular-expression-support-for-allow-and-deny-lists)
   * [Conflict resolution in label names](#conflict-resolution-in-label-names)
 * [Kube-state-metrics self metrics](#kube-state-metrics-self-metrics)
 * [Resource recommendation](#resource-recommendation)
@@ -67,9 +68,8 @@ are deleted they are no longer visible on the `/metrics` endpoint.
 #### Kubernetes Version
 
 kube-state-metrics uses [`client-go`](https://github.com/kubernetes/client-go) to talk with
-Kubernetes clusters. The supported Kubernetes cluster version is determined by `client-go`.
-The compatibility matrix for client-go and Kubernetes cluster can be found
-[here](https://github.com/kubernetes/client-go#compatibility-matrix).
+Kubernetes clusters. The supported Kubernetes cluster version is determined by
+[`client-go`](https://github.com/kubernetes/client-go#compatibility-matrix).
 All additional compatibility is only best effort, or happens to still/already be supported.
 
 #### Compatibility matrix
@@ -98,7 +98,7 @@ The latest container image can be found at:
 {{ (index . (math.Sub (len .) 2)).version -}}
 {{ end }}
 * `registry.k8s.io/kube-state-metrics/kube-state-metrics:{{ template "get-latest-release" (datasource "config").compat }}` (arch: `amd64`, `arm`, `arm64`, `ppc64le` and `s390x`)
-* View all multi-architecture images at [here](https://explore.ggcr.dev/?image=registry.k8s.io%2Fkube-state-metrics%2Fkube-state-metrics:{{ template "get-latest-release" (datasource "config").compat -}})
+* [Multi-architecture images](https://explore.ggcr.dev/?image=registry.k8s.io%2Fkube-state-metrics%2Fkube-state-metrics:{{ template "get-latest-release" (datasource "config").compat -}})
 
 ### Metrics Documentation
 
@@ -130,6 +130,10 @@ e.g. by standardizing Kubernetes labels using an
 [Admission Webhook](https://kubernetes.io/docs/reference/access-authn-authz/extensible-admission-controllers/)
 that ensures that there are no possible conflicts.
 
+#### ECMAScript regular expression support for allow and deny lists
+
+Starting from [#2616](https://github.com/kubernetes/kube-state-metrics/pull/2616/files), kube-state-metrics supports ECMAScript's `regexp` for allow and deny lists. This was incorporated as a workaround for the limitations of the `regexp` package in Go, which does not support lookarounds due to their non-linear time complexity. Please note that while lookarounds are now supported for allow and deny lists, regular expressions' evaluation time is capped at a minute to prevent performance issues.
+
 ### Kube-state-metrics self metrics
 
 kube-state-metrics exposes its own general process metrics under `--telemetry-host` and `--telemetry-port` (default 8081).
@@ -140,7 +144,7 @@ at the logs of kube-state-metrics.
 
 Example of the above mentioned metrics:
 
-```
+```prometheus
 kube_state_metrics_list_total{resource="*v1.Node",result="success"} 1
 kube_state_metrics_list_total{resource="*v1.Node",result="error"} 52
 kube_state_metrics_watch_total{resource="*v1beta1.Ingress",result="success"} 1
@@ -148,7 +152,7 @@ kube_state_metrics_watch_total{resource="*v1beta1.Ingress",result="success"} 1
 
 kube-state-metrics also exposes some http request metrics, examples of those are:
 
-```
+```prometheus
 http_request_duration_seconds_bucket{handler="metrics",method="get",le="2.5"} 30
 http_request_duration_seconds_bucket{handler="metrics",method="get",le="5"} 30
 http_request_duration_seconds_bucket{handler="metrics",method="get",le="10"} 30
@@ -159,20 +163,20 @@ http_request_duration_seconds_count{handler="metrics",method="get"} 30
 
 kube-state-metrics also exposes build and configuration metrics:
 
-```
+```prometheus
 kube_state_metrics_build_info{branch="main",goversion="go1.15.3",revision="6c9d775d",version="v2.0.0-beta"} 1
 kube_state_metrics_shard_ordinal{shard_ordinal="0"} 0
 kube_state_metrics_total_shards 1
 ```
 
 `kube_state_metrics_build_info` is used to expose version and other build information. For more usage about the info pattern,
-please check the blog post [here](https://www.robustperception.io/exposing-the-software-version-to-prometheus).
+please check this [blog post](https://www.robustperception.io/exposing-the-software-version-to-prometheus).
 Sharding metrics expose `--shard` and `--total-shards` flags and can be used to validate
 run-time configuration, see [`/examples/prometheus-alerting-rules`](./examples/prometheus-alerting-rules).
 
 kube-state-metrics also exposes metrics about it config file and the Custom Resource State config file:
 
-```
+```prometheus
 kube_state_metrics_config_hash{filename="crs.yml",type="customresourceconfig"} 2.38272279311849e+14
 kube_state_metrics_config_hash{filename="config.yml",type="config"} 2.65285922340846e+14
 kube_state_metrics_last_config_reload_success_timestamp_seconds{filename="crs.yml",type="customresourceconfig"} 1.6704882592037103e+09
@@ -199,7 +203,7 @@ Note that if CPU limits are set too low, kube-state-metrics' internal queues wil
 
 In a 100 node cluster scaling test the latency numbers were as follows:
 
-```
+```text
 "Perc50": 259615384 ns,
 "Perc90": 475000000 ns,
 "Perc99": 906666666 ns.
@@ -263,7 +267,7 @@ Each kube-state-metrics pod uses FieldSelector (spec.nodeName) to watch/list pod
 
 A daemonset kube-state-metrics example:
 
-```
+```yaml
 apiVersion: apps/v1
 kind: DaemonSet
 spec:
@@ -283,9 +287,9 @@ spec:
               fieldPath: spec.nodeName
 ```
 
-To track metrics for unassigned pods, you need to add an additional deployment and set `--node=""`, as shown in the following example:
+To track metrics for unassigned pods, you need to add an additional deployment and set `--track-unscheduled-pods`, as shown in the following example:
 
-```
+```yaml
 apiVersion: apps/v1
 kind: Deployment
 spec:
@@ -296,7 +300,7 @@ spec:
         name: kube-state-metrics
         args:
         - --resources=pods
-        - --node=""
+        - --track-unscheduled-pods
 ```
 
 Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding).
@@ -305,8 +309,8 @@ Other metrics can be sharded via [Horizontal sharding](#horizontal-sharding).
 
 Install this project to your `$GOPATH` using `go get`:
 
-```
-go get k8s.io/kube-state-metrics
+```bash
+go get k8s.io/kube-state-metrics/v2
 ```
 
 #### Building the Docker container
@@ -314,7 +318,7 @@ go get k8s.io/kube-state-metrics
 Simply run the following command in this root folder, which will create a
 self-contained, statically-linked binary and build a Docker image:
 
-```
+```bash
 make container
 ```
 
@@ -337,7 +341,7 @@ To have Prometheus discover kube-state-metrics instances it is advised to create
 
 **Note:** Google Kubernetes Engine (GKE) Users - GKE has strict role permissions that will prevent the kube-state-metrics roles and role bindings from being created. To work around this, you can give your GCP identity the cluster-admin role by running the following one-liner:
 
-```
+```bash
 kubectl create clusterrolebinding cluster-admin-binding --clusterrole=cluster-admin --user=$(gcloud info --format='value(config.account)')
 ```
 
@@ -408,17 +412,20 @@ Starting from the kube-state-metrics chart `v2.13.3` (kube-state-metrics image `
 
 #### Development
 
-When developing, test a metric dump against your local Kubernetes cluster by
-running:
+When developing, test a metric dump against your local Kubernetes cluster by running:
 
 > Users can override the apiserver address in KUBE-CONFIG file with `--apiserver` command line.
 
- go install
- kube-state-metrics --port=8080 --telemetry-port=8081 --kubeconfig=<KUBE-CONFIG> --apiserver=<APISERVER>
+```bash
+go install
+kube-state-metrics --port=8080 --telemetry-port=8081 --kubeconfig=<KUBE-CONFIG> --apiserver=<APISERVER>
+```
 
 Then curl the metrics endpoint
 
- curl localhost:8080/metrics
+```bash
+curl localhost:8080/metrics
+```
 
 To run the e2e tests locally see the documentation in [tests/README.md](./tests/README.md).
 

RELEASE.md

@@ -24,7 +24,7 @@ Maintaining the release branches for older minor releases happens on a best effo
     * Only include user relevant changes
     * Entries in the [`CHANGELOG.md`](CHANGELOG.md) are meant to be in this order:
 
-    ```
+    ```text
     [CHANGE]
     [FEATURE]
     [ENHANCEMENT]

SECURITY_CONTACTS

@@ -13,3 +13,4 @@
 dgrisonnet
 mrueg
 rexagod
+CatherineF-dev

cloudbuild.yaml

@@ -1,9 +1,9 @@
 # See https://cloud.google.com/cloud-build/docs/build-config
-timeout: 2400s
+timeout: 2700s
 options:
   substitution_option: ALLOW_LOOSE
 steps:
-  - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20230522-312425ae46'
+  - name: 'gcr.io/k8s-staging-test-infra/gcb-docker-gcloud:v20241111-71c32dbdcc'
     entrypoint: make
     env:
     - TAG=$_PULL_BASE_REF

data.yaml

@@ -1,19 +1,19 @@
 # The purpose of this config is to keep all versions in a single file and make them machine accessible
 
 # Marks the latest release
-version: "2.13.0"
+version: "2.16.0"
 
 # List at max 5 releases here + the main branch
 compat:
-  - version: "v2.9.2"
-    kubernetes: "1.26"
-  - version: "v2.10.1"
-    kubernetes: "1.27"
-  - version: "v2.11.0"
-    kubernetes: "1.28"
   - version: "v2.12.0"
     kubernetes: "1.29"
   - version: "v2.13.0"
     kubernetes: "1.30"
+  - version: "v2.14.0"
+    kubernetes: "1.31"
+  - version: "v2.15.0"
+    kubernetes: "1.32"
+  - version: "v2.16.0"
+    kubernetes: "1.32"
   - version: "main"
-    kubernetes: "1.30"
+    kubernetes: "1.33"

docs/README.md

@@ -79,13 +79,13 @@ can be used to extend single metrics output.
 This example adds `label_release` to the set of default labels of the `kube_pod_status_ready` metric
 and allows you select or group the metrics by Helm release label:
 
-```
+```promql
 kube_pod_status_ready * on (namespace, pod) group_left(label_release) kube_pod_labels
 ```
 
 Another useful example would be to query the memory usage of pods by its `phase`, such as `Running`:
 
-```
+```promql
 sum(kube_pod_container_resource_requests{resource="memory"}) by (namespace, pod, node)
   * on (namespace, pod) group_left() (sum(kube_pod_status_phase{phase="Running"}) by (pod, namespace) == 1)
 ```
@@ -97,3 +97,49 @@ See [Custom Resource State Metrics](metrics/extend/customresourcestate-metrics.m
 ## CLI Arguments
 
 Additionally, options for `kube-state-metrics` can be passed when executing as a CLI, or in a kubernetes / openshift environment. More information can be found here: [CLI Arguments](developer/cli-arguments.md)
+
+## Protecting /metrics endpoints
+
+Kube-State-Metrics' metrics can contain sensitive information about the state of the cluster, which you as an operator might want to additionally protect from unauthorized access.
+In order to achieve this, you need to enable the `--auth-filter` flag on kube-state-metrics.
+With this, kube-state-metrics will only accept authenticated and authorized requests to the /metrics endpoints.
+Kube-state-metrics uses Kubernetes' RBAC mechanisms for this, so this means that every scrape will trigger a request against the API Server for TokenReview and SubjectAccessReview.
+The clients scraping the endpoint, need to use a token which can be provided by a ServiceAccount that can be set up the following way:
+
+A ClusterRole providing access like this:
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: metrics-reader
+rules:
+- nonResourceURLs:
+  - "/metrics"
+  verbs:
+  - get
+```
+
+and a matching ClusterRoleBinding
+
+```yaml
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: metrics-reader-rolebinding
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: metrics-reader
+subjects:
+- kind: ServiceAccount
+  name: YOUR_SERVICE_ACCOUNT
+  namespace: NAMESPACE_OF_THE_SERVICE_ACCOUNT
+```
+
+Your client can then use either this ServiceAccount to gather metrics or you can create a token, that can be used to fetch data like this:
+
+```bash
+TOKEN=$(kubectl create token YOUR_SERVICE_ACCOUNT -n NAMESPACE_OF_THE_SERVICE_ACCOUNT)
+curl -H "Authorization: Bearer $TOKEN" KUBE_STATE_METRICS_URL:8080/metrics
+```

docs/design/metrics-best-practices.md

@@ -0,0 +1,72 @@
+# Kube-State-Metrics - Timeseries best practices
+
+---
+
+Author: Manuel Rüger (<manuel@rueg.eu>)
+
+Date: October 17th 2024
+
+---
+
+## Introduction
+
+Kube-State-Metrics' goal is to provide insights into the state of Kubernetes objects by exposing them as metrics.
+This document provides guidelines with the goal to create a good user experience when using these metrics.
+
+Please be aware that this document is introduced in a later stage of the project and there might be metrics that do not follow these best practices.
+Feel encouraged to report these metrics and provide a pull request to improve them.
+
+## General best practices
+
+We follow [Prometheus](https://prometheus.io/docs/practices/naming/) best practices in terms of naming and labeling.
+
+## Best practices for kube-state-metrics
+
+### Avoid pre-computation
+
+kube-state-metrics should expose metrics on an individual object level and avoid any sort of pre-computation unless it is required due to for example high cardinality on objects.
+We prefer not to add metrics that can be derived from existing raw metrics. For example, we would not want to expose a metric called `kube_pod_total` as it can be computed with `count(kube_pod_info)`.
+This way kube-state-metrics allows the user to have full control on how they want to use the metrics and gives them flexibility to do specific computation.
+
+### Static object properties
+
+An object usually has a stable set of properties that do not change during its lifecycle in Kubernetes.
+This includes properties like name, namespace, uid etc. that have a 1:1 relationship with the object.
+It is a good practice to group those together into an `_info` metric.
+If there is a 1:n relationship (e.g. a list of ports), it should be in a separate metric to avoid generating too many metrics.
+
+### Dynamic object properties
+
+An object can also have a dynamic set of properties, which are usually part of the status field.
+These change during the lifecycle of the object.
+For example a pod can be in different states like "Pending", "Running" etc.
+These should be part of a "State Set" that includes labels that identify the object as well as the dynamic property.
+
+### Linked properties
+
+If an object contains a substructure that links multiple properties together (e.g. endpoint address and port), those should be reported in the same metric.
+
+### Optional properties
+
+Some Kubernetes objects have optional fields. In case there is an optional value, the label should still be exposed, ideally as an empty string.
+
+### Timestamps
+
+Timestamps like creation time or modification time should be exposed as a value. The metric should end with `_timestamp_seconds`. The date value is represented in [UNIX epoch seconds](https://en.wikipedia.org/wiki/Unix_time).
+
+### Cardinality
+
+Some object properties can cause cardinality issues if they can contain a lot of different values or are linked together with multiple properties that also can change a lot.
+In this case it is better to limit the number of values that can be exposed within kube-state-metrics by allowing only a few of them and have a default for others.
+If for example the Kubernetes object contains a status field that contains an error message that can change a lot, it would be better to have a boolean `error="true"` label in case there is an error.
+If there are some error messages that are worth exposing, those could be exposed and for any other message, a default value could be provided.
+
+## Stability
+
+We follow the stability framework derived from Kubernetes, in which we expose experimental and stable metrics.
+Experimental metrics are recently introduced or expose alpha/beta resources in the Kubernetes API.
+They can change anytime and should be used with caution.
+They can be promoted to a stable metric once the object stabilized in the Kubernetes API or they were part of two consecutive releases and haven't observed any changes in them.
+
+Stable metrics are considered frozen with the exception of new labels being added.
+A stable metric or a label on a stable metric can be deprecated in release Major.Minor and the earliest point it will be removed is the release Major.Minor+2.

docs/design/metrics-store-performance-optimization.md

@@ -58,7 +58,7 @@ properties in memory via the Kubernetes client-go cache, use a map, addressable
 by the Kubernetes object uuid, containing all time series of that object as a
 single multi-line string.
 
-```
+```go
 var cache = map[uuid][]byte{}
 ```
 
@@ -76,7 +76,7 @@ On a scrape request on the /metrics endpoint, kube-state-metrics iterates over
 the cache map and concatenates all time series string blobs into a single
 string, which is finally passed on as a response.
 
-```
+```text
        +---------------+ +-----------+                 +---------------+         +-------------------+
        | pod_reflector | | pod_store |                 | pod_collector |         | metrics_endpoint  |
        +---------------+ +-----------+                 +---------------+         +-------------------+
@@ -116,7 +116,7 @@ string, which is finally passed on as a response.
 
 Build via [text-diagram](http://weidagang.github.io/text-diagram/)
 
-```
+```text
 object pod_reflector pod_store pod_collector metrics_endpoint
 
 note left of pod_reflector: new pod p1
@@ -163,8 +163,6 @@ pod_collector -> metrics_endpoint: concat(metrics)
   of saving unstructured strings inside the cache map, one can structure them,
   using pointers to deduplicate e.g. metric names.
 
-* ...
-
 * Kube-state-metrics does not make use of all properties of all Kubernetes
   objects. Instead of unmarshalling unused properties, their json struct tags or
   their Protobuf representation could be removed.

docs/developer/cli-arguments.md

@@ -41,9 +41,10 @@ Flags:
       --add_dir_header                             If true, adds the file directory to the header of the log messages
       --alsologtostderr                            log to standard error as well as files (no effect when -logtostderr=true)
       --apiserver string                           The URL of the apiserver to use as a master
+      --auth-filter                                If true, requires authentication and authorization through Kubernetes API to access metrics endpoints
       --auto-gomemlimit                            Automatically set GOMEMLIMIT to match container or system memory limit. (experimental)
       --auto-gomemlimit-ratio float                The ratio of reserved GOMEMLIMIT memory to the detected maximum container or system memory. (experimental) (default 0.9)
-      --config string                              Path to the kube-state-metrics options config file
+      --config string                              Path to the kube-state-metrics options config YAML file. If this flag is set, the flags defined in the file override the command line flags.
       --custom-resource-state-config string        Inline Custom Resource State Metrics config YAML (experimental)
       --custom-resource-state-config-file string   Path to a Custom Resource State Metrics config file (experimental)
       --custom-resource-state-only                 Only provide Custom Resource State metrics (experimental)
@@ -56,14 +57,15 @@ Flags:
       --log_file string                            If non-empty, use this log file (no effect when -logtostderr=true)
       --log_file_max_size uint                     Defines the maximum size a log file can grow to (no effect when -logtostderr=true). Unit is megabytes. If the value is 0, the maximum file size is unlimited. (default 1800)
       --logtostderr                                log to standard error instead of files (default true)
-      --metric-allowlist string                    Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.
+      --metric-allowlist string                    Comma-separated list of metrics to be exposed. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.
       --metric-annotations-allowlist string        Comma-separated list of Kubernetes annotations keys that will be used in the resource' labels metric. By default the annotations metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes annotation keys you would like to allow for them (Example: '=namespaces=[kubernetes.io/team,...],pods=[kubernetes.io/team],...)'. A single '*' can be provided per resource instead to allow any annotations, but that has severe performance implications (Example: '=pods=[*]').
-      --metric-denylist string                     Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or regex patterns. The allowlist and denylist are mutually exclusive.
+      --metric-denylist string                     Comma-separated list of metrics not to be enabled. This list comprises of exact metric names and/or *ECMAScript-based* regex patterns. The allowlist and denylist are mutually exclusive.
       --metric-labels-allowlist string             Comma-separated list of additional Kubernetes label keys that will be used in the resource' labels metric. By default the labels metrics are not exposed. To include them, provide a list of resource names in their plural form and Kubernetes label keys you would like to allow for them (Example: '=namespaces=[k8s-label-1,k8s-label-n,...],pods=[app],...)'. A single '*' can be provided per resource instead to allow any labels, but that has severe performance implications (Example: '=pods=[*]'). Additionally, an asterisk (*) can be provided as a key, which will resolve to all resources, i.e., assuming '--resources=deployments,pods', '=*=[*]' will resolve to '=deployments=[*],pods=[*]'.
       --metric-opt-in-list string                  Comma-separated list of metrics which are opt-in and not enabled by default. This is in addition to the metric allow- and denylists
       --namespaces string                          Comma-separated list of namespaces to be enabled. Defaults to ""
       --namespaces-denylist string                 Comma-separated list of namespaces not to be enabled. If namespaces and namespaces-denylist are both set, only namespaces that are excluded in namespaces-denylist will be used.
       --node string                                Name of the node that contains the kube-state-metrics pod. Most likely it should be passed via the downward API. This is used for daemonset sharding. Only available for resources (pod metrics) that support spec.nodeName fieldSelector. This is experimental.
+      --object-limit int                           The total number of objects to list per resource from the API Server. (experimental)
       --one_output                                 If true, only write logs to their native severity level (vs also writing to each lower severity level; no effect when -logtostderr=true)
       --pod string                                 Name of the pod that contains the kube-state-metrics container. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
       --pod-namespace string                       Name of the namespace of the pod specified by --pod. When set, it is expected that --pod and --pod-namespace are both set. Most likely this should be passed via the downward API. This is used for auto-detecting sharding. If set, this has preference over statically configured sharding. This is experimental, it may be removed without notice.
@@ -81,6 +83,7 @@ Flags:
       --telemetry-port int                         Port to expose kube-state-metrics self metrics on. (default 8081)
       --tls-config string                          Path to the TLS configuration file
       --total-shards int                           The total number of shards. Sharding is disabled when total shards is set to 1. (default 1)
+      --track-unscheduled-pods                     This configuration is used in conjunction with node configuration. When this configuration is true, node configuration is empty and the metric of unscheduled pods is fetched from the Kubernetes API Server. This is experimental.
       --use-apiserver-cache                        Sets resourceVersion=0 for ListWatch requests, using cached resources from the apiserver instead of an etcd quorum read.
   -v, --v Level                                    number for the log level verbosity
       --vmodule moduleSpec                         comma-separated list of pattern=N settings for file-filtered logging

docs/metrics/cluster/node-metrics.md

@@ -3,7 +3,7 @@
 | Metric name                  | Metric type | Description                                                                                                               | Unit (where applicable)                                                                                                                                                                  | Labels/tags                                                                                                                                                                                                                                                                                                                                                                                                                                               | Status       |
 | ---------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
 | kube_node_annotations        | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `annotation_NODE_ANNOTATION`=&lt;NODE_ANNOTATION&gt;                                                                                                                                                                                                                                                                                                                                                                     | EXPERIMENTAL |
-| kube_node_info               | Gauge       | Information about a cluster node                                                                                          |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `kernel_version`=&lt;kernel-version&gt; <br> `os_image`=&lt;os-image-name&gt; <br> `container_runtime_version`=&lt;container-runtime-and-version-combination&gt; <br> `kubelet_version`=&lt;kubelet-version&gt; <br> `kubeproxy_version`=&lt;kubeproxy-version&gt; <br> `pod_cidr`=&lt;pod-cidr&gt; <br> `provider_id`=&lt;provider-id&gt; <br> `system_uuid`=&lt;system-uuid&gt; <br> `internal_ip`=&lt;internal-ip&gt; | STABLE       |
+| kube_node_info               | Gauge       | Information about a cluster node                                                                                          |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `kernel_version`=&lt;kernel-version&gt; <br> `os_image`=&lt;os-image-name&gt; <br> `container_runtime_version`=&lt;container-runtime-and-version-combination&gt; <br> `kubelet_version`=&lt;kubelet-version&gt; <br> `kubeproxy_version`=&lt;deprecated&gt; <br> `pod_cidr`=&lt;pod-cidr&gt; <br> `provider_id`=&lt;provider-id&gt; <br> `system_uuid`=&lt;system-uuid&gt; <br> `internal_ip`=&lt;internal-ip&gt; | STABLE       |
 | kube_node_labels             | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `label_NODE_LABEL`=&lt;NODE_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                                    | STABLE       |
 | kube_node_role               | Gauge       | The role of a cluster node                                                                                                |                                                                                                                                                                                          | `node`=&lt;node-address&gt; <br> `role`=&lt;NODE_ROLE&gt;                                                                                                                                                                                                                                                                                                                                                                                                 | EXPERIMENTAL |
 | kube_node_spec_unschedulable | Gauge       | Whether a node can schedule new pods                                                                                      |                                                                                                                                                                                          | `node`=&lt;node-address&gt;                                                                                                                                                                                                                                                                                                                                                                                                                               | STABLE       |

docs/metrics/extend/customresourcestate-metrics.md

@@ -32,6 +32,7 @@ spec:
           # in YAML files, | allows a multi-line string to be passed as a flag value
           # see https://yaml-multiline.info
           -  |
+              kind: CustomResourceStateMetrics
               spec:
                 resources:
                   - groupVersionKind:
@@ -65,6 +66,7 @@ spec:
           # in YAML files, | allows a multi-line string to be passed as a flag value
           # see https://yaml-multiline.info
           -  |
+              kind: CustomResourceStateMetrics
               spec:
                 resources:
                   - groupVersionKind:
@@ -254,6 +256,73 @@ kube_customresource_ref_info{customresource_group="myteam.io", customresource_ki
 kube_customresource_ref_info{customresource_group="myteam.io", customresource_kind="Foo", customresource_version="v1", name="foo",ref="foo_with_extensions"} 1
 ```
 
+#### Same Metrics with Different Labels
+
+```yaml
+  recommendation:
+    containerRecommendations:
+    - containerName: consumer
+      lowerBound:
+        cpu: 100m
+        memory: 262144k
+```
+
+For example in VPA we have above attributes and we want to have a same metrics for both CPU and Memory, you can use below config:
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: autoscaling.k8s.io
+        kind: "VerticalPodAutoscaler"
+        version: "v1"
+      labelsFromPath:
+        verticalpodautoscaler: [metadata, name]
+        namespace: [metadata, namespace]
+        target_api_version: [apiVersion]
+        target_kind: [spec, targetRef, kind]
+        target_name: [spec, targetRef, name]
+      metrics:
+        # for memory
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound"
+          help: "Minimum memory resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [lowerBound, memory]
+        # for CPU
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound"
+          help: "Minimum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [lowerBound, cpu]
+```
+
+Produces the following metrics:
+
+```prometheus
+# HELP kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound Minimum memory resources the container can use before the VerticalPodAutoscaler updater evicts it.
+# TYPE kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound gauge
+kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="consumer",customresource_group="autoscaling.k8s.io",customresource_kind="VerticalPodAutoscaler",customresource_version="v1",namespace="namespace-example",resource="memory",target_api_version="apps/v1",target_kind="Deployment",target_name="target-name-example",unit="byte",verticalpodautoscaler="vpa-example"} 123456
+# HELP kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound Minimum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it.
+# TYPE kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound gauge
+kube_customresource_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound{container="consumer",customresource_group="autoscaling.k8s.io",customresource_kind="VerticalPodAutoscaler",customresource_version="v1",namespace="namespace-example",resource="cpu",target_api_version="apps/v1",target_kind="Deployment",target_name="target-name-example",unit="core",verticalpodautoscaler="vpa-example"} 0.1
+```
+
 #### VerticalPodAutoscaler
 
 In v2.9.0 the `vericalpodautoscalers` resource was removed from the list of default resources. In order to generate metrics for `verticalpodautoscalers`, you can use the following Custom Resource State config:
@@ -292,15 +361,226 @@ spec:
 
 The above configuration was tested on [this](https://github.com/kubernetes/autoscaler/blob/master/vertical-pod-autoscaler/examples/hamster.yaml) VPA configuration, with an added annotation (`foo: 123`).
 
+#### All VerticalPodAutoscaler Metrics
+
+As an addition for the above configuration, here's the complete `CustomResourceStateMetrics` spec to re-enable all of the VPA metrics which are removed from the list of the default resources:
+
+<details>
+
+ <summary>VPA CustomResourceStateMetrics</summary>
+
+```yaml
+kind: CustomResourceStateMetrics
+spec:
+  resources:
+    - groupVersionKind:
+        group: autoscaling.k8s.io
+        kind: "VerticalPodAutoscaler"
+        version: "v1"
+      labelsFromPath:
+        namespace: [metadata, namespace]
+        target_api_version: [spec, targetRef, apiVersion]
+        target_kind: [spec, targetRef, kind]
+        target_name: [spec, targetRef, name]
+        verticalpodautoscaler: [metadata, name]
+      metricNamePrefix: "kube"
+      metrics:
+        # kube_verticalpodautoscaler_annotations
+        - name: "verticalpodautoscaler_annotations"
+          help: "Kubernetes annotations converted to Prometheus labels."
+          each:
+            type: Info
+            info:
+              labelsFromPath:
+                annotation_*: [metadata, annotations]
+                name: [metadata, name]
+        # kube_verticalpodautoscaler_labels
+        - name: "verticalpodautoscaler_labels"
+          help: "Kubernetes labels converted to Prometheus labels."
+          each:
+            type: Info
+            info:
+              labelsFromPath:
+                label_*: [metadata, labels]
+                name: [metadata, name]
+        # kube_verticalpodautoscaler_spec_updatepolicy_updatemode
+        - name: "verticalpodautoscaler_spec_updatepolicy_updatemode"
+          help: "Update mode of the VerticalPodAutoscaler."
+          each:
+            type: StateSet
+            stateSet:
+              labelName: "update_mode"
+              path: [spec, updatePolicy, updateMode]
+              list: ["Auto", "Initial", "Off", "Recreate"]
+        # Memory kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_memory
+        - name: "verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_memory"
+          help: "Minimum memory resources the VerticalPodAutoscaler can set for containers matching the name."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [spec, resourcePolicy, containerPolicies]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [minAllowed, memory]
+        # CPU kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_cpu
+        - name: "verticalpodautoscaler_spec_resourcepolicy_container_policies_minallowed_cpu"
+          help: "Minimum cpu resources the VerticalPodAutoscaler can set for containers matching the name."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [spec, resourcePolicy, containerPolicies]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [minAllowed, cpu]
+        # Memory kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_memory
+        - name: "verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_memory"
+          help: "Maximum memory resources the VerticalPodAutoscaler can set for containers matching the name."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [spec, resourcePolicy, containerPolicies]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [maxAllowed, memory]
+        # CPU kube_verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_cpu
+        - name: "verticalpodautoscaler_spec_resourcepolicy_container_policies_maxallowed_cpu"
+          help: "Maximum cpu resources the VerticalPodAutoscaler can set for containers matching the name."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [spec, resourcePolicy, containerPolicies]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [maxAllowed, cpu]
+        # Memory kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_memory
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_memory"
+          help: "Minimum memory resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [lowerBound, memory]
+        # CPU kube_verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_cpu
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_lowerbound_cpu"
+          help: "Minimum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [lowerBound, cpu]
+        # Memory kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_memory
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_memory"
+          help: "Maximum memory resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [upperBound, memory]
+        # CPU kube_verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_cpu
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_upperbound_cpu"
+          help: "Maximum cpu resources the container can use before the VerticalPodAutoscaler updater evicts it."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [upperBound, cpu]
+        # Memory kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target_memory
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_target_memory"
+          help: "Target memory resources the VerticalPodAutoscaler recommends for the container."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [target, memory]
+        # CPU kube_verticalpodautoscaler_status_recommendation_containerrecommendations_target_cpu
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_target_cpu"
+          help: "Target cpu resources the VerticalPodAutoscaler recommends for the container."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [target, cpu]
+        # Memory kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_memory
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_memory"
+          help: "Target memory resources the VerticalPodAutoscaler recommends for the container ignoring bounds."
+          commonLabels:
+            unit: "byte"
+            resource: "memory"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [uncappedTarget, memory]
+        # CPU kube_verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_cpu
+        - name: "verticalpodautoscaler_status_recommendation_containerrecommendations_uncappedtarget_cpu"
+          help: "Target memory resources the VerticalPodAutoscaler recommends for the container ignoring bounds."
+          commonLabels:
+            unit: "core"
+            resource: "cpu"
+          each:
+            type: Gauge
+            gauge:
+              path: [status, recommendation, containerRecommendations]
+              labelsFromPath:
+                container: [containerName]
+              valueFrom: [uncappedTarget, cpu]
+```
+
+</details>
+
 ### Metric types
 
-The configuration supports three kind of metrics from the [OpenMetrics specification](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md).
+The configuration supports three kind of metrics from the [OpenMetrics specification](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md).
 
 The metric type is specified by the `type` field and its specific configuration at the types specific struct.
 
 #### Gauge
 
-> Gauges are current measurements, such as bytes of memory currently used or the number of items in a queue. For gauges the absolute value is what is of interest to a user. [[0]](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#gauge)
+> Gauges are current measurements, such as bytes of memory currently used or the number of items in a queue. For gauges the absolute value is what is of interest to a user. [[0]](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#gauge)
 
 Example:
 
@@ -337,7 +617,7 @@ Supported types are:
 * `nil` is generally mapped to `0.0` if NilIsZero is `true`, otherwise it will throw an error
 * for bool `true` is mapped to `1.0` and `false` is mapped to `0.0`
 * for string the following logic applies
-  * `"true"` and `"yes"` are mapped to `1.0` and `"false"` and `"no"` are mapped to `0.0` (all case-insensitive)
+  * `"true"` and `"yes"` are mapped to `1.0`, `"false"`, `"no"` and `"unknown"` are mapped to `0.0` (all case-insensitive)
   * RFC3339 times are parsed to float timestamp  
   * Quantities like "250m" or "512Gi" are parsed to float using <https://github.com/kubernetes/apimachinery/blob/master/pkg/api/resource/quantity.go>
   * Percentages ending with a "%" are parsed to float
@@ -386,7 +666,7 @@ kube_customresource_foo_status{customresource_group="myteam.io", customresource_
 
 #### StateSet
 
-> StateSets represent a series of related boolean values, also called a bitset. If ENUMs need to be encoded this MAY be done via StateSet. [[1]](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#stateset)
+> StateSets represent a series of related boolean values, also called a bitset. If ENUMs need to be encoded this MAY be done via StateSet. [[1]](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#stateset)
 
 ```yaml
 kind: CustomResourceStateMetrics
@@ -420,7 +700,7 @@ kube_customresource_status_phase{customresource_group="myteam.io", customresourc
 
 #### Info
 
-> Info metrics are used to expose textual information which SHOULD NOT change during process lifetime. Common examples are an application's version, revision control commit, and the version of a compiler. [[2]](https://github.com/OpenObservability/OpenMetrics/blob/main/specification/OpenMetrics.md#info)
+> Info metrics are used to expose textual information which SHOULD NOT change during process lifetime. Common examples are an application's version, revision control commit, and the version of a compiler. [[2]](https://github.com/prometheus/OpenMetrics/blob/v1.0.0/specification/OpenMetrics.md#info)
 
 Metrics of type `Info` will always have a value of 1.
 

docs/metrics/service/endpoint-metrics.md

@@ -3,10 +3,8 @@
 | Metric name                     | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                                                                  | Status       |
 | ------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
 | kube_endpoint_annotations       | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `annotation_ENDPOINT_ANNOTATION`=&lt;ENDPOINT_ANNOTATION&gt;                                                                               | EXPERIMENTAL |
-| kube_endpoint_address_not_ready | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt;                                                                                                                                                 | DEPRECATED   |
-| kube_endpoint_address_available | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt;                                                                                                                                                 | DEPRECATED   |
 | kube_endpoint_info              | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt;                                                                                                                                                 | STABLE       |
 | kube_endpoint_labels            | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `label_ENDPOINT_LABEL`=&lt;ENDPOINT_LABEL&gt;                                                                                              | STABLE       |
 | kube_endpoint_created           | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt;                                                                                                                                                 | STABLE       |
-| kube_endpoint_ports             | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `port_name`=&lt;endpoint-port-name&gt; <br> `port_protocol`=&lt;endpoint-port-protocol&gt; <br> `port_number`=&lt;endpoint-port-number&gt; | STABLE       |
-| kube_endpoint_address           | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `ip`=&lt;endpoint-ip&gt; <br> `ready`=&lt;true if available, false if unavailalbe&gt;                                                      | STABLE       |
+| kube_endpoint_ports             | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `port_name`=&lt;endpoint-port-name&gt; <br> `port_protocol`=&lt;endpoint-port-protocol&gt; <br> `port_number`=&lt;endpoint-port-number&gt; | STABLE (Deprecated from 2.14.0) |
+| kube_endpoint_address           | Gauge       |                                                                                                                           | `endpoint`=&lt;endpoint-name&gt; <br> `namespace`=&lt;endpoint-namespace&gt; <br> `ip`=&lt;endpoint-ip&gt; <br> `port_name`=&lt;endpoint-port-name&gt; <br> `port_protocol`=&lt;endpoint-port-protocol&gt; <br> `port_number`=&lt;endpoint-port-number&gt;`ready`=&lt;true if available, false if unavailalbe&gt;                                                      | STABLE       |

docs/metrics/service/ingress-metrics.md

@@ -1,11 +1,11 @@
 # Ingress Metrics
 
-| Metric name                            | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                           | Status       |
-| -------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
-| kube_ingress_annotations               | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `annotation_INGRESS_ANNOTATION`=&lt;ANNOTATION_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                               | EXPERIMENTAL |
-| kube_ingress_info                      | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `ingressclass`=&lt;ingress-class&gt; or `_default` if not set                                                                                                                                                                                                                                                                                                                                                                          | STABLE       |
-| kube_ingress_labels                    | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `label_INGRESS_LABEL`=&lt;INGRESS_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                                            | STABLE       |
-| kube_ingress_created                   | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                             | STABLE       |
-| kube_ingress_metadata_resource_version | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                             | EXPERIMENTAL |
-| kube_ingress_path                      | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `host`=&lt;ingress-host&gt; <br> `path`=&lt;ingress-path&gt; <br><i> If path served by Service Backend</i> <br> `service_name`=&lt;service name for the path&gt; <br> `service_port`=&lt;service port for the path&gt;<br><i> If path served by Resource Backend</i><br> `resource_api_group`=&lt;resource backend api group&gt; <br> `resource_kind`=&lt;resource backend kind&gt; <br> `resource_name`=&lt;resource backend name&gt; | STABLE       |
-| kube_ingress_tls                       | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `tls_host`=&lt;tls hostname&gt; <br> `secret`=&lt;tls secret name&gt;                                                                                                                                                                                                                                                                                                                                                                  | STABLE       |
+| Metric name                            | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                      | Status       |
+| -------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- |------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------| ------------ |
+| kube_ingress_annotations               | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `annotation_INGRESS_ANNOTATION`=&lt;ANNOTATION_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                                                                          | EXPERIMENTAL |
+| kube_ingress_info                      | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `ingressclass`=&lt;ingress-class&gt; or `_default` if not set                                                                                                                                                                                                                                                                                                                                                                                                                     | STABLE       |
+| kube_ingress_labels                    | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `label_INGRESS_LABEL`=&lt;INGRESS_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                       | STABLE       |
+| kube_ingress_created                   | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | STABLE       |
+| kube_ingress_metadata_resource_version | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                        | EXPERIMENTAL |
+| kube_ingress_path                      | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `host`=&lt;ingress-host&gt; <br> `path`=&lt;ingress-path&gt; <br> `path_type`=&lt;ingress-path type&gt; <br> If path served by Service Backend <br> `service_name`=&lt;service name for the path&gt; <br> `service_port`=&lt;service port for the path&gt; <br> If path served by Resource Backend <br> `resource_api_group`=&lt;resource backend api group&gt; <br> `resource_kind`=&lt;resource backend kind&gt; <br> `resource_name`=&lt;resource backend name&gt; | STABLE       |
+| kube_ingress_tls                       | Gauge       |                                                                                                                           | `ingress`=&lt;ingress-name&gt; <br> `namespace`=&lt;ingress-namespace&gt; <br> `tls_host`=&lt;tls hostname&gt; <br> `secret`=&lt;tls secret name&gt;                                                                                                                                                                                                                                                                                                                                                                                                             | STABLE       |

docs/metrics/service/service-metrics.md

@@ -3,7 +3,7 @@
 | Metric name                               | Metric type | Description                                                                                                               | Unit (where applicable) | Labels/tags                                                                                                                                                                                                                                                       | Status       |
 | ----------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | ----------------------- | ----------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
 | kube_service_annotations                  | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `annotation_SERVICE_ANNOTATION`=&lt;SERVICE_ANNOTATION&gt;                                                                                          | EXPERIMENTAL |
-| kube_service_info                         | Gauge       | Information about service                                                                                                 |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `cluster_ip`=&lt;service cluster ip&gt; <br> `external_name`=&lt;service external name&gt; <br> `load_balancer_ip`=&lt;service load balancer ip&gt; | STABLE       |
+| kube_service_info                         | Gauge       | Information about service                                                                                                 |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `cluster_ip`=&lt;service cluster ip&gt; <br> `external_name`=&lt;service external name&gt; <br> `external_traffic_policy`=&lt;service external traffic policy&gt; <br> `load_balancer_ip`=&lt;service load balancer ip&gt; | STABLE       |
 | kube_service_labels                       | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `label_SERVICE_LABEL`=&lt;SERVICE_LABEL&gt;                                                                                                         | STABLE       |
 | kube_service_created                      | Gauge       | Unix creation timestamp                                                                                                   | seconds                 | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt;                                                                                                                                                          | STABLE       |
 | kube_service_spec_type                    | Gauge       | Type about service                                                                                                        |                         | `service`=&lt;service-name&gt; <br> `namespace`=&lt;service-namespace&gt; <br> `uid`=&lt;service-uid&gt; <br> `type`=&lt;ClusterIP\|NodePort\|LoadBalancer\|ExternalName&gt;                                                                                      | STABLE       |

docs/metrics/storage/persistentvolume-metrics.md

@@ -5,9 +5,9 @@
 | kube_persistentvolume_annotations        | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) |                         | `persistentvolume`=&lt;persistentvolume-name&gt; <br> `annotation_PERSISTENTVOLUME_ANNOTATION`=&lt;PERSISTENTVOLUME_ANNOTATION&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | EXPERIMENTAL |
 | kube_persistentvolume_capacity_bytes     | Gauge       |                                                                                                                           |                         | `persistentvolume`=&lt;pv-name&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | STABLE       |
 | kube_persistentvolume_status_phase       | Gauge       |                                                                                                                           |                         | `persistentvolume`=&lt;pv-name&gt; <br>`phase`=&lt;Bound\|Failed\|Pending\|Available\|Released&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                   | STABLE       |
-| kube_persistentvolume_claim_ref          | Gauge       |                                                                                                                           |                         | `persistentvolume`=&lt;pv-name&gt; <br>`claim_namespace`=&lt;<namespace>&gt; <br>`name`=&lt;<name>&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | STABLE       |
+| kube_persistentvolume_claim_ref          | Gauge       |                                                                                                                           |                         | `persistentvolume`=&lt;pv-name&gt; <br>`claim_namespace`=`<namespace>`; <br>`name`=`<name>`;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | STABLE       |
 | kube_persistentvolume_labels             | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           |                         | `persistentvolume`=&lt;persistentvolume-name&gt; <br> `label_PERSISTENTVOLUME_LABEL`=&lt;PERSISTENTVOLUME_LABEL&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                  | STABLE       |
-| kube_persistentvolume_info               | Gauge       | Information about Persistent Volumes                                                                                      |                         | `persistentvolume`=&lt;pv-name&gt; <br> `storageclass`=&lt;storageclass-name&gt; <br> `gce_persistent_disk_name`=&lt;pd-name&gt; <br> `host_path`=&lt;path-of-a-host-volume&gt; <br> `host_path_type`=&lt;host-mount-type&gt; <br> `ebs_volume_id`=&lt;ebs-volume-id&gt; <br> `azure_disk_name`=&lt;azure-disk-name&gt; <br> `fc_wwids`=&lt;fc-wwids-comma-separated&gt; <br> `fc_lun`=&lt;fc-lun&gt; <br> `fc_target_wwns`=&lt;fc-target-wwns-comma-separated&gt; <br> `iscsi_target_portal`=&lt;iscsi-target-portal&gt; <br> `iscsi_iqn`=&lt;iscsi-iqn&gt; <br> `iscsi_lun`=&lt;iscsi-lun&gt; <br> `iscsi_initiator_name`=&lt;iscsi-initiator-name&gt; <br> `local_path`=&lt;path-of-a-local-volume&gt; <br> `local_fs`=&lt;local-volume-fs-type&gt; <br> `nfs_server`=&lt;nfs-server&gt; <br> `nfs_path`=&lt;nfs-path&gt; <br> `csi_driver`=&lt;csi-driver&gt; <br> `csi_volume_handle`=&lt;csi-volume-handle&gt; | STABLE       |
+| kube_persistentvolume_info               | Gauge       | Information about Persistent Volumes                                                                                      |                         | `persistentvolume`=&lt;pv-name&gt; <br> `storageclass`=&lt;storageclass-name&gt; <br> `gce_persistent_disk_name`=&lt;pd-name&gt; <br> `host_path`=&lt;path-of-a-host-volume&gt; <br> `host_path_type`=&lt;host-mount-type&gt; <br> `ebs_volume_id`=&lt;ebs-volume-id&gt; <br> `azure_disk_name`=&lt;azure-disk-name&gt; <br> `fc_wwids`=&lt;fc-wwids-comma-separated&gt; <br> `fc_lun`=&lt;fc-lun&gt; <br> `fc_target_wwns`=&lt;fc-target-wwns-comma-separated&gt; <br> `iscsi_target_portal`=&lt;iscsi-target-portal&gt; <br> `iscsi_iqn`=&lt;iscsi-iqn&gt; <br> `iscsi_lun`=&lt;iscsi-lun&gt; <br> `iscsi_initiator_name`=&lt;iscsi-initiator-name&gt; <br> `local_path`=&lt;path-of-a-local-volume&gt; <br> `local_fs`=&lt;local-volume-fs-type&gt; <br> `nfs_server`=&lt;nfs-server&gt; <br> `nfs_path`=&lt;nfs-path&gt; <br> `csi_driver`=&lt;csi-driver&gt; <br> `csi_volume_handle`=&lt;csi-volume-handle&gt; <br> `reclaim_policy`=&lt;reclaim-policy&gt; | STABLE       |
 | kube_persistentvolume_created            | Gauge       | Unix creation timestamp                                                                                                   | seconds                 | `persistentvolume`=&lt;persistentvolume-name&gt; <br>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | EXPERIMENTAL |
 | kube_persistentvolume_deletion_timestamp | Gauge       | Unix deletion timestamp                                                                                                   | seconds                 | `persistentvolume`=&lt;persistentvolume-name&gt; <br>                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                | EXPERIMENTAL |
 | kube_persistentvolume_csi_attributes     | Gauge       | CSI attributes of the Persistent Volume, disabled by default, manage with [--metric-opt-in-list](../../developer/cli-arguments.md))     |                         | `persistentvolume`=&lt;persistentvolume-name&gt; <br> `csi_mounter`=&lt;csi-mounter&gt; <br> `csi_map_options`=&lt;csi-map-options&gt;                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                               | EXPERIMENTAL |

docs/metrics/workload/cronjob-metrics.md

@@ -3,7 +3,7 @@
 | Metric name                                    | Metric type | Description                                                                                                               | Labels/tags                                                                                                                                                     | Status       |
 | ---------------------------------------------- | ----------- | ------------------------------------------------------------------------------------------------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------ |
 | kube_cronjob_annotations                       | Gauge       | Kubernetes annotations converted to Prometheus labels controlled via [--metric-annotations-allowlist](../../developer/cli-arguments.md) | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `annotation_CRONJOB_ANNOTATION`=&lt;CRONJOB_ANNOTATION&gt;                       | EXPERIMENTAL |
-| kube_cronjob_info                              | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `schedule`=&lt;schedule&gt; <br> `concurrency_policy`=&lt;concurrency-policy&gt; | STABLE       |
+| kube_cronjob_info                              | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `schedule`=&lt;schedule&gt; <br> `concurrency_policy`=&lt;concurrency-policy&gt;  <br> `timezone`=&lt;timezone&gt; | STABLE       |
 | kube_cronjob_labels                            | Gauge       | Kubernetes labels converted to Prometheus labels controlled via [--metric-labels-allowlist](../../developer/cli-arguments.md)           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt; <br> `label_CRONJOB_LABEL`=&lt;CRONJOB_LABEL&gt;                                      | STABLE       |
 | kube_cronjob_created                           | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | STABLE       |
 | kube_cronjob_next_schedule_time                | Gauge       |                                                                                                                           | `cronjob`=&lt;cronjob-name&gt; <br> `namespace`=&lt;cronjob-namespace&gt;                                                                                       | STABLE       |

docs/metrics/workload/deployment-metrics.md

@@ -9,7 +9,7 @@
 | kube_deployment_status_replicas_unavailable                 | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
 | kube_deployment_status_replicas_updated                     | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
 | kube_deployment_status_observed_generation                  | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
-| kube_deployment_status_condition                            | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; <br> `condition`=&lt;deployment-condition&gt; <br> `status`=&lt;true\|false\|unknown&gt; | STABLE       |
+| kube_deployment_status_condition                            |                                                     Gauge                                                                                                                            |       |  `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt; <br> `reason`=&lt;deployment-transition-reason&gt; <br> `condition`=&lt;deployment-condition&gt; <br> `status`=&lt;true\|false\|unknown&gt;                                                                                                               | STABLE       |
 | kube_deployment_spec_replicas                               | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
 | kube_deployment_spec_paused                                 | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |
 | kube_deployment_spec_strategy_rollingupdate_max_unavailable | Gauge       |                                                                                                                           | `deployment`=&lt;deployment-name&gt; <br> `namespace`=&lt;deployment-namespace&gt;                                                                                          | STABLE       |

docs/metrics/workload/job-metrics.md

@@ -17,3 +17,4 @@
 | kube_job_complete                     | Gauge       |                                                                                                                           | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; <br> `condition`=&lt;true\|false\|unknown&gt;                                                                                            | STABLE       |
 | kube_job_failed                       | Gauge       |                                                                                                                           | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt; <br> `condition`=&lt;true\|false\|unknown&gt;                                                                                            | STABLE       |
 | kube_job_created                      | Gauge       |                                                                                                                           | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt;                                                                                                                                          | STABLE       |
+| kube_job_status_suspended     | Gauge       |                                                                                                                           | `job_name`=&lt;job-name&gt; <br> `namespace`=&lt;job-namespace&gt;                                                                                                                                          | EXPERIMENTAL       |

docs/metrics/workload/pod-metrics.md

@@ -53,6 +53,7 @@
 | kube_pod_status_reason                                | Gauge       | The pod status reasons                                                                                                                                                              |                                                | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `reason`=&lt;Evicted\|NodeAffinity\|NodeLost\|Shutdown\|UnexpectedAdmissionError&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                | EXPERIMENTAL | -      |
 | kube_pod_status_scheduled_time                        | Gauge       | Unix timestamp when pod moved into scheduled status                                                                                                                                 | seconds                                        | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                                                                                                          | STABLE       | -      |
 | kube_pod_status_unschedulable                         | Gauge       | Describes the unschedulable status for the pod                                                                                                                                      |                                                | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                                                                                                          | STABLE       | -      |
+| kube_pod_status_unscheduled_time                      | Gauge       | Unix timestamp when pod moved into unscheduled status                                                                                                                               | seconds                                        | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt;                                                                                                                                                                                                                                                                          | EXPERIMENTAL | -      |
 | kube_pod_tolerations                                  | Gauge       | Information about the pod tolerations                                                                                                                                               |                                                | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; <br> `key`=&lt;toleration-key&gt; <br> `operator`=&lt;toleration-operator&gt; <br> `value`=&lt;toleration-value&gt; <br> `effect`=&lt;toleration-effect&gt; `toleration_seconds`=&lt;toleration-seconds&gt;                                                              | EXPERIMENTAL | -      |
 | kube_pod_service_account                              | Gauge       | The service account for a pod                                                                                                                                                       |                                                | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; <br> `service_account`=&lt;service_account&gt;                                                                                                                                                                                                                           | EXPERIMENTAL | -      |
 | kube_pod_scheduler                              | Gauge       | The scheduler for a pod                                                                                                                                                       |                                                | `pod`=&lt;pod-name&gt; <br> `namespace`=&lt;pod-namespace&gt; <br> `uid`=&lt;pod-uid&gt; <br> `name`=&lt;scheduler-name&gt;                                                                                                                                                                                                                           | EXPERIMENTAL | -      |

docs/metrics/workload/statefulset-metrics.md

@@ -10,7 +10,7 @@
 | kube_statefulset_status_replicas_updated                | Gauge       |                                                                                                                           | `statefulset`=&lt;statefulset-name&gt; <br> `namespace`=&lt;statefulset-namespace&gt;                                                                                                                               | STABLE       |
 | kube_statefulset_status_observed_generation             | Gauge       |                                                                                                                           | `statefulset`=&lt;statefulset-name&gt; <br> `namespace`=&lt;statefulset-namespace&gt;                                                                                                                               | STABLE       |
 | kube_statefulset_replicas                               | Gauge       |                                                                                                                           | `statefulset`=&lt;statefulset-name&gt; <br> `namespace`=&lt;statefulset-namespace&gt;                                                                                                                               | STABLE       |
-| kube_statefulset_ordinals_start                         | Gauge       |                                                                                                                           | `statefulset`=&lt;statefulset-name&gt; <br> `namespace`=&lt;statefulset-namespace&gt;                                                                                                                               | ALPHA        |
+| kube_statefulset_ordinals_start                         | Gauge       |                                                                                                                           | `statefulset`=&lt;statefulset-name&gt; <br> `namespace`=&lt;statefulset-namespace&gt;                                                                                                                               | STABLE       |
 | kube_statefulset_metadata_generation                    | Gauge       |                                                                                                                           | `statefulset`=&lt;statefulset-name&gt; <br> `namespace`=&lt;statefulset-namespace&gt;                                                                                                                               | STABLE       |
 | kube_statefulset_persistentvolumeclaim_retention_policy | Gauge       |                                                                                                                           | `statefulset`=&lt;statefulset-name&gt; <br> `namespace`=&lt;statefulset-namespace&gt; <br> `when_deleted`=&lt;statefulset-when-deleted-pvc-policy&gt; <br> `when_scaled`=&lt;statefulset-when-scaled-pvc-policy&gt; | EXPERIMENTAL |
 | kube_statefulset_created                                | Gauge       |                                                                                                                           | `statefulset`=&lt;statefulset-name&gt; <br> `namespace`=&lt;statefulset-namespace&gt;                                                                                                                               | STABLE       |

examples/autosharding/cluster-role-binding.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
 roleRef:
   apiGroup: rbac.authorization.k8s.io

examples/autosharding/cluster-role.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
 rules:
 - apiGroups:

examples/autosharding/role-binding.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system
 roleRef:

examples/autosharding/role.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system
 rules:

examples/autosharding/service-account.yaml

@@ -5,6 +5,6 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system

examples/autosharding/service.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system
 spec:

examples/autosharding/statefulset.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system
 spec:
@@ -18,7 +18,7 @@ spec:
       labels:
         app.kubernetes.io/component: exporter
         app.kubernetes.io/name: kube-state-metrics
-        app.kubernetes.io/version: 2.13.0
+        app.kubernetes.io/version: 2.16.0
     spec:
       automountServiceAccountToken: true
       containers:
@@ -34,7 +34,7 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.namespace
-        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0
+        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.16.0
         livenessProbe:
           httpGet:
             path: /livez

examples/daemonsetsharding/cluster-role-binding.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
 roleRef:
   apiGroup: rbac.authorization.k8s.io

examples/daemonsetsharding/cluster-role.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
 rules:
 - apiGroups:

examples/daemonsetsharding/daemonset-service.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics-shard
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics-shard
   namespace: kube-system
 spec:

examples/daemonsetsharding/daemonset.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics-shard
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics-shard
   namespace: kube-system
 spec:
@@ -16,7 +16,7 @@ spec:
       labels:
         app.kubernetes.io/component: exporter
         app.kubernetes.io/name: kube-state-metrics-shard
-        app.kubernetes.io/version: 2.13.0
+        app.kubernetes.io/version: 2.16.0
     spec:
       automountServiceAccountToken: true
       containers:
@@ -29,7 +29,7 @@ spec:
             fieldRef:
               apiVersion: v1
               fieldPath: spec.nodeName
-        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0
+        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.16.0
         livenessProbe:
           httpGet:
             path: /livez

examples/daemonsetsharding/deployment-service.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system
 spec:

examples/daemonsetsharding/deployment-unscheduled-pods-fetching-service.yaml

@@ -0,0 +1,20 @@
+apiVersion: v1
+kind: Service
+metadata:
+  labels:
+    app.kubernetes.io/component: exporter
+    app.kubernetes.io/name: kube-state-metrics-unscheduled-pods-fetching
+    app.kubernetes.io/version: 2.16.0
+  name: kube-state-metrics-unscheduled-pods-fetching
+  namespace: kube-system
+spec:
+  clusterIP: None
+  ports:
+  - name: http-metrics
+    port: 8080
+    targetPort: http-metrics
+  - name: telemetry
+    port: 8081
+    targetPort: telemetry
+  selector:
+    app.kubernetes.io/name: kube-state-metrics-unscheduled-pods-fetching

examples/daemonsetsharding/deployment-unscheduled-pods-fetching.yaml

@@ -3,35 +3,35 @@ kind: Deployment
 metadata:
   labels:
     app.kubernetes.io/component: exporter
-    app.kubernetes.io/name: kube-state-metrics-pods
-    app.kubernetes.io/version: 2.13.0
-  name: kube-state-metrics-pods
+    app.kubernetes.io/name: kube-state-metrics-unscheduled-pods-fetching
+    app.kubernetes.io/version: 2.16.0
+  name: kube-state-metrics-unscheduled-pods-fetching
   namespace: kube-system
 spec:
   replicas: 1
   selector:
     matchLabels:
-      app.kubernetes.io/name: kube-state-metrics
+      app.kubernetes.io/name: kube-state-metrics-unscheduled-pods-fetching
   template:
     metadata:
       labels:
         app.kubernetes.io/component: exporter
-        app.kubernetes.io/name: kube-state-metrics
-        app.kubernetes.io/version: 2.13.0
+        app.kubernetes.io/name: kube-state-metrics-unscheduled-pods-fetching
+        app.kubernetes.io/version: 2.16.0
     spec:
       automountServiceAccountToken: true
       containers:
       - args:
         - --resources=pods
-        - --node=""
-        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0
+        - --track-unscheduled-pods
+        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.16.0
         livenessProbe:
           httpGet:
             path: /livez
             port: http-metrics
           initialDelaySeconds: 5
           timeoutSeconds: 5
-        name: kube-state-metrics
+        name: kube-state-metrics-unscheduled-pods-fetching
         ports:
         - containerPort: 8080
           name: http-metrics

examples/daemonsetsharding/deployment.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system
 spec:
@@ -17,13 +17,13 @@ spec:
       labels:
         app.kubernetes.io/component: exporter
         app.kubernetes.io/name: kube-state-metrics
-        app.kubernetes.io/version: 2.13.0
+        app.kubernetes.io/version: 2.16.0
     spec:
       automountServiceAccountToken: true
       containers:
       - args:
         - --resources=certificatesigningrequests,configmaps,cronjobs,daemonsets,deployments,endpoints,horizontalpodautoscalers,ingresses,jobs,leases,limitranges,mutatingwebhookconfigurations,namespaces,networkpolicies,nodes,persistentvolumeclaims,persistentvolumes,poddisruptionbudgets,replicasets,replicationcontrollers,resourcequotas,secrets,services,statefulsets,storageclasses,validatingwebhookconfigurations,volumeattachments
-        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0
+        image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.16.0
         livenessProbe:
           httpGet:
             path: /livez

examples/daemonsetsharding/service-account.yaml

@@ -5,6 +5,6 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system

examples/standard/cluster-role-binding.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
 roleRef:
   apiGroup: rbac.authorization.k8s.io

examples/standard/cluster-role.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
 rules:
 - apiGroups:

examples/standard/deployment.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system
 spec:
@@ -17,11 +17,11 @@ spec:
       labels:
         app.kubernetes.io/component: exporter
         app.kubernetes.io/name: kube-state-metrics
-        app.kubernetes.io/version: 2.13.0
+        app.kubernetes.io/version: 2.16.0
     spec:
       automountServiceAccountToken: true
       containers:
-      - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.13.0
+      - image: registry.k8s.io/kube-state-metrics/kube-state-metrics:v2.16.0
         livenessProbe:
           httpGet:
             path: /livez

examples/standard/service-account.yaml

@@ -5,6 +5,6 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system

examples/standard/service.yaml

@@ -4,7 +4,7 @@ metadata:
   labels:
     app.kubernetes.io/component: exporter
     app.kubernetes.io/name: kube-state-metrics
-    app.kubernetes.io/version: 2.13.0
+    app.kubernetes.io/version: 2.16.0
   name: kube-state-metrics
   namespace: kube-system
 spec:

go.mod

@@ -1,100 +1,264 @@
 module k8s.io/kube-state-metrics/v2
 
-go 1.22.0
-
-toolchain go1.22.5
+go 1.24.0
 
 require (
-	github.com/KimMachineGun/automemlimit v0.6.1
+	github.com/KimMachineGun/automemlimit v0.7.4
 	github.com/dgryski/go-jump v0.0.0-20211018200510-ba001c3ffce0
-	github.com/fsnotify/fsnotify v1.7.0
-	github.com/gobuffalo/flect v1.0.2
-	github.com/google/go-cmp v0.6.0
-	github.com/oklog/run v1.1.0
-	github.com/prometheus/client_golang v1.19.1
-	github.com/prometheus/client_model v0.6.1
-	github.com/prometheus/common v0.55.0
-	github.com/prometheus/exporter-toolkit v0.11.0
+	github.com/dlclark/regexp2 v1.11.5
+	github.com/fsnotify/fsnotify v1.9.0
+	github.com/go-logr/logr v1.4.3
+	github.com/gobuffalo/flect v1.0.3
+	github.com/google/go-cmp v0.7.0
+	github.com/oklog/run v1.2.0
+	github.com/prometheus/client_golang v1.23.0
+	github.com/prometheus/client_model v0.6.2
+	github.com/prometheus/common v0.65.0
+	github.com/prometheus/exporter-toolkit v0.14.0
 	github.com/robfig/cron/v3 v3.0.1
-	github.com/spf13/cobra v1.8.1
-	github.com/spf13/viper v1.19.0
-	github.com/stretchr/testify v1.9.0
-	gopkg.in/yaml.v3 v3.0.1
-	k8s.io/api v0.30.3
-	k8s.io/apimachinery v0.30.3
-	k8s.io/client-go v0.30.3
-	k8s.io/component-base v0.30.3
+	github.com/spf13/cobra v1.9.1
+	github.com/spf13/viper v1.20.1
+	github.com/stretchr/testify v1.10.0
+	k8s.io/api v0.33.3
+	k8s.io/apimachinery v0.33.3
+	k8s.io/client-go v0.33.3
+	k8s.io/component-base v0.33.3
 	k8s.io/klog/v2 v2.130.1
-	k8s.io/sample-controller v0.30.3
-	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
+	k8s.io/sample-controller v0.33.3
+	k8s.io/utils v0.0.0-20250604170112-4c0f3b243397
+	sigs.k8s.io/controller-runtime v0.21.0
+	sigs.k8s.io/yaml v1.6.0
 )
 
 require (
+	cel.dev/expr v0.19.1 // indirect
+	cloud.google.com/go v0.116.0 // indirect
+	cloud.google.com/go/auth v0.13.0 // indirect
+	cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect
+	cloud.google.com/go/compute/metadata v0.6.0 // indirect
+	cloud.google.com/go/iam v1.2.2 // indirect
+	cloud.google.com/go/monitoring v1.21.2 // indirect
+	cloud.google.com/go/storage v1.49.0 // indirect
+	cuelang.org/go v0.11.0 // indirect
+	dario.cat/mergo v1.0.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azcore v1.16.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/azidentity v1.8.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/internal v1.10.0 // indirect
+	github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v1.5.0 // indirect
+	github.com/Azure/go-autorest v14.2.0+incompatible // indirect
+	github.com/Azure/go-autorest/autorest/to v0.4.0 // indirect
+	github.com/AzureAD/microsoft-authentication-library-for-go v1.2.2 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.25.0 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.48.1 // indirect
+	github.com/GoogleCloudPlatform/opentelemetry-operations-go/internal/resourcemapping v0.48.1 // indirect
+	github.com/Masterminds/goutils v1.1.1 // indirect
+	github.com/Masterminds/semver/v3 v3.3.1 // indirect
+	github.com/Microsoft/go-winio v0.6.2 // indirect
+	github.com/ProtonMail/go-crypto v1.0.0 // indirect
+	github.com/Shopify/ejson v1.5.3 // indirect
+	github.com/aclements/go-moremath v0.0.0-20210112150236-f10218a38794 // indirect
+	github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect
+	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 // indirect
+	github.com/antlr4-go/antlr/v4 v4.13.0 // indirect
+	github.com/armon/go-metrics v0.4.1 // indirect
+	github.com/aws/aws-sdk-go v1.55.5 // indirect
+	github.com/aws/aws-sdk-go-v2 v1.32.6 // indirect
+	github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.6.7 // indirect
+	github.com/aws/aws-sdk-go-v2/config v1.28.6 // indirect
+	github.com/aws/aws-sdk-go-v2/credentials v1.17.47 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.21 // indirect
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.17.10 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.25 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.25 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/ini v1.8.1 // indirect
+	github.com/aws/aws-sdk-go-v2/internal/v4a v1.3.25 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.12.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/checksum v1.4.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.12.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.18.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.71.0 // indirect
+	github.com/aws/aws-sdk-go-v2/service/secretsmanager v1.34.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssm v1.56.1 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sso v1.24.7 // indirect
+	github.com/aws/aws-sdk-go-v2/service/ssooidc v1.28.6 // indirect
+	github.com/aws/aws-sdk-go-v2/service/sts v1.33.2 // indirect
+	github.com/aws/smithy-go v1.22.1 // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
-	github.com/cilium/ebpf v0.9.1 // indirect
-	github.com/containerd/cgroups/v3 v3.0.1 // indirect
+	github.com/brancz/gojsontoyaml v0.1.0 // indirect
+	github.com/campoy/embedmd v1.0.0 // indirect
+	github.com/cenkalti/backoff/v4 v4.3.0 // indirect
+	github.com/census-instrumentation/opencensus-proto v0.4.1 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
+	github.com/cloudflare/circl v1.3.9 // indirect
+	github.com/cncf/xds/go v0.0.0-20240905190251-b4127c9b8d78 // indirect
+	github.com/cockroachdb/apd/v3 v3.2.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
+	github.com/cyphar/filepath-securejoin v0.2.5 // indirect
 	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
-	github.com/docker/go-units v0.4.0 // indirect
+	github.com/dustin/gojson v0.0.0-20160307161227-2e71ec9dd5ad // indirect
+	github.com/elliotchance/orderedmap/v2 v2.2.0 // indirect
 	github.com/emicklei/go-restful/v3 v3.11.0 // indirect
-	github.com/evanphx/json-patch v5.6.0+incompatible // indirect
-	github.com/go-kit/log v0.2.1 // indirect
-	github.com/go-logfmt/logfmt v0.5.1 // indirect
-	github.com/go-logr/logr v1.4.1 // indirect
-	github.com/go-openapi/jsonpointer v0.19.6 // indirect
+	github.com/emirpasic/gods v1.18.1 // indirect
+	github.com/envoyproxy/go-control-plane v0.13.1 // indirect
+	github.com/envoyproxy/protoc-gen-validate v1.1.0 // indirect
+	github.com/fatih/color v1.17.0 // indirect
+	github.com/felixge/httpsnoop v1.0.4 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/ghodss/yaml v1.0.0 // indirect
+	github.com/go-git/gcfg v1.5.1-0.20230307220236-3a3c6141e376 // indirect
+	github.com/go-git/go-billy/v5 v5.6.0 // indirect
+	github.com/go-jose/go-jose/v4 v4.0.2 // indirect
+	github.com/go-logr/stdr v1.2.2 // indirect
+	github.com/go-openapi/jsonpointer v0.21.0 // indirect
 	github.com/go-openapi/jsonreference v0.20.2 // indirect
-	github.com/go-openapi/swag v0.22.3 // indirect
-	github.com/godbus/dbus/v5 v5.0.4 // indirect
+	github.com/go-openapi/swag v0.23.0 // indirect
+	github.com/go-viper/mapstructure/v2 v2.3.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
-	github.com/golang/protobuf v1.5.4 // indirect
-	github.com/google/gnostic-models v0.6.8 // indirect
-	github.com/google/gofuzz v1.2.0 // indirect
-	github.com/google/uuid v1.4.0 // indirect
+	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
+	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/google/cel-go v0.23.2 // indirect
+	github.com/google/gnostic-models v0.6.9 // indirect
+	github.com/google/go-jsonnet v0.20.0 // indirect
+	github.com/google/s2a-go v0.1.8 // indirect
+	github.com/google/uuid v1.6.0 // indirect
+	github.com/google/wire v0.6.0 // indirect
+	github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect
+	github.com/googleapis/gax-go/v2 v2.14.1 // indirect
+	github.com/gosimple/slug v1.14.0 // indirect
+	github.com/gosimple/unidecode v1.0.1 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.24.0 // indirect
+	github.com/hack-pad/hackpadfs v0.2.4 // indirect
+	github.com/hairyhenderson/go-fsimpl v0.2.1 // indirect
+	github.com/hairyhenderson/go-git/v5 v5.12.1-0.20240530140403-1b868a7b8a3c // indirect
+	github.com/hairyhenderson/gomplate/v4 v4.3.0 // indirect
+	github.com/hairyhenderson/toml v0.4.2-0.20210923231440-40456b8e66cf // indirect
+	github.com/hairyhenderson/xignore v0.3.3-0.20230403012150-95fe86932830 // indirect
+	github.com/hairyhenderson/yaml v0.0.0-20220618171115-2d35fca545ce // indirect
+	github.com/hashicorp/consul/api v1.30.0 // indirect
+	github.com/hashicorp/errwrap v1.1.0 // indirect
+	github.com/hashicorp/go-cleanhttp v0.5.2 // indirect
+	github.com/hashicorp/go-hclog v1.6.3 // indirect
+	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
+	github.com/hashicorp/go-multierror v1.1.1 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.7 // indirect
+	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
+	github.com/hashicorp/go-secure-stdlib/awsutil v0.3.0 // indirect
+	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.8 // indirect
+	github.com/hashicorp/go-secure-stdlib/strutil v0.1.2 // indirect
+	github.com/hashicorp/go-sockaddr v1.0.7 // indirect
+	github.com/hashicorp/go-uuid v1.0.3 // indirect
+	github.com/hashicorp/golang-lru v1.0.2 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
-	github.com/imdario/mergo v0.3.6 // indirect
+	github.com/hashicorp/serf v0.10.1 // indirect
+	github.com/hashicorp/vault/api v1.15.0 // indirect
+	github.com/hashicorp/vault/api/auth/approle v0.8.0 // indirect
+	github.com/hashicorp/vault/api/auth/aws v0.8.0 // indirect
+	github.com/hashicorp/vault/api/auth/userpass v0.8.0 // indirect
 	github.com/inconshreveable/mousetrap v1.1.0 // indirect
+	github.com/itchyny/gojq v0.12.17 // indirect
+	github.com/itchyny/timefmt-go v0.1.6 // indirect
+	github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/joho/godotenv v1.5.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
 	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/magiconair/properties v1.8.7 // indirect
+	github.com/jsonnet-bundler/jsonnet-bundler v0.6.0 // indirect
+	github.com/kevinburke/ssh_config v1.2.0 // indirect
+	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lmittmann/tint v1.0.6 // indirect
 	github.com/mailru/easyjson v0.7.7 // indirect
+	github.com/mattn/go-colorable v0.1.13 // indirect
+	github.com/mattn/go-isatty v0.0.20 // indirect
+	github.com/mdlayher/socket v0.4.1 // indirect
+	github.com/mdlayher/vsock v1.2.1 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
 	github.com/modern-go/reflect2 v1.0.2 // indirect
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
-	github.com/opencontainers/runtime-spec v1.0.2 // indirect
 	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
-	github.com/pelletier/go-toml/v2 v2.2.2 // indirect
+	github.com/pelletier/go-toml/v2 v2.2.3 // indirect
+	github.com/pjbgf/sha1cd v0.3.0 // indirect
+	github.com/pkg/browser v0.0.0-20240102092130-5ac0b6a4141c // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/planetscale/vtprotobuf v0.6.1-0.20240319094008-0393e58bdf10 // indirect
 	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
-	github.com/prometheus/procfs v0.15.1 // indirect
-	github.com/sagikazarmark/locafero v0.4.0 // indirect
-	github.com/sagikazarmark/slog-shim v0.1.0 // indirect
-	github.com/sirupsen/logrus v1.8.1 // indirect
+	github.com/prometheus/procfs v0.16.1 // indirect
+	github.com/ryanuber/go-glob v1.0.0 // indirect
+	github.com/sagikazarmark/locafero v0.7.0 // indirect
+	github.com/sergi/go-diff v1.3.2-0.20230802210424-5b0b94c5c0d3 // indirect
+	github.com/skeema/knownhosts v1.2.2 // indirect
 	github.com/sourcegraph/conc v0.3.0 // indirect
-	github.com/spf13/afero v1.11.0 // indirect
-	github.com/spf13/cast v1.6.0 // indirect
-	github.com/spf13/pflag v1.0.5 // indirect
+	github.com/spf13/afero v1.12.0 // indirect
+	github.com/spf13/cast v1.7.1 // indirect
+	github.com/spf13/pflag v1.0.6 // indirect
+	github.com/stoewer/go-strcase v1.3.0 // indirect
 	github.com/subosito/gotenv v1.6.0 // indirect
+	github.com/ugorji/go/codec v1.2.12 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
+	github.com/xanzy/ssh-agent v0.3.3 // indirect
+	go.opencensus.io v0.24.0 // indirect
+	go.opentelemetry.io/auto/sdk v1.1.0 // indirect
+	go.opentelemetry.io/contrib/detectors/gcp v1.29.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.58.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.58.0 // indirect
+	go.opentelemetry.io/otel v1.33.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.33.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.33.0 // indirect
+	go.opentelemetry.io/otel/metric v1.33.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.33.0 // indirect
+	go.opentelemetry.io/otel/sdk/metric v1.29.0 // indirect
+	go.opentelemetry.io/otel/trace v1.33.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.4.0 // indirect
 	go.uber.org/multierr v1.11.0 // indirect
-	golang.org/x/crypto v0.24.0 // indirect
-	golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect
-	golang.org/x/net v0.26.0 // indirect
-	golang.org/x/oauth2 v0.21.0 // indirect
-	golang.org/x/sync v0.7.0 // indirect
-	golang.org/x/sys v0.21.0 // indirect
-	golang.org/x/term v0.21.0 // indirect
-	golang.org/x/text v0.16.0 // indirect
-	golang.org/x/time v0.5.0 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	go.yaml.in/yaml/v2 v2.4.2 // indirect
+	go.yaml.in/yaml/v3 v3.0.3 // indirect
+	go4.org/intern v0.0.0-20230525184215-6c62f75575cb // indirect
+	go4.org/netipx v0.0.0-20231129151722-fdeea329fbba // indirect
+	go4.org/unsafe/assume-no-moving-gc v0.0.0-20231121144256-b99613f794b6 // indirect
+	gocloud.dev v0.40.0 // indirect
+	golang.org/x/crypto v0.40.0 // indirect
+	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
+	golang.org/x/mod v0.25.0 // indirect
+	golang.org/x/net v0.41.0 // indirect
+	golang.org/x/oauth2 v0.30.0 // indirect
+	golang.org/x/perf v0.0.0-20250214215153-c95ad7d5b636 // indirect
+	golang.org/x/sync v0.16.0 // indirect
+	golang.org/x/sys v0.34.0 // indirect
+	golang.org/x/term v0.33.0 // indirect
+	golang.org/x/text v0.27.0 // indirect
+	golang.org/x/time v0.9.0 // indirect
+	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
+	google.golang.org/api v0.215.0 // indirect
+	google.golang.org/genproto v0.0.0-20241118233622-e639e219e697 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20241209162323-e6fa225c2576 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20241223144023-3abc09e42ca8 // indirect
+	google.golang.org/grpc v1.68.1 // indirect
+	google.golang.org/grpc/stats/opentelemetry v0.0.0-20240907200651-3ffb98b2c93a // indirect
+	google.golang.org/protobuf v1.36.6 // indirect
+	gopkg.in/alecthomas/kingpin.v2 v2.2.6 // indirect
+	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/ini.v1 v1.67.0 // indirect
+	gopkg.in/warnings.v0 v0.1.2 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
-	k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect
-	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect
-	sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect
-	sigs.k8s.io/yaml v1.3.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
+	inet.af/netaddr v0.0.0-20230525184311-b8eac61e914a // indirect
+	k8s.io/apiserver v0.33.0 // indirect
+	k8s.io/kube-openapi v0.0.0-20250318190949-c8a335a9a2ff // indirect
+	sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.31.2 // indirect
+	sigs.k8s.io/json v0.0.0-20241010143419-9aa6b5e7a4b3 // indirect
+	sigs.k8s.io/randfill v1.0.0 // indirect
+	sigs.k8s.io/structured-merge-diff/v4 v4.6.0 // indirect
+)
+
+tool (
+	github.com/brancz/gojsontoyaml
+	github.com/campoy/embedmd
+	github.com/google/go-jsonnet/cmd/jsonnet
+	github.com/hairyhenderson/gomplate/v4/cmd/gomplate
+	github.com/jsonnet-bundler/jsonnet-bundler/cmd/jb
+	golang.org/x/perf/cmd/benchstat
 )

internal/discovery/discovery.go

@@ -64,8 +64,8 @@ func (r *CRDiscoverer) StartDiscovery(ctx context.Context, config *rest.Config)
 					},
 					Plural: p,
 				}
-				r.AppendToMap(gotGVKP)
 				r.SafeWrite(func() {
+					r.AppendToMap(gotGVKP)
 					r.WasUpdated = true
 				})
 			}
@@ -89,8 +89,8 @@ func (r *CRDiscoverer) StartDiscovery(ctx context.Context, config *rest.Config)
 					},
 					Plural: p,
 				}
-				r.RemoveFromMap(gotGVKP)
 				r.SafeWrite(func() {
+					r.RemoveFromMap(gotGVKP)
 					r.WasUpdated = true
 				})
 			}
@@ -127,23 +127,20 @@ func (r *CRDiscoverer) ResolveGVKToGVKPs(gvk schema.GroupVersionKind) (resolvedG
 	hasKind := k != "" && k != "*"
 	// No need to resolve, return.
 	if hasVersion && hasKind {
-		var p string
 		for _, el := range r.Map[g][v] {
 			if el.Kind == k {
-				p = el.Plural
-				break
+				return []groupVersionKindPlural{
+					{
+						GroupVersionKind: schema.GroupVersionKind{
+							Group:   g,
+							Version: v,
+							Kind:    k,
+						},
+						Plural: el.Plural,
+					},
+				}, nil
 			}
 		}
-		return []groupVersionKindPlural{
-			{
-				GroupVersionKind: schema.GroupVersionKind{
-					Group:   g,
-					Version: v,
-					Kind:    k,
-				},
-				Plural: p,
-			},
-		}, nil
 	}
 	if hasVersion && !hasKind {
 		kinds := r.Map[g][v]
@@ -203,10 +200,6 @@ func (r *CRDiscoverer) PollForCacheUpdates(
 ) {
 	// The interval at which we will check the cache for updates.
 	t := time.NewTicker(Interval)
-	// Track previous context to allow refreshing cache.
-	olderContext, olderCancel := context.WithCancel(ctx)
-	// Prevent context leak (kill the last metric handler instance).
-	defer olderCancel()
 	generateMetrics := func() {
 		// Get families for discovered factories.
 		customFactories, err := factoryGenerator()
@@ -216,7 +209,16 @@ func (r *CRDiscoverer) PollForCacheUpdates(
 		// Update the list of enabled custom resources.
 		var enabledCustomResources []string
 		for _, factory := range customFactories {
-			gvrString := util.GVRFromType(factory.Name(), factory.ExpectedType()).String()
+			gvr, err := util.GVRFromType(factory.Name(), factory.ExpectedType())
+			if err != nil {
+				klog.ErrorS(err, "failed to update custom resource stores")
+			}
+			var gvrString string
+			if gvr != nil {
+				gvrString = gvr.String()
+			} else {
+				gvrString = factory.Name()
+			}
 			enabledCustomResources = append(enabledCustomResources, gvrString)
 		}
 		// Create clients for discovered factories.
@@ -239,21 +241,8 @@ func (r *CRDiscoverer) PollForCacheUpdates(
 		r.SafeWrite(func() {
 			r.WasUpdated = false
 		})
-		// Run the metrics handler with updated configs.
-		olderContext, olderCancel = context.WithCancel(ctx)
-		go func() {
-			// Blocks indefinitely until the unbuffered context is cancelled to serve metrics for that duration.
-			err = m.Run(olderContext)
-			if err != nil {
-				// Check if context was cancelled.
-				select {
-				case <-olderContext.Done():
-					// Context cancelled, don't really need to log this though.
-				default:
-					klog.ErrorS(err, "failed to run metrics handler")
-				}
-			}
-		}()
+		// Update metric handler with the new configs.
+		m.BuildWriters(ctx)
 	}
 	go func() {
 		for range t.C {
@@ -269,7 +258,6 @@ func (r *CRDiscoverer) PollForCacheUpdates(
 					shouldGenerateMetrics = r.WasUpdated
 				})
 				if shouldGenerateMetrics {
-					olderCancel()
 					generateMetrics()
 					klog.InfoS("discovery finished, cache updated")
 				}

internal/discovery/discovery_test.go

@@ -1,14 +1,14 @@
 /*
-Copyright 2023 The Kubernetes Authors All rights reserved.
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-    http://www.apache.org/licenses/LICENSE-2.0
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
+	Copyright 2023 The Kubernetes Authors All rights reserved.
+	Licensed under the Apache License, Version 2.0 (the "License");
+	you may not use this file except in compliance with the License.
+	You may obtain a copy of the License at
+	    http://www.apache.org/licenses/LICENSE-2.0
+	Unless required by applicable law or agreed to in writing, software
+	distributed under the License is distributed on an "AS IS" BASIS,
+	WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+	See the License for the specific language governing permissions and
+	limitations under the License.
 */
 
 package discovery
@@ -26,7 +26,6 @@ func TestGVKMapsResolveGVK(t *testing.T) {
 		desc    string
 		gvkmaps *CRDiscoverer
 		gvk     schema.GroupVersionKind
-		got     []groupVersionKindPlural
 		want    []groupVersionKindPlural
 	}
 	testcases := []testcase{
@@ -164,6 +163,23 @@ func TestGVKMapsResolveGVK(t *testing.T) {
 				},
 			},
 		},
+		{
+			desc: "fixed version and kind, no matching cache entry",
+			gvkmaps: &CRDiscoverer{
+				Map: map[string]map[string][]kindPlural{
+					"testgroup": {
+						"v1": {
+							kindPlural{
+								Kind:   "TestObject2",
+								Plural: "testobjects2",
+							},
+						},
+					},
+				},
+			},
+			gvk:  schema.GroupVersionKind{Group: "testgroup", Version: "v1", Kind: "TestObject1"},
+			want: nil,
+		},
 	}
 	for _, tc := range testcases {
 		got, err := tc.gvkmaps.ResolveGVKToGVKPs(tc.gvk)

internal/discovery/types.go

@@ -37,18 +37,20 @@ type kindPlural struct {
 
 // CRDiscoverer provides a cache of the collected GVKs, along with helper utilities.
 type CRDiscoverer struct {
-	// m is a mutex to protect the cache.
-	m sync.RWMutex
-	// Map is a cache of the collected GVKs.
-	Map map[string]map[string][]kindPlural
-	// ShouldUpdate is a flag that indicates whether the cache was updated.
-	WasUpdated bool
 	// CRDsAddEventsCounter tracks the number of times that the CRD informer triggered the "add" event.
 	CRDsAddEventsCounter prometheus.Counter
 	// CRDsDeleteEventsCounter tracks the number of times that the CRD informer triggered the "remove" event.
 	CRDsDeleteEventsCounter prometheus.Counter
 	// CRDsCacheCountGauge tracks the net amount of CRDs affecting the cache at this point.
 	CRDsCacheCountGauge prometheus.Gauge
+	// Map is a cache of the collected GVKs.
+	Map map[string]map[string][]kindPlural
+	// GVKToReflectorStopChanMap is a map of GVKs to channels that can be used to stop their corresponding reflector.
+	GVKToReflectorStopChanMap map[string]chan struct{}
+	// m is a mutex to protect the cache.
+	m sync.RWMutex
+	// ShouldUpdate is a flag that indicates whether the cache was updated.
+	WasUpdated bool
 }
 
 // SafeRead executes the given function while holding a read lock.
@@ -70,6 +72,9 @@ func (r *CRDiscoverer) AppendToMap(gvkps ...groupVersionKindPlural) {
 	if r.Map == nil {
 		r.Map = map[string]map[string][]kindPlural{}
 	}
+	if r.GVKToReflectorStopChanMap == nil {
+		r.GVKToReflectorStopChanMap = map[string]chan struct{}{}
+	}
 	for _, gvkp := range gvkps {
 		if _, ok := r.Map[gvkp.Group]; !ok {
 			r.Map[gvkp.Group] = map[string][]kindPlural{}
@@ -78,6 +83,7 @@ func (r *CRDiscoverer) AppendToMap(gvkps ...groupVersionKindPlural) {
 			r.Map[gvkp.Group][gvkp.Version] = []kindPlural{}
 		}
 		r.Map[gvkp.Group][gvkp.Version] = append(r.Map[gvkp.Group][gvkp.Version], kindPlural{Kind: gvkp.Kind, Plural: gvkp.Plural})
+		r.GVKToReflectorStopChanMap[gvkp.GroupVersionKind.String()] = make(chan struct{})
 	}
 }
 
@@ -92,6 +98,8 @@ func (r *CRDiscoverer) RemoveFromMap(gvkps ...groupVersionKindPlural) {
 		}
 		for i, el := range r.Map[gvkp.Group][gvkp.Version] {
 			if el.Kind == gvkp.Kind {
+				close(r.GVKToReflectorStopChanMap[gvkp.GroupVersionKind.String()])
+				delete(r.GVKToReflectorStopChanMap, gvkp.GroupVersionKind.String())
 				if len(r.Map[gvkp.Group][gvkp.Version]) == 1 {
 					delete(r.Map[gvkp.Group], gvkp.Version)
 					if len(r.Map[gvkp.Group]) == 0 {

internal/store/builder.go

@@ -20,7 +20,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
-	"sort"
+	"slices"
 	"strconv"
 	"strings"
 	"time"
@@ -38,6 +38,7 @@ import (
 	policyv1 "k8s.io/api/policy/v1"
 	rbacv1 "k8s.io/api/rbac/v1"
 	storagev1 "k8s.io/api/storage/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/cache"
 	"k8s.io/klog/v2"
@@ -65,24 +66,27 @@ var _ ksmtypes.BuilderInterface = &Builder{}
 // Builder helps to build store. It follows the builder pattern
 // (https://en.wikipedia.org/wiki/Builder_pattern).
 type Builder struct {
-	kubeClient            clientset.Interface
-	customResourceClients map[string]interface{}
-	namespaces            options.NamespaceList
-	// namespaceFilter is inside fieldSelectorFilter
-	fieldSelectorFilter           string
+	kubeClient                    clientset.Interface
 	ctx                           context.Context
-	enabledResources              []string
 	familyGeneratorFilter         generator.FamilyGeneratorFilter
+	customResourceClients         map[string]interface{}
 	listWatchMetrics              *watch.ListWatchMetrics
 	shardingMetrics               *sharding.Metrics
-	shard                         int32
-	totalShards                   int
 	buildStoresFunc               ksmtypes.BuildStoresFunc
 	buildCustomResourceStoresFunc ksmtypes.BuildCustomResourceStoresFunc
 	allowAnnotationsList          map[string][]string
 	allowLabelsList               map[string][]string
-	useAPIServerCache             bool
 	utilOptions                   *options.Options
+	// namespaceFilter is inside fieldSelectorFilter
+	fieldSelectorFilter string
+	namespaces          options.NamespaceList
+	enabledResources    []string
+	totalShards         int
+	shard               int32
+	useAPIServerCache   bool
+	objectLimit         int64
+
+	GVKToReflectorStopChanMap *map[string]chan struct{}
 }
 
 // NewBuilder returns a new builder.
@@ -113,12 +117,10 @@ func (b *Builder) WithEnabledResources(r []string) error {
 		}
 	}
 
-	var sortedResources []string
-	sortedResources = append(sortedResources, r...)
+	b.enabledResources = append(b.enabledResources, r...)
+	slices.Sort(b.enabledResources)
+	b.enabledResources = slices.Compact(b.enabledResources)
 
-	sort.Strings(sortedResources)
-
-	b.enabledResources = append(b.enabledResources, sortedResources...)
 	return nil
 }
 
@@ -167,6 +169,12 @@ func (b *Builder) WithUsingAPIServerCache(u bool) {
 	b.useAPIServerCache = u
 }
 
+// WithObjectLimit sets a limit on how many objects you can list from the APIServer.
+// This is to protect kube-state-metrics from running out of memory if the APIServer has a lot of objects.
+func (b *Builder) WithObjectLimit(l int64) {
+	b.objectLimit = l
+}
+
 // WithFamilyGeneratorFilter configures the family generator filter which decides which
 // metrics are to be exposed by the store build by the Builder.
 func (b *Builder) WithFamilyGeneratorFilter(l generator.FamilyGeneratorFilter) {
@@ -197,7 +205,10 @@ func (b *Builder) DefaultGenerateCustomResourceStoresFunc() ksmtypes.BuildCustom
 func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.RegistryFactory) {
 	for i := range fs {
 		f := fs[i]
-		gvr := util.GVRFromType(f.Name(), f.ExpectedType())
+		gvr, err := util.GVRFromType(f.Name(), f.ExpectedType())
+		if err != nil {
+			klog.ErrorS(err, "Failed to get GVR from type", "resourceName", f.Name(), "expectedType", f.ExpectedType())
+		}
 		var gvrString string
 		if gvr != nil {
 			gvrString = gvr.String()
@@ -214,6 +225,7 @@ func (b *Builder) WithCustomResourceStoreFactories(fs ...customresource.Registry
 				f.ExpectedType(),
 				f.ListWatch,
 				b.useAPIServerCache,
+				b.objectLimit,
 			)
 		}
 	}
@@ -362,150 +374,150 @@ func availableResources() []string {
 }
 
 func (b *Builder) buildConfigMapStores() []cache.Store {
-	return b.buildStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &v1.ConfigMap{}, createConfigMapListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(configMapMetricFamilies(b.allowAnnotationsList["configmaps"], b.allowLabelsList["configmaps"]), &v1.ConfigMap{}, createConfigMapListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildCronJobStores() []cache.Store {
-	return b.buildStoresFunc(cronJobMetricFamilies(b.allowAnnotationsList["cronjobs"], b.allowLabelsList["cronjobs"]), &batchv1.CronJob{}, createCronJobListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(cronJobMetricFamilies(b.allowAnnotationsList["cronjobs"], b.allowLabelsList["cronjobs"]), &batchv1.CronJob{}, createCronJobListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildDaemonSetStores() []cache.Store {
-	return b.buildStoresFunc(daemonSetMetricFamilies(b.allowAnnotationsList["daemonsets"], b.allowLabelsList["daemonsets"]), &appsv1.DaemonSet{}, createDaemonSetListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(daemonSetMetricFamilies(b.allowAnnotationsList["daemonsets"], b.allowLabelsList["daemonsets"]), &appsv1.DaemonSet{}, createDaemonSetListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildDeploymentStores() []cache.Store {
-	return b.buildStoresFunc(deploymentMetricFamilies(b.allowAnnotationsList["deployments"], b.allowLabelsList["deployments"]), &appsv1.Deployment{}, createDeploymentListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(deploymentMetricFamilies(b.allowAnnotationsList["deployments"], b.allowLabelsList["deployments"]), &appsv1.Deployment{}, createDeploymentListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildEndpointsStores() []cache.Store {
-	return b.buildStoresFunc(endpointMetricFamilies(b.allowAnnotationsList["endpoints"], b.allowLabelsList["endpoints"]), &v1.Endpoints{}, createEndpointsListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(endpointMetricFamilies(b.allowAnnotationsList["endpoints"], b.allowLabelsList["endpoints"]), &v1.Endpoints{}, createEndpointsListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildEndpointSlicesStores() []cache.Store {
-	return b.buildStoresFunc(endpointSliceMetricFamilies(b.allowAnnotationsList["endpointslices"], b.allowLabelsList["endpointslices"]), &discoveryv1.EndpointSlice{}, createEndpointSliceListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(endpointSliceMetricFamilies(b.allowAnnotationsList["endpointslices"], b.allowLabelsList["endpointslices"]), &discoveryv1.EndpointSlice{}, createEndpointSliceListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildHPAStores() []cache.Store {
-	return b.buildStoresFunc(hpaMetricFamilies(b.allowAnnotationsList["horizontalpodautoscalers"], b.allowLabelsList["horizontalpodautoscalers"]), &autoscaling.HorizontalPodAutoscaler{}, createHPAListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(hpaMetricFamilies(b.allowAnnotationsList["horizontalpodautoscalers"], b.allowLabelsList["horizontalpodautoscalers"]), &autoscaling.HorizontalPodAutoscaler{}, createHPAListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildIngressStores() []cache.Store {
-	return b.buildStoresFunc(ingressMetricFamilies(b.allowAnnotationsList["ingresses"], b.allowLabelsList["ingresses"]), &networkingv1.Ingress{}, createIngressListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(ingressMetricFamilies(b.allowAnnotationsList["ingresses"], b.allowLabelsList["ingresses"]), &networkingv1.Ingress{}, createIngressListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildJobStores() []cache.Store {
-	return b.buildStoresFunc(jobMetricFamilies(b.allowAnnotationsList["jobs"], b.allowLabelsList["jobs"]), &batchv1.Job{}, createJobListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(jobMetricFamilies(b.allowAnnotationsList["jobs"], b.allowLabelsList["jobs"]), &batchv1.Job{}, createJobListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildLimitRangeStores() []cache.Store {
-	return b.buildStoresFunc(limitRangeMetricFamilies, &v1.LimitRange{}, createLimitRangeListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(limitRangeMetricFamilies, &v1.LimitRange{}, createLimitRangeListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildMutatingWebhookConfigurationStores() []cache.Store {
-	return b.buildStoresFunc(mutatingWebhookConfigurationMetricFamilies, &admissionregistrationv1.MutatingWebhookConfiguration{}, createMutatingWebhookConfigurationListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(mutatingWebhookConfigurationMetricFamilies, &admissionregistrationv1.MutatingWebhookConfiguration{}, createMutatingWebhookConfigurationListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildNamespaceStores() []cache.Store {
-	return b.buildStoresFunc(namespaceMetricFamilies(b.allowAnnotationsList["namespaces"], b.allowLabelsList["namespaces"]), &v1.Namespace{}, createNamespaceListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(namespaceMetricFamilies(b.allowAnnotationsList["namespaces"], b.allowLabelsList["namespaces"]), &v1.Namespace{}, createNamespaceListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildNetworkPolicyStores() []cache.Store {
-	return b.buildStoresFunc(networkPolicyMetricFamilies(b.allowAnnotationsList["networkpolicies"], b.allowLabelsList["networkpolicies"]), &networkingv1.NetworkPolicy{}, createNetworkPolicyListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(networkPolicyMetricFamilies(b.allowAnnotationsList["networkpolicies"], b.allowLabelsList["networkpolicies"]), &networkingv1.NetworkPolicy{}, createNetworkPolicyListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildNodeStores() []cache.Store {
-	return b.buildStoresFunc(nodeMetricFamilies(b.allowAnnotationsList["nodes"], b.allowLabelsList["nodes"]), &v1.Node{}, createNodeListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(nodeMetricFamilies(b.allowAnnotationsList["nodes"], b.allowLabelsList["nodes"]), &v1.Node{}, createNodeListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildPersistentVolumeClaimStores() []cache.Store {
-	return b.buildStoresFunc(persistentVolumeClaimMetricFamilies(b.allowAnnotationsList["persistentvolumeclaims"], b.allowLabelsList["persistentvolumeclaims"]), &v1.PersistentVolumeClaim{}, createPersistentVolumeClaimListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(persistentVolumeClaimMetricFamilies(b.allowAnnotationsList["persistentvolumeclaims"], b.allowLabelsList["persistentvolumeclaims"]), &v1.PersistentVolumeClaim{}, createPersistentVolumeClaimListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildPersistentVolumeStores() []cache.Store {
-	return b.buildStoresFunc(persistentVolumeMetricFamilies(b.allowAnnotationsList["persistentvolumes"], b.allowLabelsList["persistentvolumes"]), &v1.PersistentVolume{}, createPersistentVolumeListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(persistentVolumeMetricFamilies(b.allowAnnotationsList["persistentvolumes"], b.allowLabelsList["persistentvolumes"]), &v1.PersistentVolume{}, createPersistentVolumeListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildPodDisruptionBudgetStores() []cache.Store {
-	return b.buildStoresFunc(podDisruptionBudgetMetricFamilies(b.allowAnnotationsList["poddisruptionbudgets"], b.allowLabelsList["poddisruptionbudgets"]), &policyv1.PodDisruptionBudget{}, createPodDisruptionBudgetListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(podDisruptionBudgetMetricFamilies(b.allowAnnotationsList["poddisruptionbudgets"], b.allowLabelsList["poddisruptionbudgets"]), &policyv1.PodDisruptionBudget{}, createPodDisruptionBudgetListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildReplicaSetStores() []cache.Store {
-	return b.buildStoresFunc(replicaSetMetricFamilies(b.allowAnnotationsList["replicasets"], b.allowLabelsList["replicasets"]), &appsv1.ReplicaSet{}, createReplicaSetListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(replicaSetMetricFamilies(b.allowAnnotationsList["replicasets"], b.allowLabelsList["replicasets"]), &appsv1.ReplicaSet{}, createReplicaSetListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildReplicationControllerStores() []cache.Store {
-	return b.buildStoresFunc(replicationControllerMetricFamilies, &v1.ReplicationController{}, createReplicationControllerListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(replicationControllerMetricFamilies, &v1.ReplicationController{}, createReplicationControllerListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildResourceQuotaStores() []cache.Store {
-	return b.buildStoresFunc(resourceQuotaMetricFamilies(b.allowAnnotationsList["resourcequotas"], b.allowLabelsList["resourcequotas"]), &v1.ResourceQuota{}, createResourceQuotaListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(resourceQuotaMetricFamilies(b.allowAnnotationsList["resourcequotas"], b.allowLabelsList["resourcequotas"]), &v1.ResourceQuota{}, createResourceQuotaListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildSecretStores() []cache.Store {
-	return b.buildStoresFunc(secretMetricFamilies(b.allowAnnotationsList["secrets"], b.allowLabelsList["secrets"]), &v1.Secret{}, createSecretListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(secretMetricFamilies(b.allowAnnotationsList["secrets"], b.allowLabelsList["secrets"]), &v1.Secret{}, createSecretListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildServiceAccountStores() []cache.Store {
-	return b.buildStoresFunc(serviceAccountMetricFamilies(b.allowAnnotationsList["serviceaccounts"], b.allowLabelsList["serviceaccounts"]), &v1.ServiceAccount{}, createServiceAccountListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(serviceAccountMetricFamilies(b.allowAnnotationsList["serviceaccounts"], b.allowLabelsList["serviceaccounts"]), &v1.ServiceAccount{}, createServiceAccountListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildServiceStores() []cache.Store {
-	return b.buildStoresFunc(serviceMetricFamilies(b.allowAnnotationsList["services"], b.allowLabelsList["services"]), &v1.Service{}, createServiceListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(serviceMetricFamilies(b.allowAnnotationsList["services"], b.allowLabelsList["services"]), &v1.Service{}, createServiceListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildStatefulSetStores() []cache.Store {
-	return b.buildStoresFunc(statefulSetMetricFamilies(b.allowAnnotationsList["statefulsets"], b.allowLabelsList["statefulsets"]), &appsv1.StatefulSet{}, createStatefulSetListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(statefulSetMetricFamilies(b.allowAnnotationsList["statefulsets"], b.allowLabelsList["statefulsets"]), &appsv1.StatefulSet{}, createStatefulSetListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildStorageClassStores() []cache.Store {
-	return b.buildStoresFunc(storageClassMetricFamilies(b.allowAnnotationsList["storageclasses"], b.allowLabelsList["storageclasses"]), &storagev1.StorageClass{}, createStorageClassListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(storageClassMetricFamilies(b.allowAnnotationsList["storageclasses"], b.allowLabelsList["storageclasses"]), &storagev1.StorageClass{}, createStorageClassListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildPodStores() []cache.Store {
-	return b.buildStoresFunc(podMetricFamilies(b.allowAnnotationsList["pods"], b.allowLabelsList["pods"]), &v1.Pod{}, createPodListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(podMetricFamilies(b.allowAnnotationsList["pods"], b.allowLabelsList["pods"]), &v1.Pod{}, createPodListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildCsrStores() []cache.Store {
-	return b.buildStoresFunc(csrMetricFamilies(b.allowAnnotationsList["certificatesigningrequests"], b.allowLabelsList["certificatesigningrequests"]), &certv1.CertificateSigningRequest{}, createCSRListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(csrMetricFamilies(b.allowAnnotationsList["certificatesigningrequests"], b.allowLabelsList["certificatesigningrequests"]), &certv1.CertificateSigningRequest{}, createCSRListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildValidatingWebhookConfigurationStores() []cache.Store {
-	return b.buildStoresFunc(validatingWebhookConfigurationMetricFamilies, &admissionregistrationv1.ValidatingWebhookConfiguration{}, createValidatingWebhookConfigurationListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(validatingWebhookConfigurationMetricFamilies, &admissionregistrationv1.ValidatingWebhookConfiguration{}, createValidatingWebhookConfigurationListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildVolumeAttachmentStores() []cache.Store {
-	return b.buildStoresFunc(volumeAttachmentMetricFamilies, &storagev1.VolumeAttachment{}, createVolumeAttachmentListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(volumeAttachmentMetricFamilies, &storagev1.VolumeAttachment{}, createVolumeAttachmentListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildLeasesStores() []cache.Store {
-	return b.buildStoresFunc(leaseMetricFamilies, &coordinationv1.Lease{}, createLeaseListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(leaseMetricFamilies, &coordinationv1.Lease{}, createLeaseListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildClusterRoleStores() []cache.Store {
-	return b.buildStoresFunc(clusterRoleMetricFamilies(b.allowAnnotationsList["clusterroles"], b.allowLabelsList["clusterroles"]), &rbacv1.ClusterRole{}, createClusterRoleListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(clusterRoleMetricFamilies(b.allowAnnotationsList["clusterroles"], b.allowLabelsList["clusterroles"]), &rbacv1.ClusterRole{}, createClusterRoleListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildRoleStores() []cache.Store {
-	return b.buildStoresFunc(roleMetricFamilies(b.allowAnnotationsList["roles"], b.allowLabelsList["roles"]), &rbacv1.Role{}, createRoleListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(roleMetricFamilies(b.allowAnnotationsList["roles"], b.allowLabelsList["roles"]), &rbacv1.Role{}, createRoleListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildClusterRoleBindingStores() []cache.Store {
-	return b.buildStoresFunc(clusterRoleBindingMetricFamilies(b.allowAnnotationsList["clusterrolebindings"], b.allowLabelsList["clusterrolebindings"]), &rbacv1.ClusterRoleBinding{}, createClusterRoleBindingListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(clusterRoleBindingMetricFamilies(b.allowAnnotationsList["clusterrolebindings"], b.allowLabelsList["clusterrolebindings"]), &rbacv1.ClusterRoleBinding{}, createClusterRoleBindingListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildRoleBindingStores() []cache.Store {
-	return b.buildStoresFunc(roleBindingMetricFamilies(b.allowAnnotationsList["rolebindings"], b.allowLabelsList["rolebindings"]), &rbacv1.RoleBinding{}, createRoleBindingListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(roleBindingMetricFamilies(b.allowAnnotationsList["rolebindings"], b.allowLabelsList["rolebindings"]), &rbacv1.RoleBinding{}, createRoleBindingListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildIngressClassStores() []cache.Store {
-	return b.buildStoresFunc(ingressClassMetricFamilies(b.allowAnnotationsList["ingressclasses"], b.allowLabelsList["ingressclasses"]), &networkingv1.IngressClass{}, createIngressClassListWatch, b.useAPIServerCache)
+	return b.buildStoresFunc(ingressClassMetricFamilies(b.allowAnnotationsList["ingressclasses"], b.allowLabelsList["ingressclasses"]), &networkingv1.IngressClass{}, createIngressClassListWatch, b.useAPIServerCache, b.objectLimit)
 }
 
 func (b *Builder) buildStores(
 	metricFamilies []generator.FamilyGenerator,
 	expectedType interface{},
 	listWatchFunc func(kubeClient clientset.Interface, ns string, fieldSelector string) cache.ListerWatcher,
-	useAPIServerCache bool,
+	useAPIServerCache bool, objectLimit int64,
 ) []cache.Store {
 	metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies)
 	composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies)
@@ -520,7 +532,7 @@ func (b *Builder) buildStores(
 			klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
 		}
 		listWatcher := listWatchFunc(b.kubeClient, v1.NamespaceAll, b.fieldSelectorFilter)
-		b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
+		b.startReflector(expectedType, store, listWatcher, useAPIServerCache, objectLimit)
 		return []cache.Store{store}
 	}
 
@@ -534,7 +546,7 @@ func (b *Builder) buildStores(
 			klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
 		}
 		listWatcher := listWatchFunc(b.kubeClient, ns, b.fieldSelectorFilter)
-		b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
+		b.startReflector(expectedType, store, listWatcher, useAPIServerCache, objectLimit)
 		stores = append(stores, store)
 	}
 
@@ -546,14 +558,17 @@ func (b *Builder) buildCustomResourceStores(resourceName string,
 	metricFamilies []generator.FamilyGenerator,
 	expectedType interface{},
 	listWatchFunc func(customResourceClient interface{}, ns string, fieldSelector string) cache.ListerWatcher,
-	useAPIServerCache bool,
+	useAPIServerCache bool, objectLimit int64,
 ) []cache.Store {
 	metricFamilies = generator.FilterFamilyGenerators(b.familyGeneratorFilter, metricFamilies)
 	composedMetricGenFuncs := generator.ComposeMetricGenFuncs(metricFamilies)
 
 	familyHeaders := generator.ExtractMetricFamilyHeaders(metricFamilies)
 
-	gvr := util.GVRFromType(resourceName, expectedType)
+	gvr, err := util.GVRFromType(resourceName, expectedType)
+	if err != nil {
+		klog.ErrorS(err, "Failed to get GVR from type", "resourceName", resourceName, "expectedType", expectedType)
+	}
 	var gvrString string
 	if gvr != nil {
 		gvrString = gvr.String()
@@ -575,7 +590,7 @@ func (b *Builder) buildCustomResourceStores(resourceName string,
 			klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
 		}
 		listWatcher := listWatchFunc(customResourceClient, v1.NamespaceAll, b.fieldSelectorFilter)
-		b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
+		b.startReflector(expectedType, store, listWatcher, useAPIServerCache, objectLimit)
 		return []cache.Store{store}
 	}
 
@@ -587,7 +602,7 @@ func (b *Builder) buildCustomResourceStores(resourceName string,
 		)
 		klog.InfoS("FieldSelector is used", "fieldSelector", b.fieldSelectorFilter)
 		listWatcher := listWatchFunc(customResourceClient, ns, b.fieldSelectorFilter)
-		b.startReflector(expectedType, store, listWatcher, useAPIServerCache)
+		b.startReflector(expectedType, store, listWatcher, useAPIServerCache, objectLimit)
 		stores = append(stores, store)
 	}
 
@@ -601,10 +616,15 @@ func (b *Builder) startReflector(
 	store cache.Store,
 	listWatcher cache.ListerWatcher,
 	useAPIServerCache bool,
+	objectLimit int64,
 ) {
-	instrumentedListWatch := watch.NewInstrumentedListerWatcher(listWatcher, b.listWatchMetrics, reflect.TypeOf(expectedType).String(), useAPIServerCache)
-	reflector := cache.NewReflector(sharding.NewShardedListWatch(b.shard, b.totalShards, instrumentedListWatch), expectedType, store, 0)
-	go reflector.Run(b.ctx.Done())
+	instrumentedListWatch := watch.NewInstrumentedListerWatcher(listWatcher, b.listWatchMetrics, reflect.TypeOf(expectedType).String(), useAPIServerCache, objectLimit)
+	reflector := cache.NewReflectorWithOptions(sharding.NewShardedListWatch(b.shard, b.totalShards, instrumentedListWatch), expectedType, store, cache.ReflectorOptions{ResyncPeriod: 0})
+	if cr, ok := expectedType.(*unstructured.Unstructured); ok {
+		go reflector.Run((*b.GVKToReflectorStopChanMap)[cr.GroupVersionKind().String()])
+	} else {
+		go reflector.Run(b.ctx.Done())
+	}
 }
 
 // cacheStoresToMetricStores converts []cache.Store into []*metricsstore.MetricsStore

internal/store/builder_test.go

@@ -18,6 +18,7 @@ package store
 
 import (
 	"reflect"
+	"slices"
 	"testing"
 
 	"k8s.io/kube-state-metrics/v2/pkg/options"
@@ -196,3 +197,64 @@ func TestWithAllowAnnotations(t *testing.T) {
 		}
 	}
 }
+
+func TestWithEnabledResources(t *testing.T) {
+
+	tests := []struct {
+		Desc             string
+		EnabledResources []string
+		Wanted           []string
+		err              expectedError
+	}{
+		{
+			Desc:             "sorts enabled resources",
+			EnabledResources: []string{"pods", "cronjobs", "deployments"},
+			Wanted: []string{
+				"cronjobs",
+				"deployments",
+				"pods",
+			},
+		},
+		{
+			Desc:             "de-duplicates enabled resources",
+			EnabledResources: []string{"pods", "cronjobs", "deployments", "pods"},
+			Wanted: []string{
+				"cronjobs",
+				"deployments",
+				"pods",
+			},
+		},
+		{
+			Desc:             "error if not exist",
+			EnabledResources: []string{"pods", "cronjobs", "deployments", "foo"},
+			Wanted:           []string{},
+			err: expectedError{
+				expectedResourceError: true,
+			},
+		},
+	}
+	for _, test := range tests {
+		b := NewBuilder()
+
+		// Set the enabled resources.
+		err := b.WithEnabledResources(test.EnabledResources)
+		if test.err.expectedResourceError {
+			if err == nil {
+				t.Log("Did not expect error while setting resources (--resources).")
+				t.Fatalf("Test error for Desc: %s. Got Error: %v", test.Desc, err)
+			} else {
+				return
+			}
+		}
+		if err != nil {
+			t.Log("...")
+			t.Fatal("...", test.Desc, err)
+		}
+
+		// Evaluate.
+		if !slices.Equal(b.enabledResources, test.Wanted) {
+			t.Log("Expected enabled resources to be equal.")
+			t.Errorf("Test error for Desc: %s\n Want: \n%+v\n Got: \n%#+v", test.Desc, test.Wanted, b.enabledResources)
+		}
+	}
+}

internal/store/clusterrole.go

@@ -131,7 +131,7 @@ func clusterRoleMetricFamilies(allowAnnotationsList, allowLabelsList []string) [
 			"",
 			wrapClusterRoleFunc(func(r *rbacv1.ClusterRole) *metric.Family {
 				return &metric.Family{
-					Metrics: resourceVersionMetric(r.ObjectMeta.ResourceVersion),
+					Metrics: resourceVersionMetric(r.ResourceVersion),
 				}
 			}),
 		),

internal/store/clusterrolebinding.go

@@ -133,7 +133,7 @@ func clusterRoleBindingMetricFamilies(allowAnnotationsList, allowLabelsList []st
 			"",
 			wrapClusterRoleBindingFunc(func(r *rbacv1.ClusterRoleBinding) *metric.Family {
 				return &metric.Family{
-					Metrics: resourceVersionMetric(r.ObjectMeta.ResourceVersion),
+					Metrics: resourceVersionMetric(r.ResourceVersion),
 				}
 			}),
 		),

internal/store/configmap.go

@@ -127,7 +127,7 @@ func configMapMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
 			"",
 			wrapConfigMapFunc(func(c *v1.ConfigMap) *metric.Family {
 				return &metric.Family{
-					Metrics: resourceVersionMetric(c.ObjectMeta.ResourceVersion),
+					Metrics: resourceVersionMetric(c.ResourceVersion),
 				}
 			}),
 		),

internal/store/cronjob.go

@@ -96,11 +96,15 @@ func cronJobMetricFamilies(allowAnnotationsList, allowLabelsList []string) []gen
 			basemetrics.STABLE,
 			"",
 			wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family {
+				timeZone := "local"
+				if j.Spec.TimeZone != nil {
+					timeZone = *j.Spec.TimeZone
+				}
 				return &metric.Family{
 					Metrics: []*metric.Metric{
 						{
-							LabelKeys:   []string{"schedule", "concurrency_policy"},
-							LabelValues: []string{j.Spec.Schedule, string(j.Spec.ConcurrencyPolicy)},
+							LabelKeys:   []string{"schedule", "concurrency_policy", "timezone"},
+							LabelValues: []string{j.Spec.Schedule, string(j.Spec.ConcurrencyPolicy), timeZone},
 							Value:       1,
 						},
 					},
@@ -245,7 +249,7 @@ func cronJobMetricFamilies(allowAnnotationsList, allowLabelsList []string) []gen
 				ms := []*metric.Metric{}
 
 				// If the cron job is suspended, don't track the next scheduled time
-				nextScheduledTime, err := getNextScheduledTime(j.Spec.Schedule, j.Status.LastScheduleTime, j.CreationTimestamp)
+				nextScheduledTime, err := getNextScheduledTime(j.Spec.Schedule, j.Status.LastScheduleTime, j.CreationTimestamp, j.Spec.TimeZone)
 				if err != nil {
 					panic(err)
 				} else if !*j.Spec.Suspend {
@@ -269,7 +273,7 @@ func cronJobMetricFamilies(allowAnnotationsList, allowLabelsList []string) []gen
 			"",
 			wrapCronJobFunc(func(j *batchv1.CronJob) *metric.Family {
 				return &metric.Family{
-					Metrics: resourceVersionMetric(j.ObjectMeta.ResourceVersion),
+					Metrics: resourceVersionMetric(j.ResourceVersion),
 				}
 			}),
 		),
@@ -347,10 +351,14 @@ func createCronJobListWatch(kubeClient clientset.Interface, ns string, fieldSele
 	}
 }
 
-func getNextScheduledTime(schedule string, lastScheduleTime *metav1.Time, createdTime metav1.Time) (time.Time, error) {
+func getNextScheduledTime(schedule string, lastScheduleTime *metav1.Time, createdTime metav1.Time, timeZone *string) (time.Time, error) {
+	if timeZone != nil {
+		schedule = fmt.Sprintf("CRON_TZ=%s %s", *timeZone, schedule)
+	}
+
 	sched, err := cron.ParseStandard(schedule)
 	if err != nil {
-		return time.Time{}, fmt.Errorf("Failed to parse cron job schedule '%s': %w", schedule, err)
+		return time.Time{}, fmt.Errorf("failed to parse cron job schedule '%s': %w", schedule, err)
 	}
 	if !lastScheduleTime.IsZero() {
 		return sched.Next(lastScheduleTime.Time), nil

internal/store/cronjob_test.go

@@ -40,68 +40,163 @@ var (
 	ActiveRunningCronJob1LastScheduleTime          = time.Unix(1520742896, 0)
 	SuspendedCronJob1LastScheduleTime              = time.Unix(1520742896+5.5*3600, 0) // 5.5 hours later
 	ActiveCronJob1NoLastScheduledCreationTimestamp = time.Unix(1520742896+6.5*3600, 0)
+	TimeZone                                       = "Asia/Shanghai"
 )
 
-func TestCronJobStore(t *testing.T) {
-	hour := ActiveRunningCronJob1LastScheduleTime.Hour()
-	ActiveRunningCronJob1NextScheduleTime := time.Time{}
+func calculateNextSchedule6h(timestamp time.Time, timezone string) time.Time {
+	loc, _ := time.LoadLocation(timezone)
+	hour := timestamp.In(loc).Hour()
 	switch {
 	case hour < 6:
-		ActiveRunningCronJob1NextScheduleTime = time.Date(
-			ActiveRunningCronJob1LastScheduleTime.Year(),
-			ActiveRunningCronJob1LastScheduleTime.Month(),
-			ActiveRunningCronJob1LastScheduleTime.Day(),
+		return time.Date(
+			timestamp.Year(),
+			timestamp.Month(),
+			timestamp.Day(),
 			6,
 			0,
-			0, 0, time.Local)
+			0, 0, loc)
 	case hour < 12:
-		ActiveRunningCronJob1NextScheduleTime = time.Date(
-			ActiveRunningCronJob1LastScheduleTime.Year(),
-			ActiveRunningCronJob1LastScheduleTime.Month(),
-			ActiveRunningCronJob1LastScheduleTime.Day(),
+		return time.Date(
+			timestamp.Year(),
+			timestamp.Month(),
+			timestamp.Day(),
 			12,
 			0,
-			0, 0, time.Local)
+			0, 0, loc)
 	case hour < 18:
-		ActiveRunningCronJob1NextScheduleTime = time.Date(
-			ActiveRunningCronJob1LastScheduleTime.Year(),
-			ActiveRunningCronJob1LastScheduleTime.Month(),
-			ActiveRunningCronJob1LastScheduleTime.Day(),
+		return time.Date(
+			timestamp.Year(),
+			timestamp.Month(),
+			timestamp.Day(),
 			18,
 			0,
-			0, 0, time.Local)
-	case hour < 24:
-		ActiveRunningCronJob1NextScheduleTime = time.Date(
-			ActiveRunningCronJob1LastScheduleTime.Year(),
-			ActiveRunningCronJob1LastScheduleTime.Month(),
-			ActiveRunningCronJob1LastScheduleTime.Day(),
-			24,
+			0, 0, loc)
+	default:
+		return time.Date(
+			timestamp.Year(),
+			timestamp.Month(),
+			timestamp.Day()+1,
 			0,
-			0, 0, time.Local)
+			0,
+			0, 0, loc)
 	}
+}
 
-	minute := ActiveCronJob1NoLastScheduledCreationTimestamp.Minute()
-	ActiveCronJob1NoLastScheduledNextScheduleTime := time.Time{}
+func calculateNextSchedule25m(timestamp time.Time, timezone string) time.Time {
+	loc, _ := time.LoadLocation(timezone)
+	minute := timestamp.In(loc).Minute()
 	switch {
 	case minute < 25:
-		ActiveCronJob1NoLastScheduledNextScheduleTime = time.Date(
-			ActiveCronJob1NoLastScheduledCreationTimestamp.Year(),
-			ActiveCronJob1NoLastScheduledCreationTimestamp.Month(),
-			ActiveCronJob1NoLastScheduledCreationTimestamp.Day(),
-			ActiveCronJob1NoLastScheduledCreationTimestamp.Hour(),
+		return time.Date(
+			timestamp.Year(),
+			timestamp.Month(),
+			timestamp.Day(),
+			timestamp.Hour(),
 			25,
-			0, 0, time.Local)
+			0, 0, loc)
 	default:
-		ActiveCronJob1NoLastScheduledNextScheduleTime = time.Date(
-			ActiveCronJob1NoLastScheduledNextScheduleTime.Year(),
-			ActiveCronJob1NoLastScheduledNextScheduleTime.Month(),
-			ActiveCronJob1NoLastScheduledNextScheduleTime.Day(),
-			ActiveCronJob1NoLastScheduledNextScheduleTime.Hour()+1,
+		return time.Date(
+			timestamp.Year(),
+			timestamp.Month(),
+			timestamp.Day(),
+			timestamp.Hour()+1,
 			25,
-			0, 0, time.Local)
+			0, 0, loc)
 	}
 
+}
+func TestCronJobStore(t *testing.T) {
+
+	ActiveRunningCronJob1NextScheduleTime := calculateNextSchedule6h(ActiveRunningCronJob1LastScheduleTime, "Local")
+	ActiveRunningCronJobWithTZ1NextScheduleTime := calculateNextSchedule6h(ActiveRunningCronJob1LastScheduleTime, TimeZone)
+
+	ActiveCronJob1NoLastScheduledNextScheduleTime := calculateNextSchedule25m(ActiveCronJob1NoLastScheduledCreationTimestamp, "Local")
+
 	cases := []generateMetricsTestCase{
+		{
+			AllowAnnotationsList: []string{
+				"app.k8s.io/owner",
+			},
+			Obj: &batchv1.CronJob{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:            "ActiveRunningCronJobWithTZ1",
+					Namespace:       "ns1",
+					Generation:      1,
+					ResourceVersion: "11111",
+					Labels: map[string]string{
+						"app": "example-active-running-with-tz-1",
+					},
+					Annotations: map[string]string{
+						"app":              "mysql-server",
+						"app.k8s.io/owner": "@foo",
+					},
+				},
+				Status: batchv1.CronJobStatus{
+					Active:             []v1.ObjectReference{{Name: "FakeJob1"}, {Name: "FakeJob2"}},
+					LastScheduleTime:   &metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime},
+					LastSuccessfulTime: nil,
+				},
+				Spec: batchv1.CronJobSpec{
+					StartingDeadlineSeconds:    &StartingDeadlineSeconds300,
+					ConcurrencyPolicy:          "Forbid",
+					Suspend:                    &SuspendFalse,
+					Schedule:                   "0 */6 * * *",
+					SuccessfulJobsHistoryLimit: &SuccessfulJobHistoryLimit3,
+					FailedJobsHistoryLimit:     &FailedJobHistoryLimit1,
+					TimeZone:                   &TimeZone,
+				},
+			},
+			Want: `
+				# HELP kube_cronjob_created [STABLE] Unix creation timestamp
+				# HELP kube_cronjob_info [STABLE] Info about cronjob.
+				# HELP kube_cronjob_annotations Kubernetes annotations converted to Prometheus labels.
+				# HELP kube_cronjob_labels [STABLE] Kubernetes labels converted to Prometheus labels.
+				# HELP kube_cronjob_next_schedule_time [STABLE] Next time the cronjob should be scheduled. The time after lastScheduleTime, or after the cron job's creation time if it's never been scheduled. Use this to determine if the job is delayed.
+				# HELP kube_cronjob_spec_failed_job_history_limit Failed job history limit tells the controller how many failed jobs should be preserved.
+				# HELP kube_cronjob_spec_starting_deadline_seconds [STABLE] Deadline in seconds for starting the job if it misses scheduled time for any reason.
+        		# HELP kube_cronjob_spec_successful_job_history_limit Successful job history limit tells the controller how many completed jobs should be preserved.
+				# HELP kube_cronjob_spec_suspend [STABLE] Suspend flag tells the controller to suspend subsequent executions.
+				# HELP kube_cronjob_status_active [STABLE] Active holds pointers to currently running jobs.
+                # HELP kube_cronjob_metadata_resource_version [STABLE] Resource version representing a specific version of the cronjob.
+				# HELP kube_cronjob_status_last_schedule_time [STABLE] LastScheduleTime keeps information of when was the last time the job was successfully scheduled.
+				# TYPE kube_cronjob_created gauge
+				# TYPE kube_cronjob_info gauge
+				# TYPE kube_cronjob_annotations gauge
+				# TYPE kube_cronjob_labels gauge
+				# TYPE kube_cronjob_next_schedule_time gauge
+				# TYPE kube_cronjob_spec_failed_job_history_limit gauge
+				# TYPE kube_cronjob_spec_starting_deadline_seconds gauge
+				# TYPE kube_cronjob_spec_successful_job_history_limit gauge
+				# TYPE kube_cronjob_spec_suspend gauge
+				# TYPE kube_cronjob_status_active gauge
+                # TYPE kube_cronjob_metadata_resource_version gauge
+				# TYPE kube_cronjob_status_last_schedule_time gauge
+				kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1",schedule="0 */6 * * *",timezone="Asia/Shanghai"} 1
+				kube_cronjob_annotations{annotation_app_k8s_io_owner="@foo",cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 1
+				kube_cronjob_spec_failed_job_history_limit{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 1
+				kube_cronjob_spec_starting_deadline_seconds{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 300
+				kube_cronjob_spec_successful_job_history_limit{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 3
+				kube_cronjob_spec_suspend{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 0
+				kube_cronjob_status_active{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 2
+                kube_cronjob_metadata_resource_version{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 11111
+				kube_cronjob_status_last_schedule_time{cronjob="ActiveRunningCronJobWithTZ1",namespace="ns1"} 1.520742896e+09
+` + fmt.Sprintf("kube_cronjob_next_schedule_time{cronjob=\"ActiveRunningCronJobWithTZ1\",namespace=\"ns1\"} %ve+09\n",
+				float64(ActiveRunningCronJobWithTZ1NextScheduleTime.Unix())/math.Pow10(9)),
+			MetricNames: []string{
+				"kube_cronjob_next_schedule_time",
+				"kube_cronjob_spec_starting_deadline_seconds",
+				"kube_cronjob_status_active",
+				"kube_cronjob_metadata_resource_version",
+				"kube_cronjob_spec_suspend",
+				"kube_cronjob_info",
+				"kube_cronjob_created",
+				"kube_cronjob_annotations",
+				"kube_cronjob_labels",
+				"kube_cronjob_status_last_schedule_time",
+				"kube_cronjob_spec_successful_job_history_limit",
+				"kube_cronjob_spec_failed_job_history_limit",
+			},
+		},
 		{
 			AllowAnnotationsList: []string{
 				"app.k8s.io/owner",
@@ -159,7 +254,7 @@ func TestCronJobStore(t *testing.T) {
 				# TYPE kube_cronjob_status_active gauge
                 # TYPE kube_cronjob_metadata_resource_version gauge
 				# TYPE kube_cronjob_status_last_schedule_time gauge
-				kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveRunningCronJob1",namespace="ns1",schedule="0 */6 * * *"} 1
+				kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveRunningCronJob1",namespace="ns1",schedule="0 */6 * * *",timezone="local"} 1
 				kube_cronjob_annotations{annotation_app_k8s_io_owner="@foo",cronjob="ActiveRunningCronJob1",namespace="ns1"} 1
 				kube_cronjob_spec_failed_job_history_limit{cronjob="ActiveRunningCronJob1",namespace="ns1"} 1
 				kube_cronjob_spec_starting_deadline_seconds{cronjob="ActiveRunningCronJob1",namespace="ns1"} 300
@@ -206,6 +301,7 @@ func TestCronJobStore(t *testing.T) {
 					ConcurrencyPolicy:          "Forbid",
 					Suspend:                    &SuspendTrue,
 					Schedule:                   "0 */3 * * *",
+					TimeZone:                   &TimeZone,
 					SuccessfulJobsHistoryLimit: &SuccessfulJobHistoryLimit3,
 					FailedJobsHistoryLimit:     &FailedJobHistoryLimit1,
 				},
@@ -233,7 +329,7 @@ func TestCronJobStore(t *testing.T) {
                 # TYPE kube_cronjob_metadata_resource_version gauge
 				# TYPE kube_cronjob_status_last_schedule_time gauge
 				# TYPE kube_cronjob_status_last_successful_time gauge
-				kube_cronjob_info{concurrency_policy="Forbid",cronjob="SuspendedCronJob1",namespace="ns1",schedule="0 */3 * * *"} 1
+				kube_cronjob_info{concurrency_policy="Forbid",cronjob="SuspendedCronJob1",namespace="ns1",schedule="0 */3 * * *",timezone="Asia/Shanghai"} 1
 				kube_cronjob_spec_failed_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 1
 				kube_cronjob_spec_starting_deadline_seconds{cronjob="SuspendedCronJob1",namespace="ns1"} 300
 				kube_cronjob_spec_successful_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 3
@@ -292,7 +388,7 @@ func TestCronJobStore(t *testing.T) {
                 # TYPE kube_cronjob_metadata_resource_version gauge
 				# TYPE kube_cronjob_status_last_schedule_time gauge
 				# TYPE kube_cronjob_status_last_successful_time gauge
-				kube_cronjob_info{concurrency_policy="Forbid",cronjob="SuspendedCronJob1",namespace="ns1",schedule="0 */3 * * *"} 1
+				kube_cronjob_info{concurrency_policy="Forbid",cronjob="SuspendedCronJob1",namespace="ns1",schedule="0 */3 * * *",timezone="local"} 1
 				kube_cronjob_spec_failed_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 1
 				kube_cronjob_spec_starting_deadline_seconds{cronjob="SuspendedCronJob1",namespace="ns1"} 300
 				kube_cronjob_spec_successful_job_history_limit{cronjob="SuspendedCronJob1",namespace="ns1"} 3
@@ -351,7 +447,7 @@ func TestCronJobStore(t *testing.T) {
 				# TYPE kube_cronjob_spec_successful_job_history_limit gauge
 				# TYPE kube_cronjob_spec_suspend gauge
 				# TYPE kube_cronjob_status_active gauge
-                # TYPE kube_cronjob_metadata_resource_version gauge
+                		# TYPE kube_cronjob_metadata_resource_version gauge
 				# TYPE kube_cronjob_status_last_successful_time gauge
 				kube_cronjob_spec_starting_deadline_seconds{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 300
 				kube_cronjob_status_active{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 0
@@ -359,7 +455,7 @@ func TestCronJobStore(t *testing.T) {
 				kube_cronjob_spec_failed_job_history_limit{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 1
 				kube_cronjob_spec_successful_job_history_limit{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 3
 				kube_cronjob_spec_suspend{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 0
-				kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1",schedule="25 * * * *"} 1
+				kube_cronjob_info{concurrency_policy="Forbid",cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1",schedule="25 * * * *",timezone="local"} 1
 				kube_cronjob_created{cronjob="ActiveCronJob1NoLastScheduled",namespace="ns1"} 1.520766296e+09
 ` +
 				fmt.Sprintf("kube_cronjob_next_schedule_time{cronjob=\"ActiveCronJob1NoLastScheduled\",namespace=\"ns1\"} %ve+09\n",
@@ -375,3 +471,37 @@ func TestCronJobStore(t *testing.T) {
 		}
 	}
 }
+
+func TestGetNextScheduledTime(t *testing.T) {
+
+	testCases := []struct {
+		schedule         string
+		lastScheduleTime metav1.Time
+		createdTime      metav1.Time
+		timeZone         string
+		expected         time.Time
+	}{
+		{
+			schedule:         "0 */6 * * *",
+			lastScheduleTime: metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime},
+			createdTime:      metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime},
+			timeZone:         "UTC",
+			expected:         ActiveRunningCronJob1LastScheduleTime.Add(time.Second*4 + time.Minute*25 + time.Hour),
+		},
+		{
+			schedule:         "0 */6 * * *",
+			lastScheduleTime: metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime},
+			createdTime:      metav1.Time{Time: ActiveRunningCronJob1LastScheduleTime},
+			timeZone:         TimeZone,
+			expected:         ActiveRunningCronJob1LastScheduleTime.Add(time.Second*4 + time.Minute*25 + time.Hour*5),
+		},
+	}
+
+	for _, test := range testCases {
+		actual, _ := getNextScheduledTime(test.schedule, &test.lastScheduleTime, test.createdTime, &test.timeZone) // #nosec G601
+		if !actual.Equal(test.expected) {
+			t.Fatalf("%v: expected %v, actual %v", test.schedule, test.expected, actual)
+		}
+	}
+
+}

internal/store/daemonset.go

@@ -217,7 +217,7 @@ func daemonSetMetricFamilies(allowAnnotationsList, allowLabelsList []string) []g
 						{
 							LabelKeys:   []string{},
 							LabelValues: []string{},
-							Value:       float64(d.ObjectMeta.Generation),
+							Value:       float64(d.Generation),
 						},
 					},
 				}

internal/store/deployment.go

@@ -41,6 +41,23 @@ var (
 	descDeploymentLabelsDefaultLabels = []string{"namespace", "deployment"}
 )
 
+// Reasons copied from kubernetes/pkg/controller/deployment/deployment_utils.go.
+var (
+	allowedDeploymentReasons = map[string]struct{}{
+		"ReplicaSetUpdated":          {},
+		"ReplicaSetCreateError":      {},
+		"NewReplicaSetCreated":       {},
+		"FoundNewReplicaSet":         {},
+		"NewReplicaSetAvailable":     {},
+		"ProgressDeadlineExceeded":   {},
+		"DeploymentPaused":           {},
+		"DeploymentResumed":          {},
+		"MinimumReplicasAvailable":   {},
+		"MinimumReplicasUnavailable": {},
+		"":                           {},
+	}
+)
+
 func deploymentMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generator.FamilyGenerator {
 	return []generator.FamilyGenerator{
 		*generator.NewFamilyGeneratorWithStability(
@@ -174,8 +191,13 @@ func deploymentMetricFamilies(allowAnnotationsList, allowLabelsList []string) []
 					for j, m := range conditionMetrics {
 						metric := m
 
-						metric.LabelKeys = []string{"condition", "status"}
-						metric.LabelValues = append([]string{string(c.Type)}, metric.LabelValues...)
+						reason := c.Reason
+						if _, ok := allowedDeploymentReasons[reason]; !ok {
+							reason = "unknown"
+						}
+
+						metric.LabelKeys = []string{"reason", "condition", "status"}
+						metric.LabelValues = append([]string{reason, string(c.Type)}, metric.LabelValues...)
 						ms[i*len(conditionStatuses)+j] = metric
 					}
 				}
@@ -277,7 +299,7 @@ func deploymentMetricFamilies(allowAnnotationsList, allowLabelsList []string) []
 				return &metric.Family{
 					Metrics: []*metric.Metric{
 						{
-							Value: float64(d.ObjectMeta.Generation),
+							Value: float64(d.Generation),
 						},
 					},
 				}

internal/store/deployment_test.go

@@ -31,6 +31,7 @@ import (
 var (
 	depl1Replicas int32 = 200
 	depl2Replicas int32 = 5
+	depl3Replicas int32 = 1
 
 	depl1MaxUnavailable = intstr.FromInt(10)
 	depl2MaxUnavailable = intstr.FromString("25%")
@@ -98,8 +99,8 @@ func TestDeploymentStore(t *testing.T) {
 					UpdatedReplicas:     2,
 					ObservedGeneration:  111,
 					Conditions: []v1.DeploymentCondition{
-						{Type: v1.DeploymentAvailable, Status: corev1.ConditionTrue},
-						{Type: v1.DeploymentProgressing, Status: corev1.ConditionTrue},
+						{Type: v1.DeploymentAvailable, Status: corev1.ConditionTrue, Reason: "MinimumReplicasAvailable"},
+						{Type: v1.DeploymentProgressing, Status: corev1.ConditionTrue, Reason: "NewReplicaSetAvailable"},
 					},
 				},
 				Spec: v1.DeploymentSpec{
@@ -126,12 +127,12 @@ func TestDeploymentStore(t *testing.T) {
         kube_deployment_status_replicas_updated{deployment="depl1",namespace="ns1"} 2
         kube_deployment_status_replicas{deployment="depl1",namespace="ns1"} 15
         kube_deployment_status_replicas_ready{deployment="depl1",namespace="ns1"} 10
-        kube_deployment_status_condition{deployment="depl1",namespace="ns1",condition="Available",status="true"} 1
-        kube_deployment_status_condition{deployment="depl1",namespace="ns1",condition="Progressing",status="true"} 1
-        kube_deployment_status_condition{deployment="depl1",namespace="ns1",condition="Available",status="false"} 0
-        kube_deployment_status_condition{deployment="depl1",namespace="ns1",condition="Progressing",status="false"} 0
-        kube_deployment_status_condition{deployment="depl1",namespace="ns1",condition="Available",status="unknown"} 0
-        kube_deployment_status_condition{deployment="depl1",namespace="ns1",condition="Progressing",status="unknown"} 0
+        kube_deployment_status_condition{condition="Available",deployment="depl1",namespace="ns1",reason="MinimumReplicasAvailable",status="true"} 1
+        kube_deployment_status_condition{condition="Available",deployment="depl1",namespace="ns1",reason="MinimumReplicasAvailable",status="false"} 0
+        kube_deployment_status_condition{condition="Available",deployment="depl1",namespace="ns1",reason="MinimumReplicasAvailable",status="unknown"} 0
+        kube_deployment_status_condition{condition="Progressing",deployment="depl1",namespace="ns1",reason="NewReplicaSetAvailable",status="true"} 1
+        kube_deployment_status_condition{condition="Progressing",deployment="depl1",namespace="ns1",reason="NewReplicaSetAvailable",status="false"} 0
+        kube_deployment_status_condition{condition="Progressing",deployment="depl1",namespace="ns1",reason="NewReplicaSetAvailable",status="unknown"} 0
 `,
 		},
 		{
@@ -152,9 +153,9 @@ func TestDeploymentStore(t *testing.T) {
 					UpdatedReplicas:     1,
 					ObservedGeneration:  1111,
 					Conditions: []v1.DeploymentCondition{
-						{Type: v1.DeploymentAvailable, Status: corev1.ConditionFalse},
-						{Type: v1.DeploymentProgressing, Status: corev1.ConditionFalse},
-						{Type: v1.DeploymentReplicaFailure, Status: corev1.ConditionTrue},
+						{Type: v1.DeploymentAvailable, Status: corev1.ConditionFalse, Reason: "MinimumReplicasUnavailable"},
+						{Type: v1.DeploymentProgressing, Status: corev1.ConditionFalse, Reason: "ProgressDeadlineExceeded"},
+						{Type: v1.DeploymentReplicaFailure, Status: corev1.ConditionTrue, Reason: "ReplicaSetCreateError"},
 					},
 				},
 				Spec: v1.DeploymentSpec{
@@ -180,15 +181,49 @@ func TestDeploymentStore(t *testing.T) {
         kube_deployment_status_replicas_updated{deployment="depl2",namespace="ns2"} 1
         kube_deployment_status_replicas{deployment="depl2",namespace="ns2"} 10
         kube_deployment_status_replicas_ready{deployment="depl2",namespace="ns2"} 5
-        kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="Available",status="true"} 0
-        kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="Progressing",status="true"} 0
-        kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="ReplicaFailure",status="true"} 1
-        kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="Available",status="false"} 1
-        kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="Progressing",status="false"} 1
-        kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="ReplicaFailure",status="false"} 0
-        kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="Available",status="unknown"} 0
-        kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="Progressing",status="unknown"} 0
-        kube_deployment_status_condition{deployment="depl2",namespace="ns2",condition="ReplicaFailure",status="unknown"} 0
+        kube_deployment_status_condition{condition="Available",deployment="depl2",namespace="ns2",reason="MinimumReplicasUnavailable",status="true"} 0
+        kube_deployment_status_condition{condition="Available",deployment="depl2",namespace="ns2",reason="MinimumReplicasUnavailable",status="false"} 1
+        kube_deployment_status_condition{condition="Available",deployment="depl2",namespace="ns2",reason="MinimumReplicasUnavailable",status="unknown"} 0
+        kube_deployment_status_condition{condition="Progressing",deployment="depl2",namespace="ns2",reason="ProgressDeadlineExceeded",status="true"} 0
+        kube_deployment_status_condition{condition="Progressing",deployment="depl2",namespace="ns2",reason="ProgressDeadlineExceeded",status="false"} 1
+        kube_deployment_status_condition{condition="Progressing",deployment="depl2",namespace="ns2",reason="ProgressDeadlineExceeded",status="unknown"} 0
+        kube_deployment_status_condition{condition="ReplicaFailure",deployment="depl2",namespace="ns2",reason="ReplicaSetCreateError",status="true"} 1
+        kube_deployment_status_condition{condition="ReplicaFailure",deployment="depl2",namespace="ns2",reason="ReplicaSetCreateError",status="false"} 0
+        kube_deployment_status_condition{condition="ReplicaFailure",deployment="depl2",namespace="ns2",reason="ReplicaSetCreateError",status="unknown"} 0
+`,
+		},
+		{
+			Obj: &v1.Deployment{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "depl3",
+					Namespace: "ns3",
+				},
+				Status: v1.DeploymentStatus{
+					Conditions: []v1.DeploymentCondition{
+						{Type: v1.DeploymentAvailable, Status: corev1.ConditionFalse, Reason: "ThisReasonIsNotAllowed"},
+						{Type: v1.DeploymentProgressing, Status: corev1.ConditionTrue},
+					},
+				},
+				Spec: v1.DeploymentSpec{
+					Replicas: &depl3Replicas,
+				},
+			},
+			Want: metadata + `
+        kube_deployment_metadata_generation{deployment="depl3",namespace="ns3"} 0
+        kube_deployment_spec_paused{deployment="depl3",namespace="ns3"} 0
+        kube_deployment_spec_replicas{deployment="depl3",namespace="ns3"} 1
+        kube_deployment_status_condition{condition="Available",deployment="depl3",namespace="ns3",reason="unknown",status="true"} 0
+        kube_deployment_status_condition{condition="Available",deployment="depl3",namespace="ns3",reason="unknown",status="false"} 1
+        kube_deployment_status_condition{condition="Available",deployment="depl3",namespace="ns3",reason="unknown",status="unknown"} 0
+        kube_deployment_status_observed_generation{deployment="depl3",namespace="ns3"} 0
+        kube_deployment_status_replicas{deployment="depl3",namespace="ns3"} 0
+        kube_deployment_status_replicas_available{deployment="depl3",namespace="ns3"} 0
+        kube_deployment_status_replicas_ready{deployment="depl3",namespace="ns3"} 0
+        kube_deployment_status_replicas_unavailable{deployment="depl3",namespace="ns3"} 0
+        kube_deployment_status_replicas_updated{deployment="depl3",namespace="ns3"} 0
+	    kube_deployment_status_condition{condition="Progressing",deployment="depl3",namespace="ns3",reason="",status="false"} 0
+        kube_deployment_status_condition{condition="Progressing",deployment="depl3",namespace="ns3",reason="",status="true"} 1
+        kube_deployment_status_condition{condition="Progressing",deployment="depl3",namespace="ns3",reason="",status="unknown"} 0
 `,
 		},
 	}

internal/store/endpoint.go

@@ -123,47 +123,6 @@ func endpointMetricFamilies(allowAnnotationsList, allowLabelsList []string) []ge
 				}
 			}),
 		),
-		*generator.NewFamilyGeneratorWithStability(
-			"kube_endpoint_address_available",
-			"Number of addresses available in endpoint.",
-			metric.Gauge,
-			basemetrics.ALPHA,
-			"v2.6.0",
-			wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family {
-				var available int
-				for _, s := range e.Subsets {
-					available += len(s.Addresses) * len(s.Ports)
-				}
-
-				return &metric.Family{
-					Metrics: []*metric.Metric{
-						{
-							Value: float64(available),
-						},
-					},
-				}
-			}),
-		),
-		*generator.NewFamilyGeneratorWithStability(
-			"kube_endpoint_address_not_ready",
-			"Number of addresses not ready in endpoint",
-			metric.Gauge,
-			basemetrics.ALPHA,
-			"v2.6.0",
-			wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family {
-				var notReady int
-				for _, s := range e.Subsets {
-					notReady += len(s.NotReadyAddresses) * len(s.Ports)
-				}
-				return &metric.Family{
-					Metrics: []*metric.Metric{
-						{
-							Value: float64(notReady),
-						},
-					},
-				}
-			}),
-		),
 		*generator.NewFamilyGeneratorWithStability(
 			"kube_endpoint_address",
 			"Information about Endpoint available and non available addresses.",
@@ -172,20 +131,28 @@ func endpointMetricFamilies(allowAnnotationsList, allowLabelsList []string) []ge
 			"",
 			wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family {
 				ms := []*metric.Metric{}
+				labelKeys := []string{"port_protocol", "port_number", "port_name", "ip", "ready"}
+
 				for _, s := range e.Subsets {
-					for _, available := range s.Addresses {
-						ms = append(ms, &metric.Metric{
-							LabelValues: []string{available.IP, "true"},
-							LabelKeys:   []string{"ip", "ready"},
-							Value:       1,
-						})
-					}
-					for _, notReadyAddresses := range s.NotReadyAddresses {
-						ms = append(ms, &metric.Metric{
-							LabelValues: []string{notReadyAddresses.IP, "false"},
-							LabelKeys:   []string{"ip", "ready"},
-							Value:       1,
-						})
+					for _, port := range s.Ports {
+						for _, available := range s.Addresses {
+							labelValues := []string{string(port.Protocol), strconv.FormatInt(int64(port.Port), 10), port.Name}
+
+							ms = append(ms, &metric.Metric{
+								LabelValues: append(labelValues, available.IP, "true"),
+								LabelKeys:   labelKeys,
+								Value:       1,
+							})
+						}
+						for _, notReadyAddresses := range s.NotReadyAddresses {
+							labelValues := []string{string(port.Protocol), strconv.FormatInt(int64(port.Port), 10), port.Name}
+
+							ms = append(ms, &metric.Metric{
+								LabelValues: append(labelValues, notReadyAddresses.IP, "false"),
+								LabelKeys:   labelKeys,
+								Value:       1,
+							})
+						}
 					}
 				}
 				return &metric.Family{
@@ -198,7 +165,7 @@ func endpointMetricFamilies(allowAnnotationsList, allowLabelsList []string) []ge
 			"Information about the Endpoint ports.",
 			metric.Gauge,
 			basemetrics.STABLE,
-			"",
+			"v2.14.0",
 			wrapEndpointFunc(func(e *v1.Endpoints) *metric.Family {
 				ms := []*metric.Metric{}
 				for _, s := range e.Subsets {

internal/store/endpoint_test.go

@@ -32,17 +32,13 @@ func TestEndpointStore(t *testing.T) {
 	const metadata = `
 		# HELP kube_endpoint_annotations Kubernetes annotations converted to Prometheus labels.
 		# TYPE kube_endpoint_annotations gauge
-		# HELP kube_endpoint_address_available (Deprecated since v2.6.0) Number of addresses available in endpoint.
-		# TYPE kube_endpoint_address_available gauge
-		# HELP kube_endpoint_address_not_ready (Deprecated since v2.6.0) Number of addresses not ready in endpoint
-		# TYPE kube_endpoint_address_not_ready gauge
 		# HELP kube_endpoint_created [STABLE] Unix creation timestamp
 		# TYPE kube_endpoint_created gauge
 		# HELP kube_endpoint_info [STABLE] Information about endpoint.
 		# TYPE kube_endpoint_info gauge
 		# HELP kube_endpoint_labels [STABLE] Kubernetes labels converted to Prometheus labels.
 		# TYPE kube_endpoint_labels gauge
-		# HELP kube_endpoint_ports [STABLE] Information about the Endpoint ports.
+		# HELP kube_endpoint_ports [STABLE] (Deprecated since v2.14.0) Information about the Endpoint ports.
 		# TYPE kube_endpoint_ports gauge
 		# HELP kube_endpoint_address [STABLE] Information about Endpoint available and non available addresses.
 		# TYPE kube_endpoint_address gauge
@@ -89,22 +85,26 @@ func TestEndpointStore(t *testing.T) {
 				},
 			},
 			Want: metadata + `
-				kube_endpoint_address_available{endpoint="test-endpoint",namespace="default"} 6
-				kube_endpoint_address_not_ready{endpoint="test-endpoint",namespace="default"} 6
 				kube_endpoint_created{endpoint="test-endpoint",namespace="default"} 1.5e+09
 				kube_endpoint_info{endpoint="test-endpoint",namespace="default"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="10.0.0.1",namespace="default",port_name="app",port_number="8081",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="10.0.0.1",namespace="default",port_name="http",port_number="8080",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="10.0.0.10",namespace="default",port_name="app",port_number="8081",port_protocol="TCP",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="10.0.0.10",namespace="default",port_name="http",port_number="8080",port_protocol="TCP",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="127.0.0.1",namespace="default",port_name="app",port_number="8081",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="127.0.0.1",namespace="default",port_name="http",port_number="8080",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="172.22.23.202",namespace="default",port_name="https",port_number="8443",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="172.22.23.202",namespace="default",port_name="prometheus",port_number="9090",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="192.168.1.3",namespace="default",port_name="syslog",port_number="1234",port_protocol="UDP",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="192.168.1.3",namespace="default",port_name="syslog-tcp",port_number="5678",port_protocol="TCP",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="192.168.2.2",namespace="default",port_name="syslog",port_number="1234",port_protocol="UDP",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="192.168.2.2",namespace="default",port_name="syslog-tcp",port_number="5678",port_protocol="TCP",ready="false"} 1
 				kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="http",port_protocol="TCP",port_number="8080"} 1
 				kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="app",port_protocol="TCP",port_number="8081"} 1
 				kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="https",port_protocol="TCP",port_number="8443"} 1
 				kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="prometheus",port_protocol="TCP",port_number="9090"} 1
 				kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog",port_protocol="UDP",port_number="1234"} 1
 				kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog-tcp",port_protocol="TCP",port_number="5678"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="172.22.23.202",ready="true"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.1.3",ready="false"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.2.2",ready="false"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1
 			`,
 		},
 		{
@@ -132,14 +132,12 @@ func TestEndpointStore(t *testing.T) {
 				},
 			},
 			Want: metadata + `
-				kube_endpoint_address_available{endpoint="single-port-endpoint",namespace="default"} 2
-				kube_endpoint_address_not_ready{endpoint="single-port-endpoint",namespace="default"} 1
 				kube_endpoint_created{endpoint="single-port-endpoint",namespace="default"} 1.5e+09
 				kube_endpoint_info{endpoint="single-port-endpoint",namespace="default"} 1
 				kube_endpoint_ports{endpoint="single-port-endpoint",namespace="default",port_name="",port_number="8080",port_protocol="TCP"} 1
-				kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1
-				kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1
-				kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1
+                                kube_endpoint_address{endpoint="single-port-endpoint",ip="10.0.0.1",namespace="default",port_name="",port_number="8080",port_protocol="TCP",ready="true"} 1
+                                kube_endpoint_address{endpoint="single-port-endpoint",ip="10.0.0.10",namespace="default",port_name="",port_number="8080",port_protocol="TCP",ready="false"} 1
+                                kube_endpoint_address{endpoint="single-port-endpoint",ip="127.0.0.1",namespace="default",port_name="",port_number="8080",port_protocol="TCP",ready="true"} 1
 			`,
 		},
 	}
@@ -156,10 +154,6 @@ func TestEndpointStoreWithLabels(t *testing.T) {
 	// Fixed metadata on type and help text. We prepend this to every expected
 	// output so we only have to modify a single place when doing adjustments.
 	const metadata = `
-		# HELP kube_endpoint_address_available (Deprecated since v2.6.0) Number of addresses available in endpoint.
-		# TYPE kube_endpoint_address_available gauge
-		# HELP kube_endpoint_address_not_ready (Deprecated since v2.6.0) Number of addresses not ready in endpoint
-		# TYPE kube_endpoint_address_not_ready gauge
 		# HELP kube_endpoint_annotations Kubernetes annotations converted to Prometheus labels.
 		# TYPE kube_endpoint_annotations gauge
 		# HELP kube_endpoint_created [STABLE] Unix creation timestamp
@@ -168,7 +162,7 @@ func TestEndpointStoreWithLabels(t *testing.T) {
 		# TYPE kube_endpoint_info gauge
 		# HELP kube_endpoint_labels [STABLE] Kubernetes labels converted to Prometheus labels.
 		# TYPE kube_endpoint_labels gauge
-		# HELP kube_endpoint_ports [STABLE] Information about the Endpoint ports.
+		# HELP kube_endpoint_ports [STABLE] (Deprecated since v2.14.0) Information about the Endpoint ports.
 		# TYPE kube_endpoint_ports gauge
 		# HELP kube_endpoint_address [STABLE] Information about Endpoint available and non available addresses.
 		# TYPE kube_endpoint_address gauge
@@ -218,8 +212,6 @@ func TestEndpointStoreWithLabels(t *testing.T) {
 				},
 			},
 			Want: metadata + `
-				kube_endpoint_address_available{endpoint="test-endpoint",namespace="default"} 6
-				kube_endpoint_address_not_ready{endpoint="test-endpoint",namespace="default"} 6
 				kube_endpoint_annotations{endpoint="test-endpoint",annotation_app="foobar",namespace="default"} 1
 				kube_endpoint_created{endpoint="test-endpoint",namespace="default"} 1.5e+09
 				kube_endpoint_info{endpoint="test-endpoint",namespace="default"} 1
@@ -230,12 +222,18 @@ func TestEndpointStoreWithLabels(t *testing.T) {
 				kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="prometheus",port_protocol="TCP",port_number="9090"} 1
 				kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog",port_protocol="UDP",port_number="1234"} 1
 				kube_endpoint_ports{endpoint="test-endpoint",namespace="default",port_name="syslog-tcp",port_protocol="TCP",port_number="5678"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="172.22.23.202",ready="true"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.1.3",ready="false"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="192.168.2.2",ready="false"} 1
-				kube_endpoint_address{endpoint="test-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="10.0.0.1",namespace="default",port_name="app",port_number="8081",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="10.0.0.1",namespace="default",port_name="http",port_number="8080",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="10.0.0.10",namespace="default",port_name="app",port_number="8081",port_protocol="TCP",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="10.0.0.10",namespace="default",port_name="http",port_number="8080",port_protocol="TCP",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="127.0.0.1",namespace="default",port_name="app",port_number="8081",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="127.0.0.1",namespace="default",port_name="http",port_number="8080",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="172.22.23.202",namespace="default",port_name="https",port_number="8443",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="172.22.23.202",namespace="default",port_name="prometheus",port_number="9090",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="192.168.1.3",namespace="default",port_name="syslog",port_number="1234",port_protocol="UDP",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="192.168.1.3",namespace="default",port_name="syslog-tcp",port_number="5678",port_protocol="TCP",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="192.168.2.2",namespace="default",port_name="syslog",port_number="1234",port_protocol="UDP",ready="false"} 1
+				kube_endpoint_address{endpoint="test-endpoint",ip="192.168.2.2",namespace="default",port_name="syslog-tcp",port_number="5678",port_protocol="TCP",ready="false"} 1
 			`,
 		},
 		{
@@ -267,15 +265,13 @@ func TestEndpointStoreWithLabels(t *testing.T) {
 			},
 			Want: metadata + `
 				kube_endpoint_annotations{endpoint="single-port-endpoint",annotation_app="single-foobar",namespace="default"} 1
-				kube_endpoint_address_available{endpoint="single-port-endpoint",namespace="default"} 2
-				kube_endpoint_address_not_ready{endpoint="single-port-endpoint",namespace="default"} 1
 				kube_endpoint_created{endpoint="single-port-endpoint",namespace="default"} 1.5e+09
 				kube_endpoint_info{endpoint="single-port-endpoint",namespace="default"} 1
 				kube_endpoint_labels{endpoint="single-port-endpoint",label_app="single-foobar",namespace="default"} 1
 				kube_endpoint_ports{endpoint="single-port-endpoint",namespace="default",port_name="",port_number="8080",port_protocol="TCP"} 1
-				kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="127.0.0.1",ready="true"} 1
-				kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.1",ready="true"} 1
-				kube_endpoint_address{endpoint="single-port-endpoint",namespace="default",ip="10.0.0.10",ready="false"} 1
+				kube_endpoint_address{endpoint="single-port-endpoint",ip="10.0.0.1",namespace="default",port_name="",port_number="8080",port_protocol="TCP",ready="true"} 1
+				kube_endpoint_address{endpoint="single-port-endpoint",ip="10.0.0.10",namespace="default",port_name="",port_number="8080",port_protocol="TCP",ready="false"} 1
+				kube_endpoint_address{endpoint="single-port-endpoint",ip="127.0.0.1",namespace="default",port_name="",port_number="8080",port_protocol="TCP",ready="true"} 1
 			`,
 		},
 	}

internal/store/endpointslice.go

@@ -121,54 +121,41 @@ func endpointSliceMetricFamilies(allowAnnotationsList, allowLabelsList []string)
 			wrapEndpointSliceFunc(func(e *discoveryv1.EndpointSlice) *metric.Family {
 				m := []*metric.Metric{}
 				for _, ep := range e.Endpoints {
-					var (
-						labelKeys,
-						labelValues []string
-					)
+
+					var ready, serving, terminating, hostname, targetrefKind, targetrefName, targetrefNamespace, endpointNodename, endpointZone string
 
 					if ep.Conditions.Ready != nil {
-						labelKeys = append(labelKeys, "ready")
-						labelValues = append(labelValues, strconv.FormatBool(*ep.Conditions.Ready))
-					}
-					if ep.Conditions.Serving != nil {
-						labelKeys = append(labelKeys, "serving")
-						labelValues = append(labelValues, strconv.FormatBool(*ep.Conditions.Serving))
-					}
-					if ep.Conditions.Terminating != nil {
-						labelKeys = append(labelKeys, "terminating")
-						labelValues = append(labelValues, strconv.FormatBool(*ep.Conditions.Terminating))
+						ready = strconv.FormatBool(*ep.Conditions.Ready)
 					}
 
+					if ep.Conditions.Serving != nil {
+						serving = strconv.FormatBool(*ep.Conditions.Serving)
+					}
+
+					if ep.Conditions.Terminating != nil {
+						serving = strconv.FormatBool(*ep.Conditions.Terminating)
+					}
 					if ep.Hostname != nil {
-						labelKeys = append(labelKeys, "hostname")
-						labelValues = append(labelValues, *ep.Hostname)
+						hostname = *ep.Hostname
 					}
 
 					if ep.TargetRef != nil {
-						if ep.TargetRef.Kind != "" {
-							labelKeys = append(labelKeys, "targetref_kind")
-							labelValues = append(labelValues, ep.TargetRef.Kind)
-						}
-						if ep.TargetRef.Name != "" {
-							labelKeys = append(labelKeys, "targetref_name")
-							labelValues = append(labelValues, ep.TargetRef.Name)
-						}
-						if ep.TargetRef.Namespace != "" {
-							labelKeys = append(labelKeys, "targetref_namespace")
-							labelValues = append(labelValues, ep.TargetRef.Namespace)
-						}
+						targetrefKind = ep.TargetRef.Kind
+						targetrefName = ep.TargetRef.Name
+						targetrefNamespace = ep.TargetRef.Namespace
 					}
 
 					if ep.NodeName != nil {
-						labelKeys = append(labelKeys, "endpoint_nodename")
-						labelValues = append(labelValues, *ep.NodeName)
+						endpointNodename = *ep.NodeName
 					}
 
 					if ep.Zone != nil {
-						labelKeys = append(labelKeys, "endpoint_zone")
-						labelValues = append(labelValues, *ep.Zone)
+						endpointZone = *ep.Zone
 					}
-					labelKeys = append(labelKeys, "address")
+
+					labelKeys := []string{"ready", "serving", "hostname", "terminating", "targetref_kind", "targetref_name", "targetref_namespace", "endpoint_nodename", "endpoint_zone", "address"}
+					labelValues := []string{ready, serving, terminating, hostname, targetrefKind, targetrefName, targetrefNamespace, endpointNodename, endpointZone}
+
 					for _, address := range ep.Addresses {
 						newlabelValues := make([]string, len(labelValues))
 						copy(newlabelValues, labelValues)

internal/store/endpointslice_test.go

@@ -120,8 +120,8 @@ func TestEndpointSliceStore(t *testing.T) {
 					# HELP kube_endpointslice_endpoints_hints Topology routing hints attached to endpoints
 					# TYPE kube_endpointslice_endpoints gauge
 					# TYPE kube_endpointslice_endpoints_hints gauge
-					kube_endpointslice_endpoints{address="10.0.0.1",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="host",ready="true",terminating="false",namespace="test"} 1
-					kube_endpointslice_endpoints{address="192.168.1.10",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="host",ready="true",terminating="false",namespace="test"} 1
+					kube_endpointslice_endpoints{address="10.0.0.1",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="",namespace="test",ready="true",serving="false",targetref_kind="",targetref_name="",targetref_namespace="",terminating="host"} 1
+					kube_endpointslice_endpoints{address="192.168.1.10",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="",namespace="test",ready="true",serving="false",targetref_kind="",targetref_name="",targetref_namespace="",terminating="host"} 1
 				  `,
 
 			MetricNames: []string{
@@ -159,9 +159,9 @@ func TestEndpointSliceStore(t *testing.T) {
 					# TYPE kube_endpointslice_endpoints gauge
         			        # TYPE kube_endpointslice_endpoints_hints gauge
          			kube_endpointslice_endpoints_hints{address="10.0.0.1",endpointslice="test_endpointslice-endpoints",for_zone="zone1",namespace="test"} 1
-        			kube_endpointslice_endpoints{address="10.0.0.1",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="host",ready="true",terminating="false",namespace="test"} 1
-        			kube_endpointslice_endpoints{address="192.168.1.10",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="host",ready="true",terminating="false",namespace="test"} 1
-				  `,
+        			kube_endpointslice_endpoints{address="10.0.0.1",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="",namespace="test",ready="true",serving="false",targetref_kind="",targetref_name="",targetref_namespace="",terminating="host"} 1
+				kube_endpointslice_endpoints{address="192.168.1.10",endpoint_nodename="node",endpoint_zone="west",endpointslice="test_endpointslice-endpoints",hostname="",namespace="test",ready="true",serving="false",targetref_kind="",targetref_name="",targetref_namespace="",terminating="host"} 1  
+				`,
 
 			MetricNames: []string{
 				"kube_endpointslice_endpoints",

internal/store/horizontalpodautoscaler.go

@@ -134,7 +134,7 @@ func createHPAMetaDataGeneration() generator.FamilyGenerator {
 			return &metric.Family{
 				Metrics: []*metric.Metric{
 					{
-						Value: float64(a.ObjectMeta.Generation),
+						Value: float64(a.Generation),
 					},
 				},
 			}
@@ -217,10 +217,10 @@ func createHPASpecTargetMetric() generator.FamilyGenerator {
 				}
 
 				if metricTarget.Value != nil {
-					metricMap[value] = float64(metricTarget.Value.MilliValue()) / 1000
+					metricMap[value] = convertValueToFloat64(metricTarget.Value)
 				}
 				if metricTarget.AverageValue != nil {
-					metricMap[average] = float64(metricTarget.AverageValue.MilliValue()) / 1000
+					metricMap[average] = convertValueToFloat64(metricTarget.AverageValue)
 				}
 				if metricTarget.AverageUtilization != nil {
 					metricMap[utilization] = float64(*metricTarget.AverageUtilization)
@@ -276,10 +276,10 @@ func createHPAStatusTargetMetric() generator.FamilyGenerator {
 				}
 
 				if currentMetric.Value != nil {
-					metricMap[value] = float64(currentMetric.Value.MilliValue()) / 1000
+					metricMap[value] = convertValueToFloat64(currentMetric.Value)
 				}
 				if currentMetric.AverageValue != nil {
-					metricMap[average] = float64(currentMetric.AverageValue.MilliValue()) / 1000
+					metricMap[average] = convertValueToFloat64(currentMetric.AverageValue)
 				}
 				if currentMetric.AverageUtilization != nil {
 					metricMap[utilization] = float64(*currentMetric.AverageUtilization)

internal/store/ingress.go

@@ -140,7 +140,7 @@ func ingressMetricFamilies(allowAnnotationsList, allowLabelsList []string) []gen
 			"",
 			wrapIngressFunc(func(i *networkingv1.Ingress) *metric.Family {
 				return &metric.Family{
-					Metrics: resourceVersionMetric(i.ObjectMeta.ResourceVersion),
+					Metrics: resourceVersionMetric(i.ResourceVersion),
 				}
 			}),
 		),
@@ -155,10 +155,14 @@ func ingressMetricFamilies(allowAnnotationsList, allowLabelsList []string) []gen
 				for _, rule := range i.Spec.Rules {
 					if rule.HTTP != nil {
 						for _, path := range rule.HTTP.Paths {
+							pathType := ""
+							if path.PathType != nil {
+								pathType = string(*path.PathType)
+							}
 							if path.Backend.Service != nil {
 								ms = append(ms, &metric.Metric{
-									LabelKeys:   []string{"host", "path", "service_name", "service_port"},
-									LabelValues: []string{rule.Host, path.Path, path.Backend.Service.Name, strconv.Itoa(int(path.Backend.Service.Port.Number))},
+									LabelKeys:   []string{"host", "path", "path_type", "service_name", "service_port"},
+									LabelValues: []string{rule.Host, path.Path, pathType, path.Backend.Service.Name, strconv.Itoa(int(path.Backend.Service.Port.Number))},
 									Value:       1,
 								})
 							} else {
@@ -167,8 +171,8 @@ func ingressMetricFamilies(allowAnnotationsList, allowLabelsList []string) []gen
 									apiGroup = *path.Backend.Resource.APIGroup
 								}
 								ms = append(ms, &metric.Metric{
-									LabelKeys:   []string{"host", "path", "resource_api_group", "resource_kind", "resource_name"},
-									LabelValues: []string{rule.Host, path.Path, apiGroup, path.Backend.Resource.Kind, path.Backend.Resource.Name},
+									LabelKeys:   []string{"host", "path", "path_type", "resource_api_group", "resource_kind", "resource_name"},
+									LabelValues: []string{rule.Host, path.Path, pathType, apiGroup, path.Backend.Resource.Kind, path.Backend.Resource.Name},
 									Value:       1,
 								})
 							}

internal/store/ingress_test.go

@@ -22,6 +22,7 @@ import (
 	v1 "k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 
 	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
 )
@@ -141,6 +142,7 @@ func TestIngressStore(t *testing.T) {
 													},
 												},
 											},
+											PathType: ptr.To(networkingv1.PathTypeExact),
 										},
 										{
 											Path: "/somepath2",
@@ -164,8 +166,8 @@ func TestIngressStore(t *testing.T) {
 			Want: metadata + `
 				kube_ingress_info{namespace="ns4",ingress="ingress4",ingressclass="_default"} 1
 				kube_ingress_created{namespace="ns4",ingress="ingress4"} 1.501569018e+09
-				kube_ingress_path{namespace="ns4",ingress="ingress4",host="somehost",path="/somepath",service_name="someservice",service_port="1234"} 1
-				kube_ingress_path{namespace="ns4",ingress="ingress4",host="somehost",path="/somepath2",resource_api_group="",resource_kind="somekind",resource_name="somename"} 1
+				kube_ingress_path{namespace="ns4",ingress="ingress4",host="somehost",path="/somepath",path_type="Exact",service_name="someservice",service_port="1234"} 1
+				kube_ingress_path{namespace="ns4",ingress="ingress4",host="somehost",path="/somepath2",path_type="",resource_api_group="",resource_kind="somekind",resource_name="somename"} 1
 `,
 			MetricNames: []string{"kube_ingress_info", "kube_ingress_metadata_resource_version", "kube_ingress_created", "kube_ingress_labels", "kube_ingress_path", "kube_ingress_tls"},
 		},

internal/store/job.go

@@ -26,6 +26,7 @@ import (
 	generator "k8s.io/kube-state-metrics/v2/pkg/metric_generator"
 
 	v1batch "k8s.io/api/batch/v1"
+	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/watch"
@@ -219,10 +220,10 @@ func jobMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
 					}
 				}
 
+				reasonKnown := false
 				for _, c := range j.Status.Conditions {
 					condition := c
 					if condition.Type == v1batch.JobFailed {
-						reasonKnown := false
 						for _, reason := range jobFailureReasons {
 							reasonKnown = reasonKnown || failureReason(&condition, reason)
 
@@ -233,16 +234,16 @@ func jobMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
 								Value:       boolFloat64(failureReason(&condition, reason)),
 							})
 						}
-						// for unknown reasons
-						if !reasonKnown {
-							ms = append(ms, &metric.Metric{
-								LabelKeys:   []string{"reason"},
-								LabelValues: []string{""},
-								Value:       float64(j.Status.Failed),
-							})
-						}
 					}
 				}
+				// for unknown reasons
+				if !reasonKnown {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{"reason"},
+						LabelValues: []string{""},
+						Value:       float64(j.Status.Failed),
+					})
+				}
 
 				return &metric.Family{
 					Metrics: ms,
@@ -355,6 +356,27 @@ func jobMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
 				}
 			}),
 		),
+		*generator.NewFamilyGeneratorWithStability(
+			"kube_job_status_suspended",
+			"The number of pods which reached Phase Suspended.",
+			metric.Gauge,
+			basemetrics.ALPHA,
+			"",
+			wrapJobFunc(func(j *v1batch.Job) *metric.Family {
+				ms := []*metric.Metric{}
+				for _, c := range j.Status.Conditions {
+					if c.Type == v1batch.JobSuspended {
+						ms = append(ms, &metric.Metric{
+							Value: boolFloat64(c.Status == v1.ConditionTrue),
+						})
+					}
+				}
+
+				return &metric.Family{
+					Metrics: ms,
+				}
+			}),
+		),
 		*generator.NewFamilyGeneratorWithStability(
 			"kube_job_owner",
 			"Information about the Job's owner.",

internal/store/job_test.go

@@ -44,6 +44,7 @@ var (
 
 func TestJobStore(t *testing.T) {
 	var trueValue = true
+	var falseValue = false
 
 	// Fixed metadata on type and help text. We prepend this to every expected
 	// output so we only have to modify a single place when doing adjustments.
@@ -77,7 +78,10 @@ func TestJobStore(t *testing.T) {
 		# HELP kube_job_status_start_time [STABLE] StartTime represents time when the job was acknowledged by the Job Manager.
 		# TYPE kube_job_status_start_time gauge
 		# HELP kube_job_status_succeeded [STABLE] The number of pods which reached Phase Succeeded.
-		# TYPE kube_job_status_succeeded gauge`
+		# TYPE kube_job_status_succeeded gauge
+		# HELP kube_job_status_suspended The number of pods which reached Phase Suspended.
+		# TYPE kube_job_status_suspended gauge
+		`
 
 	cases := []generateMetricsTestCase{
 		{
@@ -208,6 +212,28 @@ func TestJobStore(t *testing.T) {
 				kube_job_status_failed{job_name="FailedJob1",namespace="ns1",reason="Evicted"} 0
 				kube_job_status_start_time{job_name="FailedJob1",namespace="ns1"} 1.495807207e+09
 				kube_job_status_succeeded{job_name="FailedJob1",namespace="ns1"} 0
+`,
+		},
+		{
+			Obj: &v1batch.Job{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "FailedJobWithNoConditions",
+					Namespace: "ns1",
+				},
+				Status: v1batch.JobStatus{
+					Failed: 1,
+				},
+				Spec: v1batch.JobSpec{
+					ActiveDeadlineSeconds: &ActiveDeadlineSeconds900,
+				},
+			},
+			Want: metadata + `
+				kube_job_owner{job_name="FailedJobWithNoConditions",namespace="ns1",owner_is_controller="",owner_kind="",owner_name=""} 1
+				kube_job_info{job_name="FailedJobWithNoConditions",namespace="ns1"} 1
+				kube_job_spec_active_deadline_seconds{job_name="FailedJobWithNoConditions",namespace="ns1"} 900
+				kube_job_status_active{job_name="FailedJobWithNoConditions",namespace="ns1"} 0
+				kube_job_status_failed{job_name="FailedJobWithNoConditions",namespace="ns1",reason=""} 1
+				kube_job_status_succeeded{job_name="FailedJobWithNoConditions",namespace="ns1"} 0
 `,
 		},
 		{
@@ -250,6 +276,74 @@ func TestJobStore(t *testing.T) {
 				kube_job_status_failed{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 0
 				kube_job_status_start_time{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 1.495800607e+09
 				kube_job_status_succeeded{job_name="SuccessfulJob2NoActiveDeadlineSeconds",namespace="ns1"} 1
+`,
+		},
+		{
+			Obj: &v1batch.Job{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:       "SuspendedNoActiveDeadlineSeconds",
+					Namespace:  "ns1",
+					Generation: 1,
+				},
+				Status: v1batch.JobStatus{
+					Active:    0,
+					Failed:    0,
+					Succeeded: 0,
+					StartTime: &metav1.Time{Time: SuccessfulJob2StartTime},
+					Conditions: []v1batch.JobCondition{
+						{Type: v1batch.JobSuspended, Status: v1.ConditionTrue},
+					},
+				},
+				Spec: v1batch.JobSpec{
+					Suspend:     &trueValue,
+					Parallelism: &Parallelism1,
+					Completions: &Completions1,
+				},
+			},
+			Want: metadata + `
+				kube_job_owner{job_name="SuspendedNoActiveDeadlineSeconds",namespace="ns1",owner_is_controller="",owner_kind="",owner_name=""} 1
+				kube_job_info{job_name="SuspendedNoActiveDeadlineSeconds",namespace="ns1"} 1
+				kube_job_spec_completions{job_name="SuspendedNoActiveDeadlineSeconds",namespace="ns1"} 1
+				kube_job_spec_parallelism{job_name="SuspendedNoActiveDeadlineSeconds",namespace="ns1"} 1
+				kube_job_status_active{job_name="SuspendedNoActiveDeadlineSeconds",namespace="ns1"} 0
+				kube_job_status_failed{job_name="SuspendedNoActiveDeadlineSeconds",namespace="ns1"} 0
+				kube_job_status_start_time{job_name="SuspendedNoActiveDeadlineSeconds",namespace="ns1"} 1.495800607e+09
+				kube_job_status_succeeded{job_name="SuspendedNoActiveDeadlineSeconds",namespace="ns1"} 0
+                kube_job_status_suspended{job_name="SuspendedNoActiveDeadlineSeconds",namespace="ns1"} 1
+`,
+		},
+		{
+			Obj: &v1batch.Job{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:       "UnsuspendedNoActiveDeadlineSeconds",
+					Namespace:  "ns1",
+					Generation: 1,
+				},
+				Status: v1batch.JobStatus{
+					Active:    0,
+					Failed:    0,
+					Succeeded: 0,
+					StartTime: &metav1.Time{Time: SuccessfulJob2StartTime},
+					Conditions: []v1batch.JobCondition{
+						{Type: v1batch.JobSuspended, Status: v1.ConditionFalse},
+					},
+				},
+				Spec: v1batch.JobSpec{
+					Suspend:     &falseValue,
+					Parallelism: &Parallelism1,
+					Completions: &Completions1,
+				},
+			},
+			Want: metadata + `
+				kube_job_owner{job_name="UnsuspendedNoActiveDeadlineSeconds",namespace="ns1",owner_is_controller="",owner_kind="",owner_name=""} 1
+				kube_job_info{job_name="UnsuspendedNoActiveDeadlineSeconds",namespace="ns1"} 1
+				kube_job_spec_completions{job_name="UnsuspendedNoActiveDeadlineSeconds",namespace="ns1"} 1
+				kube_job_spec_parallelism{job_name="UnsuspendedNoActiveDeadlineSeconds",namespace="ns1"} 1
+				kube_job_status_active{job_name="UnsuspendedNoActiveDeadlineSeconds",namespace="ns1"} 0
+				kube_job_status_failed{job_name="UnsuspendedNoActiveDeadlineSeconds",namespace="ns1"} 0
+				kube_job_status_start_time{job_name="UnsuspendedNoActiveDeadlineSeconds",namespace="ns1"} 1.495800607e+09
+				kube_job_status_succeeded{job_name="UnsuspendedNoActiveDeadlineSeconds",namespace="ns1"} 0
+                kube_job_status_suspended{job_name="UnsuspendedNoActiveDeadlineSeconds",namespace="ns1"} 0
 `,
 		},
 	}

internal/store/limitrange.go

@@ -49,35 +49,35 @@ var (
 					for resource, min := range rawLimitRange.Min {
 						ms = append(ms, &metric.Metric{
 							LabelValues: []string{string(resource), string(rawLimitRange.Type), "min"},
-							Value:       float64(min.MilliValue()) / 1000,
+							Value:       convertValueToFloat64(&min),
 						})
 					}
 
 					for resource, max := range rawLimitRange.Max {
 						ms = append(ms, &metric.Metric{
 							LabelValues: []string{string(resource), string(rawLimitRange.Type), "max"},
-							Value:       float64(max.MilliValue()) / 1000,
+							Value:       convertValueToFloat64(&max),
 						})
 					}
 
 					for resource, df := range rawLimitRange.Default {
 						ms = append(ms, &metric.Metric{
 							LabelValues: []string{string(resource), string(rawLimitRange.Type), "default"},
-							Value:       float64(df.MilliValue()) / 1000,
+							Value:       convertValueToFloat64(&df),
 						})
 					}
 
 					for resource, dfR := range rawLimitRange.DefaultRequest {
 						ms = append(ms, &metric.Metric{
 							LabelValues: []string{string(resource), string(rawLimitRange.Type), "defaultRequest"},
-							Value:       float64(dfR.MilliValue()) / 1000,
+							Value:       convertValueToFloat64(&dfR),
 						})
 					}
 
 					for resource, mLR := range rawLimitRange.MaxLimitRequestRatio {
 						ms = append(ms, &metric.Metric{
 							LabelValues: []string{string(resource), string(rawLimitRange.Type), "maxLimitRequestRatio"},
-							Value:       float64(mLR.MilliValue()) / 1000,
+							Value:       convertValueToFloat64(&mLR),
 						})
 					}
 				}

internal/store/mutatingwebhookconfiguration.go

@@ -78,7 +78,7 @@ var (
 			"",
 			wrapMutatingWebhookConfigurationFunc(func(mwc *admissionregistrationv1.MutatingWebhookConfiguration) *metric.Family {
 				return &metric.Family{
-					Metrics: resourceVersionMetric(mwc.ObjectMeta.ResourceVersion),
+					Metrics: resourceVersionMetric(mwc.ResourceVersion),
 				}
 			}),
 		),

internal/store/networkpolicy_test.go

@@ -29,10 +29,6 @@ func TestNetworkPolicyStore(t *testing.T) {
 	startTime := 1501569018
 	metav1StartTime := metav1.Unix(int64(startTime), 0)
 
-	const metadata = `
-		# HELP kube_verticalpodautoscaler_labels Kubernetes labels converted to Prometheus labels.
-		# TYPE kube_verticalpodautoscaler_labels gauge
-		`
 	cases := []generateMetricsTestCase{
 		{
 			Obj: &networkingv1.NetworkPolicy{

internal/store/node.go

@@ -152,7 +152,7 @@ func createNodeInfoFamilyGenerator() generator.FamilyGenerator {
 				n.Status.NodeInfo.OSImage,
 				n.Status.NodeInfo.ContainerRuntimeVersion,
 				n.Status.NodeInfo.KubeletVersion,
-				n.Status.NodeInfo.KubeProxyVersion,
+				"deprecated",
 				n.Spec.ProviderID,
 				n.Spec.PodCIDR,
 				n.Status.NodeInfo.SystemUUID,
@@ -324,7 +324,7 @@ func createNodeStatusAllocatableFamilyGenerator() generator.FamilyGenerator {
 							SanitizeLabelName(string(resourceName)),
 							string(constant.UnitCore),
 						},
-						Value: float64(val.MilliValue()) / 1000,
+						Value: convertValueToFloat64(&val),
 					})
 				case v1.ResourceStorage:
 					fallthrough
@@ -336,7 +336,7 @@ func createNodeStatusAllocatableFamilyGenerator() generator.FamilyGenerator {
 							SanitizeLabelName(string(resourceName)),
 							string(constant.UnitByte),
 						},
-						Value: float64(val.MilliValue()) / 1000,
+						Value: convertValueToFloat64(&val),
 					})
 				case v1.ResourcePods:
 					ms = append(ms, &metric.Metric{
@@ -344,7 +344,7 @@ func createNodeStatusAllocatableFamilyGenerator() generator.FamilyGenerator {
 							SanitizeLabelName(string(resourceName)),
 							string(constant.UnitInteger),
 						},
-						Value: float64(val.MilliValue()) / 1000,
+						Value: convertValueToFloat64(&val),
 					})
 				default:
 					if isHugePageResourceName(resourceName) {
@@ -353,7 +353,7 @@ func createNodeStatusAllocatableFamilyGenerator() generator.FamilyGenerator {
 								SanitizeLabelName(string(resourceName)),
 								string(constant.UnitByte),
 							},
-							Value: float64(val.MilliValue()) / 1000,
+							Value: convertValueToFloat64(&val),
 						})
 					}
 					if isAttachableVolumeResourceName(resourceName) {
@@ -362,7 +362,7 @@ func createNodeStatusAllocatableFamilyGenerator() generator.FamilyGenerator {
 								SanitizeLabelName(string(resourceName)),
 								string(constant.UnitByte),
 							},
-							Value: float64(val.MilliValue()) / 1000,
+							Value: convertValueToFloat64(&val),
 						})
 					}
 					if isExtendedResourceName(resourceName) {
@@ -371,7 +371,7 @@ func createNodeStatusAllocatableFamilyGenerator() generator.FamilyGenerator {
 								SanitizeLabelName(string(resourceName)),
 								string(constant.UnitInteger),
 							},
-							Value: float64(val.MilliValue()) / 1000,
+							Value: convertValueToFloat64(&val),
 						})
 					}
 				}
@@ -407,7 +407,7 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
 							SanitizeLabelName(string(resourceName)),
 							string(constant.UnitCore),
 						},
-						Value: float64(val.MilliValue()) / 1000,
+						Value: convertValueToFloat64(&val),
 					})
 				case v1.ResourceStorage:
 					fallthrough
@@ -419,7 +419,7 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
 							SanitizeLabelName(string(resourceName)),
 							string(constant.UnitByte),
 						},
-						Value: float64(val.MilliValue()) / 1000,
+						Value: convertValueToFloat64(&val),
 					})
 				case v1.ResourcePods:
 					ms = append(ms, &metric.Metric{
@@ -427,7 +427,7 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
 							SanitizeLabelName(string(resourceName)),
 							string(constant.UnitInteger),
 						},
-						Value: float64(val.MilliValue()) / 1000,
+						Value: convertValueToFloat64(&val),
 					})
 				default:
 					if isHugePageResourceName(resourceName) {
@@ -436,7 +436,7 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
 								SanitizeLabelName(string(resourceName)),
 								string(constant.UnitByte),
 							},
-							Value: float64(val.MilliValue()) / 1000,
+							Value: convertValueToFloat64(&val),
 						})
 					}
 					if isAttachableVolumeResourceName(resourceName) {
@@ -445,7 +445,7 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
 								SanitizeLabelName(string(resourceName)),
 								string(constant.UnitByte),
 							},
-							Value: float64(val.MilliValue()) / 1000,
+							Value: convertValueToFloat64(&val),
 						})
 					}
 					if isExtendedResourceName(resourceName) {
@@ -454,7 +454,7 @@ func createNodeStatusCapacityFamilyGenerator() generator.FamilyGenerator {
 								SanitizeLabelName(string(resourceName)),
 								string(constant.UnitInteger),
 							},
-							Value: float64(val.MilliValue()) / 1000,
+							Value: convertValueToFloat64(&val),
 						})
 					}
 				}

internal/store/node_test.go

@@ -60,7 +60,7 @@ func TestNodeStore(t *testing.T) {
 				# TYPE kube_node_info gauge
 				# TYPE kube_node_labels gauge
 				# TYPE kube_node_spec_unschedulable gauge
-				kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="kubeproxy",node="127.0.0.1",os_image="osimage",pod_cidr="172.24.10.0/24",provider_id="provider://i-uniqueid",internal_ip="1.2.3.4",system_uuid="6a934e21-5207-4a84-baea-3a952d926c80"} 1
+				kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="deprecated",node="127.0.0.1",os_image="osimage",pod_cidr="172.24.10.0/24",provider_id="provider://i-uniqueid",internal_ip="1.2.3.4",system_uuid="6a934e21-5207-4a84-baea-3a952d926c80"} 1
 				kube_node_spec_unschedulable{node="127.0.0.1"} 0
 			`,
 			MetricNames: []string{"kube_node_spec_unschedulable", "kube_node_labels", "kube_node_info"},
@@ -75,7 +75,7 @@ func TestNodeStore(t *testing.T) {
 			Want: `
 				# HELP kube_node_info [STABLE] Information about a cluster node.
 				# TYPE kube_node_info gauge
-				kube_node_info{container_runtime_version="",kernel_version="",kubelet_version="",kubeproxy_version="",node="",os_image="",pod_cidr="",provider_id="",internal_ip="",system_uuid=""} 1
+				kube_node_info{container_runtime_version="",kernel_version="",kubelet_version="",kubeproxy_version="deprecated",node="",os_image="",pod_cidr="",provider_id="",internal_ip="",system_uuid=""} 1
 			`,
 			MetricNames: []string{"kube_node_info"},
 		},
@@ -140,7 +140,7 @@ func TestNodeStore(t *testing.T) {
 		# TYPE kube_node_status_allocatable gauge
 		# TYPE kube_node_status_capacity gauge
 		kube_node_created{node="127.0.0.1"} 1.5e+09
-        kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="kubeproxy",node="127.0.0.1",os_image="osimage",pod_cidr="172.24.10.0/24",provider_id="provider://i-randomidentifier",internal_ip="1.2.3.4",system_uuid="6a934e21-5207-4a84-baea-3a952d926c80"} 1
+        kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="deprecated",node="127.0.0.1",os_image="osimage",pod_cidr="172.24.10.0/24",provider_id="provider://i-randomidentifier",internal_ip="1.2.3.4",system_uuid="6a934e21-5207-4a84-baea-3a952d926c80"} 1
 		kube_node_role{node="127.0.0.1",role="master"} 1
         kube_node_spec_unschedulable{node="127.0.0.1"} 1
         kube_node_status_allocatable{node="127.0.0.1",resource="cpu",unit="core"} 3
@@ -298,6 +298,94 @@ func TestNodeStore(t *testing.T) {
 					`,
 			MetricNames: []string{"kube_node_status_addresses"},
 		},
+		// memory overflow test
+		{
+			Obj: &v1.Node{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "127.0.0.1",
+					CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
+					Labels: map[string]string{
+						"node-role.kubernetes.io/master": "",
+					},
+				},
+				Spec: v1.NodeSpec{
+					Unschedulable: true,
+					ProviderID:    "provider://i-randomidentifier",
+					PodCIDR:       "172.24.10.0/24",
+				},
+				Status: v1.NodeStatus{
+					NodeInfo: v1.NodeSystemInfo{
+						KernelVersion:           "kernel",
+						KubeletVersion:          "kubelet",
+						KubeProxyVersion:        "kubeproxy",
+						OSImage:                 "osimage",
+						ContainerRuntimeVersion: "rkt",
+						SystemUUID:              "6a934e21-5207-4a84-baea-3a952d926c80",
+					},
+					Addresses: []v1.NodeAddress{
+						{Type: "InternalIP", Address: "1.2.3.4"},
+					},
+					Capacity: v1.ResourceList{
+						v1.ResourceCPU: resource.MustParse("4.3"),
+						// overflow test
+						v1.ResourceMemory:                 resource.MustParse("10000T"),
+						v1.ResourcePods:                   resource.MustParse("1000"),
+						v1.ResourceStorage:                resource.MustParse("3G"),
+						v1.ResourceEphemeralStorage:       resource.MustParse("4G"),
+						v1.ResourceName("nvidia.com/gpu"): resource.MustParse("4"),
+					},
+					Allocatable: v1.ResourceList{
+						v1.ResourceCPU:                    resource.MustParse("3"),
+						v1.ResourceMemory:                 resource.MustParse("1G"),
+						v1.ResourcePods:                   resource.MustParse("555"),
+						v1.ResourceStorage:                resource.MustParse("2G"),
+						v1.ResourceEphemeralStorage:       resource.MustParse("3G"),
+						v1.ResourceName("nvidia.com/gpu"): resource.MustParse("1"),
+					},
+				},
+			},
+			Want: `
+		# HELP kube_node_created [STABLE] Unix creation timestamp
+		# HELP kube_node_info [STABLE] Information about a cluster node.
+		# HELP kube_node_labels [STABLE] Kubernetes labels converted to Prometheus labels.
+		# HELP kube_node_role The role of a cluster node.
+		# HELP kube_node_spec_unschedulable [STABLE] Whether a node can schedule new pods.
+		# HELP kube_node_status_allocatable [STABLE] The allocatable for different resources of a node that are available for scheduling.
+		# HELP kube_node_status_capacity [STABLE] The capacity for different resources of a node.
+		# TYPE kube_node_created gauge
+		# TYPE kube_node_info gauge
+		# TYPE kube_node_labels gauge
+		# TYPE kube_node_role gauge
+		# TYPE kube_node_spec_unschedulable gauge
+		# TYPE kube_node_status_allocatable gauge
+		# TYPE kube_node_status_capacity gauge
+		kube_node_created{node="127.0.0.1"} 1.5e+09
+        kube_node_info{container_runtime_version="rkt",kernel_version="kernel",kubelet_version="kubelet",kubeproxy_version="deprecated",node="127.0.0.1",os_image="osimage",pod_cidr="172.24.10.0/24",provider_id="provider://i-randomidentifier",internal_ip="1.2.3.4",system_uuid="6a934e21-5207-4a84-baea-3a952d926c80"} 1
+		kube_node_role{node="127.0.0.1",role="master"} 1
+        kube_node_spec_unschedulable{node="127.0.0.1"} 1
+        kube_node_status_allocatable{node="127.0.0.1",resource="cpu",unit="core"} 3
+        kube_node_status_allocatable{node="127.0.0.1",resource="ephemeral_storage",unit="byte"} 3e+09
+        kube_node_status_allocatable{node="127.0.0.1",resource="memory",unit="byte"} 1e+09
+        kube_node_status_allocatable{node="127.0.0.1",resource="nvidia_com_gpu",unit="integer"} 1
+        kube_node_status_allocatable{node="127.0.0.1",resource="pods",unit="integer"} 555
+        kube_node_status_allocatable{node="127.0.0.1",resource="storage",unit="byte"} 2e+09
+        kube_node_status_capacity{node="127.0.0.1",resource="cpu",unit="core"} 4.3
+        kube_node_status_capacity{node="127.0.0.1",resource="ephemeral_storage",unit="byte"} 4e+09
+        kube_node_status_capacity{node="127.0.0.1",resource="memory",unit="byte"} 1e+16
+        kube_node_status_capacity{node="127.0.0.1",resource="nvidia_com_gpu",unit="integer"} 4
+        kube_node_status_capacity{node="127.0.0.1",resource="pods",unit="integer"} 1000
+        kube_node_status_capacity{node="127.0.0.1",resource="storage",unit="byte"} 3e+09
+			`,
+			MetricNames: []string{
+				"kube_node_status_capacity",
+				"kube_node_status_allocatable",
+				"kube_node_spec_unschedulable",
+				"kube_node_labels",
+				"kube_node_role",
+				"kube_node_info",
+				"kube_node_created",
+			},
+		},
 	}
 	for i, c := range cases {
 		c.Func = generator.ComposeMetricGenFuncs(nodeMetricFamilies(nil, nil))

internal/store/persistentvolume.go

@@ -243,50 +243,50 @@ func createPersistentVolumeInfo() generator.FamilyGenerator {
 			)
 
 			switch {
-			case p.Spec.PersistentVolumeSource.GCEPersistentDisk != nil:
-				gcePDDiskName = p.Spec.PersistentVolumeSource.GCEPersistentDisk.PDName
-			case p.Spec.PersistentVolumeSource.AWSElasticBlockStore != nil:
-				ebsVolumeID = p.Spec.PersistentVolumeSource.AWSElasticBlockStore.VolumeID
-			case p.Spec.PersistentVolumeSource.AzureDisk != nil:
-				azureDiskName = p.Spec.PersistentVolumeSource.AzureDisk.DiskName
-			case p.Spec.PersistentVolumeSource.FC != nil:
-				if p.Spec.PersistentVolumeSource.FC.Lun != nil {
-					fcLun = strconv.FormatInt(int64(*p.Spec.PersistentVolumeSource.FC.Lun), 10)
+			case p.Spec.GCEPersistentDisk != nil:
+				gcePDDiskName = p.Spec.GCEPersistentDisk.PDName
+			case p.Spec.AWSElasticBlockStore != nil:
+				ebsVolumeID = p.Spec.AWSElasticBlockStore.VolumeID
+			case p.Spec.AzureDisk != nil:
+				azureDiskName = p.Spec.AzureDisk.DiskName
+			case p.Spec.FC != nil:
+				if p.Spec.FC.Lun != nil {
+					fcLun = strconv.FormatInt(int64(*p.Spec.FC.Lun), 10)
 				}
-				for _, wwn := range p.Spec.PersistentVolumeSource.FC.TargetWWNs {
+				for _, wwn := range p.Spec.FC.TargetWWNs {
 					if len(fcTargetWWNs) != 0 {
 						fcTargetWWNs += ","
 					}
 					fcTargetWWNs += wwn
 				}
-				for _, wwid := range p.Spec.PersistentVolumeSource.FC.WWIDs {
+				for _, wwid := range p.Spec.FC.WWIDs {
 					if len(fcWWIDs) != 0 {
 						fcWWIDs += ","
 					}
 					fcWWIDs += wwid
 				}
-			case p.Spec.PersistentVolumeSource.ISCSI != nil:
-				iscsiTargetPortal = p.Spec.PersistentVolumeSource.ISCSI.TargetPortal
-				iscsiIQN = p.Spec.PersistentVolumeSource.ISCSI.IQN
-				iscsiLun = strconv.FormatInt(int64(p.Spec.PersistentVolumeSource.ISCSI.Lun), 10)
-				if p.Spec.PersistentVolumeSource.ISCSI.InitiatorName != nil {
-					iscsiInitiatorName = *p.Spec.PersistentVolumeSource.ISCSI.InitiatorName
+			case p.Spec.ISCSI != nil:
+				iscsiTargetPortal = p.Spec.ISCSI.TargetPortal
+				iscsiIQN = p.Spec.ISCSI.IQN
+				iscsiLun = strconv.FormatInt(int64(p.Spec.ISCSI.Lun), 10)
+				if p.Spec.ISCSI.InitiatorName != nil {
+					iscsiInitiatorName = *p.Spec.ISCSI.InitiatorName
 				}
-			case p.Spec.PersistentVolumeSource.NFS != nil:
-				nfsServer = p.Spec.PersistentVolumeSource.NFS.Server
-				nfsPath = p.Spec.PersistentVolumeSource.NFS.Path
-			case p.Spec.PersistentVolumeSource.CSI != nil:
-				csiDriver = p.Spec.PersistentVolumeSource.CSI.Driver
-				csiVolumeHandle = p.Spec.PersistentVolumeSource.CSI.VolumeHandle
-			case p.Spec.PersistentVolumeSource.Local != nil:
-				localPath = p.Spec.PersistentVolumeSource.Local.Path
-				if p.Spec.PersistentVolumeSource.Local.FSType != nil {
-					localFS = *p.Spec.PersistentVolumeSource.Local.FSType
+			case p.Spec.NFS != nil:
+				nfsServer = p.Spec.NFS.Server
+				nfsPath = p.Spec.NFS.Path
+			case p.Spec.CSI != nil:
+				csiDriver = p.Spec.CSI.Driver
+				csiVolumeHandle = p.Spec.CSI.VolumeHandle
+			case p.Spec.Local != nil:
+				localPath = p.Spec.Local.Path
+				if p.Spec.Local.FSType != nil {
+					localFS = *p.Spec.Local.FSType
 				}
-			case p.Spec.PersistentVolumeSource.HostPath != nil:
-				hostPath = p.Spec.PersistentVolumeSource.HostPath.Path
-				if p.Spec.PersistentVolumeSource.HostPath.Type != nil {
-					hostPathType = string(*p.Spec.PersistentVolumeSource.HostPath.Type)
+			case p.Spec.HostPath != nil:
+				hostPath = p.Spec.HostPath.Path
+				if p.Spec.HostPath.Type != nil {
+					hostPathType = string(*p.Spec.HostPath.Type)
 				}
 			}
 
@@ -313,6 +313,7 @@ func createPersistentVolumeInfo() generator.FamilyGenerator {
 							"local_fs",
 							"host_path",
 							"host_path_type",
+							"reclaim_policy",
 						},
 						LabelValues: []string{
 							p.Spec.StorageClassName,
@@ -334,6 +335,7 @@ func createPersistentVolumeInfo() generator.FamilyGenerator {
 							localFS,
 							hostPath,
 							hostPathType,
+							string(p.Spec.PersistentVolumeReclaimPolicy),
 						},
 						Value: 1,
 					},
@@ -428,11 +430,12 @@ func createPersistentVolumeCSIAttributes() generator.FamilyGenerator {
 			}
 
 			var csiMounter, csiMapOptions string
-			for k, v := range p.Spec.PersistentVolumeSource.CSI.VolumeAttributes {
+			for k, v := range p.Spec.CSI.VolumeAttributes {
 				// storage attributes handled by external CEPH CSI driver
-				if k == "mapOptions" {
+				switch k {
+				case "mapOptions":
 					csiMapOptions = v
-				} else if k == "mounter" {
+				case "mounter":
 					csiMounter = v
 				}
 			}

internal/store/persistentvolume_test.go

@@ -31,6 +31,7 @@ import (
 func TestPersistentVolumeStore(t *testing.T) {
 	iscsiInitiatorName := "iqn.my.test.initiator:112233"
 	volumeMode := v1.PersistentVolumeBlock
+	var reclaimPolicy v1.PersistentVolumeReclaimPolicy = "Retain"
 	cases := []generateMetricsTestCase{
 		// Verify phase enumerations.
 		{
@@ -166,6 +167,9 @@ func TestPersistentVolumeStore(t *testing.T) {
 				ObjectMeta: metav1.ObjectMeta{
 					Name: "test-pv-available",
 				},
+				Spec: v1.PersistentVolumeSpec{
+					PersistentVolumeReclaimPolicy: reclaimPolicy,
+				},
 				Status: v1.PersistentVolumeStatus{
 					Phase: v1.VolumeAvailable,
 				},
@@ -173,7 +177,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="Retain",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -192,7 +196,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -215,7 +219,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="name",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="name",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -238,7 +242,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="aws://eu-west-1c/vol-012d34d567890123b",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="aws://eu-west-1c/vol-012d34d567890123b",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -261,7 +265,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="azure_disk_1",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="azure_disk_1",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -285,7 +289,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="123",fc_target_wwns="0123456789abcdef,abcdef0123456789",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="123",fc_target_wwns="0123456789abcdef,abcdef0123456789",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -308,7 +312,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="0123456789abcdef,abcdef0123456789",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="0123456789abcdef,abcdef0123456789",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -333,7 +337,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="iqn.my.test.server.target00",iscsi_lun="123",iscsi_target_portal="1.2.3.4:3260",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="iqn.my.test.server.target00",iscsi_lun="123",iscsi_target_portal="1.2.3.4:3260",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -359,7 +363,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="iqn.my.test.initiator:112233",iscsi_iqn="iqn.my.test.server.target00",iscsi_lun="123",iscsi_target_portal="1.2.3.4:3260",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="iqn.my.test.initiator:112233",iscsi_iqn="iqn.my.test.server.target00",iscsi_lun="123",iscsi_target_portal="1.2.3.4:3260",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -383,7 +387,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="/myPath",nfs_server="1.2.3.4",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="/myPath",nfs_server="1.2.3.4",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -407,7 +411,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="test-driver",csi_volume_handle="test-volume-handle",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="test-driver",csi_volume_handle="test-volume-handle",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -431,7 +435,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="/mnt/data",local_fs="ext4",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="/mnt/data",local_fs="ext4",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -454,7 +458,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="/mnt/data",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="/mnt/data",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -478,7 +482,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="/mnt/data",host_path_type="Directory",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="/mnt/data",host_path_type="Directory",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},
@@ -501,7 +505,7 @@ func TestPersistentVolumeStore(t *testing.T) {
 			Want: `
 					# HELP kube_persistentvolume_info [STABLE] Information about persistentvolume.
 					# TYPE kube_persistentvolume_info gauge
-					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="/mnt/data",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",storageclass=""} 1
+					kube_persistentvolume_info{azure_disk_name="",ebs_volume_id="",fc_lun="",fc_target_wwns="",fc_wwids="",gce_persistent_disk_name="",host_path="/mnt/data",host_path_type="",iscsi_initiator_name="",iscsi_iqn="",iscsi_lun="",iscsi_target_portal="",local_path="",local_fs="",nfs_path="",nfs_server="",csi_driver="",csi_volume_handle="",persistentvolume="test-pv-available",reclaim_policy="",storageclass=""} 1
 				`,
 			MetricNames: []string{"kube_persistentvolume_info"},
 		},

internal/store/pod.go

@@ -92,6 +92,7 @@ func podMetricFamilies(allowAnnotationsList, allowLabelsList []string) []generat
 		createPodStatusScheduledFamilyGenerator(),
 		createPodStatusScheduledTimeFamilyGenerator(),
 		createPodStatusUnschedulableFamilyGenerator(),
+		createPodStatusUnscheduledTimeFamilyGenerator(),
 		createPodTolerationsFamilyGenerator(),
 		createPodNodeSelectorsFamilyGenerator(),
 		createPodServiceAccountFamilyGenerator(),
@@ -182,7 +183,7 @@ func createPodContainerResourceLimitsFamilyGenerator() generator.FamilyGenerator
 					case v1.ResourceCPU:
 						ms = append(ms, &metric.Metric{
 							LabelValues: []string{c.Name, p.Spec.NodeName, SanitizeLabelName(string(resourceName)), string(constant.UnitCore)},
-							Value:       float64(val.MilliValue()) / 1000,
+							Value:       convertValueToFloat64(&val),
 						})
 					case v1.ResourceStorage:
 						fallthrough
@@ -246,7 +247,7 @@ func createPodContainerResourceRequestsFamilyGenerator() generator.FamilyGenerat
 					case v1.ResourceCPU:
 						ms = append(ms, &metric.Metric{
 							LabelValues: []string{c.Name, p.Spec.NodeName, SanitizeLabelName(string(resourceName)), string(constant.UnitCore)},
-							Value:       float64(val.MilliValue()) / 1000,
+							Value:       convertValueToFloat64(&val),
 						})
 					case v1.ResourceStorage:
 						fallthrough
@@ -749,7 +750,7 @@ func createPodInitContainerResourceLimitsFamilyGenerator() generator.FamilyGener
 					case v1.ResourceCPU:
 						ms = append(ms, &metric.Metric{
 							LabelValues: []string{c.Name, p.Spec.NodeName, SanitizeLabelName(string(resourceName)), string(constant.UnitCore)},
-							Value:       float64(val.MilliValue()) / 1000,
+							Value:       convertValueToFloat64(&val),
 						})
 					case v1.ResourceStorage:
 						fallthrough
@@ -813,7 +814,7 @@ func createPodInitContainerResourceRequestsFamilyGenerator() generator.FamilyGen
 					case v1.ResourceCPU:
 						ms = append(ms, &metric.Metric{
 							LabelValues: []string{c.Name, p.Spec.NodeName, SanitizeLabelName(string(resourceName)), string(constant.UnitCore)},
-							Value:       float64(val.MilliValue()) / 1000,
+							Value:       convertValueToFloat64(&val),
 						})
 					case v1.ResourceStorage:
 						fallthrough
@@ -1122,7 +1123,7 @@ func createPodOverheadCPUCoresFamilyGenerator() generator.FamilyGenerator {
 				for resourceName, val := range p.Spec.Overhead {
 					if resourceName == v1.ResourceCPU {
 						ms = append(ms, &metric.Metric{
-							Value: float64(val.MilliValue()) / 1000,
+							Value: convertValueToFloat64(&val),
 						})
 					}
 				}
@@ -1349,14 +1350,14 @@ func createPodStatusPhaseFamilyGenerator() generator.FamilyGenerator {
 			}
 
 			phases := []struct {
-				v bool
 				n string
+				v bool
 			}{
-				{phase == v1.PodPending, string(v1.PodPending)},
-				{phase == v1.PodSucceeded, string(v1.PodSucceeded)},
-				{phase == v1.PodFailed, string(v1.PodFailed)},
-				{phase == v1.PodUnknown, string(v1.PodUnknown)},
-				{phase == v1.PodRunning, string(v1.PodRunning)},
+				{string(v1.PodPending), phase == v1.PodPending},
+				{string(v1.PodSucceeded), phase == v1.PodSucceeded},
+				{string(v1.PodFailed), phase == v1.PodFailed},
+				{string(v1.PodUnknown), phase == v1.PodUnknown},
+				{string(v1.PodRunning), phase == v1.PodRunning},
 			}
 
 			ms := make([]*metric.Metric, len(phases))
@@ -1475,12 +1476,12 @@ func createPodStatusQosClassFamilyGenerator() generator.FamilyGenerator {
 			}
 
 			qosClasses := []struct {
-				v bool
 				n string
+				v bool
 			}{
-				{class == v1.PodQOSBestEffort, string(v1.PodQOSBestEffort)},
-				{class == v1.PodQOSBurstable, string(v1.PodQOSBurstable)},
-				{class == v1.PodQOSGuaranteed, string(v1.PodQOSGuaranteed)},
+				{string(v1.PodQOSBestEffort), class == v1.PodQOSBestEffort},
+				{string(v1.PodQOSBurstable), class == v1.PodQOSBurstable},
+				{string(v1.PodQOSGuaranteed), class == v1.PodQOSGuaranteed},
 			}
 
 			ms := make([]*metric.Metric, len(qosClasses))
@@ -1541,15 +1542,12 @@ func createPodStatusReasonFamilyGenerator() generator.FamilyGenerator {
 			ms := []*metric.Metric{}
 
 			for _, reason := range podStatusReasons {
-				metric := &metric.Metric{}
-				metric.LabelKeys = []string{"reason"}
-				metric.LabelValues = []string{reason}
-				if p.Status.Reason == reason {
-					metric.Value = boolFloat64(true)
-				} else {
-					metric.Value = boolFloat64(false)
+				m := &metric.Metric{
+					LabelKeys:   []string{"reason"},
+					LabelValues: []string{reason},
+					Value:       getPodStatusReasonValue(p, reason),
 				}
-				ms = append(ms, metric)
+				ms = append(ms, m)
 			}
 
 			return &metric.Family{
@@ -1559,6 +1557,23 @@ func createPodStatusReasonFamilyGenerator() generator.FamilyGenerator {
 	)
 }
 
+func getPodStatusReasonValue(p *v1.Pod, reason string) float64 {
+	if p.Status.Reason == reason {
+		return 1
+	}
+	for _, cond := range p.Status.Conditions {
+		if cond.Reason == reason {
+			return 1
+		}
+	}
+	for _, cs := range p.Status.ContainerStatuses {
+		if cs.State.Terminated != nil && cs.State.Terminated.Reason == reason {
+			return 1
+		}
+	}
+	return 0
+}
+
 func createPodStatusScheduledFamilyGenerator() generator.FamilyGenerator {
 	return *generator.NewFamilyGeneratorWithStability(
 		"kube_pod_status_scheduled",
@@ -1642,6 +1657,48 @@ func createPodStatusUnschedulableFamilyGenerator() generator.FamilyGenerator {
 	)
 }
 
+func createPodStatusUnscheduledTimeFamilyGenerator() generator.FamilyGenerator {
+	return *generator.NewFamilyGeneratorWithStability(
+		"kube_pod_status_unscheduled_time",
+		"Unix timestamp when pod moved into unscheduled status",
+		metric.Gauge,
+		basemetrics.ALPHA,
+		"",
+		wrapPodFunc(func(p *v1.Pod) *metric.Family {
+			ms := []*metric.Metric{}
+
+			for _, c := range p.Status.Conditions {
+				if c.Type == v1.PodScheduled && c.Status == v1.ConditionFalse {
+					ms = append(ms, &metric.Metric{
+						LabelKeys:   []string{},
+						LabelValues: []string{},
+						Value:       float64(c.LastTransitionTime.Unix()),
+					})
+				}
+			}
+
+			return &metric.Family{
+				Metrics: ms,
+			}
+		}),
+	)
+}
+
+// getUniqueTolerations takes an array
+func getUniqueTolerations(tolerations []v1.Toleration) []v1.Toleration {
+	uniqueTolerationsMap := make(map[v1.Toleration]struct{})
+	var uniqueTolerations []v1.Toleration
+
+	for _, toleration := range tolerations {
+		_, exists := uniqueTolerationsMap[toleration]
+		if !exists {
+			uniqueTolerationsMap[toleration] = struct{}{}
+			uniqueTolerations = append(uniqueTolerations, toleration)
+		}
+	}
+	return uniqueTolerations
+}
+
 func createPodTolerationsFamilyGenerator() generator.FamilyGenerator {
 	return *generator.NewFamilyGeneratorWithStability(
 		"kube_pod_tolerations",
@@ -1651,43 +1708,29 @@ func createPodTolerationsFamilyGenerator() generator.FamilyGenerator {
 		"",
 		wrapPodFunc(func(p *v1.Pod) *metric.Family {
 			var ms []*metric.Metric
+			uniqueTolerations := getUniqueTolerations(p.Spec.Tolerations)
 
-			for _, t := range p.Spec.Tolerations {
-				var labelKeys []string
-				var labelValues []string
-
-				if t.Key != "" {
-					labelKeys = append(labelKeys, "key")
-					labelValues = append(labelValues, t.Key)
-				}
+			for _, t := range uniqueTolerations {
+				var key, operator, value, effect, tolerationSeconds string
 
+				key = t.Key
 				if t.Operator != "" {
-					labelKeys = append(labelKeys, "operator")
-					labelValues = append(labelValues, string(t.Operator))
+					operator = string(t.Operator)
 				}
 
-				if t.Value != "" {
-					labelKeys = append(labelKeys, "value")
-					labelValues = append(labelValues, t.Value)
-				}
+				value = t.Value
 
 				if t.Effect != "" {
-					labelKeys = append(labelKeys, "effect")
-					labelValues = append(labelValues, string(t.Effect))
+					effect = string(t.Effect)
 				}
 
 				if t.TolerationSeconds != nil {
-					labelKeys = append(labelKeys, "toleration_seconds")
-					labelValues = append(labelValues, strconv.FormatInt(*t.TolerationSeconds, 10))
-				}
-
-				if len(labelKeys) == 0 {
-					continue
+					tolerationSeconds = strconv.FormatInt(*t.TolerationSeconds, 10)
 				}
 
 				ms = append(ms, &metric.Metric{
-					LabelKeys:   labelKeys,
-					LabelValues: labelValues,
+					LabelKeys:   []string{"key", "operator", "value", "effect", "toleration_seconds"},
+					LabelValues: []string{key, operator, value, effect, tolerationSeconds},
 					Value:       1,
 				})
 			}

internal/store/pod_test.go

@@ -2114,6 +2114,12 @@ func TestPodStore(t *testing.T) {
 							Operator: v1.TolerationOpEqual,
 							Value:    "value3",
 						},
+						{
+							// Duplicate toleration, to ensure that doesn't result in a duplicate metric
+							Key:      "key3",
+							Operator: v1.TolerationOpEqual,
+							Value:    "value3",
+						},
 						{
 							// an empty toleration to ensure that an empty toleration does not result in a metric
 						},
@@ -2123,9 +2129,10 @@ func TestPodStore(t *testing.T) {
 			Want: `
 				# HELP kube_pod_tolerations Information about the pod tolerations
 				# TYPE kube_pod_tolerations gauge
-				kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key1",operator="Equal",value="value1",effect="NoSchedule"} 1
-				kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key2",operator="Exists"} 1
-				kube_pod_tolerations{namespace="ns1",pod="pod1",uid="uid1",key="key3",operator="Equal",value="value3"} 1
+				kube_pod_tolerations{effect="",key="",namespace="ns1",operator="",pod="pod1",toleration_seconds="",uid="uid1",value=""} 1
+				kube_pod_tolerations{effect="",key="key2",namespace="ns1",operator="Exists",pod="pod1",toleration_seconds="",uid="uid1",value=""} 1
+				kube_pod_tolerations{effect="",key="key3",namespace="ns1",operator="Equal",pod="pod1",toleration_seconds="",uid="uid1",value="value3"} 1
+				kube_pod_tolerations{effect="NoSchedule",key="key1",namespace="ns1",operator="Equal",pod="pod1",toleration_seconds="",uid="uid1",value="value1"} 1
 			`,
 			MetricNames: []string{
 				"kube_pod_tolerations",
@@ -2275,7 +2282,7 @@ func BenchmarkPodStore(b *testing.B) {
 		},
 	}
 
-	expectedFamilies := 54
+	expectedFamilies := 55
 	for n := 0; n < b.N; n++ {
 		families := f(pod)
 		if len(families) != expectedFamilies {
@@ -2283,3 +2290,85 @@ func BenchmarkPodStore(b *testing.B) {
 		}
 	}
 }
+
+func TestGetPodStatusReasonValue(t *testing.T) {
+	reason := "TestReason"
+
+	tests := []struct {
+		name string
+		pod  *v1.Pod
+		want float64
+	}{
+		{
+			name: "matches Status.Reason",
+			pod: &v1.Pod{
+				Status: v1.PodStatus{
+					Reason: "TestReason",
+				},
+			},
+			want: 1,
+		},
+		{
+			name: "matches condition Reason",
+			pod: &v1.Pod{
+				Status: v1.PodStatus{
+					Conditions: []v1.PodCondition{
+						{
+							Reason: "TestReason",
+						},
+					},
+				},
+			},
+			want: 1,
+		},
+		{
+			name: "matches container terminated Reason",
+			pod: &v1.Pod{
+				Status: v1.PodStatus{
+					ContainerStatuses: []v1.ContainerStatus{
+						{
+							State: v1.ContainerState{
+								Terminated: &v1.ContainerStateTerminated{
+									Reason: "TestReason",
+								},
+							},
+						},
+					},
+				},
+			},
+			want: 1,
+		},
+		{
+			name: "no match returns 0",
+			pod: &v1.Pod{
+				Status: v1.PodStatus{
+					Reason: "OtherReason",
+					Conditions: []v1.PodCondition{
+						{
+							Reason: "NotTestReason",
+						},
+					},
+					ContainerStatuses: []v1.ContainerStatus{
+						{
+							State: v1.ContainerState{
+								Terminated: &v1.ContainerStateTerminated{
+									Reason: "AnotherReason",
+								},
+							},
+						},
+					},
+				},
+			},
+			want: 0,
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			got := getPodStatusReasonValue(tt.pod, reason)
+			if got != tt.want {
+				t.Errorf("getPodStatusReasonValue() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

internal/store/replicaset.go

@@ -158,7 +158,7 @@ func replicaSetMetricFamilies(allowAnnotationsList, allowLabelsList []string) []
 				return &metric.Family{
 					Metrics: []*metric.Metric{
 						{
-							Value: float64(r.ObjectMeta.Generation),
+							Value: float64(r.Generation),
 						},
 					},
 				}

internal/store/replicationcontroller.go

@@ -166,7 +166,7 @@ var (
 				return &metric.Family{
 					Metrics: []*metric.Metric{
 						{
-							Value: float64(r.ObjectMeta.Generation),
+							Value: float64(r.Generation),
 						},
 					},
 				}

internal/store/resourcequota.go

@@ -74,13 +74,13 @@ func resourceQuotaMetricFamilies(allowAnnotationsList, allowLabelsList []string)
 				for res, qty := range r.Status.Hard {
 					ms = append(ms, &metric.Metric{
 						LabelValues: []string{string(res), "hard"},
-						Value:       float64(qty.MilliValue()) / 1000,
+						Value:       convertValueToFloat64(&qty),
 					})
 				}
 				for res, qty := range r.Status.Used {
 					ms = append(ms, &metric.Metric{
 						LabelValues: []string{string(res), "used"},
-						Value:       float64(qty.MilliValue()) / 1000,
+						Value:       convertValueToFloat64(&qty),
 					})
 				}
 

internal/store/role.go

@@ -131,7 +131,7 @@ func roleMetricFamilies(allowAnnotationsList, allowLabelsList []string) []genera
 			"",
 			wrapRoleFunc(func(r *rbacv1.Role) *metric.Family {
 				return &metric.Family{
-					Metrics: resourceVersionMetric(r.ObjectMeta.ResourceVersion),
+					Metrics: resourceVersionMetric(r.ResourceVersion),
 				}
 			}),
 		),

internal/store/rolebinding.go

@@ -133,7 +133,7 @@ func roleBindingMetricFamilies(allowAnnotationsList, allowLabelsList []string) [
 			"",
 			wrapRoleBindingFunc(func(r *rbacv1.RoleBinding) *metric.Family {
 				return &metric.Family{
-					Metrics: resourceVersionMetric(r.ObjectMeta.ResourceVersion),
+					Metrics: resourceVersionMetric(r.ResourceVersion),
 				}
 			}),
 		),

internal/store/secret.go

@@ -150,7 +150,7 @@ func secretMetricFamilies(allowAnnotationsList, allowLabelsList []string) []gene
 			"",
 			wrapSecretFunc(func(s *v1.Secret) *metric.Family {
 				return &metric.Family{
-					Metrics: resourceVersionMetric(s.ObjectMeta.ResourceVersion),
+					Metrics: resourceVersionMetric(s.ResourceVersion),
 				}
 			}),
 		),

internal/store/service.go

@@ -49,8 +49,8 @@ func serviceMetricFamilies(allowAnnotationsList, allowLabelsList []string) []gen
 			"",
 			wrapSvcFunc(func(s *v1.Service) *metric.Family {
 				m := metric.Metric{
-					LabelKeys:   []string{"cluster_ip", "external_name", "load_balancer_ip"},
-					LabelValues: []string{s.Spec.ClusterIP, s.Spec.ExternalName, s.Spec.LoadBalancerIP},
+					LabelKeys:   []string{"cluster_ip", "external_name", "load_balancer_ip", "external_traffic_policy"},
+					LabelValues: []string{s.Spec.ClusterIP, s.Spec.ExternalName, s.Spec.LoadBalancerIP, string(s.Spec.ExternalTrafficPolicy)},
 					Value:       1,
 				}
 				return &metric.Family{Metrics: []*metric.Metric{&m}}

internal/store/service_test.go

@@ -74,7 +74,7 @@ func TestServiceStore(t *testing.T) {
 				# TYPE kube_service_labels gauge
 				# TYPE kube_service_spec_type gauge
 				kube_service_created{namespace="default",service="test-service1",uid="uid1"} 1.5e+09
-				kube_service_info{cluster_ip="1.2.3.4",external_name="",load_balancer_ip="",namespace="default",service="test-service1",uid="uid1"} 1
+				kube_service_info{cluster_ip="1.2.3.4",external_name="",external_traffic_policy="",load_balancer_ip="",namespace="default",service="test-service1",uid="uid1"} 1
 				kube_service_spec_type{namespace="default",service="test-service1",type="ClusterIP",uid="uid1"} 1
 `,
 			MetricNames: []string{
@@ -104,7 +104,7 @@ func TestServiceStore(t *testing.T) {
 			},
 			Want: metadata + `
 				kube_service_created{namespace="default",service="test-service2",uid="uid2"} 1.5e+09
-				kube_service_info{cluster_ip="1.2.3.5",external_name="",load_balancer_ip="",namespace="default",service="test-service2",uid="uid2"} 1
+				kube_service_info{cluster_ip="1.2.3.5",external_name="",external_traffic_policy="",load_balancer_ip="",namespace="default",service="test-service2",uid="uid2"} 1
 				kube_service_spec_type{namespace="default",service="test-service2",uid="uid2",type="NodePort"} 1
 `,
 		},
@@ -127,7 +127,7 @@ func TestServiceStore(t *testing.T) {
 			},
 			Want: metadata + `
 				kube_service_created{namespace="default",service="test-service3",uid="uid3"} 1.5e+09
-				kube_service_info{cluster_ip="1.2.3.6",external_name="",load_balancer_ip="1.2.3.7",namespace="default",service="test-service3",uid="uid3"} 1
+				kube_service_info{cluster_ip="1.2.3.6",external_name="",external_traffic_policy="",load_balancer_ip="1.2.3.7",namespace="default",service="test-service3",uid="uid3"} 1
 				kube_service_spec_type{namespace="default",service="test-service3",type="LoadBalancer",uid="uid3"} 1
 `,
 		},
@@ -149,7 +149,7 @@ func TestServiceStore(t *testing.T) {
 			},
 			Want: metadata + `
 				kube_service_created{namespace="default",service="test-service4",uid="uid4"} 1.5e+09
-				kube_service_info{cluster_ip="",external_name="www.example.com",load_balancer_ip="",namespace="default",service="test-service4",uid="uid4"} 1
+				kube_service_info{cluster_ip="",external_name="www.example.com",external_traffic_policy="",load_balancer_ip="",namespace="default",service="test-service4",uid="uid4"} 1
 				kube_service_spec_type{namespace="default",service="test-service4",uid="uid4",type="ExternalName"} 1
 			`,
 		},
@@ -180,7 +180,7 @@ func TestServiceStore(t *testing.T) {
 			},
 			Want: metadata + `
 				kube_service_created{namespace="default",service="test-service5",uid="uid5"} 1.5e+09
-				kube_service_info{cluster_ip="",external_name="",load_balancer_ip="",namespace="default",service="test-service5",uid="uid5"} 1
+				kube_service_info{cluster_ip="",external_name="",external_traffic_policy="",load_balancer_ip="",namespace="default",service="test-service5",uid="uid5"} 1
 				kube_service_spec_type{namespace="default",service="test-service5",type="LoadBalancer",uid="uid5"} 1
 				kube_service_status_load_balancer_ingress{hostname="www.example.com",ip="1.2.3.8",namespace="default",service="test-service5",uid="uid5"} 1
 			`,
@@ -206,12 +206,59 @@ func TestServiceStore(t *testing.T) {
 			},
 			Want: metadata + `
 				kube_service_created{namespace="default",service="test-service6",uid="uid6"} 1.5e+09
-				kube_service_info{cluster_ip="",external_name="",load_balancer_ip="",namespace="default",service="test-service6",uid="uid6"} 1
+				kube_service_info{cluster_ip="",external_name="",external_traffic_policy="",load_balancer_ip="",namespace="default",service="test-service6",uid="uid6"} 1
 				kube_service_spec_type{namespace="default",service="test-service6",uid="uid6",type="ClusterIP"} 1
 				kube_service_spec_external_ip{external_ip="1.2.3.9",namespace="default",service="test-service6",uid="uid6"} 1
 				kube_service_spec_external_ip{external_ip="1.2.3.10",namespace="default",service="test-service6",uid="uid6"} 1
 			`,
 		},
+		{
+			Obj: &v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "test-service7",
+					CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
+					Namespace:         "default",
+					UID:               "uid7",
+					Labels: map[string]string{
+						"app": "example7",
+					},
+				},
+				Spec: v1.ServiceSpec{
+					ClusterIP:             "1.2.3.11",
+					Type:                  v1.ServiceTypeClusterIP,
+					ExternalTrafficPolicy: "Cluster",
+				},
+			},
+			Want: metadata + `
+				kube_service_created{namespace="default",service="test-service7",uid="uid7"} 1.5e+09
+				kube_service_info{cluster_ip="1.2.3.11",external_name="",external_traffic_policy="Cluster",load_balancer_ip="",namespace="default",service="test-service7",uid="uid7"} 1
+				kube_service_spec_type{namespace="default",service="test-service7",uid="uid7",type="ClusterIP"} 1
+			`,
+		},
+		{
+			Obj: &v1.Service{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:              "test-service8",
+					CreationTimestamp: metav1.Time{Time: time.Unix(1500000000, 0)},
+					Namespace:         "default",
+					UID:               "uid8",
+					Labels: map[string]string{
+						"app": "example8",
+					},
+				},
+				Spec: v1.ServiceSpec{
+					ClusterIP:             "1.2.3.12",
+					LoadBalancerIP:        "1.2.3.13",
+					Type:                  v1.ServiceTypeLoadBalancer,
+					ExternalTrafficPolicy: "Local",
+				},
+			},
+			Want: metadata + `
+				kube_service_created{namespace="default",service="test-service8",uid="uid8"} 1.5e+09
+				kube_service_info{cluster_ip="1.2.3.12",external_name="",external_traffic_policy="Local",load_balancer_ip="1.2.3.13",namespace="default",service="test-service8",uid="uid8"} 1
+				kube_service_spec_type{namespace="default",service="test-service8",uid="uid8",type="LoadBalancer"} 1
+			`,
+		},
 	}
 	for i, c := range cases {
 		c.Func = generator.ComposeMetricGenFuncs(serviceMetricFamilies(nil, nil))

internal/store/serviceaccount.go

@@ -56,18 +56,16 @@ func createServiceAccountInfoFamilyGenerator() generator.FamilyGenerator {
 		basemetrics.ALPHA,
 		"",
 		wrapServiceAccountFunc(func(sa *v1.ServiceAccount) *metric.Family {
-			var labelKeys []string
-			var labelValues []string
+			var automountToken string
 
 			if sa.AutomountServiceAccountToken != nil {
-				labelKeys = append(labelKeys, "automount_token")
-				labelValues = append(labelValues, strconv.FormatBool(*sa.AutomountServiceAccountToken))
+				automountToken = strconv.FormatBool(*sa.AutomountServiceAccountToken)
 			}
 
 			return &metric.Family{
 				Metrics: []*metric.Metric{{
-					LabelKeys:   labelKeys,
-					LabelValues: labelValues,
+					LabelKeys:   []string{"automount_token"},
+					LabelValues: []string{automountToken},
 					Value:       1,
 				}},
 			}

internal/store/statefulset.go

@@ -182,7 +182,7 @@ func statefulSetMetricFamilies(allowAnnotationsList, allowLabelsList []string) [
 			"kube_statefulset_ordinals_start",
 			"Start ordinal of the StatefulSet.",
 			metric.Gauge,
-			basemetrics.ALPHA,
+			basemetrics.STABLE,
 			"",
 			wrapStatefulSetFunc(func(s *v1.StatefulSet) *metric.Family {
 				ms := []*metric.Metric{}
@@ -208,7 +208,7 @@ func statefulSetMetricFamilies(allowAnnotationsList, allowLabelsList []string) [
 				return &metric.Family{
 					Metrics: []*metric.Metric{
 						{
-							Value: float64(s.ObjectMeta.Generation),
+							Value: float64(s.Generation),
 						},
 					},
 				}