add test for webhook/util (#2138)

* add test for webhook/util and fix date logic

Signed-off-by: 7h3-3mp7y-m4n <emailtorash@gmail.com>

.codecov.yml

@@ -0,0 +1,4 @@
+ignore:
+- "pkg/client/.*"
+- "test/fuzz/.*"
+

.github/dependabot.yaml

@@ -0,0 +1,17 @@
+# This YAML configuration file is used to enable Dependabot for automated dependency management.
+# Dependabot helps keep the project's dependencies up-to-date by automatically creating pull requests
+# for outdated dependencies based on the version constraints defined in your project.
+# For more information and customization options, please refer to the Dependabot documentation:
+# Documentation: https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically
+# Configuration options: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  # Allow up to 10 open pull requests for update github-actions
+  # 5 by default
+  # see https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#open-pull-requests-limit
+  open-pull-requests-limit: 10
+  schedule:
+    # Check for updates to GitHub Actions every week
+    interval: "weekly"

.github/workflows/ci.yaml

@@ -5,13 +5,16 @@ on:
     branches:
       - master
       - release-*
-  pull_request: {}
+  pull_request: { }
-  workflow_dispatch: {}
+  workflow_dispatch: { }
+
+# Declare default permissions as read only.
+permissions: read-all
 
 env:
   # Common versions
-  GO_VERSION: '1.18'
+  GO_VERSION: '1.23'
-  GOLANGCI_VERSION: 'v1.47'
+  GOLANGCI_VERSION: 'v2.1'
   DOCKER_BUILDX_VERSION: 'v0.4.2'
 
   # Common users. We can't run a step 'if secrets.AWS_USR != ""' but we can run
@@ -23,28 +26,30 @@ env:
 jobs:
   typos-check:
     name: Spell Check with Typos
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     steps:
       - name: Checkout Actions Repository
-        uses: actions/checkout@v3
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Check spelling with custom config file
-        uses: crate-ci/typos@v1.13.10
+        uses: crate-ci/typos@0f0ccba9ed1df83948f0c15026e4f5ccfce46109 # v1.32.0
         with:
           config: ./typos.toml
 
   golangci-lint:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
+    permissions:
+      security-events: write
     steps:
       - name: Checkout Code
-        uses: actions/checkout@v3
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Cache Go Dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -53,14 +58,14 @@ jobs:
         run: |
           make generate
       - name: Lint golang code
-        uses: golangci/golangci-lint-action@v3.2.0
+        uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0
         with:
           version: ${{ env.GOLANGCI_VERSION }}
           args: --verbose
           skip-pkg-cache: true
           mod: readonly
       - name: Run Trivy vulnerability scanner in repo mode
-        uses: aquasecurity/trivy-action@master
+        uses: aquasecurity/trivy-action@77137e9dc3ab1b329b7c8a38c2eb7475850a14e8 # master
         with:
           scan-type: 'fs'
           ignore-unfixed: true
@@ -68,24 +73,24 @@ jobs:
           output: 'trivy-results.sarif'
           severity: 'CRITICAL'
       - name: Upload Trivy scan results to GitHub Security tab
-        uses: github/codeql-action/upload-sarif@v2
+        uses: github/codeql-action/upload-sarif@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
         with:
           sarif_file: 'trivy-results.sarif'
 
-  markdownlint-misspell-shellcheck:
+#  markdownlint-misspell-shellcheck:
-    runs-on: ubuntu-18.04
+#    runs-on: ubuntu-24.04
-    # this image is build from Dockerfile
+#    # this image is build from Dockerfile
-    # https://github.com/pouchcontainer/pouchlinter/blob/master/Dockerfile
+#    # https://github.com/pouchcontainer/pouchlinter/blob/master/Dockerfile
-    container: pouchcontainer/pouchlinter:v0.1.2
+#    container: pouchcontainer/pouchlinter:v0.1.2
-    steps:
+#    steps:
-      - name: Checkout
+#      - name: Checkout
-        uses: actions/checkout@v3
+#        uses: actions/checkout@v3
-      - name: Run misspell
+#      - name: Run misspell
-        run: find  ./* -name  "*"  | grep -v vendor | xargs misspell -error
+#        run: find  ./* -name  "*"  | grep -v vendor | xargs misspell -error
-      - name: Run shellcheck
+#      - name: Run shellcheck
-        run: find ./ -name "*.sh" | grep -v vendor | xargs shellcheck
+#        run: find ./ -name "*.sh" | grep -v vendor | xargs shellcheck
-      - name: Lint markdown files
+#      - name: Lint markdown files
-        run: find  ./ -name  "*.md" | grep -v vendor | grep -v commandline |  grep -v .github |  grep -v swagger |  grep -v api |  xargs mdl -r ~MD010,~MD013,~MD014,~MD022,~MD024,~MD029,~MD031,~MD032,~MD033,~MD036
+#        run: find  ./ -name  "*.md" | grep -v vendor | grep -v commandline |  grep -v .github |  grep -v swagger |  grep -v api |  xargs mdl -r ~MD010,~MD013,~MD014,~MD022,~MD024,~MD029,~MD031,~MD032,~MD033,~MD036
 #      - name: Check markdown links
 #        run: |
 #          set +e
@@ -100,19 +105,19 @@ jobs:
 #          bash -c "exit $code";
 
   unit-tests:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
       - name: Fetch History
         run: git fetch --prune --unshallow
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Cache Go Dependencies
-        uses: actions/cache@v2
+        uses: actions/cache@1bd1e32a3bdc45362d1e726936510720a7c30a57 # v4.2.0
         with:
           path: ~/go/pkg/mod
           key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
@@ -122,9 +127,42 @@ jobs:
           make test
           git status
       - name: Publish Unit Test Coverage
-        uses: codecov/codecov-action@v3
+        uses: codecov/codecov-action@18283e04ce6e62d37312384ff67231eb8fd56d24 # v5.4.3
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
         with:
+          token: ${{ secrets.CODECOV_TOKEN }}
           flags: unittests
           file: cover.out
-      - name: Check diff
+  # See: https://google.github.io/oss-fuzz/getting-started/continuous-integration/
-        run: '[[ -z $(git status -s) ]] || (printf "Existing modified/untracked files.\nPlease run \"make generate manifests\" and push again.\n"; exit 1)'
+  Fuzzing:
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+    steps:
+      - name: Build Fuzzers
+        id: build
+        uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@abe2c06d0e162320403dd10e8268adbb0b8923f8 # master
+        with:
+          oss-fuzz-project-name: 'openkruise'
+          language: go
+      - name: Run Fuzzers
+        uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@abe2c06d0e162320403dd10e8268adbb0b8923f8 # master
+        with:
+          oss-fuzz-project-name: 'openkruise'
+          language: go
+          fuzz-seconds: 1200
+          output-sarif: true
+      - name: Upload Crash
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        if: failure() && steps.build.outcome == 'success'
+        with:
+          name: artifacts
+          path: ./out/artifacts
+      - name: Upload Sarif
+        if: always() && steps.build.outcome == 'success'
+        uses: github/codeql-action/upload-sarif@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
+        with:
+          # Path to SARIF file relative to the root of the repository
+          sarif_file: cifuzz-sarif/results.sarif
+          checkout_path: cifuzz-sarif

.github/workflows/codeql.yml

@@ -0,0 +1,84 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master", "release-*"]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "master" ]
+
+
+permissions: 
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners
+    # Consider using larger runners for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
+        # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v3.29.0
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/docker-image.yaml

@@ -0,0 +1,28 @@
+name: Docker Image CI
+
+on:
+  workflow_dispatch:
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.HUB_KRIUSE }}
+      - name: Build the Docker image
+        run: |
+          docker buildx create --use --platform=linux/amd64,linux/arm64,linux/ppc64le --name multi-platform-builder
+          docker buildx ls
+          IMG=openkruise/kruise-manager:${{ github.ref_name }} make docker-multiarch

.github/workflows/e2e-1.16.yaml

@@ -1,482 +0,0 @@
-name: E2E-1.16
-
-on:
-  push:
-    branches:
-      - master
-      - release-*
-  pull_request: {}
-  workflow_dispatch: {}
-
-env:
-  # Common versions
-  GO_VERSION: '1.18'
-  KIND_VERSION: 'v0.14.0'
-  KIND_IMAGE: 'kindest/node:v1.16.15'
-  KIND_CLUSTER_NAME: 'ci-testing'
-
-jobs:
-
-  astatefulset:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
-        with:
-          node_image: ${{ env.KIND_IMAGE }}
-          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
-          config: ./test/kind-conf.yaml
-          version: ${{ env.KIND_VERSION }}
-      - name: Build image
-        run: |
-          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
-          docker build --pull --no-cache . -t $IMAGE
-          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
-      - name: Install Kruise
-        run: |
-          set -ex
-          kubectl cluster-info
-          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
-          NODES=$(kubectl get node | wc -l)
-          for ((i=1;i<10;i++));
-          do
-            set +e
-            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-            set -e
-            if [ "$PODS" -eq "$NODES" ]; then
-              break
-            fi
-            sleep 3
-          done
-          set +e
-          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-          kubectl get node -o yaml
-          kubectl get all -n kruise-system -o yaml
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
-          set -e
-          if [ "$PODS" -eq "$NODES" ]; then
-            echo "Wait for kruise-manager and kruise-daemon ready successfully"
-          else
-            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
-            exit 1
-          fi
-      - name: Run E2E Tests
-        run: |
-          export KUBECONFIG=/home/runner/.kube/config
-          make ginkgo
-          set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] StatefulSet' test/e2e
-          retVal=$?
-          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
-          if [ "${restartCount}" -eq "0" ];then
-              echo "$out"
-              echo "Kruise-manager has not restarted"
-          else
-              echo "$out"
-              echo "Kruise-manager has restarted, abort!!!"
-              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
-              exit 1
-          fi
-          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
-          do
-            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
-            if [ "${restartCount}" -eq "0" ];then
-                echo "Kruise-daemon has not restarted"
-            else
-                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
-                echo "Kruise-daemon has restarted, abort!!!"
-                kubectl logs -p -n ${ns} ${name}
-                exit 1
-            fi
-          done
-          exit $retVal
-
-  pullimages-containerrecreate:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
-        with:
-          node_image: ${{ env.KIND_IMAGE }}
-          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
-          config: ./test/kind-conf.yaml
-          version: ${{ env.KIND_VERSION }}
-      - name: Build image
-        run: |
-          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
-          docker build --pull --no-cache . -t $IMAGE
-          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
-      - name: Install Kruise
-        run: |
-          set -ex
-          kubectl cluster-info
-          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
-          NODES=$(kubectl get node | wc -l)
-          for ((i=1;i<10;i++));
-          do
-            set +e
-            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-            set -e
-            if [ "$PODS" -eq "$NODES" ]; then
-              break
-            fi
-            sleep 3
-          done
-          set +e
-          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-          kubectl get node -o yaml
-          kubectl get all -n kruise-system -o yaml
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
-          set -e
-          if [ "$PODS" -eq "$NODES" ]; then
-            echo "Wait for kruise-manager and kruise-daemon ready successfully"
-          else
-            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
-            exit 1
-          fi
-      - name: Run E2E Tests
-        run: |
-          export KUBECONFIG=/home/runner/.kube/config
-          make ginkgo
-          set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] (PullImage|ContainerRecreateRequest)' test/e2e
-          retVal=$?
-          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
-          if [ "${restartCount}" -eq "0" ];then
-              echo "Kruise-manager has not restarted"
-          else
-              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
-              echo "Kruise-manager has restarted, abort!!!"
-              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
-              exit 1
-          fi
-          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
-          do
-            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
-            if [ "${restartCount}" -eq "0" ];then
-                echo "Kruise-daemon has not restarted"
-            else
-                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
-                echo "Kruise-daemon has restarted, abort!!!"
-                kubectl logs -p -n ${ns} ${name}
-                exit 1
-            fi
-          done
-          exit $retVal
-
-  advanced-daemonset:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
-        with:
-          node_image: ${{ env.KIND_IMAGE }}
-          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
-          config: ./test/kind-conf.yaml
-          version: ${{ env.KIND_VERSION }}
-      - name: Build image
-        run: |
-          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
-          docker build --pull --no-cache . -t $IMAGE
-          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
-      - name: Install Kruise
-        run: |
-          set -ex
-          kubectl cluster-info
-          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
-          NODES=$(kubectl get node | wc -l)
-          for ((i=1;i<10;i++));
-          do
-            set +e
-            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-            set -e
-            if [ "$PODS" -eq "$NODES" ]; then
-              break
-            fi
-            sleep 3
-          done
-          set +e
-          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-          kubectl get node -o yaml
-          kubectl get all -n kruise-system -o yaml
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
-          set -e
-          if [ "$PODS" -eq "$NODES" ]; then
-            echo "Wait for kruise-manager and kruise-daemon ready successfully"
-          else
-            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
-            exit 1
-          fi
-      - name: Run E2E Tests
-        run: |
-          export KUBECONFIG=/home/runner/.kube/config
-          make ginkgo
-          set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] DaemonSet' test/e2e
-          retVal=$?
-          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
-          if [ "${restartCount}" -eq "0" ];then
-              echo "Kruise-manager has not restarted"
-          else
-              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
-              echo "Kruise-manager has restarted, abort!!!"
-              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
-              exit 1
-          fi
-          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
-          do
-            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
-            if [ "${restartCount}" -eq "0" ];then
-                echo "Kruise-daemon has not restarted"
-            else
-                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
-                echo "Kruise-daemon has restarted, abort!!!"
-                kubectl logs -p -n ${ns} ${name}
-                exit 1
-            fi
-          done
-          exit $retVal
-
-  sidecarset:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
-        with:
-          node_image: ${{ env.KIND_IMAGE }}
-          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
-          config: ./test/kind-conf.yaml
-          version: ${{ env.KIND_VERSION }}
-      - name: Build image
-        run: |
-          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
-          docker build --pull --no-cache . -t $IMAGE
-          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
-      - name: Install Kruise
-        run: |
-          set -ex
-          kubectl cluster-info
-          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
-          NODES=$(kubectl get node | wc -l)
-          for ((i=1;i<10;i++));
-          do
-            set +e
-            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-            set -e
-            if [ "$PODS" -eq "$NODES" ]; then
-              break
-            fi
-            sleep 3
-          done
-          set +e
-          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-          kubectl get node -o yaml
-          kubectl get all -n kruise-system -o yaml
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
-          set -e
-          if [ "$PODS" -eq "$NODES" ]; then
-            echo "Wait for kruise-manager and kruise-daemon ready successfully"
-          else
-            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
-            exit 1
-          fi
-      - name: Run E2E Tests
-        run: |
-          export KUBECONFIG=/home/runner/.kube/config
-          make ginkgo
-          set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] SidecarSet' test/e2e
-          retVal=$?
-          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
-          if [ "${restartCount}" -eq "0" ];then
-              echo "Kruise-manager has not restarted"
-          else
-              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
-              echo "Kruise-manager has restarted, abort!!!"
-              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
-              exit 1
-          fi
-          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
-          do
-            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
-            if [ "${restartCount}" -eq "0" ];then
-                echo "Kruise-daemon has not restarted"
-            else
-                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
-                echo "Kruise-daemon has restarted, abort!!!"
-                kubectl logs -p -n ${ns} ${name}
-                exit 1
-            fi
-          done
-          exit $retVal
-
-  podUnavailableBudget:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
-        with:
-          node_image: ${{ env.KIND_IMAGE }}
-          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
-          config: ./test/kind-conf.yaml
-          version: ${{ env.KIND_VERSION }}
-      - name: Build image
-        run: |
-          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
-          docker build --pull --no-cache . -t $IMAGE
-          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
-      - name: Install Kruise
-        run: |
-          set -ex
-          kubectl cluster-info
-          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
-          NODES=$(kubectl get node | wc -l)
-          for ((i=1;i<10;i++));
-          do
-            set +e
-            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-            set -e
-            if [ "$PODS" -eq "$NODES" ]; then
-              break
-            fi
-            sleep 3
-          done
-          set +e
-          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-          kubectl get node -o yaml
-          kubectl get all -n kruise-system -o yaml
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
-          set -e
-          if [ "$PODS" -eq "$NODES" ]; then
-            echo "Wait for kruise-manager and kruise-daemon ready successfully"
-          else
-            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
-            exit 1
-          fi
-      - name: Run E2E Tests
-        run: |
-          export KUBECONFIG=/home/runner/.kube/config
-          make ginkgo
-          set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[policy\] PodUnavailableBudget' test/e2e
-
-  other:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
-        with:
-          node_image: ${{ env.KIND_IMAGE }}
-          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
-          config: ./test/kind-conf.yaml
-          version: ${{ env.KIND_VERSION }}
-      - name: Build image
-        run: |
-          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
-          docker build --pull --no-cache . -t $IMAGE
-          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
-      - name: Install Kruise
-        run: |
-          set -ex
-          kubectl cluster-info
-          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
-          NODES=$(kubectl get node | wc -l)
-          for ((i=1;i<10;i++));
-          do
-            set +e
-            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-            set -e
-            if [ "$PODS" -eq "$NODES" ]; then
-              break
-            fi
-            sleep 3
-          done
-          set +e
-          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-          kubectl get node -o yaml
-          kubectl get all -n kruise-system -o yaml
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
-          set -e
-          if [ "$PODS" -eq "$NODES" ]; then
-            echo "Wait for kruise-manager and kruise-daemon ready successfully"
-          else
-            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
-            exit 1
-          fi
-      - name: Run E2E Tests
-        run: |
-          export KUBECONFIG=/home/runner/.kube/config
-          make ginkgo
-          set +e
-          ./bin/ginkgo -timeout 90m -v --skip='\[apps\] (StatefulSet|PullImage|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip='\[policy\] PodUnavailableBudget' test/e2e
-          retVal=$?
-          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
-          if [ "${restartCount}" -eq "0" ];then
-              echo "Kruise-manager has not restarted"
-          else
-              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
-              echo "Kruise-manager has restarted, abort!!!"
-              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
-              exit 1
-          fi
-          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
-          do
-            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
-            if [ "${restartCount}" -eq "0" ];then
-                echo "Kruise-daemon has not restarted"
-            else
-                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
-                echo "Kruise-daemon has restarted, abort!!!"
-                kubectl logs -p -n ${ns} ${name}
-                exit 1
-            fi
-          done
-          exit $retVal

.github/workflows/e2e-1.20.yaml

@@ -1,99 +0,0 @@
-name: E2E-1.20
-
-on:
-  push:
-    branches:
-      - master
-      - release-*
-  pull_request: {}
-  workflow_dispatch: {}
-
-env:
-  # Common versions
-  GO_VERSION: '1.18'
-  KIND_VERSION: 'v0.14.0'
-  KIND_IMAGE: 'kindest/node:v1.20.7'
-  KIND_CLUSTER_NAME: 'ci-testing'
-
-jobs:
-  ephemeraljob:
-    runs-on: ubuntu-18.04
-    steps:
-      - uses: actions/checkout@v3
-        with:
-          submodules: true
-      - name: Setup Go
-        uses: actions/setup-go@v3
-        with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
-        with:
-          node_image: ${{ env.KIND_IMAGE }}
-          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
-          config: ./test/kind-conf.yaml
-          version: ${{ env.KIND_VERSION }}
-      - name: Build image
-        run: |
-          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
-          docker build --pull --no-cache . -t $IMAGE
-          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
-      - name: Install Kruise
-        run: |
-          set -ex
-          kubectl cluster-info
-          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
-          NODES=$(kubectl get node | wc -l)
-          for ((i=1;i<10;i++));
-          do
-            set +e
-            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-            set -e
-            if [ "$PODS" -eq "$NODES" ]; then
-              break
-            fi
-            sleep 3
-          done
-          set +e
-          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
-          kubectl get node -o yaml
-          kubectl get all -n kruise-system -o yaml
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
-          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
-          set -e
-          if [ "$PODS" -eq "$NODES" ]; then
-            echo "Wait for kruise-manager and kruise-daemon ready successfully"
-          else
-            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
-            exit 1
-          fi
-      - name: Run E2E Tests
-        run: |
-          export KUBECONFIG=/home/runner/.kube/config
-          make ginkgo
-          set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] EphemeralJob' test/e2e
-          retVal=$?
-          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
-          if [ "${restartCount}" -eq "0" ];then
-              echo "$out"
-              echo "Kruise-manager has not restarted"
-          else
-              echo "$out"
-              echo "Kruise-manager has restarted, abort!!!"
-              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
-              exit 1
-          fi
-          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
-          do
-            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
-            if [ "${restartCount}" -eq "0" ];then
-                echo "Kruise-daemon has not restarted"
-            else
-                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
-                echo "Kruise-daemon has restarted, abort!!!"
-                kubectl logs -p -n ${ns} ${name}
-                exit 1
-            fi
-          done
-          exit $retVal

.github/workflows/e2e-1.24.yaml

@@ -8,28 +8,113 @@ on:
   pull_request: {}
   workflow_dispatch: {}
 
+# Declare default permissions as read only.
+permissions: read-all
+
 env:
   # Common versions
-  GO_VERSION: '1.18'
+  GO_VERSION: '1.23'
   KIND_ACTION_VERSION: 'v1.3.0'
   KIND_VERSION: 'v0.14.0'
-  KIND_IMAGE: 'kindest/node:v1.24.2'
+  KIND_IMAGE: 'kindest/node:v1.24.6'
   KIND_CLUSTER_NAME: 'ci-testing'
 
 jobs:
-
+  astatefulset-storage:
-  astatefulset:
+    runs-on: ubuntu-24.04
-    runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "$NODES" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-system -o yaml
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
+          set -e
+          if [ "$PODS" -eq "$NODES" ]; then
+            echo "Wait for kruise-manager and kruise-daemon ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -p -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')         
+          fi
+          exit $retVal
+
+  astatefulset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
         with:
           node_image: ${{ env.KIND_IMAGE }}
           cluster_name: ${{ env.KIND_CLUSTER_NAME }}
@@ -85,20 +170,30 @@ jobs:
               kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
               exit 1
           fi
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')             
+          fi
           exit $retVal
 
   pullimages-containerrecreate:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
         with:
           node_image: ${{ env.KIND_IMAGE }}
           cluster_name: ${{ env.KIND_CLUSTER_NAME }}
@@ -143,7 +238,7 @@ jobs:
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
           set +e
-          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] (PullImage|ContainerRecreateRequest)' test/e2e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] (PullImage|ContainerRecreateRequest|PullImages)' test/e2e
           retVal=$?
           restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
@@ -166,20 +261,31 @@ jobs:
                 exit 1
             fi
           done
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')            
+          fi
+          
           exit $retVal
 
   advanced-daemonset:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
         with:
           node_image: ${{ env.KIND_IMAGE }}
           cluster_name: ${{ env.KIND_CLUSTER_NAME }}
@@ -247,20 +353,31 @@ jobs:
                 exit 1
             fi
           done
+          
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')          
+          fi
           exit $retVal
 
   sidecarset:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
         with:
           node_image: ${{ env.KIND_IMAGE }}
           cluster_name: ${{ env.KIND_CLUSTER_NAME }}
@@ -328,20 +445,31 @@ jobs:
                 exit 1
             fi
           done
+          
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')             
+          fi
           exit $retVal
 
   ephemeraljob:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
         with:
           node_image: ${{ env.KIND_IMAGE }}
           cluster_name: ${{ env.KIND_CLUSTER_NAME }}
@@ -387,19 +515,31 @@ jobs:
           make ginkgo
           set +e
           ./bin/ginkgo -timeout 60m -v --focus='\[apps\] EphemeralJob' test/e2e
+          retVal=$?
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')           
+          fi
+          exit $retVal
 
   podUnavailableBudget:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
         with:
           node_image: ${{ env.KIND_IMAGE }}
           cluster_name: ${{ env.KIND_CLUSTER_NAME }}
@@ -445,19 +585,26 @@ jobs:
           make ginkgo
           set +e
           ./bin/ginkgo -timeout 60m -v --focus='\[policy\] PodUnavailableBudget' test/e2e
-
+          retVal=$?
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}')             
+          fi
+          exit $retVal
   other:
-    runs-on: ubuntu-18.04
+    runs-on: ubuntu-24.04
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
         with:
           submodules: true
       - name: Setup Go
-        uses: actions/setup-go@v3
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
         with:
           go-version: ${{ env.GO_VERSION }}
       - name: Setup Kind Cluster
-        uses: helm/kind-action@v1.3.0
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
         with:
           node_image: ${{ env.KIND_IMAGE }}
           cluster_name: ${{ env.KIND_CLUSTER_NAME }}
@@ -471,6 +618,8 @@ jobs:
       - name: Install Kruise
         run: |
           set -ex
+          kubectl create ns kruise-system
+          kubectl apply -f test/kruise-e2e-config.yaml
           kubectl cluster-info
           IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
           NODES=$(kubectl get node | wc -l)
@@ -502,7 +651,8 @@ jobs:
           export KUBECONFIG=/home/runner/.kube/config
           make ginkgo
           set +e
-          ./bin/ginkgo -timeout 90m -v --skip='\[apps\] (StatefulSet|PullImage|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip='\[policy\] PodUnavailableBudget' test/e2e
+          kubectl apply -f https://raw.githubusercontent.com/kubeflow/training-operator/refs/heads/v1.8-branch/manifests/base/crds/kubeflow.org_tfjobs.yaml
+          ./bin/ginkgo -timeout 90m -v --skip='\[apps\] (AppStatefulSetStorage|StatefulSet|PullImage|PullImages|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip='\[policy\] PodUnavailableBudget' test/e2e
           retVal=$?
           restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
           if [ "${restartCount}" -eq "0" ];then
@@ -525,4 +675,15 @@ jobs:
                 exit 1
             fi
           done
+          if [ "${retVal}" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')           
+          fi
+
           exit $retVal

.github/workflows/e2e-1.26.yaml

@@ -0,0 +1,688 @@
+name: E2E-1.26
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  # Common versions
+  GO_VERSION: '1.23'
+  KIND_VERSION: 'v0.18.0'
+  KIND_IMAGE: 'kindest/node:v1.26.3'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+  astatefulset-storage:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+         make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "$NODES" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-system -o yaml
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
+          set -e
+          if [ "$PODS" -eq "$NODES" ]; then
+            echo "Wait for kruise-manager and kruise-daemon ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -p -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')         
+          fi
+          exit $retVal
+
+  astatefulset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "$NODES" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-system -o yaml
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
+          set -e
+          if [ "$PODS" -eq "$NODES" ]; then
+            echo "Wait for kruise-manager and kruise-daemon ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] StatefulSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')         
+          fi
+          exit $retVal
+
+  pullimages-containerrecreate:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "$NODES" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-system -o yaml
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
+          set -e
+          if [ "$PODS" -eq "$NODES" ]; then
+            echo "Wait for kruise-manager and kruise-daemon ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] (PullImage|ContainerRecreateRequest|PullImages)' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')         
+          fi
+          
+          exit $retVal
+
+  advanced-daemonset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "$NODES" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-system -o yaml
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
+          set -e
+          if [ "$PODS" -eq "$NODES" ]; then
+            echo "Wait for kruise-manager and kruise-daemon ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] DaemonSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
+          
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')         
+          fi
+          exit $retVal
+
+  sidecarset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "$NODES" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-system -o yaml
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
+          set -e
+          if [ "$PODS" -eq "$NODES" ]; then
+            echo "Wait for kruise-manager and kruise-daemon ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] SidecarSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
+          
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')           
+          fi
+          exit $retVal
+
+  ephemeraljob:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "$NODES" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-system -o yaml
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
+          set -e
+          if [ "$PODS" -eq "$NODES" ]; then
+            echo "Wait for kruise-manager and kruise-daemon ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] EphemeralJob' test/e2e
+          retVal=$?
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')             
+          fi
+          exit $retVal
+
+  podUnavailableBudget:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl cluster-info
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "$NODES" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-system -o yaml
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
+          set -e
+          if [ "$PODS" -eq "$NODES" ]; then
+            echo "Wait for kruise-manager and kruise-daemon ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[policy\] PodUnavailableBudget' test/e2e
+          retVal=$?
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}')             
+          fi
+          exit $retVal
+  other:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          set -ex
+          kubectl create ns kruise-system
+          kubectl apply -f test/kruise-e2e-config.yaml
+          kubectl cluster-info
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} ./scripts/deploy_kind.sh
+          NODES=$(kubectl get node | wc -l)
+          for ((i=1;i<10;i++));
+          do
+            set +e
+            PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+            set -e
+            if [ "$PODS" -eq "$NODES" ]; then
+              break
+            fi
+            sleep 3
+          done
+          set +e
+          PODS=$(kubectl get pod -n kruise-system | grep '1/1' | wc -l)
+          kubectl get node -o yaml
+          kubectl get all -n kruise-system -o yaml
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system
+          kubectl get pod -n kruise-system --no-headers | grep daemon | awk '{print $1}' | xargs kubectl logs -n kruise-system --previous=true
+          set -e
+          if [ "$PODS" -eq "$NODES" ]; then
+            echo "Wait for kruise-manager and kruise-daemon ready successfully"
+          else
+            echo "Timeout to wait for kruise-manager and kruise-daemon ready"
+            exit 1
+          fi
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          kubectl apply -f https://raw.githubusercontent.com/kubeflow/training-operator/refs/heads/v1.8-branch/manifests/base/crds/kubeflow.org_tfjobs.yaml
+          ./bin/ginkgo -timeout 90m -v --skip='\[apps\] (AppStatefulSetStorage|StatefulSet|PullImage|PullImages|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip='\[policy\] PodUnavailableBudget' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
+          if [ "${retVal}" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')             
+          fi
+
+          exit $retVal

.github/workflows/e2e-1.28.yaml

@@ -0,0 +1,535 @@
+name: E2E-1.28
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  # Common versions
+  GO_VERSION: '1.23'
+  KIND_VERSION: 'v0.22.0'
+  KIND_IMAGE: 'kindest/node:v1.28.7'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+  astatefulset-storage:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -p -v --focus='\[apps\] AppStatefulSetStorage' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')         
+          fi
+          exit $retVal
+
+  astatefulset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] StatefulSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')         
+          fi
+          exit $retVal
+
+  pullimages-containerrecreate:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] (PullImage|ContainerRecreateRequest|PullImages)' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')         
+          fi
+          
+          exit $retVal
+
+  advanced-daemonset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] DaemonSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
+          
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')         
+          fi
+          exit $retVal
+
+  sidecarset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          DISABLE_E2E_CONFIG=true IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] SidecarSet' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
+          
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')           
+          fi
+          exit $retVal
+
+  ephemeraljob:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[apps\] EphemeralJob' test/e2e
+          retVal=$?
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')             
+          fi
+          exit $retVal
+
+  podUnavailableBudget:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -timeout 60m -v --focus='\[policy\] PodUnavailableBudget' test/e2e
+          retVal=$?
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}')             
+          fi
+          exit $retVal
+  clonesetAndInplace:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          ./bin/ginkgo -p -timeout 120m -v --focus='\[apps\] (InplaceVPA)' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          if [ "$retVal" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')         
+          fi
+          exit $retVal
+
+  other:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          make ginkgo
+          set +e
+          kubectl apply -f https://raw.githubusercontent.com/kubeflow/training-operator/refs/heads/v1.8-branch/manifests/base/crds/kubeflow.org_tfjobs.yaml
+          ./bin/ginkgo -timeout 90m -v --skip='\[apps\] (InplaceVPA|AppStatefulSetStorage|StatefulSet|PullImage|PullImages|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip='\[policy\] PodUnavailableBudget' test/e2e
+          retVal=$?
+          restartCount=$(kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $4}')
+          if [ "${restartCount}" -eq "0" ];then
+              echo "Kruise-manager has not restarted"
+          else
+              kubectl get pod -n kruise-system -l control-plane=controller-manager --no-headers
+              echo "Kruise-manager has restarted, abort!!!"
+              kubectl get pod -n kruise-system --no-headers -l control-plane=controller-manager | awk '{print $1}' | xargs kubectl logs -p -n kruise-system
+              exit 1
+          fi
+          kubectl get pods -n kruise-system -l control-plane=daemon -o=jsonpath="{range .items[*]}{.metadata.namespace}{\"\t\"}{.metadata.name}{\"\n\"}{end}" | while read ns name;
+          do
+            restartCount=$(kubectl get pod -n ${ns} ${name} --no-headers | awk '{print $4}')
+            if [ "${restartCount}" -eq "0" ];then
+                echo "Kruise-daemon has not restarted"
+            else
+                kubectl get pods -n ${ns} -l control-plane=daemon --no-headers
+                echo "Kruise-daemon has restarted, abort!!!"
+                kubectl logs -p -n ${ns} ${name}
+                exit 1
+            fi
+          done
+          if [ "${retVal}" -ne 0 ];then
+              echo "test fail, dump kruise-manager logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=controller-manager --no-headers | awk '{print $1}') 
+              echo "test fail, dump kruise-daemon logs"
+              while read pod; do 
+                   kubectl logs -n kruise-system $pod
+              done < <(kubectl get pods -n kruise-system -l control-plane=daemon --no-headers | awk '{print $1}')             
+          fi
+
+          exit $retVal

.github/workflows/e2e-1.30.yaml

@@ -0,0 +1,298 @@
+name: E2E-1.30
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  # Common versions
+  GO_VERSION: '1.23'
+  KIND_VERSION: 'v0.22.0'
+  KIND_IMAGE: 'kindest/node:v1.30.8'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+  astatefulset-storage:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] AppStatefulSetStorage' --print-info
+
+  astatefulset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] StatefulSet' --print-info
+
+  pullimages-containerrecreate:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] (PullImage|ContainerRecreateRequest|PullImages)' --print-info --disable-parallel
+
+  advanced-daemonset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] DaemonSet' --print-info
+
+  sidecarset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          DISABLE_E2E_CONFIG=true IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] SidecarSet' --print-info
+
+  ephemeraljob:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] EphemeralJob' --print-info
+
+  podUnavailableBudget:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[policy\] PodUnavailableBudget' --print-info
+
+  clonesetAndInplace:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] (CloneSet|InplaceVPA)' --print-info
+
+  other:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus "" --skip '\[apps\] (CloneSet|InplaceVPA|AppStatefulSetStorage|StatefulSet|PullImage|PullImages|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip '\[policy\] PodUnavailableBudget' --timeout 90m --print-info

.github/workflows/e2e-1.32.yaml

@@ -0,0 +1,298 @@
+name: E2E-1.32
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  # Common versions
+  GO_VERSION: '1.23'
+  KIND_VERSION: 'v0.22.0'
+  KIND_IMAGE: 'kindest/node:v1.32.0'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+  astatefulset-storage:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] AppStatefulSetStorage' --print-info
+
+  astatefulset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] StatefulSet' --print-info
+
+  pullimages-containerrecreate:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] (PullImage|ContainerRecreateRequest|PullImages)' --print-info --disable-parallel
+
+  advanced-daemonset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] DaemonSet' --print-info
+
+  sidecarset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          DISABLE_E2E_CONFIG=true IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] SidecarSet' --print-info
+
+  ephemeraljob:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] EphemeralJob' --print-info
+
+  podUnavailableBudget:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[policy\] PodUnavailableBudget' --print-info
+
+  clonesetAndInplace:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '\[apps\] (CloneSet|InplaceVPA)' --print-info
+
+  other:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus "" --skip '\[apps\] (CloneSet|InplaceVPA|AppStatefulSetStorage|StatefulSet|PullImage|PullImages|ContainerRecreateRequest|DaemonSet|SidecarSet|EphemeralJob)' --skip '\[policy\] PodUnavailableBudget' --timeout 90m --print-info

.github/workflows/license.yml

@@ -10,14 +10,17 @@ on:
       - master
       - release-*
 
+# Declare default permissions as read only.
+permissions: read-all
+
 jobs:
   license_check:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-24.04
     name: Check for unapproved licenses
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
       - name: Set up Ruby
-        uses: ruby/setup-ruby@v1
+        uses: ruby/setup-ruby@a4effe49ee8ee5b8b5091268c473a4628afb5651 # v1.245.0
         with:
           ruby-version: 2.6
       - name: Install dependencies

.github/workflows/scorecard.yml

@@ -0,0 +1,72 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '30 14 * * *'
+  push:
+    branches: [ "master" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@ce28f5bb42b7a9f2c824e633a3f6ee835bab6858 # v2.25.0
+        with:
+          sarif_file: results.sarif

.gitignore

@@ -28,3 +28,5 @@ test/e2e/generated/bindata.go
 .vscode
 
 .DS_Store
+
+vendor/

.golangci.yml

@@ -1,86 +1,73 @@
-# options for analysis running
+version: "2"
 run:
-  # default concurrency is a available CPU number
   concurrency: 4
-
-  # timeout for analysis, e.g. 30s, 5m, default is 1m
-  deadline: 5m
-
-  # exit code when at least one issue was found, default is 1
   issues-exit-code: 1
-
-  # include test files or not, default is true
   tests: true
 
-  # list of build tags, all linters use it. Default is empty list.
-  #build-tags:
-  #  - mytag
-
-  # which dirs to skip: they won't be analyzed;
-  # can use regexp here: generated.*, regexp is applied on full path;
-  # default value is empty list, but next dirs are always skipped independently
-  # from this option's value:
-  # third_party$, testdata$, examples$, Godeps$, builtin$
-  skip-dirs:
-    - apis
-    - pkg/client
-    - vendor
-    - test
-
-  # which files to skip: they will be analyzed, but issues from them
-  # won't be reported. Default value is empty list, but there is
-  # no need to include all autogenerated files, we confidently recognize
-  # autogenerated files. If it's not please let us know.
-  skip-files:
-  #  - ".*\\.my\\.go$"
-  #  - lib/bad.go
 
 # output configuration options
 output:
-  # colored-line-number|line-number|json|tab|checkstyle, default is "colored-line-number"
+  formats:
-  format: colored-line-number
+    text:
-
+      path: stdout
-  # print lines of code with issue, default is true
+      colors: true
-  print-issued-lines: true
-
-  # print linter name in the end of issue text, default is true
-  print-linter-name: true
-
-
-# all available settings of specific linters
-linters-settings:
-  golint:
-    # minimal confidence for issues, default is 0.8
-    min-confidence: 0.8
-  gofmt:
-    # simplify code: gofmt with `-s` option, true by default
-    simplify: true
-  goimports:
-    # put imports beginning with prefix after 3rd-party packages;
-    # it's a comma-separated list of prefixes
-    #local-prefixes: github.com/openkruise/kruise
-  misspell:
-    # Correct spellings using locale preferences for US or UK.
-    # Default is to use a neutral variety of English.
-    # Setting locale to US will correct the British spelling of 'colour' to 'color'.
-    locale: default
-    #ignore-words:
-    #  - someword
-
 linters:
-  fast: false
+  default: none
-  disable-all: true
   enable:
-    # TODO Enforce the below linters later
+    - depguard
-    - gofmt
     - govet
-    - goimports
     - ineffassign
     - misspell
-    - vet
     - unconvert
     - unused
-issues:
+  settings:
-  exclude:
+    misspell:
-    # staticcheck
+      # Correct spellings using locale preferences for US or UK.
-    - 'SA1019: package github.com/golang/protobuf/proto is deprecated: Use the "google.golang.org/protobuf/proto" package instead'
+      # Default is to use a neutral variety of English.
+      # Setting locale to US will correct the British spelling of 'colour' to 'color'.
+      locale: US
+    depguard:
+      rules:
+        forbid-pkg-errors:
+          deny:
+          - pkg: "github.com/pkg/errors"
+            desc: Should be replaced with standard lib errors or fmt.Errorf
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    rules:
+      - path: (.+)\.go$
+        text: 'SA1019: package github.com/golang/protobuf/proto is deprecated: Use the "google.golang.org/protobuf/proto" package instead'
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+      - apis
+      - pkg/client
+      - vendor
+      - test
+formatters:
+  enable:
+  - gofmt
+  - goimports
+  settings:
+    gofmt:
+      simplify: true
+    goimports:
+      # put imports beginning with prefix after 3rd-party packages;
+      local-prefixes: 
+      - github.com/openkruise/kruise
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+      - apis
+      - pkg/client
+      - vendor
+      - test

CHANGELOG.md

@@ -1,5 +1,350 @@
 # Change Log
 
+## v1.8.2
+> Change log since v1.8.1
+
+### Bug fixes
+- Fix kruise-daemon panic exception due to PodProbeMarker when container is nil. ([#1974](https://github.com/openkruise/kruise/pull/1974), [@zmberg](https://github.com/zmberg))
+
+## v1.8.1
+> Change log since v1.8.0
+
+### Bug fixes
+- JobSidecarTerminator support ignore exit code capability via env. ([#1949](https://github.com/openkruise/kruise/pull/1949), [@zmberg](https://github.com/zmberg))
+
+### Performance Improvements
+- Performance optimized PodProbeMarker to reduce many invalid patch operations ([#2007](https://github.com/openkruise/kruise/pull/2007), [@zmberg](https://github.com/zmberg))
+
+## v1.7.4
+> Change log since v1.7.3
+
+### Bug fixes
+- JobSidecarTerminator support ignore exit code capability via env. ([#1949](https://github.com/openkruise/kruise/pull/1949), [@zmberg](https://github.com/zmberg))
+
+## v1.8.0
+
+> Change log since v1.7.3
+### Upgrade Notice
+> No, really, you must read this before you upgrade
+- **Disable** the following feature gates by default: ResourcesDeletionProtection ([#1919](https://github.com/openkruise/kruise/pull/1919), [@ABNER-1](https://github.com/ABNER-1))
+- Promote these feature gates to beta:  
+  `ResourcesDeletionProtection`, `WorkloadSpread`, `PodUnavailableBudgetDeleteGate`, `InPlaceUpdateEnvFromMetadata`,
+  `StatefulSetAutoDeletePVC`,
+  `PodProbeMarkerGate` ([#1919](https://github.com/openkruise/kruise/pull/1919), [@ABNER-1](https://github.com/ABNER-1))
+- Update Kubernetes dependency to v1.30.10 and Golang to v1.22 ([#1896](https://github.com/openkruise/kruise/pull/1896), [@ABNER-1](https://github.com/ABNER-1), [#1924](https://github.com/openkruise/kruise/pull/1924), [@furykerry](https://github.com/furykerry)))
+- Prior to Kruise 1.7.3, `helm uninstall` is a **high-risk** operation that deletes Kruise, its CRDs, and associated CRs. Starting from Kruise 1.7.3, it uses a pre-delete hook to check for existing Kruise CRs before uninstallation and blocks the process to prevent accidental deletion.
+
+### Key Features
+- Support in-place expansion of StatefulSet volumes ([#1674](https://github.com/openkruise/kruise/pull/1674), [#1714](https://github.com/openkruise/kruise/pull/1714), [@ABNER-1](https://github.com/ABNER-1))
+- Enable in-place resource resizing for CloneSet, Advanced StatefulSet, and Advanced DaemonSet ([#1353](https://github.com/openkruise/kruise/pull/1353), [#1866](https://github.com/openkruise/kruise/pull/1866), [@LavenderQAQ](https://github.com/LavenderQAQ), [@ABNER-1](https://github.com/ABNER-1))
+- Support adaptive scheduling strategy for UnitedDeployment ([#1720](https://github.com/openkruise/kruise/pull/1720),  [@AiRanthem](https://github.com/AiRanthem))
+- Add WorkloadSpread support for AI workload like TFJob in KubeFlow ([#1838](https://github.com/openkruise/kruise/pull/1838), [@AiRanthem](https://github.com/AiRanthem))
+
+### Performance Improvements
+- Optimize CA bundle updates to reduce unnecessary changes ([#1717](https://github.com/openkruise/kruise/pull/1717), [@zmberg](https://github.com/zmberg))
+- Add disableDeepCopy for BroadcastJob ([#1696](https://github.com/openkruise/kruise/pull/1696), [@Prepmachine4](https://github.com/Prepmachine4))
+
+### Resilience Enhancement
+- Add Helm pre-delete hook to preserve Kruise CRs during uninstallation ([#1843](https://github.com/openkruise/kruise/pull/1843), [@AiRanthem](https://github.com/AiRanthem))
+
+### Other Notable Changes
+#### Advanced Workload
+- Add lifecycle hooks and tests for Advanced StatefulSet ([#1858](https://github.com/openkruise/kruise/pull/1858), [@mingzhou.swx](https://github.com/mingzhou.swx), [@ABNER-1](https://github.com/ABNER-1))
+- Add range-based reserveOrdinals support for Advanced StatefulSet ([#1873](https://github.com/openkruise/kruise/pull/1873), [@AiRanthem](https://github.com/AiRanthem))
+- Redefined partition semantics to represent non-updated pod count ([#1819](https://github.com/openkruise/kruise/pull/1819), [@ABNER-1](https://github.com/ABNER-1); [#1751](https://github.com/openkruise/kruise/pull/1751), [@zybtakeit](https://github.com/zybtakeit), [@ABNER-1](https://github.com/ABNER-1))
+
+#### Sidecar Management
+- Support inject both stable and updated version sidecar according to updateStrategy ([#1689](https://github.com/openkruise/kruise/pull/1689), [#1856](https://github.com/openkruise/kruise/pull/1856), [@AiRanthem](https://github.com/AiRanthem)) 
+- Refine SidecarSet initContainer handling ([#1719](https://github.com/openkruise/kruise/pull/1719), [@zmberg](https://github.com/zmberg))
+
+#### Multi-domain management
+- Introduce `pub.kruise.io/disable-fetch-replicas-from-workload=true` annotation for CRD compatibility ([#1758](https://github.com/openkruise/kruise/pull/1758), [@zmberg](https://github.com/zmberg))
+- Extend PodProbeMarker to serverless pods ([#1875](https://github.com/openkruise/kruise/pull/1875), [@zmberg](https://github.com/zmberg))
+- Enable priorityClassName patching in WorkloadSpread ([#1877](https://github.com/openkruise/kruise/pull/1877), [@AiRanthem](https://github.com/AiRanthem))
+- Sync all fields in UnitedDeployment spec to subset workload spec ([#1798](https://github.com/openkruise/kruise/pull/1798), [@AiRanthem](https://github.com/AiRanthem))
+
+### Bug Fixes
+- Resolve token permission and dependency pinning issues ([#1707](https://github.com/openkruise/kruise/pull/1707),  [@harshitasao](https://github.com/harshitasao))
+- Fix PyTorchJob pod creation failures ([#1864](https://github.com/openkruise/kruise/pull/1864), [@zmberg](https://github.com/zmberg))
+- Correct ImagePullJob timeout handling (>1800s) ([#1874](https://github.com/openkruise/kruise/pull/1874), [@zmberg](https://github.com/zmberg))
+- Resolve cri-dockerd runtime detection issues ([#1899](https://github.com/openkruise/kruise/pull/1899), [@FlikweertvisionVadym](https://github.com/FlikweertvisionVadym))
+- Remove pod ownerRef requirement in pub webhook ([#1869](https://github.com/openkruise/kruise/pull/1869), [@zmberg](https://github.com/zmberg))
+- Address maxUnavailable blocking in SidecarSet updates ([#1834](https://github.com/openkruise/kruise/pull/1834), [@zmberg](https://github.com/zmberg))
+- Fix CloneSet controller block from scale expectation leaks ([#1829](https://github.com/openkruise/kruise/pull/1829), [@zmberg](https://github.com/zmberg))
+- Enforce imagePullPolicy=Always for ImagePullJob ([#1830](https://github.com/openkruise/kruise/pull/1830), [@zmberg](https://github.com/zmberg))
+- Fix WorkloadSpread webhook panics ([#1807](https://github.com/openkruise/kruise/pull/1807), [@AiRanthem](https://github.com/AiRanthem))
+
+### Misc (Chores and tests)
+- Standardize on CRI for image pulls ([#1867](https://github.com/openkruise/kruise/pull/1867), [@furykerry](https://github.com/furykerry))
+- Introduce JSON log formatting ([#1703](https://github.com/openkruise/kruise/pull/1703), [@zmberg](https://github.com/zmberg))
+- Remove Docker runtime dependency ([#1870](https://github.com/openkruise/kruise/pull/1870),[@furykerry](https://github.com/furykerry))
+- Improve test parallelism and reliability ([#1743](https://github.com/openkruise/kruise/pull/1743), [@MichaelRren](https://github.com/MichaelRren))
+- Enhance WorkloadSpread validation logic ([#1740](https://github.com/openkruise/kruise/pull/1740), [@AiRanthem](https://github.com/AiRanthem))
+- Launche Kruise Guru on Gurubase.io ([#1800](https://github.com/openkruise/kruise/pull/1800), [@kursataktas](https://github.com/kursataktas))
+- Improve documentation accuracy ([#1824](https://github.com/openkruise/kruise/pull/1824), [@furykerry](https://github.com/furykerry))
+- Fix KIND installation issues ([#1688](https://github.com/openkruise/kruise/pull/1688),[@ABNER-1](https://github.com/ABNER-1))
+- Avoid overriding namespace config after deploying ([#1772](https://github.com/openkruise/kruise/pull/1772),[@hantmac](https://github.com/hantmac))
+- Fix WorkloadSpread test flakiness by removing dependencies ([#1895](https://github.com/openkruise/kruise/pull/1895), [@AiRanthem](https://github.com/AiRanthem))
+- Address SidecarSet e2e test failures ([#1724](https://github.com/openkruise/kruise/pull/1724), [@zmberg](https://github.com/zmberg))
+- Enhance unit test stability ([#1784](https://github.com/openkruise/kruise/pull/1784), [@AiRanthem](https://github.com/AiRanthem))
+
+## v1.7.3
+> Change log since v1.7.2
+
+### Bug fixes
+- Fix kubeflow PyTorchJob create pod failure due to pod webhook. ([#1734](https://github.com/openkruise/kruise/pull/1864), [@zmberg](https://github.com/zmberg))
+
+## v1.7.2
+> Change log since v1.7.1
+
+### Advanced Workload
+- Support specified-delete in AdvancedStatefulSet and handle specified deleted pod under maxUnavailable constrain. ([#1734](https://github.com/openkruise/kruise/pull/1734), [@ABNER-1](https://github.com/ABNER-1))
+
+## v1.6.4
+> Change log since v1.6.3
+
+### Advanced Workload
+- Support specified-delete in AdvancedStatefulSet and handle specified deleted pod under maxUnavailable constrain. ([#1734](https://github.com/openkruise/kruise/pull/1734), [@ABNER-1](https://github.com/ABNER-1))
+
+## v1.5.5
+> Chang log since v1.5.4
+
+### Advanced Workload
+- Support specified-delete in AdvancedStatefulSet and handle specified deleted pod under maxUnavailable constrain. ([#1734](https://github.com/openkruise/kruise/pull/1734), [@ABNER-1](https://github.com/ABNER-1))
+- Advanced StatefulSet maxUnavailable now counts unavailable pods with smaller ordinal in the update order during rolling upgrade. ([#1480](https://github.com/openkruise/kruise/pull/1480), [@Yesphet](https://github.com/Yesphet))
+
+## v1.7.1
+> Change log since v1.7.0
+
+### Bug fixes
+- When update crd webhook caBundle, if caBundle does not change, do not update crd again. ([#1717](https://github.com/openkruise/kruise/pull/1717), [@zmberg](https://github.com/zmberg))
+- Remove normal init container in pod's sidecarSet in-place update annotation. ([#1719](https://github.com/openkruise/kruise/pull/1719), [@zmberg](https://github.com/zmberg))
+
+## v1.7.0
+> Change log since v1.6.3
+
+### Key Features
+- When CloneSet volumeClaimTemplates changed, always recreate pods and related volumes. ([#1561](https://github.com/openkruise/kruise/pull/1561), [@ABNER-1](https://github.com/ABNER-1))
+- Bump K8s dependency to 1.28, and OpenKruise still works with Kubernetes Version >= 1.18. ([#1598](https://github.com/openkruise/kruise/pull/1598), [@ABNER-1](https://github.com/ABNER-1))
+- SidecarSet support k8s 1.28 Sidecar Containers(initContainers[x].restartPolicy=Always), and significantly improves the lifecycle management of Sidecar containers,
+  refer to the [community documentation](https://kubernetes.io/docs/concepts/workloads/pods/sidecar-containers/) for details. ([#1613](https://github.com/openkruise/kruise/pull/1613), [@zmberg](https://github.com/zmberg))
+- ImagePullJob support for credential provider plugin, e.g. aws. ([#1383](https://github.com/openkruise/kruise/pull/1383), [@Kuromesi](https://github.com/Kuromesi))
+- Advanced StatefulSet support [start ordinal](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/#start-ordinal). ([#1643](https://github.com/openkruise/kruise/pull/1643), [@ABNER-1](https://github.com/ABNER-1))
+- Support webhook CA injection using external certification management tool, e.g. [cert-manager](https://cert-manager.io/). ([#1665](https://github.com/openkruise/kruise/pull/1665), [@Kuromesi](https://github.com/Kuromesi))
+- Kruise-daemon support cri-docker.sock for kubernetes clusters that use docker runtime. ([#1631](https://github.com/openkruise/kruise/pull/1631), [@BraceCY](https://github.com/BraceCY))
+- Advanced StatefulSet add pod index label `statefulset.kubernetes.io/pod-index`. ([#1667](https://github.com/openkruise/kruise/pull/1667), [@cr7258](https://github.com/cr7258))
+- Add Structured logging support. ([#1565](https://github.com/openkruise/kruise/pull/1565), [@MajLuu](https://github.com/MajLuu)); ([#1629](https://github.com/openkruise/kruise/pull/1629), [@jairuigou](https://github.com/jairuigou)); ([#1669](https://github.com/openkruise/kruise/pull/1669), [@AiRanthem](https://github.com/AiRanthem))
+
+### Performance Enhancement
+- Optimizing Pod SidecarSet webhook and controller performance when lots of namespace scoped sidecarSet exists ([#1547](https://github.com/openkruise/kruise/pull/1547), [@ls-2018](https://github.com/ls-2018))
+- Pod readiness controller use Patch instead of Update, thus reducing updating conflict when creating a large number of Pods. ([#1560](https://github.com/openkruise/kruise/pull/1560), [@BruceAko](https://github.com/BruceAko))
+
+### Bug fixes
+- Multi-domain Management
+  - Fixes workloadSpread validation message when using adaptive strategy type. ([#1553](https://github.com/openkruise/kruise/pull/1553), [@voron](https://github.com/voron))
+  - When feature-gate WorkloadSpread=false, the WorkloadSpread Controller is no longer started and the creation of workloadSpread CR is not allowed. ([#1566](https://github.com/openkruise/kruise/pull/1566), [@ls-2018](https://github.com/ls-2018))
+- Application Protection
+  - In some extreme scenarios, fix PodUnavailableBudget blocking KCM recycling Pods. ([#1567](https://github.com/openkruise/kruise/pull/1567), [@Spground](https://github.com/Spground))
+- Sidecar Container
+  - Fix SidecarSet invalid update status. ([#1641](https://github.com/openkruise/kruise/pull/1641), [@Spground](https://github.com/Spground))
+- Advanced Workload
+  - Fix potential nil panic in CloneSet validating webhook when Pod's controller owner ref is nil. ([#1678](https://github.com/openkruise/kruise/pull/1678), [@Spground](https://github.com/Spground))
+
+### Misc (cleanup and Flake)
+- Optimized Advanced StatefulSet code structure based on upstream community code(k8s 1.28). ([#1648](https://github.com/openkruise/kruise/pull/1648), [@ABNER-1](https://github.com/ABNER-1))
+- Reduce github workflow action permission. ([#1523](https://github.com/openkruise/kruise/pull/1523), [@furykerry](https://github.com/furykerry))
+- Bug fix for Makefile envtest failed. ([#1548](https://github.com/openkruise/kruise/pull/1548), [@BH4AWS](https://github.com/BH4AWS))
+- Fix UT TestRevisionManage. ([#1555](https://github.com/openkruise/kruise/pull/1555), [@furykerry](https://github.com/furykerry))
+- Upgrade opencontainers/runc (1.1.12) and controller-gen (0.14.0). ([#1562](https://github.com/openkruise/kruise/pull/1562), [@ppbits](https://github.com/ppbits))
+- Remove vendor directory. ([#1554](https://github.com/openkruise/kruise/pull/1554), [@liangyuanpeng](https://github.com/liangyuanpeng))
+- Add dependabot config for auto-update github-actions. ([#1570](https://github.com/openkruise/kruise/pull/1570), [@liangyuanpeng](https://github.com/liangyuanpeng))
+- Add permission of security-events write for ghaction golangci-lint. ([#1582](https://github.com/openkruise/kruise/pull/1582), [@liangyuanpeng](https://github.com/liangyuanpeng))
+- Fix vendor error while running command make docker-multiarch. ([#1601](https://github.com/openkruise/kruise/pull/1601), [@MichaelRren](https://github.com/MichaelRren))
+- Change e2e centos image from 6.7 to 7, then e2e can work on arm node. ([#1623](https://github.com/openkruise/kruise/pull/1623), [@Colvin-Y](https://github.com/Colvin-Y))
+- Fix slice declarations that are not initialized with zero length. ([#1628](https://github.com/openkruise/kruise/pull/1628), [@alingse](https://github.com/alingse))
+- Fix UT TestMatchRegistryAuths failed. ([#1583](https://github.com/openkruise/kruise/pull/1583), [@ABNER-1](https://github.com/ABNER-1))
+- Changes the scorecard badge link from old format to the Standard human-readable OpenSSF Scorecard Report. ([#1657](https://github.com/openkruise/kruise/pull/1657), [@harshitasao](https://github.com/harshitasao))
+
+## v1.6.3
+> Change log since v1.6.2
+
+### CVE FIX
+- fix potential security issues of dependent packages ([#1586](https://github.com/openkruise/kruise/pull/1586), [ABNER-1](https://github.com/ABNER-1)) ([#1591](https://github.com/openkruise/kruise/pull/1591), [ABNER-1](https://github.com/ABNER-1))
+
+## v1.6.2
+> Change log since v1.6.1
+
+### CloneSet
+- Fix new version of Pods released by cloneSet that doesn't match spec.updateStrategy.partition. ([#1549](https://github.com/openkruise/kruise/pull/1549), [@qswksp](https://github.com/qswksp))
+
+## v1.5.4
+> Chang log since v1.5.3
+
+### CloneSet
+- Fix new version of Pods released by cloneSet that doesn't match spec.updateStrategy.partition. ([#1549](https://github.com/openkruise/kruise/pull/1549), [@qswksp](https://github.com/qswksp))
+
+## v1.4.2
+> Change log since v1.4.1
+
+### CloneSet
+- Fix new version of Pods released by cloneSet that doesn't match spec.updateStrategy.partition. ([#1549](https://github.com/openkruise/kruise/pull/1549), [@qswksp](https://github.com/qswksp))
+
+## v1.6.1
+> Change log since v1.6.0
+
+### Upgrade Notice
+- FeatureGate PodWebhook=false will not disable ResourcesDeletionProtection. ([#1526](https://github.com/openkruise/kruise/pull/1526), [@zmberg](https://github.com/zmberg))
+- Update go.mod require k8s version from 1.29 to 1.26, and remove go mod replace. ([#1527](https://github.com/openkruise/kruise/pull/1527), [KaiShi](https://github.com/BH4AWS))
+
+### Advanced Workload
+- Fix when StatefulSet reserveOrdinals exist and whenScaled=Delete, scale down pvc failed. ([#1531](https://github.com/openkruise/kruise/pull/1531), [@zmberg](https://github.com/zmberg))
+
+## v1.5.3
+> Chang log since v1.5.2
+
+### Advanced Workload
+- Fix when StatefulSet reserveOrdinals exist and whenScaled=Delete, scale down pvc failed. ([#1531](https://github.com/openkruise/kruise/pull/1531), [@zmberg](https://github.com/zmberg))
+
+## v1.6.0
+> Change log since v1.5.2
+
+### Upgrade Notice
+
+> No, really, you must read this before you upgrade
+- OpenKruise no longer supports Kubernetes versions 1.16, 1.17.
+  However it's still possible to use OpenKruise with Kubernetes versions 1.16 and 1.17 as long as KruiseDaemon is not enabled(install/upgrade kruise charts with featureGates="KruiseDaemon=false")
+- Kruise-Daemon will no longer support v1alpha2 CRI runtimes.
+  However it's still possible to use OpenKruise on Kubernetes with nodes that only support v1alpha2 CRI as long as KruiseDaemon is not enabled(install/upgrade kruise charts with featureGates="KruiseDaemon=false")
+- OpenKruise leader election default to use leases mode. ([#1407](https://github.com/openkruise/kruise/pull/1407), [dsxing](https://github.com/dsxing))
+  For users with OpenKruise version 1.3.0 or lower, please first upgrade your OpenKruise to version 1.4 or 1.5 before upgrading to 1.6.0, so as to avoid unexpected multiple leader problem during the installation.
+- Bump Kubernetes dependency to 1.26.10. ([#1511](https://github.com/openkruise/kruise/pull/1511), [KaiShi](https://github.com/BH4AWS))
+- To avoid potential circular dependency problem, features rely on webhook will no longer work for resources under kube-system,
+  e.g. SidecarSet, WorkloadSpread, PodUnavailableBudget, ContainerLaunchPriority and PersistentPodState. ([#92](https://github.com/openkruise/charts/pull/92), [@hantmac](https://github.com/hantmac))
+
+### Key Features
+- Fix WorkloadSpread incorrect subset allocation after workload rolling updating. ([#1197](https://github.com/openkruise/kruise/pull/1197), [veophi](https://github.com/veophi))
+- ImagePullJob support force image pulling for images with the name as previous one. ([#1384](https://github.com/openkruise/kruise/pull/1384), [ls-2018](https://github.com/ls-2018))
+- Job Sidecar Terminator reports correct pod phase for sidecar containers with non-zero exit code. ([#1303](https://github.com/openkruise/kruise/pull/1303), [@diannaowa](https://github.com/diannaowa))
+- Support the deletion protection of service and ingress resources. ([#1269](https://github.com/openkruise/kruise/pull/1269), [@kevin1689-cloud](https://github.com/kevin1689-cloud))
+
+### Performance Enhancement
+- Optimize PodProbeMarker performance. ([#1430](https://github.com/openkruise/kruise/pull/1430), [ls-2018](https://github.com/ls-2018))
+- Optimize container launch priority performance. ([#1490](https://github.com/openkruise/kruise/pull/1490), [FillZpp](https://github.com/FillZpp))
+
+### Other Changes
+
+- Enhanced Operation
+  - PodProbeMarker:  Container probe support Tcp probing. ([#1474](https://github.com/openkruise/kruise/pull/1474), [KaiShi](https://github.com/BH4AWS))
+  - PodProbeMarker:  Sync podCondition when probe message of probeStates changed. ([#1479](https://github.com/openkruise/kruise/pull/1479), [chrisliu1995](https://github.com/chrisliu1995))
+  - PersistentPodState: Fix the problem that PersistentPodState can't get spec.replicas from unstructured object. ([#1462](https://github.com/openkruise/kruise/pull/1462), [0xgj](https://github.com/0xgj))
+  - Fix PodProbeMarker feature gate dependency . ([#1429](https://github.com/openkruise/kruise/pull/1429), [ls-2018](https://github.com/ls-2018))
+
+- Advanced Workload
+  - Enforce Advanced DaemonSet spec.selector is immutable. ([#1505](https://github.com/openkruise/kruise/pull/1505), [@hantmac](https://github.com/hantmac))
+  - Advanced StatefulSet maxUnavailable now counts unavailable pods with smaller ordinal in the update order during rolling upgrade. ([#1480](https://github.com/openkruise/kruise/pull/1480), [@Yesphet](https://github.com/Yesphet))
+  - Fix EphemeralJob event handler for deleting object. ([#1401](https://github.com/openkruise/kruise/pull/1401), [FillZpp](https://github.com/FillZpp))
+
+- Sidecar Container
+  - Fix pod annotations injection abnormal for SidecarSet. ([#1453](https://github.com/openkruise/kruise/pull/1453), [@a932846905](https://github.com/a932846905))
+
+- Application Protection
+  - PodUnavailableBudget ignore deletion of not ready or inconsistent pods. ([#1512](https://github.com/openkruise/kruise/pull/1512), [Spground](https://github.com/Spground))
+
+- Others
+  - Replace 'github.com/pkg/errors' with the standard Go library 'errors'. ([#1518](https://github.com/openkruise/kruise/pull/1518), [dongjiang1989](https://github.com/dongjiang1989))
+  - Upgrade minimum docker api version from 1.23 to 1.24. ([#1510](https://github.com/openkruise/kruise/pull/1510), [hantmac](https://github.com/hantmac))
+  - Add UT in controller_revision_test file. ([#1457](https://github.com/openkruise/kruise/pull/1457), [xiangpingjiang](https://github.com/xiangpingjiang))
+  - BroadcastJob controller define some parameters as Constant. ([#1414](https://github.com/openkruise/kruise/pull/1414), [lilongfeng0902](https://github.com/lilongfeng0902))
+  - Kruise-daemon enable pprof. ([#1416](https://github.com/openkruise/kruise/pull/1416), [dsxing](https://github.com/dsxing))
+  - Remove deprecated 'io/ioutil' pkg. ([#1404](https://github.com/openkruise/kruise/pull/1404), [testwill](https://github.com/testwill))
+  - Fix unnecessary use of fmt.Sprintf. ([#1403](https://github.com/openkruise/kruise/pull/1403), [testwill](https://github.com/testwill))
+
+## v1.5.2
+> Chang log since v1.5.1
+
+### CVE FIX: Enhance kruise-daemon security ([#1482](https://github.com/openkruise/kruise/pull/1482), [veophi](https://github.com/veophi))
+
+### Start kruise-manager as a non-root user
+We start kruise-manger with a non-root user to further enhance the security of kruise-manager. ([#1491](https://github.com/openkruise/kruise/pull/1491), [@zmberg](https://github.com/zmberg))
+
+## v1.5.1
+> Chang log since v1.5.0
+
+In version 1.5.1, the focus was on enhancing UnitedDeployment and addressing various bug fixes:
+
+- Add the ability to plan the lower and upper bound of capacity to the subsets in UnitedDeployment ([#1428](https://github.com/openkruise/kruise/pull/1428), [@veophi](https://github.com/veophi))
+
+- Fix unexpected job recreation by adding controller-revision-hash label for ImageListPullJob. ([#1441](https://github.com/openkruise/kruise/pull/1428), [@veophi](https://github.com/veophi))
+
+- Add prometheus metrics for pub and deletion protection to enhance observability for pub & deletion protection ([#1398](https://github.com/openkruise/kruise/pull/1398), [@zmberg](https://github.com/zmberg))
+
+- Add enable pprof flag for kruise daemon, now you can disable the pprof of kruise daemon ([#1416](https://github.com/openkruise/kruise/pull/1416), [@chengjoey](https://github.com/chengjoey))
+
+- Fix SidecarSet upgrade exception for UpdateExpectations to solve the problem of updating the image of the sidecar container ([#1435](https://github.com/openkruise/kruise/pull/1435), [@zmberg](https://github.com/zmberg)])
+
+- add audit log for pub and deletion protection to enhance observability for pub & deletion protection  ([#1438](https://github.com/openkruise/kruise/pull/1438), [@zmberg](https://github.com/zmberg)])
+
+## v1.5.0
+> Change log since v1.4.0
+
+### Upgrade Notice
+
+> No, really, you must read this before you upgrade
+
+- **Disable** following feature-gates by default: PreDownloadImageForInPlaceUpdate([#1244](https://github.com/openkruise/kruise/pull/1224), [@zmberg](https://github.com/zmberg)), ImagePullJobGate([#1357](https://github.com/openkruise/kruise/pull/1357), [@zmberg](https://github.com/zmberg)), DeletionProtectionForCRDCascadingGate([#1365](https://github.com/openkruise/kruise/pull/1365), [@zmberg](https://github.com/zmberg)), and ResourceDistributionGate([#1360](https://github.com/openkruise/kruise/pull/1360/files), [@zmberg](https://github.com/zmberg))
+- Bump Kubernetes dependency to 1.24.16, Golang version to 1.19([#1354](https://github.com/openkruise/kruise/pull/1354), [Kuromesi](https://github.com/Kuromesi))
+
+### Key Features: Enhanced Multi-Domain Management
+- WorkloadSpread:
+  - Support any customized workloads that have `scale` sub-resource. ([#1286](https://github.com/openkruise/kruise/pull/1286), [veophi](https://github.com/veophi))
+  - Add validation for subset patch field. ([#1237](https://github.com/openkruise/kruise/pull/1237), [chengleqi](https://github.com/chengleqi))
+- UnitedDeployment:
+  - Support `scale` sub-resource. ([#1314](https://github.com/openkruise/kruise/pull/1314)), [diannaowa](https://github.com/diannaowa))
+  - Support `patch` field for each subset. ([#1266](https://github.com/openkruise/kruise/pull/1266), [chengleqi](https://github.com/chengleqi))
+  - Optimize UnitedDeployment replicas settings. ([#1247](https://github.com/openkruise/kruise/pull/1247), [y-ykcir](https://github.com/y-ykcir))
+
+### ImagePreDownload
+- ImageListPullJob:
+  - Many users have the need for batch pre-download images, and the current approach, i.e., ImagePullJob, has a relatively high threshold for use, We added a new CRD ImageListPullJob to batch pre-download images.
+    You just write a range of images in one ImageListPullJob CR, its controller will generate corresponding ImagePullJob CR for each image automatically. ([1222](https://github.com/openkruise/kruise/pull/1222), [@diannaowa](https://github.com/diannaowa))
+- ImagePullJob:
+  - Fix the matching logic for the imagePullSecret in ImagePullJob. ([#1241](https://github.com/openkruise/kruise/pull/1241), [#1357](https://github.com/openkruise/kruise/pull/1357))
+  - Advanced Workload pre-download image support attach metadata in ImagePullJob. ([#1246](https://github.com/openkruise/kruise/pull/1246), [YTGhost](https://github.com/YTGhost))
+
+### Advanced Workload
+- SidecarSet:
+  - Add condition and event for not upgradable pods when updating. ([#1309](https://github.com/openkruise/kruise/pull/1309), [MarkLux](https://github.com/MarkLux))
+  - Take effect of shareVolumePolicy on initContainers. ([#1229](https://github.com/openkruise/kruise/pull/1229), [y-ykcir](https://github.com/y-ykcir))
+  - Allow sidecar containers to mount serviceAccountToken type volume. ([#1238](https://github.com/openkruise/kruise/pull/1238), [y-ykcir](https://github.com/y-ykcir))
+  - SidecarSet updateStrategy support priorityStrategy. ([#1325](https://github.com/openkruise/kruise/pull/1325), [y-ykcir](https://github.com/y-ykcir))
+- BroadcastJob:
+  - Make OnFailure as default restartPolicy. ([#1149](https://github.com/openkruise/kruise/pull/1149), [Shubhamurkade](https://github.com/Shubhamurkade))
+  - Fix BroadcastJob doesn't make pod on node that has erased taint. ([#1204](https://github.com/openkruise/kruise/pull/1204), [weldonlwz](https://github.com/weldonlwz))
+- CloneSet & StatefulSet:
+  - Regard the pod at preparing update state as update revision when scaling. ([#1290](https://github.com/openkruise/kruise/pull/1290), [veophi](https://github.com/veophi))
+  - Add `updatedAvailableReplicas` field in status. ([#1317](https://github.com/openkruise/kruise/pull/1317), [nitishchauhan0022](https://github.com/nitishchauhan0022))
+
+### Kruise Daemon
+- Connecting to Pouch runtime via CRI interface. ([#1232](https://github.com/openkruise/kruise/pull/1232), [@zmberg](https://github.com/zmberg))
+- Compatible with v1 and v1alpha2 CRI API version. ([#1354](https://github.com/openkruise/kruise/pull/1354), [veophi](https://github.com/veophi))
+
+### ResourceProtection
+- Reject Namespace deletion when PVCs are included under NS. ([#1228](https://github.com/openkruise/kruise/pull/1228), [kevin1689-cloud](https://github.com/kevin1689-cloud))
+
+And some bugs were fixed by
+([#1238](https://github.com/openkruise/kruise/pull/1238), [y-ykcir](https://github.com/y-ykcir)),
+([#1335](https://github.com/openkruise/kruise/pull/1335), [ls-2018](https://github.com/ls-2018)),
+([#1301](https://github.com/openkruise/kruise/pull/1301), [wangwu50](https://github.com/wangwu50)),
+([#1395](https://github.com/openkruise/kruise/pull/1301), [ywdxz](https://github.com/ywdxz)),
+([#1304](https://github.com/openkruise/kruise/pull/1304), [kevin1689-cloud](https://github.com/kevin1689-cloud)),
+([#1348](https://github.com/openkruise/kruise/pull/1348), [#1343](https://github.com/openkruise/kruise/pull/1343), [Colvin-Y](https://github.com/Colvin-Y)),
+thanks!
+
+## v1.4.1
+> Change log since v1.4.0
+
+### CVE FIX: Enhance kruise-daemon security ([#1482](https://github.com/openkruise/kruise/pull/1482), [veophi](https://github.com/veophi))
+
 ## v1.4.0
 
 > Change log since v1.3.0
@@ -8,8 +353,8 @@
 
 > No, really, you must read this before you upgrade
 
-- Enable following feature-gates by default: PreDownloadImageForInPlaceUpdate, ResourcesDeletionProtection, WorkloadSpread, PodUnavailableBudgetDeleteGate, InPlaceUpdateEnvFromMetadata,
+- Enable following feature-gates by default: ResourcesDeletionProtection, WorkloadSpread, PodUnavailableBudgetDeleteGate, InPlaceUpdateEnvFromMetadata,
-StatefulSetAutoDeletePVC, PodProbeMarkerGate. ([#1214](https://github.com/openkruise/kruise/pull/1214), [@zmberg](https://github.com/zmberg))
+  StatefulSetAutoDeletePVC, PodProbeMarkerGate. ([#1214](https://github.com/openkruise/kruise/pull/1214), [@zmberg](https://github.com/zmberg))
 - Change Kruise leader election from configmap to configmapsleases, this is a smooth upgrade with no disruption to OpenKruise service.  ([#1184](https://github.com/openkruise/kruise/pull/1184), [@YTGhost](https://github.com/YTGhost))
 
 ### New Feature: JobSidecarTerminator
@@ -50,6 +395,12 @@ For more detail, please refer to its [proposal](https://github.com/openkruise/kr
 - Change kruise base image to alpine. ([#1166](https://github.com/openkruise/kruise/pull/1166), [@fengshunli](https://github.com/fengshunli))
 - PersistentPodState support custom workload (like statefulSet). ([#1063](https://github.com/openkruise/kruise/pull/1063), [@baxiaoshi](https://github.com/baxiaoshi))
 
+## v1.3.1
+
+> Change log since v1.3.0
+
+### CVE FIX: Enhance kruise-daemon security ([#1482](https://github.com/openkruise/kruise/pull/1482), [veophi](https://github.com/veophi))
+
 ## v1.3.0
 
 > Change log since v1.2.0
@@ -65,7 +416,7 @@ So the Probe capabilities provided in Kubernetes have defined specific semantics
 **In addition, there is actually a need to customize Probe semantics and related behaviors**, such as:
 - **GameServer defines Idle Probe to determine whether the Pod currently has a game match**, if not, from the perspective of cost optimization, the Pod can be scaled down.
 - **K8S Operator defines the main-secondary probe to determine the role of the current Pod (main or secondary)**. When upgrading, the secondary can be upgraded first,
-so as to achieve the behavior of selecting the main only once during the upgrade process, reducing the service interruption time during the upgrade process.
+  so as to achieve the behavior of selecting the main only once during the upgrade process, reducing the service interruption time during the upgrade process.
 
 So we provides the ability to customize the Probe and return the result to the Pod yaml.
 
@@ -612,7 +963,7 @@ spec:
 Since v0.7.0:
 
 1. OpenKruise requires Kubernetes 1.13+ because of CRD conversion.
-  Note that for Kubernetes 1.13 and 1.14, users must enable `CustomResourceWebhookConversion` feature-gate in kube-apiserver before install or upgrade Kruise.
+   Note that for Kubernetes 1.13 and 1.14, users must enable `CustomResourceWebhookConversion` feature-gate in kube-apiserver before install or upgrade Kruise.
 2. OpenKruise official image supports multi-arch, by default including linux/amd64, linux/arm64, and linux/arm platforms.
 
 ### A NEW workload controller - AdvancedCronJob
@@ -960,4 +1311,4 @@ It provides full features for more efficient, deterministic and controlled deplo
 #### Features
 
 - Add SidecarSet that automatically injects sidecar container into selected pods
-- Support sidecar update functionality for SidecarSet
+- Support sidecar update functionality for SidecarSet

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contributing to Openkruise
 
-Welcome to Openkruise! Openkruise consists several repositories under the organization.
+Welcome to Openkruise! Openkruise consists of several repositories under the organization.
 We encourage you to help out by reporting issues, improving documentation, fixing bugs, or adding new features.
 Please also take a look at our code of conduct, which details how contributors are expected to conduct themselves as part of the Openkruise community.
 
@@ -10,7 +10,7 @@ To be honest, we regard every user of Openkruise as a very kind contributor.
 After experiencing Openkruise, you may have some feedback for the project.
 Then feel free to open an issue.
 
-There are lot of cases when you could open an issue:
+There are a lot of cases when you could open an issue:
 
 - bug report
 - feature request
@@ -20,11 +20,11 @@ There are lot of cases when you could open an issue:
 - help wanted
 - doc incomplete
 - test improvement
-- any questions on project
+- any questions on the project
 - and so on
 
-Also we must remind that when filing a new issue, please remember to remove the sensitive data from your post.
+Also, we must remind you that when filing a new issue, please remember to remove the sensitive data from your post.
-Sensitive data could be password, secret key, network locations, private business data and so on.
+Sensitive data could be passwords, secret keys, network locations, private business data, and so on.
 
 ## Code and doc contribution
 
@@ -45,13 +45,14 @@ On GitHub, every improvement for Openkruise could be via a PR (short for pull re
 ### Workspace Preparation
 
 To put forward a PR, we assume you have registered a GitHub ID.
-Then you could finish the preparation in the following steps:
+Then you can finish the preparation in the following steps:
 
-1. **Fork** Fork the repository you wish to work on. You just need to click the button Fork in right-left of project repository main page. Then you will end up with your repository in your GitHub username.
+1. **Fork** Fork the repository you wish to work on. You just need to click the button Fork in the right-left of the project repository main page. Then you will end up with your repository in your GitHub username.
-2. **Clone** your own repository to develop locally. Use `git clone https://github.com/<your-username>/<project>.git` to clone repository to your local machine. Then you can create new branches to finish the change you wish to make.
+2. **Clone** your own repository to develop locally. Use `git clone https://github.com/<your-username>/<project>.git` to clone the repository to your local machine. Then you can create new branches to finish the change you wish to make.
 3. **Set remote** upstream to be `https://github.com/openkruise/<project>.git` using the following two commands:
 
 ```bash
+cd <project>
 git remote add upstream https://github.com/openkruise/<project>.git
 git remote set-url --push upstream no-pushing
 ```
@@ -60,7 +61,7 @@ Adding this, we can easily synchronize local branches with upstream branches.
 
 4. **Create a branch** to add a new feature or fix issues
 
-Update local working directory:
+Update the local working directory:
 
 ```bash
 cd <project>
@@ -79,16 +80,16 @@ Make any change on the new-branch then build and test your codes.
 
 ### PR Description
 
-PR is the only way to make change to Kruise project files.
+PR is the only way to make changes to Kruise project files.
-To help reviewers better get your purpose, PR description could not be too detailed.
+To help reviewers better understand your purpose, PR description could not be too detailed.
 We encourage contributors to follow the [PR template](./.github/PULL_REQUEST_TEMPLATE.md) to finish the pull request.
 
 ### Developing Environment
 
-As a contributor, if you want to make any contribution to Kruise project, we should reach an agreement on the version of tools used in the development environment.
+As a contributor, if you want to make any contribution to the Kruise project, we should reach an agreement on the version of tools used in the development environment.
-Here are some dependents with specific version:
+Here are some dependencies with specific versions:
 
-- Golang : v1.18+
+- Golang : v1.22+
 - Kubernetes: v1.16+
 
 ### Developing guide
@@ -106,23 +107,40 @@ make build
 make test
 ```
 
-**There are some guide documents for contributors in [./docs/contributing/](./docs/contributing), such as debug guide to help you test your own branch in a Kubernetes cluster.**
+**There are some guide documents for contributors in [./docs/contributing/](./docs/contributing), such as a debug guide to help you test your own branch in a Kubernetes cluster.**
 
 ### Proposals
 
-If you are going to contribute a feature with new API or needs significant effort, please submit a proposal in [./docs/proposals/](./docs/proposals) first.
+If you are going to contribute a feature with a new API or need significant effort, please submit a proposal in [./docs/proposals/](./docs/proposals) first.
+
+### Kruise Helm Charts
+[kruise charts](https://github.com/openkruise/charts) is the openKruise charts repo, including kruise, kruise rollout, and kruise game.
+You can add the corresponding charts package in the versions directory as follows:
+```
+ versions
+ - kruise-game
+ - kruise-rollout
+ - kruise-state-metrics
+ - kruise
+   - 1.5.0
+   - 1.5.1
+   - 1.6.0
+   - 1.6.1
+```
+
+**make generate_helm_crds** automatically generates crds files under the bin/ directory, which in turn simplifies the generation of helm charts.
 
 ## Engage to help anything
 
 We choose GitHub as the primary place for Openkruise to collaborate.
 So the latest updates of Openkruise are always here.
-Although contributions via PR is an explicit way to help, we still call for any other ways.
+Although contributions via PR are an explicit way to help, we still call for any other ways.
 
 - reply to other's issues if you could;
 - help solve other user's problems;
 - help review other's PR design;
 - help review other's codes in PR;
-- discuss about Openkruise to make things clearer;
+- discuss Openkruise to make things clearer;
 - advocate Openkruise technology beyond GitHub;
 - write blogs on Openkruise and so on.
 
@@ -130,5 +148,5 @@ In a word, **ANY HELP IS CONTRIBUTION**.
 
 ## Join Openkruise as a member
 
-It is also welcomed to join Openkruise team if you are willing to participate in Openkruise community continuously and keep active.
+It is also welcomed to join the Openkruise team if you are willing to participate in the Openkruise community continuously and keep active.
 Please read and follow the [Community Membership](https://github.com/openkruise/community/blob/master/community-membership.md).

Dockerfile

@@ -1,8 +1,7 @@
 # Build the manager and daemon binaries
 ARG BASE_IMAGE=alpine
-ARG BASE_IMAGE_VERSION=3.17
+ARG BASE_IMAGE_VERSION=3.21@sha256:56fa17d2a7e7f168a043a2712e63aed1f8543aeafdcee47c58dcffe38ed51099
-FROM golang:1.18-alpine3.17 as builder
+FROM golang:1.23.9-alpine3.21@sha256:fb7ea5cd19bc4eea3eb0d1972919ec0f6229b138985ce4b35ce5846c6bc02973 AS builder
-
 WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod
@@ -13,20 +12,37 @@ COPY main.go main.go
 COPY apis/ apis/
 COPY cmd/ cmd/
 COPY pkg/ pkg/
-COPY vendor/ vendor/
 
 # Build
-RUN CGO_ENABLED=0 GO111MODULE=on go build -mod=vendor -a -o manager main.go \
+RUN CGO_ENABLED=0 GO111MODULE=on go build -a -o manager main.go \
-  && CGO_ENABLED=0 GO111MODULE=on go build -mod=vendor -a -o daemon ./cmd/daemon/main.go
+  && CGO_ENABLED=0 GO111MODULE=on go build -a -o daemon ./cmd/daemon/main.go
 
 ARG BASE_IMAGE
 ARG BASE_IMAGE_VERSION
 FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
 
-RUN apk add --no-cache ca-certificates=~20220614-r4 bash=~5.2.15-r0 expat=~2.5.0-r0 \
-  && rm -rf /var/cache/apk/*
-
 WORKDIR /
 COPY --from=builder /workspace/manager .
 COPY --from=builder /workspace/daemon ./kruise-daemon
+
+RUN set -eux; \
+    mkdir -p /log /tmp && \
+    chown -R nobody:nobody /log && \
+    chown -R nobody:nobody /tmp && \
+    chown -R nobody:nobody /manager && \
+    apk --no-cache --update upgrade && \
+    apk --no-cache add ca-certificates && \
+    apk --no-cache add tzdata && \
+    rm -rf /var/cache/apk/* && \
+    update-ca-certificates && \
+    echo "only include root and nobody user" && \
+    echo -e "root:x:0:0:root:/root:/bin/ash\nnobody:x:65534:65534:nobody:/:/sbin/nologin" | tee /etc/passwd && \
+    echo -e "root:x:0:root\nnobody:x:65534:" | tee /etc/group && \
+    rm -rf /usr/local/sbin/* && \
+    rm -rf /usr/local/bin/* && \
+    rm -rf /usr/sbin/* && \
+    rm -rf /usr/bin/* && \
+    rm -rf /sbin/* && \
+    rm -rf /bin/*
+
 ENTRYPOINT ["/manager"]

Dockerfile_helm_hook

@@ -0,0 +1,40 @@
+ARG BASE_IMAGE=alpine
+ARG BASE_IMAGE_VERSION=3.19
+FROM golang:1.20.14-alpine3.19 AS builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+
+# Copy the go source
+COPY apis/ apis/
+COPY cmd/ cmd/
+COPY pkg/ pkg/
+# Build
+RUN --mount=type=cache,target=/go CGO_ENABLED=0 GO111MODULE=on go build -a -o helm_hook ./cmd/helm_hook/main.go
+
+FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
+WORKDIR /
+RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
+RUN set -eux; \
+    mkdir -p /log /tmp && \
+    chown -R nobody:nobody /log && \
+    chown -R nobody:nobody /tmp && \
+    apk --no-cache --update upgrade && \
+    apk --no-cache add ca-certificates && \
+    apk --no-cache add tzdata && \
+    rm -rf /var/cache/apk/* && \
+    update-ca-certificates && \
+    echo "only include root and nobody user" && \
+    echo -e "root:x:0:0:root:/root:/bin/ash\nnobody:x:65534:65534:nobody:/:/sbin/nologin" | tee /etc/passwd && \
+    echo -e "root:x:0:root\nnobody:x:65534:" | tee /etc/group
+COPY --from=builder /workspace/helm_hook .
+RUN chown -R nobody:nobody /helm_hook && \
+    rm -rf /usr/local/sbin/* && \
+    rm -rf /usr/local/bin/* && \
+    rm -rf /usr/sbin/* && \
+    rm -rf /usr/bin/* && \
+    rm -rf /sbin/* && \
+    rm -rf /bin/*
+ENTRYPOINT ["/helm_hook"]

Dockerfile_multiarch

@@ -1,7 +1,8 @@
 # Build the manager and daemon binaries
 ARG BASE_IMAGE=alpine
-ARG BASE_IMAGE_VERSION=3.17
+ARG BASE_IMAGE_VERSION=3.21@sha256:56fa17d2a7e7f168a043a2712e63aed1f8543aeafdcee47c58dcffe38ed51099
-FROM --platform=$BUILDPLATFORM golang:1.18-alpine3.17 as builder
+ARG BUILD_BASE_IMAGE=golang:1.22.11-alpine3.21@sha256:161858498a61ce093c8e2bd704299bfb23e5bff79aef99b6c40bb9c6a43acf0f
+FROM --platform=$BUILDPLATFORM ${BUILD_BASE_IMAGE} AS builder
 
 WORKDIR /workspace
 # Copy the Go Modules manifests
@@ -13,23 +14,43 @@ COPY main.go main.go
 COPY apis/ apis/
 COPY cmd/ cmd/
 COPY pkg/ pkg/
-COPY vendor/ vendor/
+
+#ENV GOPROXY=https://goproxy.cn,direct
+RUN go mod tidy
 
 # Build
 ARG TARGETOS
 ARG TARGETARCH
-RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GO111MODULE=on go build -mod=vendor -a -o manager main.go \
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GO111MODULE=on go build -a -o manager main.go \
-  && GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GO111MODULE=on go build -mod=vendor -a -o daemon ./cmd/daemon/main.go
+  && GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GO111MODULE=on go build -a -o daemon ./cmd/daemon/main.go
 
 
 ARG BASE_IMAGE
 ARG BASE_IMAGE_VERSION
 FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
 
-RUN apk add --no-cache ca-certificates=~20220614-r4 bash=~5.2.15-r0 expat=~2.5.0-r0 \
-  && rm -rf /var/cache/apk/*
-
 WORKDIR /
 COPY --from=builder /workspace/manager .
 COPY --from=builder /workspace/daemon ./kruise-daemon
+
+RUN set -eux; \
+    mkdir -p /log /tmp && \
+    chown -R nobody:nobody /log && \
+    chown -R nobody:nobody /tmp && \
+    chown -R nobody:nobody /manager && \
+    apk --no-cache --update upgrade && \
+    apk --no-cache add ca-certificates && \
+    apk --no-cache add tzdata && \
+    rm -rf /var/cache/apk/* && \
+    update-ca-certificates && \
+    echo "only include root and nobody user" && \
+    echo -e "root:x:0:0:root:/root:/bin/ash\nnobody:x:65534:65534:nobody:/:/sbin/nologin" | tee /etc/passwd && \
+    echo -e "root:x:0:root\nnobody:x:65534:" | tee /etc/group && \
+    rm -rf /usr/local/sbin/* && \
+    rm -rf /usr/local/bin/* && \
+    rm -rf /usr/sbin/* && \
+    rm -rf /usr/bin/* && \
+    rm -rf /sbin/* && \
+    rm -rf /bin/*
+
 ENTRYPOINT ["/manager"]

Dockerfile_windows

@@ -0,0 +1,11 @@
+# Build Windows image for kruise-daemon 
+
+# Using Windows HostProcess container base image: https://github.com/microsoft/windows-host-process-containers-base-image
+ARG BASE_IMAGE=mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image
+ARG BASE_IMAGE_VERSION=v1.0.0
+FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
+
+WORKDIR /
+COPY ./bin/kruise-daemon.exe .
+
+ENTRYPOINT ["kruise-daemon.exe"]

Makefile

@@ -1,7 +1,10 @@
 # Image URL to use all building/pushing image targets
 IMG ?= openkruise/kruise-manager:test
+HOOK_IMG ?= openkruise/kruise-helm-hook:test
+WIN_DAEMON_IMG ?= openkruise/kruise-daemon-win:test
 # Platforms to build the image for
 PLATFORMS ?= linux/amd64,linux/arm64,linux/ppc64le
+WIN_PLATFORMS ?= windows/amd64
 CRD_OPTIONS ?= "crd:crdVersions=v1"
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
@@ -10,6 +13,11 @@ GOBIN=$(shell go env GOPATH)/bin
 else
 GOBIN=$(shell go env GOBIN)
 endif
+GOOS ?= $(shell go env GOOS)
+
+# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
+# Run `setup-envtest list` to list available versions.
+ENVTEST_K8S_VERSION ?= 1.32.0
 
 # Setting SHELL to bash allows bash commands to be executed by recipes.
 # This is a requirement for 'setup-envtest.sh' in the test target.
@@ -22,7 +30,7 @@ all: build
 ##@ Development
 
 go_check:
-	@scripts/check_go_version "1.18.0"
+	@scripts/check_go_version "1.23"
 
 generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	@scripts/generate_client.sh
@@ -30,7 +38,7 @@ generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./apis/..."
 
 manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./apis/..." output:crd:artifacts:config=config/crd/bases
 
 fmt: go_check ## Run go fmt against code.
 	go fmt $(shell go list ./... | grep -v /vendor/)
@@ -41,16 +49,36 @@ vet: ## Run go vet against code.
 lint: golangci-lint ## Run golangci-lint against code.
 	$(GOLANGCI_LINT) run
 
-ENVTEST_ASSETS_DIR=$(shell pwd)/testbin
+test: generate fmt vet manifests envtest ## Run tests
-test: generate fmt vet manifests ## Run tests
+	echo $(ENVTEST)
-	mkdir -p ${ENVTEST_ASSETS_DIR}
+	go build -o pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin pkg/daemon/criruntime/imageruntime/fake_plugin/main.go && chmod +x pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
-	source ./scripts/setup-envtest.sh; fetch_envtest_tools $(ENVTEST_ASSETS_DIR); setup_envtest_env $(ENVTEST_ASSETS_DIR); go test ./pkg/... -coverprofile cover.out
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -race ./pkg/... -coverprofile raw-cover.out
+	rm pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
+	grep -v "pkg/client" raw-cover.out > cover.out
+
+atest: 
+	echo $(ENVTEST)
+	go build -o pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin pkg/daemon/criruntime/imageruntime/fake_plugin/main.go && chmod +x pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -race ./pkg/... -coverprofile raw-cover.out
+	rm pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
+	grep -v "pkg/client" raw-cover.out > cover.out
+
+coverage-report: ## Generate cover.html from cover.out
+	go tool cover -html=cover.out -o cover.html
+ifeq ($(GOOS), darwin)
+	open ./cover.html
+else
+	echo "open cover.html with a HTML viewer."
+endif
 
 ##@ Build
 
 build: generate fmt vet manifests ## Build manager binary.
 	go build -o bin/manager main.go
 
+build-win-daemon: ## Build Windows daemon binary.
+	GOOS=windows go build -o bin/kruise-daemon.exe ./cmd/daemon/main.go
+
 run: manifests generate fmt vet ## Run a controller from your host.
 	go run ./main.go
 
@@ -60,6 +88,9 @@ docker-build: ## Build docker image with the manager.
 docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
 
+docker-win-daemon: # Build Windows docker image with the daemon
+	docker buildx build -f ./Dockerfile_windows --pull --no-cache --platform=$(WIN_PLATFORMS) . -t $(WIN_DAEMON_IMG)
+
 # Build and push the multiarchitecture docker images and manifest.
 docker-multiarch:
 	docker buildx build -f ./Dockerfile_multiarch --pull --no-cache --platform=$(PLATFORMS) --push . -t $(IMG)
@@ -75,27 +106,28 @@ uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified
 deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
 	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
 	$(KUSTOMIZE) build config/default | kubectl apply -f -
-	echo -e "resources:\n- manager.yaml" > config/manager/kustomization.yaml
+	$(KUSTOMIZE) build config/daemonconfig | kubectl apply -f -
 
 undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
 	$(KUSTOMIZE) build config/default | kubectl delete -f -
-
+	$(KUSTOMIZE) build config/daemonconfig | kubectl delete -f -
 
 CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
 controller-gen: ## Download controller-gen locally if necessary.
-ifeq ("$(shell $(CONTROLLER_GEN) --version 2> /dev/null)", "Version: v0.7.0")
+
+# controller-gen@v0.16.5 comply with k8s.io/api v0.30.x
+ifeq ("$(shell $(CONTROLLER_GEN) --version 2> /dev/null)", "Version: v0.16.5")
 else
 	rm -rf $(CONTROLLER_GEN)
-	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.7.0)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.17.3)
 endif
-
 KUSTOMIZE = $(shell pwd)/bin/kustomize
 kustomize: ## Download kustomize locally if necessary.
 	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.5)
 
 GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
 golangci-lint: ## Download golangci-lint locally if necessary.
-	$(call go-get-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@v1.42.1)
+	$(call go-get-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2)
 
 GINKGO = $(shell pwd)/bin/ginkgo
 ginkgo: ## Download ginkgo locally if necessary.
@@ -109,8 +141,75 @@ set -e ;\
 TMP_DIR=$$(mktemp -d) ;\
 cd $$TMP_DIR ;\
 go mod init tmp ;\
-echo "Downloading $(2)" ;\
+echo "Downloading $(2) to $(PROJECT_DIR)/bin" ;\
 GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
 rm -rf $$TMP_DIR ;\
 }
 endef
+
+include tools/tools.mk
+
+## Location to install dependencies to
+TESTBIN ?= $(shell pwd)/testbin
+$(TESTBIN):
+	mkdir -p $(TESTBIN)
+
+ENVTEST ?= $(TESTBIN)/setup-envtest
+
+.PHONY: envtest
+envtest: $(TESTBIN) ## Download/update envtest-setup to latest version.
+	GOBIN=$(TESTBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+
+# create-cluster creates a kube cluster with kind.
+.PHONY: create-cluster
+create-cluster: $(tools/kind)
+	tools/hack/create-cluster.sh
+
+DISABLE_CSI ?= false
+
+.PHONY: install-csi
+install-csi:
+ifeq ($(DISABLE_CSI), true)
+	@echo "CSI is disabled, skip"
+else
+	cd tools/hack/csi-driver-host-path; ./install-snapshot.sh
+endif
+
+# delete-cluster deletes a kube cluster.
+.PHONY: delete-cluster
+delete-cluster: $(tools/kind) ## Delete kind cluster.
+	$(tools/kind) delete cluster --name ci-testing
+
+# kube-load-image loads a local built docker image into kube cluster.
+.PHONY: kube-load-image
+kube-load-image: $(tools/kind)
+	tools/hack/kind-load-image.sh $(IMG)
+
+# install-kruise install kruise with local build image to kube cluster.
+.PHONY: install-kruise
+install-kruise:
+	kubectl create namespace kruise-system;
+ifeq ($(DISABLE_E2E_CONFIG), true)
+	@echo "Skipping e2e config application...";
+else
+	@echo "Applying e2e config...";
+	kubectl apply -f test/kruise-e2e-config.yaml;
+endif
+	tools/hack/install-kruise.sh $(IMG)
+
+# run-kruise-e2e-test starts to run kruise e2e tests.
+.PHONY: run-kruise-e2e-test
+run-kruise-e2e-test:
+	@echo -e "\n\033[36mRunning kruise e2e tests...\033[0m"
+	tools/hack/run-kruise-e2e-test.sh
+
+generate_helm_crds:
+	scripts/generate_helm_crds.sh
+
+# kruise-e2e-test runs kruise e2e tests.
+.PHONY: kruise-e2e-test
+kruise-e2e-test: $(tools/kind) delete-cluster create-cluster install-csi docker-build kube-load-image install-kruise run-kruise-e2e-test delete-cluster
+
+.PHONY: docker-build-hook
+docker-build-hook:
+	docker buildx build -f ./Dockerfile_helm_hook --pull --no-cache --platform=$(PLATFORMS) --push . -t $(HOOK_IMG)

OWNERS

@@ -4,8 +4,10 @@ approvers:
   - FillZpp
   - furykerry
   - zmberg
+  - veophi
 reviewers:
   - Fei-Guo
   - FillZpp
   - furykerry
   - zmberg
+  - veophi

README-zh_CN.md

@@ -35,6 +35,7 @@ OpenKruise (官网: [https://openkruise.io](https://openkruise.io)) 是CNCF([Clo
 
   - [**SidecarSet** - 定义和升级你的 sidecar 容器](https://openkruise.io/zh/docs/user-manuals/sidecarset)
   - [**Container Launch Priority** 控制sidecar启动顺序](https://openkruise.io/zh/docs/user-manuals/containerlaunchpriority)
+  - [**Sidecar Job Terminator** 当 Job 类 Pod 主容器退出后，Terminator Sidecar容器](https://openkruise.io/zh/docs/user-manuals/jobsidecarterminator)
 
 - **多区域管理**
 
@@ -49,6 +50,9 @@ OpenKruise (官网: [https://openkruise.io](https://openkruise.io)) 是CNCF([Clo
 
   - [原地重启 pod 中的容器](https://openkruise.io/zh/docs/user-manuals/containerrecreaterequest)
   - [指定的一批节点上拉取镜像](https://openkruise.io/zh/docs/user-manuals/imagepulljob)
+  - [**ResourceDistribution** 支持 Secret、Configmaps 资源跨 Namespace 分发](https://openkruise.io/zh/docs/user-manuals/resourcedistribution)
+  - [**PersistentPodState** 保持Pod的一些状态，比如："固定IP调度"](https://openkruise.io/zh/docs/user-manuals/persistentpodstate)
+  - [**PodProbeMarker** 提供自定义Probe探测的能力](https://openkruise.io/zh/docs/user-manuals/podprobemarker)
 
 - **应用安全防护**
 
@@ -62,6 +66,14 @@ OpenKruise (官网: [https://openkruise.io](https://openkruise.io)) 是CNCF([Clo
 - 安装/升级 Kruise [稳定版本](https://openkruise.io/docs/installation)
 - 安装/升级 Kruise [最新版本（包括 alpha/beta/rc）](https://openkruise.io/docs/next/installation)
 
+### 在阿里云上快速体验
+
+- 3分钟内在阿里云上创建 Kruise 体验环境:
+
+  <a href="https://acs.console.aliyun.com/quick-deploy?repo=openkruise/charts&branch=master&paths=%5B%22versions/kruise/1.7.3%22%5D" target="_blank">
+    <img src="https://img.alicdn.com/imgextra/i1/O1CN01aiPSuA1Wiz7wkgF5u_!!6000000002823-55-tps-399-70.svg" width="200" alt="Deploy on Alibaba Cloud">
+  </a>
+
 ## 用户
 
 登记: [如果贵司正在使用 Kruise 请留言](https://github.com/openkruise/kruise/issues/289)
@@ -73,7 +85,8 @@ OpenKruise (官网: [https://openkruise.io](https://openkruise.io)) 是CNCF([Clo
 - Spectro Cloud, 艾佳生活, Arkane Systems, 滴普科技, 火花思维
 - OPPO, 苏宁, 欢聚时代, 汇量科技, 深圳凤凰木网络有限公司
 - 小米, 网易, 美团金融, 虾皮购物, e签宝
-- LinkedIn, 雪球, 兴盛优选, Wholee
+- LinkedIn, 雪球, 兴盛优选, Wholee, LilithGames, Baidu
+- Bilibili, 冠赢互娱, MeiTuan, 同城
 
 ## 贡献
 
@@ -87,10 +100,15 @@ OpenKruise (官网: [https://openkruise.io](https://openkruise.io)) 是CNCF([Clo
 - 钉钉：搜索群ID `23330762` (*Chinese*)
 - 微信：添加用户 `openkruise` 并让机器人拉你入群 (*Chinese*)
 - 社区双周会 (APAC, *Chinese*):
-  - 周四 19:00 GMT+8 (Asia/Shanghai)
+  - 周四 19:30 GMT+8 (Asia/Shanghai)
-  - [进入会议(zoom)](https://us02web.zoom.us/j/87059136652?pwd=NlI4UThFWXVRZkxIU0dtR1NINncrQT09)
+  - 进入会议(钉钉): 搜索群ID `23330762`
   - [会议纪要](https://shimo.im/docs/gXqmeQOYBehZ4vqo)
 - Bi-weekly Community Meeting (*English*): TODO
+  - [进入会议(zoom)](https://us02web.zoom.us/j/87059136652?pwd=NlI4UThFWXVRZkxIU0dtR1NINncrQT09)
+
+## 安全
+
+汇报安全漏洞请通过邮箱kubernetes-security@service.aliyun.com, 更多安全细节并参见[SECURITY.md](SECURITY.md)
 
 ## License
 

README.md

@@ -3,10 +3,11 @@
 [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
 [![Go Report Card](https://goreportcard.com/badge/github.com/openkruise/kruise)](https://goreportcard.com/report/github.com/openkruise/kruise)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2908/badge)](https://bestpractices.coreinfrastructure.org/en/projects/2908)
-[![Build Status](https://travis-ci.org/openkruise/kruise.svg?branch=master)](https://travis-ci.org/openkruise/kruise)
+[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/openkruise/kruise/badge)](https://scorecard.dev/viewer/?uri=github.com/openkruise/kruise)
 [![CircleCI](https://circleci.com/gh/openkruise/kruise.svg?style=svg)](https://circleci.com/gh/openkruise/kruise)
 [![codecov](https://codecov.io/gh/openkruise/kruise/branch/master/graph/badge.svg)](https://codecov.io/gh/openkruise/kruise)
 [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](./CODE_OF_CONDUCT.md)
+[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20Kruise%20Guru-006BFF)](https://gurubase.io/g/kruise)
 
 English | [简体中文](./README-zh_CN.md)
 
@@ -31,10 +32,11 @@ It consists of several controllers which extend and complement the [Kubernetes c
 
 - **Sidecar container Management**
 
-  Kruise simplify sidecar injection and enable sidecar in-place update. Kruise also enhance the sidecar startup and termination control.
+  Kruise simplifies sidecar injection and enables sidecar in-place update. Kruise also enhances the sidecar startup and termination control.
 
   - [**SidecarSet** for defining and upgrading your own sidecars](https://openkruise.io/docs/user-manuals/sidecarset)
   - [**Container Launch Priority** to control the container startup orders](https://openkruise.io/docs/user-manuals/containerlaunchpriority)
+  - [**Sidecar Job Terminator** terminates sidecar containers for such job-type Pods when its main containers completed.](https://openkruise.io/docs/user-manuals/jobsidecarterminator)
 
 - **Multi-domain Management**
 
@@ -48,8 +50,11 @@ It consists of several controllers which extend and complement the [Kubernetes c
 
 - **Enhanced Operations**
 
-  - [Restart containers in a running pod](https://openkruise.io/docs/user-manuals/containerrecreaterequest)
+  - [**ContainerRecreateRequest** provides a way to let users restart/recreate containers in a running pod](https://openkruise.io/docs/user-manuals/containerrecreaterequest)
-  - [Download images on specific nodes](https://openkruise.io/docs/user-manuals/imagepulljob)
+  - [**ImagePullJob** pre-download images on specific nodes](https://openkruise.io/docs/user-manuals/imagepulljob)
+  - [**ResourceDistribution** support Secret & ConfigMap resource distribution across namespaces](https://openkruise.io/docs/user-manuals/resourcedistribution)
+  - [**PersistentPodState** is able to persistent states of the Pod, such as "IP Retention"](https://openkruise.io/docs/user-manuals/persistentpodstate)
+  - [**PodProbeMarker** provides the ability to customize the Probe and return the result to the Pod](https://openkruise.io/docs/user-manuals/podprobemarker)
 
 - **Application Protection**
 
@@ -63,6 +68,14 @@ You can view the full documentation from the [OpenKruise website](https://openkr
 - Install or upgrade Kruise with [the stable version](https://openkruise.io/docs/installation).
 - Install or upgrade Kruise with [the latest version including alpha/beta/rc](https://openkruise.io/docs/next/installation).
 
+### Get Your Own Demo with Alibaba Cloud
+
+- install Kruise on a Serverless K8S cluster in 3 minutes, try:
+
+  <a href="https://acs.console.aliyun.com/quick-deploy?repo=openkruise/charts&branch=master&paths=%5B%22versions/kruise/1.8.0%22%5D" target="_blank">
+    <img src="https://img.alicdn.com/imgextra/i1/O1CN01aiPSuA1Wiz7wkgF5u_!!6000000002823-55-tps-399-70.svg" width="200" alt="Deploy on Alibaba Cloud">
+  </a>
+
 ## Users
 
 Registration: [Who is using Kruise](https://github.com/openkruise/kruise/issues/289)
@@ -74,7 +87,8 @@ Registration: [Who is using Kruise](https://github.com/openkruise/kruise/issues/
 - Spectro Cloud, ihomefnt, Arkane Systems, Deepexi, 火花思维
 - OPPO, Suning.cn, joyy, Mobvista, 深圳凤凰木网络有限公司
 - xiaomi, Netease, MeiTuan Finance, Shopee, Esign
-- LinkedIn, 雪球, 兴盛优选, Wholee
+- LinkedIn, 雪球, 兴盛优选, Wholee, LilithGames, Baidu
+- Bilibili, 冠赢互娱, MeiTuan, 同城
 
 ## Contributing
 
@@ -88,10 +102,14 @@ Active communication channels:
 - DingTalk：Search GroupID `23330762` (*Chinese*)
 - WeChat: Search User `openkruise` and let the robot invite you (*Chinese*)
 - Bi-weekly Community Meeting (APAC, *Chinese*):
-  - Thursday 19:00 GMT+8 (Asia/Shanghai), [Calendar](https://calendar.google.com/calendar/u/2?cid=MjdtbDZucXA2bjVpNTFyYTNpazV2dW8ybHNAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ)
+  - Thursday 19:30 GMT+8 (Asia/Shanghai), [Calendar](https://calendar.google.com/calendar/u/2?cid=MjdtbDZucXA2bjVpNTFyYTNpazV2dW8ybHNAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ)
-  - [Meeting Link(zoom)](https://us02web.zoom.us/j/87059136652?pwd=NlI4UThFWXVRZkxIU0dtR1NINncrQT09)
+  - Join Meeting(DingTalk): Search GroupID `23330762` (*Chinese*)
   - [Notes and agenda](https://shimo.im/docs/gXqmeQOYBehZ4vqo)
 - Bi-weekly Community Meeting (*English*): TODO
+  - [Meeting Link(zoom)](https://us02web.zoom.us/j/87059136652?pwd=NlI4UThFWXVRZkxIU0dtR1NINncrQT09)
+
+## Security
+Please report vulnerabilities by email to kubernetes-security@service.aliyun.com. Also see our [SECURITY.md](SECURITY.md) file for details.
 
 ## License
 

RELEASE-PROCESS.md

@@ -15,7 +15,7 @@ Look at [the last release](https://github.com/openkruise/kruise/releases/latest)
 
 Add a new section in [CHANGELOG.md](./CHANGELOG.md) for the new version that is being released along with the new features, patches and deprecations it introduces.
 
-It should not include every single change but solely what matters to our customers, for example issue template that has changed is not important.
+It should not include every single change but solely what matters to our customers, for example, an issue template that has changed is not important.
 
 ## 2. Publish documentation for new version
 

SECURITY.md

@@ -9,9 +9,10 @@ Here's an overview:
 
 | Version | Supported           |
 | ------- | ------------------- |
-| 0.10.x   | :white_check_mark: |
+| 1.16.x   | :white_check_mark: |
-| 0.9.x   | :white_check_mark:  |
+| 1.15.x   | :white_check_mark:  |
-| < 0.9   | :x:                 |
+| 1.14.x   | :white_check_mark:  |
+| < 1.14   | :x:                 |
 
 ## Prevention
 
@@ -26,16 +27,9 @@ Kruise maintainers are working to improve our prevention by adding additional me
 
 We strive to ship secure software, but we need the community to help us find security breaches.
 
-In case of a confirmed breach, reporters will get full credit and can be keep in the loop, if
+In case of a confirmed breach, reporters will get full credit and can be keep in the loop, if preferred.
-preferred.
 
-### Private Disclosure Processes
+DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to kubernetes-security@service.aliyun.com
-
-We ask that all suspected vulnerabilities be privately and responsibly disclosed by [contacting our maintainers](mailto:cncf-openkruise-maintainers@lists.cncf.io).
-
-### Public Disclosure Processes
-
-If you know of a publicly disclosed security vulnerability please IMMEDIATELY email the [OpenKruise maintainers](mailto:cncf-openkruise-maintainers@lists.cncf.io) to inform about the vulnerability so they may start the patch, release, and communication process.
 
 ### Compensation
 

SECURITY_CONTACTS.md

@@ -0,0 +1,10 @@
+Defined below are the security persons of contact for this project. If you have questions regarding the triaging and handling of incoming problems, they may be contacted.
+
+The following security contacts have agreed to abide by the [Embargo Policy](embargo-policy.md) and will be removed and replaced if found to be in violation of that agreement.
+
+DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, USE THE INSTRUCTIONS AT [SECURITY.md](SECURITY.md)
+
+Security Contacts:
+* [Zhen Zhang](mailto:shouchen.zz@alibaba-inc.com)
+* [Mingshan Zhao](mailto:liheng.zms@alibaba-inc.com)
+

SECURITY_RESPONSE.md

@@ -0,0 +1,84 @@
+# Incident response
+
+This serves to define how potential security issues should be triaged, how
+confirmation occurs, providing the notification, and issuing a security advisory
+as well as patch/release.
+
+## Triage
+
+### Identify the problem
+
+Triaging issues allows maintainers to focus resources on the most critically
+impacting problems. Potential security risks should be evaluated against the
+following information:
+
+* Which component(s) of the project is impacted?
+* What kind of problem is this?
+  * privilege escalation
+  * credential access
+  * code execution
+  * exfiltration
+  * lateral movement
+* How complex is the problem?
+* Is user interaction required?
+* What privileges are required for this problem to occur?
+  * admin
+  * general
+* What is the potential impact or consequence of the problem?
+* Does an exploit exist?
+
+Any potential problem that has an exploit, permits privilege escalation, is
+simple, and does not require user interaction should be evaluated immediately.
+[CVSS Version 3.1](https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator) can be
+a helpful tool in evaluating the criticality of reported issues.
+
+### Acknowledge receipt of the problem
+
+Respond to the reporter and notify them that you have received and begun reviewing the problem. Remind them of the [embargo policy](https://github.com/cncf/tag-security/blob/231b87f371274b2d68def2c6a35a719210836191/project-resources/templates/embargo-policy.md), and provide them
+information on who to contact/follow-up with if they have questions. Estimate when they can expect to receive an update. Create a calendar reminder to contact them again by that date to provide an update.
+
+### Replicate the problem
+
+Follow the instructions relayed in the problem. If the instructions are
+insufficient, contact the reporter and ask for more information.
+
+If the problem cannot be replicated, re-engage the reporter, let them know it
+cannot be replicated, and work with them to find a remediation.
+
+If the problem can be replicated, re-evaluate the criticality of the problem, and
+begin working on a remediation. Begin a draft security advisory.
+
+Notify the reporter you were able to replicate the problem and have begun working
+on a fix. Remind them of the [embargo policy](https://github.com/cncf/tag-security/blob/231b87f371274b2d68def2c6a35a719210836191/project-resources/templates/embargo-policy.md). If necessary, notify them of an
+extension (only for very complex problems where remediation cannot be issued
+within the project's specified window).
+
+#### Request a CVE number
+
+If a CVE has already been provided, be sure to include it on the advisory. If
+one has not yet been created, [GitHub functions as a CVE Numbering Authority](https://docs.github.com/en/code-security/security-advisories/about-github-security-advisories#cve-identification-numbers)
+and allows you to request one as part of the security advisory process. Provide
+all required information and as much optional information as we can. The CVE
+number is shown as reserved with no further details until notified it has been
+published.
+
+## Notification
+
+Once the problem has been replicated and a remediation is in place, notify
+subscribed parties with a security bulletin (use [this template](https://github.com/cncf/tag-security/blob/231b87f371274b2d68def2c6a35a719210836191/project-resources/templates/embargo.md)) and the expected publishing date.
+
+## Publish and release
+
+Once a CVE number has been assigned, publish and release the updated
+version/patch. Be sure to notify the CVE group when published so the CVE details
+are searchable. Be sure to give credit to the reporter by *[editing the security
+advisory](https://docs.github.com/en/github/managing-security-vulnerabilities/editing-a-security-advisory#about-credits-for-security-advisories)*
+as they took the time to notify and work with you on the problem!
+
+### Issue a security advisory
+
+Follow the instructions from [GitHub to publish the security advisory previously
+drafted](https://docs.github.com/en/github/managing-security-vulnerabilities/publishing-a-security-advisory).
+
+For more information on security advisories, please refer to the [GitHub
+Article](https://docs.github.com/en/code-security/security-advisories/about-github-security-advisories).

apis/apps/defaults/pod.go

@@ -47,20 +47,20 @@ func SetDefaultPodSpec(in *corev1.PodSpec) {
 		v1.SetDefaults_ResourceList(&a.Resources.Requests)
 		if a.LivenessProbe != nil {
 			v1.SetDefaults_Probe(a.LivenessProbe)
-			if a.LivenessProbe.Handler.HTTPGet != nil {
+			if a.LivenessProbe.ProbeHandler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.LivenessProbe.Handler.HTTPGet)
+				v1.SetDefaults_HTTPGetAction(a.LivenessProbe.ProbeHandler.HTTPGet)
 			}
 		}
 		if a.ReadinessProbe != nil {
 			v1.SetDefaults_Probe(a.ReadinessProbe)
-			if a.ReadinessProbe.Handler.HTTPGet != nil {
+			if a.ReadinessProbe.ProbeHandler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.Handler.HTTPGet)
+				v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.ProbeHandler.HTTPGet)
 			}
 		}
 		if a.StartupProbe != nil {
 			v1.SetDefaults_Probe(a.StartupProbe)
-			if a.StartupProbe.Handler.HTTPGet != nil {
+			if a.StartupProbe.ProbeHandler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.StartupProbe.Handler.HTTPGet)
+				v1.SetDefaults_HTTPGetAction(a.StartupProbe.ProbeHandler.HTTPGet)
 			}
 		}
 		if a.Lifecycle != nil {
@@ -101,20 +101,20 @@ func SetDefaultPodSpec(in *corev1.PodSpec) {
 		v1.SetDefaults_ResourceList(&a.Resources.Requests)
 		if a.LivenessProbe != nil {
 			v1.SetDefaults_Probe(a.LivenessProbe)
-			if a.LivenessProbe.Handler.HTTPGet != nil {
+			if a.LivenessProbe.ProbeHandler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.LivenessProbe.Handler.HTTPGet)
+				v1.SetDefaults_HTTPGetAction(a.LivenessProbe.ProbeHandler.HTTPGet)
 			}
 		}
 		if a.ReadinessProbe != nil {
 			v1.SetDefaults_Probe(a.ReadinessProbe)
-			if a.ReadinessProbe.Handler.HTTPGet != nil {
+			if a.ReadinessProbe.ProbeHandler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.Handler.HTTPGet)
+				v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.ProbeHandler.HTTPGet)
 			}
 		}
 		if a.StartupProbe != nil {
 			v1.SetDefaults_Probe(a.StartupProbe)
-			if a.StartupProbe.Handler.HTTPGet != nil {
+			if a.StartupProbe.ProbeHandler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.StartupProbe.Handler.HTTPGet)
+				v1.SetDefaults_HTTPGetAction(a.StartupProbe.ProbeHandler.HTTPGet)
 			}
 		}
 		if a.Lifecycle != nil {
@@ -150,20 +150,20 @@ func SetDefaultPodSpec(in *corev1.PodSpec) {
 		v1.SetDefaults_ResourceList(&a.EphemeralContainerCommon.Resources.Requests)
 		if a.EphemeralContainerCommon.LivenessProbe != nil {
 			v1.SetDefaults_Probe(a.EphemeralContainerCommon.LivenessProbe)
-			if a.EphemeralContainerCommon.LivenessProbe.Handler.HTTPGet != nil {
+			if a.EphemeralContainerCommon.LivenessProbe.ProbeHandler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.LivenessProbe.Handler.HTTPGet)
+				v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.LivenessProbe.ProbeHandler.HTTPGet)
 			}
 		}
 		if a.EphemeralContainerCommon.ReadinessProbe != nil {
 			v1.SetDefaults_Probe(a.EphemeralContainerCommon.ReadinessProbe)
-			if a.EphemeralContainerCommon.ReadinessProbe.Handler.HTTPGet != nil {
+			if a.EphemeralContainerCommon.ReadinessProbe.ProbeHandler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.ReadinessProbe.Handler.HTTPGet)
+				v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.ReadinessProbe.ProbeHandler.HTTPGet)
 			}
 		}
 		if a.EphemeralContainerCommon.StartupProbe != nil {
 			v1.SetDefaults_Probe(a.EphemeralContainerCommon.StartupProbe)
-			if a.EphemeralContainerCommon.StartupProbe.Handler.HTTPGet != nil {
+			if a.EphemeralContainerCommon.StartupProbe.ProbeHandler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.StartupProbe.Handler.HTTPGet)
+				v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.StartupProbe.ProbeHandler.HTTPGet)
 			}
 		}
 		if a.EphemeralContainerCommon.Lifecycle != nil {
@@ -192,12 +192,6 @@ func SetDefaultPodVolumes(volumes []corev1.Volume) {
 		if a.VolumeSource.Secret != nil {
 			v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret)
 		}
-		if a.VolumeSource.ISCSI != nil {
-			v1.SetDefaults_ISCSIVolumeSource(a.VolumeSource.ISCSI)
-		}
-		if a.VolumeSource.RBD != nil {
-			v1.SetDefaults_RBDVolumeSource(a.VolumeSource.RBD)
-		}
 		if a.VolumeSource.DownwardAPI != nil {
 			v1.SetDefaults_DownwardAPIVolumeSource(a.VolumeSource.DownwardAPI)
 			for j := range a.VolumeSource.DownwardAPI.Items {
@@ -210,9 +204,6 @@ func SetDefaultPodVolumes(volumes []corev1.Volume) {
 		if a.VolumeSource.ConfigMap != nil {
 			v1.SetDefaults_ConfigMapVolumeSource(a.VolumeSource.ConfigMap)
 		}
-		if a.VolumeSource.AzureDisk != nil {
-			v1.SetDefaults_AzureDiskVolumeSource(a.VolumeSource.AzureDisk)
-		}
 		if a.VolumeSource.Projected != nil {
 			v1.SetDefaults_ProjectedVolumeSource(a.VolumeSource.Projected)
 			for j := range a.VolumeSource.Projected.Sources {
@@ -230,8 +221,5 @@ func SetDefaultPodVolumes(volumes []corev1.Volume) {
 				}
 			}
 		}
-		if a.VolumeSource.ScaleIO != nil {
-			v1.SetDefaults_ScaleIOVolumeSource(a.VolumeSource.ScaleIO)
-		}
 	}
 }

apis/apps/defaults/v1alpha1.go

@@ -23,7 +23,13 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	v1 "k8s.io/kubernetes/pkg/apis/core/v1"
-	utilpointer "k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+const (
+	// ProtectionFinalizer is designed to ensure the GC of resources.
+	ProtectionFinalizer = "apps.kruise.io/deletion-protection"
 )
 
 // SetDefaults_SidecarSet set default values for SidecarSet.
@@ -31,11 +37,11 @@ func SetDefaultsSidecarSet(obj *v1alpha1.SidecarSet) {
 	setSidecarSetUpdateStrategy(&obj.Spec.UpdateStrategy)
 
 	for i := range obj.Spec.InitContainers {
-		setSidecarDefaultContainer(&obj.Spec.InitContainers[i])
+		setDefaultSidecarContainer(&obj.Spec.InitContainers[i], v1alpha1.AfterAppContainerType)
 	}
 
 	for i := range obj.Spec.Containers {
-		setDefaultSidecarContainer(&obj.Spec.Containers[i])
+		setDefaultSidecarContainer(&obj.Spec.Containers[i], v1alpha1.BeforeAppContainerType)
 	}
 
 	//default setting volumes
@@ -64,13 +70,13 @@ func SetDefaultInjectRevision(strategy *v1alpha1.SidecarSetInjectionStrategy) {
 
 func SetDefaultRevisionHistoryLimit(revisionHistoryLimit **int32) {
 	if *revisionHistoryLimit == nil {
-		*revisionHistoryLimit = utilpointer.Int32Ptr(10)
+		*revisionHistoryLimit = ptr.To(int32(10))
 	}
 }
 
-func setDefaultSidecarContainer(sidecarContainer *v1alpha1.SidecarContainer) {
+func setDefaultSidecarContainer(sidecarContainer *v1alpha1.SidecarContainer, injectPolicy v1alpha1.PodInjectPolicyType) {
 	if sidecarContainer.PodInjectPolicy == "" {
-		sidecarContainer.PodInjectPolicy = v1alpha1.BeforeAppContainerType
+		sidecarContainer.PodInjectPolicy = injectPolicy
 	}
 	if sidecarContainer.UpgradeStrategy.UpgradeType == "" {
 		sidecarContainer.UpgradeStrategy.UpgradeType = v1alpha1.SidecarContainerColdUpgrade
@@ -79,7 +85,7 @@ func setDefaultSidecarContainer(sidecarContainer *v1alpha1.SidecarContainer) {
 		sidecarContainer.ShareVolumePolicy.Type = v1alpha1.ShareVolumePolicyDisabled
 	}
 
-	setSidecarDefaultContainer(sidecarContainer)
+	setDefaultContainer(sidecarContainer)
 }
 
 func setSidecarSetUpdateStrategy(strategy *v1alpha1.SidecarSetUpdateStrategy) {
@@ -96,7 +102,7 @@ func setSidecarSetUpdateStrategy(strategy *v1alpha1.SidecarSetUpdateStrategy) {
 	}
 }
 
-func setSidecarDefaultContainer(sidecarContainer *v1alpha1.SidecarContainer) {
+func setDefaultContainer(sidecarContainer *v1alpha1.SidecarContainer) {
 	container := &sidecarContainer.Container
 	v1.SetDefaults_Container(container)
 	for i := range container.Ports {
@@ -123,14 +129,14 @@ func setSidecarDefaultContainer(sidecarContainer *v1alpha1.SidecarContainer) {
 	v1.SetDefaults_ResourceList(&container.Resources.Requests)
 	if container.LivenessProbe != nil {
 		v1.SetDefaults_Probe(container.LivenessProbe)
-		if container.LivenessProbe.Handler.HTTPGet != nil {
+		if container.LivenessProbe.ProbeHandler.HTTPGet != nil {
-			v1.SetDefaults_HTTPGetAction(container.LivenessProbe.Handler.HTTPGet)
+			v1.SetDefaults_HTTPGetAction(container.LivenessProbe.ProbeHandler.HTTPGet)
 		}
 	}
 	if container.ReadinessProbe != nil {
 		v1.SetDefaults_Probe(container.ReadinessProbe)
-		if container.ReadinessProbe.Handler.HTTPGet != nil {
+		if container.ReadinessProbe.ProbeHandler.HTTPGet != nil {
-			v1.SetDefaults_HTTPGetAction(container.ReadinessProbe.Handler.HTTPGet)
+			v1.SetDefaults_HTTPGetAction(container.ReadinessProbe.ProbeHandler.HTTPGet)
 		}
 	}
 	if container.Lifecycle != nil {
@@ -192,15 +198,17 @@ func SetDefaultsBroadcastJob(obj *v1alpha1.BroadcastJob, injectTemplateDefaults
 	if obj.Spec.FailurePolicy.Type == "" {
 		obj.Spec.FailurePolicy.Type = v1alpha1.FailurePolicyTypeFailFast
 	}
+
+	// Default to 'OnFailure' if no restartPolicy is specified
+	if obj.Spec.Template.Spec.RestartPolicy == "" {
+		obj.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyOnFailure
+	}
 }
 
 // SetDefaults_UnitedDeployment set default values for UnitedDeployment.
 func SetDefaultsUnitedDeployment(obj *v1alpha1.UnitedDeployment, injectTemplateDefaults bool) {
-	if obj.Spec.Replicas == nil {
-		obj.Spec.Replicas = utilpointer.Int32Ptr(1)
-	}
 	if obj.Spec.RevisionHistoryLimit == nil {
-		obj.Spec.RevisionHistoryLimit = utilpointer.Int32Ptr(10)
+		obj.Spec.RevisionHistoryLimit = ptr.To(int32(10))
 	}
 
 	if len(obj.Spec.UpdateStrategy.Type) == 0 {
@@ -223,15 +231,34 @@ func SetDefaultsUnitedDeployment(obj *v1alpha1.UnitedDeployment, injectTemplateD
 			}
 		}
 	}
+
+	hasReplicasSettings := false
+	hasCapacitySettings := false
+	for _, subset := range obj.Spec.Topology.Subsets {
+		if subset.Replicas != nil {
+			hasReplicasSettings = true
+		}
+		if subset.MinReplicas != nil || subset.MaxReplicas != nil {
+			hasCapacitySettings = true
+		}
+	}
+	if hasCapacitySettings && !hasReplicasSettings {
+		for i := range obj.Spec.Topology.Subsets {
+			subset := &obj.Spec.Topology.Subsets[i]
+			if subset.MinReplicas == nil {
+				subset.MinReplicas = &intstr.IntOrString{Type: intstr.Int, IntVal: 0}
+			}
+		}
+	}
 }
 
 // SetDefaults_CloneSet set default values for CloneSet.
 func SetDefaultsCloneSet(obj *v1alpha1.CloneSet, injectTemplateDefaults bool) {
 	if obj.Spec.Replicas == nil {
-		obj.Spec.Replicas = utilpointer.Int32Ptr(1)
+		obj.Spec.Replicas = ptr.To(int32(1))
 	}
 	if obj.Spec.RevisionHistoryLimit == nil {
-		obj.Spec.RevisionHistoryLimit = utilpointer.Int32Ptr(10)
+		obj.Spec.RevisionHistoryLimit = ptr.To(int32(10))
 	}
 
 	if injectTemplateDefaults {
@@ -343,15 +370,15 @@ func SetDefaultsNodeImage(obj *v1alpha1.NodeImage) {
 
 func SetDefaultsImageTagPullPolicy(obj *v1alpha1.ImageTagPullPolicy) {
 	if obj.TimeoutSeconds == nil {
-		obj.TimeoutSeconds = utilpointer.Int32Ptr(600)
+		obj.TimeoutSeconds = ptr.To(int32(600))
 	}
 	if obj.BackoffLimit == nil {
-		obj.BackoffLimit = utilpointer.Int32Ptr(3)
+		obj.BackoffLimit = ptr.To(int32(3))
 	}
 }
 
 // SetDefaults_ImagePullJob set default values for ImagePullJob.
-func SetDefaultsImagePullJob(obj *v1alpha1.ImagePullJob) {
+func SetDefaultsImagePullJob(obj *v1alpha1.ImagePullJob, addProtection bool) {
 	if obj.Spec.CompletionPolicy.Type == "" {
 		obj.Spec.CompletionPolicy.Type = v1alpha1.Always
 	}
@@ -359,9 +386,31 @@ func SetDefaultsImagePullJob(obj *v1alpha1.ImagePullJob) {
 		obj.Spec.PullPolicy = &v1alpha1.PullPolicy{}
 	}
 	if obj.Spec.PullPolicy.TimeoutSeconds == nil {
-		obj.Spec.PullPolicy.TimeoutSeconds = utilpointer.Int32Ptr(600)
+		obj.Spec.PullPolicy.TimeoutSeconds = ptr.To(int32(600))
 	}
 	if obj.Spec.PullPolicy.BackoffLimit == nil {
-		obj.Spec.PullPolicy.BackoffLimit = utilpointer.Int32Ptr(3)
+		obj.Spec.PullPolicy.BackoffLimit = ptr.To(int32(3))
+	}
+	if obj.Spec.ImagePullPolicy == "" {
+		obj.Spec.ImagePullPolicy = v1alpha1.PullIfNotPresent
+	}
+	if addProtection {
+		controllerutil.AddFinalizer(obj, ProtectionFinalizer)
+	}
+}
+
+// SetDefaultsImageListPullJob  set default values for ImageListPullJob.
+func SetDefaultsImageListPullJob(obj *v1alpha1.ImageListPullJob) {
+	if obj.Spec.CompletionPolicy.Type == "" {
+		obj.Spec.CompletionPolicy.Type = v1alpha1.Always
+	}
+	if obj.Spec.PullPolicy == nil {
+		obj.Spec.PullPolicy = &v1alpha1.PullPolicy{}
+	}
+	if obj.Spec.PullPolicy.TimeoutSeconds == nil {
+		obj.Spec.PullPolicy.TimeoutSeconds = ptr.To(int32(600))
+	}
+	if obj.Spec.PullPolicy.BackoffLimit == nil {
+		obj.Spec.PullPolicy.BackoffLimit = ptr.To(int32(3))
 	}
 }

apis/apps/defaults/v1beta1.go

@@ -17,13 +17,14 @@ limitations under the License.
 package defaults
 
 import (
-	"github.com/openkruise/kruise/apis/apps/v1beta1"
-	"github.com/openkruise/kruise/pkg/features"
-	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
 	appsv1 "k8s.io/api/apps/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	v1 "k8s.io/kubernetes/pkg/apis/core/v1"
-	utilpointer "k8s.io/utils/pointer"
+	"k8s.io/utils/ptr"
+
+	"github.com/openkruise/kruise/apis/apps/v1beta1"
+	"github.com/openkruise/kruise/pkg/features"
+	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
 )
 
 // SetDefaultsStatefulSet set default values for StatefulSet.
@@ -42,7 +43,7 @@ func SetDefaultsStatefulSet(obj *v1beta1.StatefulSet, injectTemplateDefaults boo
 			obj.Spec.UpdateStrategy.RollingUpdate = &v1beta1.RollingUpdateStatefulSetStrategy{}
 		}
 		if obj.Spec.UpdateStrategy.RollingUpdate.Partition == nil {
-			obj.Spec.UpdateStrategy.RollingUpdate.Partition = utilpointer.Int32Ptr(0)
+			obj.Spec.UpdateStrategy.RollingUpdate.Partition = ptr.To(int32(0))
 		}
 		if obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable == nil {
 			maxUnavailable := intstr.FromInt(1)
@@ -52,7 +53,7 @@ func SetDefaultsStatefulSet(obj *v1beta1.StatefulSet, injectTemplateDefaults boo
 			obj.Spec.UpdateStrategy.RollingUpdate.PodUpdatePolicy = v1beta1.RecreatePodUpdateStrategyType
 		}
 		if obj.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds == nil {
-			obj.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds = utilpointer.Int32Ptr(0)
+			obj.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds = ptr.To(int32(0))
 		}
 	}
 
@@ -68,11 +69,17 @@ func SetDefaultsStatefulSet(obj *v1beta1.StatefulSet, injectTemplateDefaults boo
 		}
 	}
 
+	if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoResizePVCGate) {
+		if obj.Spec.VolumeClaimUpdateStrategy.Type == "" {
+			obj.Spec.VolumeClaimUpdateStrategy.Type = v1beta1.OnPVCDeleteVolumeClaimUpdateStrategyType
+		}
+	}
+
 	if obj.Spec.Replicas == nil {
-		obj.Spec.Replicas = utilpointer.Int32Ptr(1)
+		obj.Spec.Replicas = ptr.To(int32(1))
 	}
 	if obj.Spec.RevisionHistoryLimit == nil {
-		obj.Spec.RevisionHistoryLimit = utilpointer.Int32Ptr(10)
+		obj.Spec.RevisionHistoryLimit = ptr.To(int32(10))
 	}
 
 	if injectTemplateDefaults {

apis/apps/pub/inplace_update.go

@@ -62,12 +62,21 @@ type InPlaceUpdateState struct {
 	// UpdateEnvFromMetadata indicates there are envs from annotations/labels that should be in-place update.
 	UpdateEnvFromMetadata bool `json:"updateEnvFromMetadata,omitempty"`
 
+	// UpdateResources indicates there are resources that should be in-place update.
+	UpdateResources bool `json:"updateResources,omitempty"`
+
+	// UpdateImages indicates there are images that should be in-place update.
+	UpdateImages bool `json:"updateImages,omitempty"`
+
 	// NextContainerImages is the containers with lower priority that waiting for in-place update images in next batch.
 	NextContainerImages map[string]string `json:"nextContainerImages,omitempty"`
 
 	// NextContainerRefMetadata is the containers with lower priority that waiting for in-place update labels/annotations in next batch.
 	NextContainerRefMetadata map[string]metav1.ObjectMeta `json:"nextContainerRefMetadata,omitempty"`
 
+	// NextContainerResources is the containers with lower priority that waiting for in-place update resources in next batch.
+	NextContainerResources map[string]v1.ResourceRequirements `json:"nextContainerResources,omitempty"`
+
 	// PreCheckBeforeNext is the pre-check that must pass before the next containers can be in-place update.
 	PreCheckBeforeNext *InPlaceUpdatePreCheckBeforeNext `json:"preCheckBeforeNext,omitempty"`
 

apis/apps/pub/launch_priority.go

@@ -29,4 +29,8 @@ const (
 	ContainerLaunchPriorityKey = "apps.kruise.io/container-launch-priority"
 	// ContainerLaunchOrdered is the annotation value that indicates containers in pod should be launched by ordinal.
 	ContainerLaunchOrdered = "Ordered"
+
+	// ContainerLaunchPriorityCompletedKey is the annotation indicates the pod has all its priorities
+	// patched into its barrier configmap.
+	ContainerLaunchPriorityCompletedKey = "apps.kruise.io/container-launch-priority-completed"
 )

apis/apps/pub/zz_generated.deepcopy.go

@@ -1,5 +1,4 @@
 //go:build !ignore_autogenerated
-// +build !ignore_autogenerated
 
 /*
 Copyright 2021 The Kruise Authors.
@@ -22,6 +21,7 @@ limitations under the License.
 package pub
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -106,6 +106,13 @@ func (in *InPlaceUpdateState) DeepCopyInto(out *InPlaceUpdateState) {
 			(*out)[key] = *val.DeepCopy()
 		}
 	}
+	if in.NextContainerResources != nil {
+		in, out := &in.NextContainerResources, &out.NextContainerResources
+		*out = make(map[string]corev1.ResourceRequirements, len(*in))
+		for key, val := range *in {
+			(*out)[key] = *val.DeepCopy()
+		}
+	}
 	if in.PreCheckBeforeNext != nil {
 		in, out := &in.PreCheckBeforeNext, &out.PreCheckBeforeNext
 		*out = new(InPlaceUpdatePreCheckBeforeNext)

apis/apps/v1alpha1/cloneset_types.go

@@ -17,10 +17,11 @@ limitations under the License.
 package v1alpha1
 
 import (
-	appspub "github.com/openkruise/kruise/apis/apps/pub"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
 )
 
 const (
@@ -176,6 +177,12 @@ type CloneSetStatus struct {
 	// indicated by updateRevision and have a Ready Condition.
 	UpdatedReadyReplicas int32 `json:"updatedReadyReplicas"`
 
+	// UpdatedAvailableReplicas is the number of Pods created by the CloneSet controller from the CloneSet version
+	// indicated by updateRevision and have a Ready Condition for at least minReadySeconds.
+	// Notice: when enable InPlaceWorkloadVerticalScaling, pod during resource resizing will also be unavailable.
+	// This means these pod will be counted in maxUnavailable.
+	UpdatedAvailableReplicas int32 `json:"updatedAvailableReplicas,omitempty"`
+
 	// ExpectedUpdatedReplicas is the number of Pods that should be updated by CloneSet controller.
 	// This field is calculated via Replicas - Partition.
 	ExpectedUpdatedReplicas int32 `json:"expectedUpdatedReplicas,omitempty"`
@@ -233,6 +240,7 @@ type CloneSetCondition struct {
 // +kubebuilder:printcolumn:name="DESIRED",type="integer",JSONPath=".spec.replicas",description="The desired number of pods."
 // +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedReplicas",description="The number of pods updated."
 // +kubebuilder:printcolumn:name="UPDATED_READY",type="integer",JSONPath=".status.updatedReadyReplicas",description="The number of pods updated and ready."
+// +kubebuilder:printcolumn:name="UPDATED_AVAILABLE",type="integer",JSONPath=".status.updatedAvailableReplicas",description="The number of pods updated and available."
 // +kubebuilder:printcolumn:name="READY",type="integer",JSONPath=".status.readyReplicas",description="The number of pods ready."
 // +kubebuilder:printcolumn:name="TOTAL",type="integer",JSONPath=".status.replicas",description="The number of currently all pods."
 // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."

apis/apps/v1alpha1/containerrecreaterequest_types.go

@@ -44,7 +44,9 @@ type ContainerRecreateRequestSpec struct {
 	// PodName is name of the Pod that owns the recreated containers.
 	PodName string `json:"podName"`
 	// Containers contains the containers that need to recreate in the Pod.
-	Containers []ContainerRecreateRequestContainer `json:"containers"`
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Containers []ContainerRecreateRequestContainer `json:"containers" patchStrategy:"merge" patchMergeKey:"name"`
 	// Strategy defines strategies for containers recreation.
 	Strategy *ContainerRecreateRequestStrategy `json:"strategy,omitempty"`
 	// ActiveDeadlineSeconds is the deadline duration of this ContainerRecreateRequest.

apis/apps/v1alpha1/daemonset_types.go

@@ -17,11 +17,12 @@ limitations under the License.
 package v1alpha1
 
 import (
-	appspub "github.com/openkruise/kruise/apis/apps/pub"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
 )
 
 // DaemonSetUpdateStrategy is a struct used to control the update strategy for a DaemonSet.
@@ -91,7 +92,7 @@ type RollingUpdateDaemonSet struct {
 	// pod is available (Ready for at least minReadySeconds) the old DaemonSet pod
 	// on that node is marked deleted. If the old pod becomes unavailable for any
 	// reason (Ready transitions to false, is evicted, or is drained) an updated
-	// pod is immediatedly created on that node without considering surge limits.
+	// pod is immediately created on that node without considering surge limits.
 	// Allowing surge implies the possibility that the resources consumed by the
 	// daemonset on any given node can double if the readiness check fails, and
 	// so resource intensive daemonsets should take into account that they may

apis/apps/v1alpha1/ephemeraljob_types.go

@@ -75,7 +75,9 @@ type EphemeralContainerTemplateSpec struct {
 	// EphemeralContainers defines ephemeral container list in match pods.
 	// +kubebuilder:pruning:PreserveUnknownFields
 	// +kubebuilder:validation:Schemaless
-	EphemeralContainers []v1.EphemeralContainer `json:"ephemeralContainers"`
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	EphemeralContainers []v1.EphemeralContainer `json:"ephemeralContainers" patchStrategy:"merge" patchMergeKey:"name"`
 }
 
 // EphemeralJobStatus defines the observed state of EphemeralJob

apis/apps/v1alpha1/imagelistpulljob_types.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2023 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ImageListPullJobSpec defines the desired state of ImageListPullJob
+type ImageListPullJobSpec struct {
+	// Images is the image list to be pulled by the job
+	Images []string `json:"images"`
+
+	ImagePullJobTemplate `json:",inline"`
+}
+
+// ImageListPullJobStatus defines the observed state of ImageListPullJob
+type ImageListPullJobStatus struct {
+	// Represents time when the job was acknowledged by the job controller.
+	// It is not guaranteed to be set in happens-before order across separate operations.
+	// It is represented in RFC3339 form and is in UTC.
+	// +optional
+	StartTime *metav1.Time `json:"startTime,omitempty"`
+
+	// Represents time when the all the image pull job was completed. It is not guaranteed to
+	// be set in happens-before order across separate operations.
+	// It is represented in RFC3339 form and is in UTC.
+	// +optional
+	CompletionTime *metav1.Time `json:"completionTime,omitempty"`
+
+	// The desired number of ImagePullJobs, this is typically equal to the number of len(spec.Images).
+	Desired int32 `json:"desired"`
+
+	// The number of running ImagePullJobs which are acknowledged by the imagepulljob controller.
+	// +optional
+	Active int32 `json:"active"`
+
+	// The number of ImagePullJobs which are finished
+	// +optional
+	Completed int32 `json:"completed"`
+
+	// The number of image pull job which are finished and status.Succeeded==status.Desired.
+	// +optional
+	Succeeded int32 `json:"succeeded"`
+
+	// The status of ImagePullJob which has the failed nodes(status.Failed>0) .
+	// +optional
+	FailedImageStatuses []*FailedImageStatus `json:"failedImageStatuses,omitempty"`
+}
+
+// FailedImageStatus the state of ImagePullJob which has the failed nodes(status.Failed>0)
+type FailedImageStatus struct {
+	// The name of ImagePullJob which has the failed nodes(status.Failed>0)
+	// +optional
+	ImagePullJob string `json:"imagePullJob,omitempty"`
+
+	// Name of the image
+	// +optional
+	Name string `json:"name,omitempty"`
+
+	// The text prompt for job running status.
+	// +optional
+	Message string `json:"message,omitempty"`
+}
+
+// +genclient
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="TOTAL",type="integer",JSONPath=".status.desired",description="Number of image pull job"
+// +kubebuilder:printcolumn:name="SUCCEEDED",type="integer",JSONPath=".status.succeeded",description="Number of image pull job succeeded"
+// +kubebuilder:printcolumn:name="COMPLETED",type="integer",JSONPath=".status.completed",description="Number of ImagePullJobs which are finished"
+// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."
+
+// ImageListPullJob is the Schema for the imagelistpulljobs API
+type ImageListPullJob struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ImageListPullJobSpec   `json:"spec,omitempty"`
+	Status ImageListPullJobStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// ImageListPullJobList contains a list of ImageListPullJob
+type ImageListPullJobList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ImageListPullJob `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&ImageListPullJob{}, &ImageListPullJobList{})
+}

apis/apps/v1alpha1/imagepulljob_types.go

@@ -27,10 +27,25 @@ const (
 	ImagePreDownloadMinUpdatedReadyPods = "apps.kruise.io/image-predownload-min-updated-ready-pods"
 )
 
+// ImagePullPolicy describes a policy for if/when to pull a container image
+// +enum
+type ImagePullPolicy string
+
+const (
+	// PullAlways means that kruise-daemon always attempts to pull the latest image.
+	PullAlways ImagePullPolicy = "Always"
+	// PullIfNotPresent means that kruise-daemon pulls if the image isn't present on disk.
+	PullIfNotPresent ImagePullPolicy = "IfNotPresent"
+)
+
 // ImagePullJobSpec defines the desired state of ImagePullJob
 type ImagePullJobSpec struct {
 	// Image is the image to be pulled by the job
-	Image string `json:"image"`
+	Image                string `json:"image"`
+	ImagePullJobTemplate `json:",inline"`
+}
+
+type ImagePullJobTemplate struct {
 
 	// ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling the image.
 	// If specified, these secrets will be passed to individual puller implementations for them to use.  For example,
@@ -65,6 +80,11 @@ type ImagePullJobSpec struct {
 	// SandboxConfig support attach metadata in PullImage CRI interface during ImagePulljobs
 	// +optional
 	SandboxConfig *SandboxConfig `json:"sandboxConfig,omitempty"`
+
+	// Image pull policy.
+	// One of Always, IfNotPresent. Defaults to IfNotPresent.
+	// +optional
+	ImagePullPolicy ImagePullPolicy `json:"imagePullPolicy,omitempty"`
 }
 
 // ImagePullJobPodSelector is a selector over pods

apis/apps/v1alpha1/node_pod_probe_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
@@ -32,6 +33,8 @@ type PodProbe struct {
 	Namespace string `json:"namespace"`
 	// pod uid
 	UID string `json:"uid"`
+	// pod ip
+	IP string `json:"IP"`
 	// Custom container probe, supports Exec, Tcp, and returns the result to Pod yaml
 	Probes []ContainerProbe `json:"probes,omitempty"`
 }
@@ -85,6 +88,17 @@ const (
 	ProbeUnknown   ProbeState = "Unknown"
 )
 
+func (p ProbeState) IsEqualPodConditionStatus(status corev1.ConditionStatus) bool {
+	switch status {
+	case corev1.ConditionTrue:
+		return p == ProbeSucceeded
+	case corev1.ConditionFalse:
+		return p == ProbeFailed
+	default:
+		return p == ProbeUnknown
+	}
+}
+
 // +genclient
 // +genclient:nonNamespaced
 // +k8s:openapi-gen=true

apis/apps/v1alpha1/nodeimage_types.go

@@ -80,6 +80,11 @@ type ImageTagSpec struct {
 	// Value must be treated as opaque by clients and .
 	// +optional
 	Version int64 `json:"version,omitempty"`
+
+	// Image pull policy.
+	// One of Always, IfNotPresent. Defaults to IfNotPresent.
+	// +optional
+	ImagePullPolicy ImagePullPolicy `json:"imagePullPolicy,omitempty"`
 }
 
 // ImageTagPullPolicy defines the policy of the pulling task
@@ -125,6 +130,10 @@ type NodeImageStatus struct {
 	// +optional
 	Pulling int32 `json:"pulling"`
 
+	// The number of pulling tasks which are waiting.
+	// +optional
+	Waiting int32 `json:"waiting"`
+
 	// all statuses of active image pulling tasks
 	ImageStatuses map[string]ImageStatus `json:"imageStatuses,omitempty"`
 

apis/apps/v1alpha1/pod_probe_marker_types.go

@@ -21,6 +21,31 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
+const (
+	// PodProbeMarkerAnnotationKey records the Probe Spec, mainly used for serverless Pod scenarios, as follows:
+	//  annotations:
+	//    kruise.io/podprobe: |
+	//      [
+	//          {
+	//              "containerName": "minecraft",
+	//              "name": "healthy",
+	//              "podConditionType": "game.kruise.io/healthy",
+	//              "probe": {
+	//                  "exec": {
+	//                      "command": [
+	//                          "bash",
+	//                          "/data/probe.sh"
+	//                      ]
+	//                  }
+	//              }
+	//          }
+	//      ]
+	PodProbeMarkerAnnotationKey = "kruise.io/podprobe"
+	// PodProbeMarkerListAnnotationKey records the injected PodProbeMarker Name List
+	// example: kruise.io/podprobemarker-list="probe-marker-1,probe-marker-2"
+	PodProbeMarkerListAnnotationKey = "kruise.io/podprobemarker-list"
+)
+
 // PodProbeMarkerSpec defines the desired state of PodProbeMarker
 type PodProbeMarkerSpec struct {
 	// Selector is a label query over pods that should exec custom probe
@@ -31,7 +56,9 @@ type PodProbeMarkerSpec struct {
 	// Probe Result will record in Pod.Status.Conditions, and condition.type=probe.name.
 	// condition.status=True indicates probe success
 	// condition.status=False indicates probe fails
-	Probes []PodContainerProbe `json:"probes"`
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Probes []PodContainerProbe `json:"probes" patchStrategy:"merge" patchMergeKey:"name"`
 }
 
 type PodContainerProbe struct {
@@ -44,7 +71,9 @@ type PodContainerProbe struct {
 	// According to the execution result of ContainerProbe, perform specific actions,
 	// such as: patch Pod labels, annotations, ReadinessGate Condition
 	// It cannot be null at the same time as PodConditionType.
-	MarkerPolicy []ProbeMarkerPolicy `json:"markerPolicy,omitempty"`
+	// +patchMergeKey=state
+	// +patchStrategy=merge
+	MarkerPolicy []ProbeMarkerPolicy `json:"markerPolicy,omitempty"  patchStrategy:"merge" patchMergeKey:"state"`
 	// If it is not empty, the Probe execution result will be recorded on the Pod condition.
 	// It cannot be null at the same time as MarkerPolicy.
 	// For example PodConditionType=game.kruise.io/healthy, pod.status.condition.type = game.kruise.io/healthy.

apis/apps/v1alpha1/resourcedistribution_types.go

@@ -69,8 +69,10 @@ type ResourceDistributionTargetNamespaces struct {
 		Pattern string `json:"pattern,omitempty"`
 	*/
 
+	// +patchMergeKey=name
+	// +patchStrategy=merge
 	// +optional
-	List []ResourceDistributionNamespace `json:"list,omitempty"`
+	List []ResourceDistributionNamespace `json:"list,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
 }
 
 // ResourceDistributionNamespace contains a namespace name

apis/apps/v1alpha1/sidecarset_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
@@ -48,15 +49,21 @@ type SidecarSetSpec struct {
 	// InitContainers is the list of init containers to be injected into the selected pod
 	// We will inject those containers by their name in ascending order
 	// We only inject init containers when a new pod is created, it does not apply to any existing pod
-	InitContainers []SidecarContainer `json:"initContainers,omitempty"`
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	InitContainers []SidecarContainer `json:"initContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
 
 	// Containers is the list of sidecar containers to be injected into the selected pod
-	Containers []SidecarContainer `json:"containers,omitempty"`
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Containers []SidecarContainer `json:"containers,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
 
 	// List of volumes that can be mounted by sidecar containers
 	// +kubebuilder:pruning:PreserveUnknownFields
 	// +kubebuilder:validation:Schemaless
-	Volumes []corev1.Volume `json:"volumes,omitempty"`
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Volumes []corev1.Volume `json:"volumes,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
 
 	// The sidecarset updateStrategy to use to replace existing pods with new ones.
 	UpdateStrategy SidecarSetUpdateStrategy `json:"updateStrategy,omitempty"`
@@ -65,7 +72,9 @@ type SidecarSetSpec struct {
 	InjectionStrategy SidecarSetInjectionStrategy `json:"injectionStrategy,omitempty"`
 
 	// List of the names of secrets required by pulling sidecar container images
-	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty"`
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
 
 	// RevisionHistoryLimit indicates the maximum quantity of stored revisions about the SidecarSet.
 	// default value is 10
@@ -123,9 +132,14 @@ type SidecarContainer struct {
 	UpgradeStrategy SidecarContainerUpgradeStrategy `json:"upgradeStrategy,omitempty"`
 
 	// If ShareVolumePolicy is enabled, the sidecar container will share the other container's VolumeMounts
-	// in the pod(don't contains the injected sidecar container).
+	// in the pod(not including the injected sidecar container).
 	ShareVolumePolicy ShareVolumePolicy `json:"shareVolumePolicy,omitempty"`
 
+	// If ShareVolumeDevicePolicy is enabled, the sidecar container will share the other container's VolumeDevices
+	// in the pod(don't contain the injected sidecar container).
+	// This is a pointer to ensure that the sidecarset-hash does not change if the user does not configure this field, mainly for compatibility with older versions.
+	ShareVolumeDevicePolicy *ShareVolumePolicy `json:"shareVolumeDevicePolicy,omitempty"`
+
 	// TransferEnv will transfer env info from other container
 	// SourceContainerName is pod.spec.container[x].name; EnvName is pod.spec.container[x].Env.name
 	TransferEnv []TransferEnvVar `json:"transferEnv,omitempty"`
@@ -207,7 +221,8 @@ type SidecarSetInjectRevision struct {
 	// + optional
 	RevisionName *string `json:"revisionName,omitempty"`
 	// Policy describes the behavior of revision injection.
-	// Defaults to Always.
+	// +kubebuilder:validation:Enum=Always;Partial;
+	// +kubebuilder:default=Always
 	Policy SidecarSetInjectRevisionPolicy `json:"policy,omitempty"`
 }
 
@@ -217,9 +232,15 @@ const (
 	// AlwaysSidecarSetInjectRevisionPolicy means the SidecarSet will always inject
 	// the specific revision to Pods when pod creating, except matching UpdateStrategy.Selector.
 	AlwaysSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Always"
-	// PartitionBasedSidecarSetInjectRevisionPolicy means the SidecarSet will inject the
+
-	// specific or the latest revision according to Partition.
+	// PartialSidecarSetInjectRevisionPolicy means the SidecarSet will inject the specific or the latest revision according to UpdateStrategy.
-	//PartitionBasedSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "PartitionBased"
+	//
+	// If UpdateStrategy.Pause is not true, only when a newly created Pod is **not** selected by the Selector explicitly
+	// configured in `UpdateStrategy` will it be injected with the specified version of the Sidecar.
+	// Under all other conditions, newly created Pods have a probability of being injected with the latest Sidecar,
+	// where the probability is `1 - UpdateStrategy.Partition`.
+	// If `Partition` is not a percentage or is not configured, its value is considered to be 0%.
+	PartialSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Partial"
 )
 
 // SidecarSetUpdateStrategy indicates the strategy that the SidecarSet
@@ -233,11 +254,15 @@ type SidecarSetUpdateStrategy struct {
 	Type SidecarSetUpdateStrategyType `json:"type,omitempty"`
 
 	// Paused indicates that the SidecarSet is paused to update the injected pods,
-	// but it don't affect the webhook inject sidecar container into the newly created pods.
+	// For the impact on the injection behavior for newly created Pods, please refer to the comments of Selector.
-	// default is false
 	Paused bool `json:"paused,omitempty"`
 
 	// If selector is not nil, this upgrade will only update the selected pods.
+	//
+	// Starting from Kruise 1.8.0, the updateStrategy.Selector affects the version of the Sidecar container
+	// injected into newly created Pods by a SidecarSet configured with an injectionStrategy.
+	// In most cases, all newly created Pods are injected with the specified Sidecar version as configured in injectionStrategy.revision,
+	// which is consistent with previous versions.
 	Selector *metav1.LabelSelector `json:"selector,omitempty"`
 
 	// Partition is the desired number of pods in old revisions. It means when partition
@@ -252,7 +277,9 @@ type SidecarSetUpdateStrategy struct {
 	// This cannot be 0.
 	// Default value is 1.
 	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
-
+	// Priorities are the rules for calculating the priority of updating pods.
+	// Each pod to be updated, will pass through these terms and get a sum of weights.
+	PriorityStrategy *appspub.UpdatePriorityStrategy `json:"priorityStrategy,omitempty"`
 	// ScatterStrategy defines the scatter rules to make pods been scattered when update.
 	// This will avoid pods with the same key-value to be updated in one batch.
 	// - Note that pods will be scattered after priority sort. So, although priority strategy and scatter strategy can be applied together, we suggest to use either one of them.

apis/apps/v1alpha1/uniteddeployment_types.go

@@ -17,10 +17,13 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"time"
+
 	"github.com/openkruise/kruise/apis/apps/v1beta1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
@@ -45,6 +48,8 @@ const (
 	SubsetUpdated UnitedDeploymentConditionType = "SubsetUpdated"
 	// SubsetFailure is added to a UnitedDeployment when one of its subsets has failure during its own reconciling.
 	SubsetFailure UnitedDeploymentConditionType = "SubsetFailure"
+	// UnitedDeploymentUpdated means currentRevision is equal to updatedRevision.
+	UnitedDeploymentUpdated UnitedDeploymentConditionType = "UnitedDeploymentUpdated"
 )
 
 // UnitedDeploymentSpec defines the desired state of UnitedDeployment.
@@ -159,8 +164,14 @@ type ManualUpdate struct {
 type Topology struct {
 	// Contains the details of each subset. Each element in this array represents one subset
 	// which will be provisioned and managed by UnitedDeployment.
+	// +patchMergeKey=name
+	// +patchStrategy=merge
 	// +optional
-	Subsets []Subset `json:"subsets,omitempty"`
+	Subsets []Subset `json:"subsets,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
+
+	// ScheduleStrategy indicates the strategy the UnitedDeployment used to preform the schedule between each of subsets.
+	// +optional
+	ScheduleStrategy UnitedDeploymentScheduleStrategy `json:"scheduleStrategy,omitempty"`
 }
 
 // Subset defines the detail of a subset.
@@ -185,8 +196,107 @@ type Subset struct {
 	// percentage like '10%', which means 10% of UnitedDeployment replicas of pods will be distributed
 	// under this subset. If nil, the number of replicas in this subset is determined by controller.
 	// Controller will try to keep all the subsets with nil replicas have average pods.
+	// Replicas and MinReplicas/MaxReplicas are mutually exclusive in a UnitedDeployment.
 	// +optional
 	Replicas *intstr.IntOrString `json:"replicas,omitempty"`
+
+	// Indicates the lower bounded replicas of the subset.
+	// MinReplicas must be more than or equal to 0 if it is set.
+	// Controller will prioritize satisfy minReplicas for each subset
+	// according to the order of Topology.Subsets.
+	// Defaults to 0.
+	// +optional
+	MinReplicas *intstr.IntOrString `json:"minReplicas,omitempty"`
+
+	// Indicates the upper bounded replicas of the subset.
+	// MaxReplicas must be more than or equal to MinReplicas.
+	// MaxReplicas == nil means no limitation.
+	// Please ensure that at least one subset has empty MaxReplicas(no limitation) to avoid stuck scaling.
+	// Defaults to nil.
+	// +optional
+	MaxReplicas *intstr.IntOrString `json:"maxReplicas,omitempty"`
+
+	// Patch indicates patching to the templateSpec.
+	// Patch takes precedence over other fields
+	// If the Patch also modifies the Replicas, NodeSelectorTerm or Tolerations, use value in the Patch
+	// +optional
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	Patch runtime.RawExtension `json:"patch,omitempty"`
+}
+
+// UnitedDeploymentScheduleStrategyType is a string enumeration type that enumerates
+// all possible schedule strategies for the UnitedDeployment controller.
+// +kubebuilder:validation:Enum=Adaptive;Fixed;""
+type UnitedDeploymentScheduleStrategyType string
+
+const (
+	// AdaptiveUnitedDeploymentScheduleStrategyType represents that when a pod is stuck in the pending status and cannot
+	// be scheduled, allow it to be rescheduled to another subset.
+	AdaptiveUnitedDeploymentScheduleStrategyType UnitedDeploymentScheduleStrategyType = "Adaptive"
+	// FixedUnitedDeploymentScheduleStrategyType represents that pods are strictly scheduled to the selected subset
+	// even if scheduling fail.
+	FixedUnitedDeploymentScheduleStrategyType UnitedDeploymentScheduleStrategyType = "Fixed"
+)
+
+const (
+	DefaultRescheduleCriticalDuration      = 30 * time.Second
+	DefaultUnschedulableStatusLastDuration = 300 * time.Second
+)
+
+// AdaptiveUnitedDeploymentStrategy is used to communicate parameters when Type is AdaptiveUnitedDeploymentScheduleStrategyType.
+type AdaptiveUnitedDeploymentStrategy struct {
+	// RescheduleCriticalSeconds indicates how long controller will reschedule a schedule failed Pod to the subset that has
+	// redundant capacity after the subset where the Pod lives. If a Pod was scheduled failed and still in an unschedulabe status
+	// over RescheduleCriticalSeconds duration, the controller will reschedule it to a suitable subset. Default is 30 seconds.
+	// +optional
+	RescheduleCriticalSeconds *int32 `json:"rescheduleCriticalSeconds,omitempty"`
+
+	// UnschedulableDuration is used to set the number of seconds for a Subset to recover from an unschedulable state,
+	// with a default value of 300 seconds.
+	// +optional
+	UnschedulableDuration *int32 `json:"unschedulableDuration,omitempty"`
+
+	// ReserveUnschedulablePods indicates whether to enable reservation rescheduling mode, which is disabled by default.
+	// If this feature is enabled, those pending pods that would otherwise be permanently transferred to other subsets
+	// due to scheduling failure will be retained, and a temporary substitute Pod will be created in another subset to take over its work.
+	// When the retained pod is successfully scheduled and ready, its temporary substitute will be deleted.
+	// +optional
+	ReserveUnschedulablePods bool `json:"reserveUnschedulablePods,omitempty"`
+}
+
+// UnitedDeploymentScheduleStrategy defines the schedule performance of UnitedDeployment.
+type UnitedDeploymentScheduleStrategy struct {
+	// Type indicates the type of the UnitedDeploymentScheduleStrategy.
+	// Default is Fixed
+	// +optional
+	Type UnitedDeploymentScheduleStrategyType `json:"type,omitempty"`
+
+	// Adaptive is used to communicate parameters when Type is AdaptiveUnitedDeploymentScheduleStrategyType.
+	// +optional
+	Adaptive *AdaptiveUnitedDeploymentStrategy `json:"adaptive,omitempty"`
+}
+
+func (s *UnitedDeploymentScheduleStrategy) IsAdaptive() bool {
+	return s.Type == AdaptiveUnitedDeploymentScheduleStrategyType
+}
+
+func (s *UnitedDeploymentScheduleStrategy) ShouldReserveUnschedulablePods() bool {
+	return s.IsAdaptive() && s.Adaptive != nil && s.Adaptive.ReserveUnschedulablePods
+}
+
+func (s *UnitedDeploymentScheduleStrategy) GetRescheduleCriticalDuration() time.Duration {
+	if s.Adaptive == nil || s.Adaptive.RescheduleCriticalSeconds == nil {
+		return DefaultRescheduleCriticalDuration
+	}
+	return time.Duration(*s.Adaptive.RescheduleCriticalSeconds) * time.Second
+}
+
+func (s *UnitedDeploymentScheduleStrategy) GetUnschedulableDuration() time.Duration {
+	if s.Adaptive == nil || s.Adaptive.UnschedulableDuration == nil {
+		return DefaultUnschedulableStatusLastDuration
+	}
+	return time.Duration(*s.Adaptive.UnschedulableDuration) * time.Second
 }
 
 // UnitedDeploymentStatus defines the observed state of UnitedDeployment.
@@ -206,6 +316,9 @@ type UnitedDeploymentStatus struct {
 	// The number of pods in current version.
 	UpdatedReplicas int32 `json:"updatedReplicas"`
 
+	// The number of reserved pods in temporary adaptive strategy.
+	ReservedPods int32 `json:"reservedPods,omitempty"`
+
 	// The number of ready current revision replicas for this UnitedDeployment.
 	// +optional
 	UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"`
@@ -223,6 +336,8 @@ type UnitedDeploymentStatus struct {
 	// +optional
 	SubsetReplicas map[string]int32 `json:"subsetReplicas,omitempty"`
 
+	// Record the conditions of each subset.
+	SubsetStatuses []UnitedDeploymentSubsetStatus `json:"subsetStatuses,omitempty"`
 	// Represents the latest available observations of a UnitedDeployment's current state.
 	// +optional
 	Conditions []UnitedDeploymentCondition `json:"conditions,omitempty"`
@@ -230,6 +345,18 @@ type UnitedDeploymentStatus struct {
 	// Records the information of update progress.
 	// +optional
 	UpdateStatus *UpdateStatus `json:"updateStatus,omitempty"`
+
+	// LabelSelector is label selectors for query over pods that should match the replica count used by HPA.
+	LabelSelector string `json:"labelSelector,omitempty"`
+}
+
+func (s *UnitedDeploymentStatus) GetSubsetStatus(subset string) *UnitedDeploymentSubsetStatus {
+	for i, subsetStatus := range s.SubsetStatuses {
+		if subsetStatus.Name == subset {
+			return &s.SubsetStatuses[i]
+		}
+	}
+	return nil
 }
 
 // UnitedDeploymentCondition describes current state of a UnitedDeployment.
@@ -246,7 +373,7 @@ type UnitedDeploymentCondition struct {
 	// The reason for the condition's last transition.
 	Reason string `json:"reason,omitempty"`
 
-	// A human readable message indicating details about the transition.
+	// A human-readable message indicating details about the transition.
 	Message string `json:"message,omitempty"`
 }
 
@@ -261,13 +388,73 @@ type UpdateStatus struct {
 	CurrentPartitions map[string]int32 `json:"currentPartitions,omitempty"`
 }
 
+type UnitedDeploymentSubsetStatus struct {
+	// Subset name specified in Topology.Subsets
+	Name string `json:"name,omitempty"`
+	// Records the current replicas. Currently unused.
+	Replicas int32 `json:"replicas,omitempty"`
+	// Records the current ready replicas. Currently unused.
+	ReadyReplicas int32 `json:"readyReplicas,omitempty"`
+	// Records the current partition. Currently unused.
+	Partition int32 `json:"partition,omitempty"`
+	// Records the reserved pods in the subset.
+	ReservedPods int32 `json:"reservedPods,omitempty"`
+	// Conditions is an array of current observed subset conditions.
+	Conditions []UnitedDeploymentSubsetCondition `json:"conditions,omitempty"`
+}
+
+func (s *UnitedDeploymentSubsetStatus) GetCondition(condType UnitedDeploymentSubsetConditionType) *UnitedDeploymentSubsetCondition {
+	for _, condition := range s.Conditions {
+		if condition.Type == condType {
+			return &condition
+		}
+	}
+	return nil
+}
+
+func (s *UnitedDeploymentSubsetStatus) SetCondition(condType UnitedDeploymentSubsetConditionType, status corev1.ConditionStatus, reason, message string) {
+	var currentCond *UnitedDeploymentSubsetCondition
+	for i, c := range s.Conditions {
+		if c.Type == condType {
+			currentCond = &s.Conditions[i]
+			break
+		}
+	}
+	if currentCond != nil && currentCond.Status == status && currentCond.Reason == reason {
+		return
+	}
+	if currentCond == nil {
+		s.Conditions = append(s.Conditions, UnitedDeploymentSubsetCondition{Type: condType})
+		currentCond = &s.Conditions[len(s.Conditions)-1]
+	}
+	currentCond.LastTransitionTime = metav1.Now()
+	currentCond.Status = status
+	currentCond.Reason = reason
+	currentCond.Message = message
+}
+
+type UnitedDeploymentSubsetConditionType string
+
+const (
+	// UnitedDeploymentSubsetSchedulable means new pods allocated into the subset will keep pending.
+	UnitedDeploymentSubsetSchedulable UnitedDeploymentSubsetConditionType = "Schedulable"
+)
+
+type UnitedDeploymentSubsetCondition struct {
+	Type               UnitedDeploymentSubsetConditionType `json:"type"`
+	Status             corev1.ConditionStatus              `json:"status"`
+	LastTransitionTime metav1.Time                         `json:"lastTransitionTime,omitempty"`
+	Reason             string                              `json:"reason,omitempty"`
+	Message            string                              `json:"message,omitempty"`
+}
+
 // +genclient
 // +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale
 // +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale
 // +k8s:openapi-gen=true
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
-// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector
+// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector
 // +kubebuilder:resource:shortName=ud
 // +kubebuilder:printcolumn:name="DESIRED",type="integer",JSONPath=".spec.replicas",description="The desired number of pods."
 // +kubebuilder:printcolumn:name="CURRENT",type="integer",JSONPath=".status.replicas",description="The number of currently all pods."

apis/apps/v1alpha1/well_know_annotations.go

@@ -0,0 +1,8 @@
+package v1alpha1
+
+const (
+	// AnnotationUsingEnhancedLiveness indicates that the enhanced liveness probe of pod is enabled.
+	AnnotationUsingEnhancedLiveness = "apps.kruise.io/using-enhanced-liveness"
+	// AnnotationUsingEnhancedLiveness indicates the backup probe (json types) of the pod native container livnessprobe configuration.
+	AnnotationNativeContainerProbeContext = "apps.kruise.io/container-probe-context"
+)

apis/apps/v1alpha1/well_known_labels.go

@@ -4,6 +4,9 @@ const (
 	// ControllerRevisionHashLabelKey is used to record the controller revision of current resource.
 	ControllerRevisionHashLabelKey = "apps.kruise.io/controller-revision-hash"
 
+	// ReservedPodLabelKey is used to mark the reserved pods.
+	ReservedPodLabelKey = "apps.kruise.io/united-deployment-reserved-pod"
+
 	// SubSetNameLabelKey is used to record the name of current subset.
 	SubSetNameLabelKey = "apps.kruise.io/subset-name"
 
@@ -15,6 +18,8 @@ const (
 
 	// ImagePreDownloadIgnoredKey indicates the images of this revision have been ignored to pre-download
 	ImagePreDownloadIgnoredKey = "apps.kruise.io/image-predownload-ignored"
+	// AnnotationSubsetPatchKey indicates the patch for every subset
+	AnnotationSubsetPatchKey = "apps.kruise.io/subset-patch"
 )
 
 // Sidecar container environment variable definitions which are used to enable SidecarTerminator to take effect on the sidecar container.
@@ -27,4 +32,7 @@ const (
 	// using in-place update strategy to kill sidecar. This image must be given if you want to use in-place update
 	// strategy to terminate sidecar containers.
 	KruiseTerminateSidecarWithImageEnv = "KRUISE_TERMINATE_SIDECAR_WHEN_JOB_EXIT_WITH_IMAGE"
+
+	// KruiseIgnoreContainerExitCodeEnv is an env name, which represents a switch to ignore the exit code of sidecar container.
+	KruiseIgnoreContainerExitCodeEnv = "KRUISE_TERMINATE_SIDECAR_IGNORE_EXIT_CODE"
 )

apis/apps/v1alpha1/workloadspread_types.go

@@ -28,8 +28,15 @@ type WorkloadSpreadSpec struct {
 	// TargetReference is the target workload that WorkloadSpread want to control.
 	TargetReference *TargetReference `json:"targetRef"`
 
+	// TargetFilter allows WorkloadSpread to manage only a portion of the Pods in the TargetReference:
+	// by specifying the criteria for the Pods to be managed through a label selector,
+	// and by specifying how to obtain the total number of these selected Pods from the workload using replicasPaths.
+	TargetFilter *TargetFilter `json:"targetFilter,omitempty"`
+
 	// Subsets describes the pods distribution details between each of subsets.
-	Subsets []WorkloadSpreadSubset `json:"subsets"`
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Subsets []WorkloadSpreadSubset `json:"subsets" patchStrategy:"merge" patchMergeKey:"name"`
 
 	// ScheduleStrategy indicates the strategy the WorkloadSpread used to preform the schedule between each of subsets.
 	// +optional
@@ -46,6 +53,58 @@ type TargetReference struct {
 	Name string `json:"name"`
 }
 
+/*
+TargetFilter is an optional parameter that allows WorkloadSpread to manage only a subset of the Pods generated by the target workload.
+
+For example, suppose a WorkloadSpread points to the following Kubeflow TFJob resource:
+
+	```yaml
+	apiVersion: kubeflow.org/v1
+	kind: TFJob
+	spec:
+	  tfReplicaSpecs:
+		PS:
+		  replicas: 1
+		  ...
+		MASTER:
+		  replicas: 1
+		  ...
+		Worker:
+		  replicas: 2
+		  ...
+	```
+
+If you want to manage only the 2 Worker Pods that are generated, you need to configure the TargetFilter as follows:
+
+	```yaml
+	targetFilter:
+	  selector:
+		matchLabels:
+		  role: worker
+	  replicasPathList:
+		- spec.tfReplicaSpecs.Worker.replicas
+	```
+
+With this configuration, the PS Pods and Master Pods generated by the TFJob will not be managed by WorkloadSpread and will not be
+counted toward the total number of replicas.
+*/
+type TargetFilter struct {
+	// Selector is used to filter the Pods to be managed.
+	//
+	//+optional
+	Selector *metav1.LabelSelector `json:"selector,omitempty"`
+
+	// ReplicasPathList is a list of resource paths used to specify how to determine the total number of replicas of
+	// the target workload after filtering. If this list is not empty, WorkloadSpread will look for the corresponding
+	// values in the target resource according to each path, and treat the sum of these values as the total number of replicas after filtering.
+	//
+	// The replicas path is a dot-separated path, similar to "spec.replicas". If there are arrays, you can use numbers to denote indexes, like "subsets.1.replicas".
+	// The real values of these paths must be integers.
+	//
+	// +optional
+	ReplicasPathList []string `json:"replicasPathList,omitempty"`
+}
+
 // WorkloadSpreadScheduleStrategyType is a string enumeration type that enumerates
 // all possible schedule strategies for the WorkloadSpread controller.
 // +kubebuilder:validation:Enum=Adaptive;Fixed;""
@@ -128,6 +187,11 @@ type WorkloadSpreadStatus struct {
 	// Contains the status of each subset. Each element in this array represents one subset
 	// +optional
 	SubsetStatuses []WorkloadSpreadSubsetStatus `json:"subsetStatuses,omitempty"`
+
+	// VersionedSubsetStatuses is to solve rolling-update problems, where the creation of new-version pod
+	// may be earlier than deletion of old-version pod. We have to calculate the pod subset distribution for
+	// each version.
+	VersionedSubsetStatuses map[string][]WorkloadSpreadSubsetStatus `json:"versionedSubsetStatuses,omitempty"`
 }
 
 type WorkloadSpreadSubsetConditionType string

apis/apps/v1alpha1/zz_generated.deepcopy.go

@@ -1,5 +1,4 @@
 //go:build !ignore_autogenerated
-// +build !ignore_autogenerated
 
 /*
 Copyright 2021 The Kruise Authors.
@@ -31,6 +30,31 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *AdaptiveUnitedDeploymentStrategy) DeepCopyInto(out *AdaptiveUnitedDeploymentStrategy) {
+	*out = *in
+	if in.RescheduleCriticalSeconds != nil {
+		in, out := &in.RescheduleCriticalSeconds, &out.RescheduleCriticalSeconds
+		*out = new(int32)
+		**out = **in
+	}
+	if in.UnschedulableDuration != nil {
+		in, out := &in.UnschedulableDuration, &out.UnschedulableDuration
+		*out = new(int32)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AdaptiveUnitedDeploymentStrategy.
+func (in *AdaptiveUnitedDeploymentStrategy) DeepCopy() *AdaptiveUnitedDeploymentStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(AdaptiveUnitedDeploymentStrategy)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *AdaptiveWorkloadSpreadStrategy) DeepCopyInto(out *AdaptiveWorkloadSpreadStrategy) {
 	*out = *in
@@ -1203,6 +1227,21 @@ func (in *EphemeralJobStatus) DeepCopy() *EphemeralJobStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *FailedImageStatus) DeepCopyInto(out *FailedImageStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new FailedImageStatus.
+func (in *FailedImageStatus) DeepCopy() *FailedImageStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(FailedImageStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *FailurePolicy) DeepCopyInto(out *FailurePolicy) {
 	*out = *in
@@ -1218,6 +1257,120 @@ func (in *FailurePolicy) DeepCopy() *FailurePolicy {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ImageListPullJob) DeepCopyInto(out *ImageListPullJob) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageListPullJob.
+func (in *ImageListPullJob) DeepCopy() *ImageListPullJob {
+	if in == nil {
+		return nil
+	}
+	out := new(ImageListPullJob)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ImageListPullJob) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ImageListPullJobList) DeepCopyInto(out *ImageListPullJobList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]ImageListPullJob, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageListPullJobList.
+func (in *ImageListPullJobList) DeepCopy() *ImageListPullJobList {
+	if in == nil {
+		return nil
+	}
+	out := new(ImageListPullJobList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *ImageListPullJobList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ImageListPullJobSpec) DeepCopyInto(out *ImageListPullJobSpec) {
+	*out = *in
+	if in.Images != nil {
+		in, out := &in.Images, &out.Images
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	in.ImagePullJobTemplate.DeepCopyInto(&out.ImagePullJobTemplate)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageListPullJobSpec.
+func (in *ImageListPullJobSpec) DeepCopy() *ImageListPullJobSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(ImageListPullJobSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ImageListPullJobStatus) DeepCopyInto(out *ImageListPullJobStatus) {
+	*out = *in
+	if in.StartTime != nil {
+		in, out := &in.StartTime, &out.StartTime
+		*out = (*in).DeepCopy()
+	}
+	if in.CompletionTime != nil {
+		in, out := &in.CompletionTime, &out.CompletionTime
+		*out = (*in).DeepCopy()
+	}
+	if in.FailedImageStatuses != nil {
+		in, out := &in.FailedImageStatuses, &out.FailedImageStatuses
+		*out = make([]*FailedImageStatus, len(*in))
+		for i := range *in {
+			if (*in)[i] != nil {
+				in, out := &(*in)[i], &(*out)[i]
+				*out = new(FailedImageStatus)
+				**out = **in
+			}
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImageListPullJobStatus.
+func (in *ImageListPullJobStatus) DeepCopy() *ImageListPullJobStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(ImageListPullJobStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ImagePullJob) DeepCopyInto(out *ImagePullJob) {
 	*out = *in
@@ -1317,37 +1470,7 @@ func (in *ImagePullJobPodSelector) DeepCopy() *ImagePullJobPodSelector {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ImagePullJobSpec) DeepCopyInto(out *ImagePullJobSpec) {
 	*out = *in
-	if in.PullSecrets != nil {
+	in.ImagePullJobTemplate.DeepCopyInto(&out.ImagePullJobTemplate)
-		in, out := &in.PullSecrets, &out.PullSecrets
-		*out = make([]string, len(*in))
-		copy(*out, *in)
-	}
-	if in.Selector != nil {
-		in, out := &in.Selector, &out.Selector
-		*out = new(ImagePullJobNodeSelector)
-		(*in).DeepCopyInto(*out)
-	}
-	if in.PodSelector != nil {
-		in, out := &in.PodSelector, &out.PodSelector
-		*out = new(ImagePullJobPodSelector)
-		(*in).DeepCopyInto(*out)
-	}
-	if in.Parallelism != nil {
-		in, out := &in.Parallelism, &out.Parallelism
-		*out = new(intstr.IntOrString)
-		**out = **in
-	}
-	if in.PullPolicy != nil {
-		in, out := &in.PullPolicy, &out.PullPolicy
-		*out = new(PullPolicy)
-		(*in).DeepCopyInto(*out)
-	}
-	in.CompletionPolicy.DeepCopyInto(&out.CompletionPolicy)
-	if in.SandboxConfig != nil {
-		in, out := &in.SandboxConfig, &out.SandboxConfig
-		*out = new(SandboxConfig)
-		(*in).DeepCopyInto(*out)
-	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePullJobSpec.
@@ -1388,6 +1511,52 @@ func (in *ImagePullJobStatus) DeepCopy() *ImagePullJobStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *ImagePullJobTemplate) DeepCopyInto(out *ImagePullJobTemplate) {
+	*out = *in
+	if in.PullSecrets != nil {
+		in, out := &in.PullSecrets, &out.PullSecrets
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+	if in.Selector != nil {
+		in, out := &in.Selector, &out.Selector
+		*out = new(ImagePullJobNodeSelector)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.PodSelector != nil {
+		in, out := &in.PodSelector, &out.PodSelector
+		*out = new(ImagePullJobPodSelector)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.Parallelism != nil {
+		in, out := &in.Parallelism, &out.Parallelism
+		*out = new(intstr.IntOrString)
+		**out = **in
+	}
+	if in.PullPolicy != nil {
+		in, out := &in.PullPolicy, &out.PullPolicy
+		*out = new(PullPolicy)
+		(*in).DeepCopyInto(*out)
+	}
+	in.CompletionPolicy.DeepCopyInto(&out.CompletionPolicy)
+	if in.SandboxConfig != nil {
+		in, out := &in.SandboxConfig, &out.SandboxConfig
+		*out = new(SandboxConfig)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ImagePullJobTemplate.
+func (in *ImagePullJobTemplate) DeepCopy() *ImagePullJobTemplate {
+	if in == nil {
+		return nil
+	}
+	out := new(ImagePullJobTemplate)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ImageSpec) DeepCopyInto(out *ImageSpec) {
 	*out = *in
@@ -2542,6 +2711,11 @@ func (in *SidecarContainer) DeepCopyInto(out *SidecarContainer) {
 	in.Container.DeepCopyInto(&out.Container)
 	out.UpgradeStrategy = in.UpgradeStrategy
 	out.ShareVolumePolicy = in.ShareVolumePolicy
+	if in.ShareVolumeDevicePolicy != nil {
+		in, out := &in.ShareVolumeDevicePolicy, &out.ShareVolumeDevicePolicy
+		*out = new(ShareVolumePolicy)
+		**out = **in
+	}
 	if in.TransferEnv != nil {
 		in, out := &in.TransferEnv, &out.TransferEnv
 		*out = make([]TransferEnvVar, len(*in))
@@ -2805,6 +2979,11 @@ func (in *SidecarSetUpdateStrategy) DeepCopyInto(out *SidecarSetUpdateStrategy)
 		*out = new(intstr.IntOrString)
 		**out = **in
 	}
+	if in.PriorityStrategy != nil {
+		in, out := &in.PriorityStrategy, &out.PriorityStrategy
+		*out = new(pub.UpdatePriorityStrategy)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.ScatterStrategy != nil {
 		in, out := &in.ScatterStrategy, &out.ScatterStrategy
 		*out = make(UpdateScatterStrategy, len(*in))
@@ -3020,6 +3199,17 @@ func (in *Subset) DeepCopyInto(out *Subset) {
 		*out = new(intstr.IntOrString)
 		**out = **in
 	}
+	if in.MinReplicas != nil {
+		in, out := &in.MinReplicas, &out.MinReplicas
+		*out = new(intstr.IntOrString)
+		**out = **in
+	}
+	if in.MaxReplicas != nil {
+		in, out := &in.MaxReplicas, &out.MaxReplicas
+		*out = new(intstr.IntOrString)
+		**out = **in
+	}
+	in.Patch.DeepCopyInto(&out.Patch)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Subset.
@@ -3083,6 +3273,31 @@ func (in *SyncStatus) DeepCopy() *SyncStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *TargetFilter) DeepCopyInto(out *TargetFilter) {
+	*out = *in
+	if in.Selector != nil {
+		in, out := &in.Selector, &out.Selector
+		*out = new(metav1.LabelSelector)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ReplicasPathList != nil {
+		in, out := &in.ReplicasPathList, &out.ReplicasPathList
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetFilter.
+func (in *TargetFilter) DeepCopy() *TargetFilter {
+	if in == nil {
+		return nil
+	}
+	out := new(TargetFilter)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *TargetReference) DeepCopyInto(out *TargetReference) {
 	*out = *in
@@ -3108,6 +3323,7 @@ func (in *Topology) DeepCopyInto(out *Topology) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	in.ScheduleStrategy.DeepCopyInto(&out.ScheduleStrategy)
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Topology.
@@ -3220,6 +3436,26 @@ func (in *UnitedDeploymentList) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UnitedDeploymentScheduleStrategy) DeepCopyInto(out *UnitedDeploymentScheduleStrategy) {
+	*out = *in
+	if in.Adaptive != nil {
+		in, out := &in.Adaptive, &out.Adaptive
+		*out = new(AdaptiveUnitedDeploymentStrategy)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnitedDeploymentScheduleStrategy.
+func (in *UnitedDeploymentScheduleStrategy) DeepCopy() *UnitedDeploymentScheduleStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(UnitedDeploymentScheduleStrategy)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *UnitedDeploymentSpec) DeepCopyInto(out *UnitedDeploymentSpec) {
 	*out = *in
@@ -3268,6 +3504,13 @@ func (in *UnitedDeploymentStatus) DeepCopyInto(out *UnitedDeploymentStatus) {
 			(*out)[key] = val
 		}
 	}
+	if in.SubsetStatuses != nil {
+		in, out := &in.SubsetStatuses, &out.SubsetStatuses
+		*out = make([]UnitedDeploymentSubsetStatus, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
 	if in.Conditions != nil {
 		in, out := &in.Conditions, &out.Conditions
 		*out = make([]UnitedDeploymentCondition, len(*in))
@@ -3292,6 +3535,44 @@ func (in *UnitedDeploymentStatus) DeepCopy() *UnitedDeploymentStatus {
 	return out
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UnitedDeploymentSubsetCondition) DeepCopyInto(out *UnitedDeploymentSubsetCondition) {
+	*out = *in
+	in.LastTransitionTime.DeepCopyInto(&out.LastTransitionTime)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnitedDeploymentSubsetCondition.
+func (in *UnitedDeploymentSubsetCondition) DeepCopy() *UnitedDeploymentSubsetCondition {
+	if in == nil {
+		return nil
+	}
+	out := new(UnitedDeploymentSubsetCondition)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UnitedDeploymentSubsetStatus) DeepCopyInto(out *UnitedDeploymentSubsetStatus) {
+	*out = *in
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]UnitedDeploymentSubsetCondition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnitedDeploymentSubsetStatus.
+func (in *UnitedDeploymentSubsetStatus) DeepCopy() *UnitedDeploymentSubsetStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(UnitedDeploymentSubsetStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *UnitedDeploymentUpdateStrategy) DeepCopyInto(out *UnitedDeploymentUpdateStrategy) {
 	*out = *in
@@ -3475,6 +3756,11 @@ func (in *WorkloadSpreadSpec) DeepCopyInto(out *WorkloadSpreadSpec) {
 		*out = new(TargetReference)
 		**out = **in
 	}
+	if in.TargetFilter != nil {
+		in, out := &in.TargetFilter, &out.TargetFilter
+		*out = new(TargetFilter)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.Subsets != nil {
 		in, out := &in.Subsets, &out.Subsets
 		*out = make([]WorkloadSpreadSubset, len(*in))
@@ -3505,6 +3791,24 @@ func (in *WorkloadSpreadStatus) DeepCopyInto(out *WorkloadSpreadStatus) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.VersionedSubsetStatuses != nil {
+		in, out := &in.VersionedSubsetStatuses, &out.VersionedSubsetStatuses
+		*out = make(map[string][]WorkloadSpreadSubsetStatus, len(*in))
+		for key, val := range *in {
+			var outVal []WorkloadSpreadSubsetStatus
+			if val == nil {
+				(*out)[key] = nil
+			} else {
+				inVal := (*in)[key]
+				in, out := &inVal, &outVal
+				*out = make([]WorkloadSpreadSubsetStatus, len(*in))
+				for i := range *in {
+					(*in)[i].DeepCopyInto(&(*out)[i])
+				}
+			}
+			(*out)[key] = outVal
+		}
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new WorkloadSpreadStatus.

apis/apps/v1beta1/statefulset_types.go

@@ -29,6 +29,39 @@ const (
 	MaxMinReadySeconds = 300
 )
 
+// VolumeClaimUpdateStrategyType defines the update strategy types for volume claims.
+// It is an enumerated type that provides two different update strategies.
+// +enum
+type VolumeClaimUpdateStrategyType string
+
+const (
+	// OnPodRollingUpdateVolumeClaimUpdateStrategyType indicates that volume claim updates are triggered when associated Pods undergo rolling updates.
+	// This strategy ensures that storage availability and integrity are maintained during the update process.
+	OnPodRollingUpdateVolumeClaimUpdateStrategyType VolumeClaimUpdateStrategyType = "OnPodRollingUpdate"
+
+	// OnPVCDeleteVolumeClaimUpdateStrategyType indicates that updates are triggered when a Persistent Volume Claim (PVC) is deleted.
+	// This strategy places full control of the update timing in the hands of the user, typically executed after ensuring data has been backed up or there are no data security concerns,
+	// allowing for storage resource management that aligns with specific user requirements and security policies.
+	OnPVCDeleteVolumeClaimUpdateStrategyType VolumeClaimUpdateStrategyType = "OnDelete"
+)
+
+// VolumeClaimStatus describes the status of a volume claim template.
+// It provides details about the compatibility and readiness of the volume claim.
+type VolumeClaimStatus struct {
+	// VolumeClaimName is the name of the volume claim.
+	// This is a unique identifier used to reference a specific volume claim.
+	VolumeClaimName string `json:"volumeClaimName"`
+	// CompatibleReplicas is the number of replicas currently compatible with the volume claim.
+	// It indicates how many replicas can function properly, being compatible with this volume claim.
+	// Compatibility is determined by whether the PVC spec storage requests are greater than or equal to the template spec storage requests
+	CompatibleReplicas int32 `json:"compatibleReplicas"`
+	// CompatibleReadyReplicas is the number of replicas that are both ready and compatible with the volume claim.
+	// It highlights that these replicas are not only compatible but also ready to be put into service immediately.
+	// Compatibility is determined by whether the pvc spec storage requests are greater than or equal to the template spec storage requests
+	// The "ready" status is determined by whether the PVC status capacity is greater than or equal to the PVC spec storage requests.
+	CompatibleReadyReplicas int32 `json:"compatibleReadyReplicas"`
+}
+
 // StatefulSetUpdateStrategy indicates the strategy that the StatefulSet
 // controller will use to perform updates. It includes any additional parameters
 // necessary to perform the update for the indicated strategy.
@@ -42,11 +75,18 @@ type StatefulSetUpdateStrategy struct {
 	RollingUpdate *RollingUpdateStatefulSetStrategy `json:"rollingUpdate,omitempty"`
 }
 
+// VolumeClaimUpdateStrategy defines the strategy for updating volume claims.
+// This structure is used to control how updates to PersistentVolumeClaims are handled during pod rolling updates or PersistentVolumeClaim deletions.
+type VolumeClaimUpdateStrategy struct {
+	// Type specifies the type of update strategy, possible values include:
+	// OnPodRollingUpdateVolumeClaimUpdateStrategyType: Apply the update strategy during pod rolling updates.
+	// OnPVCDeleteVolumeClaimUpdateStrategyType: Apply the update strategy when a PersistentVolumeClaim is deleted.
+	Type VolumeClaimUpdateStrategyType `json:"type,omitempty"`
+}
+
 // RollingUpdateStatefulSetStrategy is used to communicate parameter for RollingUpdateStatefulSetStrategyType.
 type RollingUpdateStatefulSetStrategy struct {
-	// Partition indicates the ordinal at which the StatefulSet should be partitioned by default.
+	// Partition indicates the number of pods the StatefulSet should be partitioned by default.
-	// But if unorderedUpdate has been set:
-	//   - Partition indicates the number of pods with non-updated revisions when rolling update.
 	//   - It means controller will update $(replicas - partition) number of pod.
 	// Default value is 0.
 	// +optional
@@ -128,7 +168,7 @@ const (
 )
 
 // StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs
-// created from the StatefulSet VolumeClaimTemplates.
+// created from the StatefulSet VolumeClaims.
 type StatefulSetPersistentVolumeClaimRetentionPolicy struct {
 	// WhenDeleted specifies what happens to PVCs created from StatefulSet
 	// VolumeClaimTemplates when the StatefulSet is deleted. The default policy
@@ -143,6 +183,21 @@ type StatefulSetPersistentVolumeClaimRetentionPolicy struct {
 	WhenScaled PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty"`
 }
 
+// StatefulSetOrdinals describes the policy used for replica ordinal assignment
+// in this StatefulSet.
+type StatefulSetOrdinals struct {
+	// start is the number representing the first replica's index. It may be used
+	// to number replicas from an alternate index (eg: 1-indexed) over the default
+	// 0-indexed names, or to orchestrate progressive movement of replicas from
+	// one StatefulSet to another.
+	// If set, replica indices will be in the range:
+	//   [.spec.ordinals.start, .spec.ordinals.start + .spec.replicas).
+	// If unset, defaults to 0. Replica indices will be in the range:
+	//   [0, .spec.replicas).
+	// +optional
+	Start int32 `json:"start" protobuf:"varint,1,opt,name=start"`
+}
+
 // StatefulSetSpec defines the desired state of StatefulSet
 type StatefulSetSpec struct {
 	// replicas is the desired number of replicas of the given Template.
@@ -178,6 +233,11 @@ type StatefulSetSpec struct {
 	// +kubebuilder:validation:Schemaless
 	VolumeClaimTemplates []v1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
 
+	// VolumeClaimUpdateStrategy specifies the strategy for updating VolumeClaimTemplates within a StatefulSet.
+	// This field is currently only effective if the StatefulSetAutoResizePVCGate is enabled.
+	// +optional
+	VolumeClaimUpdateStrategy VolumeClaimUpdateStrategy `json:"volumeClaimUpdateStrategy,omitempty"`
+
 	// serviceName is the name of the service that governs this StatefulSet.
 	// This service must exist before the StatefulSet, and is responsible for
 	// the network identity of the set. Pods get DNS/hostnames that follow the
@@ -214,7 +274,8 @@ type StatefulSetSpec struct {
 	//   Then controller will delete Pod-1 and create Pod-3 (existing Pods will be [0, 2, 3])
 	// - If you just want to delete Pod-1, you should set spec.reserveOrdinal to [1] and spec.replicas to 2.
 	//   Then controller will delete Pod-1 (existing Pods will be [0, 2])
-	ReserveOrdinals []int `json:"reserveOrdinals,omitempty"`
+	// You can also use ranges along with numbers, such as [1, 3-5], which is a shortcut for [1, 3, 4, 5].
+	ReserveOrdinals []intstr.IntOrString `json:"reserveOrdinals,omitempty"`
 
 	// Lifecycle defines the lifecycle hooks for Pods pre-delete, in-place update.
 	Lifecycle *appspub.Lifecycle `json:"lifecycle,omitempty"`
@@ -228,6 +289,14 @@ type StatefulSetSpec struct {
 	// StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.
 	// +optional
 	PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy `json:"persistentVolumeClaimRetentionPolicy,omitempty"`
+
+	// ordinals controls the numbering of replica indices in a StatefulSet. The
+	// default ordinals behavior assigns a "0" index to the first replica and
+	// increments the index by one for each additional replica requested. Using
+	// the ordinals field requires the StatefulSetStartOrdinal feature gate to be
+	// enabled, which is beta.
+	// +optional
+	Ordinals *StatefulSetOrdinals `json:"ordinals,omitempty"`
 }
 
 // StatefulSetScaleStrategy defines strategies for pods scale.
@@ -267,6 +336,10 @@ type StatefulSetStatus struct {
 	// updatedReadyReplicas is the number of updated Pods created by the StatefulSet controller that have a Ready Condition.
 	UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"`
 
+	// updatedAvailableReplicas is the number of updated Pods created by the StatefulSet controller that have a Ready condition
+	//for atleast minReadySeconds.
+	UpdatedAvailableReplicas int32 `json:"updatedAvailableReplicas,omitempty"`
+
 	// currentRevision, if not empty, indicates the version of the StatefulSet used to generate Pods in the
 	// sequence [0,currentReplicas).
 	CurrentRevision string `json:"currentRevision,omitempty"`
@@ -289,6 +362,12 @@ type StatefulSetStatus struct {
 
 	// LabelSelector is label selectors for query over pods that should match the replica count used by HPA.
 	LabelSelector string `json:"labelSelector,omitempty"`
+
+	// VolumeClaims represents the status of compatibility between existing PVCs
+	// and their respective templates. It tracks whether the PersistentVolumeClaims have been updated
+	// to match any changes made to the volumeClaimTemplates, ensuring synchronization
+	// between the defined templates and the actual PersistentVolumeClaims in use.
+	VolumeClaims []VolumeClaimStatus `json:"volumeClaims,omitempty"`
 }
 
 // These are valid conditions of a statefulset.

apis/apps/v1beta1/zz_generated.deepcopy.go

@@ -1,5 +1,4 @@
 //go:build !ignore_autogenerated
-// +build !ignore_autogenerated
 
 /*
 Copyright 2021 The Kruise Authors.
@@ -129,6 +128,21 @@ func (in *StatefulSetList) DeepCopyObject() runtime.Object {
 	return nil
 }
 
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSetOrdinals) DeepCopyInto(out *StatefulSetOrdinals) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetOrdinals.
+func (in *StatefulSetOrdinals) DeepCopy() *StatefulSetOrdinals {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSetOrdinals)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *StatefulSetPersistentVolumeClaimRetentionPolicy) {
 	*out = *in
@@ -185,6 +199,7 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	out.VolumeClaimUpdateStrategy = in.VolumeClaimUpdateStrategy
 	in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy)
 	if in.RevisionHistoryLimit != nil {
 		in, out := &in.RevisionHistoryLimit, &out.RevisionHistoryLimit
@@ -193,7 +208,7 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
 	}
 	if in.ReserveOrdinals != nil {
 		in, out := &in.ReserveOrdinals, &out.ReserveOrdinals
-		*out = make([]int, len(*in))
+		*out = make([]intstr.IntOrString, len(*in))
 		copy(*out, *in)
 	}
 	if in.Lifecycle != nil {
@@ -211,6 +226,11 @@ func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
 		*out = new(StatefulSetPersistentVolumeClaimRetentionPolicy)
 		**out = **in
 	}
+	if in.Ordinals != nil {
+		in, out := &in.Ordinals, &out.Ordinals
+		*out = new(StatefulSetOrdinals)
+		**out = **in
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetSpec.
@@ -238,6 +258,11 @@ func (in *StatefulSetStatus) DeepCopyInto(out *StatefulSetStatus) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
+	if in.VolumeClaims != nil {
+		in, out := &in.VolumeClaims, &out.VolumeClaims
+		*out = make([]VolumeClaimStatus, len(*in))
+		copy(*out, *in)
+	}
 }
 
 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetStatus.
@@ -289,3 +314,33 @@ func (in *UnorderedUpdateStrategy) DeepCopy() *UnorderedUpdateStrategy {
 	in.DeepCopyInto(out)
 	return out
 }
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeClaimStatus) DeepCopyInto(out *VolumeClaimStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeClaimStatus.
+func (in *VolumeClaimStatus) DeepCopy() *VolumeClaimStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeClaimStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeClaimUpdateStrategy) DeepCopyInto(out *VolumeClaimUpdateStrategy) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeClaimUpdateStrategy.
+func (in *VolumeClaimUpdateStrategy) DeepCopy() *VolumeClaimUpdateStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeClaimUpdateStrategy)
+	in.DeepCopyInto(out)
+	return out
+}

apis/policy/v1alpha1/podunavailablebudget_types.go

@@ -27,17 +27,28 @@ import (
 type PubOperation string
 
 const (
-	// PubProtectOperationAnnotation indicates the pub protected Operation[DELETE,UPDATE,EVICT]
+	// PubProtectOperationAnnotation indicates the pub protected Operation[DELETE,UPDATE,EVICT].
-	// if annotations[kruise.io/pub-protect-operations]=EVICT indicates the pub only protect evict pod
+	// if annotations[kruise.io/pub-protect-operations]=EVICT indicates the pub only protect evict pod.
-	// if the annotations do not exist, the default DELETE,EVICT,UPDATE are protected
+	// if the annotations do not exist, the default DELETE,EVICT,UPDATE are protected.
+	// RESIZE: Pod vertical scaling action. If it's enabled, all resize action will be protected. RESIZE
+	// is an extension of UPDATE, if RESIZE is disabled and UPDATE is enabled, any UPDATE operation will
+	// be protected only as it will definitely cause container restarts.
+	// UPDATE: Kruise will carefully differentiate whether this update will cause interruptions. When
+	// the FeatureGate InPlacePodVerticalScaling is enabled, pod inplace vertical scaling will be
+	// considered non-disruption only when allowedResources(cpu、memory) changes、restartPolicy
+	// is not restartContainer、is not static pod and QoS not changed. But if featureGate
+	// InPlacePodVerticalScaling is disabled, all resize action will be considered as disruption.
 	PubProtectOperationAnnotation = "kruise.io/pub-protect-operations"
 	// pod webhook operation
 	PubUpdateOperation PubOperation = "UPDATE"
 	PubDeleteOperation PubOperation = "DELETE"
 	PubEvictOperation  PubOperation = "EVICT"
-	// PubProtectTotalReplicas indicates the pub protected total replicas, rather than workload.spec.replicas.
+	PubResizeOperation PubOperation = "RESIZE"
-	// and must be used with pub.spec.selector.
+	// PubProtectTotalReplicasAnnotation is the target replicas.
-	PubProtectTotalReplicas = "pub.kruise.io/protect-total-replicas"
+	// By default, PUB will get the target replicas through workload.spec.replicas. but there are some scenarios that may workload doesn't
+	// implement scale subresources or Pod doesn't have workload management. In this scenario, you can set pub.kruise.io/protect-total-replicas
+	// in pub annotations to get the target replicas to realize the same effect of protection ability.
+	PubProtectTotalReplicasAnnotation = "pub.kruise.io/protect-total-replicas"
 	// Marked the pod will not be pub-protected, solving the scenario of force pod deletion
 	PodPubNoProtectionAnnotation = "pub.kruise.io/no-protect"
 )

apis/policy/v1alpha1/zz_generated.deepcopy.go

@@ -1,5 +1,4 @@
 //go:build !ignore_autogenerated
-// +build !ignore_autogenerated
 
 /*
 Copyright 2021 The Kruise Authors.

cmd/daemon/main.go

@@ -17,6 +17,10 @@ limitations under the License.
 package main
 
 import (
+	"os"
+
+	"k8s.io/kubernetes/pkg/credentialprovider/plugin"
+
 	"flag"
 	"math/rand"
 	"net/http"
@@ -34,11 +38,22 @@ import (
 	"github.com/openkruise/kruise/pkg/daemon"
 	"github.com/openkruise/kruise/pkg/features"
 	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
+	"github.com/openkruise/kruise/pkg/util/secret"
 )
 
 var (
-	bindAddr  = flag.String("addr", ":10221", "The address the metric endpoint and healthz binds to.")
+	bindAddr         = flag.String("addr", ":10221", "The address the metric endpoint and healthz binds to.")
-	pprofAddr = flag.String("pprof-addr", ":10222", "The address the pprof binds to.")
+	pprofAddr        = flag.String("pprof-addr", ":10222", "The address the pprof binds to.")
+	enablePprof      = flag.Bool("enable-pprof", true, "Enable pprof for daemon.")
+	pluginConfigFile = flag.String("plugin-config-file", "/kruise/CredentialProviderPlugin.yaml", "The path of plugin config file.")
+	pluginBinDir     = flag.String("plugin-bin-dir", "/kruise/plugins", "The path of directory of plugin binaries.")
+
+	// TODO: After the feature is stable, the default value should also be restricted, e.g. 5.
+
+	// Users can set this value to limit the number of workers for pulling images,
+	// preventing the consumption of all available disk IOPS or network bandwidth,
+	// which could otherwise impact the performance of other running pods.
+	maxWorkersForPullImage = flag.Int("max-workers-for-pull-image", -1, "The maximum number of workers for pulling images.")
 )
 
 func main() {
@@ -55,16 +70,32 @@ func main() {
 	if err := client.NewRegistry(cfg); err != nil {
 		klog.Fatalf("Failed to init clientset registry: %v", err)
 	}
-	go func() {
+	if enablePprof != nil && *enablePprof {
-		if err := http.ListenAndServe(*pprofAddr, nil); err != nil {
+		go func() {
-			klog.Fatal(err, "unable to start pprof")
+			if err := http.ListenAndServe(*pprofAddr, nil); err != nil {
-		}
+				klog.Fatal(err, "unable to start pprof")
-	}()
+			}
+		}()
+	}
 	ctx := signals.SetupSignalHandler()
-	d, err := daemon.NewDaemon(cfg, *bindAddr)
+	d, err := daemon.NewDaemon(cfg, *bindAddr, *maxWorkersForPullImage)
 	if err != nil {
 		klog.Fatalf("Failed to new daemon: %v", err)
 	}
+
+	if _, err := os.Stat(*pluginConfigFile); err == nil {
+		err = plugin.RegisterCredentialProviderPlugins(*pluginConfigFile, *pluginBinDir)
+		if err != nil {
+			klog.ErrorS(err, "Failed to register credential provider plugins")
+		}
+	} else if os.IsNotExist(err) {
+		klog.InfoS("No plugin config file found, skipping", "configFile", *pluginConfigFile)
+	} else {
+		klog.ErrorS(err, "Failed to check plugin config file")
+	}
+	// make sure the new docker key ring is made and set after the credential plugins are registered
+	secret.MakeAndSetKeyring()
+
 	if err := d.Run(ctx); err != nil {
 		klog.Fatalf("Failed to start daemon: %v", err)
 	}

cmd/helm_hook/main.go

@@ -0,0 +1,67 @@
+/*
+Copyright 2024 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package main
+
+import (
+	"context"
+	"log"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/client-go/rest"
+
+	kruiseclientset "github.com/openkruise/kruise/pkg/client/clientset/versioned"
+)
+
+func main() {
+	config, err := rest.InClusterConfig()
+	if err != nil {
+		panic(err)
+	}
+	kc, err := kruiseclientset.NewForConfig(config)
+	if err != nil {
+		panic(err)
+	}
+	cloneSets, err := kc.AppsV1alpha1().CloneSets("").List(context.Background(), metav1.ListOptions{Limit: 1})
+	if err != nil {
+		panic(err)
+	}
+	if len(cloneSets.Items) > 0 || cloneSets.Continue != "" {
+		log.Fatalln("there still exists some clonesets in the cluster")
+	}
+	statefulSets, err := kc.AppsV1alpha1().StatefulSets("").List(context.Background(), metav1.ListOptions{Limit: 1})
+	if err != nil {
+		panic(err)
+	}
+	if len(statefulSets.Items) > 0 || statefulSets.Continue != "" {
+		log.Fatalln("there still exists some advanced statefulsets in the cluster")
+	}
+	statefulSetsBeta1, err := kc.AppsV1beta1().StatefulSets("").List(context.Background(), metav1.ListOptions{Limit: 1})
+	if err != nil {
+		panic(err)
+	}
+	if len(statefulSetsBeta1.Items) > 0 || statefulSetsBeta1.Continue != "" {
+		log.Fatalln("there still exists some advanced statefulsets in the cluster")
+	}
+	daemonSets, err := kc.AppsV1alpha1().DaemonSets("").List(context.Background(), metav1.ListOptions{Limit: 1})
+	if err != nil {
+		panic(err)
+	}
+	if len(daemonSets.Items) > 0 || daemonSets.Continue != "" {
+		log.Fatalln("there still exists some advanced daemonsets in the cluster")
+	}
+	log.Println("cluster is clean, ready to delete kruise")
+}

config/crd/bases/apps.kruise.io_advancedcronjobs.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: advancedcronjobs.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -44,14 +42,19 @@ spec:
         description: AdvancedCronJob is the Schema for the advancedcronjobs API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -59,19 +62,21 @@ spec:
             description: AdvancedCronJobSpec defines the desired state of AdvancedCronJob
             properties:
               concurrencyPolicy:
-                description: 'Specifies how to treat concurrent executions of a Job.
+                description: |-
-                  Valid values are: - "Allow" (default): allows CronJobs to run concurrently;
+                  Specifies how to treat concurrent executions of a Job.
-                  - "Forbid": forbids concurrent runs, skipping next run if previous
+                  Valid values are:
-                  run hasn''t finished yet; - "Replace": cancels currently running
+                  - "Allow" (default): allows CronJobs to run concurrently;
-                  job and replaces it with a new one'
+                  - "Forbid": forbids concurrent runs, skipping next run if previous run hasn't finished yet;
+                  - "Replace": cancels currently running job and replaces it with a new one
                 enum:
                 - Allow
                 - Forbid
                 - Replace
                 type: string
               failedJobsHistoryLimit:
-                description: The number of failed finished jobs to retain. This is
+                description: |-
-                  a pointer to distinguish between explicit zero and not specified.
+                  The number of failed finished jobs to retain.
+                  This is a pointer to distinguish between explicit zero and not specified.
                 format: int32
                 type: integer
               paused:
@@ -82,14 +87,15 @@ spec:
                 minLength: 0
                 type: string
               startingDeadlineSeconds:
-                description: Optional deadline in seconds for starting the job if
+                description: |-
-                  it misses scheduled time for any reason.  Missed jobs executions
+                  Optional deadline in seconds for starting the job if it misses scheduled
-                  will be counted as failed ones.
+                  time for any reason.  Missed jobs executions will be counted as failed ones.
                 format: int64
                 type: integer
               successfulJobsHistoryLimit:
-                description: The number of successful finished jobs to retain. This
+                description: |-
-                  is a pointer to distinguish between explicit zero and not specified.
+                  The number of successful finished jobs to retain.
+                  This is a pointer to distinguish between explicit zero and not specified.
                 format: int32
                 type: integer
               template:
@@ -109,34 +115,34 @@ spec:
                           broadcastjob.
                         properties:
                           completionPolicy:
-                            description: CompletionPolicy indicates the completion
+                            description: |-
-                              policy of the job. Default is Always CompletionPolicyType.
+                              CompletionPolicy indicates the completion policy of the job.
+                              Default is Always CompletionPolicyType.
                             properties:
                               activeDeadlineSeconds:
-                                description: ActiveDeadlineSeconds specifies the duration
+                                description: |-
-                                  in seconds relative to the startTime that the job
+                                  ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the job may be active
-                                  may be active before the system tries to terminate
+                                  before the system tries to terminate it; value must be positive integer.
-                                  it; value must be positive integer. Only works for
+                                  Only works for Always type.
-                                  Always type.
                                 format: int64
                                 type: integer
                               ttlSecondsAfterFinished:
-                                description: ttlSecondsAfterFinished limits the lifetime
+                                description: |-
-                                  of a Job that has finished execution (either Complete
+                                  ttlSecondsAfterFinished limits the lifetime of a Job that has finished
-                                  or Failed). If this field is set, ttlSecondsAfterFinished
+                                  execution (either Complete or Failed). If this field is set,
-                                  after the Job finishes, it is eligible to be automatically
+                                  ttlSecondsAfterFinished after the Job finishes, it is eligible to be
-                                  deleted. When the Job is being deleted, its lifecycle
+                                  automatically deleted. When the Job is being deleted, its lifecycle
-                                  guarantees (e.g. finalizers) will be honored. If
+                                  guarantees (e.g. finalizers) will be honored. If this field is unset,
-                                  this field is unset, the Job won't be automatically
+                                  the Job won't be automatically deleted. If this field is set to zero,
-                                  deleted. If this field is set to zero, the Job becomes
+                                  the Job becomes eligible to be deleted immediately after it finishes.
-                                  eligible to be deleted immediately after it finishes.
+                                  This field is alpha-level and is only honored by servers that enable the
-                                  This field is alpha-level and is only honored by
+                                  TTLAfterFinished feature.
-                                  servers that enable the TTLAfterFinished feature.
                                   Only works for Always type
                                 format: int32
                                 type: integer
                               type:
-                                description: Type indicates the type of the CompletionPolicy.
+                                description: |-
+                                  Type indicates the type of the CompletionPolicy.
                                   Default is Always.
                                 type: string
                             type: object
@@ -150,7 +156,8 @@ spec:
                                 format: int32
                                 type: integer
                               type:
-                                description: Type indicates the type of FailurePolicyType.
+                                description: |-
+                                  Type indicates the type of FailurePolicyType.
                                   Default is FailurePolicyTypeFailFast.
                                 type: string
                             type: object
@@ -158,12 +165,11 @@ spec:
                             anyOf:
                             - type: integer
                             - type: string
-                            description: Parallelism specifies the maximum desired
+                            description: |-
-                              number of pods the job should run at any given time.
+                              Parallelism specifies the maximum desired number of pods the job should
-                              The actual number of pods running in steady state will
+                              run at any given time. The actual number of pods running in steady state will
-                              be less than this number when the work left to do is
+                              be less than this number when the work left to do is less than max parallelism.
-                              less than max parallelism. Not setting this value means
+                              Not setting this value means no limit.
-                              no limit.
                             x-kubernetes-int-or-string: true
                           paused:
                             description: Paused will pause the job.
@@ -182,9 +188,9 @@ spec:
                     x-kubernetes-preserve-unknown-fields: true
                 type: object
               timeZone:
-                description: The time zone name for the given schedule, see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones.
+                description: |-
-                  If not specified, this will default to the time zone of the kruise-controller-manager
+                  The time zone name for the given schedule, see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones.
-                  process.
+                  If not specified, this will default to the time zone of the kruise-controller-manager process.
                 type: string
             required:
             - schedule
@@ -196,65 +202,49 @@ spec:
               active:
                 description: A list of pointers to currently running jobs.
                 items:
-                  description: 'ObjectReference contains enough information to let
+                  description: ObjectReference contains enough information to let
-                    you inspect or modify the referred object. --- New uses of this
+                    you inspect or modify the referred object.
-                    type are discouraged because of difficulty describing its usage
-                    when embedded in APIs.  1. Ignored fields.  It includes many fields
-                    which are not generally honored.  For instance, ResourceVersion
-                    and FieldPath are both very rarely valid in actual usage.  2.
-                    Invalid usage help.  It is impossible to add specific help for
-                    individual usage.  In most embedded usages, there are particular     restrictions
-                    like, "must refer only to types A and B" or "UID not honored"
-                    or "name must be restricted".     Those cannot be well described
-                    when embedded.  3. Inconsistent validation.  Because the usages
-                    are different, the validation rules are different by usage, which
-                    makes it hard for users to predict what will happen.  4. The fields
-                    are both imprecise and overly precise.  Kind is not a precise
-                    mapping to a URL. This can produce ambiguity     during interpretation
-                    and require a REST mapping.  In most cases, the dependency is
-                    on the group,resource tuple     and the version of the actual
-                    struct is irrelevant.  5. We cannot easily change it.  Because
-                    this type is embedded in many locations, updates to this type     will
-                    affect numerous schemas.  Don''t make new APIs embed an underspecified
-                    API type they do not control. Instead of using this type, create
-                    a locally provided and used type that is well-focused on your
-                    reference. For example, ServiceReferences for admission registration:
-                    https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533
-                    .'
                   properties:
                     apiVersion:
                       description: API version of the referent.
                       type: string
                     fieldPath:
-                      description: 'If referring to a piece of an object instead of
+                      description: |-
-                        an entire object, this string should contain a valid JSON/Go
+                        If referring to a piece of an object instead of an entire object, this string
-                        field access statement, such as desiredState.manifest.containers[2].
+                        should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
-                        For example, if the object reference is to a container within
+                        For example, if the object reference is to a container within a pod, this would take on a value like:
-                        a pod, this would take on a value like: "spec.containers{name}"
+                        "spec.containers{name}" (where "name" refers to the name of the container that triggered
-                        (where "name" refers to the name of the container that triggered
+                        the event) or if no container name is specified "spec.containers[2]" (container with
-                        the event) or if no container name is specified "spec.containers[2]"
+                        index 2 in this pod). This syntax is chosen only to have some well-defined way of
-                        (container with index 2 in this pod). This syntax is chosen
+                        referencing a part of an object.
-                        only to have some well-defined way of referencing a part of
-                        an object. TODO: this design is not final and this field is
-                        subject to change in the future.'
                       type: string
                     kind:
-                      description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                      description: |-
+                        Kind of the referent.
+                        More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
                       type: string
                     name:
-                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                      description: |-
+                        Name of the referent.
+                        More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
                       type: string
                     namespace:
-                      description: 'Namespace of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                      description: |-
+                        Namespace of the referent.
+                        More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
                       type: string
                     resourceVersion:
-                      description: 'Specific resourceVersion to which this reference
+                      description: |-
-                        is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                        Specific resourceVersion to which this reference is made, if any.
+                        More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
                       type: string
                     uid:
-                      description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                      description: |-
+                        UID of the referent.
+                        More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
                       type: string
                   type: object
+                  x-kubernetes-map-type: atomic
                 type: array
               lastScheduleTime:
                 description: Information when was the last time the job was successfully
@@ -269,9 +259,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_broadcastjobs.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: broadcastjobs.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -49,14 +47,19 @@ spec:
         description: BroadcastJob is the Schema for the broadcastjobs API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -64,31 +67,34 @@ spec:
             description: BroadcastJobSpec defines the desired state of BroadcastJob
             properties:
               completionPolicy:
-                description: CompletionPolicy indicates the completion policy of the
+                description: |-
-                  job. Default is Always CompletionPolicyType.
+                  CompletionPolicy indicates the completion policy of the job.
+                  Default is Always CompletionPolicyType.
                 properties:
                   activeDeadlineSeconds:
-                    description: ActiveDeadlineSeconds specifies the duration in seconds
+                    description: |-
-                      relative to the startTime that the job may be active before
+                      ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the job may be active
-                      the system tries to terminate it; value must be positive integer.
+                      before the system tries to terminate it; value must be positive integer.
                       Only works for Always type.
                     format: int64
                     type: integer
                   ttlSecondsAfterFinished:
-                    description: ttlSecondsAfterFinished limits the lifetime of a
+                    description: |-
-                      Job that has finished execution (either Complete or Failed).
+                      ttlSecondsAfterFinished limits the lifetime of a Job that has finished
-                      If this field is set, ttlSecondsAfterFinished after the Job
+                      execution (either Complete or Failed). If this field is set,
-                      finishes, it is eligible to be automatically deleted. When the
+                      ttlSecondsAfterFinished after the Job finishes, it is eligible to be
-                      Job is being deleted, its lifecycle guarantees (e.g. finalizers)
+                      automatically deleted. When the Job is being deleted, its lifecycle
-                      will be honored. If this field is unset, the Job won't be automatically
+                      guarantees (e.g. finalizers) will be honored. If this field is unset,
-                      deleted. If this field is set to zero, the Job becomes eligible
+                      the Job won't be automatically deleted. If this field is set to zero,
-                      to be deleted immediately after it finishes. This field is alpha-level
+                      the Job becomes eligible to be deleted immediately after it finishes.
-                      and is only honored by servers that enable the TTLAfterFinished
+                      This field is alpha-level and is only honored by servers that enable the
-                      feature. Only works for Always type
+                      TTLAfterFinished feature.
+                      Only works for Always type
                     format: int32
                     type: integer
                   type:
-                    description: Type indicates the type of the CompletionPolicy.
+                    description: |-
+                      Type indicates the type of the CompletionPolicy.
                       Default is Always.
                     type: string
                 type: object
@@ -102,19 +108,20 @@ spec:
                     format: int32
                     type: integer
                   type:
-                    description: Type indicates the type of FailurePolicyType. Default
+                    description: |-
-                      is FailurePolicyTypeFailFast.
+                      Type indicates the type of FailurePolicyType.
+                      Default is FailurePolicyTypeFailFast.
                     type: string
                 type: object
               parallelism:
                 anyOf:
                 - type: integer
                 - type: string
-                description: Parallelism specifies the maximum desired number of pods
+                description: |-
-                  the job should run at any given time. The actual number of pods
+                  Parallelism specifies the maximum desired number of pods the job should
-                  running in steady state will be less than this number when the work
+                  run at any given time. The actual number of pods running in steady state will
-                  left to do is less than max parallelism. Not setting this value
+                  be less than this number when the work left to do is less than max parallelism.
-                  means no limit.
+                  Not setting this value means no limit.
                 x-kubernetes-int-or-string: true
               paused:
                 description: Paused will pause the job.
@@ -134,8 +141,9 @@ spec:
                 format: int32
                 type: integer
               completionTime:
-                description: Represents time when the job was completed. It is not
+                description: |-
-                  guaranteed to be set in happens-before order across separate operations.
+                  Represents time when the job was completed. It is not guaranteed to
+                  be set in happens-before order across separate operations.
                   It is represented in RFC3339 form and is in UTC.
                 format: date-time
                 type: string
@@ -185,10 +193,10 @@ spec:
                 description: The phase of the job.
                 type: string
               startTime:
-                description: Represents time when the job was acknowledged by the
+                description: |-
-                  job controller. It is not guaranteed to be set in happens-before
+                  Represents time when the job was acknowledged by the job controller.
-                  order across separate operations. It is represented in RFC3339 form
+                  It is not guaranteed to be set in happens-before order across separate operations.
-                  and is in UTC.
+                  It is represented in RFC3339 form and is in UTC.
                 format: date-time
                 type: string
               succeeded:
@@ -201,9 +209,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_clonesets.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: clonesets.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -31,6 +29,10 @@ spec:
       jsonPath: .status.updatedReadyReplicas
       name: UPDATED_READY
       type: integer
+    - description: The number of pods updated and available.
+      jsonPath: .status.updatedAvailableReplicas
+      name: UPDATED_AVAILABLE
+      type: integer
     - description: The number of pods ready.
       jsonPath: .status.readyReplicas
       name: READY
@@ -67,14 +69,19 @@ spec:
         description: CloneSet is the Schema for the clonesets API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -98,12 +105,12 @@ spec:
                           type: string
                         type: object
                       markPodNotReady:
-                        description: 'MarkPodNotReady = true means: - Pod will be
+                        description: |-
-                          set to ''NotReady'' at preparingDelete/preparingUpdate state.
+                          MarkPodNotReady = true means:
-                          - Pod will be restored to ''Ready'' at Updated state if
+                          - Pod will be set to 'NotReady' at preparingDelete/preparingUpdate state.
-                          it was set to ''NotReady'' at preparingUpdate state. Currently,
+                          - Pod will be restored to 'Ready' at Updated state if it was set to 'NotReady' at preparingUpdate state.
-                          MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete
+                          Currently, MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete hook.
-                          hook. Default to false.'
+                          Default to false.
                         type: boolean
                     type: object
                   preDelete:
@@ -118,12 +125,12 @@ spec:
                           type: string
                         type: object
                       markPodNotReady:
-                        description: 'MarkPodNotReady = true means: - Pod will be
+                        description: |-
-                          set to ''NotReady'' at preparingDelete/preparingUpdate state.
+                          MarkPodNotReady = true means:
-                          - Pod will be restored to ''Ready'' at Updated state if
+                          - Pod will be set to 'NotReady' at preparingDelete/preparingUpdate state.
-                          it was set to ''NotReady'' at preparingUpdate state. Currently,
+                          - Pod will be restored to 'Ready' at Updated state if it was set to 'NotReady' at preparingUpdate state.
-                          MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete
+                          Currently, MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete hook.
-                          hook. Default to false.'
+                          Default to false.
                         type: boolean
                     type: object
                   preNormal:
@@ -139,123 +146,131 @@ spec:
                           type: string
                         type: object
                       markPodNotReady:
-                        description: 'MarkPodNotReady = true means: - Pod will be
+                        description: |-
-                          set to ''NotReady'' at preparingDelete/preparingUpdate state.
+                          MarkPodNotReady = true means:
-                          - Pod will be restored to ''Ready'' at Updated state if
+                          - Pod will be set to 'NotReady' at preparingDelete/preparingUpdate state.
-                          it was set to ''NotReady'' at preparingUpdate state. Currently,
+                          - Pod will be restored to 'Ready' at Updated state if it was set to 'NotReady' at preparingUpdate state.
-                          MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete
+                          Currently, MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete hook.
-                          hook. Default to false.'
+                          Default to false.
                         type: boolean
                     type: object
                 type: object
               minReadySeconds:
-                description: Minimum number of seconds for which a newly created pod
+                description: |-
-                  should be ready without any of its container crashing, for it to
+                  Minimum number of seconds for which a newly created pod should be ready
-                  be considered available. Defaults to 0 (pod will be considered available
+                  without any of its container crashing, for it to be considered available.
-                  as soon as it is ready)
+                  Defaults to 0 (pod will be considered available as soon as it is ready)
                 format: int32
                 type: integer
               replicas:
-                description: Replicas is the desired number of replicas of the given
+                description: |-
-                  Template. These are replicas in the sense that they are instantiations
+                  Replicas is the desired number of replicas of the given Template.
-                  of the same Template. If unspecified, defaults to 1.
+                  These are replicas in the sense that they are instantiations of the
+                  same Template.
+                  If unspecified, defaults to 1.
                 format: int32
                 type: integer
               revisionHistoryLimit:
-                description: RevisionHistoryLimit is the maximum number of revisions
+                description: |-
-                  that will be maintained in the CloneSet's revision history. The
+                  RevisionHistoryLimit is the maximum number of revisions that will
-                  revision history consists of all revisions not represented by a
+                  be maintained in the CloneSet's revision history. The revision history
-                  currently applied CloneSetSpec version. The default value is 10.
+                  consists of all revisions not represented by a currently applied
+                  CloneSetSpec version. The default value is 10.
                 format: int32
                 type: integer
               scaleStrategy:
-                description: ScaleStrategy indicates the ScaleStrategy that will be
+                description: |-
-                  employed to create and delete Pods in the CloneSet.
+                  ScaleStrategy indicates the ScaleStrategy that will be employed to
+                  create and delete Pods in the CloneSet.
                 properties:
                   disablePVCReuse:
-                    description: Indicate if cloneSet will reuse already existed pvc
+                    description: |-
-                      to rebuild a new pod
+                      Indicate if cloneSet will reuse already existed pvc to
+                      rebuild a new pod
                     type: boolean
                   maxUnavailable:
                     anyOf:
                     - type: integer
                     - type: string
-                    description: The maximum number of pods that can be unavailable
+                    description: |-
-                      for scaled pods. This field can control the changes rate of
+                      The maximum number of pods that can be unavailable for scaled pods.
-                      replicas for CloneSet so as to minimize the impact for users'
+                      This field can control the changes rate of replicas for CloneSet so as to minimize the impact for users' service.
-                      service. The scale will fail if the number of unavailable pods
+                      The scale will fail if the number of unavailable pods were greater than this MaxUnavailable at scaling up.
-                      were greater than this MaxUnavailable at scaling up. MaxUnavailable
+                      MaxUnavailable works only when scaling up.
-                      works only when scaling up.
                     x-kubernetes-int-or-string: true
                   podsToDelete:
-                    description: PodsToDelete is the names of Pod should be deleted.
+                    description: |-
+                      PodsToDelete is the names of Pod should be deleted.
                       Note that this list will be truncated for non-existing pod names.
                     items:
                       type: string
                     type: array
                 type: object
               selector:
-                description: 'Selector is a label query over pods that should match
+                description: |-
-                  the replica count. It must match the pod template''s labels. More
+                  Selector is a label query over pods that should match the replica count.
-                  info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors'
+                  It must match the pod template's labels.
+                  More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
                       The requirements are ANDed.
                     items:
-                      description: A label selector requirement is a selector that
+                      description: |-
-                        contains values, a key, and an operator that relates the key
+                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        and values.
+                        relates the key and values.
                       properties:
                         key:
                           description: key is the label key that the selector applies
                             to.
                           type: string
                         operator:
-                          description: operator represents a key's relationship to
+                          description: |-
-                            a set of values. Valid operators are In, NotIn, Exists
+                            operator represents a key's relationship to a set of values.
-                            and DoesNotExist.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
                           type: string
                         values:
-                          description: values is an array of string values. If the
+                          description: |-
-                            operator is In or NotIn, the values array must be non-empty.
+                            values is an array of string values. If the operator is In or NotIn,
-                            If the operator is Exists or DoesNotExist, the values
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            array must be empty. This array is replaced during a strategic
+                            the values array must be empty. This array is replaced during a strategic
                             merge patch.
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
+                    description: |-
-                      {key,value} in the matchLabels map is equivalent to an element
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      of matchExpressions, whose key field is "key", the operator
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      is "In", and the values array contains only "value". The requirements
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                      are ANDed.
                     type: object
                 type: object
+                x-kubernetes-map-type: atomic
               template:
                 description: Template describes the pods that will be created.
                 x-kubernetes-preserve-unknown-fields: true
               updateStrategy:
-                description: UpdateStrategy indicates the UpdateStrategy that will
+                description: |-
-                  be employed to update Pods in the CloneSet when a revision is made
+                  UpdateStrategy indicates the UpdateStrategy that will be employed to
-                  to Template.
+                  update Pods in the CloneSet when a revision is made to Template.
                 properties:
                   inPlaceUpdateStrategy:
                     description: InPlaceUpdateStrategy contains strategies for in-place
                       update.
                     properties:
                       gracePeriodSeconds:
-                        description: GracePeriodSeconds is the timespan between set
+                        description: |-
-                          Pod status to not-ready and update images in Pod spec when
+                          GracePeriodSeconds is the timespan between set Pod status to not-ready and update images in Pod spec
-                          in-place update a Pod.
+                          when in-place update a Pod.
                         format: int32
                         type: integer
                     type: object
@@ -263,59 +278,64 @@ spec:
                     anyOf:
                     - type: integer
                     - type: string
-                    description: 'The maximum number of pods that can be scheduled
+                    description: |-
-                      above the desired replicas during update or specified delete.
+                      The maximum number of pods that can be scheduled above the desired replicas during update or specified delete.
-                      Value can be an absolute number (ex: 5) or a percentage of desired
+                      Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
-                      pods (ex: 10%). Absolute number is calculated from percentage
+                      Absolute number is calculated from percentage by rounding up.
-                      by rounding up. Defaults to 0.'
+                      Defaults to 0.
                     x-kubernetes-int-or-string: true
                   maxUnavailable:
                     anyOf:
                     - type: integer
                     - type: string
-                    description: 'The maximum number of pods that can be unavailable
+                    description: |-
-                      during update or scale. Value can be an absolute number (ex:
+                      The maximum number of pods that can be unavailable during update or scale.
-                      5) or a percentage of desired pods (ex: 10%). Absolute number
+                      Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
-                      is calculated from percentage by rounding up by default. When
+                      Absolute number is calculated from percentage by rounding up by default.
-                      maxSurge > 0, absolute number is calculated from percentage
+                      When maxSurge > 0, absolute number is calculated from percentage by rounding down.
-                      by rounding down. Defaults to 20%.'
+                      Defaults to 20%.
                     x-kubernetes-int-or-string: true
                   partition:
                     anyOf:
                     - type: integer
                     - type: string
-                    description: 'Partition is the desired number of pods in old revisions.
+                    description: |-
-                      Value can be an absolute number (ex: 5) or a percentage of desired
+                      Partition is the desired number of pods in old revisions.
-                      pods (ex: 10%). Absolute number is calculated from percentage
+                      Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
-                      by rounding up by default. It means when partition is set during
+                      Absolute number is calculated from percentage by rounding up by default.
-                      pods updating, (replicas - partition value) number of pods will
+                      It means when partition is set during pods updating, (replicas - partition value) number of pods will be updated.
-                      be updated. Default value is 0.'
+                      Default value is 0.
                     x-kubernetes-int-or-string: true
                   paused:
-                    description: Paused indicates that the CloneSet is paused. Default
+                    description: |-
-                      value is false
+                      Paused indicates that the CloneSet is paused.
+                      Default value is false
                     type: boolean
                   priorityStrategy:
-                    description: Priorities are the rules for calculating the priority
+                    description: |-
-                      of updating pods. Each pod to be updated, will pass through
+                      Priorities are the rules for calculating the priority of updating pods.
-                      these terms and get a sum of weights.
+                      Each pod to be updated, will pass through these terms and get a sum of weights.
                     properties:
                       orderPriority:
-                        description: 'Order priority terms, pods will be sorted by
+                        description: |-
-                          the value of orderedKey. For example: ``` orderPriority:
+                          Order priority terms, pods will be sorted by the value of orderedKey.
-                          - orderedKey: key1 - orderedKey: key2 ``` First, all pods
+                          For example:
-                          which have key1 in labels will be sorted by the value of
+                          ```
-                          key1. Then, the left pods which have no key1 but have key2
+                          orderPriority:
-                          in labels will be sorted by the value of key2 and put behind
+                          - orderedKey: key1
-                          those pods have key1.'
+                          - orderedKey: key2
+                          ```
+                          First, all pods which have key1 in labels will be sorted by the value of key1.
+                          Then, the left pods which have no key1 but have key2 in labels will be sorted by
+                          the value of key2 and put behind those pods have key1.
                         items:
                           description: UpdatePriorityOrderTerm defines order priority.
                           properties:
                             orderedKey:
-                              description: Calculate priority by value of this key.
+                              description: |-
-                                Values of this key, will be sorted by GetInt(val).
+                                Calculate priority by value of this key.
-                                GetInt method will find the last int in value, such
+                                Values of this key, will be sorted by GetInt(val). GetInt method will find the last int in value,
-                                as getting 5 in value '5', getting 10 in value 'sts-10'.
+                                such as getting 5 in value '5', getting 10 in value 'sts-10'.
                               type: string
                           required:
                           - orderedKey
@@ -335,45 +355,45 @@ spec:
                                   description: matchExpressions is a list of label
                                     selector requirements. The requirements are ANDed.
                                   items:
-                                    description: A label selector requirement is a
+                                    description: |-
-                                      selector that contains values, a key, and an
+                                      A label selector requirement is a selector that contains values, a key, and an operator that
-                                      operator that relates the key and values.
+                                      relates the key and values.
                                     properties:
                                       key:
                                         description: key is the label key that the
                                           selector applies to.
                                         type: string
                                       operator:
-                                        description: operator represents a key's relationship
+                                        description: |-
-                                          to a set of values. Valid operators are
+                                          operator represents a key's relationship to a set of values.
-                                          In, NotIn, Exists and DoesNotExist.
+                                          Valid operators are In, NotIn, Exists and DoesNotExist.
                                         type: string
                                       values:
-                                        description: values is an array of string
+                                        description: |-
-                                          values. If the operator is In or NotIn,
+                                          values is an array of string values. If the operator is In or NotIn,
-                                          the values array must be non-empty. If the
+                                          the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                          operator is Exists or DoesNotExist, the
+                                          the values array must be empty. This array is replaced during a strategic
-                                          values array must be empty. This array is
+                                          merge patch.
-                                          replaced during a strategic merge patch.
                                         items:
                                           type: string
                                         type: array
+                                        x-kubernetes-list-type: atomic
                                     required:
                                     - key
                                     - operator
                                     type: object
                                   type: array
+                                  x-kubernetes-list-type: atomic
                                 matchLabels:
                                   additionalProperties:
                                     type: string
-                                  description: matchLabels is a map of {key,value}
+                                  description: |-
-                                    pairs. A single {key,value} in the matchLabels
+                                    matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                                    map is equivalent to an element of matchExpressions,
+                                    map is equivalent to an element of matchExpressions, whose key field is "key", the
-                                    whose key field is "key", the operator is "In",
+                                    operator is "In", and the values array contains only "value". The requirements are ANDed.
-                                    and the values array contains only "value". The
-                                    requirements are ANDed.
                                   type: object
                               type: object
+                              x-kubernetes-map-type: atomic
                             weight:
                               description: Weight associated with matching the corresponding
                                 matchExpressions, in the range 1-100.
@@ -386,14 +406,11 @@ spec:
                         type: array
                     type: object
                   scatterStrategy:
-                    description: ScatterStrategy defines the scatter rules to make
+                    description: |-
-                      pods been scattered when update. This will avoid pods with the
+                      ScatterStrategy defines the scatter rules to make pods been scattered when update.
-                      same key-value to be updated in one batch. - Note that pods
+                      This will avoid pods with the same key-value to be updated in one batch.
-                      will be scattered after priority sort. So, although priority
+                      - Note that pods will be scattered after priority sort. So, although priority strategy and scatter strategy can be applied together, we suggest to use either one of them.
-                      strategy and scatter strategy can be applied together, we suggest
+                      - If scatterStrategy is used, we suggest to just use one term. Otherwise, the update order can be hard to understand.
-                      to use either one of them. - If scatterStrategy is used, we
-                      suggest to just use one term. Otherwise, the update order can
-                      be hard to understand.
                     items:
                       properties:
                         key:
@@ -406,14 +423,15 @@ spec:
                       type: object
                     type: array
                   type:
-                    description: Type indicates the type of the CloneSetUpdateStrategy.
+                    description: |-
+                      Type indicates the type of the CloneSetUpdateStrategy.
                       Default is ReCreate.
                     type: string
                 type: object
               volumeClaimTemplates:
-                description: VolumeClaimTemplates is a list of claims that pods are
+                description: |-
-                  allowed to reference. Note that PVC will be deleted when its pod
+                  VolumeClaimTemplates is a list of claims that pods are allowed to reference.
-                  has been deleted.
+                  Note that PVC will be deleted when its pod has been deleted.
                 x-kubernetes-preserve-unknown-fields: true
             required:
             - selector
@@ -428,10 +446,10 @@ spec:
                 format: int32
                 type: integer
               collisionCount:
-                description: CollisionCount is the count of hash collisions for the
+                description: |-
-                  CloneSet. The CloneSet controller uses this field as a collision
+                  CollisionCount is the count of hash collisions for the CloneSet. The CloneSet controller
-                  avoidance mechanism when it needs to create the name for the newest
+                  uses this field as a collision avoidance mechanism when it needs to create the name for the
-                  ControllerRevision.
+                  newest ControllerRevision.
                 format: int32
                 type: integer
               conditions:
@@ -469,9 +487,9 @@ spec:
                   revision version of the CloneSet.
                 type: string
               expectedUpdatedReplicas:
-                description: ExpectedUpdatedReplicas is the number of Pods that should
+                description: |-
-                  be updated by CloneSet controller. This field is calculated via
+                  ExpectedUpdatedReplicas is the number of Pods that should be updated by CloneSet controller.
-                  Replicas - Partition.
+                  This field is calculated via Replicas - Partition.
                 format: int32
                 type: integer
               labelSelector:
@@ -479,9 +497,9 @@ spec:
                   that should match the replica count used by HPA.
                 type: string
               observedGeneration:
-                description: ObservedGeneration is the most recent generation observed
+                description: |-
-                  for this CloneSet. It corresponds to the CloneSet's generation,
+                  ObservedGeneration is the most recent generation observed for this CloneSet. It corresponds to the
-                  which is updated on mutation by the API Server.
+                  CloneSet's generation, which is updated on mutation by the API Server.
                 format: int64
                 type: integer
               readyReplicas:
@@ -498,15 +516,24 @@ spec:
                 description: UpdateRevision, if not empty, indicates the latest revision
                   of the CloneSet.
                 type: string
+              updatedAvailableReplicas:
+                description: |-
+                  UpdatedAvailableReplicas is the number of Pods created by the CloneSet controller from the CloneSet version
+                  indicated by updateRevision and have a Ready Condition for at least minReadySeconds.
+                  Notice: when enable InPlaceWorkloadVerticalScaling, pod during resource resizing will also be unavailable.
+                  This means these pod will be counted in maxUnavailable.
+                format: int32
+                type: integer
               updatedReadyReplicas:
-                description: UpdatedReadyReplicas is the number of Pods created by
+                description: |-
-                  the CloneSet controller from the CloneSet version indicated by updateRevision
+                  UpdatedReadyReplicas is the number of Pods created by the CloneSet controller from the CloneSet version
-                  and have a Ready Condition.
+                  indicated by updateRevision and have a Ready Condition.
                 format: int32
                 type: integer
               updatedReplicas:
-                description: UpdatedReplicas is the number of Pods created by the
+                description: |-
-                  CloneSet controller from the CloneSet version indicated by updateRevision.
+                  UpdatedReplicas is the number of Pods created by the CloneSet controller from the CloneSet version
+                  indicated by updateRevision.
                 format: int32
                 type: integer
             required:
@@ -525,9 +552,3 @@ spec:
         specReplicasPath: .spec.replicas
         statusReplicasPath: .status.replicas
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_containerrecreaterequests.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: containerrecreaterequests.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -45,14 +43,19 @@ spec:
           API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -73,41 +76,46 @@ spec:
                     that need to recreate.
                   properties:
                     name:
-                      description: Name of the container that need to recreate. It
+                      description: |-
-                        must be existing in the real pod.Spec.Containers.
+                        Name of the container that need to recreate.
+                        It must be existing in the real pod.Spec.Containers.
                       type: string
                     ports:
-                      description: Ports is synced from the real container in Pod
+                      description: |-
-                        spec during this ContainerRecreateRequest creating. Populated
+                        Ports is synced from the real container in Pod spec during this ContainerRecreateRequest creating.
-                        by the system. Read-only.
+                        Populated by the system.
+                        Read-only.
                       items:
                         description: ContainerPort represents a network port in a
                           single container.
                         properties:
                           containerPort:
-                            description: Number of port to expose on the pod's IP
+                            description: |-
-                              address. This must be a valid port number, 0 < x < 65536.
+                              Number of port to expose on the pod's IP address.
+                              This must be a valid port number, 0 < x < 65536.
                             format: int32
                             type: integer
                           hostIP:
                             description: What host IP to bind the external port to.
                             type: string
                           hostPort:
-                            description: Number of port to expose on the host. If
+                            description: |-
-                              specified, this must be a valid port number, 0 < x <
+                              Number of port to expose on the host.
-                              65536. If HostNetwork is specified, this must match
+                              If specified, this must be a valid port number, 0 < x < 65536.
-                              ContainerPort. Most containers do not need this.
+                              If HostNetwork is specified, this must match ContainerPort.
+                              Most containers do not need this.
                             format: int32
                             type: integer
                           name:
-                            description: If specified, this must be an IANA_SVC_NAME
+                            description: |-
-                              and unique within the pod. Each named port in a pod
+                              If specified, this must be an IANA_SVC_NAME and unique within the pod. Each
-                              must have a unique name. Name for the port that can
+                              named port in a pod must have a unique name. Name for the port that can be
-                              be referred to by services.
+                              referred to by services.
                             type: string
                           protocol:
                             default: TCP
-                            description: Protocol for port. Must be UDP, TCP, or SCTP.
+                            description: |-
+                              Protocol for port. Must be UDP, TCP, or SCTP.
                               Defaults to "TCP".
                             type: string
                         required:
@@ -115,34 +123,35 @@ spec:
                         type: object
                       type: array
                     preStop:
-                      description: PreStop is synced from the real container in Pod
+                      description: |-
-                        spec during this ContainerRecreateRequest creating. Populated
+                        PreStop is synced from the real container in Pod spec during this ContainerRecreateRequest creating.
-                        by the system. Read-only.
+                        Populated by the system.
+                        Read-only.
                       properties:
                         exec:
-                          description: One and only one of the following should be
+                          description: |-
-                            specified. Exec specifies the action to take.
+                            One and only one of the following should be specified.
+                            Exec specifies the action to take.
                           properties:
                             command:
-                              description: Command is the command line to execute
+                              description: |-
-                                inside the container, the working directory for the
+                                Command is the command line to execute inside the container, the working directory for the
-                                command  is root ('/') in the container's filesystem.
+                                command  is root ('/') in the container's filesystem. The command is simply exec'd, it is
-                                The command is simply exec'd, it is not run inside
+                                not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use
-                                a shell, so traditional shell instructions ('|', etc)
+                                a shell, you need to explicitly call out to that shell.
-                                won't work. To use a shell, you need to explicitly
+                                Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
-                                call out to that shell. Exit status of 0 is treated
-                                as live/healthy and non-zero is unhealthy.
                               items:
                                 type: string
                               type: array
+                              x-kubernetes-list-type: atomic
                           type: object
                         httpGet:
                           description: HTTPGet specifies the http request to perform.
                           properties:
                             host:
-                              description: Host name to connect to, defaults to the
+                              description: |-
-                                pod IP. You probably want to set "Host" in httpHeaders
+                                Host name to connect to, defaults to the pod IP. You probably want to set
-                                instead.
+                                "Host" in httpHeaders instead.
                               type: string
                             httpHeaders:
                               description: Custom headers to set in the request. HTTP
@@ -152,7 +161,9 @@ spec:
                                   to be used in HTTP probes
                                 properties:
                                   name:
-                                    description: The header field name
+                                    description: |-
+                                      The header field name.
+                                      This will be canonicalized upon output, so case-variant names will be understood as the same header.
                                     type: string
                                   value:
                                     description: The header field value
@@ -162,6 +173,7 @@ spec:
                                 - value
                                 type: object
                               type: array
+                              x-kubernetes-list-type: atomic
                             path:
                               description: Path to access on the HTTP server.
                               type: string
@@ -169,21 +181,23 @@ spec:
                               anyOf:
                               - type: integer
                               - type: string
-                              description: Name or number of the port to access on
+                              description: |-
-                                the container. Number must be in the range 1 to 65535.
+                                Name or number of the port to access on the container.
+                                Number must be in the range 1 to 65535.
                                 Name must be an IANA_SVC_NAME.
                               x-kubernetes-int-or-string: true
                             scheme:
-                              description: Scheme to use for connecting to the host.
+                              description: |-
+                                Scheme to use for connecting to the host.
                                 Defaults to HTTP.
                               type: string
                           required:
                           - port
                           type: object
                         tcpSocket:
-                          description: 'TCPSocket specifies an action involving a
+                          description: |-
-                            TCP port. TCP hooks not yet supported TODO: implement
+                            TCPSocket specifies an action involving a TCP port.
-                            a realistic TCP lifecycle hook'
+                            TCP hooks not yet supported
                           properties:
                             host:
                               description: 'Optional: Host name to connect to, defaults
@@ -193,8 +207,9 @@ spec:
                               anyOf:
                               - type: integer
                               - type: string
-                              description: Number or name of the port to access on
+                              description: |-
-                                the container. Number must be in the range 1 to 65535.
+                                Number or name of the port to access on the container.
+                                Number must be in the range 1 to 65535.
                                 Name must be an IANA_SVC_NAME.
                               x-kubernetes-int-or-string: true
                           required:
@@ -202,20 +217,20 @@ spec:
                           type: object
                       type: object
                     statusContext:
-                      description: StatusContext is synced from the real Pod status
+                      description: |-
-                        during this ContainerRecreateRequest creating. Populated by
+                        StatusContext is synced from the real Pod status during this ContainerRecreateRequest creating.
-                        the system. Read-only.
+                        Populated by the system.
+                        Read-only.
                       properties:
                         containerID:
                           description: Container's ID in the format 'docker://<container_id>'.
                           type: string
                         restartCount:
-                          description: The number of times the container has been
+                          description: |-
-                            restarted, currently based on the number of dead containers
+                            The number of times the container has been restarted, currently based on
-                            that have not yet been removed. Note that this is calculated
+                            the number of dead containers that have not yet been removed.
-                            from dead containers. But those containers are subject
+                            Note that this is calculated from dead containers. But those containers are subject to
-                            to garbage collection. This value will get capped at 5
+                            garbage collection. This value will get capped at 5 by GC.
-                            by GC.
                           format: int32
                           type: integer
                       required:
@@ -241,10 +256,10 @@ spec:
                       container even if the previous container is starting.
                     type: boolean
                   minStartedSeconds:
-                    description: Minimum number of seconds for which a newly created
+                    description: |-
-                      container should be started and ready without any of its container
+                      Minimum number of seconds for which a newly created container should be started and ready
-                      crashing, for it to be considered Succeeded. Defaults to 0 (container
+                      without any of its container crashing, for it to be considered Succeeded.
-                      will be considered Succeeded as soon as it is started and ready)
+                      Defaults to 0 (container will be considered Succeeded as soon as it is started and ready)
                     format: int32
                     type: integer
                   orderedRecreate:
@@ -252,16 +267,15 @@ spec:
                       next container only if the previous one has recreated completely.
                     type: boolean
                   terminationGracePeriodSeconds:
-                    description: TerminationGracePeriodSeconds is the optional duration
+                    description: |-
-                      in seconds to wait the container terminating gracefully. Value
+                      TerminationGracePeriodSeconds is the optional duration in seconds to wait the container terminating gracefully.
-                      must be non-negative integer. The value zero indicates delete
+                      Value must be non-negative integer. The value zero indicates delete immediately.
-                      immediately. If this value is nil, we will use pod.Spec.TerminationGracePeriodSeconds
+                      If this value is nil, we will use pod.Spec.TerminationGracePeriodSeconds as default value.
-                      as default value.
                     format: int64
                     type: integer
                   unreadyGracePeriodSeconds:
-                    description: UnreadyGracePeriodSeconds is the optional duration
+                    description: |-
-                      in seconds to mark Pod as not ready over this duration before
+                      UnreadyGracePeriodSeconds is the optional duration in seconds to mark Pod as not ready over this duration before
                       executing preStop hook and stopping the container.
                     format: int64
                     type: integer
@@ -280,10 +294,10 @@ spec:
               of ContainerRecreateRequest
             properties:
               completionTime:
-                description: Represents time when the ContainerRecreateRequest was
+                description: |-
-                  completed. It is not guaranteed to be set in happens-before order
+                  Represents time when the ContainerRecreateRequest was completed. It is not guaranteed to
-                  across separate operations. It is represented in RFC3339 form and
+                  be set in happens-before order across separate operations.
-                  is in UTC.
+                  It is represented in RFC3339 form and is in UTC.
                 format: date-time
                 type: string
               containerRecreateStates:
@@ -327,9 +341,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_daemonsets.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: daemonsets.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -63,14 +61,19 @@ spec:
         description: DaemonSet is the Schema for the daemonsets API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -81,13 +84,14 @@ spec:
                 anyOf:
                 - type: integer
                 - type: string
-                description: BurstReplicas is a rate limiter for booting pods on a
+                description: |-
-                  lot of pods. The default value is 250
+                  BurstReplicas is a rate limiter for booting pods on a lot of pods.
+                  The default value is 250
                 x-kubernetes-int-or-string: true
               lifecycle:
-                description: Lifecycle defines the lifecycle hooks for Pods pre-delete,
+                description: |-
-                  in-place update. Currently, we only support pre-delete hook for
+                  Lifecycle defines the lifecycle hooks for Pods pre-delete, in-place update.
-                  Advanced DaemonSet.
+                  Currently, we only support pre-delete hook for Advanced DaemonSet.
                 properties:
                   inPlaceUpdate:
                     description: InPlaceUpdate is the hook before Pod to update and
@@ -102,12 +106,12 @@ spec:
                           type: string
                         type: object
                       markPodNotReady:
-                        description: 'MarkPodNotReady = true means: - Pod will be
+                        description: |-
-                          set to ''NotReady'' at preparingDelete/preparingUpdate state.
+                          MarkPodNotReady = true means:
-                          - Pod will be restored to ''Ready'' at Updated state if
+                          - Pod will be set to 'NotReady' at preparingDelete/preparingUpdate state.
-                          it was set to ''NotReady'' at preparingUpdate state. Currently,
+                          - Pod will be restored to 'Ready' at Updated state if it was set to 'NotReady' at preparingUpdate state.
-                          MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete
+                          Currently, MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete hook.
-                          hook. Default to false.'
+                          Default to false.
                         type: boolean
                     type: object
                   preDelete:
@@ -122,12 +126,12 @@ spec:
                           type: string
                         type: object
                       markPodNotReady:
-                        description: 'MarkPodNotReady = true means: - Pod will be
+                        description: |-
-                          set to ''NotReady'' at preparingDelete/preparingUpdate state.
+                          MarkPodNotReady = true means:
-                          - Pod will be restored to ''Ready'' at Updated state if
+                          - Pod will be set to 'NotReady' at preparingDelete/preparingUpdate state.
-                          it was set to ''NotReady'' at preparingUpdate state. Currently,
+                          - Pod will be restored to 'Ready' at Updated state if it was set to 'NotReady' at preparingUpdate state.
-                          MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete
+                          Currently, MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete hook.
-                          hook. Default to false.'
+                          Default to false.
                         type: boolean
                     type: object
                   preNormal:
@@ -143,79 +147,87 @@ spec:
                           type: string
                         type: object
                       markPodNotReady:
-                        description: 'MarkPodNotReady = true means: - Pod will be
+                        description: |-
-                          set to ''NotReady'' at preparingDelete/preparingUpdate state.
+                          MarkPodNotReady = true means:
-                          - Pod will be restored to ''Ready'' at Updated state if
+                          - Pod will be set to 'NotReady' at preparingDelete/preparingUpdate state.
-                          it was set to ''NotReady'' at preparingUpdate state. Currently,
+                          - Pod will be restored to 'Ready' at Updated state if it was set to 'NotReady' at preparingUpdate state.
-                          MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete
+                          Currently, MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete hook.
-                          hook. Default to false.'
+                          Default to false.
                         type: boolean
                     type: object
                 type: object
               minReadySeconds:
-                description: The minimum number of seconds for which a newly created
+                description: |-
-                  DaemonSet pod should be ready without any of its container crashing,
+                  The minimum number of seconds for which a newly created DaemonSet pod should
-                  for it to be considered available. Defaults to 0 (pod will be considered
+                  be ready without any of its container crashing, for it to be considered
-                  available as soon as it is ready).
+                  available. Defaults to 0 (pod will be considered available as soon as it
+                  is ready).
                 format: int32
                 type: integer
               revisionHistoryLimit:
-                description: The number of old history to retain to allow rollback.
+                description: |-
+                  The number of old history to retain to allow rollback.
                   This is a pointer to distinguish between explicit zero and not specified.
                   Defaults to 10.
                 format: int32
                 type: integer
               selector:
-                description: 'A label query over pods that are managed by the daemon
+                description: |-
-                  set. Must match in order to be controlled. It must match the pod
+                  A label query over pods that are managed by the daemon set.
-                  template''s labels. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors'
+                  Must match in order to be controlled.
+                  It must match the pod template's labels.
+                  More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
                       The requirements are ANDed.
                     items:
-                      description: A label selector requirement is a selector that
+                      description: |-
-                        contains values, a key, and an operator that relates the key
+                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        and values.
+                        relates the key and values.
                       properties:
                         key:
                           description: key is the label key that the selector applies
                             to.
                           type: string
                         operator:
-                          description: operator represents a key's relationship to
+                          description: |-
-                            a set of values. Valid operators are In, NotIn, Exists
+                            operator represents a key's relationship to a set of values.
-                            and DoesNotExist.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
                           type: string
                         values:
-                          description: values is an array of string values. If the
+                          description: |-
-                            operator is In or NotIn, the values array must be non-empty.
+                            values is an array of string values. If the operator is In or NotIn,
-                            If the operator is Exists or DoesNotExist, the values
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            array must be empty. This array is replaced during a strategic
+                            the values array must be empty. This array is replaced during a strategic
                             merge patch.
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
+                    description: |-
-                      {key,value} in the matchLabels map is equivalent to an element
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      of matchExpressions, whose key field is "key", the operator
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      is "In", and the values array contains only "value". The requirements
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                      are ANDed.
                     type: object
                 type: object
+                x-kubernetes-map-type: atomic
               template:
-                description: 'An object that describes the pod that will be created.
+                description: |-
-                  The DaemonSet will create exactly one copy of this pod on every
+                  An object that describes the pod that will be created.
-                  node that matches the template''s node selector (or on every node
+                  The DaemonSet will create exactly one copy of this pod on every node
-                  if no node selector is specified). More info: https://kubernetes.io/docs/concepts/workloads/controllers/replicationcontroller#pod-template'
+                  that matches the template's node selector (or on every node if no node
+                  selector is specified).
+                  More info: https://kubernetes.io/docs/concepts/workloads/controllers/replicationcontroller#pod-template
                 x-kubernetes-preserve-unknown-fields: true
               updateStrategy:
                 description: An update strategy to replace existing DaemonSet pods
@@ -229,74 +241,74 @@ spec:
                         anyOf:
                         - type: integer
                         - type: string
-                        description: 'The maximum number of nodes with an existing
+                        description: |-
-                          available DaemonSet pod that can have an updated DaemonSet
+                          The maximum number of nodes with an existing available DaemonSet pod that
-                          pod during during an update. Value can be an absolute number
+                          can have an updated DaemonSet pod during during an update.
-                          (ex: 5) or a percentage of desired pods (ex: 10%). This
+                          Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
-                          can not be 0 if MaxUnavailable is 0. Absolute number is
+                          This can not be 0 if MaxUnavailable is 0.
-                          calculated from percentage by rounding up to a minimum of
+                          Absolute number is calculated from percentage by rounding up to a minimum of 1.
-                          1. Default value is 0. Example: when this is set to 30%,
+                          Default value is 0.
-                          at most 30% of the total number of nodes that should be
+                          Example: when this is set to 30%, at most 30% of the total number of nodes
-                          running the daemon pod (i.e. status.desiredNumberScheduled)
+                          that should be running the daemon pod (i.e. status.desiredNumberScheduled)
-                          can have their a new pod created before the old pod is marked
+                          can have their a new pod created before the old pod is marked as deleted.
-                          as deleted. The update starts by launching new pods on 30%
+                          The update starts by launching new pods on 30% of nodes. Once an updated
-                          of nodes. Once an updated pod is available (Ready for at
+                          pod is available (Ready for at least minReadySeconds) the old DaemonSet pod
-                          least minReadySeconds) the old DaemonSet pod on that node
+                          on that node is marked deleted. If the old pod becomes unavailable for any
-                          is marked deleted. If the old pod becomes unavailable for
+                          reason (Ready transitions to false, is evicted, or is drained) an updated
-                          any reason (Ready transitions to false, is evicted, or is
+                          pod is immediately created on that node without considering surge limits.
-                          drained) an updated pod is immediatedly created on that
+                          Allowing surge implies the possibility that the resources consumed by the
-                          node without considering surge limits. Allowing surge implies
+                          daemonset on any given node can double if the readiness check fails, and
-                          the possibility that the resources consumed by the daemonset
+                          so resource intensive daemonsets should take into account that they may
-                          on any given node can double if the readiness check fails,
+                          cause evictions during disruption.
-                          and so resource intensive daemonsets should take into account
+                          This is beta field and enabled/disabled by DaemonSetUpdateSurge feature gate.
-                          that they may cause evictions during disruption. This is
-                          beta field and enabled/disabled by DaemonSetUpdateSurge
-                          feature gate.'
                         x-kubernetes-int-or-string: true
                       maxUnavailable:
                         anyOf:
                         - type: integer
                         - type: string
-                        description: 'The maximum number of DaemonSet pods that can
+                        description: |-
-                          be unavailable during the update. Value can be an absolute
+                          The maximum number of DaemonSet pods that can be unavailable during the
-                          number (ex: 5) or a percentage of total number of DaemonSet
+                          update. Value can be an absolute number (ex: 5) or a percentage of total
-                          pods at the start of the update (ex: 10%). Absolute number
+                          number of DaemonSet pods at the start of the update (ex: 10%). Absolute
-                          is calculated from percentage by rounding up. This cannot
+                          number is calculated from percentage by rounding up.
-                          be 0 if MaxSurge is 0 Default value is 1. Example: when
+                          This cannot be 0 if MaxSurge is 0
-                          this is set to 30%, at most 30% of the total number of nodes
+                          Default value is 1.
+                          Example: when this is set to 30%, at most 30% of the total number of nodes
                           that should be running the daemon pod (i.e. status.desiredNumberScheduled)
-                          can have their pods stopped for an update at any given time.
+                          can have their pods stopped for an update at any given time. The update
-                          The update starts by stopping at most 30% of those DaemonSet
+                          starts by stopping at most 30% of those DaemonSet pods and then brings
-                          pods and then brings up new DaemonSet pods in their place.
+                          up new DaemonSet pods in their place. Once the new pods are available,
-                          Once the new pods are available, it then proceeds onto other
+                          it then proceeds onto other DaemonSet pods, thus ensuring that at least
-                          DaemonSet pods, thus ensuring that at least 70% of original
+                          70% of original number of DaemonSet pods are available at all times during
-                          number of DaemonSet pods are available at all times during
+                          the update.
-                          the update.'
                         x-kubernetes-int-or-string: true
                       partition:
-                        description: The number of DaemonSet pods remained to be old
+                        description: |-
-                          version. Default value is 0. Maximum value is status.DesiredNumberScheduled,
+                          The number of DaemonSet pods remained to be old version.
-                          which means no pod will be updated.
+                          Default value is 0.
+                          Maximum value is status.DesiredNumberScheduled, which means no pod will be updated.
                         format: int32
                         type: integer
                       paused:
-                        description: Indicates that the daemon set is paused and will
+                        description: |-
-                          not be processed by the daemon set controller.
+                          Indicates that the daemon set is paused and will not be processed by the
+                          daemon set controller.
                         type: boolean
                       rollingUpdateType:
                         description: Type is to specify which kind of rollingUpdate.
                         type: string
                       selector:
-                        description: A label query over nodes that are managed by
+                        description: |-
-                          the daemon set RollingUpdate. Must match in order to be
+                          A label query over nodes that are managed by the daemon set RollingUpdate.
-                          controlled. It must match the node's labels.
+                          Must match in order to be controlled.
+                          It must match the node's labels.
                         properties:
                           matchExpressions:
                             description: matchExpressions is a list of label selector
                               requirements. The requirements are ANDed.
                             items:
-                              description: A label selector requirement is a selector
+                              description: |-
-                                that contains values, a key, and an operator that
+                                A label selector requirement is a selector that contains values, a key, and an operator that
                                 relates the key and values.
                               properties:
                                 key:
@@ -304,35 +316,36 @@ spec:
                                     applies to.
                                   type: string
                                 operator:
-                                  description: operator represents a key's relationship
+                                  description: |-
-                                    to a set of values. Valid operators are In, NotIn,
+                                    operator represents a key's relationship to a set of values.
-                                    Exists and DoesNotExist.
+                                    Valid operators are In, NotIn, Exists and DoesNotExist.
                                   type: string
                                 values:
-                                  description: values is an array of string values.
+                                  description: |-
-                                    If the operator is In or NotIn, the values array
+                                    values is an array of string values. If the operator is In or NotIn,
-                                    must be non-empty. If the operator is Exists or
+                                    the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                    DoesNotExist, the values array must be empty.
+                                    the values array must be empty. This array is replaced during a strategic
-                                    This array is replaced during a strategic merge
+                                    merge patch.
-                                    patch.
                                   items:
                                     type: string
                                   type: array
+                                  x-kubernetes-list-type: atomic
                               required:
                               - key
                               - operator
                               type: object
                             type: array
+                            x-kubernetes-list-type: atomic
                           matchLabels:
                             additionalProperties:
                               type: string
-                            description: matchLabels is a map of {key,value} pairs.
+                            description: |-
-                              A single {key,value} in the matchLabels map is equivalent
+                              matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                              to an element of matchExpressions, whose key field is
+                              map is equivalent to an element of matchExpressions, whose key field is "key", the
-                              "key", the operator is "In", and the values array contains
+                              operator is "In", and the values array contains only "value". The requirements are ANDed.
-                              only "value". The requirements are ANDed.
                             type: object
                         type: object
+                        x-kubernetes-map-type: atomic
                     type: object
                   type:
                     description: Type of daemon set update. Can be "RollingUpdate"
@@ -347,9 +360,10 @@ spec:
             description: DaemonSetStatus defines the observed state of DaemonSet
             properties:
               collisionCount:
-                description: Count of hash collisions for the DaemonSet. The DaemonSet
+                description: |-
-                  controller uses this field as a collision avoidance mechanism when
+                  Count of hash collisions for the DaemonSet. The DaemonSet controller
-                  it needs to create the name for the newest ControllerRevision.
+                  uses this field as a collision avoidance mechanism when it needs to
+                  create the name for the newest ControllerRevision.
                 format: int32
                 type: integer
               conditions:
@@ -383,8 +397,10 @@ spec:
                   type: object
                 type: array
               currentNumberScheduled:
-                description: 'The number of nodes that are running at least 1 daemon
+                description: |-
-                  pod and are supposed to run the daemon pod. More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/'
+                  The number of nodes that are running at least 1
+                  daemon pod and are supposed to run the daemon pod.
+                  More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/
                 format: int32
                 type: integer
               daemonSetHash:
@@ -392,31 +408,37 @@ spec:
                   represents the latest version of the DaemonSet.
                 type: string
               desiredNumberScheduled:
-                description: 'The total number of nodes that should be running the
+                description: |-
-                  daemon pod (including nodes correctly running the daemon pod). More
+                  The total number of nodes that should be running the daemon
-                  info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/'
+                  pod (including nodes correctly running the daemon pod).
+                  More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/
                 format: int32
                 type: integer
               numberAvailable:
-                description: The number of nodes that should be running the daemon
+                description: |-
-                  pod and have one or more of the daemon pod running and available
+                  The number of nodes that should be running the
-                  (ready for at least spec.minReadySeconds)
+                  daemon pod and have one or more of the daemon pod running and
+                  available (ready for at least spec.minReadySeconds)
                 format: int32
                 type: integer
               numberMisscheduled:
-                description: 'The number of nodes that are running the daemon pod,
+                description: |-
-                  but are not supposed to run the daemon pod. More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/'
+                  The number of nodes that are running the daemon pod, but are
+                  not supposed to run the daemon pod.
+                  More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/
                 format: int32
                 type: integer
               numberReady:
-                description: The number of nodes that should be running the daemon
+                description: |-
-                  pod and have one or more of the daemon pod running and ready.
+                  The number of nodes that should be running the daemon pod and have one
+                  or more of the daemon pod running and ready.
                 format: int32
                 type: integer
               numberUnavailable:
-                description: The number of nodes that should be running the daemon
+                description: |-
-                  pod and have none of the daemon pod running and available (ready
+                  The number of nodes that should be running the
-                  for at least spec.minReadySeconds)
+                  daemon pod and have none of the daemon pod running and available
+                  (ready for at least spec.minReadySeconds)
                 format: int32
                 type: integer
               observedGeneration:
@@ -442,9 +464,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_ephemeraljobs.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: ephemeraljobs.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -56,14 +54,19 @@ spec:
         description: EphemeralJob is the Schema for the ephemeraljobs API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -71,10 +74,10 @@ spec:
             description: EphemeralJobSpec defines the desired state of EphemeralJob
             properties:
               activeDeadlineSeconds:
-                description: ActiveDeadlineSeconds specifies the duration in seconds
+                description: |-
-                  relative to the startTime that the job may be active before the
+                  ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the job may be active
-                  system tries to terminate it; value must be positive integer. Only
+                  before the system tries to terminate it; value must be positive integer.
-                  works for Always type.
+                  Only works for Always type.
                 format: int64
                 type: integer
               parallelism:
@@ -86,58 +89,61 @@ spec:
                 description: Paused will pause the ephemeral job.
                 type: boolean
               replicas:
-                description: Replicas indicates a part of the quantity from matched
+                description: |-
-                  pods by selector. Usually it is used for gray scale working. if
+                  Replicas indicates a part of the quantity from matched pods by selector.
-                  Replicas exceeded the matched number by selector or not be set,
+                  Usually it is used for gray scale working.
-                  replicas will not work.
+                  if Replicas exceeded the matched number by selector or not be set, replicas will not work.
                 format: int32
                 type: integer
               selector:
-                description: 'INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
+                description: |-
+                  INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
                   Important: Run "make" to regenerate code after modifying this file
-                  Selector is a label query over pods that should match the pod labels.'
+                  Selector is a label query over pods that should match the pod labels.
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
                       The requirements are ANDed.
                     items:
-                      description: A label selector requirement is a selector that
+                      description: |-
-                        contains values, a key, and an operator that relates the key
+                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        and values.
+                        relates the key and values.
                       properties:
                         key:
                           description: key is the label key that the selector applies
                             to.
                           type: string
                         operator:
-                          description: operator represents a key's relationship to
+                          description: |-
-                            a set of values. Valid operators are In, NotIn, Exists
+                            operator represents a key's relationship to a set of values.
-                            and DoesNotExist.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
                           type: string
                         values:
-                          description: values is an array of string values. If the
+                          description: |-
-                            operator is In or NotIn, the values array must be non-empty.
+                            values is an array of string values. If the operator is In or NotIn,
-                            If the operator is Exists or DoesNotExist, the values
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            array must be empty. This array is replaced during a strategic
+                            the values array must be empty. This array is replaced during a strategic
                             merge patch.
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
+                    description: |-
-                      {key,value} in the matchLabels map is equivalent to an element
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      of matchExpressions, whose key field is "key", the operator
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      is "In", and the values array contains only "value". The requirements
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                      are ANDed.
                     type: object
                 type: object
+                x-kubernetes-map-type: atomic
               template:
                 description: Template describes the ephemeral container that will
                   be created.
@@ -150,12 +156,14 @@ spec:
                 - ephemeralContainers
                 type: object
               ttlSecondsAfterFinished:
-                description: ttlSecondsAfterFinished limits the lifetime of a Job
+                description: |-
-                  that has finished execution (either Complete or Failed). If this
+                  ttlSecondsAfterFinished limits the lifetime of a Job that has finished
-                  field is set, ttlSecondsAfterFinished after the eJob finishes, it
+                  execution (either Complete or Failed). If this field is set,
-                  is eligible to be automatically deleted. When the Job is being deleted,
+                  ttlSecondsAfterFinished after the eJob finishes, it is eligible to be
-                  its lifecycle guarantees (e.g. finalizers) will be honored. If this
+                  automatically deleted. When the Job is being deleted, its lifecycle
-                  field is unset, default value is 1800 If this field is set to zero,
+                  guarantees (e.g. finalizers) will be honored.
+                  If this field is unset, default value is 1800
+                  If this field is set to zero,
                   the Job becomes eligible to be deleted immediately after it finishes.
                 format: int32
                 type: integer
@@ -167,15 +175,16 @@ spec:
             description: EphemeralJobStatus defines the observed state of EphemeralJob
             properties:
               completionTime:
-                description: Represents time when the job was completed. It is not
+                description: |-
-                  guaranteed to be set in happens-before order across separate operations.
+                  Represents time when the job was completed. It is not guaranteed to
+                  be set in happens-before order across separate operations.
                   It is represented in RFC3339 form and is in UTC.
                 format: date-time
                 type: string
               conditions:
-                description: 'INSERT ADDITIONAL STATUS FIELD - define observed state
+                description: |-
-                  of cluster Important: Run "make" to regenerate code after modifying
+                  INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
-                  this file'
+                  Important: Run "make" to regenerate code after modifying this file
                 items:
                   description: JobCondition describes current state of a job.
                   properties:
@@ -222,10 +231,10 @@ spec:
                 format: int32
                 type: integer
               startTime:
-                description: Represents time when the job was acknowledged by the
+                description: |-
-                  job controller. It is not guaranteed to be set in happens-before
+                  Represents time when the job was acknowledged by the job controller.
-                  order across separate operations. It is represented in RFC3339 form
+                  It is not guaranteed to be set in happens-before order across separate operations.
-                  and is in UTC.
+                  It is represented in RFC3339 form and is in UTC.
                 format: date-time
                 type: string
               succeeded:
@@ -242,9 +251,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_imagelistpulljobs.yaml

@@ -0,0 +1,316 @@
+---
+apiVersion: apiextensions.k8s.io/v1
+kind: CustomResourceDefinition
+metadata:
+  annotations:
+    controller-gen.kubebuilder.io/version: v0.16.5
+  name: imagelistpulljobs.apps.kruise.io
+spec:
+  group: apps.kruise.io
+  names:
+    kind: ImageListPullJob
+    listKind: ImageListPullJobList
+    plural: imagelistpulljobs
+    singular: imagelistpulljob
+  scope: Namespaced
+  versions:
+  - additionalPrinterColumns:
+    - description: Number of image pull job
+      jsonPath: .status.desired
+      name: TOTAL
+      type: integer
+    - description: Number of image pull job succeeded
+      jsonPath: .status.succeeded
+      name: SUCCEEDED
+      type: integer
+    - description: Number of ImagePullJobs which are finished
+      jsonPath: .status.completed
+      name: COMPLETED
+      type: integer
+    - description: CreationTimestamp is a timestamp representing the server time when
+        this object was created. It is not guaranteed to be set in happens-before
+        order across separate operations. Clients may not set this value. It is represented
+        in RFC3339 form and is in UTC.
+      jsonPath: .metadata.creationTimestamp
+      name: AGE
+      type: date
+    name: v1alpha1
+    schema:
+      openAPIV3Schema:
+        description: ImageListPullJob is the Schema for the imagelistpulljobs API
+        properties:
+          apiVersion:
+            description: |-
+              APIVersion defines the versioned schema of this representation of an object.
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
+            type: string
+          kind:
+            description: |-
+              Kind is a string value representing the REST resource this object represents.
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
+            type: string
+          metadata:
+            type: object
+          spec:
+            description: ImageListPullJobSpec defines the desired state of ImageListPullJob
+            properties:
+              completionPolicy:
+                description: |-
+                  CompletionPolicy indicates the completion policy of the job.
+                  Default is Always CompletionPolicyType.
+                properties:
+                  activeDeadlineSeconds:
+                    description: |-
+                      ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the job may be active
+                      before the system tries to terminate it; value must be positive integer.
+                      Only works for Always type.
+                    format: int64
+                    type: integer
+                  ttlSecondsAfterFinished:
+                    description: |-
+                      ttlSecondsAfterFinished limits the lifetime of a Job that has finished
+                      execution (either Complete or Failed). If this field is set,
+                      ttlSecondsAfterFinished after the Job finishes, it is eligible to be
+                      automatically deleted. When the Job is being deleted, its lifecycle
+                      guarantees (e.g. finalizers) will be honored. If this field is unset,
+                      the Job won't be automatically deleted. If this field is set to zero,
+                      the Job becomes eligible to be deleted immediately after it finishes.
+                      This field is alpha-level and is only honored by servers that enable the
+                      TTLAfterFinished feature.
+                      Only works for Always type
+                    format: int32
+                    type: integer
+                  type:
+                    description: |-
+                      Type indicates the type of the CompletionPolicy.
+                      Default is Always.
+                    type: string
+                type: object
+              imagePullPolicy:
+                description: |-
+                  Image pull policy.
+                  One of Always, IfNotPresent. Defaults to IfNotPresent.
+                type: string
+              images:
+                description: Images is the image list to be pulled by the job
+                items:
+                  type: string
+                type: array
+              parallelism:
+                anyOf:
+                - type: integer
+                - type: string
+                description: |-
+                  Parallelism is the requested parallelism, it can be set to any non-negative value. If it is unspecified,
+                  it defaults to 1. If it is specified as 0, then the Job is effectively paused until it is increased.
+                x-kubernetes-int-or-string: true
+              podSelector:
+                description: |-
+                  PodSelector is a query over pods that should pull image on nodes of these pods.
+                  Mutually exclusive with Selector.
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                type: object
+                x-kubernetes-map-type: atomic
+              pullPolicy:
+                description: |-
+                  PullPolicy is an optional field to set parameters of the pulling task. If not specified,
+                  the system will use the default values.
+                properties:
+                  backoffLimit:
+                    description: |-
+                      Specifies the number of retries before marking the pulling task failed.
+                      Defaults to 3
+                    format: int32
+                    type: integer
+                  timeoutSeconds:
+                    description: |-
+                      Specifies the timeout of the pulling task.
+                      Defaults to 600
+                    format: int32
+                    type: integer
+                type: object
+              pullSecrets:
+                description: |-
+                  ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling the image.
+                  If specified, these secrets will be passed to individual puller implementations for them to use.  For example,
+                  in the case of docker, only DockerConfig type secrets are honored.
+                items:
+                  type: string
+                type: array
+              sandboxConfig:
+                description: SandboxConfig support attach metadata in PullImage CRI
+                  interface during ImagePulljobs
+                properties:
+                  annotations:
+                    additionalProperties:
+                      type: string
+                    type: object
+                  labels:
+                    additionalProperties:
+                      type: string
+                    type: object
+                type: object
+              selector:
+                description: |-
+                  Selector is a query over nodes that should match the job.
+                  nil to match all nodes.
+                properties:
+                  matchExpressions:
+                    description: matchExpressions is a list of label selector requirements.
+                      The requirements are ANDed.
+                    items:
+                      description: |-
+                        A label selector requirement is a selector that contains values, a key, and an operator that
+                        relates the key and values.
+                      properties:
+                        key:
+                          description: key is the label key that the selector applies
+                            to.
+                          type: string
+                        operator:
+                          description: |-
+                            operator represents a key's relationship to a set of values.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
+                          type: string
+                        values:
+                          description: |-
+                            values is an array of string values. If the operator is In or NotIn,
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                            the values array must be empty. This array is replaced during a strategic
+                            merge patch.
+                          items:
+                            type: string
+                          type: array
+                          x-kubernetes-list-type: atomic
+                      required:
+                      - key
+                      - operator
+                      type: object
+                    type: array
+                    x-kubernetes-list-type: atomic
+                  matchLabels:
+                    additionalProperties:
+                      type: string
+                    description: |-
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
+                    type: object
+                  names:
+                    description: Names specify a set of nodes to execute the job.
+                    items:
+                      type: string
+                    type: array
+                type: object
+                x-kubernetes-map-type: atomic
+            required:
+            - completionPolicy
+            - images
+            type: object
+          status:
+            description: ImageListPullJobStatus defines the observed state of ImageListPullJob
+            properties:
+              active:
+                description: The number of running ImagePullJobs which are acknowledged
+                  by the imagepulljob controller.
+                format: int32
+                type: integer
+              completed:
+                description: The number of ImagePullJobs which are finished
+                format: int32
+                type: integer
+              completionTime:
+                description: |-
+                  Represents time when the all the image pull job was completed. It is not guaranteed to
+                  be set in happens-before order across separate operations.
+                  It is represented in RFC3339 form and is in UTC.
+                format: date-time
+                type: string
+              desired:
+                description: The desired number of ImagePullJobs, this is typically
+                  equal to the number of len(spec.Images).
+                format: int32
+                type: integer
+              failedImageStatuses:
+                description: The status of ImagePullJob which has the failed nodes(status.Failed>0)
+                  .
+                items:
+                  description: FailedImageStatus the state of ImagePullJob which has
+                    the failed nodes(status.Failed>0)
+                  properties:
+                    imagePullJob:
+                      description: The name of ImagePullJob which has the failed nodes(status.Failed>0)
+                      type: string
+                    message:
+                      description: The text prompt for job running status.
+                      type: string
+                    name:
+                      description: Name of the image
+                      type: string
+                  type: object
+                type: array
+              startTime:
+                description: |-
+                  Represents time when the job was acknowledged by the job controller.
+                  It is not guaranteed to be set in happens-before order across separate operations.
+                  It is represented in RFC3339 form and is in UTC.
+                format: date-time
+                type: string
+              succeeded:
+                description: The number of image pull job which are finished and status.Succeeded==status.Desired.
+                format: int32
+                type: integer
+            required:
+            - desired
+            type: object
+        type: object
+    served: true
+    storage: true
+    subresources:
+      status: {}

config/crd/bases/apps.kruise.io_imagepulljobs.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: imagepulljobs.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -50,14 +48,19 @@ spec:
         description: ImagePullJob is the Schema for the imagepulljobs API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -65,113 +68,124 @@ spec:
             description: ImagePullJobSpec defines the desired state of ImagePullJob
             properties:
               completionPolicy:
-                description: CompletionPolicy indicates the completion policy of the
+                description: |-
-                  job. Default is Always CompletionPolicyType.
+                  CompletionPolicy indicates the completion policy of the job.
+                  Default is Always CompletionPolicyType.
                 properties:
                   activeDeadlineSeconds:
-                    description: ActiveDeadlineSeconds specifies the duration in seconds
+                    description: |-
-                      relative to the startTime that the job may be active before
+                      ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the job may be active
-                      the system tries to terminate it; value must be positive integer.
+                      before the system tries to terminate it; value must be positive integer.
                       Only works for Always type.
                     format: int64
                     type: integer
                   ttlSecondsAfterFinished:
-                    description: ttlSecondsAfterFinished limits the lifetime of a
+                    description: |-
-                      Job that has finished execution (either Complete or Failed).
+                      ttlSecondsAfterFinished limits the lifetime of a Job that has finished
-                      If this field is set, ttlSecondsAfterFinished after the Job
+                      execution (either Complete or Failed). If this field is set,
-                      finishes, it is eligible to be automatically deleted. When the
+                      ttlSecondsAfterFinished after the Job finishes, it is eligible to be
-                      Job is being deleted, its lifecycle guarantees (e.g. finalizers)
+                      automatically deleted. When the Job is being deleted, its lifecycle
-                      will be honored. If this field is unset, the Job won't be automatically
+                      guarantees (e.g. finalizers) will be honored. If this field is unset,
-                      deleted. If this field is set to zero, the Job becomes eligible
+                      the Job won't be automatically deleted. If this field is set to zero,
-                      to be deleted immediately after it finishes. This field is alpha-level
+                      the Job becomes eligible to be deleted immediately after it finishes.
-                      and is only honored by servers that enable the TTLAfterFinished
+                      This field is alpha-level and is only honored by servers that enable the
-                      feature. Only works for Always type
+                      TTLAfterFinished feature.
+                      Only works for Always type
                     format: int32
                     type: integer
                   type:
-                    description: Type indicates the type of the CompletionPolicy.
+                    description: |-
+                      Type indicates the type of the CompletionPolicy.
                       Default is Always.
                     type: string
                 type: object
               image:
                 description: Image is the image to be pulled by the job
                 type: string
+              imagePullPolicy:
+                description: |-
+                  Image pull policy.
+                  One of Always, IfNotPresent. Defaults to IfNotPresent.
+                type: string
               parallelism:
                 anyOf:
                 - type: integer
                 - type: string
-                description: Parallelism is the requested parallelism, it can be set
+                description: |-
-                  to any non-negative value. If it is unspecified, it defaults to
+                  Parallelism is the requested parallelism, it can be set to any non-negative value. If it is unspecified,
-                  1. If it is specified as 0, then the Job is effectively paused until
+                  it defaults to 1. If it is specified as 0, then the Job is effectively paused until it is increased.
-                  it is increased.
                 x-kubernetes-int-or-string: true
               podSelector:
-                description: PodSelector is a query over pods that should pull image
+                description: |-
-                  on nodes of these pods. Mutually exclusive with Selector.
+                  PodSelector is a query over pods that should pull image on nodes of these pods.
+                  Mutually exclusive with Selector.
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
                       The requirements are ANDed.
                     items:
-                      description: A label selector requirement is a selector that
+                      description: |-
-                        contains values, a key, and an operator that relates the key
+                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        and values.
+                        relates the key and values.
                       properties:
                         key:
                           description: key is the label key that the selector applies
                             to.
                           type: string
                         operator:
-                          description: operator represents a key's relationship to
+                          description: |-
-                            a set of values. Valid operators are In, NotIn, Exists
+                            operator represents a key's relationship to a set of values.
-                            and DoesNotExist.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
                           type: string
                         values:
-                          description: values is an array of string values. If the
+                          description: |-
-                            operator is In or NotIn, the values array must be non-empty.
+                            values is an array of string values. If the operator is In or NotIn,
-                            If the operator is Exists or DoesNotExist, the values
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            array must be empty. This array is replaced during a strategic
+                            the values array must be empty. This array is replaced during a strategic
                             merge patch.
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
+                    description: |-
-                      {key,value} in the matchLabels map is equivalent to an element
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      of matchExpressions, whose key field is "key", the operator
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      is "In", and the values array contains only "value". The requirements
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                      are ANDed.
                     type: object
                 type: object
+                x-kubernetes-map-type: atomic
               pullPolicy:
-                description: PullPolicy is an optional field to set parameters of
+                description: |-
-                  the pulling task. If not specified, the system will use the default
+                  PullPolicy is an optional field to set parameters of the pulling task. If not specified,
-                  values.
+                  the system will use the default values.
                 properties:
                   backoffLimit:
-                    description: Specifies the number of retries before marking the
+                    description: |-
-                      pulling task failed. Defaults to 3
+                      Specifies the number of retries before marking the pulling task failed.
+                      Defaults to 3
                     format: int32
                     type: integer
                   timeoutSeconds:
-                    description: Specifies the timeout of the pulling task. Defaults
+                    description: |-
-                      to 600
+                      Specifies the timeout of the pulling task.
+                      Defaults to 600
                     format: int32
                     type: integer
                 type: object
               pullSecrets:
-                description: ImagePullSecrets is an optional list of references to
+                description: |-
-                  secrets in the same namespace to use for pulling the image. If specified,
+                  ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling the image.
-                  these secrets will be passed to individual puller implementations
+                  If specified, these secrets will be passed to individual puller implementations for them to use.  For example,
-                  for them to use.  For example, in the case of docker, only DockerConfig
+                  in the case of docker, only DockerConfig type secrets are honored.
-                  type secrets are honored.
                 items:
                   type: string
                 type: array
@@ -189,48 +203,50 @@ spec:
                     type: object
                 type: object
               selector:
-                description: Selector is a query over nodes that should match the
+                description: |-
-                  job. nil to match all nodes.
+                  Selector is a query over nodes that should match the job.
+                  nil to match all nodes.
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
                       The requirements are ANDed.
                     items:
-                      description: A label selector requirement is a selector that
+                      description: |-
-                        contains values, a key, and an operator that relates the key
+                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        and values.
+                        relates the key and values.
                       properties:
                         key:
                           description: key is the label key that the selector applies
                             to.
                           type: string
                         operator:
-                          description: operator represents a key's relationship to
+                          description: |-
-                            a set of values. Valid operators are In, NotIn, Exists
+                            operator represents a key's relationship to a set of values.
-                            and DoesNotExist.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
                           type: string
                         values:
-                          description: values is an array of string values. If the
+                          description: |-
-                            operator is In or NotIn, the values array must be non-empty.
+                            values is an array of string values. If the operator is In or NotIn,
-                            If the operator is Exists or DoesNotExist, the values
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            array must be empty. This array is replaced during a strategic
+                            the values array must be empty. This array is replaced during a strategic
                             merge patch.
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
+                    description: |-
-                      {key,value} in the matchLabels map is equivalent to an element
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      of matchExpressions, whose key field is "key", the operator
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      is "In", and the values array contains only "value". The requirements
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                      are ANDed.
                     type: object
                   names:
                     description: Names specify a set of nodes to execute the job.
@@ -238,6 +254,7 @@ spec:
                       type: string
                     type: array
                 type: object
+                x-kubernetes-map-type: atomic
             required:
             - completionPolicy
             - image
@@ -250,8 +267,9 @@ spec:
                 format: int32
                 type: integer
               completionTime:
-                description: Represents time when the job was completed. It is not
+                description: |-
-                  guaranteed to be set in happens-before order across separate operations.
+                  Represents time when the job was completed. It is not guaranteed to
+                  be set in happens-before order across separate operations.
                   It is represented in RFC3339 form and is in UTC.
                 format: date-time
                 type: string
@@ -273,10 +291,10 @@ spec:
                 description: The text prompt for job running status.
                 type: string
               startTime:
-                description: Represents time when the job was acknowledged by the
+                description: |-
-                  job controller. It is not guaranteed to be set in happens-before
+                  Represents time when the job was acknowledged by the job controller.
-                  order across separate operations. It is represented in RFC3339 form
+                  It is not guaranteed to be set in happens-before order across separate operations.
-                  and is in UTC.
+                  It is represented in RFC3339 form and is in UTC.
                 format: date-time
                 type: string
               succeeded:
@@ -291,9 +309,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_nodeimages.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: nodeimages.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -46,14 +44,19 @@ spec:
         description: NodeImage is the Schema for the nodeimages API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -65,14 +68,14 @@ spec:
                   description: ImageSpec defines the pulling spec of an image
                   properties:
                     pullSecrets:
-                      description: PullSecrets is an optional list of references to
+                      description: |-
-                        secrets in the same namespace to use for pulling the image.
+                        PullSecrets is an optional list of references to secrets in the same namespace to use for pulling the image.
-                        If specified, these secrets will be passed to individual puller
+                        If specified, these secrets will be passed to individual puller implementations for them to use.  For example,
-                        implementations for them to use.  For example, in the case
+                        in the case of docker, only DockerConfig type secrets are honored.
-                        of docker, only DockerConfig type secrets are honored.
                       items:
-                        description: ReferenceObject comprises a resource name, with
+                        description: |-
-                          a mandatory namespace, rendered as "<namespace>/<name>".
+                          ReferenceObject comprises a resource name, with a mandatory namespace,
+                          rendered as "<namespace>/<name>".
                         properties:
                           name:
                             type: string
@@ -103,112 +106,90 @@ spec:
                             description: Specifies the create time of this tag
                             format: date-time
                             type: string
+                          imagePullPolicy:
+                            description: |-
+                              Image pull policy.
+                              One of Always, IfNotPresent. Defaults to IfNotPresent.
+                            type: string
                           ownerReferences:
-                            description: List of objects depended by this object.
+                            description: |-
-                              If this image is managed by a controller, then an entry
+                              List of objects depended by this object. If this image is managed by a controller,
-                              in this list will point to this controller.
+                              then an entry in this list will point to this controller.
                             items:
-                              description: 'ObjectReference contains enough information
+                              description: ObjectReference contains enough information
                                 to let you inspect or modify the referred object.
-                                --- New uses of this type are discouraged because
-                                of difficulty describing its usage when embedded in
-                                APIs.  1. Ignored fields.  It includes many fields
-                                which are not generally honored.  For instance, ResourceVersion
-                                and FieldPath are both very rarely valid in actual
-                                usage.  2. Invalid usage help.  It is impossible to
-                                add specific help for individual usage.  In most embedded
-                                usages, there are particular     restrictions like,
-                                "must refer only to types A and B" or "UID not honored"
-                                or "name must be restricted".     Those cannot be
-                                well described when embedded.  3. Inconsistent validation.  Because
-                                the usages are different, the validation rules are
-                                different by usage, which makes it hard for users
-                                to predict what will happen.  4. The fields are both
-                                imprecise and overly precise.  Kind is not a precise
-                                mapping to a URL. This can produce ambiguity     during
-                                interpretation and require a REST mapping.  In most
-                                cases, the dependency is on the group,resource tuple     and
-                                the version of the actual struct is irrelevant.  5.
-                                We cannot easily change it.  Because this type is
-                                embedded in many locations, updates to this type     will
-                                affect numerous schemas.  Don''t make new APIs embed
-                                an underspecified API type they do not control. Instead
-                                of using this type, create a locally provided and
-                                used type that is well-focused on your reference.
-                                For example, ServiceReferences for admission registration:
-                                https://github.com/kubernetes/api/blob/release-1.17/admissionregistration/v1/types.go#L533
-                                .'
                               properties:
                                 apiVersion:
                                   description: API version of the referent.
                                   type: string
                                 fieldPath:
-                                  description: 'If referring to a piece of an object
+                                  description: |-
-                                    instead of an entire object, this string should
+                                    If referring to a piece of an object instead of an entire object, this string
-                                    contain a valid JSON/Go field access statement,
+                                    should contain a valid JSON/Go field access statement, such as desiredState.manifest.containers[2].
-                                    such as desiredState.manifest.containers[2]. For
+                                    For example, if the object reference is to a container within a pod, this would take on a value like:
-                                    example, if the object reference is to a container
+                                    "spec.containers{name}" (where "name" refers to the name of the container that triggered
-                                    within a pod, this would take on a value like:
+                                    the event) or if no container name is specified "spec.containers[2]" (container with
-                                    "spec.containers{name}" (where "name" refers to
+                                    index 2 in this pod). This syntax is chosen only to have some well-defined way of
-                                    the name of the container that triggered the event)
+                                    referencing a part of an object.
-                                    or if no container name is specified "spec.containers[2]"
-                                    (container with index 2 in this pod). This syntax
-                                    is chosen only to have some well-defined way of
-                                    referencing a part of an object. TODO: this design
-                                    is not final and this field is subject to change
-                                    in the future.'
                                   type: string
                                 kind:
-                                  description: 'Kind of the referent. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+                                  description: |-
+                                    Kind of the referent.
+                                    More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
                                   type: string
                                 name:
-                                  description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names'
+                                  description: |-
+                                    Name of the referent.
+                                    More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
                                   type: string
                                 namespace:
-                                  description: 'Namespace of the referent. More info:
+                                  description: |-
-                                    https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/'
+                                    Namespace of the referent.
+                                    More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/namespaces/
                                   type: string
                                 resourceVersion:
-                                  description: 'Specific resourceVersion to which
+                                  description: |-
-                                    this reference is made, if any. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency'
+                                    Specific resourceVersion to which this reference is made, if any.
+                                    More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#concurrency-control-and-consistency
                                   type: string
                                 uid:
-                                  description: 'UID of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids'
+                                  description: |-
+                                    UID of the referent.
+                                    More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#uids
                                   type: string
                               type: object
+                              x-kubernetes-map-type: atomic
                             type: array
                           pullPolicy:
-                            description: PullPolicy is an optional field to set parameters
+                            description: |-
-                              of the pulling task. If not specified, the system will
+                              PullPolicy is an optional field to set parameters of the pulling task. If not specified,
-                              use the default values.
+                              the system will use the default values.
                             properties:
                               activeDeadlineSeconds:
-                                description: ActiveDeadlineSeconds specifies the duration
+                                description: |-
-                                  in seconds relative to the startTime that the task
+                                  ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the task may be active
-                                  may be active before the system tries to terminate
+                                  before the system tries to terminate it; value must be positive integer.
-                                  it; value must be positive integer. if not specified,
+                                  if not specified, the system will never terminate it.
-                                  the system will never terminate it.
                                 format: int64
                                 type: integer
                               backoffLimit:
-                                description: Specifies the number of retries before
+                                description: |-
-                                  marking the pulling task failed. Defaults to 3
+                                  Specifies the number of retries before marking the pulling task failed.
+                                  Defaults to 3
                                 format: int32
                                 type: integer
                               timeoutSeconds:
-                                description: Specifies the timeout of the pulling
+                                description: |-
-                                  task. Defaults to 600
+                                  Specifies the timeout of the pulling task.
+                                  Defaults to 600
                                 format: int32
                                 type: integer
                               ttlSecondsAfterFinished:
-                                description: TTLSecondsAfterFinished limits the lifetime
+                                description: |-
-                                  of a pulling task that has finished execution (either
+                                  TTLSecondsAfterFinished limits the lifetime of a pulling task that has finished execution (either Complete or Failed).
-                                  Complete or Failed). If this field is set, ttlSecondsAfterFinished
+                                  If this field is set, ttlSecondsAfterFinished after the task finishes, it is eligible to be automatically deleted.
-                                  after the task finishes, it is eligible to be automatically
+                                  If this field is unset, the task won't be automatically deleted.
-                                  deleted. If this field is unset, the task won't
+                                  If this field is set to zero, the task becomes eligible to be deleted immediately after it finishes.
-                                  be automatically deleted. If this field is set to
-                                  zero, the task becomes eligible to be deleted immediately
-                                  after it finishes.
                                 format: int32
                                 type: integer
                             type: object
@@ -216,14 +197,15 @@ spec:
                             description: Specifies the image tag
                             type: string
                           version:
-                            description: "An opaque value that represents the internal
+                            description: |-
-                              version of this tag that can be used by clients to determine
+                              An opaque value that represents the internal version of this tag that can
-                              when objects have changed. May be used for optimistic
+                              be used by clients to determine when objects have changed. May be used for optimistic
-                              concurrency, change detection, and the watch operation
+                              concurrency, change detection, and the watch operation on a resource or set of resources.
-                              on a resource or set of resources. Clients must treat
+                              Clients must treat these values as opaque and passed unmodified back to the server.
-                              these values as opaque and passed unmodified back to
+
-                              the server. \n Populated by the system. Read-only. Value
+                              Populated by the system.
-                              must be treated as opaque by clients and ."
+                              Read-only.
+                              Value must be treated as opaque by clients and .
                             format: int64
                             type: integer
                         required:
@@ -233,8 +215,9 @@ spec:
                   required:
                   - tags
                   type: object
-                description: Specifies images to be pulled on this node It can not
+                description: |-
-                  be more than 256 for each NodeImage
+                  Specifies images to be pulled on this node
+                  It can not be more than 256 for each NodeImage
                 type: object
             type: object
           status:
@@ -250,10 +233,9 @@ spec:
                 format: int32
                 type: integer
               firstSyncStatus:
-                description: The first of all job has finished on this node. When
+                description: |-
-                  a node is added to the cluster, we want to know the time when the
+                  The first of all job has finished on this node. When a node is added to the cluster, we want to know
-                  node's image pulling is completed, and use it to trigger the operation
+                  the time when the node's image pulling is completed, and use it to trigger the operation of the upper system.
-                  of the upper system.
                 properties:
                   message:
                     type: string
@@ -275,10 +257,10 @@ spec:
                           an image tag
                         properties:
                           completionTime:
-                            description: Represents time when the pulling task was
+                            description: |-
-                              completed. It is not guaranteed to be set in happens-before
+                              Represents time when the pulling task was completed. It is not guaranteed to
-                              order across separate operations. It is represented
+                              be set in happens-before order across separate operations.
-                              in RFC3339 form and is in UTC.
+                              It is represented in RFC3339 form and is in UTC.
                             format: date-time
                             type: string
                           imageID:
@@ -292,16 +274,15 @@ spec:
                             description: Represents the image pulling task phase.
                             type: string
                           progress:
-                            description: Represents the pulling progress of this tag,
+                            description: |-
-                              which is between 0-100. There is no guarantee of monotonic
+                              Represents the pulling progress of this tag, which is between 0-100. There is no guarantee
-                              consistency, and it may be a rollback due to retry during
+                              of monotonic consistency, and it may be a rollback due to retry during pulling.
-                              pulling.
                             format: int32
                             type: integer
                           startTime:
-                            description: Represents time when the pulling task was
+                            description: |-
-                              acknowledged by the image puller. It is not guaranteed
+                              Represents time when the pulling task was acknowledged by the image puller.
-                              to be set in happens-before order across separate operations.
+                              It is not guaranteed to be set in happens-before order across separate operations.
                               It is represented in RFC3339 form and is in UTC.
                             format: date-time
                             type: string
@@ -331,6 +312,10 @@ spec:
                 description: The number of pulling tasks which reached phase Succeeded.
                 format: int32
                 type: integer
+              waiting:
+                description: The number of pulling tasks which are waiting.
+                format: int32
+                type: integer
             required:
             - desired
             type: object
@@ -339,9 +324,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_nodepodprobes.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: nodepodprobes.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -22,14 +20,19 @@ spec:
         description: NodePodProbe is the Schema for the NodePodProbe API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -39,6 +42,9 @@ spec:
               podProbes:
                 items:
                   properties:
+                    IP:
+                      description: pod ip
+                      type: string
                     name:
                       description: pod name
                       type: string
@@ -60,37 +66,54 @@ spec:
                             description: container probe spec
                             properties:
                               exec:
-                                description: One and only one of the following should
+                                description: Exec specifies a command to execute in
-                                  be specified. Exec specifies the action to take.
+                                  the container.
                                 properties:
                                   command:
-                                    description: Command is the command line to execute
+                                    description: |-
-                                      inside the container, the working directory
+                                      Command is the command line to execute inside the container, the working directory for the
-                                      for the command  is root ('/') in the container's
+                                      command  is root ('/') in the container's filesystem. The command is simply exec'd, it is
-                                      filesystem. The command is simply exec'd, it
+                                      not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use
-                                      is not run inside a shell, so traditional shell
+                                      a shell, you need to explicitly call out to that shell.
-                                      instructions ('|', etc) won't work. To use a
+                                      Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
-                                      shell, you need to explicitly call out to that
-                                      shell. Exit status of 0 is treated as live/healthy
-                                      and non-zero is unhealthy.
                                     items:
                                       type: string
                                     type: array
+                                    x-kubernetes-list-type: atomic
                                 type: object
                               failureThreshold:
-                                description: Minimum consecutive failures for the
+                                description: |-
-                                  probe to be considered failed after having succeeded.
+                                  Minimum consecutive failures for the probe to be considered failed after having succeeded.
                                   Defaults to 3. Minimum value is 1.
                                 format: int32
                                 type: integer
+                              grpc:
+                                description: GRPC specifies a GRPC HealthCheckRequest.
+                                properties:
+                                  port:
+                                    description: Port number of the gRPC service.
+                                      Number must be in the range 1 to 65535.
+                                    format: int32
+                                    type: integer
+                                  service:
+                                    default: ""
+                                    description: |-
+                                      Service is the name of the service to place in the gRPC HealthCheckRequest
+                                      (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).
+
+                                      If this is not specified, the default behavior is defined by gRPC.
+                                    type: string
+                                required:
+                                - port
+                                type: object
                               httpGet:
-                                description: HTTPGet specifies the http request to
+                                description: HTTPGet specifies an HTTP GET request
-                                  perform.
+                                  to perform.
                                 properties:
                                   host:
-                                    description: Host name to connect to, defaults
+                                    description: |-
-                                      to the pod IP. You probably want to set "Host"
+                                      Host name to connect to, defaults to the pod IP. You probably want to set
-                                      in httpHeaders instead.
+                                      "Host" in httpHeaders instead.
                                     type: string
                                   httpHeaders:
                                     description: Custom headers to set in the request.
@@ -100,7 +123,9 @@ spec:
                                         to be used in HTTP probes
                                       properties:
                                         name:
-                                          description: The header field name
+                                          description: |-
+                                            The header field name.
+                                            This will be canonicalized upon output, so case-variant names will be understood as the same header.
                                           type: string
                                         value:
                                           description: The header field value
@@ -110,6 +135,7 @@ spec:
                                       - value
                                       type: object
                                     type: array
+                                    x-kubernetes-list-type: atomic
                                   path:
                                     description: Path to access on the HTTP server.
                                     type: string
@@ -117,39 +143,40 @@ spec:
                                     anyOf:
                                     - type: integer
                                     - type: string
-                                    description: Name or number of the port to access
+                                    description: |-
-                                      on the container. Number must be in the range
+                                      Name or number of the port to access on the container.
-                                      1 to 65535. Name must be an IANA_SVC_NAME.
+                                      Number must be in the range 1 to 65535.
+                                      Name must be an IANA_SVC_NAME.
                                     x-kubernetes-int-or-string: true
                                   scheme:
-                                    description: Scheme to use for connecting to the
+                                    description: |-
-                                      host. Defaults to HTTP.
+                                      Scheme to use for connecting to the host.
+                                      Defaults to HTTP.
                                     type: string
                                 required:
                                 - port
                                 type: object
                               initialDelaySeconds:
-                                description: 'Number of seconds after the container
+                                description: |-
-                                  has started before liveness probes are initiated.
+                                  Number of seconds after the container has started before liveness probes are initiated.
-                                  More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
+                                  More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
                                 format: int32
                                 type: integer
                               periodSeconds:
-                                description: How often (in seconds) to perform the
+                                description: |-
-                                  probe. Default to 10 seconds. Minimum value is 1.
+                                  How often (in seconds) to perform the probe.
+                                  Default to 10 seconds. Minimum value is 1.
                                 format: int32
                                 type: integer
                               successThreshold:
-                                description: Minimum consecutive successes for the
+                                description: |-
-                                  probe to be considered successful after having failed.
+                                  Minimum consecutive successes for the probe to be considered successful after having failed.
-                                  Defaults to 1. Must be 1 for liveness and startup.
+                                  Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
-                                  Minimum value is 1.
                                 format: int32
                                 type: integer
                               tcpSocket:
-                                description: 'TCPSocket specifies an action involving
+                                description: TCPSocket specifies a connection to a
-                                  a TCP port. TCP hooks not yet supported TODO: implement
+                                  TCP port.
-                                  a realistic TCP lifecycle hook'
                                 properties:
                                   host:
                                     description: 'Optional: Host name to connect to,
@@ -159,35 +186,33 @@ spec:
                                     anyOf:
                                     - type: integer
                                     - type: string
-                                    description: Number or name of the port to access
+                                    description: |-
-                                      on the container. Number must be in the range
+                                      Number or name of the port to access on the container.
-                                      1 to 65535. Name must be an IANA_SVC_NAME.
+                                      Number must be in the range 1 to 65535.
+                                      Name must be an IANA_SVC_NAME.
                                     x-kubernetes-int-or-string: true
                                 required:
                                 - port
                                 type: object
                               terminationGracePeriodSeconds:
-                                description: Optional duration in seconds the pod
+                                description: |-
-                                  needs to terminate gracefully upon probe failure.
+                                  Optional duration in seconds the pod needs to terminate gracefully upon probe failure.
-                                  The grace period is the duration in seconds after
+                                  The grace period is the duration in seconds after the processes running in the pod are sent
-                                  the processes running in the pod are sent a termination
+                                  a termination signal and the time when the processes are forcibly halted with a kill signal.
-                                  signal and the time when the processes are forcibly
+                                  Set this value longer than the expected cleanup time for your process.
-                                  halted with a kill signal. Set this value longer
+                                  If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this
-                                  than the expected cleanup time for your process.
+                                  value overrides the value provided by the pod spec.
-                                  If this value is nil, the pod's terminationGracePeriodSeconds
+                                  Value must be non-negative integer. The value zero indicates stop immediately via
-                                  will be used. Otherwise, this value overrides the
+                                  the kill signal (no opportunity to shut down).
-                                  value provided by the pod spec. Value must be non-negative
+                                  This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.
-                                  integer. The value zero indicates stop immediately
+                                  Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.
-                                  via the kill signal (no opportunity to shut down).
-                                  This is a beta field and requires enabling ProbeTerminationGracePeriod
-                                  feature gate. Minimum value is 1. spec.terminationGracePeriodSeconds
-                                  is used if unset.
                                 format: int64
                                 type: integer
                               timeoutSeconds:
-                                description: 'Number of seconds after which the probe
+                                description: |-
-                                  times out. Defaults to 1 second. Minimum value is
+                                  Number of seconds after which the probe times out.
-                                  1. More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
+                                  Defaults to 1 second. Minimum value is 1.
+                                  More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
                                 format: int32
                                 type: integer
                             type: object
@@ -201,6 +226,7 @@ spec:
                       description: pod uid
                       type: string
                   required:
+                  - IP
                   - name
                   - namespace
                   - uid
@@ -233,9 +259,9 @@ spec:
                             format: date-time
                             type: string
                           message:
-                            description: If Status=True, Message records the return
+                            description: |-
-                              result of Probe. If Status=False, Message records Probe's
+                              If Status=True, Message records the return result of Probe.
-                              error message
+                              If Status=False, Message records Probe's error message
                             type: string
                           name:
                             description: Name is podProbeMarker.Name#probe.Name
@@ -263,9 +289,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_persistentpodstates.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: persistentpodstates.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -22,14 +20,19 @@ spec:
         description: PersistentPodState is the Schema for the PersistentPodState API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -48,13 +51,14 @@ spec:
                   type: object
                 type: array
               persistentPodStateRetentionPolicy:
-                description: PersistentPodStateRetentionPolicy describes the policy
+                description: |-
-                  used for PodState. The default policy of 'WhenScaled' causes when
+                  PersistentPodStateRetentionPolicy describes the policy used for PodState.
-                  scale down statefulSet, deleting it.
+                  The default policy of 'WhenScaled' causes when scale down statefulSet, deleting it.
                 type: string
               preferredPersistentTopology:
-                description: Pod rebuilt topology preferred for node labels, with
+                description: |-
-                  xx weight for example  kubernetes.io/hostname, failure-domain.beta.kubernetes.io/zone
+                  Pod rebuilt topology preferred for node labels, with xx weight
+                  for example  kubernetes.io/hostname, failure-domain.beta.kubernetes.io/zone
                 items:
                   properties:
                     preference:
@@ -77,8 +81,9 @@ spec:
                   type: object
                 type: array
               requiredPersistentTopology:
-                description: Pod rebuilt topology required for node labels for example
+                description: |-
-                  kubernetes.io/hostname, failure-domain.beta.kubernetes.io/zone
+                  Pod rebuilt topology required for node labels
+                  for example kubernetes.io/hostname, failure-domain.beta.kubernetes.io/zone
                 properties:
                   nodeTopologyKeys:
                     description: A list of node selector requirements by node's labels.
@@ -89,9 +94,9 @@ spec:
                 - nodeTopologyKeys
                 type: object
               targetRef:
-                description: TargetReference contains enough information to let you
+                description: |-
-                  identify an workload for PersistentPodState Selector and TargetReference
+                  TargetReference contains enough information to let you identify an workload for PersistentPodState
-                  are mutually exclusive, TargetReference is priority to take effect
+                  Selector and TargetReference are mutually exclusive, TargetReference is priority to take effect
                   current only support StatefulSet
                 properties:
                   apiVersion:
@@ -114,9 +119,9 @@ spec:
           status:
             properties:
               observedGeneration:
-                description: observedGeneration is the most recent generation observed
+                description: |-
-                  for this PersistentPodState. It corresponds to the PersistentPodState's
+                  observedGeneration is the most recent generation observed for this PersistentPodState. It corresponds to the
-                  generation, which is updated on mutation by the API Server.
+                  PersistentPodState's generation, which is updated on mutation by the API Server.
                 format: int64
                 type: integer
               podStates:
@@ -133,12 +138,14 @@ spec:
                     nodeTopologyLabels:
                       additionalProperties:
                         type: string
-                      description: node topology labels key=value for example kubernetes.io/hostname=node-1
+                      description: |-
+                        node topology labels key=value
+                        for example kubernetes.io/hostname=node-1
                       type: object
                   type: object
-                description: 'When the pod is ready, record some status information
+                description: |-
-                  of the pod, such as: labels, annotations, topologies, etc. map[string]PodState
+                  When the pod is ready, record some status information of the pod, such as: labels, annotations, topologies, etc.
-                  -> map[Pod.Name]PodState'
+                  map[string]PodState -> map[Pod.Name]PodState
                 type: object
             required:
             - observedGeneration
@@ -148,9 +155,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_podprobemarkers.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: podprobemarkers.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -22,14 +20,19 @@ spec:
         description: PodProbeMarker is the Schema for the PodProbeMarker API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -37,20 +40,21 @@ spec:
             description: PodProbeMarkerSpec defines the desired state of PodProbeMarker
             properties:
               probes:
-                description: Custom container probe, current only support Exec().
+                description: |-
+                  Custom container probe, current only support Exec().
                   Probe Result will record in Pod.Status.Conditions, and condition.type=probe.name.
-                  condition.status=True indicates probe success condition.status=False
+                  condition.status=True indicates probe success
-                  indicates probe fails
+                  condition.status=False indicates probe fails
                 items:
                   properties:
                     containerName:
                       description: container name
                       type: string
                     markerPolicy:
-                      description: 'According to the execution result of ContainerProbe,
+                      description: |-
-                        perform specific actions, such as: patch Pod labels, annotations,
+                        According to the execution result of ContainerProbe, perform specific actions,
-                        ReadinessGate Condition It cannot be null at the same time
+                        such as: patch Pod labels, annotations, ReadinessGate Condition
-                        as PodConditionType.'
+                        It cannot be null at the same time as PodConditionType.
                       items:
                         properties:
                           annotations:
@@ -64,12 +68,11 @@ spec:
                             description: Patch Labels pod.labels
                             type: object
                           state:
-                            description: 'probe status, True or False For example:
+                            description: |-
-                              State=Succeeded, annotations[controller.kubernetes.io/pod-deletion-cost]
+                              probe status, True or False
-                              = ''10''. State=Failed, annotations[controller.kubernetes.io/pod-deletion-cost]
+                              For example: State=Succeeded, annotations[controller.kubernetes.io/pod-deletion-cost] = '10'.
-                              = ''-10''. In addition, if State=Failed is not defined,
+                              State=Failed, annotations[controller.kubernetes.io/pod-deletion-cost] = '-10'.
-                              Exec execution fails, and the annotations[controller.kubernetes.io/pod-deletion-cost]
+                              In addition, if State=Failed is not defined, Exec execution fails, and the annotations[controller.kubernetes.io/pod-deletion-cost] will be Deleted
-                              will be Deleted'
                             type: string
                         required:
                         - state
@@ -80,47 +83,63 @@ spec:
                         different containers, they cannot be the same)
                       type: string
                     podConditionType:
-                      description: If it is not empty, the Probe execution result
+                      description: |-
-                        will be recorded on the Pod condition. It cannot be null at
+                        If it is not empty, the Probe execution result will be recorded on the Pod condition.
-                        the same time as MarkerPolicy. For example PodConditionType=game.kruise.io/healthy,
+                        It cannot be null at the same time as MarkerPolicy.
-                        pod.status.condition.type = game.kruise.io/healthy. When probe
+                        For example PodConditionType=game.kruise.io/healthy, pod.status.condition.type = game.kruise.io/healthy.
-                        is Succeeded, pod.status.condition.status = True. Otherwise,
+                        When probe is Succeeded, pod.status.condition.status = True. Otherwise, when the probe fails to execute, pod.status.condition.status = False.
-                        when the probe fails to execute, pod.status.condition.status
-                        = False.
                       type: string
                     probe:
                       description: container probe spec
                       properties:
                         exec:
-                          description: One and only one of the following should be
+                          description: Exec specifies a command to execute in the
-                            specified. Exec specifies the action to take.
+                            container.
                           properties:
                             command:
-                              description: Command is the command line to execute
+                              description: |-
-                                inside the container, the working directory for the
+                                Command is the command line to execute inside the container, the working directory for the
-                                command  is root ('/') in the container's filesystem.
+                                command  is root ('/') in the container's filesystem. The command is simply exec'd, it is
-                                The command is simply exec'd, it is not run inside
+                                not run inside a shell, so traditional shell instructions ('|', etc) won't work. To use
-                                a shell, so traditional shell instructions ('|', etc)
+                                a shell, you need to explicitly call out to that shell.
-                                won't work. To use a shell, you need to explicitly
+                                Exit status of 0 is treated as live/healthy and non-zero is unhealthy.
-                                call out to that shell. Exit status of 0 is treated
-                                as live/healthy and non-zero is unhealthy.
                               items:
                                 type: string
                               type: array
+                              x-kubernetes-list-type: atomic
                           type: object
                         failureThreshold:
-                          description: Minimum consecutive failures for the probe
+                          description: |-
-                            to be considered failed after having succeeded. Defaults
+                            Minimum consecutive failures for the probe to be considered failed after having succeeded.
-                            to 3. Minimum value is 1.
+                            Defaults to 3. Minimum value is 1.
                           format: int32
                           type: integer
+                        grpc:
+                          description: GRPC specifies a GRPC HealthCheckRequest.
+                          properties:
+                            port:
+                              description: Port number of the gRPC service. Number
+                                must be in the range 1 to 65535.
+                              format: int32
+                              type: integer
+                            service:
+                              default: ""
+                              description: |-
+                                Service is the name of the service to place in the gRPC HealthCheckRequest
+                                (see https://github.com/grpc/grpc/blob/master/doc/health-checking.md).
+
+                                If this is not specified, the default behavior is defined by gRPC.
+                              type: string
+                          required:
+                          - port
+                          type: object
                         httpGet:
-                          description: HTTPGet specifies the http request to perform.
+                          description: HTTPGet specifies an HTTP GET request to perform.
                           properties:
                             host:
-                              description: Host name to connect to, defaults to the
+                              description: |-
-                                pod IP. You probably want to set "Host" in httpHeaders
+                                Host name to connect to, defaults to the pod IP. You probably want to set
-                                instead.
+                                "Host" in httpHeaders instead.
                               type: string
                             httpHeaders:
                               description: Custom headers to set in the request. HTTP
@@ -130,7 +149,9 @@ spec:
                                   to be used in HTTP probes
                                 properties:
                                   name:
-                                    description: The header field name
+                                    description: |-
+                                      The header field name.
+                                      This will be canonicalized upon output, so case-variant names will be understood as the same header.
                                     type: string
                                   value:
                                     description: The header field value
@@ -140,6 +161,7 @@ spec:
                                 - value
                                 type: object
                               type: array
+                              x-kubernetes-list-type: atomic
                             path:
                               description: Path to access on the HTTP server.
                               type: string
@@ -147,39 +169,39 @@ spec:
                               anyOf:
                               - type: integer
                               - type: string
-                              description: Name or number of the port to access on
+                              description: |-
-                                the container. Number must be in the range 1 to 65535.
+                                Name or number of the port to access on the container.
+                                Number must be in the range 1 to 65535.
                                 Name must be an IANA_SVC_NAME.
                               x-kubernetes-int-or-string: true
                             scheme:
-                              description: Scheme to use for connecting to the host.
+                              description: |-
+                                Scheme to use for connecting to the host.
                                 Defaults to HTTP.
                               type: string
                           required:
                           - port
                           type: object
                         initialDelaySeconds:
-                          description: 'Number of seconds after the container has
+                          description: |-
-                            started before liveness probes are initiated. More info:
+                            Number of seconds after the container has started before liveness probes are initiated.
-                            https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
+                            More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
                           format: int32
                           type: integer
                         periodSeconds:
-                          description: How often (in seconds) to perform the probe.
+                          description: |-
+                            How often (in seconds) to perform the probe.
                             Default to 10 seconds. Minimum value is 1.
                           format: int32
                           type: integer
                         successThreshold:
-                          description: Minimum consecutive successes for the probe
+                          description: |-
-                            to be considered successful after having failed. Defaults
+                            Minimum consecutive successes for the probe to be considered successful after having failed.
-                            to 1. Must be 1 for liveness and startup. Minimum value
+                            Defaults to 1. Must be 1 for liveness and startup. Minimum value is 1.
-                            is 1.
                           format: int32
                           type: integer
                         tcpSocket:
-                          description: 'TCPSocket specifies an action involving a
+                          description: TCPSocket specifies a connection to a TCP port.
-                            TCP port. TCP hooks not yet supported TODO: implement
-                            a realistic TCP lifecycle hook'
                           properties:
                             host:
                               description: 'Optional: Host name to connect to, defaults
@@ -189,34 +211,33 @@ spec:
                               anyOf:
                               - type: integer
                               - type: string
-                              description: Number or name of the port to access on
+                              description: |-
-                                the container. Number must be in the range 1 to 65535.
+                                Number or name of the port to access on the container.
+                                Number must be in the range 1 to 65535.
                                 Name must be an IANA_SVC_NAME.
                               x-kubernetes-int-or-string: true
                           required:
                           - port
                           type: object
                         terminationGracePeriodSeconds:
-                          description: Optional duration in seconds the pod needs
+                          description: |-
-                            to terminate gracefully upon probe failure. The grace
+                            Optional duration in seconds the pod needs to terminate gracefully upon probe failure.
-                            period is the duration in seconds after the processes
+                            The grace period is the duration in seconds after the processes running in the pod are sent
-                            running in the pod are sent a termination signal and the
+                            a termination signal and the time when the processes are forcibly halted with a kill signal.
-                            time when the processes are forcibly halted with a kill
+                            Set this value longer than the expected cleanup time for your process.
-                            signal. Set this value longer than the expected cleanup
+                            If this value is nil, the pod's terminationGracePeriodSeconds will be used. Otherwise, this
-                            time for your process. If this value is nil, the pod's
+                            value overrides the value provided by the pod spec.
-                            terminationGracePeriodSeconds will be used. Otherwise,
+                            Value must be non-negative integer. The value zero indicates stop immediately via
-                            this value overrides the value provided by the pod spec.
+                            the kill signal (no opportunity to shut down).
-                            Value must be non-negative integer. The value zero indicates
+                            This is a beta field and requires enabling ProbeTerminationGracePeriod feature gate.
-                            stop immediately via the kill signal (no opportunity to
+                            Minimum value is 1. spec.terminationGracePeriodSeconds is used if unset.
-                            shut down). This is a beta field and requires enabling
-                            ProbeTerminationGracePeriod feature gate. Minimum value
-                            is 1. spec.terminationGracePeriodSeconds is used if unset.
                           format: int64
                           type: integer
                         timeoutSeconds:
-                          description: 'Number of seconds after which the probe times
+                          description: |-
-                            out. Defaults to 1 second. Minimum value is 1. More info:
+                            Number of seconds after which the probe times out.
-                            https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes'
+                            Defaults to 1 second. Minimum value is 1.
+                            More info: https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle#container-probes
                           format: int32
                           type: integer
                       type: object
@@ -227,51 +248,54 @@ spec:
                   type: object
                 type: array
               selector:
-                description: 'Selector is a label query over pods that should exec
+                description: |-
-                  custom probe It must match the pod template''s labels. More info:
+                  Selector is a label query over pods that should exec custom probe
-                  https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors'
+                  It must match the pod template's labels.
+                  More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
                       The requirements are ANDed.
                     items:
-                      description: A label selector requirement is a selector that
+                      description: |-
-                        contains values, a key, and an operator that relates the key
+                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        and values.
+                        relates the key and values.
                       properties:
                         key:
                           description: key is the label key that the selector applies
                             to.
                           type: string
                         operator:
-                          description: operator represents a key's relationship to
+                          description: |-
-                            a set of values. Valid operators are In, NotIn, Exists
+                            operator represents a key's relationship to a set of values.
-                            and DoesNotExist.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
                           type: string
                         values:
-                          description: values is an array of string values. If the
+                          description: |-
-                            operator is In or NotIn, the values array must be non-empty.
+                            values is an array of string values. If the operator is In or NotIn,
-                            If the operator is Exists or DoesNotExist, the values
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            array must be empty. This array is replaced during a strategic
+                            the values array must be empty. This array is replaced during a strategic
                             merge patch.
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
+                    description: |-
-                      {key,value} in the matchLabels map is equivalent to an element
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      of matchExpressions, whose key field is "key", the operator
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      is "In", and the values array contains only "value". The requirements
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                      are ANDed.
                     type: object
                 type: object
+                x-kubernetes-map-type: atomic
             required:
             - probes
             - selector
@@ -283,9 +307,9 @@ spec:
                 format: int64
                 type: integer
               observedGeneration:
-                description: observedGeneration is the most recent generation observed
+                description: |-
-                  for this PodProbeMarker. It corresponds to the PodProbeMarker's
+                  observedGeneration is the most recent generation observed for this PodProbeMarker. It corresponds to the
-                  generation, which is updated on mutation by the API Server.
+                  PodProbeMarker's generation, which is updated on mutation by the API Server.
                 format: int64
                 type: integer
             required:
@@ -296,9 +320,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_resourcedistributions.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: resourcedistributions.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -38,14 +36,19 @@ spec:
           API.
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -63,14 +66,14 @@ spec:
                   to.
                 properties:
                   allNamespaces:
-                    description: If AllNamespaces is true, Resource will be distributed
+                    description: |-
-                      to the all namespaces (except some forbidden namespaces, such
+                      If AllNamespaces is true, Resource will be distributed to the all namespaces
-                      as "kube-system" and "kube-public").
+                      (except some forbidden namespaces, such as "kube-system" and "kube-public").
                     type: boolean
                   excludedNamespaces:
-                    description: If ExcludedNamespaces is not empty, Resource will
+                    description: |-
-                      never be distributed to the listed namespaces. ExcludedNamespaces
+                      If ExcludedNamespaces is not empty, Resource will never be distributed to the listed namespaces.
-                      has the highest priority.
+                      ExcludedNamespaces has the highest priority.
                     properties:
                       list:
                         items:
@@ -106,52 +109,54 @@ spec:
                         description: matchExpressions is a list of label selector
                           requirements. The requirements are ANDed.
                         items:
-                          description: A label selector requirement is a selector
+                          description: |-
-                            that contains values, a key, and an operator that relates
+                            A label selector requirement is a selector that contains values, a key, and an operator that
-                            the key and values.
+                            relates the key and values.
                           properties:
                             key:
                               description: key is the label key that the selector
                                 applies to.
                               type: string
                             operator:
-                              description: operator represents a key's relationship
+                              description: |-
-                                to a set of values. Valid operators are In, NotIn,
+                                operator represents a key's relationship to a set of values.
-                                Exists and DoesNotExist.
+                                Valid operators are In, NotIn, Exists and DoesNotExist.
                               type: string
                             values:
-                              description: values is an array of string values. If
+                              description: |-
-                                the operator is In or NotIn, the values array must
+                                values is an array of string values. If the operator is In or NotIn,
-                                be non-empty. If the operator is Exists or DoesNotExist,
+                                the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                the values array must be empty. This array is replaced
+                                the values array must be empty. This array is replaced during a strategic
-                                during a strategic merge patch.
+                                merge patch.
                               items:
                                 type: string
                               type: array
+                              x-kubernetes-list-type: atomic
                           required:
                           - key
                           - operator
                           type: object
                         type: array
+                        x-kubernetes-list-type: atomic
                       matchLabels:
                         additionalProperties:
                           type: string
-                        description: matchLabels is a map of {key,value} pairs. A
+                        description: |-
-                          single {key,value} in the matchLabels map is equivalent
+                          matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                          to an element of matchExpressions, whose key field is "key",
+                          map is equivalent to an element of matchExpressions, whose key field is "key", the
-                          the operator is "In", and the values array contains only
+                          operator is "In", and the values array contains only "value". The requirements are ANDed.
-                          "value". The requirements are ANDed.
                         type: object
                     type: object
+                    x-kubernetes-map-type: atomic
                 type: object
             required:
             - resource
             - targets
             type: object
           status:
-            description: ResourceDistributionStatus defines the observed state of
+            description: |-
-              ResourceDistribution. ResourceDistributionStatus is recorded by kruise,
+              ResourceDistributionStatus defines the observed state of ResourceDistribution.
-              users' modification is invalid and meaningless.
+              ResourceDistributionStatus is recorded by kruise, users' modification is invalid and meaningless.
             properties:
               conditions:
                 description: Conditions describe the condition when Resource creating,
@@ -209,9 +214,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_sidecarsets.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: sidecarsets.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -42,14 +40,19 @@ spec:
         description: SidecarSet is the Schema for the sidecarsets API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -63,23 +66,34 @@ spec:
                   description: SidecarContainer defines the container of Sidecar
                   properties:
                     podInjectPolicy:
-                      description: The rules that injected SidecarContainer into Pod.spec.containers,
+                      description: |-
-                        not takes effect in initContainers If BeforeAppContainer,
+                        The rules that injected SidecarContainer into Pod.spec.containers,
-                        the SidecarContainer will be injected in front of the pod.spec.containers
+                        not takes effect in initContainers
-                        otherwise it will be injected into the back. default BeforeAppContainerType
+                        If BeforeAppContainer, the SidecarContainer will be injected in front of the pod.spec.containers
+                        otherwise it will be injected into the back.
+                        default BeforeAppContainerType
                       type: string
+                    shareVolumeDevicePolicy:
+                      description: |-
+                        If ShareVolumeDevicePolicy is enabled, the sidecar container will share the other container's VolumeDevices
+                        in the pod(don't contain the injected sidecar container).
+                        This is a pointer to ensure that the sidecarset-hash does not change if the user does not configure this field, mainly for compatibility with older versions.
+                      properties:
+                        type:
+                          type: string
+                      type: object
                     shareVolumePolicy:
-                      description: If ShareVolumePolicy is enabled, the sidecar container
+                      description: |-
-                        will share the other container's VolumeMounts in the pod(don't
+                        If ShareVolumePolicy is enabled, the sidecar container will share the other container's VolumeMounts
-                        contains the injected sidecar container).
+                        in the pod(not including the injected sidecar container).
                       properties:
                         type:
                           type: string
                       type: object
                     transferEnv:
-                      description: TransferEnv will transfer env info from other container
+                      description: |-
-                        SourceContainerName is pod.spec.container[x].name; EnvName
+                        TransferEnv will transfer env info from other container
-                        is pod.spec.container[x].Env.name
+                        SourceContainerName is pod.spec.container[x].name; EnvName is pod.spec.container[x].Env.name
                       items:
                         properties:
                           envName:
@@ -107,6 +121,7 @@ spec:
                                 required:
                                 - fieldPath
                                 type: object
+                                x-kubernetes-map-type: atomic
                             type: object
                         type: object
                       type: array
@@ -115,16 +130,17 @@ spec:
                         HotUpgrade'
                       properties:
                         hotUpgradeEmptyImage:
-                          description: when HotUpgrade, HotUpgradeEmptyImage is used
+                          description: |-
-                            to complete the hot upgrading process HotUpgradeEmptyImage
+                            when HotUpgrade, HotUpgradeEmptyImage is used to complete the hot upgrading process
-                            is consistent of sidecar container in Command, Args, Liveness
+                            HotUpgradeEmptyImage is consistent of sidecar container in Command, Args, Liveness probe, etc.
-                            probe, etc. but it does no actual work.
+                            but it does no actual work.
                           type: string
                         upgradeType:
-                          description: when sidecar container is stateless, use ColdUpgrade
+                          description: |-
-                            otherwise HotUpgrade are more HotUpgrade. examples for
+                            when sidecar container is stateless, use ColdUpgrade
-                            istio envoy container is suitable for HotUpgrade default
+                            otherwise HotUpgrade are more HotUpgrade.
-                            is ColdUpgrade
+                            examples for istio envoy container is suitable for HotUpgrade
+                            default is ColdUpgrade
                           type: string
                       type: object
                   type: object
@@ -134,41 +150,59 @@ spec:
                 description: List of the names of secrets required by pulling sidecar
                   container images
                 items:
-                  description: LocalObjectReference contains enough information to
+                  description: |-
-                    let you locate the referenced object inside the same namespace.
+                    LocalObjectReference contains enough information to let you locate the
+                    referenced object inside the same namespace.
                   properties:
                     name:
-                      description: 'Name of the referent. More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
+                      default: ""
-                        TODO: Add other useful fields. apiVersion, kind, uid?'
+                      description: |-
+                        Name of the referent.
+                        This field is effectively required, but due to backwards compatibility is
+                        allowed to be empty. Instances of this type with an empty value here are
+                        almost certainly wrong.
+                        More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/names/#names
                       type: string
                   type: object
+                  x-kubernetes-map-type: atomic
                 type: array
               initContainers:
-                description: InitContainers is the list of init containers to be injected
+                description: |-
-                  into the selected pod We will inject those containers by their name
+                  InitContainers is the list of init containers to be injected into the selected pod
-                  in ascending order We only inject init containers when a new pod
+                  We will inject those containers by their name in ascending order
-                  is created, it does not apply to any existing pod
+                  We only inject init containers when a new pod is created, it does not apply to any existing pod
                 items:
                   description: SidecarContainer defines the container of Sidecar
                   properties:
                     podInjectPolicy:
-                      description: The rules that injected SidecarContainer into Pod.spec.containers,
+                      description: |-
-                        not takes effect in initContainers If BeforeAppContainer,
+                        The rules that injected SidecarContainer into Pod.spec.containers,
-                        the SidecarContainer will be injected in front of the pod.spec.containers
+                        not takes effect in initContainers
-                        otherwise it will be injected into the back. default BeforeAppContainerType
+                        If BeforeAppContainer, the SidecarContainer will be injected in front of the pod.spec.containers
+                        otherwise it will be injected into the back.
+                        default BeforeAppContainerType
                       type: string
+                    shareVolumeDevicePolicy:
+                      description: |-
+                        If ShareVolumeDevicePolicy is enabled, the sidecar container will share the other container's VolumeDevices
+                        in the pod(don't contain the injected sidecar container).
+                        This is a pointer to ensure that the sidecarset-hash does not change if the user does not configure this field, mainly for compatibility with older versions.
+                      properties:
+                        type:
+                          type: string
+                      type: object
                     shareVolumePolicy:
-                      description: If ShareVolumePolicy is enabled, the sidecar container
+                      description: |-
-                        will share the other container's VolumeMounts in the pod(don't
+                        If ShareVolumePolicy is enabled, the sidecar container will share the other container's VolumeMounts
-                        contains the injected sidecar container).
+                        in the pod(not including the injected sidecar container).
                       properties:
                         type:
                           type: string
                       type: object
                     transferEnv:
-                      description: TransferEnv will transfer env info from other container
+                      description: |-
-                        SourceContainerName is pod.spec.container[x].name; EnvName
+                        TransferEnv will transfer env info from other container
-                        is pod.spec.container[x].Env.name
+                        SourceContainerName is pod.spec.container[x].name; EnvName is pod.spec.container[x].Env.name
                       items:
                         properties:
                           envName:
@@ -196,6 +230,7 @@ spec:
                                 required:
                                 - fieldPath
                                 type: object
+                                x-kubernetes-map-type: atomic
                             type: object
                         type: object
                       type: array
@@ -204,16 +239,17 @@ spec:
                         HotUpgrade'
                       properties:
                         hotUpgradeEmptyImage:
-                          description: when HotUpgrade, HotUpgradeEmptyImage is used
+                          description: |-
-                            to complete the hot upgrading process HotUpgradeEmptyImage
+                            when HotUpgrade, HotUpgradeEmptyImage is used to complete the hot upgrading process
-                            is consistent of sidecar container in Command, Args, Liveness
+                            HotUpgradeEmptyImage is consistent of sidecar container in Command, Args, Liveness probe, etc.
-                            probe, etc. but it does no actual work.
+                            but it does no actual work.
                           type: string
                         upgradeType:
-                          description: when sidecar container is stateless, use ColdUpgrade
+                          description: |-
-                            otherwise HotUpgrade are more HotUpgrade. examples for
+                            when sidecar container is stateless, use ColdUpgrade
-                            istio envoy container is suitable for HotUpgrade default
+                            otherwise HotUpgrade are more HotUpgrade.
-                            is ColdUpgrade
+                            examples for istio envoy container is suitable for HotUpgrade
+                            default is ColdUpgrade
                           type: string
                       type: object
                   type: object
@@ -224,26 +260,30 @@ spec:
                   is injected into pods
                 properties:
                   paused:
-                    description: Paused indicates that SidecarSet will suspend injection
+                    description: |-
-                      into Pods If Paused is true, the sidecarSet will not be injected
+                      Paused indicates that SidecarSet will suspend injection into Pods
-                      to newly created Pods, but the injected sidecar container remains
+                      If Paused is true, the sidecarSet will not be injected to newly created Pods,
-                      updating and running. default is false
+                      but the injected sidecar container remains updating and running.
+                      default is false
                     type: boolean
                   revision:
-                    description: Revision can help users rolling update SidecarSet
+                    description: |-
-                      safely. If users set this filed, SidecarSet will try to inject
+                      Revision can help users rolling update SidecarSet safely. If users set
-                      specific revision according to different policies.
+                      this filed, SidecarSet will try to inject specific revision according to
+                      different policies.
                     properties:
                       customVersion:
-                        description: CustomVersion corresponds to label 'apps.kruise.io/sidecarset-custom-version'
+                        description: |-
-                          of (History) SidecarSet. SidecarSet will select the specific
+                          CustomVersion corresponds to label 'apps.kruise.io/sidecarset-custom-version' of (History) SidecarSet.
-                          ControllerRevision via this CustomVersion, and then restore
+                          SidecarSet will select the specific ControllerRevision via this CustomVersion, and then restore the
-                          the history SidecarSet to inject specific version of the
+                          history SidecarSet to inject specific version of the sidecar to pods.
-                          sidecar to pods.
                         type: string
                       policy:
+                        default: Always
                         description: Policy describes the behavior of revision injection.
-                          Defaults to Always.
+                        enum:
+                        - Always
+                        - Partial
                         type: string
                       revisionName:
                         description: RevisionName corresponds to a specific ControllerRevision
@@ -252,54 +292,58 @@ spec:
                     type: object
                 type: object
               namespace:
-                description: Namespace sidecarSet will only match the pods in the
+                description: |-
-                  namespace otherwise, match pods in all namespaces(in cluster)
+                  Namespace sidecarSet will only match the pods in the namespace
+                  otherwise, match pods in all namespaces(in cluster)
                 type: string
               namespaceSelector:
-                description: NamespaceSelector select which namespaces to inject sidecar
+                description: |-
-                  containers. Default to the empty LabelSelector, which matches everything.
+                  NamespaceSelector select which namespaces to inject sidecar containers.
+                  Default to the empty LabelSelector, which matches everything.
                 properties:
                   matchExpressions:
                     description: matchExpressions is a list of label selector requirements.
                       The requirements are ANDed.
                     items:
-                      description: A label selector requirement is a selector that
+                      description: |-
-                        contains values, a key, and an operator that relates the key
+                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        and values.
+                        relates the key and values.
                       properties:
                         key:
                           description: key is the label key that the selector applies
                             to.
                           type: string
                         operator:
-                          description: operator represents a key's relationship to
+                          description: |-
-                            a set of values. Valid operators are In, NotIn, Exists
+                            operator represents a key's relationship to a set of values.
-                            and DoesNotExist.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
                           type: string
                         values:
-                          description: values is an array of string values. If the
+                          description: |-
-                            operator is In or NotIn, the values array must be non-empty.
+                            values is an array of string values. If the operator is In or NotIn,
-                            If the operator is Exists or DoesNotExist, the values
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            array must be empty. This array is replaced during a strategic
+                            the values array must be empty. This array is replaced during a strategic
                             merge patch.
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
+                    description: |-
-                      {key,value} in the matchLabels map is equivalent to an element
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      of matchExpressions, whose key field is "key", the operator
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      is "In", and the values array contains only "value". The requirements
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                      are ANDed.
                     type: object
                 type: object
+                x-kubernetes-map-type: atomic
               patchPodMetadata:
                 description: SidecarSet support to inject & in-place update metadata
                   in pod.
@@ -311,14 +355,16 @@ spec:
                       description: annotations
                       type: object
                     patchPolicy:
-                      description: labels map[string]string `json:"labels,omitempty"`
+                      description: |-
+                        labels map[string]string `json:"labels,omitempty"`
                         patch pod metadata policy, Default is "Retain"
                       type: string
                   type: object
                 type: array
               revisionHistoryLimit:
-                description: RevisionHistoryLimit indicates the maximum quantity of
+                description: |-
-                  stored revisions about the SidecarSet. default value is 10
+                  RevisionHistoryLimit indicates the maximum quantity of stored revisions about the SidecarSet.
+                  default value is 10
                 format: int32
                 type: integer
               selector:
@@ -328,43 +374,45 @@ spec:
                     description: matchExpressions is a list of label selector requirements.
                       The requirements are ANDed.
                     items:
-                      description: A label selector requirement is a selector that
+                      description: |-
-                        contains values, a key, and an operator that relates the key
+                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        and values.
+                        relates the key and values.
                       properties:
                         key:
                           description: key is the label key that the selector applies
                             to.
                           type: string
                         operator:
-                          description: operator represents a key's relationship to
+                          description: |-
-                            a set of values. Valid operators are In, NotIn, Exists
+                            operator represents a key's relationship to a set of values.
-                            and DoesNotExist.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
                           type: string
                         values:
-                          description: values is an array of string values. If the
+                          description: |-
-                            operator is In or NotIn, the values array must be non-empty.
+                            values is an array of string values. If the operator is In or NotIn,
-                            If the operator is Exists or DoesNotExist, the values
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            array must be empty. This array is replaced during a strategic
+                            the values array must be empty. This array is replaced during a strategic
                             merge patch.
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
+                    description: |-
-                      {key,value} in the matchLabels map is equivalent to an element
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      of matchExpressions, whose key field is "key", the operator
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      is "In", and the values array contains only "value". The requirements
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                      are ANDed.
                     type: object
                 type: object
+                x-kubernetes-map-type: atomic
               updateStrategy:
                 description: The sidecarset updateStrategy to use to replace existing
                   pods with new ones.
@@ -373,36 +421,128 @@ spec:
                     anyOf:
                     - type: integer
                     - type: string
-                    description: 'The maximum number of SidecarSet pods that can be
+                    description: |-
-                      unavailable during the update. Value can be an absolute number
+                      The maximum number of SidecarSet pods that can be unavailable during the
-                      (ex: 5) or a percentage of total number of SidecarSet pods at
+                      update. Value can be an absolute number (ex: 5) or a percentage of total
-                      the start of the update (ex: 10%). Absolute number is calculated
+                      number of SidecarSet pods at the start of the update (ex: 10%). Absolute
-                      from percentage by rounding up. This cannot be 0. Default value
+                      number is calculated from percentage by rounding up.
-                      is 1.'
+                      This cannot be 0.
+                      Default value is 1.
                     x-kubernetes-int-or-string: true
                   partition:
                     anyOf:
                     - type: integer
                     - type: string
-                    description: Partition is the desired number of pods in old revisions.
+                    description: |-
-                      It means when partition is set during pods updating, (replicas
+                      Partition is the desired number of pods in old revisions. It means when partition
-                      - partition) number of pods will be updated. Default value is
+                      is set during pods updating, (replicas - partition) number of pods will be updated.
-                      0.
+                      Default value is 0.
                     x-kubernetes-int-or-string: true
                   paused:
-                    description: Paused indicates that the SidecarSet is paused to
+                    description: |-
-                      update the injected pods, but it don't affect the webhook inject
+                      Paused indicates that the SidecarSet is paused to update the injected pods,
-                      sidecar container into the newly created pods. default is false
+                      For the impact on the injection behavior for newly created Pods, please refer to the comments of Selector.
                     type: boolean
+                  priorityStrategy:
+                    description: |-
+                      Priorities are the rules for calculating the priority of updating pods.
+                      Each pod to be updated, will pass through these terms and get a sum of weights.
+                    properties:
+                      orderPriority:
+                        description: |-
+                          Order priority terms, pods will be sorted by the value of orderedKey.
+                          For example:
+                          ```
+                          orderPriority:
+                          - orderedKey: key1
+                          - orderedKey: key2
+                          ```
+                          First, all pods which have key1 in labels will be sorted by the value of key1.
+                          Then, the left pods which have no key1 but have key2 in labels will be sorted by
+                          the value of key2 and put behind those pods have key1.
+                        items:
+                          description: UpdatePriorityOrderTerm defines order priority.
+                          properties:
+                            orderedKey:
+                              description: |-
+                                Calculate priority by value of this key.
+                                Values of this key, will be sorted by GetInt(val). GetInt method will find the last int in value,
+                                such as getting 5 in value '5', getting 10 in value 'sts-10'.
+                              type: string
+                          required:
+                          - orderedKey
+                          type: object
+                        type: array
+                      weightPriority:
+                        description: Weight priority terms, pods will be sorted by
+                          the sum of all terms weight.
+                        items:
+                          description: UpdatePriorityWeightTerm defines weight priority.
+                          properties:
+                            matchSelector:
+                              description: MatchSelector is used to select by pod's
+                                labels.
+                              properties:
+                                matchExpressions:
+                                  description: matchExpressions is a list of label
+                                    selector requirements. The requirements are ANDed.
+                                  items:
+                                    description: |-
+                                      A label selector requirement is a selector that contains values, a key, and an operator that
+                                      relates the key and values.
+                                    properties:
+                                      key:
+                                        description: key is the label key that the
+                                          selector applies to.
+                                        type: string
+                                      operator:
+                                        description: |-
+                                          operator represents a key's relationship to a set of values.
+                                          Valid operators are In, NotIn, Exists and DoesNotExist.
+                                        type: string
+                                      values:
+                                        description: |-
+                                          values is an array of string values. If the operator is In or NotIn,
+                                          the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                                          the values array must be empty. This array is replaced during a strategic
+                                          merge patch.
+                                        items:
+                                          type: string
+                                        type: array
+                                        x-kubernetes-list-type: atomic
+                                    required:
+                                    - key
+                                    - operator
+                                    type: object
+                                  type: array
+                                  x-kubernetes-list-type: atomic
+                                matchLabels:
+                                  additionalProperties:
+                                    type: string
+                                  description: |-
+                                    matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                                    map is equivalent to an element of matchExpressions, whose key field is "key", the
+                                    operator is "In", and the values array contains only "value". The requirements are ANDed.
+                                  type: object
+                              type: object
+                              x-kubernetes-map-type: atomic
+                            weight:
+                              description: Weight associated with matching the corresponding
+                                matchExpressions, in the range 1-100.
+                              format: int32
+                              type: integer
+                          required:
+                          - matchSelector
+                          - weight
+                          type: object
+                        type: array
+                    type: object
                   scatterStrategy:
-                    description: ScatterStrategy defines the scatter rules to make
+                    description: |-
-                      pods been scattered when update. This will avoid pods with the
+                      ScatterStrategy defines the scatter rules to make pods been scattered when update.
-                      same key-value to be updated in one batch. - Note that pods
+                      This will avoid pods with the same key-value to be updated in one batch.
-                      will be scattered after priority sort. So, although priority
+                      - Note that pods will be scattered after priority sort. So, although priority strategy and scatter strategy can be applied together, we suggest to use either one of them.
-                      strategy and scatter strategy can be applied together, we suggest
+                      - If scatterStrategy is used, we suggest to just use one term. Otherwise, the update order can be hard to understand.
-                      to use either one of them. - If scatterStrategy is used, we
-                      suggest to just use one term. Otherwise, the update order can
-                      be hard to understand.
                     items:
                       properties:
                         key:
@@ -415,56 +555,63 @@ spec:
                       type: object
                     type: array
                   selector:
-                    description: If selector is not nil, this upgrade will only update
+                    description: |-
-                      the selected pods.
+                      If selector is not nil, this upgrade will only update the selected pods.
+
+                      Starting from Kruise 1.8.0, the updateStrategy.Selector affects the version of the Sidecar container
+                      injected into newly created Pods by a SidecarSet configured with an injectionStrategy.
+                      In most cases, all newly created Pods are injected with the specified Sidecar version as configured in injectionStrategy.revision,
+                      which is consistent with previous versions.
                     properties:
                       matchExpressions:
                         description: matchExpressions is a list of label selector
                           requirements. The requirements are ANDed.
                         items:
-                          description: A label selector requirement is a selector
+                          description: |-
-                            that contains values, a key, and an operator that relates
+                            A label selector requirement is a selector that contains values, a key, and an operator that
-                            the key and values.
+                            relates the key and values.
                           properties:
                             key:
                               description: key is the label key that the selector
                                 applies to.
                               type: string
                             operator:
-                              description: operator represents a key's relationship
+                              description: |-
-                                to a set of values. Valid operators are In, NotIn,
+                                operator represents a key's relationship to a set of values.
-                                Exists and DoesNotExist.
+                                Valid operators are In, NotIn, Exists and DoesNotExist.
                               type: string
                             values:
-                              description: values is an array of string values. If
+                              description: |-
-                                the operator is In or NotIn, the values array must
+                                values is an array of string values. If the operator is In or NotIn,
-                                be non-empty. If the operator is Exists or DoesNotExist,
+                                the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                the values array must be empty. This array is replaced
+                                the values array must be empty. This array is replaced during a strategic
-                                during a strategic merge patch.
+                                merge patch.
                               items:
                                 type: string
                               type: array
+                              x-kubernetes-list-type: atomic
                           required:
                           - key
                           - operator
                           type: object
                         type: array
+                        x-kubernetes-list-type: atomic
                       matchLabels:
                         additionalProperties:
                           type: string
-                        description: matchLabels is a map of {key,value} pairs. A
+                        description: |-
-                          single {key,value} in the matchLabels map is equivalent
+                          matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                          to an element of matchExpressions, whose key field is "key",
+                          map is equivalent to an element of matchExpressions, whose key field is "key", the
-                          the operator is "In", and the values array contains only
+                          operator is "In", and the values array contains only "value". The requirements are ANDed.
-                          "value". The requirements are ANDed.
                         type: object
                     type: object
+                    x-kubernetes-map-type: atomic
                   type:
-                    description: Type is NotUpdate, the SidecarSet don't update the
+                    description: |-
-                      injected pods, it will only inject sidecar container into the
+                      Type is NotUpdate, the SidecarSet don't update the injected pods,
-                      newly created pods. Type is RollingUpdate, the SidecarSet will
+                      it will only inject sidecar container into the newly created pods.
-                      update the injected pods to the latest version on RollingUpdate
+                      Type is RollingUpdate, the SidecarSet will update the injected pods to the latest version on RollingUpdate Strategy.
-                      Strategy. default is RollingUpdate
+                      default is RollingUpdate
                     type: string
                 type: object
               volumes:
@@ -475,10 +622,10 @@ spec:
             description: SidecarSetStatus defines the observed state of SidecarSet
             properties:
               collisionCount:
-                description: CollisionCount is the count of hash collisions for the
+                description: |-
-                  SidecarSet. The SidecarSet controller uses this field as a collision
+                  CollisionCount is the count of hash collisions for the SidecarSet. The SidecarSet controller
-                  avoidance mechanism when it needs to create the name for the newest
+                  uses this field as a collision avoidance mechanism when it needs to create the name for the
-                  ControllerRevision.
+                  newest ControllerRevision.
                 format: int32
                 type: integer
               latestRevision:
@@ -492,9 +639,9 @@ spec:
                 format: int32
                 type: integer
               observedGeneration:
-                description: observedGeneration is the most recent generation observed
+                description: |-
-                  for this SidecarSet. It corresponds to the SidecarSet's generation,
+                  observedGeneration is the most recent generation observed for this SidecarSet. It corresponds to the
-                  which is updated on mutation by the API Server.
+                  SidecarSet's generation, which is updated on mutation by the API Server.
                 format: int64
                 type: integer
               readyPods:
@@ -522,9 +669,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/apps.kruise.io_workloadspreads.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: workloadspreads.apps.kruise.io
 spec:
   group: apps.kruise.io
@@ -38,14 +36,19 @@ spec:
         description: WorkloadSpread is the Schema for the WorkloadSpread API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -61,25 +64,23 @@ spec:
                       is AdaptiveWorkloadSpreadScheduleStrategyType.
                     properties:
                       disableSimulationSchedule:
-                        description: DisableSimulationSchedule indicates whether to
+                        description: |-
-                          disable the feature of simulation schedule. Default is false.
+                          DisableSimulationSchedule indicates whether to disable the feature of simulation schedule.
-                          Webhook can take a simple general predicates to check whether
+                          Default is false.
-                          Pod can be scheduled into this subset, but it just considers
+                          Webhook can take a simple general predicates to check whether Pod can be scheduled into this subset,
-                          the Node resource and cannot replace scheduler to do richer
+                          but it just considers the Node resource and cannot replace scheduler to do richer predicates practically.
-                          predicates practically.
                         type: boolean
                       rescheduleCriticalSeconds:
-                        description: RescheduleCriticalSeconds indicates how long
+                        description: |-
-                          controller will reschedule a schedule failed Pod to the
+                          RescheduleCriticalSeconds indicates how long controller will reschedule a schedule failed Pod to the subset that has
-                          subset that has redundant capacity after the subset where
+                          redundant capacity after the subset where the Pod lives. If a Pod was scheduled failed and still in a unschedulabe status
-                          the Pod lives. If a Pod was scheduled failed and still in
+                          over RescheduleCriticalSeconds duration, the controller will reschedule it to a suitable subset.
-                          a unschedulabe status over RescheduleCriticalSeconds duration,
-                          the controller will reschedule it to a suitable subset.
                         format: int32
                         type: integer
                     type: object
                   type:
-                    description: Type indicates the type of the WorkloadSpreadScheduleStrategy.
+                    description: |-
+                      Type indicates the type of the WorkloadSpreadScheduleStrategy.
                       Default is Fixed
                     enum:
                     - Adaptive
@@ -111,10 +112,9 @@ spec:
                       description: Indicates the node preferred selector to form the
                         subset.
                       items:
-                        description: An empty preferred scheduling term matches all
+                        description: |-
-                          objects with implicit weight 0 (i.e. it's a no-op). A null
+                          An empty preferred scheduling term matches all objects with implicit weight 0
-                          preferred scheduling term matches no objects (i.e. is also
+                          (i.e. it's a no-op). A null preferred scheduling term matches no objects (i.e. is also a no-op).
-                          a no-op).
                         properties:
                           preference:
                             description: A node selector term, associated with the
@@ -124,71 +124,72 @@ spec:
                                 description: A list of node selector requirements
                                   by node's labels.
                                 items:
-                                  description: A node selector requirement is a selector
+                                  description: |-
-                                    that contains values, a key, and an operator that
+                                    A node selector requirement is a selector that contains values, a key, and an operator
-                                    relates the key and values.
+                                    that relates the key and values.
                                   properties:
                                     key:
                                       description: The label key that the selector
                                         applies to.
                                       type: string
                                     operator:
-                                      description: Represents a key's relationship
+                                      description: |-
-                                        to a set of values. Valid operators are In,
+                                        Represents a key's relationship to a set of values.
-                                        NotIn, Exists, DoesNotExist. Gt, and Lt.
+                                        Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
                                       type: string
                                     values:
-                                      description: An array of string values. If the
+                                      description: |-
-                                        operator is In or NotIn, the values array
+                                        An array of string values. If the operator is In or NotIn,
-                                        must be non-empty. If the operator is Exists
+                                        the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                        or DoesNotExist, the values array must be
+                                        the values array must be empty. If the operator is Gt or Lt, the values
-                                        empty. If the operator is Gt or Lt, the values
+                                        array must have a single element, which will be interpreted as an integer.
-                                        array must have a single element, which will
+                                        This array is replaced during a strategic merge patch.
-                                        be interpreted as an integer. This array is
-                                        replaced during a strategic merge patch.
                                       items:
                                         type: string
                                       type: array
+                                      x-kubernetes-list-type: atomic
                                   required:
                                   - key
                                   - operator
                                   type: object
                                 type: array
+                                x-kubernetes-list-type: atomic
                               matchFields:
                                 description: A list of node selector requirements
                                   by node's fields.
                                 items:
-                                  description: A node selector requirement is a selector
+                                  description: |-
-                                    that contains values, a key, and an operator that
+                                    A node selector requirement is a selector that contains values, a key, and an operator
-                                    relates the key and values.
+                                    that relates the key and values.
                                   properties:
                                     key:
                                       description: The label key that the selector
                                         applies to.
                                       type: string
                                     operator:
-                                      description: Represents a key's relationship
+                                      description: |-
-                                        to a set of values. Valid operators are In,
+                                        Represents a key's relationship to a set of values.
-                                        NotIn, Exists, DoesNotExist. Gt, and Lt.
+                                        Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
                                       type: string
                                     values:
-                                      description: An array of string values. If the
+                                      description: |-
-                                        operator is In or NotIn, the values array
+                                        An array of string values. If the operator is In or NotIn,
-                                        must be non-empty. If the operator is Exists
+                                        the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                        or DoesNotExist, the values array must be
+                                        the values array must be empty. If the operator is Gt or Lt, the values
-                                        empty. If the operator is Gt or Lt, the values
+                                        array must have a single element, which will be interpreted as an integer.
-                                        array must have a single element, which will
+                                        This array is replaced during a strategic merge patch.
-                                        be interpreted as an integer. This array is
-                                        replaced during a strategic merge patch.
                                       items:
                                         type: string
                                       type: array
+                                      x-kubernetes-list-type: atomic
                                   required:
                                   - key
                                   - operator
                                   type: object
                                 type: array
+                                x-kubernetes-list-type: atomic
                             type: object
+                            x-kubernetes-map-type: atomic
                           weight:
                             description: Weight associated with matching the corresponding
                               nodeSelectorTerm, in the range 1-100.
@@ -207,109 +208,109 @@ spec:
                           description: A list of node selector requirements by node's
                             labels.
                           items:
-                            description: A node selector requirement is a selector
+                            description: |-
-                              that contains values, a key, and an operator that relates
+                              A node selector requirement is a selector that contains values, a key, and an operator
-                              the key and values.
+                              that relates the key and values.
                             properties:
                               key:
                                 description: The label key that the selector applies
                                   to.
                                 type: string
                               operator:
-                                description: Represents a key's relationship to a
+                                description: |-
-                                  set of values. Valid operators are In, NotIn, Exists,
+                                  Represents a key's relationship to a set of values.
-                                  DoesNotExist. Gt, and Lt.
+                                  Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
                                 type: string
                               values:
-                                description: An array of string values. If the operator
+                                description: |-
-                                  is In or NotIn, the values array must be non-empty.
+                                  An array of string values. If the operator is In or NotIn,
-                                  If the operator is Exists or DoesNotExist, the values
+                                  the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                  array must be empty. If the operator is Gt or Lt,
+                                  the values array must be empty. If the operator is Gt or Lt, the values
-                                  the values array must have a single element, which
+                                  array must have a single element, which will be interpreted as an integer.
-                                  will be interpreted as an integer. This array is
+                                  This array is replaced during a strategic merge patch.
-                                  replaced during a strategic merge patch.
                                 items:
                                   type: string
                                 type: array
+                                x-kubernetes-list-type: atomic
                             required:
                             - key
                             - operator
                             type: object
                           type: array
+                          x-kubernetes-list-type: atomic
                         matchFields:
                           description: A list of node selector requirements by node's
                             fields.
                           items:
-                            description: A node selector requirement is a selector
+                            description: |-
-                              that contains values, a key, and an operator that relates
+                              A node selector requirement is a selector that contains values, a key, and an operator
-                              the key and values.
+                              that relates the key and values.
                             properties:
                               key:
                                 description: The label key that the selector applies
                                   to.
                                 type: string
                               operator:
-                                description: Represents a key's relationship to a
+                                description: |-
-                                  set of values. Valid operators are In, NotIn, Exists,
+                                  Represents a key's relationship to a set of values.
-                                  DoesNotExist. Gt, and Lt.
+                                  Valid operators are In, NotIn, Exists, DoesNotExist. Gt, and Lt.
                                 type: string
                               values:
-                                description: An array of string values. If the operator
+                                description: |-
-                                  is In or NotIn, the values array must be non-empty.
+                                  An array of string values. If the operator is In or NotIn,
-                                  If the operator is Exists or DoesNotExist, the values
+                                  the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                                  array must be empty. If the operator is Gt or Lt,
+                                  the values array must be empty. If the operator is Gt or Lt, the values
-                                  the values array must have a single element, which
+                                  array must have a single element, which will be interpreted as an integer.
-                                  will be interpreted as an integer. This array is
+                                  This array is replaced during a strategic merge patch.
-                                  replaced during a strategic merge patch.
                                 items:
                                   type: string
                                 type: array
+                                x-kubernetes-list-type: atomic
                             required:
                             - key
                             - operator
                             type: object
                           type: array
+                          x-kubernetes-list-type: atomic
                       type: object
+                      x-kubernetes-map-type: atomic
                     tolerations:
                       description: Indicates the tolerations the pods under this subset
                         have.
                       items:
-                        description: The pod this Toleration is attached to tolerates
+                        description: |-
-                          any taint that matches the triple <key,value,effect> using
+                          The pod this Toleration is attached to tolerates any taint that matches
-                          the matching operator <operator>.
+                          the triple <key,value,effect> using the matching operator <operator>.
                         properties:
                           effect:
-                            description: Effect indicates the taint effect to match.
+                            description: |-
-                              Empty means match all taint effects. When specified,
+                              Effect indicates the taint effect to match. Empty means match all taint effects.
-                              allowed values are NoSchedule, PreferNoSchedule and
+                              When specified, allowed values are NoSchedule, PreferNoSchedule and NoExecute.
-                              NoExecute.
                             type: string
                           key:
-                            description: Key is the taint key that the toleration
+                            description: |-
-                              applies to. Empty means match all taint keys. If the
+                              Key is the taint key that the toleration applies to. Empty means match all taint keys.
-                              key is empty, operator must be Exists; this combination
+                              If the key is empty, operator must be Exists; this combination means to match all values and all keys.
-                              means to match all values and all keys.
                             type: string
                           operator:
-                            description: Operator represents a key's relationship
+                            description: |-
-                              to the value. Valid operators are Exists and Equal.
+                              Operator represents a key's relationship to the value.
-                              Defaults to Equal. Exists is equivalent to wildcard
+                              Valid operators are Exists and Equal. Defaults to Equal.
-                              for value, so that a pod can tolerate all taints of
+                              Exists is equivalent to wildcard for value, so that a pod can
-                              a particular category.
+                              tolerate all taints of a particular category.
                             type: string
                           tolerationSeconds:
-                            description: TolerationSeconds represents the period of
+                            description: |-
-                              time the toleration (which must be of effect NoExecute,
+                              TolerationSeconds represents the period of time the toleration (which must be
-                              otherwise this field is ignored) tolerates the taint.
+                              of effect NoExecute, otherwise this field is ignored) tolerates the taint. By default,
-                              By default, it is not set, which means tolerate the
+                              it is not set, which means tolerate the taint forever (do not evict). Zero and
-                              taint forever (do not evict). Zero and negative values
+                              negative values will be treated as 0 (evict immediately) by the system.
-                              will be treated as 0 (evict immediately) by the system.
                             format: int64
                             type: integer
                           value:
-                            description: Value is the taint value the toleration matches
+                            description: |-
-                              to. If the operator is Exists, the value should be empty,
+                              Value is the taint value the toleration matches to.
-                              otherwise just a regular string.
+                              If the operator is Exists, the value should be empty, otherwise just a regular string.
                             type: string
                         type: object
                       type: array
@@ -317,6 +318,70 @@ spec:
                   - name
                   type: object
                 type: array
+              targetFilter:
+                description: |-
+                  TargetFilter allows WorkloadSpread to manage only a portion of the Pods in the TargetReference:
+                  by specifying the criteria for the Pods to be managed through a label selector,
+                  and by specifying how to obtain the total number of these selected Pods from the workload using replicasPaths.
+                properties:
+                  replicasPathList:
+                    description: |-
+                      ReplicasPathList is a list of resource paths used to specify how to determine the total number of replicas of
+                      the target workload after filtering. If this list is not empty, WorkloadSpread will look for the corresponding
+                      values in the target resource according to each path, and treat the sum of these values as the total number of replicas after filtering.
+
+                      The replicas path is a dot-separated path, similar to "spec.replicas". If there are arrays, you can use numbers to denote indexes, like "subsets.1.replicas".
+                      The real values of these paths must be integers.
+                    items:
+                      type: string
+                    type: array
+                  selector:
+                    description: Selector is used to filter the Pods to be managed.
+                    properties:
+                      matchExpressions:
+                        description: matchExpressions is a list of label selector
+                          requirements. The requirements are ANDed.
+                        items:
+                          description: |-
+                            A label selector requirement is a selector that contains values, a key, and an operator that
+                            relates the key and values.
+                          properties:
+                            key:
+                              description: key is the label key that the selector
+                                applies to.
+                              type: string
+                            operator:
+                              description: |-
+                                operator represents a key's relationship to a set of values.
+                                Valid operators are In, NotIn, Exists and DoesNotExist.
+                              type: string
+                            values:
+                              description: |-
+                                values is an array of string values. If the operator is In or NotIn,
+                                the values array must be non-empty. If the operator is Exists or DoesNotExist,
+                                the values array must be empty. This array is replaced during a strategic
+                                merge patch.
+                              items:
+                                type: string
+                              type: array
+                              x-kubernetes-list-type: atomic
+                          required:
+                          - key
+                          - operator
+                          type: object
+                        type: array
+                        x-kubernetes-list-type: atomic
+                      matchLabels:
+                        additionalProperties:
+                          type: string
+                        description: |-
+                          matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
+                          map is equivalent to an element of matchExpressions, whose key field is "key", the
+                          operator is "In", and the values array contains only "value". The requirements are ANDed.
+                        type: object
+                    type: object
+                    x-kubernetes-map-type: atomic
+                type: object
               targetRef:
                 description: TargetReference is the target workload that WorkloadSpread
                   want to control.
@@ -343,9 +408,9 @@ spec:
             description: WorkloadSpreadStatus defines the observed state of WorkloadSpread.
             properties:
               observedGeneration:
-                description: ObservedGeneration is the most recent generation observed
+                description: |-
-                  for this WorkloadSpread. It corresponds to the WorkloadSpread's
+                  ObservedGeneration is the most recent generation observed for this WorkloadSpread. It corresponds to the
-                  generation, which is updated on mutation by the API Server.
+                  WorkloadSpread's generation, which is updated on mutation by the API Server.
                 format: int64
                 type: integer
               subsetStatuses:
@@ -388,18 +453,17 @@ spec:
                       additionalProperties:
                         format: date-time
                         type: string
-                      description: CreatingPods contains information about pods whose
+                      description: |-
-                        creation was processed by the webhook handler but not yet
+                        CreatingPods contains information about pods whose creation was processed by
-                        been observed by the WorkloadSpread controller. A pod will
+                        the webhook handler but not yet been observed by the WorkloadSpread controller.
-                        be in this map from the time when the webhook handler processed
+                        A pod will be in this map from the time when the webhook handler processed the
-                        the creation request to the time when the pod is seen by controller.
+                        creation request to the time when the pod is seen by controller.
-                        The key in the map is the name of the pod and the value is
+                        The key in the map is the name of the pod and the value is the time when the webhook
-                        the time when the webhook handler process the creation request.
+                        handler process the creation request. If the real creation didn't happen and a pod is
-                        If the real creation didn't happen and a pod is still in this
+                        still in this map, it will be removed from the list automatically by WorkloadSpread controller
-                        map, it will be removed from the list automatically by WorkloadSpread
+                        after some time.
-                        controller after some time. If everything goes smooth this
+                        If everything goes smooth this map should be empty for the most of the time.
-                        map should be empty for the most of the time. Large number
+                        Large number of entries in the map may indicate problems with pod creations.
-                        of entries in the map may indicate problems with pod creations.
                       type: object
                     deletingPods:
                       additionalProperties:
@@ -409,13 +473,11 @@ spec:
                         contains information about pod deletion.
                       type: object
                     missingReplicas:
-                      description: MissingReplicas is the number of active replicas
+                      description: |-
-                        belong to this subset not be found. MissingReplicas > 0 indicates
+                        MissingReplicas is the number of active replicas belong to this subset not be found.
-                        the subset is still missing MissingReplicas pods to create
+                        MissingReplicas > 0 indicates the subset is still missing MissingReplicas pods to create
-                        MissingReplicas = 0 indicates the subset already has enough
+                        MissingReplicas = 0 indicates the subset already has enough pods, there is no need to create
-                        pods, there is no need to create MissingReplicas = -1 indicates
+                        MissingReplicas = -1 indicates the subset's MaxReplicas not set, then there is no limit for pods number
-                        the subset's MaxReplicas not set, then there is no limit for
-                        pods number
                       format: int32
                       type: integer
                     name:
@@ -433,15 +495,95 @@ spec:
                   - replicas
                   type: object
                 type: array
+              versionedSubsetStatuses:
+                additionalProperties:
+                  items:
+                    description: WorkloadSpreadSubsetStatus defines the observed state
+                      of subset
+                    properties:
+                      conditions:
+                        description: Conditions is an array of current observed subset
+                          conditions.
+                        items:
+                          properties:
+                            lastTransitionTime:
+                              description: Last time the condition transitioned from
+                                one status to another.
+                              format: date-time
+                              type: string
+                            message:
+                              description: A human readable message indicating details
+                                about the transition.
+                              type: string
+                            reason:
+                              description: The reason for the condition's last transition.
+                              type: string
+                            status:
+                              description: Status of the condition, one of True, False,
+                                Unknown.
+                              type: string
+                            type:
+                              description: Type of in place set condition.
+                              type: string
+                          required:
+                          - status
+                          - type
+                          type: object
+                        type: array
+                      creatingPods:
+                        additionalProperties:
+                          format: date-time
+                          type: string
+                        description: |-
+                          CreatingPods contains information about pods whose creation was processed by
+                          the webhook handler but not yet been observed by the WorkloadSpread controller.
+                          A pod will be in this map from the time when the webhook handler processed the
+                          creation request to the time when the pod is seen by controller.
+                          The key in the map is the name of the pod and the value is the time when the webhook
+                          handler process the creation request. If the real creation didn't happen and a pod is
+                          still in this map, it will be removed from the list automatically by WorkloadSpread controller
+                          after some time.
+                          If everything goes smooth this map should be empty for the most of the time.
+                          Large number of entries in the map may indicate problems with pod creations.
+                        type: object
+                      deletingPods:
+                        additionalProperties:
+                          format: date-time
+                          type: string
+                        description: DeletingPods is similar with CreatingPods and
+                          it contains information about pod deletion.
+                        type: object
+                      missingReplicas:
+                        description: |-
+                          MissingReplicas is the number of active replicas belong to this subset not be found.
+                          MissingReplicas > 0 indicates the subset is still missing MissingReplicas pods to create
+                          MissingReplicas = 0 indicates the subset already has enough pods, there is no need to create
+                          MissingReplicas = -1 indicates the subset's MaxReplicas not set, then there is no limit for pods number
+                        format: int32
+                        type: integer
+                      name:
+                        description: Name should be unique between all of the subsets
+                          under one WorkloadSpread.
+                        type: string
+                      replicas:
+                        description: Replicas is the most recently observed number
+                          of active replicas for subset.
+                        format: int32
+                        type: integer
+                    required:
+                    - missingReplicas
+                    - name
+                    - replicas
+                    type: object
+                  type: array
+                description: |-
+                  VersionedSubsetStatuses is to solve rolling-update problems, where the creation of new-version pod
+                  may be earlier than deletion of old-version pod. We have to calculate the pod subset distribution for
+                  each version.
+                type: object
             type: object
         type: object
     served: true
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/bases/policy.kruise.io_podunavailablebudgets.yaml

@@ -1,11 +1,9 @@
-
 ---
 apiVersion: apiextensions.k8s.io/v1
 kind: CustomResourceDefinition
 metadata:
   annotations:
-    controller-gen.kubebuilder.io/version: v0.7.0
+    controller-gen.kubebuilder.io/version: v0.16.5
-  creationTimestamp: null
   name: podunavailablebudgets.policy.kruise.io
 spec:
   group: policy.kruise.io
@@ -43,14 +41,19 @@ spec:
           API
         properties:
           apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
+            description: |-
-              of an object. Servers should convert recognized schemas to the latest
+              APIVersion defines the versioned schema of this representation of an object.
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
+              Servers should convert recognized schemas to the latest internal value, and
+              may reject unrecognized values.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources
             type: string
           kind:
-            description: 'Kind is a string value representing the REST resource this
+            description: |-
-              object represents. Servers may infer this from the endpoint the client
+              Kind is a string value representing the REST resource this object represents.
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
+              Servers may infer this from the endpoint the client submits requests to.
+              Cannot be updated.
+              In CamelCase.
+              More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds
             type: string
           metadata:
             type: object
@@ -61,20 +64,18 @@ spec:
                 anyOf:
                 - type: integer
                 - type: string
-                description: Delete pod, evict pod or update pod specification is
+                description: |-
-                  allowed if at most "maxUnavailable" pods selected by "selector"
+                  Delete pod, evict pod or update pod specification is allowed if at most "maxUnavailable" pods selected by
-                  or "targetRef"  are unavailable after the above operation for pod.
+                  "selector" or "targetRef"  are unavailable after the above operation for pod.
-                  MaxUnavailable and MinAvailable are mutually exclusive, MaxUnavailable
+                  MaxUnavailable and MinAvailable are mutually exclusive, MaxUnavailable is priority to take effect
-                  is priority to take effect
                 x-kubernetes-int-or-string: true
               minAvailable:
                 anyOf:
                 - type: integer
                 - type: string
-                description: Delete pod, evict pod or update pod specification is
+                description: |-
-                  allowed if at least "minAvailable" pods selected by "selector" or
+                  Delete pod, evict pod or update pod specification is allowed if at least "minAvailable" pods selected by
-                  "targetRef" will still be available after the above operation for
+                  "selector" or "targetRef" will still be available after the above operation for pod.
-                  pod.
                 x-kubernetes-int-or-string: true
               selector:
                 description: Selector label query over pods managed by the budget
@@ -83,47 +84,49 @@ spec:
                     description: matchExpressions is a list of label selector requirements.
                       The requirements are ANDed.
                     items:
-                      description: A label selector requirement is a selector that
+                      description: |-
-                        contains values, a key, and an operator that relates the key
+                        A label selector requirement is a selector that contains values, a key, and an operator that
-                        and values.
+                        relates the key and values.
                       properties:
                         key:
                           description: key is the label key that the selector applies
                             to.
                           type: string
                         operator:
-                          description: operator represents a key's relationship to
+                          description: |-
-                            a set of values. Valid operators are In, NotIn, Exists
+                            operator represents a key's relationship to a set of values.
-                            and DoesNotExist.
+                            Valid operators are In, NotIn, Exists and DoesNotExist.
                           type: string
                         values:
-                          description: values is an array of string values. If the
+                          description: |-
-                            operator is In or NotIn, the values array must be non-empty.
+                            values is an array of string values. If the operator is In or NotIn,
-                            If the operator is Exists or DoesNotExist, the values
+                            the values array must be non-empty. If the operator is Exists or DoesNotExist,
-                            array must be empty. This array is replaced during a strategic
+                            the values array must be empty. This array is replaced during a strategic
                             merge patch.
                           items:
                             type: string
                           type: array
+                          x-kubernetes-list-type: atomic
                       required:
                       - key
                       - operator
                       type: object
                     type: array
+                    x-kubernetes-list-type: atomic
                   matchLabels:
                     additionalProperties:
                       type: string
-                    description: matchLabels is a map of {key,value} pairs. A single
+                    description: |-
-                      {key,value} in the matchLabels map is equivalent to an element
+                      matchLabels is a map of {key,value} pairs. A single {key,value} in the matchLabels
-                      of matchExpressions, whose key field is "key", the operator
+                      map is equivalent to an element of matchExpressions, whose key field is "key", the
-                      is "In", and the values array contains only "value". The requirements
+                      operator is "In", and the values array contains only "value". The requirements are ANDed.
-                      are ANDed.
                     type: object
                 type: object
+                x-kubernetes-map-type: atomic
               targetRef:
-                description: TargetReference contains enough information to let you
+                description: |-
-                  identify an workload for PodUnavailableBudget Selector and TargetReference
+                  TargetReference contains enough information to let you identify an workload for PodUnavailableBudget
-                  are mutually exclusive, TargetReference is priority to take effect
+                  Selector and TargetReference are mutually exclusive, TargetReference is priority to take effect
                 properties:
                   apiVersion:
                     description: API version of the referent.
@@ -153,14 +156,14 @@ spec:
                 additionalProperties:
                   format: date-time
                   type: string
-                description: DisruptedPods contains information about pods whose eviction
+                description: |-
-                  or deletion was processed by the API handler but has not yet been
+                  DisruptedPods contains information about pods whose eviction or deletion was
-                  observed by the PodUnavailableBudget.
+                  processed by the API handler but has not yet been observed by the PodUnavailableBudget.
                 type: object
               observedGeneration:
-                description: Most recent generation observed when updating this PUB
+                description: |-
-                  status. UnavailableAllowed and other status information is valid
+                  Most recent generation observed when updating this PUB status. UnavailableAllowed and other
-                  only if observedGeneration equals to PUB's object generation.
+                  status information is valid only if observedGeneration equals to PUB's object generation.
                 format: int64
                 type: integer
               totalReplicas:
@@ -177,9 +180,9 @@ spec:
                 additionalProperties:
                   format: date-time
                   type: string
-                description: UnavailablePods contains information about pods whose
+                description: |-
-                  specification changed(inplace-update pod), once pod is available(consistent
+                  UnavailablePods contains information about pods whose specification changed(inplace-update pod),
-                  and ready) again, it will be removed from the list.
+                  once pod is available(consistent and ready) again, it will be removed from the list.
                 type: object
             required:
             - currentAvailable
@@ -192,9 +195,3 @@ spec:
     storage: true
     subresources:
       status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

config/crd/kustomization.yaml

@@ -19,6 +19,7 @@ resources:
 - bases/apps.kruise.io_persistentpodstates.yaml
 - bases/apps.kruise.io_podprobemarkers.yaml
 - bases/apps.kruise.io_nodepodprobes.yaml
+- bases/apps.kruise.io_imagelistpulljobs.yaml
 # +kubebuilder:scaffold:crdkustomizeresource
 
 patchesStrategicMerge:

config/daemonconfig/config/kustomization.yaml

@@ -0,0 +1,3 @@
+resources:
+- namespace.yaml
+- rbac.yaml

config/daemonconfig/config/namespace.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  name: kruise-daemon-config

config/daemonconfig/config/rbac.yaml

@@ -0,0 +1,29 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: Role
+metadata:
+  creationTimestamp: null
+  name: kruise-daemon-secret-role
+  namespace: kruise-daemon-config
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+    verbs:
+      - get
+      - list
+      - watch
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: kruise-daemon-secret-rolebinding
+  namespace: kruise-daemon-config
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: Role
+  name: kruise-daemon-secret-role
+subjects:
+  - kind: ServiceAccount
+    name: kruise-daemon
+    namespace: kruise-system

config/daemonconfig/kustomization.yaml

@@ -0,0 +1,8 @@
+namespace: kruise-daemon-config
+# Value of this field is prepended to the
+# names of all resources, e.g. a deployment named
+# "wordpress" becomes "alices-wordpress".
+# Note that it should also match with the prefix (text before '-') of the namespace
+# field above.
+bases:
+  - config

config/default/kruise-daemon-config.yaml

@@ -0,0 +1,6 @@
+apiVersion: v1
+kind: Namespace
+metadata:
+  labels:
+    control-plane: controller-manager
+  name: kruise-daemon-config

config/default/kustomization.yaml

@@ -1,6 +1,3 @@
-# Adds namespace to all resources.
-namespace: kruise-system
-
 # Value of this field is prepended to the
 # names of all resources, e.g. a deployment named
 # "wordpress" becomes "alices-wordpress".
@@ -12,16 +9,19 @@ namePrefix: kruise-
 #commonLabels:
 #  someName: someValue
 
+resources:
+  - kruise-daemon-config.yaml
+
 bases:
 - ../crd
 - ../rbac
 - ../manager
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 - ../webhook
 # [CERTMANAGER] To enable cert-manager, uncomment all sections with 'CERTMANAGER'. 'WEBHOOK' components are required.
 #- ../certmanager
-# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'. 
+# [PROMETHEUS] To enable prometheus monitor, uncomment all sections with 'PROMETHEUS'.
 #- ../prometheus
 
 patchesStrategicMerge:
@@ -30,7 +30,7 @@ patchesStrategicMerge:
   # endpoint w/o any authn/z, please comment the following line.
 # - manager_auth_proxy_patch.yaml
 
-# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in 
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix including the one in
 # crd/kustomization.yaml
 - manager_webhook_patch.yaml
 

config/manager/kustomization.yaml

@@ -1,2 +1,5 @@
+# Adds namespace to all resources.
+namespace: kruise-system
+
 resources:
 - manager.yaml

config/manager/manager.yaml

@@ -35,9 +35,17 @@ spec:
         - --enable-leader-election
         - --logtostderr=true
         - --v=5
-        - --feature-gates=AllAlpha=true
+        - --feature-gates=AllAlpha=true,AllBeta=true,EnableExternalCerts=false
         image: controller:latest
         imagePullPolicy: Always
+        securityContext:
+          capabilities:
+            drop:
+              - all
+            add: [ 'NET_BIND_SERVICE' ]
+          allowPrivilegeEscalation: false
+          runAsNonRoot: true
+          runAsUser: 65534
         name: manager
         env:
           - name: KUBE_CACHE_MUTATION_DETECTOR
@@ -52,8 +60,8 @@ spec:
             port: 8000
         resources:
           limits:
-            cpu: 100m
+            cpu: 2
-            memory: 200Mi
+            memory: 2Gi
           requests:
             cpu: 100m
             memory: 200Mi
@@ -94,9 +102,16 @@ spec:
         args:
         - --logtostderr=true
         - -v=5
-        - --feature-gates=AllAlpha=true
+        - --feature-gates=AllAlpha=true,AllBeta=true
+        - --max-workers-for-pull-image=2
         image: controller:latest
         imagePullPolicy: Always
+        securityContext:
+          capabilities:
+            drop:
+              - all
+            add: [ 'NET_BIND_SERVICE' ]
+          allowPrivilegeEscalation: false
         name: daemon
         env:
         - name: KUBE_CACHE_MUTATION_DETECTOR

config/rbac/daemon_role.yaml

@@ -53,8 +53,6 @@ rules:
   verbs:
   - get
   - list
-  - patch
-  - update
   - watch
 - apiGroups:
   - ""
@@ -64,14 +62,6 @@ rules:
   - get
   - patch
   - update
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - get
-  - list
-  - watch
 - apiGroups:
   - apps.kruise.io
   resources:

config/rbac/kustomization.yaml

@@ -1,3 +1,6 @@
+# Adds namespace to all resources.
+namespace: kruise-system
+
 resources:
 - role.yaml
 - role_binding.yaml

config/rbac/role.yaml

@@ -1,30 +1,43 @@
-
 ---
 apiVersion: rbac.authorization.k8s.io/v1
 kind: ClusterRole
 metadata:
-  creationTimestamp: null
   name: manager-role
 rules:
 - apiGroups:
   - ""
   resources:
   - configmaps
+  - events
+  - persistentvolumeclaims
+  - pods
   verbs:
   - create
   - delete
   - get
   - list
+  - patch
   - update
   - watch
 - apiGroups:
   - ""
   resources:
   - namespaces
+  - nodes
   verbs:
   - get
   - list
   - watch
+- apiGroups:
+  - ""
+  resources:
+  - pods/ephemeralcontainers
+  - pods/status
+  - pods/resize
+  verbs:
+  - get
+  - patch
+  - update
 - apiGroups:
   - ""
   resources:
@@ -54,15 +67,6 @@ rules:
   - admissionregistration.k8s.io
   resources:
   - mutatingwebhookconfigurations
-  verbs:
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - admissionregistration.k8s.io
-  resources:
   - validatingwebhookconfigurations
   verbs:
   - get
@@ -84,18 +88,8 @@ rules:
   - apps
   resources:
   - controllerrevisions
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps
-  resources:
   - deployments
+  - statefulsets
   verbs:
   - create
   - delete
@@ -108,6 +102,7 @@ rules:
   - apps
   resources:
   - deployments/status
+  - statefulsets/status
   verbs:
   - get
   - patch
@@ -126,30 +121,23 @@ rules:
   - replicasets/status
   verbs:
   - get
-- apiGroups:
-  - apps
-  resources:
-  - statefulsets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps
-  resources:
-  - statefulsets/status
-  verbs:
-  - get
-  - patch
-  - update
 - apiGroups:
   - apps.kruise.io
   resources:
   - advancedcronjobs
+  - broadcastjobs
+  - clonesets
+  - containerrecreaterequests
+  - daemonsets
+  - imagelistpulljobs
+  - imagepulljobs
+  - nodeimages
+  - nodepodprobes
+  - persistentpodstates
+  - podprobemarkers
+  - sidecarsets
+  - statefulsets
+  - uniteddeployments
   verbs:
   - create
   - delete
@@ -158,90 +146,48 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - apps.kruise.io
+  resources:
+  - advancedcronjobs/finalizers
+  - broadcastjobs/finalizers
+  - clonesets/finalizers
+  - containerrecreaterequests/finalizers
+  - daemonsets/finalizers
+  - imagelistpulljobs/finalizers
+  - imagepulljobs/finalizers
+  - nodeimages/finalizers
+  - nodepodprobes/finalizers
+  - persistentpodstates/finalizers
+  - podprobemarkers/finalizers
+  - resourcedistributions/finalizers
+  - sidecarsets/finalizers
+  - statefulsets/finalizers
+  - uniteddeployments/finalizers
+  - workloadspreads/finalizers
+  verbs:
+  - update
 - apiGroups:
   - apps.kruise.io
   resources:
   - advancedcronjobs/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - broadcastjobs
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
   - broadcastjobs/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - clonesets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
   - clonesets/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - containerrecreaterequests
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
   - containerrecreaterequests/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - daemonsets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
   - daemonsets/status
+  - ephemeraljobs/finalizers
+  - ephemeraljobs/status
+  - imagelistpulljobs/status
+  - imagepulljobs/status
+  - nodeimages/status
+  - nodepodprobes/status
+  - persistentpodstates/status
+  - podprobemarkers/status
+  - resourcedistributions/status
+  - sidecarsets/status
+  - statefulsets/status
+  - uniteddeployments/status
+  - workloadspreads/status
   verbs:
   - get
   - patch
@@ -257,114 +203,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - ephemeraljobs/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - imagepulljobs
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - imagepulljobs/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - nodeimages
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - nodeimages/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - nodepodprobes
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - nodepodprobes/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - persistentpodstates
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - persistentpodstates/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - podprobemarkers
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - podprobemarkers/status
-  verbs:
-  - get
-  - patch
-  - update
 - apiGroups:
   - apps.kruise.io
   resources:
@@ -373,74 +211,6 @@ rules:
   - get
   - list
   - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - resourcedistributions/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - sidecarsets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - sidecarsets/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - statefulsets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - statefulsets/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - uniteddeployments
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - uniteddeployments/status
-  verbs:
-  - get
-  - patch
-  - update
 - apiGroups:
   - apps.kruise.io
   resources:
@@ -451,14 +221,6 @@ rules:
   - patch
   - update
   - watch
-- apiGroups:
-  - apps.kruise.io
-  resources:
-  - workloadspreads/status
-  verbs:
-  - get
-  - patch
-  - update
 - apiGroups:
   - batch
   resources:
@@ -479,90 +241,6 @@ rules:
   - get
   - patch
   - update
-- apiGroups:
-  - ""
-  resources:
-  - configmaps
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - events
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - nodes
-  verbs:
-  - get
-  - list
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - persistentvolumeclaims
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
-- apiGroups:
-  - ""
-  resources:
-  - pods/ephemeralcontainers
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - pods/status
-  verbs:
-  - get
-  - patch
-  - update
-- apiGroups:
-  - ""
-  resources:
-  - secrets
-  verbs:
-  - create
-  - delete
-  - get
-  - list
-  - patch
-  - update
-  - watch
 - apiGroups:
   - policy.kruise.io
   resources:
@@ -575,6 +253,12 @@ rules:
   - patch
   - update
   - watch
+- apiGroups:
+  - policy.kruise.io
+  resources:
+  - podunavailablebudgets/finalizers
+  verbs:
+  - update
 - apiGroups:
   - policy.kruise.io
   resources:
@@ -583,3 +267,11 @@ rules:
   - get
   - patch
   - update
+- apiGroups:
+  - storage.k8s.io
+  resources:
+  - storageclasses
+  verbs:
+  - get
+  - list
+  - watch

config/webhook/kustomization.yaml

@@ -1,3 +1,6 @@
+# Adds namespace to all resources.
+namespace: kruise-system
+
 resources:
 - manifests.yaml
 - service.yaml

config/webhook/manifests.yaml

@@ -1,9 +1,7 @@
-
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
 metadata:
-  creationTimestamp: null
   name: mutating-webhook-configuration
 webhooks:
 - admissionReviewVersions:
@@ -131,6 +129,27 @@ webhooks:
     resources:
     - daemonsets
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /mutate-apps-kruise-io-v1alpha1-imagelistpulljob
+  failurePolicy: Fail
+  name: mimagelistpulljob.kb.io
+  rules:
+  - apiGroups:
+    - apps.kruise.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - imagelistpulljobs
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   - v1beta1
@@ -237,12 +256,10 @@ webhooks:
     resources:
     - uniteddeployments
   sideEffects: None
-
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: ValidatingWebhookConfiguration
 metadata:
-  creationTimestamp: null
   name: validating-webhook-configuration
 webhooks:
 - admissionReviewVersions:
@@ -453,6 +470,48 @@ webhooks:
     resources:
     - daemonsets
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-apps-kruise-io-v1alpha1-ephemeraljob
+  failurePolicy: Fail
+  name: vephemeraljobs.kb.io
+  rules:
+  - apiGroups:
+    - apps.kruise.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - ephemeraljobs
+  sideEffects: None
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-apps-kruise-io-v1alpha1-imagelistpulljob
+  failurePolicy: Fail
+  name: vimagelistpulljob.kb.io
+  rules:
+  - apiGroups:
+    - apps.kruise.io
+    apiVersions:
+    - v1alpha1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - imagelistpulljobs
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   - v1beta1
@@ -474,6 +533,27 @@ webhooks:
     resources:
     - imagepulljobs
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-ingress
+  failurePolicy: Fail
+  name: vingress.kb.io
+  rules:
+  - apiGroups:
+    - networking.k8s.io
+    apiVersions:
+    - v1
+    - v1beta1
+    operations:
+    - DELETE
+    resources:
+    - ingresses
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   - v1beta1
@@ -619,6 +699,26 @@ webhooks:
     resources:
     - podunavailablebudgets
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: webhook-service
+      namespace: system
+      path: /validate-service
+  failurePolicy: Fail
+  name: vservice.kb.io
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - DELETE
+    resources:
+    - services
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   - v1beta1

config/webhook/patch_manifests.yaml

@@ -39,6 +39,16 @@ webhooks:
     matchExpressions:
     - key: policy.kruise.io/delete-protection
       operator: Exists
+- name: vservice.kb.io
+  objectSelector:
+    matchExpressions:
+      - key: policy.kruise.io/delete-protection
+        operator: Exists
+- name: vingress.kb.io
+  objectSelector:
+    matchExpressions:
+      - key: policy.kruise.io/delete-protection
+        operator: Exists
 - name: vpod.kb.io
   namespaceSelector:
     matchExpressions:

docs/README.md

@@ -5,11 +5,11 @@ like StatefulSet, Deployment, DaemonSet for instances. While at the same time, m
 express more and more diverse requirements for workload upgrade and deployment, which
 in many cases, cannot be satisfied by the default workload controllers.
 
-Kruise attempts to fill such gap by offering a set of controllers as the supplement
+Kruise attempts to fill such a gap by offering a set of controllers as the supplement
 to manage new workloads in Kubernetes. The target use cases are representative,
 originally collected from the users of Alibaba cloud container services and the
 developers of the in-house large scale on-line/off-line container applications.
-Most of the use cases can be easily applied to other similar cloud user scenarios.
+Most of the use cases can be easily applied to other similar scenarios for cloud users.
 
 Currently, Kruise supports the following workloads.
 

docs/proposals/20210713-sidecarsetimagepullsecrects.md

@@ -36,7 +36,7 @@ One of the most common-used features of it is to pull images from private reposi
 
 ## Proposal
 **Main idea**: In this design, we separate the logic of `Secret` and `SidecarSet`.
-In `SidecarSet` part, we only consider injecting their `imagePullSecrets` feilds into Pod.
+In `SidecarSet` part, we only consider injecting their `imagePullSecrets` fields into Pod.
 Users should manually  distribute the required `Secrets` to all the namespaces that the `SidecarSet` may be instantiated.
 
 ### API Definition