test(lifecycle): add unit tests for lifecycle utils and improve nil-checks (#2143)

Signed-off-by: kincoy <1152072645@qq.com>

.circleci/config.yml

@@ -1,61 +0,0 @@
-# Golang CircleCI 2.0 configuration file
-#
-# Check https://circleci.com/docs/2.0/language-go/ for more details
-
-version: 2
-jobs:
-  markdownlint-misspell-shellcheck:
-    docker:
-      # this image is build from Dockerfile 
-      # https://github.com/pouchcontainer/pouchlinter/blob/master/Dockerfile
-      - image: pouchcontainer/pouchlinter:v0.1.2
-    working_directory: /go/src/github.com/openkruise/kruise
-    steps:
-      - checkout
-      - run:
-          name: use markdownlint v0.5.0 to lint markdown file (https://github.com/markdownlint/markdownlint)
-          command: |
-            find  ./ -name  "*.md" | grep -v vendor | grep -v commandline |  grep -v .github |  grep -v swagger |  grep -v api |  xargs mdl -r ~MD010,~MD013,~MD024,~MD029,~MD033,~MD036
-      - run:
-          name: use markdown-link-check(https://github.com/tcort/markdown-link-check) to check links in markdown files
-          command: |
-            set +e
-            for name in $(find . -name \*.md | grep -v vendor | grep -v CHANGELOG); do 
-              if [ -f $name ]; then 
-                markdown-link-check -q $name -c .circleci/markdown-link-check.config.json;
-                if [ $? -ne 0 ]; then
-                  code=1
-                fi
-              fi 
-            done 
-            bash -c "exit $code";
-      - run:
-          name: use opensource tool client9/misspell to correct commonly misspelled English words 
-          command: |
-            find  ./* -name  "*"  | grep -v vendor | xargs misspell -error
-      - run:
-          name: use ShellCheck (https://github.com/koalaman/shellcheck) to check the validateness of shell scripts in pouch repo
-          command: |
-            find ./ -name "*.sh" | grep -v vendor | xargs shellcheck
-  code-check:
-    docker: 
-      - image: golangci/golangci-lint:v1.21
-    working_directory: /go/src/github.com/openkruise/kruise
-    steps:
-      - checkout
-      - run: 
-          name: validate go code with golangci-lint
-          command: |
-            golangci-lint run --skip-dirs=apis,pkg/client --disable-all -E gofmt -E goimports -E golint -E ineffassign -E misspell -E vet
-      
-      - run:
-          name: detect deadcode without test folder
-          command: |
-            golangci-lint run --disable-all --skip-dirs=vendor,test,pkg/client -E deadcode
-
-workflows:
-  version: 2
-  ci:
-    jobs:
-      - markdownlint-misspell-shellcheck
-      - code-check

.circleci/markdown-link-check.config.json

@@ -1,7 +0,0 @@
-{
-  "ignorePatterns": [
-    {
-      "pattern": "^https://codecov.io"
-    }
-  ]
-}

.codecov.yml

@@ -0,0 +1,6 @@
+ignore:
+- "pkg/client/.*"
+- "test/fuzz/.*"
+- "test/e2e/.*"
+
+

.github/ISSUE_TEMPLATE/bug-report.md

@@ -1,7 +1,7 @@
 ---
 name: Bug Report
 about: Create a report to help us improve
-title: "[BUG]"
+title: "[BUG] YOUR_TITLE"
 labels: kind/bug
 assignees: FillZpp
 
@@ -20,7 +20,5 @@ assignees: FillZpp
 **Environment**:
 - Kruise version:
 - Kubernetes version (use `kubectl version`):
-- OS (e.g: `cat /etc/os-release`):
-- Kernel (e.g. `uname -a`):
-- Install tools:
+- Install details (e.g. helm install args):
 - Others:

.github/ISSUE_TEMPLATE/feature-request.md

@@ -1,9 +1,9 @@
 ---
 name: Feature Request
 about: Suggest an idea for this project
-title: "[feature request]"
+title: "[feature request] YOUR_TITLE"
 labels: kind/feature-request
-assignees: jian-he
+assignees: FillZpp
 
 ---
 

.github/PULL_REQUEST_TEMPLATE.md

@@ -8,13 +8,8 @@ https://github.com/openkruise/kruise/blob/master/CONTRIBUTING.md -->
 ### Ⅱ. Does this pull request fix one issue?
 <!--If so, add "fixes #xxxx" below in the next line, for example, fixes #15. Otherwise, add "NONE" -->
 
-
-### Ⅲ. List the added test cases (unit test/integration test) if any, please explain if no tests are needed.
+### Ⅲ. Describe how to verify it
 
 
-### Ⅳ. Describe how to verify it
-
-
-### Ⅴ. Special notes for reviews
-
+### Ⅳ. Special notes for reviews
 

.github/dependabot.yaml

@@ -0,0 +1,17 @@
+# This YAML configuration file is used to enable Dependabot for automated dependency management.
+# Dependabot helps keep the project's dependencies up-to-date by automatically creating pull requests
+# for outdated dependencies based on the version constraints defined in your project.
+# For more information and customization options, please refer to the Dependabot documentation:
+# Documentation: https://docs.github.com/en/code-security/supply-chain-security/keeping-your-dependencies-updated-automatically
+# Configuration options: https://docs.github.com/en/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file
+version: 2
+updates:
+- package-ecosystem: "github-actions"
+  directory: "/"
+  # Allow up to 10 open pull requests for update github-actions
+  # 5 by default
+  # see https://docs.github.com/code-security/dependabot/dependabot-version-updates/configuration-options-for-the-dependabot.yml-file#open-pull-requests-limit
+  open-pull-requests-limit: 10
+  schedule:
+    # Check for updates to GitHub Actions every week
+    interval: "weekly"

.github/stale.yml

@@ -0,0 +1,61 @@
+# Configuration for probot-stale - https://github.com/probot/stale
+
+# Number of days of inactivity before an Issue or Pull Request becomes stale
+daysUntilStale: 90
+
+# Number of days of inactivity before an Issue or Pull Request with the stale label is closed.
+# Set to false to disable. If disabled, issues still need to be closed manually, but will remain marked as stale.
+daysUntilClose: 7
+
+# Only issues or pull requests with all of these labels are check if stale. Defaults to `[]` (disabled)
+onlyLabels: []
+
+# Issues or Pull Requests with these labels will never be considered stale. Set to `[]` to disable
+exemptLabels:
+  - pinned
+  - security
+  - "[Status] Maybe Later"
+
+# Set to true to ignore issues in a project (defaults to false)
+exemptProjects: true
+
+# Set to true to ignore issues in a milestone (defaults to false)
+exemptMilestones: false
+
+# Set to true to ignore issues with an assignee (defaults to false)
+exemptAssignees: false
+
+# Label to use when marking as stale
+staleLabel: wontfix
+
+# Comment to post when marking as stale. Set to `false` to disable
+markComment: >
+  This issue has been automatically marked as stale because it has not had
+  recent activity. It will be closed if no further activity occurs. Thank you
+  for your contributions.
+
+# Comment to post when removing the stale label.
+# unmarkComment: >
+#   Your comment here.
+
+# Comment to post when closing a stale Issue or Pull Request.
+# closeComment: >
+#   Your comment here.
+
+# Limit the number of actions per hour, from 1-30. Default is 30
+limitPerRun: 3
+
+# Limit to only `issues` or `pulls`
+# only: issues
+
+# Optionally, specify configuration settings that are specific to just 'issues' or 'pulls':
+# pulls:
+#   daysUntilStale: 30
+#   markComment: >
+#     This pull request has been automatically marked as stale because it has not had
+#     recent activity. It will be closed if no further activity occurs. Thank you
+#     for your contributions.
+
+# issues:
+#   exemptLabels:
+#     - confirmed

.github/workflows/ci.yaml

@@ -0,0 +1,168 @@
+name: CI
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: { }
+  workflow_dispatch: { }
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  # Common versions
+  GO_VERSION: '1.23'
+  GOLANGCI_VERSION: 'v2.1'
+  DOCKER_BUILDX_VERSION: 'v0.4.2'
+
+  # Common users. We can't run a step 'if secrets.AWS_USR != ""' but we can run
+  # a step 'if env.AWS_USR' != ""', so we copy these to succinctly test whether
+  # credentials have been provided before trying to run steps that need them.
+  DOCKER_USR: ${{ secrets.DOCKER_USR }}
+  AWS_USR: ${{ secrets.AWS_USR }}
+
+jobs:
+  typos-check:
+    name: Spell Check with Typos
+    runs-on: ubuntu-24.04
+    steps:
+      - name: Checkout Actions Repository
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Check spelling with custom config file
+        uses: crate-ci/typos@a67079b4ae32e18c3f53d75368c52ce53b5fb56b # v1.35.4
+        with:
+          config: ./typos.toml
+
+  golangci-lint:
+    runs-on: ubuntu-24.04
+    permissions:
+      security-events: write
+    steps:
+      - name: Checkout Code
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Cache Go Dependencies
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-
+      - name: Code generate
+        run: |
+          make generate
+      - name: Lint golang code
+        uses: golangci/golangci-lint-action@4afd733a84b1f43292c63897423277bb7f4313a9 # v8.0.0
+        with:
+          version: ${{ env.GOLANGCI_VERSION }}
+          args: --verbose
+          skip-pkg-cache: true
+          mod: readonly
+      - name: Run Trivy vulnerability scanner in repo mode
+        uses: aquasecurity/trivy-action@77137e9dc3ab1b329b7c8a38c2eb7475850a14e8 # master
+        with:
+          scan-type: 'fs'
+          ignore-unfixed: true
+          format: 'sarif'
+          output: 'trivy-results.sarif'
+          severity: 'CRITICAL'
+      - name: Upload Trivy scan results to GitHub Security tab
+        uses: github/codeql-action/upload-sarif@df559355d593797519d70b90fc8edd5db049e7a2 # v3.29.9
+        with:
+          sarif_file: 'trivy-results.sarif'
+
+#  markdownlint-misspell-shellcheck:
+#    runs-on: ubuntu-24.04
+#    # this image is build from Dockerfile
+#    # https://github.com/pouchcontainer/pouchlinter/blob/master/Dockerfile
+#    container: pouchcontainer/pouchlinter:v0.1.2
+#    steps:
+#      - name: Checkout
+#        uses: actions/checkout@v3
+#      - name: Run misspell
+#        run: find  ./* -name  "*"  | grep -v vendor | xargs misspell -error
+#      - name: Run shellcheck
+#        run: find ./ -name "*.sh" | grep -v vendor | xargs shellcheck
+#      - name: Lint markdown files
+#        run: find  ./ -name  "*.md" | grep -v vendor | grep -v commandline |  grep -v .github |  grep -v swagger |  grep -v api |  xargs mdl -r ~MD010,~MD013,~MD014,~MD022,~MD024,~MD029,~MD031,~MD032,~MD033,~MD036
+#      - name: Check markdown links
+#        run: |
+#          set +e
+#          for name in $(find . -name \*.md | grep -v vendor | grep -v CHANGELOG); do
+#            if [ -f $name ]; then
+#              markdown-link-check -q $name -c .github/workflows/markdown-link-check.config.json;
+#              if [ $? -ne 0 ]; then
+#                code=1
+#              fi
+#            fi
+#          done
+#          bash -c "exit $code";
+
+  unit-tests:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Fetch History
+        run: git fetch --prune --unshallow
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Cache Go Dependencies
+        uses: actions/cache@0400d5f644dc74513175e3cd8d07132dd4860809 # v4.2.4
+        with:
+          path: ~/go/pkg/mod
+          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+          restore-keys: ${{ runner.os }}-go-
+      - name: Run Unit Tests
+        run: |
+          make test
+          git status
+      - name: Publish Unit Test Coverage
+        uses: codecov/codecov-action@fdcc8476540edceab3de004e990f80d881c6cc00 # v5.5.0
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+        with:
+          token: ${{ secrets.CODECOV_TOKEN }}
+          flags: unittests
+          file: cover.out
+  # See: https://google.github.io/oss-fuzz/getting-started/continuous-integration/
+  Fuzzing:
+    runs-on: ubuntu-latest
+    permissions:
+      security-events: write
+    steps:
+      - name: Build Fuzzers
+        id: build
+        uses: google/oss-fuzz/infra/cifuzz/actions/build_fuzzers@abe2c06d0e162320403dd10e8268adbb0b8923f8 # master
+        with:
+          oss-fuzz-project-name: 'openkruise'
+          language: go
+      - name: Run Fuzzers
+        uses: google/oss-fuzz/infra/cifuzz/actions/run_fuzzers@abe2c06d0e162320403dd10e8268adbb0b8923f8 # master
+        with:
+          oss-fuzz-project-name: 'openkruise'
+          language: go
+          fuzz-seconds: 1200
+          output-sarif: true
+      - name: Upload Crash
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        if: failure() && steps.build.outcome == 'success'
+        with:
+          name: artifacts
+          path: ./out/artifacts
+      - name: Upload Sarif
+        if: always() && steps.build.outcome == 'success'
+        uses: github/codeql-action/upload-sarif@df559355d593797519d70b90fc8edd5db049e7a2 # v3.29.9
+        with:
+          # Path to SARIF file relative to the root of the repository
+          sarif_file: cifuzz-sarif/results.sarif
+          checkout_path: cifuzz-sarif

.github/workflows/codeql.yml

@@ -0,0 +1,84 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "CodeQL"
+
+on:
+  push:
+    branches: [ "master", "release-*"]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "master" ]
+
+
+permissions: 
+  contents: read
+
+jobs:
+  analyze:
+    name: Analyze
+    # Runner size impacts CodeQL analysis time. To learn more, please see:
+    #   - https://gh.io/recommended-hardware-resources-for-running-codeql
+    #   - https://gh.io/supported-runners-and-hardware-resources
+    #   - https://gh.io/using-larger-runners
+    # Consider using larger runners for possible analysis time improvements.
+    runs-on: ${{ (matrix.language == 'swift' && 'macos-latest') || 'ubuntu-latest' }}
+    timeout-minutes: ${{ (matrix.language == 'swift' && 120) || 360 }}
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'c-cpp', 'csharp', 'go', 'java-kotlin', 'javascript-typescript', 'python', 'ruby', 'swift' ]
+        # Use only 'java-kotlin' to analyze code written in Java, Kotlin or both
+        # Use only 'javascript-typescript' to analyze code written in JavaScript, TypeScript or both
+        # Learn more about CodeQL language support at https://aka.ms/codeql-docs/language-support
+
+    steps:
+    - name: Checkout repository
+      uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+
+    # Initializes the CodeQL tools for scanning.
+    - name: Initialize CodeQL
+      uses: github/codeql-action/init@df559355d593797519d70b90fc8edd5db049e7a2 # v3.29.9
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+
+        # For more details on CodeQL's query packs, refer to: https://docs.github.com/en/code-security/code-scanning/automatically-scanning-your-code-for-vulnerabilities-and-errors/configuring-code-scanning#using-queries-in-ql-packs
+        # queries: security-extended,security-and-quality
+
+
+    # Autobuild attempts to build any compiled languages (C/C++, C#, Go, Java, or Swift).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: Autobuild
+      uses: github/codeql-action/autobuild@df559355d593797519d70b90fc8edd5db049e7a2 # v3.29.9
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 See https://docs.github.com/en/actions/using-workflows/workflow-syntax-for-github-actions#jobsjob_idstepsrun
+
+    #   If the Autobuild fails above, remove it and uncomment the following three lines.
+    #   modify them (or add more) to build your code if your project, please refer to the EXAMPLE below for guidance.
+
+    # - run: |
+    #     echo "Run, Build Application using script"
+    #     ./location_of_script_within_repo/buildscript.sh
+
+    - name: Perform CodeQL Analysis
+      uses: github/codeql-action/analyze@df559355d593797519d70b90fc8edd5db049e7a2 # v3.29.9
+      with:
+        category: "/language:${{matrix.language}}"

.github/workflows/docker-image.yaml

@@ -0,0 +1,28 @@
+name: Docker Image CI
+
+on:
+  workflow_dispatch:
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+
+  build:
+
+    runs-on: ubuntu-latest
+
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Set up Docker Buildx
+        uses: docker/setup-buildx-action@v3
+      - name: Login to Docker Hub
+        uses: docker/login-action@v3
+        with:
+          username: ${{ vars.DOCKERHUB_USERNAME }}
+          password: ${{ secrets.HUB_KRIUSE }}
+      - name: Build the Docker image
+        run: |
+          docker buildx create --use --platform=linux/amd64,linux/arm64,linux/ppc64le --name multi-platform-builder
+          docker buildx ls
+          IMG=openkruise/kruise-manager:${{ github.ref_name }} make docker-multiarch

.github/workflows/e2e-1.24.yaml

@@ -0,0 +1,298 @@
+name: E2E-1.24
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  # Common versions
+  GO_VERSION: '1.23'
+  KIND_ACTION_VERSION: 'v1.3.0'
+  KIND_VERSION: 'v0.14.0'
+  KIND_IMAGE: 'kindest/node:v1.24.6'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+  astatefulset-storage:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'VolumeExpansion' --print-info
+  astatefulset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'StatefulSet' --print-info
+
+  cloneset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(CloneSet|ContainerMeta)' --print-info
+
+  operation:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(PullImage|ContainerRecreateRequest|PullImages|ResourceDistribution|PersistentPodState|PodProbeMarker)' --print-info
+
+  advanced-daemonset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'DaemonSet' --print-info
+
+  sidecar:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'SidecarSet|SidecarTerminator|ContainerPriority' --print-info
+
+  job-workload:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(EphemeralJob|BroadcastJob)' --print-info
+
+
+  policy:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(PodUnavailableBudget|DeletionProtection)' --print-info
+  multidomain:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          ENABLE_E2E_CONFIG=true IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(UnitedDeployment|WorkloadSpread)' --print-info

.github/workflows/e2e-1.26.yaml

@@ -0,0 +1,295 @@
+name: E2E-1.26
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  # Common versions
+  GO_VERSION: '1.23'
+  KIND_VERSION: 'v0.18.0'
+  KIND_IMAGE: 'kindest/node:v1.26.3'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+  astatefulset-storage:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+         make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'VolumeExpansion' --print-info
+  astatefulset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'StatefulSet' --print-info
+  cloneset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(CloneSet|ContainerMeta)' --print-info
+
+  operation:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(PullImage|ContainerRecreateRequest|PullImages|ResourceDistribution|PersistentPodState|PodProbeMarker)' --print-info
+
+  advanced-daemonset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'DaemonSet' --print-info
+
+  sidecar:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'SidecarSet|SidecarTerminator|ContainerPriority' --print-info
+
+  job-workload:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(EphemeralJob|BroadcastJob)' --print-info
+
+  policy:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(PodUnavailableBudget|DeletionProtection)' --print-info
+  multidomain:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          ENABLE_E2E_CONFIG=true IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(UnitedDeployment|WorkloadSpread)' --print-info

.github/workflows/e2e-1.28.yaml

@@ -0,0 +1,298 @@
+name: E2E-1.28
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  # Common versions
+  GO_VERSION: '1.23'
+  KIND_VERSION: 'v0.22.0'
+  KIND_IMAGE: 'kindest/node:v1.28.7'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+  astatefulset-storage:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'VolumeExpansion' --print-info
+
+  astatefulset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'StatefulSet' --print-info
+
+  operation:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(PullImage|ContainerRecreateRequest|PullImages|ResourceDistribution|PersistentPodState|PodProbeMarker)' --print-info
+
+  advanced-daemonset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'DaemonSet' --print-info
+
+  sidecar:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+         IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'SidecarSet|SidecarTerminator|ContainerPriority' --print-info
+
+  job-workload:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(EphemeralJob|BroadcastJob)' --print-info
+
+  policy:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(PodUnavailableBudget|DeletionProtection)' --print-info
+
+  cloneset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          ENABLE_E2E_CONFIG=true IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(CloneSet|ContainerMeta|InplaceVPA)' --print-info
+
+  multidomain:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          ENABLE_E2E_CONFIG=true IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(UnitedDeployment|WorkloadSpread)' --print-info

.github/workflows/e2e-1.30.yaml

@@ -0,0 +1,298 @@
+name: E2E-1.30
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  # Common versions
+  GO_VERSION: '1.23'
+  KIND_VERSION: 'v0.22.0'
+  KIND_IMAGE: 'kindest/node:v1.30.8'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+  astatefulset-storage:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'VolumeExpansion' --print-info
+
+  astatefulset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'StatefulSet' --print-info
+
+  operation:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(PullImage|ContainerRecreateRequest|PullImages|ResourceDistribution|PersistentPodState|PodProbeMarker)' --print-info
+
+  advanced-daemonset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'DaemonSet' --print-info
+
+  sidecar:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'SidecarSet|SidecarTerminator|ContainerPriority' --print-info
+
+  job-workload:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(EphemeralJob|BroadcastJob)' --print-info
+
+  policy:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(PodUnavailableBudget|DeletionProtection)' --print-info
+
+  cloneset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(CloneSet|ContainerMeta|InplaceVPA)' --print-info
+
+  multidomain:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          ENABLE_E2E_CONFIG=true IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(UnitedDeployment|WorkloadSpread)' --print-info

.github/workflows/e2e-1.32.yaml

@@ -0,0 +1,298 @@
+name: E2E-1.32
+
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  pull_request: {}
+  workflow_dispatch: {}
+
+# Declare default permissions as read only.
+permissions: read-all
+
+env:
+  # Common versions
+  GO_VERSION: '1.23'
+  KIND_VERSION: 'v0.22.0'
+  KIND_IMAGE: 'kindest/node:v1.32.0'
+  KIND_CLUSTER_NAME: 'ci-testing'
+
+jobs:
+  astatefulset-storage:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'VolumeExpansion' --print-info
+
+  astatefulset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'StatefulSet' --print-info
+
+  operation:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(PullImage|ContainerRecreateRequest|PullImages|ResourceDistribution|PersistentPodState|PodProbeMarker)' --print-info
+
+  advanced-daemonset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'DaemonSet' --print-info
+
+  sidecar:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus 'SidecarSet|SidecarTerminator|ContainerPriority' --print-info
+
+  job-workload:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(EphemeralJob|BroadcastJob)' --print-info
+
+  policy:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(PodUnavailableBudget|DeletionProtection)' --print-info
+
+  cloneset:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Install-CSI
+        run: |
+          make install-csi
+
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(CloneSet|ContainerMeta|InplaceVPA)' --print-info
+
+  multidomain:
+    runs-on: ubuntu-24.04
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          submodules: true
+      - name: Setup Go
+        uses: actions/setup-go@0aaccfd150d50ccaeb58ebd88d36e91967a5f35b # v5.4.0
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Setup Kind Cluster
+        uses: helm/kind-action@a1b0e391336a6ee6713a0583f8c6240d70863de3 # v1.12.0
+        with:
+          node_image: ${{ env.KIND_IMAGE }}
+          cluster_name: ${{ env.KIND_CLUSTER_NAME }}
+          config: ./test/kind-conf-with-vpa.yaml
+          version: ${{ env.KIND_VERSION }}
+      - name: Build image
+        run: |
+          export IMAGE="openkruise/kruise-manager:e2e-${GITHUB_RUN_ID}"
+          docker build --pull --no-cache . -t $IMAGE
+          kind load docker-image --name=${KIND_CLUSTER_NAME} $IMAGE || { echo >&2 "kind not installed or error loading image: $IMAGE"; exit 1; }
+      - name: Install Kruise
+        run: |
+          ENABLE_E2E_CONFIG=true IMG=openkruise/kruise-manager:e2e-${GITHUB_RUN_ID} make install-kruise
+      - name: Run E2E Tests
+        run: |
+          export KUBECONFIG=/home/runner/.kube/config
+          tools/hack/run-kruise-e2e-test.sh --focus '(UnitedDeployment|WorkloadSpread)' --print-info

.github/workflows/e2e.yml

@@ -1,24 +0,0 @@
-name: "Create cluster using KinD"
-on: push
-
-jobs:
-  kind:
-    runs-on: ubuntu-latest
-    steps:
-    - uses: actions/checkout@master
-    - uses: engineerd/setup-kind@v0.1.0
-    - name: setup-k8s and install kruise
-      run: |
-        export KUBECONFIG="$(kind get kubeconfig-path)"
-        echo $KUBECONFIG
-        kubectl cluster-info
-        kubectl get pods -n kube-system
-        kubectl apply -f ./config/crds/
-        kubectl apply -f ./config/manager/all_in_one.yaml
-    - name: e2e-test
-      run: |
-        APP_PATH=/home/runner/go/src/github.com/openkruise/kruise
-        mkdir -p $APP_PATH
-        cp -r ./ $APP_PATH
-        cd $APP_PATH
-        go test ./test/e2e/ -timeout 60m -v -args "-kubeconfig=/home/runner/.kube/kind-config-kind"

.github/workflows/license.yml

@@ -0,0 +1,29 @@
+name: License
+on:
+  push:
+    branches:
+      - master
+      - release-*
+  workflow_dispatch: {}
+  pull_request:
+    branches:
+      - master
+      - release-*
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  license_check:
+    runs-on: ubuntu-24.04
+    name: Check for unapproved licenses
+    steps:
+      - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+      - name: Set up Ruby
+        uses: ruby/setup-ruby@efbf473cab83af4468e8606cc33eca9281bb213f # v1.256.0
+        with:
+          ruby-version: 2.6
+      - name: Install dependencies
+        run: gem install license_finder
+      - name: Run tests
+        run: license_finder --decisions_file .license/dependency_decisions.yml

.github/workflows/markdown-link-check.config.json

@@ -0,0 +1,10 @@
+{
+  "ignorePatterns": [
+    {
+      "pattern": "^https://codecov.io"
+    },
+    {
+      "pattern": "^https://openkruise.io"
+    }
+  ]
+}

.github/workflows/scorecard.yml

@@ -0,0 +1,72 @@
+# This workflow uses actions that are not certified by GitHub. They are provided
+# by a third-party and are governed by separate terms of service, privacy
+# policy, and support documentation.
+
+name: Scorecard supply-chain security
+on:
+  # For Branch-Protection check. Only the default branch is supported. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#branch-protection
+  branch_protection_rule:
+  # To guarantee Maintained check is occasionally updated. See
+  # https://github.com/ossf/scorecard/blob/main/docs/checks.md#maintained
+  schedule:
+    - cron: '30 14 * * *'
+  push:
+    branches: [ "master" ]
+
+# Declare default permissions as read only.
+permissions: read-all
+
+jobs:
+  analysis:
+    name: Scorecard analysis
+    runs-on: ubuntu-latest
+    permissions:
+      # Needed to upload the results to code-scanning dashboard.
+      security-events: write
+      # Needed to publish results and get a badge (see publish_results below).
+      id-token: write
+      # Uncomment the permissions below if installing in a private repository.
+      # contents: read
+      # actions: read
+
+    steps:
+      - name: "Checkout code"
+        uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
+        with:
+          persist-credentials: false
+
+      - name: "Run analysis"
+        uses: ossf/scorecard-action@05b42c624433fc40578a4040d5cf5e36ddca8cde # v2.4.2
+        with:
+          results_file: results.sarif
+          results_format: sarif
+          # (Optional) "write" PAT token. Uncomment the `repo_token` line below if:
+          # - you want to enable the Branch-Protection check on a *public* repository, or
+          # - you are installing Scorecard on a *private* repository
+          # To create the PAT, follow the steps in https://github.com/ossf/scorecard-action#authentication-with-pat.
+          # repo_token: ${{ secrets.SCORECARD_TOKEN }}
+
+          # Public repositories:
+          #   - Publish results to OpenSSF REST API for easy access by consumers
+          #   - Allows the repository to include the Scorecard badge.
+          #   - See https://github.com/ossf/scorecard-action#publishing-results.
+          # For private repositories:
+          #   - `publish_results` will always be set to `false`, regardless
+          #     of the value entered here.
+          publish_results: true
+
+      # Upload the results as artifacts (optional). Commenting out will disable uploads of run results in SARIF
+      # format to the repository Actions tab.
+      - name: "Upload artifact"
+        uses: actions/upload-artifact@65c4c4a1ddee5b72f698fdd19549f0f0fb45cf08 # v4.6.0
+        with:
+          name: SARIF file
+          path: results.sarif
+          retention-days: 5
+
+      # Upload the results to GitHub's code scanning dashboard.
+      - name: "Upload to code-scanning"
+        uses: github/codeql-action/upload-sarif@df559355d593797519d70b90fc8edd5db049e7a2 # v2.25.0
+        with:
+          sarif_file: results.sarif

.gitignore

@@ -5,10 +5,13 @@
 *.dll
 *.so
 *.dylib
-bin
+bin/
+testbin/
+.temp
 
 # Test binary, build with `go test -c`
 *.test
+test/e2e/generated/bindata.go
 
 # Output of the go coverage tool, specifically when used with LiteIDE
 *.out
@@ -22,5 +25,8 @@ bin
 *.swp
 *.swo
 *~
+.vscode
 
 .DS_Store
+
+vendor/

.golangci.yml

@@ -0,0 +1,73 @@
+version: "2"
+run:
+  concurrency: 4
+  issues-exit-code: 1
+  tests: true
+
+
+# output configuration options
+output:
+  formats:
+    text:
+      path: stdout
+      colors: true
+linters:
+  default: none
+  enable:
+    - depguard
+    - govet
+    - ineffassign
+    - misspell
+    - unconvert
+    - unused
+  settings:
+    misspell:
+      # Correct spellings using locale preferences for US or UK.
+      # Default is to use a neutral variety of English.
+      # Setting locale to US will correct the British spelling of 'colour' to 'color'.
+      locale: US
+    depguard:
+      rules:
+        forbid-pkg-errors:
+          deny:
+          - pkg: "github.com/pkg/errors"
+            desc: Should be replaced with standard lib errors or fmt.Errorf
+  exclusions:
+    generated: lax
+    presets:
+      - comments
+      - common-false-positives
+      - legacy
+      - std-error-handling
+    rules:
+      - path: (.+)\.go$
+        text: 'SA1019: package github.com/golang/protobuf/proto is deprecated: Use the "google.golang.org/protobuf/proto" package instead'
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+      - apis
+      - pkg/client
+      - vendor
+      - test
+formatters:
+  enable:
+  - gofmt
+  - goimports
+  settings:
+    gofmt:
+      simplify: true
+    goimports:
+      # put imports beginning with prefix after 3rd-party packages;
+      local-prefixes: 
+      - github.com/openkruise/kruise
+  exclusions:
+    generated: lax
+    paths:
+      - third_party$
+      - builtin$
+      - examples$
+      - apis
+      - pkg/client
+      - vendor
+      - test

.license/README.md

@@ -0,0 +1,19 @@
+# License Checker
+
+Our license checker CI rely on [LicenseFinder](https://github.com/pivotal/LicenseFinder).
+
+## How to add a new license
+
+LicenseFinder is a ruby project, so make sure you have ruby installed.
+
+### Install the tool
+
+```shell
+gem install license_finder
+```
+
+### Add a license
+
+```shell
+license_finder  permitted_licenses add MIT --decisions_file .license/dependency_decisions.yml
+```

.license/dependency_decisions.yml

@@ -0,0 +1,43 @@
+---
+- - :permit
+  - MIT
+  - :who: 
+    :why: 
+    :versions: []
+    :when: 2021-03-12 07:35:34.645031000 Z
+- - :permit
+  - Apache 2.0
+  - :who: 
+    :why: 
+    :versions: []
+    :when: 2021-03-12 07:19:18.243194000 Z
+- - :permit
+  - New BSD
+  - :who: 
+    :why: 
+    :versions: []
+    :when: 2021-03-12 07:19:28.540675000 Z
+- - :permit
+  - Simplified BSD
+  - :who: 
+    :why: 
+    :versions: []
+    :when: 2021-03-12 07:20:01.774212000 Z
+- - :permit
+  - Mozilla Public License 2.0
+  - :who: 
+    :why: 
+    :versions: []
+    :when: 2021-03-12 07:21:05.194536000 Z
+- - :permit
+  - unknown
+  - :who: 
+    :why: 
+    :versions: []
+    :when: 2021-03-12 07:21:43.379269000 Z
+- - :permit
+  - ISC
+  - :who: 
+    :why: 
+    :versions: []
+    :when: 2021-03-12 07:22:07.265966000 Z

.lift.toml

@@ -0,0 +1,4 @@
+# Ignore results from vendor directories
+ignoreFiles = """
+vendor/
+"""

.muse.toml

@@ -0,0 +1,5 @@
+# Ignore results from test and vendor directories
+ignoreFiles = """
+vendor/
+test/
+"""

.travis.yml

@@ -1,13 +0,0 @@
-language: go
-
-go:
-  - "1.14"
-
-go_import_path: github.com/openkruise/kruise
-
-script:
-  - make all
-
-after_success:
-  - ./hack/docker_build_daily.sh
-  - bash <(curl -s https://codecov.io/bash) || echo "Failed to report code coverage!"

ADOPTERS.md

@@ -1 +0,0 @@
-# Adopters

BUILDING_MULTI_ARCH.md

@@ -0,0 +1,39 @@
+# BUILDING MULTI-ARCHITECTURE IMAGES
+
+While the test Docker images for OpenKruise can be built by running:
+
+```
+make docker-build
+```
+
+the stable images are build for multiple architectures (and automatically pushed to the repository) by
+
+```
+make docker-multiarch
+```
+
+However, there are several options and requirements for building these images, which are explained here.
+
+## BUILDERS
+
+We use `docker buildx` to build images for multiple architectures. At the moment, this is considered an
+experimental feature by Docker, and as such it requires access to experimental features to be enabled
+in the daemon.json file.
+
+By default, this will use BuildKit and QEMU emulation support to build the image for each architecture
+locally. However, for faster building, it is possible to use native nodes to build each architecture,
+either simple remote Docker instances or a Kubernetes cluster. For information on how to set this up,
+please see the blog article here:
+
+<https://www.docker.com/blog/multi-arch-images/>
+
+And/or the command reference here:
+
+<https://docs.docker.com/engine/reference/commandline/buildx_create/>
+
+## BUILD OPTIONS
+
+### PLATFORMS
+
+The PLATFORMS variable can be set to a comma-separated list of platforms for which images should be
+built when running the multi-arch build. If unset, it defaults to _linux/amd64,linux/arm64,linux/arm_.

CODE_OF_CONDUCT.md

@@ -1,132 +1,3 @@
+# Code of Conduct
 
-# Contributor Covenant Code of Conduct
-
-This document defines the code of conduct that applies to the entire Openkruise project hosted and managed by the [Openkruise organization](https://github.com/openkruise).
-
-## Our Pledge
-
-We as members, contributors, and maintainers pledge to make participation in our
-community a harassment-free experience for everyone, regardless of age, body
-size, visible or invisible disability, ethnicity, sex characteristics, gender
-identity and expression, level of experience, education, social-economic status,
-nationality, personal appearance, race, religion, or sexual identity
-and orientation.
-
-We pledge to act and interact in ways that contribute to an open, welcoming,
-diverse, inclusive, and healthy community.
-
-## Our Standards
-
-Examples of behavior that contributes to a positive environment for our
-community include:
-
-* Demonstrating empathy and kindness toward other people
-* Being respectful of differing opinions, viewpoints, and experiences
-* Giving and gracefully accepting constructive feedback
-* Accepting responsibility and apologizing to those affected by our mistakes,
-  and learning from the experience
-* Focusing on what is best not just for us as individuals, but for the
-  overall community
-
-Examples of unacceptable behavior include:
-
-* The use of sexualized language or imagery, and sexual attention or
-  advances of any kind
-* Personal attacks
-* Trolling, insulting or derogatory comments, and personal or political attacks
-* Public or private harassment
-* Publishing others' private information, such as a physical or email
-  address, without their explicit permission
-* Other conduct which could reasonably be considered inappropriate in a
-  professional setting
-
-## Enforcement Responsibilities
-
-Project maintainers are responsible for clarifying and enforcing our standards of
-acceptable behavior and will take appropriate and fair corrective action in
-response to any behavior that they deem inappropriate, threatening, offensive,
-or harmful.
-
-Project maintainers have the right and responsibility to remove, edit, or reject
-comments, commits, code, wiki edits, issues, and other contributions that are
-not aligned to this Code of Conduct, and will communicate reasons for moderation
-decisions when appropriate.
-
-## Scope
-
-This Code of Conduct applies within all community spaces, and also applies when
-an individual is officially representing the community in public spaces.
-Examples of representing our community include using an official e-mail address,
-posting via an official social media account, or acting as an appointed
-representative at an online or offline event.
-
-## Enforcement
-
-Instances of abusive, harassing, or otherwise unacceptable behavior may be
-reported to the project maintainers responsible for enforcement at
-openkruise-user@googlegroups.com.
-All complaints will be reviewed and investigated promptly and fairly.
-
-All project maintainers are obligated to respect the privacy and security of the
-reporter of any incident.
-
-## Enforcement Guidelines
-
-Project maintainers will follow these Community Impact Guidelines in determining
-the consequences for any action they deem in violation of this Code of Conduct:
-
-### 1. Correction
-
-**Community Impact**: Use of inappropriate language or other behavior deemed
-unprofessional or unwelcome in the community.
-
-**Consequence**: A private, written warning from project maintainers, providing
-clarity around the nature of the violation and an explanation of why the
-behavior was inappropriate. A public apology may be requested.
-
-### 2. Warning
-
-**Community Impact**: A violation through a single incident or series
-of actions.
-
-**Consequence**: A warning with consequences for continued behavior. No
-interaction with the people involved, including unsolicited interaction with
-those enforcing the Code of Conduct, for a specified period of time. This
-includes avoiding interactions in community spaces as well as external channels
-like social media. Violating these terms may lead to a temporary or
-permanent ban.
-
-### 3. Temporary Ban
-
-**Community Impact**: A serious violation of community standards, including
-sustained inappropriate behavior.
-
-**Consequence**: A temporary ban from any sort of interaction or public
-communication with the community for a specified period of time. No public or
-private interaction with the people involved, including unsolicited interaction
-with those enforcing the Code of Conduct, is allowed during this period.
-Violating these terms may lead to a permanent ban.
-
-### 4. Permanent Ban
-
-**Community Impact**: Demonstrating a pattern of violation of community
-standards, including sustained inappropriate behavior,  harassment of an
-individual, or aggression toward or disparagement of classes of individuals.
-
-**Consequence**: A permanent ban from any sort of public interaction within
-the community.
-
-## Attribution
-
-This Code of Conduct is adapted from the [Contributor Covenant][homepage],
-version 2.0, available at
-[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html](https://www.contributor-covenant.org/version/2/0/code_of_conduct.html).
-
-Community Impact Guidelines were inspired by [Mozilla's code of conduct
-enforcement ladder](https://github.com/mozilla/diversity).
-
-[homepage]: https://www.contributor-covenant.org
-
-For answers to common questions about this code of conduct, see the FAQ at
-[https://www.contributor-covenant.org/faq](https://www.contributor-covenant.org/faq).
-Translations are available at [https://www.contributor-covenant.org/translations](https://www.contributor-covenant.org/translations).
+OpenKruise follows the [CNCF Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).

CONTRIBUTING.md

@@ -1,6 +1,6 @@
 # Contributing to Openkruise
 
-Welcome to Openkruise! Openkruise consists several repositories under the organization.
+Welcome to Openkruise! Openkruise consists of several repositories under the organization.
 We encourage you to help out by reporting issues, improving documentation, fixing bugs, or adding new features.
 Please also take a look at our code of conduct, which details how contributors are expected to conduct themselves as part of the Openkruise community.
 
@@ -10,7 +10,7 @@ To be honest, we regard every user of Openkruise as a very kind contributor.
 After experiencing Openkruise, you may have some feedback for the project.
 Then feel free to open an issue.
 
-There are lot of cases when you could open an issue:
+There are a lot of cases when you could open an issue:
 
 - bug report
 - feature request
@@ -20,11 +20,11 @@ There are lot of cases when you could open an issue:
 - help wanted
 - doc incomplete
 - test improvement
-- any questions on project
+- any questions on the project
 - and so on
 
-Also we must remind that when filing a new issue, please remember to remove the sensitive data from your post.
-Sensitive data could be password, secret key, network locations, private business data and so on.
+Also, we must remind you that when filing a new issue, please remember to remove the sensitive data from your post.
+Sensitive data could be passwords, secret keys, network locations, private business data, and so on.
 
 ## Code and doc contribution
 
@@ -45,13 +45,14 @@ On GitHub, every improvement for Openkruise could be via a PR (short for pull re
 ### Workspace Preparation
 
 To put forward a PR, we assume you have registered a GitHub ID.
-Then you could finish the preparation in the following steps:
+Then you can finish the preparation in the following steps:
 
-1. **Fork** Fork the repository you wish to work on. You just need to click the button Fork in right-left of project repository main page. Then you will end up with your repository in your GitHub username.
-2. **Clone** your own repository to develop locally. Use `git clone https://github.com/<your-username>/<project>.git` to clone repository to your local machine. Then you can create new branches to finish the change you wish to make.
+1. **Fork** Fork the repository you wish to work on. You just need to click the button Fork in the right-left of the project repository main page. Then you will end up with your repository in your GitHub username.
+2. **Clone** your own repository to develop locally. Use `git clone https://github.com/<your-username>/<project>.git` to clone the repository to your local machine. Then you can create new branches to finish the change you wish to make.
 3. **Set remote** upstream to be `https://github.com/openkruise/<project>.git` using the following two commands:
 
 ```bash
+cd <project>
 git remote add upstream https://github.com/openkruise/<project>.git
 git remote set-url --push upstream no-pushing
 ```
@@ -60,7 +61,7 @@ Adding this, we can easily synchronize local branches with upstream branches.
 
 4. **Create a branch** to add a new feature or fix issues
 
-Update local working directory:
+Update the local working directory:
 
 ```bash
 cd <project>
@@ -79,46 +80,67 @@ Make any change on the new-branch then build and test your codes.
 
 ### PR Description
 
-PR is the only way to make change to Kruise project files.
-To help reviewers better get your purpose, PR description could not be too detailed.
+PR is the only way to make changes to Kruise project files.
+To help reviewers better understand your purpose, PR description could not be too detailed.
 We encourage contributors to follow the [PR template](./.github/PULL_REQUEST_TEMPLATE.md) to finish the pull request.
 
 ### Developing Environment
 
-As a contributor, if you want to make any contribution to Kruise project, we should reach an agreement on the version of tools used in the development environment.
-Here are some dependents with specific version:
+As a contributor, if you want to make any contribution to the Kruise project, we should reach an agreement on the version of tools used in the development environment.
+Here are some dependencies with specific versions:
 
-- Golang : v1.13+ (1.14 is best)
-- Kubernetes: v1.12+
+- Golang : v1.22+
+- Kubernetes: v1.16+
 
 ### Developing guide
 
 There's a `Makefile` in the root folder which describes the options to build and install. Here are some common ones:
 
 ```bash
-# Build the controller manager binary
-make manager
-
-# Run the tests
-make test
-
-# Generate manifests e.g. CRD, RBAC YAML files etc
+# Generate code and manifests e.g. CRD, RBAC YAML files etc
 make manifests
+
+# Build the controller manager binary
+make build
+
+# Run the unit tests
+make test
 ```
 
-If you want to start kruise-controller-manager locally to work with a Kubernetes cluster, you should follow the [debug guide](./docs/debug/README.md).
+**There are some guide documents for contributors in [./docs/contributing/](./docs/contributing), such as a debug guide to help you test your own branch in a Kubernetes cluster.**
+
+### Proposals
+
+If you are going to contribute a feature with a new API or need significant effort, please submit a proposal in [./docs/proposals/](./docs/proposals) first.
+
+### Kruise Helm Charts
+[kruise charts](https://github.com/openkruise/charts) is the openKruise charts repo, including kruise, kruise rollout, and kruise game.
+You can add the corresponding charts package in the versions directory as follows:
+```
+ versions
+ - kruise-game
+ - kruise-rollout
+ - kruise-state-metrics
+ - kruise
+   - 1.5.0
+   - 1.5.1
+   - 1.6.0
+   - 1.6.1
+```
+
+**make generate_helm_crds** automatically generates crds files under the bin/ directory, which in turn simplifies the generation of helm charts.
 
 ## Engage to help anything
 
 We choose GitHub as the primary place for Openkruise to collaborate.
 So the latest updates of Openkruise are always here.
-Although contributions via PR is an explicit way to help, we still call for any other ways.
+Although contributions via PR are an explicit way to help, we still call for any other ways.
 
 - reply to other's issues if you could;
 - help solve other user's problems;
 - help review other's PR design;
 - help review other's codes in PR;
-- discuss about Openkruise to make things clearer;
+- discuss Openkruise to make things clearer;
 - advocate Openkruise technology beyond GitHub;
 - write blogs on Openkruise and so on.
 
@@ -126,21 +148,5 @@ In a word, **ANY HELP IS CONTRIBUTION**.
 
 ## Join Openkruise as a member
 
-It is also welcomed to join Openkruise team if you are willing to participate in Openkruise community continuously and keep active.
-
-### Requirements
-
-- Have read the [Contributing to Openkruise](./CONTRIBUTING.md) carefully
-- Have read the [Contributor Covenant Code of Conduct](./CODE_OF_CONDUCT.md)
-- Have submitted multi PRs to the community
-- Be active in the community, may including but not limited
-  - Submitting or commenting on issues
-  - Contributing PRs to the community
-  - Reviewing PRs in the community
-
-### How to do it
-
-You can do it in either of two ways:
-
-- Submit a PR in the project repo
-- Contact via the [community](./README.md#community) channels offline
+It is also welcomed to join the Openkruise team if you are willing to participate in the Openkruise community continuously and keep active.
+Please read and follow the [Community Membership](https://github.com/openkruise/community/blob/master/community-membership.md).

Dockerfile

@@ -1,27 +1,48 @@
-# Build the manager binary
-FROM golang:1.14 as builder
-
+# Build the manager and daemon binaries
+ARG BASE_IMAGE=alpine
+ARG BASE_IMAGE_VERSION=3.21@sha256:56fa17d2a7e7f168a043a2712e63aed1f8543aeafdcee47c58dcffe38ed51099
+FROM golang:1.23.9-alpine3.21@sha256:fb7ea5cd19bc4eea3eb0d1972919ec0f6229b138985ce4b35ce5846c6bc02973 AS builder
 WORKDIR /workspace
 # Copy the Go Modules manifests
 COPY go.mod go.mod
 COPY go.sum go.sum
-# cache deps before building and copying source so that we don't need to re-download as much
-# and so that source changes don't invalidate our downloaded layer
-#RUN go mod download
 
 # Copy the go source
 COPY main.go main.go
 COPY apis/ apis/
+COPY cmd/ cmd/
 COPY pkg/ pkg/
-COPY vendor/ vendor/
 
 # Build
-RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 GO111MODULE=on go build -mod=vendor -a -o manager main.go
+RUN CGO_ENABLED=0 GO111MODULE=on go build -a -o manager main.go \
+  && CGO_ENABLED=0 GO111MODULE=on go build -a -o daemon ./cmd/daemon/main.go
+
+ARG BASE_IMAGE
+ARG BASE_IMAGE_VERSION
+FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
 
-# Use distroless as minimal base image to package the manager binary
-# Refer to https://github.com/GoogleContainerTools/distroless for more details
-#FROM gcr.io/distroless/static:nonroot
-FROM ubuntu:latest
 WORKDIR /
 COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/daemon ./kruise-daemon
+
+RUN set -eux; \
+    mkdir -p /log /tmp && \
+    chown -R nobody:nobody /log && \
+    chown -R nobody:nobody /tmp && \
+    chown -R nobody:nobody /manager && \
+    apk --no-cache --update upgrade && \
+    apk --no-cache add ca-certificates && \
+    apk --no-cache add tzdata && \
+    rm -rf /var/cache/apk/* && \
+    update-ca-certificates && \
+    echo "only include root and nobody user" && \
+    echo -e "root:x:0:0:root:/root:/bin/ash\nnobody:x:65534:65534:nobody:/:/sbin/nologin" | tee /etc/passwd && \
+    echo -e "root:x:0:root\nnobody:x:65534:" | tee /etc/group && \
+    rm -rf /usr/local/sbin/* && \
+    rm -rf /usr/local/bin/* && \
+    rm -rf /usr/sbin/* && \
+    rm -rf /usr/bin/* && \
+    rm -rf /sbin/* && \
+    rm -rf /bin/*
+
 ENTRYPOINT ["/manager"]

Dockerfile_helm_hook

@@ -0,0 +1,40 @@
+ARG BASE_IMAGE=alpine
+ARG BASE_IMAGE_VERSION=3.19
+FROM golang:1.20.14-alpine3.19 AS builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+
+# Copy the go source
+COPY apis/ apis/
+COPY cmd/ cmd/
+COPY pkg/ pkg/
+# Build
+RUN --mount=type=cache,target=/go CGO_ENABLED=0 GO111MODULE=on go build -a -o helm_hook ./cmd/helm_hook/main.go
+
+FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
+WORKDIR /
+RUN sed -i 's/dl-cdn.alpinelinux.org/mirrors.aliyun.com/g' /etc/apk/repositories
+RUN set -eux; \
+    mkdir -p /log /tmp && \
+    chown -R nobody:nobody /log && \
+    chown -R nobody:nobody /tmp && \
+    apk --no-cache --update upgrade && \
+    apk --no-cache add ca-certificates && \
+    apk --no-cache add tzdata && \
+    rm -rf /var/cache/apk/* && \
+    update-ca-certificates && \
+    echo "only include root and nobody user" && \
+    echo -e "root:x:0:0:root:/root:/bin/ash\nnobody:x:65534:65534:nobody:/:/sbin/nologin" | tee /etc/passwd && \
+    echo -e "root:x:0:root\nnobody:x:65534:" | tee /etc/group
+COPY --from=builder /workspace/helm_hook .
+RUN chown -R nobody:nobody /helm_hook && \
+    rm -rf /usr/local/sbin/* && \
+    rm -rf /usr/local/bin/* && \
+    rm -rf /usr/sbin/* && \
+    rm -rf /usr/bin/* && \
+    rm -rf /sbin/* && \
+    rm -rf /bin/*
+ENTRYPOINT ["/helm_hook"]

Dockerfile_multiarch

@@ -0,0 +1,56 @@
+# Build the manager and daemon binaries
+ARG BASE_IMAGE=alpine
+ARG BASE_IMAGE_VERSION=3.21@sha256:56fa17d2a7e7f168a043a2712e63aed1f8543aeafdcee47c58dcffe38ed51099
+ARG BUILD_BASE_IMAGE=golang:1.22.11-alpine3.21@sha256:161858498a61ce093c8e2bd704299bfb23e5bff79aef99b6c40bb9c6a43acf0f
+FROM --platform=$BUILDPLATFORM ${BUILD_BASE_IMAGE} AS builder
+
+WORKDIR /workspace
+# Copy the Go Modules manifests
+COPY go.mod go.mod
+COPY go.sum go.sum
+
+# Copy the go source
+COPY main.go main.go
+COPY apis/ apis/
+COPY cmd/ cmd/
+COPY pkg/ pkg/
+
+#ENV GOPROXY=https://goproxy.cn,direct
+RUN go mod tidy
+
+# Build
+ARG TARGETOS
+ARG TARGETARCH
+RUN GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GO111MODULE=on go build -a -o manager main.go \
+  && GOOS=${TARGETOS} GOARCH=${TARGETARCH} CGO_ENABLED=0 GO111MODULE=on go build -a -o daemon ./cmd/daemon/main.go
+
+
+ARG BASE_IMAGE
+ARG BASE_IMAGE_VERSION
+FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
+
+WORKDIR /
+COPY --from=builder /workspace/manager .
+COPY --from=builder /workspace/daemon ./kruise-daemon
+
+RUN set -eux; \
+    mkdir -p /log /tmp && \
+    chown -R nobody:nobody /log && \
+    chown -R nobody:nobody /tmp && \
+    chown -R nobody:nobody /manager && \
+    apk --no-cache --update upgrade && \
+    apk --no-cache add ca-certificates && \
+    apk --no-cache add tzdata && \
+    rm -rf /var/cache/apk/* && \
+    update-ca-certificates && \
+    echo "only include root and nobody user" && \
+    echo -e "root:x:0:0:root:/root:/bin/ash\nnobody:x:65534:65534:nobody:/:/sbin/nologin" | tee /etc/passwd && \
+    echo -e "root:x:0:root\nnobody:x:65534:" | tee /etc/group && \
+    rm -rf /usr/local/sbin/* && \
+    rm -rf /usr/local/bin/* && \
+    rm -rf /usr/sbin/* && \
+    rm -rf /usr/bin/* && \
+    rm -rf /sbin/* && \
+    rm -rf /bin/*
+
+ENTRYPOINT ["/manager"]

Dockerfile_windows

@@ -0,0 +1,11 @@
+# Build Windows image for kruise-daemon 
+
+# Using Windows HostProcess container base image: https://github.com/microsoft/windows-host-process-containers-base-image
+ARG BASE_IMAGE=mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image
+ARG BASE_IMAGE_VERSION=v1.0.0
+FROM ${BASE_IMAGE}:${BASE_IMAGE_VERSION}
+
+WORKDIR /
+COPY ./bin/kruise-daemon.exe .
+
+ENTRYPOINT ["kruise-daemon.exe"]

FAQ.md

@@ -1,20 +0,0 @@
-# Frequently Asked Questions (FAQ)
-
-FAQ contains some frequently asked questions about two aspects:
-
-- First, user-facing functionalities.
-- Second, underlying concept and theory.
-
-Technical questions will not be included in openkruise/kruise's FAQ.
-
-## Advanced Statefulset
-
-To be added.
-
-## Sidecar Set
-
-To be added.
-
-## Broadcast Job
-
-To be added.

GOVERNANCE.md

@@ -1,47 +1,22 @@
 # Governance
 
-This document defines governance policies for the entire Openkruise project hosted and managed by the [Openkruise organization](https://github.com/openkruise).
+The governance model adopted in OpenKruise is influenced by many CNCF projects.
 
 ## Principles
 
 - **Open**: Openkruise is an open source community.
-- **Welcoming and respectful**: See [Code of Conduct](CODE_OF_CONDUCT.md).
+- **Welcoming and respectful**: See [Code of Conduct](https://github.com/cncf/foundation/blob/master/code-of-conduct.md).
 - **Transparent and accessible**: Work and collaboration should be done in public.
 - **Merit**: Ideas and contributions are accepted according to their technical merit
   and alignment with project objectives, scope and design principles.
 
-## Project Maintainers
+## Code of Conduct
 
-Maintainers are the first and foremost contributors that are committed to the success of Openkruise project.
-They normally take the following responsibilities:
+The OpenKruise [Code of Conduct](CODE_OF_CONDUCT.md) is aligned with the CNCF Code of Conduct.
 
-- Classify GitHub issues and perform pull request reviews for other maintainers and the community.
-- During GitHub issue classification, apply all applicable [labels](https://github.com/openkruise/kruise/labels)
-  to each new issue. Labels are extremely useful for follow-up of future issues. Which labels to apply
-  is somewhat subjective so just use your best judgment.
-- Make sure that ongoing PRs are moving forward at the right pace or closing them if they are not
-  moving in a productive direction.
-- Participate when called upon in the security release process. Note
-  that although this should be a rare occurrence, if a serious vulnerability is found, the process
-  may take up to several full days of work to implement.
+## Community Membership
 
-## Process of becoming a maintainer
-
-- Talk to one of the existing project [maintainers](MAINTAINERS.md) that you are interested in becoming a
-  maintainer, and he will nominate you as a new maintainer. After nomination, you will need to
-  create a PR to update the list in [MAINTAINERS.md](MAINTAINERS.md).
-- We will expect you to start contributing increasingly complicated PRs, under the guidance
-  of the existing maintainers.
-- We may ask you to do some PRs from our backlog. As you gain experience with the code base and our standards,
-  we will ask you to do code reviews for incoming PRs.
-- Once the existing maintainers have made a consensus that the nominating maintainer has deep understanding
-  about the project and is able to independently take the maintainer responsibilities,
-  the PR will be approved and the new maintainer becomes active.
-
-## When does a maintainer lose maintainer status
-
-- If a maintainer is no longer interested or cannot perform the maintainer duties listed above, they should volunteer to be moved to emeritus status.
-- In extreme cases this can also occur by a vote of the maintainers per the voting process. The voting process is a simple majority in which each maintainer receives one vote.
+See [community membership](https://github.com/openkruise/community/blob/master/community-membership.md).
 
 ## Decision making process
 
@@ -53,10 +28,6 @@ If a dispute cannot be decided independently, get a third-party maintainer (e.g.
 on the issue, but not involved in the conflict) to intercede and the final decision will be made.
 Decision making process should be transparent to adhere to the principles of Openkruise project.
 
-## Code of Conduct
-
-The Openkruise [Code of Conduct](CODE_OF_CONDUCT.md) is aligned with the CNCF Code of Conduct.
-
 ## Credits
 
 Some contents in this documents have been borrowed from [BFE](https://github.com/bfenetworks/bfe/blob/develop/GOVERNANCE.md) and [CoreDNS](https://github.com/coredns/coredns/blob/master/GOVERNANCE.md) projects.

MAINTAINERS.md

@@ -1,13 +0,0 @@
-# The OpenKruise Maintainers
-
-This file lists the maintainers of the OpenKruise project. The responsibilities of maintainers are listed in the [GOVERNANCE.md](GOVERNANCE.md) file.
-
-## Project Maintainers
-
-| Name | GitHub ID | Affiliation |
-| ---- | --------- | ----------- |
-| [Fei Guo](mailto:f.guo@alibaba-inc.com) | [Fei-Guo](https://github.com/Fei-Guo) | Alibaba |
-| [Siyu Wang](mailto:jiuzhu.wsy@alibaba-inc.com) | [FillZpp](https://github.com/FillZpp) | Alibaba |
-| [Zhen Zhang](mailto:shouchen.zz@alibaba-inc.com) | [furykerry](https://github.com/furykerry) | Alibaba |
-| [Robert Everson](mailto:robert@reverson.net) | [reverson](https://github.com/reverson) | Lyft |
-| [Henry Wang](mailto:henrywangx@163.com) | [henrywangx](https://github.com/henrywangx) | Tencent |

Makefile

@@ -1,8 +1,11 @@
-
 # Image URL to use all building/pushing image targets
 IMG ?= openkruise/kruise-manager:test
-# Produce CRDs that work back to Kubernetes 1.11 (no version conversion)
-CRD_OPTIONS ?= "crd:trivialVersions=true"
+HOOK_IMG ?= openkruise/kruise-helm-hook:test
+WIN_DAEMON_IMG ?= openkruise/kruise-daemon-win:test
+# Platforms to build the image for
+PLATFORMS ?= linux/amd64,linux/arm64,linux/ppc64le
+WIN_PLATFORMS ?= windows/amd64
+CRD_OPTIONS ?= "crd:crdVersions=v1"
 
 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
 ifeq (,$(shell go env GOBIN))
@@ -10,77 +13,203 @@ GOBIN=$(shell go env GOPATH)/bin
 else
 GOBIN=$(shell go env GOBIN)
 endif
+GOOS ?= $(shell go env GOOS)
 
-all: manager
+# ENVTEST_K8S_VERSION refers to the version of kubebuilder assets to be downloaded by envtest binary.
+# Run `setup-envtest list` to list available versions.
+ENVTEST_K8S_VERSION ?= 1.32.0
+
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# This is a requirement for 'setup-envtest.sh' in the test target.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+all: build
+
+##@ Development
 
 go_check:
-	@scripts/check_go_version "1.13.0"
+	@scripts/check_go_version "1.23"
 
-# Run tests
-test: generate fmt vet manifests
-	go test ./pkg/... -coverprofile cover.out
-
-# Build manager binary
-manager: generate fmt vet
-	go build -o bin/manager main.go
-
-# Run against the configured Kubernetes cluster in ~/.kube/config
-run: generate fmt vet manifests
-	go run ./main.go --enable-leader-election=false
-
-# Install CRDs into a cluster
-install: manifests
-	kustomize build config/crd | kubectl apply -f -
-
-# Uninstall CRDs from a cluster
-uninstall: manifests
-	kustomize build config/crd | kubectl delete -f -
-
-# Deploy controller in the configured Kubernetes cluster in ~/.kube/config
-deploy: manifests
-	cd config/manager && kustomize edit set image controller=${IMG}
-	kustomize build config/default | kubectl apply -f -
-
-# Generate manifests e.g. CRD, RBAC etc.
-manifests: controller-gen
-	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./..." output:crd:artifacts:config=config/crd/bases
-
-# Run go fmt against code
-fmt: go_check
-	go fmt ./...
-
-# Run go vet against code
-vet:
-	go vet ./...
-
-# Generate code
-generate: controller-gen
+generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
 	@scripts/generate_client.sh
+	@scripts/generate_openapi.sh
 	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./apis/..."
 
-# Build the docker image
-docker-build: test
-	docker build . -t ${IMG}
+manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
+	$(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role webhook paths="./apis/..." output:crd:artifacts:config=config/crd/bases
 
-# Push the docker image
-docker-push:
+fmt: go_check ## Run go fmt against code.
+	go fmt $(shell go list ./... | grep -v /vendor/)
+
+vet: ## Run go vet against code.
+	go vet $(shell go list ./... | grep -v /vendor/)
+
+lint: golangci-lint ## Run golangci-lint against code.
+	$(GOLANGCI_LINT) run
+
+test: generate fmt vet manifests envtest ## Run tests
+	echo $(ENVTEST)
+	go build -o pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin pkg/daemon/criruntime/imageruntime/fake_plugin/main.go && chmod +x pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -race ./pkg/... -coverprofile raw-cover.out
+	rm pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
+	grep -v "pkg/client" raw-cover.out > cover.out
+
+atest: 
+	echo $(ENVTEST)
+	go build -o pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin pkg/daemon/criruntime/imageruntime/fake_plugin/main.go && chmod +x pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
+	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) -p path)" go test -race ./pkg/... -coverprofile raw-cover.out
+	rm pkg/daemon/criruntime/imageruntime/fake_plugin/fake-credential-plugin
+	grep -v "pkg/client" raw-cover.out > cover.out
+
+coverage-report: ## Generate cover.html from cover.out
+	go tool cover -html=cover.out -o cover.html
+ifeq ($(GOOS), darwin)
+	open ./cover.html
+else
+	echo "open cover.html with a HTML viewer."
+endif
+
+##@ Build
+
+build: generate fmt vet manifests ## Build manager binary.
+	go build -o bin/manager main.go
+
+build-win-daemon: ## Build Windows daemon binary.
+	GOOS=windows go build -o bin/kruise-daemon.exe ./cmd/daemon/main.go
+
+run: manifests generate fmt vet ## Run a controller from your host.
+	go run ./main.go
+
+docker-build: ## Build docker image with the manager.
+	docker build --pull --no-cache . -t ${IMG}
+
+docker-push: ## Push docker image with the manager.
 	docker push ${IMG}
 
-# find or download controller-gen
-# download controller-gen if necessary
-controller-gen:
-ifeq (, $(shell which controller-gen-kruise))
-	@{ \
-	set -e ;\
-	CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\
-	cd $$CONTROLLER_GEN_TMP_DIR ;\
-	go mod init tmp ;\
-	echo "replace sigs.k8s.io/controller-tools => github.com/openkruise/controller-tools v0.2.9-kruise" >> go.mod ;\
-	go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.2.9 ;\
-	rm -rf $$CONTROLLER_GEN_TMP_DIR ;\
-	mv $(GOPATH)/bin/controller-gen $(GOPATH)/bin/controller-gen-kruise ;\
-	}
-CONTROLLER_GEN=$(GOPATH)/bin/controller-gen-kruise
+docker-win-daemon: # Build Windows docker image with the daemon
+	docker buildx build -f ./Dockerfile_windows --pull --no-cache --platform=$(WIN_PLATFORMS) . -t $(WIN_DAEMON_IMG)
+
+# Build and push the multiarchitecture docker images and manifest.
+docker-multiarch:
+	docker buildx build -f ./Dockerfile_multiarch --pull --no-cache --platform=$(PLATFORMS) --push . -t $(IMG)
+
+##@ Deployment
+
+install: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl apply -f -
+
+uninstall: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | kubectl delete -f -
+
+deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/default | kubectl apply -f -
+	$(KUSTOMIZE) build config/daemonconfig | kubectl apply -f -
+
+undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/default | kubectl delete -f -
+	$(KUSTOMIZE) build config/daemonconfig | kubectl delete -f -
+
+CONTROLLER_GEN = $(shell pwd)/bin/controller-gen
+controller-gen: ## Download controller-gen locally if necessary.
+
+# controller-gen@v0.16.5 comply with k8s.io/api v0.30.x
+ifeq ("$(shell $(CONTROLLER_GEN) --version 2> /dev/null)", "Version: v0.16.5")
 else
-CONTROLLER_GEN=$(shell which controller-gen-kruise)
+	rm -rf $(CONTROLLER_GEN)
+	$(call go-get-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen@v0.17.3)
 endif
+KUSTOMIZE = $(shell pwd)/bin/kustomize
+kustomize: ## Download kustomize locally if necessary.
+	$(call go-get-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v4@v4.5.5)
+
+GOLANGCI_LINT = $(shell pwd)/bin/golangci-lint
+golangci-lint: ## Download golangci-lint locally if necessary.
+	$(call go-get-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint@v1.51.2)
+
+GINKGO = $(shell pwd)/bin/ginkgo
+ginkgo: ## Download ginkgo locally if necessary.
+	$(call go-get-tool,$(GINKGO),github.com/onsi/ginkgo/v2/ginkgo@latest)
+
+# go-get-tool will 'go get' any package $2 and install it to $1.
+PROJECT_DIR := $(shell dirname $(abspath $(lastword $(MAKEFILE_LIST))))
+define go-get-tool
+@[ -f $(1) ] || { \
+set -e ;\
+TMP_DIR=$$(mktemp -d) ;\
+cd $$TMP_DIR ;\
+go mod init tmp ;\
+echo "Downloading $(2) to $(PROJECT_DIR)/bin" ;\
+GOBIN=$(PROJECT_DIR)/bin go install $(2) ;\
+rm -rf $$TMP_DIR ;\
+}
+endef
+
+include tools/tools.mk
+
+## Location to install dependencies to
+TESTBIN ?= $(shell pwd)/testbin
+$(TESTBIN):
+	mkdir -p $(TESTBIN)
+
+ENVTEST ?= $(TESTBIN)/setup-envtest
+
+.PHONY: envtest
+envtest: $(TESTBIN) ## Download/update envtest-setup to latest version.
+	GOBIN=$(TESTBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+
+# create-cluster creates a kube cluster with kind.
+.PHONY: create-cluster
+create-cluster: $(tools/kind)
+	tools/hack/create-cluster.sh
+
+DISABLE_CSI ?= false
+
+.PHONY: install-csi
+install-csi:
+ifeq ($(DISABLE_CSI), true)
+	@echo "CSI is disabled, skip"
+else
+	cd tools/hack/csi-driver-host-path; ./install-snapshot.sh
+endif
+
+# delete-cluster deletes a kube cluster.
+.PHONY: delete-cluster
+delete-cluster: $(tools/kind) ## Delete kind cluster.
+	$(tools/kind) delete cluster --name ci-testing
+
+# kube-load-image loads a local built docker image into kube cluster.
+.PHONY: kube-load-image
+kube-load-image: $(tools/kind)
+	tools/hack/kind-load-image.sh $(IMG)
+
+# install-kruise install kruise with local build image to kube cluster.
+.PHONY: install-kruise
+install-kruise:
+	kubectl create namespace kruise-system;
+ifeq ($(ENABLE_E2E_CONFIG), true)
+	@echo "Applying e2e config...";
+	kubectl apply -f test/kruise-e2e-config.yaml;
+else
+	@echo "Skipping e2e config application...";
+endif
+	tools/hack/install-kruise.sh $(IMG)
+
+# run-kruise-e2e-test starts to run kruise e2e tests.
+.PHONY: run-kruise-e2e-test
+run-kruise-e2e-test:
+	@echo -e "\n\033[36mRunning kruise e2e tests...\033[0m"
+	tools/hack/run-kruise-e2e-test.sh
+
+generate_helm_crds:
+	scripts/generate_helm_crds.sh
+
+# kruise-e2e-test runs kruise e2e tests.
+.PHONY: kruise-e2e-test
+kruise-e2e-test: $(tools/kind) delete-cluster create-cluster install-csi docker-build kube-load-image install-kruise run-kruise-e2e-test delete-cluster
+
+.PHONY: docker-build-hook
+docker-build-hook:
+	docker buildx build -f ./Dockerfile_helm_hook --pull --no-cache --platform=$(PLATFORMS) --push . -t $(HOOK_IMG)

OWNERS

@@ -2,10 +2,12 @@
 approvers:
   - Fei-Guo
   - FillZpp
-  - jian-he
-  - jzwlqx # for BroadcastJob
+  - furykerry
+  - zmberg
+  - veophi
 reviewers:
   - Fei-Guo
   - FillZpp
-  - jian-he
-  - jzwlqx # for BroadcastJob
+  - furykerry
+  - zmberg
+  - veophi

PROJECT

@@ -1,29 +1,117 @@
 domain: kruise.io
+layout:
+- go.kubebuilder.io/v3
 multigroup: true
+projectName: kruise
 repo: github.com/openkruise/kruise
 resources:
-- group: apps
-  kind: CloneSet
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: apps
+  kind: AdvancedCronJob
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
   version: v1alpha1
-- group: apps
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: apps
   kind: BroadcastJob
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
   version: v1alpha1
-- group: apps
-  kind: SidecarSet
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: apps
+  kind: CloneSet
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
   version: v1alpha1
-- group: apps
-  kind: StatefulSet
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: apps
+  kind: ContainerRecreateRequest
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
   version: v1alpha1
-- group: apps
-  kind: UnitedDeployment
-  version: v1alpha1
-- group: apps
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: apps
   kind: DaemonSet
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
   version: v1alpha1
-- group: apps
-  kind: NodeImage
-  version: v1alpha1
-- group: apps
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: apps
   kind: ImagePullJob
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
   version: v1alpha1
-version: "2"
+- api:
+    crdVersion: v1
+  domain: kruise.io
+  group: apps
+  kind: NodeImage
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+  domain: kruise.io
+  group: apps
+  kind: ResourceDistribution
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+  domain: kruise.io
+  group: apps
+  kind: SidecarSet
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: apps
+  kind: StatefulSet
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: apps
+  kind: UnitedDeployment
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: apps
+  kind: WorkloadSpread
+  path: github.com/openkruise/kruise/apis/apps/v1alpha1
+  version: v1alpha1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: apps
+  kind: StatefulSet
+  path: github.com/openkruise/kruise/apis/apps/v1beta1
+  version: v1beta1
+- api:
+    crdVersion: v1
+    namespaced: true
+  domain: kruise.io
+  group: policy
+  kind: PodUnavailableBudget
+  path: github.com/openkruise/kruise/apis/policy/v1alpha1
+  version: v1alpha1
+version: "3"

README-zh_CN.md

@@ -1,4 +1,4 @@
-# OpenKruise/Kruise
+# Kruise
 
 [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
 [![Go Report Card](https://goreportcard.com/badge/github.com/openkruise/kruise)](https://goreportcard.com/report/github.com/openkruise/kruise)
@@ -10,72 +10,83 @@
 
 [English](./README.md) | 简体中文
 
-|![notification](docs/img/bell-outline-badge.svg) 最新进展：|
-|------------------|
-|Oct 1st, 2020. Kruise v0.6.1 发布! 提供一系列增强 feature 和 bugfix 比如 CloneSet lifecycle hook 、UnitedDeployment 支持 CloneSet 等，详情参见 [CHANGELOG](CHANGELOG.md).|
-|Aug 19th, 2020. Kruise v0.6.0 发布! 升级新的项目结构和依赖，并提供新的 Advanced DaemonSet 控制器，详情参见 [CHANGELOG](CHANGELOG.md).|
-|May 19th, 2020. Kruise v0.5.0 发布! CloneSet 支持 `maxSurge` 策略、为 StatefulSet/SidecarSet 修复部分 bug，详情参见 [CHANGELOG](CHANGELOG.md).|
-|Mar 20th, 2020. Kruise v0.4.1 发布! 为 Advanced StatefulSet 和 CloneSet 提供了 **优雅原地升级** 功能，详情参见 [CHANGELOG](CHANGELOG.md).|
-
 ## 介绍
 
-Kruise 是 OpenKruise (官网: [https://openkruise.io](https://openkruise.io)) 中的核心项目之一，它提供一套在[Kubernetes核心控制器](https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/)之外的扩展 workload 管理和实现。
+OpenKruise (官网: [https://openkruise.io](https://openkruise.io)) 是CNCF([Cloud Native Computing Foundation](https://cncf.io/)) 的孵化项目。
+它提供一套在 [Kubernetes核心控制器](https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/) 之外的扩展工作负载、应用管理能力。
 
-目前，Kruise 提供了以下 workload 控制器：
+## 核心能力
 
-- [CloneSet](https://openkruise.io/zh-cn/docs/cloneset.html): 提供了更加高效、确定可控的应用管理和部署能力，支持优雅原地升级、指定删除、发布顺序可配置、并行/灰度发布等丰富的策略，可以满足更多样化的应用场景。
+- **高级工作负载**
 
-- [Advanced StatefulSet](https://openkruise.io/zh-cn/docs/advanced_statefulset.html): 基于原生 [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) 之上的增强版本，默认行为与原生完全一致，在此之外提供了原地升级、并行发布（最大不可用）、发布暂停等功能。
+  通用工作负载能帮助你管理 stateless(无状态)、stateful(有状态)、daemon 类型和作业类的应用。
 
-- [SidecarSet](https://openkruise.io/zh-cn/docs/sidecarset.html): 对 sidecar 容器做统一管理，在满足 selector 条件的 Pod 中注入指定的 sidecar 容器。
+  它们不仅支持类似于 Kubernetes 原生 Workloads 的基础功能，还提供了如 **原地升级**、**可配置的扩缩容/发布策略**、**并发操作** 等。
 
-- [UnitedDeployment](https://openkruise.io/zh-cn/docs/uniteddeployment.html): 通过多个 subset workload 将应用部署到多个可用区。
+  - [**CloneSet** - 无状态应用](https://openkruise.io/zh/docs/user-manuals/cloneset/)
+  - [**Advanced StatefulSet** - 有状态应用](https://openkruise.io/zh/docs/user-manuals/advancedstatefulset)
+  - [**Advanced DaemonSet** - daemon 类型应用](https://openkruise.io/zh/docs/user-manuals/advanceddaemonset)
+  - [**BroadcastJob** - 部署任务到一批特定节点上](https://openkruise.io/zh/docs/user-manuals/broadcastjob)
+  - [**AdvancedCronJob** - 周期性地创建 Job 或 BroadcastJob](https://openkruise.io/zh/docs/user-manuals/advancedcronjob)
 
-- [BroadcastJob](https://openkruise.io/zh-cn/docs/broadcastjob.html): 配置一个 job，在集群中所有满足条件的 Node 上都跑一个 Pod 任务。
+- **Sidecar 容器管理**
 
-- [Advanced DaemonSet](https://openkruise.io/zh-cn/docs/advanced_daemonset.html): 基于原生 DaemonSet 之上的增强版本，默认行为与原生一致，在此之外提供了灰度分批、按 Node label 选择、暂停、热升级等发布策略。
+  Kruise通过**SidecarSet**简化了Sidecar的注入， 并提供了sidecar原地升级的能力。另外， Kruise提供了增强的sidecar启动、退出的控制
 
-项目的 **roadmap** 参考[这里](https://github.com/openkruise/kruise/projects)。
-[Video](https://www.youtube.com/watch?v=elB7reZ6eAQ) by [Lachlan Evenson](https://github.com/lachie83) 是一个对于新人很友好的 demo。
+  - [**SidecarSet** - 定义和升级你的 sidecar 容器](https://openkruise.io/zh/docs/user-manuals/sidecarset)
+  - [**Container Launch Priority** 控制sidecar启动顺序](https://openkruise.io/zh/docs/user-manuals/containerlaunchpriority)
+  - [**Sidecar Job Terminator** 当 Job 类 Pod 主容器退出后，Terminator Sidecar容器](https://openkruise.io/zh/docs/user-manuals/jobsidecarterminator)
 
-## 核心功能
+- **多区域管理**
 
-- **原地升级**
+  它可以帮助你在一个 Kubernetes 集群中的多个区域上部署应用，比如 不同的 node 资源池、可用区、机型架构（x86 & arm）、节点类型（kubelet & virtual kubelet）等。
 
-    原地升级是一种可以避免删除、新建 Pod 的升级镜像能力。它比原生 Deployment/StatefulSet 的重建 Pod 升级更快、更高效，并且避免对 Pod 中其他不需要更新的容器造成干扰。
+  这里我们提供两种不同的方式：
 
-- **Sidecar 管理**
+  - [**WorkloadSpread** - 旁路地分发 workload 创建的 pods](https://openkruise.io/zh/docs/user-manuals/workloadspread)
+  - [**UnitedDeployment** - 一个新的 workload 来管理多个下属的 workloads](https://openkruise.io/zh/docs/user-manuals/uniteddeployment)
 
-    支持在一个单独的 CR 中定义 sidecar 容器，OpenKruise 能够帮你把这些 Sidecar 容器注入到所有符合条件的 Pod 中。这个过程和 Istio 的注入很相似，但是你可以管理任意你关心的 Sidecar。
+- **增强运维能力**
 
-- **跨多可用区部署**
+  - [原地重启 pod 中的容器](https://openkruise.io/zh/docs/user-manuals/containerrecreaterequest)
+  - [指定的一批节点上拉取镜像](https://openkruise.io/zh/docs/user-manuals/imagepulljob)
+  - [**ResourceDistribution** 支持 Secret、Configmaps 资源跨 Namespace 分发](https://openkruise.io/zh/docs/user-manuals/resourcedistribution)
+  - [**PersistentPodState** 保持Pod的一些状态，比如："固定IP调度"](https://openkruise.io/zh/docs/user-manuals/persistentpodstate)
+  - [**PodProbeMarker** 提供自定义Probe探测的能力](https://openkruise.io/zh/docs/user-manuals/podprobemarker)
 
-    定义一个跨多个可用区的全局 workload，容器，OpenKruise 会帮你在每个可用区创建一个对应的下属 workload。你可以统一管理他们的副本数、版本、甚至针对不同可用区采用不同的发布策略。
+- **应用安全防护**
 
-- **...**
+  - [保护 Kubernetes 资源及应用 pods 不被级联删除](https://openkruise.io/zh/docs/user-manuals/deletionprotection)
+  - [**PodUnavailableBudget** - 覆盖更多的 Voluntary Disruption 场景，提供应用更加强大的防护能力](https://openkruise.io/zh/docs/user-manuals/podunavailablebudget)
 
 ## 快速开始
 
-想要快速使用 OpenKruise 非常简单！
-对于版本高于 v1.12+ 的 Kubernetes 集群来说，只要使用 helm v3.1.0+ 执行安装即可：
+你可以在 [OpenKruise website](https://openkruise.io/zh/docs/) 查看到完整的文档集。
 
-```bash
-# Kubernetes 版本低于 1.14
-helm install kruise https://github.com/openkruise/kruise/releases/download/v0.6.1/kruise-chart.tgz --disable-openapi-validation
+- 安装/升级 Kruise [稳定版本](https://openkruise.io/docs/installation)
+- 安装/升级 Kruise [最新版本（包括 alpha/beta/rc）](https://openkruise.io/docs/next/installation)
 
-# Kubernetes 版本大于等于 1.15
-helm install kruise https://github.com/openkruise/kruise/releases/download/v0.6.1/kruise-chart.tgz
-```
+### 在阿里云上快速体验
 
-注意直接安装 chart 会使用默认的 template values，你也可以根据你的集群情况指定一些特殊配置，比如修改 resources 限制或者只启用某些特定的控制器能力。
+- 3分钟内在阿里云上创建 Kruise 体验环境:
 
-更多细节可以查看 [quick-start手册](https://openkruise.io/zh-cn/docs/quick_start.html)
+  <a href="https://acs.console.aliyun.com/quick-deploy?repo=openkruise/charts&branch=master&paths=%5B%22versions/kruise/1.7.3%22%5D" target="_blank">
+    <img src="https://img.alicdn.com/imgextra/i1/O1CN01aiPSuA1Wiz7wkgF5u_!!6000000002823-55-tps-399-70.svg" width="200" alt="Deploy on Alibaba Cloud">
+  </a>
 
-## 文档
+## 用户
 
-你可以在 [OpenKruise website](https://openkruise.io/zh-cn/docs/what_is_openkruise.html) 查看到完整的文档集。
+登记: [如果贵司正在使用 Kruise 请留言](https://github.com/openkruise/kruise/issues/289)
 
-我们也提供了 [**tutorials**](./docs/tutorial/README.md) 来示范如何使用 Kruise 控制器。
+- 阿里巴巴集团, 蚂蚁集团, 斗鱼TV, 申通, Boss直聘
+- 杭银消费, 万翼科技, 多点, Bringg, 佐疆科技
+- Lyft, 携程, 享住智慧, VIPKID, 掌门1对1
+- 小红书, 比心, 永辉科技中心, 跟谁学, 哈啰出行
+- Spectro Cloud, 艾佳生活, Arkane Systems, 滴普科技, 火花思维
+- OPPO, 苏宁, 欢聚时代, 汇量科技, 深圳凤凰木网络有限公司
+- 小米, 网易, 美团金融, 虾皮购物, e签宝
+- LinkedIn, 雪球, 兴盛优选, Wholee, LilithGames, Baidu
+- Bilibili, 冠赢互娱, MeiTuan, 同城
 
 ## 贡献
 
@@ -85,13 +96,20 @@ helm install kruise https://github.com/openkruise/kruise/releases/download/v0.6.
 
 活跃的社区途径：
 
-- Slack: [channel address](https://join.slack.com/t/kruise-workspace/shared_invite/enQtNjU5NzQ0ODcyNjYzLWJlZGJiZjUwNGU5Y2U2ODI3N2JiODI4N2M1OWFlOTgzMDgyOWVkZGRjNzdmZTBjYzgxZmM5MjAyNjhhZTdmMjQ)
-- 钉钉讨论群
+- Slack: [OpenKruise channel](https://kubernetes.slack.com/channels/openkruise) (*English*)
+- 钉钉：搜索群ID `23330762` (*Chinese*)
+- 微信：添加用户 `openkruise` 并让机器人拉你入群 (*Chinese*)
+- 社区双周会 (APAC, *Chinese*):
+  - 周四 19:30 GMT+8 (Asia/Shanghai)
+  - 进入会议(钉钉): 搜索群ID `23330762`
+  - [会议纪要](https://shimo.im/docs/gXqmeQOYBehZ4vqo)
+- Bi-weekly Community Meeting (*English*): TODO
+  - [进入会议(zoom)](https://us02web.zoom.us/j/87059136652?pwd=NlI4UThFWXVRZkxIU0dtR1NINncrQT09)
 
-<div>
-  <img src="docs/img/openkruise-dev-group.JPG" width="280" title="dingtalk">
-</div>
+## 安全
+
+汇报安全漏洞请通过邮箱kubernetes-security@service.aliyun.com, 更多安全细节并参见[SECURITY.md](SECURITY.md)
 
 ## License
 
-Kruise is licensed under the Apache License, Version 2.0. See [LICENSE](./LICENSE.md) for the full license text.
+Kruise is licensed under the Apache License, Version 2.0. See [LICENSE](./LICENSE.md) for the full license text.

README.md

@@ -1,79 +1,94 @@
-# OpenKruise/Kruise
+# Kruise
 
 [![License](https://img.shields.io/badge/license-Apache%202-4EB1BA.svg)](https://www.apache.org/licenses/LICENSE-2.0.html)
 [![Go Report Card](https://goreportcard.com/badge/github.com/openkruise/kruise)](https://goreportcard.com/report/github.com/openkruise/kruise)
 [![CII Best Practices](https://bestpractices.coreinfrastructure.org/projects/2908/badge)](https://bestpractices.coreinfrastructure.org/en/projects/2908)
-[![Build Status](https://travis-ci.org/openkruise/kruise.svg?branch=master)](https://travis-ci.org/openkruise/kruise)
+[![OpenSSF Scorecard](https://api.scorecard.dev/projects/github.com/openkruise/kruise/badge)](https://scorecard.dev/viewer/?uri=github.com/openkruise/kruise)
 [![CircleCI](https://circleci.com/gh/openkruise/kruise.svg?style=svg)](https://circleci.com/gh/openkruise/kruise)
 [![codecov](https://codecov.io/gh/openkruise/kruise/branch/master/graph/badge.svg)](https://codecov.io/gh/openkruise/kruise)
 [![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-v2.0%20adopted-ff69b4.svg)](./CODE_OF_CONDUCT.md)
+[![Gurubase](https://img.shields.io/badge/Gurubase-Ask%20Kruise%20Guru-006BFF)](https://gurubase.io/g/kruise)
 
 English | [简体中文](./README-zh_CN.md)
 
-|![notification](docs/img/bell-outline-badge.svg) What is NEW!|
-|------------------|
-|Oct 1st, 2020. Kruise v0.6.1 is **RELEASED**! It provides various features and bugfix, such as CloneSet lifecycle hook and UnitedDeployment supported CloneSet, please check the [CHANGELOG](CHANGELOG.md) for details.|
-|Aug 19th, 2020. Kruise v0.6.0 is **RELEASED**! It updates Kubernetes dependency and switches to new controller runtime framework. It also supports a new controller called Advanced DaemonSet, please check the [CHANGELOG](CHANGELOG.md) for details.|
-|May 19th, 2020. Kruise v0.5.0 is **RELEASED**! It supports `maxSurge` for CloneSet and fixes bugs for StatefulSet/SidecarSet, please check the [CHANGELOG](CHANGELOG.md) for details.|
-|Mar 20th, 2020. Kruise v0.4.1 is **RELEASED**! It provides **graceful in-place update** for Advanced StatefulSet and CloneSet, please check the [CHANGELOG](CHANGELOG.md) for details.|
-
 ## Introduction
 
-Kruise is the core of the OpenKruise (official site: [https://openkruise.io](https://openkruise.io)) project. It consists of several controllers which extend and complement the [Kubernetes core controllers](https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/) for workload management.
-
-As of now, Kruise offers these workload controllers:
-
-- [CloneSet](https://openkruise.io/en-us/docs/cloneset.html): CloneSet is a workload that mainly focuses on managing stateless applications. It provides a rich set of features for more efficient, deterministic and controlled management, such as in-place update, specified Pod deletion, configurable priority/scatter based update, preUpdate/postUpdate hooks, etc. This [post](https://thenewstack.io/introducing-cloneset-production-grade-kubernetes-deployment-crd/) provides more details about why CloneSet is useful.
-
-- [Advanced StatefulSet](https://openkruise.io/en-us/docs/advanced_statefulset.html): An enhanced version of default [StatefulSet](https://kubernetes.io/docs/concepts/workloads/controllers/statefulset/) with extra functionalities such as `in-place update`, `pause` and `maxUnavailable`.
-
-- [SidecarSet](https://openkruise.io/en-us/docs/sidecarset.html): A controller that injects sidecar containers into the Pod spec based on the Pod selectors. The controller is also responsible for upgrading the sidecar containers.
-
-- [UnitedDeployment](https://openkruise.io/en-us/docs/uniteddeployment.html): This controller manages application Pods spread in multiple fault domains by using multiple workloads.
-
-- [BroadcastJob](https://openkruise.io/en-us/docs/broadcastjob.html): A job that runs Pods to completion across all the nodes in the cluster.
-
-- [Advanced DaemonSet](https://openkruise.io/en-us/docs/advanced_daemonset.html): An enhanced version of default [DaemonSet](https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/) with extra upgrade strategies such as partition, node selector, pause and surging.
-
-The project **roadmap** is actively updated in [here](https://github.com/openkruise/kruise/projects).
-This [video](https://www.youtube.com/watch?v=elB7reZ6eAQ) demo by [Lachlan Evenson](https://github.com/lachie83) is a good introduction for new users.
+OpenKruise  (official site: [https://openkruise.io](https://openkruise.io)) is a CNCF([Cloud Native Computing Foundation](https://cncf.io/)) incubating project.
+It consists of several controllers which extend and complement the [Kubernetes core controllers](https://kubernetes.io/docs/concepts/overview/what-is-kubernetes/) for workload and application management.
 
 ## Key Features
 
-- **In-place update**
+- **Advance Workloads**
 
-    In-place update provides an alternative to update container images without deleting and recreating the Pod. It is much faster compared to the recreate update used by the native Deployment/StatefulSet and has almost no side effects on other running containers.
+  Advance Workloads can help you manage applications of stateless, stateful, daemon and job.
 
-- **Sidecar containers management**
+  They all support not only the basic features which are similar to the original Workloads in Kubernetes, but also more advanced abilities like **in-place update**, **configurable scale/upgrade strategies**, **parallel operations**.
 
-    The Sidecar containers can be simply defined in the SidecarSet custom resource and the controller will inject them into all Pods matched. The implementation is done by using Kubernetes mutating webhooks, similar to what [istio](https://istio.io/latest/docs/setup/additional-setup/sidecar-injection/) does. However, SidecarSet allows you to explicitly manage your own sidecars.
+  - [**CloneSet** for stateless applications](https://openkruise.io/docs/user-manuals/cloneset/)
+  - [**Advanced StatefulSet** for stateful applications](https://openkruise.io/docs/user-manuals/advancedstatefulset)
+  - [**Advanced DaemonSet** for daemon applications](https://openkruise.io/docs/user-manuals/advanceddaemonset)
+  - [**BroadcastJob** for deploying jobs over specific nodes](https://openkruise.io/docs/user-manuals/broadcastjob)
+  - [**AdvancedCronJob** for creating Job or BroadcastJob periodically](https://openkruise.io/docs/user-manuals/advancedcronjob)
 
-- **Multiple fault domains deployment**
+- **Sidecar container Management**
 
-    A global workload can be defined over multiple fault domains, and the Kruise controller will spread a sub workload in each domain. You can manage the domain replicas, sub workload template and update strategies uniformly using the global workload.
+  Kruise simplifies sidecar injection and enables sidecar in-place update. Kruise also enhances the sidecar startup and termination control.
+
+  - [**SidecarSet** for defining and upgrading your own sidecars](https://openkruise.io/docs/user-manuals/sidecarset)
+  - [**Container Launch Priority** to control the container startup orders](https://openkruise.io/docs/user-manuals/containerlaunchpriority)
+  - [**Sidecar Job Terminator** terminates sidecar containers for such job-type Pods when its main containers completed.](https://openkruise.io/docs/user-manuals/jobsidecarterminator)
+
+- **Multi-domain Management**
+
+  This can help you manage applications over nodes with multiple domains,
+  such as different node pools, available zones, architectures(x86 & arm) or node types(kubelet & virtual kubelet).
+
+  Here we provide two different ways:
+
+  - [**WorkloadSpread** for bypass distributing pods in workloads](https://openkruise.io/docs/user-manuals/workloadspread)
+  - [**UnitedDeployment**, a new workload to manage multiple sub-workloads](https://openkruise.io/docs/user-manuals/uniteddeployment)
+
+- **Enhanced Operations**
+
+  - [**ContainerRecreateRequest** provides a way to let users restart/recreate containers in a running pod](https://openkruise.io/docs/user-manuals/containerrecreaterequest)
+  - [**ImagePullJob** pre-download images on specific nodes](https://openkruise.io/docs/user-manuals/imagepulljob)
+  - [**ResourceDistribution** support Secret & ConfigMap resource distribution across namespaces](https://openkruise.io/docs/user-manuals/resourcedistribution)
+  - [**PersistentPodState** is able to persistent states of the Pod, such as "IP Retention"](https://openkruise.io/docs/user-manuals/persistentpodstate)
+  - [**PodProbeMarker** provides the ability to customize the Probe and return the result to the Pod](https://openkruise.io/docs/user-manuals/podprobemarker)
+
+- **Application Protection**
+
+  - [Protect Kubernetes resources and applications' availability from the cascading deletion](https://openkruise.io/docs/user-manuals/deletionprotection)
+  - [**PodUnavailableBudget** for achieving the effect of preventing application disruption or SLA degradation](https://openkruise.io/docs/user-manuals/podunavailablebudget)
 
 ## Quick Start
 
-For a Kubernetes cluster with its version higher than v1.12, you can simply install Kruise with helm v3.1.0+:
+You can view the full documentation from the [OpenKruise website](https://openkruise.io/docs/).
 
-```bash
-# Kubernetes 1.14 and older versions
-helm install kruise https://github.com/openkruise/kruise/releases/download/v0.6.1/kruise-chart.tgz --disable-openapi-validation
+- Install or upgrade Kruise with [the stable version](https://openkruise.io/docs/installation).
+- Install or upgrade Kruise with [the latest version including alpha/beta/rc](https://openkruise.io/docs/next/installation).
 
-# Kubernetes 1.15 and newer versions
-helm install kruise https://github.com/openkruise/kruise/releases/download/v0.6.1/kruise-chart.tgz
-```
+### Get Your Own Demo with Alibaba Cloud
 
-Note that installing this chart directly means it will use the default template values for the kruise-manager.
-You may have to set your specific configurations when it is deployed into a production cluster or you want to enable/disable specific controllers.
+- install Kruise on a Serverless K8S cluster in 3 minutes, try:
 
-For more details, see [quick-start](https://openkruise.io/en-us/docs/quick_start.html).
+  <a href="https://acs.console.aliyun.com/quick-deploy?repo=openkruise/charts&branch=master&paths=%5B%22versions/kruise/1.8.0%22%5D" target="_blank">
+    <img src="https://img.alicdn.com/imgextra/i1/O1CN01aiPSuA1Wiz7wkgF5u_!!6000000002823-55-tps-399-70.svg" width="200" alt="Deploy on Alibaba Cloud">
+  </a>
 
-## Documentation
+## Users
 
-You can view the full documentation from the [OpenKruise website](https://openkruise.io/en-us/docs/what_is_openkruise.html).
+Registration: [Who is using Kruise](https://github.com/openkruise/kruise/issues/289)
 
-We also provide [**tutorials**](./docs/tutorial/README.md) for **ALL** Kruise controllers to demonstrate how to use them.
+- Alibaba Group, Ant Group, DouyuTV, Sto, Boss直聘
+- hangyinxiaofei, vanyitech, Dmall, Bringg, 佐疆科技
+- Lyft, Ctrip, 享住智慧, VIPKID, zhangmen
+- xiaohongshu, bixin, 永辉科技中心, 跟谁学, 哈啰出行
+- Spectro Cloud, ihomefnt, Arkane Systems, Deepexi, 火花思维
+- OPPO, Suning.cn, joyy, Mobvista, 深圳凤凰木网络有限公司
+- xiaomi, Netease, MeiTuan Finance, Shopee, Esign
+- LinkedIn, 雪球, 兴盛优选, Wholee, LilithGames, Baidu
+- Bilibili, 冠赢互娱, MeiTuan, 同城
 
 ## Contributing
 
@@ -83,13 +98,18 @@ You are warmly welcome to hack on Kruise. We have prepared a detailed guide [CON
 
 Active communication channels:
 
-- Slack: [channel address](https://join.slack.com/t/kruise-workspace/shared_invite/enQtNjU5NzQ0ODcyNjYzLWJlZGJiZjUwNGU5Y2U2ODI3N2JiODI4N2M1OWFlOTgzMDgyOWVkZGRjNzdmZTBjYzgxZmM5MjAyNjhhZTdmMjQ)
-- Mailing List: todo
-- Dingtalk Group(钉钉讨论群)
+- Slack: [OpenKruise channel](https://kubernetes.slack.com/channels/openkruise) (*English*)
+- DingTalk：Search GroupID `23330762` (*Chinese*)
+- WeChat: Search User `openkruise` and let the robot invite you (*Chinese*)
+- Bi-weekly Community Meeting (APAC, *Chinese*):
+  - Thursday 19:30 GMT+8 (Asia/Shanghai), [Calendar](https://calendar.google.com/calendar/u/2?cid=MjdtbDZucXA2bjVpNTFyYTNpazV2dW8ybHNAZ3JvdXAuY2FsZW5kYXIuZ29vZ2xlLmNvbQ)
+  - Join Meeting(DingTalk): Search GroupID `23330762` (*Chinese*)
+  - [Notes and agenda](https://shimo.im/docs/gXqmeQOYBehZ4vqo)
+- Bi-weekly Community Meeting (*English*): TODO
+  - [Meeting Link(zoom)](https://us02web.zoom.us/j/87059136652?pwd=NlI4UThFWXVRZkxIU0dtR1NINncrQT09)
 
-<div>
-  <img src="docs/img/openkruise-dev-group.JPG" width="280" title="dingtalk">
-</div>
+## Security
+Please report vulnerabilities by email to kubernetes-security@service.aliyun.com. Also see our [SECURITY.md](SECURITY.md) file for details.
 
 ## License
 

RELEASE-PROCESS.md

@@ -0,0 +1,72 @@
+# Release Process
+
+The release process of a new version of Kruise involves the following:
+
+*(Currently only the [maintainers](https://github.com/openkruise/community/blob/master/MAINTAINERS.md) are eligible to release a new version)*
+
+## 0. Prerequisites
+
+Look at [the last release](https://github.com/openkruise/kruise/releases/latest) in the releases page:
+
+- For example, at the time of writing, it was v1.2.0
+- The next version will thus be v1.3.0
+
+## 1. Changelog
+
+Add a new section in [CHANGELOG.md](./CHANGELOG.md) for the new version that is being released along with the new features, patches and deprecations it introduces.
+
+It should not include every single change but solely what matters to our customers, for example, an issue template that has changed is not important.
+
+## 2. Publish documentation for new version
+
+Publish documentation for new version on [https://openkruise.io](https://openkruise.io).
+
+Fork [openkruise/openkruise.io](https://github.com/openkruise/openkruise.io), create a version from current using `yarn run docusaurus docs:version <VERSION>`.
+
+## 3. Create Kruise release on GitHub
+
+Creating a new release in the [releases page](https://github.com/openkruise/kruise/releases) will trigger a GitHub workflow which will create a new image with the latest code and tagged with the next version (in this example v1.3.0).
+
+## 4. Release template
+
+Every release should use the template provided below to create the GitHub release.
+
+Here's the template:
+
+```
+#### To install or upgrade to the old version, see [installation doc](https://openkruise.io/docs/installation/).
+
+## Changes since v1.2.0
+
+### New CRD and Controller: XXX
+
+### CloneSet
+
+### XXX
+
+### Others
+
+Thanks to all our contributors! 😊
+```
+
+## 5. Prepare our Helm Chart
+
+Before we can release our new Helm chart version, we need to prepare it:
+
+1. Create a new chart version with the updated version and appVersion in our [chart repository](https://github.com/openkruise/charts/tree/master/versions/kruise).
+2. Update the CRDs & Kubernetes resources based on the release artifact (YAML)
+
+## 6. Ship new Helm chart
+
+Submit a PR to merge the new release,
+and then the [Publish action](https://github.com/openkruise/charts/actions/workflows/publish.yaml) will automatically package and publish it.
+
+## 7. Prepare next release
+
+As per our [release governance](./RELEASES.md), we need to create a new shipping cycle in our project settings with a target date in 2 to 3 months after the last cycle.
+
+Lastly, a new [milestone](https://github.com/openkruise/kruise/milestones) should be created to maintain the changes of next release.
+
+## 8. Announcement
+
+Announce the new release in Slack channel, DingTalk and WeChat groups.

RELEASES.md

@@ -0,0 +1,37 @@
+# RELEASES
+
+This document describes how Kruise handles versioning, releases new versions and communicates about them.
+
+Kruise uses semantic versioning so that their users can reliably upgrade according to their needs.
+
+## Release Cycle
+
+Kruise is using a release schedule of 2 to 3 months and uses [GitHub milestones](https://github.com/openkruise/kruise/milestones) to provide an indication of when the next release is planned.
+It can, however, happen that releases get delayed which will be reflected on the milestone.
+
+## Release Process
+
+You can learn about our release process [here](./RELEASE-PROCESS.md).
+
+## Keeping Track Of Changes
+
+We provide a changelog that includes every change per version must be available as part of the repo, for example, here is the [changelog for Kruise](./CHANGELOG.md).
+
+Next to that, every closed issue and PR should be added to a GitHub Milestone that provides an overview of everything that will be included and provide an indication of what the current intended release date is.
+However, the release date is subject to change and only provides an indication.
+
+Lastly, every new release will come with a GitHub Release which provides the exact changelog for that release.
+
+## Get Notified For New Releases
+
+If you want to be notified of new releases, we recommend [watching](https://docs.github.com/en/github/managing-subscriptions-and-notifications-on-github/setting-up-notifications/configuring-notifications#configuring-your-watch-settings-for-an-individual-repository) our [openkruise/kruise repository](https://github.com/openkruise/kruise) on GitHub and ensure you are subscribing for releases.
+
+Next to that, you can join the conversation for every new release on GitHub Discussions.
+
+## Support
+
+- [User Documentation](https://openkruise.io/docs/)
+- Community
+  - [GitHub Discussions](https://github.com/openkruise/kruise/discussions/new)
+  - [Channel #openkruise in Kubernetes Slack](https://kubernetes.slack.com/channels/openkruise) ([registration](http://slack.k8s.io/))
+  - DingTalk：Search GroupID 23330762 (Chinese)

SECURITY.md

@@ -0,0 +1,42 @@
+# Security Policy
+
+## Supported Versions
+
+Kruise commits to supporting the n-2 version minor version of the current major release;
+as well as the last minor version of the previous major release.
+
+Here's an overview:
+
+| Version | Supported           |
+| ------- | ------------------- |
+| 1.16.x   | :white_check_mark: |
+| 1.15.x   | :white_check_mark:  |
+| 1.14.x   | :white_check_mark:  |
+| < 1.14   | :x:                 |
+
+## Prevention
+
+Container images are scanned in every pull request (PR) with [Snyk](https://snyk.io/) to detect new vulnerabilities.
+
+Kruise maintainers are working to improve our prevention by adding additional measures:
+
+- Scan code in master/nightly build and PR/master/nightly for Go.
+- Scan published container images on GitHub Container Registry.
+
+## Disclosures
+
+We strive to ship secure software, but we need the community to help us find security breaches.
+
+In case of a confirmed breach, reporters will get full credit and can be keep in the loop, if preferred.
+
+DO NOT CREATE AN ISSUE to report a security problem. Instead, please send an email to kubernetes-security@service.aliyun.com
+
+### Compensation
+
+We do not provide compensations for reporting vulnerabilities except for eternal gratitude.
+
+## Communication
+
+[GitHub Security Advisory](https://github.com/openkruise/kruise/security/advisories) will be used to communicate during the process of identifying, fixing & shipping the mitigation of the vulnerability.
+
+The advisory will only be made public when the patched version is released to inform the community of the breach and its potential security impact.

SECURITY_CONTACTS.md

@@ -0,0 +1,10 @@
+Defined below are the security persons of contact for this project. If you have questions regarding the triaging and handling of incoming problems, they may be contacted.
+
+The following security contacts have agreed to abide by the [Embargo Policy](embargo-policy.md) and will be removed and replaced if found to be in violation of that agreement.
+
+DO NOT REPORT SECURITY VULNERABILITIES DIRECTLY TO THESE NAMES, USE THE INSTRUCTIONS AT [SECURITY.md](SECURITY.md)
+
+Security Contacts:
+* [Zhen Zhang](mailto:shouchen.zz@alibaba-inc.com)
+* [Mingshan Zhao](mailto:liheng.zms@alibaba-inc.com)
+

SECURITY_RESPONSE.md

@@ -0,0 +1,84 @@
+# Incident response
+
+This serves to define how potential security issues should be triaged, how
+confirmation occurs, providing the notification, and issuing a security advisory
+as well as patch/release.
+
+## Triage
+
+### Identify the problem
+
+Triaging issues allows maintainers to focus resources on the most critically
+impacting problems. Potential security risks should be evaluated against the
+following information:
+
+* Which component(s) of the project is impacted?
+* What kind of problem is this?
+  * privilege escalation
+  * credential access
+  * code execution
+  * exfiltration
+  * lateral movement
+* How complex is the problem?
+* Is user interaction required?
+* What privileges are required for this problem to occur?
+  * admin
+  * general
+* What is the potential impact or consequence of the problem?
+* Does an exploit exist?
+
+Any potential problem that has an exploit, permits privilege escalation, is
+simple, and does not require user interaction should be evaluated immediately.
+[CVSS Version 3.1](https://nvd.nist.gov/vuln-metrics/cvss/v3-calculator) can be
+a helpful tool in evaluating the criticality of reported issues.
+
+### Acknowledge receipt of the problem
+
+Respond to the reporter and notify them that you have received and begun reviewing the problem. Remind them of the [embargo policy](https://github.com/cncf/tag-security/blob/231b87f371274b2d68def2c6a35a719210836191/project-resources/templates/embargo-policy.md), and provide them
+information on who to contact/follow-up with if they have questions. Estimate when they can expect to receive an update. Create a calendar reminder to contact them again by that date to provide an update.
+
+### Replicate the problem
+
+Follow the instructions relayed in the problem. If the instructions are
+insufficient, contact the reporter and ask for more information.
+
+If the problem cannot be replicated, re-engage the reporter, let them know it
+cannot be replicated, and work with them to find a remediation.
+
+If the problem can be replicated, re-evaluate the criticality of the problem, and
+begin working on a remediation. Begin a draft security advisory.
+
+Notify the reporter you were able to replicate the problem and have begun working
+on a fix. Remind them of the [embargo policy](https://github.com/cncf/tag-security/blob/231b87f371274b2d68def2c6a35a719210836191/project-resources/templates/embargo-policy.md). If necessary, notify them of an
+extension (only for very complex problems where remediation cannot be issued
+within the project's specified window).
+
+#### Request a CVE number
+
+If a CVE has already been provided, be sure to include it on the advisory. If
+one has not yet been created, [GitHub functions as a CVE Numbering Authority](https://docs.github.com/en/code-security/security-advisories/about-github-security-advisories#cve-identification-numbers)
+and allows you to request one as part of the security advisory process. Provide
+all required information and as much optional information as we can. The CVE
+number is shown as reserved with no further details until notified it has been
+published.
+
+## Notification
+
+Once the problem has been replicated and a remediation is in place, notify
+subscribed parties with a security bulletin (use [this template](https://github.com/cncf/tag-security/blob/231b87f371274b2d68def2c6a35a719210836191/project-resources/templates/embargo.md)) and the expected publishing date.
+
+## Publish and release
+
+Once a CVE number has been assigned, publish and release the updated
+version/patch. Be sure to notify the CVE group when published so the CVE details
+are searchable. Be sure to give credit to the reporter by *[editing the security
+advisory](https://docs.github.com/en/github/managing-security-vulnerabilities/editing-a-security-advisory#about-credits-for-security-advisories)*
+as they took the time to notify and work with you on the problem!
+
+### Issue a security advisory
+
+Follow the instructions from [GitHub to publish the security advisory previously
+drafted](https://docs.github.com/en/github/managing-security-vulnerabilities/publishing-a-security-advisory).
+
+For more information on security advisories, please refer to the [GitHub
+Article](https://docs.github.com/en/code-security/security-advisories/about-github-security-advisories).

apis/.gitignore

@@ -0,0 +1 @@
+openapi_generated.go

apis/addtoscheme_apps_v1beta1.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apis
+
+import (
+	"github.com/openkruise/kruise/apis/apps/v1beta1"
+)
+
+func init() {
+	// Register the types with the Scheme so the components can map objects to GroupVersionKinds and back
+	AddToSchemes = append(AddToSchemes, v1beta1.SchemeBuilder.AddToScheme)
+}

apis/addtoscheme_policy_v1alpha1.go

@@ -0,0 +1,26 @@
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package apis
+
+import (
+	"github.com/openkruise/kruise/apis/policy/v1alpha1"
+)
+
+func init() {
+	// Register the types with the Scheme so the components can map objects to GroupVersionKinds and back
+	AddToSchemes = append(AddToSchemes, v1alpha1.SchemeBuilder.AddToScheme)
+}

apis/apps/defaults/pod.go

@@ -0,0 +1,225 @@
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package defaults
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	v1 "k8s.io/kubernetes/pkg/apis/core/v1"
+)
+
+// SetDefaultPodSpec sets default pod spec
+func SetDefaultPodSpec(in *corev1.PodSpec) {
+	v1.SetDefaults_PodSpec(in)
+	//default pod volumes
+	SetDefaultPodVolumes(in.Volumes)
+	for i := range in.InitContainers {
+		a := &in.InitContainers[i]
+		v1.SetDefaults_Container(a)
+		for j := range a.Ports {
+			b := &a.Ports[j]
+			if b.Protocol == "" {
+				b.Protocol = "TCP"
+			}
+		}
+		for j := range a.Env {
+			b := &a.Env[j]
+			if b.ValueFrom != nil {
+				if b.ValueFrom.FieldRef != nil {
+					v1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef)
+				}
+			}
+		}
+		v1.SetDefaults_ResourceList(&a.Resources.Limits)
+		v1.SetDefaults_ResourceList(&a.Resources.Requests)
+		if a.LivenessProbe != nil {
+			v1.SetDefaults_Probe(a.LivenessProbe)
+			if a.LivenessProbe.ProbeHandler.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(a.LivenessProbe.ProbeHandler.HTTPGet)
+			}
+		}
+		if a.ReadinessProbe != nil {
+			v1.SetDefaults_Probe(a.ReadinessProbe)
+			if a.ReadinessProbe.ProbeHandler.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.ProbeHandler.HTTPGet)
+			}
+		}
+		if a.StartupProbe != nil {
+			v1.SetDefaults_Probe(a.StartupProbe)
+			if a.StartupProbe.ProbeHandler.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(a.StartupProbe.ProbeHandler.HTTPGet)
+			}
+		}
+		if a.Lifecycle != nil {
+			if a.Lifecycle.PostStart != nil {
+				if a.Lifecycle.PostStart.HTTPGet != nil {
+					v1.SetDefaults_HTTPGetAction(a.Lifecycle.PostStart.HTTPGet)
+				}
+			}
+			if a.Lifecycle.PreStop != nil {
+				if a.Lifecycle.PreStop.HTTPGet != nil {
+					v1.SetDefaults_HTTPGetAction(a.Lifecycle.PreStop.HTTPGet)
+				}
+			}
+		}
+	}
+	for i := range in.Containers {
+		a := &in.Containers[i]
+		// For in-place update, we set default imagePullPolicy to Always
+		if a.ImagePullPolicy == "" {
+			a.ImagePullPolicy = corev1.PullAlways
+		}
+		v1.SetDefaults_Container(a)
+		for j := range a.Ports {
+			b := &a.Ports[j]
+			if b.Protocol == "" {
+				b.Protocol = "TCP"
+			}
+		}
+		for j := range a.Env {
+			b := &a.Env[j]
+			if b.ValueFrom != nil {
+				if b.ValueFrom.FieldRef != nil {
+					v1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef)
+				}
+			}
+		}
+		v1.SetDefaults_ResourceList(&a.Resources.Limits)
+		v1.SetDefaults_ResourceList(&a.Resources.Requests)
+		if a.LivenessProbe != nil {
+			v1.SetDefaults_Probe(a.LivenessProbe)
+			if a.LivenessProbe.ProbeHandler.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(a.LivenessProbe.ProbeHandler.HTTPGet)
+			}
+		}
+		if a.ReadinessProbe != nil {
+			v1.SetDefaults_Probe(a.ReadinessProbe)
+			if a.ReadinessProbe.ProbeHandler.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.ProbeHandler.HTTPGet)
+			}
+		}
+		if a.StartupProbe != nil {
+			v1.SetDefaults_Probe(a.StartupProbe)
+			if a.StartupProbe.ProbeHandler.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(a.StartupProbe.ProbeHandler.HTTPGet)
+			}
+		}
+		if a.Lifecycle != nil {
+			if a.Lifecycle.PostStart != nil {
+				if a.Lifecycle.PostStart.HTTPGet != nil {
+					v1.SetDefaults_HTTPGetAction(a.Lifecycle.PostStart.HTTPGet)
+				}
+			}
+			if a.Lifecycle.PreStop != nil {
+				if a.Lifecycle.PreStop.HTTPGet != nil {
+					v1.SetDefaults_HTTPGetAction(a.Lifecycle.PreStop.HTTPGet)
+				}
+			}
+		}
+	}
+	for i := range in.EphemeralContainers {
+		a := &in.EphemeralContainers[i]
+		for j := range a.EphemeralContainerCommon.Ports {
+			b := &a.EphemeralContainerCommon.Ports[j]
+			if b.Protocol == "" {
+				b.Protocol = "TCP"
+			}
+		}
+		for j := range a.EphemeralContainerCommon.Env {
+			b := &a.EphemeralContainerCommon.Env[j]
+			if b.ValueFrom != nil {
+				if b.ValueFrom.FieldRef != nil {
+					v1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef)
+				}
+			}
+		}
+		v1.SetDefaults_ResourceList(&a.EphemeralContainerCommon.Resources.Limits)
+		v1.SetDefaults_ResourceList(&a.EphemeralContainerCommon.Resources.Requests)
+		if a.EphemeralContainerCommon.LivenessProbe != nil {
+			v1.SetDefaults_Probe(a.EphemeralContainerCommon.LivenessProbe)
+			if a.EphemeralContainerCommon.LivenessProbe.ProbeHandler.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.LivenessProbe.ProbeHandler.HTTPGet)
+			}
+		}
+		if a.EphemeralContainerCommon.ReadinessProbe != nil {
+			v1.SetDefaults_Probe(a.EphemeralContainerCommon.ReadinessProbe)
+			if a.EphemeralContainerCommon.ReadinessProbe.ProbeHandler.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.ReadinessProbe.ProbeHandler.HTTPGet)
+			}
+		}
+		if a.EphemeralContainerCommon.StartupProbe != nil {
+			v1.SetDefaults_Probe(a.EphemeralContainerCommon.StartupProbe)
+			if a.EphemeralContainerCommon.StartupProbe.ProbeHandler.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.StartupProbe.ProbeHandler.HTTPGet)
+			}
+		}
+		if a.EphemeralContainerCommon.Lifecycle != nil {
+			if a.EphemeralContainerCommon.Lifecycle.PostStart != nil {
+				if a.EphemeralContainerCommon.Lifecycle.PostStart.HTTPGet != nil {
+					v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.Lifecycle.PostStart.HTTPGet)
+				}
+			}
+			if a.EphemeralContainerCommon.Lifecycle.PreStop != nil {
+				if a.EphemeralContainerCommon.Lifecycle.PreStop.HTTPGet != nil {
+					v1.SetDefaults_HTTPGetAction(a.EphemeralContainerCommon.Lifecycle.PreStop.HTTPGet)
+				}
+			}
+		}
+	}
+	v1.SetDefaults_ResourceList(&in.Overhead)
+}
+
+func SetDefaultPodVolumes(volumes []corev1.Volume) {
+	for i := range volumes {
+		a := &volumes[i]
+		v1.SetDefaults_Volume(a)
+		if a.VolumeSource.HostPath != nil {
+			v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath)
+		}
+		if a.VolumeSource.Secret != nil {
+			v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret)
+		}
+		if a.VolumeSource.DownwardAPI != nil {
+			v1.SetDefaults_DownwardAPIVolumeSource(a.VolumeSource.DownwardAPI)
+			for j := range a.VolumeSource.DownwardAPI.Items {
+				b := &a.VolumeSource.DownwardAPI.Items[j]
+				if b.FieldRef != nil {
+					v1.SetDefaults_ObjectFieldSelector(b.FieldRef)
+				}
+			}
+		}
+		if a.VolumeSource.ConfigMap != nil {
+			v1.SetDefaults_ConfigMapVolumeSource(a.VolumeSource.ConfigMap)
+		}
+		if a.VolumeSource.Projected != nil {
+			v1.SetDefaults_ProjectedVolumeSource(a.VolumeSource.Projected)
+			for j := range a.VolumeSource.Projected.Sources {
+				b := &a.VolumeSource.Projected.Sources[j]
+				if b.DownwardAPI != nil {
+					for k := range b.DownwardAPI.Items {
+						c := &b.DownwardAPI.Items[k]
+						if c.FieldRef != nil {
+							v1.SetDefaults_ObjectFieldSelector(c.FieldRef)
+						}
+					}
+				}
+				if b.ServiceAccountToken != nil {
+					v1.SetDefaults_ServiceAccountTokenProjection(b.ServiceAccountToken)
+				}
+			}
+		}
+	}
+}

apis/apps/defaults/v1alpha1.go

@@ -0,0 +1,416 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package defaults
+
+import (
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
+	"github.com/openkruise/kruise/apis/apps/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	v1 "k8s.io/kubernetes/pkg/apis/core/v1"
+	"k8s.io/utils/ptr"
+	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+)
+
+const (
+	// ProtectionFinalizer is designed to ensure the GC of resources.
+	ProtectionFinalizer = "apps.kruise.io/deletion-protection"
+)
+
+// SetDefaults_SidecarSet set default values for SidecarSet.
+func SetDefaultsSidecarSet(obj *v1alpha1.SidecarSet) {
+	setSidecarSetUpdateStrategy(&obj.Spec.UpdateStrategy)
+
+	for i := range obj.Spec.InitContainers {
+		setDefaultSidecarContainer(&obj.Spec.InitContainers[i], v1alpha1.AfterAppContainerType)
+	}
+
+	for i := range obj.Spec.Containers {
+		setDefaultSidecarContainer(&obj.Spec.Containers[i], v1alpha1.BeforeAppContainerType)
+	}
+
+	//default setting volumes
+	SetDefaultPodVolumes(obj.Spec.Volumes)
+
+	//default setting history revision limitation
+	SetDefaultRevisionHistoryLimit(&obj.Spec.RevisionHistoryLimit)
+
+	// default patchPolicy is 'Retain'
+	for i := range obj.Spec.PatchPodMetadata {
+		patch := &obj.Spec.PatchPodMetadata[i]
+		if patch.PatchPolicy == "" {
+			patch.PatchPolicy = v1alpha1.SidecarSetRetainPatchPolicy
+		}
+	}
+
+	//default setting injectRevisionStrategy
+	SetDefaultInjectRevision(&obj.Spec.InjectionStrategy)
+}
+
+func SetDefaultInjectRevision(strategy *v1alpha1.SidecarSetInjectionStrategy) {
+	if strategy.Revision != nil && strategy.Revision.Policy == "" {
+		strategy.Revision.Policy = v1alpha1.AlwaysSidecarSetInjectRevisionPolicy
+	}
+}
+
+func SetDefaultRevisionHistoryLimit(revisionHistoryLimit **int32) {
+	if *revisionHistoryLimit == nil {
+		*revisionHistoryLimit = ptr.To(int32(10))
+	}
+}
+
+func setDefaultSidecarContainer(sidecarContainer *v1alpha1.SidecarContainer, injectPolicy v1alpha1.PodInjectPolicyType) {
+	if sidecarContainer.PodInjectPolicy == "" {
+		sidecarContainer.PodInjectPolicy = injectPolicy
+	}
+	if sidecarContainer.UpgradeStrategy.UpgradeType == "" {
+		sidecarContainer.UpgradeStrategy.UpgradeType = v1alpha1.SidecarContainerColdUpgrade
+	}
+	if sidecarContainer.ShareVolumePolicy.Type == "" {
+		sidecarContainer.ShareVolumePolicy.Type = v1alpha1.ShareVolumePolicyDisabled
+	}
+
+	setDefaultContainer(sidecarContainer)
+}
+
+func setSidecarSetUpdateStrategy(strategy *v1alpha1.SidecarSetUpdateStrategy) {
+	if strategy.Type == "" {
+		strategy.Type = v1alpha1.RollingUpdateSidecarSetStrategyType
+	}
+	if strategy.MaxUnavailable == nil {
+		maxUnavailable := intstr.FromInt(1)
+		strategy.MaxUnavailable = &maxUnavailable
+	}
+	if strategy.Partition == nil {
+		partition := intstr.FromInt(0)
+		strategy.Partition = &partition
+	}
+}
+
+func setDefaultContainer(sidecarContainer *v1alpha1.SidecarContainer) {
+	container := &sidecarContainer.Container
+	v1.SetDefaults_Container(container)
+	for i := range container.Ports {
+		p := &container.Ports[i]
+		if p.Protocol == "" {
+			p.Protocol = "TCP"
+		}
+	}
+	for i := range sidecarContainer.TransferEnv {
+		tEnv := &sidecarContainer.TransferEnv[i]
+		if tEnv.SourceContainerNameFrom != nil {
+			v1.SetDefaults_ObjectFieldSelector(tEnv.SourceContainerNameFrom.FieldRef)
+		}
+	}
+	for i := range container.Env {
+		e := &container.Env[i]
+		if e.ValueFrom != nil {
+			if e.ValueFrom.FieldRef != nil {
+				v1.SetDefaults_ObjectFieldSelector(e.ValueFrom.FieldRef)
+			}
+		}
+	}
+	v1.SetDefaults_ResourceList(&container.Resources.Limits)
+	v1.SetDefaults_ResourceList(&container.Resources.Requests)
+	if container.LivenessProbe != nil {
+		v1.SetDefaults_Probe(container.LivenessProbe)
+		if container.LivenessProbe.ProbeHandler.HTTPGet != nil {
+			v1.SetDefaults_HTTPGetAction(container.LivenessProbe.ProbeHandler.HTTPGet)
+		}
+	}
+	if container.ReadinessProbe != nil {
+		v1.SetDefaults_Probe(container.ReadinessProbe)
+		if container.ReadinessProbe.ProbeHandler.HTTPGet != nil {
+			v1.SetDefaults_HTTPGetAction(container.ReadinessProbe.ProbeHandler.HTTPGet)
+		}
+	}
+	if container.Lifecycle != nil {
+		if container.Lifecycle.PostStart != nil {
+			if container.Lifecycle.PostStart.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(container.Lifecycle.PostStart.HTTPGet)
+			}
+		}
+		if container.Lifecycle.PreStop != nil {
+			if container.Lifecycle.PreStop.HTTPGet != nil {
+				v1.SetDefaults_HTTPGetAction(container.Lifecycle.PreStop.HTTPGet)
+			}
+		}
+	}
+}
+
+// SetDefaults_AdvancedCronJob set default values for BroadcastJob.
+func SetDefaultsAdvancedCronJob(obj *v1alpha1.AdvancedCronJob, injectTemplateDefaults bool) {
+	if obj.Spec.Template.JobTemplate != nil && injectTemplateDefaults {
+		SetDefaultPodSpec(&obj.Spec.Template.JobTemplate.Spec.Template.Spec)
+	}
+
+	if obj.Spec.Template.BroadcastJobTemplate != nil && injectTemplateDefaults {
+		SetDefaultPodSpec(&obj.Spec.Template.BroadcastJobTemplate.Spec.Template.Spec)
+	}
+
+	if obj.Spec.ConcurrencyPolicy == "" {
+		obj.Spec.ConcurrencyPolicy = v1alpha1.AllowConcurrent
+	}
+	if obj.Spec.Paused == nil {
+		obj.Spec.Paused = new(bool)
+	}
+
+	if obj.Spec.SuccessfulJobsHistoryLimit == nil {
+		obj.Spec.SuccessfulJobsHistoryLimit = new(int32)
+		*obj.Spec.SuccessfulJobsHistoryLimit = 3
+	}
+	if obj.Spec.FailedJobsHistoryLimit == nil {
+		obj.Spec.FailedJobsHistoryLimit = new(int32)
+		*obj.Spec.FailedJobsHistoryLimit = 1
+	}
+}
+
+// SetDefaults_BroadcastJob set default values for BroadcastJob.
+func SetDefaultsBroadcastJob(obj *v1alpha1.BroadcastJob, injectTemplateDefaults bool) {
+	if injectTemplateDefaults {
+		SetDefaultPodSpec(&obj.Spec.Template.Spec)
+	}
+	if obj.Spec.CompletionPolicy.Type == "" {
+		obj.Spec.CompletionPolicy.Type = v1alpha1.Always
+	}
+
+	if obj.Spec.Parallelism == nil {
+		parallelism := int32(1<<31 - 1)
+		parallelismIntStr := intstr.FromInt(int(parallelism))
+		obj.Spec.Parallelism = &parallelismIntStr
+	}
+
+	if obj.Spec.FailurePolicy.Type == "" {
+		obj.Spec.FailurePolicy.Type = v1alpha1.FailurePolicyTypeFailFast
+	}
+
+	// Default to 'OnFailure' if no restartPolicy is specified
+	if obj.Spec.Template.Spec.RestartPolicy == "" {
+		obj.Spec.Template.Spec.RestartPolicy = corev1.RestartPolicyOnFailure
+	}
+}
+
+// SetDefaults_UnitedDeployment set default values for UnitedDeployment.
+func SetDefaultsUnitedDeployment(obj *v1alpha1.UnitedDeployment, injectTemplateDefaults bool) {
+	if obj.Spec.RevisionHistoryLimit == nil {
+		obj.Spec.RevisionHistoryLimit = ptr.To(int32(10))
+	}
+
+	if len(obj.Spec.UpdateStrategy.Type) == 0 {
+		obj.Spec.UpdateStrategy.Type = v1alpha1.ManualUpdateStrategyType
+	}
+
+	if obj.Spec.UpdateStrategy.Type == v1alpha1.ManualUpdateStrategyType && obj.Spec.UpdateStrategy.ManualUpdate == nil {
+		obj.Spec.UpdateStrategy.ManualUpdate = &v1alpha1.ManualUpdate{}
+	}
+
+	if obj.Spec.Template.StatefulSetTemplate != nil {
+		if injectTemplateDefaults {
+			SetDefaultPodSpec(&obj.Spec.Template.StatefulSetTemplate.Spec.Template.Spec)
+			for i := range obj.Spec.Template.StatefulSetTemplate.Spec.VolumeClaimTemplates {
+				a := &obj.Spec.Template.StatefulSetTemplate.Spec.VolumeClaimTemplates[i]
+				v1.SetDefaults_PersistentVolumeClaim(a)
+				v1.SetDefaults_ResourceList(&a.Spec.Resources.Limits)
+				v1.SetDefaults_ResourceList(&a.Spec.Resources.Requests)
+				v1.SetDefaults_ResourceList(&a.Status.Capacity)
+			}
+		}
+	}
+
+	hasReplicasSettings := false
+	hasCapacitySettings := false
+	for _, subset := range obj.Spec.Topology.Subsets {
+		if subset.Replicas != nil {
+			hasReplicasSettings = true
+		}
+		if subset.MinReplicas != nil || subset.MaxReplicas != nil {
+			hasCapacitySettings = true
+		}
+	}
+	if hasCapacitySettings && !hasReplicasSettings {
+		for i := range obj.Spec.Topology.Subsets {
+			subset := &obj.Spec.Topology.Subsets[i]
+			if subset.MinReplicas == nil {
+				subset.MinReplicas = &intstr.IntOrString{Type: intstr.Int, IntVal: 0}
+			}
+		}
+	}
+}
+
+// SetDefaults_CloneSet set default values for CloneSet.
+func SetDefaultsCloneSet(obj *v1alpha1.CloneSet, injectTemplateDefaults bool) {
+	if obj.Spec.Replicas == nil {
+		obj.Spec.Replicas = ptr.To(int32(1))
+	}
+	if obj.Spec.RevisionHistoryLimit == nil {
+		obj.Spec.RevisionHistoryLimit = ptr.To(int32(10))
+	}
+
+	if injectTemplateDefaults {
+		SetDefaultPodSpec(&obj.Spec.Template.Spec)
+		for i := range obj.Spec.VolumeClaimTemplates {
+			a := &obj.Spec.VolumeClaimTemplates[i]
+			v1.SetDefaults_PersistentVolumeClaim(a)
+			v1.SetDefaults_ResourceList(&a.Spec.Resources.Limits)
+			v1.SetDefaults_ResourceList(&a.Spec.Resources.Requests)
+			v1.SetDefaults_ResourceList(&a.Status.Capacity)
+		}
+	}
+
+	switch obj.Spec.UpdateStrategy.Type {
+	case "":
+		obj.Spec.UpdateStrategy.Type = v1alpha1.RecreateCloneSetUpdateStrategyType
+	case v1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType, v1alpha1.InPlaceOnlyCloneSetUpdateStrategyType:
+		if obj.Spec.UpdateStrategy.InPlaceUpdateStrategy == nil {
+			obj.Spec.UpdateStrategy.InPlaceUpdateStrategy = &appspub.InPlaceUpdateStrategy{}
+		}
+	}
+
+	if obj.Spec.UpdateStrategy.Partition == nil {
+		partition := intstr.FromInt(0)
+		obj.Spec.UpdateStrategy.Partition = &partition
+	}
+	if obj.Spec.UpdateStrategy.MaxUnavailable == nil {
+		maxUnavailable := intstr.FromString(v1alpha1.DefaultCloneSetMaxUnavailable)
+		obj.Spec.UpdateStrategy.MaxUnavailable = &maxUnavailable
+	}
+	if obj.Spec.UpdateStrategy.MaxSurge == nil {
+		maxSurge := intstr.FromInt(0)
+		obj.Spec.UpdateStrategy.MaxSurge = &maxSurge
+	}
+}
+
+// SetDefaults_DaemonSet set default values for DaemonSet.
+func SetDefaultsDaemonSet(obj *v1alpha1.DaemonSet) {
+	if obj.Spec.BurstReplicas == nil {
+		BurstReplicas := intstr.FromInt(250)
+		obj.Spec.BurstReplicas = &BurstReplicas
+	}
+
+	if obj.Spec.UpdateStrategy.Type == "" {
+		obj.Spec.UpdateStrategy.Type = v1alpha1.RollingUpdateDaemonSetStrategyType
+	}
+	if obj.Spec.UpdateStrategy.Type == v1alpha1.RollingUpdateDaemonSetStrategyType {
+		if obj.Spec.UpdateStrategy.RollingUpdate == nil {
+			obj.Spec.UpdateStrategy.RollingUpdate = &v1alpha1.RollingUpdateDaemonSet{}
+		}
+
+		// Make it compatible with the predicated Surging
+		if obj.Spec.UpdateStrategy.RollingUpdate.Type == v1alpha1.DeprecatedSurgingRollingUpdateType {
+			if obj.Spec.UpdateStrategy.RollingUpdate.MaxSurge == nil {
+				maxSurge := intstr.FromInt(1)
+				obj.Spec.UpdateStrategy.RollingUpdate.MaxSurge = &maxSurge
+			}
+			if obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable == nil {
+				maxUnavailable := intstr.FromInt(0)
+				obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable = &maxUnavailable
+			}
+		}
+
+		// Default and convert to Standard
+		if obj.Spec.UpdateStrategy.RollingUpdate.Type == "" || obj.Spec.UpdateStrategy.RollingUpdate.Type == v1alpha1.DeprecatedSurgingRollingUpdateType {
+			obj.Spec.UpdateStrategy.RollingUpdate.Type = v1alpha1.StandardRollingUpdateType
+		}
+
+		if obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable == nil && obj.Spec.UpdateStrategy.RollingUpdate.MaxSurge == nil {
+			maxUnavailable := intstr.FromInt(1)
+			obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable = &maxUnavailable
+			MaxSurge := intstr.FromInt(0)
+			obj.Spec.UpdateStrategy.RollingUpdate.MaxSurge = &MaxSurge
+		}
+	}
+
+	if obj.Spec.RevisionHistoryLimit == nil {
+		obj.Spec.RevisionHistoryLimit = new(int32)
+		*obj.Spec.RevisionHistoryLimit = 10
+	}
+}
+
+// SetDefaultPod sets default pod
+func SetDefaultPod(in *corev1.Pod) {
+	SetDefaultPodSpec(&in.Spec)
+	if in.Spec.EnableServiceLinks == nil {
+		enableServiceLinks := corev1.DefaultEnableServiceLinks
+		in.Spec.EnableServiceLinks = &enableServiceLinks
+	}
+}
+
+// SetDefaults_NodeImage set default values for NodeImage.
+func SetDefaultsNodeImage(obj *v1alpha1.NodeImage) {
+	now := metav1.Now()
+	for name, imageSpec := range obj.Spec.Images {
+		for i := range imageSpec.Tags {
+			tagSpec := &imageSpec.Tags[i]
+			if tagSpec.CreatedAt == nil {
+				tagSpec.CreatedAt = &now
+			}
+			if tagSpec.PullPolicy == nil {
+				tagSpec.PullPolicy = &v1alpha1.ImageTagPullPolicy{}
+			}
+			SetDefaultsImageTagPullPolicy(tagSpec.PullPolicy)
+		}
+		obj.Spec.Images[name] = imageSpec
+	}
+}
+
+func SetDefaultsImageTagPullPolicy(obj *v1alpha1.ImageTagPullPolicy) {
+	if obj.TimeoutSeconds == nil {
+		obj.TimeoutSeconds = ptr.To(int32(600))
+	}
+	if obj.BackoffLimit == nil {
+		obj.BackoffLimit = ptr.To(int32(3))
+	}
+}
+
+// SetDefaults_ImagePullJob set default values for ImagePullJob.
+func SetDefaultsImagePullJob(obj *v1alpha1.ImagePullJob, addProtection bool) {
+	if obj.Spec.CompletionPolicy.Type == "" {
+		obj.Spec.CompletionPolicy.Type = v1alpha1.Always
+	}
+	if obj.Spec.PullPolicy == nil {
+		obj.Spec.PullPolicy = &v1alpha1.PullPolicy{}
+	}
+	if obj.Spec.PullPolicy.TimeoutSeconds == nil {
+		obj.Spec.PullPolicy.TimeoutSeconds = ptr.To(int32(600))
+	}
+	if obj.Spec.PullPolicy.BackoffLimit == nil {
+		obj.Spec.PullPolicy.BackoffLimit = ptr.To(int32(3))
+	}
+	if obj.Spec.ImagePullPolicy == "" {
+		obj.Spec.ImagePullPolicy = v1alpha1.PullIfNotPresent
+	}
+	if addProtection {
+		controllerutil.AddFinalizer(obj, ProtectionFinalizer)
+	}
+}
+
+// SetDefaultsImageListPullJob  set default values for ImageListPullJob.
+func SetDefaultsImageListPullJob(obj *v1alpha1.ImageListPullJob) {
+	if obj.Spec.CompletionPolicy.Type == "" {
+		obj.Spec.CompletionPolicy.Type = v1alpha1.Always
+	}
+	if obj.Spec.PullPolicy == nil {
+		obj.Spec.PullPolicy = &v1alpha1.PullPolicy{}
+	}
+	if obj.Spec.PullPolicy.TimeoutSeconds == nil {
+		obj.Spec.PullPolicy.TimeoutSeconds = ptr.To(int32(600))
+	}
+	if obj.Spec.PullPolicy.BackoffLimit == nil {
+		obj.Spec.PullPolicy.BackoffLimit = ptr.To(int32(3))
+	}
+}

apis/apps/defaults/v1beta1.go

@@ -0,0 +1,95 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package defaults
+
+import (
+	appsv1 "k8s.io/api/apps/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	v1 "k8s.io/kubernetes/pkg/apis/core/v1"
+	"k8s.io/utils/ptr"
+
+	"github.com/openkruise/kruise/apis/apps/v1beta1"
+	"github.com/openkruise/kruise/pkg/features"
+	utilfeature "github.com/openkruise/kruise/pkg/util/feature"
+)
+
+// SetDefaultsStatefulSet set default values for StatefulSet.
+func SetDefaultsStatefulSet(obj *v1beta1.StatefulSet, injectTemplateDefaults bool) {
+	if len(obj.Spec.PodManagementPolicy) == 0 {
+		obj.Spec.PodManagementPolicy = appsv1.OrderedReadyPodManagement
+	}
+
+	if obj.Spec.UpdateStrategy.Type == "" {
+		obj.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType
+	}
+
+	if obj.Spec.UpdateStrategy.Type == appsv1.RollingUpdateStatefulSetStrategyType {
+		if obj.Spec.UpdateStrategy.RollingUpdate == nil {
+			// UpdateStrategy.RollingUpdate will take default values below.
+			obj.Spec.UpdateStrategy.RollingUpdate = &v1beta1.RollingUpdateStatefulSetStrategy{}
+		}
+		if obj.Spec.UpdateStrategy.RollingUpdate.Partition == nil {
+			obj.Spec.UpdateStrategy.RollingUpdate.Partition = ptr.To(int32(0))
+		}
+		if obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable == nil {
+			maxUnavailable := intstr.FromInt(1)
+			obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable = &maxUnavailable
+		}
+		if obj.Spec.UpdateStrategy.RollingUpdate.PodUpdatePolicy == "" {
+			obj.Spec.UpdateStrategy.RollingUpdate.PodUpdatePolicy = v1beta1.RecreatePodUpdateStrategyType
+		}
+		if obj.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds == nil {
+			obj.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds = ptr.To(int32(0))
+		}
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoDeletePVC) {
+		if obj.Spec.PersistentVolumeClaimRetentionPolicy == nil {
+			obj.Spec.PersistentVolumeClaimRetentionPolicy = &v1beta1.StatefulSetPersistentVolumeClaimRetentionPolicy{}
+		}
+		if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted) == 0 {
+			obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenDeleted = v1beta1.RetainPersistentVolumeClaimRetentionPolicyType
+		}
+		if len(obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled) == 0 {
+			obj.Spec.PersistentVolumeClaimRetentionPolicy.WhenScaled = v1beta1.RetainPersistentVolumeClaimRetentionPolicyType
+		}
+	}
+
+	if utilfeature.DefaultFeatureGate.Enabled(features.StatefulSetAutoResizePVCGate) {
+		if obj.Spec.VolumeClaimUpdateStrategy.Type == "" {
+			obj.Spec.VolumeClaimUpdateStrategy.Type = v1beta1.OnPVCDeleteVolumeClaimUpdateStrategyType
+		}
+	}
+
+	if obj.Spec.Replicas == nil {
+		obj.Spec.Replicas = ptr.To(int32(1))
+	}
+	if obj.Spec.RevisionHistoryLimit == nil {
+		obj.Spec.RevisionHistoryLimit = ptr.To(int32(10))
+	}
+
+	if injectTemplateDefaults {
+		SetDefaultPodSpec(&obj.Spec.Template.Spec)
+		for i := range obj.Spec.VolumeClaimTemplates {
+			a := &obj.Spec.VolumeClaimTemplates[i]
+			v1.SetDefaults_PersistentVolumeClaim(a)
+			v1.SetDefaults_ResourceList(&a.Spec.Resources.Limits)
+			v1.SetDefaults_ResourceList(&a.Spec.Resources.Requests)
+			v1.SetDefaults_ResourceList(&a.Status.Capacity)
+		}
+	}
+}

apis/apps/pub/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:openapi-gen=true
+// +kubebuilder:object:generate=true
+package pub

apis/apps/pub/inplace_update.go

@@ -0,0 +1,168 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pub
+
+import (
+	"encoding/json"
+
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	// InPlaceUpdateReady must be added into template.spec.readinessGates when pod podUpdatePolicy
+	// is InPlaceIfPossible or InPlaceOnly. The condition in podStatus will be updated to False before in-place
+	// updating and updated to True after the update is finished. This ensures pod to remain at NotReady state while
+	// in-place update is happening.
+	InPlaceUpdateReady v1.PodConditionType = "InPlaceUpdateReady"
+
+	// InPlaceUpdateStateKey records the state of inplace-update.
+	// The value of annotation is InPlaceUpdateState.
+	InPlaceUpdateStateKey string = "apps.kruise.io/inplace-update-state"
+	// TODO: will be removed since v1.0.0
+	InPlaceUpdateStateKeyOld string = "inplace-update-state"
+
+	// InPlaceUpdateGraceKey records the spec that Pod should be updated when
+	// grace period ends.
+	InPlaceUpdateGraceKey string = "apps.kruise.io/inplace-update-grace"
+	// TODO: will be removed since v1.0.0
+	InPlaceUpdateGraceKeyOld string = "inplace-update-grace"
+
+	// RuntimeContainerMetaKey is a key in pod annotations. Kruise-daemon should report the
+	// states of runtime containers into its value, which is a structure JSON of RuntimeContainerMetaSet type.
+	RuntimeContainerMetaKey = "apps.kruise.io/runtime-containers-meta"
+)
+
+// InPlaceUpdateState records latest inplace-update state, including old statuses of containers.
+type InPlaceUpdateState struct {
+	// Revision is the updated revision hash.
+	Revision string `json:"revision"`
+
+	// UpdateTimestamp is the start time when the in-place update happens.
+	UpdateTimestamp metav1.Time `json:"updateTimestamp"`
+
+	// LastContainerStatuses records the before-in-place-update container statuses. It is a map from ContainerName
+	// to InPlaceUpdateContainerStatus
+	LastContainerStatuses map[string]InPlaceUpdateContainerStatus `json:"lastContainerStatuses"`
+
+	// UpdateEnvFromMetadata indicates there are envs from annotations/labels that should be in-place update.
+	UpdateEnvFromMetadata bool `json:"updateEnvFromMetadata,omitempty"`
+
+	// UpdateResources indicates there are resources that should be in-place update.
+	UpdateResources bool `json:"updateResources,omitempty"`
+
+	// UpdateImages indicates there are images that should be in-place update.
+	UpdateImages bool `json:"updateImages,omitempty"`
+
+	// NextContainerImages is the containers with lower priority that waiting for in-place update images in next batch.
+	NextContainerImages map[string]string `json:"nextContainerImages,omitempty"`
+
+	// NextContainerRefMetadata is the containers with lower priority that waiting for in-place update labels/annotations in next batch.
+	NextContainerRefMetadata map[string]metav1.ObjectMeta `json:"nextContainerRefMetadata,omitempty"`
+
+	// NextContainerResources is the containers with lower priority that waiting for in-place update resources in next batch.
+	NextContainerResources map[string]v1.ResourceRequirements `json:"nextContainerResources,omitempty"`
+
+	// PreCheckBeforeNext is the pre-check that must pass before the next containers can be in-place update.
+	PreCheckBeforeNext *InPlaceUpdatePreCheckBeforeNext `json:"preCheckBeforeNext,omitempty"`
+
+	// ContainerBatchesRecord records the update batches that have patched in this revision.
+	ContainerBatchesRecord []InPlaceUpdateContainerBatch `json:"containerBatchesRecord,omitempty"`
+}
+
+// InPlaceUpdatePreCheckBeforeNext contains the pre-check that must pass before the next containers can be in-place update.
+type InPlaceUpdatePreCheckBeforeNext struct {
+	ContainersRequiredReady []string `json:"containersRequiredReady,omitempty"`
+}
+
+// InPlaceUpdateContainerBatch indicates the timestamp and containers for a batch update
+type InPlaceUpdateContainerBatch struct {
+	// Timestamp is the time for this update batch
+	Timestamp metav1.Time `json:"timestamp"`
+	// Containers is the name list of containers for this update batch
+	Containers []string `json:"containers"`
+}
+
+// InPlaceUpdateContainerStatus records the statuses of the container that are mainly used
+// to determine whether the InPlaceUpdate is completed.
+type InPlaceUpdateContainerStatus struct {
+	ImageID string `json:"imageID,omitempty"`
+}
+
+// InPlaceUpdateStrategy defines the strategies for in-place update.
+type InPlaceUpdateStrategy struct {
+	// GracePeriodSeconds is the timespan between set Pod status to not-ready and update images in Pod spec
+	// when in-place update a Pod.
+	GracePeriodSeconds int32 `json:"gracePeriodSeconds,omitempty"`
+}
+
+func GetInPlaceUpdateState(obj metav1.Object) (string, bool) {
+	if v, ok := obj.GetAnnotations()[InPlaceUpdateStateKey]; ok {
+		return v, ok
+	}
+	v, ok := obj.GetAnnotations()[InPlaceUpdateStateKeyOld]
+	return v, ok
+}
+
+func GetInPlaceUpdateGrace(obj metav1.Object) (string, bool) {
+	if v, ok := obj.GetAnnotations()[InPlaceUpdateGraceKey]; ok {
+		return v, ok
+	}
+	v, ok := obj.GetAnnotations()[InPlaceUpdateGraceKeyOld]
+	return v, ok
+}
+
+func RemoveInPlaceUpdateGrace(obj metav1.Object) {
+	delete(obj.GetAnnotations(), InPlaceUpdateGraceKey)
+	delete(obj.GetAnnotations(), InPlaceUpdateGraceKeyOld)
+}
+
+// RuntimeContainerMetaSet contains all the containers' meta of the Pod.
+type RuntimeContainerMetaSet struct {
+	Containers []RuntimeContainerMeta `json:"containers"`
+}
+
+// RuntimeContainerMeta contains the meta data of a runtime container.
+type RuntimeContainerMeta struct {
+	Name         string                 `json:"name"`
+	ContainerID  string                 `json:"containerID"`
+	RestartCount int32                  `json:"restartCount"`
+	Hashes       RuntimeContainerHashes `json:"hashes"`
+}
+
+// RuntimeContainerHashes contains the hashes of such container.
+type RuntimeContainerHashes struct {
+	// PlainHash is the hash that directly calculated from pod.spec.container[x].
+	// Usually it is calculated by Kubelet and will be in annotation of each runtime container.
+	PlainHash uint64 `json:"plainHash"`
+	// ExtractedEnvFromMetadataHash is the hash that calculated from pod.spec.container[x],
+	// whose envs from annotations/labels have already been extracted to the real values.
+	ExtractedEnvFromMetadataHash uint64 `json:"extractedEnvFromMetadataHash,omitempty"`
+}
+
+func GetRuntimeContainerMetaSet(obj metav1.Object) (*RuntimeContainerMetaSet, error) {
+	str, ok := obj.GetAnnotations()[RuntimeContainerMetaKey]
+	if !ok {
+		return nil, nil
+	}
+
+	s := RuntimeContainerMetaSet{}
+	if err := json.Unmarshal([]byte(str), &s); err != nil {
+		return nil, err
+	}
+	return &s, nil
+}

apis/apps/pub/launch_priority.go

@@ -0,0 +1,36 @@
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pub
+
+const (
+	// ContainerLaunchPriorityEnvName is the env name that users have to define in pod container
+	// to identity the launch priority of this container.
+	ContainerLaunchPriorityEnvName = "KRUISE_CONTAINER_PRIORITY"
+	// ContainerLaunchBarrierEnvName is the env name that Kruise webhook will inject into containers
+	// if the pod have configured launch priority.
+	ContainerLaunchBarrierEnvName = "KRUISE_CONTAINER_BARRIER"
+
+	// ContainerLaunchPriorityKey is the annotation key that users could define in pod annotation
+	// to make containers in pod launched by ordinal.
+	ContainerLaunchPriorityKey = "apps.kruise.io/container-launch-priority"
+	// ContainerLaunchOrdered is the annotation value that indicates containers in pod should be launched by ordinal.
+	ContainerLaunchOrdered = "Ordered"
+
+	// ContainerLaunchPriorityCompletedKey is the annotation indicates the pod has all its priorities
+	// patched into its barrier configmap.
+	ContainerLaunchPriorityCompletedKey = "apps.kruise.io/container-launch-priority-completed"
+)

apis/apps/pub/lifecycle.go

@@ -0,0 +1,63 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pub
+
+const (
+	LifecycleStateKey     = "lifecycle.apps.kruise.io/state"
+	LifecycleTimestampKey = "lifecycle.apps.kruise.io/timestamp"
+
+	// LifecycleStatePreparingNormal means the Pod is created but unavailable.
+	// It will translate to Normal state if Lifecycle.PreNormal is hooked.
+	LifecycleStatePreparingNormal LifecycleStateType = "PreparingNormal"
+	// LifecycleStateNormal is a necessary condition for Pod to be available.
+	LifecycleStateNormal LifecycleStateType = "Normal"
+	// LifecycleStatePreparingUpdate means pod is being prepared to update.
+	// It will translate to Updating state if Lifecycle.InPlaceUpdate is Not hooked.
+	LifecycleStatePreparingUpdate LifecycleStateType = "PreparingUpdate"
+	// LifecycleStateUpdating means the Pod is being updated.
+	// It will translate to Updated state if the in-place update of the Pod is done.
+	LifecycleStateUpdating LifecycleStateType = "Updating"
+	// LifecycleStateUpdated means the Pod is updated, but unavailable.
+	// It will translate to Normal state if Lifecycle.InPlaceUpdate is hooked.
+	LifecycleStateUpdated LifecycleStateType = "Updated"
+	// LifecycleStatePreparingDelete means the Pod is prepared to delete.
+	// The Pod will be deleted by workload if Lifecycle.PreDelete is Not hooked.
+	LifecycleStatePreparingDelete LifecycleStateType = "PreparingDelete"
+)
+
+type LifecycleStateType string
+
+// Lifecycle contains the hooks for Pod lifecycle.
+type Lifecycle struct {
+	// PreDelete is the hook before Pod to be deleted.
+	PreDelete *LifecycleHook `json:"preDelete,omitempty"`
+	// InPlaceUpdate is the hook before Pod to update and after Pod has been updated.
+	InPlaceUpdate *LifecycleHook `json:"inPlaceUpdate,omitempty"`
+	// PreNormal is the hook after Pod to be created and ready to be Normal.
+	PreNormal *LifecycleHook `json:"preNormal,omitempty"`
+}
+
+type LifecycleHook struct {
+	LabelsHandler     map[string]string `json:"labelsHandler,omitempty"`
+	FinalizersHandler []string          `json:"finalizersHandler,omitempty"`
+	// MarkPodNotReady = true means:
+	// - Pod will be set to 'NotReady' at preparingDelete/preparingUpdate state.
+	// - Pod will be restored to 'Ready' at Updated state if it was set to 'NotReady' at preparingUpdate state.
+	// Currently, MarkPodNotReady only takes effect on InPlaceUpdate & PreDelete hook.
+	// Default to false.
+	MarkPodNotReady bool `json:"markPodNotReady,omitempty"`
+}

apis/apps/pub/pod_readiness_gate.go

@@ -0,0 +1,30 @@
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pub
+
+import v1 "k8s.io/api/core/v1"
+
+const (
+	// KruisePodReadyConditionType can support multiple writers, such as:
+	// - ContainerRecreateRequest;
+	// - Workload controller, including CloneSet, Advanced StatefulSet, Advanced Daemonset.
+	//
+	// If its corresponding condition status was set to "False" by multiple writers,
+	// the condition status will be considered as "True" only when all these writers
+	// set it to "True".
+	KruisePodReadyConditionType v1.PodConditionType = "KruisePodReady"
+)

apis/apps/pub/pod_unavailable_label.go

@@ -0,0 +1,40 @@
+/*
+Copyright 2022 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pub
+
+import (
+	"strings"
+)
+
+const (
+	// PubUnavailablePodLabelPrefix indicates if the pod has this label, both kruise workload and
+	// pub will determine that the pod is unavailable, even if pod.status.ready=true.
+	// Main users non-destructive offline and other scenarios
+	PubUnavailablePodLabelPrefix = "unavailable-pod.kruise.io/"
+)
+
+func HasUnavailableLabel(labels map[string]string) bool {
+	if len(labels) == 0 {
+		return false
+	}
+	for key := range labels {
+		if strings.HasPrefix(key, PubUnavailablePodLabelPrefix) {
+			return true
+		}
+	}
+	return false
+}

apis/apps/pub/update_priority.go

@@ -0,0 +1,88 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package pub
+
+import (
+	"fmt"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// UpdatePriorityStrategy is the strategy to define priority for pods update.
+// Only one of orderPriority and weightPriority can be set.
+type UpdatePriorityStrategy struct {
+	// Order priority terms, pods will be sorted by the value of orderedKey.
+	// For example:
+	// ```
+	// orderPriority:
+	// - orderedKey: key1
+	// - orderedKey: key2
+	// ```
+	// First, all pods which have key1 in labels will be sorted by the value of key1.
+	// Then, the left pods which have no key1 but have key2 in labels will be sorted by
+	// the value of key2 and put behind those pods have key1.
+	OrderPriority []UpdatePriorityOrderTerm `json:"orderPriority,omitempty"`
+	// Weight priority terms, pods will be sorted by the sum of all terms weight.
+	WeightPriority []UpdatePriorityWeightTerm `json:"weightPriority,omitempty"`
+}
+
+// UpdatePriorityOrderTerm defines order priority.
+type UpdatePriorityOrderTerm struct {
+	// Calculate priority by value of this key.
+	// Values of this key, will be sorted by GetInt(val). GetInt method will find the last int in value,
+	// such as getting 5 in value '5', getting 10 in value 'sts-10'.
+	OrderedKey string `json:"orderedKey"`
+}
+
+// UpdatePriorityWeightTerm defines weight priority.
+type UpdatePriorityWeightTerm struct {
+	// Weight associated with matching the corresponding matchExpressions, in the range 1-100.
+	Weight int32 `json:"weight"`
+	// MatchSelector is used to select by pod's labels.
+	MatchSelector metav1.LabelSelector `json:"matchSelector"`
+}
+
+// FieldsValidation checks invalid fields in UpdatePriorityStrategy.
+func (strategy *UpdatePriorityStrategy) FieldsValidation() error {
+	if strategy == nil {
+		return nil
+	}
+
+	if len(strategy.WeightPriority) > 0 && len(strategy.OrderPriority) > 0 {
+		return fmt.Errorf("only one of weightPriority and orderPriority can be used")
+	}
+
+	for _, w := range strategy.WeightPriority {
+		if w.Weight < 0 || w.Weight > 100 {
+			return fmt.Errorf("weight must be valid number in the range 1-100")
+		}
+		if w.MatchSelector.Size() == 0 {
+			return fmt.Errorf("selector can not be empty")
+		}
+		if _, err := metav1.LabelSelectorAsSelector(&w.MatchSelector); err != nil {
+			return fmt.Errorf("invalid selector %v", err)
+		}
+	}
+
+	for _, o := range strategy.OrderPriority {
+		if len(o.OrderedKey) == 0 {
+			return fmt.Errorf("order key can not be empty")
+		}
+	}
+
+	return nil
+}

apis/apps/pub/zz_generated.deepcopy.go

@@ -0,0 +1,319 @@
+//go:build !ignore_autogenerated
+
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by controller-gen. DO NOT EDIT.
+
+package pub
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *InPlaceUpdateContainerBatch) DeepCopyInto(out *InPlaceUpdateContainerBatch) {
+	*out = *in
+	in.Timestamp.DeepCopyInto(&out.Timestamp)
+	if in.Containers != nil {
+		in, out := &in.Containers, &out.Containers
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InPlaceUpdateContainerBatch.
+func (in *InPlaceUpdateContainerBatch) DeepCopy() *InPlaceUpdateContainerBatch {
+	if in == nil {
+		return nil
+	}
+	out := new(InPlaceUpdateContainerBatch)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *InPlaceUpdateContainerStatus) DeepCopyInto(out *InPlaceUpdateContainerStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InPlaceUpdateContainerStatus.
+func (in *InPlaceUpdateContainerStatus) DeepCopy() *InPlaceUpdateContainerStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(InPlaceUpdateContainerStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *InPlaceUpdatePreCheckBeforeNext) DeepCopyInto(out *InPlaceUpdatePreCheckBeforeNext) {
+	*out = *in
+	if in.ContainersRequiredReady != nil {
+		in, out := &in.ContainersRequiredReady, &out.ContainersRequiredReady
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InPlaceUpdatePreCheckBeforeNext.
+func (in *InPlaceUpdatePreCheckBeforeNext) DeepCopy() *InPlaceUpdatePreCheckBeforeNext {
+	if in == nil {
+		return nil
+	}
+	out := new(InPlaceUpdatePreCheckBeforeNext)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *InPlaceUpdateState) DeepCopyInto(out *InPlaceUpdateState) {
+	*out = *in
+	in.UpdateTimestamp.DeepCopyInto(&out.UpdateTimestamp)
+	if in.LastContainerStatuses != nil {
+		in, out := &in.LastContainerStatuses, &out.LastContainerStatuses
+		*out = make(map[string]InPlaceUpdateContainerStatus, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.NextContainerImages != nil {
+		in, out := &in.NextContainerImages, &out.NextContainerImages
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.NextContainerRefMetadata != nil {
+		in, out := &in.NextContainerRefMetadata, &out.NextContainerRefMetadata
+		*out = make(map[string]v1.ObjectMeta, len(*in))
+		for key, val := range *in {
+			(*out)[key] = *val.DeepCopy()
+		}
+	}
+	if in.NextContainerResources != nil {
+		in, out := &in.NextContainerResources, &out.NextContainerResources
+		*out = make(map[string]corev1.ResourceRequirements, len(*in))
+		for key, val := range *in {
+			(*out)[key] = *val.DeepCopy()
+		}
+	}
+	if in.PreCheckBeforeNext != nil {
+		in, out := &in.PreCheckBeforeNext, &out.PreCheckBeforeNext
+		*out = new(InPlaceUpdatePreCheckBeforeNext)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ContainerBatchesRecord != nil {
+		in, out := &in.ContainerBatchesRecord, &out.ContainerBatchesRecord
+		*out = make([]InPlaceUpdateContainerBatch, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InPlaceUpdateState.
+func (in *InPlaceUpdateState) DeepCopy() *InPlaceUpdateState {
+	if in == nil {
+		return nil
+	}
+	out := new(InPlaceUpdateState)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *InPlaceUpdateStrategy) DeepCopyInto(out *InPlaceUpdateStrategy) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new InPlaceUpdateStrategy.
+func (in *InPlaceUpdateStrategy) DeepCopy() *InPlaceUpdateStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(InPlaceUpdateStrategy)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *Lifecycle) DeepCopyInto(out *Lifecycle) {
+	*out = *in
+	if in.PreDelete != nil {
+		in, out := &in.PreDelete, &out.PreDelete
+		*out = new(LifecycleHook)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.InPlaceUpdate != nil {
+		in, out := &in.InPlaceUpdate, &out.InPlaceUpdate
+		*out = new(LifecycleHook)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.PreNormal != nil {
+		in, out := &in.PreNormal, &out.PreNormal
+		*out = new(LifecycleHook)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Lifecycle.
+func (in *Lifecycle) DeepCopy() *Lifecycle {
+	if in == nil {
+		return nil
+	}
+	out := new(Lifecycle)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *LifecycleHook) DeepCopyInto(out *LifecycleHook) {
+	*out = *in
+	if in.LabelsHandler != nil {
+		in, out := &in.LabelsHandler, &out.LabelsHandler
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.FinalizersHandler != nil {
+		in, out := &in.FinalizersHandler, &out.FinalizersHandler
+		*out = make([]string, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new LifecycleHook.
+func (in *LifecycleHook) DeepCopy() *LifecycleHook {
+	if in == nil {
+		return nil
+	}
+	out := new(LifecycleHook)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RuntimeContainerHashes) DeepCopyInto(out *RuntimeContainerHashes) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeContainerHashes.
+func (in *RuntimeContainerHashes) DeepCopy() *RuntimeContainerHashes {
+	if in == nil {
+		return nil
+	}
+	out := new(RuntimeContainerHashes)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RuntimeContainerMeta) DeepCopyInto(out *RuntimeContainerMeta) {
+	*out = *in
+	out.Hashes = in.Hashes
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeContainerMeta.
+func (in *RuntimeContainerMeta) DeepCopy() *RuntimeContainerMeta {
+	if in == nil {
+		return nil
+	}
+	out := new(RuntimeContainerMeta)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RuntimeContainerMetaSet) DeepCopyInto(out *RuntimeContainerMetaSet) {
+	*out = *in
+	if in.Containers != nil {
+		in, out := &in.Containers, &out.Containers
+		*out = make([]RuntimeContainerMeta, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RuntimeContainerMetaSet.
+func (in *RuntimeContainerMetaSet) DeepCopy() *RuntimeContainerMetaSet {
+	if in == nil {
+		return nil
+	}
+	out := new(RuntimeContainerMetaSet)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UpdatePriorityOrderTerm) DeepCopyInto(out *UpdatePriorityOrderTerm) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdatePriorityOrderTerm.
+func (in *UpdatePriorityOrderTerm) DeepCopy() *UpdatePriorityOrderTerm {
+	if in == nil {
+		return nil
+	}
+	out := new(UpdatePriorityOrderTerm)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UpdatePriorityStrategy) DeepCopyInto(out *UpdatePriorityStrategy) {
+	*out = *in
+	if in.OrderPriority != nil {
+		in, out := &in.OrderPriority, &out.OrderPriority
+		*out = make([]UpdatePriorityOrderTerm, len(*in))
+		copy(*out, *in)
+	}
+	if in.WeightPriority != nil {
+		in, out := &in.WeightPriority, &out.WeightPriority
+		*out = make([]UpdatePriorityWeightTerm, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdatePriorityStrategy.
+func (in *UpdatePriorityStrategy) DeepCopy() *UpdatePriorityStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(UpdatePriorityStrategy)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UpdatePriorityWeightTerm) DeepCopyInto(out *UpdatePriorityWeightTerm) {
+	*out = *in
+	in.MatchSelector.DeepCopyInto(&out.MatchSelector)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UpdatePriorityWeightTerm.
+func (in *UpdatePriorityWeightTerm) DeepCopy() *UpdatePriorityWeightTerm {
+	if in == nil {
+		return nil
+	}
+	out := new(UpdatePriorityWeightTerm)
+	in.DeepCopyInto(out)
+	return out
+}

apis/apps/v1alpha1/advancedcronjob_types.go

@@ -0,0 +1,163 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	batchv1 "k8s.io/api/batch/v1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const AdvancedCronJobKind = "AdvancedCronJob"
+
+// AdvancedCronJobSpec defines the desired state of AdvancedCronJob
+type AdvancedCronJobSpec struct {
+	// +kubebuilder:validation:MinLength=0
+
+	// The schedule in Cron format, see https://en.wikipedia.org/wiki/Cron.
+	Schedule string `json:"schedule" protobuf:"bytes,1,opt,name=schedule"`
+
+	// The time zone name for the given schedule, see https://en.wikipedia.org/wiki/List_of_tz_database_time_zones.
+	// If not specified, this will default to the time zone of the kruise-controller-manager process.
+	// +optional
+	TimeZone *string `json:"timeZone,omitempty" protobuf:"bytes,8,opt,name=timeZone"`
+
+	// Optional deadline in seconds for starting the job if it misses scheduled
+	// time for any reason.  Missed jobs executions will be counted as failed ones.
+	// +optional
+	StartingDeadlineSeconds *int64 `json:"startingDeadlineSeconds,omitempty" protobuf:"varint,2,opt,name=startingDeadlineSeconds"`
+
+	// Specifies how to treat concurrent executions of a Job.
+	// Valid values are:
+	// - "Allow" (default): allows CronJobs to run concurrently;
+	// - "Forbid": forbids concurrent runs, skipping next run if previous run hasn't finished yet;
+	// - "Replace": cancels currently running job and replaces it with a new one
+	// +optional
+	ConcurrencyPolicy ConcurrencyPolicy `json:"concurrencyPolicy,omitempty" protobuf:"bytes,3,opt,name=concurrencyPolicy"`
+
+	// Paused will pause the cron job.
+	// +optional
+	Paused *bool `json:"paused,omitempty" protobuf:"bytes,4,opt,name=paused"`
+
+	// The number of successful finished jobs to retain.
+	// This is a pointer to distinguish between explicit zero and not specified.
+	// +optional
+	SuccessfulJobsHistoryLimit *int32 `json:"successfulJobsHistoryLimit,omitempty" protobuf:"varint,5,opt,name=successfulJobsHistoryLimit"`
+
+	// The number of failed finished jobs to retain.
+	// This is a pointer to distinguish between explicit zero and not specified.
+	// +optional
+	FailedJobsHistoryLimit *int32 `json:"failedJobsHistoryLimit,omitempty" protobuf:"varint,6,opt,name=failedJobsHistoryLimit"`
+
+	// Specifies the job that will be created when executing a CronJob.
+	Template CronJobTemplate `json:"template" protobuf:"bytes,7,opt,name=template"`
+}
+
+type CronJobTemplate struct {
+	// Specifies the job that will be created when executing a CronJob.
+	// +optional
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	JobTemplate *batchv1.JobTemplateSpec `json:"jobTemplate,omitempty" protobuf:"bytes,1,opt,name=jobTemplate"`
+
+	// Specifies the broadcastjob that will be created when executing a BroadcastCronJob.
+	// +optional
+	BroadcastJobTemplate *BroadcastJobTemplateSpec `json:"broadcastJobTemplate,omitempty" protobuf:"bytes,2,opt,name=broadcastJobTemplate"`
+}
+
+type TemplateKind string
+
+const (
+	JobTemplate TemplateKind = "Job"
+
+	BroadcastJobTemplate TemplateKind = "BroadcastJob"
+)
+
+// JobTemplateSpec describes the data a Job should have when created from a template
+type BroadcastJobTemplateSpec struct {
+	// Standard object's metadata of the jobs created from this template.
+	// +optional
+	metav1.ObjectMeta `json:"metadata,omitempty" protobuf:"bytes,1,opt,name=metadata"`
+
+	// Specification of the desired behavior of the broadcastjob.
+	// +optional
+	Spec BroadcastJobSpec `json:"spec,omitempty" protobuf:"bytes,2,opt,name=spec"`
+}
+
+// ConcurrencyPolicy describes how the job will be handled.
+// Only one of the following concurrent policies may be specified.
+// If none of the following policies is specified, the default one
+// is AllowConcurrent.
+// +kubebuilder:validation:Enum=Allow;Forbid;Replace
+type ConcurrencyPolicy string
+
+const (
+	// AllowConcurrent allows CronJobs to run concurrently.
+	AllowConcurrent ConcurrencyPolicy = "Allow"
+
+	// ForbidConcurrent forbids concurrent runs, skipping next run if previous
+	// hasn't finished yet.
+	ForbidConcurrent ConcurrencyPolicy = "Forbid"
+
+	// ReplaceConcurrent cancels currently running job and replaces it with a new one.
+	ReplaceConcurrent ConcurrencyPolicy = "Replace"
+)
+
+// AdvancedCronJobStatus defines the observed state of AdvancedCronJob
+type AdvancedCronJobStatus struct {
+	Type TemplateKind `json:"type,omitempty"`
+
+	// A list of pointers to currently running jobs.
+	// +optional
+	Active []corev1.ObjectReference `json:"active,omitempty"`
+
+	// Information when was the last time the job was successfully scheduled.
+	// +optional
+	LastScheduleTime *metav1.Time `json:"lastScheduleTime,omitempty"`
+}
+
+// +genclient
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:shortName=acj
+// +kubebuilder:printcolumn:name="Schedule",type="string",JSONPath=".spec.schedule",description="The schedule of advanced cron job."
+// +kubebuilder:printcolumn:name="Type",type="string",JSONPath=".status.type",description="Type of cron job."
+// +kubebuilder:printcolumn:name="LastScheduleTime",type="date",JSONPath=".status.lastScheduleTime",description="The last time at which job was scheduled."
+// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."
+
+// AdvancedCronJob is the Schema for the advancedcronjobs API
+type AdvancedCronJob struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   AdvancedCronJobSpec   `json:"spec,omitempty"`
+	Status AdvancedCronJobStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// AdvancedCronJobList contains a list of AdvancedCronJob
+type AdvancedCronJobList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []AdvancedCronJob `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&AdvancedCronJob{}, &AdvancedCronJobList{})
+}

apis/apps/v1alpha1/broadcastjob_types.go

@@ -32,10 +32,12 @@ type BroadcastJobSpec struct {
 	Parallelism *intstr.IntOrString `json:"parallelism,omitempty" protobuf:"varint,1,opt,name=parallelism"`
 
 	// Template describes the pod that will be created when executing a job.
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
 	Template v1.PodTemplateSpec `json:"template" protobuf:"bytes,2,opt,name=template"`
 
 	// CompletionPolicy indicates the completion policy of the job.
-	// Default is Always CompletionPolicyType
+	// Default is Always CompletionPolicyType.
 	// +optional
 	CompletionPolicy CompletionPolicy `json:"completionPolicy" protobuf:"bytes,3,opt,name=completionPolicy"`
 
@@ -50,8 +52,8 @@ type BroadcastJobSpec struct {
 
 // CompletionPolicy indicates the completion policy for the job
 type CompletionPolicy struct {
-	// Type indicates the type of the CompletionPolicy
-	// Default is Always
+	// Type indicates the type of the CompletionPolicy.
+	// Default is Always.
 	Type CompletionPolicyType `json:"type,omitempty" protobuf:"bytes,1,opt,name=type,casttype=CompletionPolicyType"`
 
 	// ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the job may be active
@@ -81,8 +83,8 @@ const (
 	// Always means the job will eventually finish on these conditions:
 	// 1) after all pods on the desired nodes are completed (regardless succeeded or failed),
 	// 2) exceeds ActiveDeadlineSeconds,
-	// 3) exceeds BackoffLimit.
-	// This is the default CompletionPolicyType
+	// 3) exceeds RestartLimit.
+	// This is the default CompletionPolicyType.
 	Always CompletionPolicyType = "Always"
 
 	// Never means the job will be kept alive after all pods on the desired nodes are completed.
@@ -151,6 +153,7 @@ const (
 // FailurePolicy indicates the behavior of the job, when failed pod is found.
 type FailurePolicy struct {
 	// Type indicates the type of FailurePolicyType.
+	// Default is FailurePolicyTypeFailFast.
 	Type FailurePolicyType `json:"type,omitempty" protobuf:"bytes,1,opt,name=type,casttype=FailurePolicyType"`
 
 	// RestartLimit specifies the number of retries before marking the pod failed.
@@ -165,9 +168,10 @@ const (
 	FailurePolicyTypeContinue FailurePolicyType = "Continue"
 
 	// FailurePolicyTypeFailFast means the job will be failed, when failed pod is found.
+	// This is the default FailurePolicyType.
 	FailurePolicyTypeFailFast FailurePolicyType = "FailFast"
 
-	// FailurePolicyTypePause means the the job will be paused, when failed pod is found.
+	// FailurePolicyTypePause means the job will be paused, when failed pod is found.
 	FailurePolicyTypePause FailurePolicyType = "Pause"
 )
 
@@ -183,7 +187,7 @@ const (
 	JobComplete JobConditionType = "Complete"
 
 	// JobFailed means the job has failed its execution. A failed job means the job has either exceeded the
-	// ActiveDeadlineSeconds limit, or the aggregated number of container restarts for all pods have exceeded the BackoffLimit.
+	// ActiveDeadlineSeconds limit, or the aggregated number of container restarts for all pods have exceeded the RestartLimit.
 	JobFailed JobConditionType = "Failed"
 )
 
@@ -208,6 +212,7 @@ type JobCondition struct {
 }
 
 // +genclient
+// +k8s:openapi-gen=true
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:resource:shortName=bcj

apis/apps/v1alpha1/cloneset_types.go

@@ -20,6 +20,8 @@ import (
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
 )
 
 const (
@@ -29,6 +31,10 @@ const (
 
 	// DefaultCloneSetMaxUnavailable is the default value of maxUnavailable for CloneSet update strategy.
 	DefaultCloneSetMaxUnavailable = "20%"
+
+	// CloneSetScalingExcludePreparingDeleteKey is the label key that enables scalingExcludePreparingDelete
+	// only for this CloneSet, which means it will calculate scale number excluding Pods in PreparingDelete state.
+	CloneSetScalingExcludePreparingDeleteKey = "apps.kruise.io/cloneset-scaling-exclude-preparing-delete"
 )
 
 // CloneSetSpec defines the desired state of CloneSet
@@ -45,10 +51,14 @@ type CloneSetSpec struct {
 	Selector *metav1.LabelSelector `json:"selector"`
 
 	// Template describes the pods that will be created.
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
 	Template v1.PodTemplateSpec `json:"template"`
 
 	// VolumeClaimTemplates is a list of claims that pods are allowed to reference.
 	// Note that PVC will be deleted when its pod has been deleted.
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
 	VolumeClaimTemplates []v1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
 
 	// ScaleStrategy indicates the ScaleStrategy that will be employed to
@@ -70,8 +80,8 @@ type CloneSetSpec struct {
 	// Defaults to 0 (pod will be considered available as soon as it is ready)
 	MinReadySeconds int32 `json:"minReadySeconds,omitempty"`
 
-	// Lifecycle defines the lifecycle hooks for Pods pre-delete, in-place update.
-	Lifecycle *Lifecycle `json:"lifecycle,omitempty"`
+	// Lifecycle defines the lifecycle hooks for Pods pre-available(pre-normal), pre-delete, in-place update.
+	Lifecycle *appspub.Lifecycle `json:"lifecycle,omitempty"`
 }
 
 // CloneSetScaleStrategy defines strategies for pods scale.
@@ -79,6 +89,15 @@ type CloneSetScaleStrategy struct {
 	// PodsToDelete is the names of Pod should be deleted.
 	// Note that this list will be truncated for non-existing pod names.
 	PodsToDelete []string `json:"podsToDelete,omitempty"`
+	// The maximum number of pods that can be unavailable for scaled pods.
+	// This field can control the changes rate of replicas for CloneSet so as to minimize the impact for users' service.
+	// The scale will fail if the number of unavailable pods were greater than this MaxUnavailable at scaling up.
+	// MaxUnavailable works only when scaling up.
+	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+
+	// Indicate if cloneSet will reuse already existed pvc to
+	// rebuild a new pod
+	DisablePVCReuse bool `json:"disablePVCReuse,omitempty"`
 }
 
 // CloneSetUpdateStrategy defines strategies for pods update.
@@ -86,17 +105,19 @@ type CloneSetUpdateStrategy struct {
 	// Type indicates the type of the CloneSetUpdateStrategy.
 	// Default is ReCreate.
 	Type CloneSetUpdateStrategyType `json:"type,omitempty"`
-	// Partition is the desired number of pods in old revisions. It means when partition
-	// is set during pods updating, (replicas - partition) number of pods will be updated.
+	// Partition is the desired number of pods in old revisions.
+	// Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
+	// Absolute number is calculated from percentage by rounding up by default.
+	// It means when partition is set during pods updating, (replicas - partition value) number of pods will be updated.
 	// Default value is 0.
-	Partition *int32 `json:"partition,omitempty"`
-	// The maximum number of pods that can be unavailable during the update.
+	Partition *intstr.IntOrString `json:"partition,omitempty"`
+	// The maximum number of pods that can be unavailable during update or scale.
 	// Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
 	// Absolute number is calculated from percentage by rounding up by default.
 	// When maxSurge > 0, absolute number is calculated from percentage by rounding down.
 	// Defaults to 20%.
 	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
-	// The maximum number of pods that can be scheduled above the desired replicas during the update.
+	// The maximum number of pods that can be scheduled above the desired replicas during update or specified delete.
 	// Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
 	// Absolute number is calculated from percentage by rounding up.
 	// Defaults to 0.
@@ -106,14 +127,14 @@ type CloneSetUpdateStrategy struct {
 	Paused bool `json:"paused,omitempty"`
 	// Priorities are the rules for calculating the priority of updating pods.
 	// Each pod to be updated, will pass through these terms and get a sum of weights.
-	PriorityStrategy *UpdatePriorityStrategy `json:"priorityStrategy,omitempty"`
+	PriorityStrategy *appspub.UpdatePriorityStrategy `json:"priorityStrategy,omitempty"`
 	// ScatterStrategy defines the scatter rules to make pods been scattered when update.
 	// This will avoid pods with the same key-value to be updated in one batch.
 	// - Note that pods will be scattered after priority sort. So, although priority strategy and scatter strategy can be applied together, we suggest to use either one of them.
 	// - If scatterStrategy is used, we suggest to just use one term. Otherwise, the update order can be hard to understand.
-	ScatterStrategy CloneSetUpdateScatterStrategy `json:"scatterStrategy,omitempty"`
+	ScatterStrategy UpdateScatterStrategy `json:"scatterStrategy,omitempty"`
 	// InPlaceUpdateStrategy contains strategies for in-place update.
-	InPlaceUpdateStrategy *InPlaceUpdateStrategy `json:"inPlaceUpdateStrategy,omitempty"`
+	InPlaceUpdateStrategy *appspub.InPlaceUpdateStrategy `json:"inPlaceUpdateStrategy,omitempty"`
 }
 
 // CloneSetUpdateStrategyType defines strategies for pods in-place update.
@@ -133,21 +154,6 @@ const (
 	InPlaceOnlyCloneSetUpdateStrategyType CloneSetUpdateStrategyType = "InPlaceOnly"
 )
 
-// CloneSetUpdateScatterStrategy defines a map for label key-value. Pods matches the key-value will be scattered when update.
-//
-// Example1: [{"Key": "labelA", "Value": "AAA"}]
-// It means all pods with label labelA=AAA will be scattered when update.
-//
-// Example2: [{"Key": "labelA", "Value": "AAA"}, {"Key": "labelB", "Value": "BBB"}]
-// Controller will calculate the two sums of pods with labelA=AAA and with labelB=BBB,
-// pods with the label that has bigger amount will be scattered first, then pods with the other label will be scattered.
-type CloneSetUpdateScatterStrategy []CloneSetUpdateScatterTerm
-
-type CloneSetUpdateScatterTerm struct {
-	Key   string `json:"key"`
-	Value string `json:"value"`
-}
-
 // CloneSetStatus defines the observed state of CloneSet
 type CloneSetStatus struct {
 	// ObservedGeneration is the most recent generation observed for this CloneSet. It corresponds to the
@@ -171,9 +177,22 @@ type CloneSetStatus struct {
 	// indicated by updateRevision and have a Ready Condition.
 	UpdatedReadyReplicas int32 `json:"updatedReadyReplicas"`
 
+	// UpdatedAvailableReplicas is the number of Pods created by the CloneSet controller from the CloneSet version
+	// indicated by updateRevision and have a Ready Condition for at least minReadySeconds.
+	// Notice: when enable InPlaceWorkloadVerticalScaling, pod during resource resizing will also be unavailable.
+	// This means these pod will be counted in maxUnavailable.
+	UpdatedAvailableReplicas int32 `json:"updatedAvailableReplicas,omitempty"`
+
+	// ExpectedUpdatedReplicas is the number of Pods that should be updated by CloneSet controller.
+	// This field is calculated via Replicas - Partition.
+	ExpectedUpdatedReplicas int32 `json:"expectedUpdatedReplicas,omitempty"`
+
 	// UpdateRevision, if not empty, indicates the latest revision of the CloneSet.
 	UpdateRevision string `json:"updateRevision,omitempty"`
 
+	// currentRevision, if not empty, indicates the current revision version of the CloneSet.
+	CurrentRevision string `json:"currentRevision,omitempty"`
+
 	// CollisionCount is the count of hash collisions for the CloneSet. The CloneSet controller
 	// uses this field as a collision avoidance mechanism when it needs to create the name for the
 	// newest ControllerRevision.
@@ -211,6 +230,9 @@ type CloneSetCondition struct {
 }
 
 // +genclient
+// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale
+// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale
+// +k8s:openapi-gen=true
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector
@@ -218,9 +240,13 @@ type CloneSetCondition struct {
 // +kubebuilder:printcolumn:name="DESIRED",type="integer",JSONPath=".spec.replicas",description="The desired number of pods."
 // +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedReplicas",description="The number of pods updated."
 // +kubebuilder:printcolumn:name="UPDATED_READY",type="integer",JSONPath=".status.updatedReadyReplicas",description="The number of pods updated and ready."
+// +kubebuilder:printcolumn:name="UPDATED_AVAILABLE",type="integer",JSONPath=".status.updatedAvailableReplicas",description="The number of pods updated and available."
 // +kubebuilder:printcolumn:name="READY",type="integer",JSONPath=".status.readyReplicas",description="The number of pods ready."
 // +kubebuilder:printcolumn:name="TOTAL",type="integer",JSONPath=".status.replicas",description="The number of currently all pods."
 // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."
+// +kubebuilder:printcolumn:name="CONTAINERS",type="string",priority=1,JSONPath=".spec.template.spec.containers[*].name",description="The containers of currently cloneset."
+// +kubebuilder:printcolumn:name="IMAGES",type="string",priority=1,JSONPath=".spec.template.spec.containers[*].image",description="The images of currently cloneset."
+// +kubebuilder:printcolumn:name="SELECTOR",type="string",priority=1,JSONPath=".status.labelSelector",description="The selector of currently cloneset."
 
 // CloneSet is the Schema for the clonesets API
 type CloneSet struct {

apis/apps/v1alpha1/containerrecreaterequest_types.go

@@ -0,0 +1,211 @@
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	// [Immutable] Pod UID of this ContainerRecreateRequest.
+	ContainerRecreateRequestPodUIDKey = "crr.apps.kruise.io/pod-uid"
+	// [Immutable] Node name of this ContainerRecreateRequest.
+	ContainerRecreateRequestNodeNameKey = "crr.apps.kruise.io/node-name"
+
+	// ContainerRecreateRequestActiveKey indicates if this ContainerRecreateRequest is active.
+	// It will be removed in labels since a ContainerRecreateRequest has completed.
+	// We use it mainly to make kruise-daemon do not list watch those ContainerRecreateRequests that have completed.
+	ContainerRecreateRequestActiveKey = "crr.apps.kruise.io/active"
+	// ContainerRecreateRequestSyncContainerStatusesKey contains the container statuses in current Pod.
+	// It is only synchronized during a ContainerRecreateRequest in Recreating phase.
+	ContainerRecreateRequestSyncContainerStatusesKey = "crr.apps.kruise.io/sync-container-statuses"
+	// ContainerRecreateRequestUnreadyAcquiredKey indicates the Pod has been forced to not-ready.
+	// It is required if the unreadyGracePeriodSeconds is set in ContainerRecreateRequests.
+	ContainerRecreateRequestUnreadyAcquiredKey = "crr.apps.kruise.io/unready-acquired"
+)
+
+// ContainerRecreateRequestSpec defines the desired state of ContainerRecreateRequest
+type ContainerRecreateRequestSpec struct {
+	// PodName is name of the Pod that owns the recreated containers.
+	PodName string `json:"podName"`
+	// Containers contains the containers that need to recreate in the Pod.
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Containers []ContainerRecreateRequestContainer `json:"containers" patchStrategy:"merge" patchMergeKey:"name"`
+	// Strategy defines strategies for containers recreation.
+	Strategy *ContainerRecreateRequestStrategy `json:"strategy,omitempty"`
+	// ActiveDeadlineSeconds is the deadline duration of this ContainerRecreateRequest.
+	ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty"`
+	// TTLSecondsAfterFinished is the TTL duration after this ContainerRecreateRequest has completed.
+	TTLSecondsAfterFinished *int32 `json:"ttlSecondsAfterFinished,omitempty"`
+}
+
+// ContainerRecreateRequestContainer defines the container that need to recreate.
+type ContainerRecreateRequestContainer struct {
+	// Name of the container that need to recreate.
+	// It must be existing in the real pod.Spec.Containers.
+	Name string `json:"name"`
+	// PreStop is synced from the real container in Pod spec during this ContainerRecreateRequest creating.
+	// Populated by the system.
+	// Read-only.
+	PreStop *ProbeHandler `json:"preStop,omitempty"`
+	// Ports is synced from the real container in Pod spec during this ContainerRecreateRequest creating.
+	// Populated by the system.
+	// Read-only.
+	Ports []v1.ContainerPort `json:"ports,omitempty"`
+	// StatusContext is synced from the real Pod status during this ContainerRecreateRequest creating.
+	// Populated by the system.
+	// Read-only.
+	StatusContext *ContainerRecreateRequestContainerContext `json:"statusContext,omitempty"`
+}
+
+// ProbeHandler defines a specific action that should be taken
+// TODO(FillZpp): improve the definition when openkruise/kruise updates to k8s 1.23
+type ProbeHandler struct {
+	// One and only one of the following should be specified.
+	// Exec specifies the action to take.
+	// +optional
+	Exec *v1.ExecAction `json:"exec,omitempty" protobuf:"bytes,1,opt,name=exec"`
+	// HTTPGet specifies the http request to perform.
+	// +optional
+	HTTPGet *v1.HTTPGetAction `json:"httpGet,omitempty" protobuf:"bytes,2,opt,name=httpGet"`
+	// TCPSocket specifies an action involving a TCP port.
+	// TCP hooks not yet supported
+	// TODO: implement a realistic TCP lifecycle hook
+	// +optional
+	TCPSocket *v1.TCPSocketAction `json:"tcpSocket,omitempty" protobuf:"bytes,3,opt,name=tcpSocket"`
+}
+
+// ContainerRecreateRequestContainerContext contains context status of the container that need to recreate.
+type ContainerRecreateRequestContainerContext struct {
+	// Container's ID in the format 'docker://<container_id>'.
+	ContainerID string `json:"containerID"`
+	// The number of times the container has been restarted, currently based on
+	// the number of dead containers that have not yet been removed.
+	// Note that this is calculated from dead containers. But those containers are subject to
+	// garbage collection. This value will get capped at 5 by GC.
+	RestartCount int32 `json:"restartCount"`
+}
+
+// ContainerRecreateRequestStrategy contains the strategies for containers recreation.
+type ContainerRecreateRequestStrategy struct {
+	// FailurePolicy decides whether to continue if one container fails to recreate
+	FailurePolicy ContainerRecreateRequestFailurePolicyType `json:"failurePolicy,omitempty"`
+	// OrderedRecreate indicates whether to recreate the next container only if the previous one has recreated completely.
+	OrderedRecreate bool `json:"orderedRecreate,omitempty"`
+	// ForceRecreate indicates whether to force kill the container even if the previous container is starting.
+	ForceRecreate bool `json:"forceRecreate,omitempty"`
+	// TerminationGracePeriodSeconds is the optional duration in seconds to wait the container terminating gracefully.
+	// Value must be non-negative integer. The value zero indicates delete immediately.
+	// If this value is nil, we will use pod.Spec.TerminationGracePeriodSeconds as default value.
+	TerminationGracePeriodSeconds *int64 `json:"terminationGracePeriodSeconds,omitempty"`
+	// UnreadyGracePeriodSeconds is the optional duration in seconds to mark Pod as not ready over this duration before
+	// executing preStop hook and stopping the container.
+	UnreadyGracePeriodSeconds *int64 `json:"unreadyGracePeriodSeconds,omitempty"`
+	// Minimum number of seconds for which a newly created container should be started and ready
+	// without any of its container crashing, for it to be considered Succeeded.
+	// Defaults to 0 (container will be considered Succeeded as soon as it is started and ready)
+	MinStartedSeconds int32 `json:"minStartedSeconds,omitempty"`
+}
+
+type ContainerRecreateRequestFailurePolicyType string
+
+const (
+	ContainerRecreateRequestFailurePolicyFail   ContainerRecreateRequestFailurePolicyType = "Fail"
+	ContainerRecreateRequestFailurePolicyIgnore ContainerRecreateRequestFailurePolicyType = "Ignore"
+)
+
+// ContainerRecreateRequestStatus defines the observed state of ContainerRecreateRequest
+type ContainerRecreateRequestStatus struct {
+	// Phase of this ContainerRecreateRequest, e.g. Pending, Recreating, Completed
+	Phase ContainerRecreateRequestPhase `json:"phase"`
+	// Represents time when the ContainerRecreateRequest was completed. It is not guaranteed to
+	// be set in happens-before order across separate operations.
+	// It is represented in RFC3339 form and is in UTC.
+	CompletionTime *metav1.Time `json:"completionTime,omitempty"`
+	// A human readable message indicating details about this ContainerRecreateRequest.
+	Message string `json:"message,omitempty"`
+	// ContainerRecreateStates contains the recreation states of the containers.
+	ContainerRecreateStates []ContainerRecreateRequestContainerRecreateState `json:"containerRecreateStates,omitempty"`
+}
+
+type ContainerRecreateRequestPhase string
+
+const (
+	ContainerRecreateRequestPending    ContainerRecreateRequestPhase = "Pending"
+	ContainerRecreateRequestRecreating ContainerRecreateRequestPhase = "Recreating"
+	ContainerRecreateRequestSucceeded  ContainerRecreateRequestPhase = "Succeeded"
+	ContainerRecreateRequestFailed     ContainerRecreateRequestPhase = "Failed"
+	ContainerRecreateRequestCompleted  ContainerRecreateRequestPhase = "Completed"
+)
+
+// ContainerRecreateRequestContainerRecreateState contains the recreation state of the container.
+type ContainerRecreateRequestContainerRecreateState struct {
+	// Name of the container.
+	Name string `json:"name"`
+	// Phase indicates the recreation phase of the container.
+	Phase ContainerRecreateRequestPhase `json:"phase"`
+	// A human readable message indicating details about this state.
+	Message string `json:"message,omitempty"`
+	// Containers are killed by kruise daemon
+	IsKilled bool `json:"isKilled,omitempty"`
+}
+
+// ContainerRecreateRequestSyncContainerStatus only uses in the annotation `crr.apps.kruise.io/sync-container-statuses`.
+type ContainerRecreateRequestSyncContainerStatus struct {
+	Name string `json:"name"`
+	// Specifies whether the container has passed its readiness probe.
+	Ready bool `json:"ready"`
+	// The number of times the container has been restarted, currently based on
+	// the number of dead containers that have not yet been removed.
+	RestartCount int32 `json:"restartCount"`
+	// Container's ID in the format 'docker://<container_id>'.
+	ContainerID string `json:"containerID,omitempty"`
+}
+
+// +genclient
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:shortName=crr
+// +kubebuilder:printcolumn:name="PHASE",type="string",JSONPath=".status.phase",description="Phase of this ContainerRecreateRequest."
+// +kubebuilder:printcolumn:name="POD",type="string",JSONPath=".spec.podName",description="Pod name of this ContainerRecreateRequest."
+// +kubebuilder:printcolumn:name="NODE",type="string",JSONPath=".metadata.labels.crr\\.apps\\.kruise\\.io/node-name",description="Pod name of this ContainerRecreateRequest."
+// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."
+
+// ContainerRecreateRequest is the Schema for the containerrecreaterequests API
+type ContainerRecreateRequest struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ContainerRecreateRequestSpec   `json:"spec,omitempty"`
+	Status ContainerRecreateRequestStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// ContainerRecreateRequestList contains a list of ContainerRecreateRequest
+type ContainerRecreateRequestList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ContainerRecreateRequest `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&ContainerRecreateRequest{}, &ContainerRecreateRequestList{})
+}

apis/apps/v1alpha1/daemonset_types.go

@@ -17,20 +17,23 @@ limitations under the License.
 package v1alpha1
 
 import (
+	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
+
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
 )
 
 // DaemonSetUpdateStrategy is a struct used to control the update strategy for a DaemonSet.
 type DaemonSetUpdateStrategy struct {
 	// Type of daemon set update. Can be "RollingUpdate" or "OnDelete". Default is RollingUpdate.
 	// +optional
-	Type DaemonSetUpdateStrategyType `json:"type,omitempty" protobuf:"bytes,1,opt,name=type"`
+	Type DaemonSetUpdateStrategyType `json:"type,omitempty"`
 
 	// Rolling update config params. Present only if type = "RollingUpdate".
 	// +optional
-	RollingUpdate *RollingUpdateDaemonSet `json:"rollingUpdate,omitempty" protobuf:"bytes,2,opt,name=rollingUpdate"`
+	RollingUpdate *RollingUpdateDaemonSet `json:"rollingUpdate,omitempty"`
 }
 
 type DaemonSetUpdateStrategyType string
@@ -43,69 +46,76 @@ const (
 	// Replace the old daemons only when it's killed
 	OnDeleteDaemonSetStrategyType DaemonSetUpdateStrategyType = "OnDelete"
 
-	// StandardRollingUpdateType replace the old daemons by new ones using rolling update i.e replace them on each node one after the other.
-	// this is the default type for RollingUpdate.
+	// StandardRollingUpdateType is the Standard way that update pods with recreation that sames to the upstream DaemonSet.
 	StandardRollingUpdateType RollingUpdateType = "Standard"
 
-	// Replace container image without killing the pod.
-	//InplaceRollingUpdateType RollingUpdateType = "Inplace"
+	// InplaceRollingUpdateType update container image without killing the pod if possible.
+	InplaceRollingUpdateType RollingUpdateType = "InPlaceIfPossible"
 
-	// SurgingRollingUpdateType replaces the old daemons by new ones using rolling update i.e replace them on each node one
-	// after the other, creating the new pod and then killing the old one.
-	SurgingRollingUpdateType RollingUpdateType = "Surging"
+	// DeprecatedSurgingRollingUpdateType is a depreciated alias for Standard.
+	// Deprecated: Just use Standard instead.
+	DeprecatedSurgingRollingUpdateType RollingUpdateType = "Surging"
 )
 
 // Spec to control the desired behavior of daemon set rolling update.
 type RollingUpdateDaemonSet struct {
 	// Type is to specify which kind of rollingUpdate.
-	Type RollingUpdateType `json:"rollingUpdateType,omitempty" protobuf:"bytes,1,opt,name=rollingUpdateType"`
+	Type RollingUpdateType `json:"rollingUpdateType,omitempty"`
 
 	// The maximum number of DaemonSet pods that can be unavailable during the
 	// update. Value can be an absolute number (ex: 5) or a percentage of total
 	// number of DaemonSet pods at the start of the update (ex: 10%). Absolute
 	// number is calculated from percentage by rounding up.
-	// This cannot be 0.
+	// This cannot be 0 if MaxSurge is 0
 	// Default value is 1.
 	// Example: when this is set to 30%, at most 30% of the total number of nodes
 	// that should be running the daemon pod (i.e. status.desiredNumberScheduled)
-	// can have their pods stopped for an update at any given
-	// time. The update starts by stopping at most 30% of those DaemonSet pods
-	// and then brings up new DaemonSet pods in their place. Once the new pods
-	// are available, it then proceeds onto other DaemonSet pods, thus ensuring
-	// that at least 70% of original number of DaemonSet pods are available at
-	// all times during the update.
+	// can have their pods stopped for an update at any given time. The update
+	// starts by stopping at most 30% of those DaemonSet pods and then brings
+	// up new DaemonSet pods in their place. Once the new pods are available,
+	// it then proceeds onto other DaemonSet pods, thus ensuring that at least
+	// 70% of original number of DaemonSet pods are available at all times during
+	// the update.
 	// +optional
-	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty" protobuf:"bytes,2,opt,name=maxUnavailable"`
+	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+
+	// The maximum number of nodes with an existing available DaemonSet pod that
+	// can have an updated DaemonSet pod during during an update.
+	// Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
+	// This can not be 0 if MaxUnavailable is 0.
+	// Absolute number is calculated from percentage by rounding up to a minimum of 1.
+	// Default value is 0.
+	// Example: when this is set to 30%, at most 30% of the total number of nodes
+	// that should be running the daemon pod (i.e. status.desiredNumberScheduled)
+	// can have their a new pod created before the old pod is marked as deleted.
+	// The update starts by launching new pods on 30% of nodes. Once an updated
+	// pod is available (Ready for at least minReadySeconds) the old DaemonSet pod
+	// on that node is marked deleted. If the old pod becomes unavailable for any
+	// reason (Ready transitions to false, is evicted, or is drained) an updated
+	// pod is immediately created on that node without considering surge limits.
+	// Allowing surge implies the possibility that the resources consumed by the
+	// daemonset on any given node can double if the readiness check fails, and
+	// so resource intensive daemonsets should take into account that they may
+	// cause evictions during disruption.
+	// This is beta field and enabled/disabled by DaemonSetUpdateSurge feature gate.
+	// +optional
+	MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty"`
 
 	// A label query over nodes that are managed by the daemon set RollingUpdate.
 	// Must match in order to be controlled.
 	// It must match the node's labels.
-	Selector *metav1.LabelSelector `json:"selector,omitempty" protobuf:"bytes,3,opt,name=selector"`
+	Selector *metav1.LabelSelector `json:"selector,omitempty"`
 
 	// The number of DaemonSet pods remained to be old version.
 	// Default value is 0.
 	// Maximum value is status.DesiredNumberScheduled, which means no pod will be updated.
 	// +optional
-	Partition *int32 `json:"partition,omitempty" protobuf:"varint,4,opt,name=partition"`
+	Partition *int32 `json:"partition,omitempty"`
 
 	// Indicates that the daemon set is paused and will not be processed by the
 	// daemon set controller.
 	// +optional
-	Paused *bool `json:"paused,omitempty" protobuf:"varint,5,opt,name=paused"`
-
-	// Only when type=SurgingRollingUpdateType, it works.
-	// The maximum number of DaemonSet pods that can be scheduled above the desired number of pods
-	// during the update. Value can be an absolute number (ex: 5) or a percentage of the total number
-	// of DaemonSet pods at the start of the update (ex: 10%). The absolute number is calculated from
-	// the percentage by rounding up. This cannot be 0. The default value is 1. Example: when this is
-	// set to 30%, at most 30% of the total number of nodes that should be running the daemon pod
-	// (i.e. status.desiredNumberScheduled) can have 2 pods running at any given time. The update
-	// starts by starting replacements for at most 30% of those DaemonSet pods. Once the new pods are
-	// available it then stops the existing pods before proceeding onto other DaemonSet pods, thus
-	// ensuring that at most 130% of the desired final number of DaemonSet  pods are running at all
-	// times during the update.
-	// +optional
-	MaxSurge *intstr.IntOrString `json:"maxSurge,omitempty" protobuf:"bytes,7,opt,name=maxSurge"`
+	Paused *bool `json:"paused,omitempty"`
 }
 
 // DaemonSetSpec defines the desired state of DaemonSet
@@ -117,35 +127,42 @@ type DaemonSetSpec struct {
 	// Must match in order to be controlled.
 	// It must match the pod template's labels.
 	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
-	Selector *metav1.LabelSelector `json:"selector" protobuf:"bytes,1,opt,name=selector"`
+	Selector *metav1.LabelSelector `json:"selector"`
 
 	// An object that describes the pod that will be created.
 	// The DaemonSet will create exactly one copy of this pod on every node
 	// that matches the template's node selector (or on every node if no node
 	// selector is specified).
 	// More info: https://kubernetes.io/docs/concepts/workloads/controllers/replicationcontroller#pod-template
-	Template corev1.PodTemplateSpec `json:"template" protobuf:"bytes,2,opt,name=template"`
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	Template corev1.PodTemplateSpec `json:"template"`
 
 	// An update strategy to replace existing DaemonSet pods with new pods.
 	// +optional
-	UpdateStrategy DaemonSetUpdateStrategy `json:"updateStrategy,omitempty" protobuf:"bytes,3,opt,name=updateStrategy"`
+	UpdateStrategy DaemonSetUpdateStrategy `json:"updateStrategy,omitempty"`
 
 	// The minimum number of seconds for which a newly created DaemonSet pod should
 	// be ready without any of its container crashing, for it to be considered
 	// available. Defaults to 0 (pod will be considered available as soon as it
 	// is ready).
 	// +optional
-	MinReadySeconds int32 `json:"minReadySeconds,omitempty" protobuf:"varint,4,opt,name=minReadySeconds"`
+	MinReadySeconds int32 `json:"minReadySeconds,omitempty"`
 
 	// BurstReplicas is a rate limiter for booting pods on a lot of pods.
 	// The default value is 250
-	BurstReplicas *intstr.IntOrString `json:"burstReplicas,omitempty" protobuf:"bytes,5,opt,name=burstReplicas"`
+	BurstReplicas *intstr.IntOrString `json:"burstReplicas,omitempty"`
 
 	// The number of old history to retain to allow rollback.
 	// This is a pointer to distinguish between explicit zero and not specified.
 	// Defaults to 10.
 	// +optional
-	RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty" protobuf:"varint,6,opt,name=revisionHistoryLimit"`
+	RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"`
+
+	// Lifecycle defines the lifecycle hooks for Pods pre-delete, in-place update.
+	// Currently, we only support pre-delete hook for Advanced DaemonSet.
+	// +optional
+	Lifecycle *appspub.Lifecycle `json:"lifecycle,omitempty"`
 }
 
 // DaemonSetStatus defines the observed state of DaemonSet
@@ -156,93 +173,70 @@ type DaemonSetStatus struct {
 	// The number of nodes that are running at least 1
 	// daemon pod and are supposed to run the daemon pod.
 	// More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/
-	CurrentNumberScheduled int32 `json:"currentNumberScheduled" protobuf:"varint,1,opt,name=currentNumberScheduled"`
+	CurrentNumberScheduled int32 `json:"currentNumberScheduled"`
 
 	// The number of nodes that are running the daemon pod, but are
 	// not supposed to run the daemon pod.
 	// More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/
-	NumberMisscheduled int32 `json:"numberMisscheduled" protobuf:"varint,2,opt,name=numberMisscheduled"`
+	NumberMisscheduled int32 `json:"numberMisscheduled"`
 
 	// The total number of nodes that should be running the daemon
 	// pod (including nodes correctly running the daemon pod).
 	// More info: https://kubernetes.io/docs/concepts/workloads/controllers/daemonset/
-	DesiredNumberScheduled int32 `json:"desiredNumberScheduled" protobuf:"varint,3,opt,name=desiredNumberScheduled"`
+	DesiredNumberScheduled int32 `json:"desiredNumberScheduled"`
 
 	// The number of nodes that should be running the daemon pod and have one
 	// or more of the daemon pod running and ready.
-	NumberReady int32 `json:"numberReady" protobuf:"varint,4,opt,name=numberReady"`
+	NumberReady int32 `json:"numberReady"`
 
 	// The most recent generation observed by the daemon set controller.
 	// +optional
-	ObservedGeneration int64 `json:"observedGeneration,omitempty" protobuf:"varint,5,opt,name=observedGeneration"`
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 
 	// The total number of nodes that are running updated daemon pod
-	UpdatedNumberScheduled int32 `json:"updatedNumberScheduled" protobuf:"varint,6,opt,name=updatedNumberScheduled"`
+	UpdatedNumberScheduled int32 `json:"updatedNumberScheduled"`
 
 	// The number of nodes that should be running the
 	// daemon pod and have one or more of the daemon pod running and
 	// available (ready for at least spec.minReadySeconds)
 	// +optional
-	NumberAvailable int32 `json:"numberAvailable,omitempty" protobuf:"varint,7,opt,name=numberAvailable"`
+	NumberAvailable int32 `json:"numberAvailable,omitempty"`
 
 	// The number of nodes that should be running the
 	// daemon pod and have none of the daemon pod running and available
 	// (ready for at least spec.minReadySeconds)
 	// +optional
-	NumberUnavailable int32 `json:"numberUnavailable,omitempty" protobuf:"varint,8,opt,name=numberUnavailable"`
+	NumberUnavailable int32 `json:"numberUnavailable,omitempty"`
 
 	// Count of hash collisions for the DaemonSet. The DaemonSet controller
 	// uses this field as a collision avoidance mechanism when it needs to
 	// create the name for the newest ControllerRevision.
 	// +optional
-	CollisionCount *int32 `json:"collisionCount,omitempty" protobuf:"varint,9,opt,name=collisionCount"`
+	CollisionCount *int32 `json:"collisionCount,omitempty"`
 
 	// Represents the latest available observations of a DaemonSet's current state.
 	// +optional
 	// +patchMergeKey=type
 	// +patchStrategy=merge
-	Conditions []DaemonSetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,10,rep,name=conditions"`
+	Conditions []appsv1.DaemonSetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
 
 	// DaemonSetHash is the controller-revision-hash, which represents the latest version of the DaemonSet.
-	DaemonSetHash string `json:"daemonSetHash" protobuf:"bytes,11,opt,name=daemonSetHash"`
+	DaemonSetHash string `json:"daemonSetHash"`
 }
 
-type DaemonSetConditionType string
-
-// TODO: Add valid condition types of a DaemonSet.
-
-// DaemonSetCondition describes the state of a DaemonSet at a certain point.
-type DaemonSetCondition struct {
-	// Type of DaemonSet condition.
-	Type DaemonSetConditionType `json:"type" protobuf:"bytes,1,opt,name=type,casttype=DaemonSetConditionType"`
-	// Status of the condition, one of True, False, Unknown.
-	Status corev1.ConditionStatus `json:"status" protobuf:"bytes,2,opt,name=status,casttype=k8s.io/api/core/v1.ConditionStatus"`
-	// Last time the condition transitioned from one status to another.
-	// +optional
-	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty" protobuf:"bytes,3,opt,name=lastTransitionTime"`
-	// The reason for the condition's last transition.
-	// +optional
-	Reason string `json:"reason,omitempty" protobuf:"bytes,4,opt,name=reason"`
-	// A human readable message indicating details about the transition.
-	// +optional
-	Message string `json:"message,omitempty" protobuf:"bytes,5,opt,name=message"`
-}
-
-const (
-	// DefaultDaemonSetUniqueLabelKey is the default label key that is added
-	// to existing DaemonSet pods to distinguish between old and new
-	// DaemonSet pods during DaemonSet template updates.
-	DefaultDaemonSetUniqueLabelKey = ControllerRevisionHashLabelKey
-)
-
 // +genclient
+// +k8s:openapi-gen=true
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
-// +kubebuilder:resource:shortName=daemon
-// +kubebuilder:printcolumn:name="DesiredNumber",type="integer",JSONPath=".status.desiredNumberScheduled",description="The desired number of pods."
-// +kubebuilder:printcolumn:name="CurrentNumber",type="integer",JSONPath=".status.currentNumberScheduled",description="The current number of pods."
-// +kubebuilder:printcolumn:name="UpdatedNumberScheduled",type="integer",JSONPath=".status.updatedNumberScheduled",description="The updated number of pods."
+// +kubebuilder:resource:shortName=daemon;ads
+// +kubebuilder:printcolumn:name="DESIRED",type="integer",JSONPath=".status.desiredNumberScheduled",description="The desired number of pods."
+// +kubebuilder:printcolumn:name="CURRENT",type="integer",JSONPath=".status.currentNumberScheduled",description="The current number of pods."
+// +kubebuilder:printcolumn:name="READY",type="integer",JSONPath=".status.numberReady",description="The ready number of pods."
+// +kubebuilder:printcolumn:name="UP-TO-DATE",type="integer",JSONPath=".status.updatedNumberScheduled",description="The updated number of pods."
+// +kubebuilder:printcolumn:name="AVAILABLE",type="integer",JSONPath=".status.numberAvailable",description="The updated number of pods."
 // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."
+// +kubebuilder:printcolumn:name="CONTAINERS",type="string",priority=1,JSONPath=".spec.template.spec.containers[*].name",description="The containers of currently  daemonset."
+// +kubebuilder:printcolumn:name="IMAGES",type="string",priority=1,JSONPath=".spec.template.spec.containers[*].image",description="The images of currently advanced daemonset."
 
 // DaemonSet is the Schema for the daemonsets API
 type DaemonSet struct {

apis/apps/v1alpha1/defaults.go

@@ -1,426 +0,0 @@
-/*
-Copyright 2020 The Kruise Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1alpha1
-
-import (
-	appsv1 "k8s.io/api/apps/v1"
-	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
-	"k8s.io/kubernetes/pkg/apis/core/v1"
-	utilpointer "k8s.io/utils/pointer"
-)
-
-// SetDefaults_SidecarSet set default values for SidecarSet.
-func SetDefaultsSidecarSet(obj *SidecarSet) {
-	setSidecarSetUpdateStratety(&obj.Spec.Strategy)
-
-	for i := range obj.Spec.InitContainers {
-		setSidecarDefaultContainer(&obj.Spec.InitContainers[i])
-	}
-
-	for i := range obj.Spec.Containers {
-		setSidecarDefaultContainer(&obj.Spec.Containers[i])
-	}
-}
-
-func setSidecarSetUpdateStratety(strategy *SidecarSetUpdateStrategy) {
-	if strategy.RollingUpdate == nil {
-		rollingUpdate := RollingUpdateSidecarSet{}
-		strategy.RollingUpdate = &rollingUpdate
-	}
-	if strategy.RollingUpdate.MaxUnavailable == nil {
-		maxUnavailable := intstr.FromInt(1)
-		strategy.RollingUpdate.MaxUnavailable = &maxUnavailable
-	}
-}
-
-func setSidecarDefaultContainer(sidecarContainer *SidecarContainer) {
-	container := &sidecarContainer.Container
-	v1.SetDefaults_Container(container)
-	for i := range container.Ports {
-		p := &container.Ports[i]
-		v1.SetDefaults_ContainerPort(p)
-	}
-	for i := range container.Env {
-		e := &container.Env[i]
-		if e.ValueFrom != nil {
-			if e.ValueFrom.FieldRef != nil {
-				v1.SetDefaults_ObjectFieldSelector(e.ValueFrom.FieldRef)
-			}
-		}
-	}
-	v1.SetDefaults_ResourceList(&container.Resources.Limits)
-	v1.SetDefaults_ResourceList(&container.Resources.Requests)
-	if container.LivenessProbe != nil {
-		v1.SetDefaults_Probe(container.LivenessProbe)
-		if container.LivenessProbe.Handler.HTTPGet != nil {
-			v1.SetDefaults_HTTPGetAction(container.LivenessProbe.Handler.HTTPGet)
-		}
-	}
-	if container.ReadinessProbe != nil {
-		v1.SetDefaults_Probe(container.ReadinessProbe)
-		if container.ReadinessProbe.Handler.HTTPGet != nil {
-			v1.SetDefaults_HTTPGetAction(container.ReadinessProbe.Handler.HTTPGet)
-		}
-	}
-	if container.Lifecycle != nil {
-		if container.Lifecycle.PostStart != nil {
-			if container.Lifecycle.PostStart.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(container.Lifecycle.PostStart.HTTPGet)
-			}
-		}
-		if container.Lifecycle.PreStop != nil {
-			if container.Lifecycle.PreStop.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(container.Lifecycle.PreStop.HTTPGet)
-			}
-		}
-	}
-}
-
-// SetDefaults_BroadcastJob set default values for BroadcastJob.
-func SetDefaultsBroadcastJob(obj *BroadcastJob) {
-	SetDefaultPodSpec(&obj.Spec.Template.Spec)
-	if obj.Spec.CompletionPolicy.Type == "" {
-		obj.Spec.CompletionPolicy.Type = Always
-	}
-
-	if obj.Spec.Parallelism == nil {
-		parallelism := int32(1<<31 - 1)
-		parallelismIntStr := intstr.FromInt(int(parallelism))
-		obj.Spec.Parallelism = &parallelismIntStr
-	}
-
-	if obj.Spec.FailurePolicy.Type == "" {
-		obj.Spec.FailurePolicy.Type = FailurePolicyTypeFailFast
-	}
-}
-
-// SetDefaults_StatefulSet set default values for StatefulSet.
-func SetDefaultsStatefulSet(obj *StatefulSet) {
-	if len(obj.Spec.PodManagementPolicy) == 0 {
-		obj.Spec.PodManagementPolicy = appsv1.OrderedReadyPodManagement
-	}
-
-	if obj.Spec.UpdateStrategy.Type == "" {
-		obj.Spec.UpdateStrategy.Type = appsv1.RollingUpdateStatefulSetStrategyType
-
-		// UpdateStrategy.RollingUpdate will take default values below.
-		obj.Spec.UpdateStrategy.RollingUpdate = &RollingUpdateStatefulSetStrategy{}
-	}
-
-	if obj.Spec.UpdateStrategy.Type == appsv1.RollingUpdateStatefulSetStrategyType {
-		if obj.Spec.UpdateStrategy.RollingUpdate == nil {
-			obj.Spec.UpdateStrategy.RollingUpdate = &RollingUpdateStatefulSetStrategy{}
-		}
-		if obj.Spec.UpdateStrategy.RollingUpdate.Partition == nil {
-			obj.Spec.UpdateStrategy.RollingUpdate.Partition = utilpointer.Int32Ptr(0)
-		}
-		if obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable == nil {
-			maxUnavailable := intstr.FromInt(1)
-			obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable = &maxUnavailable
-		}
-		if obj.Spec.UpdateStrategy.RollingUpdate.PodUpdatePolicy == "" {
-			obj.Spec.UpdateStrategy.RollingUpdate.PodUpdatePolicy = RecreatePodUpdateStrategyType
-		}
-		if obj.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds == nil {
-			obj.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds = utilpointer.Int32Ptr(0)
-		}
-	}
-
-	if obj.Spec.Replicas == nil {
-		obj.Spec.Replicas = utilpointer.Int32Ptr(1)
-	}
-	if obj.Spec.RevisionHistoryLimit == nil {
-		obj.Spec.RevisionHistoryLimit = utilpointer.Int32Ptr(10)
-	}
-
-	SetDefaultPodSpec(&obj.Spec.Template.Spec)
-	for i := range obj.Spec.VolumeClaimTemplates {
-		a := &obj.Spec.VolumeClaimTemplates[i]
-		v1.SetDefaults_PersistentVolumeClaim(a)
-		v1.SetDefaults_ResourceList(&a.Spec.Resources.Limits)
-		v1.SetDefaults_ResourceList(&a.Spec.Resources.Requests)
-		v1.SetDefaults_ResourceList(&a.Status.Capacity)
-	}
-}
-
-// SetDefaults_UnitedDeployment set default values for UnitedDeployment.
-func SetDefaultsUnitedDeployment(obj *UnitedDeployment) {
-	if obj.Spec.Replicas == nil {
-		obj.Spec.Replicas = utilpointer.Int32Ptr(1)
-	}
-	if obj.Spec.RevisionHistoryLimit == nil {
-		obj.Spec.RevisionHistoryLimit = utilpointer.Int32Ptr(10)
-	}
-
-	if len(obj.Spec.UpdateStrategy.Type) == 0 {
-		obj.Spec.UpdateStrategy.Type = ManualUpdateStrategyType
-	}
-
-	if obj.Spec.UpdateStrategy.Type == ManualUpdateStrategyType && obj.Spec.UpdateStrategy.ManualUpdate == nil {
-		obj.Spec.UpdateStrategy.ManualUpdate = &ManualUpdate{}
-	}
-
-	if obj.Spec.Template.StatefulSetTemplate != nil {
-		SetDefaultPodSpec(&obj.Spec.Template.StatefulSetTemplate.Spec.Template.Spec)
-		for i := range obj.Spec.Template.StatefulSetTemplate.Spec.VolumeClaimTemplates {
-			a := &obj.Spec.Template.StatefulSetTemplate.Spec.VolumeClaimTemplates[i]
-			v1.SetDefaults_PersistentVolumeClaim(a)
-			v1.SetDefaults_ResourceList(&a.Spec.Resources.Limits)
-			v1.SetDefaults_ResourceList(&a.Spec.Resources.Requests)
-			v1.SetDefaults_ResourceList(&a.Status.Capacity)
-		}
-	}
-}
-
-// SetDefaults_CloneSet set default values for CloneSet.
-func SetDefaultsCloneSet(obj *CloneSet) {
-	if obj.Spec.Replicas == nil {
-		obj.Spec.Replicas = utilpointer.Int32Ptr(1)
-	}
-	if obj.Spec.RevisionHistoryLimit == nil {
-		obj.Spec.RevisionHistoryLimit = utilpointer.Int32Ptr(10)
-	}
-
-	SetDefaultPodSpec(&obj.Spec.Template.Spec)
-	for i := range obj.Spec.VolumeClaimTemplates {
-		a := &obj.Spec.VolumeClaimTemplates[i]
-		v1.SetDefaults_PersistentVolumeClaim(a)
-		v1.SetDefaults_ResourceList(&a.Spec.Resources.Limits)
-		v1.SetDefaults_ResourceList(&a.Spec.Resources.Requests)
-		v1.SetDefaults_ResourceList(&a.Status.Capacity)
-	}
-
-	switch obj.Spec.UpdateStrategy.Type {
-	case "":
-		obj.Spec.UpdateStrategy.Type = RecreateCloneSetUpdateStrategyType
-	case InPlaceIfPossibleCloneSetUpdateStrategyType, InPlaceOnlyCloneSetUpdateStrategyType:
-		if obj.Spec.UpdateStrategy.InPlaceUpdateStrategy == nil {
-			obj.Spec.UpdateStrategy.InPlaceUpdateStrategy = &InPlaceUpdateStrategy{}
-		}
-	}
-
-	if obj.Spec.UpdateStrategy.Partition == nil {
-		obj.Spec.UpdateStrategy.Partition = utilpointer.Int32Ptr(0)
-	}
-	if obj.Spec.UpdateStrategy.MaxUnavailable == nil {
-		maxUnavailable := intstr.FromString(DefaultCloneSetMaxUnavailable)
-		obj.Spec.UpdateStrategy.MaxUnavailable = &maxUnavailable
-	}
-	if obj.Spec.UpdateStrategy.MaxSurge == nil {
-		maxSurge := intstr.FromInt(0)
-		obj.Spec.UpdateStrategy.MaxSurge = &maxSurge
-	}
-}
-
-// SetDefaults_DaemonSet set default values for DaemonSet.
-func SetDefaultsDaemonSet(obj *DaemonSet) {
-	if obj.Spec.BurstReplicas == nil {
-		BurstReplicas := intstr.FromInt(250)
-		obj.Spec.BurstReplicas = &BurstReplicas
-	}
-
-	if obj.Spec.UpdateStrategy.Type == "" {
-		obj.Spec.UpdateStrategy.Type = RollingUpdateDaemonSetStrategyType
-
-		// UpdateStrategy.RollingUpdate will take default values below.
-		obj.Spec.UpdateStrategy.RollingUpdate = &RollingUpdateDaemonSet{}
-	}
-
-	if obj.Spec.UpdateStrategy.Type == RollingUpdateDaemonSetStrategyType {
-		if obj.Spec.UpdateStrategy.RollingUpdate == nil {
-			obj.Spec.UpdateStrategy.RollingUpdate = &RollingUpdateDaemonSet{}
-		}
-		if obj.Spec.UpdateStrategy.RollingUpdate.Partition == nil {
-			obj.Spec.UpdateStrategy.RollingUpdate.Partition = new(int32)
-			*obj.Spec.UpdateStrategy.RollingUpdate.Partition = 0
-		}
-		if obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable == nil {
-			maxUnavailable := intstr.FromInt(1)
-			obj.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable = &maxUnavailable
-		}
-
-		if obj.Spec.UpdateStrategy.RollingUpdate.Type == "" {
-			obj.Spec.UpdateStrategy.RollingUpdate.Type = StandardRollingUpdateType
-		}
-		// Only when RollingUpdate Type is SurgingRollingUpdateType, it need to initialize the MaxSurge.
-		if obj.Spec.UpdateStrategy.RollingUpdate.Type == SurgingRollingUpdateType {
-			if obj.Spec.UpdateStrategy.RollingUpdate.MaxSurge == nil {
-				MaxSurge := intstr.FromInt(1)
-				obj.Spec.UpdateStrategy.RollingUpdate.MaxSurge = &MaxSurge
-			}
-		}
-	}
-
-	if obj.Spec.RevisionHistoryLimit == nil {
-		obj.Spec.RevisionHistoryLimit = new(int32)
-		*obj.Spec.RevisionHistoryLimit = 10
-	}
-}
-
-// SetDefaultPod sets default pod
-func SetDefaultPod(in *corev1.Pod) {
-	SetDefaultPodSpec(&in.Spec)
-	if in.Spec.EnableServiceLinks == nil {
-		enableServiceLinks := corev1.DefaultEnableServiceLinks
-		in.Spec.EnableServiceLinks = &enableServiceLinks
-	}
-}
-
-// SetDefaultPodSpec sets default pod spec
-func SetDefaultPodSpec(in *corev1.PodSpec) {
-	v1.SetDefaults_PodSpec(in)
-	for i := range in.Volumes {
-		a := &in.Volumes[i]
-		v1.SetDefaults_Volume(a)
-		if a.VolumeSource.HostPath != nil {
-			v1.SetDefaults_HostPathVolumeSource(a.VolumeSource.HostPath)
-		}
-		if a.VolumeSource.Secret != nil {
-			v1.SetDefaults_SecretVolumeSource(a.VolumeSource.Secret)
-		}
-		if a.VolumeSource.ISCSI != nil {
-			v1.SetDefaults_ISCSIVolumeSource(a.VolumeSource.ISCSI)
-		}
-		if a.VolumeSource.RBD != nil {
-			v1.SetDefaults_RBDVolumeSource(a.VolumeSource.RBD)
-		}
-		if a.VolumeSource.DownwardAPI != nil {
-			v1.SetDefaults_DownwardAPIVolumeSource(a.VolumeSource.DownwardAPI)
-			for j := range a.VolumeSource.DownwardAPI.Items {
-				b := &a.VolumeSource.DownwardAPI.Items[j]
-				if b.FieldRef != nil {
-					v1.SetDefaults_ObjectFieldSelector(b.FieldRef)
-				}
-			}
-		}
-		if a.VolumeSource.ConfigMap != nil {
-			v1.SetDefaults_ConfigMapVolumeSource(a.VolumeSource.ConfigMap)
-		}
-		if a.VolumeSource.AzureDisk != nil {
-			v1.SetDefaults_AzureDiskVolumeSource(a.VolumeSource.AzureDisk)
-		}
-		if a.VolumeSource.Projected != nil {
-			v1.SetDefaults_ProjectedVolumeSource(a.VolumeSource.Projected)
-			for j := range a.VolumeSource.Projected.Sources {
-				b := &a.VolumeSource.Projected.Sources[j]
-				if b.DownwardAPI != nil {
-					for k := range b.DownwardAPI.Items {
-						c := &b.DownwardAPI.Items[k]
-						if c.FieldRef != nil {
-							v1.SetDefaults_ObjectFieldSelector(c.FieldRef)
-						}
-					}
-				}
-				if b.ServiceAccountToken != nil {
-					v1.SetDefaults_ServiceAccountTokenProjection(b.ServiceAccountToken)
-				}
-			}
-		}
-		if a.VolumeSource.ScaleIO != nil {
-			v1.SetDefaults_ScaleIOVolumeSource(a.VolumeSource.ScaleIO)
-		}
-	}
-	for i := range in.InitContainers {
-		a := &in.InitContainers[i]
-		v1.SetDefaults_Container(a)
-		for j := range a.Ports {
-			b := &a.Ports[j]
-			v1.SetDefaults_ContainerPort(b)
-		}
-		for j := range a.Env {
-			b := &a.Env[j]
-			if b.ValueFrom != nil {
-				if b.ValueFrom.FieldRef != nil {
-					v1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef)
-				}
-			}
-		}
-		v1.SetDefaults_ResourceList(&a.Resources.Limits)
-		v1.SetDefaults_ResourceList(&a.Resources.Requests)
-		if a.LivenessProbe != nil {
-			v1.SetDefaults_Probe(a.LivenessProbe)
-			if a.LivenessProbe.Handler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.LivenessProbe.Handler.HTTPGet)
-			}
-		}
-		if a.ReadinessProbe != nil {
-			v1.SetDefaults_Probe(a.ReadinessProbe)
-			if a.ReadinessProbe.Handler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.Handler.HTTPGet)
-			}
-		}
-		if a.Lifecycle != nil {
-			if a.Lifecycle.PostStart != nil {
-				if a.Lifecycle.PostStart.HTTPGet != nil {
-					v1.SetDefaults_HTTPGetAction(a.Lifecycle.PostStart.HTTPGet)
-				}
-			}
-			if a.Lifecycle.PreStop != nil {
-				if a.Lifecycle.PreStop.HTTPGet != nil {
-					v1.SetDefaults_HTTPGetAction(a.Lifecycle.PreStop.HTTPGet)
-				}
-			}
-		}
-	}
-	for i := range in.Containers {
-		a := &in.Containers[i]
-		// For in-place update, we set default imagePullPolicy to Always
-		if a.ImagePullPolicy == "" {
-			a.ImagePullPolicy = corev1.PullAlways
-		}
-		v1.SetDefaults_Container(a)
-		for j := range a.Ports {
-			b := &a.Ports[j]
-			v1.SetDefaults_ContainerPort(b)
-		}
-		for j := range a.Env {
-			b := &a.Env[j]
-			if b.ValueFrom != nil {
-				if b.ValueFrom.FieldRef != nil {
-					v1.SetDefaults_ObjectFieldSelector(b.ValueFrom.FieldRef)
-				}
-			}
-		}
-		v1.SetDefaults_ResourceList(&a.Resources.Limits)
-		v1.SetDefaults_ResourceList(&a.Resources.Requests)
-		if a.LivenessProbe != nil {
-			v1.SetDefaults_Probe(a.LivenessProbe)
-			if a.LivenessProbe.Handler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.LivenessProbe.Handler.HTTPGet)
-			}
-		}
-		if a.ReadinessProbe != nil {
-			v1.SetDefaults_Probe(a.ReadinessProbe)
-			if a.ReadinessProbe.Handler.HTTPGet != nil {
-				v1.SetDefaults_HTTPGetAction(a.ReadinessProbe.Handler.HTTPGet)
-			}
-		}
-		if a.Lifecycle != nil {
-			if a.Lifecycle.PostStart != nil {
-				if a.Lifecycle.PostStart.HTTPGet != nil {
-					v1.SetDefaults_HTTPGetAction(a.Lifecycle.PostStart.HTTPGet)
-				}
-			}
-			if a.Lifecycle.PreStop != nil {
-				if a.Lifecycle.PreStop.HTTPGet != nil {
-					v1.SetDefaults_HTTPGetAction(a.Lifecycle.PreStop.HTTPGet)
-				}
-			}
-		}
-	}
-}

apis/apps/v1alpha1/doc.go

@@ -14,5 +14,6 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
+// +k8s:openapi-gen=true
 // +groupName=apps.kruise.io
 package v1alpha1

apis/apps/v1alpha1/ephemeraljob_types.go

@@ -0,0 +1,229 @@
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+const (
+	EphemeralContainerEnvKey = "KRUISE_EJOB_ID"
+)
+
+// EphemeralJobSpec defines the desired state of EphemeralJob
+type EphemeralJobSpec struct {
+	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+	// Selector is a label query over pods that should match the pod labels.
+	Selector *metav1.LabelSelector `json:"selector"`
+
+	// Replicas indicates a part of the quantity from matched pods by selector.
+	// Usually it is used for gray scale working.
+	// if Replicas exceeded the matched number by selector or not be set, replicas will not work.
+	Replicas *int32 `json:"replicas,omitempty"`
+
+	// Parallelism specifies the maximum desired number of pods which matches running ephemeral containers.
+	// +optional
+	Parallelism *int32 `json:"parallelism,omitempty" protobuf:"varint,1,opt,name=parallelism"`
+
+	// Template describes the ephemeral container that will be created.
+	Template EphemeralContainerTemplateSpec `json:"template"`
+
+	// Paused will pause the ephemeral job.
+	// +optional
+	Paused bool `json:"paused,omitempty" protobuf:"bytes,4,opt,name=paused"`
+
+	// ActiveDeadlineSeconds specifies the duration in seconds relative to the startTime that the job may be active
+	// before the system tries to terminate it; value must be positive integer.
+	// Only works for Always type.
+	// +optional
+	ActiveDeadlineSeconds *int64 `json:"activeDeadlineSeconds,omitempty" protobuf:"varint,2,opt,name=activeDeadlineSeconds"`
+
+	// ttlSecondsAfterFinished limits the lifetime of a Job that has finished
+	// execution (either Complete or Failed). If this field is set,
+	// ttlSecondsAfterFinished after the eJob finishes, it is eligible to be
+	// automatically deleted. When the Job is being deleted, its lifecycle
+	// guarantees (e.g. finalizers) will be honored.
+	// If this field is unset, default value is 1800
+	// If this field is set to zero,
+	// the Job becomes eligible to be deleted immediately after it finishes.
+	// +optional
+	TTLSecondsAfterFinished *int32 `json:"ttlSecondsAfterFinished,omitempty" protobuf:"varint,4,opt,name=ttlSecondsAfterFinished"`
+}
+
+// EphemeralContainerTemplateSpec describes template spec of ephemeral containers
+type EphemeralContainerTemplateSpec struct {
+
+	// EphemeralContainers defines ephemeral container list in match pods.
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	EphemeralContainers []v1.EphemeralContainer `json:"ephemeralContainers" patchStrategy:"merge" patchMergeKey:"name"`
+}
+
+// EphemeralJobStatus defines the observed state of EphemeralJob
+type EphemeralJobStatus struct {
+	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+	// +optional
+	// +patchMergeKey=type
+	// +patchStrategy=merge
+	Conditions []EphemeralJobCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type" protobuf:"bytes,1,rep,name=conditions"`
+
+	// Represents time when the job was acknowledged by the job controller.
+	// It is not guaranteed to be set in happens-before order across separate operations.
+	// It is represented in RFC3339 form and is in UTC.
+	// +optional
+	StartTime *metav1.Time `json:"startTime,omitempty" protobuf:"bytes,2,opt,name=startTime"`
+
+	// Represents time when the job was completed. It is not guaranteed to
+	// be set in happens-before order across separate operations.
+	// It is represented in RFC3339 form and is in UTC.
+	// +optional
+	CompletionTime *metav1.Time `json:"completionTime,omitempty" protobuf:"bytes,3,opt,name=completionTime"`
+
+	// The phase of the job.
+	// +optional
+	Phase EphemeralJobPhase `json:"phase" protobuf:"varint,8,opt,name=phase"`
+
+	// The number of total matched pods.
+	// +optional
+	Matches int32 `json:"match" protobuf:"varint,4,opt,name=match"`
+
+	// The number of actively running pods.
+	// +optional
+	Running int32 `json:"running" protobuf:"varint,4,opt,name=running"`
+
+	// The number of pods which reached phase Succeeded.
+	// +optional
+	Succeeded int32 `json:"succeeded" protobuf:"varint,5,opt,name=completed"`
+
+	// The number of waiting pods.
+	// +optional
+	Waiting int32 `json:"waiting" protobuf:"varint,4,opt,name=waiting"`
+
+	// The number of pods which reached phase Failed.
+	// +optional
+	Failed int32 `json:"failed" protobuf:"varint,6,opt,name=failed"`
+}
+
+// JobCondition describes current state of a job.
+type EphemeralJobCondition struct {
+	// Type of job condition, Complete or Failed.
+	Type EphemeralJobConditionType `json:"type" protobuf:"bytes,1,opt,name=type,casttype=EphemeralJobConditionType"`
+	// Status of the condition, one of True, False, Unknown.
+	Status v1.ConditionStatus `json:"status" protobuf:"bytes,2,opt,name=status,casttype=k8s.io/api/core/v1.ConditionStatus"`
+	// Last time the condition was checked.
+	// +optional
+	LastProbeTime metav1.Time `json:"lastProbeTime,omitempty" protobuf:"bytes,3,opt,name=lastProbeTime"`
+	// Last time the condition transit from one status to another.
+	// +optional
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty" protobuf:"bytes,4,opt,name=lastTransitionTime"`
+	// (brief) reason for the condition's last transition.
+	// +optional
+	Reason string `json:"reason,omitempty" protobuf:"bytes,5,opt,name=reason"`
+	// Human readable message indicating details about last transition.
+	// +optional
+	Message string `json:"message,omitempty" protobuf:"bytes,6,opt,name=message"`
+}
+
+// JobConditionType indicates valid conditions type of a job
+type EphemeralJobConditionType string
+
+// These are valid conditions of a job.
+const (
+	// EJobSucceeded means the ephemeral job has succeeded completed its execution. A succeeded job means pods have been
+	// successful finished on all tasks on eligible nodes.
+	EJobSucceeded EphemeralJobConditionType = "JobSucceeded"
+
+	// EJobFailed means there are some ephemeral containers matched by ephemeral job failed.
+	EJobFailed EphemeralJobConditionType = "JobFailed"
+
+	// EJobError means some ephemeral containers matched by ephemeral job run error.
+	EJobError EphemeralJobConditionType = "JobError"
+
+	// EJobInitialized means ephemeral job has created and initialized by controller.
+	EJobInitialized EphemeralJobConditionType = "Initialized"
+
+	// EJobMatchedEmpty means the ephemeral job has not matched the target pods.
+	EJobMatchedEmpty EphemeralJobConditionType = "MatchedEmpty"
+)
+
+// EphemeralJobPhase indicates the type of EphemeralJobPhase.
+type EphemeralJobPhase string
+
+const (
+	// EphemeralJobSucceeded means the job has succeeded.
+	EphemeralJobSucceeded EphemeralJobPhase = "Succeeded"
+
+	// EphemeralJobFailed means the job has failed.
+	EphemeralJobFailed EphemeralJobPhase = "Failed"
+
+	// EphemeralJobWaiting means the job is waiting.
+	EphemeralJobWaiting EphemeralJobPhase = "Waiting"
+
+	// EphemeralJobRunning means the job is running.
+	EphemeralJobRunning EphemeralJobPhase = "Running"
+
+	// EphemeralJobPause means the ephemeral job paused.
+	EphemeralJobPause EphemeralJobPhase = "Paused"
+
+	// EphemeralJobError means the ephemeral  paused.
+	EphemeralJobError EphemeralJobPhase = "Error"
+
+	EphemeralJobUnknown EphemeralJobPhase = "Unknown"
+)
+
+// +genclient
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:shortName=ejob
+// +kubebuilder:printcolumn:name="STATUS",type="string",JSONPath=".status.phase",description="The status of ephemeral job"
+// +kubebuilder:printcolumn:name="MATCH",type="integer",JSONPath=".status.match",description="Number of ephemeral container matched by this job"
+// +kubebuilder:printcolumn:name="SUCCEED",type="integer",JSONPath=".status.succeeded",description="Number of succeed ephemeral containers"
+// +kubebuilder:printcolumn:name="FAILED",type="integer",JSONPath=".status.failed",description="Number of failed ephemeral containers"
+// +kubebuilder:printcolumn:name="RUNNING",type="integer",JSONPath=".status.running",description="Number of running ephemeral containers"
+// +kubebuilder:printcolumn:name="WAITING",type="integer",JSONPath=".status.waiting",description="Number of waiting ephemeral containers"
+// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."
+
+// EphemeralJob is the Schema for the ephemeraljobs API
+type EphemeralJob struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   EphemeralJobSpec   `json:"spec,omitempty"`
+	Status EphemeralJobStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// EphemeralJobList contains a list of EphemeralJob
+type EphemeralJobList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []EphemeralJob `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&EphemeralJob{}, &EphemeralJobList{})
+}

apis/apps/v1alpha1/imagelistpulljob_types.go

@@ -0,0 +1,109 @@
+/*
+Copyright 2023 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// ImageListPullJobSpec defines the desired state of ImageListPullJob
+type ImageListPullJobSpec struct {
+	// Images is the image list to be pulled by the job
+	Images []string `json:"images"`
+
+	ImagePullJobTemplate `json:",inline"`
+}
+
+// ImageListPullJobStatus defines the observed state of ImageListPullJob
+type ImageListPullJobStatus struct {
+	// Represents time when the job was acknowledged by the job controller.
+	// It is not guaranteed to be set in happens-before order across separate operations.
+	// It is represented in RFC3339 form and is in UTC.
+	// +optional
+	StartTime *metav1.Time `json:"startTime,omitempty"`
+
+	// Represents time when the all the image pull job was completed. It is not guaranteed to
+	// be set in happens-before order across separate operations.
+	// It is represented in RFC3339 form and is in UTC.
+	// +optional
+	CompletionTime *metav1.Time `json:"completionTime,omitempty"`
+
+	// The desired number of ImagePullJobs, this is typically equal to the number of len(spec.Images).
+	Desired int32 `json:"desired"`
+
+	// The number of running ImagePullJobs which are acknowledged by the imagepulljob controller.
+	// +optional
+	Active int32 `json:"active"`
+
+	// The number of ImagePullJobs which are finished
+	// +optional
+	Completed int32 `json:"completed"`
+
+	// The number of image pull job which are finished and status.Succeeded==status.Desired.
+	// +optional
+	Succeeded int32 `json:"succeeded"`
+
+	// The status of ImagePullJob which has the failed nodes(status.Failed>0) .
+	// +optional
+	FailedImageStatuses []*FailedImageStatus `json:"failedImageStatuses,omitempty"`
+}
+
+// FailedImageStatus the state of ImagePullJob which has the failed nodes(status.Failed>0)
+type FailedImageStatus struct {
+	// The name of ImagePullJob which has the failed nodes(status.Failed>0)
+	// +optional
+	ImagePullJob string `json:"imagePullJob,omitempty"`
+
+	// Name of the image
+	// +optional
+	Name string `json:"name,omitempty"`
+
+	// The text prompt for job running status.
+	// +optional
+	Message string `json:"message,omitempty"`
+}
+
+// +genclient
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:name="TOTAL",type="integer",JSONPath=".status.desired",description="Number of image pull job"
+// +kubebuilder:printcolumn:name="SUCCEEDED",type="integer",JSONPath=".status.succeeded",description="Number of image pull job succeeded"
+// +kubebuilder:printcolumn:name="COMPLETED",type="integer",JSONPath=".status.completed",description="Number of ImagePullJobs which are finished"
+// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."
+
+// ImageListPullJob is the Schema for the imagelistpulljobs API
+type ImageListPullJob struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ImageListPullJobSpec   `json:"spec,omitempty"`
+	Status ImageListPullJobStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// ImageListPullJobList contains a list of ImageListPullJob
+type ImageListPullJobList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ImageListPullJob `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&ImageListPullJob{}, &ImageListPullJobList{})
+}

apis/apps/v1alpha1/imagepulljob_types.go

@@ -21,10 +21,31 @@ import (
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+const (
+	ImagePreDownloadParallelismKey      = "apps.kruise.io/image-predownload-parallelism"
+	ImagePreDownloadTimeoutSecondsKey   = "apps.kruise.io/image-predownload-timeout-seconds"
+	ImagePreDownloadMinUpdatedReadyPods = "apps.kruise.io/image-predownload-min-updated-ready-pods"
+)
+
+// ImagePullPolicy describes a policy for if/when to pull a container image
+// +enum
+type ImagePullPolicy string
+
+const (
+	// PullAlways means that kruise-daemon always attempts to pull the latest image.
+	PullAlways ImagePullPolicy = "Always"
+	// PullIfNotPresent means that kruise-daemon pulls if the image isn't present on disk.
+	PullIfNotPresent ImagePullPolicy = "IfNotPresent"
+)
+
 // ImagePullJobSpec defines the desired state of ImagePullJob
 type ImagePullJobSpec struct {
 	// Image is the image to be pulled by the job
-	Image string `json:"image"`
+	Image                string `json:"image"`
+	ImagePullJobTemplate `json:",inline"`
+}
+
+type ImagePullJobTemplate struct {
 
 	// ImagePullSecrets is an optional list of references to secrets in the same namespace to use for pulling the image.
 	// If specified, these secrets will be passed to individual puller implementations for them to use.  For example,
@@ -35,7 +56,12 @@ type ImagePullJobSpec struct {
 	// Selector is a query over nodes that should match the job.
 	// nil to match all nodes.
 	// +optional
-	Selector *NodeSelector `json:"selector,omitempty"`
+	Selector *ImagePullJobNodeSelector `json:"selector,omitempty"`
+
+	// PodSelector is a query over pods that should pull image on nodes of these pods.
+	// Mutually exclusive with Selector.
+	// +optional
+	PodSelector *ImagePullJobPodSelector `json:"podSelector,omitempty"`
 
 	// Parallelism is the requested parallelism, it can be set to any non-negative value. If it is unspecified,
 	// it defaults to 1. If it is specified as 0, then the Job is effectively paused until it is increased.
@@ -50,10 +76,26 @@ type ImagePullJobSpec struct {
 	// CompletionPolicy indicates the completion policy of the job.
 	// Default is Always CompletionPolicyType.
 	CompletionPolicy CompletionPolicy `json:"completionPolicy"`
+
+	// SandboxConfig support attach metadata in PullImage CRI interface during ImagePulljobs
+	// +optional
+	SandboxConfig *SandboxConfig `json:"sandboxConfig,omitempty"`
+
+	// Image pull policy.
+	// One of Always, IfNotPresent. Defaults to IfNotPresent.
+	// +optional
+	ImagePullPolicy ImagePullPolicy `json:"imagePullPolicy,omitempty"`
 }
 
-// NodeSelector is a selector over nodes
-type NodeSelector struct {
+// ImagePullJobPodSelector is a selector over pods
+type ImagePullJobPodSelector struct {
+	// LabelSelector is a label query over pods that should match the job.
+	// +optional
+	metav1.LabelSelector `json:",inline"`
+}
+
+// ImagePullJobNodeSelector is a selector over nodes
+type ImagePullJobNodeSelector struct {
 	// Names specify a set of nodes to execute the job.
 	// +optional
 	Names []string `json:"names,omitempty"`
@@ -115,6 +157,7 @@ type ImagePullJobStatus struct {
 }
 
 // +genclient
+// +k8s:openapi-gen=true
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:printcolumn:name="TOTAL",type="integer",JSONPath=".status.desired",description="Number of all nodes matched by this job"

apis/apps/v1alpha1/inplace_update.go

@@ -1,64 +0,0 @@
-/*
-Copyright 2020 The Kruise Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1alpha1
-
-import (
-	v1 "k8s.io/api/core/v1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-const (
-	// InPlaceUpdateReady must be added into template.spec.readinessGates when pod podUpdatePolicy
-	// is InPlaceIfPossible or InPlaceOnly. The condition in podStatus will be updated to False before in-place
-	// updating and updated to True after the update is finished. This ensures pod to remain at NotReady state while
-	// in-place update is happening.
-	InPlaceUpdateReady v1.PodConditionType = "InPlaceUpdateReady"
-
-	// InPlaceUpdateStateKey records the state of inplace-update.
-	// The value of annotation is InPlaceUpdateState.
-	InPlaceUpdateStateKey string = "inplace-update-state"
-
-	// InPlaceUpdateGraceKey records the spec that Pod should be updated when
-	// grace period ends.
-	InPlaceUpdateGraceKey string = "inplace-update-grace"
-)
-
-// InPlaceUpdateState records latest inplace-update state, including old statuses of containers.
-type InPlaceUpdateState struct {
-	// Revision is the updated revision hash.
-	Revision string `json:"revision"`
-
-	// UpdateTimestamp is the time when the in-place update happens.
-	UpdateTimestamp metav1.Time `json:"updateTimestamp"`
-
-	// LastContainerStatuses records the before-in-place-update container statuses. It is a map from ContainerName
-	// to InPlaceUpdateContainerStatus
-	LastContainerStatuses map[string]InPlaceUpdateContainerStatus `json:"lastContainerStatuses"`
-}
-
-// InPlaceUpdateContainerStatus records the statuses of the container that are mainly used
-// to determine whether the InPlaceUpdate is completed.
-type InPlaceUpdateContainerStatus struct {
-	ImageID string `json:"imageID,omitempty"`
-}
-
-// InPlaceUpdateStrategy defines the strategies for in-place update.
-type InPlaceUpdateStrategy struct {
-	// GracePeriodSeconds is the timespan between set Pod status to not-ready and update images in Pod spec
-	// when in-place update a Pod.
-	GracePeriodSeconds int32 `json:"gracePeriodSeconds,omitempty"`
-}

apis/apps/v1alpha1/lifecycle.go

@@ -1,43 +0,0 @@
-/*
-Copyright 2020 The Kruise Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1alpha1
-
-const (
-	LifecycleStateKey     = "lifecycle.apps.kruise.io/state"
-	LifecycleTimestampKey = "lifecycle.apps.kruise.io/timestamp"
-
-	LifecycleStateNormal          LifecycleStateType = "Normal"
-	LifecycleStatePreparingUpdate LifecycleStateType = "PreparingUpdate"
-	LifecycleStateUpdating        LifecycleStateType = "Updating"
-	LifecycleStateUpdated         LifecycleStateType = "Updated"
-	LifecycleStatePreparingDelete LifecycleStateType = "PreparingDelete"
-)
-
-type LifecycleStateType string
-
-// Lifecycle contains the hooks for Pod lifecycle.
-type Lifecycle struct {
-	// PreDelete is the hook before Pod to be deleted.
-	PreDelete *LifecycleHook `json:"preDelete,omitempty"`
-	// InPlaceUpdate is the hook before Pod to update and after Pod has been updated.
-	InPlaceUpdate *LifecycleHook `json:"inPlaceUpdate,omitempty"`
-}
-
-type LifecycleHook struct {
-	LabelsHandler     map[string]string `json:"labelsHandler,omitempty"`
-	FinalizersHandler []string          `json:"finalizersHandler,omitempty"`
-}

apis/apps/v1alpha1/node_pod_probe_types.go

@@ -0,0 +1,129 @@
+/*
+Copyright 2022 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless persistent by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// NodePodProbeSpec defines the desired state of NodePodProbe
+type NodePodProbeSpec struct {
+	PodProbes []PodProbe `json:"podProbes,omitempty"`
+}
+
+type PodProbe struct {
+	// pod name
+	Name string `json:"name"`
+	// pod namespace
+	Namespace string `json:"namespace"`
+	// pod uid
+	UID string `json:"uid"`
+	// pod ip
+	IP string `json:"IP"`
+	// Custom container probe, supports Exec, Tcp, and returns the result to Pod yaml
+	Probes []ContainerProbe `json:"probes,omitempty"`
+}
+
+type ContainerProbe struct {
+	// Name is podProbeMarker.Name#probe.Name
+	Name string `json:"name"`
+	// container name
+	ContainerName string `json:"containerName"`
+	// container probe spec
+	Probe ContainerProbeSpec `json:"probe"`
+}
+
+type NodePodProbeStatus struct {
+	// pod probe results
+	PodProbeStatuses []PodProbeStatus `json:"podProbeStatuses,omitempty"`
+}
+
+type PodProbeStatus struct {
+	// pod name
+	Name string `json:"name"`
+	// pod namespace
+	Namespace string `json:"namespace"`
+	// pod uid
+	UID string `json:"uid"`
+	// pod probe result
+	ProbeStates []ContainerProbeState `json:"probeStates,omitempty"`
+}
+
+type ContainerProbeState struct {
+	// Name is podProbeMarker.Name#probe.Name
+	Name string `json:"name"`
+	// container probe exec state, True or False
+	State ProbeState `json:"state"`
+	// Last time we probed the condition.
+	// +optional
+	LastProbeTime metav1.Time `json:"lastProbeTime,omitempty"`
+	// Last time the condition transitioned from one status to another.
+	// +optional
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+	// If Status=True, Message records the return result of Probe.
+	// If Status=False, Message records Probe's error message
+	Message string `json:"message,omitempty"`
+}
+
+type ProbeState string
+
+const (
+	ProbeSucceeded ProbeState = "Succeeded"
+	ProbeFailed    ProbeState = "Failed"
+	ProbeUnknown   ProbeState = "Unknown"
+)
+
+func (p ProbeState) IsEqualPodConditionStatus(status corev1.ConditionStatus) bool {
+	switch status {
+	case corev1.ConditionTrue:
+		return p == ProbeSucceeded
+	case corev1.ConditionFalse:
+		return p == ProbeFailed
+	default:
+		return p == ProbeUnknown
+	}
+}
+
+// +genclient
+// +genclient:nonNamespaced
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+// +kubebuilder:resource:scope=Cluster
+// +kubebuilder:subresource:status
+
+// NodePodProbe is the Schema for the NodePodProbe API
+type NodePodProbe struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   NodePodProbeSpec   `json:"spec,omitempty"`
+	Status NodePodProbeStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// NodePodProbeList contains a list of NodePodProbe
+type NodePodProbeList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []NodePodProbe `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&NodePodProbe{}, &NodePodProbeList{})
+}

apis/apps/v1alpha1/nodeimage_types.go

@@ -24,6 +24,7 @@ import (
 // NodeImageSpec defines the desired state of NodeImage
 type NodeImageSpec struct {
 	// Specifies images to be pulled on this node
+	// It can not be more than 256 for each NodeImage
 	Images map[string]ImageSpec `json:"images,omitempty"`
 }
 
@@ -37,6 +38,10 @@ type ImageSpec struct {
 
 	// Tags is a list of versions of this image
 	Tags []ImageTagSpec `json:"tags"`
+
+	// SandboxConfig support attach metadata in PullImage CRI interface during ImagePulljobs
+	// +optional
+	SandboxConfig *SandboxConfig `json:"sandboxConfig,omitempty"`
 }
 
 // ReferenceObject comprises a resource name, with a mandatory namespace,
@@ -75,6 +80,11 @@ type ImageTagSpec struct {
 	// Value must be treated as opaque by clients and .
 	// +optional
 	Version int64 `json:"version,omitempty"`
+
+	// Image pull policy.
+	// One of Always, IfNotPresent. Defaults to IfNotPresent.
+	// +optional
+	ImagePullPolicy ImagePullPolicy `json:"imagePullPolicy,omitempty"`
 }
 
 // ImageTagPullPolicy defines the policy of the pulling task
@@ -120,6 +130,10 @@ type NodeImageStatus struct {
 	// +optional
 	Pulling int32 `json:"pulling"`
 
+	// The number of pulling tasks which are waiting.
+	// +optional
+	Waiting int32 `json:"waiting"`
+
 	// all statuses of active image pulling tasks
 	ImageStatuses map[string]ImageStatus `json:"imageStatuses,omitempty"`
 
@@ -143,7 +157,7 @@ type ImageTagStatus struct {
 	// Represents the image pulling task phase.
 	Phase ImagePullPhase `json:"phase"`
 
-	// Represents the pulling progress of this tag, which is beetween 0-100. There is no guarantee
+	// Represents the pulling progress of this tag, which is between 0-100. There is no guarantee
 	// of monotonic consistency, and it may be a rollback due to retry during pulling.
 	Progress int32 `json:"progress,omitempty"`
 
@@ -167,7 +181,7 @@ type ImageTagStatus struct {
 	// +optional
 	ImageID string `json:"imageID,omitempty"`
 
-	// Represents the summary informations of this node
+	// Represents the summary information of this node
 	// +optional
 	Message string `json:"message,omitempty"`
 }
@@ -205,6 +219,7 @@ const (
 
 // +genclient
 // +genclient:nonNamespaced
+// +k8s:openapi-gen=true
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:scope=Cluster
 // +kubebuilder:subresource:status

apis/apps/v1alpha1/persistent_pod_state_types.go

@@ -0,0 +1,133 @@
+/*
+Copyright 2022 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless persistent by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are persistent.  Any new fields you add must have json tags for the fields to be serialized.
+
+const (
+	// AnnotationAutoGeneratePersistentPodState indicates kruise will auto generate PersistentPodState object
+	// Need to work with AnnotationRequiredPersistentTopology and AnnotationPreferredPersistentTopology
+	AnnotationAutoGeneratePersistentPodState = "kruise.io/auto-generate-persistent-pod-state"
+	// AnnotationRequiredPersistentTopology Pod rebuilt topology required for node labels
+	// for example kruise.io/required-persistent-topology: topology.kubernetes.io/zone[,xxx]
+	// optional
+	AnnotationRequiredPersistentTopology = "kruise.io/required-persistent-topology"
+	// AnnotationPreferredPersistentTopology Pod rebuilt topology preferred for node labels and default with 100 weight
+	// for example kruise.io/preferred-persistent-topology: kubernetes.io/hostname[,xxx]
+	// optional
+	AnnotationPreferredPersistentTopology = "kruise.io/preferred-persistent-topology"
+	// AnnotationPersistentPodAnnotations Pod needs persistent annotations
+	// for example kruise.io/persistent-pod-annotations: cni.projectcalico.org/podIP[,xxx]
+	// optional
+	AnnotationPersistentPodAnnotations = "kruise.io/persistent-pod-annotations"
+)
+
+// PersistentPodStateSpec defines the desired state of PersistentPodState
+type PersistentPodStateSpec struct {
+	// TargetReference contains enough information to let you identify an workload for PersistentPodState
+	// Selector and TargetReference are mutually exclusive, TargetReference is priority to take effect
+	// current only support StatefulSet
+	TargetReference TargetReference `json:"targetRef"`
+
+	// Persist the annotations information of the pods that need to be saved
+	PersistentPodAnnotations []PersistentPodAnnotation `json:"persistentPodAnnotations,omitempty"`
+
+	// Pod rebuilt topology required for node labels
+	// for example kubernetes.io/hostname, failure-domain.beta.kubernetes.io/zone
+	RequiredPersistentTopology *NodeTopologyTerm `json:"requiredPersistentTopology,omitempty"`
+
+	// Pod rebuilt topology preferred for node labels, with xx weight
+	// for example  kubernetes.io/hostname, failure-domain.beta.kubernetes.io/zone
+	PreferredPersistentTopology []PreferredTopologyTerm `json:"preferredPersistentTopology,omitempty"`
+
+	// PersistentPodStateRetentionPolicy describes the policy used for PodState.
+	// The default policy of 'WhenScaled' causes when scale down statefulSet, deleting it.
+	// +optional
+	PersistentPodStateRetentionPolicy PersistentPodStateRetentionPolicyType `json:"persistentPodStateRetentionPolicy,omitempty"`
+}
+
+type PreferredTopologyTerm struct {
+	Weight     int32            `json:"weight"`
+	Preference NodeTopologyTerm `json:"preference"`
+}
+type NodeTopologyTerm struct {
+	// A list of node selector requirements by node's labels.
+	NodeTopologyKeys []string `json:"nodeTopologyKeys"`
+}
+
+type PersistentPodAnnotation struct {
+	Key string `json:"key"`
+}
+
+type PersistentPodStateRetentionPolicyType string
+
+const (
+	// PersistentPodStateRetentionPolicyWhenScaled specifies when scale down statefulSet, deleting podState record.
+	PersistentPodStateRetentionPolicyWhenScaled = "WhenScaled"
+	// PersistentPodStateRetentionPolicyWhenDeleted specifies when delete statefulSet, deleting podState record.
+	PersistentPodStateRetentionPolicyWhenDeleted = "WhenDeleted"
+)
+
+type PersistentPodStateStatus struct {
+	// observedGeneration is the most recent generation observed for this PersistentPodState. It corresponds to the
+	// PersistentPodState's generation, which is updated on mutation by the API Server.
+	ObservedGeneration int64 `json:"observedGeneration"`
+	// When the pod is ready, record some status information of the pod, such as: labels, annotations, topologies, etc.
+	// map[string]PodState -> map[Pod.Name]PodState
+	PodStates map[string]PodState `json:"podStates,omitempty"`
+}
+
+type PodState struct {
+	// pod.spec.nodeName
+	NodeName string `json:"nodeName,omitempty"`
+	// node topology labels key=value
+	// for example kubernetes.io/hostname=node-1
+	NodeTopologyLabels map[string]string `json:"nodeTopologyLabels,omitempty"`
+	// pod persistent annotations
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+// +genclient
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+
+// PersistentPodState is the Schema for the PersistentPodState API
+type PersistentPodState struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   PersistentPodStateSpec   `json:"spec,omitempty"`
+	Status PersistentPodStateStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// PersistentPodStateList contains a list of PersistentPodState
+type PersistentPodStateList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []PersistentPodState `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&PersistentPodState{}, &PersistentPodStateList{})
+}

apis/apps/v1alpha1/pod_probe_marker_types.go

@@ -0,0 +1,132 @@
+/*
+Copyright 2022 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless persistent by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
+
+const (
+	// PodProbeMarkerAnnotationKey records the Probe Spec, mainly used for serverless Pod scenarios, as follows:
+	//  annotations:
+	//    kruise.io/podprobe: |
+	//      [
+	//          {
+	//              "containerName": "minecraft",
+	//              "name": "healthy",
+	//              "podConditionType": "game.kruise.io/healthy",
+	//              "probe": {
+	//                  "exec": {
+	//                      "command": [
+	//                          "bash",
+	//                          "/data/probe.sh"
+	//                      ]
+	//                  }
+	//              }
+	//          }
+	//      ]
+	PodProbeMarkerAnnotationKey = "kruise.io/podprobe"
+	// PodProbeMarkerListAnnotationKey records the injected PodProbeMarker Name List
+	// example: kruise.io/podprobemarker-list="probe-marker-1,probe-marker-2"
+	PodProbeMarkerListAnnotationKey = "kruise.io/podprobemarker-list"
+)
+
+// PodProbeMarkerSpec defines the desired state of PodProbeMarker
+type PodProbeMarkerSpec struct {
+	// Selector is a label query over pods that should exec custom probe
+	// It must match the pod template's labels.
+	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
+	Selector *metav1.LabelSelector `json:"selector"`
+	// Custom container probe, current only support Exec().
+	// Probe Result will record in Pod.Status.Conditions, and condition.type=probe.name.
+	// condition.status=True indicates probe success
+	// condition.status=False indicates probe fails
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Probes []PodContainerProbe `json:"probes" patchStrategy:"merge" patchMergeKey:"name"`
+}
+
+type PodContainerProbe struct {
+	// probe name, unique within the Pod(Even between different containers, they cannot be the same)
+	Name string `json:"name"`
+	// container name
+	ContainerName string `json:"containerName"`
+	// container probe spec
+	Probe ContainerProbeSpec `json:"probe"`
+	// According to the execution result of ContainerProbe, perform specific actions,
+	// such as: patch Pod labels, annotations, ReadinessGate Condition
+	// It cannot be null at the same time as PodConditionType.
+	// +patchMergeKey=state
+	// +patchStrategy=merge
+	MarkerPolicy []ProbeMarkerPolicy `json:"markerPolicy,omitempty"  patchStrategy:"merge" patchMergeKey:"state"`
+	// If it is not empty, the Probe execution result will be recorded on the Pod condition.
+	// It cannot be null at the same time as MarkerPolicy.
+	// For example PodConditionType=game.kruise.io/healthy, pod.status.condition.type = game.kruise.io/healthy.
+	// When probe is Succeeded, pod.status.condition.status = True. Otherwise, when the probe fails to execute, pod.status.condition.status = False.
+	PodConditionType string `json:"podConditionType,omitempty"`
+}
+
+type ContainerProbeSpec struct {
+	v1.Probe `json:",inline"`
+}
+
+type ProbeMarkerPolicy struct {
+	// probe status, True or False
+	// For example: State=Succeeded, annotations[controller.kubernetes.io/pod-deletion-cost] = '10'.
+	// State=Failed, annotations[controller.kubernetes.io/pod-deletion-cost] = '-10'.
+	// In addition, if State=Failed is not defined, Exec execution fails, and the annotations[controller.kubernetes.io/pod-deletion-cost] will be Deleted
+	State ProbeState `json:"state"`
+	// Patch Labels pod.labels
+	Labels map[string]string `json:"labels,omitempty"`
+	// Patch annotations pod.annotations
+	Annotations map[string]string `json:"annotations,omitempty"`
+}
+
+type PodProbeMarkerStatus struct {
+	// observedGeneration is the most recent generation observed for this PodProbeMarker. It corresponds to the
+	// PodProbeMarker's generation, which is updated on mutation by the API Server.
+	ObservedGeneration int64 `json:"observedGeneration"`
+	// matched Pods
+	MatchedPods int64 `json:"matchedPods,omitempty"`
+}
+
+// +genclient
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+
+// PodProbeMarker is the Schema for the PodProbeMarker API
+type PodProbeMarker struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   PodProbeMarkerSpec   `json:"spec,omitempty"`
+	Status PodProbeMarkerStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// PodProbeMarkerList contains a list of PodProbeMarker
+type PodProbeMarkerList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []PodProbeMarker `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&PodProbeMarker{}, &PodProbeMarkerList{})
+}

apis/apps/v1alpha1/resourcedistribution_types.go

@@ -0,0 +1,191 @@
+/*
+Copyright 2021 The Kruise Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+// ResourceDistributionSpec defines the desired state of ResourceDistribution.
+type ResourceDistributionSpec struct {
+	// INSERT ADDITIONAL SPEC FIELDS - desired state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+
+	// Resource must be the complete yaml that users want to distribute.
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:EmbeddedResource
+	Resource runtime.RawExtension `json:"resource"`
+
+	// Targets defines the namespaces that users want to distribute to.
+	Targets ResourceDistributionTargets `json:"targets"`
+}
+
+// ResourceDistributionTargets defines the targets of Resource.
+// Four options are provided to select target namespaces.
+type ResourceDistributionTargets struct {
+	// Priority: ExcludedNamespaces > AllNamespaces = IncludedNamespaces = NamespaceLabelSelector.
+	// Firstly, ResourceDistributionTargets will parse AllNamespaces, IncludedNamespaces, and NamespaceLabelSelector,
+	// then calculate their union,
+	// At last ExcludedNamespaces will act on the union to remove and exclude the designated namespaces from it.
+
+	// If AllNamespaces is true, Resource will be distributed to the all namespaces
+	// (except some forbidden namespaces, such as "kube-system" and "kube-public").
+	// +optional
+	AllNamespaces bool `json:"allNamespaces,omitempty"`
+
+	// If ExcludedNamespaces is not empty, Resource will never be distributed to the listed namespaces.
+	// ExcludedNamespaces has the highest priority.
+	// +optional
+	ExcludedNamespaces ResourceDistributionTargetNamespaces `json:"excludedNamespaces,omitempty"`
+
+	// If IncludedNamespaces is not empty, Resource will be distributed to the listed namespaces.
+	// +optional
+	IncludedNamespaces ResourceDistributionTargetNamespaces `json:"includedNamespaces,omitempty"`
+
+	// If NamespaceLabelSelector is not empty, Resource will be distributed to the matched namespaces.
+	// +optional
+	NamespaceLabelSelector metav1.LabelSelector `json:"namespaceLabelSelector,omitempty"`
+}
+
+type ResourceDistributionTargetNamespaces struct {
+	/*
+		// TODO: support regular expression in the future
+		// +optional
+		Pattern string `json:"pattern,omitempty"`
+	*/
+
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	// +optional
+	List []ResourceDistributionNamespace `json:"list,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
+}
+
+// ResourceDistributionNamespace contains a namespace name
+type ResourceDistributionNamespace struct {
+	// Namespace name
+	Name string `json:"name,omitempty"`
+}
+
+// ResourceDistributionStatus defines the observed state of ResourceDistribution.
+// ResourceDistributionStatus is recorded by kruise, users' modification is invalid and meaningless.
+type ResourceDistributionStatus struct {
+	// INSERT ADDITIONAL STATUS FIELD - define observed state of cluster
+	// Important: Run "make" to regenerate code after modifying this file
+
+	// Desired represents the number of total target namespaces.
+	Desired int32 `json:"desired,omitempty"`
+
+	// Succeeded represents the number of successful distributions.
+	Succeeded int32 `json:"succeeded,omitempty"`
+
+	// Failed represents the number of failed distributions.
+	Failed int32 `json:"failed,omitempty"`
+
+	// ObservedGeneration represents the .metadata.generation that the condition was set based upon.
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// Conditions describe the condition when Resource creating, updating and deleting.
+	Conditions []ResourceDistributionCondition `json:"conditions,omitempty"`
+}
+
+// ResourceDistributionCondition allows a row to be marked with additional information.
+type ResourceDistributionCondition struct {
+	// Type of ResourceDistributionCondition.
+	Type ResourceDistributionConditionType `json:"type"`
+
+	// Status of the condition, one of True, False, Unknown.
+	Status ResourceDistributionConditionStatus `json:"status"`
+
+	// LastTransitionTime is the last time the condition transitioned from one status to another.
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+
+	// Reason describe human readable message indicating details about last transition.
+	Reason string `json:"reason,omitempty"`
+
+	// FailedNamespaces describe all failed namespaces when Status is False
+	FailedNamespaces []string `json:"failedNamespace,omitempty"`
+}
+
+type ResourceDistributionConditionType string
+
+// These are valid conditions of a ResourceDistribution.
+const (
+	// ResourceDistributionConflictOccurred means there are conflict with existing resources when reconciling.
+	ResourceDistributionConflictOccurred ResourceDistributionConditionType = "ConflictOccurred"
+
+	// ResourceDistributionNamespaceNotExists means some target namespaces not exist.
+	ResourceDistributionNamespaceNotExists ResourceDistributionConditionType = "NamespaceNotExists"
+
+	// ResourceDistributionGetResourceFailed means some get operations about Resource are failed.
+	ResourceDistributionGetResourceFailed ResourceDistributionConditionType = "GetResourceFailed"
+
+	// ResourceDistributionCreateResourceFailed means some create operations about Resource are failed.
+	ResourceDistributionCreateResourceFailed ResourceDistributionConditionType = "CreateResourceFailed"
+
+	// ResourceDistributionUpdateResourceFailed means some update operations about Resource are failed.
+	ResourceDistributionUpdateResourceFailed ResourceDistributionConditionType = "UpdateResourceFailed"
+
+	// ResourceDistributionDeleteResourceFailed means some delete operations about Resource are failed.
+	ResourceDistributionDeleteResourceFailed ResourceDistributionConditionType = "DeleteResourceFailed"
+)
+
+type ResourceDistributionConditionStatus string
+
+// These are valid condition statuses.
+const (
+	// ResourceDistributionConditionTrue means a resource is in the condition.
+	ResourceDistributionConditionTrue ResourceDistributionConditionStatus = "True"
+
+	// ResourceDistributionConditionFalse means a resource is not in the condition.
+	ResourceDistributionConditionFalse ResourceDistributionConditionStatus = "False"
+
+	// ResourceDistributionConditionUnknown means kubernetes can't decide if a resource is in the condition or not.
+	ResourceDistributionConditionUnknown ResourceDistributionConditionStatus = "Unknown"
+)
+
+// +genclient
+// +genclient:nonNamespaced
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:scope=Cluster,shortName=distributor
+// +kubebuilder:printcolumn:name="TOTAL",type="integer",JSONPath=".status.desired",description="The desired number of desired distribution and syncs."
+// +kubebuilder:printcolumn:name="SUCCEED",type="integer",JSONPath=".status.succeeded",description="The number of successful distribution and syncs."
+// +kubebuilder:printcolumn:name="FAILED",type="integer",JSONPath=".status.failed",description="The number of failed distributions and syncs."
+
+// ResourceDistribution is the Schema for the resourcedistributions API.
+type ResourceDistribution struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   ResourceDistributionSpec   `json:"spec,omitempty"`
+	Status ResourceDistributionStatus `json:"status,omitempty"`
+}
+
+//+kubebuilder:object:root=true
+
+// ResourceDistributionList contains a list of ResourceDistribution.
+type ResourceDistributionList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []ResourceDistribution `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&ResourceDistribution{}, &ResourceDistributionList{})
+}

apis/apps/v1alpha1/sandboxconfig_types.go

@@ -0,0 +1,25 @@
+/*
+Copyright 2023 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+// SandboxConfig support attach metadata in PullImage CRI interface during ImagePulljobs
+type SandboxConfig struct {
+	// +optional
+	Labels map[string]string `json:"labels,omitempty"`
+	// +optional
+	Annotations map[string]string `json:"annotations,omitempty"`
+}

apis/apps/v1alpha1/scatter_strategy.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import "fmt"
+
+// UpdateScatterStrategy defines a map for label key-value. Pods matches the key-value will be scattered when update.
+//
+// Example1: [{"Key": "labelA", "Value": "AAA"}]
+// It means all pods with label labelA=AAA will be scattered when update.
+//
+// Example2: [{"Key": "labelA", "Value": "AAA"}, {"Key": "labelB", "Value": "BBB"}]
+// Controller will calculate the two sums of pods with labelA=AAA and with labelB=BBB,
+// pods with the label that has bigger amount will be scattered first, then pods with the other label will be scattered.
+type UpdateScatterStrategy []UpdateScatterTerm
+
+type UpdateScatterTerm struct {
+	Key   string `json:"key"`
+	Value string `json:"value"`
+}
+
+// FieldsValidation checks invalid fields in UpdateScatterStrategy.
+func (strategy UpdateScatterStrategy) FieldsValidation() error {
+	if len(strategy) == 0 {
+		return nil
+	}
+
+	m := make(map[string]struct{}, len(strategy))
+	for _, term := range strategy {
+		if term.Key == "" {
+			return fmt.Errorf("key should not be empty")
+		}
+		id := term.Key + ":" + term.Value
+		if _, ok := m[id]; !ok {
+			m[id] = struct{}{}
+		} else {
+			return fmt.Errorf("duplicated key=%v value=%v", term.Key, term.Value)
+		}
+	}
+
+	return nil
+}

apis/apps/v1alpha1/sidecarset_types.go

@@ -17,50 +17,282 @@ limitations under the License.
 package v1alpha1
 
 import (
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+const (
+	// SidecarSetCustomVersionLabel is designed to record and label the controllerRevision of sidecarSet.
+	// This label will be passed from SidecarSet to its corresponding ControllerRevision, users can use
+	// this label to selector the ControllerRevision they want.
+	// For example, users can update the label from "version-1" to "version-2" when they upgrade the
+	// sidecarSet to "version-2", and they write the "version-2" to InjectionStrategy.Revision.CustomVersion
+	// when they decided to promote the "version-2", to avoid some risks about gray deployment of SidecarSet.
+	SidecarSetCustomVersionLabel = "apps.kruise.io/sidecarset-custom-version"
+)
+
 // SidecarSetSpec defines the desired state of SidecarSet
 type SidecarSetSpec struct {
 	// selector is a label query over pods that should be injected
 	Selector *metav1.LabelSelector `json:"selector,omitempty"`
 
-	// Containers is the list of init containers to be injected into the selected pod
+	// Namespace sidecarSet will only match the pods in the namespace
+	// otherwise, match pods in all namespaces(in cluster)
+	Namespace string `json:"namespace,omitempty"`
+
+	// NamespaceSelector select which namespaces to inject sidecar containers.
+	// Default to the empty LabelSelector, which matches everything.
+	NamespaceSelector *metav1.LabelSelector `json:"namespaceSelector,omitempty"`
+
+	// InitContainers is the list of init containers to be injected into the selected pod
 	// We will inject those containers by their name in ascending order
 	// We only inject init containers when a new pod is created, it does not apply to any existing pod
-	InitContainers []SidecarContainer `json:"initContainers,omitempty"`
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	InitContainers []SidecarContainer `json:"initContainers,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
 
 	// Containers is the list of sidecar containers to be injected into the selected pod
-	Containers []SidecarContainer `json:"containers,omitempty"`
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Containers []SidecarContainer `json:"containers,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
 
 	// List of volumes that can be mounted by sidecar containers
-	Volumes []corev1.Volume `json:"volumes,omitempty"`
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Volumes []corev1.Volume `json:"volumes,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
 
-	// Paused indicates that the sidecarset is paused and will not be processed by the sidecarset controller.
-	Paused bool `json:"paused,omitempty"`
+	// The sidecarset updateStrategy to use to replace existing pods with new ones.
+	UpdateStrategy SidecarSetUpdateStrategy `json:"updateStrategy,omitempty"`
 
-	// The sidecarset strategy to use to replace existing pods with new ones.
-	Strategy SidecarSetUpdateStrategy `json:"strategy,omitempty"`
+	// InjectionStrategy describe the strategy when sidecarset is injected into pods
+	InjectionStrategy SidecarSetInjectionStrategy `json:"injectionStrategy,omitempty"`
+
+	// List of the names of secrets required by pulling sidecar container images
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	ImagePullSecrets []corev1.LocalObjectReference `json:"imagePullSecrets,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
+
+	// RevisionHistoryLimit indicates the maximum quantity of stored revisions about the SidecarSet.
+	// default value is 10
+	RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"`
+
+	// SidecarSet support to inject & in-place update metadata in pod.
+	PatchPodMetadata []SidecarSetPatchPodMetadata `json:"patchPodMetadata,omitempty"`
 }
 
+type SidecarSetPatchPodMetadata struct {
+	// annotations
+	Annotations map[string]string `json:"annotations,omitempty"`
+
+	// labels map[string]string `json:"labels,omitempty"`
+	// patch pod metadata policy, Default is "Retain"
+	PatchPolicy SidecarSetPatchPolicyType `json:"patchPolicy,omitempty"`
+}
+
+type SidecarSetPatchPolicyType string
+
+var (
+	// SidecarSetRetainPatchPolicy indicates if PatchPodFields conflicts with Pod,
+	// will ignore PatchPodFields, and retain the corresponding fields of pods.
+	// SidecarSet webhook cannot allow the conflict of PatchPodFields between SidecarSets under this policy type.
+	// Note: Retain is only supported for injection, and the Metadata will not be updated when upgrading the Sidecar Container in-place.
+	SidecarSetRetainPatchPolicy SidecarSetPatchPolicyType = "Retain"
+
+	// SidecarSetOverwritePatchPolicy indicates if PatchPodFields conflicts with Pod,
+	// SidecarSet will apply PatchPodFields to overwrite the corresponding fields of pods.
+	// SidecarSet webhook cannot allow the conflict of PatchPodFields between SidecarSets under this policy type.
+	// Overwrite support to inject and in-place metadata.
+	SidecarSetOverwritePatchPolicy SidecarSetPatchPolicyType = "Overwrite"
+
+	// SidecarSetMergePatchJsonPatchPolicy indicate that sidecarSet use application/merge-patch+json to patch annotation value,
+	// for example, A patch annotation[oom-score] = '{"log-agent": 1}' and B patch annotation[oom-score] = '{"envoy": 2}'
+	// result pod annotation[oom-score] = '{"log-agent": 1, "envoy": 2}'
+	// MergePatchJson support to inject and in-place metadata.
+	SidecarSetMergePatchJsonPatchPolicy SidecarSetPatchPolicyType = "MergePatchJson"
+)
+
 // SidecarContainer defines the container of Sidecar
 type SidecarContainer struct {
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
 	corev1.Container `json:",inline"`
+
+	// The rules that injected SidecarContainer into Pod.spec.containers,
+	// not takes effect in initContainers
+	// If BeforeAppContainer, the SidecarContainer will be injected in front of the pod.spec.containers
+	// otherwise it will be injected into the back.
+	// default BeforeAppContainerType
+	PodInjectPolicy PodInjectPolicyType `json:"podInjectPolicy,omitempty"`
+
+	// sidecarContainer upgrade strategy, include: ColdUpgrade, HotUpgrade
+	UpgradeStrategy SidecarContainerUpgradeStrategy `json:"upgradeStrategy,omitempty"`
+
+	// If ShareVolumePolicy is enabled, the sidecar container will share the other container's VolumeMounts
+	// in the pod(not including the injected sidecar container).
+	ShareVolumePolicy ShareVolumePolicy `json:"shareVolumePolicy,omitempty"`
+
+	// If ShareVolumeDevicePolicy is enabled, the sidecar container will share the other container's VolumeDevices
+	// in the pod(don't contain the injected sidecar container).
+	// This is a pointer to ensure that the sidecarset-hash does not change if the user does not configure this field, mainly for compatibility with older versions.
+	ShareVolumeDevicePolicy *ShareVolumePolicy `json:"shareVolumeDevicePolicy,omitempty"`
+
+	// TransferEnv will transfer env info from other container
+	// SourceContainerName is pod.spec.container[x].name; EnvName is pod.spec.container[x].Env.name
+	TransferEnv []TransferEnvVar `json:"transferEnv,omitempty"`
 }
 
+type ShareVolumePolicy struct {
+	Type ShareVolumePolicyType `json:"type,omitempty"`
+}
+
+type PodInjectPolicyType string
+
+const (
+	BeforeAppContainerType PodInjectPolicyType = "BeforeAppContainer"
+	AfterAppContainerType  PodInjectPolicyType = "AfterAppContainer"
+)
+
+type ShareVolumePolicyType string
+
+const (
+	ShareVolumePolicyEnabled  ShareVolumePolicyType = "enabled"
+	ShareVolumePolicyDisabled ShareVolumePolicyType = "disabled"
+)
+
+type TransferEnvVar struct {
+	SourceContainerName string `json:"sourceContainerName,omitempty"`
+	// +optional
+	SourceContainerNameFrom *SourceContainerNameSource `json:"sourceContainerNameFrom,omitempty"`
+	EnvName                 string                     `json:"envName,omitempty"`
+	// +optional
+	EnvNames []string `json:"envNames,omitempty"`
+}
+
+type SourceContainerNameSource struct {
+	// Selects a field of the pod: supports metadata.name, `metadata.labels['<KEY>']`, `metadata.annotations['<KEY>']`,
+	FieldRef *corev1.ObjectFieldSelector `json:"fieldRef,omitempty"`
+}
+
+type SidecarContainerUpgradeType string
+
+const (
+	SidecarContainerColdUpgrade SidecarContainerUpgradeType = "ColdUpgrade"
+	SidecarContainerHotUpgrade  SidecarContainerUpgradeType = "HotUpgrade"
+)
+
+type SidecarContainerUpgradeStrategy struct {
+	// when sidecar container is stateless, use ColdUpgrade
+	// otherwise HotUpgrade are more HotUpgrade.
+	// examples for istio envoy container is suitable for HotUpgrade
+	// default is ColdUpgrade
+	UpgradeType SidecarContainerUpgradeType `json:"upgradeType,omitempty"`
+
+	// when HotUpgrade, HotUpgradeEmptyImage is used to complete the hot upgrading process
+	// HotUpgradeEmptyImage is consistent of sidecar container in Command, Args, Liveness probe, etc.
+	// but it does no actual work.
+	HotUpgradeEmptyImage string `json:"hotUpgradeEmptyImage,omitempty"`
+}
+
+// SidecarSetInjectionStrategy indicates the injection strategy of SidecarSet.
+type SidecarSetInjectionStrategy struct {
+	// Paused indicates that SidecarSet will suspend injection into Pods
+	// If Paused is true, the sidecarSet will not be injected to newly created Pods,
+	// but the injected sidecar container remains updating and running.
+	// default is false
+	Paused bool `json:"paused,omitempty"`
+
+	// Revision can help users rolling update SidecarSet safely. If users set
+	// this filed, SidecarSet will try to inject specific revision according to
+	// different policies.
+	Revision *SidecarSetInjectRevision `json:"revision,omitempty"`
+}
+
+type SidecarSetInjectRevision struct {
+	// CustomVersion corresponds to label 'apps.kruise.io/sidecarset-custom-version' of (History) SidecarSet.
+	// SidecarSet will select the specific ControllerRevision via this CustomVersion, and then restore the
+	// history SidecarSet to inject specific version of the sidecar to pods.
+	// + optional
+	CustomVersion *string `json:"customVersion,omitempty"`
+	// RevisionName corresponds to a specific ControllerRevision name of SidecarSet that you want to inject to Pods.
+	// + optional
+	RevisionName *string `json:"revisionName,omitempty"`
+	// Policy describes the behavior of revision injection.
+	// +kubebuilder:validation:Enum=Always;Partial;
+	// +kubebuilder:default=Always
+	Policy SidecarSetInjectRevisionPolicy `json:"policy,omitempty"`
+}
+
+type SidecarSetInjectRevisionPolicy string
+
+const (
+	// AlwaysSidecarSetInjectRevisionPolicy means the SidecarSet will always inject
+	// the specific revision to Pods when pod creating, except matching UpdateStrategy.Selector.
+	AlwaysSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Always"
+
+	// PartialSidecarSetInjectRevisionPolicy means the SidecarSet will inject the specific or the latest revision according to UpdateStrategy.
+	//
+	// If UpdateStrategy.Pause is not true, only when a newly created Pod is **not** selected by the Selector explicitly
+	// configured in `UpdateStrategy` will it be injected with the specified version of the Sidecar.
+	// Under all other conditions, newly created Pods have a probability of being injected with the latest Sidecar,
+	// where the probability is `1 - UpdateStrategy.Partition`.
+	// If `Partition` is not a percentage or is not configured, its value is considered to be 0%.
+	PartialSidecarSetInjectRevisionPolicy SidecarSetInjectRevisionPolicy = "Partial"
+)
+
 // SidecarSetUpdateStrategy indicates the strategy that the SidecarSet
 // controller will use to perform updates. It includes any additional parameters
 // necessary to perform the update for the indicated strategy.
 type SidecarSetUpdateStrategy struct {
-	RollingUpdate *RollingUpdateSidecarSet `json:"rollingUpdate,omitempty"`
+	// Type is NotUpdate, the SidecarSet don't update the injected pods,
+	// it will only inject sidecar container into the newly created pods.
+	// Type is RollingUpdate, the SidecarSet will update the injected pods to the latest version on RollingUpdate Strategy.
+	// default is RollingUpdate
+	Type SidecarSetUpdateStrategyType `json:"type,omitempty"`
+
+	// Paused indicates that the SidecarSet is paused to update the injected pods,
+	// For the impact on the injection behavior for newly created Pods, please refer to the comments of Selector.
+	Paused bool `json:"paused,omitempty"`
+
+	// If selector is not nil, this upgrade will only update the selected pods.
+	//
+	// Starting from Kruise 1.8.0, the updateStrategy.Selector affects the version of the Sidecar container
+	// injected into newly created Pods by a SidecarSet configured with an injectionStrategy.
+	// In most cases, all newly created Pods are injected with the specified Sidecar version as configured in injectionStrategy.revision,
+	// which is consistent with previous versions.
+	Selector *metav1.LabelSelector `json:"selector,omitempty"`
+
+	// Partition is the desired number of pods in old revisions. It means when partition
+	// is set during pods updating, (replicas - partition) number of pods will be updated.
+	// Default value is 0.
+	Partition *intstr.IntOrString `json:"partition,omitempty"`
+
+	// The maximum number of SidecarSet pods that can be unavailable during the
+	// update. Value can be an absolute number (ex: 5) or a percentage of total
+	// number of SidecarSet pods at the start of the update (ex: 10%). Absolute
+	// number is calculated from percentage by rounding up.
+	// This cannot be 0.
+	// Default value is 1.
+	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+	// Priorities are the rules for calculating the priority of updating pods.
+	// Each pod to be updated, will pass through these terms and get a sum of weights.
+	PriorityStrategy *appspub.UpdatePriorityStrategy `json:"priorityStrategy,omitempty"`
+	// ScatterStrategy defines the scatter rules to make pods been scattered when update.
+	// This will avoid pods with the same key-value to be updated in one batch.
+	// - Note that pods will be scattered after priority sort. So, although priority strategy and scatter strategy can be applied together, we suggest to use either one of them.
+	// - If scatterStrategy is used, we suggest to just use one term. Otherwise, the update order can be hard to understand.
+	ScatterStrategy UpdateScatterStrategy `json:"scatterStrategy,omitempty"`
 }
 
-// RollingUpdateSidecarSet is used to communicate parameter
-type RollingUpdateSidecarSet struct {
-	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
-}
+type SidecarSetUpdateStrategyType string
+
+const (
+	NotUpdateSidecarSetStrategyType     SidecarSetUpdateStrategyType = "NotUpdate"
+	RollingUpdateSidecarSetStrategyType SidecarSetUpdateStrategyType = "RollingUpdate"
+)
 
 // SidecarSetStatus defines the observed state of SidecarSet
 type SidecarSetStatus struct {
@@ -76,10 +308,22 @@ type SidecarSetStatus struct {
 
 	// readyPods is the number of matched Pods that have a ready condition
 	ReadyPods int32 `json:"readyPods"`
+
+	// updatedReadyPods is the number of matched pods that updated and ready
+	UpdatedReadyPods int32 `json:"updatedReadyPods,omitempty"`
+
+	// LatestRevision, if not empty, indicates the latest controllerRevision name of the SidecarSet.
+	LatestRevision string `json:"latestRevision,omitempty"`
+
+	// CollisionCount is the count of hash collisions for the SidecarSet. The SidecarSet controller
+	// uses this field as a collision avoidance mechanism when it needs to create the name for the
+	// newest ControllerRevision.
+	CollisionCount *int32 `json:"collisionCount,omitempty"`
 }
 
 // +genclient
 // +genclient:nonNamespaced
+// +k8s:openapi-gen=true
 // +kubebuilder:object:root=true
 // +kubebuilder:resource:scope=Cluster
 // +kubebuilder:subresource:status

apis/apps/v1alpha1/statefulset_conversion.go

@@ -0,0 +1,137 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"fmt"
+
+	"github.com/openkruise/kruise/apis/apps/v1beta1"
+	"sigs.k8s.io/controller-runtime/pkg/conversion"
+)
+
+func (sts *StatefulSet) ConvertTo(dst conversion.Hub) error {
+	switch t := dst.(type) {
+	case *v1beta1.StatefulSet:
+		stsv1beta1 := dst.(*v1beta1.StatefulSet)
+		stsv1beta1.ObjectMeta = sts.ObjectMeta
+
+		// spec
+		stsv1beta1.Spec = v1beta1.StatefulSetSpec{
+			Replicas:             sts.Spec.Replicas,
+			Selector:             sts.Spec.Selector,
+			Template:             sts.Spec.Template,
+			VolumeClaimTemplates: sts.Spec.VolumeClaimTemplates,
+			ServiceName:          sts.Spec.ServiceName,
+			PodManagementPolicy:  sts.Spec.PodManagementPolicy,
+			UpdateStrategy: v1beta1.StatefulSetUpdateStrategy{
+				Type: sts.Spec.UpdateStrategy.Type,
+			},
+			RevisionHistoryLimit: sts.Spec.RevisionHistoryLimit,
+		}
+		if sts.Spec.UpdateStrategy.RollingUpdate != nil {
+			stsv1beta1.Spec.UpdateStrategy.RollingUpdate = &v1beta1.RollingUpdateStatefulSetStrategy{
+				Partition:             sts.Spec.UpdateStrategy.RollingUpdate.Partition,
+				MaxUnavailable:        sts.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable,
+				PodUpdatePolicy:       v1beta1.PodUpdateStrategyType(sts.Spec.UpdateStrategy.RollingUpdate.PodUpdatePolicy),
+				Paused:                sts.Spec.UpdateStrategy.RollingUpdate.Paused,
+				InPlaceUpdateStrategy: sts.Spec.UpdateStrategy.RollingUpdate.InPlaceUpdateStrategy,
+				MinReadySeconds:       sts.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds,
+			}
+			if sts.Spec.UpdateStrategy.RollingUpdate.UnorderedUpdate != nil {
+				stsv1beta1.Spec.UpdateStrategy.RollingUpdate.UnorderedUpdate = &v1beta1.UnorderedUpdateStrategy{
+					PriorityStrategy: sts.Spec.UpdateStrategy.RollingUpdate.UnorderedUpdate.PriorityStrategy,
+				}
+			}
+		}
+
+		// status
+		stsv1beta1.Status = v1beta1.StatefulSetStatus{
+			ObservedGeneration: sts.Status.ObservedGeneration,
+			Replicas:           sts.Status.Replicas,
+			ReadyReplicas:      sts.Status.ReadyReplicas,
+			AvailableReplicas:  sts.Status.AvailableReplicas,
+			CurrentReplicas:    sts.Status.CurrentReplicas,
+			UpdatedReplicas:    sts.Status.UpdatedReplicas,
+			CurrentRevision:    sts.Status.CurrentRevision,
+			UpdateRevision:     sts.Status.UpdateRevision,
+			CollisionCount:     sts.Status.CollisionCount,
+			Conditions:         sts.Status.Conditions,
+			LabelSelector:      sts.Status.LabelSelector,
+		}
+
+		return nil
+
+	default:
+		return fmt.Errorf("unsupported type %v", t)
+	}
+}
+
+func (sts *StatefulSet) ConvertFrom(src conversion.Hub) error {
+	switch t := src.(type) {
+	case *v1beta1.StatefulSet:
+		stsv1beta1 := src.(*v1beta1.StatefulSet)
+		sts.ObjectMeta = stsv1beta1.ObjectMeta
+
+		// spec
+		sts.Spec = StatefulSetSpec{
+			Replicas:             stsv1beta1.Spec.Replicas,
+			Selector:             stsv1beta1.Spec.Selector,
+			Template:             stsv1beta1.Spec.Template,
+			VolumeClaimTemplates: stsv1beta1.Spec.VolumeClaimTemplates,
+			ServiceName:          stsv1beta1.Spec.ServiceName,
+			PodManagementPolicy:  stsv1beta1.Spec.PodManagementPolicy,
+			UpdateStrategy: StatefulSetUpdateStrategy{
+				Type: stsv1beta1.Spec.UpdateStrategy.Type,
+			},
+			RevisionHistoryLimit: stsv1beta1.Spec.RevisionHistoryLimit,
+		}
+		if stsv1beta1.Spec.UpdateStrategy.RollingUpdate != nil {
+			sts.Spec.UpdateStrategy.RollingUpdate = &RollingUpdateStatefulSetStrategy{
+				Partition:             stsv1beta1.Spec.UpdateStrategy.RollingUpdate.Partition,
+				MaxUnavailable:        stsv1beta1.Spec.UpdateStrategy.RollingUpdate.MaxUnavailable,
+				PodUpdatePolicy:       PodUpdateStrategyType(stsv1beta1.Spec.UpdateStrategy.RollingUpdate.PodUpdatePolicy),
+				Paused:                stsv1beta1.Spec.UpdateStrategy.RollingUpdate.Paused,
+				InPlaceUpdateStrategy: stsv1beta1.Spec.UpdateStrategy.RollingUpdate.InPlaceUpdateStrategy,
+				MinReadySeconds:       stsv1beta1.Spec.UpdateStrategy.RollingUpdate.MinReadySeconds,
+			}
+			if stsv1beta1.Spec.UpdateStrategy.RollingUpdate.UnorderedUpdate != nil {
+				sts.Spec.UpdateStrategy.RollingUpdate.UnorderedUpdate = &UnorderedUpdateStrategy{
+					PriorityStrategy: stsv1beta1.Spec.UpdateStrategy.RollingUpdate.UnorderedUpdate.PriorityStrategy,
+				}
+			}
+		}
+
+		// status
+		sts.Status = StatefulSetStatus{
+			ObservedGeneration: stsv1beta1.Status.ObservedGeneration,
+			Replicas:           stsv1beta1.Status.Replicas,
+			ReadyReplicas:      stsv1beta1.Status.ReadyReplicas,
+			AvailableReplicas:  stsv1beta1.Status.AvailableReplicas,
+			CurrentReplicas:    stsv1beta1.Status.CurrentReplicas,
+			UpdatedReplicas:    stsv1beta1.Status.UpdatedReplicas,
+			CurrentRevision:    stsv1beta1.Status.CurrentRevision,
+			UpdateRevision:     stsv1beta1.Status.UpdateRevision,
+			CollisionCount:     stsv1beta1.Status.CollisionCount,
+			Conditions:         stsv1beta1.Status.Conditions,
+			LabelSelector:      stsv1beta1.Status.LabelSelector,
+		}
+
+		return nil
+	default:
+		return fmt.Errorf("unsupported type %v", t)
+	}
+}

apis/apps/v1alpha1/statefulset_types.go

@@ -17,6 +17,7 @@ limitations under the License.
 package v1alpha1
 
 import (
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
 	apps "k8s.io/api/apps/v1"
 	v1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -72,7 +73,7 @@ type RollingUpdateStatefulSetStrategy struct {
 	UnorderedUpdate *UnorderedUpdateStrategy `json:"unorderedUpdate,omitempty"`
 	// InPlaceUpdateStrategy contains strategies for in-place update.
 	// +optional
-	InPlaceUpdateStrategy *InPlaceUpdateStrategy `json:"inPlaceUpdateStrategy,omitempty"`
+	InPlaceUpdateStrategy *appspub.InPlaceUpdateStrategy `json:"inPlaceUpdateStrategy,omitempty"`
 	// MinReadySeconds indicates how long will the pod be considered ready after it's updated.
 	// MinReadySeconds works with both OrderedReady and Parallel podManagementPolicy.
 	// It affects the pod scale up speed when the podManagementPolicy is set to be OrderedReady.
@@ -87,7 +88,7 @@ type UnorderedUpdateStrategy struct {
 	// Priorities are the rules for calculating the priority of updating pods.
 	// Each pod to be updated, will pass through these terms and get a sum of weights.
 	// +optional
-	PriorityStrategy *UpdatePriorityStrategy `json:"priorityStrategy,omitempty"`
+	PriorityStrategy *appspub.UpdatePriorityStrategy `json:"priorityStrategy,omitempty"`
 }
 
 // PodUpdateStrategyType is a string enumeration type that enumerates
@@ -127,6 +128,8 @@ type StatefulSetSpec struct {
 	// insufficient replicas are detected. Each pod stamped out by the StatefulSet
 	// will fulfill this Template, but have a unique identity from the rest
 	// of the StatefulSet.
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
 	Template v1.PodTemplateSpec `json:"template"`
 
 	// volumeClaimTemplates is a list of claims that pods are allowed to reference.
@@ -137,6 +140,8 @@ type StatefulSetSpec struct {
 	// any volumes in the template, with the same name.
 	// TODO: Define the behavior if a claim already exists with the same name.
 	// +optional
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
 	VolumeClaimTemplates []v1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
 
 	// serviceName is the name of the service that governs this StatefulSet.
@@ -225,6 +230,9 @@ const (
 )
 
 // +genclient
+// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale
+// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale
+// +k8s:openapi-gen=true
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
 // +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector
@@ -235,6 +243,8 @@ const (
 // +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedReplicas",description="The number of pods updated."
 // +kubebuilder:printcolumn:name="READY",type="integer",JSONPath=".status.readyReplicas",description="The number of pods ready."
 // +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."
+// +kubebuilder:printcolumn:name="CONTAINERS",type="string",priority=1,JSONPath=".spec.template.spec.containers[*].name",description="The containers of currently advanced statefulset."
+// +kubebuilder:printcolumn:name="IMAGES",type="string",priority=1,JSONPath=".spec.template.spec.containers[*].image",description="The images of currently advanced statefulset."
 
 // StatefulSet is the Schema for the statefulsets API
 type StatefulSet struct {

apis/apps/v1alpha1/uniteddeployment_types.go

@@ -17,9 +17,13 @@ limitations under the License.
 package v1alpha1
 
 import (
+	"time"
+
+	"github.com/openkruise/kruise/apis/apps/v1beta1"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
@@ -44,6 +48,8 @@ const (
 	SubsetUpdated UnitedDeploymentConditionType = "SubsetUpdated"
 	// SubsetFailure is added to a UnitedDeployment when one of its subsets has failure during its own reconciling.
 	SubsetFailure UnitedDeploymentConditionType = "SubsetFailure"
+	// UnitedDeploymentUpdated means currentRevision is equal to updatedRevision.
+	UnitedDeploymentUpdated UnitedDeploymentConditionType = "UnitedDeploymentUpdated"
 )
 
 // UnitedDeploymentSpec defines the desired state of UnitedDeployment.
@@ -90,25 +96,48 @@ type SubsetTemplate struct {
 	// CloneSet template
 	// +optional
 	CloneSetTemplate *CloneSetTemplateSpec `json:"cloneSetTemplate,omitempty"`
+
+	// Deployment template
+	// +optional
+	DeploymentTemplate *DeploymentTemplateSpec `json:"deploymentTemplate,omitempty"`
 }
 
 // StatefulSetTemplateSpec defines the subset template of StatefulSet.
 type StatefulSetTemplateSpec struct {
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
 	metav1.ObjectMeta `json:"metadata,omitempty"`
-	Spec              appsv1.StatefulSetSpec `json:"spec"`
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	Spec appsv1.StatefulSetSpec `json:"spec"`
 }
 
 // AdvancedStatefulSetTemplateSpec defines the subset template of AdvancedStatefulSet.
 type AdvancedStatefulSetTemplateSpec struct {
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
 	metav1.ObjectMeta `json:"metadata,omitempty"`
-	Spec              StatefulSetSpec `json:"spec"`
+	Spec              v1beta1.StatefulSetSpec `json:"spec"`
 }
 
+// CloneSetTemplateSpec defines the subset template of CloneSet.
 type CloneSetTemplateSpec struct {
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
 	metav1.ObjectMeta `json:"metadata,omitempty"`
 	Spec              CloneSetSpec `json:"spec"`
 }
 
+// DeploymentTemplateSpec defines the subset template of Deployment.
+type DeploymentTemplateSpec struct {
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	Spec appsv1.DeploymentSpec `json:"spec"`
+}
+
 // UnitedDeploymentUpdateStrategy defines the update performance
 // when template of UnitedDeployment is changed.
 type UnitedDeploymentUpdateStrategy struct {
@@ -135,8 +164,14 @@ type ManualUpdate struct {
 type Topology struct {
 	// Contains the details of each subset. Each element in this array represents one subset
 	// which will be provisioned and managed by UnitedDeployment.
+	// +patchMergeKey=name
+	// +patchStrategy=merge
 	// +optional
-	Subsets []Subset `json:"subsets,omitempty"`
+	Subsets []Subset `json:"subsets,omitempty" patchStrategy:"merge" patchMergeKey:"name"`
+
+	// ScheduleStrategy indicates the strategy the UnitedDeployment used to preform the schedule between each of subsets.
+	// +optional
+	ScheduleStrategy UnitedDeploymentScheduleStrategy `json:"scheduleStrategy,omitempty"`
 }
 
 // Subset defines the detail of a subset.
@@ -161,8 +196,107 @@ type Subset struct {
 	// percentage like '10%', which means 10% of UnitedDeployment replicas of pods will be distributed
 	// under this subset. If nil, the number of replicas in this subset is determined by controller.
 	// Controller will try to keep all the subsets with nil replicas have average pods.
+	// Replicas and MinReplicas/MaxReplicas are mutually exclusive in a UnitedDeployment.
 	// +optional
 	Replicas *intstr.IntOrString `json:"replicas,omitempty"`
+
+	// Indicates the lower bounded replicas of the subset.
+	// MinReplicas must be more than or equal to 0 if it is set.
+	// Controller will prioritize satisfy minReplicas for each subset
+	// according to the order of Topology.Subsets.
+	// Defaults to 0.
+	// +optional
+	MinReplicas *intstr.IntOrString `json:"minReplicas,omitempty"`
+
+	// Indicates the upper bounded replicas of the subset.
+	// MaxReplicas must be more than or equal to MinReplicas.
+	// MaxReplicas == nil means no limitation.
+	// Please ensure that at least one subset has empty MaxReplicas(no limitation) to avoid stuck scaling.
+	// Defaults to nil.
+	// +optional
+	MaxReplicas *intstr.IntOrString `json:"maxReplicas,omitempty"`
+
+	// Patch indicates patching to the templateSpec.
+	// Patch takes precedence over other fields
+	// If the Patch also modifies the Replicas, NodeSelectorTerm or Tolerations, use value in the Patch
+	// +optional
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	Patch runtime.RawExtension `json:"patch,omitempty"`
+}
+
+// UnitedDeploymentScheduleStrategyType is a string enumeration type that enumerates
+// all possible schedule strategies for the UnitedDeployment controller.
+// +kubebuilder:validation:Enum=Adaptive;Fixed;""
+type UnitedDeploymentScheduleStrategyType string
+
+const (
+	// AdaptiveUnitedDeploymentScheduleStrategyType represents that when a pod is stuck in the pending status and cannot
+	// be scheduled, allow it to be rescheduled to another subset.
+	AdaptiveUnitedDeploymentScheduleStrategyType UnitedDeploymentScheduleStrategyType = "Adaptive"
+	// FixedUnitedDeploymentScheduleStrategyType represents that pods are strictly scheduled to the selected subset
+	// even if scheduling fail.
+	FixedUnitedDeploymentScheduleStrategyType UnitedDeploymentScheduleStrategyType = "Fixed"
+)
+
+const (
+	DefaultRescheduleCriticalDuration      = 30 * time.Second
+	DefaultUnschedulableStatusLastDuration = 300 * time.Second
+)
+
+// AdaptiveUnitedDeploymentStrategy is used to communicate parameters when Type is AdaptiveUnitedDeploymentScheduleStrategyType.
+type AdaptiveUnitedDeploymentStrategy struct {
+	// RescheduleCriticalSeconds indicates how long controller will reschedule a schedule failed Pod to the subset that has
+	// redundant capacity after the subset where the Pod lives. If a Pod was scheduled failed and still in an unschedulabe status
+	// over RescheduleCriticalSeconds duration, the controller will reschedule it to a suitable subset. Default is 30 seconds.
+	// +optional
+	RescheduleCriticalSeconds *int32 `json:"rescheduleCriticalSeconds,omitempty"`
+
+	// UnschedulableDuration is used to set the number of seconds for a Subset to recover from an unschedulable state,
+	// with a default value of 300 seconds.
+	// +optional
+	UnschedulableDuration *int32 `json:"unschedulableDuration,omitempty"`
+
+	// ReserveUnschedulablePods indicates whether to enable reservation rescheduling mode, which is disabled by default.
+	// If this feature is enabled, those pending pods that would otherwise be permanently transferred to other subsets
+	// due to scheduling failure will be retained, and a temporary substitute Pod will be created in another subset to take over its work.
+	// When the retained pod is successfully scheduled and ready, its temporary substitute will be deleted.
+	// +optional
+	ReserveUnschedulablePods bool `json:"reserveUnschedulablePods,omitempty"`
+}
+
+// UnitedDeploymentScheduleStrategy defines the schedule performance of UnitedDeployment.
+type UnitedDeploymentScheduleStrategy struct {
+	// Type indicates the type of the UnitedDeploymentScheduleStrategy.
+	// Default is Fixed
+	// +optional
+	Type UnitedDeploymentScheduleStrategyType `json:"type,omitempty"`
+
+	// Adaptive is used to communicate parameters when Type is AdaptiveUnitedDeploymentScheduleStrategyType.
+	// +optional
+	Adaptive *AdaptiveUnitedDeploymentStrategy `json:"adaptive,omitempty"`
+}
+
+func (s *UnitedDeploymentScheduleStrategy) IsAdaptive() bool {
+	return s.Type == AdaptiveUnitedDeploymentScheduleStrategyType
+}
+
+func (s *UnitedDeploymentScheduleStrategy) ShouldReserveUnschedulablePods() bool {
+	return s.IsAdaptive() && s.Adaptive != nil && s.Adaptive.ReserveUnschedulablePods
+}
+
+func (s *UnitedDeploymentScheduleStrategy) GetRescheduleCriticalDuration() time.Duration {
+	if s.Adaptive == nil || s.Adaptive.RescheduleCriticalSeconds == nil {
+		return DefaultRescheduleCriticalDuration
+	}
+	return time.Duration(*s.Adaptive.RescheduleCriticalSeconds) * time.Second
+}
+
+func (s *UnitedDeploymentScheduleStrategy) GetUnschedulableDuration() time.Duration {
+	if s.Adaptive == nil || s.Adaptive.UnschedulableDuration == nil {
+		return DefaultUnschedulableStatusLastDuration
+	}
+	return time.Duration(*s.Adaptive.UnschedulableDuration) * time.Second
 }
 
 // UnitedDeploymentStatus defines the observed state of UnitedDeployment.
@@ -182,6 +316,9 @@ type UnitedDeploymentStatus struct {
 	// The number of pods in current version.
 	UpdatedReplicas int32 `json:"updatedReplicas"`
 
+	// The number of reserved pods in temporary adaptive strategy.
+	ReservedPods int32 `json:"reservedPods,omitempty"`
+
 	// The number of ready current revision replicas for this UnitedDeployment.
 	// +optional
 	UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"`
@@ -199,6 +336,8 @@ type UnitedDeploymentStatus struct {
 	// +optional
 	SubsetReplicas map[string]int32 `json:"subsetReplicas,omitempty"`
 
+	// Record the conditions of each subset.
+	SubsetStatuses []UnitedDeploymentSubsetStatus `json:"subsetStatuses,omitempty"`
 	// Represents the latest available observations of a UnitedDeployment's current state.
 	// +optional
 	Conditions []UnitedDeploymentCondition `json:"conditions,omitempty"`
@@ -206,6 +345,18 @@ type UnitedDeploymentStatus struct {
 	// Records the information of update progress.
 	// +optional
 	UpdateStatus *UpdateStatus `json:"updateStatus,omitempty"`
+
+	// LabelSelector is label selectors for query over pods that should match the replica count used by HPA.
+	LabelSelector string `json:"labelSelector,omitempty"`
+}
+
+func (s *UnitedDeploymentStatus) GetSubsetStatus(subset string) *UnitedDeploymentSubsetStatus {
+	for i, subsetStatus := range s.SubsetStatuses {
+		if subsetStatus.Name == subset {
+			return &s.SubsetStatuses[i]
+		}
+	}
+	return nil
 }
 
 // UnitedDeploymentCondition describes current state of a UnitedDeployment.
@@ -222,7 +373,7 @@ type UnitedDeploymentCondition struct {
 	// The reason for the condition's last transition.
 	Reason string `json:"reason,omitempty"`
 
-	// A human readable message indicating details about the transition.
+	// A human-readable message indicating details about the transition.
 	Message string `json:"message,omitempty"`
 }
 
@@ -237,10 +388,73 @@ type UpdateStatus struct {
 	CurrentPartitions map[string]int32 `json:"currentPartitions,omitempty"`
 }
 
+type UnitedDeploymentSubsetStatus struct {
+	// Subset name specified in Topology.Subsets
+	Name string `json:"name,omitempty"`
+	// Records the current replicas. Currently unused.
+	Replicas int32 `json:"replicas,omitempty"`
+	// Records the current ready replicas. Currently unused.
+	ReadyReplicas int32 `json:"readyReplicas,omitempty"`
+	// Records the current partition. Currently unused.
+	Partition int32 `json:"partition,omitempty"`
+	// Records the reserved pods in the subset.
+	ReservedPods int32 `json:"reservedPods,omitempty"`
+	// Conditions is an array of current observed subset conditions.
+	Conditions []UnitedDeploymentSubsetCondition `json:"conditions,omitempty"`
+}
+
+func (s *UnitedDeploymentSubsetStatus) GetCondition(condType UnitedDeploymentSubsetConditionType) *UnitedDeploymentSubsetCondition {
+	for _, condition := range s.Conditions {
+		if condition.Type == condType {
+			return &condition
+		}
+	}
+	return nil
+}
+
+func (s *UnitedDeploymentSubsetStatus) SetCondition(condType UnitedDeploymentSubsetConditionType, status corev1.ConditionStatus, reason, message string) {
+	var currentCond *UnitedDeploymentSubsetCondition
+	for i, c := range s.Conditions {
+		if c.Type == condType {
+			currentCond = &s.Conditions[i]
+			break
+		}
+	}
+	if currentCond != nil && currentCond.Status == status && currentCond.Reason == reason {
+		return
+	}
+	if currentCond == nil {
+		s.Conditions = append(s.Conditions, UnitedDeploymentSubsetCondition{Type: condType})
+		currentCond = &s.Conditions[len(s.Conditions)-1]
+	}
+	currentCond.LastTransitionTime = metav1.Now()
+	currentCond.Status = status
+	currentCond.Reason = reason
+	currentCond.Message = message
+}
+
+type UnitedDeploymentSubsetConditionType string
+
+const (
+	// UnitedDeploymentSubsetSchedulable means new pods allocated into the subset will keep pending.
+	UnitedDeploymentSubsetSchedulable UnitedDeploymentSubsetConditionType = "Schedulable"
+)
+
+type UnitedDeploymentSubsetCondition struct {
+	Type               UnitedDeploymentSubsetConditionType `json:"type"`
+	Status             corev1.ConditionStatus              `json:"status"`
+	LastTransitionTime metav1.Time                         `json:"lastTransitionTime,omitempty"`
+	Reason             string                              `json:"reason,omitempty"`
+	Message            string                              `json:"message,omitempty"`
+}
+
 // +genclient
+// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale
+// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale
+// +k8s:openapi-gen=true
 // +kubebuilder:object:root=true
 // +kubebuilder:subresource:status
-// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.selector
+// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector
 // +kubebuilder:resource:shortName=ud
 // +kubebuilder:printcolumn:name="DESIRED",type="integer",JSONPath=".spec.replicas",description="The desired number of pods."
 // +kubebuilder:printcolumn:name="CURRENT",type="integer",JSONPath=".status.replicas",description="The number of currently all pods."

apis/apps/v1alpha1/update_priority.go

@@ -1,55 +0,0 @@
-/*
-Copyright 2020 The Kruise Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1alpha1
-
-import (
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// UpdatePriorityStrategy is the strategy to define priority for pods update.
-// Only one of orderPriority and weightPriority can be set.
-type UpdatePriorityStrategy struct {
-	// Order priority terms, pods will be sorted by the value of orderedKey.
-	// For example:
-	// ```
-	// orderPriority:
-	// - orderedKey: key1
-	// - orderedKey: key2
-	// ```
-	// First, all pods which have key1 in labels will be sorted by the value of key1.
-	// Then, the left pods which have no key1 but have key2 in labels will be sorted by
-	// the value of key2 and put behind those pods have key1.
-	OrderPriority []UpdatePriorityOrderTerm `json:"orderPriority,omitempty"`
-	// Weight priority terms, pods will be sorted by the sum of all terms weight.
-	WeightPriority []UpdatePriorityWeightTerm `json:"weightPriority,omitempty"`
-}
-
-// UpdatePriorityOrder defines order priority.
-type UpdatePriorityOrderTerm struct {
-	// Calculate priority by value of this key.
-	// Values of this key, will be sorted by GetInt(val). GetInt method will find the last int in value,
-	// such as getting 5 in value '5', getting 10 in value 'sts-10'.
-	OrderedKey string `json:"orderedKey"`
-}
-
-// UpdatePriorityWeightTerm defines weight priority.
-type UpdatePriorityWeightTerm struct {
-	// Weight associated with matching the corresponding matchExpressions, in the range 1-100.
-	Weight int32 `json:"weight"`
-	// MatchSelector is used to select by pod's labels.
-	MatchSelector metav1.LabelSelector `json:"matchSelector"`
-}

apis/apps/v1alpha1/validation.go

@@ -1,76 +0,0 @@
-/*
-Copyright 2020 The Kruise Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1alpha1
-
-import (
-	"fmt"
-
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// FieldsValidation checks invalid fields in UpdatePriorityStrategy.
-func (strategy *UpdatePriorityStrategy) FieldsValidation() error {
-	if strategy == nil {
-		return nil
-	}
-
-	if len(strategy.WeightPriority) > 0 && len(strategy.OrderPriority) > 0 {
-		return fmt.Errorf("only one of weightPriority and orderPriority can be used")
-	}
-
-	for _, w := range strategy.WeightPriority {
-		if w.Weight < 0 || w.Weight > 100 {
-			return fmt.Errorf("weight must be valid number in the range 1-100")
-		}
-		if w.MatchSelector.Size() == 0 {
-			return fmt.Errorf("selector can not be empty")
-		}
-		if _, err := metav1.LabelSelectorAsSelector(&w.MatchSelector); err != nil {
-			return fmt.Errorf("invalid selector %v", err)
-		}
-	}
-
-	for _, o := range strategy.OrderPriority {
-		if len(o.OrderedKey) == 0 {
-			return fmt.Errorf("order key can not be empty")
-		}
-	}
-
-	return nil
-}
-
-// FieldsValidation checks invalid fields in CloneSetUpdateScatterStrategy.
-func (strategy CloneSetUpdateScatterStrategy) FieldsValidation() error {
-	if len(strategy) == 0 {
-		return nil
-	}
-
-	m := make(map[string]struct{}, len(strategy))
-	for _, term := range strategy {
-		if term.Key == "" {
-			return fmt.Errorf("key should not be empty")
-		}
-		id := term.Key + ":" + term.Value
-		if _, ok := m[id]; !ok {
-			m[id] = struct{}{}
-		} else {
-			return fmt.Errorf("duplicated key=%v value=%v", term.Key, term.Value)
-		}
-	}
-
-	return nil
-}

apis/apps/v1alpha1/well_know_annotations.go

@@ -0,0 +1,8 @@
+package v1alpha1
+
+const (
+	// AnnotationUsingEnhancedLiveness indicates that the enhanced liveness probe of pod is enabled.
+	AnnotationUsingEnhancedLiveness = "apps.kruise.io/using-enhanced-liveness"
+	// AnnotationUsingEnhancedLiveness indicates the backup probe (json types) of the pod native container livnessprobe configuration.
+	AnnotationNativeContainerProbeContext = "apps.kruise.io/container-probe-context"
+)

apis/apps/v1alpha1/well_known_labels.go

@@ -4,9 +4,35 @@ const (
 	// ControllerRevisionHashLabelKey is used to record the controller revision of current resource.
 	ControllerRevisionHashLabelKey = "apps.kruise.io/controller-revision-hash"
 
+	// ReservedPodLabelKey is used to mark the reserved pods.
+	ReservedPodLabelKey = "apps.kruise.io/united-deployment-reserved-pod"
+
 	// SubSetNameLabelKey is used to record the name of current subset.
 	SubSetNameLabelKey = "apps.kruise.io/subset-name"
 
 	// SpecifiedDeleteKey indicates this object should be deleted, and the value could be the deletion option.
 	SpecifiedDeleteKey = "apps.kruise.io/specified-delete"
+
+	// ImagePreDownloadCreatedKey indicates the images of this revision have been pre-downloaded
+	ImagePreDownloadCreatedKey = "apps.kruise.io/pre-predownload-created"
+
+	// ImagePreDownloadIgnoredKey indicates the images of this revision have been ignored to pre-download
+	ImagePreDownloadIgnoredKey = "apps.kruise.io/image-predownload-ignored"
+	// AnnotationSubsetPatchKey indicates the patch for every subset
+	AnnotationSubsetPatchKey = "apps.kruise.io/subset-patch"
+)
+
+// Sidecar container environment variable definitions which are used to enable SidecarTerminator to take effect on the sidecar container.
+const (
+	// KruiseTerminateSidecarEnv is an env name, which represents a switch to enable sidecar terminator.
+	// The corresponding value is "true", which means apply a crr to kill sidecar using kruise-daemon.
+	KruiseTerminateSidecarEnv = "KRUISE_TERMINATE_SIDECAR_WHEN_JOB_EXIT"
+
+	// KruiseTerminateSidecarWithImageEnv is an env name, which refers to an image that will replace the original image
+	// using in-place update strategy to kill sidecar. This image must be given if you want to use in-place update
+	// strategy to terminate sidecar containers.
+	KruiseTerminateSidecarWithImageEnv = "KRUISE_TERMINATE_SIDECAR_WHEN_JOB_EXIT_WITH_IMAGE"
+
+	// KruiseIgnoreContainerExitCodeEnv is an env name, which represents a switch to ignore the exit code of sidecar container.
+	KruiseIgnoreContainerExitCodeEnv = "KRUISE_TERMINATE_SIDECAR_IGNORE_EXIT_CODE"
 )

apis/apps/v1alpha1/workloadspread_types.go

@@ -0,0 +1,294 @@
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/intstr"
+)
+
+// WorkloadSpreadSpec defines the desired state of WorkloadSpread.
+type WorkloadSpreadSpec struct {
+	// TargetReference is the target workload that WorkloadSpread want to control.
+	TargetReference *TargetReference `json:"targetRef"`
+
+	// TargetFilter allows WorkloadSpread to manage only a portion of the Pods in the TargetReference:
+	// by specifying the criteria for the Pods to be managed through a label selector,
+	// and by specifying how to obtain the total number of these selected Pods from the workload using replicasPaths.
+	TargetFilter *TargetFilter `json:"targetFilter,omitempty"`
+
+	// Subsets describes the pods distribution details between each of subsets.
+	// +patchMergeKey=name
+	// +patchStrategy=merge
+	Subsets []WorkloadSpreadSubset `json:"subsets" patchStrategy:"merge" patchMergeKey:"name"`
+
+	// ScheduleStrategy indicates the strategy the WorkloadSpread used to preform the schedule between each of subsets.
+	// +optional
+	ScheduleStrategy WorkloadSpreadScheduleStrategy `json:"scheduleStrategy,omitempty"`
+}
+
+// TargetReference contains enough information to let you identify an workload
+type TargetReference struct {
+	// API version of the referent.
+	APIVersion string `json:"apiVersion"`
+	// Kind of the referent.
+	Kind string `json:"kind"`
+	// Name of the referent.
+	Name string `json:"name"`
+}
+
+/*
+TargetFilter is an optional parameter that allows WorkloadSpread to manage only a subset of the Pods generated by the target workload.
+
+For example, suppose a WorkloadSpread points to the following Kubeflow TFJob resource:
+
+	```yaml
+	apiVersion: kubeflow.org/v1
+	kind: TFJob
+	spec:
+	  tfReplicaSpecs:
+		PS:
+		  replicas: 1
+		  ...
+		MASTER:
+		  replicas: 1
+		  ...
+		Worker:
+		  replicas: 2
+		  ...
+	```
+
+If you want to manage only the 2 Worker Pods that are generated, you need to configure the TargetFilter as follows:
+
+	```yaml
+	targetFilter:
+	  selector:
+		matchLabels:
+		  role: worker
+	  replicasPathList:
+		- spec.tfReplicaSpecs.Worker.replicas
+	```
+
+With this configuration, the PS Pods and Master Pods generated by the TFJob will not be managed by WorkloadSpread and will not be
+counted toward the total number of replicas.
+*/
+type TargetFilter struct {
+	// Selector is used to filter the Pods to be managed.
+	//
+	//+optional
+	Selector *metav1.LabelSelector `json:"selector,omitempty"`
+
+	// ReplicasPathList is a list of resource paths used to specify how to determine the total number of replicas of
+	// the target workload after filtering. If this list is not empty, WorkloadSpread will look for the corresponding
+	// values in the target resource according to each path, and treat the sum of these values as the total number of replicas after filtering.
+	//
+	// The replicas path is a dot-separated path, similar to "spec.replicas". If there are arrays, you can use numbers to denote indexes, like "subsets.1.replicas".
+	// The real values of these paths must be integers.
+	//
+	// +optional
+	ReplicasPathList []string `json:"replicasPathList,omitempty"`
+}
+
+// WorkloadSpreadScheduleStrategyType is a string enumeration type that enumerates
+// all possible schedule strategies for the WorkloadSpread controller.
+// +kubebuilder:validation:Enum=Adaptive;Fixed;""
+type WorkloadSpreadScheduleStrategyType string
+
+const (
+	// AdaptiveWorkloadSpreadScheduleStrategyType represents that user can permit that controller reschedule Pod
+	// or simulate a schedule process to check whether Pod can run in some subset.
+	AdaptiveWorkloadSpreadScheduleStrategyType WorkloadSpreadScheduleStrategyType = "Adaptive"
+	// FixedWorkloadSpreadScheduleStrategyType represents to give up reschedule and simulation schedule feature.
+	FixedWorkloadSpreadScheduleStrategyType WorkloadSpreadScheduleStrategyType = "Fixed"
+)
+
+// WorkloadSpreadScheduleStrategy defines the schedule performance of WorkloadSpread
+type WorkloadSpreadScheduleStrategy struct {
+	// Type indicates the type of the WorkloadSpreadScheduleStrategy.
+	// Default is Fixed
+	// +optional
+	Type WorkloadSpreadScheduleStrategyType `json:"type,omitempty"`
+
+	// Adaptive is used to communicate parameters when Type is AdaptiveWorkloadSpreadScheduleStrategyType.
+	// +optional
+	Adaptive *AdaptiveWorkloadSpreadStrategy `json:"adaptive,omitempty"`
+}
+
+// AdaptiveWorkloadSpreadStrategy is used to communicate parameters when Type is AdaptiveWorkloadSpreadScheduleStrategyType.
+type AdaptiveWorkloadSpreadStrategy struct {
+	// DisableSimulationSchedule indicates whether to disable the feature of simulation schedule.
+	// Default is false.
+	// Webhook can take a simple general predicates to check whether Pod can be scheduled into this subset,
+	// but it just considers the Node resource and cannot replace scheduler to do richer predicates practically.
+	// +optional
+	DisableSimulationSchedule bool `json:"disableSimulationSchedule,omitempty"`
+
+	// RescheduleCriticalSeconds indicates how long controller will reschedule a schedule failed Pod to the subset that has
+	// redundant capacity after the subset where the Pod lives. If a Pod was scheduled failed and still in a unschedulabe status
+	// over RescheduleCriticalSeconds duration, the controller will reschedule it to a suitable subset.
+	// +optional
+	RescheduleCriticalSeconds *int32 `json:"rescheduleCriticalSeconds,omitempty"`
+}
+
+// WorkloadSpreadSubset defines the details of a subset.
+type WorkloadSpreadSubset struct {
+	// Name should be unique between all of the subsets under one WorkloadSpread.
+	Name string `json:"name"`
+
+	// Indicates the node required selector to form the subset.
+	// +optional
+	RequiredNodeSelectorTerm *corev1.NodeSelectorTerm `json:"requiredNodeSelectorTerm,omitempty"`
+
+	// Indicates the node preferred selector to form the subset.
+	// +optional
+	PreferredNodeSelectorTerms []corev1.PreferredSchedulingTerm `json:"preferredNodeSelectorTerms,omitempty"`
+
+	// Indicates the tolerations the pods under this subset have.
+	// +optional
+	Tolerations []corev1.Toleration `json:"tolerations,omitempty"`
+
+	// MaxReplicas indicates the desired max replicas of this subset.
+	// +optional
+	MaxReplicas *intstr.IntOrString `json:"maxReplicas,omitempty"`
+
+	// Patch indicates patching podTemplate to the Pod.
+	// +optional
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	Patch runtime.RawExtension `json:"patch,omitempty"`
+}
+
+// WorkloadSpreadStatus defines the observed state of WorkloadSpread.
+type WorkloadSpreadStatus struct {
+	// ObservedGeneration is the most recent generation observed for this WorkloadSpread. It corresponds to the
+	// WorkloadSpread's generation, which is updated on mutation by the API Server.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// ObservedGeneration is the most recent replicas of target workload observed for this WorkloadSpread.
+	//ObservedWorkloadReplicas int32 `json:"observedWorkloadReplicas"`
+
+	// Contains the status of each subset. Each element in this array represents one subset
+	// +optional
+	SubsetStatuses []WorkloadSpreadSubsetStatus `json:"subsetStatuses,omitempty"`
+
+	// VersionedSubsetStatuses is to solve rolling-update problems, where the creation of new-version pod
+	// may be earlier than deletion of old-version pod. We have to calculate the pod subset distribution for
+	// each version.
+	VersionedSubsetStatuses map[string][]WorkloadSpreadSubsetStatus `json:"versionedSubsetStatuses,omitempty"`
+}
+
+type WorkloadSpreadSubsetConditionType string
+
+const (
+	// SubsetSchedulable means the nodes in this subset have sufficient resources to schedule a fixed number of Pods of a workload.
+	// When one or more one pods in a subset have condition[PodScheduled] = false, the subset is considered temporarily unschedulable.
+	// The condition[PodScheduled] = false, which means the cluster does not have enough resources to schedule at this moment,
+	// even if only single Pod scheduled fails.
+	// After a period of time(e.g. 5m), the controller will recover the subset to be schedulable.
+	SubsetSchedulable WorkloadSpreadSubsetConditionType = "Schedulable"
+)
+
+type WorkloadSpreadSubsetCondition struct {
+	// Type of in place set condition.
+	Type WorkloadSpreadSubsetConditionType `json:"type"`
+
+	// Status of the condition, one of True, False, Unknown.
+	Status corev1.ConditionStatus `json:"status"`
+
+	// Last time the condition transitioned from one status to another.
+	// +optional
+	LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
+
+	// The reason for the condition's last transition.
+	// +optional
+	Reason string `json:"reason,omitempty"`
+
+	// A human readable message indicating details about the transition.
+	// +optional
+	Message string `json:"message,omitempty"`
+}
+
+// WorkloadSpreadSubsetStatus defines the observed state of subset
+type WorkloadSpreadSubsetStatus struct {
+	// Name should be unique between all of the subsets under one WorkloadSpread.
+	Name string `json:"name"`
+
+	// Replicas is the most recently observed number of active replicas for subset.
+	Replicas int32 `json:"replicas"`
+
+	// Conditions is an array of current observed subset conditions.
+	// +optional
+	Conditions []WorkloadSpreadSubsetCondition `json:"conditions,omitempty"`
+
+	// MissingReplicas is the number of active replicas belong to this subset not be found.
+	// MissingReplicas > 0 indicates the subset is still missing MissingReplicas pods to create
+	// MissingReplicas = 0 indicates the subset already has enough pods, there is no need to create
+	// MissingReplicas = -1 indicates the subset's MaxReplicas not set, then there is no limit for pods number
+	MissingReplicas int32 `json:"missingReplicas"`
+
+	// CreatingPods contains information about pods whose creation was processed by
+	// the webhook handler but not yet been observed by the WorkloadSpread controller.
+	// A pod will be in this map from the time when the webhook handler processed the
+	// creation request to the time when the pod is seen by controller.
+	// The key in the map is the name of the pod and the value is the time when the webhook
+	// handler process the creation request. If the real creation didn't happen and a pod is
+	// still in this map, it will be removed from the list automatically by WorkloadSpread controller
+	// after some time.
+	// If everything goes smooth this map should be empty for the most of the time.
+	// Large number of entries in the map may indicate problems with pod creations.
+	// +optional
+	CreatingPods map[string]metav1.Time `json:"creatingPods,omitempty"`
+
+	// DeletingPods is similar with CreatingPods and it contains information about pod deletion.
+	// +optional
+	DeletingPods map[string]metav1.Time `json:"deletingPods,omitempty"`
+}
+
+// +genclient
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:shortName=ws
+// +kubebuilder:printcolumn:name=WorkloadName,type=string,JSONPath=".spec.targetRef.name"
+// +kubebuilder:printcolumn:name=WorkloadKind,type=string,JSONPath=".spec.targetRef.kind"
+// +kubebuilder:printcolumn:name=Adaptive,type=boolean,JSONPath=`.spec.scheduleStrategy.type[?(@ == "Adaptive")]`,description="Whether use the adaptive reschedule strategy"
+// +kubebuilder:printcolumn:name="Age",type="date",JSONPath=".metadata.creationTimestamp"
+
+// WorkloadSpread is the Schema for the WorkloadSpread API
+type WorkloadSpread struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   WorkloadSpreadSpec   `json:"spec,omitempty"`
+	Status WorkloadSpreadStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// WorkloadSpreadList contains a list of WorkloadSpread
+type WorkloadSpreadList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []WorkloadSpread `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&WorkloadSpread{}, &WorkloadSpreadList{})
+}

apis/apps/v1beta1/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:openapi-gen=true
+// +groupName=apps.kruise.io
+package v1beta1

apis/apps/v1beta1/groupversion_info.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package v1beta1 contains API Schema definitions for the apps v1beta1 API group
+// +kubebuilder:object:generate=true
+// +groupName=apps.kruise.io
+package v1beta1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: "apps.kruise.io", Version: "v1beta1"}
+
+	SchemeGroupVersion = GroupVersion
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)
+
+// Resource is required by pkg/client/listers/...
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}

apis/apps/v1beta1/statefulset_conversion.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+func (*StatefulSet) Hub() {}

apis/apps/v1beta1/statefulset_types.go

@@ -0,0 +1,417 @@
+/*
+Copyright 2020 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1beta1
+
+import (
+	appspub "github.com/openkruise/kruise/apis/apps/pub"
+	apps "k8s.io/api/apps/v1"
+	v1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+)
+
+const (
+	// MaxMinReadySeconds is the max value of MinReadySeconds
+	MaxMinReadySeconds = 300
+)
+
+// VolumeClaimUpdateStrategyType defines the update strategy types for volume claims.
+// It is an enumerated type that provides two different update strategies.
+// +enum
+type VolumeClaimUpdateStrategyType string
+
+const (
+	// OnPodRollingUpdateVolumeClaimUpdateStrategyType indicates that volume claim updates are triggered when associated Pods undergo rolling updates.
+	// This strategy ensures that storage availability and integrity are maintained during the update process.
+	OnPodRollingUpdateVolumeClaimUpdateStrategyType VolumeClaimUpdateStrategyType = "OnPodRollingUpdate"
+
+	// OnPVCDeleteVolumeClaimUpdateStrategyType indicates that updates are triggered when a Persistent Volume Claim (PVC) is deleted.
+	// This strategy places full control of the update timing in the hands of the user, typically executed after ensuring data has been backed up or there are no data security concerns,
+	// allowing for storage resource management that aligns with specific user requirements and security policies.
+	OnPVCDeleteVolumeClaimUpdateStrategyType VolumeClaimUpdateStrategyType = "OnDelete"
+)
+
+// VolumeClaimStatus describes the status of a volume claim template.
+// It provides details about the compatibility and readiness of the volume claim.
+type VolumeClaimStatus struct {
+	// VolumeClaimName is the name of the volume claim.
+	// This is a unique identifier used to reference a specific volume claim.
+	VolumeClaimName string `json:"volumeClaimName"`
+	// CompatibleReplicas is the number of replicas currently compatible with the volume claim.
+	// It indicates how many replicas can function properly, being compatible with this volume claim.
+	// Compatibility is determined by whether the PVC spec storage requests are greater than or equal to the template spec storage requests
+	CompatibleReplicas int32 `json:"compatibleReplicas"`
+	// CompatibleReadyReplicas is the number of replicas that are both ready and compatible with the volume claim.
+	// It highlights that these replicas are not only compatible but also ready to be put into service immediately.
+	// Compatibility is determined by whether the pvc spec storage requests are greater than or equal to the template spec storage requests
+	// The "ready" status is determined by whether the PVC status capacity is greater than or equal to the PVC spec storage requests.
+	CompatibleReadyReplicas int32 `json:"compatibleReadyReplicas"`
+}
+
+// StatefulSetUpdateStrategy indicates the strategy that the StatefulSet
+// controller will use to perform updates. It includes any additional parameters
+// necessary to perform the update for the indicated strategy.
+type StatefulSetUpdateStrategy struct {
+	// Type indicates the type of the StatefulSetUpdateStrategy.
+	// Default is RollingUpdate.
+	// +optional
+	Type apps.StatefulSetUpdateStrategyType `json:"type,omitempty"`
+	// RollingUpdate is used to communicate parameters when Type is RollingUpdateStatefulSetStrategyType.
+	// +optional
+	RollingUpdate *RollingUpdateStatefulSetStrategy `json:"rollingUpdate,omitempty"`
+}
+
+// VolumeClaimUpdateStrategy defines the strategy for updating volume claims.
+// This structure is used to control how updates to PersistentVolumeClaims are handled during pod rolling updates or PersistentVolumeClaim deletions.
+type VolumeClaimUpdateStrategy struct {
+	// Type specifies the type of update strategy, possible values include:
+	// OnPodRollingUpdateVolumeClaimUpdateStrategyType: Apply the update strategy during pod rolling updates.
+	// OnPVCDeleteVolumeClaimUpdateStrategyType: Apply the update strategy when a PersistentVolumeClaim is deleted.
+	Type VolumeClaimUpdateStrategyType `json:"type,omitempty"`
+}
+
+// RollingUpdateStatefulSetStrategy is used to communicate parameter for RollingUpdateStatefulSetStrategyType.
+type RollingUpdateStatefulSetStrategy struct {
+	// Partition indicates the number of pods the StatefulSet should be partitioned by default.
+	//   - It means controller will update $(replicas - partition) number of pod.
+	// Default value is 0.
+	// +optional
+	Partition *int32 `json:"partition,omitempty"`
+	// The maximum number of pods that can be unavailable during the update.
+	// Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
+	// Absolute number is calculated from percentage by rounding down.
+	// Also, maxUnavailable can just be allowed to work with Parallel podManagementPolicy.
+	// Defaults to 1.
+	// +optional
+	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+	// PodUpdatePolicy indicates how pods should be updated
+	// Default value is "ReCreate"
+	// +optional
+	PodUpdatePolicy PodUpdateStrategyType `json:"podUpdatePolicy,omitempty"`
+	// Paused indicates that the StatefulSet is paused.
+	// Default value is false
+	// +optional
+	Paused bool `json:"paused,omitempty"`
+	// UnorderedUpdate contains strategies for non-ordered update.
+	// If it is not nil, pods will be updated with non-ordered sequence.
+	// Noted that UnorderedUpdate can only be allowed to work with Parallel podManagementPolicy
+	// +optional
+	UnorderedUpdate *UnorderedUpdateStrategy `json:"unorderedUpdate,omitempty"`
+	// InPlaceUpdateStrategy contains strategies for in-place update.
+	// +optional
+	InPlaceUpdateStrategy *appspub.InPlaceUpdateStrategy `json:"inPlaceUpdateStrategy,omitempty"`
+	// MinReadySeconds indicates how long will the pod be considered ready after it's updated.
+	// MinReadySeconds works with both OrderedReady and Parallel podManagementPolicy.
+	// It affects the pod scale up speed when the podManagementPolicy is set to be OrderedReady.
+	// Combined with MaxUnavailable, it affects the pod update speed regardless of podManagementPolicy.
+	// Default value is 0, max is 300.
+	// +optional
+	MinReadySeconds *int32 `json:"minReadySeconds,omitempty"`
+}
+
+// UnorderedUpdateStrategy defines strategies for non-ordered update.
+type UnorderedUpdateStrategy struct {
+	// Priorities are the rules for calculating the priority of updating pods.
+	// Each pod to be updated, will pass through these terms and get a sum of weights.
+	// +optional
+	PriorityStrategy *appspub.UpdatePriorityStrategy `json:"priorityStrategy,omitempty"`
+}
+
+// PodUpdateStrategyType is a string enumeration type that enumerates
+// all possible ways we can update a Pod when updating application
+type PodUpdateStrategyType string
+
+const (
+	// RecreatePodUpdateStrategyType indicates that we always delete Pod and create new Pod
+	// during Pod update, which is the default behavior
+	RecreatePodUpdateStrategyType PodUpdateStrategyType = "ReCreate"
+	// InPlaceIfPossiblePodUpdateStrategyType indicates that we try to in-place update Pod instead of
+	// recreating Pod when possible. Currently, only image update of pod spec is allowed. Any other changes to the pod
+	// spec will fall back to ReCreate PodUpdateStrategyType where pod will be recreated.
+	InPlaceIfPossiblePodUpdateStrategyType PodUpdateStrategyType = "InPlaceIfPossible"
+	// InPlaceOnlyPodUpdateStrategyType indicates that we will in-place update Pod instead of
+	// recreating pod. Currently we only allow image update for pod spec. Any other changes to the pod spec will be
+	// rejected by kube-apiserver
+	InPlaceOnlyPodUpdateStrategyType PodUpdateStrategyType = "InPlaceOnly"
+)
+
+// PersistentVolumeClaimRetentionPolicyType is a string enumeration of the policies that will determine
+// when volumes from the VolumeClaimTemplates will be deleted when the controlling StatefulSet is
+// deleted or scaled down.
+type PersistentVolumeClaimRetentionPolicyType string
+
+const (
+	// RetainPersistentVolumeClaimRetentionPolicyType is the default
+	// PersistentVolumeClaimRetentionPolicy and specifies that
+	// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
+	// will not be deleted.
+	RetainPersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Retain"
+	// DeletePersistentVolumeClaimRetentionPolicyType specifies that
+	// PersistentVolumeClaims associated with StatefulSet VolumeClaimTemplates
+	// will be deleted in the scenario specified in
+	// StatefulSetPersistentVolumeClaimPolicy.
+	DeletePersistentVolumeClaimRetentionPolicyType PersistentVolumeClaimRetentionPolicyType = "Delete"
+)
+
+// StatefulSetPersistentVolumeClaimRetentionPolicy describes the policy used for PVCs
+// created from the StatefulSet VolumeClaims.
+type StatefulSetPersistentVolumeClaimRetentionPolicy struct {
+	// WhenDeleted specifies what happens to PVCs created from StatefulSet
+	// VolumeClaimTemplates when the StatefulSet is deleted. The default policy
+	// of `Retain` causes PVCs to not be affected by StatefulSet deletion. The
+	// `Delete` policy causes those PVCs to be deleted.
+	WhenDeleted PersistentVolumeClaimRetentionPolicyType `json:"whenDeleted,omitempty"`
+	// WhenScaled specifies what happens to PVCs created from StatefulSet
+	// VolumeClaimTemplates when the StatefulSet is scaled down. The default
+	// policy of `Retain` causes PVCs to not be affected by a scaledown. The
+	// `Delete` policy causes the associated PVCs for any excess pods above
+	// the replica count to be deleted.
+	WhenScaled PersistentVolumeClaimRetentionPolicyType `json:"whenScaled,omitempty"`
+}
+
+// StatefulSetOrdinals describes the policy used for replica ordinal assignment
+// in this StatefulSet.
+type StatefulSetOrdinals struct {
+	// start is the number representing the first replica's index. It may be used
+	// to number replicas from an alternate index (eg: 1-indexed) over the default
+	// 0-indexed names, or to orchestrate progressive movement of replicas from
+	// one StatefulSet to another.
+	// If set, replica indices will be in the range:
+	//   [.spec.ordinals.start, .spec.ordinals.start + .spec.replicas).
+	// If unset, defaults to 0. Replica indices will be in the range:
+	//   [0, .spec.replicas).
+	// +optional
+	Start int32 `json:"start" protobuf:"varint,1,opt,name=start"`
+}
+
+// StatefulSetSpec defines the desired state of StatefulSet
+type StatefulSetSpec struct {
+	// replicas is the desired number of replicas of the given Template.
+	// These are replicas in the sense that they are instantiations of the
+	// same Template, but individual replicas also have a consistent identity.
+	// If unspecified, defaults to 1.
+	// TODO: Consider a rename of this field.
+	// +optional
+	Replicas *int32 `json:"replicas,omitempty"`
+
+	// selector is a label query over pods that should match the replica count.
+	// It must match the pod template's labels.
+	// More info: https://kubernetes.io/docs/concepts/overview/working-with-objects/labels/#label-selectors
+	Selector *metav1.LabelSelector `json:"selector"`
+
+	// template is the object that describes the pod that will be created if
+	// insufficient replicas are detected. Each pod stamped out by the StatefulSet
+	// will fulfill this Template, but have a unique identity from the rest
+	// of the StatefulSet.
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	Template v1.PodTemplateSpec `json:"template"`
+
+	// volumeClaimTemplates is a list of claims that pods are allowed to reference.
+	// The StatefulSet controller is responsible for mapping network identities to
+	// claims in a way that maintains the identity of a pod. Every claim in
+	// this list must have at least one matching (by name) volumeMount in one
+	// container in the template. A claim in this list takes precedence over
+	// any volumes in the template, with the same name.
+	// TODO: Define the behavior if a claim already exists with the same name.
+	// +optional
+	// +kubebuilder:pruning:PreserveUnknownFields
+	// +kubebuilder:validation:Schemaless
+	VolumeClaimTemplates []v1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
+
+	// VolumeClaimUpdateStrategy specifies the strategy for updating VolumeClaimTemplates within a StatefulSet.
+	// This field is currently only effective if the StatefulSetAutoResizePVCGate is enabled.
+	// +optional
+	VolumeClaimUpdateStrategy VolumeClaimUpdateStrategy `json:"volumeClaimUpdateStrategy,omitempty"`
+
+	// serviceName is the name of the service that governs this StatefulSet.
+	// This service must exist before the StatefulSet, and is responsible for
+	// the network identity of the set. Pods get DNS/hostnames that follow the
+	// pattern: pod-specific-string.serviceName.default.svc.cluster.local
+	// where "pod-specific-string" is managed by the StatefulSet controller.
+	ServiceName string `json:"serviceName,omitempty"`
+
+	// podManagementPolicy controls how pods are created during initial scale up,
+	// when replacing pods on nodes, or when scaling down. The default policy is
+	// `OrderedReady`, where pods are created in increasing order (pod-0, then
+	// pod-1, etc) and the controller will wait until each pod is ready before
+	// continuing. When scaling down, the pods are removed in the opposite order.
+	// The alternative policy is `Parallel` which will create pods in parallel
+	// to match the desired scale without waiting, and on scale down will delete
+	// all pods at once.
+	// +optional
+	PodManagementPolicy apps.PodManagementPolicyType `json:"podManagementPolicy,omitempty"`
+
+	// updateStrategy indicates the StatefulSetUpdateStrategy that will be
+	// employed to update Pods in the StatefulSet when a revision is made to
+	// Template.
+	UpdateStrategy StatefulSetUpdateStrategy `json:"updateStrategy,omitempty"`
+
+	// revisionHistoryLimit is the maximum number of revisions that will
+	// be maintained in the StatefulSet's revision history. The revision history
+	// consists of all revisions not represented by a currently applied
+	// StatefulSetSpec version. The default value is 10.
+	RevisionHistoryLimit *int32 `json:"revisionHistoryLimit,omitempty"`
+
+	// reserveOrdinals controls the ordinal numbers that should be reserved, and the replicas
+	// will always be the expectation number of running Pods.
+	// For a sts with replicas=3 and its Pods in [0, 1, 2]:
+	// - If you want to migrate Pod-1 and reserve this ordinal, just set spec.reserveOrdinal to [1].
+	//   Then controller will delete Pod-1 and create Pod-3 (existing Pods will be [0, 2, 3])
+	// - If you just want to delete Pod-1, you should set spec.reserveOrdinal to [1] and spec.replicas to 2.
+	//   Then controller will delete Pod-1 (existing Pods will be [0, 2])
+	// You can also use ranges along with numbers, such as [1, 3-5], which is a shortcut for [1, 3, 4, 5].
+	ReserveOrdinals []intstr.IntOrString `json:"reserveOrdinals,omitempty"`
+
+	// Lifecycle defines the lifecycle hooks for Pods pre-delete, in-place update.
+	Lifecycle *appspub.Lifecycle `json:"lifecycle,omitempty"`
+
+	// scaleStrategy indicates the StatefulSetScaleStrategy that will be
+	// employed to scale Pods in the StatefulSet.
+	ScaleStrategy *StatefulSetScaleStrategy `json:"scaleStrategy,omitempty"`
+
+	// PersistentVolumeClaimRetentionPolicy describes the policy used for PVCs created from
+	// the StatefulSet VolumeClaimTemplates. This requires the
+	// StatefulSetAutoDeletePVC feature gate to be enabled, which is alpha.
+	// +optional
+	PersistentVolumeClaimRetentionPolicy *StatefulSetPersistentVolumeClaimRetentionPolicy `json:"persistentVolumeClaimRetentionPolicy,omitempty"`
+
+	// ordinals controls the numbering of replica indices in a StatefulSet. The
+	// default ordinals behavior assigns a "0" index to the first replica and
+	// increments the index by one for each additional replica requested. Using
+	// the ordinals field requires the StatefulSetStartOrdinal feature gate to be
+	// enabled, which is beta.
+	// +optional
+	Ordinals *StatefulSetOrdinals `json:"ordinals,omitempty"`
+}
+
+// StatefulSetScaleStrategy defines strategies for pods scale.
+type StatefulSetScaleStrategy struct {
+	// The maximum number of pods that can be unavailable during scaling.
+	// Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
+	// Absolute number is calculated from percentage by rounding down.
+	// It can just be allowed to work with Parallel podManagementPolicy.
+	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+}
+
+// StatefulSetStatus defines the observed state of StatefulSet
+type StatefulSetStatus struct {
+	// observedGeneration is the most recent generation observed for this StatefulSet. It corresponds to the
+	// StatefulSet's generation, which is updated on mutation by the API Server.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
+
+	// replicas is the number of Pods created by the StatefulSet controller.
+	Replicas int32 `json:"replicas"`
+
+	// readyReplicas is the number of Pods created by the StatefulSet controller that have a Ready Condition.
+	ReadyReplicas int32 `json:"readyReplicas"`
+
+	// AvailableReplicas is the number of Pods created by the StatefulSet controller that have been ready for
+	//minReadySeconds.
+	AvailableReplicas int32 `json:"availableReplicas"`
+
+	// currentReplicas is the number of Pods created by the StatefulSet controller from the StatefulSet version
+	// indicated by currentRevision.
+	CurrentReplicas int32 `json:"currentReplicas"`
+
+	// updatedReplicas is the number of Pods created by the StatefulSet controller from the StatefulSet version
+	// indicated by updateRevision.
+	UpdatedReplicas int32 `json:"updatedReplicas"`
+
+	// updatedReadyReplicas is the number of updated Pods created by the StatefulSet controller that have a Ready Condition.
+	UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"`
+
+	// updatedAvailableReplicas is the number of updated Pods created by the StatefulSet controller that have a Ready condition
+	//for atleast minReadySeconds.
+	UpdatedAvailableReplicas int32 `json:"updatedAvailableReplicas,omitempty"`
+
+	// currentRevision, if not empty, indicates the version of the StatefulSet used to generate Pods in the
+	// sequence [0,currentReplicas).
+	CurrentRevision string `json:"currentRevision,omitempty"`
+
+	// updateRevision, if not empty, indicates the version of the StatefulSet used to generate Pods in the sequence
+	// [replicas-updatedReplicas,replicas)
+	UpdateRevision string `json:"updateRevision,omitempty"`
+
+	// collisionCount is the count of hash collisions for the StatefulSet. The StatefulSet controller
+	// uses this field as a collision avoidance mechanism when it needs to create the name for the
+	// newest ControllerRevision.
+	// +optional
+	CollisionCount *int32 `json:"collisionCount,omitempty"`
+
+	// Represents the latest available observations of a statefulset's current state.
+	// +optional
+	// +patchMergeKey=type
+	// +patchStrategy=merge
+	Conditions []apps.StatefulSetCondition `json:"conditions,omitempty" patchStrategy:"merge" patchMergeKey:"type"`
+
+	// LabelSelector is label selectors for query over pods that should match the replica count used by HPA.
+	LabelSelector string `json:"labelSelector,omitempty"`
+
+	// VolumeClaims represents the status of compatibility between existing PVCs
+	// and their respective templates. It tracks whether the PersistentVolumeClaims have been updated
+	// to match any changes made to the volumeClaimTemplates, ensuring synchronization
+	// between the defined templates and the actual PersistentVolumeClaims in use.
+	VolumeClaims []VolumeClaimStatus `json:"volumeClaims,omitempty"`
+}
+
+// These are valid conditions of a statefulset.
+const (
+	FailedCreatePod apps.StatefulSetConditionType = "FailedCreatePod"
+	FailedUpdatePod apps.StatefulSetConditionType = "FailedUpdatePod"
+)
+
+// +genclient
+// +genclient:method=GetScale,verb=get,subresource=scale,result=k8s.io/api/autoscaling/v1.Scale
+// +genclient:method=UpdateScale,verb=update,subresource=scale,input=k8s.io/api/autoscaling/v1.Scale,result=k8s.io/api/autoscaling/v1.Scale
+// +k8s:openapi-gen=true
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector
+// +kubebuilder:resource:shortName=sts;asts
+// +kubebuilder:storageversion
+// +kubebuilder:subresource:scale:specpath=.spec.replicas,statuspath=.status.replicas,selectorpath=.status.labelSelector
+// +kubebuilder:printcolumn:name="DESIRED",type="integer",JSONPath=".spec.replicas",description="The desired number of pods."
+// +kubebuilder:printcolumn:name="CURRENT",type="integer",JSONPath=".status.replicas",description="The number of currently all pods."
+// +kubebuilder:printcolumn:name="UPDATED",type="integer",JSONPath=".status.updatedReplicas",description="The number of pods updated."
+// +kubebuilder:printcolumn:name="READY",type="integer",JSONPath=".status.readyReplicas",description="The number of pods ready."
+// +kubebuilder:printcolumn:name="AGE",type="date",JSONPath=".metadata.creationTimestamp",description="CreationTimestamp is a timestamp representing the server time when this object was created. It is not guaranteed to be set in happens-before order across separate operations. Clients may not set this value. It is represented in RFC3339 form and is in UTC."
+// +kubebuilder:printcolumn:name="CONTAINERS",type="string",priority=1,JSONPath=".spec.template.spec.containers[*].name",description="The containers of currently advanced statefulset."
+// +kubebuilder:printcolumn:name="IMAGES",type="string",priority=1,JSONPath=".spec.template.spec.containers[*].image",description="The images of currently advanced statefulset."
+
+// StatefulSet is the Schema for the statefulsets API
+type StatefulSet struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   StatefulSetSpec   `json:"spec,omitempty"`
+	Status StatefulSetStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// StatefulSetList contains a list of StatefulSet
+type StatefulSetList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []StatefulSet `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&StatefulSet{}, &StatefulSetList{})
+}

apis/apps/v1beta1/zz_generated.deepcopy.go

@@ -0,0 +1,346 @@
+//go:build !ignore_autogenerated
+
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Code generated by controller-gen. DO NOT EDIT.
+
+package v1beta1
+
+import (
+	"github.com/openkruise/kruise/apis/apps/pub"
+	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/apis/meta/v1"
+	runtime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/intstr"
+)
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *RollingUpdateStatefulSetStrategy) DeepCopyInto(out *RollingUpdateStatefulSetStrategy) {
+	*out = *in
+	if in.Partition != nil {
+		in, out := &in.Partition, &out.Partition
+		*out = new(int32)
+		**out = **in
+	}
+	if in.MaxUnavailable != nil {
+		in, out := &in.MaxUnavailable, &out.MaxUnavailable
+		*out = new(intstr.IntOrString)
+		**out = **in
+	}
+	if in.UnorderedUpdate != nil {
+		in, out := &in.UnorderedUpdate, &out.UnorderedUpdate
+		*out = new(UnorderedUpdateStrategy)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.InPlaceUpdateStrategy != nil {
+		in, out := &in.InPlaceUpdateStrategy, &out.InPlaceUpdateStrategy
+		*out = new(pub.InPlaceUpdateStrategy)
+		**out = **in
+	}
+	if in.MinReadySeconds != nil {
+		in, out := &in.MinReadySeconds, &out.MinReadySeconds
+		*out = new(int32)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RollingUpdateStatefulSetStrategy.
+func (in *RollingUpdateStatefulSetStrategy) DeepCopy() *RollingUpdateStatefulSetStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(RollingUpdateStatefulSetStrategy)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSet) DeepCopyInto(out *StatefulSet) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
+	in.Spec.DeepCopyInto(&out.Spec)
+	in.Status.DeepCopyInto(&out.Status)
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSet.
+func (in *StatefulSet) DeepCopy() *StatefulSet {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSet)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *StatefulSet) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSetList) DeepCopyInto(out *StatefulSetList) {
+	*out = *in
+	out.TypeMeta = in.TypeMeta
+	in.ListMeta.DeepCopyInto(&out.ListMeta)
+	if in.Items != nil {
+		in, out := &in.Items, &out.Items
+		*out = make([]StatefulSet, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetList.
+func (in *StatefulSetList) DeepCopy() *StatefulSetList {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSetList)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
+func (in *StatefulSetList) DeepCopyObject() runtime.Object {
+	if c := in.DeepCopy(); c != nil {
+		return c
+	}
+	return nil
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSetOrdinals) DeepCopyInto(out *StatefulSetOrdinals) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetOrdinals.
+func (in *StatefulSetOrdinals) DeepCopy() *StatefulSetOrdinals {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSetOrdinals)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopyInto(out *StatefulSetPersistentVolumeClaimRetentionPolicy) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetPersistentVolumeClaimRetentionPolicy.
+func (in *StatefulSetPersistentVolumeClaimRetentionPolicy) DeepCopy() *StatefulSetPersistentVolumeClaimRetentionPolicy {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSetPersistentVolumeClaimRetentionPolicy)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSetScaleStrategy) DeepCopyInto(out *StatefulSetScaleStrategy) {
+	*out = *in
+	if in.MaxUnavailable != nil {
+		in, out := &in.MaxUnavailable, &out.MaxUnavailable
+		*out = new(intstr.IntOrString)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetScaleStrategy.
+func (in *StatefulSetScaleStrategy) DeepCopy() *StatefulSetScaleStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSetScaleStrategy)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSetSpec) DeepCopyInto(out *StatefulSetSpec) {
+	*out = *in
+	if in.Replicas != nil {
+		in, out := &in.Replicas, &out.Replicas
+		*out = new(int32)
+		**out = **in
+	}
+	if in.Selector != nil {
+		in, out := &in.Selector, &out.Selector
+		*out = new(v1.LabelSelector)
+		(*in).DeepCopyInto(*out)
+	}
+	in.Template.DeepCopyInto(&out.Template)
+	if in.VolumeClaimTemplates != nil {
+		in, out := &in.VolumeClaimTemplates, &out.VolumeClaimTemplates
+		*out = make([]corev1.PersistentVolumeClaim, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	out.VolumeClaimUpdateStrategy = in.VolumeClaimUpdateStrategy
+	in.UpdateStrategy.DeepCopyInto(&out.UpdateStrategy)
+	if in.RevisionHistoryLimit != nil {
+		in, out := &in.RevisionHistoryLimit, &out.RevisionHistoryLimit
+		*out = new(int32)
+		**out = **in
+	}
+	if in.ReserveOrdinals != nil {
+		in, out := &in.ReserveOrdinals, &out.ReserveOrdinals
+		*out = make([]intstr.IntOrString, len(*in))
+		copy(*out, *in)
+	}
+	if in.Lifecycle != nil {
+		in, out := &in.Lifecycle, &out.Lifecycle
+		*out = new(pub.Lifecycle)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.ScaleStrategy != nil {
+		in, out := &in.ScaleStrategy, &out.ScaleStrategy
+		*out = new(StatefulSetScaleStrategy)
+		(*in).DeepCopyInto(*out)
+	}
+	if in.PersistentVolumeClaimRetentionPolicy != nil {
+		in, out := &in.PersistentVolumeClaimRetentionPolicy, &out.PersistentVolumeClaimRetentionPolicy
+		*out = new(StatefulSetPersistentVolumeClaimRetentionPolicy)
+		**out = **in
+	}
+	if in.Ordinals != nil {
+		in, out := &in.Ordinals, &out.Ordinals
+		*out = new(StatefulSetOrdinals)
+		**out = **in
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetSpec.
+func (in *StatefulSetSpec) DeepCopy() *StatefulSetSpec {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSetSpec)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSetStatus) DeepCopyInto(out *StatefulSetStatus) {
+	*out = *in
+	if in.CollisionCount != nil {
+		in, out := &in.CollisionCount, &out.CollisionCount
+		*out = new(int32)
+		**out = **in
+	}
+	if in.Conditions != nil {
+		in, out := &in.Conditions, &out.Conditions
+		*out = make([]appsv1.StatefulSetCondition, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+	if in.VolumeClaims != nil {
+		in, out := &in.VolumeClaims, &out.VolumeClaims
+		*out = make([]VolumeClaimStatus, len(*in))
+		copy(*out, *in)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetStatus.
+func (in *StatefulSetStatus) DeepCopy() *StatefulSetStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSetStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *StatefulSetUpdateStrategy) DeepCopyInto(out *StatefulSetUpdateStrategy) {
+	*out = *in
+	if in.RollingUpdate != nil {
+		in, out := &in.RollingUpdate, &out.RollingUpdate
+		*out = new(RollingUpdateStatefulSetStrategy)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StatefulSetUpdateStrategy.
+func (in *StatefulSetUpdateStrategy) DeepCopy() *StatefulSetUpdateStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(StatefulSetUpdateStrategy)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *UnorderedUpdateStrategy) DeepCopyInto(out *UnorderedUpdateStrategy) {
+	*out = *in
+	if in.PriorityStrategy != nil {
+		in, out := &in.PriorityStrategy, &out.PriorityStrategy
+		*out = new(pub.UpdatePriorityStrategy)
+		(*in).DeepCopyInto(*out)
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new UnorderedUpdateStrategy.
+func (in *UnorderedUpdateStrategy) DeepCopy() *UnorderedUpdateStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(UnorderedUpdateStrategy)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeClaimStatus) DeepCopyInto(out *VolumeClaimStatus) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeClaimStatus.
+func (in *VolumeClaimStatus) DeepCopy() *VolumeClaimStatus {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeClaimStatus)
+	in.DeepCopyInto(out)
+	return out
+}
+
+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *VolumeClaimUpdateStrategy) DeepCopyInto(out *VolumeClaimUpdateStrategy) {
+	*out = *in
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new VolumeClaimUpdateStrategy.
+func (in *VolumeClaimUpdateStrategy) DeepCopy() *VolumeClaimUpdateStrategy {
+	if in == nil {
+		return nil
+	}
+	out := new(VolumeClaimUpdateStrategy)
+	in.DeepCopyInto(out)
+	return out
+}

apis/policy/v1alpha1/doc.go

@@ -0,0 +1,19 @@
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// +k8s:openapi-gen=true
+// +groupName=policy.kruise.io
+package v1alpha1

apis/policy/v1alpha1/groupversion_info.go

@@ -0,0 +1,43 @@
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package v1alpha1 contains API Schema definitions for the policy v1alpha1 API group
+// +kubebuilder:object:generate=true
+// +groupName=policy.kruise.io
+package v1alpha1
+
+import (
+	"k8s.io/apimachinery/pkg/runtime/schema"
+	"sigs.k8s.io/controller-runtime/pkg/scheme"
+)
+
+var (
+	// GroupVersion is group version used to register these objects
+	GroupVersion = schema.GroupVersion{Group: "policy.kruise.io", Version: "v1alpha1"}
+
+	SchemeGroupVersion = GroupVersion
+
+	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
+	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
+
+	// AddToScheme adds the types in this group-version to the given scheme.
+	AddToScheme = SchemeBuilder.AddToScheme
+)
+
+// Resource is required by pkg/client/listers/...
+func Resource(resource string) schema.GroupResource {
+	return SchemeGroupVersion.WithResource(resource).GroupResource()
+}

apis/policy/v1alpha1/podunavailablebudget_types.go

@@ -0,0 +1,144 @@
+/*
+Copyright 2021 The Kruise Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/util/intstr"
+)
+
+// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
+// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
+
+type PubOperation string
+
+const (
+	// PubProtectOperationAnnotation indicates the pub protected Operation[DELETE,UPDATE,EVICT].
+	// if annotations[kruise.io/pub-protect-operations]=EVICT indicates the pub only protect evict pod.
+	// if the annotations do not exist, the default DELETE,EVICT,UPDATE are protected.
+	// RESIZE: Pod vertical scaling action. If it's enabled, all resize action will be protected. RESIZE
+	// is an extension of UPDATE, if RESIZE is disabled and UPDATE is enabled, any UPDATE operation will
+	// be protected only as it will definitely cause container restarts.
+	// UPDATE: Kruise will carefully differentiate whether this update will cause interruptions. When
+	// the FeatureGate InPlacePodVerticalScaling is enabled, pod inplace vertical scaling will be
+	// considered non-disruption only when allowedResources(cpu、memory) changes、restartPolicy
+	// is not restartContainer、is not static pod and QoS not changed. But if featureGate
+	// InPlacePodVerticalScaling is disabled, all resize action will be considered as disruption.
+	PubProtectOperationAnnotation = "kruise.io/pub-protect-operations"
+	// pod webhook operation
+	PubUpdateOperation PubOperation = "UPDATE"
+	PubDeleteOperation PubOperation = "DELETE"
+	PubEvictOperation  PubOperation = "EVICT"
+	PubResizeOperation PubOperation = "RESIZE"
+	// PubProtectTotalReplicasAnnotation is the target replicas.
+	// By default, PUB will get the target replicas through workload.spec.replicas. but there are some scenarios that may workload doesn't
+	// implement scale subresources or Pod doesn't have workload management. In this scenario, you can set pub.kruise.io/protect-total-replicas
+	// in pub annotations to get the target replicas to realize the same effect of protection ability.
+	PubProtectTotalReplicasAnnotation = "pub.kruise.io/protect-total-replicas"
+	// Marked the pod will not be pub-protected, solving the scenario of force pod deletion
+	PodPubNoProtectionAnnotation = "pub.kruise.io/no-protect"
+)
+
+// PodUnavailableBudgetSpec defines the desired state of PodUnavailableBudget
+type PodUnavailableBudgetSpec struct {
+	// Selector label query over pods managed by the budget
+	Selector *metav1.LabelSelector `json:"selector,omitempty"`
+
+	// TargetReference contains enough information to let you identify an workload for PodUnavailableBudget
+	// Selector and TargetReference are mutually exclusive, TargetReference is priority to take effect
+	TargetReference *TargetReference `json:"targetRef,omitempty"`
+
+	// Delete pod, evict pod or update pod specification is allowed if at most "maxUnavailable" pods selected by
+	// "selector" or "targetRef"  are unavailable after the above operation for pod.
+	// MaxUnavailable and MinAvailable are mutually exclusive, MaxUnavailable is priority to take effect
+	MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
+
+	// Delete pod, evict pod or update pod specification is allowed if at least "minAvailable" pods selected by
+	// "selector" or "targetRef" will still be available after the above operation for pod.
+	MinAvailable *intstr.IntOrString `json:"minAvailable,omitempty"`
+}
+
+// TargetReference contains enough information to let you identify an workload for PodUnavailableBudget
+type TargetReference struct {
+	// API version of the referent.
+	APIVersion string `json:"apiVersion,omitempty"`
+	// Kind of the referent.
+	Kind string `json:"kind,omitempty"`
+	// Name of the referent.
+	Name string `json:"name,omitempty"`
+}
+
+// PodUnavailableBudgetStatus defines the observed state of PodUnavailableBudget
+type PodUnavailableBudgetStatus struct {
+	// Most recent generation observed when updating this PUB status. UnavailableAllowed and other
+	// status information is valid only if observedGeneration equals to PUB's object generation.
+	// +optional
+	ObservedGeneration int64 `json:"observedGeneration"`
+
+	// DisruptedPods contains information about pods whose eviction or deletion was
+	// processed by the API handler but has not yet been observed by the PodUnavailableBudget.
+	// +optional
+	DisruptedPods map[string]metav1.Time `json:"disruptedPods,omitempty"`
+
+	// UnavailablePods contains information about pods whose specification changed(inplace-update pod),
+	// once pod is available(consistent and ready) again, it will be removed from the list.
+	// +optional
+	UnavailablePods map[string]metav1.Time `json:"unavailablePods,omitempty"`
+
+	// UnavailableAllowed number of pod unavailable that are currently allowed
+	UnavailableAllowed int32 `json:"unavailableAllowed"`
+
+	// CurrentAvailable current number of available pods
+	CurrentAvailable int32 `json:"currentAvailable"`
+
+	// DesiredAvailable minimum desired number of available pods
+	DesiredAvailable int32 `json:"desiredAvailable"`
+
+	// TotalReplicas total number of pods counted by this unavailable budget
+	TotalReplicas int32 `json:"totalReplicas"`
+}
+
+// +genclient
+// +kubebuilder:object:root=true
+// +kubebuilder:subresource:status
+// +kubebuilder:resource:shortName=pub
+// +kubebuilder:printcolumn:name="Allowed",type="integer",JSONPath=".status.unavailableAllowed",description="UnavailableAllowed number of pod unavailable that are currently allowed"
+// +kubebuilder:printcolumn:name="Current",type="integer",JSONPath=".status.currentAvailable",description="CurrentAvailable current number of available pods"
+// +kubebuilder:printcolumn:name="Desired",type="integer",JSONPath=".status.desiredAvailable",description="DesiredAvailable minimum desired number of available pods"
+// +kubebuilder:printcolumn:name="Total",type="integer",JSONPath=".status.totalReplicas",description="TotalReplicas total number of pods counted by this budget"
+
+// PodUnavailableBudget is the Schema for the podunavailablebudgets API
+type PodUnavailableBudget struct {
+	metav1.TypeMeta   `json:",inline"`
+	metav1.ObjectMeta `json:"metadata,omitempty"`
+
+	Spec   PodUnavailableBudgetSpec   `json:"spec,omitempty"`
+	Status PodUnavailableBudgetStatus `json:"status,omitempty"`
+}
+
+// +kubebuilder:object:root=true
+
+// PodUnavailableBudgetList contains a list of PodUnavailableBudget
+type PodUnavailableBudgetList struct {
+	metav1.TypeMeta `json:",inline"`
+	metav1.ListMeta `json:"metadata,omitempty"`
+	Items           []PodUnavailableBudget `json:"items"`
+}
+
+func init() {
+	SchemeBuilder.Register(&PodUnavailableBudget{}, &PodUnavailableBudgetList{})
+}