Update the process to build api-docs, generate CRD manifests and code (#2046)

* Update .gitignore Signed-off-by: Yi Chen <github@chenyicn.net> * Update .dockerignore Signed-off-by: Yi Chen <github@chenyicn.net> * Update Makefile Signed-off-by: Yi Chen <github@chenyicn.net> * Update the process to generate api docs Signed-off-by: Yi Chen <github@chenyicn.net> * Update the workflow to generate api docs Signed-off-by: Yi Chen <github@chenyicn.net> * Use controller-gen to generate CRD and deep copy related methods Signed-off-by: Yi Chen <github@chenyicn.net> * Update helm chart CRDs Signed-off-by: Yi Chen <github@chenyicn.net> * Update workflow for building spark operator Signed-off-by: Yi Chen <github@chenyicn.net> * Update README.md Signed-off-by: Yi Chen <github@chenyicn.net> --------- Signed-off-by: Yi Chen <github@chenyicn.net>
2024-07-22 13:10:00 +08:00 · 2024-07-22 13:10:00 +08:00 · 779ea3debc
parent 8894a4fedc
commit 779ea3debc
29 changed files with 46645 additions and 18090 deletions
--- a/.dockerignore
+++ b/.dockerignore
@ -1 +1,31 @@
-vendor
+.github/
+.idea/
+.vscode/
+bin/
+charts/
+docs/
+config/
+examples/
+hack/
+manifest/
+spark-docker/
+sparkctl/
+test/
+vendor/
+.dockerignore
+.DS_Store
+.gitignore
+.gitlab-ci.yaml
+.golangci.yaml
+.pre-commit-config.yaml
+ADOPTERS.md
+CODE_OF_CONDUCT.md
+codecov.ymal
+CONTRIBUTING.md
+cover.out
+Dockerfile
+LICENSE
+OWNERS
+PROJECT
+README.md
+test.sh
--- a/.github/workflows/main.yaml
+++ b/.github/workflows/main.yaml
@ -18,6 +18,11 @@ jobs:
        with:
          fetch-depth: "0"

+      - name: Set up Go
+        uses: actions/setup-go@v5
+        with:
+          go-version-file: go.mod
+
      - name: The API documentation hasn't changed
        run: |
          make build-api-docs
@ -42,7 +47,7 @@ jobs:

      - name: build sparkctl
        run: |
-          make all
+          make build-sparkctl

  build-spark-operator:
    runs-on: ubuntu-latest
@ -57,18 +62,17 @@ jobs:
        with:
          go-version-file: "go.mod"

-      - name: Run gofmt check
-        run: make fmt-check
+      - name: Run go fmt check
+        run: make go-fmt

-      - name: Run static analysis
-        run: make static-analysis
+      - name: Run go vet
+        run: make go-vet

      - name: Run unit tests
        run: make unit-test

      - name: Build Spark-Operator Docker Image
-        run: |
-          docker build -t docker.io/kubeflow/spark-operator:latest .
+        run: make docker-build IMAGE_TAG=latest

      - name: Check changes in resources used in docker file
        run: |
--- a/.gitignore
+++ b/.gitignore
@ -1,9 +1,11 @@
-.vscode/
+bin/
 vendor/
-spark-operator
-.idea/
-**/*.iml
+cover.out
 sparkctl/sparkctl
-spark-on-k8s-operator
 sparkctl/sparkctl-linux-amd64
 sparkctl/sparkctl-darwin-amd64
+**/*.iml
+
+# Various IDEs
+.idea/
+.vscode/
--- a/307
+++ b/307
@ -1,16 +1,116 @@
-
 .SILENT:
-.PHONY: clean-sparkctl

+# Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set)
+ifeq (,$(shell go env GOBIN))
+GOBIN=$(shell go env GOPATH)/bin
+else
+GOBIN=$(shell go env GOBIN)
+endif
+
+# Setting SHELL to bash allows bash commands to be executed by recipes.
+# Options are set to exit when a recipe line exits non-zero or a piped command fails.
+SHELL = /usr/bin/env bash -o pipefail
+.SHELLFLAGS = -ec
+
+REPO=github.com/kubeflow/spark-operator
 SPARK_OPERATOR_GOPATH=/go/src/github.com/kubeflow/spark-operator
+SPARK_OPERATOR_CHART_PATH=charts/spark-operator-chart
+OPERATOR_VERSION ?= $$(grep appVersion $(SPARK_OPERATOR_CHART_PATH)/Chart.yaml | awk '{print $$2}')
 DEP_VERSION:=`grep DEP_VERSION= Dockerfile | awk -F\" '{print $$2}'`
 BUILDER=`grep "FROM golang:" Dockerfile | awk '{print $$2}'`
 UNAME:=`uname | tr '[:upper:]' '[:lower:]'`
-REPO=github.com/kubeflow/spark-operator

-all: clean-sparkctl build-sparkctl install-sparkctl
+# CONTAINER_TOOL defines the container tool to be used for building images.
+# Be aware that the target commands are only tested with Docker which is
+# scaffolded by default. However, you might want to replace it to use other
+# tools. (i.e. podman)
+CONTAINER_TOOL ?= docker

-build-sparkctl:
+# Image URL to use all building/pushing image targets
+IMAGE_REPOSITORY ?= docker.io/kubeflow/spark-operator
+IMAGE_TAG ?= $(OPERATOR_VERSION)
+OPERATOR_IMAGE ?= $(IMAGE_REPOSITORY):$(IMAGE_TAG)
+
+##@ General
+
+# The help target prints out all targets with their descriptions organized
+# beneath their categories. The categories are represented by '##@' and the
+# target descriptions by '##'. The awk command is responsible for reading the
+# entire set of makefiles included in this invocation, looking for lines of the
+# file as xyz: ## something, and then pretty-format the target and help. Then,
+# if there's a line with ##@ something, that gets pretty-printed as a category.
+# More info on the usage of ANSI control characters for terminal formatting:
+# https://en.wikipedia.org/wiki/ANSI_escape_code#SGR_parameters
+# More info on the awk command:
+# http://linuxcommand.org/lc3_adv_awk.php
+
+.PHONY: help
+help: ## Display this help.
+	@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n  make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf "  \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
+
+##@ Development
+
+.PHONY: manifests
+manifests: controller-gen ## Generate CustomResourceDefinition, RBAC and WebhookConfiguration manifests.
+	$(CONTROLLER_GEN) crd rbac:roleName=spark-operator-controller webhook paths="./..." output:crd:artifacts:config=config/crd/bases
+
+.PHONY: generate
+generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.
+	$(CONTROLLER_GEN) object:headerFile="hack/boilerplate.go.txt" paths="./..."
+
+.PHONY: update-crd
+update-crd: manifests ## Update CRD files in the Helm chart.
+	cp config/crd/bases/* charts/spark-operator-chart/crds/
+
+.PHONY: clean
+clean: ## Clean up caches and output.
+	@echo "cleaning up caches and output"
+	go clean -cache -testcache -r -x 2>&1 >/dev/null
+	-rm -rf _output
+
+.PHONY: go-fmt
+go-fmt: ## Run go fmt against code.
+	@echo "Running go fmt..."
+	if [ -n "$(shell go fmt ./...)" ]; then \
+		echo "Go code is not formatted, need to run \"make go-fmt\" and commit the changes."; \
+		false; \
+	else \
+	    echo "Go code is formatted."; \
+	fi
+
+.PHONY: go-vet
+go-vet: ## Run go vet against code.
+	@echo "Running go vet..."
+	go vet ./...
+
+.PHONY: lint
+lint: golangci-lint ## Run golangci-lint linter.
+	@echo "Running golangci-lint run..."
+	$(GOLANGCI_LINT) run
+
+.PHONY: lint-fix
+lint-fix: golangci-lint ## Run golangci-lint linter and perform fixes.
+	@echo "Running golangci-lint run --fix..."
+	$(GOLANGCI_LINT) run --fix
+
+.PHONY: unit-test
+unit-test: clean ## Run go unit tests.
+	@echo "running unit tests"
+	go test $(shell go list ./... | grep -v /e2e) -coverprofile cover.out
+
+.PHONY: e2e-test
+e2e-test: clean ## Run go integration tests.
+	@echo "running integration tests"
+	go test -v ./test/e2e/ --kubeconfig "$(HOME)/.kube/config" --operator-image=docker.io/spark-operator/spark-operator:local
+
+##@ Build
+
+.PHONY: build-operator
+build-operator: ## Build spark-operator binary.
+	go build -o bin/spark-operator main.go
+
+.PHONY: build-sparkctl
+build-sparkctl: ## Build sparkctl binary.
 	[ ! -f "sparkctl/sparkctl-darwin-amd64" ] || [ ! -f "sparkctl/sparkctl-linux-amd64" ] && \
 	echo building using $(BUILDER) && \
 	docker run -w $(SPARK_OPERATOR_GOPATH) \
@ -19,10 +119,8 @@ build-sparkctl:
 	cd sparkctl && \
 	./build.sh" || true

-clean-sparkctl:
-	rm -f sparkctl/sparkctl-darwin-amd64 sparkctl/sparkctl-linux-amd64
-
-install-sparkctl: | sparkctl/sparkctl-darwin-amd64 sparkctl/sparkctl-linux-amd64
+.PHONY: install-sparkctl
+install-sparkctl: | sparkctl/sparkctl-darwin-amd64 sparkctl/sparkctl-linux-amd64 ## Install sparkctl binary.
 	@if [ "$(UNAME)" = "linux" ]; then \
 		echo "installing linux binary to /usr/local/bin/sparkctl"; \
 		sudo cp sparkctl/sparkctl-linux-amd64 /usr/local/bin/sparkctl; \
@ -35,52 +133,161 @@ install-sparkctl: | sparkctl/sparkctl-darwin-amd64 sparkctl/sparkctl-linux-amd64
 		echo "$(UNAME) not supported"; \
 	fi

-build-api-docs:
-	docker build -t temp-api-ref-docs hack/api-docs
-	docker run -v $$(pwd):/repo/ temp-api-ref-docs \
-		sh -c "cd /repo/ && /go/bin/gen-crd-api-reference-docs \
-			-config /repo/hack/api-docs/api-docs-config.json \
-			-api-dir github.com/kubeflow/spark-operator/pkg/apis/sparkoperator.k8s.io/v1beta2 \
-			-template-dir /repo/hack/api-docs/api-docs-template \
-			-out-file /repo/docs/api-docs.md"
+.PHONY: clean-sparkctl
+clean-sparkctl: ## Clean sparkctl binary.
+	rm -f sparkctl/sparkctl-darwin-amd64 sparkctl/sparkctl-linux-amd64

-helm-unittest:
+.PHONY: build-api-docs
+build-api-docs: gen-crd-api-reference-docs ## Build api documentaion.
+	$(GEN_CRD_API_REFERENCE_DOCS) \
+	-config hack/api-docs/config.json \
+	-api-dir github.com/kubeflow/spark-operator/pkg/apis/sparkoperator.k8s.io/v1beta2 \
+	-template-dir hack/api-docs/template \
+	-out-file docs/api-docs.md
+
+# If you wish to build the operator image targeting other platforms you can use the --platform flag.
+# (i.e. docker build --platform linux/arm64). However, you must enable docker buildKit for it.
+# More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+.PHONY: docker-build
+docker-build: ## Build docker image with the operator.
+	$(CONTAINER_TOOL) build -t ${IMAGE_REPOSITORY}:${IMAGE_TAG} .
+
+.PHONY: docker-push
+docker-push: ## Push docker image with the operator.
+	$(CONTAINER_TOOL) push ${IMAGE_REPOSITORY}:${IMAGE_TAG}
+
+# PLATFORMS defines the target platforms for the operator image be built to provide support to multiple
+# architectures. (i.e. make docker-buildx IMG=myregistry/mypoperator:0.0.1). To use this option you need to:
+# - be able to use docker buildx. More info: https://docs.docker.com/build/buildx/
+# - have enabled BuildKit. More info: https://docs.docker.com/develop/develop-images/build_enhancements/
+# - be able to push the image to your registry (i.e. if you do not set a valid value via IMG=<myregistry/image:<tag>> then the export will fail)
+# To adequately provide solutions that are compatible with multiple platforms, you should consider using this option.
+PLATFORMS ?= linux/amd64,linux/arm64
+.PHONY: docker-buildx
+docker-buildx: ## Build and push docker image for the operator for cross-platform support.
+	# copy existing Dockerfile and insert --platform=${BUILDPLATFORM} into Dockerfile.cross, and preserve the original Dockerfile
+	sed -e '1 s/\(^FROM\)/FROM --platform=\$$\{BUILDPLATFORM\}/; t' -e ' 1,// s//FROM --platform=\$$\{BUILDPLATFORM\}/' Dockerfile > Dockerfile.cross
+	- $(CONTAINER_TOOL) buildx create --name spark-operator-builder
+	$(CONTAINER_TOOL) buildx use spark-operator-builder
+	- $(CONTAINER_TOOL) buildx build --push --platform=$(PLATFORMS) --tag ${IMAGE_REPOSITORY}:${IMAGE_TAG} -f Dockerfile.cross .
+	- $(CONTAINER_TOOL) buildx rm spark-operator-builder
+	rm Dockerfile.cross
+
+##@ Helm
+
+.PHONY: detect-crds-drift
+detect-crds-drift:
+	diff -q charts/spark-operator-chart/crds config/crd/bases
+
+.PHONY: helm-unittest
+helm-unittest: helm-unittest-plugin ## Run Helm chart unittests.
 	helm unittest charts/spark-operator-chart --strict --file "tests/**/*_test.yaml"

-helm-lint:
-	docker run --rm --workdir /workspace --volume "$$(pwd):/workspace" quay.io/helmpack/chart-testing:latest ct lint
+.PHONY: helm-lint
+helm-lint: ## Run Helm chart lint test.
+	docker run --rm --workdir /workspace --volume "$$(pwd):/workspace" quay.io/helmpack/chart-testing:latest ct lint --target-branch master

-helm-docs:
+.PHONY: helm-docs
+helm-docs: ## Generates markdown documentation for helm charts from requirements and values files.
 	docker run --rm --volume "$$(pwd):/helm-docs" -u "$(id -u)" jnorwood/helm-docs:latest

-fmt-check: clean
-	@echo "running fmt check"; cd "$(dirname $0)"; \
-	if [ -n "$(go fmt ./...)" ]; \
-	then \
-		echo "Go code is not formatted, please run 'go fmt ./...'." >&2; \
-		exit 1; \
-	else \
-		echo "Go code is formatted"; \
+##@ Deployment
+
+ifndef ignore-not-found
+  ignore-not-found = false
+endif
+
+.PHONY: install-crds
+install-crds: manifests kustomize ## Install CRDs into the K8s cluster specified in ~/.kube/config.
+	$(KUSTOMIZE) build config/crd | $(KUBECTL) create -f -
+
+.PHONY: uninstall-crds
+uninstall-crds: manifests kustomize ## Uninstall CRDs from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	$(KUSTOMIZE) build config/crd | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
+
+.PHONY: deploy
+deploy: manifests kustomize ## Deploy controller to the K8s cluster specified in ~/.kube/config.
+	cd config/manager && $(KUSTOMIZE) edit set image controller=${IMG}
+	$(KUSTOMIZE) build config/default | $(KUBECTL) apply -f -
+
+.PHONY: undeploy
+undeploy: kustomize ## Undeploy controller from the K8s cluster specified in ~/.kube/config. Call with ignore-not-found=true to ignore resource not found errors during deletion.
+	$(KUSTOMIZE) build config/default | $(KUBECTL) delete --ignore-not-found=$(ignore-not-found) -f -
+
+##@ Dependencies
+
+## Location to install dependencies to
+LOCALBIN ?= $(shell pwd)/bin
+$(LOCALBIN):
+	mkdir -p $(LOCALBIN)
+
+## Tool Binaries
+KUBECTL ?= kubectl
+KUSTOMIZE ?= $(LOCALBIN)/kustomize-$(KUSTOMIZE_VERSION)
+CONTROLLER_GEN ?= $(LOCALBIN)/controller-gen-$(CONTROLLER_TOOLS_VERSION)
+KIND ?= $(LOCALBIN)/kind-$(KIND_VERSION)
+ENVTEST ?= $(LOCALBIN)/setup-envtest-$(ENVTEST_VERSION)
+GOLANGCI_LINT = $(LOCALBIN)/golangci-lint-$(GOLANGCI_LINT_VERSION)
+GEN_CRD_API_REFERENCE_DOCS ?= $(LOCALBIN)/gen-crd-api-reference-docs-$(GEN_CRD_API_REFERENCE_DOCS_VERSION)
+HELM ?= helm
+HELM_UNITTEST ?= unittest
+
+## Tool Versions
+KUSTOMIZE_VERSION ?= v5.4.1
+CONTROLLER_TOOLS_VERSION ?= v0.15.0
+KIND_VERSION ?= v0.23.0
+ENVTEST_VERSION ?= release-0.18
+GOLANGCI_LINT_VERSION ?= v1.57.2
+GEN_CRD_API_REFERENCE_DOCS_VERSION ?= v0.3.0
+HELM_UNITTEST_VERSION ?= 0.5.1
+
+.PHONY: kustomize
+kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
+$(KUSTOMIZE): $(LOCALBIN)
+	$(call go-install-tool,$(KUSTOMIZE),sigs.k8s.io/kustomize/kustomize/v5,$(KUSTOMIZE_VERSION))
+
+.PHONY: controller-gen
+controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
+$(CONTROLLER_GEN): $(LOCALBIN)
+	$(call go-install-tool,$(CONTROLLER_GEN),sigs.k8s.io/controller-tools/cmd/controller-gen,$(CONTROLLER_TOOLS_VERSION))
+
+.PHONY: kind
+kind: $(KIND) ## Download kind locally if necessary.
+$(KIND): $(LOCALBIN)
+	$(call go-install-tool,$(KIND),sigs.k8s.io/kind,$(KIND_VERSION))
+
+.PHONY: envtest
+envtest: $(ENVTEST) ## Download setup-envtest locally if necessary.
+$(ENVTEST): $(LOCALBIN)
+	$(call go-install-tool,$(ENVTEST),sigs.k8s.io/controller-runtime/tools/setup-envtest,$(ENVTEST_VERSION))
+
+.PHONY: golangci-lint
+golangci-lint: $(GOLANGCI_LINT) ## Download golangci-lint locally if necessary.
+$(GOLANGCI_LINT): $(LOCALBIN)
+	$(call go-install-tool,$(GOLANGCI_LINT),github.com/golangci/golangci-lint/cmd/golangci-lint,${GOLANGCI_LINT_VERSION})
+
+.PHONY: gen-crd-api-reference-docs
+gen-crd-api-reference-docs: $(GEN_CRD_API_REFERENCE_DOCS) ## Download gen-crd-api-reference-docs locally if necessary.
+$(GEN_CRD_API_REFERENCE_DOCS): $(LOCALBIN)
+	$(call go-install-tool,$(GEN_CRD_API_REFERENCE_DOCS),github.com/ahmetb/gen-crd-api-reference-docs,$(GEN_CRD_API_REFERENCE_DOCS_VERSION))
+
+.PHONY: helm-unittest-plugin
+helm-unittest-plugin: ## Download helm unittest plugin locally if necessary.
+	if [ -z "$(shell helm plugin list | grep unittest)" ]; then \
+		echo "Installing helm unittest plugin..."; \
+		helm plugin install https://github.com/helm-unittest/helm-unittest.git --version $(HELM_UNITTEST_VERSION); \
 	fi

-detect-crds-drift:
-	diff -q charts/spark-operator-chart/crds manifest/crds --exclude=kustomization.yaml
-
-clean:
-	@echo "cleaning up caches and output"
-	go clean -cache -testcache -r -x 2>&1 >/dev/null
-	-rm -rf _output
-
-unit-test: clean
-	@echo "running unit tests"
-	go test -v ./... -covermode=atomic
-
-integration-test: clean
-	@echo "running integration tests"
-	go test -v ./test/e2e/ --kubeconfig "$(HOME)/.kube/config" --operator-image=gcr.io/spark-operator/spark-operator:local
-
-static-analysis:
-	@echo "running go vet"
-	# echo "Building using $(BUILDER)"
-	# go vet ./...
-	go vet $(REPO)...
+# go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
+# $1 - target path with name of binary (ideally with version)
+# $2 - package url which can be installed
+# $3 - specific version of package
+define go-install-tool
+@[ -f $(1) ] || { \
+set -e; \
+package=$(2)@$(3) ;\
+echo "Downloading $${package}" ;\
+GOBIN=$(LOCALBIN) go install $${package} ;\
+mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\
+}
+endef
--- a/README.md
+++ b/README.md
@ -31,7 +31,7 @@ The Kubernetes Operator for Apache Spark currently supports the following list o

 **Current API version:** *`v1beta2`*

-**If you are currently using the `v1beta1` version of the APIs in your manifests, please update them to use the `v1beta2` version by changing `apiVersion: "sparkoperator.k8s.io/<version>"` to `apiVersion: "sparkoperator.k8s.io/v1beta2"`. You will also need to delete the `previous` version of the CustomResourceDefinitions named `sparkapplications.sparkoperator.k8s.io` and `scheduledsparkapplications.sparkoperator.k8s.io`, and replace them with the `v1beta2` version either by installing the latest version of the operator or by running `kubectl create -f manifest/crds`.**
+**If you are currently using the `v1beta1` version of the APIs in your manifests, please update them to use the `v1beta2` version by changing `apiVersion: "sparkoperator.k8s.io/<version>"` to `apiVersion: "sparkoperator.k8s.io/v1beta2"`. You will also need to delete the `previous` version of the CustomResourceDefinitions named `sparkapplications.sparkoperator.k8s.io` and `scheduledsparkapplications.sparkoperator.k8s.io`, and replace them with the `v1beta2` version either by installing the latest version of the operator or by running `kubectl create -f config/crd/bases`.**

 ## Prerequisites

--- a/charts/spark-operator-chart/Chart.yaml
+++ b/charts/spark-operator-chart/Chart.yaml
@ -1,8 +1,8 @@
 apiVersion: v2
 name: spark-operator
 description: A Helm chart for Spark on Kubernetes operator
-version: 1.4.4
-appVersion: v1beta2-1.6.1-3.5.0
+version: 1.4.5
+appVersion: v1beta2-1.6.2-3.5.0
 keywords:
  - spark
 home: https://github.com/kubeflow/spark-operator
--- a/charts/spark-operator-chart/README.md
+++ b/charts/spark-operator-chart/README.md
@ -1,6 +1,6 @@
 # spark-operator

-![Version: 1.4.4](https://img.shields.io/badge/Version-1.4.4-informational?style=flat-square) ![AppVersion: v1beta2-1.6.1-3.5.0](https://img.shields.io/badge/AppVersion-v1beta2--1.6.1--3.5.0-informational?style=flat-square)
+![Version: 1.4.5](https://img.shields.io/badge/Version-1.4.5-informational?style=flat-square) ![AppVersion: v1beta2-1.6.2-3.5.0](https://img.shields.io/badge/AppVersion-v1beta2--1.6.2--3.5.0-informational?style=flat-square)

 A Helm chart for Spark on Kubernetes operator

--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
--- a/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
+++ b/charts/spark-operator-chart/crds/sparkoperator.k8s.io_sparkapplications.yaml
--- a/config/crd/bases/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+++ b/config/crd/bases/sparkoperator.k8s.io_scheduledsparkapplications.yaml
--- a/config/crd/bases/sparkoperator.k8s.io_sparkapplications.yaml
+++ b/config/crd/bases/sparkoperator.k8s.io_sparkapplications.yaml
--- a/config/crd/kustomization.yaml
+++ b/config/crd/kustomization.yaml
@ -0,0 +1,24 @@
+# This kustomization.yaml is not intended to be run by itself,
+# since it depends on service name and namespace that are out of this kustomize package.
+# It should be run by config/default
+resources:
+- bases/sparkoperator.k8s.io_sparkapplications.yaml
+- bases/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+# +kubebuilder:scaffold:crdkustomizeresource
+
+patches:
+# [WEBHOOK] To enable webhook, uncomment all the sections with [WEBHOOK] prefix.
+# patches here are for enabling the conversion webhook for each CRD
+# +kubebuilder:scaffold:crdkustomizewebhookpatch
+
+# [CERTMANAGER] To enable cert-manager, uncomment all the sections with [CERTMANAGER] prefix.
+# patches here are for enabling the CA injection for each CRD
+#- path: patches/cainjection_in_sparkapplications.yaml
+#- path: patches/cainjection_in_scheduledsparkapplications.yaml
+# +kubebuilder:scaffold:crdkustomizecainjectionpatch
+
+# [WEBHOOK] To enable webhook, uncomment the following section
+# the following config is for teaching kustomize how to do kustomization for CRDs.
+
+#configurations:
+#- kustomizeconfig.yaml
--- a/config/crd/kustomizeconfig.yaml
+++ b/config/crd/kustomizeconfig.yaml
@ -0,0 +1,19 @@
+# This file is for teaching kustomize how to substitute name and namespace reference in CRD
+nameReference:
+- kind: Service
+  version: v1
+  fieldSpecs:
+  - kind: CustomResourceDefinition
+    version: v1
+    group: apiextensions.k8s.io
+    path: spec/conversion/webhook/clientConfig/service/name
+
+namespace:
+- kind: CustomResourceDefinition
+  version: v1
+  group: apiextensions.k8s.io
+  path: spec/conversion/webhook/clientConfig/service/namespace
+  create: false
+
+varReference:
+- path: metadata/annotations
--- a/docs/api-docs.md
+++ b/docs/api-docs.md
@ -47,7 +47,7 @@ string
 <td>
 <code>metadata</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#objectmeta-v1-meta">
 Kubernetes meta/v1.ObjectMeta
 </a>
 </em>
@ -197,7 +197,7 @@ string
 <td>
 <code>metadata</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#objectmeta-v1-meta">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#objectmeta-v1-meta">
 Kubernetes meta/v1.ObjectMeta
 </a>
 </em>
@ -401,7 +401,7 @@ The controller will add environment variable HADOOP_CONF_DIR to the path where t
 <td>
 <code>volumes</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volume-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#volume-v1-core">
 []Kubernetes core/v1.Volume
 </a>
 </em>
@ -768,7 +768,7 @@ string
 <td>
 <code>resources</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#resourcelist-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#resourcelist-v1-core">
 Kubernetes core/v1.ResourceList
 </a>
 </em>
@ -1048,7 +1048,7 @@ This may be useful for sidecar proxies like Envoy injected by Istio which requir
 <td>
 <code>serviceType</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#servicetype-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#servicetype-v1-core">
 Kubernetes core/v1.ServiceType
 </a>
 </em>
@ -1109,7 +1109,7 @@ map[string]string
 <td>
 <code>ingressTLS</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#ingresstls-v1-networking">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#ingresstls-v1-networking">
 []Kubernetes networking/v1.IngressTLS
 </a>
 </em>
@ -1197,7 +1197,7 @@ GC settings or other logging.</p>
 <td>
 <code>lifecycle</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#lifecycle-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#lifecycle-v1-core">
 Kubernetes core/v1.Lifecycle
 </a>
 </em>
@ -1438,7 +1438,7 @@ GC settings or other logging.</p>
 <td>
 <code>lifecycle</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#lifecycle-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#lifecycle-v1-core">
 Kubernetes core/v1.Lifecycle
 </a>
 </em>
@ -2066,7 +2066,7 @@ Defaults to 1.</p>
 <td>
 <code>lastRun</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#time-v1-meta">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#time-v1-meta">
 Kubernetes meta/v1.Time
 </a>
 </em>
@ -2079,7 +2079,7 @@ Kubernetes meta/v1.Time
 <td>
 <code>nextRun</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#time-v1-meta">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#time-v1-meta">
 Kubernetes meta/v1.Time
 </a>
 </em>
@ -2423,7 +2423,7 @@ The controller will add environment variable HADOOP_CONF_DIR to the path where t
 <td>
 <code>volumes</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volume-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#volume-v1-core">
 []Kubernetes core/v1.Volume
 </a>
 </em>
@ -2694,7 +2694,7 @@ string
 <td>
 <code>lastSubmissionAttemptTime</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#time-v1-meta">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#time-v1-meta">
 Kubernetes meta/v1.Time
 </a>
 </em>
@ -2707,7 +2707,7 @@ Kubernetes meta/v1.Time
 <td>
 <code>terminationTime</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#time-v1-meta">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#time-v1-meta">
 Kubernetes meta/v1.Time
 </a>
 </em>
@ -2929,7 +2929,7 @@ string
 <td>
 <code>env</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#envvar-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#envvar-v1-core">
 []Kubernetes core/v1.EnvVar
 </a>
 </em>
@ -2956,7 +2956,7 @@ Deprecated. Consider using <code>env</code> instead.</p>
 <td>
 <code>envFrom</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#envfromsource-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#envfromsource-v1-core">
 []Kubernetes core/v1.EnvFromSource
 </a>
 </em>
@ -3009,7 +3009,7 @@ map[string]string
 <td>
 <code>volumeMounts</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#volumemount-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#volumemount-v1-core">
 []Kubernetes core/v1.VolumeMount
 </a>
 </em>
@ -3023,7 +3023,7 @@ map[string]string
 <td>
 <code>affinity</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#affinity-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#affinity-v1-core">
 Kubernetes core/v1.Affinity
 </a>
 </em>
@ -3037,7 +3037,7 @@ Kubernetes core/v1.Affinity
 <td>
 <code>tolerations</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#toleration-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#toleration-v1-core">
 []Kubernetes core/v1.Toleration
 </a>
 </em>
@ -3051,7 +3051,7 @@ Kubernetes core/v1.Affinity
 <td>
 <code>podSecurityContext</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#podsecuritycontext-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#podsecuritycontext-v1-core">
 Kubernetes core/v1.PodSecurityContext
 </a>
 </em>
@ -3065,7 +3065,7 @@ Kubernetes core/v1.PodSecurityContext
 <td>
 <code>securityContext</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#securitycontext-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#securitycontext-v1-core">
 Kubernetes core/v1.SecurityContext
 </a>
 </em>
@ -3091,7 +3091,7 @@ string
 <td>
 <code>sidecars</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core">
 []Kubernetes core/v1.Container
 </a>
 </em>
@ -3105,7 +3105,7 @@ string
 <td>
 <code>initContainers</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#container-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#container-v1-core">
 []Kubernetes core/v1.Container
 </a>
 </em>
@ -3144,7 +3144,7 @@ This field is mutually exclusive with nodeSelector at SparkApplication level (wh
 <td>
 <code>dnsConfig</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#poddnsconfig-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#poddnsconfig-v1-core">
 Kubernetes core/v1.PodDNSConfig
 </a>
 </em>
@ -3182,7 +3182,7 @@ string
 <td>
 <code>hostAliases</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#hostalias-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#hostalias-v1-core">
 []Kubernetes core/v1.HostAlias
 </a>
 </em>
@ -3253,7 +3253,7 @@ Defaults to spark-driver-ui-port.</p>
 <td>
 <code>serviceType</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#servicetype-v1-core">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#servicetype-v1-core">
 Kubernetes core/v1.ServiceType
 </a>
 </em>
@ -3303,7 +3303,7 @@ map[string]string
 <td>
 <code>ingressTLS</code><br/>
 <em>
-<a href="https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#ingresstls-v1-networking">
+<a href="https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#ingresstls-v1-networking">
 []Kubernetes networking/v1.IngressTLS
 </a>
 </em>
@ -3317,5 +3317,5 @@ map[string]string
 </table>
 <hr/>
 <p><em>
-Generated with <code>https://github.com/ahmetb/gen-crd-api-reference-docs.git</code> on git commit <code>ccf856504caaeac38151b57a950d3f8a7942b9db</code>.
+Generated with <code>gen-crd-api-reference-docs</code>.
 </em></p>
--- a/hack/api-docs/Dockerfile
+++ b/hack/api-docs/Dockerfile
@ -1,18 +0,0 @@
-#
-# Copyright 2021 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-FROM golang:1.22.2-alpine
-RUN go install github.com/ahmetb/gen-crd-api-reference-docs@latest
--- a/hack/api-docs/api-docs-config.json
+++ b/hack/api-docs/api-docs-config.json
@ -1,28 +0,0 @@
-{
-    "hideMemberFields": [
-        "TypeMeta"
-    ],
-    "hideTypePatterns": [
-        "ParseError$",
-        "List$"
-    ],
-    "externalPackages": [
-        {
-            "typeMatchPrefix": "^k8s\\.io/apimachinery/pkg/apis/meta/v1\\.Duration$",
-            "docsURLTemplate": "https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Duration"
-        },
-        {
-            "typeMatchPrefix": "^k8s\\.io/(api|apimachinery/pkg/apis)/",
-            "docsURLTemplate": "https://v1-18.docs.kubernetes.io/docs/reference/generated/kubernetes-api/v1.18/#{{lower .TypeIdentifier}}-{{arrIndex .PackageSegments -1}}-{{arrIndex .PackageSegments -2}}"
-        },
-        {
-            "typeMatchPrefix": "^github\\.com/knative/pkg/apis/duck/",
-            "docsURLTemplate": "https://pkg.go.dev/github.com/knative/pkg/apis/duck/{{arrIndex .PackageSegments -1}}#{{.TypeIdentifier}}"
-        }
-    ],
-    "typeDisplayNamePrefixOverrides": {
-        "k8s.io/api/": "Kubernetes ",
-        "k8s.io/apimachinery/pkg/apis/": "Kubernetes "
-    },
-    "markdownDisabled": false
-}
--- a/hack/api-docs/config.json
+++ b/hack/api-docs/config.json
@ -0,0 +1,28 @@
+{
+  "hideMemberFields": [
+    "TypeMeta"
+  ],
+  "hideTypePatterns": [
+    "ParseError$",
+    "List$"
+  ],
+  "externalPackages": [
+    {
+      "typeMatchPrefix": "^k8s\\.io/apimachinery/pkg/apis/meta/v1\\.Duration$",
+      "docsURLTemplate": "https://pkg.go.dev/k8s.io/apimachinery/pkg/apis/meta/v1#Duration"
+    },
+    {
+      "typeMatchPrefix": "^k8s\\.io/(api|apimachinery/pkg/apis)/",
+      "docsURLTemplate": "https://kubernetes.io/docs/reference/generated/kubernetes-api/v1.24/#{{lower .TypeIdentifier}}-{{arrIndex .PackageSegments -1}}-{{arrIndex .PackageSegments -2}}"
+    },
+    {
+      "typeMatchPrefix": "^github\\.com/knative/pkg/apis/duck/",
+      "docsURLTemplate": "https://pkg.go.dev/github.com/knative/pkg/apis/duck/{{arrIndex .PackageSegments -1}}#{{.TypeIdentifier}}"
+    }
+  ],
+  "typeDisplayNamePrefixOverrides": {
+    "k8s.io/api/": "Kubernetes ",
+    "k8s.io/apimachinery/pkg/apis/": "Kubernetes "
+  },
+  "markdownDisabled": false
+}
--- a/hack/api-docs/api-docs-template/members.tpl
+++ b/hack/api-docs/api-docs-template/members.tpl
--- a/hack/api-docs/api-docs-template/pkg.tpl
+++ b/hack/api-docs/api-docs-template/pkg.tpl
@ -42,7 +42,7 @@
 {{ end }}

 <p><em>
-    Generated with <code>https://github.com/ahmetb/gen-crd-api-reference-docs.git</code> on git commit <code>ccf856504caaeac38151b57a950d3f8a7942b9db</code>.
+    Generated with <code>gen-crd-api-reference-docs</code>.
 </em></p>

 {{ end }}
--- a/hack/api-docs/api-docs-template/placeholder.go
+++ b/hack/api-docs/api-docs-template/placeholder.go
--- a/hack/api-docs/api-docs-template/type.tpl
+++ b/hack/api-docs/api-docs-template/type.tpl
--- a/hack/custom-boilerplate.go.txt
+++ b/hack/custom-boilerplate.go.txt
@ -1,7 +1,5 @@
-// Code generated by k8s code-generator DO NOT EDIT.
-
 /*
-Copyright 2018 Google LLC
+Copyright 2024 The Kubeflow authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@ -14,4 +12,4 @@ distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 See the License for the specific language governing permissions and
 limitations under the License.
-*/
+*/
--- a/manifest/crds/kustomization.yaml
+++ b/manifest/crds/kustomization.yaml
@ -1,21 +0,0 @@
-#
-# Copyright 2018 Google LLC
-#
-# Licensed under the Apache License, Version 2.0 (the "License");
-# you may not use this file except in compliance with the License.
-# You may obtain a copy of the License at
-#
-#     https://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-apiVersion: kustomize.config.k8s.io/v1beta1
-kind: Kustomization
-resources:
-  - sparkoperator.k8s.io_sparkapplications.yaml
-  - sparkoperator.k8s.io_scheduledsparkapplications.yaml
--- a/manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
+++ b/manifest/crds/sparkoperator.k8s.io_scheduledsparkapplications.yaml
--- a/manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml
+++ b/manifest/crds/sparkoperator.k8s.io_sparkapplications.yaml
--- a/pkg/apis/sparkoperator.k8s.io/v1beta1/types.go
+++ b/pkg/apis/sparkoperator.k8s.io/v1beta1/types.go
@ -14,6 +14,8 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

+// +kubebuilder:skip
+
 package v1beta1

 import (
@ -409,7 +411,7 @@ type SparkPodSpec struct {

 // DriverSpec is specification of the driver.
 type DriverSpec struct {
-	SparkPodSpec
+	SparkPodSpec `json:",inline"`
 	// PodName is the name of the driver pod that the user creates. This is used for the
 	// in-cluster client mode in which the user creates a client pod where the driver of
 	// the user application runs. It's an error to set this field if Mode is not
@ -426,7 +428,7 @@ type DriverSpec struct {

 // ExecutorSpec is specification of the executor.
 type ExecutorSpec struct {
-	SparkPodSpec
+	SparkPodSpec `json:",inline"`
 	// Instances is the number of executor instances.
 	// Optional.
 	Instances *int32 `json:"instances,omitempty"`
--- a/pkg/apis/sparkoperator.k8s.io/v1beta1/zz_generated.deepcopy.go
+++ b/pkg/apis/sparkoperator.k8s.io/v1beta1/zz_generated.deepcopy.go
@ -1,10 +1,7 @@
 //go:build !ignore_autogenerated
-// +build !ignore_autogenerated
-
-// Code generated by k8s code-generator DO NOT EDIT.

 /*
-Copyright 2018 Google LLC
+Copyright 2024 The Kubeflow authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@ -19,19 +16,18 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-// Code generated by deepcopy-gen. DO NOT EDIT.
+// Code generated by controller-gen. DO NOT EDIT.

 package v1beta1

 import (
-	v1 "k8s.io/api/core/v1"
-	runtime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/runtime"
 )

 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ApplicationState) DeepCopyInto(out *ApplicationState) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplicationState.
@ -82,7 +78,6 @@ func (in *Dependencies) DeepCopyInto(out *Dependencies) {
 		*out = new(int32)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Dependencies.
@ -98,7 +93,6 @@ func (in *Dependencies) DeepCopy() *Dependencies {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DriverInfo) DeepCopyInto(out *DriverInfo) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DriverInfo.
@ -130,7 +124,6 @@ func (in *DriverSpec) DeepCopyInto(out *DriverSpec) {
 		*out = new(string)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DriverSpec.
@ -162,7 +155,6 @@ func (in *ExecutorSpec) DeepCopyInto(out *ExecutorSpec) {
 		*out = new(string)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecutorSpec.
@ -178,7 +170,6 @@ func (in *ExecutorSpec) DeepCopy() *ExecutorSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GPUSpec) DeepCopyInto(out *GPUSpec) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUSpec.
@ -204,7 +195,6 @@ func (in *MonitoringSpec) DeepCopyInto(out *MonitoringSpec) {
 		*out = new(PrometheusSpec)
 		(*in).DeepCopyInto(*out)
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MonitoringSpec.
@ -220,7 +210,6 @@ func (in *MonitoringSpec) DeepCopy() *MonitoringSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NameKey) DeepCopyInto(out *NameKey) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NameKey.
@ -236,7 +225,6 @@ func (in *NameKey) DeepCopy() *NameKey {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NamePath) DeepCopyInto(out *NamePath) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamePath.
@ -267,7 +255,6 @@ func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) {
 		*out = new(string)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusSpec.
@ -303,7 +290,6 @@ func (in *RestartPolicy) DeepCopyInto(out *RestartPolicy) {
 		*out = new(int64)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestartPolicy.
@ -323,7 +309,6 @@ func (in *ScheduledSparkApplication) DeepCopyInto(out *ScheduledSparkApplication
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
 	in.Spec.DeepCopyInto(&out.Spec)
 	in.Status.DeepCopyInto(&out.Status)
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplication.
@ -356,7 +341,6 @@ func (in *ScheduledSparkApplicationList) DeepCopyInto(out *ScheduledSparkApplica
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplicationList.
@ -396,7 +380,6 @@ func (in *ScheduledSparkApplicationSpec) DeepCopyInto(out *ScheduledSparkApplica
 		*out = new(int32)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplicationSpec.
@ -424,7 +407,6 @@ func (in *ScheduledSparkApplicationStatus) DeepCopyInto(out *ScheduledSparkAppli
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplicationStatus.
@ -440,7 +422,6 @@ func (in *ScheduledSparkApplicationStatus) DeepCopy() *ScheduledSparkApplication
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SecretInfo) DeepCopyInto(out *SecretInfo) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretInfo.
@ -460,7 +441,6 @@ func (in *SparkApplication) DeepCopyInto(out *SparkApplication) {
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
 	in.Spec.DeepCopyInto(&out.Spec)
 	in.Status.DeepCopyInto(&out.Status)
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkApplication.
@ -493,7 +473,6 @@ func (in *SparkApplicationList) DeepCopyInto(out *SparkApplicationList) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkApplicationList.
@ -624,7 +603,6 @@ func (in *SparkApplicationSpec) DeepCopyInto(out *SparkApplicationSpec) {
 		*out = new(string)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkApplicationSpec.
@ -651,7 +629,6 @@ func (in *SparkApplicationStatus) DeepCopyInto(out *SparkApplicationStatus) {
 			(*out)[key] = val
 		}
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkApplicationStatus.
@ -788,7 +765,6 @@ func (in *SparkPodSpec) DeepCopyInto(out *SparkPodSpec) {
 		*out = new(v1.PodDNSConfig)
 		(*in).DeepCopyInto(*out)
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkPodSpec.
--- a/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
+++ b/pkg/apis/sparkoperator.k8s.io/v1beta2/types.go
@ -84,8 +84,14 @@ const (
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +k8s:defaulter-gen=true
-// +kubebuilder:subresource:status
+// +kubebuilder:metadata:annotations="api-approved.kubernetes.io=https://github.com/kubeflow/spark-operator/pull/1298"
 // +kubebuilder:resource:scope=Namespaced,shortName=scheduledsparkapp,singular=scheduledsparkapplication
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:JSONPath=.spec.schedule,name=Schedule,type=string
+// +kubebuilder:printcolumn:JSONPath=.spec.suspend,name=Suspend,type=string
+// +kubebuilder:printcolumn:JSONPath=.status.lastRun,name=Last Run,type=date
+// +kubebuilder:printcolumn:JSONPath=.status.lastRunName,name=Last Run Name,type=string
+// +kubebuilder:printcolumn:JSONPath=.metadata.creationTimestamp,name=Age,type=date

 type ScheduledSparkApplication struct {
 	metav1.TypeMeta   `json:",inline"`
@ -165,8 +171,14 @@ type ScheduledSparkApplicationList struct {
 // +genclient
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
 // +k8s:defaulter-gen=true
-// +kubebuilder:subresource:status
+// +kubebuilder:metadata:annotations="api-approved.kubernetes.io=https://github.com/kubeflow/spark-operator/pull/1298"
 // +kubebuilder:resource:scope=Namespaced,shortName=sparkapp,singular=sparkapplication
+// +kubebuilder:subresource:status
+// +kubebuilder:printcolumn:JSONPath=.status.applicationState.state,name=Status,type=string
+// +kubebuilder:printcolumn:JSONPath=.status.executionAttempts,name=Attempts,type=string
+// +kubebuilder:printcolumn:JSONPath=.status.lastSubmissionAttemptTime,name=Start,type=string
+// +kubebuilder:printcolumn:JSONPath=.status.terminationTime,name=Finish,type=string
+// +kubebuilder:printcolumn:JSONPath=.metadata.creationTimestamp,name=Age,type=date

 // SparkApplication represents a Spark application running on and using Kubernetes as a cluster manager.
 type SparkApplication struct {
--- a/pkg/apis/sparkoperator.k8s.io/v1beta2/zz_generated.deepcopy.go
+++ b/pkg/apis/sparkoperator.k8s.io/v1beta2/zz_generated.deepcopy.go
@ -1,10 +1,7 @@
 //go:build !ignore_autogenerated
-// +build !ignore_autogenerated
-
-// Code generated by k8s code-generator DO NOT EDIT.

 /*
-Copyright 2018 Google LLC
+Copyright 2024 The Kubeflow authors.

 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
@ -19,20 +16,19 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */

-// Code generated by deepcopy-gen. DO NOT EDIT.
+// Code generated by controller-gen. DO NOT EDIT.

 package v1beta2

 import (
-	v1 "k8s.io/api/core/v1"
+	"k8s.io/api/core/v1"
 	networkingv1 "k8s.io/api/networking/v1"
-	runtime "k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/runtime"
 )

 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *ApplicationState) DeepCopyInto(out *ApplicationState) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ApplicationState.
@ -65,7 +61,6 @@ func (in *BatchSchedulerConfiguration) DeepCopyInto(out *BatchSchedulerConfigura
 			(*out)[key] = val.DeepCopy()
 		}
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new BatchSchedulerConfiguration.
@ -111,7 +106,6 @@ func (in *Dependencies) DeepCopyInto(out *Dependencies) {
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Dependencies.
@ -127,7 +121,6 @@ func (in *Dependencies) DeepCopy() *Dependencies {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DriverInfo) DeepCopyInto(out *DriverInfo) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DriverInfo.
@ -140,6 +133,64 @@ func (in *DriverInfo) DeepCopy() *DriverInfo {
 	return out
 }

+// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
+func (in *DriverIngressConfiguration) DeepCopyInto(out *DriverIngressConfiguration) {
+	*out = *in
+	if in.ServicePort != nil {
+		in, out := &in.ServicePort, &out.ServicePort
+		*out = new(int32)
+		**out = **in
+	}
+	if in.ServicePortName != nil {
+		in, out := &in.ServicePortName, &out.ServicePortName
+		*out = new(string)
+		**out = **in
+	}
+	if in.ServiceType != nil {
+		in, out := &in.ServiceType, &out.ServiceType
+		*out = new(v1.ServiceType)
+		**out = **in
+	}
+	if in.ServiceAnnotations != nil {
+		in, out := &in.ServiceAnnotations, &out.ServiceAnnotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.ServiceLabels != nil {
+		in, out := &in.ServiceLabels, &out.ServiceLabels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.IngressAnnotations != nil {
+		in, out := &in.IngressAnnotations, &out.IngressAnnotations
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
+	if in.IngressTLS != nil {
+		in, out := &in.IngressTLS, &out.IngressTLS
+		*out = make([]networkingv1.IngressTLS, len(*in))
+		for i := range *in {
+			(*in)[i].DeepCopyInto(&(*out)[i])
+		}
+	}
+}
+
+// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DriverIngressConfiguration.
+func (in *DriverIngressConfiguration) DeepCopy() *DriverIngressConfiguration {
+	if in == nil {
+		return nil
+	}
+	out := new(DriverIngressConfiguration)
+	in.DeepCopyInto(out)
+	return out
+}
+
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *DriverSpec) DeepCopyInto(out *DriverSpec) {
 	*out = *in
@ -176,12 +227,18 @@ func (in *DriverSpec) DeepCopyInto(out *DriverSpec) {
 			(*out)[key] = val
 		}
 	}
+	if in.ServiceLabels != nil {
+		in, out := &in.ServiceLabels, &out.ServiceLabels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.Ports != nil {
 		in, out := &in.Ports, &out.Ports
 		*out = make([]Port, len(*in))
 		copy(*out, *in)
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DriverSpec.
@ -217,7 +274,6 @@ func (in *DynamicAllocation) DeepCopyInto(out *DynamicAllocation) {
 		*out = new(int64)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DynamicAllocation.
@ -249,6 +305,11 @@ func (in *ExecutorSpec) DeepCopyInto(out *ExecutorSpec) {
 		*out = new(string)
 		**out = **in
 	}
+	if in.Lifecycle != nil {
+		in, out := &in.Lifecycle, &out.Lifecycle
+		*out = new(v1.Lifecycle)
+		(*in).DeepCopyInto(*out)
+	}
 	if in.DeleteOnTermination != nil {
 		in, out := &in.DeleteOnTermination, &out.DeleteOnTermination
 		*out = new(bool)
@ -259,7 +320,6 @@ func (in *ExecutorSpec) DeepCopyInto(out *ExecutorSpec) {
 		*out = make([]Port, len(*in))
 		copy(*out, *in)
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ExecutorSpec.
@ -275,7 +335,6 @@ func (in *ExecutorSpec) DeepCopy() *ExecutorSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *GPUSpec) DeepCopyInto(out *GPUSpec) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GPUSpec.
@ -306,7 +365,6 @@ func (in *MonitoringSpec) DeepCopyInto(out *MonitoringSpec) {
 		*out = new(PrometheusSpec)
 		(*in).DeepCopyInto(*out)
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new MonitoringSpec.
@ -322,7 +380,6 @@ func (in *MonitoringSpec) DeepCopy() *MonitoringSpec {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NameKey) DeepCopyInto(out *NameKey) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NameKey.
@ -338,7 +395,6 @@ func (in *NameKey) DeepCopy() *NameKey {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *NamePath) DeepCopyInto(out *NamePath) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new NamePath.
@ -354,7 +410,6 @@ func (in *NamePath) DeepCopy() *NamePath {
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *Port) DeepCopyInto(out *Port) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Port.
@ -390,7 +445,6 @@ func (in *PrometheusSpec) DeepCopyInto(out *PrometheusSpec) {
 		*out = new(string)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PrometheusSpec.
@ -426,7 +480,6 @@ func (in *RestartPolicy) DeepCopyInto(out *RestartPolicy) {
 		*out = new(int64)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new RestartPolicy.
@ -446,7 +499,6 @@ func (in *ScheduledSparkApplication) DeepCopyInto(out *ScheduledSparkApplication
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
 	in.Spec.DeepCopyInto(&out.Spec)
 	in.Status.DeepCopyInto(&out.Status)
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplication.
@ -479,7 +531,6 @@ func (in *ScheduledSparkApplicationList) DeepCopyInto(out *ScheduledSparkApplica
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplicationList.
@ -519,7 +570,6 @@ func (in *ScheduledSparkApplicationSpec) DeepCopyInto(out *ScheduledSparkApplica
 		*out = new(int32)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplicationSpec.
@ -547,7 +597,6 @@ func (in *ScheduledSparkApplicationStatus) DeepCopyInto(out *ScheduledSparkAppli
 		*out = make([]string, len(*in))
 		copy(*out, *in)
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ScheduledSparkApplicationStatus.
@ -563,7 +612,6 @@ func (in *ScheduledSparkApplicationStatus) DeepCopy() *ScheduledSparkApplication
 // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
 func (in *SecretInfo) DeepCopyInto(out *SecretInfo) {
 	*out = *in
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SecretInfo.
@ -583,7 +631,6 @@ func (in *SparkApplication) DeepCopyInto(out *SparkApplication) {
 	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
 	in.Spec.DeepCopyInto(&out.Spec)
 	in.Status.DeepCopyInto(&out.Status)
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkApplication.
@ -616,7 +663,6 @@ func (in *SparkApplicationList) DeepCopyInto(out *SparkApplicationList) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkApplicationList.
@ -774,7 +820,6 @@ func (in *SparkApplicationSpec) DeepCopyInto(out *SparkApplicationSpec) {
 		*out = new(DynamicAllocation)
 		(*in).DeepCopyInto(*out)
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkApplicationSpec.
@ -801,7 +846,6 @@ func (in *SparkApplicationStatus) DeepCopyInto(out *SparkApplicationStatus) {
 			(*out)[key] = val
 		}
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkApplicationStatus.
@ -986,7 +1030,6 @@ func (in *SparkPodSpec) DeepCopyInto(out *SparkPodSpec) {
 		*out = new(bool)
 		**out = **in
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkPodSpec.
@ -1024,6 +1067,13 @@ func (in *SparkUIConfiguration) DeepCopyInto(out *SparkUIConfiguration) {
 			(*out)[key] = val
 		}
 	}
+	if in.ServiceLabels != nil {
+		in, out := &in.ServiceLabels, &out.ServiceLabels
+		*out = make(map[string]string, len(*in))
+		for key, val := range *in {
+			(*out)[key] = val
+		}
+	}
 	if in.IngressAnnotations != nil {
 		in, out := &in.IngressAnnotations, &out.IngressAnnotations
 		*out = make(map[string]string, len(*in))
@ -1038,7 +1088,6 @@ func (in *SparkUIConfiguration) DeepCopyInto(out *SparkUIConfiguration) {
 			(*in)[i].DeepCopyInto(&(*out)[i])
 		}
 	}
-	return
 }

 // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new SparkUIConfiguration.
@ -1050,56 +1099,3 @@ func (in *SparkUIConfiguration) DeepCopy() *SparkUIConfiguration {
 	in.DeepCopyInto(out)
 	return out
 }
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DriverIngressConfiguration) DeepCopyInto(out *DriverIngressConfiguration) {
-	*out = *in
-	if in.ServicePort != nil {
-		in, out := &in.ServicePort, &out.ServicePort
-		*out = new(int32)
-		**out = **in
-	}
-	if in.ServicePortName != nil {
-		in, out := &in.ServicePortName, &out.ServicePortName
-		*out = new(string)
-		**out = **in
-	}
-	if in.ServiceType != nil {
-		in, out := &in.ServiceType, &out.ServiceType
-		*out = new(v1.ServiceType)
-		**out = **in
-	}
-	if in.ServiceAnnotations != nil {
-		in, out := &in.ServiceAnnotations, &out.ServiceAnnotations
-		*out = make(map[string]string, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val
-		}
-	}
-	out.IngressURLFormat = in.IngressURLFormat
-	if in.IngressAnnotations != nil {
-		in, out := &in.IngressAnnotations, &out.IngressAnnotations
-		*out = make(map[string]string, len(*in))
-		for key, val := range *in {
-			(*out)[key] = val
-		}
-	}
-	if in.IngressTLS != nil {
-		in, out := &in.IngressTLS, &out.IngressTLS
-		*out = make([]networkingv1.IngressTLS, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
-	return
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DriverIngressConfiguration.
-func (in *DriverIngressConfiguration) DeepCopy() *DriverIngressConfiguration {
-	if in == nil {
-		return nil
-	}
-	out := new(DriverIngressConfiguration)
-	in.DeepCopyInto(out)
-	return out
-}