deploy karmada-aggregated-apiserver

Signed-off-by: changzhen <changzhen5@huawei.com>

.gitignore

@@ -20,6 +20,7 @@
 # karamada binary
 /karmadactl
 /karmada-agent
+/karmada-aggregated-apiserver
 /karmada-controller-manager
 /karmada-interpreter-webhook-example
 /karmada-scheduler

Makefile

@@ -39,7 +39,13 @@ ifeq ($(VERSION), "")
     endif
 endif
 
-all: karmada-controller-manager karmada-scheduler karmadactl kubectl-karmada karmada-webhook karmada-agent karmada-scheduler-estimator karmada-interpreter-webhook-example
+all: karmada-aggregated-apiserver karmada-controller-manager karmada-scheduler karmadactl kubectl-karmada karmada-webhook karmada-agent karmada-scheduler-estimator karmada-interpreter-webhook-example
+
+karmada-aggregated-apiserver: $(SOURCES)
+	CGO_ENABLED=0 GOOS=$(GOOS) go build \
+		-ldflags $(LDFLAGS) \
+		-o karmada-aggregated-apiserver \
+		cmd/aggregated-apiserver/main.go
 
 karmada-controller-manager: $(SOURCES)
 	CGO_ENABLED=0 GOOS=$(GOOS) go build \
@@ -90,7 +96,7 @@ karmada-interpreter-webhook-example: $(SOURCES)
 		examples/customresourceinterpreter/webhook/main.go
 
 clean:
-	rm -rf karmada-controller-manager karmada-scheduler karmadactl kubectl-karmada karmada-webhook karmada-agent karmada-scheduler-estimator karmada-interpreter-webhook-example
+	rm -rf karmada-aggregated-apiserver karmada-controller-manager karmada-scheduler karmadactl kubectl-karmada karmada-webhook karmada-agent karmada-scheduler-estimator karmada-interpreter-webhook-example
 
 .PHONY: update
 update:
@@ -106,7 +112,10 @@ test:
 	go test --race --v ./cmd/...
 	go test --race --v ./examples/...
 
-images: image-karmada-controller-manager image-karmada-scheduler image-karmada-webhook image-karmada-agent image-karmada-scheduler-estimator image-karmada-interpreter-webhook-example
+images: image-karmada-aggregated-apiserver image-karmada-controller-manager image-karmada-scheduler image-karmada-webhook image-karmada-agent image-karmada-scheduler-estimator image-karmada-interpreter-webhook-example
+
+image-karmada-aggregated-apiserver: karmada-aggregated-apiserver
+	VERSION=$(VERSION) hack/docker.sh karmada-aggregated-apiserver
 
 image-karmada-controller-manager: karmada-controller-manager
 	VERSION=$(VERSION) hack/docker.sh karmada-controller-manager
@@ -137,3 +146,4 @@ endif
 	docker push ${REGISTRY}/karmada-agent:${VERSION}
 	docker push ${REGISTRY}/karmada-scheduler-estimator:${VERSION}
 	docker push ${REGISTRY}/karmada-interpreter-webhook-example:${VERSION}
+	docker push ${REGISTRY}/karmada-aggregated-apiserver:${VERSION}

artifacts/deploy/apiservice.yaml

@@ -0,0 +1,25 @@
+apiVersion: apiregistration.k8s.io/v1
+kind: APIService
+metadata:
+  name: v1alpha1.cluster.karmada.io
+  labels:
+    app: karmada-aggregated-apiserver
+    apiserver: "true"
+spec:
+  insecureSkipTLSVerify: true
+  group: cluster.karmada.io
+  groupPriorityMinimum: 2000
+  service:
+    name: karmada-aggregated-apiserver
+    namespace: karmada-system
+  version: v1alpha1
+  versionPriority: 10
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: karmada-aggregated-apiserver
+  namespace: karmada-system
+spec:
+  type: ExternalName
+  externalName: karmada-aggregated-apiserver.karmada-system.svc.cluster.local

artifacts/deploy/karmada-aggregated-apiserver.yaml

@@ -0,0 +1,74 @@
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: karmada-aggregated-apiserver
+  namespace: karmada-system
+  labels:
+    app: karmada-aggregated-apiserver
+    apiserver: "true"
+spec:
+  selector:
+    matchLabels:
+      app: karmada-aggregated-apiserver
+      apiserver: "true"
+  replicas: 1
+  template:
+    metadata:
+      labels:
+        app: karmada-aggregated-apiserver
+        apiserver: "true"
+    spec:
+      automountServiceAccountToken: false
+      containers:
+        - name: karmada-aggregated-apiserver
+          image: swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-aggregated-apiserver:latest
+          imagePullPolicy: IfNotPresent
+          volumeMounts:
+            - name: k8s-certs
+              mountPath: /etc/kubernetes/pki
+              readOnly: true
+            - name: kubeconfig
+              subPath: kubeconfig
+              mountPath: /etc/kubeconfig
+          command:
+            - /bin/karmada-aggregated-apiserver
+            - --kubeconfig=/etc/kubeconfig
+            - --authentication-kubeconfig=/etc/kubeconfig
+            - --authorization-kubeconfig=/etc/kubeconfig
+            - --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379
+            - --etcd-cafile=/etc/kubernetes/pki/server-ca.crt
+            - --etcd-certfile=/etc/kubernetes/pki/karmada.crt
+            - --etcd-keyfile=/etc/kubernetes/pki/karmada.key
+            - --tls-cert-file=/etc/kubernetes/pki/karmada.crt
+            - --tls-private-key-file=/etc/kubernetes/pki/karmada.key
+            - --audit-log-path=-
+            - --feature-gates=APIPriorityAndFairness=false
+            - --audit-log-maxage=0
+            - --audit-log-maxbackup=0
+          resources:
+            requests:
+              cpu: 100m
+      volumes:
+        - name: k8s-certs
+          secret:
+            secretName: karmada-cert-secret
+        - name: kubeconfig
+          secret:
+            secretName: kubeconfig
+---
+apiVersion: v1
+kind: Service
+metadata:
+  name: karmada-aggregated-apiserver
+  namespace: karmada-system
+  labels:
+    app: karmada-aggregated-apiserver
+    apiserver: "true"
+spec:
+  ports:
+    - port: 443
+      protocol: TCP
+      targetPort: 443
+  selector:
+    app: karmada-aggregated-apiserver

artifacts/deploy/karmada-apiserver.yaml

@@ -54,10 +54,10 @@ spec:
             - --service-account-key-file=/etc/kubernetes/pki/karmada.key
             - --service-account-signing-key-file=/etc/kubernetes/pki/karmada.key
             - --service-cluster-ip-range=10.96.0.0/12
-            - --proxy-client-cert-file=/etc/kubernetes/pki/karmada.crt
+            - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt
-            - --proxy-client-key-file=/etc/kubernetes/pki/karmada.key
+            - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key
             - --requestheader-allowed-names=front-proxy-client
-            - --requestheader-client-ca-file=/etc/kubernetes/pki/server-ca.crt
+            - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt
             - --requestheader-extra-headers-prefix=X-Remote-Extra-
             - --requestheader-group-headers=X-Remote-Group
             - --requestheader-username-headers=X-Remote-User
@@ -111,9 +111,7 @@ spec:
         - name: k8s-certs
           secret:
             secretName: karmada-cert-secret
-
 ---
-
 apiVersion: v1
 kind: Service
 metadata:

artifacts/deploy/karmada-cert-secret.yaml

@@ -11,3 +11,9 @@ data:
     {{client_cer}}
   karmada.key: |
     {{client_key}}
+  front-proxy-ca.crt: |
+    {{front_proxy_ca_crt}}
+  front-proxy-client.crt: |
+    {{front_proxy_client_crt}}
+  front-proxy-client.key: |
+    {{front_proxy_client_key}}

charts/_crds/bases/cluster.karmada.io_clusters.yaml

@@ -1,310 +0,0 @@
-
----
-apiVersion: apiextensions.k8s.io/v1
-kind: CustomResourceDefinition
-metadata:
-  annotations:
-    controller-gen.kubebuilder.io/version: v0.6.2
-  creationTimestamp: null
-  name: clusters.cluster.karmada.io
-spec:
-  group: cluster.karmada.io
-  names:
-    kind: Cluster
-    listKind: ClusterList
-    plural: clusters
-    singular: cluster
-  scope: Cluster
-  versions:
-  - additionalPrinterColumns:
-    - jsonPath: .status.kubernetesVersion
-      name: Version
-      type: string
-    - jsonPath: .spec.syncMode
-      name: Mode
-      type: string
-    - jsonPath: .status.conditions[?(@.type=="Ready")].status
-      name: Ready
-      type: string
-    - jsonPath: .metadata.creationTimestamp
-      name: Age
-      type: date
-    name: v1alpha1
-    schema:
-      openAPIV3Schema:
-        description: Cluster represents the desire state and status of a member cluster.
-        properties:
-          apiVersion:
-            description: 'APIVersion defines the versioned schema of this representation
-              of an object. Servers should convert recognized schemas to the latest
-              internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
-            type: string
-          kind:
-            description: 'Kind is a string value representing the REST resource this
-              object represents. Servers may infer this from the endpoint the client
-              submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
-            type: string
-          metadata:
-            type: object
-          spec:
-            description: Spec represents the specification of the desired behavior
-              of member cluster.
-            properties:
-              apiEndpoint:
-                description: The API endpoint of the member cluster. This can be a
-                  hostname, hostname:port, IP or IP:port.
-                type: string
-              insecureSkipTLSVerification:
-                description: InsecureSkipTLSVerification indicates that the karmada
-                  control plane should not confirm the validity of the serving certificate
-                  of the cluster it is connecting to. This will make the HTTPS connection
-                  between the karmada control plane and the member cluster insecure.
-                  Defaults to false.
-                type: boolean
-              provider:
-                description: Provider represents the cloud provider name of the member
-                  cluster.
-                type: string
-              proxyURL:
-                description: 'ProxyURL is the proxy URL for the cluster. If not empty,
-                  the karmada control plane will use this proxy to talk to the cluster.
-                  More details please refer to: https://github.com/kubernetes/client-go/issues/351'
-                type: string
-              region:
-                description: Region represents the region of the member cluster locate
-                  in.
-                type: string
-              secretRef:
-                description: 'SecretRef represents the secret contains mandatory credentials
-                  to access the member cluster. The secret should hold credentials
-                  as follows: - secret.data.token - secret.data.caBundle'
-                properties:
-                  name:
-                    description: Name is the name of resource being referenced.
-                    type: string
-                  namespace:
-                    description: Namespace is the namespace for the resource being
-                      referenced.
-                    type: string
-                required:
-                - name
-                - namespace
-                type: object
-              syncMode:
-                description: SyncMode describes how a cluster sync resources from
-                  karmada control plane.
-                enum:
-                - Push
-                - Pull
-                type: string
-              taints:
-                description: Taints attached to the member cluster. Taints on the
-                  cluster have the "effect" on any resource that does not tolerate
-                  the Taint.
-                items:
-                  description: The node this Taint is attached to has the "effect"
-                    on any pod that does not tolerate the Taint.
-                  properties:
-                    effect:
-                      description: Required. The effect of the taint on pods that
-                        do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule
-                        and NoExecute.
-                      type: string
-                    key:
-                      description: Required. The taint key to be applied to a node.
-                      type: string
-                    timeAdded:
-                      description: TimeAdded represents the time at which the taint
-                        was added. It is only written for NoExecute taints.
-                      format: date-time
-                      type: string
-                    value:
-                      description: The taint value corresponding to the taint key.
-                      type: string
-                  required:
-                  - effect
-                  - key
-                  type: object
-                type: array
-              zone:
-                description: Zone represents the zone of the member cluster locate
-                  in.
-                type: string
-            required:
-            - syncMode
-            type: object
-          status:
-            description: Status represents the status of member cluster.
-            properties:
-              apiEnablements:
-                description: APIEnablements represents the list of APIs installed
-                  in the member cluster.
-                items:
-                  description: APIEnablement is a list of API resource, it is used
-                    to expose the name of the resources supported in a specific group
-                    and version.
-                  properties:
-                    groupVersion:
-                      description: GroupVersion is the group and version this APIEnablement
-                        is for.
-                      type: string
-                    resources:
-                      description: Resources is a list of APIResource.
-                      items:
-                        description: APIResource specifies the name and kind names
-                          for the resource.
-                        properties:
-                          kind:
-                            description: Kind is the kind for the resource (e.g. 'Deployment'
-                              is the kind for resource 'deployments')
-                            type: string
-                          name:
-                            description: Name is the plural name of the resource.
-                            type: string
-                        required:
-                        - kind
-                        - name
-                        type: object
-                      type: array
-                  required:
-                  - groupVersion
-                  type: object
-                type: array
-              conditions:
-                description: Conditions is an array of current cluster conditions.
-                items:
-                  description: "Condition contains details for one aspect of the current
-                    state of this API Resource. --- This struct is intended for direct
-                    use as an array at the field path .status.conditions.  For example,
-                    type FooStatus struct{     // Represents the observations of a
-                    foo's current state.     // Known .status.conditions.type are:
-                    \"Available\", \"Progressing\", and \"Degraded\"     // +patchMergeKey=type
-                    \    // +patchStrategy=merge     // +listType=map     // +listMapKey=type
-                    \    Conditions []metav1.Condition `json:\"conditions,omitempty\"
-                    patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"`
-                    \n     // other fields }"
-                  properties:
-                    lastTransitionTime:
-                      description: lastTransitionTime is the last time the condition
-                        transitioned from one status to another. This should be when
-                        the underlying condition changed.  If that is not known, then
-                        using the time when the API field changed is acceptable.
-                      format: date-time
-                      type: string
-                    message:
-                      description: message is a human readable message indicating
-                        details about the transition. This may be an empty string.
-                      maxLength: 32768
-                      type: string
-                    observedGeneration:
-                      description: observedGeneration represents the .metadata.generation
-                        that the condition was set based upon. For instance, if .metadata.generation
-                        is currently 12, but the .status.conditions[x].observedGeneration
-                        is 9, the condition is out of date with respect to the current
-                        state of the instance.
-                      format: int64
-                      minimum: 0
-                      type: integer
-                    reason:
-                      description: reason contains a programmatic identifier indicating
-                        the reason for the condition's last transition. Producers
-                        of specific condition types may define expected values and
-                        meanings for this field, and whether the values are considered
-                        a guaranteed API. The value should be a CamelCase string.
-                        This field may not be empty.
-                      maxLength: 1024
-                      minLength: 1
-                      pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$
-                      type: string
-                    status:
-                      description: status of the condition, one of True, False, Unknown.
-                      enum:
-                      - "True"
-                      - "False"
-                      - Unknown
-                      type: string
-                    type:
-                      description: type of condition in CamelCase or in foo.example.com/CamelCase.
-                        --- Many .condition.type values are consistent across resources
-                        like Available, but because arbitrary conditions can be useful
-                        (see .node.status.conditions), the ability to deconflict is
-                        important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt)
-                      maxLength: 316
-                      pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$
-                      type: string
-                  required:
-                  - lastTransitionTime
-                  - message
-                  - reason
-                  - status
-                  - type
-                  type: object
-                type: array
-              kubernetesVersion:
-                description: KubernetesVersion represents version of the member cluster.
-                type: string
-              nodeSummary:
-                description: NodeSummary represents the summary of nodes status in
-                  the member cluster.
-                properties:
-                  readyNum:
-                    description: ReadyNum is the number of ready nodes in the cluster.
-                    format: int32
-                    type: integer
-                  totalNum:
-                    description: TotalNum is the total number of nodes in the cluster.
-                    format: int32
-                    type: integer
-                type: object
-              resourceSummary:
-                description: ResourceSummary represents the summary of resources in
-                  the member cluster.
-                properties:
-                  allocatable:
-                    additionalProperties:
-                      anyOf:
-                      - type: integer
-                      - type: string
-                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
-                      x-kubernetes-int-or-string: true
-                    description: Allocatable represents the resources of a cluster
-                      that are available for scheduling. Total amount of allocatable
-                      resources on all nodes.
-                    type: object
-                  allocated:
-                    additionalProperties:
-                      anyOf:
-                      - type: integer
-                      - type: string
-                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
-                      x-kubernetes-int-or-string: true
-                    description: Allocated represents the resources of a cluster that
-                      have been scheduled. Total amount of required resources of all
-                      Pods that have been scheduled to nodes.
-                    type: object
-                  allocating:
-                    additionalProperties:
-                      anyOf:
-                      - type: integer
-                      - type: string
-                      pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$
-                      x-kubernetes-int-or-string: true
-                    description: Allocating represents the resources of a cluster
-                      that are pending for scheduling. Total amount of required resources
-                      of all Pods that are waiting for scheduling.
-                    type: object
-                type: object
-            type: object
-        required:
-        - spec
-        type: object
-    served: true
-    storage: true
-    subresources:
-      status: {}
-status:
-  acceptedNames:
-    kind: ""
-    plural: ""
-  conditions: []
-  storedVersions: []

charts/_crds/kustomization.yaml

@@ -1,5 +1,4 @@
 resources:
-- bases/cluster.karmada.io_clusters.yaml
 - bases/multicluster.x-k8s.io_serviceexports.yaml
 - bases/multicluster.x-k8s.io_serviceimports.yaml
 - bases/policy.karmada.io_clusteroverridepolicies.yaml

cluster/images/karmada-aggregated-apiserver/Dockerfile

@@ -0,0 +1,7 @@
+FROM alpine:3.7
+
+RUN apk add --no-cache ca-certificates
+
+ADD karmada-aggregated-apiserver /bin/
+
+CMD ["/bin/karmada-aggregated-apiserver"]

go.mod

@@ -35,6 +35,6 @@ require (
 	sigs.k8s.io/controller-runtime v0.10.3
 	sigs.k8s.io/kind v0.11.1
 	sigs.k8s.io/mcs-api v0.1.0
-	sigs.k8s.io/yaml v1.3.0
 	sigs.k8s.io/structured-merge-diff/v4 v4.1.2
+	sigs.k8s.io/yaml v1.3.0
 )

hack/deploy-karmada.sh

@@ -86,6 +86,10 @@ function generate_cert_secret {
     sed -i'' -e "s/{{client_cer}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
     sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
 
+    sed -i'' -e "s/{{front_proxy_ca_crt}}/${FRONT_PROXY_CA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
+    sed -i'' -e "s/{{front_proxy_client_crt}}/${FRONT_PROXY_CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
+    sed -i'' -e "s/{{front_proxy_client_key}}/${FRONT_PROXY_CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml
+
     sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/secret-tmp.yaml
     sed -i'' -e "s/{{client_cer}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/secret-tmp.yaml
     sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/secret-tmp.yaml
@@ -113,8 +117,10 @@ util::cmd_must_exist "openssl"
 util::cmd_must_exist_cfssl ${CFSSL_VERSION}
 # create CA signers
 util::create_signing_certkey "" "${CERT_DIR}" server '"client auth","server auth"'
+util::create_signing_certkey "" "${CERT_DIR}" front-proxy '"client auth","server auth"'
 # signs a certificate
 util::create_certkey "" "${CERT_DIR}" "server-ca" karmada system:admin kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1"
+util::create_certkey "" "${CERT_DIR}" "front-proxy-ca" front-proxy-client front-proxy-client kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1"
 
 # create namespace for control plane components
 kubectl apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml"
@@ -126,6 +132,9 @@ kubectl apply -f "${REPO_ROOT}/artifacts/deploy/clusterrolebinding.yaml"
 
 KARMADA_CRT=$(base64 "${CERT_DIR}/karmada.crt" | tr -d '\r\n')
 KARMADA_KEY=$(base64 "${CERT_DIR}/karmada.key" | tr -d '\r\n')
+FRONT_PROXY_CA_CRT=$(base64 "${CERT_DIR}/front-proxy-ca.crt" | tr -d '\r\n')
+FRONT_PROXY_CLIENT_CRT=$(base64 "${CERT_DIR}/front-proxy-client.crt" | tr -d '\r\n')
+FRONT_PROXY_CLIENT_KEY=$(base64 "${CERT_DIR}/front-proxy-client.key" | tr -d '\r\n')
 generate_cert_secret
 
 # deploy karmada etcd
@@ -191,6 +200,9 @@ util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/karmada
 
 # deploy kube controller manager
 kubectl apply -f "${REPO_ROOT}/artifacts/deploy/kube-controller-manager.yaml"
+# deploy aggregated-apiserver on host cluster
+kubectl apply -f "${REPO_ROOT}/artifacts/deploy/karmada-aggregated-apiserver.yaml"
+util::wait_pod_ready "${KARMADA_AGGREGATION_APISERVER_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}"
 
 # install CRD APIs on karmada apiserver.
 if ! kubectl config use-context karmada-apiserver > /dev/null 2>&1;
@@ -210,6 +222,11 @@ rm -rf "${TEMP_PATH_CRDS}"
 # deploy webhook configurations on karmada apiserver
 util::deploy_webhook_configuration "${ROOT_CA_FILE}" "${REPO_ROOT}/artifacts/deploy/webhook-configuration.yaml"
 
+# deploy APIService on karmada apiserver for karmada-aggregated-apiserver
+kubectl apply -f "${REPO_ROOT}/artifacts/deploy/apiservice.yaml"
+# make sure apiservice for v1alpha1.cluster.karmada.io is Available
+util::wait_apiservice_ready "${KARMADA_AGGREGATION_APISERVER_LABEL}"
+
 kubectl config use-context "${HOST_CLUSTER_NAME}"
 
 # deploy controller-manager on host cluster

hack/local-up-karmada.sh

@@ -47,6 +47,7 @@ dockerfile_list=( # Dockerfile files need to be replaced
   "cluster/images/karmada-scheduler-estimator/Dockerfile"
   "cluster/images/karmada-scheduler/Dockerfile"
   "cluster/images/karmada-webhook/Dockerfile"
+  "cluster/images/karmada-aggregated-apiserver/Dockerfile"
 )
 for dockerfile in "${dockerfile_list[@]}"; do
   grep 'mirrors.ustc.edu.cn' ${REPO_ROOT}/${dockerfile} > /dev/null || sed -i'' -e "s#FROM alpine:3.7#FROM alpine:3.7\nRUN echo -e http://mirrors.ustc.edu.cn/alpine/v3.7/main/ > /etc/apk/repositories#" ${REPO_ROOT}/${dockerfile}
@@ -122,6 +123,7 @@ kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --nam
 kind load docker-image "${REGISTRY}/karmada-scheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}"
 kind load docker-image "${REGISTRY}/karmada-webhook:${VERSION}" --name="${HOST_CLUSTER_NAME}"
 kind load docker-image "${REGISTRY}/karmada-scheduler-estimator:${VERSION}" --name="${HOST_CLUSTER_NAME}"
+kind load docker-image "${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" --name="${HOST_CLUSTER_NAME}"
 
 #step5. install karmada control plane components
 "${REPO_ROOT}"/hack/deploy-karmada.sh "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}"

hack/update-codegen.sh

@@ -18,6 +18,11 @@ deepcopy-gen \
   --input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1 \
   --output-package=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1 \
   --output-file-base=zz_generated.deepcopy
+deepcopy-gen \
+  --go-header-file hack/boilerplate/boilerplate.go.txt \
+  --input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster \
+  --output-package=github.com/karmada-io/karmada/pkg/apis/cluster \
+  --output-file-base=zz_generated.deepcopy
 deepcopy-gen \
   --go-header-file hack/boilerplate/boilerplate.go.txt \
   --input-dirs=github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1 \
@@ -77,6 +82,14 @@ register-gen \
   --output-package=github.com/karmada-io/karmada/examples/customresourceinterpreter/apis/workload/v1alpha1 \
   --output-file-base=zz_generated.register
 
+echo "Generating with conversion-gen"
+GO111MODULE=on go install k8s.io/code-generator/cmd/conversion-gen
+conversion-gen \
+  --go-header-file hack/boilerplate/boilerplate.go.txt \
+  --input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1 \
+  --output-package=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1 \
+  --output-file-base=zz_generated.conversion
+
 echo "Generating with client-gen"
 GO111MODULE=on go install k8s.io/code-generator/cmd/client-gen
 client-gen \
@@ -101,4 +114,3 @@ informer-gen \
   --versioned-clientset-package=github.com/karmada-io/karmada/pkg/generated/clientset/versioned \
   --listers-package=github.com/karmada-io/karmada/pkg/generated/listers \
   --output-package=github.com/karmada-io/karmada/pkg/generated/informers
-

hack/update-crdgen.sh

@@ -13,5 +13,7 @@ echo "Generating with controller-gen"
 util::install_tools ${CONTROLLER_GEN_PKG} ${CONTROLLER_GEN_VER} >/dev/null 2>&1
 
 # Unify the crds used by helm chart and the installation scripts
-controller-gen crd paths=./pkg/apis/... output:crd:dir=./charts/_crds/bases
+controller-gen crd paths=./pkg/apis/config/... output:crd:dir=./charts/_crds/bases
+controller-gen crd paths=./pkg/apis/policy/... output:crd:dir=./charts/_crds/bases
+controller-gen crd paths=./pkg/apis/work/... output:crd:dir=./charts/_crds/bases
 controller-gen crd paths=./examples/customresourceinterpreter/apis/... output:crd:dir=./examples/customresourceinterpreter/apis/

hack/util.sh

@@ -10,6 +10,7 @@ KARMADA_SYSTEM_NAMESPACE="karmada-system"
 ETCD_POD_LABEL="etcd"
 APISERVER_POD_LABEL="karmada-apiserver"
 KUBE_CONTROLLER_POD_LABEL="kube-controller-manager"
+KARMADA_AGGREGATION_APISERVER_LABEL="karmada-aggregated-apiserver"
 KARMADA_CONTROLLER_LABEL="karmada-controller-manager"
 KARMADA_SCHEDULER_LABEL="karmada-scheduler"
 KARMADA_WEBHOOK_LABEL="karmada-webhook"
@@ -301,6 +302,25 @@ function util::wait_pod_ready() {
     return ${ret}
 }
 
+# util::wait_apiservice_ready waits for apiservice state becomes Available until timeout.
+# Parmeters:
+#  - $1: apiservice label, such as "app=etcd"
+#  - $3: time out, such as "200s"
+function util::wait_apiservice_ready() {
+    local apiservice_label=$1
+
+    echo "wait the $apiservice_label Available..."
+    set +e
+    util::kubectl_with_retry wait --for=condition=Available --timeout=30s apiservices -l app=${apiservice_label}
+    ret=$?
+    set -e
+    if [ $ret -ne 0 ];then
+      echo "kubectl describe info:"
+      kubectl describe apiservices -l app=${apiservice_label}
+    fi
+    return ${ret}
+}
+
 # util::kubectl_with_retry will retry if execute kubectl command failed
 # tolerate kubectl command failure that may happen before the pod is created by  StatefulSet/Deployment.
 function util::kubectl_with_retry() {