diff --git a/.gitignore b/.gitignore index 298cd81a7..71046180c 100644 --- a/.gitignore +++ b/.gitignore @@ -20,6 +20,7 @@ # karamada binary /karmadactl /karmada-agent +/karmada-aggregated-apiserver /karmada-controller-manager /karmada-interpreter-webhook-example /karmada-scheduler diff --git a/Makefile b/Makefile index fc4f4ab01..71dc5dc1d 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,13 @@ ifeq ($(VERSION), "") endif endif -all: karmada-controller-manager karmada-scheduler karmadactl kubectl-karmada karmada-webhook karmada-agent karmada-scheduler-estimator karmada-interpreter-webhook-example +all: karmada-aggregated-apiserver karmada-controller-manager karmada-scheduler karmadactl kubectl-karmada karmada-webhook karmada-agent karmada-scheduler-estimator karmada-interpreter-webhook-example + +karmada-aggregated-apiserver: $(SOURCES) + CGO_ENABLED=0 GOOS=$(GOOS) go build \ + -ldflags $(LDFLAGS) \ + -o karmada-aggregated-apiserver \ + cmd/aggregated-apiserver/main.go karmada-controller-manager: $(SOURCES) CGO_ENABLED=0 GOOS=$(GOOS) go build \ @@ -90,7 +96,7 @@ karmada-interpreter-webhook-example: $(SOURCES) examples/customresourceinterpreter/webhook/main.go clean: - rm -rf karmada-controller-manager karmada-scheduler karmadactl kubectl-karmada karmada-webhook karmada-agent karmada-scheduler-estimator karmada-interpreter-webhook-example + rm -rf karmada-aggregated-apiserver karmada-controller-manager karmada-scheduler karmadactl kubectl-karmada karmada-webhook karmada-agent karmada-scheduler-estimator karmada-interpreter-webhook-example .PHONY: update update: @@ -106,7 +112,10 @@ test: go test --race --v ./cmd/... go test --race --v ./examples/... -images: image-karmada-controller-manager image-karmada-scheduler image-karmada-webhook image-karmada-agent image-karmada-scheduler-estimator image-karmada-interpreter-webhook-example +images: image-karmada-aggregated-apiserver image-karmada-controller-manager image-karmada-scheduler image-karmada-webhook image-karmada-agent image-karmada-scheduler-estimator image-karmada-interpreter-webhook-example + +image-karmada-aggregated-apiserver: karmada-aggregated-apiserver + VERSION=$(VERSION) hack/docker.sh karmada-aggregated-apiserver image-karmada-controller-manager: karmada-controller-manager VERSION=$(VERSION) hack/docker.sh karmada-controller-manager @@ -137,3 +146,4 @@ endif docker push ${REGISTRY}/karmada-agent:${VERSION} docker push ${REGISTRY}/karmada-scheduler-estimator:${VERSION} docker push ${REGISTRY}/karmada-interpreter-webhook-example:${VERSION} + docker push ${REGISTRY}/karmada-aggregated-apiserver:${VERSION} diff --git a/artifacts/deploy/apiservice.yaml b/artifacts/deploy/apiservice.yaml new file mode 100644 index 000000000..7fab8df2c --- /dev/null +++ b/artifacts/deploy/apiservice.yaml @@ -0,0 +1,25 @@ +apiVersion: apiregistration.k8s.io/v1 +kind: APIService +metadata: + name: v1alpha1.cluster.karmada.io + labels: + app: karmada-aggregated-apiserver + apiserver: "true" +spec: + insecureSkipTLSVerify: true + group: cluster.karmada.io + groupPriorityMinimum: 2000 + service: + name: karmada-aggregated-apiserver + namespace: karmada-system + version: v1alpha1 + versionPriority: 10 +--- +apiVersion: v1 +kind: Service +metadata: + name: karmada-aggregated-apiserver + namespace: karmada-system +spec: + type: ExternalName + externalName: karmada-aggregated-apiserver.karmada-system.svc.cluster.local diff --git a/artifacts/deploy/karmada-aggregated-apiserver.yaml b/artifacts/deploy/karmada-aggregated-apiserver.yaml new file mode 100644 index 000000000..fbd74634e --- /dev/null +++ b/artifacts/deploy/karmada-aggregated-apiserver.yaml @@ -0,0 +1,74 @@ +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: karmada-aggregated-apiserver + namespace: karmada-system + labels: + app: karmada-aggregated-apiserver + apiserver: "true" +spec: + selector: + matchLabels: + app: karmada-aggregated-apiserver + apiserver: "true" + replicas: 1 + template: + metadata: + labels: + app: karmada-aggregated-apiserver + apiserver: "true" + spec: + automountServiceAccountToken: false + containers: + - name: karmada-aggregated-apiserver + image: swr.ap-southeast-1.myhuaweicloud.com/karmada/karmada-aggregated-apiserver:latest + imagePullPolicy: IfNotPresent + volumeMounts: + - name: k8s-certs + mountPath: /etc/kubernetes/pki + readOnly: true + - name: kubeconfig + subPath: kubeconfig + mountPath: /etc/kubeconfig + command: + - /bin/karmada-aggregated-apiserver + - --kubeconfig=/etc/kubeconfig + - --authentication-kubeconfig=/etc/kubeconfig + - --authorization-kubeconfig=/etc/kubeconfig + - --etcd-servers=https://etcd-client.karmada-system.svc.cluster.local:2379 + - --etcd-cafile=/etc/kubernetes/pki/server-ca.crt + - --etcd-certfile=/etc/kubernetes/pki/karmada.crt + - --etcd-keyfile=/etc/kubernetes/pki/karmada.key + - --tls-cert-file=/etc/kubernetes/pki/karmada.crt + - --tls-private-key-file=/etc/kubernetes/pki/karmada.key + - --audit-log-path=- + - --feature-gates=APIPriorityAndFairness=false + - --audit-log-maxage=0 + - --audit-log-maxbackup=0 + resources: + requests: + cpu: 100m + volumes: + - name: k8s-certs + secret: + secretName: karmada-cert-secret + - name: kubeconfig + secret: + secretName: kubeconfig +--- +apiVersion: v1 +kind: Service +metadata: + name: karmada-aggregated-apiserver + namespace: karmada-system + labels: + app: karmada-aggregated-apiserver + apiserver: "true" +spec: + ports: + - port: 443 + protocol: TCP + targetPort: 443 + selector: + app: karmada-aggregated-apiserver diff --git a/artifacts/deploy/karmada-apiserver.yaml b/artifacts/deploy/karmada-apiserver.yaml index 72a8f26b9..50e15efb2 100644 --- a/artifacts/deploy/karmada-apiserver.yaml +++ b/artifacts/deploy/karmada-apiserver.yaml @@ -54,10 +54,10 @@ spec: - --service-account-key-file=/etc/kubernetes/pki/karmada.key - --service-account-signing-key-file=/etc/kubernetes/pki/karmada.key - --service-cluster-ip-range=10.96.0.0/12 - - --proxy-client-cert-file=/etc/kubernetes/pki/karmada.crt - - --proxy-client-key-file=/etc/kubernetes/pki/karmada.key + - --proxy-client-cert-file=/etc/kubernetes/pki/front-proxy-client.crt + - --proxy-client-key-file=/etc/kubernetes/pki/front-proxy-client.key - --requestheader-allowed-names=front-proxy-client - - --requestheader-client-ca-file=/etc/kubernetes/pki/server-ca.crt + - --requestheader-client-ca-file=/etc/kubernetes/pki/front-proxy-ca.crt - --requestheader-extra-headers-prefix=X-Remote-Extra- - --requestheader-group-headers=X-Remote-Group - --requestheader-username-headers=X-Remote-User @@ -111,9 +111,7 @@ spec: - name: k8s-certs secret: secretName: karmada-cert-secret - --- - apiVersion: v1 kind: Service metadata: diff --git a/artifacts/deploy/karmada-cert-secret.yaml b/artifacts/deploy/karmada-cert-secret.yaml index dafeaecd0..92cda6ea1 100644 --- a/artifacts/deploy/karmada-cert-secret.yaml +++ b/artifacts/deploy/karmada-cert-secret.yaml @@ -11,3 +11,9 @@ data: {{client_cer}} karmada.key: | {{client_key}} + front-proxy-ca.crt: | + {{front_proxy_ca_crt}} + front-proxy-client.crt: | + {{front_proxy_client_crt}} + front-proxy-client.key: | + {{front_proxy_client_key}} diff --git a/charts/_crds/bases/cluster.karmada.io_clusters.yaml b/charts/_crds/bases/cluster.karmada.io_clusters.yaml deleted file mode 100644 index 94ad8eea1..000000000 --- a/charts/_crds/bases/cluster.karmada.io_clusters.yaml +++ /dev/null @@ -1,310 +0,0 @@ - ---- -apiVersion: apiextensions.k8s.io/v1 -kind: CustomResourceDefinition -metadata: - annotations: - controller-gen.kubebuilder.io/version: v0.6.2 - creationTimestamp: null - name: clusters.cluster.karmada.io -spec: - group: cluster.karmada.io - names: - kind: Cluster - listKind: ClusterList - plural: clusters - singular: cluster - scope: Cluster - versions: - - additionalPrinterColumns: - - jsonPath: .status.kubernetesVersion - name: Version - type: string - - jsonPath: .spec.syncMode - name: Mode - type: string - - jsonPath: .status.conditions[?(@.type=="Ready")].status - name: Ready - type: string - - jsonPath: .metadata.creationTimestamp - name: Age - type: date - name: v1alpha1 - schema: - openAPIV3Schema: - description: Cluster represents the desire state and status of a member cluster. - properties: - apiVersion: - description: 'APIVersion defines the versioned schema of this representation - of an object. Servers should convert recognized schemas to the latest - internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' - type: string - kind: - description: 'Kind is a string value representing the REST resource this - object represents. Servers may infer this from the endpoint the client - submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' - type: string - metadata: - type: object - spec: - description: Spec represents the specification of the desired behavior - of member cluster. - properties: - apiEndpoint: - description: The API endpoint of the member cluster. This can be a - hostname, hostname:port, IP or IP:port. - type: string - insecureSkipTLSVerification: - description: InsecureSkipTLSVerification indicates that the karmada - control plane should not confirm the validity of the serving certificate - of the cluster it is connecting to. This will make the HTTPS connection - between the karmada control plane and the member cluster insecure. - Defaults to false. - type: boolean - provider: - description: Provider represents the cloud provider name of the member - cluster. - type: string - proxyURL: - description: 'ProxyURL is the proxy URL for the cluster. If not empty, - the karmada control plane will use this proxy to talk to the cluster. - More details please refer to: https://github.com/kubernetes/client-go/issues/351' - type: string - region: - description: Region represents the region of the member cluster locate - in. - type: string - secretRef: - description: 'SecretRef represents the secret contains mandatory credentials - to access the member cluster. The secret should hold credentials - as follows: - secret.data.token - secret.data.caBundle' - properties: - name: - description: Name is the name of resource being referenced. - type: string - namespace: - description: Namespace is the namespace for the resource being - referenced. - type: string - required: - - name - - namespace - type: object - syncMode: - description: SyncMode describes how a cluster sync resources from - karmada control plane. - enum: - - Push - - Pull - type: string - taints: - description: Taints attached to the member cluster. Taints on the - cluster have the "effect" on any resource that does not tolerate - the Taint. - items: - description: The node this Taint is attached to has the "effect" - on any pod that does not tolerate the Taint. - properties: - effect: - description: Required. The effect of the taint on pods that - do not tolerate the taint. Valid effects are NoSchedule, PreferNoSchedule - and NoExecute. - type: string - key: - description: Required. The taint key to be applied to a node. - type: string - timeAdded: - description: TimeAdded represents the time at which the taint - was added. It is only written for NoExecute taints. - format: date-time - type: string - value: - description: The taint value corresponding to the taint key. - type: string - required: - - effect - - key - type: object - type: array - zone: - description: Zone represents the zone of the member cluster locate - in. - type: string - required: - - syncMode - type: object - status: - description: Status represents the status of member cluster. - properties: - apiEnablements: - description: APIEnablements represents the list of APIs installed - in the member cluster. - items: - description: APIEnablement is a list of API resource, it is used - to expose the name of the resources supported in a specific group - and version. - properties: - groupVersion: - description: GroupVersion is the group and version this APIEnablement - is for. - type: string - resources: - description: Resources is a list of APIResource. - items: - description: APIResource specifies the name and kind names - for the resource. - properties: - kind: - description: Kind is the kind for the resource (e.g. 'Deployment' - is the kind for resource 'deployments') - type: string - name: - description: Name is the plural name of the resource. - type: string - required: - - kind - - name - type: object - type: array - required: - - groupVersion - type: object - type: array - conditions: - description: Conditions is an array of current cluster conditions. - items: - description: "Condition contains details for one aspect of the current - state of this API Resource. --- This struct is intended for direct - use as an array at the field path .status.conditions. For example, - type FooStatus struct{ // Represents the observations of a - foo's current state. // Known .status.conditions.type are: - \"Available\", \"Progressing\", and \"Degraded\" // +patchMergeKey=type - \ // +patchStrategy=merge // +listType=map // +listMapKey=type - \ Conditions []metav1.Condition `json:\"conditions,omitempty\" - patchStrategy:\"merge\" patchMergeKey:\"type\" protobuf:\"bytes,1,rep,name=conditions\"` - \n // other fields }" - properties: - lastTransitionTime: - description: lastTransitionTime is the last time the condition - transitioned from one status to another. This should be when - the underlying condition changed. If that is not known, then - using the time when the API field changed is acceptable. - format: date-time - type: string - message: - description: message is a human readable message indicating - details about the transition. This may be an empty string. - maxLength: 32768 - type: string - observedGeneration: - description: observedGeneration represents the .metadata.generation - that the condition was set based upon. For instance, if .metadata.generation - is currently 12, but the .status.conditions[x].observedGeneration - is 9, the condition is out of date with respect to the current - state of the instance. - format: int64 - minimum: 0 - type: integer - reason: - description: reason contains a programmatic identifier indicating - the reason for the condition's last transition. Producers - of specific condition types may define expected values and - meanings for this field, and whether the values are considered - a guaranteed API. The value should be a CamelCase string. - This field may not be empty. - maxLength: 1024 - minLength: 1 - pattern: ^[A-Za-z]([A-Za-z0-9_,:]*[A-Za-z0-9_])?$ - type: string - status: - description: status of the condition, one of True, False, Unknown. - enum: - - "True" - - "False" - - Unknown - type: string - type: - description: type of condition in CamelCase or in foo.example.com/CamelCase. - --- Many .condition.type values are consistent across resources - like Available, but because arbitrary conditions can be useful - (see .node.status.conditions), the ability to deconflict is - important. The regex it matches is (dns1123SubdomainFmt/)?(qualifiedNameFmt) - maxLength: 316 - pattern: ^([a-z0-9]([-a-z0-9]*[a-z0-9])?(\.[a-z0-9]([-a-z0-9]*[a-z0-9])?)*/)?(([A-Za-z0-9][-A-Za-z0-9_.]*)?[A-Za-z0-9])$ - type: string - required: - - lastTransitionTime - - message - - reason - - status - - type - type: object - type: array - kubernetesVersion: - description: KubernetesVersion represents version of the member cluster. - type: string - nodeSummary: - description: NodeSummary represents the summary of nodes status in - the member cluster. - properties: - readyNum: - description: ReadyNum is the number of ready nodes in the cluster. - format: int32 - type: integer - totalNum: - description: TotalNum is the total number of nodes in the cluster. - format: int32 - type: integer - type: object - resourceSummary: - description: ResourceSummary represents the summary of resources in - the member cluster. - properties: - allocatable: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Allocatable represents the resources of a cluster - that are available for scheduling. Total amount of allocatable - resources on all nodes. - type: object - allocated: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Allocated represents the resources of a cluster that - have been scheduled. Total amount of required resources of all - Pods that have been scheduled to nodes. - type: object - allocating: - additionalProperties: - anyOf: - - type: integer - - type: string - pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ - x-kubernetes-int-or-string: true - description: Allocating represents the resources of a cluster - that are pending for scheduling. Total amount of required resources - of all Pods that are waiting for scheduling. - type: object - type: object - type: object - required: - - spec - type: object - served: true - storage: true - subresources: - status: {} -status: - acceptedNames: - kind: "" - plural: "" - conditions: [] - storedVersions: [] diff --git a/charts/_crds/kustomization.yaml b/charts/_crds/kustomization.yaml index 1246f3ac9..57ab9b487 100644 --- a/charts/_crds/kustomization.yaml +++ b/charts/_crds/kustomization.yaml @@ -1,5 +1,4 @@ resources: -- bases/cluster.karmada.io_clusters.yaml - bases/multicluster.x-k8s.io_serviceexports.yaml - bases/multicluster.x-k8s.io_serviceimports.yaml - bases/policy.karmada.io_clusteroverridepolicies.yaml diff --git a/cluster/images/karmada-aggregated-apiserver/Dockerfile b/cluster/images/karmada-aggregated-apiserver/Dockerfile new file mode 100644 index 000000000..acd47f6fe --- /dev/null +++ b/cluster/images/karmada-aggregated-apiserver/Dockerfile @@ -0,0 +1,7 @@ +FROM alpine:3.7 + +RUN apk add --no-cache ca-certificates + +ADD karmada-aggregated-apiserver /bin/ + +CMD ["/bin/karmada-aggregated-apiserver"] diff --git a/go.mod b/go.mod index ccc674c86..5e9079932 100644 --- a/go.mod +++ b/go.mod @@ -35,6 +35,6 @@ require ( sigs.k8s.io/controller-runtime v0.10.3 sigs.k8s.io/kind v0.11.1 sigs.k8s.io/mcs-api v0.1.0 - sigs.k8s.io/yaml v1.3.0 sigs.k8s.io/structured-merge-diff/v4 v4.1.2 + sigs.k8s.io/yaml v1.3.0 ) diff --git a/hack/deploy-karmada.sh b/hack/deploy-karmada.sh index 6b699482d..d0107ad8a 100755 --- a/hack/deploy-karmada.sh +++ b/hack/deploy-karmada.sh @@ -86,6 +86,10 @@ function generate_cert_secret { sed -i'' -e "s/{{client_cer}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml + sed -i'' -e "s/{{front_proxy_ca_crt}}/${FRONT_PROXY_CA_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml + sed -i'' -e "s/{{front_proxy_client_crt}}/${FRONT_PROXY_CLIENT_CRT}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml + sed -i'' -e "s/{{front_proxy_client_key}}/${FRONT_PROXY_CLIENT_KEY}/g" "${TEMP_PATH}"/karmada-cert-secret-tmp.yaml + sed -i'' -e "s/{{ca_crt}}/${karmada_ca}/g" "${TEMP_PATH}"/secret-tmp.yaml sed -i'' -e "s/{{client_cer}}/${KARMADA_CRT}/g" "${TEMP_PATH}"/secret-tmp.yaml sed -i'' -e "s/{{client_key}}/${KARMADA_KEY}/g" "${TEMP_PATH}"/secret-tmp.yaml @@ -113,8 +117,10 @@ util::cmd_must_exist "openssl" util::cmd_must_exist_cfssl ${CFSSL_VERSION} # create CA signers util::create_signing_certkey "" "${CERT_DIR}" server '"client auth","server auth"' +util::create_signing_certkey "" "${CERT_DIR}" front-proxy '"client auth","server auth"' # signs a certificate util::create_certkey "" "${CERT_DIR}" "server-ca" karmada system:admin kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" +util::create_certkey "" "${CERT_DIR}" "front-proxy-ca" front-proxy-client front-proxy-client kubernetes.default.svc "*.etcd.karmada-system.svc.cluster.local" "*.karmada-system.svc.cluster.local" "*.karmada-system.svc" "localhost" "127.0.0.1" # create namespace for control plane components kubectl apply -f "${REPO_ROOT}/artifacts/deploy/namespace.yaml" @@ -126,6 +132,9 @@ kubectl apply -f "${REPO_ROOT}/artifacts/deploy/clusterrolebinding.yaml" KARMADA_CRT=$(base64 "${CERT_DIR}/karmada.crt" | tr -d '\r\n') KARMADA_KEY=$(base64 "${CERT_DIR}/karmada.key" | tr -d '\r\n') +FRONT_PROXY_CA_CRT=$(base64 "${CERT_DIR}/front-proxy-ca.crt" | tr -d '\r\n') +FRONT_PROXY_CLIENT_CRT=$(base64 "${CERT_DIR}/front-proxy-client.crt" | tr -d '\r\n') +FRONT_PROXY_CLIENT_KEY=$(base64 "${CERT_DIR}/front-proxy-client.key" | tr -d '\r\n') generate_cert_secret # deploy karmada etcd @@ -191,6 +200,9 @@ util::append_client_kubeconfig "${HOST_CLUSTER_KUBECONFIG}" "${CERT_DIR}/karmada # deploy kube controller manager kubectl apply -f "${REPO_ROOT}/artifacts/deploy/kube-controller-manager.yaml" +# deploy aggregated-apiserver on host cluster +kubectl apply -f "${REPO_ROOT}/artifacts/deploy/karmada-aggregated-apiserver.yaml" +util::wait_pod_ready "${KARMADA_AGGREGATION_APISERVER_LABEL}" "${KARMADA_SYSTEM_NAMESPACE}" # install CRD APIs on karmada apiserver. if ! kubectl config use-context karmada-apiserver > /dev/null 2>&1; @@ -210,6 +222,11 @@ rm -rf "${TEMP_PATH_CRDS}" # deploy webhook configurations on karmada apiserver util::deploy_webhook_configuration "${ROOT_CA_FILE}" "${REPO_ROOT}/artifacts/deploy/webhook-configuration.yaml" +# deploy APIService on karmada apiserver for karmada-aggregated-apiserver +kubectl apply -f "${REPO_ROOT}/artifacts/deploy/apiservice.yaml" +# make sure apiservice for v1alpha1.cluster.karmada.io is Available +util::wait_apiservice_ready "${KARMADA_AGGREGATION_APISERVER_LABEL}" + kubectl config use-context "${HOST_CLUSTER_NAME}" # deploy controller-manager on host cluster diff --git a/hack/local-up-karmada.sh b/hack/local-up-karmada.sh index 0f64753db..b2379a12f 100755 --- a/hack/local-up-karmada.sh +++ b/hack/local-up-karmada.sh @@ -47,6 +47,7 @@ dockerfile_list=( # Dockerfile files need to be replaced "cluster/images/karmada-scheduler-estimator/Dockerfile" "cluster/images/karmada-scheduler/Dockerfile" "cluster/images/karmada-webhook/Dockerfile" + "cluster/images/karmada-aggregated-apiserver/Dockerfile" ) for dockerfile in "${dockerfile_list[@]}"; do grep 'mirrors.ustc.edu.cn' ${REPO_ROOT}/${dockerfile} > /dev/null || sed -i'' -e "s#FROM alpine:3.7#FROM alpine:3.7\nRUN echo -e http://mirrors.ustc.edu.cn/alpine/v3.7/main/ > /etc/apk/repositories#" ${REPO_ROOT}/${dockerfile} @@ -122,6 +123,7 @@ kind load docker-image "${REGISTRY}/karmada-controller-manager:${VERSION}" --nam kind load docker-image "${REGISTRY}/karmada-scheduler:${VERSION}" --name="${HOST_CLUSTER_NAME}" kind load docker-image "${REGISTRY}/karmada-webhook:${VERSION}" --name="${HOST_CLUSTER_NAME}" kind load docker-image "${REGISTRY}/karmada-scheduler-estimator:${VERSION}" --name="${HOST_CLUSTER_NAME}" +kind load docker-image "${REGISTRY}/karmada-aggregated-apiserver:${VERSION}" --name="${HOST_CLUSTER_NAME}" #step5. install karmada control plane components "${REPO_ROOT}"/hack/deploy-karmada.sh "${MAIN_KUBECONFIG}" "${HOST_CLUSTER_NAME}" diff --git a/hack/update-codegen.sh b/hack/update-codegen.sh index 9d9f97a94..cd0f91a46 100755 --- a/hack/update-codegen.sh +++ b/hack/update-codegen.sh @@ -18,6 +18,11 @@ deepcopy-gen \ --input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1 \ --output-package=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1 \ --output-file-base=zz_generated.deepcopy +deepcopy-gen \ + --go-header-file hack/boilerplate/boilerplate.go.txt \ + --input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster \ + --output-package=github.com/karmada-io/karmada/pkg/apis/cluster \ + --output-file-base=zz_generated.deepcopy deepcopy-gen \ --go-header-file hack/boilerplate/boilerplate.go.txt \ --input-dirs=github.com/karmada-io/karmada/pkg/apis/policy/v1alpha1 \ @@ -77,6 +82,14 @@ register-gen \ --output-package=github.com/karmada-io/karmada/examples/customresourceinterpreter/apis/workload/v1alpha1 \ --output-file-base=zz_generated.register +echo "Generating with conversion-gen" +GO111MODULE=on go install k8s.io/code-generator/cmd/conversion-gen +conversion-gen \ + --go-header-file hack/boilerplate/boilerplate.go.txt \ + --input-dirs=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1 \ + --output-package=github.com/karmada-io/karmada/pkg/apis/cluster/v1alpha1 \ + --output-file-base=zz_generated.conversion + echo "Generating with client-gen" GO111MODULE=on go install k8s.io/code-generator/cmd/client-gen client-gen \ @@ -101,4 +114,3 @@ informer-gen \ --versioned-clientset-package=github.com/karmada-io/karmada/pkg/generated/clientset/versioned \ --listers-package=github.com/karmada-io/karmada/pkg/generated/listers \ --output-package=github.com/karmada-io/karmada/pkg/generated/informers - diff --git a/hack/update-crdgen.sh b/hack/update-crdgen.sh index 59ef69045..d7b586c6d 100755 --- a/hack/update-crdgen.sh +++ b/hack/update-crdgen.sh @@ -13,5 +13,7 @@ echo "Generating with controller-gen" util::install_tools ${CONTROLLER_GEN_PKG} ${CONTROLLER_GEN_VER} >/dev/null 2>&1 # Unify the crds used by helm chart and the installation scripts -controller-gen crd paths=./pkg/apis/... output:crd:dir=./charts/_crds/bases +controller-gen crd paths=./pkg/apis/config/... output:crd:dir=./charts/_crds/bases +controller-gen crd paths=./pkg/apis/policy/... output:crd:dir=./charts/_crds/bases +controller-gen crd paths=./pkg/apis/work/... output:crd:dir=./charts/_crds/bases controller-gen crd paths=./examples/customresourceinterpreter/apis/... output:crd:dir=./examples/customresourceinterpreter/apis/ diff --git a/hack/util.sh b/hack/util.sh index ad17781ca..008031985 100755 --- a/hack/util.sh +++ b/hack/util.sh @@ -10,6 +10,7 @@ KARMADA_SYSTEM_NAMESPACE="karmada-system" ETCD_POD_LABEL="etcd" APISERVER_POD_LABEL="karmada-apiserver" KUBE_CONTROLLER_POD_LABEL="kube-controller-manager" +KARMADA_AGGREGATION_APISERVER_LABEL="karmada-aggregated-apiserver" KARMADA_CONTROLLER_LABEL="karmada-controller-manager" KARMADA_SCHEDULER_LABEL="karmada-scheduler" KARMADA_WEBHOOK_LABEL="karmada-webhook" @@ -301,6 +302,25 @@ function util::wait_pod_ready() { return ${ret} } +# util::wait_apiservice_ready waits for apiservice state becomes Available until timeout. +# Parmeters: +# - $1: apiservice label, such as "app=etcd" +# - $3: time out, such as "200s" +function util::wait_apiservice_ready() { + local apiservice_label=$1 + + echo "wait the $apiservice_label Available..." + set +e + util::kubectl_with_retry wait --for=condition=Available --timeout=30s apiservices -l app=${apiservice_label} + ret=$? + set -e + if [ $ret -ne 0 ];then + echo "kubectl describe info:" + kubectl describe apiservices -l app=${apiservice_label} + fi + return ${ret} +} + # util::kubectl_with_retry will retry if execute kubectl command failed # tolerate kubectl command failure that may happen before the pod is created by StatefulSet/Deployment. function util::kubectl_with_retry() {