diff --git a/.github/workflows/e2e-kruise.yaml b/.github/workflows/e2e-kruise.yaml index e14a1df..321ebef 100644 --- a/.github/workflows/e2e-kruise.yaml +++ b/.github/workflows/e2e-kruise.yaml @@ -17,6 +17,7 @@ env: KIND_VERSION: 'v0.18.0' KIND_VERSION_FOR_HIGHER: 'v0.22.0' KIND_CLUSTER_NAME: 'ci-testing' + CERT_MANAGER_VERSION: 'v1.18.2' # todo: add kruise e2e here jobs: # 1.27- @@ -36,6 +37,10 @@ jobs: cluster_name: ${{ env.KIND_CLUSTER_NAME }} config: ./test/kind-conf.yaml version: ${{ env.KIND_VERSION }} + - name: Install Cert-Manager + run: | + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/${{ env.CERT_MANAGER_VERSION }}/cert-manager.yaml + kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s - name: Install Kruise run: | make install-kruise-from-local @@ -67,6 +72,10 @@ jobs: cluster_name: ${{ env.KIND_CLUSTER_NAME }} config: ./test/kind-conf-with-vpa.yaml version: ${{ env.KIND_VERSION_FOR_HIGHER }} + - name: Install Cert-Manager + run: | + kubectl apply -f https://github.com/cert-manager/cert-manager/releases/download/${{ env.CERT_MANAGER_VERSION }}/cert-manager.yaml + kubectl -n cert-manager rollout status deploy/cert-manager-webhook --timeout=180s - name: Install Kruise run: | make install-kruise-from-local diff --git a/versions/kruise-game/next/Chart.yaml b/versions/kruise-game/next/Chart.yaml index 533c6cd..a5f9fb3 100644 --- a/versions/kruise-game/next/Chart.yaml +++ b/versions/kruise-game/next/Chart.yaml @@ -8,4 +8,5 @@ sources: - https://github.com/openkruise/kruise-game annotations: artifacthub.io/changes: | - - "[Changed]: https://github.com/openkruise/kruise-game/blob/master/CHANGELOG.md" \ No newline at end of file + - "[Changed]: https://github.com/openkruise/kruise-game/blob/master/CHANGELOG.md" + - "[Added]: Support for cert-manager with CA injection" \ No newline at end of file diff --git a/versions/kruise-game/next/README.md b/versions/kruise-game/next/README.md index 5d95275..c6ee716 100644 --- a/versions/kruise-game/next/README.md +++ b/versions/kruise-game/next/README.md @@ -31,10 +31,56 @@ The following table lists the configurable parameters of the kruise-game chart a | `network.totalWaitTime` | Maximum time to wait for network ready, the unit is seconds | `60` | | `network.probeIntervalTime` | Time interval for detecting network status, the unit is seconds | `5` | | `cloudProvider.installCRD` | Whether to install CloudProvider CRD | `true` | +| `certificates.autoGenerated` | Whether to auto-generate webhook certificates | `true` | +| `certificates.secretName` | Name of the secret containing webhook certificates | `kruise-game-certs` | +| `certificates.mountPath` | Path to mount webhook certificates in container | `/tmp/webhook-certs/` | +| `certificates.certManager.enabled` | Whether to use cert-manager for certificate management | `false` | +| `certificates.certManager.duration` | Certificate validity duration | `8760h0m0s` | +| `certificates.certManager.renewBefore` | Time before expiry to renew certificate | `5840h0m0s` | +| `certificates.certManager.generateCA` | Whether to generate a Certificate Authority | `true` | +| `certificates.certManager.caSecretName` | Name of the secret containing the CA certificate | `kruise-game-ca` | +| `certificates.certManager.issuer.generate` | Whether to generate the issuer automatically | `true` | +| `certificates.certManager.issuer.name` | Name of the certificate issuer | `kruise-ca` | +| `certificates.certManager.issuer.kind` | Type of the certificate issuer | `ClusterIssuer` | +| `certificates.certManager.issuer.group` | API group of the certificate issuer | `cert-manager.io` | Specify each parameter using the `--set key=value[,key=value]` argument to `helm install`. For example, +### Certificate Management + +> **Important**: Kruise Game webhook requires TLS certificates for secure communication. Regardless of which certificate management method you choose, you must ensure that the webhook certificate is signed by a trusted CA certificate, and that the CA certificate is properly configured in the Kubernetes cluster so that the API Server can verify the webhook's identity. + +Kruise Game supports two methods for webhook certificate management: + +#### Auto-generated Certificates (Default) + +By default, kruise-game uses auto-generated certificates for webhook TLS: + +```bash +$ helm install kruise-game https://... --set certificates.autoGenerated=true +``` + +#### cert-manager Integration + +For production environments, you can use cert-manager to manage webhook certificates: + +```bash +$ helm install kruise-game https://... \ + --set certificates.autoGenerated=false \ + --set certificates.certManager.enabled=true \ +``` + +You can also use a custom issuer instead of generating one: + +```bash +$ helm install kruise-game https://... \ + --set certificates.certManager.enabled=true \ + --set certificates.certManager.issuer.generate=false \ + --set certificates.certManager.issuer.name=my-custom-issuer \ + --set certificates.certManager.issuer.kind=Issuer +``` + ### Optional: the local image for China If you are in China and have problem to pull image from official DockerHub, you can use the registry hosted on Alibaba Cloud: diff --git a/versions/kruise-game/next/templates/cert-manager/okg-issuer.yaml b/versions/kruise-game/next/templates/cert-manager/okg-issuer.yaml new file mode 100644 index 0000000..efeb542 --- /dev/null +++ b/versions/kruise-game/next/templates/cert-manager/okg-issuer.yaml @@ -0,0 +1,10 @@ +{{- if and .Values.certificates.certManager.enabled .Values.certificates.certManager.issuer.generate }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + name: {{ .Values.kruiseGame.fullname }}-issuer + namespace: {{ .Values.installation.namespace }} +spec: + ca: + secretName: {{ .Values.certificates.certManager.caSecretName }} +{{- end }} diff --git a/versions/kruise-game/next/templates/cert-manager/okg-tls-certificate.yaml b/versions/kruise-game/next/templates/cert-manager/okg-tls-certificate.yaml new file mode 100644 index 0000000..cb441ff --- /dev/null +++ b/versions/kruise-game/next/templates/cert-manager/okg-tls-certificate.yaml @@ -0,0 +1,39 @@ +{{- if .Values.certificates.certManager.enabled }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.kruiseGame.fullname }}-cert + namespace: {{ .Values.installation.namespace }} +spec: + commonName: {{ .Values.kruiseGame.fullname }} + dnsNames: + - {{ .Values.kruiseGame.webhook.serviceName }}.{{ .Values.installation.namespace }} + - {{ .Values.kruiseGame.webhook.serviceName }}.{{ .Values.installation.namespace }}.svc + - {{ .Values.kruiseGame.webhook.serviceName }}.{{ .Values.installation.namespace }}.svc.{{ .Values.clusterDomain }} + secretName: {{ .Values.certificates.secretName }} + usages: + - server auth + - client auth + privateKey: + rotationPolicy: Always + algorithm: RSA + size: 2048 + duration: {{ .Values.certificates.certManager.duration }} + renewBefore: {{ .Values.certificates.certManager.renewBefore }} + issuerRef: + {{- if .Values.certificates.certManager.issuer.generate }} + name: {{ .Values.kruiseGame.fullname }}-issuer + kind: Issuer + group: cert-manager.io + {{- else }} + {{- if .Values.certificates.certManager.issuer.name }} + name: {{ .Values.certificates.certManager.issuer.name }} + {{- end }} + {{- if .Values.certificates.certManager.issuer.kind }} + kind: {{ .Values.certificates.certManager.issuer.kind }} + {{- end }} + {{- if .Values.certificates.certManager.issuer.group }} + group: {{ .Values.certificates.certManager.issuer.group }} + {{- end }} + {{- end }} +{{- end }} diff --git a/versions/kruise-game/next/templates/cert-manager/self-ca.yaml b/versions/kruise-game/next/templates/cert-manager/self-ca.yaml new file mode 100644 index 0000000..82fc605 --- /dev/null +++ b/versions/kruise-game/next/templates/cert-manager/self-ca.yaml @@ -0,0 +1,21 @@ +{{- if and .Values.certificates.certManager.enabled .Values.certificates.certManager.generateCA .Values.certificates.certManager.issuer.generate }} +apiVersion: cert-manager.io/v1 +kind: Certificate +metadata: + name: {{ .Values.kruiseGame.fullname }}-ca + namespace: {{ .Values.installation.namespace }} +spec: + isCA: true + commonName: {{ .Values.kruiseGame.fullname }} + secretName: {{ .Values.certificates.certManager.caSecretName }} + privateKey: + rotationPolicy: Always + algorithm: RSA + size: 2048 + duration: 8760h0m0s # 1 year + renewBefore: 720h0m0s # 1 month + issuerRef: + name: {{ .Values.kruiseGame.fullname }}-selfsigned-issuer + kind: Issuer + group: cert-manager.io +{{- end }} diff --git a/versions/kruise-game/next/templates/cert-manager/self-issuer.yaml b/versions/kruise-game/next/templates/cert-manager/self-issuer.yaml new file mode 100644 index 0000000..d6e1904 --- /dev/null +++ b/versions/kruise-game/next/templates/cert-manager/self-issuer.yaml @@ -0,0 +1,13 @@ +{{- if and .Values.certificates.certManager.enabled .Values.certificates.certManager.generateCA .Values.certificates.certManager.issuer.generate }} +apiVersion: cert-manager.io/v1 +kind: Issuer +metadata: + {{- with .Values.additionalAnnotations }} + annotations: + {{- toYaml . | nindent 4 }} + {{- end }} + name: {{ .Values.kruiseGame.fullname }}-selfsigned-issuer + namespace: {{ .Values.installation.namespace }} +spec: + selfSigned: {} +{{- end }} diff --git a/versions/kruise-game/next/templates/manager.yaml b/versions/kruise-game/next/templates/manager.yaml index 6e56313..9495145 100644 --- a/versions/kruise-game/next/templates/manager.yaml +++ b/versions/kruise-game/next/templates/manager.yaml @@ -59,10 +59,15 @@ spec: - --provider-config=/etc/kruise-game/config.toml - --api-server-qps={{ .Values.kruiseGame.apiServerQps }} - --api-server-qps-burst={{ .Values.kruiseGame.apiServerQpsBurst }} + - --gameserver-workers={{ .Values.kruiseGame.gameserverWorkers }} + - --gameserverset-workers={{ .Values.kruiseGame.gameserversetWorkers }} - --scale-server-bind-address=:{{ .Values.scale.service.targetPort }} {{- if .Values.prometheus.enabled }} - --metrics-bind-address=:{{ .Values.prometheus.monitorService.port }} {{- end }} + {{- if not .Values.certificates.autoGenerated }} + - --enable-cert-generation={{ .Values.certificates.autoGenerated }} + {{- end }} image: "{{ .Values.image.repository }}:{{ .Values.image.tag | default .Chart.AppVersion }}" imagePullPolicy: {{ .Values.image.pullPolicy }} name: manager @@ -98,17 +103,10 @@ spec: volumeMounts: - mountPath: /etc/kruise-game name: provider-config - topologySpreadConstraints: - - labelSelector: - matchLabels: - control-plane: {{ .Values.kruiseGame.fullname }} -{{- if and ( eq (int .Capabilities.KubeVersion.Major) 1) ( gt (int .Capabilities.KubeVersion.Minor) 26 ) }} - matchLabelKeys: - - pod-template-hash -{{- end }} - maxSkew: 1 - topologyKey: topology.kubernetes.io/zone - whenUnsatisfiable: ScheduleAnyway + {{- if not .Values.certificates.autoGenerated }} + - mountPath: {{ .Values.certificates.mountPath }} + name: certificates + {{- end }} serviceAccountName: {{ .Values.kruiseGame.fullname }} terminationGracePeriodSeconds: 10 volumes: @@ -119,3 +117,10 @@ spec: path: config.toml name: kruise-game-manager-config name: provider-config + {{- if not .Values.certificates.autoGenerated }} + - name: certificates + secret: + defaultMode: 420 + secretName: {{ .Values.certificates.secretName}} + optional: {{ and .Values.certificates.autoGenerated ( not .Values.certificates.certManager.enabled ) }} + {{- end }} \ No newline at end of file diff --git a/versions/kruise-game/next/templates/webhooks/mutatingconfiguration.yaml b/versions/kruise-game/next/templates/webhooks/mutatingconfiguration.yaml new file mode 100644 index 0000000..3cbacb9 --- /dev/null +++ b/versions/kruise-game/next/templates/webhooks/mutatingconfiguration.yaml @@ -0,0 +1,44 @@ +{{- if not .Values.certificates.autoGenerated }} +apiVersion: admissionregistration.k8s.io/v1 +kind: MutatingWebhookConfiguration +metadata: + annotations: + {{- if .Values.certificates.certManager.enabled }} + {{- if and (not .Values.certificates.certManager.generateCA) .Values.certificates.certManager.issuer.generate }} + cert-manager.io/inject-ca-from-secret: {{ .Values.installation.namespace }}/{{ .Values.certificates.certManager.caSecretName }} + {{- else }} + cert-manager.io/inject-ca-from: {{ .Values.installation.namespace }}/{{ .Values.kruiseGame.fullname }}-cert + {{- end }} + {{- end }} + labels: + app.kubernetes.io/name: {{ .Values.kruiseGame.fullname }} + name: kruise-game-mutating-webhook +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: {{ .Values.kruiseGame.webhook.serviceName }} + namespace: {{ .Values.installation.namespace }} + path: /mutate-v1-pod + failurePolicy: {{ .Values.kruiseGame.webhook.failurePolicy }} + matchPolicy: Equivalent + name: mgameserverset.kb.io + rules: + - operations: + - CREATE + - UPDATE + - DELETE + apiGroups: + - "" + apiVersions: + - v1 + resources: + - pods + objectSelector: + matchExpressions: + - key: game.kruise.io/owner-gss + operator: Exists + sideEffects: None +{{- end}} \ No newline at end of file diff --git a/versions/kruise-game/next/templates/webhook_service.yaml b/versions/kruise-game/next/templates/webhooks/service.yaml similarity index 68% rename from versions/kruise-game/next/templates/webhook_service.yaml rename to versions/kruise-game/next/templates/webhooks/service.yaml index c8d6178..9a334f6 100644 --- a/versions/kruise-game/next/templates/webhook_service.yaml +++ b/versions/kruise-game/next/templates/webhooks/service.yaml @@ -2,7 +2,7 @@ apiVersion: v1 kind: Service metadata: - name: kruise-game-webhook-service + name: {{ .Values.kruiseGame.webhook.serviceName }} namespace: {{ .Values.installation.namespace }} spec: ports: diff --git a/versions/kruise-game/next/templates/webhooks/validatingconfiguration.yaml b/versions/kruise-game/next/templates/webhooks/validatingconfiguration.yaml new file mode 100644 index 0000000..daed2da --- /dev/null +++ b/versions/kruise-game/next/templates/webhooks/validatingconfiguration.yaml @@ -0,0 +1,42 @@ +{{- if not .Values.certificates.autoGenerated }} +apiVersion: admissionregistration.k8s.io/v1 +kind: ValidatingWebhookConfiguration +metadata: + annotations: + {{- if .Values.certificates.certManager.enabled }} + {{- if and (not .Values.certificates.certManager.generateCA) .Values.certificates.certManager.issuer.generate }} + cert-manager.io/inject-ca-from-secret: {{ .Values.installation.namespace }}/{{ .Values.certificates.certManager.caSecretName }} + {{- else }} + cert-manager.io/inject-ca-from: {{ .Values.installation.namespace }}/{{ .Values.kruiseGame.fullname }}-cert + {{- end }} + {{- end }} + labels: + app.kubernetes.io/name: {{ .Values.kruiseGame.fullname }} + name: kruise-game-validating-webhook +webhooks: +- admissionReviewVersions: + - v1 + - v1beta1 + clientConfig: + service: + name: {{ .Values.kruiseGame.webhook.serviceName }} + namespace: {{ .Values.installation.namespace }} + path: /validate-v1alpha1-gss + failurePolicy: {{ .Values.kruiseGame.webhook.failurePolicy }} + matchPolicy: Equivalent + name: vgameserverset.kb.io + namespaceSelector: {} + objectSelector: {} + rules: + - apiGroups: + - game.kruise.io + apiVersions: + - v1alpha1 + operations: + - CREATE + - UPDATE + resources: + - gameserversets + sideEffects: None + timeoutSeconds: 10 +{{- end}} \ No newline at end of file diff --git a/versions/kruise-game/next/values.yaml b/versions/kruise-game/next/values.yaml index 296c01a..35a5196 100644 --- a/versions/kruise-game/next/values.yaml +++ b/versions/kruise-game/next/values.yaml @@ -11,16 +11,20 @@ kruiseGame: fullname: kruise-game-controller-manager healthBindPort: "8082" webhook: + serviceName: kruise-game-webhook-service port: 443 targetPort: 9876 + failurePolicy: Fail apiServerQps: 5 apiServerQpsBurst: 10 + gameserverWorkers: 10 + gameserversetWorkers: 10 replicaCount: 1 image: repository: openkruise/kruise-game-manager - tag: v0.9.0 + tag: v1.0.0 pullPolicy: Always # Overrides the image tag whose default is the chart appVersion. @@ -63,4 +67,27 @@ network: probeIntervalTime: 5 cloudProvider: - installCRD: true \ No newline at end of file + installCRD: true + +indexOffsetScheduler: + enabled: false + +# Kubernetes cluster domain +clusterDomain: cluster.local + +certificates: + autoGenerated: true + secretName: kruise-game-certs + mountPath: /tmp/webhook-certs/ + certManager: + enabled: false + duration: 8760h0m0s # 1 year + renewBefore: 5840h0m0s # 8 months + generateCA: true + caSecretName: "kruise-game-ca" + # -- Reference to custom Issuer. If issuer.generate is false, then issuer.group, issuer.kind and issuer.name are required + issuer: + generate: true + name: kruise-ca + kind: ClusterIssuer + group: cert-manager.io \ No newline at end of file