feat: add networkType kubernetes-ingress (#54)

* feat: add networkType kubernetes-ingress

Signed-off-by: ChrisLiu <chrisliu1995@163.com>

* feat: network plugin kubernetes-ingress support multi paths

Signed-off-by: ChrisLiu <chrisliu1995@163.com>

* feat: add param <id> for host of Kuberntes-Ingress plugin

Signed-off-by: ChrisLiu <chrisliu1995@163.com>

---------

Signed-off-by: ChrisLiu <chrisliu1995@163.com>
This commit is contained in:
ChrisLiu 2023-05-22 16:38:45 +08:00 committed by GitHub
parent ab6aff7c0d
commit 07a26a9b81
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 1234 additions and 40 deletions

View File

@ -0,0 +1,385 @@
package kubernetes
import (
"context"
"encoding/json"
"fmt"
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider"
cperrors "github.com/openkruise/kruise-game/cloudprovider/errors"
"github.com/openkruise/kruise-game/cloudprovider/utils"
"github.com/openkruise/kruise-game/pkg/util"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/networking/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/pointer"
"sigs.k8s.io/controller-runtime/pkg/client"
"strconv"
"strings"
)
const (
IngressNetwork = "Kubernetes-Ingress"
// PathTypeKey determines the interpretation of the Path matching, which is same as PathType in HTTPIngressPath.
PathTypeKey = "PathType"
// PathKey is matched against the path of an incoming request, the meaning of which is same as Path in HTTPIngressPath.
// Users can add <id> to any position of the path, and the network plugin will generate the path corresponding to the game server.
// e.g. /game<id>(/|$)(.*) The ingress path of GameServer 0 is /game0(/|$)(.*), the ingress path of GameServer 1 is /game1(/|$)(.*), and so on.
PathKey = "Path"
// PortKey indicates the exposed port value of game server.
PortKey = "Port"
// IngressClassNameKey indicates the name of the IngressClass cluster resource, which is same as IngressClassName in IngressSpec.
IngressClassNameKey = "IngressClassName"
// HostKey indicates domain name, which is same as Host in IngressRule.
HostKey = "Host"
// TlsHostsKey indicates hosts that included in the TLS certificate, the meaning of which is the same as that of Hosts in IngressTLS.
// Its corresponding value format is as follows, host1,host2,... e.g. xxx.xx.com
TlsHostsKey = "TlsHosts"
// TlsSecretNameKey indicates the name of the secret used to terminate TLS traffic on port 443, which is same as SecretName in IngressTLS.
TlsSecretNameKey = "TlsSecretName"
// AnnotationKey is the key of an annotation for ingress.
// Its corresponding value format is as follows, key: value(note that there is a space after: ) e.g. nginx.ingress.kubernetes.io/rewrite-target: /$2
// If the user wants to fill in multiple annotations, just fill in multiple AnnotationKeys in the network config.
AnnotationKey = "Annotation"
)
const (
SvcSelectorKey = "statefulset.kubernetes.io/pod-name"
IngressHashKey = "game.kruise.io/ingress-hash"
ServiceHashKey = "game.kruise.io/svc-hash"
)
const paramsError = "Network Config Params Error"
func init() {
kubernetesProvider.registerPlugin(&IngressPlugin{})
}
type IngressPlugin struct {
}
func (i IngressPlugin) Name() string {
return IngressNetwork
}
func (i IngressPlugin) Alias() string {
return ""
}
func (i IngressPlugin) Init(client client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error {
return nil
}
func (i IngressPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
networkManager := utils.NewNetworkManager(pod, c)
conf := networkManager.GetNetworkConfig()
ic, err := parseIngConfig(conf, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ParameterError, err.Error())
}
err = c.Create(ctx, consSvc(ic, pod))
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
err = c.Create(ctx, consIngress(ic, pod))
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
return pod, nil
}
func (i IngressPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
networkManager := utils.NewNetworkManager(pod, c)
networkStatus, _ := networkManager.GetNetworkStatus()
if networkStatus == nil {
pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{
CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady,
}, pod)
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
}
conf := networkManager.GetNetworkConfig()
ic, err := parseIngConfig(conf, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ParameterError, err.Error())
}
// get svc
svc := &corev1.Service{}
err = c.Get(ctx, types.NamespacedName{
Name: pod.GetName(),
Namespace: pod.GetNamespace(),
}, svc)
if err != nil {
if errors.IsNotFound(err) {
return pod, cperrors.ToPluginError(c.Create(ctx, consSvc(ic, pod)), cperrors.ApiCallError)
}
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
// update svc
if util.GetHash(ic.ports) != svc.GetAnnotations()[ServiceHashKey] {
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}
newSvc := consSvc(ic, pod)
patchSvc := map[string]interface{}{"metadata": map[string]map[string]string{"annotations": newSvc.Annotations}, "spec": newSvc.Spec}
patchSvcBytes, err := json.Marshal(patchSvc)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}
return pod, cperrors.ToPluginError(c.Patch(ctx, svc, client.RawPatch(types.MergePatchType, patchSvcBytes)), cperrors.ApiCallError)
}
// get ingress
ing := &v1.Ingress{}
err = c.Get(ctx, types.NamespacedName{
Name: pod.GetName(),
Namespace: pod.GetNamespace(),
}, ing)
if err != nil {
if errors.IsNotFound(err) {
return pod, cperrors.ToPluginError(c.Create(ctx, consIngress(ic, pod)), cperrors.ApiCallError)
}
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
// update ingress
if util.GetHash(ic) != ing.GetAnnotations()[IngressHashKey] {
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}
return pod, cperrors.ToPluginError(c.Update(ctx, consIngress(ic, pod)), cperrors.ApiCallError)
}
// network not ready
if ing.Status.LoadBalancer.Ingress == nil {
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
}
// network ready
internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)
externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)
networkPorts := make([]gamekruiseiov1alpha1.NetworkPort, 0)
for _, p := range ing.Spec.Rules[0].HTTP.Paths {
instrIPort := intstr.FromInt(int(p.Backend.Service.Port.Number))
networkPort := gamekruiseiov1alpha1.NetworkPort{
Name: p.Path,
Port: &instrIPort,
}
networkPorts = append(networkPorts, networkPort)
}
internalAddress := gamekruiseiov1alpha1.NetworkAddress{
IP: pod.Status.PodIP,
Ports: networkPorts,
}
externalAddress := gamekruiseiov1alpha1.NetworkAddress{
IP: ing.Status.LoadBalancer.Ingress[0].IP,
EndPoint: ing.Spec.Rules[0].Host,
Ports: networkPorts,
}
networkStatus.InternalAddresses = append(internalAddresses, internalAddress)
networkStatus.ExternalAddresses = append(externalAddresses, externalAddress)
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
}
func (i IngressPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError {
return nil
}
type ingConfig struct {
paths []string
pathTypes []*v1.PathType
ports []int32
host string
ingressClassName *string
tlsHosts []string
tlsSecretName string
annotations map[string]string
}
func parseIngConfig(conf []gamekruiseiov1alpha1.NetworkConfParams, pod *corev1.Pod) (ingConfig, error) {
var ic ingConfig
ic.annotations = make(map[string]string)
ic.paths = make([]string, 0)
ic.pathTypes = make([]*v1.PathType, 0)
ic.ports = make([]int32, 0)
id := util.GetIndexFromGsName(pod.GetName())
for _, c := range conf {
switch c.Name {
case PathTypeKey:
pathType := v1.PathType(c.Value)
ic.pathTypes = append(ic.pathTypes, &pathType)
case PortKey:
port, _ := strconv.ParseInt(c.Value, 10, 32)
ic.ports = append(ic.ports, int32(port))
case HostKey:
strs := strings.Split(c.Value, "<id>")
switch len(strs) {
case 2:
ic.host = strs[0] + strconv.Itoa(id) + strs[1]
case 1:
ic.host = strs[0]
default:
return ingConfig{}, fmt.Errorf("%s", paramsError)
}
case IngressClassNameKey:
ic.ingressClassName = pointer.String(c.Value)
case TlsSecretNameKey:
ic.tlsSecretName = c.Value
case TlsHostsKey:
ic.tlsHosts = strings.Split(c.Value, ",")
case PathKey:
strs := strings.Split(c.Value, "<id>")
switch len(strs) {
case 2:
ic.paths = append(ic.paths, strs[0]+strconv.Itoa(id)+strs[1])
case 1:
ic.paths = append(ic.paths, strs[0])
default:
return ingConfig{}, fmt.Errorf("%s", paramsError)
}
case AnnotationKey:
kv := strings.Split(c.Value, ": ")
if len(kv) != 2 {
return ingConfig{}, fmt.Errorf("%s", paramsError)
}
ic.annotations[kv[0]] = kv[1]
}
}
if len(ic.paths) == 0 || len(ic.pathTypes) == 0 || len(ic.ports) == 0 {
return ingConfig{}, fmt.Errorf("%s", paramsError)
}
return ic, nil
}
func consIngress(ic ingConfig, pod *corev1.Pod) *v1.Ingress {
pathSlice := ic.paths
pathTypeSlice := ic.pathTypes
pathPortSlice := ic.ports
lenPathTypeSlice := len(pathTypeSlice)
lenPathPortSlice := len(pathPortSlice)
for i := 0; i < len(pathSlice)-lenPathTypeSlice; i++ {
pathTypeSlice = append(pathTypeSlice, pathTypeSlice[0])
}
for i := 0; i < len(pathSlice)-lenPathPortSlice; i++ {
pathPortSlice = append(pathPortSlice, pathPortSlice[0])
}
ingAnnotations := ic.annotations
if ingAnnotations == nil {
ingAnnotations = make(map[string]string)
}
ingAnnotations[IngressHashKey] = util.GetHash(ic)
ingPaths := make([]v1.HTTPIngressPath, 0)
for i := 0; i < len(pathSlice); i++ {
ingPath := v1.HTTPIngressPath{
Path: pathSlice[i],
PathType: pathTypeSlice[i],
Backend: v1.IngressBackend{
Service: &v1.IngressServiceBackend{
Name: pod.GetName(),
Port: v1.ServiceBackendPort{
Number: pathPortSlice[i],
},
},
},
}
ingPaths = append(ingPaths, ingPath)
}
ing := &v1.Ingress{
ObjectMeta: metav1.ObjectMeta{
Name: pod.GetName(),
Namespace: pod.GetNamespace(),
Annotations: ingAnnotations,
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: pod.APIVersion,
Kind: pod.Kind,
Name: pod.GetName(),
UID: pod.GetUID(),
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
},
},
},
Spec: v1.IngressSpec{
IngressClassName: ic.ingressClassName,
Rules: []v1.IngressRule{
{
Host: ic.host,
IngressRuleValue: v1.IngressRuleValue{
HTTP: &v1.HTTPIngressRuleValue{
Paths: ingPaths,
},
},
},
},
},
}
if ic.tlsHosts != nil || ic.tlsSecretName != "" {
ing.Spec.TLS = []v1.IngressTLS{
{
SecretName: ic.tlsSecretName,
Hosts: ic.tlsHosts,
},
}
}
return ing
}
func consSvc(ic ingConfig, pod *corev1.Pod) *corev1.Service {
annoatations := make(map[string]string)
annoatations[ServiceHashKey] = util.GetHash(ic.ports)
ports := make([]corev1.ServicePort, 0)
for _, p := range ic.ports {
port := corev1.ServicePort{
Port: p,
Name: strconv.Itoa(int(p)),
}
ports = append(ports, port)
}
return &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: pod.GetName(),
Namespace: pod.GetNamespace(),
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: pod.APIVersion,
Kind: pod.Kind,
Name: pod.GetName(),
UID: pod.GetUID(),
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
},
},
Annotations: annoatations,
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Selector: map[string]string{
SvcSelectorKey: pod.GetName(),
},
Ports: ports,
},
}
}

View File

@ -0,0 +1,434 @@
package kubernetes
import (
"fmt"
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/pkg/util"
corev1 "k8s.io/api/core/v1"
v1 "k8s.io/api/networking/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/utils/pointer"
"reflect"
"testing"
)
func TestParseIngConfig(t *testing.T) {
pathTypePrefix := v1.PathTypePrefix
tests := []struct {
conf []gamekruiseiov1alpha1.NetworkConfParams
pod *corev1.Pod
ic ingConfig
err error
}{
{
conf: []gamekruiseiov1alpha1.NetworkConfParams{
{
Name: PathKey,
Value: "/game<id>(/|$)(.*)",
},
{
Name: AnnotationKey,
Value: "nginx.ingress.kubernetes.io/rewrite-target: /$2",
},
{
Name: AnnotationKey,
Value: "alb.ingress.kubernetes.io/server-snippets: |\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";",
},
{
Name: TlsHostsKey,
Value: "xxx-xxx.com,xxx-xx.com",
},
{
Name: PortKey,
Value: "8080",
},
{
Name: PathTypeKey,
Value: string(v1.PathTypePrefix),
},
},
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod-3",
},
},
ic: ingConfig{
paths: []string{"/game3(/|$)(.*)"},
tlsHosts: []string{
"xxx-xxx.com",
"xxx-xx.com",
},
annotations: map[string]string{
"nginx.ingress.kubernetes.io/rewrite-target": "/$2",
"alb.ingress.kubernetes.io/server-snippets": "|\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";",
},
ports: []int32{8080},
pathTypes: []*v1.PathType{&pathTypePrefix},
},
},
{
conf: []gamekruiseiov1alpha1.NetworkConfParams{
{
Name: PathKey,
Value: "/game<id>",
},
{
Name: AnnotationKey,
Value: "nginx.ingress.kubernetes.io/rewrite-target: /$2",
},
{
Name: TlsHostsKey,
Value: "xxx-xxx.com,xxx-xx.com",
},
{
Name: PathTypeKey,
Value: string(v1.PathTypePrefix),
},
},
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod-3",
},
},
err: fmt.Errorf("%s", paramsError),
},
{
conf: []gamekruiseiov1alpha1.NetworkConfParams{
{
Name: PathKey,
Value: "/game",
},
{
Name: PortKey,
Value: "8080",
},
{
Name: PathTypeKey,
Value: string(v1.PathTypePrefix),
},
{
Name: HostKey,
Value: "instance<id>.xxx.xxx.com",
},
},
pod: &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "pod-2",
},
},
ic: ingConfig{
paths: []string{"/game"},
ports: []int32{8080},
pathTypes: []*v1.PathType{&pathTypePrefix},
host: "instance2.xxx.xxx.com",
annotations: map[string]string{},
},
},
}
for i, test := range tests {
expect := test.ic
actual, err := parseIngConfig(test.conf, test.pod)
if !reflect.DeepEqual(err, test.err) {
t.Errorf("case %d: expect err: %v , but actual: %v", i, test.err, err)
}
if !reflect.DeepEqual(actual, expect) {
if !reflect.DeepEqual(expect.paths, actual.paths) {
t.Errorf("case %d: expect paths: %v , but actual: %v", i, expect.paths, actual.paths)
}
if !reflect.DeepEqual(expect.ports, actual.ports) {
t.Errorf("case %d: expect ports: %v , but actual: %v", i, expect.ports, actual.ports)
}
if !reflect.DeepEqual(expect.pathTypes, actual.pathTypes) {
t.Errorf("case %d: expect annotations: %v , but actual: %v", i, expect.pathTypes, actual.pathTypes)
}
if !reflect.DeepEqual(expect.host, actual.host) {
t.Errorf("case %d: expect host: %v , but actual: %v", i, expect.host, actual.host)
}
if !reflect.DeepEqual(expect.tlsHosts, actual.tlsHosts) {
t.Errorf("case %d: expect tlsHosts: %v , but actual: %v", i, expect.tlsHosts, actual.tlsHosts)
}
if !reflect.DeepEqual(expect.tlsSecretName, actual.tlsSecretName) {
t.Errorf("case %d: expect tlsSecretName: %v , but actual: %v", i, expect.tlsSecretName, actual.tlsSecretName)
}
if !reflect.DeepEqual(expect.ingressClassName, actual.ingressClassName) {
t.Errorf("case %d: expect ingressClassName: %v , but actual: %v", i, expect.ingressClassName, actual.ingressClassName)
}
if !reflect.DeepEqual(expect.annotations, actual.annotations) {
t.Errorf("case %d: expect annotations: %v , but actual: %v", i, expect.annotations, actual.annotations)
}
}
}
}
func TestConsIngress(t *testing.T) {
pathTypePrefix := v1.PathTypePrefix
pathTypeImplementationSpecific := v1.PathTypeImplementationSpecific
ingressClassNameNginx := "nginx"
pod := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Pod",
},
ObjectMeta: metav1.ObjectMeta{
Name: "pod-3",
Namespace: "ns",
UID: "bff0afd6-bb30-4641-8607-8329547324eb",
},
}
baseIngObjectMeta := metav1.ObjectMeta{
Name: "pod-3",
Namespace: "ns",
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "v1",
Kind: "Pod",
Name: "pod-3",
UID: "bff0afd6-bb30-4641-8607-8329547324eb",
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
},
},
}
// case 0
icCase0 := ingConfig{
ports: []int32{
int32(80),
},
pathTypes: []*v1.PathType{
&pathTypePrefix,
},
paths: []string{
"/path1-3",
"/path2-3",
},
host: "xxx.xx.com",
ingressClassName: &ingressClassNameNginx,
annotations: map[string]string{
"nginx.ingress.kubernetes.io/rewrite-target": "/$2",
},
}
ingObjectMetaCase0 := baseIngObjectMeta
ingObjectMetaCase0.Annotations = map[string]string{
"nginx.ingress.kubernetes.io/rewrite-target": "/$2",
IngressHashKey: util.GetHash(icCase0),
}
ingCase0 := &v1.Ingress{
ObjectMeta: ingObjectMetaCase0,
Spec: v1.IngressSpec{
IngressClassName: &ingressClassNameNginx,
Rules: []v1.IngressRule{
{
Host: "xxx.xx.com",
IngressRuleValue: v1.IngressRuleValue{
HTTP: &v1.HTTPIngressRuleValue{
Paths: []v1.HTTPIngressPath{
{
Path: "/path1-3",
PathType: &pathTypePrefix,
Backend: v1.IngressBackend{
Service: &v1.IngressServiceBackend{
Name: "pod-3",
Port: v1.ServiceBackendPort{
Number: int32(80),
},
},
},
},
{
Path: "/path2-3",
PathType: &pathTypePrefix,
Backend: v1.IngressBackend{
Service: &v1.IngressServiceBackend{
Name: "pod-3",
Port: v1.ServiceBackendPort{
Number: int32(80),
},
},
},
},
},
},
},
},
},
},
}
// case 1
icCase1 := ingConfig{
ports: []int32{
int32(80),
int32(8080),
},
pathTypes: []*v1.PathType{
&pathTypePrefix,
&pathTypeImplementationSpecific,
},
paths: []string{
"/path1-3",
"/path2-3",
"/path3-3",
},
host: "xxx.xx.com",
ingressClassName: &ingressClassNameNginx,
}
ingObjectMetaCase1 := baseIngObjectMeta
ingObjectMetaCase1.Annotations = map[string]string{
IngressHashKey: util.GetHash(icCase1),
}
ingCase1 := &v1.Ingress{
ObjectMeta: ingObjectMetaCase1,
Spec: v1.IngressSpec{
IngressClassName: &ingressClassNameNginx,
Rules: []v1.IngressRule{
{
Host: "xxx.xx.com",
IngressRuleValue: v1.IngressRuleValue{
HTTP: &v1.HTTPIngressRuleValue{
Paths: []v1.HTTPIngressPath{
{
Path: "/path1-3",
PathType: &pathTypePrefix,
Backend: v1.IngressBackend{
Service: &v1.IngressServiceBackend{
Name: "pod-3",
Port: v1.ServiceBackendPort{
Number: int32(80),
},
},
},
},
{
Path: "/path2-3",
PathType: &pathTypeImplementationSpecific,
Backend: v1.IngressBackend{
Service: &v1.IngressServiceBackend{
Name: "pod-3",
Port: v1.ServiceBackendPort{
Number: int32(8080),
},
},
},
},
{
Path: "/path3-3",
PathType: &pathTypePrefix,
Backend: v1.IngressBackend{
Service: &v1.IngressServiceBackend{
Name: "pod-3",
Port: v1.ServiceBackendPort{
Number: int32(80),
},
},
},
},
},
},
},
},
},
},
}
tests := []struct {
ic ingConfig
ing *v1.Ingress
}{
{
ic: icCase0,
ing: ingCase0,
},
{
ic: icCase1,
ing: ingCase1,
},
}
for i, test := range tests {
actual := consIngress(test.ic, pod)
if !reflect.DeepEqual(actual, test.ing) {
t.Errorf("case %d: expect ingress: %v , but actual: %v", i, test.ing, actual)
}
}
}
func TestConsSvc(t *testing.T) {
pod := &corev1.Pod{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Pod",
},
ObjectMeta: metav1.ObjectMeta{
Name: "pod-3",
Namespace: "ns",
UID: "bff0afd6-bb30-4641-8607-8329547324eb",
},
}
baseSvcObjectMeta := metav1.ObjectMeta{
Name: "pod-3",
Namespace: "ns",
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "v1",
Kind: "Pod",
Name: "pod-3",
UID: "bff0afd6-bb30-4641-8607-8329547324eb",
Controller: pointer.BoolPtr(true),
BlockOwnerDeletion: pointer.BoolPtr(true),
},
},
}
// case 0
icCase0 := ingConfig{
ports: []int32{
int32(80),
int32(8080),
},
}
svcObjectMetaCase0 := baseSvcObjectMeta
svcObjectMetaCase0.Annotations = map[string]string{
ServiceHashKey: util.GetHash(icCase0.ports),
}
svcCase0 := &corev1.Service{
ObjectMeta: svcObjectMetaCase0,
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeClusterIP,
Selector: map[string]string{
SvcSelectorKey: "pod-3",
},
Ports: []corev1.ServicePort{
{
Name: "80",
Port: int32(80),
},
{
Name: "8080",
Port: int32(8080),
},
},
},
}
tests := []struct {
ic ingConfig
svc *corev1.Service
}{
{
ic: icCase0,
svc: svcCase0,
},
}
for i, test := range tests {
actual := consSvc(test.ic, pod)
if !reflect.DeepEqual(actual, test.svc) {
t.Errorf("case %d: expect service: %v , but actual: %v", i, test.svc, actual)
}
}
}

View File

@ -196,3 +196,23 @@ rules:
- get - get
- patch - patch
- update - update
- apiGroups:
- networking.k8s.io
resources:
- ingresses
verbs:
- create
- delete
- get
- list
- patch
- update
- watch
- apiGroups:
- networking.k8s.io
resources:
- ingresses/status
verbs:
- get
- patch
- update

View File

@ -136,22 +136,24 @@ OpenKruiseGame supports the following network plugins:
--- ---
### Plugin name ### Kubernetes-HostPort
#### Plugin name
`Kubernetes-HostPort` `Kubernetes-HostPort`
### Cloud Provider #### Cloud Provider
Kubernetes Kubernetes
### Plugin description #### Plugin description
- HostPort enables game servers to be accessed from the Internet by forwarding Internet traffic to the game servers by using the external IP address and ports exposed by the host where the game servers are located. The exposed IP address of the host must be a public IP address so that the host can be accessed from the Internet. - HostPort enables game servers to be accessed from the Internet by forwarding Internet traffic to the game servers by using the external IP address and ports exposed by the host where the game servers are located. The exposed IP address of the host must be a public IP address so that the host can be accessed from the Internet.
- In the configuration file, you can specify a custom range of available host ports. The default port range is 8000 to 9000. This network plugin can help you allocate and manage host ports to prevent port conflicts. - In the configuration file, you can specify a custom range of available host ports. The default port range is 8000 to 9000. This network plugin can help you allocate and manage host ports to prevent port conflicts.
- This network plugin does not support network isolation. - This network plugin does not support network isolation.
### Network parameters #### Network parameters
ContainerPorts ContainerPorts
@ -159,7 +161,7 @@ ContainerPorts
- Value: in the format of containerName:port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters. Example: `game-server:25565/TCP`. - Value: in the format of containerName:port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters. Example: `game-server:25565/TCP`.
- Configuration change supported or not: no. The value of this parameter is effective until the pod lifecycle ends. - Configuration change supported or not: no. The value of this parameter is effective until the pod lifecycle ends.
### Plugin configuration #### Plugin configuration
``` ```
[kubernetes] [kubernetes]
@ -172,21 +174,193 @@ min_port = 8000
--- ---
### Plugin name ### Kubernetes-Ingress
#### Plugin name
`Kubernetes-Ingress`
#### Cloud Provider
Kubernetes
#### Plugin description
- OKG provides the Ingress network model for games such as H5 games that require the application layer network model. This plugin will automatically set the corresponding path for each game server, which is related to the game server ID and is unique for each game server.
- This network plugin does not support network isolation.
#### Network parameters
PathType
- Meaning: Path type. Same as the PathType field in HTTPIngressPath.
- Value format: Same as the PathType field in HTTPIngressPath.
- Configuration change supported or not: yes.
Path
- Meaning: Access path. Each game server has its own access path based on its ID.
- Value format: Add \<id> to any position in the original path(consistent with the Path field in HTTPIngressPath), and the plugin will generate the path corresponding to the game server ID. For example, when setting the path to /game\<id>, the path for game server 0 is /game0, the path for game server 1 is /game1, and so on.
- Configuration change supported or not: yes.
Port
- Meaning: Port value exposed by the game server.
- Value format: port number
- Configuration change supported or not: yes.
IngressClassName
- Meaning: Specify the name of the IngressClass. Same as the IngressClassName field in IngressSpec.
- Value format: Same as the IngressClassName field in IngressSpec.
- Configuration change supported or not: yes.
Host
- Meaning: Domain name. Same as the Host field in IngressRule.
- Value format: Same as the Host field in IngressRule.
- Configuration change supported or not: yes.
TlsHosts
- Meaning: List of hosts containing TLS certificates. Similar to the Hosts field in IngressTLS.
- Value format: host1,host2,... For example, xxx.xx1.com,xxx.xx2.com
- Configuration change supported or not: yes.
TlsSecretName
- Meaning: Same as the SecretName field in IngressTLS.
- Value format: Same as the SecretName field in IngressTLS.
- Configuration change supported or not: yes.
Annotation
- Meaning: as an annotation of the Ingress object
- Value format: key: value (note the space after the colon), for example: nginx.ingress.kubernetes.io/rewrite-target: /$2
- Configuration change supported or not: yes.
_additional explanation_
- If you want to fill in multiple annotations, you can define multiple slices named Annotation in the networkConf.
- Supports filling in multiple paths. The path, path type, and port correspond one-to-one in the order of filling. When the number of paths is greater than the number of path types(or port), non-corresponding paths will match the path type(or port) that was filled in first.
#### Plugin configuration
None
#### Example
Set GameServerSet.Spec.Network:
```yaml
network:
networkConf:
- name: IngressClassName
value: nginx
- name: Port
value: "80"
- name: Path
value: /game<id>(/|$)(.*)
- name: Path
value: /test-<id>
- name: Host
value: test.xxx.cn-hangzhou.ali.com
- name: PathType
value: ImplementationSpecific
- name: TlsHosts
value: xxx.xx1.com,xxx.xx2.com
- name: Annotation
value: 'nginx.ingress.kubernetes.io/rewrite-target: /$2'
- name: Annotation
value: 'nginx.ingress.kubernetes.io/random: xxx'
networkType: Kubernetes-Ingress
```
This will generate a service and an ingress object for each replica of GameServerSet. The configuration for the ingress of the 0th game server is shown below:
```yaml
spec:
ingressClassName: nginx
rules:
- host: test.xxx.cn-hangzhou.ali.com
http:
paths:
- backend:
service:
name: ing-nginx-0
port:
number: 80
path: /game0(/|$)(.*)
pathType: ImplementationSpecific
- backend:
service:
name: ing-nginx-0
port:
number: 80
path: /test-0
pathType: ImplementationSpecific
tls:
- hosts:
- xxx.xx1.com
- xxx.xx2.com
status:
loadBalancer:
ingress:
- ip: 47.xx.xxx.xxx
```
The other GameServers only have different path fields and service names, while the other generated parameters are the same.
The network status of GameServer is as follows:
```yaml
networkStatus:
createTime: "2023-04-28T14:00:30Z"
currentNetworkState: Ready
desiredNetworkState: Ready
externalAddresses:
- ip: 47.xx.xxx.xxx
ports:
- name: /game0(/|$)(.*)
port: 80
protocol: TCP
- name: /test-0
port: 80
protocol: TCP
internalAddresses:
- ip: 10.xxx.x.xxx
ports:
- name: /game0(/|$)(.*)
port: 80
protocol: TCP
- name: /test-0
port: 80
protocol: TCP
lastTransitionTime: "2023-04-28T14:00:30Z"
networkType: Kubernetes-Ingress
```
---
### AlibabaCloud-NATGW
#### Plugin name
`AlibabaCloud-NATGW` `AlibabaCloud-NATGW`
### Cloud Provider #### Cloud Provider
AlibabaCloud AlibabaCloud
### Plugin description #### Plugin description
- AlibabaCloud-NATGW enables game servers to be accessed from the Internet by using an Internet NAT gateway of Alibaba Cloud. Internet traffic is forwarded to the corresponding game servers based on DNAT rules. - AlibabaCloud-NATGW enables game servers to be accessed from the Internet by using an Internet NAT gateway of Alibaba Cloud. Internet traffic is forwarded to the corresponding game servers based on DNAT rules.
- This network plugin does not support network isolation. - This network plugin does not support network isolation.
### Network parameters #### Network parameters
Ports Ports
@ -206,21 +380,23 @@ Fixed
- Value: false or true. - Value: false or true.
- Configuration change supported or not: no. - Configuration change supported or not: no.
### Plugin configuration #### Plugin configuration
None None
--- ---
### Plugin name ### AlibabaCloud-SLB
#### Plugin name
`AlibabaCloud-SLB` `AlibabaCloud-SLB`
### Cloud Provider #### Cloud Provider
AlibabaCloud AlibabaCloud
### Plugin description #### Plugin description
- AlibabaCloud-SLB enables game servers to be accessed from the Internet by using Layer 4 Classic Load Balancer (CLB) of Alibaba Cloud. CLB is a type of Server Load Balancer (SLB). AlibabaCloud-SLB uses different ports of the same CLB instance to forward Internet traffic to different game servers. The CLB instance only forwards traffic, but does not implement load balancing. - AlibabaCloud-SLB enables game servers to be accessed from the Internet by using Layer 4 Classic Load Balancer (CLB) of Alibaba Cloud. CLB is a type of Server Load Balancer (SLB). AlibabaCloud-SLB uses different ports of the same CLB instance to forward Internet traffic to different game servers. The CLB instance only forwards traffic, but does not implement load balancing.
@ -228,7 +404,7 @@ AlibabaCloud
Related design: https://github.com/openkruise/kruise-game/issues/20 Related design: https://github.com/openkruise/kruise-game/issues/20
### Network parameters #### Network parameters
SlbIds SlbIds
@ -248,7 +424,7 @@ Fixed
- Value: false or true. - Value: false or true.
- Configuration change supported or not: no. - Configuration change supported or not: no.
### Plugin configuration #### Plugin configuration
``` ```
[alibabacloud] [alibabacloud]
enable = true enable = true
@ -260,22 +436,24 @@ min_port = 500
--- ---
### Plugin name ### AlibabaCloud-SLB-SharedPort
#### Plugin name
`AlibabaCloud-SLB-SharedPort` `AlibabaCloud-SLB-SharedPort`
### Cloud Provider #### Cloud Provider
AlibabaCloud AlibabaCloud
### Plugin description #### Plugin description
- AlibabaCloud-SLB-SharedPort enables game servers to be accessed from the Internet by using Layer 4 CLB of Alibaba Cloud. Unlike AlibabaCloud-SLB, `AlibabaCloud-SLB-SharedPort` uses the same port of a CLB instance to forward traffic to game servers, and the CLB instance implements load balancing. - AlibabaCloud-SLB-SharedPort enables game servers to be accessed from the Internet by using Layer 4 CLB of Alibaba Cloud. Unlike AlibabaCloud-SLB, `AlibabaCloud-SLB-SharedPort` uses the same port of a CLB instance to forward traffic to game servers, and the CLB instance implements load balancing.
This network plugin applies to stateless network services, such as proxy or gateway, in gaming scenarios. This network plugin applies to stateless network services, such as proxy or gateway, in gaming scenarios.
- This network plugin supports network isolation. - This network plugin supports network isolation.
### Network parameters #### Network parameters
SlbIds SlbIds
@ -289,6 +467,6 @@ PortProtocols
- Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters. - Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters.
- Configuration change supported or not: no. The configuration change can be supported in future. - Configuration change supported or not: no. The configuration change can be supported in future.
### Plugin configuration #### Plugin configuration
None None

View File

@ -136,22 +136,24 @@ EOF
--- ---
### 插件名称 ### Kubernetes-HostPort
#### 插件名称
`Kubernetes-HostPort` `Kubernetes-HostPort`
### Cloud Provider #### Cloud Provider
Kubernetes Kubernetes
### 插件说明 #### 插件说明
- Kubernetes-HostPort利用宿主机网络通过主机上的端口转发实现游戏服对外暴露服务。宿主机需要配置公网IP有被公网访问的能力。 - Kubernetes-HostPort利用宿主机网络通过主机上的端口转发实现游戏服对外暴露服务。宿主机需要配置公网IP有被公网访问的能力。
- 用户在配置文件中可自定义宿主机开放的端口段默认为8000-9000该网络插件可以帮助用户分配管理宿主机端口尽量避免端口冲突。 - 用户在配置文件中可自定义宿主机开放的端口段默认为8000-9000该网络插件可以帮助用户分配管理宿主机端口尽量避免端口冲突。
- 该插件不支持网络隔离。 - 该插件不支持网络隔离。
### 网络参数 #### 网络参数
ContainerPorts ContainerPorts
@ -159,7 +161,7 @@ ContainerPorts
- 填写格式containerName:port1/protocol1,port2/protocol2,...(协议需大写) 比如:`game-server:25565/TCP` - 填写格式containerName:port1/protocol1,port2/protocol2,...(协议需大写) 比如:`game-server:25565/TCP`
- 是否支持变更不支持在创建时即永久生效随pod生命周期结束而结束 - 是否支持变更不支持在创建时即永久生效随pod生命周期结束而结束
### 插件配置 #### 插件配置
``` ```
[kubernetes] [kubernetes]
@ -172,21 +174,190 @@ min_port = 8000
--- ---
### 插件名称 ### Kubernetes-Ingress
#### 插件名称
`Kubernetes-Ingress`
#### Cloud Provider
Kubernetes
#### 插件说明
- 针对页游等需要七层网络模型的游戏场景OKG提供了Ingress网络模型。该插件将会自动地为每个游戏服设置对应的访问路径该路径与游戏服ID相关每个游戏服各不相同。
- 是否支持网络隔离:否
#### 网络参数
Path
- 含义访问路径。每个游戏服依据ID拥有各自的访问路径。
- 填写格式:将\<id>添加到原始路径(与HTTPIngressPath中Path一致)的任意位置该插件将会生成游戏服ID对应的路径。例如当设置路径为 /game\<id>游戏服0对应路径为/game0游戏服1对应路径为/game1以此类推。
- 是否支持变更:支持
PathType
- 含义路径类型。与HTTPIngressPath的PathType字段一致。
- 填写格式与HTTPIngressPath的PathType字段一致。
- 是否支持变更:支持
Port
- 含义:游戏服暴露的端口值。
- 填写格式:端口数字
- 是否支持变更:支持
IngressClassName
- 含义指定IngressClass的名称。与IngressSpec的IngressClassName字段一致。
- 填写格式与IngressSpec的IngressClassName字段一致。
- 是否支持变更:支持
Host
- 含义域名。与IngressRule的Host字段一致。
- 填写格式与IngressRule的Host字段一致。
- 是否支持变更:支持
TlsHosts
- 含义包含TLS证书的host列表。含义与IngressTLS的Hosts字段类似。
- 填写格式host1,host2,... 例如xxx.xx1.com,xxx.xx2.com
- 是否支持变更:支持
TlsSecretName
- 含义与IngressTLS的SecretName字段一致。
- 填写格式与IngressTLS的SecretName字段一致。
- 是否支持变更:支持
Annotation
- 含义作为ingress对象的annotation
- 格式key: value注意:后有空格例如nginx.ingress.kubernetes.io/rewrite-target: /$2
- 是否支持变更:支持
_补充说明_
- 支持填写多个annotation在networkConf中填写多个Annotation以及对应值即可不区分填写顺序。
- 支持填写多个路径。路径、路径类型、端口按照填写顺序一一对应。当路径数目大于路径类型数目(或端口数目)时,无法找到对应关系的路径按照率先填写的路径类型(或端口)匹配。
#### 插件配置
#### 示例说明
GameServerSet中network字段声明如下
```yaml
network:
networkConf:
- name: IngressClassName
value: nginx
- name: Port
value: "80"
- name: Path
value: /game<id>(/|$)(.*)
- name: Path
value: /test-<id>
- name: Host
value: test.xxx.cn-hangzhou.ali.com
- name: PathType
value: ImplementationSpecific
- name: TlsHosts
value: xxx.xx1.com,xxx.xx2.com
- name: Annotation
value: 'nginx.ingress.kubernetes.io/rewrite-target: /$2'
- name: Annotation
value: 'nginx.ingress.kubernetes.io/random: xxx'
networkType: Kubernetes-Ingress
```
则会生成gss replicas对应数目的service与ingress对象。0号游戏服生成的ingress字段如下所示
```yaml
spec:
ingressClassName: nginx
rules:
- host: test.xxx.cn-hangzhou.ali.com
http:
paths:
- backend:
service:
name: ing-nginx-0
port:
number: 80
path: /game0(/|$)(.*)
pathType: ImplementationSpecific
- backend:
service:
name: ing-nginx-0
port:
number: 80
path: /test-0
pathType: ImplementationSpecific
tls:
- hosts:
- xxx.xx1.com
- xxx.xx2.com
status:
loadBalancer:
ingress:
- ip: 47.xx.xxx.xxx
```
其他序号的游戏服只有path字段与service name不同生成的其他参数都相同。
对应的0号GameServer的networkStatus如下
```yaml
networkStatus:
createTime: "2023-04-28T14:00:30Z"
currentNetworkState: Ready
desiredNetworkState: Ready
externalAddresses:
- ip: 47.xx.xxx.xxx
ports:
- name: /game0(/|$)(.*)
port: 80
protocol: TCP
- name: /test-0
port: 80
protocol: TCP
internalAddresses:
- ip: 10.xxx.x.xxx
ports:
- name: /game0(/|$)(.*)
port: 80
protocol: TCP
- name: /test-0
port: 80
protocol: TCP
lastTransitionTime: "2023-04-28T14:00:30Z"
networkType: Kubernetes-Ingress
```
---
### AlibabaCloud-NATGW
#### 插件名称
`AlibabaCloud-NATGW` `AlibabaCloud-NATGW`
### Cloud Provider #### Cloud Provider
AlibabaCloud AlibabaCloud
### 插件说明 #### 插件说明
- AlibabaCloud-NATGW 使用阿里云公网网关作为游戏服对外服务的承载实体外网流量通过DNAT规则转发至对应的游戏服中。 - AlibabaCloud-NATGW 使用阿里云公网网关作为游戏服对外服务的承载实体外网流量通过DNAT规则转发至对应的游戏服中。
- 是否支持网络隔离:否 - 是否支持网络隔离:否
### 网络参数 #### 网络参数
Ports Ports
@ -206,21 +377,23 @@ Fixed
- 填写格式false / true - 填写格式false / true
- 是否支持变更:不支持 - 是否支持变更:不支持
### 插件配置 #### 插件配置
--- ---
### 插件名称 ### AlibabaCloud-SLB
#### 插件名称
`AlibabaCloud-SLB` `AlibabaCloud-SLB`
### Cloud Provider #### Cloud Provider
AlibabaCloud AlibabaCloud
### 插件说明 #### 插件说明
- AlibabaCloud-SLB 使用阿里云经典四层负载均衡SLB又称CLB作为对外服务的承载实体在此模式下不同游戏服将使用同一SLB的不同端口此时SLB只做转发并未均衡流量。 - AlibabaCloud-SLB 使用阿里云经典四层负载均衡SLB又称CLB作为对外服务的承载实体在此模式下不同游戏服将使用同一SLB的不同端口此时SLB只做转发并未均衡流量。
@ -228,7 +401,7 @@ AlibabaCloud
相关设计https://github.com/openkruise/kruise-game/issues/20 相关设计https://github.com/openkruise/kruise-game/issues/20
### 网络参数 #### 网络参数
SlbIds SlbIds
@ -248,7 +421,7 @@ Fixed
- 填写格式false / true - 填写格式false / true
- 是否支持变更:不支持 - 是否支持变更:不支持
### 插件配置 #### 插件配置
``` ```
[alibabacloud] [alibabacloud]
enable = true enable = true
@ -260,22 +433,24 @@ min_port = 500
--- ---
### 插件名称 ### AlibabaCloud-SLB-SharedPort
#### 插件名称
`AlibabaCloud-SLB-SharedPort` `AlibabaCloud-SLB-SharedPort`
### Cloud Provider #### Cloud Provider
AlibabaCloud AlibabaCloud
### 插件说明 #### 插件说明
- AlibabaCloud-SLB-SharedPort 使用阿里云经典四层负载均衡SLB又称CLB作为对外服务的承载实体。但与AlibabaCloud-SLB不同`AlibabaCloud-SLB-SharedPort` 使用SLB同一端口转发流量具有负载均衡的特点。 - AlibabaCloud-SLB-SharedPort 使用阿里云经典四层负载均衡SLB又称CLB作为对外服务的承载实体。但与AlibabaCloud-SLB不同`AlibabaCloud-SLB-SharedPort` 使用SLB同一端口转发流量具有负载均衡的特点。
适用于游戏场景下代理proxy或网关等无状态网络服务。 适用于游戏场景下代理proxy或网关等无状态网络服务。
- 是否支持网络隔离:是 - 是否支持网络隔离:是
### 网络参数 #### 网络参数
SlbIds SlbIds
@ -289,6 +464,6 @@ PortProtocols
- 格式port1/protocol1,port2/protocol2,...(协议需大写) - 格式port1/protocol1,port2/protocol2,...(协议需大写)
- 是否支持变更:暂不支持。未来将支持 - 是否支持变更:暂不支持。未来将支持
### 插件配置 #### 插件配置

View File

@ -63,6 +63,8 @@ func init() {
// +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch // +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch // +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete
// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch
// +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch
// +kubebuilder:rbac:groups=core,resources=nodes/status,verbs=get // +kubebuilder:rbac:groups=core,resources=nodes/status,verbs=get
// +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations,verbs=create;get;list;watch;update;patch // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations,verbs=create;get;list;watch;update;patch