diff --git a/cloudprovider/kubernetes/ingress.go b/cloudprovider/kubernetes/ingress.go new file mode 100644 index 0000000..5c2edb5 --- /dev/null +++ b/cloudprovider/kubernetes/ingress.go @@ -0,0 +1,385 @@ +package kubernetes + +import ( + "context" + "encoding/json" + "fmt" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/cloudprovider" + cperrors "github.com/openkruise/kruise-game/cloudprovider/errors" + "github.com/openkruise/kruise-game/cloudprovider/utils" + "github.com/openkruise/kruise-game/pkg/util" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/networking/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" + "strconv" + "strings" +) + +const ( + IngressNetwork = "Kubernetes-Ingress" + // PathTypeKey determines the interpretation of the Path matching, which is same as PathType in HTTPIngressPath. + PathTypeKey = "PathType" + // PathKey is matched against the path of an incoming request, the meaning of which is same as Path in HTTPIngressPath. + // Users can add to any position of the path, and the network plugin will generate the path corresponding to the game server. + // e.g. /game(/|$)(.*) The ingress path of GameServer 0 is /game0(/|$)(.*), the ingress path of GameServer 1 is /game1(/|$)(.*), and so on. + PathKey = "Path" + // PortKey indicates the exposed port value of game server. + PortKey = "Port" + // IngressClassNameKey indicates the name of the IngressClass cluster resource, which is same as IngressClassName in IngressSpec. + IngressClassNameKey = "IngressClassName" + // HostKey indicates domain name, which is same as Host in IngressRule. + HostKey = "Host" + // TlsHostsKey indicates hosts that included in the TLS certificate, the meaning of which is the same as that of Hosts in IngressTLS. + // Its corresponding value format is as follows, host1,host2,... e.g. xxx.xx.com + TlsHostsKey = "TlsHosts" + // TlsSecretNameKey indicates the name of the secret used to terminate TLS traffic on port 443, which is same as SecretName in IngressTLS. + TlsSecretNameKey = "TlsSecretName" + // AnnotationKey is the key of an annotation for ingress. + // Its corresponding value format is as follows, key: value(note that there is a space after: ) e.g. nginx.ingress.kubernetes.io/rewrite-target: /$2 + // If the user wants to fill in multiple annotations, just fill in multiple AnnotationKeys in the network config. + AnnotationKey = "Annotation" +) + +const ( + SvcSelectorKey = "statefulset.kubernetes.io/pod-name" + IngressHashKey = "game.kruise.io/ingress-hash" + ServiceHashKey = "game.kruise.io/svc-hash" +) + +const paramsError = "Network Config Params Error" + +func init() { + kubernetesProvider.registerPlugin(&IngressPlugin{}) +} + +type IngressPlugin struct { +} + +func (i IngressPlugin) Name() string { + return IngressNetwork +} + +func (i IngressPlugin) Alias() string { + return "" +} + +func (i IngressPlugin) Init(client client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error { + return nil +} + +func (i IngressPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + networkManager := utils.NewNetworkManager(pod, c) + conf := networkManager.GetNetworkConfig() + ic, err := parseIngConfig(conf, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ParameterError, err.Error()) + } + + err = c.Create(ctx, consSvc(ic, pod)) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + + err = c.Create(ctx, consIngress(ic, pod)) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + + return pod, nil +} + +func (i IngressPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + networkManager := utils.NewNetworkManager(pod, c) + networkStatus, _ := networkManager.GetNetworkStatus() + if networkStatus == nil { + pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{ + CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady, + }, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + + conf := networkManager.GetNetworkConfig() + ic, err := parseIngConfig(conf, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ParameterError, err.Error()) + } + + // get svc + svc := &corev1.Service{} + err = c.Get(ctx, types.NamespacedName{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + }, svc) + if err != nil { + if errors.IsNotFound(err) { + return pod, cperrors.ToPluginError(c.Create(ctx, consSvc(ic, pod)), cperrors.ApiCallError) + } + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + // update svc + if util.GetHash(ic.ports) != svc.GetAnnotations()[ServiceHashKey] { + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + + newSvc := consSvc(ic, pod) + patchSvc := map[string]interface{}{"metadata": map[string]map[string]string{"annotations": newSvc.Annotations}, "spec": newSvc.Spec} + patchSvcBytes, err := json.Marshal(patchSvc) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + + return pod, cperrors.ToPluginError(c.Patch(ctx, svc, client.RawPatch(types.MergePatchType, patchSvcBytes)), cperrors.ApiCallError) + } + + // get ingress + ing := &v1.Ingress{} + err = c.Get(ctx, types.NamespacedName{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + }, ing) + if err != nil { + if errors.IsNotFound(err) { + return pod, cperrors.ToPluginError(c.Create(ctx, consIngress(ic, pod)), cperrors.ApiCallError) + } + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + + // update ingress + if util.GetHash(ic) != ing.GetAnnotations()[IngressHashKey] { + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + return pod, cperrors.ToPluginError(c.Update(ctx, consIngress(ic, pod)), cperrors.ApiCallError) + } + + // network not ready + if ing.Status.LoadBalancer.Ingress == nil { + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + + // network ready + internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) + externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) + networkPorts := make([]gamekruiseiov1alpha1.NetworkPort, 0) + + for _, p := range ing.Spec.Rules[0].HTTP.Paths { + instrIPort := intstr.FromInt(int(p.Backend.Service.Port.Number)) + networkPort := gamekruiseiov1alpha1.NetworkPort{ + Name: p.Path, + Port: &instrIPort, + } + networkPorts = append(networkPorts, networkPort) + } + + internalAddress := gamekruiseiov1alpha1.NetworkAddress{ + IP: pod.Status.PodIP, + Ports: networkPorts, + } + externalAddress := gamekruiseiov1alpha1.NetworkAddress{ + IP: ing.Status.LoadBalancer.Ingress[0].IP, + EndPoint: ing.Spec.Rules[0].Host, + Ports: networkPorts, + } + + networkStatus.InternalAddresses = append(internalAddresses, internalAddress) + networkStatus.ExternalAddresses = append(externalAddresses, externalAddress) + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) +} + +func (i IngressPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError { + return nil +} + +type ingConfig struct { + paths []string + pathTypes []*v1.PathType + ports []int32 + host string + ingressClassName *string + tlsHosts []string + tlsSecretName string + annotations map[string]string +} + +func parseIngConfig(conf []gamekruiseiov1alpha1.NetworkConfParams, pod *corev1.Pod) (ingConfig, error) { + var ic ingConfig + ic.annotations = make(map[string]string) + ic.paths = make([]string, 0) + ic.pathTypes = make([]*v1.PathType, 0) + ic.ports = make([]int32, 0) + id := util.GetIndexFromGsName(pod.GetName()) + for _, c := range conf { + switch c.Name { + case PathTypeKey: + pathType := v1.PathType(c.Value) + ic.pathTypes = append(ic.pathTypes, &pathType) + case PortKey: + port, _ := strconv.ParseInt(c.Value, 10, 32) + ic.ports = append(ic.ports, int32(port)) + case HostKey: + strs := strings.Split(c.Value, "") + switch len(strs) { + case 2: + ic.host = strs[0] + strconv.Itoa(id) + strs[1] + case 1: + ic.host = strs[0] + default: + return ingConfig{}, fmt.Errorf("%s", paramsError) + } + case IngressClassNameKey: + ic.ingressClassName = pointer.String(c.Value) + case TlsSecretNameKey: + ic.tlsSecretName = c.Value + case TlsHostsKey: + ic.tlsHosts = strings.Split(c.Value, ",") + case PathKey: + strs := strings.Split(c.Value, "") + switch len(strs) { + case 2: + ic.paths = append(ic.paths, strs[0]+strconv.Itoa(id)+strs[1]) + case 1: + ic.paths = append(ic.paths, strs[0]) + default: + return ingConfig{}, fmt.Errorf("%s", paramsError) + } + case AnnotationKey: + kv := strings.Split(c.Value, ": ") + if len(kv) != 2 { + return ingConfig{}, fmt.Errorf("%s", paramsError) + } + ic.annotations[kv[0]] = kv[1] + } + } + + if len(ic.paths) == 0 || len(ic.pathTypes) == 0 || len(ic.ports) == 0 { + return ingConfig{}, fmt.Errorf("%s", paramsError) + } + + return ic, nil +} + +func consIngress(ic ingConfig, pod *corev1.Pod) *v1.Ingress { + pathSlice := ic.paths + pathTypeSlice := ic.pathTypes + pathPortSlice := ic.ports + lenPathTypeSlice := len(pathTypeSlice) + lenPathPortSlice := len(pathPortSlice) + for i := 0; i < len(pathSlice)-lenPathTypeSlice; i++ { + pathTypeSlice = append(pathTypeSlice, pathTypeSlice[0]) + } + for i := 0; i < len(pathSlice)-lenPathPortSlice; i++ { + pathPortSlice = append(pathPortSlice, pathPortSlice[0]) + } + + ingAnnotations := ic.annotations + if ingAnnotations == nil { + ingAnnotations = make(map[string]string) + } + ingAnnotations[IngressHashKey] = util.GetHash(ic) + ingPaths := make([]v1.HTTPIngressPath, 0) + for i := 0; i < len(pathSlice); i++ { + ingPath := v1.HTTPIngressPath{ + Path: pathSlice[i], + PathType: pathTypeSlice[i], + Backend: v1.IngressBackend{ + Service: &v1.IngressServiceBackend{ + Name: pod.GetName(), + Port: v1.ServiceBackendPort{ + Number: pathPortSlice[i], + }, + }, + }, + } + ingPaths = append(ingPaths, ingPath) + } + ing := &v1.Ingress{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + Annotations: ingAnnotations, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.GetName(), + UID: pod.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + }, + }, + Spec: v1.IngressSpec{ + IngressClassName: ic.ingressClassName, + Rules: []v1.IngressRule{ + { + Host: ic.host, + IngressRuleValue: v1.IngressRuleValue{ + HTTP: &v1.HTTPIngressRuleValue{ + Paths: ingPaths, + }, + }, + }, + }, + }, + } + if ic.tlsHosts != nil || ic.tlsSecretName != "" { + ing.Spec.TLS = []v1.IngressTLS{ + { + SecretName: ic.tlsSecretName, + Hosts: ic.tlsHosts, + }, + } + } + return ing +} + +func consSvc(ic ingConfig, pod *corev1.Pod) *corev1.Service { + annoatations := make(map[string]string) + annoatations[ServiceHashKey] = util.GetHash(ic.ports) + ports := make([]corev1.ServicePort, 0) + for _, p := range ic.ports { + port := corev1.ServicePort{ + Port: p, + Name: strconv.Itoa(int(p)), + } + ports = append(ports, port) + } + + return &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.GetName(), + UID: pod.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + }, + Annotations: annoatations, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + SvcSelectorKey: pod.GetName(), + }, + Ports: ports, + }, + } +} diff --git a/cloudprovider/kubernetes/ingress_test.go b/cloudprovider/kubernetes/ingress_test.go new file mode 100644 index 0000000..bb6b3d9 --- /dev/null +++ b/cloudprovider/kubernetes/ingress_test.go @@ -0,0 +1,434 @@ +package kubernetes + +import ( + "fmt" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" + corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/networking/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/utils/pointer" + "reflect" + "testing" +) + +func TestParseIngConfig(t *testing.T) { + pathTypePrefix := v1.PathTypePrefix + tests := []struct { + conf []gamekruiseiov1alpha1.NetworkConfParams + pod *corev1.Pod + ic ingConfig + err error + }{ + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: PathKey, + Value: "/game(/|$)(.*)", + }, + { + Name: AnnotationKey, + Value: "nginx.ingress.kubernetes.io/rewrite-target: /$2", + }, + { + Name: AnnotationKey, + Value: "alb.ingress.kubernetes.io/server-snippets: |\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";", + }, + { + Name: TlsHostsKey, + Value: "xxx-xxx.com,xxx-xx.com", + }, + { + Name: PortKey, + Value: "8080", + }, + { + Name: PathTypeKey, + Value: string(v1.PathTypePrefix), + }, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-3", + }, + }, + ic: ingConfig{ + paths: []string{"/game3(/|$)(.*)"}, + tlsHosts: []string{ + "xxx-xxx.com", + "xxx-xx.com", + }, + annotations: map[string]string{ + "nginx.ingress.kubernetes.io/rewrite-target": "/$2", + "alb.ingress.kubernetes.io/server-snippets": "|\n proxy_set_header Upgrade $http_upgrade;\n proxy_set_header Connection \"upgrade\";", + }, + ports: []int32{8080}, + pathTypes: []*v1.PathType{&pathTypePrefix}, + }, + }, + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: PathKey, + Value: "/game", + }, + { + Name: AnnotationKey, + Value: "nginx.ingress.kubernetes.io/rewrite-target: /$2", + }, + { + Name: TlsHostsKey, + Value: "xxx-xxx.com,xxx-xx.com", + }, + { + Name: PathTypeKey, + Value: string(v1.PathTypePrefix), + }, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-3", + }, + }, + err: fmt.Errorf("%s", paramsError), + }, + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: PathKey, + Value: "/game", + }, + { + Name: PortKey, + Value: "8080", + }, + { + Name: PathTypeKey, + Value: string(v1.PathTypePrefix), + }, + { + Name: HostKey, + Value: "instance.xxx.xxx.com", + }, + }, + pod: &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-2", + }, + }, + ic: ingConfig{ + paths: []string{"/game"}, + ports: []int32{8080}, + pathTypes: []*v1.PathType{&pathTypePrefix}, + host: "instance2.xxx.xxx.com", + annotations: map[string]string{}, + }, + }, + } + + for i, test := range tests { + expect := test.ic + actual, err := parseIngConfig(test.conf, test.pod) + if !reflect.DeepEqual(err, test.err) { + t.Errorf("case %d: expect err: %v , but actual: %v", i, test.err, err) + } + if !reflect.DeepEqual(actual, expect) { + if !reflect.DeepEqual(expect.paths, actual.paths) { + t.Errorf("case %d: expect paths: %v , but actual: %v", i, expect.paths, actual.paths) + } + if !reflect.DeepEqual(expect.ports, actual.ports) { + t.Errorf("case %d: expect ports: %v , but actual: %v", i, expect.ports, actual.ports) + } + if !reflect.DeepEqual(expect.pathTypes, actual.pathTypes) { + t.Errorf("case %d: expect annotations: %v , but actual: %v", i, expect.pathTypes, actual.pathTypes) + } + if !reflect.DeepEqual(expect.host, actual.host) { + t.Errorf("case %d: expect host: %v , but actual: %v", i, expect.host, actual.host) + } + if !reflect.DeepEqual(expect.tlsHosts, actual.tlsHosts) { + t.Errorf("case %d: expect tlsHosts: %v , but actual: %v", i, expect.tlsHosts, actual.tlsHosts) + } + if !reflect.DeepEqual(expect.tlsSecretName, actual.tlsSecretName) { + t.Errorf("case %d: expect tlsSecretName: %v , but actual: %v", i, expect.tlsSecretName, actual.tlsSecretName) + } + if !reflect.DeepEqual(expect.ingressClassName, actual.ingressClassName) { + t.Errorf("case %d: expect ingressClassName: %v , but actual: %v", i, expect.ingressClassName, actual.ingressClassName) + } + if !reflect.DeepEqual(expect.annotations, actual.annotations) { + t.Errorf("case %d: expect annotations: %v , but actual: %v", i, expect.annotations, actual.annotations) + } + } + } +} + +func TestConsIngress(t *testing.T) { + pathTypePrefix := v1.PathTypePrefix + pathTypeImplementationSpecific := v1.PathTypeImplementationSpecific + ingressClassNameNginx := "nginx" + + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-3", + Namespace: "ns", + UID: "bff0afd6-bb30-4641-8607-8329547324eb", + }, + } + + baseIngObjectMeta := metav1.ObjectMeta{ + Name: "pod-3", + Namespace: "ns", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "Pod", + Name: "pod-3", + UID: "bff0afd6-bb30-4641-8607-8329547324eb", + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + }, + } + + // case 0 + icCase0 := ingConfig{ + ports: []int32{ + int32(80), + }, + pathTypes: []*v1.PathType{ + &pathTypePrefix, + }, + paths: []string{ + "/path1-3", + "/path2-3", + }, + host: "xxx.xx.com", + ingressClassName: &ingressClassNameNginx, + annotations: map[string]string{ + "nginx.ingress.kubernetes.io/rewrite-target": "/$2", + }, + } + ingObjectMetaCase0 := baseIngObjectMeta + ingObjectMetaCase0.Annotations = map[string]string{ + "nginx.ingress.kubernetes.io/rewrite-target": "/$2", + IngressHashKey: util.GetHash(icCase0), + } + ingCase0 := &v1.Ingress{ + ObjectMeta: ingObjectMetaCase0, + Spec: v1.IngressSpec{ + IngressClassName: &ingressClassNameNginx, + Rules: []v1.IngressRule{ + { + Host: "xxx.xx.com", + IngressRuleValue: v1.IngressRuleValue{ + HTTP: &v1.HTTPIngressRuleValue{ + Paths: []v1.HTTPIngressPath{ + { + Path: "/path1-3", + PathType: &pathTypePrefix, + Backend: v1.IngressBackend{ + Service: &v1.IngressServiceBackend{ + Name: "pod-3", + Port: v1.ServiceBackendPort{ + Number: int32(80), + }, + }, + }, + }, + { + Path: "/path2-3", + PathType: &pathTypePrefix, + Backend: v1.IngressBackend{ + Service: &v1.IngressServiceBackend{ + Name: "pod-3", + Port: v1.ServiceBackendPort{ + Number: int32(80), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + // case 1 + icCase1 := ingConfig{ + ports: []int32{ + int32(80), + int32(8080), + }, + pathTypes: []*v1.PathType{ + &pathTypePrefix, + &pathTypeImplementationSpecific, + }, + paths: []string{ + "/path1-3", + "/path2-3", + "/path3-3", + }, + host: "xxx.xx.com", + ingressClassName: &ingressClassNameNginx, + } + ingObjectMetaCase1 := baseIngObjectMeta + ingObjectMetaCase1.Annotations = map[string]string{ + IngressHashKey: util.GetHash(icCase1), + } + ingCase1 := &v1.Ingress{ + ObjectMeta: ingObjectMetaCase1, + Spec: v1.IngressSpec{ + IngressClassName: &ingressClassNameNginx, + Rules: []v1.IngressRule{ + { + Host: "xxx.xx.com", + IngressRuleValue: v1.IngressRuleValue{ + HTTP: &v1.HTTPIngressRuleValue{ + Paths: []v1.HTTPIngressPath{ + { + Path: "/path1-3", + PathType: &pathTypePrefix, + Backend: v1.IngressBackend{ + Service: &v1.IngressServiceBackend{ + Name: "pod-3", + Port: v1.ServiceBackendPort{ + Number: int32(80), + }, + }, + }, + }, + { + Path: "/path2-3", + PathType: &pathTypeImplementationSpecific, + Backend: v1.IngressBackend{ + Service: &v1.IngressServiceBackend{ + Name: "pod-3", + Port: v1.ServiceBackendPort{ + Number: int32(8080), + }, + }, + }, + }, + { + Path: "/path3-3", + PathType: &pathTypePrefix, + Backend: v1.IngressBackend{ + Service: &v1.IngressServiceBackend{ + Name: "pod-3", + Port: v1.ServiceBackendPort{ + Number: int32(80), + }, + }, + }, + }, + }, + }, + }, + }, + }, + }, + } + + tests := []struct { + ic ingConfig + ing *v1.Ingress + }{ + { + ic: icCase0, + ing: ingCase0, + }, + { + ic: icCase1, + ing: ingCase1, + }, + } + + for i, test := range tests { + actual := consIngress(test.ic, pod) + if !reflect.DeepEqual(actual, test.ing) { + t.Errorf("case %d: expect ingress: %v , but actual: %v", i, test.ing, actual) + } + } +} + +func TestConsSvc(t *testing.T) { + pod := &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Pod", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "pod-3", + Namespace: "ns", + UID: "bff0afd6-bb30-4641-8607-8329547324eb", + }, + } + + baseSvcObjectMeta := metav1.ObjectMeta{ + Name: "pod-3", + Namespace: "ns", + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "Pod", + Name: "pod-3", + UID: "bff0afd6-bb30-4641-8607-8329547324eb", + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + }, + } + + // case 0 + icCase0 := ingConfig{ + ports: []int32{ + int32(80), + int32(8080), + }, + } + svcObjectMetaCase0 := baseSvcObjectMeta + svcObjectMetaCase0.Annotations = map[string]string{ + ServiceHashKey: util.GetHash(icCase0.ports), + } + svcCase0 := &corev1.Service{ + ObjectMeta: svcObjectMetaCase0, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeClusterIP, + Selector: map[string]string{ + SvcSelectorKey: "pod-3", + }, + Ports: []corev1.ServicePort{ + { + Name: "80", + Port: int32(80), + }, + { + Name: "8080", + Port: int32(8080), + }, + }, + }, + } + + tests := []struct { + ic ingConfig + svc *corev1.Service + }{ + { + ic: icCase0, + svc: svcCase0, + }, + } + + for i, test := range tests { + actual := consSvc(test.ic, pod) + if !reflect.DeepEqual(actual, test.svc) { + t.Errorf("case %d: expect service: %v , but actual: %v", i, test.svc, actual) + } + } +} diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 36f0ab0..41c4d22 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -196,3 +196,23 @@ rules: - get - patch - update +- apiGroups: + - networking.k8s.io + resources: + - ingresses + verbs: + - create + - delete + - get + - list + - patch + - update + - watch +- apiGroups: + - networking.k8s.io + resources: + - ingresses/status + verbs: + - get + - patch + - update diff --git a/docs/en/user_manuals/network.md b/docs/en/user_manuals/network.md index 7067a67..a61d824 100644 --- a/docs/en/user_manuals/network.md +++ b/docs/en/user_manuals/network.md @@ -136,22 +136,24 @@ OpenKruiseGame supports the following network plugins: --- -### Plugin name +### Kubernetes-HostPort + +#### Plugin name `Kubernetes-HostPort` -### Cloud Provider +#### Cloud Provider Kubernetes -### Plugin description +#### Plugin description - HostPort enables game servers to be accessed from the Internet by forwarding Internet traffic to the game servers by using the external IP address and ports exposed by the host where the game servers are located. The exposed IP address of the host must be a public IP address so that the host can be accessed from the Internet. - In the configuration file, you can specify a custom range of available host ports. The default port range is 8000 to 9000. This network plugin can help you allocate and manage host ports to prevent port conflicts. - This network plugin does not support network isolation. -### Network parameters +#### Network parameters ContainerPorts @@ -159,7 +161,7 @@ ContainerPorts - Value: in the format of containerName:port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters. Example: `game-server:25565/TCP`. - Configuration change supported or not: no. The value of this parameter is effective until the pod lifecycle ends. -### Plugin configuration +#### Plugin configuration ``` [kubernetes] @@ -172,21 +174,193 @@ min_port = 8000 --- -### Plugin name +### Kubernetes-Ingress + +#### Plugin name + +`Kubernetes-Ingress` + +#### Cloud Provider + +Kubernetes + +#### Plugin description + +- OKG provides the Ingress network model for games such as H5 games that require the application layer network model. This plugin will automatically set the corresponding path for each game server, which is related to the game server ID and is unique for each game server. + +- This network plugin does not support network isolation. + +#### Network parameters + +PathType + +- Meaning: Path type. Same as the PathType field in HTTPIngressPath. +- Value format: Same as the PathType field in HTTPIngressPath. +- Configuration change supported or not: yes. + +Path + +- Meaning: Access path. Each game server has its own access path based on its ID. +- Value format: Add \ to any position in the original path(consistent with the Path field in HTTPIngressPath), and the plugin will generate the path corresponding to the game server ID. For example, when setting the path to /game\, the path for game server 0 is /game0, the path for game server 1 is /game1, and so on. +- Configuration change supported or not: yes. + +Port + +- Meaning: Port value exposed by the game server. +- Value format: port number +- Configuration change supported or not: yes. + +IngressClassName + +- Meaning: Specify the name of the IngressClass. Same as the IngressClassName field in IngressSpec. +- Value format: Same as the IngressClassName field in IngressSpec. +- Configuration change supported or not: yes. + +Host + +- Meaning: Domain name. Same as the Host field in IngressRule. +- Value format: Same as the Host field in IngressRule. +- Configuration change supported or not: yes. + +TlsHosts + +- Meaning: List of hosts containing TLS certificates. Similar to the Hosts field in IngressTLS. +- Value format: host1,host2,... For example, xxx.xx1.com,xxx.xx2.com +- Configuration change supported or not: yes. + +TlsSecretName + +- Meaning: Same as the SecretName field in IngressTLS. +- Value format: Same as the SecretName field in IngressTLS. +- Configuration change supported or not: yes. + +Annotation + +- Meaning: as an annotation of the Ingress object +- Value format: key: value (note the space after the colon), for example: nginx.ingress.kubernetes.io/rewrite-target: /$2 +- Configuration change supported or not: yes. + +_additional explanation_ + +- If you want to fill in multiple annotations, you can define multiple slices named Annotation in the networkConf. +- Supports filling in multiple paths. The path, path type, and port correspond one-to-one in the order of filling. When the number of paths is greater than the number of path types(or port), non-corresponding paths will match the path type(or port) that was filled in first. + + +#### Plugin configuration + +None + +#### Example + +Set GameServerSet.Spec.Network: + +```yaml + network: + networkConf: + - name: IngressClassName + value: nginx + - name: Port + value: "80" + - name: Path + value: /game(/|$)(.*) + - name: Path + value: /test- + - name: Host + value: test.xxx.cn-hangzhou.ali.com + - name: PathType + value: ImplementationSpecific + - name: TlsHosts + value: xxx.xx1.com,xxx.xx2.com + - name: Annotation + value: 'nginx.ingress.kubernetes.io/rewrite-target: /$2' + - name: Annotation + value: 'nginx.ingress.kubernetes.io/random: xxx' + networkType: Kubernetes-Ingress + +``` +This will generate a service and an ingress object for each replica of GameServerSet. The configuration for the ingress of the 0th game server is shown below: + +```yaml +spec: + ingressClassName: nginx + rules: + - host: test.xxx.cn-hangzhou.ali.com + http: + paths: + - backend: + service: + name: ing-nginx-0 + port: + number: 80 + path: /game0(/|$)(.*) + pathType: ImplementationSpecific + - backend: + service: + name: ing-nginx-0 + port: + number: 80 + path: /test-0 + pathType: ImplementationSpecific + tls: + - hosts: + - xxx.xx1.com + - xxx.xx2.com +status: + loadBalancer: + ingress: + - ip: 47.xx.xxx.xxx +``` + +The other GameServers only have different path fields and service names, while the other generated parameters are the same. + +The network status of GameServer is as follows: + +```yaml + networkStatus: + createTime: "2023-04-28T14:00:30Z" + currentNetworkState: Ready + desiredNetworkState: Ready + externalAddresses: + - ip: 47.xx.xxx.xxx + ports: + - name: /game0(/|$)(.*) + port: 80 + protocol: TCP + - name: /test-0 + port: 80 + protocol: TCP + internalAddresses: + - ip: 10.xxx.x.xxx + ports: + - name: /game0(/|$)(.*) + port: 80 + protocol: TCP + - name: /test-0 + port: 80 + protocol: TCP + lastTransitionTime: "2023-04-28T14:00:30Z" + networkType: Kubernetes-Ingress +``` + +--- + +### AlibabaCloud-NATGW + +#### Plugin name `AlibabaCloud-NATGW` -### Cloud Provider +#### Cloud Provider AlibabaCloud -### Plugin description +#### Plugin description - AlibabaCloud-NATGW enables game servers to be accessed from the Internet by using an Internet NAT gateway of Alibaba Cloud. Internet traffic is forwarded to the corresponding game servers based on DNAT rules. - This network plugin does not support network isolation. -### Network parameters +#### Network parameters Ports @@ -206,21 +380,23 @@ Fixed - Value: false or true. - Configuration change supported or not: no. -### Plugin configuration +#### Plugin configuration None --- -### Plugin name +### AlibabaCloud-SLB + +#### Plugin name `AlibabaCloud-SLB` -### Cloud Provider +#### Cloud Provider AlibabaCloud -### Plugin description +#### Plugin description - AlibabaCloud-SLB enables game servers to be accessed from the Internet by using Layer 4 Classic Load Balancer (CLB) of Alibaba Cloud. CLB is a type of Server Load Balancer (SLB). AlibabaCloud-SLB uses different ports of the same CLB instance to forward Internet traffic to different game servers. The CLB instance only forwards traffic, but does not implement load balancing. @@ -228,7 +404,7 @@ AlibabaCloud Related design: https://github.com/openkruise/kruise-game/issues/20 -### Network parameters +#### Network parameters SlbIds @@ -248,7 +424,7 @@ Fixed - Value: false or true. - Configuration change supported or not: no. -### Plugin configuration +#### Plugin configuration ``` [alibabacloud] enable = true @@ -260,22 +436,24 @@ min_port = 500 --- -### Plugin name +### AlibabaCloud-SLB-SharedPort + +#### Plugin name `AlibabaCloud-SLB-SharedPort` -### Cloud Provider +#### Cloud Provider AlibabaCloud -### Plugin description +#### Plugin description - AlibabaCloud-SLB-SharedPort enables game servers to be accessed from the Internet by using Layer 4 CLB of Alibaba Cloud. Unlike AlibabaCloud-SLB, `AlibabaCloud-SLB-SharedPort` uses the same port of a CLB instance to forward traffic to game servers, and the CLB instance implements load balancing. This network plugin applies to stateless network services, such as proxy or gateway, in gaming scenarios. - This network plugin supports network isolation. -### Network parameters +#### Network parameters SlbIds @@ -289,6 +467,6 @@ PortProtocols - Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters. - Configuration change supported or not: no. The configuration change can be supported in future. -### Plugin configuration +#### Plugin configuration None diff --git a/docs/中文/用户手册/网络模型.md b/docs/中文/用户手册/网络模型.md index 66950b4..39a8721 100644 --- a/docs/中文/用户手册/网络模型.md +++ b/docs/中文/用户手册/网络模型.md @@ -136,22 +136,24 @@ EOF --- -### 插件名称 +### Kubernetes-HostPort + +#### 插件名称 `Kubernetes-HostPort` -### Cloud Provider +#### Cloud Provider Kubernetes -### 插件说明 +#### 插件说明 - Kubernetes-HostPort利用宿主机网络,通过主机上的端口转发实现游戏服对外暴露服务。宿主机需要配置公网IP,有被公网访问的能力。 - 用户在配置文件中可自定义宿主机开放的端口段(默认为8000-9000),该网络插件可以帮助用户分配管理宿主机端口,尽量避免端口冲突。 - 该插件不支持网络隔离。 -### 网络参数 +#### 网络参数 ContainerPorts @@ -159,7 +161,7 @@ ContainerPorts - 填写格式:containerName:port1/protocol1,port2/protocol2,...(协议需大写) 比如:`game-server:25565/TCP` - 是否支持变更:不支持,在创建时即永久生效,随pod生命周期结束而结束 -### 插件配置 +#### 插件配置 ``` [kubernetes] @@ -172,21 +174,190 @@ min_port = 8000 --- -### 插件名称 +### Kubernetes-Ingress + +#### 插件名称 + +`Kubernetes-Ingress` + +#### Cloud Provider + +Kubernetes + +#### 插件说明 + +- 针对页游等需要七层网络模型的游戏场景,OKG提供了Ingress网络模型。该插件将会自动地为每个游戏服设置对应的访问路径,该路径与游戏服ID相关,每个游戏服各不相同。 +- 是否支持网络隔离:否 + +#### 网络参数 + +Path + +- 含义:访问路径。每个游戏服依据ID拥有各自的访问路径。 +- 填写格式:将\添加到原始路径(与HTTPIngressPath中Path一致)的任意位置,该插件将会生成游戏服ID对应的路径。例如,当设置路径为 /game\,游戏服0对应路径为/game0,游戏服1对应路径为/game1,以此类推。 +- 是否支持变更:支持 + +PathType + +- 含义:路径类型。与HTTPIngressPath的PathType字段一致。 +- 填写格式:与HTTPIngressPath的PathType字段一致。 +- 是否支持变更:支持 + +Port + +- 含义:游戏服暴露的端口值。 +- 填写格式:端口数字 +- 是否支持变更:支持 + +IngressClassName + +- 含义:指定IngressClass的名称。与IngressSpec的IngressClassName字段一致。 +- 填写格式:与IngressSpec的IngressClassName字段一致。 +- 是否支持变更:支持 + +Host + +- 含义:域名。与IngressRule的Host字段一致。 +- 填写格式:与IngressRule的Host字段一致。 +- 是否支持变更:支持 + +TlsHosts + +- 含义:包含TLS证书的host列表。含义与IngressTLS的Hosts字段类似。 +- 填写格式:host1,host2,... 例如,xxx.xx1.com,xxx.xx2.com +- 是否支持变更:支持 + +TlsSecretName + +- 含义:与IngressTLS的SecretName字段一致。 +- 填写格式:与IngressTLS的SecretName字段一致。 +- 是否支持变更:支持 + +Annotation + +- 含义:作为ingress对象的annotation +- 格式:key: value(注意:后有空格),例如:nginx.ingress.kubernetes.io/rewrite-target: /$2 +- 是否支持变更:支持 + +_补充说明_ + +- 支持填写多个annotation,在networkConf中填写多个Annotation以及对应值即可,不区分填写顺序。 +- 支持填写多个路径。路径、路径类型、端口按照填写顺序一一对应。当路径数目大于路径类型数目(或端口数目)时,无法找到对应关系的路径按照率先填写的路径类型(或端口)匹配。 + + +#### 插件配置 + +无 + +#### 示例说明 + +GameServerSet中network字段声明如下: + +```yaml + network: + networkConf: + - name: IngressClassName + value: nginx + - name: Port + value: "80" + - name: Path + value: /game(/|$)(.*) + - name: Path + value: /test- + - name: Host + value: test.xxx.cn-hangzhou.ali.com + - name: PathType + value: ImplementationSpecific + - name: TlsHosts + value: xxx.xx1.com,xxx.xx2.com + - name: Annotation + value: 'nginx.ingress.kubernetes.io/rewrite-target: /$2' + - name: Annotation + value: 'nginx.ingress.kubernetes.io/random: xxx' + networkType: Kubernetes-Ingress +``` + +则会生成gss replicas对应数目的service与ingress对象。0号游戏服生成的ingress字段如下所示: +```yaml +spec: + ingressClassName: nginx + rules: + - host: test.xxx.cn-hangzhou.ali.com + http: + paths: + - backend: + service: + name: ing-nginx-0 + port: + number: 80 + path: /game0(/|$)(.*) + pathType: ImplementationSpecific + - backend: + service: + name: ing-nginx-0 + port: + number: 80 + path: /test-0 + pathType: ImplementationSpecific + tls: + - hosts: + - xxx.xx1.com + - xxx.xx2.com +status: + loadBalancer: + ingress: + - ip: 47.xx.xxx.xxx +``` + +其他序号的游戏服只有path字段与service name不同,生成的其他参数都相同。 + +对应的0号GameServer的networkStatus如下: +```yaml + networkStatus: + createTime: "2023-04-28T14:00:30Z" + currentNetworkState: Ready + desiredNetworkState: Ready + externalAddresses: + - ip: 47.xx.xxx.xxx + ports: + - name: /game0(/|$)(.*) + port: 80 + protocol: TCP + - name: /test-0 + port: 80 + protocol: TCP + internalAddresses: + - ip: 10.xxx.x.xxx + ports: + - name: /game0(/|$)(.*) + port: 80 + protocol: TCP + - name: /test-0 + port: 80 + protocol: TCP + lastTransitionTime: "2023-04-28T14:00:30Z" + networkType: Kubernetes-Ingress +``` + +--- + +### AlibabaCloud-NATGW + +#### 插件名称 `AlibabaCloud-NATGW` -### Cloud Provider +#### Cloud Provider AlibabaCloud -### 插件说明 +#### 插件说明 - AlibabaCloud-NATGW 使用阿里云公网网关作为游戏服对外服务的承载实体,外网流量通过DNAT规则转发至对应的游戏服中。 - 是否支持网络隔离:否 -### 网络参数 +#### 网络参数 Ports @@ -206,21 +377,23 @@ Fixed - 填写格式:false / true - 是否支持变更:不支持 -### 插件配置 +#### 插件配置 无 --- -### 插件名称 +### AlibabaCloud-SLB + +#### 插件名称 `AlibabaCloud-SLB` -### Cloud Provider +#### Cloud Provider AlibabaCloud -### 插件说明 +#### 插件说明 - AlibabaCloud-SLB 使用阿里云经典四层负载均衡(SLB,又称CLB)作为对外服务的承载实体,在此模式下,不同游戏服将使用同一SLB的不同端口,此时SLB只做转发,并未均衡流量。 @@ -228,7 +401,7 @@ AlibabaCloud 相关设计:https://github.com/openkruise/kruise-game/issues/20 -### 网络参数 +#### 网络参数 SlbIds @@ -248,7 +421,7 @@ Fixed - 填写格式:false / true - 是否支持变更:不支持 -### 插件配置 +#### 插件配置 ``` [alibabacloud] enable = true @@ -260,22 +433,24 @@ min_port = 500 --- -### 插件名称 +### AlibabaCloud-SLB-SharedPort + +#### 插件名称 `AlibabaCloud-SLB-SharedPort` -### Cloud Provider +#### Cloud Provider AlibabaCloud -### 插件说明 +#### 插件说明 - AlibabaCloud-SLB-SharedPort 使用阿里云经典四层负载均衡(SLB,又称CLB)作为对外服务的承载实体。但与AlibabaCloud-SLB不同,`AlibabaCloud-SLB-SharedPort` 使用SLB同一端口转发流量,具有负载均衡的特点。 适用于游戏场景下代理(proxy)或网关等无状态网络服务。 - 是否支持网络隔离:是 -### 网络参数 +#### 网络参数 SlbIds @@ -289,6 +464,6 @@ PortProtocols - 格式:port1/protocol1,port2/protocol2,...(协议需大写) - 是否支持变更:暂不支持。未来将支持 -### 插件配置 +#### 插件配置 无 diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 47e3254..d7653a0 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -63,6 +63,8 @@ func init() { // +kubebuilder:rbac:groups=core,resources=pods/status,verbs=get;update;patch // +kubebuilder:rbac:groups=core,resources=services,verbs=get;list;watch;create;update;patch;delete // +kubebuilder:rbac:groups=core,resources=services/status,verbs=get;update;patch +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses,verbs=get;list;watch;create;update;patch;delete +// +kubebuilder:rbac:groups=networking.k8s.io,resources=ingresses/status,verbs=get;update;patch // +kubebuilder:rbac:groups=core,resources=nodes,verbs=get;list;watch // +kubebuilder:rbac:groups=core,resources=nodes/status,verbs=get // +kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=mutatingwebhookconfigurations,verbs=create;get;list;watch;update;patch