openkruise
/
kruise-game
mirror of https://github.com/openkruise/kruise-game.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

support Huawei CCE EIB and EIP (#261) 

Co-authored-by: bing.ma <bing.ma@daocloud.io>
This commit is contained in:
usernameisnull 2025-07-24 10:26:35 +08:00 committed by GitHub
parent afb9896ebf
commit b83e3e3d0a
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
12 changed files with 3035 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

116
cloudprovider/hwcloud/eip_cce.go Normal file
View File

@ -0,0 +1,116 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package hwcloud
import (
"context"
corev1 "k8s.io/api/core/v1"
log "k8s.io/klog/v2"
"sigs.k8s.io/controller-runtime/pkg/client"
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider"
"github.com/openkruise/kruise-game/cloudprovider/errors"
"github.com/openkruise/kruise-game/cloudprovider/utils"
)
const (
EIPNetwork = "HwCloud-CCE-EIP"
AliasSEIP = "CCE-EIP-Network"
)
var allowedAnnotations = []string{
"yangtse.io/pod-with-eip",
"yangtse.io/eip-bandwidth-size",
"yangtse.io/eip-network-type",
"yangtse.io/eip-charge-mode",
"yangtse.io/eip-bandwidth-name",
"yangtse.io/eip-network-type",
"yangtse.io/eip-bandwidth-id",
"yangtse.io/eip-id",
"yangtse.io/security-group-ids",
"yangtse.io/additional-security-group-ids",
}
func init() {
eipPlugin := EipPlugin{}
hwCloudProvider.registerPlugin(&eipPlugin)
}
type EipPlugin struct{}
func (E EipPlugin) Name() string {
return EIPNetwork
}
func (E EipPlugin) Alias() string {
return AliasSEIP
}
func (E EipPlugin) Init(client client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error {
return nil
}
func (E EipPlugin) OnPodAdded(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, errors.PluginError) {
networkManager := utils.NewNetworkManager(pod, client)
conf := networkManager.GetNetworkConfig()
log.Infof("pod %s/%s network config: %#v", pod.Namespace, pod.Name, conf)
if networkManager.GetNetworkType() != EIPNetwork {
log.Infof("pod %s/%s network type is not %s, skipping", pod.Namespace, pod.Name, EIPNetwork)
return pod, nil
}
allowedAnnotationsMap := make(map[string]struct{})
for _, item := range allowedAnnotations {
allowedAnnotationsMap[item] = struct{}{}
}
for _, c := range conf {
_, ok := allowedAnnotationsMap[c.Name]
if ok {
pod.Annotations[c.Name] = c.Value
} else {
log.Warningf("pod %s/%s network config %s is not allowed", pod.Namespace, pod.Name, c.Name)
}
}
return pod, nil
}
func (E EipPlugin) OnPodUpdated(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, errors.PluginError) {
networkManager := utils.NewNetworkManager(pod, client)
if networkManager.GetNetworkType() != EIPNetwork {
log.Infof("pod %s/%s network type is not %s, skipping", pod.Namespace, pod.Name, EIPNetwork)
return pod, nil
}
networkStatus, _ := networkManager.GetNetworkStatus()
if networkStatus == nil {
pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{
CurrentNetworkState: gamekruiseiov1alpha1.NetworkWaiting,
}, pod)
return pod, errors.ToPluginError(err, errors.InternalError)
}
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady
pod, err := networkManager.UpdateNetworkStatus(*networkStatus, pod)
return pod, errors.ToPluginError(err, errors.InternalError)
}
func (E EipPlugin) OnPodDeleted(client client.Client, pod *corev1.Pod, ctx context.Context) errors.PluginError {
return nil
}

178
cloudprovider/hwcloud/eip_cce_test.go Normal file
View File

@ -0,0 +1,178 @@
package hwcloud
import (
"context"
"encoding/json"
"testing"
"github.com/stretchr/testify/assert"
corev1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"github.com/openkruise/kruise-game/apis/v1alpha1"
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider/alibabacloud/apis/v1beta1"
)
func TestEipPlugin_Init(t *testing.T) {
plugin := EipPlugin{}
assert.Equal(t, EIPNetwork, plugin.Name())
assert.Equal(t, AliasSEIP, plugin.Alias())
err := plugin.Init(nil, nil, context.Background())
assert.NoError(t, err)
}
func TestEipPlugin_OnPodAdded_UseExistingEIP(t *testing.T) {
// create test pod
var networkConf []v1alpha1.NetworkConfParams
networkConf = append(networkConf, v1alpha1.NetworkConfParams{
Name: "yangtse.io/eip-id",
Value: "huawei-eip-12345",
})
jsonStr, _ := json.Marshal(networkConf)
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
Annotations: map[string]string{
v1alpha1.GameServerNetworkType: EIPNetwork,
v1alpha1.GameServerNetworkConf: string(jsonStr),
},
},
}
// create fake client.
scheme := runtime.NewScheme()
_ = corev1.AddToScheme(scheme)
fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
// execute the test
plugin := EipPlugin{}
updatedPod, err := plugin.OnPodAdded(fakeClient, pod, context.Background())
// check the result.
assert.NoError(t, err)
assert.Equal(t, EIPNetwork, updatedPod.Annotations[v1alpha1.GameServerNetworkType])
unmarshalErr := json.Unmarshal([]byte(updatedPod.Annotations[v1alpha1.GameServerNetworkConf]), &networkConf)
assert.NoError(t, unmarshalErr)
assert.Equal(t, "huawei-eip-12345", networkConf[0].Value)
}
func addKvToParams(networkConf []v1alpha1.NetworkConfParams, keys []string, values []string) []v1alpha1.NetworkConfParams {
for i := 0; i < len(keys); i++ {
networkConf = append(networkConf, v1alpha1.NetworkConfParams{
Name: keys[i],
Value: values[i],
})
}
return networkConf
}
func TestEipPlugin_OnPodAdded_NewEIP(t *testing.T) {
var networkConf []v1alpha1.NetworkConfParams
networkConf = addKvToParams(networkConf,
[]string{
"name",
"yangtse.io/pod-with-eip",
"yangtse.io/eip-bandwidth-size",
"yangtse.io/eip-network-type",
"yangtse.io/eip-charge-mode",
},
[]string{
"huawei-eip-demo",
"true",
"5",
"5-bgp",
"traffic",
},
)
jsonStr, _ := json.Marshal(networkConf)
// create test Pod and add related annotations.
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
Annotations: map[string]string{
v1alpha1.GameServerNetworkType: EIPNetwork,
v1alpha1.GameServerNetworkConf: string(jsonStr),
},
},
}
// create fake client.
scheme := runtime.NewScheme()
_ = corev1.AddToScheme(scheme)
fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build()
// execute the test.
plugin := EipPlugin{}
updatedPod, err := plugin.OnPodAdded(fakeClient, pod, context.Background())
// check the result.
assert.NoError(t, err)
assert.Equal(t, EIPNetwork, updatedPod.Annotations[v1alpha1.GameServerNetworkType])
assert.Equal(t, "true", updatedPod.Annotations["yangtse.io/pod-with-eip"])
assert.Equal(t, "5", updatedPod.Annotations["yangtse.io/eip-bandwidth-size"])
assert.Equal(t, "5-bgp", updatedPod.Annotations["yangtse.io/eip-network-type"])
assert.Equal(t, "traffic", updatedPod.Annotations["yangtse.io/eip-charge-mode"])
}
func TestEipPlugin_OnPodUpdated_WithNetworkStatus(t *testing.T) {
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
Annotations: map[string]string{
v1alpha1.GameServerNetworkType: EIPNetwork,
"cloud.kruise.io/network-status": `{"currentNetworkState":"Waiting"}`,
},
},
Status: corev1.PodStatus{},
}
scheme := runtime.NewScheme()
_ = corev1.AddToScheme(scheme)
_ = v1beta1.AddToScheme(scheme)
_ = gamekruiseiov1alpha1.AddToScheme(scheme)
fakeClient := fake.NewClientBuilder().
WithScheme(scheme).
WithObjects(pod).
Build()
plugin := EipPlugin{}
networkStatus := &v1alpha1.NetworkStatus{}
networkStatus.ExternalAddresses = []v1alpha1.NetworkAddress{{IP: "203.0.113.1"}}
networkStatus.InternalAddresses = []v1alpha1.NetworkAddress{{IP: "10.0.0.1"}}
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady
networkStatusBytes, jErr := json.Marshal(networkStatus)
assert.NoError(t, jErr)
pod.Annotations[v1alpha1.GameServerNetworkStatus] = string(networkStatusBytes)
updatedPod, err := plugin.OnPodUpdated(fakeClient, pod, context.Background())
assert.NoError(t, err)
jErr = json.Unmarshal([]byte(updatedPod.Annotations[v1alpha1.GameServerNetworkStatus]), &networkStatus)
assert.NoError(t, jErr)
assert.Contains(t, updatedPod.Annotations[v1alpha1.GameServerNetworkStatus], "Ready")
assert.Contains(t, updatedPod.Annotations[v1alpha1.GameServerNetworkStatus], "203.0.113.1")
assert.Contains(t, updatedPod.Annotations[v1alpha1.GameServerNetworkStatus], "10.0.0.1")
}
func TestEipPlugin_OnPodDeleted(t *testing.T) {
plugin := EipPlugin{}
pod := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
Annotations: map[string]string{
v1alpha1.GameServerNetworkType: EIPNetwork,
"cloud.kruise.io/network-status": `{"currentNetworkState":"Waiting"}`,
},
},
Status: corev1.PodStatus{},
}
err := plugin.OnPodDeleted(nil, pod, context.Background())
assert.Nil(t, err)
}

678
cloudprovider/hwcloud/elb_cce.go Normal file
View File

@ -0,0 +1,678 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package hwcloud
import (
"context"
"fmt"
"strconv"
"strings"
"sync"
"time"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
log "k8s.io/klog/v2"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider"
cperrors "github.com/openkruise/kruise-game/cloudprovider/errors"
provideroptions "github.com/openkruise/kruise-game/cloudprovider/options"
"github.com/openkruise/kruise-game/cloudprovider/utils"
"github.com/openkruise/kruise-game/pkg/util"
)
const (
ElbAutocreateAnnotationKey = "kubernetes.io/elb.autocreate"
CCEElbNetwork = "HwCloud-CCE-ELB"
AliasCCEELB = "CCE-ELB-Network"
)
func init() {
elbPlugin := CCEElbPlugin{
mutex: sync.RWMutex{},
}
hwCloudProvider.registerPlugin(&elbPlugin)
}
type cceElbConfig struct {
elbIds []string
targetPorts []int
protocols []corev1.Protocol
isFixed bool
externalTrafficPolicyType corev1.ServiceExternalTrafficPolicyType
hwOptions map[string]string
}
func (e cceElbConfig) isAutoCreateElb() bool {
// auto create elb mode annotation
jsonValue, ok := e.hwOptions[ElbAutocreateAnnotationKey]
return ok && jsonValue != "" && len(e.elbIds) == 0
}
type ccePortAllocated map[int32]bool
type CCEElbPlugin struct {
maxPort int32
minPort int32
blockPorts []int32
cache map[string]ccePortAllocated
podAllocate map[string]string
mutex sync.RWMutex
}
func (s *CCEElbPlugin) Name() string {
return CCEElbNetwork
}
func (s *CCEElbPlugin) Alias() string {
return AliasCCEELB
}
func (s *CCEElbPlugin) Init(c client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error {
s.mutex.Lock()
defer s.mutex.Unlock()
elbOptions := options.(provideroptions.HwCloudOptions).CCEELBOptions
s.minPort = elbOptions.ELBOptions.MinPort
s.maxPort = elbOptions.ELBOptions.MaxPort
s.blockPorts = elbOptions.ELBOptions.BlockPorts
// get all service
svcList := &corev1.ServiceList{}
err := c.List(ctx, svcList)
if err != nil {
return err
}
s.cache, s.podAllocate = initCCELbCache(svcList.Items, s.minPort, s.maxPort, s.blockPorts)
log.Infof("[%s] podAllocate cache complete initialization: %v", CCEElbNetwork, s.podAllocate)
return nil
}
// fillCache: you need to add lock before calling this function
func (s *CCEElbPlugin) fillCache(lbId string, usedPorts []int32) {
if s.cache == nil {
s.cache = make(map[string]ccePortAllocated)
}
if s.cache[lbId] != nil {
return
}
alloc := make(ccePortAllocated, s.maxPort-s.minPort+1)
for port := s.minPort; port <= s.maxPort; port++ {
alloc[port] = false
}
for _, port := range s.blockPorts {
if port >= s.minPort && port <= s.maxPort {
alloc[port] = true
}
}
s.cache[lbId] = alloc
for _, port := range usedPorts {
s.cache[lbId][port] = true
}
}
func (s *CCEElbPlugin) updateCachesAfterAutoCreateElb(c client.Client, name, namespace string,
interval, totalTimeout time.Duration) {
if interval > totalTimeout {
panic("interval must be lesser than timeout")
}
log.Infof("Starting periodic cache update for %s/%s (interval: %s, timeout: %s)",
namespace, name, interval, totalTimeout)
timeoutCtx, cancel := context.WithTimeout(context.Background(), totalTimeout)
defer cancel()
ticker := time.NewTicker(interval)
defer ticker.Stop()
var (
attempt int
lastError error
)
for {
select {
case <-timeoutCtx.Done():
log.Warningf("Cache update failed for %s/%s after %d attempts. Last error: %v",
namespace, name, attempt, lastError)
return
case <-ticker.C:
attempt++
log.Infof("Attempt #%d: updating cache for %s/%s", attempt, namespace, name)
svc := &corev1.Service{}
err := c.Get(timeoutCtx, types.NamespacedName{
Name: name,
Namespace: namespace,
}, svc)
if err != nil {
log.Errorf("failed to get Service: %s", err)
continue
}
elbId := svc.Annotations[ElbIdAnnotationKey]
usedPorts := getCCEPorts(svc.Spec.Ports)
if elbId == "" || len(usedPorts) == 0 {
continue
}
s.mutex.Lock()
s.fillCache(elbId, usedPorts)
if s.podAllocate == nil {
s.podAllocate = make(map[string]string)
}
s.podAllocate[newPodAllocateKey(name, namespace)] = newPodAllocateValue(elbId, usedPorts)
s.mutex.Unlock()
log.Infof("Attempt #%d success: updated cache for %s/%s with ELB %s and %d ports",
attempt, namespace, name, elbId, len(usedPorts))
return
}
}
}
func (s *CCEElbPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
return pod, nil
}
func (s *CCEElbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
log.Infof("on update pod begin")
networkManager := utils.NewNetworkManager(pod, c)
if networkManager.GetNetworkType() != CCEElbNetwork {
log.Infof("pod %s/%s network type is not %s, skipping", pod.Namespace, pod.Name, CCEElbNetwork)
return pod, nil
}
networkStatus, _ := networkManager.GetNetworkStatus()
if networkStatus == nil {
log.Warningf("network status is nil")
pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{
CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady,
}, pod)
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
}
networkConfig := networkManager.GetNetworkConfig()
sc, err := parseCCELbConfig(networkConfig)
if err != nil {
log.Errorf("parse elb config failed: %s, network configuration: %#v", err, networkConfig)
return pod, cperrors.ToPluginError(err, cperrors.ParameterError)
}
log.Infof("creating svc %s/%s", pod.GetNamespace(), pod.GetName())
// get svc
svc := &corev1.Service{}
err = c.Get(ctx, types.NamespacedName{
Name: pod.GetName(),
Namespace: pod.GetNamespace(),
}, svc)
if err != nil {
if k8serrors.IsNotFound(err) {
log.Infof("svc %s/%s not found, will create it", pod.GetNamespace(), pod.GetName())
service, err := s.consSvc(sc, pod, c, ctx)
if err != nil {
return pod, cperrors.ToPluginError(err, cperrors.ParameterError)
}
if err = c.Create(ctx, service); err != nil {
log.Errorf("create svc %s/%s failed: %s", pod.GetNamespace(), pod.GetName(), err)
return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
}
log.Infof("create svc %s/%s success", pod.GetNamespace(), pod.GetName())
if sc.isAutoCreateElb() {
go s.updateCachesAfterAutoCreateElb(c, pod.Name, pod.Namespace, 5*time.Second, 10*time.Minute)
}
return pod, cperrors.ToPluginError(nil, cperrors.ApiCallError)
}
log.Errorf("get svc %s/%s failed: %s", pod.GetNamespace(), pod.GetName(), err)
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
// old svc remain
if svc.OwnerReferences[0].Kind == "Pod" && svc.OwnerReferences[0].UID != pod.UID {
log.Infof("[%s] waitting old svc %s/%s deleted. old owner pod uid is %s, but now is %s", CCEElbNetwork, svc.Namespace, svc.Name, svc.OwnerReferences[0].UID, pod.UID)
return pod, nil
}
// update svc
if util.GetHash(sc) != svc.GetAnnotations()[ElbConfigHashKey] {
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}
service, err := s.consSvc(sc, pod, c, ctx)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ParameterError, err.Error())
}
return pod, cperrors.ToPluginError(c.Update(ctx, service), cperrors.ApiCallError)
}
// disable network
if networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeLoadBalancer {
svc.Spec.Type = corev1.ServiceTypeClusterIP
return pod, cperrors.ToPluginError(c.Update(ctx, svc), cperrors.ApiCallError)
}
// enable network
if !networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeClusterIP {
svc.Spec.Type = corev1.ServiceTypeLoadBalancer
return pod, cperrors.ToPluginError(c.Update(ctx, svc), cperrors.ApiCallError)
}
// network not ready
if svc.Status.LoadBalancer.Ingress == nil {
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
}
// allow not ready containers
if util.IsAllowNotReadyContainers(networkManager.GetNetworkConfig()) {
toUpDateSvc, err := utils.AllowNotReadyContainers(c, ctx, pod, svc, false)
if err != nil {
return pod, err
}
if toUpDateSvc {
err := c.Update(ctx, svc)
if err != nil {
return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
}
}
}
// network ready
internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)
externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)
for _, port := range svc.Spec.Ports {
instrIPort := port.TargetPort
instrEPort := intstr.FromInt(int(port.Port))
internalAddress := gamekruiseiov1alpha1.NetworkAddress{
IP: pod.Status.PodIP,
Ports: []gamekruiseiov1alpha1.NetworkPort{
{
Name: instrIPort.String(),
Port: &instrIPort,
Protocol: port.Protocol,
},
},
}
externalAddress := gamekruiseiov1alpha1.NetworkAddress{
IP: svc.Status.LoadBalancer.Ingress[0].IP,
Ports: []gamekruiseiov1alpha1.NetworkPort{
{
Name: instrIPort.String(),
Port: &instrEPort,
Protocol: port.Protocol,
},
},
}
internalAddresses = append(internalAddresses, internalAddress)
externalAddresses = append(externalAddresses, externalAddress)
}
networkStatus.InternalAddresses = internalAddresses
networkStatus.ExternalAddresses = externalAddresses
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
}
func (s *CCEElbPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError {
networkManager := utils.NewNetworkManager(pod, c)
networkConfig := networkManager.GetNetworkConfig()
if networkManager.GetNetworkType() != CCEElbNetwork {
log.Infof("pod %s/%s network type is not %s, skipping", pod.Namespace, pod.Name, CCEElbNetwork)
return nil
}
sc, err := parseCCELbConfig(networkConfig)
if err != nil {
return cperrors.NewPluginError(cperrors.ParameterError, err.Error())
}
var podKeys []string
if sc.isFixed {
gss, err := util.GetGameServerSetOfPod(pod, c, ctx)
if err != nil && !k8serrors.IsNotFound(err) {
return cperrors.ToPluginError(err, cperrors.ApiCallError)
}
// gss exists in cluster, do not deAllocate.
if err == nil && gss.GetDeletionTimestamp() == nil {
return nil
}
// gss not exists in cluster, deAllocate all the ports related to it.
for key := range s.podAllocate {
gssName := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]
if strings.Contains(key, pod.GetNamespace()+"/"+gssName) {
podKeys = append(podKeys, key)
}
}
} else {
podKeys = append(podKeys, pod.GetNamespace()+"/"+pod.GetName())
}
for _, podKey := range podKeys {
s.deAllocate(podKey)
}
return nil
}
func (s *CCEElbPlugin) allocate(lbIds []string, num int, podKey string) (string, []int32) {
s.mutex.Lock()
defer s.mutex.Unlock()
var ports []int32
var lbId string
// find lb with adequate ports
for _, elbId := range lbIds {
sum := 0
for i := s.minPort; i <= s.maxPort; i++ {
if !s.cache[elbId][i] {
sum++
}
if sum >= num {
lbId = elbId
break
}
}
}
if lbId == "" {
return "", nil
}
// select ports
for i := 0; i < num; i++ {
var port int32
s.fillCache(lbId, nil)
for p, allocated := range s.cache[lbId] {
if !allocated {
port = p
break
}
}
s.cache[lbId][port] = true
ports = append(ports, port)
}
s.podAllocate[podKey] = newPodAllocateValue(lbId, ports)
log.Infof("pod %s allocate elb %s ports %v", podKey, lbId, ports)
return lbId, ports
}
func (s *CCEElbPlugin) deAllocate(nsSvcKey string) {
s.mutex.Lock()
defer s.mutex.Unlock()
allocatedPorts, exist := s.podAllocate[nsSvcKey]
if !exist {
return
}
elbPorts := strings.Split(allocatedPorts, ":")
lbId := elbPorts[0]
ports := util.StringToInt32Slice(elbPorts[1], ",")
for _, port := range ports {
s.cache[lbId][port] = false
}
// block ports
for _, blockPort := range s.blockPorts {
s.cache[lbId][blockPort] = true
}
delete(s.podAllocate, nsSvcKey)
log.Infof("pod %s deallocate elb %s ports %v", nsSvcKey, lbId, ports)
}
func (s *CCEElbPlugin) consSvc(sc *cceElbConfig, pod *corev1.Pod, c client.Client, ctx context.Context) (*corev1.Service, error) {
var ports []int32
var lbId string
podKey := pod.GetNamespace() + "/" + pod.GetName()
allocatedPorts, exist := s.podAllocate[podKey]
if exist {
elbPorts := strings.Split(allocatedPorts, ":")
ports = util.StringToInt32Slice(elbPorts[1], ",")
} else {
if sc.isAutoCreateElb() {
lbId, ports = "", s.getPortFromHead(len(sc.targetPorts))
} else {
lbId, ports = s.allocate(sc.elbIds, len(sc.targetPorts), podKey)
}
if lbId == "" && ports == nil {
return nil, fmt.Errorf("there are no avaliable ports for %v", sc.elbIds)
}
}
svcPorts := make([]corev1.ServicePort, 0)
for i := 0; i < len(sc.targetPorts); i++ {
if sc.protocols[i] == ProtocolTCPUDP {
svcPorts = append(svcPorts, corev1.ServicePort{
Name: fmt.Sprintf("%s-%s", strconv.Itoa(sc.targetPorts[i]), strings.ToLower(string(corev1.ProtocolTCP))),
Port: ports[i],
Protocol: corev1.ProtocolTCP,
TargetPort: intstr.FromInt(sc.targetPorts[i]),
})
svcPorts = append(svcPorts, corev1.ServicePort{
Name: fmt.Sprintf("%s-%s", strconv.Itoa(sc.targetPorts[i]), strings.ToLower(string(corev1.ProtocolUDP))),
Port: ports[i],
Protocol: corev1.ProtocolUDP,
TargetPort: intstr.FromInt(sc.targetPorts[i]),
})
} else {
svcPorts = append(svcPorts, corev1.ServicePort{
Name: fmt.Sprintf("%s-%s", strconv.Itoa(sc.targetPorts[i]), strings.ToLower(string(sc.protocols[i]))),
Port: ports[i],
Protocol: sc.protocols[i],
TargetPort: intstr.FromInt(sc.targetPorts[i]),
})
}
}
svcAnnotations := make(map[string]string, 0)
for k, v := range sc.hwOptions {
svcAnnotations[k] = v
}
// add hash to svc, otherwise, the status of GS will remain in NetworkNotReady.
svcAnnotations[ElbConfigHashKey] = util.GetHash(sc)
svc := &corev1.Service{
ObjectMeta: metav1.ObjectMeta{
Name: pod.GetName(),
Namespace: pod.GetNamespace(),
Annotations: svcAnnotations,
OwnerReferences: getCCESvcOwnerReference(c, ctx, pod, sc.isFixed),
},
Spec: corev1.ServiceSpec{
Type: corev1.ServiceTypeLoadBalancer,
ExternalTrafficPolicy: sc.externalTrafficPolicyType,
Selector: map[string]string{
SvcSelectorKey: pod.GetName(),
},
Ports: svcPorts,
},
}
return svc, nil
}
func (s *CCEElbPlugin) getPortFromHead(num int) []int32 {
res := make([]int32, 0)
blocked := make(map[int32]struct{})
for _, port := range s.blockPorts {
blocked[port] = struct{}{}
}
count := 0
for i := s.minPort; i <= s.maxPort && count < num; i++ {
if _, exist := blocked[i]; exist {
continue
}
count++
res = append(res, i)
}
return res
}
func getCCESvcOwnerReference(c client.Client, ctx context.Context, pod *corev1.Pod, isFixed bool) []metav1.OwnerReference {
ownerReferences := []metav1.OwnerReference{
{
APIVersion: pod.APIVersion,
Kind: pod.Kind,
Name: pod.GetName(),
UID: pod.GetUID(),
Controller: ptr.To[bool](true),
BlockOwnerDeletion: ptr.To[bool](true),
},
}
if isFixed {
gss, err := util.GetGameServerSetOfPod(pod, c, ctx)
if err == nil {
ownerReferences = []metav1.OwnerReference{
{
APIVersion: gss.APIVersion,
Kind: gss.Kind,
Name: gss.GetName(),
UID: gss.GetUID(),
Controller: ptr.To[bool](true),
BlockOwnerDeletion: ptr.To[bool](true),
},
}
}
}
return ownerReferences
}
func newPodAllocateKey(name, namespace string) string {
return namespace + "/" + name
}
func newPodAllocateValue(elbId string, ports []int32) string {
return elbId + ":" + util.Int32SliceToString(ports, ",")
}
func getCCEPorts(ports []corev1.ServicePort) []int32 {
var ret []int32
for _, port := range ports {
ret = append(ret, port.Port)
}
return ret
}
func initCCELbCache(svcList []corev1.Service, minPort, maxPort int32, blockPorts []int32) (map[string]ccePortAllocated, map[string]string) {
newCache := make(map[string]ccePortAllocated)
newPodAllocate := make(map[string]string)
for _, svc := range svcList {
lbId := svc.Annotations[ElbIdAnnotationKey]
// Associate an existing ELB.
if lbId != "" && svc.Spec.Type == corev1.ServiceTypeLoadBalancer {
// init cache for that lb
if newCache[lbId] == nil {
newCache[lbId] = make(ccePortAllocated, maxPort-minPort+1)
for i := minPort; i <= maxPort; i++ {
newCache[lbId][i] = false
}
}
// block ports
for _, blockPort := range blockPorts {
newCache[lbId][blockPort] = true
}
// fill in cache for that lb
var ports []int32
for _, port := range getCCEPorts(svc.Spec.Ports) {
if port <= maxPort && port >= minPort {
value, ok := newCache[lbId][port]
if !ok || !value {
newCache[lbId][port] = true
ports = append(ports, port)
}
}
}
if len(ports) != 0 {
newPodAllocate[newPodAllocateKey(svc.GetName(), svc.GetNamespace())] = newPodAllocateValue(lbId, ports)
log.Infof("svc %s/%s allocate elb %s ports %v", svc.Namespace, svc.Name, lbId, ports)
}
}
}
return newCache, newPodAllocate
}
func parseCCELbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) (*cceElbConfig, error) {
res := &cceElbConfig{
targetPorts: make([]int, 0),
protocols: make([]corev1.Protocol, 0),
isFixed: false,
externalTrafficPolicyType: corev1.ServiceExternalTrafficPolicyTypeCluster,
hwOptions: make(map[string]string),
}
specifyElbId := false
autoCreateElb := false
for _, c := range conf {
switch c.Name {
case ElbIdAnnotationKey:
if autoCreateElb {
return nil, fmt.Errorf("%s and %s cannot be filled in simultaneously",
ElbIdAnnotationKey, ElbAutocreateAnnotationKey)
}
specifyElbId = true
// huawei only supports one elb id
if c.Value == "" {
return nil, fmt.Errorf("no elb id found, must specify at least one elb id")
}
res.elbIds = []string{c.Value}
res.hwOptions[c.Name] = c.Value
case ElbAutocreateAnnotationKey:
if specifyElbId {
return nil, fmt.Errorf("%s and %s cannot be filled in simultaneously",
ElbIdAnnotationKey, ElbAutocreateAnnotationKey)
}
autoCreateElb = true
res.hwOptions[c.Name] = c.Value
case PortProtocolsConfigName:
for _, pp := range strings.Split(c.Value, ",") {
ppSlice := strings.Split(pp, "/")
port, err := strconv.Atoi(ppSlice[0])
if err != nil {
continue
}
res.targetPorts = append(res.targetPorts, port)
if len(ppSlice) != 2 {
res.protocols = append(res.protocols, corev1.ProtocolTCP)
} else {
res.protocols = append(res.protocols, corev1.Protocol(ppSlice[1]))
}
}
case FixedConfigName:
v, err := strconv.ParseBool(c.Value)
if err != nil {
continue
}
res.isFixed = v
case ExternalTrafficPolicyTypeConfigName:
if strings.EqualFold(c.Value, string(corev1.ServiceExternalTrafficPolicyTypeLocal)) {
res.externalTrafficPolicyType = corev1.ServiceExternalTrafficPolicyTypeLocal
}
default:
res.hwOptions[c.Name] = c.Value
}
}
return res, nil
}

936
cloudprovider/hwcloud/elb_cce_test.go Normal file
View File

@ -0,0 +1,936 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package hwcloud
import (
"context"
"fmt"
"reflect"
"sync"
"testing"
"time"
"github.com/openkruise/kruise-game/cloudprovider"
"github.com/openkruise/kruise-game/cloudprovider/options"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/mock"
corev1 "k8s.io/api/core/v1"
k8serrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider/errors"
)
var (
fakeSvcTemplate = corev1.Service{
TypeMeta: metav1.TypeMeta{
APIVersion: "v1",
Kind: "Service",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-0",
Namespace: "default",
Annotations: map[string]string{
"game.kruise.io/network-config-hash": "2536262647",
"kubernetes.io/elb.class": "performance",
"kubernetes.io/elb.connection-drain-enable": "true",
"kubernetes.io/elb.connection-drain-timeout": "300",
"kubernetes.io/elb.id": "8f4cf216-a659-40dc-8c77-6068b036ba56",
"kubernetes.io/elb.mark": "0",
},
CreationTimestamp: metav1.Time{},
Finalizers: []string{"service.kubernetes.io/load-balancer-cleanup"},
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: "v1",
Kind: "Pod",
Name: "test-pod-0",
UID: "53cb0992-c720-4ae4-9af9-3cc7e2bf3660",
},
},
ResourceVersion: "9867633",
UID: "eccc0de3-ea09-4554-8710-4319eb551237",
},
Spec: corev1.ServiceSpec{
Ports: []corev1.ServicePort{
{
Name: "80-tcp",
Protocol: corev1.ProtocolTCP,
Port: 500,
TargetPort: intstr.FromInt(80),
NodePort: 31749,
},
},
Selector: map[string]string{
"statefulset.kubernetes.io/pod-name": "gs-elb-performance-2-0",
},
ClusterIP: "10.247.83.164",
ClusterIPs: []string{"10.247.83.164"},
Type: corev1.ServiceTypeLoadBalancer,
ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeCluster,
LoadBalancerIP: "192.168.0.147",
IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol},
SessionAffinity: corev1.ServiceAffinityNone,
},
Status: corev1.ServiceStatus{
LoadBalancer: corev1.LoadBalancerStatus{
Ingress: []corev1.LoadBalancerIngress{
{IP: "192.168.0.147"},
{IP: "189.1.225.136"},
},
},
},
}
)
func TestAllocateDeAllocate(t *testing.T) {
test := struct {
lbIds []string
elb *CCEElbPlugin
num int
podKey string
}{
lbIds: []string{"cce-lb-xxxx"},
elb: &CCEElbPlugin{
maxPort: int32(712),
minPort: int32(512),
cache: make(map[string]ccePortAllocated),
podAllocate: make(map[string]string),
mutex: sync.RWMutex{},
},
podKey: "xxx/xxx",
num: 3,
}
lbId, ports := test.elb.allocate(test.lbIds, test.num, test.podKey)
if _, exist := test.elb.podAllocate[test.podKey]; !exist {
t.Errorf("podAllocate[%s] is empty after allocated", test.podKey)
}
for _, port := range ports {
if port > test.elb.maxPort || port < test.elb.minPort {
t.Errorf("allocate port %d, unexpected", port)
}
if test.elb.cache[lbId][port] == false {
t.Errorf("Allocate port %d failed", port)
}
}
test.elb.deAllocate(test.podKey)
for _, port := range ports {
if test.elb.cache[lbId][port] == true {
t.Errorf("deAllocate port %d failed", port)
}
}
if _, exist := test.elb.podAllocate[test.podKey]; exist {
t.Errorf("podAllocate[%s] is not empty after deallocated", test.podKey)
}
}
func TestParseLbConfig(t *testing.T) {
tests := []struct {
conf []gamekruiseiov1alpha1.NetworkConfParams
slbConfig *cceElbConfig
}{
{
conf: []gamekruiseiov1alpha1.NetworkConfParams{
{
Name: PortProtocolsConfigName,
Value: "80",
},
{
Name: "kubernetes.io/elb.class",
Value: "performance",
},
{
Name: "kubernetes.io/elb.id",
Value: "c1d8f4c6-7aef-4596-8c7c-xxxxxxxxxxxx",
},
{
Name: "kubernetes.io/elb.enterpriseID",
Value: "ff97261-4dbd-4593-8236-xxxxxxxxxxxx",
},
{
Name: "kubernetes.io/elb.lb-algorithm",
Value: "ROUND_ROBIN",
},
{
Name: "kubernetes.io/elb.protocol-port",
Value: "https:443,http:80",
},
{
Name: "kubernetes.io/elb.x-forwarded-port",
Value: "true",
},
},
slbConfig: &cceElbConfig{
elbIds: []string{"c1d8f4c6-7aef-4596-8c7c-xxxxxxxxxxxx"},
targetPorts: []int{80},
protocols: []corev1.Protocol{corev1.ProtocolTCP},
externalTrafficPolicyType: corev1.ServiceExternalTrafficPolicyTypeCluster,
isFixed: false,
hwOptions: map[string]string{
"kubernetes.io/elb.class": "performance",
"kubernetes.io/elb.id": "c1d8f4c6-7aef-4596-8c7c-xxxxxxxxxxxx",
"kubernetes.io/elb.enterpriseID": "ff97261-4dbd-4593-8236-xxxxxxxxxxxx",
"kubernetes.io/elb.lb-algorithm": "ROUND_ROBIN",
"kubernetes.io/elb.protocol-port": "https:443,http:80",
"kubernetes.io/elb.x-forwarded-port": "true",
},
},
},
{
conf: []gamekruiseiov1alpha1.NetworkConfParams{
{
Name: PortProtocolsConfigName,
Value: "81/UDP,82,83/TCP",
},
{
Name: "kubernetes.io/elb.class",
Value: "union",
},
{
Name: "kubernetes.io/elb.id",
Value: "c1d8f4c6-7aef-4596-8c7c-yyyyyyyyyyyy",
},
{
Name: "kubernetes.io/elb.enterpriseID",
Value: "ff97261-4dbd-4593-8236-yyyyyyyyyyyy",
},
{
Name: "kubernetes.io/elb.cert-id",
Value: "17e3b4f4bc40471c86741dc3aa211379",
},
{
Name: "kubernetes.io/elb.tls-certificate-ids",
Value: "5196aa70b0f143189e4cb54991ba2286,8125d71fcc124aabbe007610cba42d60",
},
{
Name: "kubernetes.io/elb.multicluster",
Value: "true",
},
{
Name: "kubernetes.io/elb.keepalive_timeout",
Value: "400s",
},
{
Name: "kubernetes.io/elb.client_timeout",
Value: "50s",
},
{
Name: "kubernetes.io/elb.member_timeout",
Value: "50s",
},
{
Name: ExternalTrafficPolicyTypeConfigName,
Value: "Local",
},
},
slbConfig: &cceElbConfig{
elbIds: []string{"c1d8f4c6-7aef-4596-8c7c-yyyyyyyyyyyy"},
targetPorts: []int{81, 82, 83},
protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP},
externalTrafficPolicyType: corev1.ServiceExternalTrafficPolicyTypeLocal,
isFixed: false,
hwOptions: map[string]string{
"kubernetes.io/elb.class": "union",
"kubernetes.io/elb.id": "c1d8f4c6-7aef-4596-8c7c-yyyyyyyyyyyy",
"kubernetes.io/elb.enterpriseID": "ff97261-4dbd-4593-8236-yyyyyyyyyyyy",
"kubernetes.io/elb.cert-id": "17e3b4f4bc40471c86741dc3aa211379",
"kubernetes.io/elb.tls-certificate-ids": "5196aa70b0f143189e4cb54991ba2286,8125d71fcc124aabbe007610cba42d60",
"kubernetes.io/elb.multicluster": "true",
"kubernetes.io/elb.keepalive_timeout": "400s",
"kubernetes.io/elb.client_timeout": "50s",
"kubernetes.io/elb.member_timeout": "50s",
},
},
},
}
for i, test := range tests {
sc, err := parseCCELbConfig(test.conf)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(test.slbConfig, sc) {
t.Errorf("case %d: lbId expect: %v, actual: %v", i, test.slbConfig, sc)
}
}
}
func TestInitLbCache(t *testing.T) {
svcA := fakeSvcTemplate.DeepCopy()
svcA.Annotations[ElbIdAnnotationKey] = "elb-id-A"
svcA.Name = "svc-A"
svcA.Namespace = "ns-A"
svcA.Spec.Ports[0].Port = 555
svcB := fakeSvcTemplate.DeepCopy()
svcB.Annotations[ElbIdAnnotationKey] = "elb-id-B"
svcB.Name = "svc-B"
svcB.Namespace = "ns-B"
svcB.Spec.Ports[0].Port = 666
test := struct {
svcList []corev1.Service
minPort int32
maxPort int32
blockPorts []int32
cache map[string]ccePortAllocated
podAllocate map[string]string
}{
minPort: 512,
maxPort: 712,
blockPorts: []int32{593},
cache: map[string]ccePortAllocated{
"elb-id-A": map[int32]bool{
555: true,
593: true,
},
"elb-id-B": map[int32]bool{
666: true,
593: true,
},
},
podAllocate: map[string]string{
"ns-A/svc-A": "elb-id-A:555",
"ns-B/svc-B": "elb-id-B:666",
},
svcList: []corev1.Service{
*svcA,
*svcB,
},
}
actualCache, actualPodAllocate := initCCELbCache(test.svcList, test.minPort, test.maxPort, test.blockPorts)
for lb, pa := range test.cache {
for port, isAllocated := range pa {
if actualCache[lb][port] != isAllocated {
t.Errorf("lb %s port %d isAllocated, expect: %t, actual: %t", lb, port, isAllocated, actualCache[lb][port])
}
}
}
if !reflect.DeepEqual(actualPodAllocate, test.podAllocate) {
t.Errorf("podAllocate expect %v, but actully got %v", test.podAllocate, actualPodAllocate)
}
}
func TestElbPlugin_OnPodUpdated(t *testing.T) {
type fields struct {
maxPort int32
minPort int32
blockPorts []int32
cache map[string]ccePortAllocated
podAllocate map[string]string
}
type args struct {
pod func() *corev1.Pod
ctx context.Context
}
fakePodTemplate := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-0",
Namespace: "default",
UID: "53cb0992-c720-4ae4-9af9-3cc7e2bf3660",
Annotations: map[string]string{
"game.kruise.io/network-type": CCEElbNetwork,
"game.kruise.io/network-conf": `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"},{"name":"kubernetes.io/elb.connection-drain-enable","value":"true"},{"name":"kubernetes.io/elb.connection-drain-timeout","value":"300"}]`,
},
},
}
tests := []struct {
name string
fields fields
args args
setup func(*MockClient)
want func() *corev1.Pod
want1 errors.PluginError
}{
{
name: "network is not ready",
fields: fields{
maxPort: 500,
minPort: 502,
blockPorts: []int32{501},
cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}},
podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:501"},
},
args: args{
pod: func() *corev1.Pod {
return fakePodTemplate
},
ctx: context.Background(),
},
want: func() *corev1.Pod {
res := fakePodTemplate.DeepCopy()
res.Annotations["game.kruise.io/network-status"] = `{"currentNetworkState":"NotReady","createTime":null,"lastTransitionTime":null}`
return res
},
want1: nil,
},
{
name: "network is ready",
fields: fields{
maxPort: 500,
minPort: 502,
blockPorts: []int32{501},
cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}},
podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500,501"},
},
args: args{
pod: func() *corev1.Pod {
res := fakePodTemplate.DeepCopy()
res.Annotations["game.kruise.io/network-status"] = `{"internalAddresses":[{"ip":"192.168.1.38","ports":[{"name":"80","protocol":"TCP","port":80}]}],"externalAddresses":[{"ip":"192.168.0.147","ports":[{"name":"80","protocol":"TCP","port":500}]}],"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}`
return res
},
ctx: context.Background(),
},
setup: func(clientMock *MockClient) {
clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
service := args[2].(*corev1.Service)
*service = fakeSvcTemplate
}).Return(nil)
},
want: func() *corev1.Pod {
res := fakePodTemplate.DeepCopy()
res.Annotations["game.kruise.io/network-status"] = `{"internalAddresses":[{"ip":"","ports":[{"name":"80","protocol":"TCP","port":80}]}],"externalAddresses":[{"ip":"192.168.0.147","ports":[{"name":"80","protocol":"TCP","port":500}]}],"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}`
res.Annotations["game.kruise.io/network-conf"] = `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"},{"name":"kubernetes.io/elb.connection-drain-enable","value":"true"},{"name":"kubernetes.io/elb.connection-drain-timeout","value":"300"}]`
return res
},
want1: nil,
},
{
name: "svc is not exist",
fields: fields{
maxPort: 500,
minPort: 502,
blockPorts: []int32{501},
cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}},
podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500,501"},
},
args: args{
pod: func() *corev1.Pod {
res := fakePodTemplate.DeepCopy()
res.Annotations["game.kruise.io/network-status"] = `{"currentNetworkState":"NotReady","createTime":null,"lastTransitionTime":null}`
res.Annotations["game.kruise.io/network-conf"] = `[{"name":"Fixed", "value":"true"},{"name":"PortProtocols","value":"80/TCPUDP"},{"name":"kubernetes.io/elb.class","value":"union"},{"name":"kubernetes.io/elb.id","value":"c1d8f4c6-7aef-4596-8c7c-2de87ff89545"}]`
return res
},
ctx: context.Background(),
},
setup: func(clientMock *MockClient) {
clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(k8serrors.NewNotFound(schema.GroupResource{}, "test-pod-0"))
clientMock.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(nil)
},
want: func() *corev1.Pod {
res := fakePodTemplate.DeepCopy()
res.Annotations["game.kruise.io/network-status"] = `{"currentNetworkState":"NotReady","createTime":null,"lastTransitionTime":null}`
res.Annotations["game.kruise.io/network-conf"] = `[{"name":"Fixed", "value":"true"},{"name":"PortProtocols","value":"80/TCPUDP"},{"name":"kubernetes.io/elb.class","value":"union"},{"name":"kubernetes.io/elb.id","value":"c1d8f4c6-7aef-4596-8c7c-2de87ff89545"}]`
return res
},
want1: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
clientMock := new(MockClient)
if tt.setup != nil {
tt.setup(clientMock)
}
if tt.want == nil {
t.Fatal("want is nil, set the function")
}
if tt.args.pod == nil {
t.Fatal("pod is nil, set the function")
}
s := &CCEElbPlugin{
maxPort: tt.fields.maxPort,
minPort: tt.fields.minPort,
blockPorts: tt.fields.blockPorts,
cache: tt.fields.cache,
podAllocate: tt.fields.podAllocate,
mutex: sync.RWMutex{},
}
got, got1 := s.OnPodUpdated(clientMock, tt.args.pod(), tt.args.ctx)
assert.Equalf(t, tt.want().Annotations["game.kruise.io/network-status"], got.Annotations["game.kruise.io/network-status"], "OnPodUpdated(%v, %v, %v)", clientMock, tt.args.pod, tt.args.ctx)
assert.Equalf(t, tt.want().Annotations["game.kruise.io/network-type"], got.Annotations["game.kruise.io/network-type"], "OnPodUpdated(%v, %v, %v)", clientMock, tt.args.pod, tt.args.ctx)
assert.Equalf(t, tt.want().Annotations["game.kruise.io/network-conf"], got.Annotations["game.kruise.io/network-conf"], "OnPodUpdated(%v, %v, %v)", clientMock, tt.args.pod, tt.args.ctx)
assert.Equalf(t, tt.want1, got1, "OnPodUpdated(%v, %v, %v)", clientMock, tt.args.pod, tt.args.ctx)
})
}
}
func TestElbPlugin_Init(t *testing.T) {
type fields struct {
maxPort int32
minPort int32
blockPorts []int32
cache map[string]ccePortAllocated
podAllocate map[string]string
}
type args struct {
c client.Client
options cloudprovider.CloudProviderOptions
ctx context.Context
}
tests := []struct {
name string
fields fields
args args
setup func(clientMock *MockClient)
want *CCEElbPlugin
wantErr assert.ErrorAssertionFunc
}{
{
name: "success",
fields: fields{},
args: args{
c: nil,
options: options.HwCloudOptions{
Enable: true,
CCEELBOptions: options.CCEELBOptions{
ELBOptions: options.ELBOptions{
MaxPort: 503,
MinPort: 500,
BlockPorts: []int32{501},
},
},
},
ctx: context.Background(),
},
setup: func(clientMock *MockClient) {
clientMock.On("List", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
res := args[1].(*corev1.ServiceList)
*res = corev1.ServiceList{
Items: []corev1.Service{
fakeSvcTemplate,
},
}
}).Return(nil)
},
want: &CCEElbPlugin{
maxPort: 503,
minPort: 500,
blockPorts: []int32{501},
cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false, 503: false}},
podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"},
mutex: sync.RWMutex{},
},
wantErr: assert.NoError,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := &CCEElbPlugin{
maxPort: tt.fields.maxPort,
minPort: tt.fields.minPort,
blockPorts: tt.fields.blockPorts,
cache: tt.fields.cache,
podAllocate: tt.fields.podAllocate,
mutex: sync.RWMutex{},
}
clientMock := new(MockClient)
tt.args.c = clientMock
if tt.setup != nil {
tt.setup(clientMock)
}
tt.wantErr(t, s.Init(tt.args.c, tt.args.options, tt.args.ctx), fmt.Sprintf("Init(%v, %v, %v)", tt.args.c, tt.args.options, tt.args.ctx))
assert.Equal(t, s.cache, tt.want.cache)
assert.Equal(t, s.podAllocate, tt.want.podAllocate)
assert.Equal(t, s.minPort, tt.want.minPort)
assert.Equal(t, s.maxPort, tt.want.maxPort)
assert.Equal(t, s.blockPorts, tt.want.blockPorts)
})
}
}
func TestElbPlugin_updateCachesAfterAutoCreateElb(t *testing.T) {
type fields struct {
maxPort int32
minPort int32
blockPorts []int32
cache map[string]ccePortAllocated
podAllocate map[string]string
}
type args struct {
name string
namespace string
interval time.Duration
totalTimeout time.Duration
}
tests := []struct {
name string
fields fields
args args
setup func(clientMock *MockClient)
wantCache map[string]ccePortAllocated
wantPodAllocate map[string]string
}{
{
name: "success",
fields: fields{
minPort: 500,
maxPort: 502,
blockPorts: []int32{501},
},
args: args{
name: "test-pod-0",
namespace: "default",
interval: 1 * time.Second,
totalTimeout: 2 * time.Second,
},
setup: func(clientMock *MockClient) {
clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
service := args[2].(*corev1.Service)
*service = fakeSvcTemplate
}).Return(nil)
},
wantCache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}},
wantPodAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"},
},
{
name: "timeout",
fields: fields{
minPort: 500,
maxPort: 502,
blockPorts: []int32{501},
},
args: args{
name: "test-pod-0",
namespace: "default",
interval: 1 * time.Second,
totalTimeout: 2 * time.Second,
},
setup: func(clientMock *MockClient) {
clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("some error"))
},
wantCache: nil,
wantPodAllocate: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
clientMock := new(MockClient)
if tt.setup != nil {
tt.setup(clientMock)
}
s := &CCEElbPlugin{
maxPort: tt.fields.maxPort,
minPort: tt.fields.minPort,
blockPorts: tt.fields.blockPorts,
cache: tt.fields.cache,
podAllocate: tt.fields.podAllocate,
mutex: sync.RWMutex{},
}
s.updateCachesAfterAutoCreateElb(clientMock, tt.args.name, tt.args.namespace, tt.args.interval, tt.args.totalTimeout)
assert.Equal(t, s.cache, tt.wantCache)
assert.Equal(t, s.podAllocate, tt.wantPodAllocate)
})
}
}
func TestElbPlugin_OnPodDeleted(t *testing.T) {
podForTest := &corev1.Pod{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-0",
Namespace: "default",
Annotations: map[string]string{
"kubernetes.io/elb.id": "8f4cf216-a659-40dc-8c77-6068b036ba56",
"game.kruise.io/network-conf": `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"}]`,
"game.kruise.io/network-type": CCEElbNetwork,
"game.kruise.io/network-status": `{"internalAddresses":[{"ip":"192.168.1.219","ports":[{"name":"80","protocol":"TCP","port":80}]}],"externalAddresses":[{"ip":"159.138.146.2","ports":[{"name":"80","protocol":"TCP","port":500}]}],"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}`,
},
Labels: map[string]string{
"game.kruise.io/owner-gss": "test-pod",
},
},
}
deleteTimetamp := metav1.NewTime(time.Now())
type fields struct {
maxPort int32
minPort int32
blockPorts []int32
cache map[string]ccePortAllocated
podAllocate map[string]string
}
type args struct {
c client.Client
pod func(p *corev1.Pod) *corev1.Pod
ctx context.Context
}
tests := []struct {
name string
fields fields
args args
setup func(clientMock *MockClient)
wantCache map[string]ccePortAllocated
wantPodAllocate map[string]string
wantError errors.PluginError
}{
{
name: "success",
fields: fields{
maxPort: 502,
minPort: 500,
blockPorts: []int32{501},
cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}},
podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"},
},
args: args{
c: nil,
pod: func(p *corev1.Pod) *corev1.Pod {
return p.DeepCopy()
},
ctx: context.Background(),
},
wantCache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: false, 501: true, 502: false}},
wantPodAllocate: map[string]string{},
wantError: nil,
},
{
name: "fixed, get gss failed",
fields: fields{
maxPort: 502,
minPort: 500,
blockPorts: []int32{501},
cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}},
podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"},
},
args: args{
c: nil,
pod: func(p *corev1.Pod) *corev1.Pod {
res := p.DeepCopy()
res.Annotations["game.kruise.io/network-conf"] = `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"}, {"name":"Fixed","value":"true"}]`
res.Labels["game.kruise.io/owner-gss"] = "test-pod"
return res
},
ctx: context.Background(),
},
setup: func(clientMock *MockClient) {
clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("some error"))
},
wantCache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}},
wantPodAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"},
wantError: errors.ToPluginError(fmt.Errorf("some error"), errors.ApiCallError),
},
{
name: "fixed, gss exists",
fields: fields{
maxPort: 502,
minPort: 500,
blockPorts: []int32{501},
cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}},
podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"},
},
args: args{
c: nil,
pod: func(p *corev1.Pod) *corev1.Pod {
res := p.DeepCopy()
res.Annotations["game.kruise.io/network-conf"] = `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"}, {"name":"Fixed","value":"true"}]`
res.Labels["game.kruise.io/owner-gss"] = "test-pod"
return res
},
ctx: context.Background(),
},
setup: func(clientMock *MockClient) {
clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
gss := args[2].(*gamekruiseiov1alpha1.GameServerSet)
*gss = gamekruiseiov1alpha1.GameServerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
Annotations: map[string]string{},
},
}
}).Return(nil)
},
wantCache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}},
wantPodAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"},
wantError: nil,
},
{
name: "fixed, gss deleted",
fields: fields{
maxPort: 502,
minPort: 500,
blockPorts: []int32{501},
cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}},
podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"},
},
args: args{
c: nil,
pod: func(p *corev1.Pod) *corev1.Pod {
res := p.DeepCopy()
res.Annotations["game.kruise.io/network-conf"] = `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"}, {"name":"Fixed","value":"true"}]`
res.Labels["game.kruise.io/owner-gss"] = "test-pod"
return res
},
ctx: context.Background(),
},
setup: func(clientMock *MockClient) {
clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
gss := args[2].(*gamekruiseiov1alpha1.GameServerSet)
*gss = gamekruiseiov1alpha1.GameServerSet{
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
Annotations: map[string]string{},
DeletionTimestamp: &deleteTimetamp,
},
}
}).Return(nil)
},
wantCache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: false, 501: true, 502: false}},
wantPodAllocate: map[string]string{},
wantError: nil,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
clientMock := new(MockClient)
tt.args.c = clientMock
if tt.setup != nil {
tt.setup(clientMock)
}
s := &CCEElbPlugin{
maxPort: tt.fields.maxPort,
minPort: tt.fields.minPort,
blockPorts: tt.fields.blockPorts,
cache: tt.fields.cache,
podAllocate: tt.fields.podAllocate,
mutex: sync.RWMutex{},
}
assert.Equalf(t, tt.wantError, s.OnPodDeleted(tt.args.c, tt.args.pod(podForTest), tt.args.ctx), "OnPodDeleted(%v, %v, %v)", tt.args.c, tt.args.pod, tt.args.ctx)
assert.Equal(t, s.cache, tt.wantCache)
assert.Equal(t, s.podAllocate, tt.wantPodAllocate)
})
}
}
func Test_getSvcOwnerReference(t *testing.T) {
type args struct {
ctx context.Context
pod *corev1.Pod
isFixed bool
}
tests := []struct {
name string
args args
want []metav1.OwnerReference
setup func(clientMock *MockClient)
}{
{
name: "fixed, success",
args: args{
ctx: context.Background(),
pod: &corev1.Pod{
TypeMeta: metav1.TypeMeta{
Kind: "Pod",
APIVersion: "v1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod-0",
Namespace: "default",
UID: "pod-uuid-xxxx",
},
},
isFixed: true,
},
setup: func(clientMock *MockClient) {
clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) {
gss := args[2].(*gamekruiseiov1alpha1.GameServerSet)
*gss = gamekruiseiov1alpha1.GameServerSet{
TypeMeta: metav1.TypeMeta{
Kind: "GameServerSet",
APIVersion: "game.kruise.io/v1alpha1",
},
ObjectMeta: metav1.ObjectMeta{
Name: "test-pod",
Namespace: "default",
UID: "gss-uuid-xxxx",
},
}
}).Return(nil)
},
want: []metav1.OwnerReference{
{
APIVersion: "game.kruise.io/v1alpha1",
Kind: "GameServerSet",
Name: "test-pod",
UID: "gss-uuid-xxxx",
Controller: ptr.To[bool](true),
BlockOwnerDeletion: ptr.To[bool](true),
},
},
},
}
for _, tt := range tests {
clientMock := new(MockClient)
if tt.setup != nil {
tt.setup(clientMock)
}
t.Run(tt.name, func(t *testing.T) {
assert.Equalf(t, tt.want, getCCESvcOwnerReference(clientMock, tt.args.ctx, tt.args.pod, tt.args.isFixed), "getCCESvcOwnerReference(%v, %v, %v, %v)", clientMock, tt.args.ctx, tt.args.pod, tt.args.isFixed)
})
}
}
func TestElbPlugin_getPortFromHead(t *testing.T) {
type fields struct {
maxPort int32
minPort int32
blockPorts []int32
cache map[string]ccePortAllocated
podAllocate map[string]string
}
type args struct {
num int
}
tests := []struct {
name string
fields fields
args args
want []int32
}{
{
name: "success",
fields: fields{
maxPort: 505,
minPort: 500,
blockPorts: []int32{501},
},
args: args{
num: 2,
},
want: []int32{500, 502},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
s := &CCEElbPlugin{
maxPort: tt.fields.maxPort,
minPort: tt.fields.minPort,
blockPorts: tt.fields.blockPorts,
cache: tt.fields.cache,
podAllocate: tt.fields.podAllocate,
mutex: sync.RWMutex{},
}
assert.Equalf(t, tt.want, s.getPortFromHead(tt.args.num), "getPortFromHead(%v)", tt.args.num)
})
}
}

80
cloudprovider/hwcloud/elb_test_mock.go Normal file
View File

@ -0,0 +1,80 @@
package hwcloud
import (
"context"
"github.com/stretchr/testify/mock"
"k8s.io/apimachinery/pkg/api/meta"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/client"
)
type MockClient struct {
mock.Mock
}
func (m *MockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error {
args := m.Called(ctx, list, opts)
return args.Error(0)
}
func (m *MockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error {
//TODO implement me
panic("implement me")
}
func (m *MockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error {
//TODO implement me
panic("implement me")
}
func (m *MockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error {
//TODO implement me
panic("implement me")
}
func (m *MockClient) Status() client.SubResourceWriter {
//TODO implement me
panic("implement me")
}
func (m *MockClient) SubResource(subResource string) client.SubResourceClient {
//TODO implement me
panic("implement me")
}
func (m *MockClient) Scheme() *runtime.Scheme {
//TODO implement me
panic("implement me")
}
func (m *MockClient) RESTMapper() meta.RESTMapper {
//TODO implement me
panic("implement me")
}
func (m *MockClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) {
//TODO implement me
panic("implement me")
}
func (m *MockClient) IsObjectNamespaced(obj runtime.Object) (bool, error) {
//TODO implement me
panic("implement me")
}
func (m *MockClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error {
args := m.Called(ctx, key, obj, opts)
return args.Error(0)
}
func (m *MockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error {
args := m.Called(ctx, obj)
return args.Error(0)
}
func (m *MockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error {
args := m.Called(ctx, obj)
return args.Error(0)
}

16
cloudprovider/hwcloud/hwcloud.go
View File

@ -1,3 +1,19 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package hwcloud
import (

25
cloudprovider/options/hwcloud_options.go
View File

@ -1,7 +1,12 @@
package options
type HwCloudOptions struct {
Enable bool `toml:"enable"`
Enable bool `toml:"enable"`
ELBOptions ELBOptions `toml:"elb"`
CCEELBOptions CCEELBOptions `toml:"cce"`
}
type CCEELBOptions struct {
ELBOptions ELBOptions `toml:"elb"`
}
@ -11,22 +16,28 @@ type ELBOptions struct {
BlockPorts []int32 `toml:"block_ports"`
}
func (o HwCloudOptions) Valid() bool {
elbOptions := o.ELBOptions
for _, blockPort := range elbOptions.BlockPorts {
if blockPort >= elbOptions.MaxPort || blockPort <= elbOptions.MinPort {
func (e ELBOptions) valid(skipPortRangeCheck bool) bool {
for _, blockPort := range e.BlockPorts {
if blockPort >= e.MaxPort || blockPort <= e.MinPort {
return false
}
}
if int(elbOptions.MaxPort-elbOptions.MinPort)-len(elbOptions.BlockPorts) > 200 {
// old elb plugin only allow 200 ports.
if !skipPortRangeCheck && int(e.MaxPort-e.MinPort)-len(e.BlockPorts) > 200 {
return false
}
if elbOptions.MinPort <= 0 {
if e.MinPort <= 0 || e.MaxPort > 65535 {
return false
}
return true
}
func (o HwCloudOptions) Valid() bool {
elbOptions := o.ELBOptions
cceElbOptions := o.CCEELBOptions
return elbOptions.valid(false) && cceElbOptions.ELBOptions.valid(true)
}
func (o HwCloudOptions) Enabled() bool {
return o.Enable
}

4
config/manager/config.toml
View File

@ -21,6 +21,10 @@ enable = true
max_port = 700
min_port = 500
block_ports = []
[hwcloud.cce.elb]
max_port = 65535
min_port = 32768
block_ports = []
[volcengine]
enable = true

501
docs/en/user_manuals/network.md
View File

@ -134,6 +134,9 @@ OpenKruiseGame supports the following network plugins:
- AlibabaCloud-SLB
- AlibabaCloud-SLB-SharedPort
- Volcengine-EIP
- HwCloud-ELB
- HwCloud-CCE-ELB
- HwCloud-CCE-EIP
---
@ -1217,9 +1220,507 @@ max_port = 700
min_port = 500
block_ports = []
```
---
### HwCloud-CCE-ELB
#### Plugin name
`HwCloud-CCE-ELB`
**Note**:
- This plugin is only applicable to Huawei Cloud's CCE Standard and CCE Turbo clusters.
- If using an existing ELB, ensure its VPC matches the CCE cluster's VPC; otherwise, access will fail.
#### Cloud Provider
HuaweiCloud
#### Plugin description
- HwCloud-ELB uses Huawei Cloud Load Balancer (ELB) as the entity for external service hosting. It distributes external traffic to multiple Pods within the cluster through Elastic Load Balancing (ELB), providing higher reliability compared to the NodePort type.
- Supported annotations, please refer to the documentation: https://support.huaweicloud.com/usermanual-cce/cce_10_0681.html
- The exposed public network access port is consistent with the port being listened to in the container.
- You can bind security groups for management ([Use annotations to bind security groups to Pods](https://support.huaweicloud.com/usermanual-cce/cce_10_0897.html)), which is only supported in CCE Turbo clusters.
- The network interface of the Pod uses the security group configured via the annotation: `yangtse.io/security-group-ids`.
- The Pod's network interface will use the existing security groups and additionally include the security group configured via the annotation: `yangtse.io/additional-security-group-ids`.
- Supports Network Isolation: Yes.
#### Network parameters
PortProtocols
- Meaning: Exposed ports and protocols of the Pod. Supports multiple ports/protocols.
- Format: port1/protocol1,port2/protocol2,... (Protocols must be uppercase).
- Supports Modification: Yes.
Fixed
- Meaning: Whether to retain fixed access IP/port. If enabled, the external/internal mapping relationship remains unchanged even if Pods are recreated.
- Format: false / true
- Supports Modification: Yes.
AllowNotReadyContainers
- Meaning: Container names allowed to maintain traffic flow during in-place upgrades.
- Format: {containerName_0},{containerName_1},... e.g., sidecar
- Supports Modification: Not modifiable during in-place upgrades.
ExternalTrafficPolicyType
- Meaning: Determines whether Service LB forwards traffic only to local instances. Setting to Local creates a Local-type Service and retains client source IP addresses when configured with cloud-manager.
- Format: Local / Cluster (Default: Cluster)
- Supports Modification: No. Due to dependencies on fixed IP/port settings, modification is not recommended.
Other Huawei CCE Cluster Parameters
Refer to annotations' keys/values in the documentation:
- [LoadBalancer](https://support.huaweicloud.com/usermanual-cce/cce_10_0014.html)
#### Plugin configuration
The port range here can be configured according to your business requirements. For block_ports, please refer to this issue: https://github.com/openkruise/kruise-game/issues/174
```
[hwcloud]
enable = true
[hwcloud.cce.elb]
max_port = 65535
min_port = 32768
block_ports = []
```
---
#### Example
Using Existing ELB
https://support.huaweicloud.com/usermanual-cce/cce_10_0385.html#section1
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: hw-cce-elb-nginx
namespace: default
spec:
replicas: 2
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: HwCloud-CCE-ELB
networkConf:
- name: PortProtocols
value: "80/TCP"
- name: kubernetes.io/elb.class # The type of the ELB instance
value: performance
- name: kubernetes.io/elb.id # The ID of the ELB instance
value: 8f4cf216-a659-40dc-8c77-xxxx
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
The generated svc is shown below. As you can see, both svcs point to the same ELB.
```yaml
apiVersion: v1
kind: Service
metadata:
annotations:
game.kruise.io/network-config-hash: "3594992400"
kubernetes.io/elb.class: performance
kubernetes.io/elb.connection-drain-enable: "true"
kubernetes.io/elb.connection-drain-timeout: "300"
kubernetes.io/elb.id: 8f4cf216-a659-40dc-8c77-xxxx
kubernetes.io/elb.mark: "0"
creationTimestamp: "2025-07-23T08:15:09Z"
finalizers:
- service.kubernetes.io/load-balancer-cleanup
name: hw-cce-elb-nginx-0
namespace: kruise-game-system
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Pod
name: hw-cce-elb-nginx-0
uid: 4f9f37f9-16d4-4ee7-b553-9b6e0039c5d5
resourceVersion: "13369506"
uid: 23815818-a626-4be3-b31f-4b95a4f89786
spec:
allocateLoadBalancerNodePorts: true
clusterIP: 10.247.213.xxx
clusterIPs:
- 10.247.213.xxx
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
loadBalancerIP: 192.168.0.xxx
ports:
- name: 80-tcp
nodePort: 30622
port: 3308
protocol: TCP
targetPort: 80
- name: 80-udp
nodePort: 30622
port: 3308
protocol: UDP
targetPort: 80
selector:
statefulset.kubernetes.io/pod-name: hw-cce-elb-nginx-0
sessionAffinity: None
type: LoadBalancer
status:
loadBalancer:
ingress:
- ip: 192.168.0.xxx
- ip: 189.1.225.xxx
---
apiVersion: v1
kind: Service
metadata:
annotations:
game.kruise.io/network-config-hash: "3594992400"
kubernetes.io/elb.class: performance
kubernetes.io/elb.connection-drain-enable: "true"
kubernetes.io/elb.connection-drain-timeout: "300"
kubernetes.io/elb.id: 8f4cf216-a659-40dc-8c77-xxxx
kubernetes.io/elb.mark: "0"
creationTimestamp: "2025-07-23T08:15:08Z"
finalizers:
- service.kubernetes.io/load-balancer-cleanup
name: hw-cce-elb-nginx-1
namespace: kruise-game-system
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Pod
name: hw-cce-elb-nginx-1
uid: 0f42b430-49ba-4203-8b50-4be059619b79
resourceVersion: "13369489"
uid: 92a56054-ad92-4dbd-9d1b-e717e0a14af2
spec:
allocateLoadBalancerNodePorts: true
clusterIP: 10.247.14.xxx
clusterIPs:
- 10.247.14.xxx
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
loadBalancerIP: 192.168.0.xxx
ports:
- name: 80-tcp
nodePort: 32227
port: 3611
protocol: TCP
targetPort: 80
- name: 80-udp
nodePort: 32227
port: 3611
protocol: UDP
targetPort: 80
selector:
statefulset.kubernetes.io/pod-name: hw-cce-elb-nginx-1
sessionAffinity: None
type: LoadBalancer
status:
loadBalancer:
ingress:
- ip: 192.168.0.xxx
- ip: 189.1.225.xxx
```
The generated svc is shown below. As you can see, both svcs point to the same IP address, differing only in their ports:
```bash
kubectl get svc |grep hw-cce-elb-nginx
hw-cce-elb-nginx-0 LoadBalancer 10.247.213.xxx 189.1.225.xxx,192.168.0.xxx 3308:30622/TCP,3308:30622/UDP 2m3s
hw-cce-elb-nginx-1 LoadBalancer 10.247.14.xxx 189.1.225.xxx,192.168.0.xxx 3611:32227/TCP,3611:32227/UDP 2m4s
```
---
Automatically create an ELB and bind it to the created pod.
**Note**:
- When ELBs are automatically created for multiple replicas, each svc will use its own auto-created ELB. Each ELB will have a unique ID and a distinct external IP address.
- When the svc is deleted, the associated auto-created ELB will also be deleted.
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: hw-cce-elb-auto-performance
namespace: kruise-game-system
spec:
replicas: 2
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: HwCloud-CCE-ELB
networkConf:
- name: PortProtocols
value: "80/TCP"
- name: kubernetes.io/elb.class
value: performance # The type of the ELB instance.
- name: kubernetes.io/elb.autocreate # Options for automatically creating an ELB: https://support.huaweicloud.com/usermanual-cce/cce_10_0385.html#section21
value: '{
"type": "public",
"bandwidth_name": "bandwidth-xxxx",
"bandwidth_chargemode": "traffic",
"bandwidth_size": 5,
"bandwidth_sharetype": "PER",
"eip_type": "5_bgp",
"available_zone": [
"ap-southeast-1a",
"ap-southeast-1b"
],
"l4_flavor_name": "L4_flavor.elb.s1.small"
}'
- name: kubernetes.io/elb.enterpriseID # The enterprise project ID to which the created load balancer belongs.
value: 'aff97261-4dbd-4593-8236-xxxx'
- name: kubernetes.io/elb.lb-algorithm
value: ROUND_ROBIN # Load balancer algorithm
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
The generated svc is shown below. As you can see, both svcs point to different ELBs.
```yaml
apiVersion: v1
kind: Service
metadata:
annotations:
game.kruise.io/network-config-hash: "3090934611"
kubernetes.io/elb.autocreate: '{ "type": "public", "bandwidth_name": "bandwidth-89f0",
"bandwidth_chargemode": "traffic", "bandwidth_size": 5, "bandwidth_sharetype":
"PER", "eip_type": "5_bgp", "available_zone": [ "ap-southeast-1a", "ap-southeast-1b"
], "l4_flavor_name": "L4_flavor.elb.s1.small" }'
kubernetes.io/elb.class: performance
kubernetes.io/elb.eip-id: 566d5f4c-3484-4d7e-aa6b-xxxx
kubernetes.io/elb.enterpriseID: aff97261-4dbd-4593-8236-xxxx
kubernetes.io/elb.id: 75e06e8b-a246-48cb-b05c-xxxx
kubernetes.io/elb.lb-algorithm: ROUND_ROBIN
kubernetes.io/elb.mark: "0"
creationTimestamp: "2025-07-23T09:25:01Z"
finalizers:
- service.kubernetes.io/load-balancer-cleanup
name: hw-cce-elb-auto-performance-0
namespace: kruise-game-system
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Pod
name: hw-cce-elb-auto-performance-0
uid: 1da0edf4-f45d-4635-8db0-ed5ccea2441d
resourceVersion: "13401553"
uid: 13efd440-65a7-4b45-bafc-2268102a4fd7
spec:
allocateLoadBalancerNodePorts: true
clusterIP: 10.247.50.xxx
clusterIPs:
- 10.247.50.xxx
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
loadBalancerIP: 49.0.251.xxx
ports:
- name: 80-tcp
nodePort: 30918
port: 1
protocol: TCP
targetPort: 80
selector:
statefulset.kubernetes.io/pod-name: hw-cce-elb-auto-performance-0
sessionAffinity: None
type: LoadBalancer
status:
loadBalancer:
ingress:
- ip: 49.0.251.xxx
- ip: 192.168.1.xxx
---
apiVersion: v1
kind: Service
metadata:
annotations:
game.kruise.io/network-config-hash: "3090934611"
kubernetes.io/elb.autocreate: '{ "type": "public", "bandwidth_name": "bandwidth-89f0",
"bandwidth_chargemode": "traffic", "bandwidth_size": 5, "bandwidth_sharetype":
"PER", "eip_type": "5_bgp", "available_zone": [ "ap-southeast-1a", "ap-southeast-1b"
], "l4_flavor_name": "L4_flavor.elb.s1.small" }'
kubernetes.io/elb.class: performance
kubernetes.io/elb.eip-id: 4a5396b1-e750-4ba5-a5d3-xxxx
kubernetes.io/elb.enterpriseID: aff97261-4dbd-4593-8236-xxxx
kubernetes.io/elb.id: b093db79-3c3e-4e77-a2ee-xxxx
kubernetes.io/elb.lb-algorithm: ROUND_ROBIN
kubernetes.io/elb.mark: "0"
creationTimestamp: "2025-07-23T09:25:01Z"
finalizers:
- service.kubernetes.io/load-balancer-cleanup
name: hw-cce-elb-auto-performance-1
namespace: kruise-game-system
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Pod
name: hw-cce-elb-auto-performance-1
uid: abfc9ad1-1ae3-45fa-b956-4617c465a44f
resourceVersion: "13401664"
uid: 01dd8e13-b1c8-4d9f-8b1c-13c2f001c614
spec:
allocateLoadBalancerNodePorts: true
clusterIP: 10.247.196.xxx
clusterIPs:
- 10.247.196.xxx
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
loadBalancerIP: 150.40.245.xxx
ports:
- name: 80-tcp
nodePort: 30942
port: 1
protocol: TCP
targetPort: 80
selector:
statefulset.kubernetes.io/pod-name: hw-cce-elb-auto-performance-1
sessionAffinity: None
type: LoadBalancer
status:
loadBalancer:
ingress:
- ip: 150.40.245.xxx
- ip: 192.168.1.xxx
```
The generated svc is shown below. As you can see, both svcs are assigned different external IPs:
```bash
kubectl get svc |grep hw-cce-elb-auto-performance
hw-cce-elb-auto-performance-0 LoadBalancer 10.247.50.xxx 192.168.1.xxx,49.0.251.xxx 1:30918/TCP 4m29s
hw-cce-elb-auto-performance-1 LoadBalancer 10.247.196.xxx 150.40.245.xxx,192.168.1.xxx 1:30942/TCP 4m29s
```
#### Plugin Name
`HwCloud-EIP`
**Note**: This plugin is only applicable to Huawei Cloud's CCE Turbo clusters.
#### Cloud Provider
HuaweiCloud
#### Plugin Description
- Only Huawei Cloud CCE Turbo clusters are supported: https://support.huaweicloud.com/usermanual-cce/cce_10_0284.html#section1
- Assigns a separate Elastic IP (EIP) to each pod.
- The exposed public network access port is consistent with the port being listened to in the container. Security groups can be bound for management ([Binding Security Groups to Pods Using Annotations](https://support.huaweicloud.com/usermanual-cce/cce_10_0897.html))
- The Pod's network interface uses the security group configured via the annotation: `yangtse.io/security-group-ids`.
- The Pod's network interface will use the existing security groups while additionally applying the security group configured via the annotation: `yangtse.io/additional-security-group-ids`
- The automatically created EIP does not support specifying the 'enterprise project' during creation.
#### Network Parameters
Refer to Huawei Cloud documentation: https://support.huaweicloud.com/usermanual-cce/cce_10_0734.html. This plugin supports all annotations on this page.
#### Plugin Configuration
None
#### Example
Exclusive Bandwidth EIP Created with Pod
Note: The EIP created here belongs to the `default` enterprise project. Huawei Cloud currently does not support specifying enterprise projects in this mode.
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: hwcloud-cce-eip-performance
namespace: default
spec:
replicas: 2
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: HwCloud-CCE-EIP
networkConf:
# https://support.huaweicloud.com/usermanual-cce/cce_10_0734.html
- name: yangtse.io/pod-with-eip
value: "true"
- name: yangtse.io/eip-bandwidth-size
value: "5"
- name: yangtse.io/eip-network-type
value: "5_bgp"
- name: yangtse.io/eip-charge-mode
value: "traffic"
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
Generated Pod Annotations:
`yangtse.io/allocated-eip-id` corresponds to the EIP viewable in Huawei Cloud's Elastic IP details.
`yangtse.io/allocated-ipv4-eip` is the pod's EIP.
```yaml
apiVersion: v1
kind: Pod
metadata:
annotations:
apps.kruise.io/runtime-containers-meta: '{"containers":[{"name":"nginx","containerID":"containerd://302f710dc7fb5771be5b16a31de84ff457fd84c9aa1ce00b7e7f2ddc3b7c3978","restartCount":0,"hashes":{"plainHash":2641665875,"plainHashWithoutResources":0,"extractedEnvFromMetadataHash":86995377}}]}'
game.kruise.io/network-conf: '[{"name":"yangtse.io/pod-with-eip","value":"true"},{"name":"yangtse.io/eip-bandwidth-size","value":"5"},{"name":"yangtse.io/eip-network-type","value":"5_bgp"},{"name":"yangtse.io/eip-charge-mode","value":"traffic"}]'
game.kruise.io/network-status: '{"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}'
game.kruise.io/network-trigger-time: "2025-07-16 17:03:07"
game.kruise.io/network-type: HwCloud-EIP
game.kruise.io/opsState-last-changed-time: "2025-07-16 17:03:07"
game.kruise.io/state-last-changed-time: "2025-07-16 09:03:13"
lifecycle.apps.kruise.io/timestamp: "2025-07-16T09:03:03Z"
yangtse.io/allocated-eip-id: 3a52ca79-d78d-4fc2-8590-xxx
yangtse.io/allocated-ipv4-eip: 94.74.110.xxx
yangtse.io/eip-bandwidth-size: "5"
yangtse.io/eip-charge-mode: traffic
yangtse.io/eip-network-type: 5_bgp
yangtse.io/pod-with-eip: "true"
```
To use an existing EIP, add yangtse.io/eip-id in spec.network.networkConf. You need to create the EIP in Huawei Cloud in advance.
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: hw-cce-eip-exist
namespace: kruise-game-system
spec:
replicas: 1
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: HwCloud-CCE-EIP
networkConf:
- name: yangtse.io/eip-id
value: "7ec474aa-3bd9-46a2-a45c-xxx" # Use an existing EIP.
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
In the pod's YAML, you can see that the yangtse.io/allocated-eip-id in the pod's annotations corresponds to the EIP we specified.
By logging into the Huawei Cloud EIP console, you can verify that this EIP is already bound to the pod.
```yaml
apiVersion: v1
kind: Pod
metadata:
annotations:
apps.kruise.io/runtime-containers-meta: '{"containers":[{"name":"nginx","containerID":"containerd://0fc9de69e30b48cf13ad2d2c6f5fe3be86e48e922a982dbb77b53ffd0ca6f54b","restartCount":0,"hashes":{"plainHash":2957831032,"plainHashWithoutResources":0,"extractedEnvFromMetadataHash":86995377}}]}'
game.kruise.io/network-conf: '[{"name":"yangtse.io/eip-id","value":"7ec474aa-3bd9-46a2-a45c-xxxx"}]'
game.kruise.io/network-status: '{"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}'
game.kruise.io/network-trigger-time: "2025-07-18 15:38:21"
game.kruise.io/network-type: HwCloud-EIP
game.kruise.io/opsState-last-changed-time: "2025-07-18 15:38:21"
game.kruise.io/state-last-changed-time: "2025-07-18 15:38:31"
lifecycle.apps.kruise.io/timestamp: "2025-07-18T07:38:13Z"
yangtse.io/allocated-eip-id: 7ec474aa-3bd9-46a2-a45c-xxxx
yangtse.io/allocated-ipv4-eip: 159.138.21.xxx
yangtse.io/eip-id: 7ec474aa-3bd9-46a2-a45c-xxxx
creationTimestamp: "2025-07-18T07:38:14Z
# other info ignored
```
### Volcengine-EIP
#### Plugin name

503
docs/中文/用户手册/网络模型.md
View File

@ -140,7 +140,9 @@ EOF
- AlibabaCloud-NLB-SharedPort
- Volcengine-CLB
- Volcengine-EIP
- HwCloud-ELB
- HwCloud-CCE-ELB
- HwCloud-CCE-EIP
---
### Kubernetes-HostPort
@ -1342,7 +1344,506 @@ spec:
lastTransitionTime: "2024-10-28T03:16:20Z"
networkType: TencentCloud-CLB
```
---
### HwCloud-CCE-ELB
#### 插件名称
`HwCloud-CCE-ELB`
**注意**:
- 此插件仅仅适用于华为云的CCE Standard和CCE Turbo集群.
- 如果使用已有的ELB, 请保证ELB的VPC和CCE集群的VPC一致, 否则无法访问.
#### Cloud Provider
HuaweiCloud
#### 插件说明
- HwCloud-ELB 使用华为云负载均衡器（ELB）作为对外服务的承载实体，可以通过弹性负载均衡（ELB）将外部流量向集群内的多个Pod进行分发，与NodePort类型相比提供了高可靠的保障。
- 支持的annotations,可以参看文档: https://support.huaweicloud.com/usermanual-cce/cce_10_0681.html
- 暴露的公网访问端口与容器中监听的端口一致.
- 可以绑定安全组进行管理([使用注解为Pod绑定安全组](https://support.huaweicloud.com/usermanual-cce/cce_10_0897.html)),需要CCE Turbo集群才支持。
- Pod的网卡使用annotation配置的安全组: `yangtse.io/security-group-ids`
- Pod的网卡在使用已有安全组的基础上，额外再增加annotation配置的安全组: `yangtse.io/additional-security-group-ids`
- 是否支持网络隔离：是。
#### 网络参数
PortProtocols
- 含义：pod暴露的端口及协议，支持填写多个端口/协议。
- 格式：port1/protocol1,port2/protocol2,...（协议需大写）
- 是否支持变更：支持。
Fixed
- 含义：是否固定访问IP/端口。若是，即使pod删除重建，网络内外映射关系不会改变
- 填写格式：false / true
- 是否支持变更：支持
AllowNotReadyContainers
- 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个
- 格式：{containerName_0},{containerName_1},... 例如：sidecar
- 是否支持变更：在原地升级过程中不可变更。
ExternalTrafficPolicyType
- 含义：Service LB 是否只转发给本地实例。若是Local， 创建Local类型Service, 配合cloud-manager只配置对应Node，可以保留客户端源IP地址
- 填写格式: Local/Cluster 默认Cluster
- 是否支持变更：不支持。跟是否固定IP/端口有关系，建议不更改
其他与华为CCE集群相关的参数
参考文档里的annotation的key与value进行填写
- [负载均衡（LoadBalancer）](https://support.huaweicloud.com/usermanual-cce/cce_10_0014.html)
#### Plugin configuration
这里的端口范围可以根据您的业务范围自行配置, block_ports请参考这个issue: https://github.com/openkruise/kruise-game/issues/174
```
[hwcloud]
enable = true
[hwcloud.cce.elb]
max_port = 65535
min_port = 32768
block_ports = []
```
---
#### 示例说明
使用已有的elb(https://support.huaweicloud.com/usermanual-cce/cce_10_0385.html#section1),其他可用的annotation请参考华为云文档
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: hw-cce-elb-nginx
namespace: default
spec:
replicas: 2
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: HwCloud-CCE-ELB
networkConf:
- name: PortProtocols
value: "80/TCP"
- name: kubernetes.io/elb.class # ELB实例的类型
value: performance
- name: kubernetes.io/elb.id # ELB实例的ID
value: 8f4cf216-a659-40dc-8c77-xxxx
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
生成的svc如下所示,可以看到2个svc都指向了同一个elb:
```yaml
apiVersion: v1
kind: Service
metadata:
annotations:
game.kruise.io/network-config-hash: "3594992400"
kubernetes.io/elb.class: performance
kubernetes.io/elb.connection-drain-enable: "true"
kubernetes.io/elb.connection-drain-timeout: "300"
kubernetes.io/elb.id: 8f4cf216-a659-40dc-8c77-xxxx
kubernetes.io/elb.mark: "0"
creationTimestamp: "2025-07-23T08:15:09Z"
finalizers:
- service.kubernetes.io/load-balancer-cleanup
name: hw-cce-elb-nginx-0
namespace: kruise-game-system
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Pod
name: hw-cce-elb-nginx-0
uid: 4f9f37f9-16d4-4ee7-b553-9b6e0039c5d5
resourceVersion: "13369506"
uid: 23815818-a626-4be3-b31f-4b95a4f89786
spec:
allocateLoadBalancerNodePorts: true
clusterIP: 10.247.213.xxx
clusterIPs:
- 10.247.213.xxx
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
loadBalancerIP: 192.168.0.xxx
ports:
- name: 80-tcp
nodePort: 30622
port: 3308
protocol: TCP
targetPort: 80
- name: 80-udp
nodePort: 30622
port: 3308
protocol: UDP
targetPort: 80
selector:
statefulset.kubernetes.io/pod-name: hw-cce-elb-nginx-0
sessionAffinity: None
type: LoadBalancer
status:
loadBalancer:
ingress:
- ip: 192.168.0.xxx
- ip: 189.1.225.xxx
---
apiVersion: v1
kind: Service
metadata:
annotations:
game.kruise.io/network-config-hash: "3594992400"
kubernetes.io/elb.class: performance
kubernetes.io/elb.connection-drain-enable: "true"
kubernetes.io/elb.connection-drain-timeout: "300"
kubernetes.io/elb.id: 8f4cf216-a659-40dc-8c77-xxxx
kubernetes.io/elb.mark: "0"
creationTimestamp: "2025-07-23T08:15:08Z"
finalizers:
- service.kubernetes.io/load-balancer-cleanup
name: hw-cce-elb-nginx-1
namespace: kruise-game-system
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Pod
name: hw-cce-elb-nginx-1
uid: 0f42b430-49ba-4203-8b50-4be059619b79
resourceVersion: "13369489"
uid: 92a56054-ad92-4dbd-9d1b-e717e0a14af2
spec:
allocateLoadBalancerNodePorts: true
clusterIP: 10.247.14.xxx
clusterIPs:
- 10.247.14.xxx
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
loadBalancerIP: 192.168.0.xxx
ports:
- name: 80-tcp
nodePort: 32227
port: 3611
protocol: TCP
targetPort: 80
- name: 80-udp
nodePort: 32227
port: 3611
protocol: UDP
targetPort: 80
selector:
statefulset.kubernetes.io/pod-name: hw-cce-elb-nginx-1
sessionAffinity: None
type: LoadBalancer
status:
loadBalancer:
ingress:
- ip: 192.168.0.xxx
- ip: 189.1.225.xxx
```
生成的svc如下,可以看到2个svc指向了同一个ip, 只是端口不同:
```bash
kubectl get svc |grep hw-cce-elb-nginx
hw-cce-elb-nginx-0 LoadBalancer 10.247.213.xxx 189.1.225.xxx,192.168.0.xxx 3308:30622/TCP,3308:30622/UDP 2m3s
hw-cce-elb-nginx-1 LoadBalancer 10.247.14.xxx 189.1.225.xxx,192.168.0.xxx 3611:32227/TCP,3611:32227/UDP 2m4s
```
---
自动创建ELB并绑定到创建出来的svc上面
**注意**:
- 自动创建elb,如果为多副本,每个svc都使用自动创建的elb,每个elb的id是不一样的,暴露的external ip也是不一样的.
- 当删除svc时, 关联的自动创建的elb也会被删除.
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: hw-cce-elb-auto-performance
namespace: kruise-game-system
spec:
replicas: 2
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: HwCloud-CCE-ELB
networkConf:
- name: PortProtocols
value: "80/TCP"
- name: kubernetes.io/elb.class
value: performance # ELB实例的类型
- name: kubernetes.io/elb.autocreate # 自动创建ELB的选项: https://support.huaweicloud.com/usermanual-cce/cce_10_0385.html#section21
value: '{
"type": "public",
"bandwidth_name": "bandwidth-xxxx",
"bandwidth_chargemode": "traffic",
"bandwidth_size": 5,
"bandwidth_sharetype": "PER",
"eip_type": "5_bgp",
"available_zone": [
"ap-southeast-1a",
"ap-southeast-1b"
],
"l4_flavor_name": "L4_flavor.elb.s1.small"
}'
- name: kubernetes.io/elb.enterpriseID # 创建出来的负载均衡所属企业项目ID
value: 'aff97261-4dbd-4593-8236-xxxx'
- name: kubernetes.io/elb.lb-algorithm
value: ROUND_ROBIN # 负载均衡器算法
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
生成的svc如下所示,可以看到2个svc都指向不同的elb:
```yaml
apiVersion: v1
kind: Service
metadata:
annotations:
game.kruise.io/network-config-hash: "3090934611"
kubernetes.io/elb.autocreate: '{ "type": "public", "bandwidth_name": "bandwidth-89f0",
"bandwidth_chargemode": "traffic", "bandwidth_size": 5, "bandwidth_sharetype":
"PER", "eip_type": "5_bgp", "available_zone": [ "ap-southeast-1a", "ap-southeast-1b"
], "l4_flavor_name": "L4_flavor.elb.s1.small" }'
kubernetes.io/elb.class: performance
kubernetes.io/elb.eip-id: 566d5f4c-3484-4d7e-aa6b-xxxx
kubernetes.io/elb.enterpriseID: aff97261-4dbd-4593-8236-xxxx
kubernetes.io/elb.id: 75e06e8b-a246-48cb-b05c-xxxx
kubernetes.io/elb.lb-algorithm: ROUND_ROBIN
kubernetes.io/elb.mark: "0"
creationTimestamp: "2025-07-23T09:25:01Z"
finalizers:
- service.kubernetes.io/load-balancer-cleanup
name: hw-cce-elb-auto-performance-0
namespace: kruise-game-system
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Pod
name: hw-cce-elb-auto-performance-0
uid: 1da0edf4-f45d-4635-8db0-ed5ccea2441d
resourceVersion: "13401553"
uid: 13efd440-65a7-4b45-bafc-2268102a4fd7
spec:
allocateLoadBalancerNodePorts: true
clusterIP: 10.247.50.xxx
clusterIPs:
- 10.247.50.xxx
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
loadBalancerIP: 49.0.251.xxx
ports:
- name: 80-tcp
nodePort: 30918
port: 1
protocol: TCP
targetPort: 80
selector:
statefulset.kubernetes.io/pod-name: hw-cce-elb-auto-performance-0
sessionAffinity: None
type: LoadBalancer
status:
loadBalancer:
ingress:
- ip: 49.0.251.xxx
- ip: 192.168.1.xxx
---
apiVersion: v1
kind: Service
metadata:
annotations:
game.kruise.io/network-config-hash: "3090934611"
kubernetes.io/elb.autocreate: '{ "type": "public", "bandwidth_name": "bandwidth-89f0",
"bandwidth_chargemode": "traffic", "bandwidth_size": 5, "bandwidth_sharetype":
"PER", "eip_type": "5_bgp", "available_zone": [ "ap-southeast-1a", "ap-southeast-1b"
], "l4_flavor_name": "L4_flavor.elb.s1.small" }'
kubernetes.io/elb.class: performance
kubernetes.io/elb.eip-id: 4a5396b1-e750-4ba5-a5d3-xxxx
kubernetes.io/elb.enterpriseID: aff97261-4dbd-4593-8236-xxxx
kubernetes.io/elb.id: b093db79-3c3e-4e77-a2ee-xxxx
kubernetes.io/elb.lb-algorithm: ROUND_ROBIN
kubernetes.io/elb.mark: "0"
creationTimestamp: "2025-07-23T09:25:01Z"
finalizers:
- service.kubernetes.io/load-balancer-cleanup
name: hw-cce-elb-auto-performance-1
namespace: kruise-game-system
ownerReferences:
- apiVersion: v1
blockOwnerDeletion: true
controller: true
kind: Pod
name: hw-cce-elb-auto-performance-1
uid: abfc9ad1-1ae3-45fa-b956-4617c465a44f
resourceVersion: "13401664"
uid: 01dd8e13-b1c8-4d9f-8b1c-13c2f001c614
spec:
allocateLoadBalancerNodePorts: true
clusterIP: 10.247.196.xxx
clusterIPs:
- 10.247.196.xxx
externalTrafficPolicy: Cluster
internalTrafficPolicy: Cluster
ipFamilies:
- IPv4
ipFamilyPolicy: SingleStack
loadBalancerIP: 150.40.245.xxx
ports:
- name: 80-tcp
nodePort: 30942
port: 1
protocol: TCP
targetPort: 80
selector:
statefulset.kubernetes.io/pod-name: hw-cce-elb-auto-performance-1
sessionAffinity: None
type: LoadBalancer
status:
loadBalancer:
ingress:
- ip: 150.40.245.xxx
- ip: 192.168.1.xxx
```
生成的svc如下,可以看到2个svc指向不同的external ip:
```bash
kubectl get svc |grep hw-cce-elb-auto-performance
hw-cce-elb-auto-performance-0 LoadBalancer 10.247.50.xxx 192.168.1.xxx,49.0.251.xxx 1:30918/TCP 4m29s
hw-cce-elb-auto-performance-1 LoadBalancer 10.247.196.xxx 150.40.245.xxx,192.168.1.xxx 1:30942/TCP 4m29s
```
### HwCloud-CCE-EIP
#### 插件名称
`HwCloud-CCE-EIP`
**注意**: 此插件仅仅适用于华为云的CCE Turbo集群
#### Cloud Provider
HuaweiCloud
#### 插件说明
- 仅适用于华为云CCE Turbo集群: https://support.huaweicloud.com/usermanual-cce/cce_10_0284.html#section1
- 为每个pod单独分配EIP
- 暴露的公网访问端口与容器中监听的端口一致，可以绑定安全组进行管理([使用注解为Pod绑定安全组](https://support.huaweicloud.com/usermanual-cce/cce_10_0897.html))
- Pod的网卡使用annotation配置的安全组: `yangtse.io/security-group-ids`
- Pod的网卡在使用已有安全组的基础上，额外再增加annotation配置的安全组: `yangtse.io/additional-security-group-ids`
- 支持自动创建EIP并绑定到对应的pod上, 或者使用已有的EIP
- 自动创建EIP时不支持指定`企业项目`
#### 网络参数
参考华为云的文档: https://support.huaweicloud.com/usermanual-cce/cce_10_0734.html, 本插件支持这个页面所有的annotation.
#### 插件配置
无
#### 示例说明
独占带宽EIP跟随Pod创建, 其他可用的annotation请参考华为云文档
注意: 这里创建出来的EIP属于`default`这个企业项目, 华为云CCE Turbo集群暂时不支持在这种模式下指定`企业项目`
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: hw-cce-eip-performance
namespace: default
spec:
replicas: 2
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: HwCloud-CCE-EIP
networkConf:
# https://support.huaweicloud.com/usermanual-cce/cce_10_0734.html
- name: yangtse.io/pod-with-eip
value: "true"
- name: yangtse.io/eip-bandwidth-size
value: "5"
- name: yangtse.io/eip-network-type
value: "5_bgp"
- name: yangtse.io/eip-charge-mode
value: "traffic"
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
生成的pod的annotation如下,`yangtse.io/allocated-eip-id`对应的eip可以在华为云的弹性公网IP详情中查看, `yangtse.io/allocated-ipv4-eip`
即为pod的eip
```yaml
apiVersion: v1
kind: Pod
metadata:
annotations:
apps.kruise.io/runtime-containers-meta: '{"containers":[{"name":"nginx","containerID":"containerd://302f710dc7fb5771be5b16a31de84ff457fd84c9aa1ce00b7e7f2ddc3b7c3978","restartCount":0,"hashes":{"plainHash":2641665875,"plainHashWithoutResources":0,"extractedEnvFromMetadataHash":86995377}}]}'
game.kruise.io/network-conf: '[{"name":"yangtse.io/pod-with-eip","value":"true"},{"name":"yangtse.io/eip-bandwidth-size","value":"5"},{"name":"yangtse.io/eip-network-type","value":"5_bgp"},{"name":"yangtse.io/eip-charge-mode","value":"traffic"}]'
game.kruise.io/network-status: '{"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}'
game.kruise.io/network-trigger-time: "2025-07-16 17:03:07"
game.kruise.io/network-type: HwCloud-EIP
game.kruise.io/opsState-last-changed-time: "2025-07-16 17:03:07"
game.kruise.io/state-last-changed-time: "2025-07-16 09:03:13"
lifecycle.apps.kruise.io/timestamp: "2025-07-16T09:03:03Z"
yangtse.io/allocated-eip-id: 3a52ca79-d78d-4fc2-8590-xxxx
yangtse.io/allocated-ipv4-eip: 94.74.110.xxx
yangtse.io/eip-bandwidth-size: "5"
yangtse.io/eip-charge-mode: traffic
yangtse.io/eip-network-type: 5_bgp
yangtse.io/pod-with-eip: "true"
```
使用已有的EIP, spec.network.networkConf添加`yangtse.io/eip-id`,你需要事先在华为云创建好EIP
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: hw-cce-eip-exist
namespace: kruise-game-system
spec:
replicas: 1
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: HwCloud-CCE-EIP
networkConf:
- name: yangtse.io/eip-id
value: "7ec474aa-3bd9-46a2-a45c-xxxx" # 使用已有的EIP
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
pod的yaml, 可以看到pod的annotations里`yangtse.io/allocated-eip-id`为我们指定的EIP,登录华为云EIP控制台,可以看到这个EIP已经绑定了pod
```yaml
apiVersion: v1
kind: Pod
metadata:
annotations:
apps.kruise.io/runtime-containers-meta: '{"containers":[{"name":"nginx","containerID":"containerd://0fc9de69e30b48cf13ad2d2c6f5fe3be86e48e922a982dbb77b53ffd0ca6f54b","restartCount":0,"hashes":{"plainHash":2957831032,"plainHashWithoutResources":0,"extractedEnvFromMetadataHash":86995377}}]}'
game.kruise.io/network-conf: '[{"name":"yangtse.io/eip-id","value":"7ec474aa-3bd9-46a2-a45c-xxxx"}]'
game.kruise.io/network-status: '{"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}'
game.kruise.io/network-trigger-time: "2025-07-18 15:38:21"
game.kruise.io/network-type: HwCloud-EIP
game.kruise.io/opsState-last-changed-time: "2025-07-18 15:38:21"
game.kruise.io/state-last-changed-time: "2025-07-18 15:38:31"
lifecycle.apps.kruise.io/timestamp: "2025-07-18T07:38:13Z"
yangtse.io/allocated-eip-id: 7ec474aa-3bd9-46a2-a45c-xxxx
yangtse.io/allocated-ipv4-eip: 159.138.21.xxx
yangtse.io/eip-id: 7ec474aa-3bd9-46a2-a45c-xxxx
creationTimestamp: "2025-07-18T07:38:14Z
# other info ignored
```
## 获取网络信息
GameServer Network Status可以通过两种方式获取

3
go.mod
View File

@ -13,7 +13,7 @@ require (
github.com/onsi/gomega v1.32.0
github.com/openkruise/kruise-api v1.8.0
github.com/prometheus/client_golang v1.18.0
github.com/stretchr/testify v1.8.4
github.com/stretchr/testify v1.10.0
golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e
google.golang.org/grpc v1.58.3
google.golang.org/protobuf v1.33.0
@ -65,6 +65,7 @@ require (
github.com/prometheus/procfs v0.12.0 // indirect
github.com/rogpeppe/go-internal v1.10.0 // indirect
github.com/spf13/pflag v1.0.5 // indirect
github.com/stretchr/objx v0.5.2 // indirect
go.uber.org/multierr v1.11.0 // indirect
go.uber.org/zap v1.26.0 // indirect
golang.org/x/mod v0.17.0 // indirect

6
go.sum
View File

@ -133,13 +133,15 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw=
github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo=
github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY=
github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA=
github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU=
github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk=
github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=