diff --git a/cloudprovider/hwcloud/eip_cce.go b/cloudprovider/hwcloud/eip_cce.go new file mode 100644 index 0000000..16d728c --- /dev/null +++ b/cloudprovider/hwcloud/eip_cce.go @@ -0,0 +1,116 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hwcloud + +import ( + "context" + + corev1 "k8s.io/api/core/v1" + log "k8s.io/klog/v2" + "sigs.k8s.io/controller-runtime/pkg/client" + + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/cloudprovider" + "github.com/openkruise/kruise-game/cloudprovider/errors" + "github.com/openkruise/kruise-game/cloudprovider/utils" +) + +const ( + EIPNetwork = "HwCloud-CCE-EIP" + AliasSEIP = "CCE-EIP-Network" +) + +var allowedAnnotations = []string{ + "yangtse.io/pod-with-eip", + "yangtse.io/eip-bandwidth-size", + "yangtse.io/eip-network-type", + "yangtse.io/eip-charge-mode", + "yangtse.io/eip-bandwidth-name", + "yangtse.io/eip-network-type", + "yangtse.io/eip-bandwidth-id", + "yangtse.io/eip-id", + "yangtse.io/security-group-ids", + "yangtse.io/additional-security-group-ids", +} + +func init() { + eipPlugin := EipPlugin{} + hwCloudProvider.registerPlugin(&eipPlugin) +} + +type EipPlugin struct{} + +func (E EipPlugin) Name() string { + return EIPNetwork +} + +func (E EipPlugin) Alias() string { + return AliasSEIP +} + +func (E EipPlugin) Init(client client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error { + return nil +} + +func (E EipPlugin) OnPodAdded(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, errors.PluginError) { + networkManager := utils.NewNetworkManager(pod, client) + conf := networkManager.GetNetworkConfig() + log.Infof("pod %s/%s network config: %#v", pod.Namespace, pod.Name, conf) + + if networkManager.GetNetworkType() != EIPNetwork { + log.Infof("pod %s/%s network type is not %s, skipping", pod.Namespace, pod.Name, EIPNetwork) + return pod, nil + } + allowedAnnotationsMap := make(map[string]struct{}) + for _, item := range allowedAnnotations { + allowedAnnotationsMap[item] = struct{}{} + } + for _, c := range conf { + _, ok := allowedAnnotationsMap[c.Name] + if ok { + pod.Annotations[c.Name] = c.Value + } else { + log.Warningf("pod %s/%s network config %s is not allowed", pod.Namespace, pod.Name, c.Name) + } + } + + return pod, nil +} + +func (E EipPlugin) OnPodUpdated(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, errors.PluginError) { + networkManager := utils.NewNetworkManager(pod, client) + if networkManager.GetNetworkType() != EIPNetwork { + log.Infof("pod %s/%s network type is not %s, skipping", pod.Namespace, pod.Name, EIPNetwork) + return pod, nil + } + networkStatus, _ := networkManager.GetNetworkStatus() + if networkStatus == nil { + pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{ + CurrentNetworkState: gamekruiseiov1alpha1.NetworkWaiting, + }, pod) + return pod, errors.ToPluginError(err, errors.InternalError) + } + + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady + + pod, err := networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, errors.ToPluginError(err, errors.InternalError) +} + +func (E EipPlugin) OnPodDeleted(client client.Client, pod *corev1.Pod, ctx context.Context) errors.PluginError { + return nil +} diff --git a/cloudprovider/hwcloud/eip_cce_test.go b/cloudprovider/hwcloud/eip_cce_test.go new file mode 100644 index 0000000..c6573ab --- /dev/null +++ b/cloudprovider/hwcloud/eip_cce_test.go @@ -0,0 +1,178 @@ +package hwcloud + +import ( + "context" + "encoding/json" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + "sigs.k8s.io/controller-runtime/pkg/client/fake" + + "github.com/openkruise/kruise-game/apis/v1alpha1" + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/cloudprovider/alibabacloud/apis/v1beta1" +) + +func TestEipPlugin_Init(t *testing.T) { + plugin := EipPlugin{} + assert.Equal(t, EIPNetwork, plugin.Name()) + assert.Equal(t, AliasSEIP, plugin.Alias()) + err := plugin.Init(nil, nil, context.Background()) + assert.NoError(t, err) +} + +func TestEipPlugin_OnPodAdded_UseExistingEIP(t *testing.T) { + // create test pod + var networkConf []v1alpha1.NetworkConfParams + networkConf = append(networkConf, v1alpha1.NetworkConfParams{ + Name: "yangtse.io/eip-id", + Value: "huawei-eip-12345", + }) + jsonStr, _ := json.Marshal(networkConf) + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + v1alpha1.GameServerNetworkType: EIPNetwork, + v1alpha1.GameServerNetworkConf: string(jsonStr), + }, + }, + } + + // create fake client. + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + + // execute the test + plugin := EipPlugin{} + updatedPod, err := plugin.OnPodAdded(fakeClient, pod, context.Background()) + + // check the result. + assert.NoError(t, err) + assert.Equal(t, EIPNetwork, updatedPod.Annotations[v1alpha1.GameServerNetworkType]) + unmarshalErr := json.Unmarshal([]byte(updatedPod.Annotations[v1alpha1.GameServerNetworkConf]), &networkConf) + assert.NoError(t, unmarshalErr) + assert.Equal(t, "huawei-eip-12345", networkConf[0].Value) +} + +func addKvToParams(networkConf []v1alpha1.NetworkConfParams, keys []string, values []string) []v1alpha1.NetworkConfParams { + for i := 0; i < len(keys); i++ { + networkConf = append(networkConf, v1alpha1.NetworkConfParams{ + Name: keys[i], + Value: values[i], + }) + } + return networkConf +} +func TestEipPlugin_OnPodAdded_NewEIP(t *testing.T) { + var networkConf []v1alpha1.NetworkConfParams + networkConf = addKvToParams(networkConf, + []string{ + "name", + "yangtse.io/pod-with-eip", + "yangtse.io/eip-bandwidth-size", + "yangtse.io/eip-network-type", + "yangtse.io/eip-charge-mode", + }, + []string{ + "huawei-eip-demo", + "true", + "5", + "5-bgp", + "traffic", + }, + ) + jsonStr, _ := json.Marshal(networkConf) + // create test Pod and add related annotations. + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + v1alpha1.GameServerNetworkType: EIPNetwork, + v1alpha1.GameServerNetworkConf: string(jsonStr), + }, + }, + } + + // create fake client. + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + fakeClient := fake.NewClientBuilder().WithScheme(scheme).Build() + + // execute the test. + plugin := EipPlugin{} + updatedPod, err := plugin.OnPodAdded(fakeClient, pod, context.Background()) + + // check the result. + assert.NoError(t, err) + assert.Equal(t, EIPNetwork, updatedPod.Annotations[v1alpha1.GameServerNetworkType]) + assert.Equal(t, "true", updatedPod.Annotations["yangtse.io/pod-with-eip"]) + assert.Equal(t, "5", updatedPod.Annotations["yangtse.io/eip-bandwidth-size"]) + assert.Equal(t, "5-bgp", updatedPod.Annotations["yangtse.io/eip-network-type"]) + assert.Equal(t, "traffic", updatedPod.Annotations["yangtse.io/eip-charge-mode"]) +} + +func TestEipPlugin_OnPodUpdated_WithNetworkStatus(t *testing.T) { + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + v1alpha1.GameServerNetworkType: EIPNetwork, + "cloud.kruise.io/network-status": `{"currentNetworkState":"Waiting"}`, + }, + }, + Status: corev1.PodStatus{}, + } + + scheme := runtime.NewScheme() + _ = corev1.AddToScheme(scheme) + _ = v1beta1.AddToScheme(scheme) + _ = gamekruiseiov1alpha1.AddToScheme(scheme) + fakeClient := fake.NewClientBuilder(). + WithScheme(scheme). + WithObjects(pod). + Build() + + plugin := EipPlugin{} + networkStatus := &v1alpha1.NetworkStatus{} + networkStatus.ExternalAddresses = []v1alpha1.NetworkAddress{{IP: "203.0.113.1"}} + networkStatus.InternalAddresses = []v1alpha1.NetworkAddress{{IP: "10.0.0.1"}} + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady + networkStatusBytes, jErr := json.Marshal(networkStatus) + assert.NoError(t, jErr) + pod.Annotations[v1alpha1.GameServerNetworkStatus] = string(networkStatusBytes) + updatedPod, err := plugin.OnPodUpdated(fakeClient, pod, context.Background()) + assert.NoError(t, err) + + jErr = json.Unmarshal([]byte(updatedPod.Annotations[v1alpha1.GameServerNetworkStatus]), &networkStatus) + assert.NoError(t, jErr) + + assert.Contains(t, updatedPod.Annotations[v1alpha1.GameServerNetworkStatus], "Ready") + assert.Contains(t, updatedPod.Annotations[v1alpha1.GameServerNetworkStatus], "203.0.113.1") + assert.Contains(t, updatedPod.Annotations[v1alpha1.GameServerNetworkStatus], "10.0.0.1") +} + +func TestEipPlugin_OnPodDeleted(t *testing.T) { + plugin := EipPlugin{} + pod := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + v1alpha1.GameServerNetworkType: EIPNetwork, + "cloud.kruise.io/network-status": `{"currentNetworkState":"Waiting"}`, + }, + }, + Status: corev1.PodStatus{}, + } + + err := plugin.OnPodDeleted(nil, pod, context.Background()) + assert.Nil(t, err) +} diff --git a/cloudprovider/hwcloud/elb_cce.go b/cloudprovider/hwcloud/elb_cce.go new file mode 100644 index 0000000..d2688ec --- /dev/null +++ b/cloudprovider/hwcloud/elb_cce.go @@ -0,0 +1,678 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hwcloud + +import ( + "context" + "fmt" + "strconv" + "strings" + "sync" + "time" + + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + log "k8s.io/klog/v2" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/cloudprovider" + cperrors "github.com/openkruise/kruise-game/cloudprovider/errors" + provideroptions "github.com/openkruise/kruise-game/cloudprovider/options" + "github.com/openkruise/kruise-game/cloudprovider/utils" + "github.com/openkruise/kruise-game/pkg/util" +) + +const ( + ElbAutocreateAnnotationKey = "kubernetes.io/elb.autocreate" + + CCEElbNetwork = "HwCloud-CCE-ELB" + AliasCCEELB = "CCE-ELB-Network" +) + +func init() { + elbPlugin := CCEElbPlugin{ + mutex: sync.RWMutex{}, + } + hwCloudProvider.registerPlugin(&elbPlugin) +} + +type cceElbConfig struct { + elbIds []string + targetPorts []int + protocols []corev1.Protocol + isFixed bool + externalTrafficPolicyType corev1.ServiceExternalTrafficPolicyType + hwOptions map[string]string +} + +func (e cceElbConfig) isAutoCreateElb() bool { + // auto create elb mode annotation + jsonValue, ok := e.hwOptions[ElbAutocreateAnnotationKey] + return ok && jsonValue != "" && len(e.elbIds) == 0 +} + +type ccePortAllocated map[int32]bool + +type CCEElbPlugin struct { + maxPort int32 + minPort int32 + blockPorts []int32 + cache map[string]ccePortAllocated + podAllocate map[string]string + mutex sync.RWMutex +} + +func (s *CCEElbPlugin) Name() string { + return CCEElbNetwork +} + +func (s *CCEElbPlugin) Alias() string { + return AliasCCEELB +} + +func (s *CCEElbPlugin) Init(c client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error { + s.mutex.Lock() + defer s.mutex.Unlock() + elbOptions := options.(provideroptions.HwCloudOptions).CCEELBOptions + s.minPort = elbOptions.ELBOptions.MinPort + s.maxPort = elbOptions.ELBOptions.MaxPort + s.blockPorts = elbOptions.ELBOptions.BlockPorts + + // get all service + svcList := &corev1.ServiceList{} + err := c.List(ctx, svcList) + if err != nil { + return err + } + + s.cache, s.podAllocate = initCCELbCache(svcList.Items, s.minPort, s.maxPort, s.blockPorts) + log.Infof("[%s] podAllocate cache complete initialization: %v", CCEElbNetwork, s.podAllocate) + return nil +} + +// fillCache: you need to add lock before calling this function +func (s *CCEElbPlugin) fillCache(lbId string, usedPorts []int32) { + if s.cache == nil { + s.cache = make(map[string]ccePortAllocated) + } + if s.cache[lbId] != nil { + return + } + alloc := make(ccePortAllocated, s.maxPort-s.minPort+1) + for port := s.minPort; port <= s.maxPort; port++ { + alloc[port] = false + } + for _, port := range s.blockPorts { + if port >= s.minPort && port <= s.maxPort { + alloc[port] = true + } + } + s.cache[lbId] = alloc + for _, port := range usedPorts { + s.cache[lbId][port] = true + } +} + +func (s *CCEElbPlugin) updateCachesAfterAutoCreateElb(c client.Client, name, namespace string, + interval, totalTimeout time.Duration) { + if interval > totalTimeout { + panic("interval must be lesser than timeout") + } + log.Infof("Starting periodic cache update for %s/%s (interval: %s, timeout: %s)", + namespace, name, interval, totalTimeout) + timeoutCtx, cancel := context.WithTimeout(context.Background(), totalTimeout) + defer cancel() + + ticker := time.NewTicker(interval) + defer ticker.Stop() + + var ( + attempt int + lastError error + ) + + for { + select { + case <-timeoutCtx.Done(): + log.Warningf("Cache update failed for %s/%s after %d attempts. Last error: %v", + namespace, name, attempt, lastError) + return + + case <-ticker.C: + attempt++ + log.Infof("Attempt #%d: updating cache for %s/%s", attempt, namespace, name) + + svc := &corev1.Service{} + err := c.Get(timeoutCtx, types.NamespacedName{ + Name: name, + Namespace: namespace, + }, svc) + + if err != nil { + log.Errorf("failed to get Service: %s", err) + continue + } + + elbId := svc.Annotations[ElbIdAnnotationKey] + usedPorts := getCCEPorts(svc.Spec.Ports) + + if elbId == "" || len(usedPorts) == 0 { + continue + } + + s.mutex.Lock() + s.fillCache(elbId, usedPorts) + if s.podAllocate == nil { + s.podAllocate = make(map[string]string) + } + s.podAllocate[newPodAllocateKey(name, namespace)] = newPodAllocateValue(elbId, usedPorts) + s.mutex.Unlock() + + log.Infof("Attempt #%d success: updated cache for %s/%s with ELB %s and %d ports", + attempt, namespace, name, elbId, len(usedPorts)) + return + } + } +} + +func (s *CCEElbPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + return pod, nil +} + +func (s *CCEElbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + log.Infof("on update pod begin") + networkManager := utils.NewNetworkManager(pod, c) + if networkManager.GetNetworkType() != CCEElbNetwork { + log.Infof("pod %s/%s network type is not %s, skipping", pod.Namespace, pod.Name, CCEElbNetwork) + return pod, nil + } + networkStatus, _ := networkManager.GetNetworkStatus() + if networkStatus == nil { + log.Warningf("network status is nil") + pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{ + CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady, + }, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + networkConfig := networkManager.GetNetworkConfig() + sc, err := parseCCELbConfig(networkConfig) + if err != nil { + log.Errorf("parse elb config failed: %s, network configuration: %#v", err, networkConfig) + return pod, cperrors.ToPluginError(err, cperrors.ParameterError) + } + log.Infof("creating svc %s/%s", pod.GetNamespace(), pod.GetName()) + // get svc + svc := &corev1.Service{} + err = c.Get(ctx, types.NamespacedName{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + }, svc) + if err != nil { + if k8serrors.IsNotFound(err) { + log.Infof("svc %s/%s not found, will create it", pod.GetNamespace(), pod.GetName()) + service, err := s.consSvc(sc, pod, c, ctx) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.ParameterError) + } + if err = c.Create(ctx, service); err != nil { + log.Errorf("create svc %s/%s failed: %s", pod.GetNamespace(), pod.GetName(), err) + return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) + } + log.Infof("create svc %s/%s success", pod.GetNamespace(), pod.GetName()) + if sc.isAutoCreateElb() { + go s.updateCachesAfterAutoCreateElb(c, pod.Name, pod.Namespace, 5*time.Second, 10*time.Minute) + } + return pod, cperrors.ToPluginError(nil, cperrors.ApiCallError) + } + log.Errorf("get svc %s/%s failed: %s", pod.GetNamespace(), pod.GetName(), err) + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + + // old svc remain + if svc.OwnerReferences[0].Kind == "Pod" && svc.OwnerReferences[0].UID != pod.UID { + log.Infof("[%s] waitting old svc %s/%s deleted. old owner pod uid is %s, but now is %s", CCEElbNetwork, svc.Namespace, svc.Name, svc.OwnerReferences[0].UID, pod.UID) + return pod, nil + } + + // update svc + if util.GetHash(sc) != svc.GetAnnotations()[ElbConfigHashKey] { + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + service, err := s.consSvc(sc, pod, c, ctx) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.ParameterError, err.Error()) + } + return pod, cperrors.ToPluginError(c.Update(ctx, service), cperrors.ApiCallError) + } + + // disable network + if networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeLoadBalancer { + svc.Spec.Type = corev1.ServiceTypeClusterIP + return pod, cperrors.ToPluginError(c.Update(ctx, svc), cperrors.ApiCallError) + } + + // enable network + if !networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeClusterIP { + svc.Spec.Type = corev1.ServiceTypeLoadBalancer + return pod, cperrors.ToPluginError(c.Update(ctx, svc), cperrors.ApiCallError) + } + + // network not ready + if svc.Status.LoadBalancer.Ingress == nil { + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + + // allow not ready containers + if util.IsAllowNotReadyContainers(networkManager.GetNetworkConfig()) { + toUpDateSvc, err := utils.AllowNotReadyContainers(c, ctx, pod, svc, false) + if err != nil { + return pod, err + } + + if toUpDateSvc { + err := c.Update(ctx, svc) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) + } + } + } + + // network ready + internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) + externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) + for _, port := range svc.Spec.Ports { + instrIPort := port.TargetPort + instrEPort := intstr.FromInt(int(port.Port)) + internalAddress := gamekruiseiov1alpha1.NetworkAddress{ + IP: pod.Status.PodIP, + Ports: []gamekruiseiov1alpha1.NetworkPort{ + { + Name: instrIPort.String(), + Port: &instrIPort, + Protocol: port.Protocol, + }, + }, + } + externalAddress := gamekruiseiov1alpha1.NetworkAddress{ + IP: svc.Status.LoadBalancer.Ingress[0].IP, + Ports: []gamekruiseiov1alpha1.NetworkPort{ + { + Name: instrIPort.String(), + Port: &instrEPort, + Protocol: port.Protocol, + }, + }, + } + internalAddresses = append(internalAddresses, internalAddress) + externalAddresses = append(externalAddresses, externalAddress) + } + networkStatus.InternalAddresses = internalAddresses + networkStatus.ExternalAddresses = externalAddresses + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) +} + +func (s *CCEElbPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError { + networkManager := utils.NewNetworkManager(pod, c) + networkConfig := networkManager.GetNetworkConfig() + if networkManager.GetNetworkType() != CCEElbNetwork { + log.Infof("pod %s/%s network type is not %s, skipping", pod.Namespace, pod.Name, CCEElbNetwork) + return nil + } + sc, err := parseCCELbConfig(networkConfig) + if err != nil { + return cperrors.NewPluginError(cperrors.ParameterError, err.Error()) + } + + var podKeys []string + if sc.isFixed { + gss, err := util.GetGameServerSetOfPod(pod, c, ctx) + if err != nil && !k8serrors.IsNotFound(err) { + return cperrors.ToPluginError(err, cperrors.ApiCallError) + } + // gss exists in cluster, do not deAllocate. + if err == nil && gss.GetDeletionTimestamp() == nil { + return nil + } + // gss not exists in cluster, deAllocate all the ports related to it. + for key := range s.podAllocate { + gssName := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey] + if strings.Contains(key, pod.GetNamespace()+"/"+gssName) { + podKeys = append(podKeys, key) + } + } + } else { + podKeys = append(podKeys, pod.GetNamespace()+"/"+pod.GetName()) + } + + for _, podKey := range podKeys { + s.deAllocate(podKey) + } + + return nil +} + +func (s *CCEElbPlugin) allocate(lbIds []string, num int, podKey string) (string, []int32) { + s.mutex.Lock() + defer s.mutex.Unlock() + + var ports []int32 + var lbId string + + // find lb with adequate ports + for _, elbId := range lbIds { + sum := 0 + for i := s.minPort; i <= s.maxPort; i++ { + if !s.cache[elbId][i] { + sum++ + } + if sum >= num { + lbId = elbId + break + } + } + } + if lbId == "" { + return "", nil + } + + // select ports + for i := 0; i < num; i++ { + var port int32 + s.fillCache(lbId, nil) + for p, allocated := range s.cache[lbId] { + if !allocated { + port = p + break + } + } + s.cache[lbId][port] = true + ports = append(ports, port) + } + s.podAllocate[podKey] = newPodAllocateValue(lbId, ports) + log.Infof("pod %s allocate elb %s ports %v", podKey, lbId, ports) + return lbId, ports +} + +func (s *CCEElbPlugin) deAllocate(nsSvcKey string) { + s.mutex.Lock() + defer s.mutex.Unlock() + + allocatedPorts, exist := s.podAllocate[nsSvcKey] + if !exist { + return + } + + elbPorts := strings.Split(allocatedPorts, ":") + lbId := elbPorts[0] + ports := util.StringToInt32Slice(elbPorts[1], ",") + for _, port := range ports { + s.cache[lbId][port] = false + } + // block ports + for _, blockPort := range s.blockPorts { + s.cache[lbId][blockPort] = true + } + + delete(s.podAllocate, nsSvcKey) + log.Infof("pod %s deallocate elb %s ports %v", nsSvcKey, lbId, ports) +} + +func (s *CCEElbPlugin) consSvc(sc *cceElbConfig, pod *corev1.Pod, c client.Client, ctx context.Context) (*corev1.Service, error) { + var ports []int32 + var lbId string + podKey := pod.GetNamespace() + "/" + pod.GetName() + allocatedPorts, exist := s.podAllocate[podKey] + if exist { + elbPorts := strings.Split(allocatedPorts, ":") + ports = util.StringToInt32Slice(elbPorts[1], ",") + } else { + if sc.isAutoCreateElb() { + lbId, ports = "", s.getPortFromHead(len(sc.targetPorts)) + } else { + lbId, ports = s.allocate(sc.elbIds, len(sc.targetPorts), podKey) + } + if lbId == "" && ports == nil { + return nil, fmt.Errorf("there are no avaliable ports for %v", sc.elbIds) + } + } + + svcPorts := make([]corev1.ServicePort, 0) + for i := 0; i < len(sc.targetPorts); i++ { + if sc.protocols[i] == ProtocolTCPUDP { + svcPorts = append(svcPorts, corev1.ServicePort{ + Name: fmt.Sprintf("%s-%s", strconv.Itoa(sc.targetPorts[i]), strings.ToLower(string(corev1.ProtocolTCP))), + Port: ports[i], + Protocol: corev1.ProtocolTCP, + TargetPort: intstr.FromInt(sc.targetPorts[i]), + }) + + svcPorts = append(svcPorts, corev1.ServicePort{ + Name: fmt.Sprintf("%s-%s", strconv.Itoa(sc.targetPorts[i]), strings.ToLower(string(corev1.ProtocolUDP))), + Port: ports[i], + Protocol: corev1.ProtocolUDP, + TargetPort: intstr.FromInt(sc.targetPorts[i]), + }) + + } else { + svcPorts = append(svcPorts, corev1.ServicePort{ + Name: fmt.Sprintf("%s-%s", strconv.Itoa(sc.targetPorts[i]), strings.ToLower(string(sc.protocols[i]))), + Port: ports[i], + Protocol: sc.protocols[i], + TargetPort: intstr.FromInt(sc.targetPorts[i]), + }) + } + } + svcAnnotations := make(map[string]string, 0) + for k, v := range sc.hwOptions { + svcAnnotations[k] = v + } + // add hash to svc, otherwise, the status of GS will remain in NetworkNotReady. + svcAnnotations[ElbConfigHashKey] = util.GetHash(sc) + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + Annotations: svcAnnotations, + OwnerReferences: getCCESvcOwnerReference(c, ctx, pod, sc.isFixed), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: sc.externalTrafficPolicyType, + Selector: map[string]string{ + SvcSelectorKey: pod.GetName(), + }, + Ports: svcPorts, + }, + } + return svc, nil +} + +func (s *CCEElbPlugin) getPortFromHead(num int) []int32 { + res := make([]int32, 0) + blocked := make(map[int32]struct{}) + for _, port := range s.blockPorts { + blocked[port] = struct{}{} + } + count := 0 + for i := s.minPort; i <= s.maxPort && count < num; i++ { + if _, exist := blocked[i]; exist { + continue + } + count++ + res = append(res, i) + } + return res +} + +func getCCESvcOwnerReference(c client.Client, ctx context.Context, pod *corev1.Pod, isFixed bool) []metav1.OwnerReference { + ownerReferences := []metav1.OwnerReference{ + { + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.GetName(), + UID: pod.GetUID(), + Controller: ptr.To[bool](true), + BlockOwnerDeletion: ptr.To[bool](true), + }, + } + if isFixed { + gss, err := util.GetGameServerSetOfPod(pod, c, ctx) + if err == nil { + ownerReferences = []metav1.OwnerReference{ + { + APIVersion: gss.APIVersion, + Kind: gss.Kind, + Name: gss.GetName(), + UID: gss.GetUID(), + Controller: ptr.To[bool](true), + BlockOwnerDeletion: ptr.To[bool](true), + }, + } + } + } + return ownerReferences +} + +func newPodAllocateKey(name, namespace string) string { + return namespace + "/" + name +} + +func newPodAllocateValue(elbId string, ports []int32) string { + return elbId + ":" + util.Int32SliceToString(ports, ",") +} + +func getCCEPorts(ports []corev1.ServicePort) []int32 { + var ret []int32 + for _, port := range ports { + ret = append(ret, port.Port) + } + return ret +} + +func initCCELbCache(svcList []corev1.Service, minPort, maxPort int32, blockPorts []int32) (map[string]ccePortAllocated, map[string]string) { + newCache := make(map[string]ccePortAllocated) + newPodAllocate := make(map[string]string) + for _, svc := range svcList { + lbId := svc.Annotations[ElbIdAnnotationKey] + // Associate an existing ELB. + if lbId != "" && svc.Spec.Type == corev1.ServiceTypeLoadBalancer { + // init cache for that lb + if newCache[lbId] == nil { + newCache[lbId] = make(ccePortAllocated, maxPort-minPort+1) + for i := minPort; i <= maxPort; i++ { + newCache[lbId][i] = false + } + } + + // block ports + for _, blockPort := range blockPorts { + newCache[lbId][blockPort] = true + } + + // fill in cache for that lb + var ports []int32 + for _, port := range getCCEPorts(svc.Spec.Ports) { + if port <= maxPort && port >= minPort { + value, ok := newCache[lbId][port] + if !ok || !value { + newCache[lbId][port] = true + ports = append(ports, port) + } + } + } + if len(ports) != 0 { + newPodAllocate[newPodAllocateKey(svc.GetName(), svc.GetNamespace())] = newPodAllocateValue(lbId, ports) + log.Infof("svc %s/%s allocate elb %s ports %v", svc.Namespace, svc.Name, lbId, ports) + } + } + } + return newCache, newPodAllocate +} + +func parseCCELbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) (*cceElbConfig, error) { + res := &cceElbConfig{ + targetPorts: make([]int, 0), + protocols: make([]corev1.Protocol, 0), + isFixed: false, + externalTrafficPolicyType: corev1.ServiceExternalTrafficPolicyTypeCluster, + hwOptions: make(map[string]string), + } + specifyElbId := false + autoCreateElb := false + for _, c := range conf { + switch c.Name { + case ElbIdAnnotationKey: + if autoCreateElb { + return nil, fmt.Errorf("%s and %s cannot be filled in simultaneously", + ElbIdAnnotationKey, ElbAutocreateAnnotationKey) + } + specifyElbId = true + // huawei only supports one elb id + if c.Value == "" { + return nil, fmt.Errorf("no elb id found, must specify at least one elb id") + } + res.elbIds = []string{c.Value} + res.hwOptions[c.Name] = c.Value + case ElbAutocreateAnnotationKey: + if specifyElbId { + return nil, fmt.Errorf("%s and %s cannot be filled in simultaneously", + ElbIdAnnotationKey, ElbAutocreateAnnotationKey) + } + autoCreateElb = true + res.hwOptions[c.Name] = c.Value + case PortProtocolsConfigName: + for _, pp := range strings.Split(c.Value, ",") { + ppSlice := strings.Split(pp, "/") + port, err := strconv.Atoi(ppSlice[0]) + if err != nil { + continue + } + res.targetPorts = append(res.targetPorts, port) + if len(ppSlice) != 2 { + res.protocols = append(res.protocols, corev1.ProtocolTCP) + } else { + res.protocols = append(res.protocols, corev1.Protocol(ppSlice[1])) + } + } + case FixedConfigName: + v, err := strconv.ParseBool(c.Value) + if err != nil { + continue + } + res.isFixed = v + case ExternalTrafficPolicyTypeConfigName: + if strings.EqualFold(c.Value, string(corev1.ServiceExternalTrafficPolicyTypeLocal)) { + res.externalTrafficPolicyType = corev1.ServiceExternalTrafficPolicyTypeLocal + } + default: + res.hwOptions[c.Name] = c.Value + } + } + return res, nil +} diff --git a/cloudprovider/hwcloud/elb_cce_test.go b/cloudprovider/hwcloud/elb_cce_test.go new file mode 100644 index 0000000..402bb08 --- /dev/null +++ b/cloudprovider/hwcloud/elb_cce_test.go @@ -0,0 +1,936 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package hwcloud + +import ( + "context" + "fmt" + "reflect" + "sync" + "testing" + "time" + + "github.com/openkruise/kruise-game/cloudprovider" + "github.com/openkruise/kruise-game/cloudprovider/options" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + corev1 "k8s.io/api/core/v1" + k8serrors "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime/schema" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/ptr" + "sigs.k8s.io/controller-runtime/pkg/client" + + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/cloudprovider/errors" +) + +var ( + fakeSvcTemplate = corev1.Service{ + TypeMeta: metav1.TypeMeta{ + APIVersion: "v1", + Kind: "Service", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-0", + Namespace: "default", + Annotations: map[string]string{ + "game.kruise.io/network-config-hash": "2536262647", + "kubernetes.io/elb.class": "performance", + "kubernetes.io/elb.connection-drain-enable": "true", + "kubernetes.io/elb.connection-drain-timeout": "300", + "kubernetes.io/elb.id": "8f4cf216-a659-40dc-8c77-6068b036ba56", + "kubernetes.io/elb.mark": "0", + }, + CreationTimestamp: metav1.Time{}, + Finalizers: []string{"service.kubernetes.io/load-balancer-cleanup"}, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "Pod", + Name: "test-pod-0", + UID: "53cb0992-c720-4ae4-9af9-3cc7e2bf3660", + }, + }, + ResourceVersion: "9867633", + UID: "eccc0de3-ea09-4554-8710-4319eb551237", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: "80-tcp", + Protocol: corev1.ProtocolTCP, + Port: 500, + TargetPort: intstr.FromInt(80), + NodePort: 31749, + }, + }, + Selector: map[string]string{ + "statefulset.kubernetes.io/pod-name": "gs-elb-performance-2-0", + }, + ClusterIP: "10.247.83.164", + ClusterIPs: []string{"10.247.83.164"}, + Type: corev1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeCluster, + LoadBalancerIP: "192.168.0.147", + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, + SessionAffinity: corev1.ServiceAffinityNone, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{ + Ingress: []corev1.LoadBalancerIngress{ + {IP: "192.168.0.147"}, + {IP: "189.1.225.136"}, + }, + }, + }, + } +) + +func TestAllocateDeAllocate(t *testing.T) { + test := struct { + lbIds []string + elb *CCEElbPlugin + num int + podKey string + }{ + lbIds: []string{"cce-lb-xxxx"}, + elb: &CCEElbPlugin{ + maxPort: int32(712), + minPort: int32(512), + cache: make(map[string]ccePortAllocated), + podAllocate: make(map[string]string), + mutex: sync.RWMutex{}, + }, + podKey: "xxx/xxx", + num: 3, + } + + lbId, ports := test.elb.allocate(test.lbIds, test.num, test.podKey) + if _, exist := test.elb.podAllocate[test.podKey]; !exist { + t.Errorf("podAllocate[%s] is empty after allocated", test.podKey) + } + for _, port := range ports { + if port > test.elb.maxPort || port < test.elb.minPort { + t.Errorf("allocate port %d, unexpected", port) + } + if test.elb.cache[lbId][port] == false { + t.Errorf("Allocate port %d failed", port) + } + } + test.elb.deAllocate(test.podKey) + for _, port := range ports { + if test.elb.cache[lbId][port] == true { + t.Errorf("deAllocate port %d failed", port) + } + } + if _, exist := test.elb.podAllocate[test.podKey]; exist { + t.Errorf("podAllocate[%s] is not empty after deallocated", test.podKey) + } +} + +func TestParseLbConfig(t *testing.T) { + tests := []struct { + conf []gamekruiseiov1alpha1.NetworkConfParams + slbConfig *cceElbConfig + }{ + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: PortProtocolsConfigName, + Value: "80", + }, + { + Name: "kubernetes.io/elb.class", + Value: "performance", + }, + { + Name: "kubernetes.io/elb.id", + Value: "c1d8f4c6-7aef-4596-8c7c-xxxxxxxxxxxx", + }, + { + Name: "kubernetes.io/elb.enterpriseID", + Value: "ff97261-4dbd-4593-8236-xxxxxxxxxxxx", + }, + { + Name: "kubernetes.io/elb.lb-algorithm", + Value: "ROUND_ROBIN", + }, + { + Name: "kubernetes.io/elb.protocol-port", + Value: "https:443,http:80", + }, + { + Name: "kubernetes.io/elb.x-forwarded-port", + Value: "true", + }, + }, + slbConfig: &cceElbConfig{ + elbIds: []string{"c1d8f4c6-7aef-4596-8c7c-xxxxxxxxxxxx"}, + targetPorts: []int{80}, + protocols: []corev1.Protocol{corev1.ProtocolTCP}, + externalTrafficPolicyType: corev1.ServiceExternalTrafficPolicyTypeCluster, + isFixed: false, + hwOptions: map[string]string{ + "kubernetes.io/elb.class": "performance", + "kubernetes.io/elb.id": "c1d8f4c6-7aef-4596-8c7c-xxxxxxxxxxxx", + "kubernetes.io/elb.enterpriseID": "ff97261-4dbd-4593-8236-xxxxxxxxxxxx", + "kubernetes.io/elb.lb-algorithm": "ROUND_ROBIN", + "kubernetes.io/elb.protocol-port": "https:443,http:80", + "kubernetes.io/elb.x-forwarded-port": "true", + }, + }, + }, + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: PortProtocolsConfigName, + Value: "81/UDP,82,83/TCP", + }, + { + Name: "kubernetes.io/elb.class", + Value: "union", + }, + { + Name: "kubernetes.io/elb.id", + Value: "c1d8f4c6-7aef-4596-8c7c-yyyyyyyyyyyy", + }, + { + Name: "kubernetes.io/elb.enterpriseID", + Value: "ff97261-4dbd-4593-8236-yyyyyyyyyyyy", + }, + { + Name: "kubernetes.io/elb.cert-id", + Value: "17e3b4f4bc40471c86741dc3aa211379", + }, + { + Name: "kubernetes.io/elb.tls-certificate-ids", + Value: "5196aa70b0f143189e4cb54991ba2286,8125d71fcc124aabbe007610cba42d60", + }, + { + Name: "kubernetes.io/elb.multicluster", + Value: "true", + }, + { + Name: "kubernetes.io/elb.keepalive_timeout", + Value: "400s", + }, + { + Name: "kubernetes.io/elb.client_timeout", + Value: "50s", + }, + { + Name: "kubernetes.io/elb.member_timeout", + Value: "50s", + }, + { + Name: ExternalTrafficPolicyTypeConfigName, + Value: "Local", + }, + }, + slbConfig: &cceElbConfig{ + elbIds: []string{"c1d8f4c6-7aef-4596-8c7c-yyyyyyyyyyyy"}, + targetPorts: []int{81, 82, 83}, + protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP}, + externalTrafficPolicyType: corev1.ServiceExternalTrafficPolicyTypeLocal, + isFixed: false, + hwOptions: map[string]string{ + "kubernetes.io/elb.class": "union", + "kubernetes.io/elb.id": "c1d8f4c6-7aef-4596-8c7c-yyyyyyyyyyyy", + "kubernetes.io/elb.enterpriseID": "ff97261-4dbd-4593-8236-yyyyyyyyyyyy", + "kubernetes.io/elb.cert-id": "17e3b4f4bc40471c86741dc3aa211379", + "kubernetes.io/elb.tls-certificate-ids": "5196aa70b0f143189e4cb54991ba2286,8125d71fcc124aabbe007610cba42d60", + "kubernetes.io/elb.multicluster": "true", + "kubernetes.io/elb.keepalive_timeout": "400s", + "kubernetes.io/elb.client_timeout": "50s", + "kubernetes.io/elb.member_timeout": "50s", + }, + }, + }, + } + + for i, test := range tests { + sc, err := parseCCELbConfig(test.conf) + if err != nil { + t.Error(err) + } + if !reflect.DeepEqual(test.slbConfig, sc) { + t.Errorf("case %d: lbId expect: %v, actual: %v", i, test.slbConfig, sc) + } + } +} + +func TestInitLbCache(t *testing.T) { + svcA := fakeSvcTemplate.DeepCopy() + svcA.Annotations[ElbIdAnnotationKey] = "elb-id-A" + svcA.Name = "svc-A" + svcA.Namespace = "ns-A" + svcA.Spec.Ports[0].Port = 555 + + svcB := fakeSvcTemplate.DeepCopy() + svcB.Annotations[ElbIdAnnotationKey] = "elb-id-B" + svcB.Name = "svc-B" + svcB.Namespace = "ns-B" + svcB.Spec.Ports[0].Port = 666 + test := struct { + svcList []corev1.Service + minPort int32 + maxPort int32 + blockPorts []int32 + cache map[string]ccePortAllocated + podAllocate map[string]string + }{ + minPort: 512, + maxPort: 712, + blockPorts: []int32{593}, + cache: map[string]ccePortAllocated{ + "elb-id-A": map[int32]bool{ + 555: true, + 593: true, + }, + "elb-id-B": map[int32]bool{ + 666: true, + 593: true, + }, + }, + podAllocate: map[string]string{ + "ns-A/svc-A": "elb-id-A:555", + "ns-B/svc-B": "elb-id-B:666", + }, + svcList: []corev1.Service{ + *svcA, + *svcB, + }, + } + + actualCache, actualPodAllocate := initCCELbCache(test.svcList, test.minPort, test.maxPort, test.blockPorts) + for lb, pa := range test.cache { + for port, isAllocated := range pa { + if actualCache[lb][port] != isAllocated { + t.Errorf("lb %s port %d isAllocated, expect: %t, actual: %t", lb, port, isAllocated, actualCache[lb][port]) + } + } + } + if !reflect.DeepEqual(actualPodAllocate, test.podAllocate) { + t.Errorf("podAllocate expect %v, but actully got %v", test.podAllocate, actualPodAllocate) + } +} + +func TestElbPlugin_OnPodUpdated(t *testing.T) { + type fields struct { + maxPort int32 + minPort int32 + blockPorts []int32 + cache map[string]ccePortAllocated + podAllocate map[string]string + } + type args struct { + pod func() *corev1.Pod + ctx context.Context + } + fakePodTemplate := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-0", + Namespace: "default", + UID: "53cb0992-c720-4ae4-9af9-3cc7e2bf3660", + Annotations: map[string]string{ + "game.kruise.io/network-type": CCEElbNetwork, + "game.kruise.io/network-conf": `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"},{"name":"kubernetes.io/elb.connection-drain-enable","value":"true"},{"name":"kubernetes.io/elb.connection-drain-timeout","value":"300"}]`, + }, + }, + } + tests := []struct { + name string + fields fields + args args + setup func(*MockClient) + want func() *corev1.Pod + want1 errors.PluginError + }{ + { + name: "network is not ready", + fields: fields{ + maxPort: 500, + minPort: 502, + blockPorts: []int32{501}, + cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}}, + podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:501"}, + }, + args: args{ + pod: func() *corev1.Pod { + return fakePodTemplate + }, + ctx: context.Background(), + }, + want: func() *corev1.Pod { + res := fakePodTemplate.DeepCopy() + res.Annotations["game.kruise.io/network-status"] = `{"currentNetworkState":"NotReady","createTime":null,"lastTransitionTime":null}` + return res + }, + want1: nil, + }, + { + name: "network is ready", + fields: fields{ + maxPort: 500, + minPort: 502, + blockPorts: []int32{501}, + cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}}, + podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500,501"}, + }, + args: args{ + pod: func() *corev1.Pod { + res := fakePodTemplate.DeepCopy() + res.Annotations["game.kruise.io/network-status"] = `{"internalAddresses":[{"ip":"192.168.1.38","ports":[{"name":"80","protocol":"TCP","port":80}]}],"externalAddresses":[{"ip":"192.168.0.147","ports":[{"name":"80","protocol":"TCP","port":500}]}],"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}` + return res + }, + ctx: context.Background(), + }, + setup: func(clientMock *MockClient) { + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + service := args[2].(*corev1.Service) + *service = fakeSvcTemplate + }).Return(nil) + }, + want: func() *corev1.Pod { + res := fakePodTemplate.DeepCopy() + res.Annotations["game.kruise.io/network-status"] = `{"internalAddresses":[{"ip":"","ports":[{"name":"80","protocol":"TCP","port":80}]}],"externalAddresses":[{"ip":"192.168.0.147","ports":[{"name":"80","protocol":"TCP","port":500}]}],"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}` + res.Annotations["game.kruise.io/network-conf"] = `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"},{"name":"kubernetes.io/elb.connection-drain-enable","value":"true"},{"name":"kubernetes.io/elb.connection-drain-timeout","value":"300"}]` + return res + }, + want1: nil, + }, + { + name: "svc is not exist", + fields: fields{ + maxPort: 500, + minPort: 502, + blockPorts: []int32{501}, + cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}}, + podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500,501"}, + }, + args: args{ + pod: func() *corev1.Pod { + res := fakePodTemplate.DeepCopy() + res.Annotations["game.kruise.io/network-status"] = `{"currentNetworkState":"NotReady","createTime":null,"lastTransitionTime":null}` + res.Annotations["game.kruise.io/network-conf"] = `[{"name":"Fixed", "value":"true"},{"name":"PortProtocols","value":"80/TCPUDP"},{"name":"kubernetes.io/elb.class","value":"union"},{"name":"kubernetes.io/elb.id","value":"c1d8f4c6-7aef-4596-8c7c-2de87ff89545"}]` + return res + }, + ctx: context.Background(), + }, + setup: func(clientMock *MockClient) { + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(k8serrors.NewNotFound(schema.GroupResource{}, "test-pod-0")) + clientMock.On("Create", mock.Anything, mock.Anything, mock.Anything).Return(nil) + }, + want: func() *corev1.Pod { + res := fakePodTemplate.DeepCopy() + res.Annotations["game.kruise.io/network-status"] = `{"currentNetworkState":"NotReady","createTime":null,"lastTransitionTime":null}` + res.Annotations["game.kruise.io/network-conf"] = `[{"name":"Fixed", "value":"true"},{"name":"PortProtocols","value":"80/TCPUDP"},{"name":"kubernetes.io/elb.class","value":"union"},{"name":"kubernetes.io/elb.id","value":"c1d8f4c6-7aef-4596-8c7c-2de87ff89545"}]` + return res + }, + want1: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientMock := new(MockClient) + if tt.setup != nil { + tt.setup(clientMock) + } + if tt.want == nil { + t.Fatal("want is nil, set the function") + } + if tt.args.pod == nil { + t.Fatal("pod is nil, set the function") + } + s := &CCEElbPlugin{ + maxPort: tt.fields.maxPort, + minPort: tt.fields.minPort, + blockPorts: tt.fields.blockPorts, + cache: tt.fields.cache, + podAllocate: tt.fields.podAllocate, + mutex: sync.RWMutex{}, + } + got, got1 := s.OnPodUpdated(clientMock, tt.args.pod(), tt.args.ctx) + assert.Equalf(t, tt.want().Annotations["game.kruise.io/network-status"], got.Annotations["game.kruise.io/network-status"], "OnPodUpdated(%v, %v, %v)", clientMock, tt.args.pod, tt.args.ctx) + assert.Equalf(t, tt.want().Annotations["game.kruise.io/network-type"], got.Annotations["game.kruise.io/network-type"], "OnPodUpdated(%v, %v, %v)", clientMock, tt.args.pod, tt.args.ctx) + assert.Equalf(t, tt.want().Annotations["game.kruise.io/network-conf"], got.Annotations["game.kruise.io/network-conf"], "OnPodUpdated(%v, %v, %v)", clientMock, tt.args.pod, tt.args.ctx) + assert.Equalf(t, tt.want1, got1, "OnPodUpdated(%v, %v, %v)", clientMock, tt.args.pod, tt.args.ctx) + }) + } +} + +func TestElbPlugin_Init(t *testing.T) { + type fields struct { + maxPort int32 + minPort int32 + blockPorts []int32 + cache map[string]ccePortAllocated + podAllocate map[string]string + } + type args struct { + c client.Client + options cloudprovider.CloudProviderOptions + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + setup func(clientMock *MockClient) + want *CCEElbPlugin + wantErr assert.ErrorAssertionFunc + }{ + { + name: "success", + fields: fields{}, + args: args{ + c: nil, + options: options.HwCloudOptions{ + Enable: true, + CCEELBOptions: options.CCEELBOptions{ + ELBOptions: options.ELBOptions{ + MaxPort: 503, + MinPort: 500, + BlockPorts: []int32{501}, + }, + }, + }, + ctx: context.Background(), + }, + setup: func(clientMock *MockClient) { + clientMock.On("List", mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + res := args[1].(*corev1.ServiceList) + *res = corev1.ServiceList{ + Items: []corev1.Service{ + fakeSvcTemplate, + }, + } + }).Return(nil) + }, + want: &CCEElbPlugin{ + maxPort: 503, + minPort: 500, + blockPorts: []int32{501}, + cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false, 503: false}}, + podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"}, + mutex: sync.RWMutex{}, + }, + wantErr: assert.NoError, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &CCEElbPlugin{ + maxPort: tt.fields.maxPort, + minPort: tt.fields.minPort, + blockPorts: tt.fields.blockPorts, + cache: tt.fields.cache, + podAllocate: tt.fields.podAllocate, + mutex: sync.RWMutex{}, + } + clientMock := new(MockClient) + tt.args.c = clientMock + if tt.setup != nil { + tt.setup(clientMock) + } + tt.wantErr(t, s.Init(tt.args.c, tt.args.options, tt.args.ctx), fmt.Sprintf("Init(%v, %v, %v)", tt.args.c, tt.args.options, tt.args.ctx)) + assert.Equal(t, s.cache, tt.want.cache) + assert.Equal(t, s.podAllocate, tt.want.podAllocate) + assert.Equal(t, s.minPort, tt.want.minPort) + assert.Equal(t, s.maxPort, tt.want.maxPort) + assert.Equal(t, s.blockPorts, tt.want.blockPorts) + }) + } +} + +func TestElbPlugin_updateCachesAfterAutoCreateElb(t *testing.T) { + type fields struct { + maxPort int32 + minPort int32 + blockPorts []int32 + cache map[string]ccePortAllocated + podAllocate map[string]string + } + type args struct { + name string + namespace string + interval time.Duration + totalTimeout time.Duration + } + tests := []struct { + name string + fields fields + args args + setup func(clientMock *MockClient) + wantCache map[string]ccePortAllocated + wantPodAllocate map[string]string + }{ + { + name: "success", + fields: fields{ + minPort: 500, + maxPort: 502, + blockPorts: []int32{501}, + }, + args: args{ + name: "test-pod-0", + namespace: "default", + interval: 1 * time.Second, + totalTimeout: 2 * time.Second, + }, + setup: func(clientMock *MockClient) { + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + service := args[2].(*corev1.Service) + *service = fakeSvcTemplate + }).Return(nil) + }, + wantCache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}}, + wantPodAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"}, + }, + { + name: "timeout", + fields: fields{ + minPort: 500, + maxPort: 502, + blockPorts: []int32{501}, + }, + args: args{ + name: "test-pod-0", + namespace: "default", + interval: 1 * time.Second, + totalTimeout: 2 * time.Second, + }, + setup: func(clientMock *MockClient) { + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("some error")) + }, + wantCache: nil, + wantPodAllocate: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientMock := new(MockClient) + if tt.setup != nil { + tt.setup(clientMock) + } + s := &CCEElbPlugin{ + maxPort: tt.fields.maxPort, + minPort: tt.fields.minPort, + blockPorts: tt.fields.blockPorts, + cache: tt.fields.cache, + podAllocate: tt.fields.podAllocate, + mutex: sync.RWMutex{}, + } + s.updateCachesAfterAutoCreateElb(clientMock, tt.args.name, tt.args.namespace, tt.args.interval, tt.args.totalTimeout) + assert.Equal(t, s.cache, tt.wantCache) + assert.Equal(t, s.podAllocate, tt.wantPodAllocate) + }) + } +} + +func TestElbPlugin_OnPodDeleted(t *testing.T) { + podForTest := &corev1.Pod{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-0", + Namespace: "default", + Annotations: map[string]string{ + "kubernetes.io/elb.id": "8f4cf216-a659-40dc-8c77-6068b036ba56", + "game.kruise.io/network-conf": `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"}]`, + "game.kruise.io/network-type": CCEElbNetwork, + "game.kruise.io/network-status": `{"internalAddresses":[{"ip":"192.168.1.219","ports":[{"name":"80","protocol":"TCP","port":80}]}],"externalAddresses":[{"ip":"159.138.146.2","ports":[{"name":"80","protocol":"TCP","port":500}]}],"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}`, + }, + Labels: map[string]string{ + "game.kruise.io/owner-gss": "test-pod", + }, + }, + } + deleteTimetamp := metav1.NewTime(time.Now()) + type fields struct { + maxPort int32 + minPort int32 + blockPorts []int32 + cache map[string]ccePortAllocated + podAllocate map[string]string + } + type args struct { + c client.Client + pod func(p *corev1.Pod) *corev1.Pod + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + setup func(clientMock *MockClient) + wantCache map[string]ccePortAllocated + wantPodAllocate map[string]string + wantError errors.PluginError + }{ + { + name: "success", + fields: fields{ + maxPort: 502, + minPort: 500, + blockPorts: []int32{501}, + cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}}, + podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"}, + }, + args: args{ + c: nil, + pod: func(p *corev1.Pod) *corev1.Pod { + return p.DeepCopy() + }, + ctx: context.Background(), + }, + wantCache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: false, 501: true, 502: false}}, + wantPodAllocate: map[string]string{}, + wantError: nil, + }, + { + name: "fixed, get gss failed", + fields: fields{ + maxPort: 502, + minPort: 500, + blockPorts: []int32{501}, + cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}}, + podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"}, + }, + args: args{ + c: nil, + pod: func(p *corev1.Pod) *corev1.Pod { + res := p.DeepCopy() + res.Annotations["game.kruise.io/network-conf"] = `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"}, {"name":"Fixed","value":"true"}]` + res.Labels["game.kruise.io/owner-gss"] = "test-pod" + return res + }, + ctx: context.Background(), + }, + setup: func(clientMock *MockClient) { + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(fmt.Errorf("some error")) + }, + wantCache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}}, + wantPodAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"}, + wantError: errors.ToPluginError(fmt.Errorf("some error"), errors.ApiCallError), + }, + { + name: "fixed, gss exists", + fields: fields{ + maxPort: 502, + minPort: 500, + blockPorts: []int32{501}, + cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}}, + podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"}, + }, + args: args{ + c: nil, + pod: func(p *corev1.Pod) *corev1.Pod { + res := p.DeepCopy() + res.Annotations["game.kruise.io/network-conf"] = `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"}, {"name":"Fixed","value":"true"}]` + res.Labels["game.kruise.io/owner-gss"] = "test-pod" + return res + }, + ctx: context.Background(), + }, + setup: func(clientMock *MockClient) { + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + gss := args[2].(*gamekruiseiov1alpha1.GameServerSet) + *gss = gamekruiseiov1alpha1.GameServerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{}, + }, + } + }).Return(nil) + }, + wantCache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}}, + wantPodAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"}, + wantError: nil, + }, + { + name: "fixed, gss deleted", + fields: fields{ + maxPort: 502, + minPort: 500, + blockPorts: []int32{501}, + cache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: true, 501: true, 502: false}}, + podAllocate: map[string]string{"default/test-pod-0": "8f4cf216-a659-40dc-8c77-6068b036ba56:500"}, + }, + args: args{ + c: nil, + pod: func(p *corev1.Pod) *corev1.Pod { + res := p.DeepCopy() + res.Annotations["game.kruise.io/network-conf"] = `[{"name":"PortProtocols","value":"80/TCP"},{"name":"kubernetes.io/elb.class","value":"performance"},{"name":"kubernetes.io/elb.id","value":"8f4cf216-a659-40dc-8c77-6068b036ba56"}, {"name":"Fixed","value":"true"}]` + res.Labels["game.kruise.io/owner-gss"] = "test-pod" + return res + }, + ctx: context.Background(), + }, + setup: func(clientMock *MockClient) { + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + gss := args[2].(*gamekruiseiov1alpha1.GameServerSet) + *gss = gamekruiseiov1alpha1.GameServerSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{}, + DeletionTimestamp: &deleteTimetamp, + }, + } + }).Return(nil) + }, + wantCache: map[string]ccePortAllocated{"8f4cf216-a659-40dc-8c77-6068b036ba56": map[int32]bool{500: false, 501: true, 502: false}}, + wantPodAllocate: map[string]string{}, + wantError: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + clientMock := new(MockClient) + tt.args.c = clientMock + if tt.setup != nil { + tt.setup(clientMock) + } + s := &CCEElbPlugin{ + maxPort: tt.fields.maxPort, + minPort: tt.fields.minPort, + blockPorts: tt.fields.blockPorts, + cache: tt.fields.cache, + podAllocate: tt.fields.podAllocate, + mutex: sync.RWMutex{}, + } + assert.Equalf(t, tt.wantError, s.OnPodDeleted(tt.args.c, tt.args.pod(podForTest), tt.args.ctx), "OnPodDeleted(%v, %v, %v)", tt.args.c, tt.args.pod, tt.args.ctx) + assert.Equal(t, s.cache, tt.wantCache) + assert.Equal(t, s.podAllocate, tt.wantPodAllocate) + }) + } +} + +func Test_getSvcOwnerReference(t *testing.T) { + type args struct { + ctx context.Context + pod *corev1.Pod + isFixed bool + } + tests := []struct { + name string + args args + want []metav1.OwnerReference + setup func(clientMock *MockClient) + }{ + { + name: "fixed, success", + args: args{ + ctx: context.Background(), + pod: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "Pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod-0", + Namespace: "default", + UID: "pod-uuid-xxxx", + }, + }, + isFixed: true, + }, + setup: func(clientMock *MockClient) { + clientMock.On("Get", mock.Anything, mock.Anything, mock.Anything, mock.Anything).Run(func(args mock.Arguments) { + gss := args[2].(*gamekruiseiov1alpha1.GameServerSet) + *gss = gamekruiseiov1alpha1.GameServerSet{ + TypeMeta: metav1.TypeMeta{ + Kind: "GameServerSet", + APIVersion: "game.kruise.io/v1alpha1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + UID: "gss-uuid-xxxx", + }, + } + }).Return(nil) + }, + want: []metav1.OwnerReference{ + { + APIVersion: "game.kruise.io/v1alpha1", + Kind: "GameServerSet", + Name: "test-pod", + UID: "gss-uuid-xxxx", + Controller: ptr.To[bool](true), + BlockOwnerDeletion: ptr.To[bool](true), + }, + }, + }, + } + for _, tt := range tests { + clientMock := new(MockClient) + if tt.setup != nil { + tt.setup(clientMock) + } + t.Run(tt.name, func(t *testing.T) { + assert.Equalf(t, tt.want, getCCESvcOwnerReference(clientMock, tt.args.ctx, tt.args.pod, tt.args.isFixed), "getCCESvcOwnerReference(%v, %v, %v, %v)", clientMock, tt.args.ctx, tt.args.pod, tt.args.isFixed) + }) + } +} + +func TestElbPlugin_getPortFromHead(t *testing.T) { + type fields struct { + maxPort int32 + minPort int32 + blockPorts []int32 + cache map[string]ccePortAllocated + podAllocate map[string]string + } + type args struct { + num int + } + tests := []struct { + name string + fields fields + args args + want []int32 + }{ + { + name: "success", + fields: fields{ + maxPort: 505, + minPort: 500, + blockPorts: []int32{501}, + }, + args: args{ + num: 2, + }, + want: []int32{500, 502}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + s := &CCEElbPlugin{ + maxPort: tt.fields.maxPort, + minPort: tt.fields.minPort, + blockPorts: tt.fields.blockPorts, + cache: tt.fields.cache, + podAllocate: tt.fields.podAllocate, + mutex: sync.RWMutex{}, + } + assert.Equalf(t, tt.want, s.getPortFromHead(tt.args.num), "getPortFromHead(%v)", tt.args.num) + }) + } +} diff --git a/cloudprovider/hwcloud/elb_test_mock.go b/cloudprovider/hwcloud/elb_test_mock.go new file mode 100644 index 0000000..b0569ad --- /dev/null +++ b/cloudprovider/hwcloud/elb_test_mock.go @@ -0,0 +1,80 @@ +package hwcloud + +import ( + "context" + + "github.com/stretchr/testify/mock" + "k8s.io/apimachinery/pkg/api/meta" + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/apimachinery/pkg/runtime/schema" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +type MockClient struct { + mock.Mock +} + +func (m *MockClient) List(ctx context.Context, list client.ObjectList, opts ...client.ListOption) error { + args := m.Called(ctx, list, opts) + return args.Error(0) +} + +func (m *MockClient) Delete(ctx context.Context, obj client.Object, opts ...client.DeleteOption) error { + //TODO implement me + panic("implement me") +} + +func (m *MockClient) Patch(ctx context.Context, obj client.Object, patch client.Patch, opts ...client.PatchOption) error { + //TODO implement me + panic("implement me") +} + +func (m *MockClient) DeleteAllOf(ctx context.Context, obj client.Object, opts ...client.DeleteAllOfOption) error { + //TODO implement me + panic("implement me") +} + +func (m *MockClient) Status() client.SubResourceWriter { + //TODO implement me + panic("implement me") +} + +func (m *MockClient) SubResource(subResource string) client.SubResourceClient { + //TODO implement me + panic("implement me") +} + +func (m *MockClient) Scheme() *runtime.Scheme { + //TODO implement me + panic("implement me") +} + +func (m *MockClient) RESTMapper() meta.RESTMapper { + //TODO implement me + panic("implement me") +} + +func (m *MockClient) GroupVersionKindFor(obj runtime.Object) (schema.GroupVersionKind, error) { + //TODO implement me + panic("implement me") +} + +func (m *MockClient) IsObjectNamespaced(obj runtime.Object) (bool, error) { + //TODO implement me + panic("implement me") +} + +func (m *MockClient) Get(ctx context.Context, key client.ObjectKey, obj client.Object, opts ...client.GetOption) error { + args := m.Called(ctx, key, obj, opts) + return args.Error(0) +} + +func (m *MockClient) Create(ctx context.Context, obj client.Object, opts ...client.CreateOption) error { + args := m.Called(ctx, obj) + return args.Error(0) +} + +func (m *MockClient) Update(ctx context.Context, obj client.Object, opts ...client.UpdateOption) error { + args := m.Called(ctx, obj) + return args.Error(0) +} diff --git a/cloudprovider/hwcloud/hwcloud.go b/cloudprovider/hwcloud/hwcloud.go index 60fb167..9329491 100644 --- a/cloudprovider/hwcloud/hwcloud.go +++ b/cloudprovider/hwcloud/hwcloud.go @@ -1,3 +1,19 @@ +/* +Copyright 2022 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package hwcloud import ( diff --git a/cloudprovider/options/hwcloud_options.go b/cloudprovider/options/hwcloud_options.go index 32ff5fd..9f9b783 100644 --- a/cloudprovider/options/hwcloud_options.go +++ b/cloudprovider/options/hwcloud_options.go @@ -1,7 +1,12 @@ package options type HwCloudOptions struct { - Enable bool `toml:"enable"` + Enable bool `toml:"enable"` + ELBOptions ELBOptions `toml:"elb"` + CCEELBOptions CCEELBOptions `toml:"cce"` +} + +type CCEELBOptions struct { ELBOptions ELBOptions `toml:"elb"` } @@ -11,22 +16,28 @@ type ELBOptions struct { BlockPorts []int32 `toml:"block_ports"` } -func (o HwCloudOptions) Valid() bool { - elbOptions := o.ELBOptions - for _, blockPort := range elbOptions.BlockPorts { - if blockPort >= elbOptions.MaxPort || blockPort <= elbOptions.MinPort { +func (e ELBOptions) valid(skipPortRangeCheck bool) bool { + for _, blockPort := range e.BlockPorts { + if blockPort >= e.MaxPort || blockPort <= e.MinPort { return false } } - if int(elbOptions.MaxPort-elbOptions.MinPort)-len(elbOptions.BlockPorts) > 200 { + // old elb plugin only allow 200 ports. + if !skipPortRangeCheck && int(e.MaxPort-e.MinPort)-len(e.BlockPorts) > 200 { return false } - if elbOptions.MinPort <= 0 { + if e.MinPort <= 0 || e.MaxPort > 65535 { return false } return true } +func (o HwCloudOptions) Valid() bool { + elbOptions := o.ELBOptions + cceElbOptions := o.CCEELBOptions + return elbOptions.valid(false) && cceElbOptions.ELBOptions.valid(true) +} + func (o HwCloudOptions) Enabled() bool { return o.Enable } diff --git a/config/manager/config.toml b/config/manager/config.toml index 26568e7..6de5dd8 100644 --- a/config/manager/config.toml +++ b/config/manager/config.toml @@ -21,6 +21,10 @@ enable = true max_port = 700 min_port = 500 block_ports = [] +[hwcloud.cce.elb] +max_port = 65535 +min_port = 32768 +block_ports = [] [volcengine] enable = true diff --git a/docs/en/user_manuals/network.md b/docs/en/user_manuals/network.md index bdf659e..925ebc2 100644 --- a/docs/en/user_manuals/network.md +++ b/docs/en/user_manuals/network.md @@ -134,6 +134,9 @@ OpenKruiseGame supports the following network plugins: - AlibabaCloud-SLB - AlibabaCloud-SLB-SharedPort - Volcengine-EIP +- HwCloud-ELB +- HwCloud-CCE-ELB +- HwCloud-CCE-EIP --- @@ -1217,9 +1220,507 @@ max_port = 700 min_port = 500 block_ports = [] ``` +--- +### HwCloud-CCE-ELB +#### Plugin name +`HwCloud-CCE-ELB` +**Note**: +- This plugin is only applicable to Huawei Cloud's CCE Standard and CCE Turbo clusters. +- If using an existing ELB, ensure its VPC matches the CCE cluster's VPC; otherwise, access will fail. + +#### Cloud Provider +HuaweiCloud + +#### Plugin description +- HwCloud-ELB uses Huawei Cloud Load Balancer (ELB) as the entity for external service hosting. It distributes external traffic to multiple Pods within the cluster through Elastic Load Balancing (ELB), providing higher reliability compared to the NodePort type. +- Supported annotations, please refer to the documentation: https://support.huaweicloud.com/usermanual-cce/cce_10_0681.html +- The exposed public network access port is consistent with the port being listened to in the container. +- You can bind security groups for management ([Use annotations to bind security groups to Pods](https://support.huaweicloud.com/usermanual-cce/cce_10_0897.html)), which is only supported in CCE Turbo clusters. + - The network interface of the Pod uses the security group configured via the annotation: `yangtse.io/security-group-ids`. + - The Pod's network interface will use the existing security groups and additionally include the security group configured via the annotation: `yangtse.io/additional-security-group-ids`. +- Supports Network Isolation: Yes. + +#### Network parameters +PortProtocols + +- Meaning: Exposed ports and protocols of the Pod. Supports multiple ports/protocols. +- Format: port1/protocol1,port2/protocol2,... (Protocols must be uppercase). +- Supports Modification: Yes. + +Fixed +- Meaning: Whether to retain fixed access IP/port. If enabled, the external/internal mapping relationship remains unchanged even if Pods are recreated. +- Format: false / true +- Supports Modification: Yes. + +AllowNotReadyContainers +- Meaning: Container names allowed to maintain traffic flow during in-place upgrades. +- Format: {containerName_0},{containerName_1},... e.g., sidecar +- Supports Modification: Not modifiable during in-place upgrades. + +ExternalTrafficPolicyType +- Meaning: Determines whether Service LB forwards traffic only to local instances. Setting to Local creates a Local-type Service and retains client source IP addresses when configured with cloud-manager. +- Format: Local / Cluster (Default: Cluster) +- Supports Modification: No. Due to dependencies on fixed IP/port settings, modification is not recommended. + +Other Huawei CCE Cluster Parameters +Refer to annotations' keys/values in the documentation: +- [LoadBalancer](https://support.huaweicloud.com/usermanual-cce/cce_10_0014.html) + + +#### Plugin configuration +The port range here can be configured according to your business requirements. For block_ports, please refer to this issue: https://github.com/openkruise/kruise-game/issues/174 +``` +[hwcloud] +enable = true +[hwcloud.cce.elb] +max_port = 65535 +min_port = 32768 +block_ports = [] +``` --- +#### Example +Using Existing ELB +https://support.huaweicloud.com/usermanual-cce/cce_10_0385.html#section1 +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: hw-cce-elb-nginx + namespace: default +spec: + replicas: 2 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + network: + networkType: HwCloud-CCE-ELB + networkConf: + - name: PortProtocols + value: "80/TCP" + - name: kubernetes.io/elb.class # The type of the ELB instance + value: performance + - name: kubernetes.io/elb.id # The ID of the ELB instance + value: 8f4cf216-a659-40dc-8c77-xxxx + gameServerTemplate: + spec: + containers: + - image: nginx + name: nginx +``` +The generated svc is shown below. As you can see, both svcs point to the same ELB. +```yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + game.kruise.io/network-config-hash: "3594992400" + kubernetes.io/elb.class: performance + kubernetes.io/elb.connection-drain-enable: "true" + kubernetes.io/elb.connection-drain-timeout: "300" + kubernetes.io/elb.id: 8f4cf216-a659-40dc-8c77-xxxx + kubernetes.io/elb.mark: "0" + creationTimestamp: "2025-07-23T08:15:09Z" + finalizers: + - service.kubernetes.io/load-balancer-cleanup + name: hw-cce-elb-nginx-0 + namespace: kruise-game-system + ownerReferences: + - apiVersion: v1 + blockOwnerDeletion: true + controller: true + kind: Pod + name: hw-cce-elb-nginx-0 + uid: 4f9f37f9-16d4-4ee7-b553-9b6e0039c5d5 + resourceVersion: "13369506" + uid: 23815818-a626-4be3-b31f-4b95a4f89786 +spec: + allocateLoadBalancerNodePorts: true + clusterIP: 10.247.213.xxx + clusterIPs: + - 10.247.213.xxx + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + loadBalancerIP: 192.168.0.xxx + ports: + - name: 80-tcp + nodePort: 30622 + port: 3308 + protocol: TCP + targetPort: 80 + - name: 80-udp + nodePort: 30622 + port: 3308 + protocol: UDP + targetPort: 80 + selector: + statefulset.kubernetes.io/pod-name: hw-cce-elb-nginx-0 + sessionAffinity: None + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 192.168.0.xxx + - ip: 189.1.225.xxx + +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + game.kruise.io/network-config-hash: "3594992400" + kubernetes.io/elb.class: performance + kubernetes.io/elb.connection-drain-enable: "true" + kubernetes.io/elb.connection-drain-timeout: "300" + kubernetes.io/elb.id: 8f4cf216-a659-40dc-8c77-xxxx + kubernetes.io/elb.mark: "0" + creationTimestamp: "2025-07-23T08:15:08Z" + finalizers: + - service.kubernetes.io/load-balancer-cleanup + name: hw-cce-elb-nginx-1 + namespace: kruise-game-system + ownerReferences: + - apiVersion: v1 + blockOwnerDeletion: true + controller: true + kind: Pod + name: hw-cce-elb-nginx-1 + uid: 0f42b430-49ba-4203-8b50-4be059619b79 + resourceVersion: "13369489" + uid: 92a56054-ad92-4dbd-9d1b-e717e0a14af2 +spec: + allocateLoadBalancerNodePorts: true + clusterIP: 10.247.14.xxx + clusterIPs: + - 10.247.14.xxx + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + loadBalancerIP: 192.168.0.xxx + ports: + - name: 80-tcp + nodePort: 32227 + port: 3611 + protocol: TCP + targetPort: 80 + - name: 80-udp + nodePort: 32227 + port: 3611 + protocol: UDP + targetPort: 80 + selector: + statefulset.kubernetes.io/pod-name: hw-cce-elb-nginx-1 + sessionAffinity: None + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 192.168.0.xxx + - ip: 189.1.225.xxx +``` +The generated svc is shown below. As you can see, both svcs point to the same IP address, differing only in their ports: +```bash +kubectl get svc |grep hw-cce-elb-nginx +hw-cce-elb-nginx-0 LoadBalancer 10.247.213.xxx 189.1.225.xxx,192.168.0.xxx 3308:30622/TCP,3308:30622/UDP 2m3s +hw-cce-elb-nginx-1 LoadBalancer 10.247.14.xxx 189.1.225.xxx,192.168.0.xxx 3611:32227/TCP,3611:32227/UDP 2m4s +``` +--- +Automatically create an ELB and bind it to the created pod. +**Note**: +- When ELBs are automatically created for multiple replicas, each svc will use its own auto-created ELB. Each ELB will have a unique ID and a distinct external IP address. +- When the svc is deleted, the associated auto-created ELB will also be deleted. +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: hw-cce-elb-auto-performance + namespace: kruise-game-system +spec: + replicas: 2 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + network: + networkType: HwCloud-CCE-ELB + networkConf: + - name: PortProtocols + value: "80/TCP" + - name: kubernetes.io/elb.class + value: performance # The type of the ELB instance. + - name: kubernetes.io/elb.autocreate # Options for automatically creating an ELB: https://support.huaweicloud.com/usermanual-cce/cce_10_0385.html#section21 + value: '{ + "type": "public", + "bandwidth_name": "bandwidth-xxxx", + "bandwidth_chargemode": "traffic", + "bandwidth_size": 5, + "bandwidth_sharetype": "PER", + "eip_type": "5_bgp", + "available_zone": [ + "ap-southeast-1a", + "ap-southeast-1b" + ], + "l4_flavor_name": "L4_flavor.elb.s1.small" + }' + - name: kubernetes.io/elb.enterpriseID # The enterprise project ID to which the created load balancer belongs. + value: 'aff97261-4dbd-4593-8236-xxxx' + - name: kubernetes.io/elb.lb-algorithm + value: ROUND_ROBIN # Load balancer algorithm + gameServerTemplate: + spec: + containers: + - image: nginx + name: nginx + +``` +The generated svc is shown below. As you can see, both svcs point to different ELBs. +```yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + game.kruise.io/network-config-hash: "3090934611" + kubernetes.io/elb.autocreate: '{ "type": "public", "bandwidth_name": "bandwidth-89f0", + "bandwidth_chargemode": "traffic", "bandwidth_size": 5, "bandwidth_sharetype": + "PER", "eip_type": "5_bgp", "available_zone": [ "ap-southeast-1a", "ap-southeast-1b" + ], "l4_flavor_name": "L4_flavor.elb.s1.small" }' + kubernetes.io/elb.class: performance + kubernetes.io/elb.eip-id: 566d5f4c-3484-4d7e-aa6b-xxxx + kubernetes.io/elb.enterpriseID: aff97261-4dbd-4593-8236-xxxx + kubernetes.io/elb.id: 75e06e8b-a246-48cb-b05c-xxxx + kubernetes.io/elb.lb-algorithm: ROUND_ROBIN + kubernetes.io/elb.mark: "0" + creationTimestamp: "2025-07-23T09:25:01Z" + finalizers: + - service.kubernetes.io/load-balancer-cleanup + name: hw-cce-elb-auto-performance-0 + namespace: kruise-game-system + ownerReferences: + - apiVersion: v1 + blockOwnerDeletion: true + controller: true + kind: Pod + name: hw-cce-elb-auto-performance-0 + uid: 1da0edf4-f45d-4635-8db0-ed5ccea2441d + resourceVersion: "13401553" + uid: 13efd440-65a7-4b45-bafc-2268102a4fd7 +spec: + allocateLoadBalancerNodePorts: true + clusterIP: 10.247.50.xxx + clusterIPs: + - 10.247.50.xxx + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + loadBalancerIP: 49.0.251.xxx + ports: + - name: 80-tcp + nodePort: 30918 + port: 1 + protocol: TCP + targetPort: 80 + selector: + statefulset.kubernetes.io/pod-name: hw-cce-elb-auto-performance-0 + sessionAffinity: None + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 49.0.251.xxx + - ip: 192.168.1.xxx +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + game.kruise.io/network-config-hash: "3090934611" + kubernetes.io/elb.autocreate: '{ "type": "public", "bandwidth_name": "bandwidth-89f0", + "bandwidth_chargemode": "traffic", "bandwidth_size": 5, "bandwidth_sharetype": + "PER", "eip_type": "5_bgp", "available_zone": [ "ap-southeast-1a", "ap-southeast-1b" + ], "l4_flavor_name": "L4_flavor.elb.s1.small" }' + kubernetes.io/elb.class: performance + kubernetes.io/elb.eip-id: 4a5396b1-e750-4ba5-a5d3-xxxx + kubernetes.io/elb.enterpriseID: aff97261-4dbd-4593-8236-xxxx + kubernetes.io/elb.id: b093db79-3c3e-4e77-a2ee-xxxx + kubernetes.io/elb.lb-algorithm: ROUND_ROBIN + kubernetes.io/elb.mark: "0" + creationTimestamp: "2025-07-23T09:25:01Z" + finalizers: + - service.kubernetes.io/load-balancer-cleanup + name: hw-cce-elb-auto-performance-1 + namespace: kruise-game-system + ownerReferences: + - apiVersion: v1 + blockOwnerDeletion: true + controller: true + kind: Pod + name: hw-cce-elb-auto-performance-1 + uid: abfc9ad1-1ae3-45fa-b956-4617c465a44f + resourceVersion: "13401664" + uid: 01dd8e13-b1c8-4d9f-8b1c-13c2f001c614 +spec: + allocateLoadBalancerNodePorts: true + clusterIP: 10.247.196.xxx + clusterIPs: + - 10.247.196.xxx + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + loadBalancerIP: 150.40.245.xxx + ports: + - name: 80-tcp + nodePort: 30942 + port: 1 + protocol: TCP + targetPort: 80 + selector: + statefulset.kubernetes.io/pod-name: hw-cce-elb-auto-performance-1 + sessionAffinity: None + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 150.40.245.xxx + - ip: 192.168.1.xxx +``` +The generated svc is shown below. As you can see, both svcs are assigned different external IPs: +```bash +kubectl get svc |grep hw-cce-elb-auto-performance +hw-cce-elb-auto-performance-0 LoadBalancer 10.247.50.xxx 192.168.1.xxx,49.0.251.xxx 1:30918/TCP 4m29s +hw-cce-elb-auto-performance-1 LoadBalancer 10.247.196.xxx 150.40.245.xxx,192.168.1.xxx 1:30942/TCP 4m29s +``` + +#### Plugin Name +`HwCloud-EIP` +**Note**: This plugin is only applicable to Huawei Cloud's CCE Turbo clusters. + +#### Cloud Provider +HuaweiCloud + +#### Plugin Description +- Only Huawei Cloud CCE Turbo clusters are supported: https://support.huaweicloud.com/usermanual-cce/cce_10_0284.html#section1 +- Assigns a separate Elastic IP (EIP) to each pod. +- The exposed public network access port is consistent with the port being listened to in the container. Security groups can be bound for management ([Binding Security Groups to Pods Using Annotations](https://support.huaweicloud.com/usermanual-cce/cce_10_0897.html)) + - The Pod's network interface uses the security group configured via the annotation: `yangtse.io/security-group-ids`. + - The Pod's network interface will use the existing security groups while additionally applying the security group configured via the annotation: `yangtse.io/additional-security-group-ids` +- The automatically created EIP does not support specifying the 'enterprise project' during creation. + +#### Network Parameters +Refer to Huawei Cloud documentation: https://support.huaweicloud.com/usermanual-cce/cce_10_0734.html. This plugin supports all annotations on this page. + +#### Plugin Configuration +None + +#### Example +Exclusive Bandwidth EIP Created with Pod +Note: The EIP created here belongs to the `default` enterprise project. Huawei Cloud currently does not support specifying enterprise projects in this mode. +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: hwcloud-cce-eip-performance + namespace: default +spec: + replicas: 2 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + network: + networkType: HwCloud-CCE-EIP + networkConf: + # https://support.huaweicloud.com/usermanual-cce/cce_10_0734.html + - name: yangtse.io/pod-with-eip + value: "true" + - name: yangtse.io/eip-bandwidth-size + value: "5" + - name: yangtse.io/eip-network-type + value: "5_bgp" + - name: yangtse.io/eip-charge-mode + value: "traffic" + gameServerTemplate: + spec: + containers: + - image: nginx + name: nginx +``` + +Generated Pod Annotations: +`yangtse.io/allocated-eip-id` corresponds to the EIP viewable in Huawei Cloud's Elastic IP details. +`yangtse.io/allocated-ipv4-eip` is the pod's EIP. +```yaml +apiVersion: v1 +kind: Pod +metadata: + annotations: + apps.kruise.io/runtime-containers-meta: '{"containers":[{"name":"nginx","containerID":"containerd://302f710dc7fb5771be5b16a31de84ff457fd84c9aa1ce00b7e7f2ddc3b7c3978","restartCount":0,"hashes":{"plainHash":2641665875,"plainHashWithoutResources":0,"extractedEnvFromMetadataHash":86995377}}]}' + game.kruise.io/network-conf: '[{"name":"yangtse.io/pod-with-eip","value":"true"},{"name":"yangtse.io/eip-bandwidth-size","value":"5"},{"name":"yangtse.io/eip-network-type","value":"5_bgp"},{"name":"yangtse.io/eip-charge-mode","value":"traffic"}]' + game.kruise.io/network-status: '{"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}' + game.kruise.io/network-trigger-time: "2025-07-16 17:03:07" + game.kruise.io/network-type: HwCloud-EIP + game.kruise.io/opsState-last-changed-time: "2025-07-16 17:03:07" + game.kruise.io/state-last-changed-time: "2025-07-16 09:03:13" + lifecycle.apps.kruise.io/timestamp: "2025-07-16T09:03:03Z" + yangtse.io/allocated-eip-id: 3a52ca79-d78d-4fc2-8590-xxx + yangtse.io/allocated-ipv4-eip: 94.74.110.xxx + yangtse.io/eip-bandwidth-size: "5" + yangtse.io/eip-charge-mode: traffic + yangtse.io/eip-network-type: 5_bgp + yangtse.io/pod-with-eip: "true" +``` + +To use an existing EIP, add yangtse.io/eip-id in spec.network.networkConf. You need to create the EIP in Huawei Cloud in advance. +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: hw-cce-eip-exist + namespace: kruise-game-system +spec: + replicas: 1 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + network: + networkType: HwCloud-CCE-EIP + networkConf: + - name: yangtse.io/eip-id + value: "7ec474aa-3bd9-46a2-a45c-xxx" # Use an existing EIP. + gameServerTemplate: + spec: + containers: + - image: nginx + name: nginx +``` +In the pod's YAML, you can see that the yangtse.io/allocated-eip-id in the pod's annotations corresponds to the EIP we specified. +By logging into the Huawei Cloud EIP console, you can verify that this EIP is already bound to the pod. +```yaml +apiVersion: v1 +kind: Pod +metadata: + annotations: + apps.kruise.io/runtime-containers-meta: '{"containers":[{"name":"nginx","containerID":"containerd://0fc9de69e30b48cf13ad2d2c6f5fe3be86e48e922a982dbb77b53ffd0ca6f54b","restartCount":0,"hashes":{"plainHash":2957831032,"plainHashWithoutResources":0,"extractedEnvFromMetadataHash":86995377}}]}' + game.kruise.io/network-conf: '[{"name":"yangtse.io/eip-id","value":"7ec474aa-3bd9-46a2-a45c-xxxx"}]' + game.kruise.io/network-status: '{"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}' + game.kruise.io/network-trigger-time: "2025-07-18 15:38:21" + game.kruise.io/network-type: HwCloud-EIP + game.kruise.io/opsState-last-changed-time: "2025-07-18 15:38:21" + game.kruise.io/state-last-changed-time: "2025-07-18 15:38:31" + lifecycle.apps.kruise.io/timestamp: "2025-07-18T07:38:13Z" + yangtse.io/allocated-eip-id: 7ec474aa-3bd9-46a2-a45c-xxxx + yangtse.io/allocated-ipv4-eip: 159.138.21.xxx + yangtse.io/eip-id: 7ec474aa-3bd9-46a2-a45c-xxxx + creationTimestamp: "2025-07-18T07:38:14Z +# other info ignored +``` ### Volcengine-EIP #### Plugin name diff --git a/docs/中文/用户手册/网络模型.md b/docs/中文/用户手册/网络模型.md index 9365315..20cfe02 100644 --- a/docs/中文/用户手册/网络模型.md +++ b/docs/中文/用户手册/网络模型.md @@ -140,7 +140,9 @@ EOF - AlibabaCloud-NLB-SharedPort - Volcengine-CLB - Volcengine-EIP - +- HwCloud-ELB +- HwCloud-CCE-ELB +- HwCloud-CCE-EIP --- ### Kubernetes-HostPort @@ -1342,7 +1344,506 @@ spec: lastTransitionTime: "2024-10-28T03:16:20Z" networkType: TencentCloud-CLB ``` +--- +### HwCloud-CCE-ELB +#### 插件名称 +`HwCloud-CCE-ELB` +**注意**: +- 此插件仅仅适用于华为云的CCE Standard和CCE Turbo集群. +- 如果使用已有的ELB, 请保证ELB的VPC和CCE集群的VPC一致, 否则无法访问. + +#### Cloud Provider +HuaweiCloud + +#### 插件说明 +- HwCloud-ELB 使用华为云负载均衡器（ELB）作为对外服务的承载实体，可以通过弹性负载均衡（ELB）将外部流量向集群内的多个Pod进行分发，与NodePort类型相比提供了高可靠的保障。 +- 支持的annotations,可以参看文档: https://support.huaweicloud.com/usermanual-cce/cce_10_0681.html +- 暴露的公网访问端口与容器中监听的端口一致. +- 可以绑定安全组进行管理([使用注解为Pod绑定安全组](https://support.huaweicloud.com/usermanual-cce/cce_10_0897.html)),需要CCE Turbo集群才支持。 + - Pod的网卡使用annotation配置的安全组: `yangtse.io/security-group-ids` + - Pod的网卡在使用已有安全组的基础上，额外再增加annotation配置的安全组: `yangtse.io/additional-security-group-ids` +- 是否支持网络隔离：是。 + +#### 网络参数 +PortProtocols +- 含义：pod暴露的端口及协议，支持填写多个端口/协议。 +- 格式：port1/protocol1,port2/protocol2,...（协议需大写） +- 是否支持变更：支持。 + +Fixed +- 含义：是否固定访问IP/端口。若是，即使pod删除重建，网络内外映射关系不会改变 +- 填写格式：false / true +- 是否支持变更：支持 + +AllowNotReadyContainers +- 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个 +- 格式：{containerName_0},{containerName_1},... 例如：sidecar +- 是否支持变更：在原地升级过程中不可变更。 + +ExternalTrafficPolicyType +- 含义：Service LB 是否只转发给本地实例。若是Local， 创建Local类型Service, 配合cloud-manager只配置对应Node，可以保留客户端源IP地址 +- 填写格式: Local/Cluster 默认Cluster +- 是否支持变更：不支持。跟是否固定IP/端口有关系，建议不更改 + +其他与华为CCE集群相关的参数 +参考文档里的annotation的key与value进行填写 +- [负载均衡（LoadBalancer）](https://support.huaweicloud.com/usermanual-cce/cce_10_0014.html) + + +#### Plugin configuration +这里的端口范围可以根据您的业务范围自行配置, block_ports请参考这个issue: https://github.com/openkruise/kruise-game/issues/174 +``` +[hwcloud] +enable = true +[hwcloud.cce.elb] +max_port = 65535 +min_port = 32768 +block_ports = [] +``` + +--- +#### 示例说明 +使用已有的elb(https://support.huaweicloud.com/usermanual-cce/cce_10_0385.html#section1),其他可用的annotation请参考华为云文档 +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: hw-cce-elb-nginx + namespace: default +spec: + replicas: 2 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + network: + networkType: HwCloud-CCE-ELB + networkConf: + - name: PortProtocols + value: "80/TCP" + - name: kubernetes.io/elb.class # ELB实例的类型 + value: performance + - name: kubernetes.io/elb.id # ELB实例的ID + value: 8f4cf216-a659-40dc-8c77-xxxx + gameServerTemplate: + spec: + containers: + - image: nginx + name: nginx +``` + +生成的svc如下所示,可以看到2个svc都指向了同一个elb: +```yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + game.kruise.io/network-config-hash: "3594992400" + kubernetes.io/elb.class: performance + kubernetes.io/elb.connection-drain-enable: "true" + kubernetes.io/elb.connection-drain-timeout: "300" + kubernetes.io/elb.id: 8f4cf216-a659-40dc-8c77-xxxx + kubernetes.io/elb.mark: "0" + creationTimestamp: "2025-07-23T08:15:09Z" + finalizers: + - service.kubernetes.io/load-balancer-cleanup + name: hw-cce-elb-nginx-0 + namespace: kruise-game-system + ownerReferences: + - apiVersion: v1 + blockOwnerDeletion: true + controller: true + kind: Pod + name: hw-cce-elb-nginx-0 + uid: 4f9f37f9-16d4-4ee7-b553-9b6e0039c5d5 + resourceVersion: "13369506" + uid: 23815818-a626-4be3-b31f-4b95a4f89786 +spec: + allocateLoadBalancerNodePorts: true + clusterIP: 10.247.213.xxx + clusterIPs: + - 10.247.213.xxx + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + loadBalancerIP: 192.168.0.xxx + ports: + - name: 80-tcp + nodePort: 30622 + port: 3308 + protocol: TCP + targetPort: 80 + - name: 80-udp + nodePort: 30622 + port: 3308 + protocol: UDP + targetPort: 80 + selector: + statefulset.kubernetes.io/pod-name: hw-cce-elb-nginx-0 + sessionAffinity: None + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 192.168.0.xxx + - ip: 189.1.225.xxx + +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + game.kruise.io/network-config-hash: "3594992400" + kubernetes.io/elb.class: performance + kubernetes.io/elb.connection-drain-enable: "true" + kubernetes.io/elb.connection-drain-timeout: "300" + kubernetes.io/elb.id: 8f4cf216-a659-40dc-8c77-xxxx + kubernetes.io/elb.mark: "0" + creationTimestamp: "2025-07-23T08:15:08Z" + finalizers: + - service.kubernetes.io/load-balancer-cleanup + name: hw-cce-elb-nginx-1 + namespace: kruise-game-system + ownerReferences: + - apiVersion: v1 + blockOwnerDeletion: true + controller: true + kind: Pod + name: hw-cce-elb-nginx-1 + uid: 0f42b430-49ba-4203-8b50-4be059619b79 + resourceVersion: "13369489" + uid: 92a56054-ad92-4dbd-9d1b-e717e0a14af2 +spec: + allocateLoadBalancerNodePorts: true + clusterIP: 10.247.14.xxx + clusterIPs: + - 10.247.14.xxx + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + loadBalancerIP: 192.168.0.xxx + ports: + - name: 80-tcp + nodePort: 32227 + port: 3611 + protocol: TCP + targetPort: 80 + - name: 80-udp + nodePort: 32227 + port: 3611 + protocol: UDP + targetPort: 80 + selector: + statefulset.kubernetes.io/pod-name: hw-cce-elb-nginx-1 + sessionAffinity: None + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 192.168.0.xxx + - ip: 189.1.225.xxx +``` +生成的svc如下,可以看到2个svc指向了同一个ip, 只是端口不同: +```bash +kubectl get svc |grep hw-cce-elb-nginx +hw-cce-elb-nginx-0 LoadBalancer 10.247.213.xxx 189.1.225.xxx,192.168.0.xxx 3308:30622/TCP,3308:30622/UDP 2m3s +hw-cce-elb-nginx-1 LoadBalancer 10.247.14.xxx 189.1.225.xxx,192.168.0.xxx 3611:32227/TCP,3611:32227/UDP 2m4s +``` +--- +自动创建ELB并绑定到创建出来的svc上面 +**注意**: +- 自动创建elb,如果为多副本,每个svc都使用自动创建的elb,每个elb的id是不一样的,暴露的external ip也是不一样的. +- 当删除svc时, 关联的自动创建的elb也会被删除. +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: hw-cce-elb-auto-performance + namespace: kruise-game-system +spec: + replicas: 2 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + network: + networkType: HwCloud-CCE-ELB + networkConf: + - name: PortProtocols + value: "80/TCP" + - name: kubernetes.io/elb.class + value: performance # ELB实例的类型 + - name: kubernetes.io/elb.autocreate # 自动创建ELB的选项: https://support.huaweicloud.com/usermanual-cce/cce_10_0385.html#section21 + value: '{ + "type": "public", + "bandwidth_name": "bandwidth-xxxx", + "bandwidth_chargemode": "traffic", + "bandwidth_size": 5, + "bandwidth_sharetype": "PER", + "eip_type": "5_bgp", + "available_zone": [ + "ap-southeast-1a", + "ap-southeast-1b" + ], + "l4_flavor_name": "L4_flavor.elb.s1.small" + }' + - name: kubernetes.io/elb.enterpriseID # 创建出来的负载均衡所属企业项目ID + value: 'aff97261-4dbd-4593-8236-xxxx' + - name: kubernetes.io/elb.lb-algorithm + value: ROUND_ROBIN # 负载均衡器算法 + gameServerTemplate: + spec: + containers: + - image: nginx + name: nginx + +``` +生成的svc如下所示,可以看到2个svc都指向不同的elb: +```yaml +apiVersion: v1 +kind: Service +metadata: + annotations: + game.kruise.io/network-config-hash: "3090934611" + kubernetes.io/elb.autocreate: '{ "type": "public", "bandwidth_name": "bandwidth-89f0", + "bandwidth_chargemode": "traffic", "bandwidth_size": 5, "bandwidth_sharetype": + "PER", "eip_type": "5_bgp", "available_zone": [ "ap-southeast-1a", "ap-southeast-1b" + ], "l4_flavor_name": "L4_flavor.elb.s1.small" }' + kubernetes.io/elb.class: performance + kubernetes.io/elb.eip-id: 566d5f4c-3484-4d7e-aa6b-xxxx + kubernetes.io/elb.enterpriseID: aff97261-4dbd-4593-8236-xxxx + kubernetes.io/elb.id: 75e06e8b-a246-48cb-b05c-xxxx + kubernetes.io/elb.lb-algorithm: ROUND_ROBIN + kubernetes.io/elb.mark: "0" + creationTimestamp: "2025-07-23T09:25:01Z" + finalizers: + - service.kubernetes.io/load-balancer-cleanup + name: hw-cce-elb-auto-performance-0 + namespace: kruise-game-system + ownerReferences: + - apiVersion: v1 + blockOwnerDeletion: true + controller: true + kind: Pod + name: hw-cce-elb-auto-performance-0 + uid: 1da0edf4-f45d-4635-8db0-ed5ccea2441d + resourceVersion: "13401553" + uid: 13efd440-65a7-4b45-bafc-2268102a4fd7 +spec: + allocateLoadBalancerNodePorts: true + clusterIP: 10.247.50.xxx + clusterIPs: + - 10.247.50.xxx + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + loadBalancerIP: 49.0.251.xxx + ports: + - name: 80-tcp + nodePort: 30918 + port: 1 + protocol: TCP + targetPort: 80 + selector: + statefulset.kubernetes.io/pod-name: hw-cce-elb-auto-performance-0 + sessionAffinity: None + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 49.0.251.xxx + - ip: 192.168.1.xxx +--- +apiVersion: v1 +kind: Service +metadata: + annotations: + game.kruise.io/network-config-hash: "3090934611" + kubernetes.io/elb.autocreate: '{ "type": "public", "bandwidth_name": "bandwidth-89f0", + "bandwidth_chargemode": "traffic", "bandwidth_size": 5, "bandwidth_sharetype": + "PER", "eip_type": "5_bgp", "available_zone": [ "ap-southeast-1a", "ap-southeast-1b" + ], "l4_flavor_name": "L4_flavor.elb.s1.small" }' + kubernetes.io/elb.class: performance + kubernetes.io/elb.eip-id: 4a5396b1-e750-4ba5-a5d3-xxxx + kubernetes.io/elb.enterpriseID: aff97261-4dbd-4593-8236-xxxx + kubernetes.io/elb.id: b093db79-3c3e-4e77-a2ee-xxxx + kubernetes.io/elb.lb-algorithm: ROUND_ROBIN + kubernetes.io/elb.mark: "0" + creationTimestamp: "2025-07-23T09:25:01Z" + finalizers: + - service.kubernetes.io/load-balancer-cleanup + name: hw-cce-elb-auto-performance-1 + namespace: kruise-game-system + ownerReferences: + - apiVersion: v1 + blockOwnerDeletion: true + controller: true + kind: Pod + name: hw-cce-elb-auto-performance-1 + uid: abfc9ad1-1ae3-45fa-b956-4617c465a44f + resourceVersion: "13401664" + uid: 01dd8e13-b1c8-4d9f-8b1c-13c2f001c614 +spec: + allocateLoadBalancerNodePorts: true + clusterIP: 10.247.196.xxx + clusterIPs: + - 10.247.196.xxx + externalTrafficPolicy: Cluster + internalTrafficPolicy: Cluster + ipFamilies: + - IPv4 + ipFamilyPolicy: SingleStack + loadBalancerIP: 150.40.245.xxx + ports: + - name: 80-tcp + nodePort: 30942 + port: 1 + protocol: TCP + targetPort: 80 + selector: + statefulset.kubernetes.io/pod-name: hw-cce-elb-auto-performance-1 + sessionAffinity: None + type: LoadBalancer +status: + loadBalancer: + ingress: + - ip: 150.40.245.xxx + - ip: 192.168.1.xxx +``` +生成的svc如下,可以看到2个svc指向不同的external ip: +```bash +kubectl get svc |grep hw-cce-elb-auto-performance +hw-cce-elb-auto-performance-0 LoadBalancer 10.247.50.xxx 192.168.1.xxx,49.0.251.xxx 1:30918/TCP 4m29s +hw-cce-elb-auto-performance-1 LoadBalancer 10.247.196.xxx 150.40.245.xxx,192.168.1.xxx 1:30942/TCP 4m29s +``` + +### HwCloud-CCE-EIP +#### 插件名称 +`HwCloud-CCE-EIP` +**注意**: 此插件仅仅适用于华为云的CCE Turbo集群 + +#### Cloud Provider +HuaweiCloud + +#### 插件说明 +- 仅适用于华为云CCE Turbo集群: https://support.huaweicloud.com/usermanual-cce/cce_10_0284.html#section1 +- 为每个pod单独分配EIP +- 暴露的公网访问端口与容器中监听的端口一致，可以绑定安全组进行管理([使用注解为Pod绑定安全组](https://support.huaweicloud.com/usermanual-cce/cce_10_0897.html)) + - Pod的网卡使用annotation配置的安全组: `yangtse.io/security-group-ids` + - Pod的网卡在使用已有安全组的基础上，额外再增加annotation配置的安全组: `yangtse.io/additional-security-group-ids` +- 支持自动创建EIP并绑定到对应的pod上, 或者使用已有的EIP +- 自动创建EIP时不支持指定`企业项目` +#### 网络参数 +参考华为云的文档: https://support.huaweicloud.com/usermanual-cce/cce_10_0734.html, 本插件支持这个页面所有的annotation. + +#### 插件配置 +无 + +#### 示例说明 +独占带宽EIP跟随Pod创建, 其他可用的annotation请参考华为云文档 +注意: 这里创建出来的EIP属于`default`这个企业项目, 华为云CCE Turbo集群暂时不支持在这种模式下指定`企业项目` +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: hw-cce-eip-performance + namespace: default +spec: + replicas: 2 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + network: + networkType: HwCloud-CCE-EIP + networkConf: + # https://support.huaweicloud.com/usermanual-cce/cce_10_0734.html + - name: yangtse.io/pod-with-eip + value: "true" + - name: yangtse.io/eip-bandwidth-size + value: "5" + - name: yangtse.io/eip-network-type + value: "5_bgp" + - name: yangtse.io/eip-charge-mode + value: "traffic" + gameServerTemplate: + spec: + containers: + - image: nginx + name: nginx +``` + +生成的pod的annotation如下,`yangtse.io/allocated-eip-id`对应的eip可以在华为云的弹性公网IP详情中查看, `yangtse.io/allocated-ipv4-eip` +即为pod的eip +```yaml +apiVersion: v1 +kind: Pod +metadata: + annotations: + apps.kruise.io/runtime-containers-meta: '{"containers":[{"name":"nginx","containerID":"containerd://302f710dc7fb5771be5b16a31de84ff457fd84c9aa1ce00b7e7f2ddc3b7c3978","restartCount":0,"hashes":{"plainHash":2641665875,"plainHashWithoutResources":0,"extractedEnvFromMetadataHash":86995377}}]}' + game.kruise.io/network-conf: '[{"name":"yangtse.io/pod-with-eip","value":"true"},{"name":"yangtse.io/eip-bandwidth-size","value":"5"},{"name":"yangtse.io/eip-network-type","value":"5_bgp"},{"name":"yangtse.io/eip-charge-mode","value":"traffic"}]' + game.kruise.io/network-status: '{"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}' + game.kruise.io/network-trigger-time: "2025-07-16 17:03:07" + game.kruise.io/network-type: HwCloud-EIP + game.kruise.io/opsState-last-changed-time: "2025-07-16 17:03:07" + game.kruise.io/state-last-changed-time: "2025-07-16 09:03:13" + lifecycle.apps.kruise.io/timestamp: "2025-07-16T09:03:03Z" + yangtse.io/allocated-eip-id: 3a52ca79-d78d-4fc2-8590-xxxx + yangtse.io/allocated-ipv4-eip: 94.74.110.xxx + yangtse.io/eip-bandwidth-size: "5" + yangtse.io/eip-charge-mode: traffic + yangtse.io/eip-network-type: 5_bgp + yangtse.io/pod-with-eip: "true" +``` + +使用已有的EIP, spec.network.networkConf添加`yangtse.io/eip-id`,你需要事先在华为云创建好EIP +```yaml +apiVersion: game.kruise.io/v1alpha1 +kind: GameServerSet +metadata: + name: hw-cce-eip-exist + namespace: kruise-game-system +spec: + replicas: 1 + updateStrategy: + rollingUpdate: + podUpdatePolicy: InPlaceIfPossible + network: + networkType: HwCloud-CCE-EIP + networkConf: + - name: yangtse.io/eip-id + value: "7ec474aa-3bd9-46a2-a45c-xxxx" # 使用已有的EIP + gameServerTemplate: + spec: + containers: + - image: nginx + name: nginx +``` + +pod的yaml, 可以看到pod的annotations里`yangtse.io/allocated-eip-id`为我们指定的EIP,登录华为云EIP控制台,可以看到这个EIP已经绑定了pod +```yaml +apiVersion: v1 +kind: Pod +metadata: + annotations: + apps.kruise.io/runtime-containers-meta: '{"containers":[{"name":"nginx","containerID":"containerd://0fc9de69e30b48cf13ad2d2c6f5fe3be86e48e922a982dbb77b53ffd0ca6f54b","restartCount":0,"hashes":{"plainHash":2957831032,"plainHashWithoutResources":0,"extractedEnvFromMetadataHash":86995377}}]}' + game.kruise.io/network-conf: '[{"name":"yangtse.io/eip-id","value":"7ec474aa-3bd9-46a2-a45c-xxxx"}]' + game.kruise.io/network-status: '{"currentNetworkState":"Ready","createTime":null,"lastTransitionTime":null}' + game.kruise.io/network-trigger-time: "2025-07-18 15:38:21" + game.kruise.io/network-type: HwCloud-EIP + game.kruise.io/opsState-last-changed-time: "2025-07-18 15:38:21" + game.kruise.io/state-last-changed-time: "2025-07-18 15:38:31" + lifecycle.apps.kruise.io/timestamp: "2025-07-18T07:38:13Z" + yangtse.io/allocated-eip-id: 7ec474aa-3bd9-46a2-a45c-xxxx + yangtse.io/allocated-ipv4-eip: 159.138.21.xxx + yangtse.io/eip-id: 7ec474aa-3bd9-46a2-a45c-xxxx + creationTimestamp: "2025-07-18T07:38:14Z +# other info ignored +``` ## 获取网络信息 GameServer Network Status可以通过两种方式获取 diff --git a/go.mod b/go.mod index 0cf99b8..9e9b64e 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/onsi/gomega v1.32.0 github.com/openkruise/kruise-api v1.8.0 github.com/prometheus/client_golang v1.18.0 - github.com/stretchr/testify v1.8.4 + github.com/stretchr/testify v1.10.0 golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e google.golang.org/grpc v1.58.3 google.golang.org/protobuf v1.33.0 @@ -65,6 +65,7 @@ require ( github.com/prometheus/procfs v0.12.0 // indirect github.com/rogpeppe/go-internal v1.10.0 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/stretchr/objx v0.5.2 // indirect go.uber.org/multierr v1.11.0 // indirect go.uber.org/zap v1.26.0 // indirect golang.org/x/mod v0.17.0 // indirect diff --git a/go.sum b/go.sum index 81c62cb..2db60bc 100644 --- a/go.sum +++ b/go.sum @@ -133,13 +133,15 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= +github.com/stretchr/objx v0.5.2 h1:xuMeJ0Sdp5ZMRXx/aWO6RZxdr3beISkG5/G/aIRr3pY= +github.com/stretchr/objx v0.5.2/go.mod h1:FRsXN1f5AsAjCGJKqEizvkpNtU+EGNCLh3NxZ/8L+MA= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= -github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= +github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY=