升级 TencentCloud-CLB 插件 (#239)

* upgrade tencentcloud clb plugin

* deprecate DedicatedCLBListener CRD
* use CLBPortPool's pod annotation

Signed-off-by: roc <roc@imroc.cc>

* add comments

Signed-off-by: roc <roc@imroc.cc>

---------

Signed-off-by: roc <roc@imroc.cc>

cloudprovider/options/tencentcloud_options.go

@@ -2,28 +2,12 @@ package options
 
 type TencentCloudOptions struct {
 	Enable bool `toml:"enable"`
-	CLBOptions TencentCloudCLBOptions `toml:"clb"`
-}
-
-type TencentCloudCLBOptions struct {
-	MaxPort int32 `toml:"max_port"`
-	MinPort int32 `toml:"min_port"`
-}
-
-func (o TencentCloudOptions) Valid() bool {
-	clbOptions := o.CLBOptions
-
-	if clbOptions.MaxPort > 65535 {
-		return false
-	}
-
-	if clbOptions.MinPort < 1 {
-		return false
-	}
-
-	return true
 }
 
 func (o TencentCloudOptions) Enabled() bool {
 	return o.Enable
 }
+
+func (o TencentCloudOptions) Valid() bool {
+	return true
+}

cloudprovider/tencentcloud/apis/v1alpha1/dedicatedclblistener_types.go

@@ -1,93 +0,0 @@
-/*
-Copyright 2024.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package v1alpha1
-
-import (
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-)
-
-// EDIT THIS FILE!  THIS IS SCAFFOLDING FOR YOU TO OWN!
-// NOTE: json tags are required.  Any new fields you add must have json tags for the fields to be serialized.
-
-// DedicatedCLBListenerSpec defines the desired state of DedicatedCLBListener
-type DedicatedCLBListenerSpec struct {
-	// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
-	LbId string `json:"lbId"`
-	// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
-	// +optional
-	LbRegion string `json:"lbRegion,omitempty"`
-	// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
-	LbPort int64 `json:"lbPort"`
-	// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
-	// +kubebuilder:validation:Enum=TCP;UDP
-	Protocol string `json:"protocol"`
-	// +optional
-	ExtensiveParameters string `json:"extensiveParameters,omitempty"`
-	// +optional
-	TargetPod *TargetPod `json:"targetPod,omitempty"`
-}
-
-type TargetPod struct {
-	PodName    string `json:"podName"`
-	TargetPort int64  `json:"targetPort"`
-}
-
-// DedicatedCLBListenerStatus defines the observed state of DedicatedCLBListener
-type DedicatedCLBListenerStatus struct {
-	ListenerId string `json:"listenerId,omitempty"`
-	// +kubebuilder:validation:Enum=Bound;Available;Pending;Failed;Deleting
-	State   string `json:"state,omitempty"`
-	Message string `json:"message,omitempty"`
-	Address string `json:"address,omitempty"`
-}
-
-const (
-	DedicatedCLBListenerStateBound     = "Bound"
-	DedicatedCLBListenerStateAvailable = "Available"
-	DedicatedCLBListenerStatePending   = "Pending"
-	DedicatedCLBListenerStateFailed    = "Failed"
-	DedicatedCLBListenerStateDeleting  = "Deleting"
-)
-
-// +kubebuilder:object:root=true
-// +kubebuilder:subresource:status
-// +kubebuilder:printcolumn:name="LbId",type="string",JSONPath=".spec.lbId",description="CLB ID"
-// +kubebuilder:printcolumn:name="LbPort",type="integer",JSONPath=".spec.lbPort",description="Port of CLB Listener"
-// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.targetPod.podName",description="Pod name of target pod"
-// +kubebuilder:printcolumn:name="State",type="string",JSONPath=".status.state",description="State of the dedicated clb listener"
-
-// DedicatedCLBListener is the Schema for the dedicatedclblisteners API
-type DedicatedCLBListener struct {
-	metav1.TypeMeta   `json:",inline"`
-	metav1.ObjectMeta `json:"metadata,omitempty"`
-
-	Spec   DedicatedCLBListenerSpec   `json:"spec,omitempty"`
-	Status DedicatedCLBListenerStatus `json:"status,omitempty"`
-}
-
-// +kubebuilder:object:root=true
-
-// DedicatedCLBListenerList contains a list of DedicatedCLBListener
-type DedicatedCLBListenerList struct {
-	metav1.TypeMeta `json:",inline"`
-	metav1.ListMeta `json:"metadata,omitempty"`
-	Items           []DedicatedCLBListener `json:"items"`
-}
-
-func init() {
-	SchemeBuilder.Register(&DedicatedCLBListener{}, &DedicatedCLBListenerList{})
-}

cloudprovider/tencentcloud/apis/v1alpha1/doc.go

@@ -1,4 +0,0 @@
-// Package v1alpha1 contains API Schema definitions for the tencentcloud v1alpha1 API group
-// +k8s:deepcopy-gen=package,register
-// +groupName=networking.cloud.tencent.com
-package v1alpha1

cloudprovider/tencentcloud/apis/v1alpha1/register.go

@@ -1,38 +0,0 @@
-/*
-Copyright 2024.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Package v1alpha1 contains API Schema definitions for the networking v1alpha1 API group
-// +kubebuilder:validation:Required
-// +kubebuilder:object:generate=true
-// +groupName=networking.cloud.tencent.com
-
-package v1alpha1
-
-import (
-	"k8s.io/apimachinery/pkg/runtime/schema"
-	"sigs.k8s.io/controller-runtime/pkg/scheme"
-)
-
-var (
-	// GroupVersion is group version used to register these objects
-	GroupVersion = schema.GroupVersion{Group: "networking.cloud.tencent.com", Version: "v1alpha1"}
-
-	// SchemeBuilder is used to add go types to the GroupVersionKind scheme
-	SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
-
-	// AddToScheme adds the types in this group-version to the given scheme.
-	AddToScheme = SchemeBuilder.AddToScheme
-)

cloudprovider/tencentcloud/apis/v1alpha1/zz_generated.deepcopy.go

@@ -1,134 +0,0 @@
-//go:build !ignore_autogenerated
-
-/*
-Copyright 2022 The Kruise Authors.
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-// Code generated by controller-gen. DO NOT EDIT.
-
-package v1alpha1
-
-import (
-	runtime "k8s.io/apimachinery/pkg/runtime"
-)
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DedicatedCLBListener) DeepCopyInto(out *DedicatedCLBListener) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
-	in.Spec.DeepCopyInto(&out.Spec)
-	out.Status = in.Status
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListener.
-func (in *DedicatedCLBListener) DeepCopy() *DedicatedCLBListener {
-	if in == nil {
-		return nil
-	}
-	out := new(DedicatedCLBListener)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *DedicatedCLBListener) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DedicatedCLBListenerList) DeepCopyInto(out *DedicatedCLBListenerList) {
-	*out = *in
-	out.TypeMeta = in.TypeMeta
-	in.ListMeta.DeepCopyInto(&out.ListMeta)
-	if in.Items != nil {
-		in, out := &in.Items, &out.Items
-		*out = make([]DedicatedCLBListener, len(*in))
-		for i := range *in {
-			(*in)[i].DeepCopyInto(&(*out)[i])
-		}
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerList.
-func (in *DedicatedCLBListenerList) DeepCopy() *DedicatedCLBListenerList {
-	if in == nil {
-		return nil
-	}
-	out := new(DedicatedCLBListenerList)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
-func (in *DedicatedCLBListenerList) DeepCopyObject() runtime.Object {
-	if c := in.DeepCopy(); c != nil {
-		return c
-	}
-	return nil
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DedicatedCLBListenerSpec) DeepCopyInto(out *DedicatedCLBListenerSpec) {
-	*out = *in
-	if in.TargetPod != nil {
-		in, out := &in.TargetPod, &out.TargetPod
-		*out = new(TargetPod)
-		**out = **in
-	}
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerSpec.
-func (in *DedicatedCLBListenerSpec) DeepCopy() *DedicatedCLBListenerSpec {
-	if in == nil {
-		return nil
-	}
-	out := new(DedicatedCLBListenerSpec)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *DedicatedCLBListenerStatus) DeepCopyInto(out *DedicatedCLBListenerStatus) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerStatus.
-func (in *DedicatedCLBListenerStatus) DeepCopy() *DedicatedCLBListenerStatus {
-	if in == nil {
-		return nil
-	}
-	out := new(DedicatedCLBListenerStatus)
-	in.DeepCopyInto(out)
-	return out
-}
-
-// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
-func (in *TargetPod) DeepCopyInto(out *TargetPod) {
-	*out = *in
-}
-
-// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetPod.
-func (in *TargetPod) DeepCopy() *TargetPod {
-	if in == nil {
-		return nil
-	}
-	out := new(TargetPod)
-	in.DeepCopyInto(out)
-	return out
-}

cloudprovider/tencentcloud/clb.go

@@ -2,26 +2,21 @@ package tencentcloud
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
-	"reflect"
-	"slices"
 	"strconv"
 	"strings"
-	"sync"
+
+	"github.com/openkruise/kruise-game/pkg/util"
+
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/util/intstr"
 
 	kruisev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
 	"github.com/openkruise/kruise-game/cloudprovider"
 	cperrors "github.com/openkruise/kruise-game/cloudprovider/errors"
-	provideroptions "github.com/openkruise/kruise-game/cloudprovider/options"
-	"github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1"
 	"github.com/openkruise/kruise-game/cloudprovider/utils"
-	"github.com/openkruise/kruise-game/pkg/util"
 	corev1 "k8s.io/api/core/v1"
-	"k8s.io/apimachinery/pkg/api/errors"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
-	"k8s.io/apimachinery/pkg/util/intstr"
-	log "k8s.io/klog/v2"
-	"k8s.io/utils/ptr"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
 
@@ -30,21 +25,13 @@ const (
 	AliasCLB                        = "CLB-Network"
 	ClbIdsConfigName                = "ClbIds"
 	PortProtocolsConfigName         = "PortProtocols"
-	MinPortConfigName       = "MinPort"
+	CLBPortMappingAnnotation        = "networking.cloud.tencent.com/clb-port-mapping"
-	MaxPortConfigName       = "MaxPort"
+	EnableCLBPortMappingAnnotation  = "networking.cloud.tencent.com/enable-clb-port-mapping"
-	OwnerPodKey             = "game.kruise.io/owner-pod"
+	CLBPortMappingResultAnnotation  = "networking.cloud.tencent.com/clb-port-mapping-result"
-	TargetPortKey           = "game.kruise.io/target-port"
+	CLBPortMappingStatuslAnnotation = "networking.cloud.tencent.com/clb-port-mapping-status"
 )
 
-type portAllocated map[int32]bool
+type ClbPlugin struct{}
-
-type ClbPlugin struct {
-	maxPort     int32
-	minPort     int32
-	cache       map[string]portAllocated
-	podAllocate map[string][]string
-	mutex       sync.RWMutex
-}
 
 type portProtocol struct {
 	port     int
@@ -52,10 +39,15 @@ type portProtocol struct {
 }
 
 type clbConfig struct {
-	lbIds       []string
 	targetPorts []portProtocol
 }
 
+type portMapping struct {
+	Port     int    `json:"port"`
+	Protocol string `json:"protocol"`
+	Address  string `json:"address"`
+}
+
 func (p *ClbPlugin) Name() string {
 	return ClbNetwork
 }
@@ -65,83 +57,27 @@ func (p *ClbPlugin) Alias() string {
 }
 
 func (p *ClbPlugin) Init(c client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error {
-	p.mutex.Lock()
-	defer p.mutex.Unlock()
-	clbOptions := options.(provideroptions.TencentCloudOptions).CLBOptions
-	p.minPort = clbOptions.MinPort
-	p.maxPort = clbOptions.MaxPort
-
-	listenerList := &v1alpha1.DedicatedCLBListenerList{}
-	err := c.List(ctx, listenerList)
-	if err != nil {
-		return err
-	}
-	p.cache, p.podAllocate = initLbCache(listenerList.Items, p.minPort, p.maxPort)
-	log.Infof("[%s] podAllocate cache complete initialization: %v", ClbNetwork, p.podAllocate)
 	return nil
 }
 
-func initLbCache(listenerList []v1alpha1.DedicatedCLBListener, minPort, maxPort int32) (map[string]portAllocated, map[string][]string) {
-	newCache := make(map[string]portAllocated)
-	newPodAllocate := make(map[string][]string)
-	for _, lis := range listenerList {
-		podName, exist := lis.GetLabels()[OwnerPodKey]
-		if !exist || podName == "" {
-			continue
-		}
-		if lis.Spec.LbPort > int64(maxPort) || lis.Spec.LbPort < int64(minPort) {
-			continue
-		}
-		lbId := lis.Spec.LbId
-		if newCache[lbId] == nil {
-			newCache[lbId] = make(portAllocated, maxPort-minPort)
-			for i := minPort; i < maxPort; i++ {
-				newCache[lbId][i] = false
-			}
-		}
-		newCache[lbId][int32(lis.Spec.LbPort)] = true
-		podKey := lis.GetNamespace() + "/" + podName
-		newPodAllocate[podKey] = append(newPodAllocate[podKey], fmt.Sprintf("%s:%d", lbId, lis.Spec.LbPort))
-	}
-	return newCache, newPodAllocate
-}
-
 func (p *ClbPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
-	return pod, nil
+	return p.reconcile(c, pod, ctx)
-}
-
-func (p *ClbPlugin) deleteListener(ctx context.Context, c client.Client, lis *v1alpha1.DedicatedCLBListener) cperrors.PluginError {
-	err := c.Delete(ctx, lis)
-	if err != nil {
-		return cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
-	}
-	if pm := p.cache[lis.Spec.LbId]; pm != nil {
-		pm[int32(lis.Spec.LbPort)] = false
-	}
-	var podName string
-	if targetPod := lis.Spec.TargetPod; targetPod != nil {
-		podName = targetPod.PodName
-	} else if lis.Labels != nil && lis.Labels[TargetPortKey] != "" && lis.Labels[OwnerPodKey] != "" {
-		podName = lis.Labels[OwnerPodKey]
-	} else {
-		return nil
-	}
-	target := fmt.Sprintf("%s/%d", lis.Spec.LbId, lis.Spec.LbPort)
-	p.podAllocate[podName] = slices.DeleteFunc(p.podAllocate[podName], func(el string) bool {
-		return el == target
-	})
-	return nil
 }
 
 func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
 	if pod.DeletionTimestamp != nil {
 		return pod, nil
 	}
+	return p.reconcile(c, pod, ctx)
+}
+
+// Ensure the annotation of pod is correct.
+func (p *ClbPlugin) reconcile(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
 	networkManager := utils.NewNetworkManager(pod, c)
 	networkStatus, _ := networkManager.GetNetworkStatus()
 	if networkStatus == nil {
 		pod, err := networkManager.UpdateNetworkStatus(kruisev1alpha1.NetworkStatus{
-			CurrentNetworkState: kruisev1alpha1.NetworkNotReady,
+			CurrentNetworkState: kruisev1alpha1.NetworkWaiting,
 		}, pod)
 		return pod, cperrors.ToPluginError(err, cperrors.InternalError)
 	}
@@ -150,263 +86,101 @@ func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C
 	if err != nil {
 		return pod, cperrors.ToPluginError(err, cperrors.ParameterError)
 	}
-
 	gss, err := util.GetGameServerSetOfPod(pod, c, ctx)
-	if err != nil && !errors.IsNotFound(err) {
+	if err != nil && !apierrors.IsNotFound(err) {
 		return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
 	}
-
+	if pod.Annotations == nil {
-	// get related dedicated clb listeners
+		pod.Annotations = make(map[string]string)
-	listeners := &v1alpha1.DedicatedCLBListenerList{}
-	if err := c.List(
-		ctx, listeners,
-		client.InNamespace(pod.Namespace),
-		client.MatchingLabels{
-			OwnerPodKey:                          pod.Name,
-			kruisev1alpha1.GameServerOwnerGssKey: gss.Name,
-		},
-	); err != nil {
-		return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
 	}
-
+	pod.Annotations[CLBPortMappingAnnotation] = getClbPortMappingAnnotation(clbConf, gss)
-	// reconcile
+	enableCLBPortMapping := "true"
-	lisMap := make(map[portProtocol]v1alpha1.DedicatedCLBListener)
+	if networkManager.GetNetworkDisabled() {
-
+		enableCLBPortMapping = "false"
-	for _, lis := range listeners.Items {
-		// ignore deleting dedicated clb listener
-		if lis.DeletionTimestamp != nil {
-			continue
 	}
-		// old dedicated clb listener remain
+	pod.Annotations[EnableCLBPortMappingAnnotation] = enableCLBPortMapping
-		if lis.OwnerReferences[0].Kind == "Pod" && lis.OwnerReferences[0].UID != pod.UID {
+	if pod.Annotations[CLBPortMappingStatuslAnnotation] == "Ready" {
-			log.Infof("[%s] waitting old dedicated clb listener %s/%s deleted. old owner pod uid is %s, but now is %s", ClbNetwork, lis.Namespace, lis.Name, lis.OwnerReferences[0].UID, pod.UID)
+		if result := pod.Annotations[CLBPortMappingResultAnnotation]; result != "" {
-			return pod, nil
+			mappings := []portMapping{}
+			if err := json.Unmarshal([]byte(result), &mappings); err != nil {
+				return pod, cperrors.ToPluginError(err, cperrors.InternalError)
 			}
-
+			if len(mappings) != 0 {
-		targetPod := lis.Spec.TargetPod
-		if targetPod != nil && targetPod.PodName == pod.Name {
-			port := portProtocol{
-				port:     int(targetPod.TargetPort),
-				protocol: lis.Spec.Protocol,
-			}
-			lisMap[port] = lis
-		} else if targetPod == nil && (lis.Labels != nil && lis.Labels[TargetPortKey] != "") {
-			targetPort, err := strconv.Atoi(lis.Labels[TargetPortKey])
-			if err != nil {
-				log.Warningf("[%s] invalid dedicated clb listener target port annotation %s/%s: %s", ClbNetwork, lis.Namespace, lis.Name, err.Error())
-				continue
-			}
-			port := portProtocol{
-				port:     targetPort,
-				protocol: lis.Spec.Protocol,
-			}
-			// lower priority than targetPod is not nil
-			if _, exists := lisMap[port]; !exists {
-				lisMap[port] = lis
-			}
-		}
-	}
-
 				internalAddresses := make([]kruisev1alpha1.NetworkAddress, 0)
 				externalAddresses := make([]kruisev1alpha1.NetworkAddress, 0)
-
+				for _, mapping := range mappings {
-	for _, port := range clbConf.targetPorts {
+					ss := strings.Split(mapping.Address, ":")
-		if lis, ok := lisMap[port]; !ok { // no dedicated clb listener, try to create one
+					if len(ss) != 2 {
-			if networkManager.GetNetworkDisabled() {
 						continue
 					}
-			// ensure not ready while creating the listener
+					lbIP := ss[0]
-			networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
-			pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
-			if err != nil {
-				return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
-			}
-			// allocate and create new listener bound to pod
-			newLis, err := p.consLis(clbConf, pod, port, gss.Name)
-			if err != nil {
-				return pod, cperrors.ToPluginError(err, cperrors.InternalError)
-			}
-			err = c.Create(ctx, newLis)
-			if err != nil {
-				return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
-			}
-		} else { // already created dedicated clb listener bound to pod
-			delete(lisMap, port)
-			if networkManager.GetNetworkDisabled() { // disable network
-				// deregister pod if networkDisabled is true
-				if lis.Spec.TargetPod != nil {
-					lis.Spec.TargetPod = nil
-					err = c.Update(ctx, &lis)
-					if err != nil {
-						return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
-					}
-				}
-			} else { // enable network
-				if lis.Spec.TargetPod == nil { // ensure target pod is bound to dedicated clb listener
-					lis.Spec.TargetPod = &v1alpha1.TargetPod{
-						PodName:    pod.Name,
-						TargetPort: int64(port.port),
-					}
-					err = c.Update(ctx, &lis)
-					if err != nil {
-						return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
-					}
-				} else {
-					//  recreate dedicated clb listener if necessary (config changed)
-					if !slices.Contains(clbConf.lbIds, lis.Spec.LbId) || lis.Spec.LbPort > int64(p.maxPort) || lis.Spec.LbPort < int64(p.minPort) || lis.Spec.Protocol != port.protocol || lis.Spec.TargetPod.TargetPort != int64(port.port) {
-						// ensure not ready while recreating the listener
-						networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
-						pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
-						if err != nil {
-							return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
-						}
-
-						// delete old listener
-						err := p.deleteListener(ctx, c, &lis)
-						if err != nil {
-							return pod, err
-						}
-
-						// allocate and create new listener bound to pod
-						if newLis, err := p.consLis(clbConf, pod, port, gss.Name); err != nil {
-							return pod, cperrors.ToPluginError(err, cperrors.InternalError)
-						} else {
-							err := c.Create(ctx, newLis)
-							if err != nil {
-								return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
-							}
-						}
-					} else { // dedicated clb listener is desired, check status
-						if lis.Status.State == v1alpha1.DedicatedCLBListenerStateBound && lis.Status.Address != "" { // network ready
-							ss := strings.Split(lis.Status.Address, ":")
-							if len(ss) != 2 {
-								return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener address %s", lis.Status.Address))
-							}
 					lbPort, err := strconv.Atoi(ss[1])
 					if err != nil {
-								return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener port %s", ss[1]))
+						continue
 					}
-							instrIPort := intstr.FromInt(int(port.port))
+					port := mapping.Port
+					instrIPort := intstr.FromInt(port)
 					instrEPort := intstr.FromInt(lbPort)
+					portName := instrIPort.String()
+					protocol := corev1.Protocol(mapping.Protocol)
 					internalAddresses = append(internalAddresses, kruisev1alpha1.NetworkAddress{
 						IP: pod.Status.PodIP,
 						Ports: []kruisev1alpha1.NetworkPort{
 							{
-										Name:     instrIPort.String(),
+								Name:     portName,
 								Port:     &instrIPort,
-										Protocol: corev1.Protocol(port.protocol),
+								Protocol: protocol,
 							},
 						},
 					})
 					externalAddresses = append(externalAddresses, kruisev1alpha1.NetworkAddress{
-								IP: ss[0],
+						IP: lbIP,
 						Ports: []kruisev1alpha1.NetworkPort{
 							{
-										Name:     instrIPort.String(),
+								Name:     portName,
 								Port:     &instrEPort,
-										Protocol: corev1.Protocol(port.protocol),
+								Protocol: protocol,
 							},
 						},
 					})
-						} else { // network not ready
-							networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
-							pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
-							if err != nil {
-								return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
-							}
-						}
-					}
-				}
-			}
-		}
-	}
-
-	// other dedicated clb listener is not used, delete it
-	for _, lis := range lisMap {
-		err := p.deleteListener(ctx, c, &lis)
-		if err != nil {
-			return pod, err
-		}
-	}
-
-	// set network status to ready when all lb port is ready
-	if len(externalAddresses) == len(clbConf.targetPorts) {
-		// change network status to ready if necessary
-		if !reflect.DeepEqual(externalAddresses, networkStatus.ExternalAddresses) || networkStatus.CurrentNetworkState != kruisev1alpha1.NetworkReady {
 					networkStatus.InternalAddresses = internalAddresses
 					networkStatus.ExternalAddresses = externalAddresses
 					networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkReady
 					pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
 					if err != nil {
-				return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
+						return pod, cperrors.ToPluginError(err, cperrors.InternalError)
+					}
+				}
 			}
 		}
 	}
-
 	return pod, nil
 }
 
 func (p *ClbPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError {
-	p.deAllocate(pod.GetNamespace() + "/" + pod.GetName())
 	return nil
 }
 
-func (p *ClbPlugin) consLis(clbConf *clbConfig, pod *corev1.Pod, port portProtocol, gssName string) (*v1alpha1.DedicatedCLBListener, error) {
-	lbId, lbPort := p.allocate(clbConf.lbIds, pod.GetNamespace()+"/"+pod.GetName())
-	if lbId == "" {
-		return nil, fmt.Errorf("there are no avaialable ports for %v", clbConf.lbIds)
-	}
-	lis := &v1alpha1.DedicatedCLBListener{
-		ObjectMeta: metav1.ObjectMeta{
-			GenerateName: pod.Name + "-",
-			Namespace:    pod.Namespace,
-			Labels: map[string]string{
-				OwnerPodKey:                          pod.Name,                // used to select pod related dedicated clb listener
-				TargetPortKey:                        strconv.Itoa(port.port), // used to recover clb pod binding when networkDisabled set from true to false
-				kruisev1alpha1.GameServerOwnerGssKey: gssName,
-			},
-			OwnerReferences: []metav1.OwnerReference{
-				{
-					APIVersion:         pod.APIVersion,
-					Kind:               pod.Kind,
-					Name:               pod.GetName(),
-					UID:                pod.GetUID(),
-					Controller:         ptr.To[bool](true),
-					BlockOwnerDeletion: ptr.To[bool](true),
-				},
-			},
-		},
-		Spec: v1alpha1.DedicatedCLBListenerSpec{
-			LbId:     lbId,
-			LbPort:   int64(lbPort),
-			Protocol: port.protocol,
-			TargetPod: &v1alpha1.TargetPod{
-				PodName:    pod.Name,
-				TargetPort: int64(port.port),
-			},
-		},
-	}
-	return lis, nil
-}
-
 func init() {
-	clbPlugin := ClbPlugin{
+	clbPlugin := ClbPlugin{}
-		mutex: sync.RWMutex{},
-	}
 	tencentCloudProvider.registerPlugin(&clbPlugin)
 }
 
+func getClbPortMappingAnnotation(clbConf *clbConfig, gss *kruisev1alpha1.GameServerSet) string {
+	poolName := fmt.Sprintf("%s-%s", gss.Namespace, gss.Name)
+	var buf strings.Builder
+	for _, pp := range clbConf.targetPorts {
+		buf.WriteString(fmt.Sprintf("%d %s %s\n", pp.port, pp.protocol, poolName))
+	}
+	return buf.String()
+}
+
+var ErrMissingPortProtocolsConfig = fmt.Errorf("missing %s config", PortProtocolsConfigName)
+
 func parseLbConfig(conf []kruisev1alpha1.NetworkConfParams) (*clbConfig, error) {
-	var lbIds []string
 	ports := []portProtocol{}
 	for _, c := range conf {
 		switch c.Name {
-		case ClbIdsConfigName:
-			for _, clbId := range strings.Split(c.Value, ",") {
-				if clbId != "" {
-					lbIds = append(lbIds, clbId)
-				}
-			}
 		case PortProtocolsConfigName:
 			for _, pp := range strings.Split(c.Value, ",") {
 				ppSlice := strings.Split(pp, "/")
@@ -425,67 +199,10 @@ func parseLbConfig(conf []kruisev1alpha1.NetworkConfParams) (*clbConfig, error)
 			}
 		}
 	}
+	if len(ports) == 0 {
+		return nil, ErrMissingPortProtocolsConfig
+	}
 	return &clbConfig{
-		lbIds:       lbIds,
 		targetPorts: ports,
 	}, nil
 }
-
-func (p *ClbPlugin) allocate(lbIds []string, podKey string) (string, int32) {
-	p.mutex.Lock()
-	defer p.mutex.Unlock()
-
-	var lbId string
-	var port int32
-
-	// find avaialable port
-	for _, clbId := range lbIds {
-		for i := p.minPort; i < p.maxPort; i++ {
-			if !p.cache[clbId][i] {
-				lbId = clbId
-				port = i
-				break
-			}
-		}
-	}
-	// update cache
-	if lbId != "" {
-		if p.cache[lbId] == nil { // init lb cache if not exist
-			p.cache[lbId] = make(portAllocated, p.maxPort-p.minPort)
-			for i := p.minPort; i < p.maxPort; i++ {
-				p.cache[lbId][i] = false
-			}
-		}
-		p.cache[lbId][port] = true
-		p.podAllocate[podKey] = append(p.podAllocate[podKey], fmt.Sprintf("%s:%d", lbId, port))
-		log.Infof("pod %s allocate clb %s port %d", podKey, lbId, port)
-	}
-	return lbId, port
-}
-
-func (p *ClbPlugin) deAllocate(podKey string) {
-	p.mutex.Lock()
-	defer p.mutex.Unlock()
-
-	allocatedPorts, exist := p.podAllocate[podKey]
-	if !exist {
-		return
-	}
-
-	for _, port := range allocatedPorts {
-		ss := strings.Split(port, ":")
-		if len(ss) != 2 {
-			log.Errorf("bad allocated port cache format %s", port)
-			continue
-		}
-		lbId := ss[0]
-		lbPort, err := strconv.Atoi(ss[1])
-		if err != nil {
-			log.Errorf("failed to parse allocated port %s: %s", port, err.Error())
-			continue
-		}
-		p.cache[lbId][int32(lbPort)] = false
-		log.Infof("pod %s deallocate clb %s ports %d", podKey, lbId, lbPort)
-	}
-	delete(p.podAllocate, podKey)
-}

cloudprovider/tencentcloud/clb_test.go

@@ -2,50 +2,11 @@ package tencentcloud
 
 import (
 	"reflect"
-	"sync"
 	"testing"
 
 	kruisev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
-	"github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1"
-	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
-func TestAllocateDeAllocate(t *testing.T) {
-	test := struct {
-		lbIds  []string
-		clb    *ClbPlugin
-		podKey string
-	}{
-		lbIds: []string{"lb-xxx"},
-		clb: &ClbPlugin{
-			maxPort:     int32(712),
-			minPort:     int32(512),
-			cache:       make(map[string]portAllocated),
-			podAllocate: make(map[string][]string),
-			mutex:       sync.RWMutex{},
-		},
-		podKey: "xxx/xxx",
-	}
-
-	lbId, port := test.clb.allocate(test.lbIds, test.podKey)
-	if _, exist := test.clb.podAllocate[test.podKey]; !exist {
-		t.Errorf("podAllocate[%s] is empty after allocated", test.podKey)
-	}
-	if port > test.clb.maxPort || port < test.clb.minPort {
-		t.Errorf("allocate port %d, unexpected", port)
-	}
-	if test.clb.cache[lbId][port] == false {
-		t.Errorf("Allocate port %d failed", port)
-	}
-	test.clb.deAllocate(test.podKey)
-	if test.clb.cache[lbId][port] == true {
-		t.Errorf("deAllocate port %d failed", port)
-	}
-	if _, exist := test.clb.podAllocate[test.podKey]; exist {
-		t.Errorf("podAllocate[%s] is not empty after deallocated", test.podKey)
-	}
-}
-
 func TestParseLbConfig(t *testing.T) {
 	tests := []struct {
 		conf      []kruisev1alpha1.NetworkConfParams
@@ -63,7 +24,6 @@ func TestParseLbConfig(t *testing.T) {
 				},
 			},
 			clbConfig: &clbConfig{
-				lbIds: []string{"xxx-A"},
 				targetPorts: []portProtocol{
 					{
 						port:     80,
@@ -84,7 +44,6 @@ func TestParseLbConfig(t *testing.T) {
 				},
 			},
 			clbConfig: &clbConfig{
-				lbIds: []string{"xxx-A", "xxx-B"},
 				targetPorts: []portProtocol{
 					{
 						port:     81,
@@ -113,78 +72,3 @@ func TestParseLbConfig(t *testing.T) {
 		}
 	}
 }
-
-func TestInitLbCache(t *testing.T) {
-	test := struct {
-		listenerList []v1alpha1.DedicatedCLBListener
-		minPort      int32
-		maxPort      int32
-		cache        map[string]portAllocated
-		podAllocate  map[string][]string
-	}{
-		minPort: 512,
-		maxPort: 712,
-		cache: map[string]portAllocated{
-			"xxx-A": map[int32]bool{
-				666: true,
-			},
-			"xxx-B": map[int32]bool{
-				555: true,
-			},
-		},
-		podAllocate: map[string][]string{
-			"ns-0/name-0": {"xxx-A:666"},
-			"ns-1/name-1": {"xxx-B:555"},
-		},
-		listenerList: []v1alpha1.DedicatedCLBListener{
-			{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						OwnerPodKey: "name-0",
-					},
-					Namespace: "ns-0",
-					Name:      "name-0-xxx",
-				},
-				Spec: v1alpha1.DedicatedCLBListenerSpec{
-					LbId:     "xxx-A",
-					LbPort:   666,
-					Protocol: "TCP",
-					TargetPod: &v1alpha1.TargetPod{
-						PodName:    "name-0",
-						TargetPort: 80,
-					},
-				},
-			},
-			{
-				ObjectMeta: metav1.ObjectMeta{
-					Labels: map[string]string{
-						OwnerPodKey: "name-1",
-					},
-					Namespace: "ns-1",
-					Name:      "name-1-xxx",
-				},
-				Spec: v1alpha1.DedicatedCLBListenerSpec{
-					LbId:     "xxx-B",
-					LbPort:   555,
-					Protocol: "TCP",
-					TargetPod: &v1alpha1.TargetPod{
-						PodName:    "name-1",
-						TargetPort: 80,
-					},
-				},
-			},
-		},
-	}
-
-	actualCache, actualPodAllocate := initLbCache(test.listenerList, test.minPort, test.maxPort)
-	for lb, pa := range test.cache {
-		for port, isAllocated := range pa {
-			if actualCache[lb][port] != isAllocated {
-				t.Errorf("lb %s port %d isAllocated, expect: %t, actual: %t", lb, port, isAllocated, actualCache[lb][port])
-			}
-		}
-	}
-	if !reflect.DeepEqual(actualPodAllocate, test.podAllocate) {
-		t.Errorf("podAllocate expect %v, but actully got %v", test.podAllocate, actualPodAllocate)
-	}
-}

main.go

@@ -44,7 +44,6 @@ import (
 	"github.com/openkruise/kruise-game/cloudprovider"
 	aliv1beta1 "github.com/openkruise/kruise-game/cloudprovider/alibabacloud/apis/v1beta1"
 	cpmanager "github.com/openkruise/kruise-game/cloudprovider/manager"
-	tencentv1alpha1 "github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1"
 	kruisegameclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned"
 	kruisegamevisions "github.com/openkruise/kruise-game/pkg/client/informers/externalversions"
 	controller "github.com/openkruise/kruise-game/pkg/controllers"
@@ -72,7 +71,6 @@ func init() {
 	utilruntime.Must(kruiseV1alpha1.AddToScheme(scheme))
 
 	utilruntime.Must(aliv1beta1.AddToScheme(scheme))
-	utilruntime.Must(tencentv1alpha1.AddToScheme(scheme))
 
 	utilruntime.Must(ackv1alpha1.AddToScheme(scheme))
 	utilruntime.Must(elbv2api.AddToScheme(scheme))