diff --git a/cloudprovider/options/tencentcloud_options.go b/cloudprovider/options/tencentcloud_options.go index e0ff8cd..f93bf1c 100644 --- a/cloudprovider/options/tencentcloud_options.go +++ b/cloudprovider/options/tencentcloud_options.go @@ -1,29 +1,13 @@ package options type TencentCloudOptions struct { - Enable bool `toml:"enable"` - CLBOptions TencentCloudCLBOptions `toml:"clb"` -} - -type TencentCloudCLBOptions struct { - MaxPort int32 `toml:"max_port"` - MinPort int32 `toml:"min_port"` -} - -func (o TencentCloudOptions) Valid() bool { - clbOptions := o.CLBOptions - - if clbOptions.MaxPort > 65535 { - return false - } - - if clbOptions.MinPort < 1 { - return false - } - - return true + Enable bool `toml:"enable"` } func (o TencentCloudOptions) Enabled() bool { return o.Enable } + +func (o TencentCloudOptions) Valid() bool { + return true +} diff --git a/cloudprovider/tencentcloud/apis/v1alpha1/dedicatedclblistener_types.go b/cloudprovider/tencentcloud/apis/v1alpha1/dedicatedclblistener_types.go deleted file mode 100644 index fd390bf..0000000 --- a/cloudprovider/tencentcloud/apis/v1alpha1/dedicatedclblistener_types.go +++ /dev/null @@ -1,93 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package v1alpha1 - -import ( - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" -) - -// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN! -// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized. - -// DedicatedCLBListenerSpec defines the desired state of DedicatedCLBListener -type DedicatedCLBListenerSpec struct { - // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable" - LbId string `json:"lbId"` - // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable" - // +optional - LbRegion string `json:"lbRegion,omitempty"` - // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable" - LbPort int64 `json:"lbPort"` - // +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable" - // +kubebuilder:validation:Enum=TCP;UDP - Protocol string `json:"protocol"` - // +optional - ExtensiveParameters string `json:"extensiveParameters,omitempty"` - // +optional - TargetPod *TargetPod `json:"targetPod,omitempty"` -} - -type TargetPod struct { - PodName string `json:"podName"` - TargetPort int64 `json:"targetPort"` -} - -// DedicatedCLBListenerStatus defines the observed state of DedicatedCLBListener -type DedicatedCLBListenerStatus struct { - ListenerId string `json:"listenerId,omitempty"` - // +kubebuilder:validation:Enum=Bound;Available;Pending;Failed;Deleting - State string `json:"state,omitempty"` - Message string `json:"message,omitempty"` - Address string `json:"address,omitempty"` -} - -const ( - DedicatedCLBListenerStateBound = "Bound" - DedicatedCLBListenerStateAvailable = "Available" - DedicatedCLBListenerStatePending = "Pending" - DedicatedCLBListenerStateFailed = "Failed" - DedicatedCLBListenerStateDeleting = "Deleting" -) - -// +kubebuilder:object:root=true -// +kubebuilder:subresource:status -// +kubebuilder:printcolumn:name="LbId",type="string",JSONPath=".spec.lbId",description="CLB ID" -// +kubebuilder:printcolumn:name="LbPort",type="integer",JSONPath=".spec.lbPort",description="Port of CLB Listener" -// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.targetPod.podName",description="Pod name of target pod" -// +kubebuilder:printcolumn:name="State",type="string",JSONPath=".status.state",description="State of the dedicated clb listener" - -// DedicatedCLBListener is the Schema for the dedicatedclblisteners API -type DedicatedCLBListener struct { - metav1.TypeMeta `json:",inline"` - metav1.ObjectMeta `json:"metadata,omitempty"` - - Spec DedicatedCLBListenerSpec `json:"spec,omitempty"` - Status DedicatedCLBListenerStatus `json:"status,omitempty"` -} - -// +kubebuilder:object:root=true - -// DedicatedCLBListenerList contains a list of DedicatedCLBListener -type DedicatedCLBListenerList struct { - metav1.TypeMeta `json:",inline"` - metav1.ListMeta `json:"metadata,omitempty"` - Items []DedicatedCLBListener `json:"items"` -} - -func init() { - SchemeBuilder.Register(&DedicatedCLBListener{}, &DedicatedCLBListenerList{}) -} diff --git a/cloudprovider/tencentcloud/apis/v1alpha1/doc.go b/cloudprovider/tencentcloud/apis/v1alpha1/doc.go deleted file mode 100644 index ac49e5d..0000000 --- a/cloudprovider/tencentcloud/apis/v1alpha1/doc.go +++ /dev/null @@ -1,4 +0,0 @@ -// Package v1alpha1 contains API Schema definitions for the tencentcloud v1alpha1 API group -// +k8s:deepcopy-gen=package,register -// +groupName=networking.cloud.tencent.com -package v1alpha1 diff --git a/cloudprovider/tencentcloud/apis/v1alpha1/register.go b/cloudprovider/tencentcloud/apis/v1alpha1/register.go deleted file mode 100644 index 86e8445..0000000 --- a/cloudprovider/tencentcloud/apis/v1alpha1/register.go +++ /dev/null @@ -1,38 +0,0 @@ -/* -Copyright 2024. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Package v1alpha1 contains API Schema definitions for the networking v1alpha1 API group -// +kubebuilder:validation:Required -// +kubebuilder:object:generate=true -// +groupName=networking.cloud.tencent.com - -package v1alpha1 - -import ( - "k8s.io/apimachinery/pkg/runtime/schema" - "sigs.k8s.io/controller-runtime/pkg/scheme" -) - -var ( - // GroupVersion is group version used to register these objects - GroupVersion = schema.GroupVersion{Group: "networking.cloud.tencent.com", Version: "v1alpha1"} - - // SchemeBuilder is used to add go types to the GroupVersionKind scheme - SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion} - - // AddToScheme adds the types in this group-version to the given scheme. - AddToScheme = SchemeBuilder.AddToScheme -) diff --git a/cloudprovider/tencentcloud/apis/v1alpha1/zz_generated.deepcopy.go b/cloudprovider/tencentcloud/apis/v1alpha1/zz_generated.deepcopy.go deleted file mode 100644 index 64eb451..0000000 --- a/cloudprovider/tencentcloud/apis/v1alpha1/zz_generated.deepcopy.go +++ /dev/null @@ -1,134 +0,0 @@ -//go:build !ignore_autogenerated - -/* -Copyright 2022 The Kruise Authors. - -Licensed under the Apache License, Version 2.0 (the "License"); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an "AS IS" BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -// Code generated by controller-gen. DO NOT EDIT. - -package v1alpha1 - -import ( - runtime "k8s.io/apimachinery/pkg/runtime" -) - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DedicatedCLBListener) DeepCopyInto(out *DedicatedCLBListener) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ObjectMeta.DeepCopyInto(&out.ObjectMeta) - in.Spec.DeepCopyInto(&out.Spec) - out.Status = in.Status -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListener. -func (in *DedicatedCLBListener) DeepCopy() *DedicatedCLBListener { - if in == nil { - return nil - } - out := new(DedicatedCLBListener) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *DedicatedCLBListener) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DedicatedCLBListenerList) DeepCopyInto(out *DedicatedCLBListenerList) { - *out = *in - out.TypeMeta = in.TypeMeta - in.ListMeta.DeepCopyInto(&out.ListMeta) - if in.Items != nil { - in, out := &in.Items, &out.Items - *out = make([]DedicatedCLBListener, len(*in)) - for i := range *in { - (*in)[i].DeepCopyInto(&(*out)[i]) - } - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerList. -func (in *DedicatedCLBListenerList) DeepCopy() *DedicatedCLBListenerList { - if in == nil { - return nil - } - out := new(DedicatedCLBListenerList) - in.DeepCopyInto(out) - return out -} - -// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object. -func (in *DedicatedCLBListenerList) DeepCopyObject() runtime.Object { - if c := in.DeepCopy(); c != nil { - return c - } - return nil -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DedicatedCLBListenerSpec) DeepCopyInto(out *DedicatedCLBListenerSpec) { - *out = *in - if in.TargetPod != nil { - in, out := &in.TargetPod, &out.TargetPod - *out = new(TargetPod) - **out = **in - } -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerSpec. -func (in *DedicatedCLBListenerSpec) DeepCopy() *DedicatedCLBListenerSpec { - if in == nil { - return nil - } - out := new(DedicatedCLBListenerSpec) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *DedicatedCLBListenerStatus) DeepCopyInto(out *DedicatedCLBListenerStatus) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerStatus. -func (in *DedicatedCLBListenerStatus) DeepCopy() *DedicatedCLBListenerStatus { - if in == nil { - return nil - } - out := new(DedicatedCLBListenerStatus) - in.DeepCopyInto(out) - return out -} - -// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. -func (in *TargetPod) DeepCopyInto(out *TargetPod) { - *out = *in -} - -// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetPod. -func (in *TargetPod) DeepCopy() *TargetPod { - if in == nil { - return nil - } - out := new(TargetPod) - in.DeepCopyInto(out) - return out -} diff --git a/cloudprovider/tencentcloud/clb.go b/cloudprovider/tencentcloud/clb.go index c82375e..445fb30 100644 --- a/cloudprovider/tencentcloud/clb.go +++ b/cloudprovider/tencentcloud/clb.go @@ -2,49 +2,36 @@ package tencentcloud import ( "context" + "encoding/json" "fmt" - "reflect" - "slices" "strconv" "strings" - "sync" + + "github.com/openkruise/kruise-game/pkg/util" + + apierrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/util/intstr" kruisev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" "github.com/openkruise/kruise-game/cloudprovider" cperrors "github.com/openkruise/kruise-game/cloudprovider/errors" - provideroptions "github.com/openkruise/kruise-game/cloudprovider/options" - "github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1" "github.com/openkruise/kruise-game/cloudprovider/utils" - "github.com/openkruise/kruise-game/pkg/util" corev1 "k8s.io/api/core/v1" - "k8s.io/apimachinery/pkg/api/errors" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/util/intstr" - log "k8s.io/klog/v2" - "k8s.io/utils/ptr" "sigs.k8s.io/controller-runtime/pkg/client" ) const ( - ClbNetwork = "TencentCloud-CLB" - AliasCLB = "CLB-Network" - ClbIdsConfigName = "ClbIds" - PortProtocolsConfigName = "PortProtocols" - MinPortConfigName = "MinPort" - MaxPortConfigName = "MaxPort" - OwnerPodKey = "game.kruise.io/owner-pod" - TargetPortKey = "game.kruise.io/target-port" + ClbNetwork = "TencentCloud-CLB" + AliasCLB = "CLB-Network" + ClbIdsConfigName = "ClbIds" + PortProtocolsConfigName = "PortProtocols" + CLBPortMappingAnnotation = "networking.cloud.tencent.com/clb-port-mapping" + EnableCLBPortMappingAnnotation = "networking.cloud.tencent.com/enable-clb-port-mapping" + CLBPortMappingResultAnnotation = "networking.cloud.tencent.com/clb-port-mapping-result" + CLBPortMappingStatuslAnnotation = "networking.cloud.tencent.com/clb-port-mapping-status" ) -type portAllocated map[int32]bool - -type ClbPlugin struct { - maxPort int32 - minPort int32 - cache map[string]portAllocated - podAllocate map[string][]string - mutex sync.RWMutex -} +type ClbPlugin struct{} type portProtocol struct { port int @@ -52,10 +39,15 @@ type portProtocol struct { } type clbConfig struct { - lbIds []string targetPorts []portProtocol } +type portMapping struct { + Port int `json:"port"` + Protocol string `json:"protocol"` + Address string `json:"address"` +} + func (p *ClbPlugin) Name() string { return ClbNetwork } @@ -65,83 +57,27 @@ func (p *ClbPlugin) Alias() string { } func (p *ClbPlugin) Init(c client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error { - p.mutex.Lock() - defer p.mutex.Unlock() - clbOptions := options.(provideroptions.TencentCloudOptions).CLBOptions - p.minPort = clbOptions.MinPort - p.maxPort = clbOptions.MaxPort - - listenerList := &v1alpha1.DedicatedCLBListenerList{} - err := c.List(ctx, listenerList) - if err != nil { - return err - } - p.cache, p.podAllocate = initLbCache(listenerList.Items, p.minPort, p.maxPort) - log.Infof("[%s] podAllocate cache complete initialization: %v", ClbNetwork, p.podAllocate) return nil } -func initLbCache(listenerList []v1alpha1.DedicatedCLBListener, minPort, maxPort int32) (map[string]portAllocated, map[string][]string) { - newCache := make(map[string]portAllocated) - newPodAllocate := make(map[string][]string) - for _, lis := range listenerList { - podName, exist := lis.GetLabels()[OwnerPodKey] - if !exist || podName == "" { - continue - } - if lis.Spec.LbPort > int64(maxPort) || lis.Spec.LbPort < int64(minPort) { - continue - } - lbId := lis.Spec.LbId - if newCache[lbId] == nil { - newCache[lbId] = make(portAllocated, maxPort-minPort) - for i := minPort; i < maxPort; i++ { - newCache[lbId][i] = false - } - } - newCache[lbId][int32(lis.Spec.LbPort)] = true - podKey := lis.GetNamespace() + "/" + podName - newPodAllocate[podKey] = append(newPodAllocate[podKey], fmt.Sprintf("%s:%d", lbId, lis.Spec.LbPort)) - } - return newCache, newPodAllocate -} - func (p *ClbPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { - return pod, nil -} - -func (p *ClbPlugin) deleteListener(ctx context.Context, c client.Client, lis *v1alpha1.DedicatedCLBListener) cperrors.PluginError { - err := c.Delete(ctx, lis) - if err != nil { - return cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) - } - if pm := p.cache[lis.Spec.LbId]; pm != nil { - pm[int32(lis.Spec.LbPort)] = false - } - var podName string - if targetPod := lis.Spec.TargetPod; targetPod != nil { - podName = targetPod.PodName - } else if lis.Labels != nil && lis.Labels[TargetPortKey] != "" && lis.Labels[OwnerPodKey] != "" { - podName = lis.Labels[OwnerPodKey] - } else { - return nil - } - target := fmt.Sprintf("%s/%d", lis.Spec.LbId, lis.Spec.LbPort) - p.podAllocate[podName] = slices.DeleteFunc(p.podAllocate[podName], func(el string) bool { - return el == target - }) - return nil + return p.reconcile(c, pod, ctx) } func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { if pod.DeletionTimestamp != nil { return pod, nil } + return p.reconcile(c, pod, ctx) +} + +// Ensure the annotation of pod is correct. +func (p *ClbPlugin) reconcile(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { networkManager := utils.NewNetworkManager(pod, c) networkStatus, _ := networkManager.GetNetworkStatus() if networkStatus == nil { pod, err := networkManager.UpdateNetworkStatus(kruisev1alpha1.NetworkStatus{ - CurrentNetworkState: kruisev1alpha1.NetworkNotReady, + CurrentNetworkState: kruisev1alpha1.NetworkWaiting, }, pod) return pod, cperrors.ToPluginError(err, cperrors.InternalError) } @@ -150,263 +86,101 @@ func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C if err != nil { return pod, cperrors.ToPluginError(err, cperrors.ParameterError) } - gss, err := util.GetGameServerSetOfPod(pod, c, ctx) - if err != nil && !errors.IsNotFound(err) { + if err != nil && !apierrors.IsNotFound(err) { return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) } - - // get related dedicated clb listeners - listeners := &v1alpha1.DedicatedCLBListenerList{} - if err := c.List( - ctx, listeners, - client.InNamespace(pod.Namespace), - client.MatchingLabels{ - OwnerPodKey: pod.Name, - kruisev1alpha1.GameServerOwnerGssKey: gss.Name, - }, - ); err != nil { - return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + if pod.Annotations == nil { + pod.Annotations = make(map[string]string) } - - // reconcile - lisMap := make(map[portProtocol]v1alpha1.DedicatedCLBListener) - - for _, lis := range listeners.Items { - // ignore deleting dedicated clb listener - if lis.DeletionTimestamp != nil { - continue - } - // old dedicated clb listener remain - if lis.OwnerReferences[0].Kind == "Pod" && lis.OwnerReferences[0].UID != pod.UID { - log.Infof("[%s] waitting old dedicated clb listener %s/%s deleted. old owner pod uid is %s, but now is %s", ClbNetwork, lis.Namespace, lis.Name, lis.OwnerReferences[0].UID, pod.UID) - return pod, nil - } - - targetPod := lis.Spec.TargetPod - if targetPod != nil && targetPod.PodName == pod.Name { - port := portProtocol{ - port: int(targetPod.TargetPort), - protocol: lis.Spec.Protocol, - } - lisMap[port] = lis - } else if targetPod == nil && (lis.Labels != nil && lis.Labels[TargetPortKey] != "") { - targetPort, err := strconv.Atoi(lis.Labels[TargetPortKey]) - if err != nil { - log.Warningf("[%s] invalid dedicated clb listener target port annotation %s/%s: %s", ClbNetwork, lis.Namespace, lis.Name, err.Error()) - continue - } - port := portProtocol{ - port: targetPort, - protocol: lis.Spec.Protocol, - } - // lower priority than targetPod is not nil - if _, exists := lisMap[port]; !exists { - lisMap[port] = lis - } - } + pod.Annotations[CLBPortMappingAnnotation] = getClbPortMappingAnnotation(clbConf, gss) + enableCLBPortMapping := "true" + if networkManager.GetNetworkDisabled() { + enableCLBPortMapping = "false" } - - internalAddresses := make([]kruisev1alpha1.NetworkAddress, 0) - externalAddresses := make([]kruisev1alpha1.NetworkAddress, 0) - - for _, port := range clbConf.targetPorts { - if lis, ok := lisMap[port]; !ok { // no dedicated clb listener, try to create one - if networkManager.GetNetworkDisabled() { - continue - } - // ensure not ready while creating the listener - networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady - pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) - if err != nil { - return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) - } - // allocate and create new listener bound to pod - newLis, err := p.consLis(clbConf, pod, port, gss.Name) - if err != nil { + pod.Annotations[EnableCLBPortMappingAnnotation] = enableCLBPortMapping + if pod.Annotations[CLBPortMappingStatuslAnnotation] == "Ready" { + if result := pod.Annotations[CLBPortMappingResultAnnotation]; result != "" { + mappings := []portMapping{} + if err := json.Unmarshal([]byte(result), &mappings); err != nil { return pod, cperrors.ToPluginError(err, cperrors.InternalError) } - err = c.Create(ctx, newLis) - if err != nil { - return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) - } - } else { // already created dedicated clb listener bound to pod - delete(lisMap, port) - if networkManager.GetNetworkDisabled() { // disable network - // deregister pod if networkDisabled is true - if lis.Spec.TargetPod != nil { - lis.Spec.TargetPod = nil - err = c.Update(ctx, &lis) + if len(mappings) != 0 { + internalAddresses := make([]kruisev1alpha1.NetworkAddress, 0) + externalAddresses := make([]kruisev1alpha1.NetworkAddress, 0) + for _, mapping := range mappings { + ss := strings.Split(mapping.Address, ":") + if len(ss) != 2 { + continue + } + lbIP := ss[0] + lbPort, err := strconv.Atoi(ss[1]) if err != nil { - return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) + continue } - } - } else { // enable network - if lis.Spec.TargetPod == nil { // ensure target pod is bound to dedicated clb listener - lis.Spec.TargetPod = &v1alpha1.TargetPod{ - PodName: pod.Name, - TargetPort: int64(port.port), - } - err = c.Update(ctx, &lis) + port := mapping.Port + instrIPort := intstr.FromInt(port) + instrEPort := intstr.FromInt(lbPort) + portName := instrIPort.String() + protocol := corev1.Protocol(mapping.Protocol) + internalAddresses = append(internalAddresses, kruisev1alpha1.NetworkAddress{ + IP: pod.Status.PodIP, + Ports: []kruisev1alpha1.NetworkPort{ + { + Name: portName, + Port: &instrIPort, + Protocol: protocol, + }, + }, + }) + externalAddresses = append(externalAddresses, kruisev1alpha1.NetworkAddress{ + IP: lbIP, + Ports: []kruisev1alpha1.NetworkPort{ + { + Name: portName, + Port: &instrEPort, + Protocol: protocol, + }, + }, + }) + networkStatus.InternalAddresses = internalAddresses + networkStatus.ExternalAddresses = externalAddresses + networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) if err != nil { - return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) - } - } else { - // recreate dedicated clb listener if necessary (config changed) - if !slices.Contains(clbConf.lbIds, lis.Spec.LbId) || lis.Spec.LbPort > int64(p.maxPort) || lis.Spec.LbPort < int64(p.minPort) || lis.Spec.Protocol != port.protocol || lis.Spec.TargetPod.TargetPort != int64(port.port) { - // ensure not ready while recreating the listener - networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady - pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) - if err != nil { - return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) - } - - // delete old listener - err := p.deleteListener(ctx, c, &lis) - if err != nil { - return pod, err - } - - // allocate and create new listener bound to pod - if newLis, err := p.consLis(clbConf, pod, port, gss.Name); err != nil { - return pod, cperrors.ToPluginError(err, cperrors.InternalError) - } else { - err := c.Create(ctx, newLis) - if err != nil { - return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) - } - } - } else { // dedicated clb listener is desired, check status - if lis.Status.State == v1alpha1.DedicatedCLBListenerStateBound && lis.Status.Address != "" { // network ready - ss := strings.Split(lis.Status.Address, ":") - if len(ss) != 2 { - return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener address %s", lis.Status.Address)) - } - lbPort, err := strconv.Atoi(ss[1]) - if err != nil { - return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener port %s", ss[1])) - } - instrIPort := intstr.FromInt(int(port.port)) - instrEPort := intstr.FromInt(lbPort) - internalAddresses = append(internalAddresses, kruisev1alpha1.NetworkAddress{ - IP: pod.Status.PodIP, - Ports: []kruisev1alpha1.NetworkPort{ - { - Name: instrIPort.String(), - Port: &instrIPort, - Protocol: corev1.Protocol(port.protocol), - }, - }, - }) - externalAddresses = append(externalAddresses, kruisev1alpha1.NetworkAddress{ - IP: ss[0], - Ports: []kruisev1alpha1.NetworkPort{ - { - Name: instrIPort.String(), - Port: &instrEPort, - Protocol: corev1.Protocol(port.protocol), - }, - }, - }) - } else { // network not ready - networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady - pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) - if err != nil { - return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) - } - } + return pod, cperrors.ToPluginError(err, cperrors.InternalError) } } } } } - - // other dedicated clb listener is not used, delete it - for _, lis := range lisMap { - err := p.deleteListener(ctx, c, &lis) - if err != nil { - return pod, err - } - } - - // set network status to ready when all lb port is ready - if len(externalAddresses) == len(clbConf.targetPorts) { - // change network status to ready if necessary - if !reflect.DeepEqual(externalAddresses, networkStatus.ExternalAddresses) || networkStatus.CurrentNetworkState != kruisev1alpha1.NetworkReady { - networkStatus.InternalAddresses = internalAddresses - networkStatus.ExternalAddresses = externalAddresses - networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkReady - pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) - if err != nil { - return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) - } - } - } - return pod, nil } func (p *ClbPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError { - p.deAllocate(pod.GetNamespace() + "/" + pod.GetName()) return nil } -func (p *ClbPlugin) consLis(clbConf *clbConfig, pod *corev1.Pod, port portProtocol, gssName string) (*v1alpha1.DedicatedCLBListener, error) { - lbId, lbPort := p.allocate(clbConf.lbIds, pod.GetNamespace()+"/"+pod.GetName()) - if lbId == "" { - return nil, fmt.Errorf("there are no avaialable ports for %v", clbConf.lbIds) - } - lis := &v1alpha1.DedicatedCLBListener{ - ObjectMeta: metav1.ObjectMeta{ - GenerateName: pod.Name + "-", - Namespace: pod.Namespace, - Labels: map[string]string{ - OwnerPodKey: pod.Name, // used to select pod related dedicated clb listener - TargetPortKey: strconv.Itoa(port.port), // used to recover clb pod binding when networkDisabled set from true to false - kruisev1alpha1.GameServerOwnerGssKey: gssName, - }, - OwnerReferences: []metav1.OwnerReference{ - { - APIVersion: pod.APIVersion, - Kind: pod.Kind, - Name: pod.GetName(), - UID: pod.GetUID(), - Controller: ptr.To[bool](true), - BlockOwnerDeletion: ptr.To[bool](true), - }, - }, - }, - Spec: v1alpha1.DedicatedCLBListenerSpec{ - LbId: lbId, - LbPort: int64(lbPort), - Protocol: port.protocol, - TargetPod: &v1alpha1.TargetPod{ - PodName: pod.Name, - TargetPort: int64(port.port), - }, - }, - } - return lis, nil -} - func init() { - clbPlugin := ClbPlugin{ - mutex: sync.RWMutex{}, - } + clbPlugin := ClbPlugin{} tencentCloudProvider.registerPlugin(&clbPlugin) } +func getClbPortMappingAnnotation(clbConf *clbConfig, gss *kruisev1alpha1.GameServerSet) string { + poolName := fmt.Sprintf("%s-%s", gss.Namespace, gss.Name) + var buf strings.Builder + for _, pp := range clbConf.targetPorts { + buf.WriteString(fmt.Sprintf("%d %s %s\n", pp.port, pp.protocol, poolName)) + } + return buf.String() +} + +var ErrMissingPortProtocolsConfig = fmt.Errorf("missing %s config", PortProtocolsConfigName) + func parseLbConfig(conf []kruisev1alpha1.NetworkConfParams) (*clbConfig, error) { - var lbIds []string ports := []portProtocol{} for _, c := range conf { switch c.Name { - case ClbIdsConfigName: - for _, clbId := range strings.Split(c.Value, ",") { - if clbId != "" { - lbIds = append(lbIds, clbId) - } - } case PortProtocolsConfigName: for _, pp := range strings.Split(c.Value, ",") { ppSlice := strings.Split(pp, "/") @@ -425,67 +199,10 @@ func parseLbConfig(conf []kruisev1alpha1.NetworkConfParams) (*clbConfig, error) } } } + if len(ports) == 0 { + return nil, ErrMissingPortProtocolsConfig + } return &clbConfig{ - lbIds: lbIds, targetPorts: ports, }, nil } - -func (p *ClbPlugin) allocate(lbIds []string, podKey string) (string, int32) { - p.mutex.Lock() - defer p.mutex.Unlock() - - var lbId string - var port int32 - - // find avaialable port - for _, clbId := range lbIds { - for i := p.minPort; i < p.maxPort; i++ { - if !p.cache[clbId][i] { - lbId = clbId - port = i - break - } - } - } - // update cache - if lbId != "" { - if p.cache[lbId] == nil { // init lb cache if not exist - p.cache[lbId] = make(portAllocated, p.maxPort-p.minPort) - for i := p.minPort; i < p.maxPort; i++ { - p.cache[lbId][i] = false - } - } - p.cache[lbId][port] = true - p.podAllocate[podKey] = append(p.podAllocate[podKey], fmt.Sprintf("%s:%d", lbId, port)) - log.Infof("pod %s allocate clb %s port %d", podKey, lbId, port) - } - return lbId, port -} - -func (p *ClbPlugin) deAllocate(podKey string) { - p.mutex.Lock() - defer p.mutex.Unlock() - - allocatedPorts, exist := p.podAllocate[podKey] - if !exist { - return - } - - for _, port := range allocatedPorts { - ss := strings.Split(port, ":") - if len(ss) != 2 { - log.Errorf("bad allocated port cache format %s", port) - continue - } - lbId := ss[0] - lbPort, err := strconv.Atoi(ss[1]) - if err != nil { - log.Errorf("failed to parse allocated port %s: %s", port, err.Error()) - continue - } - p.cache[lbId][int32(lbPort)] = false - log.Infof("pod %s deallocate clb %s ports %d", podKey, lbId, lbPort) - } - delete(p.podAllocate, podKey) -} diff --git a/cloudprovider/tencentcloud/clb_test.go b/cloudprovider/tencentcloud/clb_test.go index b502113..dc7c1ef 100644 --- a/cloudprovider/tencentcloud/clb_test.go +++ b/cloudprovider/tencentcloud/clb_test.go @@ -2,50 +2,11 @@ package tencentcloud import ( "reflect" - "sync" "testing" kruisev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" - "github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1" - metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -func TestAllocateDeAllocate(t *testing.T) { - test := struct { - lbIds []string - clb *ClbPlugin - podKey string - }{ - lbIds: []string{"lb-xxx"}, - clb: &ClbPlugin{ - maxPort: int32(712), - minPort: int32(512), - cache: make(map[string]portAllocated), - podAllocate: make(map[string][]string), - mutex: sync.RWMutex{}, - }, - podKey: "xxx/xxx", - } - - lbId, port := test.clb.allocate(test.lbIds, test.podKey) - if _, exist := test.clb.podAllocate[test.podKey]; !exist { - t.Errorf("podAllocate[%s] is empty after allocated", test.podKey) - } - if port > test.clb.maxPort || port < test.clb.minPort { - t.Errorf("allocate port %d, unexpected", port) - } - if test.clb.cache[lbId][port] == false { - t.Errorf("Allocate port %d failed", port) - } - test.clb.deAllocate(test.podKey) - if test.clb.cache[lbId][port] == true { - t.Errorf("deAllocate port %d failed", port) - } - if _, exist := test.clb.podAllocate[test.podKey]; exist { - t.Errorf("podAllocate[%s] is not empty after deallocated", test.podKey) - } -} - func TestParseLbConfig(t *testing.T) { tests := []struct { conf []kruisev1alpha1.NetworkConfParams @@ -63,7 +24,6 @@ func TestParseLbConfig(t *testing.T) { }, }, clbConfig: &clbConfig{ - lbIds: []string{"xxx-A"}, targetPorts: []portProtocol{ { port: 80, @@ -84,7 +44,6 @@ func TestParseLbConfig(t *testing.T) { }, }, clbConfig: &clbConfig{ - lbIds: []string{"xxx-A", "xxx-B"}, targetPorts: []portProtocol{ { port: 81, @@ -113,78 +72,3 @@ func TestParseLbConfig(t *testing.T) { } } } - -func TestInitLbCache(t *testing.T) { - test := struct { - listenerList []v1alpha1.DedicatedCLBListener - minPort int32 - maxPort int32 - cache map[string]portAllocated - podAllocate map[string][]string - }{ - minPort: 512, - maxPort: 712, - cache: map[string]portAllocated{ - "xxx-A": map[int32]bool{ - 666: true, - }, - "xxx-B": map[int32]bool{ - 555: true, - }, - }, - podAllocate: map[string][]string{ - "ns-0/name-0": {"xxx-A:666"}, - "ns-1/name-1": {"xxx-B:555"}, - }, - listenerList: []v1alpha1.DedicatedCLBListener{ - { - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - OwnerPodKey: "name-0", - }, - Namespace: "ns-0", - Name: "name-0-xxx", - }, - Spec: v1alpha1.DedicatedCLBListenerSpec{ - LbId: "xxx-A", - LbPort: 666, - Protocol: "TCP", - TargetPod: &v1alpha1.TargetPod{ - PodName: "name-0", - TargetPort: 80, - }, - }, - }, - { - ObjectMeta: metav1.ObjectMeta{ - Labels: map[string]string{ - OwnerPodKey: "name-1", - }, - Namespace: "ns-1", - Name: "name-1-xxx", - }, - Spec: v1alpha1.DedicatedCLBListenerSpec{ - LbId: "xxx-B", - LbPort: 555, - Protocol: "TCP", - TargetPod: &v1alpha1.TargetPod{ - PodName: "name-1", - TargetPort: 80, - }, - }, - }, - }, - } - - actualCache, actualPodAllocate := initLbCache(test.listenerList, test.minPort, test.maxPort) - for lb, pa := range test.cache { - for port, isAllocated := range pa { - if actualCache[lb][port] != isAllocated { - t.Errorf("lb %s port %d isAllocated, expect: %t, actual: %t", lb, port, isAllocated, actualCache[lb][port]) - } - } - } - if !reflect.DeepEqual(actualPodAllocate, test.podAllocate) { - t.Errorf("podAllocate expect %v, but actully got %v", test.podAllocate, actualPodAllocate) - } -} diff --git a/main.go b/main.go index 8f72cca..461f3a5 100644 --- a/main.go +++ b/main.go @@ -44,7 +44,6 @@ import ( "github.com/openkruise/kruise-game/cloudprovider" aliv1beta1 "github.com/openkruise/kruise-game/cloudprovider/alibabacloud/apis/v1beta1" cpmanager "github.com/openkruise/kruise-game/cloudprovider/manager" - tencentv1alpha1 "github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1" kruisegameclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" kruisegamevisions "github.com/openkruise/kruise-game/pkg/client/informers/externalversions" controller "github.com/openkruise/kruise-game/pkg/controllers" @@ -72,7 +71,6 @@ func init() { utilruntime.Must(kruiseV1alpha1.AddToScheme(scheme)) utilruntime.Must(aliv1beta1.AddToScheme(scheme)) - utilruntime.Must(tencentv1alpha1.AddToScheme(scheme)) utilruntime.Must(ackv1alpha1.AddToScheme(scheme)) utilruntime.Must(elbv2api.AddToScheme(scheme))