feat(*): add tencent cloud provider and clb plugin (#179)

* feat(*): add tencent cloud provider and clb plugin

Signed-off-by: rockerchen <rockerchen@tencent.com>
This commit is contained in:
roc 2024-10-29 14:13:11 +08:00 committed by GitHub
parent e121bcc109
commit c680b411b7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
14 changed files with 1332 additions and 6 deletions

View File

@ -17,13 +17,13 @@ limitations under the License.
package cloudprovider package cloudprovider
import ( import (
"flag"
"github.com/BurntSushi/toml" "github.com/BurntSushi/toml"
"github.com/openkruise/kruise-game/cloudprovider/options" "github.com/openkruise/kruise-game/cloudprovider/options"
"k8s.io/klog/v2" "k8s.io/klog/v2"
) )
import "flag"
var Opt *Options var Opt *Options
type Options struct { type Options struct {
@ -47,6 +47,7 @@ type CloudProviderConfig struct {
AlibabaCloudOptions CloudProviderOptions AlibabaCloudOptions CloudProviderOptions
VolcengineOptions CloudProviderOptions VolcengineOptions CloudProviderOptions
AmazonsWebServicesOptions CloudProviderOptions AmazonsWebServicesOptions CloudProviderOptions
TencentCloudOptions CloudProviderOptions
} }
type tomlConfigs struct { type tomlConfigs struct {
@ -54,10 +55,10 @@ type tomlConfigs struct {
AlibabaCloud options.AlibabaCloudOptions `toml:"alibabacloud"` AlibabaCloud options.AlibabaCloudOptions `toml:"alibabacloud"`
Volcengine options.VolcengineOptions `toml:"volcengine"` Volcengine options.VolcengineOptions `toml:"volcengine"`
AmazonsWebServices options.AmazonsWebServicesOptions `toml:"aws"` AmazonsWebServices options.AmazonsWebServicesOptions `toml:"aws"`
TencentCloud options.TencentCloudOptions `toml:"tencentcloud"`
} }
func (cf *ConfigFile) Parse() *CloudProviderConfig { func (cf *ConfigFile) Parse() *CloudProviderConfig {
var config tomlConfigs var config tomlConfigs
if _, err := toml.DecodeFile(cf.Path, &config); err != nil { if _, err := toml.DecodeFile(cf.Path, &config); err != nil {
klog.Fatal(err) klog.Fatal(err)
@ -68,6 +69,7 @@ func (cf *ConfigFile) Parse() *CloudProviderConfig {
AlibabaCloudOptions: config.AlibabaCloud, AlibabaCloudOptions: config.AlibabaCloud,
VolcengineOptions: config.Volcengine, VolcengineOptions: config.Volcengine,
AmazonsWebServicesOptions: config.AmazonsWebServices, AmazonsWebServicesOptions: config.AmazonsWebServices,
TencentCloudOptions: config.TencentCloud,
} }
} }

View File

@ -18,11 +18,13 @@ package manager
import ( import (
"context" "context"
"github.com/openkruise/kruise-game/apis/v1alpha1" "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider" "github.com/openkruise/kruise-game/cloudprovider"
"github.com/openkruise/kruise-game/cloudprovider/alibabacloud" "github.com/openkruise/kruise-game/cloudprovider/alibabacloud"
aws "github.com/openkruise/kruise-game/cloudprovider/amazonswebservices" aws "github.com/openkruise/kruise-game/cloudprovider/amazonswebservices"
"github.com/openkruise/kruise-game/cloudprovider/kubernetes" "github.com/openkruise/kruise-game/cloudprovider/kubernetes"
"github.com/openkruise/kruise-game/cloudprovider/tencentcloud"
volcengine "github.com/openkruise/kruise-game/cloudprovider/volcengine" volcengine "github.com/openkruise/kruise-game/cloudprovider/volcengine"
corev1 "k8s.io/api/core/v1" corev1 "k8s.io/api/core/v1"
log "k8s.io/klog/v2" log "k8s.io/klog/v2"
@ -138,5 +140,15 @@ func NewProviderManager() (*ProviderManager, error) {
} }
} }
if configs.TencentCloudOptions.Valid() && configs.TencentCloudOptions.Enabled() {
// build and register tencent cloud provider
tcp, err := tencentcloud.NewTencentCloudProvider()
if err != nil {
log.Errorf("Failed to initialize tencentcloud provider.because of %s", err.Error())
} else {
pm.RegisterCloudProvider(tcp, configs.TencentCloudOptions)
}
}
return pm, nil return pm, nil
} }

View File

@ -0,0 +1,29 @@
package options
type TencentCloudOptions struct {
Enable bool `toml:"enable"`
CLBOptions TencentCloudCLBOptions `toml:"clb"`
}
type TencentCloudCLBOptions struct {
MaxPort int32 `toml:"max_port"`
MinPort int32 `toml:"min_port"`
}
func (o TencentCloudOptions) Valid() bool {
clbOptions := o.CLBOptions
if clbOptions.MaxPort > 65535 {
return false
}
if clbOptions.MinPort < 1 {
return false
}
return true
}
func (o TencentCloudOptions) Enabled() bool {
return o.Enable
}

View File

@ -0,0 +1,93 @@
/*
Copyright 2024.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package v1alpha1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
// DedicatedCLBListenerSpec defines the desired state of DedicatedCLBListener
type DedicatedCLBListenerSpec struct {
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
LbId string `json:"lbId"`
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
// +optional
LbRegion string `json:"lbRegion,omitempty"`
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
LbPort int64 `json:"lbPort"`
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
// +kubebuilder:validation:Enum=TCP;UDP
Protocol string `json:"protocol"`
// +optional
ExtensiveParameters string `json:"extensiveParameters,omitempty"`
// +optional
TargetPod *TargetPod `json:"targetPod,omitempty"`
}
type TargetPod struct {
PodName string `json:"podName"`
TargetPort int64 `json:"targetPort"`
}
// DedicatedCLBListenerStatus defines the observed state of DedicatedCLBListener
type DedicatedCLBListenerStatus struct {
ListenerId string `json:"listenerId,omitempty"`
// +kubebuilder:validation:Enum=Bound;Available;Pending;Failed;Deleting
State string `json:"state,omitempty"`
Message string `json:"message,omitempty"`
Address string `json:"address,omitempty"`
}
const (
DedicatedCLBListenerStateBound = "Bound"
DedicatedCLBListenerStateAvailable = "Available"
DedicatedCLBListenerStatePending = "Pending"
DedicatedCLBListenerStateFailed = "Failed"
DedicatedCLBListenerStateDeleting = "Deleting"
)
// +kubebuilder:object:root=true
// +kubebuilder:subresource:status
// +kubebuilder:printcolumn:name="LbId",type="string",JSONPath=".spec.lbId",description="CLB ID"
// +kubebuilder:printcolumn:name="LbPort",type="integer",JSONPath=".spec.lbPort",description="Port of CLB Listener"
// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.targetPod.podName",description="Pod name of target pod"
// +kubebuilder:printcolumn:name="State",type="string",JSONPath=".status.state",description="State of the dedicated clb listener"
// DedicatedCLBListener is the Schema for the dedicatedclblisteners API
type DedicatedCLBListener struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec DedicatedCLBListenerSpec `json:"spec,omitempty"`
Status DedicatedCLBListenerStatus `json:"status,omitempty"`
}
// +kubebuilder:object:root=true
// DedicatedCLBListenerList contains a list of DedicatedCLBListener
type DedicatedCLBListenerList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []DedicatedCLBListener `json:"items"`
}
func init() {
SchemeBuilder.Register(&DedicatedCLBListener{}, &DedicatedCLBListenerList{})
}

View File

@ -0,0 +1,4 @@
// Package v1alpha1 contains API Schema definitions for the tencentcloud v1alpha1 API group
// +k8s:deepcopy-gen=package,register
// +groupName=networking.cloud.tencent.com
package v1alpha1

View File

@ -0,0 +1,38 @@
/*
Copyright 2024.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Package v1alpha1 contains API Schema definitions for the networking v1alpha1 API group
// +kubebuilder:validation:Required
// +kubebuilder:object:generate=true
// +groupName=networking.cloud.tencent.com
package v1alpha1
import (
"k8s.io/apimachinery/pkg/runtime/schema"
"sigs.k8s.io/controller-runtime/pkg/scheme"
)
var (
// GroupVersion is group version used to register these objects
GroupVersion = schema.GroupVersion{Group: "networking.cloud.tencent.com", Version: "v1alpha1"}
// SchemeBuilder is used to add go types to the GroupVersionKind scheme
SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
// AddToScheme adds the types in this group-version to the given scheme.
AddToScheme = SchemeBuilder.AddToScheme
)

View File

@ -0,0 +1,135 @@
//go:build !ignore_autogenerated
// +build !ignore_autogenerated
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
// Code generated by controller-gen. DO NOT EDIT.
package v1alpha1
import (
runtime "k8s.io/apimachinery/pkg/runtime"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DedicatedCLBListener) DeepCopyInto(out *DedicatedCLBListener) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
in.Spec.DeepCopyInto(&out.Spec)
out.Status = in.Status
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListener.
func (in *DedicatedCLBListener) DeepCopy() *DedicatedCLBListener {
if in == nil {
return nil
}
out := new(DedicatedCLBListener)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *DedicatedCLBListener) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DedicatedCLBListenerList) DeepCopyInto(out *DedicatedCLBListenerList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]DedicatedCLBListener, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerList.
func (in *DedicatedCLBListenerList) DeepCopy() *DedicatedCLBListenerList {
if in == nil {
return nil
}
out := new(DedicatedCLBListenerList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *DedicatedCLBListenerList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DedicatedCLBListenerSpec) DeepCopyInto(out *DedicatedCLBListenerSpec) {
*out = *in
if in.TargetPod != nil {
in, out := &in.TargetPod, &out.TargetPod
*out = new(TargetPod)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerSpec.
func (in *DedicatedCLBListenerSpec) DeepCopy() *DedicatedCLBListenerSpec {
if in == nil {
return nil
}
out := new(DedicatedCLBListenerSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *DedicatedCLBListenerStatus) DeepCopyInto(out *DedicatedCLBListenerStatus) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerStatus.
func (in *DedicatedCLBListenerStatus) DeepCopy() *DedicatedCLBListenerStatus {
if in == nil {
return nil
}
out := new(DedicatedCLBListenerStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *TargetPod) DeepCopyInto(out *TargetPod) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetPod.
func (in *TargetPod) DeepCopy() *TargetPod {
if in == nil {
return nil
}
out := new(TargetPod)
in.DeepCopyInto(out)
return out
}

View File

@ -0,0 +1,433 @@
package tencentcloud
import (
"context"
"fmt"
"reflect"
"slices"
"strconv"
"strings"
"sync"
kruisev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider"
cperrors "github.com/openkruise/kruise-game/cloudprovider/errors"
provideroptions "github.com/openkruise/kruise-game/cloudprovider/options"
"github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider/utils"
"github.com/openkruise/kruise-game/pkg/util"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/intstr"
log "k8s.io/klog/v2"
"k8s.io/utils/ptr"
"sigs.k8s.io/controller-runtime/pkg/client"
)
const (
ClbNetwork = "TencentCloud-CLB"
AliasCLB = "CLB-Network"
ClbIdsConfigName = "ClbIds"
PortProtocolsConfigName = "PortProtocols"
MinPortConfigName = "MinPort"
MaxPortConfigName = "MaxPort"
OwnerPodKey = "game.kruise.io/owner-pod"
)
type portAllocated map[int32]bool
type ClbPlugin struct {
maxPort int32
minPort int32
cache map[string]portAllocated
podAllocate map[string][]string
mutex sync.RWMutex
}
type portProtocol struct {
port int
protocol string
}
type clbConfig struct {
lbIds []string
targetPorts []portProtocol
}
func (p *ClbPlugin) Name() string {
return ClbNetwork
}
func (p *ClbPlugin) Alias() string {
return AliasCLB
}
func (p *ClbPlugin) Init(c client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error {
p.mutex.Lock()
defer p.mutex.Unlock()
clbOptions := options.(provideroptions.TencentCloudOptions).CLBOptions
p.minPort = clbOptions.MinPort
p.maxPort = clbOptions.MaxPort
listenerList := &v1alpha1.DedicatedCLBListenerList{}
err := c.List(ctx, listenerList)
if err != nil {
return err
}
p.cache, p.podAllocate = initLbCache(listenerList.Items, p.minPort, p.maxPort)
log.Infof("[%s] podAllocate cache complete initialization: %v", ClbNetwork, p.podAllocate)
return nil
}
func initLbCache(listenerList []v1alpha1.DedicatedCLBListener, minPort, maxPort int32) (map[string]portAllocated, map[string][]string) {
newCache := make(map[string]portAllocated)
newPodAllocate := make(map[string][]string)
for _, lis := range listenerList {
podName, exist := lis.GetLabels()[OwnerPodKey]
if !exist || podName == "" {
continue
}
if lis.Spec.LbPort > int64(maxPort) || lis.Spec.LbPort < int64(minPort) {
continue
}
lbId := lis.Spec.LbId
if newCache[lbId] == nil {
newCache[lbId] = make(portAllocated, maxPort-minPort)
for i := minPort; i < maxPort; i++ {
newCache[lbId][i] = false
}
}
newCache[lbId][int32(lis.Spec.LbPort)] = true
podKey := lis.GetNamespace() + "/" + podName
newPodAllocate[podKey] = append(newPodAllocate[podKey], fmt.Sprintf("%s:%d", lbId, lis.Spec.LbPort))
}
return newCache, newPodAllocate
}
func (p *ClbPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
return pod, nil
}
func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
if pod.DeletionTimestamp != nil {
return pod, nil
}
networkManager := utils.NewNetworkManager(pod, c)
networkStatus, _ := networkManager.GetNetworkStatus()
if networkStatus == nil {
pod, err := networkManager.UpdateNetworkStatus(kruisev1alpha1.NetworkStatus{
CurrentNetworkState: kruisev1alpha1.NetworkNotReady,
}, pod)
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
}
networkConfig := networkManager.GetNetworkConfig()
clbConf, err := parseLbConfig(networkConfig)
if err != nil {
return pod, cperrors.ToPluginError(err, cperrors.ParameterError)
}
gss, err := util.GetGameServerSetOfPod(pod, c, ctx)
if err != nil && !errors.IsNotFound(err) {
return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
}
// get related dedicated clb listeners
listeners := &v1alpha1.DedicatedCLBListenerList{}
if err := c.List(
ctx, listeners,
client.InNamespace(pod.Namespace),
client.MatchingLabels{
OwnerPodKey: pod.Name,
kruisev1alpha1.GameServerOwnerGssKey: gss.Name,
},
); err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
// reconcile
lisMap := make(map[portProtocol]v1alpha1.DedicatedCLBListener)
for _, lis := range listeners.Items {
// ignore deleting dedicated clb listener
if lis.DeletionTimestamp != nil {
continue
}
// old dedicated clb listener remain
if lis.OwnerReferences[0].Kind == "Pod" && lis.OwnerReferences[0].UID != pod.UID {
log.Infof("[%s] waitting old dedicated clb listener %s/%s deleted. old owner pod uid is %s, but now is %s", ClbNetwork, lis.Namespace, lis.Name, lis.OwnerReferences[0].UID, pod.UID)
return pod, nil
}
// delete dedicated clb listener if necessary (target pod not match)
targetPod := lis.Spec.TargetPod
if targetPod == nil || targetPod.PodName != pod.Name {
err := c.Delete(ctx, &lis)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
continue
}
port := portProtocol{
port: int(targetPod.TargetPort),
protocol: lis.Spec.Protocol,
}
lisMap[port] = lis
}
internalAddresses := make([]kruisev1alpha1.NetworkAddress, 0)
externalAddresses := make([]kruisev1alpha1.NetworkAddress, 0)
for _, port := range clbConf.targetPorts {
if lis, ok := lisMap[port]; !ok { // no dedicated clb listener, try to create one
// ensure not ready while creating the listener
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}
// allocate and create new listener bound to pod
newLis, err := p.consLis(clbConf, pod, port, gss.Name)
if err != nil {
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
}
err = c.Create(ctx, newLis)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
} else { // already created dedicated clb listener bound to pod
delete(lisMap, port)
// recreate dedicated clb listener if necessary (config changed)
if !slices.Contains(clbConf.lbIds, lis.Spec.LbId) || lis.Spec.LbPort > int64(p.maxPort) || lis.Spec.LbPort < int64(p.minPort) || lis.Spec.Protocol != port.protocol || lis.Spec.TargetPod.TargetPort != int64(port.port) {
// ensure not ready while recreating the listener
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}
// delete old listener
err := c.Delete(ctx, &lis)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
// allocate and create new listener bound to pod
newLis, err := p.consLis(clbConf, pod, port, gss.Name)
if err != nil {
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
}
err = c.Create(ctx, newLis)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
} else { // dedicated clb listener is desired, check status
if lis.Status.State == v1alpha1.DedicatedCLBListenerStateBound && lis.Status.Address != "" { // network ready
ss := strings.Split(lis.Status.Address, ":")
if len(ss) != 2 {
return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener address %s", lis.Status.Address))
}
lbPort, err := strconv.Atoi(ss[1])
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener port %s", ss[1]))
}
instrIPort := intstr.FromInt(int(port.port))
instrEPort := intstr.FromInt(lbPort)
internalAddresses = append(internalAddresses, kruisev1alpha1.NetworkAddress{
IP: pod.Status.PodIP,
Ports: []kruisev1alpha1.NetworkPort{
{
Name: instrIPort.String(),
Port: &instrIPort,
Protocol: corev1.Protocol(port.protocol),
},
},
})
externalAddresses = append(externalAddresses, kruisev1alpha1.NetworkAddress{
IP: ss[0],
Ports: []kruisev1alpha1.NetworkPort{
{
Name: instrIPort.String(),
Port: &instrEPort,
Protocol: corev1.Protocol(port.protocol),
},
},
})
} else { // network not ready
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}
}
}
}
}
// other dedicated clb listener is not used, delete it
for _, lis := range lisMap {
err := c.Delete(ctx, &lis)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
}
}
// set network status to ready when all lb port is ready
if len(externalAddresses) == len(clbConf.targetPorts) {
// change network status to ready if necessary
if !reflect.DeepEqual(externalAddresses, networkStatus.ExternalAddresses) || networkStatus.CurrentNetworkState != kruisev1alpha1.NetworkReady {
networkStatus.InternalAddresses = internalAddresses
networkStatus.ExternalAddresses = externalAddresses
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
if err != nil {
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
}
}
}
return pod, nil
}
func (p *ClbPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError {
p.deAllocate(pod.GetNamespace() + "/" + pod.GetName())
return nil
}
func (p *ClbPlugin) consLis(clbConf *clbConfig, pod *corev1.Pod, port portProtocol, gssName string) (*v1alpha1.DedicatedCLBListener, error) {
lbId, lbPort := p.allocate(clbConf.lbIds, pod.GetNamespace()+"/"+pod.GetName())
if lbId == "" {
return nil, fmt.Errorf("there are no avaialable ports for %v", clbConf.lbIds)
}
lis := &v1alpha1.DedicatedCLBListener{
ObjectMeta: metav1.ObjectMeta{
GenerateName: pod.Name + "-",
Namespace: pod.Namespace,
Labels: map[string]string{
OwnerPodKey: pod.Name,
kruisev1alpha1.GameServerOwnerGssKey: gssName,
},
OwnerReferences: []metav1.OwnerReference{
{
APIVersion: pod.APIVersion,
Kind: pod.Kind,
Name: pod.GetName(),
UID: pod.GetUID(),
Controller: ptr.To[bool](true),
BlockOwnerDeletion: ptr.To[bool](true),
},
},
},
Spec: v1alpha1.DedicatedCLBListenerSpec{
LbId: lbId,
LbPort: int64(lbPort),
Protocol: port.protocol,
TargetPod: &v1alpha1.TargetPod{
PodName: pod.Name,
TargetPort: int64(port.port),
},
},
}
return lis, nil
}
func init() {
clbPlugin := ClbPlugin{
mutex: sync.RWMutex{},
}
tencentCloudProvider.registerPlugin(&clbPlugin)
}
func parseLbConfig(conf []kruisev1alpha1.NetworkConfParams) (*clbConfig, error) {
var lbIds []string
ports := []portProtocol{}
for _, c := range conf {
switch c.Name {
case ClbIdsConfigName:
for _, clbId := range strings.Split(c.Value, ",") {
if clbId != "" {
lbIds = append(lbIds, clbId)
}
}
case PortProtocolsConfigName:
for _, pp := range strings.Split(c.Value, ",") {
ppSlice := strings.Split(pp, "/")
port, err := strconv.Atoi(ppSlice[0])
if err != nil {
continue
}
protocol := "TCP"
if len(ppSlice) == 2 {
protocol = ppSlice[1]
}
ports = append(ports, portProtocol{
port: port,
protocol: protocol,
})
}
}
}
return &clbConfig{
lbIds: lbIds,
targetPorts: ports,
}, nil
}
func (p *ClbPlugin) allocate(lbIds []string, podKey string) (string, int32) {
p.mutex.Lock()
defer p.mutex.Unlock()
var lbId string
var port int32
// find avaialable port
for _, clbId := range lbIds {
for i := p.minPort; i < p.maxPort; i++ {
if !p.cache[clbId][i] {
lbId = clbId
port = i
break
}
}
}
// update cache
if lbId != "" {
if p.cache[lbId] == nil { // init lb cache if not exist
p.cache[lbId] = make(portAllocated, p.maxPort-p.minPort)
for i := p.minPort; i < p.maxPort; i++ {
p.cache[lbId][i] = false
}
}
p.cache[lbId][port] = true
p.podAllocate[podKey] = append(p.podAllocate[podKey], fmt.Sprintf("%s:%d", lbId, port))
log.Infof("pod %s allocate clb %s port %d", podKey, lbId, port)
}
return lbId, port
}
func (p *ClbPlugin) deAllocate(podKey string) {
p.mutex.Lock()
defer p.mutex.Unlock()
allocatedPorts, exist := p.podAllocate[podKey]
if !exist {
return
}
for _, port := range allocatedPorts {
ss := strings.Split(port, ":")
if len(ss) != 2 {
log.Errorf("bad allocated port cache format %s", port)
continue
}
lbId := ss[0]
lbPort, err := strconv.Atoi(ss[1])
if err != nil {
log.Errorf("failed to parse allocated port %s: %s", port, err.Error())
continue
}
p.cache[lbId][int32(lbPort)] = false
log.Infof("pod %s deallocate clb %s ports %d", podKey, lbId, lbPort)
}
delete(p.podAllocate, podKey)
}

View File

@ -0,0 +1,190 @@
package tencentcloud
import (
"reflect"
"sync"
"testing"
kruisev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
func TestAllocateDeAllocate(t *testing.T) {
test := struct {
lbIds []string
clb *ClbPlugin
podKey string
}{
lbIds: []string{"lb-xxx"},
clb: &ClbPlugin{
maxPort: int32(712),
minPort: int32(512),
cache: make(map[string]portAllocated),
podAllocate: make(map[string][]string),
mutex: sync.RWMutex{},
},
podKey: "xxx/xxx",
}
lbId, port := test.clb.allocate(test.lbIds, test.podKey)
if _, exist := test.clb.podAllocate[test.podKey]; !exist {
t.Errorf("podAllocate[%s] is empty after allocated", test.podKey)
}
if port > test.clb.maxPort || port < test.clb.minPort {
t.Errorf("allocate port %d, unexpected", port)
}
if test.clb.cache[lbId][port] == false {
t.Errorf("Allocate port %d failed", port)
}
test.clb.deAllocate(test.podKey)
if test.clb.cache[lbId][port] == true {
t.Errorf("deAllocate port %d failed", port)
}
if _, exist := test.clb.podAllocate[test.podKey]; exist {
t.Errorf("podAllocate[%s] is not empty after deallocated", test.podKey)
}
}
func TestParseLbConfig(t *testing.T) {
tests := []struct {
conf []kruisev1alpha1.NetworkConfParams
clbConfig *clbConfig
}{
{
conf: []kruisev1alpha1.NetworkConfParams{
{
Name: ClbIdsConfigName,
Value: "xxx-A",
},
{
Name: PortProtocolsConfigName,
Value: "80",
},
},
clbConfig: &clbConfig{
lbIds: []string{"xxx-A"},
targetPorts: []portProtocol{
{
port: 80,
protocol: "TCP",
},
},
},
},
{
conf: []kruisev1alpha1.NetworkConfParams{
{
Name: ClbIdsConfigName,
Value: "xxx-A,xxx-B,",
},
{
Name: PortProtocolsConfigName,
Value: "81/UDP,82,83/TCP",
},
},
clbConfig: &clbConfig{
lbIds: []string{"xxx-A", "xxx-B"},
targetPorts: []portProtocol{
{
port: 81,
protocol: "UDP",
},
{
port: 82,
protocol: "TCP",
},
{
port: 83,
protocol: "TCP",
},
},
},
},
}
for i, test := range tests {
lc, err := parseLbConfig(test.conf)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(test.clbConfig, lc) {
t.Errorf("case %d: lbId expect: %v, actual: %v", i, test.clbConfig, lc)
}
}
}
func TestInitLbCache(t *testing.T) {
test := struct {
listenerList []v1alpha1.DedicatedCLBListener
minPort int32
maxPort int32
cache map[string]portAllocated
podAllocate map[string][]string
}{
minPort: 512,
maxPort: 712,
cache: map[string]portAllocated{
"xxx-A": map[int32]bool{
666: true,
},
"xxx-B": map[int32]bool{
555: true,
},
},
podAllocate: map[string][]string{
"ns-0/name-0": {"xxx-A:666"},
"ns-1/name-1": {"xxx-B:555"},
},
listenerList: []v1alpha1.DedicatedCLBListener{
{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
OwnerPodKey: "name-0",
},
Namespace: "ns-0",
Name: "name-0-xxx",
},
Spec: v1alpha1.DedicatedCLBListenerSpec{
LbId: "xxx-A",
LbPort: 666,
Protocol: "TCP",
TargetPod: &v1alpha1.TargetPod{
PodName: "name-0",
TargetPort: 80,
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Labels: map[string]string{
OwnerPodKey: "name-1",
},
Namespace: "ns-1",
Name: "name-1-xxx",
},
Spec: v1alpha1.DedicatedCLBListenerSpec{
LbId: "xxx-B",
LbPort: 555,
Protocol: "TCP",
TargetPod: &v1alpha1.TargetPod{
PodName: "name-1",
TargetPort: 80,
},
},
},
},
}
actualCache, actualPodAllocate := initLbCache(test.listenerList, test.minPort, test.maxPort)
for lb, pa := range test.cache {
for port, isAllocated := range pa {
if actualCache[lb][port] != isAllocated {
t.Errorf("lb %s port %d isAllocated, expect: %t, actual: %t", lb, port, isAllocated, actualCache[lb][port])
}
}
}
if !reflect.DeepEqual(actualPodAllocate, test.podAllocate) {
t.Errorf("podAllocate expect %v, but actully got %v", test.podAllocate, actualPodAllocate)
}
}

View File

@ -0,0 +1,59 @@
/*
Copyright 2022 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package tencentcloud
import (
"github.com/openkruise/kruise-game/cloudprovider"
"k8s.io/klog/v2"
)
const (
TencentCloud = "TencentCloud"
)
var tencentCloudProvider = &Provider{
plugins: make(map[string]cloudprovider.Plugin),
}
type Provider struct {
plugins map[string]cloudprovider.Plugin
}
func (ap *Provider) Name() string {
return TencentCloud
}
func (ap *Provider) ListPlugins() (map[string]cloudprovider.Plugin, error) {
if ap.plugins == nil {
return make(map[string]cloudprovider.Plugin), nil
}
return ap.plugins, nil
}
// register plugin of cloud provider and different cloud providers
func (ap *Provider) registerPlugin(plugin cloudprovider.Plugin) {
name := plugin.Name()
if name == "" {
klog.Fatal("empty plugin name")
}
ap.plugins[name] = plugin
}
func NewTencentCloudProvider() (cloudprovider.CloudProvider, error) {
return tencentCloudProvider, nil
}

View File

@ -0,0 +1,121 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.9.0
creationTimestamp: null
name: dedicatedclblisteners.networking.cloud.tencent.com
spec:
group: networking.cloud.tencent.com
names:
kind: DedicatedCLBListener
listKind: DedicatedCLBListenerList
plural: dedicatedclblisteners
singular: dedicatedclblistener
scope: Namespaced
versions:
- additionalPrinterColumns:
- description: CLB ID
jsonPath: .spec.lbId
name: LbId
type: string
- description: Port of CLB Listener
jsonPath: .spec.lbPort
name: LbPort
type: integer
- description: Pod name of target pod
jsonPath: .spec.targetPod.podName
name: Pod
type: string
- description: State of the dedicated clb listener
jsonPath: .status.state
name: State
type: string
name: v1alpha1
schema:
openAPIV3Schema:
description: DedicatedCLBListener is the Schema for the dedicatedclblisteners
API
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
description: DedicatedCLBListenerSpec defines the desired state of DedicatedCLBListener
properties:
extensiveParameters:
type: string
lbId:
type: string
x-kubernetes-validations:
- message: Value is immutable
rule: self == oldSelf
lbPort:
format: int64
type: integer
x-kubernetes-validations:
- message: Value is immutable
rule: self == oldSelf
lbRegion:
type: string
x-kubernetes-validations:
- message: Value is immutable
rule: self == oldSelf
protocol:
enum:
- TCP
- UDP
type: string
x-kubernetes-validations:
- message: Value is immutable
rule: self == oldSelf
targetPod:
properties:
podName:
type: string
targetPort:
format: int64
type: integer
required:
- podName
- targetPort
type: object
required:
- lbId
- lbPort
- protocol
type: object
status:
description: DedicatedCLBListenerStatus defines the observed state of
DedicatedCLBListener
properties:
address:
type: string
listenerId:
type: string
message:
type: string
state:
enum:
- Bound
- Available
- Pending
- Failed
- Deleting
type: string
type: object
type: object
served: true
storage: true
subresources:
status: {}

View File

@ -969,4 +969,109 @@ nlb-26jbknebrjlejt5abu 192.168.0.8:80,192.168.0.82:80,192.168.63.228:80
``` ```
After waiting for the entire update process to end, you can find that there are no changes in the ep, indicating that no extraction has been performed. After waiting for the entire update process to end, you can find that there are no changes in the ep, indicating that no extraction has been performed.
---
### TencentCloud-CLB
#### Plugin name
`TencentCloud-CLB`
#### Cloud Provider
TencentCloud
#### Plugin description
- TencentCloud-CLB enables game servers to be accessed from the Internet by using Cloud Load Balancer (CLB) of Tencent Cloud. CLB is a type of Server Load Balancer (CLB). TencentCloud-CLB uses different ports for different game servers. The CLB instance only forwards traffic, but does not implement load balancing.
The [tke-extend-network-controller](https://github.com/tkestack/tke-extend-network-controller) network plugin needs to be installed (can be installed through the TKE application market).
- This network plugin does not support network isolation.
#### Network parameters
ClbIds
- Meaning: the CLB instance ID. You can fill in multiple ids.
- Value: in the format of slbId-0,slbId-1,... An example value can be "lb-9zeo7prq1m25ctpfrw1m7,lb-bp1qz7h50yd3w58h2f8je"
- Configuration change supported or not: yes. You can add new slbIds at the end. However, it is recommended not to change existing slbId that is in use.
PortProtocols
- Meaning: the ports in the pod to be exposed and the protocols. You can specify multiple ports and protocols.
- Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters.
- Configuration change supported or not: yes.
#### Plugin configuration
```
[tencentcloud]
enable = true
[tencentcloud.clb]
# Specify the range of available ports of the CLB instance. Ports in this range can be used to forward Internet traffic to pods. In this example, the range includes 200 ports.
min_port = 1000
max_port = 1100
```
#### Example
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: clb-nginx
namespace: default
spec:
replicas: 1
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: TencentCloud-CLB
networkConf:
- name: ClbIds
value: "lb-3ip9k5kr,lb-4ia8k0yh"
- name: PortProtocols
value: "80/TCP,7777/UDP"
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
The network status of GameServer would be as follows:
```yaml
networkStatus:
createTime: "2024-10-28T03:16:20Z"
currentNetworkState: Ready
desiredNetworkState: Ready
externalAddresses:
- ip: 139.155.64.52
ports:
- name: "80"
port: 1002
protocol: TCP
- ip: 139.155.64.52
ports:
- name: "7777"
port: 1003
protocol: UDP
internalAddresses:
- ip: 172.16.7.106
ports:
- name: "80"
port: 80
protocol: TCP
- ip: 172.16.7.106
ports:
- name: "7777"
port: 7777
protocol: UDP
lastTransitionTime: "2024-10-28T03:16:20Z"
networkType: TencentCloud-CLB
```

View File

@ -972,6 +972,109 @@ nlb-26jbknebrjlejt5abu 192.168.0.8:80,192.168.0.82:80,192.168.63.228:80
等待整个更新过程结束可以发现ep没有任何变化说明并未进行摘流。 等待整个更新过程结束可以发现ep没有任何变化说明并未进行摘流。
---
### TencentCloud-CLB
#### 插件名称
`TencentCloud-CLB`
#### Cloud Provider
TencentCloud
#### 插件说明
- TencentCloud-CLB 使用腾讯云负载均衡器CLB作为对外服务的承载实体在此模式下不同游戏服使用 CLB 的不同端口对外暴露,此时 CLB 只做转发,并未均衡流量。
- 需安装 [tke-extend-network-controller](https://github.com/tkestack/tke-extend-network-controller) 网络插件(可通过 TKE 应用市场安装)。
- 是否支持网络隔离:否。
#### 网络参数
ClbIds
- 含义填写clb的id。可填写多个。
- 填写格式各个clbId用,分割。例如lb-xxxx,lb-yyyy,...
- 是否支持变更:支持。
PortProtocols
- 含义pod暴露的端口及协议支持填写多个端口/协议。
- 格式port1/protocol1,port2/protocol2,...(协议需大写)
- 是否支持变更:支持。
#### 插件配置
```
[tencentcloud]
enable = true
[tencentcloud.clb]
# 填写clb可使用的空闲端口段用于为pod分配外部接入端口
min_port = 1000
max_port = 1100
```
#### 示例说明
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: clb-nginx
namespace: default
spec:
replicas: 1
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: TencentCloud-CLB
networkConf:
- name: ClbIds
value: "lb-3ip9k5kr,lb-4ia8k0yh"
- name: PortProtocols
value: "80/TCP,7777/UDP"
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
生成的 gameserver clb-nginx-0 networkStatus 字段如下所示:
```yaml
networkStatus:
createTime: "2024-10-28T03:16:20Z"
currentNetworkState: Ready
desiredNetworkState: Ready
externalAddresses:
- ip: 139.155.64.52
ports:
- name: "80"
port: 1002
protocol: TCP
- ip: 139.155.64.52
ports:
- name: "7777"
port: 1003
protocol: UDP
internalAddresses:
- ip: 172.16.7.106
ports:
- name: "80"
port: 80
protocol: TCP
- ip: 172.16.7.106
ports:
- name: "7777"
port: 7777
protocol: UDP
lastTransitionTime: "2024-10-28T03:16:20Z"
networkType: TencentCloud-CLB
```
## 获取网络信息 ## 获取网络信息
GameServer Network Status可以通过两种方式获取 GameServer Network Status可以通过两种方式获取
@ -1051,4 +1154,4 @@ func getNetwork() {
// 访问networkStatus各个字段 // 访问networkStatus各个字段
} }
``` ```

View File

@ -29,6 +29,7 @@ import (
"k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime"
utilruntime "k8s.io/apimachinery/pkg/util/runtime" utilruntime "k8s.io/apimachinery/pkg/util/runtime"
clientgoscheme "k8s.io/client-go/kubernetes/scheme" clientgoscheme "k8s.io/client-go/kubernetes/scheme"
// Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.) // Import all Kubernetes client auth plugins (e.g. Azure, GCP, OIDC, etc.)
// to ensure that exec-entrypoint and run can make use of them. // to ensure that exec-entrypoint and run can make use of them.
_ "k8s.io/client-go/plugin/pkg/client/auth" _ "k8s.io/client-go/plugin/pkg/client/auth"
@ -42,6 +43,7 @@ import (
"github.com/openkruise/kruise-game/cloudprovider" "github.com/openkruise/kruise-game/cloudprovider"
aliv1beta1 "github.com/openkruise/kruise-game/cloudprovider/alibabacloud/apis/v1beta1" aliv1beta1 "github.com/openkruise/kruise-game/cloudprovider/alibabacloud/apis/v1beta1"
cpmanager "github.com/openkruise/kruise-game/cloudprovider/manager" cpmanager "github.com/openkruise/kruise-game/cloudprovider/manager"
tencentv1alpha1 "github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1"
kruisegameclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned" kruisegameclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned"
kruisegamevisions "github.com/openkruise/kruise-game/pkg/client/informers/externalversions" kruisegamevisions "github.com/openkruise/kruise-game/pkg/client/informers/externalversions"
controller "github.com/openkruise/kruise-game/pkg/controllers" controller "github.com/openkruise/kruise-game/pkg/controllers"
@ -67,6 +69,7 @@ func init() {
utilruntime.Must(kruiseV1alpha1.AddToScheme(scheme)) utilruntime.Must(kruiseV1alpha1.AddToScheme(scheme))
utilruntime.Must(aliv1beta1.AddToScheme(scheme)) utilruntime.Must(aliv1beta1.AddToScheme(scheme))
utilruntime.Must(tencentv1alpha1.AddToScheme(scheme))
utilruntime.Must(ackv1alpha1.AddToScheme(scheme)) utilruntime.Must(ackv1alpha1.AddToScheme(scheme))
utilruntime.Must(elbv2api.AddToScheme(scheme)) utilruntime.Must(elbv2api.AddToScheme(scheme))
@ -138,7 +141,6 @@ func main() {
SyncPeriod: syncPeriod, SyncPeriod: syncPeriod,
NewClient: utilclient.NewClient, NewClient: utilclient.NewClient,
}) })
if err != nil { if err != nil {
setupLog.Error(err, "unable to start kruise-game-manager") setupLog.Error(err, "unable to start kruise-game-manager")
os.Exit(1) os.Exit(1)