升级 TencentCloud-CLB 插件 (#239)
* upgrade tencentcloud clb plugin * deprecate DedicatedCLBListener CRD * use CLBPortPool's pod annotation Signed-off-by: roc <roc@imroc.cc> * add comments Signed-off-by: roc <roc@imroc.cc> --------- Signed-off-by: roc <roc@imroc.cc>
This commit is contained in:
roc
GitHub
parent
7136738627
commit
1414654f46
|
|
@ -1,29 +1,13 @@
|
|||
package options
|
||||
|
||||
type TencentCloudOptions struct {
|
||||
Enable bool `toml:"enable"`
|
||||
CLBOptions TencentCloudCLBOptions `toml:"clb"`
|
||||
}
|
||||
|
||||
type TencentCloudCLBOptions struct {
|
||||
MaxPort int32 `toml:"max_port"`
|
||||
MinPort int32 `toml:"min_port"`
|
||||
}
|
||||
|
||||
func (o TencentCloudOptions) Valid() bool {
|
||||
clbOptions := o.CLBOptions
|
||||
|
||||
if clbOptions.MaxPort > 65535 {
|
||||
return false
|
||||
}
|
||||
|
||||
if clbOptions.MinPort < 1 {
|
||||
return false
|
||||
}
|
||||
|
||||
return true
|
||||
Enable bool `toml:"enable"`
|
||||
}
|
||||
|
||||
func (o TencentCloudOptions) Enabled() bool {
|
||||
return o.Enable
|
||||
}
|
||||
|
||||
func (o TencentCloudOptions) Valid() bool {
|
||||
return true
|
||||
}
|
||||
|
|
|
|||
|
|
@ -1,93 +0,0 @@
|
|||
/*
|
||||
Copyright 2024.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
// EDIT THIS FILE! THIS IS SCAFFOLDING FOR YOU TO OWN!
|
||||
// NOTE: json tags are required. Any new fields you add must have json tags for the fields to be serialized.
|
||||
|
||||
// DedicatedCLBListenerSpec defines the desired state of DedicatedCLBListener
|
||||
type DedicatedCLBListenerSpec struct {
|
||||
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
|
||||
LbId string `json:"lbId"`
|
||||
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
|
||||
// +optional
|
||||
LbRegion string `json:"lbRegion,omitempty"`
|
||||
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
|
||||
LbPort int64 `json:"lbPort"`
|
||||
// +kubebuilder:validation:XValidation:rule="self == oldSelf", message="Value is immutable"
|
||||
// +kubebuilder:validation:Enum=TCP;UDP
|
||||
Protocol string `json:"protocol"`
|
||||
// +optional
|
||||
ExtensiveParameters string `json:"extensiveParameters,omitempty"`
|
||||
// +optional
|
||||
TargetPod *TargetPod `json:"targetPod,omitempty"`
|
||||
}
|
||||
|
||||
type TargetPod struct {
|
||||
PodName string `json:"podName"`
|
||||
TargetPort int64 `json:"targetPort"`
|
||||
}
|
||||
|
||||
// DedicatedCLBListenerStatus defines the observed state of DedicatedCLBListener
|
||||
type DedicatedCLBListenerStatus struct {
|
||||
ListenerId string `json:"listenerId,omitempty"`
|
||||
// +kubebuilder:validation:Enum=Bound;Available;Pending;Failed;Deleting
|
||||
State string `json:"state,omitempty"`
|
||||
Message string `json:"message,omitempty"`
|
||||
Address string `json:"address,omitempty"`
|
||||
}
|
||||
|
||||
const (
|
||||
DedicatedCLBListenerStateBound = "Bound"
|
||||
DedicatedCLBListenerStateAvailable = "Available"
|
||||
DedicatedCLBListenerStatePending = "Pending"
|
||||
DedicatedCLBListenerStateFailed = "Failed"
|
||||
DedicatedCLBListenerStateDeleting = "Deleting"
|
||||
)
|
||||
|
||||
// +kubebuilder:object:root=true
|
||||
// +kubebuilder:subresource:status
|
||||
// +kubebuilder:printcolumn:name="LbId",type="string",JSONPath=".spec.lbId",description="CLB ID"
|
||||
// +kubebuilder:printcolumn:name="LbPort",type="integer",JSONPath=".spec.lbPort",description="Port of CLB Listener"
|
||||
// +kubebuilder:printcolumn:name="Pod",type="string",JSONPath=".spec.targetPod.podName",description="Pod name of target pod"
|
||||
// +kubebuilder:printcolumn:name="State",type="string",JSONPath=".status.state",description="State of the dedicated clb listener"
|
||||
|
||||
// DedicatedCLBListener is the Schema for the dedicatedclblisteners API
|
||||
type DedicatedCLBListener struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ObjectMeta `json:"metadata,omitempty"`
|
||||
|
||||
Spec DedicatedCLBListenerSpec `json:"spec,omitempty"`
|
||||
Status DedicatedCLBListenerStatus `json:"status,omitempty"`
|
||||
}
|
||||
|
||||
// +kubebuilder:object:root=true
|
||||
|
||||
// DedicatedCLBListenerList contains a list of DedicatedCLBListener
|
||||
type DedicatedCLBListenerList struct {
|
||||
metav1.TypeMeta `json:",inline"`
|
||||
metav1.ListMeta `json:"metadata,omitempty"`
|
||||
Items []DedicatedCLBListener `json:"items"`
|
||||
}
|
||||
|
||||
func init() {
|
||||
SchemeBuilder.Register(&DedicatedCLBListener{}, &DedicatedCLBListenerList{})
|
||||
}
|
||||
|
|
@ -1,4 +0,0 @@
|
|||
// Package v1alpha1 contains API Schema definitions for the tencentcloud v1alpha1 API group
|
||||
// +k8s:deepcopy-gen=package,register
|
||||
// +groupName=networking.cloud.tencent.com
|
||||
package v1alpha1
|
||||
|
|
@ -1,38 +0,0 @@
|
|||
/*
|
||||
Copyright 2024.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Package v1alpha1 contains API Schema definitions for the networking v1alpha1 API group
|
||||
// +kubebuilder:validation:Required
|
||||
// +kubebuilder:object:generate=true
|
||||
// +groupName=networking.cloud.tencent.com
|
||||
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
"k8s.io/apimachinery/pkg/runtime/schema"
|
||||
"sigs.k8s.io/controller-runtime/pkg/scheme"
|
||||
)
|
||||
|
||||
var (
|
||||
// GroupVersion is group version used to register these objects
|
||||
GroupVersion = schema.GroupVersion{Group: "networking.cloud.tencent.com", Version: "v1alpha1"}
|
||||
|
||||
// SchemeBuilder is used to add go types to the GroupVersionKind scheme
|
||||
SchemeBuilder = &scheme.Builder{GroupVersion: GroupVersion}
|
||||
|
||||
// AddToScheme adds the types in this group-version to the given scheme.
|
||||
AddToScheme = SchemeBuilder.AddToScheme
|
||||
)
|
||||
|
|
@ -1,134 +0,0 @@
|
|||
//go:build !ignore_autogenerated
|
||||
|
||||
/*
|
||||
Copyright 2022 The Kruise Authors.
|
||||
|
||||
Licensed under the Apache License, Version 2.0 (the "License");
|
||||
you may not use this file except in compliance with the License.
|
||||
You may obtain a copy of the License at
|
||||
|
||||
http://www.apache.org/licenses/LICENSE-2.0
|
||||
|
||||
Unless required by applicable law or agreed to in writing, software
|
||||
distributed under the License is distributed on an "AS IS" BASIS,
|
||||
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
See the License for the specific language governing permissions and
|
||||
limitations under the License.
|
||||
*/
|
||||
|
||||
// Code generated by controller-gen. DO NOT EDIT.
|
||||
|
||||
package v1alpha1
|
||||
|
||||
import (
|
||||
runtime "k8s.io/apimachinery/pkg/runtime"
|
||||
)
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *DedicatedCLBListener) DeepCopyInto(out *DedicatedCLBListener) {
|
||||
*out = *in
|
||||
out.TypeMeta = in.TypeMeta
|
||||
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
|
||||
in.Spec.DeepCopyInto(&out.Spec)
|
||||
out.Status = in.Status
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListener.
|
||||
func (in *DedicatedCLBListener) DeepCopy() *DedicatedCLBListener {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(DedicatedCLBListener)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
|
||||
func (in *DedicatedCLBListener) DeepCopyObject() runtime.Object {
|
||||
if c := in.DeepCopy(); c != nil {
|
||||
return c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *DedicatedCLBListenerList) DeepCopyInto(out *DedicatedCLBListenerList) {
|
||||
*out = *in
|
||||
out.TypeMeta = in.TypeMeta
|
||||
in.ListMeta.DeepCopyInto(&out.ListMeta)
|
||||
if in.Items != nil {
|
||||
in, out := &in.Items, &out.Items
|
||||
*out = make([]DedicatedCLBListener, len(*in))
|
||||
for i := range *in {
|
||||
(*in)[i].DeepCopyInto(&(*out)[i])
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerList.
|
||||
func (in *DedicatedCLBListenerList) DeepCopy() *DedicatedCLBListenerList {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(DedicatedCLBListenerList)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
|
||||
func (in *DedicatedCLBListenerList) DeepCopyObject() runtime.Object {
|
||||
if c := in.DeepCopy(); c != nil {
|
||||
return c
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *DedicatedCLBListenerSpec) DeepCopyInto(out *DedicatedCLBListenerSpec) {
|
||||
*out = *in
|
||||
if in.TargetPod != nil {
|
||||
in, out := &in.TargetPod, &out.TargetPod
|
||||
*out = new(TargetPod)
|
||||
**out = **in
|
||||
}
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerSpec.
|
||||
func (in *DedicatedCLBListenerSpec) DeepCopy() *DedicatedCLBListenerSpec {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(DedicatedCLBListenerSpec)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *DedicatedCLBListenerStatus) DeepCopyInto(out *DedicatedCLBListenerStatus) {
|
||||
*out = *in
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DedicatedCLBListenerStatus.
|
||||
func (in *DedicatedCLBListenerStatus) DeepCopy() *DedicatedCLBListenerStatus {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(DedicatedCLBListenerStatus)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (in *TargetPod) DeepCopyInto(out *TargetPod) {
|
||||
*out = *in
|
||||
}
|
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new TargetPod.
|
||||
func (in *TargetPod) DeepCopy() *TargetPod {
|
||||
if in == nil {
|
||||
return nil
|
||||
}
|
||||
out := new(TargetPod)
|
||||
in.DeepCopyInto(out)
|
||||
return out
|
||||
}
|
||||
|
|
@ -2,49 +2,36 @@ package tencentcloud
|
|||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"slices"
|
||||
"strconv"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/openkruise/kruise-game/pkg/util"
|
||||
|
||||
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
|
||||
kruisev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
|
||||
"github.com/openkruise/kruise-game/cloudprovider"
|
||||
cperrors "github.com/openkruise/kruise-game/cloudprovider/errors"
|
||||
provideroptions "github.com/openkruise/kruise-game/cloudprovider/options"
|
||||
"github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1"
|
||||
"github.com/openkruise/kruise-game/cloudprovider/utils"
|
||||
"github.com/openkruise/kruise-game/pkg/util"
|
||||
corev1 "k8s.io/api/core/v1"
|
||||
"k8s.io/apimachinery/pkg/api/errors"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
"k8s.io/apimachinery/pkg/util/intstr"
|
||||
log "k8s.io/klog/v2"
|
||||
"k8s.io/utils/ptr"
|
||||
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||
)
|
||||
|
||||
const (
|
||||
ClbNetwork = "TencentCloud-CLB"
|
||||
AliasCLB = "CLB-Network"
|
||||
ClbIdsConfigName = "ClbIds"
|
||||
PortProtocolsConfigName = "PortProtocols"
|
||||
MinPortConfigName = "MinPort"
|
||||
MaxPortConfigName = "MaxPort"
|
||||
OwnerPodKey = "game.kruise.io/owner-pod"
|
||||
TargetPortKey = "game.kruise.io/target-port"
|
||||
ClbNetwork = "TencentCloud-CLB"
|
||||
AliasCLB = "CLB-Network"
|
||||
ClbIdsConfigName = "ClbIds"
|
||||
PortProtocolsConfigName = "PortProtocols"
|
||||
CLBPortMappingAnnotation = "networking.cloud.tencent.com/clb-port-mapping"
|
||||
EnableCLBPortMappingAnnotation = "networking.cloud.tencent.com/enable-clb-port-mapping"
|
||||
CLBPortMappingResultAnnotation = "networking.cloud.tencent.com/clb-port-mapping-result"
|
||||
CLBPortMappingStatuslAnnotation = "networking.cloud.tencent.com/clb-port-mapping-status"
|
||||
)
|
||||
|
||||
type portAllocated map[int32]bool
|
||||
|
||||
type ClbPlugin struct {
|
||||
maxPort int32
|
||||
minPort int32
|
||||
cache map[string]portAllocated
|
||||
podAllocate map[string][]string
|
||||
mutex sync.RWMutex
|
||||
}
|
||||
type ClbPlugin struct{}
|
||||
|
||||
type portProtocol struct {
|
||||
port int
|
||||
|
|
@ -52,10 +39,15 @@ type portProtocol struct {
|
|||
}
|
||||
|
||||
type clbConfig struct {
|
||||
lbIds []string
|
||||
targetPorts []portProtocol
|
||||
}
|
||||
|
||||
type portMapping struct {
|
||||
Port int `json:"port"`
|
||||
Protocol string `json:"protocol"`
|
||||
Address string `json:"address"`
|
||||
}
|
||||
|
||||
func (p *ClbPlugin) Name() string {
|
||||
return ClbNetwork
|
||||
}
|
||||
|
|
@ -65,83 +57,27 @@ func (p *ClbPlugin) Alias() string {
|
|||
}
|
||||
|
||||
func (p *ClbPlugin) Init(c client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error {
|
||||
p.mutex.Lock()
|
||||
defer p.mutex.Unlock()
|
||||
clbOptions := options.(provideroptions.TencentCloudOptions).CLBOptions
|
||||
p.minPort = clbOptions.MinPort
|
||||
p.maxPort = clbOptions.MaxPort
|
||||
|
||||
listenerList := &v1alpha1.DedicatedCLBListenerList{}
|
||||
err := c.List(ctx, listenerList)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
p.cache, p.podAllocate = initLbCache(listenerList.Items, p.minPort, p.maxPort)
|
||||
log.Infof("[%s] podAllocate cache complete initialization: %v", ClbNetwork, p.podAllocate)
|
||||
return nil
|
||||
}
|
||||
|
||||
func initLbCache(listenerList []v1alpha1.DedicatedCLBListener, minPort, maxPort int32) (map[string]portAllocated, map[string][]string) {
|
||||
newCache := make(map[string]portAllocated)
|
||||
newPodAllocate := make(map[string][]string)
|
||||
for _, lis := range listenerList {
|
||||
podName, exist := lis.GetLabels()[OwnerPodKey]
|
||||
if !exist || podName == "" {
|
||||
continue
|
||||
}
|
||||
if lis.Spec.LbPort > int64(maxPort) || lis.Spec.LbPort < int64(minPort) {
|
||||
continue
|
||||
}
|
||||
lbId := lis.Spec.LbId
|
||||
if newCache[lbId] == nil {
|
||||
newCache[lbId] = make(portAllocated, maxPort-minPort)
|
||||
for i := minPort; i < maxPort; i++ {
|
||||
newCache[lbId][i] = false
|
||||
}
|
||||
}
|
||||
newCache[lbId][int32(lis.Spec.LbPort)] = true
|
||||
podKey := lis.GetNamespace() + "/" + podName
|
||||
newPodAllocate[podKey] = append(newPodAllocate[podKey], fmt.Sprintf("%s:%d", lbId, lis.Spec.LbPort))
|
||||
}
|
||||
return newCache, newPodAllocate
|
||||
}
|
||||
|
||||
func (p *ClbPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
|
||||
return pod, nil
|
||||
}
|
||||
|
||||
func (p *ClbPlugin) deleteListener(ctx context.Context, c client.Client, lis *v1alpha1.DedicatedCLBListener) cperrors.PluginError {
|
||||
err := c.Delete(ctx, lis)
|
||||
if err != nil {
|
||||
return cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
|
||||
}
|
||||
if pm := p.cache[lis.Spec.LbId]; pm != nil {
|
||||
pm[int32(lis.Spec.LbPort)] = false
|
||||
}
|
||||
var podName string
|
||||
if targetPod := lis.Spec.TargetPod; targetPod != nil {
|
||||
podName = targetPod.PodName
|
||||
} else if lis.Labels != nil && lis.Labels[TargetPortKey] != "" && lis.Labels[OwnerPodKey] != "" {
|
||||
podName = lis.Labels[OwnerPodKey]
|
||||
} else {
|
||||
return nil
|
||||
}
|
||||
target := fmt.Sprintf("%s/%d", lis.Spec.LbId, lis.Spec.LbPort)
|
||||
p.podAllocate[podName] = slices.DeleteFunc(p.podAllocate[podName], func(el string) bool {
|
||||
return el == target
|
||||
})
|
||||
return nil
|
||||
return p.reconcile(c, pod, ctx)
|
||||
}
|
||||
|
||||
func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
|
||||
if pod.DeletionTimestamp != nil {
|
||||
return pod, nil
|
||||
}
|
||||
return p.reconcile(c, pod, ctx)
|
||||
}
|
||||
|
||||
// Ensure the annotation of pod is correct.
|
||||
func (p *ClbPlugin) reconcile(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
|
||||
networkManager := utils.NewNetworkManager(pod, c)
|
||||
networkStatus, _ := networkManager.GetNetworkStatus()
|
||||
if networkStatus == nil {
|
||||
pod, err := networkManager.UpdateNetworkStatus(kruisev1alpha1.NetworkStatus{
|
||||
CurrentNetworkState: kruisev1alpha1.NetworkNotReady,
|
||||
CurrentNetworkState: kruisev1alpha1.NetworkWaiting,
|
||||
}, pod)
|
||||
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
|
||||
}
|
||||
|
|
@ -150,263 +86,101 @@ func (p *ClbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C
|
|||
if err != nil {
|
||||
return pod, cperrors.ToPluginError(err, cperrors.ParameterError)
|
||||
}
|
||||
|
||||
gss, err := util.GetGameServerSetOfPod(pod, c, ctx)
|
||||
if err != nil && !errors.IsNotFound(err) {
|
||||
if err != nil && !apierrors.IsNotFound(err) {
|
||||
return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
|
||||
}
|
||||
|
||||
// get related dedicated clb listeners
|
||||
listeners := &v1alpha1.DedicatedCLBListenerList{}
|
||||
if err := c.List(
|
||||
ctx, listeners,
|
||||
client.InNamespace(pod.Namespace),
|
||||
client.MatchingLabels{
|
||||
OwnerPodKey: pod.Name,
|
||||
kruisev1alpha1.GameServerOwnerGssKey: gss.Name,
|
||||
},
|
||||
); err != nil {
|
||||
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
|
||||
if pod.Annotations == nil {
|
||||
pod.Annotations = make(map[string]string)
|
||||
}
|
||||
|
||||
// reconcile
|
||||
lisMap := make(map[portProtocol]v1alpha1.DedicatedCLBListener)
|
||||
|
||||
for _, lis := range listeners.Items {
|
||||
// ignore deleting dedicated clb listener
|
||||
if lis.DeletionTimestamp != nil {
|
||||
continue
|
||||
}
|
||||
// old dedicated clb listener remain
|
||||
if lis.OwnerReferences[0].Kind == "Pod" && lis.OwnerReferences[0].UID != pod.UID {
|
||||
log.Infof("[%s] waitting old dedicated clb listener %s/%s deleted. old owner pod uid is %s, but now is %s", ClbNetwork, lis.Namespace, lis.Name, lis.OwnerReferences[0].UID, pod.UID)
|
||||
return pod, nil
|
||||
}
|
||||
|
||||
targetPod := lis.Spec.TargetPod
|
||||
if targetPod != nil && targetPod.PodName == pod.Name {
|
||||
port := portProtocol{
|
||||
port: int(targetPod.TargetPort),
|
||||
protocol: lis.Spec.Protocol,
|
||||
}
|
||||
lisMap[port] = lis
|
||||
} else if targetPod == nil && (lis.Labels != nil && lis.Labels[TargetPortKey] != "") {
|
||||
targetPort, err := strconv.Atoi(lis.Labels[TargetPortKey])
|
||||
if err != nil {
|
||||
log.Warningf("[%s] invalid dedicated clb listener target port annotation %s/%s: %s", ClbNetwork, lis.Namespace, lis.Name, err.Error())
|
||||
continue
|
||||
}
|
||||
port := portProtocol{
|
||||
port: targetPort,
|
||||
protocol: lis.Spec.Protocol,
|
||||
}
|
||||
// lower priority than targetPod is not nil
|
||||
if _, exists := lisMap[port]; !exists {
|
||||
lisMap[port] = lis
|
||||
}
|
||||
}
|
||||
pod.Annotations[CLBPortMappingAnnotation] = getClbPortMappingAnnotation(clbConf, gss)
|
||||
enableCLBPortMapping := "true"
|
||||
if networkManager.GetNetworkDisabled() {
|
||||
enableCLBPortMapping = "false"
|
||||
}
|
||||
|
||||
internalAddresses := make([]kruisev1alpha1.NetworkAddress, 0)
|
||||
externalAddresses := make([]kruisev1alpha1.NetworkAddress, 0)
|
||||
|
||||
for _, port := range clbConf.targetPorts {
|
||||
if lis, ok := lisMap[port]; !ok { // no dedicated clb listener, try to create one
|
||||
if networkManager.GetNetworkDisabled() {
|
||||
continue
|
||||
}
|
||||
// ensure not ready while creating the listener
|
||||
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
|
||||
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
|
||||
if err != nil {
|
||||
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
|
||||
}
|
||||
// allocate and create new listener bound to pod
|
||||
newLis, err := p.consLis(clbConf, pod, port, gss.Name)
|
||||
if err != nil {
|
||||
pod.Annotations[EnableCLBPortMappingAnnotation] = enableCLBPortMapping
|
||||
if pod.Annotations[CLBPortMappingStatuslAnnotation] == "Ready" {
|
||||
if result := pod.Annotations[CLBPortMappingResultAnnotation]; result != "" {
|
||||
mappings := []portMapping{}
|
||||
if err := json.Unmarshal([]byte(result), &mappings); err != nil {
|
||||
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
|
||||
}
|
||||
err = c.Create(ctx, newLis)
|
||||
if err != nil {
|
||||
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
|
||||
}
|
||||
} else { // already created dedicated clb listener bound to pod
|
||||
delete(lisMap, port)
|
||||
if networkManager.GetNetworkDisabled() { // disable network
|
||||
// deregister pod if networkDisabled is true
|
||||
if lis.Spec.TargetPod != nil {
|
||||
lis.Spec.TargetPod = nil
|
||||
err = c.Update(ctx, &lis)
|
||||
if len(mappings) != 0 {
|
||||
internalAddresses := make([]kruisev1alpha1.NetworkAddress, 0)
|
||||
externalAddresses := make([]kruisev1alpha1.NetworkAddress, 0)
|
||||
for _, mapping := range mappings {
|
||||
ss := strings.Split(mapping.Address, ":")
|
||||
if len(ss) != 2 {
|
||||
continue
|
||||
}
|
||||
lbIP := ss[0]
|
||||
lbPort, err := strconv.Atoi(ss[1])
|
||||
if err != nil {
|
||||
return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
|
||||
continue
|
||||
}
|
||||
}
|
||||
} else { // enable network
|
||||
if lis.Spec.TargetPod == nil { // ensure target pod is bound to dedicated clb listener
|
||||
lis.Spec.TargetPod = &v1alpha1.TargetPod{
|
||||
PodName: pod.Name,
|
||||
TargetPort: int64(port.port),
|
||||
}
|
||||
err = c.Update(ctx, &lis)
|
||||
port := mapping.Port
|
||||
instrIPort := intstr.FromInt(port)
|
||||
instrEPort := intstr.FromInt(lbPort)
|
||||
portName := instrIPort.String()
|
||||
protocol := corev1.Protocol(mapping.Protocol)
|
||||
internalAddresses = append(internalAddresses, kruisev1alpha1.NetworkAddress{
|
||||
IP: pod.Status.PodIP,
|
||||
Ports: []kruisev1alpha1.NetworkPort{
|
||||
{
|
||||
Name: portName,
|
||||
Port: &instrIPort,
|
||||
Protocol: protocol,
|
||||
},
|
||||
},
|
||||
})
|
||||
externalAddresses = append(externalAddresses, kruisev1alpha1.NetworkAddress{
|
||||
IP: lbIP,
|
||||
Ports: []kruisev1alpha1.NetworkPort{
|
||||
{
|
||||
Name: portName,
|
||||
Port: &instrEPort,
|
||||
Protocol: protocol,
|
||||
},
|
||||
},
|
||||
})
|
||||
networkStatus.InternalAddresses = internalAddresses
|
||||
networkStatus.ExternalAddresses = externalAddresses
|
||||
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkReady
|
||||
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
|
||||
if err != nil {
|
||||
return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
|
||||
}
|
||||
} else {
|
||||
// recreate dedicated clb listener if necessary (config changed)
|
||||
if !slices.Contains(clbConf.lbIds, lis.Spec.LbId) || lis.Spec.LbPort > int64(p.maxPort) || lis.Spec.LbPort < int64(p.minPort) || lis.Spec.Protocol != port.protocol || lis.Spec.TargetPod.TargetPort != int64(port.port) {
|
||||
// ensure not ready while recreating the listener
|
||||
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
|
||||
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
|
||||
if err != nil {
|
||||
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
|
||||
}
|
||||
|
||||
// delete old listener
|
||||
err := p.deleteListener(ctx, c, &lis)
|
||||
if err != nil {
|
||||
return pod, err
|
||||
}
|
||||
|
||||
// allocate and create new listener bound to pod
|
||||
if newLis, err := p.consLis(clbConf, pod, port, gss.Name); err != nil {
|
||||
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
|
||||
} else {
|
||||
err := c.Create(ctx, newLis)
|
||||
if err != nil {
|
||||
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
|
||||
}
|
||||
}
|
||||
} else { // dedicated clb listener is desired, check status
|
||||
if lis.Status.State == v1alpha1.DedicatedCLBListenerStateBound && lis.Status.Address != "" { // network ready
|
||||
ss := strings.Split(lis.Status.Address, ":")
|
||||
if len(ss) != 2 {
|
||||
return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener address %s", lis.Status.Address))
|
||||
}
|
||||
lbPort, err := strconv.Atoi(ss[1])
|
||||
if err != nil {
|
||||
return pod, cperrors.NewPluginError(cperrors.InternalError, fmt.Sprintf("invalid dedicated clb listener port %s", ss[1]))
|
||||
}
|
||||
instrIPort := intstr.FromInt(int(port.port))
|
||||
instrEPort := intstr.FromInt(lbPort)
|
||||
internalAddresses = append(internalAddresses, kruisev1alpha1.NetworkAddress{
|
||||
IP: pod.Status.PodIP,
|
||||
Ports: []kruisev1alpha1.NetworkPort{
|
||||
{
|
||||
Name: instrIPort.String(),
|
||||
Port: &instrIPort,
|
||||
Protocol: corev1.Protocol(port.protocol),
|
||||
},
|
||||
},
|
||||
})
|
||||
externalAddresses = append(externalAddresses, kruisev1alpha1.NetworkAddress{
|
||||
IP: ss[0],
|
||||
Ports: []kruisev1alpha1.NetworkPort{
|
||||
{
|
||||
Name: instrIPort.String(),
|
||||
Port: &instrEPort,
|
||||
Protocol: corev1.Protocol(port.protocol),
|
||||
},
|
||||
},
|
||||
})
|
||||
} else { // network not ready
|
||||
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkNotReady
|
||||
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
|
||||
if err != nil {
|
||||
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
|
||||
}
|
||||
}
|
||||
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// other dedicated clb listener is not used, delete it
|
||||
for _, lis := range lisMap {
|
||||
err := p.deleteListener(ctx, c, &lis)
|
||||
if err != nil {
|
||||
return pod, err
|
||||
}
|
||||
}
|
||||
|
||||
// set network status to ready when all lb port is ready
|
||||
if len(externalAddresses) == len(clbConf.targetPorts) {
|
||||
// change network status to ready if necessary
|
||||
if !reflect.DeepEqual(externalAddresses, networkStatus.ExternalAddresses) || networkStatus.CurrentNetworkState != kruisev1alpha1.NetworkReady {
|
||||
networkStatus.InternalAddresses = internalAddresses
|
||||
networkStatus.ExternalAddresses = externalAddresses
|
||||
networkStatus.CurrentNetworkState = kruisev1alpha1.NetworkReady
|
||||
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
|
||||
if err != nil {
|
||||
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return pod, nil
|
||||
}
|
||||
|
||||
func (p *ClbPlugin) OnPodDeleted(c client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError {
|
||||
p.deAllocate(pod.GetNamespace() + "/" + pod.GetName())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (p *ClbPlugin) consLis(clbConf *clbConfig, pod *corev1.Pod, port portProtocol, gssName string) (*v1alpha1.DedicatedCLBListener, error) {
|
||||
lbId, lbPort := p.allocate(clbConf.lbIds, pod.GetNamespace()+"/"+pod.GetName())
|
||||
if lbId == "" {
|
||||
return nil, fmt.Errorf("there are no avaialable ports for %v", clbConf.lbIds)
|
||||
}
|
||||
lis := &v1alpha1.DedicatedCLBListener{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
GenerateName: pod.Name + "-",
|
||||
Namespace: pod.Namespace,
|
||||
Labels: map[string]string{
|
||||
OwnerPodKey: pod.Name, // used to select pod related dedicated clb listener
|
||||
TargetPortKey: strconv.Itoa(port.port), // used to recover clb pod binding when networkDisabled set from true to false
|
||||
kruisev1alpha1.GameServerOwnerGssKey: gssName,
|
||||
},
|
||||
OwnerReferences: []metav1.OwnerReference{
|
||||
{
|
||||
APIVersion: pod.APIVersion,
|
||||
Kind: pod.Kind,
|
||||
Name: pod.GetName(),
|
||||
UID: pod.GetUID(),
|
||||
Controller: ptr.To[bool](true),
|
||||
BlockOwnerDeletion: ptr.To[bool](true),
|
||||
},
|
||||
},
|
||||
},
|
||||
Spec: v1alpha1.DedicatedCLBListenerSpec{
|
||||
LbId: lbId,
|
||||
LbPort: int64(lbPort),
|
||||
Protocol: port.protocol,
|
||||
TargetPod: &v1alpha1.TargetPod{
|
||||
PodName: pod.Name,
|
||||
TargetPort: int64(port.port),
|
||||
},
|
||||
},
|
||||
}
|
||||
return lis, nil
|
||||
}
|
||||
|
||||
func init() {
|
||||
clbPlugin := ClbPlugin{
|
||||
mutex: sync.RWMutex{},
|
||||
}
|
||||
clbPlugin := ClbPlugin{}
|
||||
tencentCloudProvider.registerPlugin(&clbPlugin)
|
||||
}
|
||||
|
||||
func getClbPortMappingAnnotation(clbConf *clbConfig, gss *kruisev1alpha1.GameServerSet) string {
|
||||
poolName := fmt.Sprintf("%s-%s", gss.Namespace, gss.Name)
|
||||
var buf strings.Builder
|
||||
for _, pp := range clbConf.targetPorts {
|
||||
buf.WriteString(fmt.Sprintf("%d %s %s\n", pp.port, pp.protocol, poolName))
|
||||
}
|
||||
return buf.String()
|
||||
}
|
||||
|
||||
var ErrMissingPortProtocolsConfig = fmt.Errorf("missing %s config", PortProtocolsConfigName)
|
||||
|
||||
func parseLbConfig(conf []kruisev1alpha1.NetworkConfParams) (*clbConfig, error) {
|
||||
var lbIds []string
|
||||
ports := []portProtocol{}
|
||||
for _, c := range conf {
|
||||
switch c.Name {
|
||||
case ClbIdsConfigName:
|
||||
for _, clbId := range strings.Split(c.Value, ",") {
|
||||
if clbId != "" {
|
||||
lbIds = append(lbIds, clbId)
|
||||
}
|
||||
}
|
||||
case PortProtocolsConfigName:
|
||||
for _, pp := range strings.Split(c.Value, ",") {
|
||||
ppSlice := strings.Split(pp, "/")
|
||||
|
|
@ -425,67 +199,10 @@ func parseLbConfig(conf []kruisev1alpha1.NetworkConfParams) (*clbConfig, error)
|
|||
}
|
||||
}
|
||||
}
|
||||
if len(ports) == 0 {
|
||||
return nil, ErrMissingPortProtocolsConfig
|
||||
}
|
||||
return &clbConfig{
|
||||
lbIds: lbIds,
|
||||
targetPorts: ports,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (p *ClbPlugin) allocate(lbIds []string, podKey string) (string, int32) {
|
||||
p.mutex.Lock()
|
||||
defer p.mutex.Unlock()
|
||||
|
||||
var lbId string
|
||||
var port int32
|
||||
|
||||
// find avaialable port
|
||||
for _, clbId := range lbIds {
|
||||
for i := p.minPort; i < p.maxPort; i++ {
|
||||
if !p.cache[clbId][i] {
|
||||
lbId = clbId
|
||||
port = i
|
||||
break
|
||||
}
|
||||
}
|
||||
}
|
||||
// update cache
|
||||
if lbId != "" {
|
||||
if p.cache[lbId] == nil { // init lb cache if not exist
|
||||
p.cache[lbId] = make(portAllocated, p.maxPort-p.minPort)
|
||||
for i := p.minPort; i < p.maxPort; i++ {
|
||||
p.cache[lbId][i] = false
|
||||
}
|
||||
}
|
||||
p.cache[lbId][port] = true
|
||||
p.podAllocate[podKey] = append(p.podAllocate[podKey], fmt.Sprintf("%s:%d", lbId, port))
|
||||
log.Infof("pod %s allocate clb %s port %d", podKey, lbId, port)
|
||||
}
|
||||
return lbId, port
|
||||
}
|
||||
|
||||
func (p *ClbPlugin) deAllocate(podKey string) {
|
||||
p.mutex.Lock()
|
||||
defer p.mutex.Unlock()
|
||||
|
||||
allocatedPorts, exist := p.podAllocate[podKey]
|
||||
if !exist {
|
||||
return
|
||||
}
|
||||
|
||||
for _, port := range allocatedPorts {
|
||||
ss := strings.Split(port, ":")
|
||||
if len(ss) != 2 {
|
||||
log.Errorf("bad allocated port cache format %s", port)
|
||||
continue
|
||||
}
|
||||
lbId := ss[0]
|
||||
lbPort, err := strconv.Atoi(ss[1])
|
||||
if err != nil {
|
||||
log.Errorf("failed to parse allocated port %s: %s", port, err.Error())
|
||||
continue
|
||||
}
|
||||
p.cache[lbId][int32(lbPort)] = false
|
||||
log.Infof("pod %s deallocate clb %s ports %d", podKey, lbId, lbPort)
|
||||
}
|
||||
delete(p.podAllocate, podKey)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -2,50 +2,11 @@ package tencentcloud
|
|||
|
||||
import (
|
||||
"reflect"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
kruisev1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
|
||||
"github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1"
|
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||
)
|
||||
|
||||
func TestAllocateDeAllocate(t *testing.T) {
|
||||
test := struct {
|
||||
lbIds []string
|
||||
clb *ClbPlugin
|
||||
podKey string
|
||||
}{
|
||||
lbIds: []string{"lb-xxx"},
|
||||
clb: &ClbPlugin{
|
||||
maxPort: int32(712),
|
||||
minPort: int32(512),
|
||||
cache: make(map[string]portAllocated),
|
||||
podAllocate: make(map[string][]string),
|
||||
mutex: sync.RWMutex{},
|
||||
},
|
||||
podKey: "xxx/xxx",
|
||||
}
|
||||
|
||||
lbId, port := test.clb.allocate(test.lbIds, test.podKey)
|
||||
if _, exist := test.clb.podAllocate[test.podKey]; !exist {
|
||||
t.Errorf("podAllocate[%s] is empty after allocated", test.podKey)
|
||||
}
|
||||
if port > test.clb.maxPort || port < test.clb.minPort {
|
||||
t.Errorf("allocate port %d, unexpected", port)
|
||||
}
|
||||
if test.clb.cache[lbId][port] == false {
|
||||
t.Errorf("Allocate port %d failed", port)
|
||||
}
|
||||
test.clb.deAllocate(test.podKey)
|
||||
if test.clb.cache[lbId][port] == true {
|
||||
t.Errorf("deAllocate port %d failed", port)
|
||||
}
|
||||
if _, exist := test.clb.podAllocate[test.podKey]; exist {
|
||||
t.Errorf("podAllocate[%s] is not empty after deallocated", test.podKey)
|
||||
}
|
||||
}
|
||||
|
||||
func TestParseLbConfig(t *testing.T) {
|
||||
tests := []struct {
|
||||
conf []kruisev1alpha1.NetworkConfParams
|
||||
|
|
@ -63,7 +24,6 @@ func TestParseLbConfig(t *testing.T) {
|
|||
},
|
||||
},
|
||||
clbConfig: &clbConfig{
|
||||
lbIds: []string{"xxx-A"},
|
||||
targetPorts: []portProtocol{
|
||||
{
|
||||
port: 80,
|
||||
|
|
@ -84,7 +44,6 @@ func TestParseLbConfig(t *testing.T) {
|
|||
},
|
||||
},
|
||||
clbConfig: &clbConfig{
|
||||
lbIds: []string{"xxx-A", "xxx-B"},
|
||||
targetPorts: []portProtocol{
|
||||
{
|
||||
port: 81,
|
||||
|
|
@ -113,78 +72,3 @@ func TestParseLbConfig(t *testing.T) {
|
|||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestInitLbCache(t *testing.T) {
|
||||
test := struct {
|
||||
listenerList []v1alpha1.DedicatedCLBListener
|
||||
minPort int32
|
||||
maxPort int32
|
||||
cache map[string]portAllocated
|
||||
podAllocate map[string][]string
|
||||
}{
|
||||
minPort: 512,
|
||||
maxPort: 712,
|
||||
cache: map[string]portAllocated{
|
||||
"xxx-A": map[int32]bool{
|
||||
666: true,
|
||||
},
|
||||
"xxx-B": map[int32]bool{
|
||||
555: true,
|
||||
},
|
||||
},
|
||||
podAllocate: map[string][]string{
|
||||
"ns-0/name-0": {"xxx-A:666"},
|
||||
"ns-1/name-1": {"xxx-B:555"},
|
||||
},
|
||||
listenerList: []v1alpha1.DedicatedCLBListener{
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
OwnerPodKey: "name-0",
|
||||
},
|
||||
Namespace: "ns-0",
|
||||
Name: "name-0-xxx",
|
||||
},
|
||||
Spec: v1alpha1.DedicatedCLBListenerSpec{
|
||||
LbId: "xxx-A",
|
||||
LbPort: 666,
|
||||
Protocol: "TCP",
|
||||
TargetPod: &v1alpha1.TargetPod{
|
||||
PodName: "name-0",
|
||||
TargetPort: 80,
|
||||
},
|
||||
},
|
||||
},
|
||||
{
|
||||
ObjectMeta: metav1.ObjectMeta{
|
||||
Labels: map[string]string{
|
||||
OwnerPodKey: "name-1",
|
||||
},
|
||||
Namespace: "ns-1",
|
||||
Name: "name-1-xxx",
|
||||
},
|
||||
Spec: v1alpha1.DedicatedCLBListenerSpec{
|
||||
LbId: "xxx-B",
|
||||
LbPort: 555,
|
||||
Protocol: "TCP",
|
||||
TargetPod: &v1alpha1.TargetPod{
|
||||
PodName: "name-1",
|
||||
TargetPort: 80,
|
||||
},
|
||||
},
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
actualCache, actualPodAllocate := initLbCache(test.listenerList, test.minPort, test.maxPort)
|
||||
for lb, pa := range test.cache {
|
||||
for port, isAllocated := range pa {
|
||||
if actualCache[lb][port] != isAllocated {
|
||||
t.Errorf("lb %s port %d isAllocated, expect: %t, actual: %t", lb, port, isAllocated, actualCache[lb][port])
|
||||
}
|
||||
}
|
||||
}
|
||||
if !reflect.DeepEqual(actualPodAllocate, test.podAllocate) {
|
||||
t.Errorf("podAllocate expect %v, but actully got %v", test.podAllocate, actualPodAllocate)
|
||||
}
|
||||
}
|
||||
|
|
|
|||
2
main.go
2
main.go
|
|
@ -44,7 +44,6 @@ import (
|
|||
"github.com/openkruise/kruise-game/cloudprovider"
|
||||
aliv1beta1 "github.com/openkruise/kruise-game/cloudprovider/alibabacloud/apis/v1beta1"
|
||||
cpmanager "github.com/openkruise/kruise-game/cloudprovider/manager"
|
||||
tencentv1alpha1 "github.com/openkruise/kruise-game/cloudprovider/tencentcloud/apis/v1alpha1"
|
||||
kruisegameclientset "github.com/openkruise/kruise-game/pkg/client/clientset/versioned"
|
||||
kruisegamevisions "github.com/openkruise/kruise-game/pkg/client/informers/externalversions"
|
||||
controller "github.com/openkruise/kruise-game/pkg/controllers"
|
||||
|
|
@ -72,7 +71,6 @@ func init() {
|
|||
utilruntime.Must(kruiseV1alpha1.AddToScheme(scheme))
|
||||
|
||||
utilruntime.Must(aliv1beta1.AddToScheme(scheme))
|
||||
utilruntime.Must(tencentv1alpha1.AddToScheme(scheme))
|
||||
|
||||
utilruntime.Must(ackv1alpha1.AddToScheme(scheme))
|
||||
utilruntime.Must(elbv2api.AddToScheme(scheme))
|
||||
|
|
|
|||
Loading…
Reference in New Issue