diff --git a/cloudprovider/config.go b/cloudprovider/config.go index 4da5bfa..e29101a 100644 --- a/cloudprovider/config.go +++ b/cloudprovider/config.go @@ -45,11 +45,13 @@ type ConfigFile struct { type CloudProviderConfig struct { KubernetesOptions CloudProviderOptions AlibabaCloudOptions CloudProviderOptions + VolcengineOptions CloudProviderOptions } type tomlConfigs struct { Kubernetes options.KubernetesOptions `toml:"kubernetes"` AlibabaCloud options.AlibabaCloudOptions `toml:"alibabacloud"` + Volcengine options.VolcengineOptions `toml:"volcengine"` } func (cf *ConfigFile) Parse() *CloudProviderConfig { @@ -62,6 +64,7 @@ func (cf *ConfigFile) Parse() *CloudProviderConfig { return &CloudProviderConfig{ KubernetesOptions: config.Kubernetes, AlibabaCloudOptions: config.AlibabaCloud, + VolcengineOptions: config.Volcengine, } } diff --git a/cloudprovider/manager/provider_manager.go b/cloudprovider/manager/provider_manager.go index 2e23171..060d5b6 100644 --- a/cloudprovider/manager/provider_manager.go +++ b/cloudprovider/manager/provider_manager.go @@ -22,6 +22,7 @@ import ( "github.com/openkruise/kruise-game/cloudprovider" "github.com/openkruise/kruise-game/cloudprovider/alibabacloud" "github.com/openkruise/kruise-game/cloudprovider/kubernetes" + volcengine "github.com/openkruise/kruise-game/cloudprovider/volcengine" corev1 "k8s.io/api/core/v1" log "k8s.io/klog/v2" "sigs.k8s.io/controller-runtime/pkg/client" @@ -116,5 +117,15 @@ func NewProviderManager() (*ProviderManager, error) { } } + if configs.VolcengineOptions.Valid() && configs.VolcengineOptions.Enabled() { + // build and register volcengine cloud provider + vcp, err := volcengine.NewVolcengineProvider() + if err != nil { + log.Errorf("Failed to initialize volcengine provider.because of %s", err.Error()) + } else { + pm.RegisterCloudProvider(vcp, configs.VolcengineOptions) + } + } + return pm, nil } diff --git a/cloudprovider/options/volcenginecloud_options.go b/cloudprovider/options/volcenginecloud_options.go new file mode 100644 index 0000000..210e5d9 --- /dev/null +++ b/cloudprovider/options/volcenginecloud_options.go @@ -0,0 +1,31 @@ +package options + +type VolcengineOptions struct { + Enable bool `toml:"enable"` + CLBOptions CLBOptions `toml:"clb"` +} + +type CLBOptions struct { + MaxPort int32 `toml:"max_port"` + MinPort int32 `toml:"min_port"` +} + +func (v VolcengineOptions) Valid() bool { + clbOptions := v.CLBOptions + if clbOptions.MaxPort-clbOptions.MinPort > 200 { + return false + } + + if clbOptions.MaxPort > 65535 { + return false + } + + if clbOptions.MinPort < 1 { + return false + } + return true +} + +func (v VolcengineOptions) Enabled() bool { + return v.Enable +} diff --git a/cloudprovider/volcengine/README.md b/cloudprovider/volcengine/README.md new file mode 100644 index 0000000..144ff05 --- /dev/null +++ b/cloudprovider/volcengine/README.md @@ -0,0 +1,108 @@ +English | [中文](./README.zh_CN.md) + +The Volcaengine Kubernetes Engine supports the CLB reuse mechanism in k8s. Different SVCs can use different ports of the same CLB. Therefore, the Volcengine-CLB network plugin will record the port allocation corresponding to each CLB. For the specified network type as Volcengine-CLB, the Volcengine-CLB network plugin will automatically allocate a port and create a service object. Wait for the svc ingress field. After the public network IP is successfully created, the GameServer network is in the Ready state and the process is completed. +![image](https://github.com/lizhipeng629/kruise-game/assets/110802158/209de309-b9b7-4ba8-b2fb-da0d299e2edb) + +## Volcengine-CLB configuration +### plugin configuration +```toml +[volcengine] +enable = true +[volcengine.clb] +#Fill in the free port segment that clb can use to allocate external access ports to pods, The maximum port range is 200. +max_port = 700 +min_port = 500 +``` +### Parameter +#### ClbIds +- Meaning：fill in the id of the clb. You can fill in more than one. You need to create the clb in [Volcano Engine]. +- Value：each clbId is divided by `,` . For example: `clb-9zeo7prq1m25ctpfrw1m7`,`clb-bp1qz7h50yd3w58h2f8je`,... +- Configurable：Y + +#### PortProtocols +- Meaning：the ports and protocols exposed by the pod, support filling in multiple ports/protocols +- Value：`port1/protocol1`,`port2/protocol2`,... The protocol names must be in uppercase letters. +- Configurable：Y + +#### Fixed +- Meaning：whether the mapping relationship is fixed. If the mapping relationship is fixed, the mapping relationship remains unchanged even if the pod is deleted and recreated. +- Value：false / true +- Configurable：Y + +#### AllowNotReadyContainers +- Meaning：the container names that are allowed not ready when inplace updating, when traffic will not be cut. +- Value：{containerName_0},{containerName_1},... eg：sidecar +- Configurable：It cannot be changed during the in-place updating process. + + +### Example +```yaml +cat <= minPort { + newCache[lbId][port] = true + ports = append(ports, port) + } + } + if len(ports) != 0 { + newPodAllocate[svc.GetNamespace()+"/"+svc.GetName()] = lbId + ":" + util.Int32SliceToString(ports, ",") + } + } + } + log.Infof("[%s] podAllocate cache complete initialization: %v", ClbNetwork, newPodAllocate) + return newCache, newPodAllocate +} + +func (c *ClbPlugin) OnPodAdded(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + networkManager := utils.NewNetworkManager(pod, client) + networkConfig := networkManager.GetNetworkConfig() + sc := parseLbConfig(networkConfig) + err := client.Create(ctx, c.consSvc(sc, pod, client, ctx)) + return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) +} + +func (c *ClbPlugin) OnPodUpdated(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) { + networkManager := utils.NewNetworkManager(pod, client) + + networkStatus, err := networkManager.GetNetworkStatus() + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + networkConfig := networkManager.GetNetworkConfig() + config := parseLbConfig(networkConfig) + if networkStatus == nil { + pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{ + CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady, + }, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + + // get svc + svc := &corev1.Service{} + err = client.Get(ctx, types.NamespacedName{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + }, svc) + if err != nil { + if errors.IsNotFound(err) { + return pod, cperrors.ToPluginError(client.Create(ctx, c.consSvc(config, pod, client, ctx)), cperrors.ApiCallError) + } + return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error()) + } + + // update svc + if util.GetHash(config) != svc.GetAnnotations()[ClbConfigHashKey] { + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + if err != nil { + return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error()) + } + return pod, cperrors.ToPluginError(client.Update(ctx, c.consSvc(config, pod, client, ctx)), cperrors.ApiCallError) + } + + // disable network + if networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeLoadBalancer { + svc.Spec.Type = corev1.ServiceTypeClusterIP + return pod, cperrors.ToPluginError(client.Update(ctx, svc), cperrors.ApiCallError) + } + + // enable network + if !networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeClusterIP { + svc.Spec.Type = corev1.ServiceTypeLoadBalancer + return pod, cperrors.ToPluginError(client.Update(ctx, svc), cperrors.ApiCallError) + } + + // network not ready + if len(svc.Status.LoadBalancer.Ingress) == 0 { + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) + } + + // allow not ready containers + if util.IsAllowNotReadyContainers(networkManager.GetNetworkConfig()) { + toUpDateSvc, err := utils.AllowNotReadyContainers(client, ctx, pod, svc, false) + if err != nil { + return pod, err + } + + if toUpDateSvc { + err := client.Update(ctx, svc) + if err != nil { + return pod, cperrors.ToPluginError(err, cperrors.ApiCallError) + } + } + } + + // network ready + internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) + externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0) + for _, port := range svc.Spec.Ports { + instrIPort := port.TargetPort + instrEPort := intstr.FromInt(int(port.Port)) + internalAddress := gamekruiseiov1alpha1.NetworkAddress{ + IP: pod.Status.PodIP, + Ports: []gamekruiseiov1alpha1.NetworkPort{ + { + Name: instrIPort.String(), + Port: &instrIPort, + Protocol: port.Protocol, + }, + }, + } + externalAddress := gamekruiseiov1alpha1.NetworkAddress{ + IP: svc.Status.LoadBalancer.Ingress[0].IP, + Ports: []gamekruiseiov1alpha1.NetworkPort{ + { + Name: instrIPort.String(), + Port: &instrEPort, + Protocol: port.Protocol, + }, + }, + } + internalAddresses = append(internalAddresses, internalAddress) + externalAddresses = append(externalAddresses, externalAddress) + } + networkStatus.InternalAddresses = internalAddresses + networkStatus.ExternalAddresses = externalAddresses + networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady + pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod) + return pod, cperrors.ToPluginError(err, cperrors.InternalError) +} + +func (c *ClbPlugin) OnPodDeleted(client client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError { + networkManager := utils.NewNetworkManager(pod, client) + networkConfig := networkManager.GetNetworkConfig() + sc := parseLbConfig(networkConfig) + + var podKeys []string + if sc.isFixed { + gss, err := util.GetGameServerSetOfPod(pod, client, ctx) + if err != nil && !errors.IsNotFound(err) { + return cperrors.ToPluginError(err, cperrors.ApiCallError) + } + // gss exists in cluster, do not deAllocate. + if err == nil && gss.GetDeletionTimestamp() == nil { + return nil + } + // gss not exists in cluster, deAllocate all the ports related to it. + for key := range c.podAllocate { + gssName := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey] + if strings.Contains(key, pod.GetNamespace()+"/"+gssName) { + podKeys = append(podKeys, key) + } + } + } else { + podKeys = append(podKeys, pod.GetNamespace()+"/"+pod.GetName()) + } + + for _, podKey := range podKeys { + c.deAllocate(podKey) + } + + return nil +} + +func (c *ClbPlugin) allocate(lbIds []string, num int, nsName string) (string, []int32) { + c.mutex.Lock() + defer c.mutex.Unlock() + + var ports []int32 + var lbId string + + // find lb with adequate ports + for _, clbId := range lbIds { + sum := 0 + for i := c.minPort; i < c.maxPort; i++ { + if !c.cache[clbId][i] { + sum++ + } + if sum >= num { + lbId = clbId + break + } + } + } + + // select ports + for i := 0; i < num; i++ { + var port int32 + if c.cache[lbId] == nil { + c.cache[lbId] = make(portAllocated, c.maxPort-c.minPort) + for i := c.minPort; i < c.maxPort; i++ { + c.cache[lbId][i] = false + } + } + + for p, allocated := range c.cache[lbId] { + if !allocated { + port = p + break + } + } + c.cache[lbId][port] = true + ports = append(ports, port) + } + + c.podAllocate[nsName] = lbId + ":" + util.Int32SliceToString(ports, ",") + log.Infof("pod %s allocate clb %s ports %v", nsName, lbId, ports) + return lbId, ports +} + +func (c *ClbPlugin) deAllocate(nsName string) { + c.mutex.Lock() + defer c.mutex.Unlock() + + allocatedPorts, exist := c.podAllocate[nsName] + if !exist { + return + } + + clbPorts := strings.Split(allocatedPorts, ":") + lbId := clbPorts[0] + ports := util.StringToInt32Slice(clbPorts[1], ",") + for _, port := range ports { + c.cache[lbId][port] = false + } + + delete(c.podAllocate, nsName) + log.Infof("pod %s deallocate clb %s ports %v", nsName, lbId, ports) +} + +func init() { + clbPlugin := ClbPlugin{ + mutex: sync.RWMutex{}, + } + volcengineProvider.registerPlugin(&clbPlugin) +} + +func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *clbConfig { + var lbIds []string + ports := make([]int, 0) + protocols := make([]corev1.Protocol, 0) + isFixed := false + for _, c := range conf { + switch c.Name { + case ClbIdsConfigName: + for _, clbId := range strings.Split(c.Value, ",") { + if clbId != "" { + lbIds = append(lbIds, clbId) + } + } + case PortProtocolsConfigName: + for _, pp := range strings.Split(c.Value, ",") { + ppSlice := strings.Split(pp, "/") + port, err := strconv.Atoi(ppSlice[0]) + if err != nil { + continue + } + ports = append(ports, port) + if len(ppSlice) != 2 { + protocols = append(protocols, corev1.ProtocolTCP) + } else { + protocols = append(protocols, corev1.Protocol(ppSlice[1])) + } + } + case FixedConfigName: + v, err := strconv.ParseBool(c.Value) + if err != nil { + continue + } + isFixed = v + } + } + return &clbConfig{ + lbIds: lbIds, + protocols: protocols, + targetPorts: ports, + isFixed: isFixed, + } +} + +func getPorts(ports []corev1.ServicePort) []int32 { + var ret []int32 + for _, port := range ports { + ret = append(ret, port.Port) + } + return ret +} + +func (c *ClbPlugin) consSvc(config *clbConfig, pod *corev1.Pod, client client.Client, ctx context.Context) *corev1.Service { + var ports []int32 + var lbId string + podKey := pod.GetNamespace() + "/" + pod.GetName() + allocatedPorts, exist := c.podAllocate[podKey] + if exist { + clbPorts := strings.Split(allocatedPorts, ":") + lbId = clbPorts[0] + ports = util.StringToInt32Slice(clbPorts[1], ",") + } else { + lbId, ports = c.allocate(config.lbIds, len(config.targetPorts), podKey) + } + + svcPorts := make([]corev1.ServicePort, 0) + for i := 0; i < len(config.targetPorts); i++ { + svcPorts = append(svcPorts, corev1.ServicePort{ + Name: strconv.Itoa(config.targetPorts[i]), + Port: ports[i], + Protocol: config.protocols[i], + TargetPort: intstr.FromInt(config.targetPorts[i]), + }) + } + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: pod.GetName(), + Namespace: pod.GetNamespace(), + Annotations: map[string]string{ + ClbSchedulerKey: ClbSchedulerWRR, + ClbAddressTypeKey: ClbAddressTypePublic, + ClbIdAnnotationKey: lbId, + ClbConfigHashKey: util.GetHash(config), + }, + OwnerReferences: getSvcOwnerReference(client, ctx, pod, config.isFixed), + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{ + SvcSelectorKey: pod.GetName(), + }, + Ports: svcPorts, + }, + } + return svc +} + +func getSvcOwnerReference(c client.Client, ctx context.Context, pod *corev1.Pod, isFixed bool) []metav1.OwnerReference { + ownerReferences := []metav1.OwnerReference{ + { + APIVersion: pod.APIVersion, + Kind: pod.Kind, + Name: pod.GetName(), + UID: pod.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + } + if isFixed { + gss, err := util.GetGameServerSetOfPod(pod, c, ctx) + if err == nil { + ownerReferences = []metav1.OwnerReference{ + { + APIVersion: gss.APIVersion, + Kind: gss.Kind, + Name: gss.GetName(), + UID: gss.GetUID(), + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + } + } + } + return ownerReferences +} diff --git a/cloudprovider/volcengine/clb_test.go b/cloudprovider/volcengine/clb_test.go new file mode 100644 index 0000000..028106f --- /dev/null +++ b/cloudprovider/volcengine/clb_test.go @@ -0,0 +1,335 @@ +/* +Copyright 2024 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volcengine + +import ( + "context" + "reflect" + "sync" + "testing" + + gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1" + "github.com/openkruise/kruise-game/pkg/util" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/utils/pointer" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func TestAllocateDeAllocate(t *testing.T) { + test := struct { + lbIds []string + clb *ClbPlugin + num int + podKey string + }{ + lbIds: []string{"xxx-A"}, + clb: &ClbPlugin{ + maxPort: int32(712), + minPort: int32(512), + cache: make(map[string]portAllocated), + podAllocate: make(map[string]string), + mutex: sync.RWMutex{}, + }, + podKey: "xxx/xxx", + num: 3, + } + + lbId, ports := test.clb.allocate(test.lbIds, test.num, test.podKey) + if _, exist := test.clb.podAllocate[test.podKey]; !exist { + t.Errorf("podAllocate[%s] is empty after allocated", test.podKey) + } + for _, port := range ports { + if port > test.clb.maxPort || port < test.clb.minPort { + t.Errorf("allocate port %d, unexpected", port) + } + if test.clb.cache[lbId][port] == false { + t.Errorf("Allocate port %d failed", port) + } + } + + test.clb.deAllocate(test.podKey) + for _, port := range ports { + if test.clb.cache[lbId][port] == true { + t.Errorf("deAllocate port %d failed", port) + } + } + if _, exist := test.clb.podAllocate[test.podKey]; exist { + t.Errorf("podAllocate[%s] is not empty after deallocated", test.podKey) + } +} + +func TestParseLbConfig(t *testing.T) { + tests := []struct { + conf []gamekruiseiov1alpha1.NetworkConfParams + lbIds []string + ports []int + protocols []corev1.Protocol + isFixed bool + }{ + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: ClbIdsConfigName, + Value: "xxx-A", + }, + { + Name: PortProtocolsConfigName, + Value: "80", + }, + }, + lbIds: []string{"xxx-A"}, + ports: []int{80}, + protocols: []corev1.Protocol{corev1.ProtocolTCP}, + isFixed: false, + }, + { + conf: []gamekruiseiov1alpha1.NetworkConfParams{ + { + Name: ClbIdsConfigName, + Value: "xxx-A,xxx-B,", + }, + { + Name: PortProtocolsConfigName, + Value: "81/UDP,82,83/TCP", + }, + { + Name: FixedConfigName, + Value: "true", + }, + }, + lbIds: []string{"xxx-A", "xxx-B"}, + ports: []int{81, 82, 83}, + protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP}, + isFixed: true, + }, + } + + for _, test := range tests { + sc := parseLbConfig(test.conf) + if !reflect.DeepEqual(test.lbIds, sc.lbIds) { + t.Errorf("lbId expect: %v, actual: %v", test.lbIds, sc.lbIds) + } + if !util.IsSliceEqual(test.ports, sc.targetPorts) { + t.Errorf("ports expect: %v, actual: %v", test.ports, sc.targetPorts) + } + if !reflect.DeepEqual(test.protocols, sc.protocols) { + t.Errorf("protocols expect: %v, actual: %v", test.protocols, sc.protocols) + } + if test.isFixed != sc.isFixed { + t.Errorf("isFixed expect: %v, actual: %v", test.isFixed, sc.isFixed) + } + } +} + +func TestInitLbCache(t *testing.T) { + test := struct { + svcList []corev1.Service + minPort int32 + maxPort int32 + cache map[string]portAllocated + podAllocate map[string]string + }{ + minPort: 512, + maxPort: 712, + cache: map[string]portAllocated{ + "xxx-A": map[int32]bool{ + 666: true, + }, + "xxx-B": map[int32]bool{ + 555: true, + }, + }, + podAllocate: map[string]string{ + "ns-0/name-0": "xxx-A:666", + "ns-1/name-1": "xxx-B:555", + }, + svcList: []corev1.Service{ + { + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + ClbIdLabelKey: "xxx-A", + }, + Namespace: "ns-0", + Name: "name-0", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{ + SvcSelectorKey: "pod-A", + }, + Ports: []corev1.ServicePort{ + { + TargetPort: intstr.FromInt(80), + Port: 666, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + { + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + ClbIdLabelKey: "xxx-B", + }, + Namespace: "ns-1", + Name: "name-1", + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{ + SvcSelectorKey: "pod-B", + }, + Ports: []corev1.ServicePort{ + { + TargetPort: intstr.FromInt(8080), + Port: 555, + Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + } + + actualCache, actualPodAllocate := initLbCache(test.svcList, test.minPort, test.maxPort) + for lb, pa := range test.cache { + for port, isAllocated := range pa { + if actualCache[lb][port] != isAllocated { + t.Errorf("lb %s port %d isAllocated, expect: %t, actual: %t", lb, port, isAllocated, actualCache[lb][port]) + } + } + } + if !reflect.DeepEqual(actualPodAllocate, test.podAllocate) { + t.Errorf("podAllocate expect %v, but actully got %v", test.podAllocate, actualPodAllocate) + } +} + +func TestClbPlugin_consSvc(t *testing.T) { + type fields struct { + maxPort int32 + minPort int32 + cache map[string]portAllocated + podAllocate map[string]string + } + type args struct { + config *clbConfig + pod *corev1.Pod + client client.Client + ctx context.Context + } + tests := []struct { + name string + fields fields + args args + want *corev1.Service + }{ + { + name: "convert svc cache exist", + fields: fields{ + maxPort: 3000, + minPort: 1, + cache: map[string]portAllocated{ + "default/test-pod": map[int32]bool{}, + }, + podAllocate: map[string]string{ + "default/test-pod": "clb-xxx:80,81", + }, + }, + args: args{ + config: &clbConfig{ + lbIds: []string{"clb-xxx"}, + targetPorts: []int{82}, + protocols: []corev1.Protocol{ + corev1.ProtocolTCP, + }, + isFixed: false, + }, + pod: &corev1.Pod{ + TypeMeta: metav1.TypeMeta{ + Kind: "pod", + APIVersion: "v1", + }, + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + UID: "32fqwfqfew", + }, + }, + client: nil, + ctx: context.Background(), + }, + want: &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "test-pod", + Namespace: "default", + Annotations: map[string]string{ + ClbSchedulerKey: ClbSchedulerWRR, + ClbAddressTypeKey: ClbAddressTypePublic, + ClbIdAnnotationKey: "clb-xxx", + ClbConfigHashKey: util.GetHash(&clbConfig{ + lbIds: []string{"clb-xxx"}, + targetPorts: []int{82}, + protocols: []corev1.Protocol{ + corev1.ProtocolTCP, + }, + isFixed: false, + }), + }, + OwnerReferences: []metav1.OwnerReference{ + { + APIVersion: "v1", + Kind: "pod", + Name: "test-pod", + UID: "32fqwfqfew", + Controller: pointer.BoolPtr(true), + BlockOwnerDeletion: pointer.BoolPtr(true), + }, + }, + }, + Spec: corev1.ServiceSpec{ + Type: corev1.ServiceTypeLoadBalancer, + Selector: map[string]string{ + SvcSelectorKey: "test-pod", + }, + Ports: []corev1.ServicePort{{ + Name: "82", + Port: 80, + Protocol: "TCP", + TargetPort: intstr.IntOrString{ + Type: 0, + IntVal: 82, + }, + }, + }, + }, + }, + }, + } + for _, tt := range tests { + c := &ClbPlugin{ + maxPort: tt.fields.maxPort, + minPort: tt.fields.minPort, + cache: tt.fields.cache, + podAllocate: tt.fields.podAllocate, + } + if got := c.consSvc(tt.args.config, tt.args.pod, tt.args.client, tt.args.ctx); !reflect.DeepEqual(got, tt.want) { + t.Errorf("consSvc() = %v, want %v", got, tt.want) + } + } +} diff --git a/cloudprovider/volcengine/volcengine.go b/cloudprovider/volcengine/volcengine.go new file mode 100644 index 0000000..f840036 --- /dev/null +++ b/cloudprovider/volcengine/volcengine.go @@ -0,0 +1,61 @@ +/* +Copyright 2024 The Kruise Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package volcengine + +import ( + "github.com/openkruise/kruise-game/cloudprovider" + "k8s.io/klog/v2" +) + +const ( + Volcengine = "Volcengine" +) + +var ( + volcengineProvider = &Provider{ + plugins: make(map[string]cloudprovider.Plugin), + } +) + +type Provider struct { + plugins map[string]cloudprovider.Plugin +} + +func (vp *Provider) Name() string { + return Volcengine +} + +func (vp *Provider) ListPlugins() (map[string]cloudprovider.Plugin, error) { + if vp.plugins == nil { + return make(map[string]cloudprovider.Plugin), nil + } + + return vp.plugins, nil +} + +// register plugin of cloud provider and different cloud providers +func (vp *Provider) registerPlugin(plugin cloudprovider.Plugin) { + name := plugin.Name() + if name == "" { + klog.Fatal("empty plugin name") + } + vp.plugins[name] = plugin +} + +func NewVolcengineProvider() (cloudprovider.CloudProvider, error) { + return volcengineProvider, nil +} diff --git a/config/manager/config.toml b/config/manager/config.toml index 69a125b..c6c43e1 100644 --- a/config/manager/config.toml +++ b/config/manager/config.toml @@ -9,3 +9,9 @@ enable = true [alibabacloud.slb] max_port = 700 min_port = 500 + +[volcengine] +enable = true +[volcengine.clb] +max_port = 700 +min_port = 500