feat: add jdcloud provider and the nlb&eip plugin (#180)
This commit is contained in:
hhr
GitHub
parent
468b2c77fb
commit
6bba287858
|
|
@ -48,6 +48,7 @@ type CloudProviderConfig struct {
|
||||||
VolcengineOptions CloudProviderOptions
|
VolcengineOptions CloudProviderOptions
|
||||||
AmazonsWebServicesOptions CloudProviderOptions
|
AmazonsWebServicesOptions CloudProviderOptions
|
||||||
TencentCloudOptions CloudProviderOptions
|
TencentCloudOptions CloudProviderOptions
|
||||||
|
JdCloudOptions CloudProviderOptions
|
||||||
}
|
}
|
||||||
|
|
||||||
type tomlConfigs struct {
|
type tomlConfigs struct {
|
||||||
|
|
@ -56,6 +57,7 @@ type tomlConfigs struct {
|
||||||
Volcengine options.VolcengineOptions `toml:"volcengine"`
|
Volcengine options.VolcengineOptions `toml:"volcengine"`
|
||||||
AmazonsWebServices options.AmazonsWebServicesOptions `toml:"aws"`
|
AmazonsWebServices options.AmazonsWebServicesOptions `toml:"aws"`
|
||||||
TencentCloud options.TencentCloudOptions `toml:"tencentcloud"`
|
TencentCloud options.TencentCloudOptions `toml:"tencentcloud"`
|
||||||
|
JdCloud options.JdCloudOptions `toml:"jdcloud"`
|
||||||
}
|
}
|
||||||
|
|
||||||
func (cf *ConfigFile) Parse() *CloudProviderConfig {
|
func (cf *ConfigFile) Parse() *CloudProviderConfig {
|
||||||
|
|
@ -70,6 +72,7 @@ func (cf *ConfigFile) Parse() *CloudProviderConfig {
|
||||||
VolcengineOptions: config.Volcengine,
|
VolcengineOptions: config.Volcengine,
|
||||||
AmazonsWebServicesOptions: config.AmazonsWebServices,
|
AmazonsWebServicesOptions: config.AmazonsWebServices,
|
||||||
TencentCloudOptions: config.TencentCloud,
|
TencentCloudOptions: config.TencentCloud,
|
||||||
|
JdCloudOptions: config.JdCloud,
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,193 @@
|
||||||
|
English | [中文](./README.md)
|
||||||
|
|
||||||
|
Based on JdCloud Container Service, for game scenarios, combine OKG to provide various network model plugins.
|
||||||
|
|
||||||
|
## JdCloud-NLB configuration
|
||||||
|
|
||||||
|
JdCloud Container Service supports the reuse of NLB (Network Load Balancer) in Kubernetes. Different services (svcs) can use different ports of the same NLB. As a result, the JdCloud-NLB network plugin will record the port allocation for each NLB. For services that specify the network type as JdCloud-NLB, the JdCloud-NLB network plugin will automatically allocate a port and create a service object. Once it detects that the public IP of the svc has been successfully created, the GameServer's network will transition to the Ready state, completing the process.
|
||||||
|
|
||||||
|
### plugin configuration
|
||||||
|
```toml
|
||||||
|
[jdcloud]
|
||||||
|
enable = true
|
||||||
|
[jdcloud.nlb]
|
||||||
|
#To allocate external access ports for Pods, you need to define the idle port ranges that the NLB (Network Load Balancer) can use. The maximum range for each port segment is 200 ports.
|
||||||
|
max_port = 700
|
||||||
|
min_port = 500
|
||||||
|
```
|
||||||
|
|
||||||
|
### Parameter
|
||||||
|
#### NlbIds
|
||||||
|
- Meaning：fill in the id of the clb. You can fill in more than one. You need to create the clb in [JdCloud].
|
||||||
|
- Value：each clbId is divided by `,` . For example：`netlb-aaa,netlb-bbb,...`
|
||||||
|
- Configurable：Y
|
||||||
|
|
||||||
|
#### PortProtocols
|
||||||
|
- Meaning：the ports and protocols exposed by the pod, support filling in multiple ports/protocols
|
||||||
|
- Value：`port1/protocol1`,`port2/protocol2`,... The protocol names must be in uppercase letters.
|
||||||
|
- Configurable：Y
|
||||||
|
|
||||||
|
#### Fixed
|
||||||
|
- Meaning：whether the mapping relationship is fixed. If the mapping relationship is fixed, the mapping relationship remains unchanged even if the pod is deleted and recreated.
|
||||||
|
- Value：false / true
|
||||||
|
- Configurable：Y
|
||||||
|
|
||||||
|
#### AllocateLoadBalancerNodePorts
|
||||||
|
- Meaning：Whether the generated service is assigned nodeport, this can be set to false only in nlb passthrough mode
|
||||||
|
- Value：false / true
|
||||||
|
- Configurable：Y
|
||||||
|
|
||||||
|
#### AllowNotReadyContainers
|
||||||
|
- Meaning：the container names that are allowed not ready when inplace updating, when traffic will not be cut.
|
||||||
|
- Value：{containerName_0},{containerName_1},... eg：sidecar
|
||||||
|
- Configurable：It cannot be changed during the in-place updating process.
|
||||||
|
|
||||||
|
#### Annotations
|
||||||
|
- Meaning：the anno added to the service
|
||||||
|
- Value：key1:value1,key2:value2...
|
||||||
|
- Configurable：Y
|
||||||
|
|
||||||
|
|
||||||
|
### Example
|
||||||
|
```yaml
|
||||||
|
cat <<EOF | kubectl apply -f -
|
||||||
|
apiVersion: game.kruise.io/v1alpha1
|
||||||
|
kind: GameServerSet
|
||||||
|
metadata:
|
||||||
|
name: nlb
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
replicas: 3
|
||||||
|
updateStrategy:
|
||||||
|
rollingUpdate:
|
||||||
|
podUpdatePolicy: InPlaceIfPossible
|
||||||
|
network:
|
||||||
|
networkType: JdCloud-NLB
|
||||||
|
networkConf:
|
||||||
|
- name: NlbIds
|
||||||
|
#Fill in Jdcloud Cloud LoadBalancer Id here
|
||||||
|
value: netlb-xxxxx
|
||||||
|
- name: PortProtocols
|
||||||
|
#Fill in the exposed ports and their corresponding protocols here.
|
||||||
|
#If there are multiple ports, the format is as follows: {port1}/{protocol1},{port2}/{protocol2}...
|
||||||
|
#If the protocol is not filled in, the default is TCP
|
||||||
|
value: 80/TCP
|
||||||
|
- name: AllocateLoadBalancerNodePorts
|
||||||
|
# Whether the generated service is assigned nodeport.
|
||||||
|
value: "true"
|
||||||
|
- name: Fixed
|
||||||
|
#Fill in here whether a fixed IP is required [optional] ; Default is false
|
||||||
|
value: "false"
|
||||||
|
- name: Annotations
|
||||||
|
#Fill in the anno related to clb on the service
|
||||||
|
#The format is as follows: {key1}:{value1},{key2}:{value2}...
|
||||||
|
value: "key1:value1,key2:value2"
|
||||||
|
gameServerTemplate:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- args:
|
||||||
|
- /data/server/start.sh
|
||||||
|
command:
|
||||||
|
- /bin/bash
|
||||||
|
image: gss-cn-north-1.jcr.service.jdcloud.com/gsshosting/pal:v1
|
||||||
|
name: game-server
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
Check the network status in GameServer:
|
||||||
|
```
|
||||||
|
networkStatus:
|
||||||
|
createTime: "2024-11-04T08:00:20Z"
|
||||||
|
currentNetworkState: Ready
|
||||||
|
desiredNetworkState: Ready
|
||||||
|
externalAddresses:
|
||||||
|
- ip: xxx.xxx.xxx.xxx
|
||||||
|
ports:
|
||||||
|
- name: "8211"
|
||||||
|
port: 531
|
||||||
|
protocol: UDP
|
||||||
|
internalAddresses:
|
||||||
|
- ip: 10.0.0.95
|
||||||
|
ports:
|
||||||
|
- name: "8211"
|
||||||
|
port: 8211
|
||||||
|
protocol: UDP
|
||||||
|
lastTransitionTime: "2024-11-04T08:00:20Z"
|
||||||
|
networkType: JdCloud-NLB
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
## JdCloud-EIP configuration
|
||||||
|
JdCloud Container Service supports binding an Elastic Public IP directly to a pod in Kubernetes, allowing the pod to communicate directly with the external network.
|
||||||
|
- The cluster's network plugin uses Yunjian-CNI and cannot use Flannel to create the cluster.
|
||||||
|
- For specific usage restrictions of Elastic Public IPs, please refer to the JdCloud Elastic Public IP product documentation.
|
||||||
|
- Install the EIP-Controller component.
|
||||||
|
- The Elastic Public IP will not be deleted when the pod is destroyed.
|
||||||
|
|
||||||
|
### Parameter
|
||||||
|
|
||||||
|
#### BandwidthConfigName
|
||||||
|
- Meaning：The bandwidth of the Elastic Public IP, measured in Mbps, has a value range of [1, 1024].
|
||||||
|
- Value：Must be an integer
|
||||||
|
- Configurable：Y
|
||||||
|
|
||||||
|
#### ChargeTypeConfigName
|
||||||
|
- Meaning：The billing method for the Elastic Public IP
|
||||||
|
- Value：string, `postpaid_by_usage`/`postpaid_by_duration`
|
||||||
|
- Configurable：Y
|
||||||
|
|
||||||
|
#### FixedEIPConfigName
|
||||||
|
- Meaning：Whether to fixed the Elastic Public IP,if so, the EIP will not be changed when the pod is recreated.
|
||||||
|
- Value：string, "false" / "true"
|
||||||
|
- Configurable：Y
|
||||||
|
|
||||||
|
#### AssignEIPConfigName
|
||||||
|
- Meaning：Whether to designate a specific Elastic Public IP. If true, provide the ID of the Elastic Public IP; otherwise, an EIP will be automatically allocated.
|
||||||
|
- Value：string, "false" / "true"
|
||||||
|
|
||||||
|
#### EIPIdConfigName
|
||||||
|
- Meaning：If a specific Elastic Public IP is designated, the ID of the Elastic Public IP must be provided, and the component will automatically perform the lookup and binding.
|
||||||
|
- Value：string，for example：`fip-xxxxxxxx`
|
||||||
|
|
||||||
|
### Example
|
||||||
|
```yaml
|
||||||
|
cat <<EOF | kubectl apply -f -
|
||||||
|
apiVersion: game.kruise.io/v1alpha1
|
||||||
|
kind: GameServerSet
|
||||||
|
metadata:
|
||||||
|
name: eip
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- args:
|
||||||
|
- /data/server/start.sh
|
||||||
|
command:
|
||||||
|
- /bin/bash
|
||||||
|
image: gss-cn-north-1.jcr.service.jdcloud.com/gsshosting/pal:v1
|
||||||
|
name: game-server
|
||||||
|
network:
|
||||||
|
networkType: JdCloud-EIP
|
||||||
|
networkConf:
|
||||||
|
- name: "BandWidth"
|
||||||
|
value: "10"
|
||||||
|
- name: "ChargeType"
|
||||||
|
value: postpaid_by_usage
|
||||||
|
- name: "Fixed"
|
||||||
|
value: "false"
|
||||||
|
replicas: 3
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
Check the network status in GameServer:
|
||||||
|
```
|
||||||
|
networkStatus:
|
||||||
|
createTime: "2024-11-04T10:53:14Z"
|
||||||
|
currentNetworkState: Ready
|
||||||
|
desiredNetworkState: Ready
|
||||||
|
externalAddresses:
|
||||||
|
- ip: xxx.xxx.xxx.xxx
|
||||||
|
internalAddresses:
|
||||||
|
- ip: 10.0.0.95
|
||||||
|
lastTransitionTime: "2024-11-04T10:53:14Z"
|
||||||
|
networkType: JdCloud-EIP
|
||||||
|
```
|
||||||
|
|
@ -0,0 +1,192 @@
|
||||||
|
中文 | [English](./README.md)
|
||||||
|
|
||||||
|
基于京东云容器服务，针对游戏场景，结合OKG提供各网络模型插件。
|
||||||
|
|
||||||
|
## JdCloud-NLB 相关配置
|
||||||
|
|
||||||
|
京东云容器服务支持在k8s中对NLB复用的机制，不同的svc可以使用同一个NLB的不同端口。由此，JdCloud-NLB network plugin将记录各NLB对应的端口分配情况，对于指定了网络类型为JdCloud-NLB，JdCloud-NLB网络插件将会自动分配一个端口并创建一个service对象，待检测到svc公网IP创建成功后，GameServer的网络变为Ready状态，该过程执行完成。
|
||||||
|
|
||||||
|
### plugin配置
|
||||||
|
```toml
|
||||||
|
[jdcloud]
|
||||||
|
enable = true
|
||||||
|
[jdcloud.nlb]
|
||||||
|
#填写nlb可使用的空闲端口段，用于为pod分配外部接入端口，范围最大为200
|
||||||
|
max_port = 700
|
||||||
|
min_port = 500
|
||||||
|
```
|
||||||
|
### 参数
|
||||||
|
#### NlbIds
|
||||||
|
- 含义：填写nlb的id，可填写多个，需要先在【京东云】中创建好nlb。
|
||||||
|
- 填写格式：各个nlbId用,分割。例如：netlb-aaa,netlb-bbb,...
|
||||||
|
- 是否支持变更：是
|
||||||
|
|
||||||
|
#### PortProtocols
|
||||||
|
- 含义：pod暴露的端口及协议，支持填写多个端口/协议
|
||||||
|
- 填写格式：port1/protocol1,port2/protocol2,...（协议需大写）
|
||||||
|
- 是否支持变更：是
|
||||||
|
|
||||||
|
#### Fixed
|
||||||
|
- 含义：是否固定访问IP/端口。若是，即使pod删除重建，网络内外映射关系不会改变
|
||||||
|
- 填写格式：false / true
|
||||||
|
- 是否支持变更：是
|
||||||
|
|
||||||
|
#### AllocateLoadBalancerNodePorts
|
||||||
|
- 含义：生成的service是否分配nodeport, 仅在nlb的直通模式（passthrough）下，才能设置为false
|
||||||
|
- 填写格式：true/false
|
||||||
|
- 是否支持变更：是
|
||||||
|
|
||||||
|
#### AllowNotReadyContainers
|
||||||
|
- 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个
|
||||||
|
- 填写格式：{containerName_0},{containerName_1},... 例如：sidecar
|
||||||
|
- 是否支持变更：在原地升级过程中不可变更
|
||||||
|
|
||||||
|
#### Annotations
|
||||||
|
- 含义：添加在service上的anno，可填写多个
|
||||||
|
- 填写格式：key1:value1,key2:value2...
|
||||||
|
- 是否支持变更：是
|
||||||
|
|
||||||
|
|
||||||
|
### 使用示例
|
||||||
|
```yaml
|
||||||
|
cat <<EOF | kubectl apply -f -
|
||||||
|
apiVersion: game.kruise.io/v1alpha1
|
||||||
|
kind: GameServerSet
|
||||||
|
metadata:
|
||||||
|
name: nlb
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
replicas: 3
|
||||||
|
updateStrategy:
|
||||||
|
rollingUpdate:
|
||||||
|
podUpdatePolicy: InPlaceIfPossible
|
||||||
|
network:
|
||||||
|
networkType: JdCloud-NLB
|
||||||
|
networkConf:
|
||||||
|
- name: NlbIds
|
||||||
|
#Fill in Jdcloud Cloud LoadBalancer Id here
|
||||||
|
value: netlb-xxxxx
|
||||||
|
- name: PortProtocols
|
||||||
|
#Fill in the exposed ports and their corresponding protocols here.
|
||||||
|
#If there are multiple ports, the format is as follows: {port1}/{protocol1},{port2}/{protocol2}...
|
||||||
|
#If the protocol is not filled in, the default is TCP
|
||||||
|
value: 80/TCP
|
||||||
|
- name: AllocateLoadBalancerNodePorts
|
||||||
|
# Whether the generated service is assigned nodeport.
|
||||||
|
value: "true"
|
||||||
|
- name: Fixed
|
||||||
|
#Fill in here whether a fixed IP is required [optional] ; Default is false
|
||||||
|
value: "false"
|
||||||
|
- name: Annotations
|
||||||
|
#Fill in the anno related to clb on the service
|
||||||
|
#The format is as follows: {key1}:{value1},{key2}:{value2}...
|
||||||
|
value: "key1:value1,key2:value2"
|
||||||
|
gameServerTemplate:
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- args:
|
||||||
|
- /data/server/start.sh
|
||||||
|
command:
|
||||||
|
- /bin/bash
|
||||||
|
image: gss-cn-north-1.jcr.service.jdcloud.com/gsshosting/pal:v1
|
||||||
|
name: game-server
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
检查GameServer中的网络状态:
|
||||||
|
```
|
||||||
|
networkStatus:
|
||||||
|
createTime: "2024-11-04T08:00:20Z"
|
||||||
|
currentNetworkState: Ready
|
||||||
|
desiredNetworkState: Ready
|
||||||
|
externalAddresses:
|
||||||
|
- ip: xxx.xxx.xxx.xxx
|
||||||
|
ports:
|
||||||
|
- name: "8211"
|
||||||
|
port: 531
|
||||||
|
protocol: UDP
|
||||||
|
internalAddresses:
|
||||||
|
- ip: 10.0.0.95
|
||||||
|
ports:
|
||||||
|
- name: "8211"
|
||||||
|
port: 8211
|
||||||
|
protocol: UDP
|
||||||
|
lastTransitionTime: "2024-11-04T08:00:20Z"
|
||||||
|
networkType: JdCloud-NLB
|
||||||
|
```
|
||||||
|
|
||||||
|
|
||||||
|
## JdCloud-EIP 相关配置
|
||||||
|
京东云容器服务支持在k8s中，让一个 pod 和弹性公网 IP 直接进行绑定，可以让 pod 直接与外部网络进行通信。
|
||||||
|
- 集群的网络插件使用 yunjian-CNI，不可使用 flannel 创建集群
|
||||||
|
- 弹性公网 IP 使用限制请具体参考京东云弹性公网 IP 产品文档
|
||||||
|
- 安装 EIP-Controller 组件
|
||||||
|
- 弹性公网 IP 不会随 POD 的销毁而删除
|
||||||
|
|
||||||
|
### 参数
|
||||||
|
|
||||||
|
#### BandwidthConfigName
|
||||||
|
- 含义：弹性公网IP的带宽，单位为 Mbps，取值范围为 [1,1024]
|
||||||
|
- 填写格式：必须填整数，且不带单位
|
||||||
|
- 是否支持变更：是
|
||||||
|
|
||||||
|
#### ChargeTypeConfigName
|
||||||
|
- 含义：弹性公网IP的计费方式，取值：按量计费：postpaid_by_usage，包年包月：postpaid_by_duration
|
||||||
|
- 填写格式：字符串
|
||||||
|
- 是否支持变更：是
|
||||||
|
|
||||||
|
#### FixedEIPConfigName
|
||||||
|
- 含义：是否固定弹性公网IP。若是，即使pod删除重建，弹性公网IP也不会改变
|
||||||
|
- 填写格式："false" / "true"，字符串
|
||||||
|
- 是否支持变更：是
|
||||||
|
|
||||||
|
#### AssignEIPConfigName
|
||||||
|
- 含义：是否指定使用某个弹性公网IP，请填写 true，否则自动分配一个EIP
|
||||||
|
- 填写格式："false" / "true"，字符串
|
||||||
|
|
||||||
|
#### EIPIdConfigName
|
||||||
|
- 含义：若指定使用某个弹性公网IP，则必须填写弹性公网IP的ID，，组件会自动进行进行查询和绑定
|
||||||
|
- 填写格式：字符串，例如：fip-xxxxxxxx
|
||||||
|
|
||||||
|
### 使用示例
|
||||||
|
```yaml
|
||||||
|
cat <<EOF | kubectl apply -f -
|
||||||
|
apiVersion: game.kruise.io/v1alpha1
|
||||||
|
kind: GameServerSet
|
||||||
|
metadata:
|
||||||
|
name: eip
|
||||||
|
namespace: default
|
||||||
|
spec:
|
||||||
|
containers:
|
||||||
|
- args:
|
||||||
|
- /data/server/start.sh
|
||||||
|
command:
|
||||||
|
- /bin/bash
|
||||||
|
image: gss-cn-north-1.jcr.service.jdcloud.com/gsshosting/pal:v1
|
||||||
|
name: game-server
|
||||||
|
network:
|
||||||
|
networkType: JdCloud-EIP
|
||||||
|
networkConf:
|
||||||
|
- name: "BandWidth"
|
||||||
|
value: "10"
|
||||||
|
- name: "ChargeType"
|
||||||
|
value: postpaid_by_usage
|
||||||
|
- name: "Fixed"
|
||||||
|
value: "false"
|
||||||
|
replicas: 3
|
||||||
|
EOF
|
||||||
|
```
|
||||||
|
|
||||||
|
检查GameServer中的网络状态:
|
||||||
|
```
|
||||||
|
networkStatus:
|
||||||
|
createTime: "2024-11-04T10:53:14Z"
|
||||||
|
currentNetworkState: Ready
|
||||||
|
desiredNetworkState: Ready
|
||||||
|
externalAddresses:
|
||||||
|
- ip: xxx.xxx.xxx.xxx
|
||||||
|
internalAddresses:
|
||||||
|
- ip: 10.0.0.95
|
||||||
|
lastTransitionTime: "2024-11-04T10:53:14Z"
|
||||||
|
networkType: JdCloud-EIP
|
||||||
|
```
|
||||||
|
|
@ -0,0 +1,111 @@
|
||||||
|
package jdcloud
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
cerr "errors"
|
||||||
|
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
|
||||||
|
"github.com/openkruise/kruise-game/cloudprovider"
|
||||||
|
"github.com/openkruise/kruise-game/cloudprovider/errors"
|
||||||
|
"github.com/openkruise/kruise-game/cloudprovider/utils"
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
EIPNetwork = "JdCloud-EIP"
|
||||||
|
AliasSEIP = "EIP-Network"
|
||||||
|
EIPIdConfigName = "EIPId"
|
||||||
|
EIPIdAnnotationKey = "jdos.jd.com/eip.id"
|
||||||
|
EIPIfaceAnnotationKey = "jdos.jd.com/eip.iface"
|
||||||
|
EIPAnnotationKey = "jdos.jd.com/eip.ip"
|
||||||
|
|
||||||
|
BandwidthConfigName = "Bandwidth"
|
||||||
|
BandwidthAnnotationkey = "jdos.jd.com/eip.bandwith"
|
||||||
|
ChargeTypeConfigName = "ChargeType"
|
||||||
|
ChargeTypeAnnotationkey = "jdos.jd.com/eip.chargeMode"
|
||||||
|
EnableEIPAnnotationKey = "jdos.jd.com/eip.enable"
|
||||||
|
FixedEIPConfigName = "Fixed"
|
||||||
|
FixedEIPAnnotationKey = "jdos.jd.com/eip.static"
|
||||||
|
EIPNameAnnotationKey = "jdos.jd.com/eip-name"
|
||||||
|
AssignEIPConfigName = "AssignEIP"
|
||||||
|
AssignEIPAnnotationKey = "jdos.jd.com/eip.userAssign"
|
||||||
|
)
|
||||||
|
|
||||||
|
type EipPlugin struct {
|
||||||
|
}
|
||||||
|
|
||||||
|
func (E EipPlugin) Name() string {
|
||||||
|
return EIPNetwork
|
||||||
|
}
|
||||||
|
|
||||||
|
func (E EipPlugin) Alias() string {
|
||||||
|
return AliasSEIP
|
||||||
|
}
|
||||||
|
|
||||||
|
func (E EipPlugin) Init(client client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (E EipPlugin) OnPodAdded(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, errors.PluginError) {
|
||||||
|
networkManager := utils.NewNetworkManager(pod, client)
|
||||||
|
conf := networkManager.GetNetworkConfig()
|
||||||
|
|
||||||
|
pod.Annotations[EnableEIPAnnotationKey] = "true"
|
||||||
|
pod.Annotations[EIPNameAnnotationKey] = pod.GetNamespace() + "/" + pod.GetName()
|
||||||
|
//parse network configuration
|
||||||
|
for _, c := range conf {
|
||||||
|
switch c.Name {
|
||||||
|
case BandwidthConfigName:
|
||||||
|
pod.Annotations[BandwidthAnnotationkey] = c.Value
|
||||||
|
case ChargeTypeConfigName:
|
||||||
|
pod.Annotations[ChargeTypeAnnotationkey] = c.Value
|
||||||
|
case FixedEIPConfigName:
|
||||||
|
pod.Annotations[FixedEIPAnnotationKey] = c.Value
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return pod, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (E EipPlugin) OnPodUpdated(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, errors.PluginError) {
|
||||||
|
networkManager := utils.NewNetworkManager(pod, client)
|
||||||
|
|
||||||
|
networkStatus, _ := networkManager.GetNetworkStatus()
|
||||||
|
if networkStatus == nil {
|
||||||
|
pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{
|
||||||
|
CurrentNetworkState: gamekruiseiov1alpha1.NetworkWaiting,
|
||||||
|
}, pod)
|
||||||
|
return pod, errors.ToPluginError(err, errors.InternalError)
|
||||||
|
}
|
||||||
|
|
||||||
|
if enable, ok := pod.Annotations[EnableEIPAnnotationKey]; !ok || (ok && enable != "true") {
|
||||||
|
return pod, errors.ToPluginError(cerr.New("eip plugin is not enabled"), errors.InternalError)
|
||||||
|
}
|
||||||
|
if _, ok := pod.Annotations[EIPIdAnnotationKey]; !ok {
|
||||||
|
return pod, nil
|
||||||
|
}
|
||||||
|
if _, ok := pod.Annotations[EIPAnnotationKey]; !ok {
|
||||||
|
return pod, nil
|
||||||
|
}
|
||||||
|
networkStatus.ExternalAddresses = []gamekruiseiov1alpha1.NetworkAddress{
|
||||||
|
{
|
||||||
|
IP: pod.Annotations[EIPAnnotationKey],
|
||||||
|
},
|
||||||
|
}
|
||||||
|
networkStatus.InternalAddresses = []gamekruiseiov1alpha1.NetworkAddress{
|
||||||
|
{
|
||||||
|
IP: pod.Status.PodIP,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady
|
||||||
|
|
||||||
|
pod, err := networkManager.UpdateNetworkStatus(*networkStatus, pod)
|
||||||
|
return pod, errors.ToPluginError(err, errors.InternalError)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (E EipPlugin) OnPodDeleted(client client.Client, pod *corev1.Pod, ctx context.Context) errors.PluginError {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
jdcloudProvider.registerPlugin(&EipPlugin{})
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1 @@
|
||||||
|
package jdcloud
|
||||||
|
|
@ -0,0 +1,61 @@
|
||||||
|
/*
|
||||||
|
Copyright 2024 The Kruise Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package jdcloud
|
||||||
|
|
||||||
|
import (
|
||||||
|
"github.com/openkruise/kruise-game/cloudprovider"
|
||||||
|
"k8s.io/klog/v2"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
Jdcloud = "Jdcloud"
|
||||||
|
)
|
||||||
|
|
||||||
|
var (
|
||||||
|
jdcloudProvider = &Provider{
|
||||||
|
plugins: make(map[string]cloudprovider.Plugin),
|
||||||
|
}
|
||||||
|
)
|
||||||
|
|
||||||
|
type Provider struct {
|
||||||
|
plugins map[string]cloudprovider.Plugin
|
||||||
|
}
|
||||||
|
|
||||||
|
func (jp *Provider) Name() string {
|
||||||
|
return Jdcloud
|
||||||
|
}
|
||||||
|
|
||||||
|
func (jp *Provider) ListPlugins() (map[string]cloudprovider.Plugin, error) {
|
||||||
|
if jp.plugins == nil {
|
||||||
|
return make(map[string]cloudprovider.Plugin), nil
|
||||||
|
}
|
||||||
|
|
||||||
|
return jp.plugins, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
// register plugin of cloud provider and different cloud providers
|
||||||
|
func (jp *Provider) registerPlugin(plugin cloudprovider.Plugin) {
|
||||||
|
name := plugin.Name()
|
||||||
|
if name == "" {
|
||||||
|
klog.Fatal("empty plugin name")
|
||||||
|
}
|
||||||
|
jp.plugins[name] = plugin
|
||||||
|
}
|
||||||
|
|
||||||
|
func NewJdcloudProvider() (cloudprovider.CloudProvider, error) {
|
||||||
|
return jdcloudProvider, nil
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,581 @@
|
||||||
|
/*
|
||||||
|
Copyright 2024 The Kruise Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package jdcloud
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"encoding/json"
|
||||||
|
"fmt"
|
||||||
|
"strconv"
|
||||||
|
"strings"
|
||||||
|
"sync"
|
||||||
|
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/api/errors"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/types"
|
||||||
|
"k8s.io/apimachinery/pkg/util/intstr"
|
||||||
|
log "k8s.io/klog/v2"
|
||||||
|
"k8s.io/utils/ptr"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||||
|
|
||||||
|
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
|
||||||
|
"github.com/openkruise/kruise-game/cloudprovider"
|
||||||
|
cperrors "github.com/openkruise/kruise-game/cloudprovider/errors"
|
||||||
|
provideroptions "github.com/openkruise/kruise-game/cloudprovider/options"
|
||||||
|
"github.com/openkruise/kruise-game/cloudprovider/utils"
|
||||||
|
"github.com/openkruise/kruise-game/pkg/util"
|
||||||
|
)
|
||||||
|
|
||||||
|
const (
|
||||||
|
LbType_NLB = "nlb"
|
||||||
|
)
|
||||||
|
|
||||||
|
type JdNLBElasticIp struct {
|
||||||
|
ElasticIpId string `json:"elasticIpId"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type JdNLBAlgorithm string
|
||||||
|
|
||||||
|
const (
|
||||||
|
JdNLBDefaultConnIdleTime int = 600
|
||||||
|
JdNLBAlgorithmRoundRobin JdNLBAlgorithm = "RoundRobin"
|
||||||
|
JdNLBAlgorithmLeastConn JdNLBAlgorithm = "LeastConn"
|
||||||
|
JdNLBAlgorithmIpHash JdNLBAlgorithm = "IpHash"
|
||||||
|
)
|
||||||
|
|
||||||
|
type JdNLBListenerBackend struct {
|
||||||
|
ProxyProtocol bool `json:"proxyProtocol"`
|
||||||
|
Algorithm JdNLBAlgorithm `json:"algorithm"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type JdNLBListener struct {
|
||||||
|
Protocol string `json:"protocol"`
|
||||||
|
ConnectionIdleTimeSeconds int `json:"connectionIdleTimeSeconds"`
|
||||||
|
Backend *JdNLBListenerBackend `json:"backend"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type JdNLB struct {
|
||||||
|
Version string `json:"version"`
|
||||||
|
LoadBalancerId string `json:"loadBalancerId"`
|
||||||
|
LoadBalancerType string `json:"loadBalancerType"`
|
||||||
|
Internal bool `json:"internal"`
|
||||||
|
Listeners []*JdNLBListener `json:"listeners"`
|
||||||
|
}
|
||||||
|
|
||||||
|
const (
|
||||||
|
JdNLBVersion = "v1"
|
||||||
|
NlbNetwork = "JdCloud-NLB"
|
||||||
|
AliasNLB = "NLB-Network"
|
||||||
|
NlbIdLabelKey = "service.beta.kubernetes.io/jdcloud-loadbalancer-id"
|
||||||
|
NlbIdsConfigName = "NlbIds"
|
||||||
|
PortProtocolsConfigName = "PortProtocols"
|
||||||
|
FixedConfigName = "Fixed"
|
||||||
|
AllocateLoadBalancerNodePorts = "AllocateLoadBalancerNodePorts"
|
||||||
|
NlbAnnotations = "Annotations"
|
||||||
|
NlbConfigHashKey = "game.kruise.io/network-config-hash"
|
||||||
|
NlbSpecAnnotationKey = "service.beta.kubernetes.io/jdcloud-load-balancer-spec"
|
||||||
|
SvcSelectorKey = "statefulset.kubernetes.io/pod-name"
|
||||||
|
NlbAlgorithm = "service.beta.kubernetes.io/jdcloud-lb-algorithm"
|
||||||
|
NlbConnectionIdleTime = "service.beta.kubernetes.io/jdcloud-lb-idle-time"
|
||||||
|
)
|
||||||
|
|
||||||
|
type portAllocated map[int32]bool
|
||||||
|
|
||||||
|
type NlbPlugin struct {
|
||||||
|
maxPort int32
|
||||||
|
minPort int32
|
||||||
|
cache map[string]portAllocated
|
||||||
|
podAllocate map[string]string
|
||||||
|
mutex sync.RWMutex
|
||||||
|
}
|
||||||
|
|
||||||
|
type nlbConfig struct {
|
||||||
|
lbIds []string
|
||||||
|
targetPorts []int
|
||||||
|
protocols []corev1.Protocol
|
||||||
|
isFixed bool
|
||||||
|
annotations map[string]string
|
||||||
|
allocateLoadBalancerNodePorts bool
|
||||||
|
algorithm string
|
||||||
|
connIdleTimeSeconds int
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *NlbPlugin) Name() string {
|
||||||
|
return NlbNetwork
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *NlbPlugin) Alias() string {
|
||||||
|
return AliasNLB
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *NlbPlugin) Init(client client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error {
|
||||||
|
c.mutex.Lock()
|
||||||
|
defer c.mutex.Unlock()
|
||||||
|
nlbOptions, ok := options.(provideroptions.JdCloudOptions)
|
||||||
|
if !ok {
|
||||||
|
return cperrors.ToPluginError(fmt.Errorf("failed to convert options to nlbOptions"), cperrors.InternalError)
|
||||||
|
}
|
||||||
|
c.minPort = nlbOptions.NLBOptions.MinPort
|
||||||
|
c.maxPort = nlbOptions.NLBOptions.MaxPort
|
||||||
|
|
||||||
|
svcList := &corev1.ServiceList{}
|
||||||
|
err := client.List(ctx, svcList)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
|
c.cache, c.podAllocate = initLbCache(svcList.Items, c.minPort, c.maxPort)
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func initLbCache(svcList []corev1.Service, minPort, maxPort int32) (map[string]portAllocated, map[string]string) {
|
||||||
|
newCache := make(map[string]portAllocated)
|
||||||
|
newPodAllocate := make(map[string]string)
|
||||||
|
for _, svc := range svcList {
|
||||||
|
lbId := svc.Labels[NlbIdLabelKey]
|
||||||
|
if lbId != "" && svc.Spec.Type == corev1.ServiceTypeLoadBalancer {
|
||||||
|
if newCache[lbId] == nil {
|
||||||
|
newCache[lbId] = make(portAllocated, maxPort-minPort)
|
||||||
|
for i := minPort; i < maxPort; i++ {
|
||||||
|
newCache[lbId][i] = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
var ports []int32
|
||||||
|
for _, port := range getPorts(svc.Spec.Ports) {
|
||||||
|
if port <= maxPort && port >= minPort {
|
||||||
|
newCache[lbId][port] = true
|
||||||
|
ports = append(ports, port)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if len(ports) != 0 {
|
||||||
|
newPodAllocate[svc.GetNamespace()+"/"+svc.GetName()] = lbId + ":" + util.Int32SliceToString(ports, ",")
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
log.Infof("[%s] podAllocate cache complete initialization: %v", NlbNetwork, newPodAllocate)
|
||||||
|
return newCache, newPodAllocate
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *NlbPlugin) OnPodAdded(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
|
||||||
|
return pod, nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *NlbPlugin) OnPodUpdated(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
|
||||||
|
networkManager := utils.NewNetworkManager(pod, client)
|
||||||
|
|
||||||
|
networkStatus, err := networkManager.GetNetworkStatus()
|
||||||
|
if err != nil {
|
||||||
|
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
|
||||||
|
}
|
||||||
|
networkConfig := networkManager.GetNetworkConfig()
|
||||||
|
config := parseLbConfig(networkConfig)
|
||||||
|
if networkStatus == nil {
|
||||||
|
pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{
|
||||||
|
CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady,
|
||||||
|
}, pod)
|
||||||
|
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
|
||||||
|
}
|
||||||
|
|
||||||
|
// get svc
|
||||||
|
svc := &corev1.Service{}
|
||||||
|
err = client.Get(ctx, types.NamespacedName{
|
||||||
|
Name: pod.GetName(),
|
||||||
|
Namespace: pod.GetNamespace(),
|
||||||
|
}, svc)
|
||||||
|
if err != nil {
|
||||||
|
if errors.IsNotFound(err) {
|
||||||
|
return pod, cperrors.ToPluginError(client.Create(ctx, c.consSvc(config, pod, client, ctx)), cperrors.ApiCallError)
|
||||||
|
}
|
||||||
|
return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
|
||||||
|
}
|
||||||
|
|
||||||
|
// update svc
|
||||||
|
if util.GetHash(config) != svc.GetAnnotations()[NlbConfigHashKey] {
|
||||||
|
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady
|
||||||
|
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
|
||||||
|
if err != nil {
|
||||||
|
return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
|
||||||
|
}
|
||||||
|
return pod, cperrors.ToPluginError(client.Update(ctx, c.consSvc(config, pod, client, ctx)), cperrors.ApiCallError)
|
||||||
|
}
|
||||||
|
|
||||||
|
// disable network
|
||||||
|
if networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeLoadBalancer {
|
||||||
|
svc.Spec.Type = corev1.ServiceTypeClusterIP
|
||||||
|
return pod, cperrors.ToPluginError(client.Update(ctx, svc), cperrors.ApiCallError)
|
||||||
|
}
|
||||||
|
|
||||||
|
// enable network
|
||||||
|
if !networkManager.GetNetworkDisabled() && svc.Spec.Type == corev1.ServiceTypeClusterIP {
|
||||||
|
svc.Spec.Type = corev1.ServiceTypeLoadBalancer
|
||||||
|
return pod, cperrors.ToPluginError(client.Update(ctx, svc), cperrors.ApiCallError)
|
||||||
|
}
|
||||||
|
|
||||||
|
// network not ready
|
||||||
|
if len(svc.Status.LoadBalancer.Ingress) == 0 {
|
||||||
|
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady
|
||||||
|
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
|
||||||
|
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
|
||||||
|
}
|
||||||
|
|
||||||
|
// allow not ready containers
|
||||||
|
if util.IsAllowNotReadyContainers(networkManager.GetNetworkConfig()) {
|
||||||
|
toUpDateSvc, err := utils.AllowNotReadyContainers(client, ctx, pod, svc, false)
|
||||||
|
if err != nil {
|
||||||
|
return pod, err
|
||||||
|
}
|
||||||
|
|
||||||
|
if toUpDateSvc {
|
||||||
|
err := client.Update(ctx, svc)
|
||||||
|
if err != nil {
|
||||||
|
return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// network ready
|
||||||
|
internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)
|
||||||
|
externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)
|
||||||
|
for _, port := range svc.Spec.Ports {
|
||||||
|
instrIPort := port.TargetPort
|
||||||
|
instrEPort := intstr.FromInt(int(port.Port))
|
||||||
|
internalAddress := gamekruiseiov1alpha1.NetworkAddress{
|
||||||
|
IP: pod.Status.PodIP,
|
||||||
|
Ports: []gamekruiseiov1alpha1.NetworkPort{
|
||||||
|
{
|
||||||
|
Name: instrIPort.String(),
|
||||||
|
Port: &instrIPort,
|
||||||
|
Protocol: port.Protocol,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
externalAddress := gamekruiseiov1alpha1.NetworkAddress{
|
||||||
|
IP: svc.Status.LoadBalancer.Ingress[0].IP,
|
||||||
|
Ports: []gamekruiseiov1alpha1.NetworkPort{
|
||||||
|
{
|
||||||
|
Name: instrIPort.String(),
|
||||||
|
Port: &instrEPort,
|
||||||
|
Protocol: port.Protocol,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
internalAddresses = append(internalAddresses, internalAddress)
|
||||||
|
externalAddresses = append(externalAddresses, externalAddress)
|
||||||
|
}
|
||||||
|
networkStatus.InternalAddresses = internalAddresses
|
||||||
|
networkStatus.ExternalAddresses = externalAddresses
|
||||||
|
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady
|
||||||
|
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
|
||||||
|
return pod, cperrors.ToPluginError(err, cperrors.InternalError)
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *NlbPlugin) OnPodDeleted(client client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError {
|
||||||
|
networkManager := utils.NewNetworkManager(pod, client)
|
||||||
|
networkConfig := networkManager.GetNetworkConfig()
|
||||||
|
sc := parseLbConfig(networkConfig)
|
||||||
|
|
||||||
|
var podKeys []string
|
||||||
|
if sc.isFixed {
|
||||||
|
gss, err := util.GetGameServerSetOfPod(pod, client, ctx)
|
||||||
|
if err != nil && !errors.IsNotFound(err) {
|
||||||
|
return cperrors.ToPluginError(err, cperrors.ApiCallError)
|
||||||
|
}
|
||||||
|
// gss exists in cluster, do not deAllocate.
|
||||||
|
if err == nil && gss.GetDeletionTimestamp() == nil {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
// gss not exists in cluster, deAllocate all the ports related to it.
|
||||||
|
for key := range c.podAllocate {
|
||||||
|
gssName := pod.GetLabels()[gamekruiseiov1alpha1.GameServerOwnerGssKey]
|
||||||
|
if strings.Contains(key, pod.GetNamespace()+"/"+gssName) {
|
||||||
|
podKeys = append(podKeys, key)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
} else {
|
||||||
|
podKeys = append(podKeys, pod.GetNamespace()+"/"+pod.GetName())
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, podKey := range podKeys {
|
||||||
|
c.deAllocate(podKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *NlbPlugin) allocate(lbIds []string, num int, nsName string) (string, []int32) {
|
||||||
|
c.mutex.Lock()
|
||||||
|
defer c.mutex.Unlock()
|
||||||
|
|
||||||
|
var ports []int32
|
||||||
|
var lbId string
|
||||||
|
|
||||||
|
// find lb with adequate ports
|
||||||
|
for _, nlbId := range lbIds {
|
||||||
|
sum := 0
|
||||||
|
for i := c.minPort; i < c.maxPort; i++ {
|
||||||
|
if !c.cache[nlbId][i] {
|
||||||
|
sum++
|
||||||
|
}
|
||||||
|
if sum >= num {
|
||||||
|
lbId = nlbId
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
// select ports
|
||||||
|
for i := 0; i < num; i++ {
|
||||||
|
var port int32
|
||||||
|
if c.cache[lbId] == nil {
|
||||||
|
c.cache[lbId] = make(portAllocated, c.maxPort-c.minPort)
|
||||||
|
for i := c.minPort; i < c.maxPort; i++ {
|
||||||
|
c.cache[lbId][i] = false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
for p, allocated := range c.cache[lbId] {
|
||||||
|
if !allocated {
|
||||||
|
port = p
|
||||||
|
break
|
||||||
|
}
|
||||||
|
}
|
||||||
|
c.cache[lbId][port] = true
|
||||||
|
ports = append(ports, port)
|
||||||
|
}
|
||||||
|
|
||||||
|
c.podAllocate[nsName] = lbId + ":" + util.Int32SliceToString(ports, ",")
|
||||||
|
log.Infof("pod %s allocate nlb %s ports %v", nsName, lbId, ports)
|
||||||
|
return lbId, ports
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *NlbPlugin) deAllocate(nsName string) {
|
||||||
|
c.mutex.Lock()
|
||||||
|
defer c.mutex.Unlock()
|
||||||
|
|
||||||
|
allocatedPorts, exist := c.podAllocate[nsName]
|
||||||
|
if !exist {
|
||||||
|
return
|
||||||
|
}
|
||||||
|
|
||||||
|
nlbPorts := strings.Split(allocatedPorts, ":")
|
||||||
|
lbId := nlbPorts[0]
|
||||||
|
ports := util.StringToInt32Slice(nlbPorts[1], ",")
|
||||||
|
for _, port := range ports {
|
||||||
|
c.cache[lbId][port] = false
|
||||||
|
}
|
||||||
|
|
||||||
|
delete(c.podAllocate, nsName)
|
||||||
|
log.Infof("pod %s deallocate nlb %s ports %v", nsName, lbId, ports)
|
||||||
|
}
|
||||||
|
|
||||||
|
func init() {
|
||||||
|
JdNlbPlugin := NlbPlugin{
|
||||||
|
mutex: sync.RWMutex{},
|
||||||
|
}
|
||||||
|
jdcloudProvider.registerPlugin(&JdNlbPlugin)
|
||||||
|
}
|
||||||
|
|
||||||
|
func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *nlbConfig {
|
||||||
|
var lbIds []string
|
||||||
|
ports := make([]int, 0)
|
||||||
|
protocols := make([]corev1.Protocol, 0)
|
||||||
|
isFixed := false
|
||||||
|
allocateLoadBalancerNodePorts := true
|
||||||
|
annotations := map[string]string{}
|
||||||
|
algo := string(JdNLBAlgorithmRoundRobin)
|
||||||
|
idleTime := JdNLBDefaultConnIdleTime
|
||||||
|
for _, c := range conf {
|
||||||
|
switch c.Name {
|
||||||
|
case NlbIdsConfigName:
|
||||||
|
for _, clbId := range strings.Split(c.Value, ",") {
|
||||||
|
if clbId != "" {
|
||||||
|
lbIds = append(lbIds, clbId)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case PortProtocolsConfigName:
|
||||||
|
for _, pp := range strings.Split(c.Value, ",") {
|
||||||
|
ppSlice := strings.Split(pp, "/")
|
||||||
|
port, err := strconv.Atoi(ppSlice[0])
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
ports = append(ports, port)
|
||||||
|
if len(ppSlice) != 2 {
|
||||||
|
protocols = append(protocols, corev1.ProtocolTCP)
|
||||||
|
} else {
|
||||||
|
protocols = append(protocols, corev1.Protocol(ppSlice[1]))
|
||||||
|
}
|
||||||
|
}
|
||||||
|
case FixedConfigName:
|
||||||
|
v, err := strconv.ParseBool(c.Value)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
isFixed = v
|
||||||
|
case NlbAlgorithm:
|
||||||
|
algo = c.Value
|
||||||
|
case NlbConnectionIdleTime:
|
||||||
|
t, err := strconv.Atoi(c.Value)
|
||||||
|
if err == nil {
|
||||||
|
idleTime = t
|
||||||
|
}
|
||||||
|
case AllocateLoadBalancerNodePorts:
|
||||||
|
v, err := strconv.ParseBool(c.Value)
|
||||||
|
if err != nil {
|
||||||
|
continue
|
||||||
|
}
|
||||||
|
allocateLoadBalancerNodePorts = v
|
||||||
|
case NlbAnnotations:
|
||||||
|
for _, anno := range strings.Split(c.Value, ",") {
|
||||||
|
annoKV := strings.Split(anno, ":")
|
||||||
|
if len(annoKV) == 2 {
|
||||||
|
annotations[annoKV[0]] = annoKV[1]
|
||||||
|
} else {
|
||||||
|
log.Warningf("nlb annotation %s is invalid", annoKV[0])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &nlbConfig{
|
||||||
|
lbIds: lbIds,
|
||||||
|
protocols: protocols,
|
||||||
|
targetPorts: ports,
|
||||||
|
isFixed: isFixed,
|
||||||
|
annotations: annotations,
|
||||||
|
allocateLoadBalancerNodePorts: allocateLoadBalancerNodePorts,
|
||||||
|
algorithm: algo,
|
||||||
|
connIdleTimeSeconds: idleTime,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func getPorts(ports []corev1.ServicePort) []int32 {
|
||||||
|
var ret []int32
|
||||||
|
for _, port := range ports {
|
||||||
|
ret = append(ret, port.Port)
|
||||||
|
}
|
||||||
|
return ret
|
||||||
|
}
|
||||||
|
|
||||||
|
func (c *NlbPlugin) consSvc(config *nlbConfig, pod *corev1.Pod, client client.Client, ctx context.Context) *corev1.Service {
|
||||||
|
var ports []int32
|
||||||
|
var lbId string
|
||||||
|
podKey := pod.GetNamespace() + "/" + pod.GetName()
|
||||||
|
allocatedPorts, exist := c.podAllocate[podKey]
|
||||||
|
if exist {
|
||||||
|
nlbPorts := strings.Split(allocatedPorts, ":")
|
||||||
|
lbId = nlbPorts[0]
|
||||||
|
ports = util.StringToInt32Slice(nlbPorts[1], ",")
|
||||||
|
} else {
|
||||||
|
lbId, ports = c.allocate(config.lbIds, len(config.targetPorts), podKey)
|
||||||
|
}
|
||||||
|
|
||||||
|
svcPorts := make([]corev1.ServicePort, 0)
|
||||||
|
for i := 0; i < len(config.targetPorts); i++ {
|
||||||
|
svcPorts = append(svcPorts, corev1.ServicePort{
|
||||||
|
Name: strconv.Itoa(config.targetPorts[i]),
|
||||||
|
Port: ports[i],
|
||||||
|
Protocol: config.protocols[i],
|
||||||
|
TargetPort: intstr.FromInt(config.targetPorts[i]),
|
||||||
|
})
|
||||||
|
}
|
||||||
|
|
||||||
|
annotations := map[string]string{
|
||||||
|
NlbIdLabelKey: lbId,
|
||||||
|
NlbSpecAnnotationKey: getNLBSpec(svcPorts, lbId, config.algorithm, config.connIdleTimeSeconds),
|
||||||
|
NlbConfigHashKey: util.GetHash(config),
|
||||||
|
}
|
||||||
|
for key, value := range config.annotations {
|
||||||
|
annotations[key] = value
|
||||||
|
}
|
||||||
|
|
||||||
|
svc := &corev1.Service{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: pod.GetName(),
|
||||||
|
Namespace: pod.GetNamespace(),
|
||||||
|
Annotations: annotations,
|
||||||
|
OwnerReferences: getSvcOwnerReference(client, ctx, pod, config.isFixed),
|
||||||
|
},
|
||||||
|
Spec: corev1.ServiceSpec{
|
||||||
|
Type: corev1.ServiceTypeLoadBalancer,
|
||||||
|
Selector: map[string]string{
|
||||||
|
SvcSelectorKey: pod.GetName(),
|
||||||
|
},
|
||||||
|
Ports: svcPorts,
|
||||||
|
AllocateLoadBalancerNodePorts: ptr.To[bool](config.allocateLoadBalancerNodePorts),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
return svc
|
||||||
|
}
|
||||||
|
|
||||||
|
func getNLBSpec(ports []corev1.ServicePort, lbId, algorithm string, connIdleTimeSeconds int) string {
|
||||||
|
jdNlb := _getNLBSpec(ports, lbId, algorithm, connIdleTimeSeconds)
|
||||||
|
bytes, err := json.Marshal(jdNlb)
|
||||||
|
if err != nil {
|
||||||
|
return ""
|
||||||
|
}
|
||||||
|
return string(bytes)
|
||||||
|
}
|
||||||
|
|
||||||
|
func _getNLBSpec(ports []corev1.ServicePort, lbId, algorithm string, connIdleTimeSeconds int) *JdNLB {
|
||||||
|
var listeners = make([]*JdNLBListener, len(ports))
|
||||||
|
for k, v := range ports {
|
||||||
|
listeners[k] = &JdNLBListener{
|
||||||
|
Protocol: strings.ToUpper(string(v.Protocol)),
|
||||||
|
ConnectionIdleTimeSeconds: connIdleTimeSeconds,
|
||||||
|
Backend: &JdNLBListenerBackend{
|
||||||
|
Algorithm: JdNLBAlgorithm(algorithm),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return &JdNLB{
|
||||||
|
Version: JdNLBVersion,
|
||||||
|
LoadBalancerId: lbId,
|
||||||
|
LoadBalancerType: LbType_NLB,
|
||||||
|
Internal: false,
|
||||||
|
Listeners: listeners,
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func getSvcOwnerReference(c client.Client, ctx context.Context, pod *corev1.Pod, isFixed bool) []metav1.OwnerReference {
|
||||||
|
ownerReferences := []metav1.OwnerReference{
|
||||||
|
{
|
||||||
|
APIVersion: pod.APIVersion,
|
||||||
|
Kind: pod.Kind,
|
||||||
|
Name: pod.GetName(),
|
||||||
|
UID: pod.GetUID(),
|
||||||
|
Controller: ptr.To[bool](true),
|
||||||
|
BlockOwnerDeletion: ptr.To[bool](true),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
if isFixed {
|
||||||
|
gss, err := util.GetGameServerSetOfPod(pod, c, ctx)
|
||||||
|
if err == nil {
|
||||||
|
ownerReferences = []metav1.OwnerReference{
|
||||||
|
{
|
||||||
|
APIVersion: gss.APIVersion,
|
||||||
|
Kind: gss.Kind,
|
||||||
|
Name: gss.GetName(),
|
||||||
|
UID: gss.GetUID(),
|
||||||
|
Controller: ptr.To[bool](true),
|
||||||
|
BlockOwnerDeletion: ptr.To[bool](true),
|
||||||
|
},
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return ownerReferences
|
||||||
|
}
|
||||||
|
|
@ -0,0 +1,344 @@
|
||||||
|
/*
|
||||||
|
Copyright 2024 The Kruise Authors.
|
||||||
|
|
||||||
|
Licensed under the Apache License, Version 2.0 (the "License");
|
||||||
|
you may not use this file except in compliance with the License.
|
||||||
|
You may obtain a copy of the License at
|
||||||
|
|
||||||
|
http://www.apache.org/licenses/LICENSE-2.0
|
||||||
|
|
||||||
|
Unless required by applicable law or agreed to in writing, software
|
||||||
|
distributed under the License is distributed on an "AS IS" BASIS,
|
||||||
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||||
|
See the License for the specific language governing permissions and
|
||||||
|
limitations under the License.
|
||||||
|
*/
|
||||||
|
|
||||||
|
package jdcloud
|
||||||
|
|
||||||
|
import (
|
||||||
|
"context"
|
||||||
|
"k8s.io/utils/ptr"
|
||||||
|
"reflect"
|
||||||
|
"sync"
|
||||||
|
"testing"
|
||||||
|
|
||||||
|
corev1 "k8s.io/api/core/v1"
|
||||||
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
||||||
|
"k8s.io/apimachinery/pkg/util/intstr"
|
||||||
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
||||||
|
|
||||||
|
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
|
||||||
|
"github.com/openkruise/kruise-game/pkg/util"
|
||||||
|
)
|
||||||
|
|
||||||
|
func TestAllocateDeAllocate(t *testing.T) {
|
||||||
|
test := struct {
|
||||||
|
lbIds []string
|
||||||
|
nlb *NlbPlugin
|
||||||
|
num int
|
||||||
|
podKey string
|
||||||
|
}{
|
||||||
|
lbIds: []string{"xxx-A"},
|
||||||
|
nlb: &NlbPlugin{
|
||||||
|
maxPort: int32(700),
|
||||||
|
minPort: int32(500),
|
||||||
|
cache: make(map[string]portAllocated),
|
||||||
|
podAllocate: make(map[string]string),
|
||||||
|
mutex: sync.RWMutex{},
|
||||||
|
},
|
||||||
|
podKey: "xxx/xxx",
|
||||||
|
num: 3,
|
||||||
|
}
|
||||||
|
|
||||||
|
lbId, ports := test.nlb.allocate(test.lbIds, test.num, test.podKey)
|
||||||
|
if _, exist := test.nlb.podAllocate[test.podKey]; !exist {
|
||||||
|
t.Errorf("podAllocate[%s] is empty after allocated", test.podKey)
|
||||||
|
}
|
||||||
|
for _, port := range ports {
|
||||||
|
if port > test.nlb.maxPort || port < test.nlb.minPort {
|
||||||
|
t.Errorf("allocate port %d, unexpected", port)
|
||||||
|
}
|
||||||
|
if test.nlb.cache[lbId][port] == false {
|
||||||
|
t.Errorf("Allocate port %d failed", port)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
test.nlb.deAllocate(test.podKey)
|
||||||
|
for _, port := range ports {
|
||||||
|
if test.nlb.cache[lbId][port] == true {
|
||||||
|
t.Errorf("deAllocate port %d failed", port)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if _, exist := test.nlb.podAllocate[test.podKey]; exist {
|
||||||
|
t.Errorf("podAllocate[%s] is not empty after deallocated", test.podKey)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestParseLbConfig(t *testing.T) {
|
||||||
|
tests := []struct {
|
||||||
|
conf []gamekruiseiov1alpha1.NetworkConfParams
|
||||||
|
lbIds []string
|
||||||
|
ports []int
|
||||||
|
protocols []corev1.Protocol
|
||||||
|
isFixed bool
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
conf: []gamekruiseiov1alpha1.NetworkConfParams{
|
||||||
|
{
|
||||||
|
Name: NlbIdsConfigName,
|
||||||
|
Value: "xxx-A",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: PortProtocolsConfigName,
|
||||||
|
Value: "80",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
lbIds: []string{"xxx-A"},
|
||||||
|
ports: []int{80},
|
||||||
|
protocols: []corev1.Protocol{corev1.ProtocolTCP},
|
||||||
|
isFixed: false,
|
||||||
|
},
|
||||||
|
{
|
||||||
|
conf: []gamekruiseiov1alpha1.NetworkConfParams{
|
||||||
|
{
|
||||||
|
Name: NlbIdsConfigName,
|
||||||
|
Value: "xxx-A,xxx-B,",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: PortProtocolsConfigName,
|
||||||
|
Value: "81/UDP,82,83/TCP",
|
||||||
|
},
|
||||||
|
{
|
||||||
|
Name: FixedConfigName,
|
||||||
|
Value: "true",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
lbIds: []string{"xxx-A", "xxx-B"},
|
||||||
|
ports: []int{81, 82, 83},
|
||||||
|
protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP, corev1.ProtocolTCP},
|
||||||
|
isFixed: true,
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
for _, test := range tests {
|
||||||
|
sc := parseLbConfig(test.conf)
|
||||||
|
if !reflect.DeepEqual(test.lbIds, sc.lbIds) {
|
||||||
|
t.Errorf("lbId expect: %v, actual: %v", test.lbIds, sc.lbIds)
|
||||||
|
}
|
||||||
|
if !util.IsSliceEqual(test.ports, sc.targetPorts) {
|
||||||
|
t.Errorf("ports expect: %v, actual: %v", test.ports, sc.targetPorts)
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(test.protocols, sc.protocols) {
|
||||||
|
t.Errorf("protocols expect: %v, actual: %v", test.protocols, sc.protocols)
|
||||||
|
}
|
||||||
|
if test.isFixed != sc.isFixed {
|
||||||
|
t.Errorf("isFixed expect: %v, actual: %v", test.isFixed, sc.isFixed)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestInitLbCache(t *testing.T) {
|
||||||
|
test := struct {
|
||||||
|
svcList []corev1.Service
|
||||||
|
minPort int32
|
||||||
|
maxPort int32
|
||||||
|
cache map[string]portAllocated
|
||||||
|
podAllocate map[string]string
|
||||||
|
}{
|
||||||
|
minPort: 500,
|
||||||
|
maxPort: 700,
|
||||||
|
cache: map[string]portAllocated{
|
||||||
|
"xxx-A": map[int32]bool{
|
||||||
|
666: true,
|
||||||
|
},
|
||||||
|
"xxx-B": map[int32]bool{
|
||||||
|
555: true,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
podAllocate: map[string]string{
|
||||||
|
"ns-0/name-0": "xxx-A:666",
|
||||||
|
"ns-1/name-1": "xxx-B:555",
|
||||||
|
},
|
||||||
|
svcList: []corev1.Service{
|
||||||
|
{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Labels: map[string]string{
|
||||||
|
NlbIdLabelKey: "xxx-A",
|
||||||
|
},
|
||||||
|
Namespace: "ns-0",
|
||||||
|
Name: "name-0",
|
||||||
|
},
|
||||||
|
Spec: corev1.ServiceSpec{
|
||||||
|
Type: corev1.ServiceTypeLoadBalancer,
|
||||||
|
Selector: map[string]string{
|
||||||
|
SvcSelectorKey: "pod-A",
|
||||||
|
},
|
||||||
|
Ports: []corev1.ServicePort{
|
||||||
|
{
|
||||||
|
TargetPort: intstr.FromInt(80),
|
||||||
|
Port: 666,
|
||||||
|
Protocol: corev1.ProtocolTCP,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Labels: map[string]string{
|
||||||
|
NlbIdLabelKey: "xxx-B",
|
||||||
|
},
|
||||||
|
Namespace: "ns-1",
|
||||||
|
Name: "name-1",
|
||||||
|
},
|
||||||
|
Spec: corev1.ServiceSpec{
|
||||||
|
Type: corev1.ServiceTypeLoadBalancer,
|
||||||
|
Selector: map[string]string{
|
||||||
|
SvcSelectorKey: "pod-B",
|
||||||
|
},
|
||||||
|
Ports: []corev1.ServicePort{
|
||||||
|
{
|
||||||
|
TargetPort: intstr.FromInt(8080),
|
||||||
|
Port: 555,
|
||||||
|
Protocol: corev1.ProtocolTCP,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
|
||||||
|
actualCache, actualPodAllocate := initLbCache(test.svcList, test.minPort, test.maxPort)
|
||||||
|
for lb, pa := range test.cache {
|
||||||
|
for port, isAllocated := range pa {
|
||||||
|
if actualCache[lb][port] != isAllocated {
|
||||||
|
t.Errorf("lb %s port %d isAllocated, expect: %t, actual: %t", lb, port, isAllocated, actualCache[lb][port])
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
if !reflect.DeepEqual(actualPodAllocate, test.podAllocate) {
|
||||||
|
t.Errorf("podAllocate expect %v, but actully got %v", test.podAllocate, actualPodAllocate)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
|
func TestNlbPlugin_consSvc(t *testing.T) {
|
||||||
|
type fields struct {
|
||||||
|
maxPort int32
|
||||||
|
minPort int32
|
||||||
|
cache map[string]portAllocated
|
||||||
|
podAllocate map[string]string
|
||||||
|
}
|
||||||
|
type args struct {
|
||||||
|
config *nlbConfig
|
||||||
|
pod *corev1.Pod
|
||||||
|
client client.Client
|
||||||
|
ctx context.Context
|
||||||
|
}
|
||||||
|
tests := []struct {
|
||||||
|
name string
|
||||||
|
fields fields
|
||||||
|
args args
|
||||||
|
want *corev1.Service
|
||||||
|
}{
|
||||||
|
{
|
||||||
|
name: "convert svc cache exist",
|
||||||
|
fields: fields{
|
||||||
|
maxPort: 3000,
|
||||||
|
minPort: 1,
|
||||||
|
cache: map[string]portAllocated{
|
||||||
|
"default/test-pod": map[int32]bool{},
|
||||||
|
},
|
||||||
|
podAllocate: map[string]string{
|
||||||
|
"default/test-pod": "nlb-xxx:80,81",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
args: args{
|
||||||
|
config: &nlbConfig{
|
||||||
|
lbIds: []string{"nlb-xxx"},
|
||||||
|
targetPorts: []int{82},
|
||||||
|
protocols: []corev1.Protocol{
|
||||||
|
corev1.ProtocolTCP,
|
||||||
|
},
|
||||||
|
isFixed: false,
|
||||||
|
annotations: map[string]string{
|
||||||
|
"service.beta.kubernetes.io/jdcloud-load-balancer-spec": "{}",
|
||||||
|
},
|
||||||
|
allocateLoadBalancerNodePorts: true,
|
||||||
|
},
|
||||||
|
pod: &corev1.Pod{
|
||||||
|
TypeMeta: metav1.TypeMeta{
|
||||||
|
Kind: "pod",
|
||||||
|
APIVersion: "v1",
|
||||||
|
},
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "test-pod",
|
||||||
|
Namespace: "default",
|
||||||
|
UID: "32fqwfqfew",
|
||||||
|
},
|
||||||
|
},
|
||||||
|
client: nil,
|
||||||
|
ctx: context.Background(),
|
||||||
|
},
|
||||||
|
want: &corev1.Service{
|
||||||
|
ObjectMeta: metav1.ObjectMeta{
|
||||||
|
Name: "test-pod",
|
||||||
|
Namespace: "default",
|
||||||
|
Annotations: map[string]string{
|
||||||
|
NlbConfigHashKey: util.GetHash(&nlbConfig{
|
||||||
|
lbIds: []string{"nlb-xxx"},
|
||||||
|
targetPorts: []int{82},
|
||||||
|
protocols: []corev1.Protocol{
|
||||||
|
corev1.ProtocolTCP,
|
||||||
|
},
|
||||||
|
isFixed: false,
|
||||||
|
annotations: map[string]string{
|
||||||
|
"service.beta.kubernetes.io/jdcloud-load-balancer-spec": "{}",
|
||||||
|
},
|
||||||
|
allocateLoadBalancerNodePorts: true,
|
||||||
|
}),
|
||||||
|
"service.beta.kubernetes.io/jdcloud-load-balancer-spec": "{}",
|
||||||
|
"service.beta.kubernetes.io/jdcloud-loadbalancer-id": "nlb-xxx",
|
||||||
|
},
|
||||||
|
OwnerReferences: []metav1.OwnerReference{
|
||||||
|
{
|
||||||
|
APIVersion: "v1",
|
||||||
|
Kind: "pod",
|
||||||
|
Name: "test-pod",
|
||||||
|
UID: "32fqwfqfew",
|
||||||
|
Controller: ptr.To[bool](true),
|
||||||
|
BlockOwnerDeletion: ptr.To[bool](true),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
Spec: corev1.ServiceSpec{
|
||||||
|
Type: corev1.ServiceTypeLoadBalancer,
|
||||||
|
Selector: map[string]string{
|
||||||
|
SvcSelectorKey: "test-pod",
|
||||||
|
},
|
||||||
|
Ports: []corev1.ServicePort{{
|
||||||
|
Name: "82",
|
||||||
|
Port: 80,
|
||||||
|
Protocol: "TCP",
|
||||||
|
TargetPort: intstr.IntOrString{
|
||||||
|
Type: 0,
|
||||||
|
IntVal: 82,
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
AllocateLoadBalancerNodePorts: ptr.To[bool](true),
|
||||||
|
},
|
||||||
|
},
|
||||||
|
},
|
||||||
|
}
|
||||||
|
for _, tt := range tests {
|
||||||
|
c := &NlbPlugin{
|
||||||
|
maxPort: tt.fields.maxPort,
|
||||||
|
minPort: tt.fields.minPort,
|
||||||
|
cache: tt.fields.cache,
|
||||||
|
podAllocate: tt.fields.podAllocate,
|
||||||
|
}
|
||||||
|
if got := c.consSvc(tt.args.config, tt.args.pod, tt.args.client, tt.args.ctx); !reflect.DeepEqual(got, tt.want) {
|
||||||
|
t.Errorf("consSvc() = %v, want %v", got, tt.want)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
@ -18,6 +18,7 @@ package manager
|
||||||
|
|
||||||
import (
|
import (
|
||||||
"context"
|
"context"
|
||||||
|
"github.com/openkruise/kruise-game/cloudprovider/jdcloud"
|
||||||
|
|
||||||
"github.com/openkruise/kruise-game/apis/v1alpha1"
|
"github.com/openkruise/kruise-game/apis/v1alpha1"
|
||||||
"github.com/openkruise/kruise-game/cloudprovider"
|
"github.com/openkruise/kruise-game/cloudprovider"
|
||||||
|
|
@ -150,5 +151,15 @@ func NewProviderManager() (*ProviderManager, error) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
if configs.JdCloudOptions.Valid() && configs.JdCloudOptions.Enabled() {
|
||||||
|
// build and register tencent cloud provider
|
||||||
|
tcp, err := jdcloud.NewJdcloudProvider()
|
||||||
|
if err != nil {
|
||||||
|
log.Errorf("Failed to initialize jdcloud provider.because of %s", err.Error())
|
||||||
|
} else {
|
||||||
|
pm.RegisterCloudProvider(tcp, configs.JdCloudOptions)
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
return pm, nil
|
return pm, nil
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -0,0 +1,28 @@
|
||||||
|
package options
|
||||||
|
|
||||||
|
type JdCloudOptions struct {
|
||||||
|
Enable bool `toml:"enable"`
|
||||||
|
NLBOptions JdNLBOptions `toml:"nlb"`
|
||||||
|
}
|
||||||
|
|
||||||
|
type JdNLBOptions struct {
|
||||||
|
MaxPort int32 `toml:"max_port"`
|
||||||
|
MinPort int32 `toml:"min_port"`
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v JdCloudOptions) Valid() bool {
|
||||||
|
nlbOptions := v.NLBOptions
|
||||||
|
|
||||||
|
if nlbOptions.MaxPort > 65535 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
|
||||||
|
if nlbOptions.MinPort < 1 {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
||||||
|
func (v JdCloudOptions) Enabled() bool {
|
||||||
|
return v.Enable
|
||||||
|
}
|
||||||
|
|
@ -26,3 +26,10 @@ enable = false
|
||||||
[aws.nlb]
|
[aws.nlb]
|
||||||
max_port = 30050
|
max_port = 30050
|
||||||
min_port = 30001
|
min_port = 30001
|
||||||
|
|
||||||
|
[jdcloud]
|
||||||
|
enable = false
|
||||||
|
[jdcloud.nlb]
|
||||||
|
max_port = 700
|
||||||
|
min_port = 500
|
||||||
|
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue