add network plugin AlibabaCloud-EIP (#86)

Signed-off-by: ChrisLiu <chrisliu1995@163.com>
This commit is contained in:
ChrisLiu 2023-07-24 14:49:34 +08:00 committed by GitHub
parent defcb15f02
commit 6f89f6e637
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
8 changed files with 713 additions and 0 deletions

View File

@ -0,0 +1,96 @@
package v1beta1
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// PodEIPSpec defines the desired state of PodEIP
type PodEIPSpec struct {
// +kubebuilder:validation:Required
AllocationID string `json:"allocationID"`
BandwidthPackageID string `json:"bandwidthPackageID,omitempty"`
// +kubebuilder:validation:Required
AllocationType AllocationType `json:"allocationType"`
}
// AllocationType ip type and release strategy
type AllocationType struct {
// +kubebuilder:default:=Auto
// +kubebuilder:validation:Required
Type IPAllocType `json:"type"`
// +kubebuilder:validation:Required
// +kubebuilder:validation:Enum=Follow;TTL;Never
ReleaseStrategy ReleaseStrategy `json:"releaseStrategy"`
ReleaseAfter string `json:"releaseAfter,omitempty"` // go type 5m0s
}
// +kubebuilder:validation:Enum=Auto;Static
// IPAllocType is the type for eip alloc strategy
type IPAllocType string
// IPAllocType
const (
IPAllocTypeAuto IPAllocType = "Auto"
IPAllocTypeStatic IPAllocType = "Static"
)
// +kubebuilder:validation:Enum=Follow;TTL;Never
// ReleaseStrategy is the type for eip release strategy
type ReleaseStrategy string
// ReleaseStrategy
const (
ReleaseStrategyFollow ReleaseStrategy = "Follow" // default policy
ReleaseStrategyTTL ReleaseStrategy = "TTL"
ReleaseStrategyNever ReleaseStrategy = "Never"
)
// PodEIPStatus defines the observed state of PodEIP
type PodEIPStatus struct {
// eni
NetworkInterfaceID string `json:"networkInterfaceID,omitempty"`
PrivateIPAddress string `json:"privateIPAddress,omitempty"`
// eip
EipAddress string `json:"eipAddress,omitempty"`
ISP string `json:"isp,omitempty"`
InternetChargeType string `json:"internetChargeType,omitempty"`
ResourceGroupID string `json:"resourceGroupID,omitempty"`
Name string `json:"name,omitempty"`
PublicIpAddressPoolID string `json:"publicIpAddressPoolID,omitempty"`
Status string `json:"status,omitempty"`
// BandwidthPackageID
BandwidthPackageID string `json:"bandwidthPackageID,omitempty"`
// PodLastSeen is the timestamp when pod resource last seen
PodLastSeen metav1.Time `json:"podLastSeen,omitempty"`
}
//+kubebuilder:object:root=true
//+kubebuilder:subresource:status
// PodEIP is the Schema for the podeips API
type PodEIP struct {
metav1.TypeMeta `json:",inline"`
metav1.ObjectMeta `json:"metadata,omitempty"`
Spec PodEIPSpec `json:"spec,omitempty"`
Status PodEIPStatus `json:"status,omitempty"`
}
//+kubebuilder:object:root=true
// PodEIPList contains a list of PodEIP
type PodEIPList struct {
metav1.TypeMeta `json:",inline"`
metav1.ListMeta `json:"metadata,omitempty"`
Items []PodEIP `json:"items"`
}
func init() {
SchemeBuilder.Register(&PodEIP{}, &PodEIPList{})
}

View File

@ -25,6 +25,21 @@ import (
runtime "k8s.io/apimachinery/pkg/runtime"
)
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *AllocationType) DeepCopyInto(out *AllocationType) {
*out = *in
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AllocationType.
func (in *AllocationType) DeepCopy() *AllocationType {
if in == nil {
return nil
}
out := new(AllocationType)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *Entry) DeepCopyInto(out *Entry) {
*out = *in
@ -194,6 +209,97 @@ func (in *PodDNATStatus) DeepCopy() *PodDNATStatus {
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PodEIP) DeepCopyInto(out *PodEIP) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ObjectMeta.DeepCopyInto(&out.ObjectMeta)
out.Spec = in.Spec
in.Status.DeepCopyInto(&out.Status)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodEIP.
func (in *PodEIP) DeepCopy() *PodEIP {
if in == nil {
return nil
}
out := new(PodEIP)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *PodEIP) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PodEIPList) DeepCopyInto(out *PodEIPList) {
*out = *in
out.TypeMeta = in.TypeMeta
in.ListMeta.DeepCopyInto(&out.ListMeta)
if in.Items != nil {
in, out := &in.Items, &out.Items
*out = make([]PodEIP, len(*in))
for i := range *in {
(*in)[i].DeepCopyInto(&(*out)[i])
}
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodEIPList.
func (in *PodEIPList) DeepCopy() *PodEIPList {
if in == nil {
return nil
}
out := new(PodEIPList)
in.DeepCopyInto(out)
return out
}
// DeepCopyObject is an autogenerated deepcopy function, copying the receiver, creating a new runtime.Object.
func (in *PodEIPList) DeepCopyObject() runtime.Object {
if c := in.DeepCopy(); c != nil {
return c
}
return nil
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PodEIPSpec) DeepCopyInto(out *PodEIPSpec) {
*out = *in
out.AllocationType = in.AllocationType
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodEIPSpec.
func (in *PodEIPSpec) DeepCopy() *PodEIPSpec {
if in == nil {
return nil
}
out := new(PodEIPSpec)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PodEIPStatus) DeepCopyInto(out *PodEIPStatus) {
*out = *in
in.PodLastSeen.DeepCopyInto(&out.PodLastSeen)
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new PodEIPStatus.
func (in *PodEIPStatus) DeepCopy() *PodEIPStatus {
if in == nil {
return nil
}
out := new(PodEIPStatus)
in.DeepCopyInto(out)
return out
}
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
func (in *PortMapping) DeepCopyInto(out *PortMapping) {
*out = *in

View File

@ -0,0 +1,117 @@
package alibabacloud
import (
"context"
gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
"github.com/openkruise/kruise-game/cloudprovider"
"github.com/openkruise/kruise-game/cloudprovider/alibabacloud/apis/v1beta1"
"github.com/openkruise/kruise-game/cloudprovider/errors"
"github.com/openkruise/kruise-game/cloudprovider/utils"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
)
const (
EIPNetwork = "AlibabaCloud-EIP"
AliasSEIP = "EIP-Network"
ReleaseStrategyConfigName = "ReleaseStrategy"
PoolIdConfigName = "PoolId"
ResourceGroupIdConfigName = "ResourceGroupId"
BandwidthConfigName = "Bandwidth"
BandwidthPackageIdConfigName = "BandwidthPackageId"
ChargeTypeConfigName = "ChargeType"
WithEIPAnnotationKey = "k8s.aliyun.com/pod-with-eip"
ReleaseStrategyAnnotationkey = "k8s.aliyun.com/pod-eip-release-strategy"
PoolIdAnnotationkey = "k8s.aliyun.com/eip-public-ip-address-pool-id"
ResourceGroupIdAnnotationkey = "k8s.aliyun.com/eip-resource-group-id"
BandwidthAnnotationkey = "k8s.aliyun.com/eip-bandwidth"
BandwidthPackageIdAnnotationkey = "k8s.aliyun.com/eip-common-bandwidth-package-id"
ChargeTypeConfigAnnotationkey = "k8s.aliyun.com/eip-internet-charge-type"
)
type EipPlugin struct {
}
func (E EipPlugin) Name() string {
return EIPNetwork
}
func (E EipPlugin) Alias() string {
return AliasSEIP
}
func (E EipPlugin) Init(client client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error {
return nil
}
func (E EipPlugin) OnPodAdded(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, errors.PluginError) {
networkManager := utils.NewNetworkManager(pod, client)
conf := networkManager.GetNetworkConfig()
pod.Annotations[WithEIPAnnotationKey] = "true"
// parse network configuration
for _, c := range conf {
switch c.Name {
case ReleaseStrategyConfigName:
pod.Annotations[ReleaseStrategyAnnotationkey] = c.Value
case PoolIdConfigName:
pod.Annotations[PoolIdAnnotationkey] = c.Value
case ResourceGroupIdConfigName:
pod.Annotations[ResourceGroupIdAnnotationkey] = c.Value
case BandwidthConfigName:
pod.Annotations[BandwidthAnnotationkey] = c.Value
case BandwidthPackageIdConfigName:
pod.Annotations[BandwidthPackageIdAnnotationkey] = c.Value
case ChargeTypeConfigName:
pod.Annotations[ChargeTypeConfigAnnotationkey] = c.Value
}
}
return pod, nil
}
func (E EipPlugin) OnPodUpdated(client client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, errors.PluginError) {
networkManager := utils.NewNetworkManager(pod, client)
networkStatus, _ := networkManager.GetNetworkStatus()
if networkStatus == nil {
pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{
CurrentNetworkState: gamekruiseiov1alpha1.NetworkWaiting,
}, pod)
return pod, errors.ToPluginError(err, errors.InternalError)
}
podEip := &v1beta1.PodEIP{}
err := client.Get(ctx, types.NamespacedName{
Name: pod.GetName(),
Namespace: pod.GetNamespace(),
}, podEip)
if err != nil || podEip.Status.EipAddress == "" {
return pod, nil
}
networkStatus.InternalAddresses = []gamekruiseiov1alpha1.NetworkAddress{
{
IP: podEip.Status.PrivateIPAddress,
},
}
networkStatus.ExternalAddresses = []gamekruiseiov1alpha1.NetworkAddress{
{
IP: podEip.Status.EipAddress,
},
}
networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady
pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
return pod, errors.ToPluginError(err, errors.InternalError)
}
func (E EipPlugin) OnPodDeleted(client client.Client, pod *corev1.Pod, ctx context.Context) errors.PluginError {
return nil
}
func init() {
alibabaCloudProvider.registerPlugin(&EipPlugin{})
}

View File

@ -0,0 +1,109 @@
---
apiVersion: apiextensions.k8s.io/v1
kind: CustomResourceDefinition
metadata:
annotations:
controller-gen.kubebuilder.io/version: v0.9.0
creationTimestamp: null
name: podeips.alibabacloud.com
spec:
group: alibabacloud.com
names:
kind: PodEIP
listKind: PodEIPList
plural: podeips
singular: podeip
scope: Namespaced
versions:
- name: v1beta1
schema:
openAPIV3Schema:
description: PodEIP is the Schema for the podeips API
properties:
apiVersion:
description: 'APIVersion defines the versioned schema of this representation
of an object. Servers should convert recognized schemas to the latest
internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources'
type: string
kind:
description: 'Kind is a string value representing the REST resource this
object represents. Servers may infer this from the endpoint the client
submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds'
type: string
metadata:
type: object
spec:
description: PodEIPSpec defines the desired state of PodEIP
properties:
allocationID:
type: string
allocationType:
description: AllocationType ip type and release strategy
properties:
releaseAfter:
type: string
releaseStrategy:
allOf:
- enum:
- Follow
- TTL
- Never
- enum:
- Follow
- TTL
- Never
description: ReleaseStrategy is the type for eip release strategy
type: string
type:
default: Auto
description: IPAllocType is the type for eip alloc strategy
enum:
- Auto
- Static
type: string
required:
- releaseStrategy
- type
type: object
bandwidthPackageID:
type: string
required:
- allocationID
- allocationType
type: object
status:
description: PodEIPStatus defines the observed state of PodEIP
properties:
bandwidthPackageID:
description: BandwidthPackageID
type: string
eipAddress:
description: eip
type: string
internetChargeType:
type: string
isp:
type: string
name:
type: string
networkInterfaceID:
description: eni
type: string
podLastSeen:
description: PodLastSeen is the timestamp when pod resource last seen
format: date-time
type: string
privateIPAddress:
type: string
publicIpAddressPoolID:
type: string
resourceGroupID:
type: string
status:
type: string
type: object
type: object
served: true
storage: true
subresources:
status: {}

View File

@ -48,6 +48,20 @@ rules:
- poddnats/status
verbs:
- get
- apiGroups:
- alibabacloud.com
resources:
- podeips
verbs:
- get
- list
- watch
- apiGroups:
- alibabacloud.com
resources:
- podeips/status
verbs:
- get
- apiGroups:
- apiextensions.k8s.io
resources:

View File

@ -476,3 +476,137 @@ PortProtocols
#### Plugin configuration
None
### AlibabaCloud-EIP
#### Plugin name
`AlibabaCloud-EIP`
#### Cloud Provider
AlibabaCloud
#### Plugin description
- Allocate a separate EIP for each GameServer
- The exposed public access port is consistent with the port monitored in the container, which is managed by security group.
- It is necessary to install the latest version of the ack-extend-network-controller component in the ACK cluster. For details, please refer to the [component description page](https://cs.console.aliyun.com/#/next/app-catalog/ack/incubator/ack-extend-network-controller).
#### Network parameters
ReleaseStrategy
- Meaning: Specifies the EIP release policy.
- Value:
- Follow: follows the lifecycle of the pod that is associated with the EIP. This is the default value.
- Never: does not release the EIP. You need to manually release the EIP when you no longer need the EIP. ( By 'kubectl delete podeip {gameserver name} -n {gameserver namespace}')
- You can also specify the timeout period of the EIP. For example, if you set the time period to 5m30s, the EIP is released 5.5 minutes after the pod is deleted. Time expressions written in Go are supported.
- Configuration change supported or not: no.
PoolId
- Meaning: Specifies the EIP address pool. For more information. It could be nil.
- Configuration change supported or not: no.
ResourceGroupId
- Meaning: Specifies the resource group to which the EIP belongs. It could be nil.
- Configuration change supported or not: no.
Bandwidth
- Meaning: Specifies the maximum bandwidth of the EIP. Unit: Mbit/s. It could be nil. Default is 5.
- Configuration change supported or not: no.
BandwidthPackageId
- Meaning: Specifies the EIP bandwidth plan that you want to use.
- Configuration change supported or not: no.
ChargeType
- Meaning: Specifies the metering method of the EIP.
- Value
- PayByTraffic: Fees are charged based on data transfer.
- PayByBandwidth: Fees are charged based on bandwidth usage.
- Configuration change supported or not: no.
#### Plugin configuration
None
#### Example
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: eip-nginx
namespace: default
spec:
replicas: 1
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: AlibabaCloud-EIP
networkConf:
- name: ReleaseStrategy
value: Never
- name: Bandwidth
value: "3"
- name: ChargeType
value: PayByTraffic
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
The network status of GameServer would be as follows:
```yaml
networkStatus:
createTime: "2023-07-17T10:10:18Z"
currentNetworkState: Ready
desiredNetworkState: Ready
externalAddresses:
- ip: 47.98.xxx.xxx
internalAddresses:
- ip: 192.168.1.51
lastTransitionTime: "2023-07-17T10:10:18Z"
networkType: AlibabaCloud-EIP
```
The generated podeip eip-nginx-0 would be as follows
```yaml
apiVersion: alibabacloud.com/v1beta1
kind: PodEIP
metadata:
annotations:
k8s.aliyun.com/eip-controller: ack-extend-network-controller
creationTimestamp: "2023-07-17T09:58:12Z"
finalizers:
- podeip-controller.alibabacloud.com/finalizer
generation: 1
name: eip-nginx-1
namespace: default
resourceVersion: "41443319"
uid: 105a9575-998e-4e17-ab91-8f2597eeb55f
spec:
allocationID: eip-xxx
allocationType:
releaseStrategy: Never
type: Auto
status:
eipAddress: 47.98.xxx.xxx
internetChargeType: PayByTraffic
isp: BGP
networkInterfaceID: eni-xxx
podLastSeen: "2023-07-17T10:36:02Z"
privateIPAddress: 192.168.1.51
resourceGroupID: rg-xxx
status: InUse
```

View File

@ -474,6 +474,141 @@ PortProtocols
### AlibabaCloud-EIP
#### 插件名称
`AlibabaCloud-EIP`
#### Cloud Provider
AlibabaCloud
#### 插件说明
- 为每个GameServer单独分配EIP
- 暴露的公网访问端口与容器中监听的端口一致,通过安全组管理
- 需要在ACK集群安装最新版本ack-extend-network-controller组件详情请见[组件说明页](https://cs.console.aliyun.com/#/next/app-catalog/ack/incubator/ack-extend-network-controller)
#### 网络参数
ReleaseStrategy
- 含义EIP回收策略。
- 填写格式:
- Follow默认值跟随游戏服生命周期。当游戏服被删除时EIP也将被回收。
- Never不删除podEIP。当不需要时需要手动删除这个podEIP。(通过kubectl delete podeip {游戏服name} -n {游戏服所在namespace})
- 可直接配置过期时间例如5m30s表示Pod删除5.5分钟后删除podEIP。支持Go类型时间表达式。
- 是否支持变更:否
PoolId
- 含义EIP地址池ID。可为空则不使用EIP地址池。
- 是否支持变更:否
ResourceGroupId
- 含义EIP资源组ID。可为空则使用默认资源组。
- 是否支持变更:否
Bandwidth
- 含义峰值带宽。单位Mbps。可为空默认为5
- 是否支持变更:否
BandwidthPackageId
- 含义要绑定已有的共享带宽包ID。可为空则EIP不绑定共享带宽包。
- 是否支持变更:否
ChargeType
- 含义EIP的计费方式。
- 填写格式:
- PayByTraffic按使用流量计费。
- PayByBandwidth按带宽计费为默认值。
- 是否支持变更:否
#### 插件配置
#### 示例说明
```yaml
apiVersion: game.kruise.io/v1alpha1
kind: GameServerSet
metadata:
name: eip-nginx
namespace: default
spec:
replicas: 1
updateStrategy:
rollingUpdate:
podUpdatePolicy: InPlaceIfPossible
network:
networkType: AlibabaCloud-EIP
networkConf:
- name: ReleaseStrategy
value: Never
- name: Bandwidth
value: "3"
- name: ChargeType
value: PayByTraffic
gameServerTemplate:
spec:
containers:
- image: nginx
name: nginx
```
生成的gameserver eip-nginx-0 networkStatus字段如下所示
```yaml
networkStatus:
createTime: "2023-07-17T10:10:18Z"
currentNetworkState: Ready
desiredNetworkState: Ready
externalAddresses:
- ip: 47.98.xxx.xxx
internalAddresses:
- ip: 192.168.1.51
lastTransitionTime: "2023-07-17T10:10:18Z"
networkType: AlibabaCloud-EIP
```
生成对应的podeip eip-nginx-0 如下所示:
```yaml
apiVersion: alibabacloud.com/v1beta1
kind: PodEIP
metadata:
annotations:
k8s.aliyun.com/eip-controller: ack-extend-network-controller
creationTimestamp: "2023-07-17T09:58:12Z"
finalizers:
- podeip-controller.alibabacloud.com/finalizer
generation: 1
name: eip-nginx-1
namespace: default
resourceVersion: "41443319"
uid: 105a9575-998e-4e17-ab91-8f2597eeb55f
spec:
allocationID: eip-xxx
allocationType:
releaseStrategy: Never
type: Auto
status:
eipAddress: 47.98.xxx.xxx
internetChargeType: PayByTraffic
isp: BGP
networkInterfaceID: eni-xxx
podLastSeen: "2023-07-17T10:36:02Z"
privateIPAddress: 192.168.1.51
resourceGroupID: rg-xxx
status: InUse
```
## 获取网络信息
GameServer Network Status可以通过两种方式获取

View File

@ -72,6 +72,8 @@ func init() {
// +kubebuilder:rbac:groups=apiextensions.k8s.io,resources=customresourcedefinitions,verbs=get;list;watch;update;patch
// +kubebuilder:rbac:groups=alibabacloud.com,resources=poddnats,verbs=get;list;watch
// +kubebuilder:rbac:groups=alibabacloud.com,resources=poddnats/status,verbs=get
// +kubebuilder:rbac:groups=alibabacloud.com,resources=podeips,verbs=get;list;watch
// +kubebuilder:rbac:groups=alibabacloud.com,resources=podeips/status,verbs=get
// +kubebuilder:rbac:groups="",resources=events,verbs=create;patch
type Webhook struct {