feat: add ReserveIds ScaleDownStrategyType (#52)

* feat: add ReserveIds ScaleDownStrategyType

Signed-off-by: ChrisLiu <chrisliu1995@163.com>

* add docs for CRD description

Signed-off-by: ChrisLiu <chrisliu1995@163.com>

---------

Signed-off-by: ChrisLiu <chrisliu1995@163.com>
This commit is contained in:
ChrisLiu 2023-04-27 14:05:16 +08:00 committed by GitHub
parent af327dc8a6
commit 4eaed8df57
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
7 changed files with 423 additions and 39 deletions

View File

@ -41,7 +41,7 @@ help: ## Display this help.
.PHONY: manifests
manifests: controller-gen ## Generate WebhookConfiguration, ClusterRole and CustomResourceDefinition objects.
$(CONTROLLER_GEN) rbac:roleName=manager-role crd webhook paths="./..." output:crd:artifacts:config=config/crd/bases
$(CONTROLLER_GEN) rbac:roleName=manager-role crd:generateEmbeddedObjectMeta=true webhook paths="./..." output:crd:artifacts:config=config/crd/bases
.PHONY: generate
generate: controller-gen ## Generate code containing DeepCopy, DeepCopyInto, and DeepCopyObject method implementations.

View File

@ -124,8 +124,29 @@ type RollingUpdateStatefulSetStrategy struct {
type ScaleStrategy struct {
kruiseV1beta1.StatefulSetScaleStrategy `json:",inline"`
// ScaleDownStrategyType indicates the scaling down strategy.
// Default is GeneralScaleDownStrategyType
// +optional
ScaleDownStrategyType ScaleDownStrategyType `json:"scaleDownStrategyType,omitempty"`
}
// ScaleDownStrategyType is a string enumeration type that enumerates
// all possible scale down strategies for the GameServerSet controller.
// +enum
type ScaleDownStrategyType string
const (
// GeneralScaleDownStrategyType will first consider the ReserveGameServerIds
// field when game server scaling down. When the number of reserved game servers
// does not meet the scale down number, continue to select and delete the game
// servers from the current game server list.
GeneralScaleDownStrategyType ScaleDownStrategyType = "General"
// ReserveIdsScaleDownStrategyType will backfill the sequence numbers into
// ReserveGameServerIds field when GameServers scale down, whether set by
// ReserveGameServerIds field or the GameServerSet controller chooses to remove it.
ReserveIdsScaleDownStrategyType ScaleDownStrategyType = "ReserveIds"
)
// GameServerSetStatus defines the observed state of GameServerSet
type GameServerSetStatus struct {
// The generation observed by the controller.

View File

@ -89,6 +89,23 @@ spec:
type: string
metadata:
description: 'Standard object''s metadata. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#metadata'
properties:
annotations:
additionalProperties:
type: string
type: object
finalizers:
items:
type: string
type: array
labels:
additionalProperties:
type: string
type: object
name:
type: string
namespace:
type: string
type: object
spec:
description: 'spec defines the desired characteristics of
@ -400,6 +417,10 @@ spec:
from percentage by rounding down. It can just be allowed to
work with Parallel podManagementPolicy.'
x-kubernetes-int-or-string: true
scaleDownStrategyType:
description: ScaleDownStrategyType indicates the scaling down
strategy. Default is GeneralScaleDownStrategyType
type: string
type: object
serviceQualities:
items:

View File

@ -0,0 +1,276 @@
## GameServerSet
### GameServerSetSpec
```
type GameServerSetSpec struct {
// The number of game servers. Must be specified, with a minimum value of 0.
Replicas *int32 `json:"replicas"`
// Game server template. The new game server will be created with the parameters defined in GameServerTemplate.
GameServerTemplate GameServerTemplate `json:"gameServerTemplate,omitempty"`
// Reserved game server IDs, optional. If specified, existing game servers with those IDs will be deleted,
// and new game servers will not be created with those IDs.
ReserveGameServerIds []int `json:"reserveGameServerIds,omitempty"`
// Custom service qualities for game servers.
ServiceQualities []ServiceQuality `json:"serviceQualities,omitempty"`
// Batch update strategy for game servers.
UpdateStrategy UpdateStrategy `json:"updateStrategy,omitempty"`
// Horizontal scaling strategy for game servers.
ScaleStrategy ScaleStrategy `json:"scaleStrategy,omitempty"`
// Network settings for game server access layer.
Network *Network `json:"network,omitempty"`
}
```
#### GameServerTemplate
```yaml
type GameServerTemplate struct {
// All fields inherited from PodTemplateSpec.
corev1.PodTemplateSpec `json:",inline"`
// Requests and claims for persistent volumes.
VolumeClaimTemplates []corev1.PersistentVolumeClaim `json:"volumeClaimTemplates,omitempty"`
}
```
#### UpdateStrategy
```
type UpdateStrategy struct {
// Type indicates the type of the StatefulSetUpdateStrategy.
// Default is RollingUpdate.
// +optional
Type apps.StatefulSetUpdateStrategyType `json:"type,omitempty"`
// RollingUpdate is used to communicate parameters when Type is RollingUpdateStatefulSetStrategyType.
// +optional
RollingUpdate *RollingUpdateStatefulSetStrategy `json:"rollingUpdate,omitempty"`
}
type RollingUpdateStatefulSetStrategy struct {
// Partition indicates the ordinal at which the StatefulSet should be partitioned by default.
// But if unorderedUpdate has been set:
// - Partition indicates the number of pods with non-updated revisions when rolling update.
// - It means controller will update $(replicas - partition) number of pod.
// Default value is 0.
// +optional
Partition *int32 `json:"partition,omitempty"`
// The maximum number of pods that can be unavailable during the update.
// Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
// Absolute number is calculated from percentage by rounding down.
// Also, maxUnavailable can just be allowed to work with Parallel podManagementPolicy.
// Defaults to 1.
// +optional
MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
// PodUpdatePolicy indicates how pods should be updated
// Default value is "ReCreate"
// +optional
PodUpdatePolicy kruiseV1beta1.PodUpdateStrategyType `json:"podUpdatePolicy,omitempty"`
// Paused indicates that the StatefulSet is paused.
// Default value is false
// +optional
Paused bool `json:"paused,omitempty"`
// UnorderedUpdate contains strategies for non-ordered update.
// If it is not nil, pods will be updated with non-ordered sequence.
// Noted that UnorderedUpdate can only be allowed to work with Parallel podManagementPolicy
// +optional
// UnorderedUpdate *kruiseV1beta1.UnorderedUpdateStrategy `json:"unorderedUpdate,omitempty"`
// InPlaceUpdateStrategy contains strategies for in-place update.
// +optional
InPlaceUpdateStrategy *appspub.InPlaceUpdateStrategy `json:"inPlaceUpdateStrategy,omitempty"`
// MinReadySeconds indicates how long will the pod be considered ready after it's updated.
// MinReadySeconds works with both OrderedReady and Parallel podManagementPolicy.
// It affects the pod scale up speed when the podManagementPolicy is set to be OrderedReady.
// Combined with MaxUnavailable, it affects the pod update speed regardless of podManagementPolicy.
// Default value is 0, max is 300.
// +optional
MinReadySeconds *int32 `json:"minReadySeconds,omitempty"`
}
type InPlaceUpdateStrategy struct {
// GracePeriodSeconds is the timespan between set Pod status to not-ready and update images in Pod spec
// when in-place update a Pod.
GracePeriodSeconds int32 `json:"gracePeriodSeconds,omitempty"`
}
```
#### ScaleStrategy
```
type ScaleStrategy struct {
// The maximum number of pods that can be unavailable during scaling.
// Value can be an absolute number (ex: 5) or a percentage of desired pods (ex: 10%).
// Absolute number is calculated from percentage by rounding down.
// It can just be allowed to work with Parallel podManagementPolicy.
MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
// ScaleDownStrategyType indicates the scaling down strategy, include two types: General & ReserveIds
// General will first consider the ReserveGameServerIds field when game server scaling down.
// When the number of reserved game servers does not meet the scale down number, continue to
// select and delete the game servers from the current game server list.
// ReserveIds will backfill the sequence numbers into ReserveGameServerIds field when
// GameServers scale down, whether set by ReserveGameServerIds field or the GameServerSet
// controller chooses to remove it.
// Default is General
// +optional
ScaleDownStrategyType ScaleDownStrategyType `json:"scaleDownStrategyType,omitempty"`
}
```
#### ServiceQualities
```
type ServiceQuality struct {
// Inherits all fields from corev1.Probe
corev1.Probe `json:",inline"`
// Custom name for the service quality, distinguishes different service qualities that are defined.
Name string `json:"name"`
// Name of the container to be probed.
ContainerName string `json:"containerName,omitempty"`
// Whether to make GameServerSpec not change after the ServiceQualityAction is executed.
// When Permanent is true, regardless of the detection results, ServiceQualityAction will only be executed once.
// When Permanent is false, ServiceQualityAction can be executed again even though ServiceQualityAction has been executed.
Permanent bool `json:"permanent"`
// Corresponding actions to be executed for the service quality.
ServiceQualityAction []ServiceQualityAction `json:"serviceQualityAction,omitempty"`
}
type ServiceQualityAction struct {
// Defines to change the GameServerSpec field when the detection is true/false.
State bool `json:"state"`
GameServerSpec `json:",inline"`
}
```
#### Network
```
type Network struct {
// Different network types correspond to different network plugins.
NetworkType string `json:"networkType,omitempty"`
// Different network types need to fill in different network parameters.
NetworkConf []NetworkConfParams `json:"networkConf,omitempty"`
}
type NetworkConfParams KVParams
type KVParams struct {
// Parameter name, the name is determined by the network plugin
Name string `json:"name,omitempty"`
// Parameter value, the format is determined by the network plugin
Value string `json:"value,omitempty"`
}
```
### GameServerSetStatus
```yaml
type GameServerSetStatus struct {
// The iteration version of the GameServerSet observed by the controller.
ObservedGeneration int64 `json:"observedGeneration,omitempty"`
// The number of game servers.
Replicas int32 `json:"replicas"`
// The number of game servers that are ready.
ReadyReplicas int32 `json:"readyReplicas"`
// The number of game servers that are available.
AvailableReplicas int32 `json:"availableReplicas"`
// The current number of game servers.
CurrentReplicas int32 `json:"currentReplicas"`
// The number of game servers that have been updated.
UpdatedReplicas int32 `json:"updatedReplicas"`
// The number of game servers that have been updated and are ready.
UpdatedReadyReplicas int32 `json:"updatedReadyReplicas,omitempty"`
// The number of game servers that are in Maintaining state.
MaintainingReplicas *int32 `json:"maintainingReplicas,omitempty"`
// The number of game servers that are in WaitToBeDeleted state.
WaitToBeDeletedReplicas *int32 `json:"waitToBeDeletedReplicas,omitempty"`
// The label selector used to query game servers that should match the replica count used by HPA.
LabelSelector string `json:"labelSelector,omitempty"`
}
```
## GameServer
### GameServerSpec
```
type GameServerSpec struct {
// The O&M state of the game server, not pod runtime state, more biased towards the state of the game itself.
// Currently, the states that can be specified are: None / WaitToBeDeleted / Maintaining.
// Default is None
OpsState OpsState `json:"opsState,omitempty"`
// Update priority. If the priority is higher, it will be updated first.
UpdatePriority *intstr.IntOrString `json:"updatePriority,omitempty"`
// Deletion priority. If the priority is higher, it will be deleted first.
DeletionPriority *intstr.IntOrString `json:"deletionPriority,omitempty"`
// Whether to perform network isolation and cut off the access layer network
// Default is false
NetworkDisabled bool `json:"networkDisabled,omitempty"`
}
```
### GameServerStatus
```
type GameServerStatus struct {
// Expected game server status
DesiredState GameServerState `json:"desiredState,omitempty"`
// The actual status of the current game server
CurrentState GameServerState `json:"currentState,omitempty"`
// network status information
NetworkStatus NetworkStatus `json:"networkStatus,omitempty"`
// The game server corresponds to the pod status
PodStatus corev1.PodStatus `json:"podStatus,omitempty"`
// Service quality status of game server
ServiceQualitiesCondition []ServiceQualityCondition `json:"serviceQualitiesConditions,omitempty"`
// Current update priority
UpdatePriority *intstr.IntOrString `json:"updatePriority,omitempty"`
// Current deletion priority
DeletionPriority *intstr.IntOrString `json:"deletionPriority,omitempty"`
// Last change time
LastTransitionTime metav1.Time `json:"lastTransitionTime,omitempty"`
}
```

View File

@ -7,13 +7,13 @@ type GameServerSetSpec struct {
// 游戏服数目必须指定最小值为0
Replicas *int32 `json:"replicas"`
// 游戏服模版
// 游戏服模版,新生成的游戏服将以模版定义的参数创建
GameServerTemplate GameServerTemplate `json:"gameServerTemplate,omitempty"`
// 保留的游戏服序号,可选项。若指定了该序号,已经存在的游戏服将被删除;而未存在的游戏服,新建时将跳过、不创建该序号
ReserveGameServerIds []int `json:"reserveGameServerIds,omitempty"`
// 游戏服自定义服务质量
// 游戏服自定义服务质量。用户通过该字段实现游戏服自动化状态感知。
ServiceQualities []ServiceQuality `json:"serviceQualities,omitempty"`
// 游戏服批量更新策略
@ -121,6 +121,13 @@ type InPlaceUpdateStrategy struct {
type ScaleStrategy struct {
// 扩缩期间游戏服最大不可用的数量,可为绝对值或百分比
MaxUnavailable *intstr.IntOrString `json:"maxUnavailable,omitempty"`
// 缩容策略类型目前支持两种General 与 ReserveIds。
// 默认为General缩容时优先考虑reserveGameServerIds字段
// 当预留的GameServer数量不满足缩减数量时继续从当前游戏服务器列表中选择并删除GameServer。
// 当该字段设置为ReserveIds时无论是保留的游戏服还是控制器按照优先级删除的游戏服
// 被删除的游戏服的序号都会回填至ReserveGameServerIds字段。
ScaleDownStrategyType ScaleDownStrategyType `json:"scaleDownStrategyType,omitempty"`
}
```

View File

@ -107,7 +107,8 @@ func (manager *GameServerSetManager) GameServerScale() error {
klog.Infof("GameServers %s/%s already has %d replicas, expect to have %d replicas.", gss.GetNamespace(), gss.GetName(), currentReplicas, expectedReplicas)
manager.eventRecorder.Eventf(gss, corev1.EventTypeNormal, ScaleReason, "scale from %d to %d", currentReplicas, expectedReplicas)
asts.Spec.ReserveOrdinals = append(gssReserveIds, computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, gsList)...)
newReserveIds := computeToScaleGs(gssReserveIds, reserveIds, notExistIds, expectedReplicas, gsList, gss.Spec.ScaleStrategy.ScaleDownStrategyType)
asts.Spec.ReserveOrdinals = newReserveIds
asts.Spec.Replicas = gss.Spec.Replicas
asts.Spec.ScaleStrategy = &kruiseV1beta1.StatefulSetScaleStrategy{
MaxUnavailable: gss.Spec.ScaleStrategy.MaxUnavailable,
@ -118,9 +119,12 @@ func (manager *GameServerSetManager) GameServerScale() error {
return err
}
if gss.Spec.ScaleStrategy.ScaleDownStrategyType == gameKruiseV1alpha1.ReserveIdsScaleDownStrategyType {
gssReserveIds = newReserveIds
}
gssAnnotations := make(map[string]string)
gssAnnotations[gameKruiseV1alpha1.GameServerSetReserveIdsKey] = util.IntSliceToString(gssReserveIds, ",")
patchGss := map[string]interface{}{"metadata": map[string]map[string]string{"annotations": gssAnnotations}}
patchGss := map[string]interface{}{"spec": map[string]interface{}{"reserveGameServerIds": gssReserveIds}, "metadata": map[string]map[string]string{"annotations": gssAnnotations}}
patchGssBytes, _ := json.Marshal(patchGss)
err = c.Patch(ctx, gss, client.RawPatch(types.MergePatchType, patchGssBytes))
if err != nil {
@ -131,7 +135,7 @@ func (manager *GameServerSetManager) GameServerScale() error {
return nil
}
func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod) []int {
func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedReplicas int, pods []corev1.Pod, scaleDownType gameKruiseV1alpha1.ScaleDownStrategyType) []int {
workloadManageIds := util.GetIndexListFromPodList(pods)
var toAdd []int
@ -189,19 +193,23 @@ func computeToScaleGs(gssReserveIds, reserveIds, notExistIds []int, expectedRepl
}
}
if scaleDownType == gameKruiseV1alpha1.ReserveIdsScaleDownStrategyType {
return append(gssReserveIds, util.GetSliceInANotInB(toDelete, gssReserveIds)...)
}
newManageIds := append(workloadManageIds, util.GetSliceInANotInB(toAdd, workloadManageIds)...)
newManageIds = util.GetSliceInANotInB(newManageIds, toDelete)
var newNotExistIds []int
var newReserveIds []int
if len(newManageIds) != 0 {
sort.Ints(newManageIds)
for i := 0; i < newManageIds[len(newManageIds)-1]; i++ {
if !util.IsNumInList(i, newManageIds) && !util.IsNumInList(i, gssReserveIds) {
newNotExistIds = append(newNotExistIds, i)
if !util.IsNumInList(i, newManageIds) {
newReserveIds = append(newReserveIds, i)
}
}
}
return newNotExistIds
return newReserveIds
}
func (manager *GameServerSetManager) SyncGameServerReplicas() error {

View File

@ -35,14 +35,16 @@ func TestComputeToScaleGs(t *testing.T) {
oldGssreserveIds []int
notExistIds []int
expectedReplicas int
scaleDownStrategyType gameKruiseV1alpha1.ScaleDownStrategyType
pods []corev1.Pod
newNotExistIds []int
newReserveIds []int
}{
{
newGssReserveIds: []int{2, 3, 4},
oldGssreserveIds: []int{2, 3},
notExistIds: []int{5},
expectedReplicas: 3,
scaleDownStrategyType: gameKruiseV1alpha1.GeneralScaleDownStrategyType,
pods: []corev1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
@ -81,13 +83,14 @@ func TestComputeToScaleGs(t *testing.T) {
},
},
},
newNotExistIds: []int{5},
newReserveIds: []int{2, 3, 4, 5},
},
{
newGssReserveIds: []int{0, 2, 3},
oldGssreserveIds: []int{0, 4, 5},
notExistIds: []int{},
expectedReplicas: 3,
scaleDownStrategyType: gameKruiseV1alpha1.GeneralScaleDownStrategyType,
pods: []corev1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
@ -135,13 +138,14 @@ func TestComputeToScaleGs(t *testing.T) {
},
},
},
newNotExistIds: []int{4, 5},
newReserveIds: []int{0, 2, 3, 4, 5},
},
{
newGssReserveIds: []int{0},
oldGssreserveIds: []int{0, 4, 5},
notExistIds: []int{},
expectedReplicas: 1,
scaleDownStrategyType: gameKruiseV1alpha1.GeneralScaleDownStrategyType,
pods: []corev1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
@ -189,13 +193,14 @@ func TestComputeToScaleGs(t *testing.T) {
},
},
},
newNotExistIds: nil,
newReserveIds: []int{0},
},
{
newGssReserveIds: []int{0, 2, 3},
oldGssreserveIds: []int{0, 4, 5},
notExistIds: []int{},
expectedReplicas: 4,
scaleDownStrategyType: gameKruiseV1alpha1.GeneralScaleDownStrategyType,
pods: []corev1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
@ -243,14 +248,60 @@ func TestComputeToScaleGs(t *testing.T) {
},
},
},
newNotExistIds: []int{5},
newReserveIds: []int{0, 2, 3, 5},
},
{
newGssReserveIds: []int{0, 3, 5},
oldGssreserveIds: []int{0, 3, 5},
notExistIds: []int{},
expectedReplicas: 1,
scaleDownStrategyType: gameKruiseV1alpha1.ReserveIdsScaleDownStrategyType,
pods: []corev1.Pod{
{
ObjectMeta: metav1.ObjectMeta{
Name: "xxx-1",
Labels: map[string]string{
gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.None),
gameKruiseV1alpha1.GameServerDeletePriorityKey: "0",
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "xxx-2",
Labels: map[string]string{
gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.None),
gameKruiseV1alpha1.GameServerDeletePriorityKey: "0",
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "xxx-4",
Labels: map[string]string{
gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.None),
gameKruiseV1alpha1.GameServerDeletePriorityKey: "0",
},
},
},
{
ObjectMeta: metav1.ObjectMeta{
Name: "xxx-6",
Labels: map[string]string{
gameKruiseV1alpha1.GameServerOpsStateKey: string(gameKruiseV1alpha1.None),
gameKruiseV1alpha1.GameServerDeletePriorityKey: "0",
},
},
},
},
newReserveIds: []int{0, 3, 5, 2, 4, 6},
},
}
for i, test := range tests {
newNotExistIds := computeToScaleGs(test.newGssReserveIds, test.oldGssreserveIds, test.notExistIds, test.expectedReplicas, test.pods)
if !util.IsSliceEqual(newNotExistIds, test.newNotExistIds) {
t.Errorf("case %d: expect newNotExistIds %v but got %v", i, test.newNotExistIds, newNotExistIds)
newReserveIds := computeToScaleGs(test.newGssReserveIds, test.oldGssreserveIds, test.notExistIds, test.expectedReplicas, test.pods, test.scaleDownStrategyType)
if !util.IsSliceEqual(newReserveIds, test.newReserveIds) {
t.Errorf("case %d: expect newNotExistIds %v but got %v", i, test.newReserveIds, newReserveIds)
}
}
}