enhance: add annotations config in clb plugin (#137)

Co-authored-by: 李志朋 <lizhipeng.629@bytedance.com>

Makefile

@@ -130,7 +130,7 @@ $(CONTROLLER_GEN): $(LOCALBIN)
 .PHONY: envtest
 envtest: $(ENVTEST) ## Download envtest-setup locally if necessary.
 $(ENVTEST): $(LOCALBIN)
-	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
+	GOBIN=$(LOCALBIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@c7e1dc9b5302d649d5531e19168dd7ea0013736d
 
 HELM = $(shell pwd)/bin/helm
 helm: ## Download helm locally if necessary.

cloudprovider/volcengine/README.md

@@ -34,6 +34,10 @@ min_port = 500
 - Value：{containerName_0},{containerName_1},... eg：sidecar
 - Configurable：It cannot be changed during the in-place updating process.
 
+#### Annotations
+- Meaning：the anno added to the service
+- Value：key1:value1,key2:value2...
+- Configurable：Y
 
 ### Example
 ```yaml
@@ -62,6 +66,10 @@ spec:
       - name: Fixed
         #Fill in here whether a fixed IP is required [optional] ; Default is false
         value: "false"
+      - name: Annotations
+        #Fill in the anno related to clb on the service
+        #The format is as follows: {key1}:{value1},{key2}:{value2}...
+        value: "key1:value1,key2:value2"
   gameServerTemplate:
     spec:
       containers:

cloudprovider/volcengine/README.zh_CN.md

@@ -32,7 +32,12 @@ min_port = 500
 #### AllowNotReadyContainers
 - 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个
 - 填写格式：{containerName_0},{containerName_1},... 例如：sidecar
-- 是否支持变更：在原地升级过程中不可变更。
+- 是否支持变更：在原地升级过程中不可变更
+
+#### Annotations
+- 含义：添加在service上的anno，可填写多个
+- 填写格式：key1:value1,key2:value2...
+- 是否支持变更：是
 
 
 ### 使用示例
@@ -62,6 +67,10 @@ spec:
       - name: Fixed
         #Fill in here whether a fixed IP is required [optional] ; Default is false
         value: "false"
+      - name: Annotations
+        #Fill in the anno related to clb on the service
+        #The format is as follows: {key1}:{value1},{key2}:{value2}...
+        value: "key1:value1,key2:value2"
   gameServerTemplate:
     spec:
       containers:

cloudprovider/volcengine/clb.go

@@ -45,6 +45,7 @@ const (
 	ClbIdsConfigName        = "ClbIds"
 	PortProtocolsConfigName = "PortProtocols"
 	FixedConfigName         = "Fixed"
+	ClbAnnotations          = "Annotations"
 	ClbConfigHashKey        = "game.kruise.io/network-config-hash"
 	ClbIdAnnotationKey      = "service.beta.kubernetes.io/volcengine-loadbalancer-id"
 	ClbAddressTypeKey       = "service.beta.kubernetes.io/volcengine-loadbalancer-address-type"
@@ -69,6 +70,7 @@ type clbConfig struct {
 	targetPorts []int
 	protocols   []corev1.Protocol
 	isFixed     bool
+	annotations map[string]string
 }
 
 func (c *ClbPlugin) Name() string {
@@ -351,6 +353,7 @@ func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *clbConfig {
 	ports := make([]int, 0)
 	protocols := make([]corev1.Protocol, 0)
 	isFixed := false
+	annotations := map[string]string{}
 	for _, c := range conf {
 		switch c.Name {
 		case ClbIdsConfigName:
@@ -379,6 +382,15 @@ func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *clbConfig {
 				continue
 			}
 			isFixed = v
+		case ClbAnnotations:
+			for _, anno := range strings.Split(c.Value, ",") {
+				annoKV := strings.Split(anno, ":")
+				if len(annoKV) == 2 {
+					annotations[annoKV[0]] = annoKV[1]
+				} else {
+					log.Warningf("clb annotation %s is invalid", annoKV[0])
+				}
+			}
 		}
 	}
 	return &clbConfig{
@@ -386,6 +398,7 @@ func parseLbConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *clbConfig {
 		protocols:   protocols,
 		targetPorts: ports,
 		isFixed:     isFixed,
+		annotations: annotations,
 	}
 }
 
@@ -420,16 +433,21 @@ func (c *ClbPlugin) consSvc(config *clbConfig, pod *corev1.Pod, client client.Cl
 		})
 	}
 
+	annotations := map[string]string{
+		ClbSchedulerKey:    ClbSchedulerWRR,
+		ClbAddressTypeKey:  ClbAddressTypePublic,
+		ClbIdAnnotationKey: lbId,
+		ClbConfigHashKey:   util.GetHash(config),
+	}
+	for key, value := range config.annotations {
+		annotations[key] = value
+	}
+
 	svc := &corev1.Service{
 		ObjectMeta: metav1.ObjectMeta{
-			Name:      pod.GetName(),
-			Namespace: pod.GetNamespace(),
-			Annotations: map[string]string{
-				ClbSchedulerKey:    ClbSchedulerWRR,
-				ClbAddressTypeKey:  ClbAddressTypePublic,
-				ClbIdAnnotationKey: lbId,
-				ClbConfigHashKey:   util.GetHash(config),
-			},
+			Name:            pod.GetName(),
+			Namespace:       pod.GetNamespace(),
+			Annotations:     annotations,
 			OwnerReferences: getSvcOwnerReference(client, ctx, pod, config.isFixed),
 		},
 		Spec: corev1.ServiceSpec{

cloudprovider/volcengine/clb_test.go

@@ -259,6 +259,12 @@ func TestClbPlugin_consSvc(t *testing.T) {
 						corev1.ProtocolTCP,
 					},
 					isFixed: false,
+					annotations: map[string]string{
+						"service.beta.kubernetes.io/volcengine-loadbalancer-health-check-flag": "on",
+						"service.beta.kubernetes.io/volcengine-loadbalancer-healthy-threshold": "3",
+						"service.beta.kubernetes.io/volcengine-loadbalancer-scheduler":         "wrr",
+						"service.beta.kubernetes.io/volcengine-loadbalancer-pass-through":      "true",
+					},
 				},
 				pod: &corev1.Pod{
 					TypeMeta: metav1.TypeMeta{
@@ -289,7 +295,16 @@ func TestClbPlugin_consSvc(t *testing.T) {
 								corev1.ProtocolTCP,
 							},
 							isFixed: false,
+							annotations: map[string]string{
+								"service.beta.kubernetes.io/volcengine-loadbalancer-health-check-flag": "on",
+								"service.beta.kubernetes.io/volcengine-loadbalancer-healthy-threshold": "3",
+								"service.beta.kubernetes.io/volcengine-loadbalancer-scheduler":         "wrr",
+								"service.beta.kubernetes.io/volcengine-loadbalancer-pass-through":      "true",
+							},
 						}),
+						"service.beta.kubernetes.io/volcengine-loadbalancer-health-check-flag": "on",
+						"service.beta.kubernetes.io/volcengine-loadbalancer-healthy-threshold": "3",
+						"service.beta.kubernetes.io/volcengine-loadbalancer-pass-through":      "true",
 					},
 					OwnerReferences: []metav1.OwnerReference{
 						{