feat: add network plugin AlibabaCloud-NLB-SharedPort & support AllowNotReadyContainers (#98)

Signed-off-by: ChrisLiu <chrisliu1995@163.com>

apis/v1alpha1/gameserverset_types.go

@@ -36,6 +36,10 @@ const (
 	GsTemplateMetadataHashKey  = "game.kruise.io/gsTemplate-metadata-hash"
 )
 
+const (
+	InplaceUpdateNotReadyBlocker = "game.kruise.io/inplace-update-not-ready-blocker"
+)
+
 // GameServerSetSpec defines the desired state of GameServerSet
 type GameServerSetSpec struct {
 	// replicas is the desired number of replicas of the given Template.
@@ -69,6 +73,10 @@ type Network struct {
 
 type NetworkConfParams KVParams
 
+const (
+	AllowNotReadyContainersNetworkConfName = "AllowNotReadyContainers"
+)
+
 type KVParams struct {
 	Name  string `json:"name,omitempty"`
 	Value string `json:"value,omitempty"`

cloudprovider/alibabacloud/nlb_sp.go

@@ -0,0 +1,239 @@
+package alibabacloud
+
+import (
+	"context"
+	gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
+	"github.com/openkruise/kruise-game/cloudprovider"
+	cperrors "github.com/openkruise/kruise-game/cloudprovider/errors"
+	"github.com/openkruise/kruise-game/cloudprovider/utils"
+	"github.com/openkruise/kruise-game/pkg/util"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/apimachinery/pkg/util/intstr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"strconv"
+)
+
+const (
+	NlbSPNetwork     = "AlibabaCloud-NLB-SharedPort"
+	NlbIdsConfigName = "NlbIds"
+)
+
+func init() {
+	alibabaCloudProvider.registerPlugin(&NlbSpPlugin{})
+}
+
+type NlbSpPlugin struct {
+}
+
+func (N *NlbSpPlugin) Name() string {
+	return NlbSPNetwork
+}
+
+func (N *NlbSpPlugin) Alias() string {
+	return ""
+}
+
+func (N *NlbSpPlugin) Init(client client.Client, options cloudprovider.CloudProviderOptions, ctx context.Context) error {
+	return nil
+}
+
+func (N *NlbSpPlugin) OnPodAdded(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
+	networkManager := utils.NewNetworkManager(pod, c)
+	podNetConfig := parseNLbSpConfig(networkManager.GetNetworkConfig())
+
+	pod.Labels[SlbIdLabelKey] = podNetConfig.lbId
+
+	// Get Svc
+	svc := &corev1.Service{}
+	err := c.Get(ctx, types.NamespacedName{
+		Namespace: pod.GetNamespace(),
+		Name:      podNetConfig.lbId,
+	}, svc)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			// Create Svc
+			return pod, cperrors.ToPluginError(c.Create(ctx, consNlbSvc(podNetConfig, pod, c, ctx)), cperrors.ApiCallError)
+		}
+		return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
+	}
+	return pod, nil
+}
+
+func (N *NlbSpPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.Context) (*corev1.Pod, cperrors.PluginError) {
+	networkManager := utils.NewNetworkManager(pod, c)
+	networkStatus, _ := networkManager.GetNetworkStatus()
+	if networkStatus == nil {
+		pod, err := networkManager.UpdateNetworkStatus(gamekruiseiov1alpha1.NetworkStatus{
+			CurrentNetworkState: gamekruiseiov1alpha1.NetworkNotReady,
+		}, pod)
+		return pod, cperrors.ToPluginError(err, cperrors.InternalError)
+	}
+
+	networkConfig := networkManager.GetNetworkConfig()
+	podNetConfig := parseNLbSpConfig(networkConfig)
+
+	// Get Svc
+	svc := &corev1.Service{}
+	err := c.Get(context.Background(), types.NamespacedName{
+		Namespace: pod.GetNamespace(),
+		Name:      podNetConfig.lbId,
+	}, svc)
+	if err != nil {
+		if errors.IsNotFound(err) {
+			// Create Svc
+			return pod, cperrors.ToPluginError(c.Create(ctx, consNlbSvc(podNetConfig, pod, c, ctx)), cperrors.ApiCallError)
+		}
+		return pod, cperrors.NewPluginError(cperrors.ApiCallError, err.Error())
+	}
+
+	// update svc
+	if util.GetHash(podNetConfig) != svc.GetAnnotations()[SlbConfigHashKey] {
+		networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady
+		pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
+		if err != nil {
+			return pod, cperrors.NewPluginError(cperrors.InternalError, err.Error())
+		}
+		return pod, cperrors.ToPluginError(c.Update(ctx, consNlbSvc(podNetConfig, pod, c, ctx)), cperrors.ApiCallError)
+	}
+
+	_, hasLabel := pod.Labels[SlbIdLabelKey]
+	// disable network
+	if networkManager.GetNetworkDisabled() && hasLabel {
+		newLabels := pod.GetLabels()
+		delete(newLabels, SlbIdLabelKey)
+		pod.Labels = newLabels
+		networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkNotReady
+		pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
+		return pod, cperrors.ToPluginError(err, cperrors.InternalError)
+	}
+
+	// enable network
+	if !networkManager.GetNetworkDisabled() && !hasLabel {
+		pod.Labels[SlbIdLabelKey] = podNetConfig.lbId
+		networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady
+		pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
+		return pod, cperrors.ToPluginError(err, cperrors.InternalError)
+	}
+
+	// network not ready
+	if svc.Status.LoadBalancer.Ingress == nil {
+		return pod, nil
+	}
+
+	// allow not ready containers
+	if util.IsAllowNotReadyContainers(networkConfig) {
+		toUpDateSvc, err := utils.AllowNotReadyContainers(c, ctx, pod, svc, true)
+		if err != nil {
+			return pod, err
+		}
+
+		if toUpDateSvc {
+			err := c.Update(ctx, svc)
+			if err != nil {
+				return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
+			}
+		}
+	}
+
+	// network ready
+	internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)
+	externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)
+	for _, port := range svc.Spec.Ports {
+		instrIPort := port.TargetPort
+		instrEPort := intstr.FromInt(int(port.Port))
+		internalAddress := gamekruiseiov1alpha1.NetworkAddress{
+			IP: pod.Status.PodIP,
+			Ports: []gamekruiseiov1alpha1.NetworkPort{
+				{
+					Name:     instrIPort.String(),
+					Port:     &instrIPort,
+					Protocol: port.Protocol,
+				},
+			},
+		}
+		externalAddress := gamekruiseiov1alpha1.NetworkAddress{
+			EndPoint: svc.Status.LoadBalancer.Ingress[0].Hostname,
+			Ports: []gamekruiseiov1alpha1.NetworkPort{
+				{
+					Name:     instrIPort.String(),
+					Port:     &instrEPort,
+					Protocol: port.Protocol,
+				},
+			},
+		}
+		internalAddresses = append(internalAddresses, internalAddress)
+		externalAddresses = append(externalAddresses, externalAddress)
+	}
+	networkStatus.InternalAddresses = internalAddresses
+	networkStatus.ExternalAddresses = externalAddresses
+	networkStatus.CurrentNetworkState = gamekruiseiov1alpha1.NetworkReady
+	pod, err = networkManager.UpdateNetworkStatus(*networkStatus, pod)
+	return pod, cperrors.ToPluginError(err, cperrors.InternalError)
+}
+
+func (N *NlbSpPlugin) OnPodDeleted(client client.Client, pod *corev1.Pod, ctx context.Context) cperrors.PluginError {
+	return nil
+}
+
+type nlbConfig struct {
+	lbId      string
+	ports     []int
+	protocols []corev1.Protocol
+}
+
+func parseNLbSpConfig(conf []gamekruiseiov1alpha1.NetworkConfParams) *nlbConfig {
+	var lbIds string
+	var ports []int
+	var protocols []corev1.Protocol
+	for _, c := range conf {
+		switch c.Name {
+		case NlbIdsConfigName:
+			lbIds = c.Value
+		case PortProtocolsConfigName:
+			ports, protocols = parsePortProtocols(c.Value)
+		}
+	}
+	return &nlbConfig{
+		lbId:      lbIds,
+		ports:     ports,
+		protocols: protocols,
+	}
+}
+
+func consNlbSvc(nc *nlbConfig, pod *corev1.Pod, c client.Client, ctx context.Context) *corev1.Service {
+	svcPorts := make([]corev1.ServicePort, 0)
+	for i := 0; i < len(nc.ports); i++ {
+		svcPorts = append(svcPorts, corev1.ServicePort{
+			Name:       strconv.Itoa(nc.ports[i]),
+			Port:       int32(nc.ports[i]),
+			Protocol:   nc.protocols[i],
+			TargetPort: intstr.FromInt(nc.ports[i]),
+		})
+	}
+	loadBalancerClass := "alibabacloud.com/nlb"
+
+	svc := &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name:      nc.lbId,
+			Namespace: pod.GetNamespace(),
+			Annotations: map[string]string{
+				SlbListenerOverrideKey: "true",
+				SlbIdAnnotationKey:     nc.lbId,
+				SlbConfigHashKey:       util.GetHash(nc),
+			},
+			OwnerReferences: getSvcOwnerReference(c, ctx, pod, true),
+		},
+		Spec: corev1.ServiceSpec{
+			Type: corev1.ServiceTypeLoadBalancer,
+			Selector: map[string]string{
+				SlbIdLabelKey: nc.lbId,
+			},
+			Ports:             svcPorts,
+			LoadBalancerClass: &loadBalancerClass,
+		},
+	}
+	return svc
+}

cloudprovider/alibabacloud/nlb_sp_test.go

@@ -0,0 +1,58 @@
+package alibabacloud
+
+import (
+	gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	"reflect"
+	"testing"
+)
+
+func TestParseNLbSpConfig(t *testing.T) {
+	tests := []struct {
+		conf []gamekruiseiov1alpha1.NetworkConfParams
+		nc   *nlbConfig
+	}{
+		{
+			conf: []gamekruiseiov1alpha1.NetworkConfParams{
+				{
+					Name:  NlbIdsConfigName,
+					Value: "nlb-xxx",
+				},
+				{
+					Name:  PortProtocolsConfigName,
+					Value: "80/UDP",
+				},
+			},
+			nc: &nlbConfig{
+				protocols: []corev1.Protocol{corev1.ProtocolUDP},
+				ports:     []int{80},
+				lbId:      "nlb-xxx",
+			},
+		},
+		{
+			conf: []gamekruiseiov1alpha1.NetworkConfParams{
+				{
+					Name:  NlbIdsConfigName,
+					Value: "nlb-xxx",
+				},
+				{
+					Name:  PortProtocolsConfigName,
+					Value: "80",
+				},
+			},
+			nc: &nlbConfig{
+				protocols: []corev1.Protocol{corev1.ProtocolTCP},
+				ports:     []int{80},
+				lbId:      "nlb-xxx",
+			},
+		},
+	}
+
+	for i, test := range tests {
+		expect := test.nc
+		actual := parseNLbSpConfig(test.conf)
+		if !reflect.DeepEqual(expect, actual) {
+			t.Errorf("case %d: expect nlbConfig is %v, but actually is %v", i, expect, actual)
+		}
+	}
+}

cloudprovider/alibabacloud/slb.go

@@ -182,6 +182,21 @@ func (s *SlbPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context.C
 		return pod, cperrors.ToPluginError(err, cperrors.InternalError)
 	}
 
+	// allow not ready containers
+	if util.IsAllowNotReadyContainers(networkManager.GetNetworkConfig()) {
+		toUpDateSvc, err := utils.AllowNotReadyContainers(c, ctx, pod, svc, false)
+		if err != nil {
+			return pod, err
+		}
+
+		if toUpDateSvc {
+			err := c.Update(ctx, svc)
+			if err != nil {
+				return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
+			}
+		}
+	}
+
 	// network ready
 	internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)
 	externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)

cloudprovider/alibabacloud/slb_sp.go

@@ -7,6 +7,7 @@ import (
 	"github.com/openkruise/kruise-game/cloudprovider"
 	cperrors "github.com/openkruise/kruise-game/cloudprovider/errors"
 	"github.com/openkruise/kruise-game/cloudprovider/utils"
+	"github.com/openkruise/kruise-game/pkg/util"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -130,6 +131,21 @@ func (s *SlbSpPlugin) OnPodUpdated(c client.Client, pod *corev1.Pod, ctx context
 		return pod, nil
 	}
 
+	// allow not ready containers
+	if util.IsAllowNotReadyContainers(networkManager.GetNetworkConfig()) {
+		toUpDateSvc, err := utils.AllowNotReadyContainers(c, ctx, pod, svc, true)
+		if err != nil {
+			return pod, err
+		}
+
+		if toUpDateSvc {
+			err := c.Update(ctx, svc)
+			if err != nil {
+				return pod, cperrors.ToPluginError(err, cperrors.ApiCallError)
+			}
+		}
+	}
+
 	// network ready
 	internalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)
 	externalAddresses := make([]gamekruiseiov1alpha1.NetworkAddress, 0)

cloudprovider/alibabacloud/slb_sp_test.go

@@ -3,6 +3,7 @@ package alibabacloud
 import (
 	"fmt"
 	gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
+	"github.com/openkruise/kruise-game/pkg/util"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"reflect"
@@ -121,3 +122,32 @@ func TestParseLbSpConfig(t *testing.T) {
 		}
 	}
 }
+
+func TestParsePortProtocols(t *testing.T) {
+	tests := []struct {
+		value     string
+		ports     []int
+		protocols []corev1.Protocol
+	}{
+		{
+			value:     "80",
+			ports:     []int{80},
+			protocols: []corev1.Protocol{corev1.ProtocolTCP},
+		},
+		{
+			value:     "8080/UDP,80/TCP",
+			ports:     []int{8080, 80},
+			protocols: []corev1.Protocol{corev1.ProtocolUDP, corev1.ProtocolTCP},
+		},
+	}
+
+	for i, test := range tests {
+		actualPorts, actualProtocols := parsePortProtocols(test.value)
+		if !util.IsSliceEqual(actualPorts, test.ports) {
+			t.Errorf("case %d: expect ports is %v, but actually is %v", i, test.ports, actualPorts)
+		}
+		if !reflect.DeepEqual(actualProtocols, test.protocols) {
+			t.Errorf("case %d: expect protocols is %v, but actually is %v", i, test.protocols, actualProtocols)
+		}
+	}
+}

cloudprovider/utils/service.go

@@ -0,0 +1,85 @@
+package utils
+
+import (
+	"context"
+	kruisePub "github.com/openkruise/kruise-api/apps/pub"
+	gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
+	cperrors "github.com/openkruise/kruise-game/cloudprovider/errors"
+	"github.com/openkruise/kruise-game/pkg/util"
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/labels"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"strings"
+)
+
+func AllowNotReadyContainers(c client.Client, ctx context.Context, pod *corev1.Pod, svc *corev1.Service, isSvcShared bool) (bool, cperrors.PluginError) {
+	// get lifecycleState
+	lifecycleState, exist := pod.GetLabels()[kruisePub.LifecycleStateKey]
+
+	// get gss
+	gss, err := util.GetGameServerSetOfPod(pod, c, ctx)
+	if err != nil {
+		return false, cperrors.ToPluginError(err, cperrors.ApiCallError)
+	}
+
+	// get allowNotReadyContainers
+	var allowNotReadyContainers []string
+	for _, kv := range gss.Spec.Network.NetworkConf {
+		if kv.Name == gamekruiseiov1alpha1.AllowNotReadyContainersNetworkConfName {
+			for _, allowNotReadyContainer := range strings.Split(kv.Value, ",") {
+				if allowNotReadyContainer != "" {
+					allowNotReadyContainers = append(allowNotReadyContainers, allowNotReadyContainer)
+				}
+			}
+		}
+	}
+
+	// PreInplaceUpdating
+	if exist && lifecycleState == string(kruisePub.LifecycleStatePreparingUpdate) {
+		// ensure PublishNotReadyAddresses is true when containers pre-updating
+		if !svc.Spec.PublishNotReadyAddresses && util.IsContainersPreInplaceUpdating(pod, gss, allowNotReadyContainers) {
+			svc.Spec.PublishNotReadyAddresses = true
+			return true, nil
+		}
+
+		// ensure remove finalizer
+		if svc.Spec.PublishNotReadyAddresses || !util.IsContainersPreInplaceUpdating(pod, gss, allowNotReadyContainers) {
+			pod.GetLabels()[gamekruiseiov1alpha1.InplaceUpdateNotReadyBlocker] = "false"
+		}
+	} else {
+		pod.GetLabels()[gamekruiseiov1alpha1.InplaceUpdateNotReadyBlocker] = "true"
+		if !svc.Spec.PublishNotReadyAddresses {
+			return false, nil
+		}
+		if isSvcShared {
+			// ensure PublishNotReadyAddresses is false when all pods are updated
+			if gss.Status.UpdatedReplicas == gss.Status.Replicas {
+				podList := &corev1.PodList{}
+				err := c.List(ctx, podList, &client.ListOptions{
+					Namespace: gss.GetNamespace(),
+					LabelSelector: labels.SelectorFromSet(map[string]string{
+						gamekruiseiov1alpha1.GameServerOwnerGssKey: gss.GetName(),
+					})})
+				if err != nil {
+					return false, cperrors.ToPluginError(err, cperrors.ApiCallError)
+				}
+				for _, p := range podList.Items {
+					_, condition := util.GetPodConditionFromList(p.Status.Conditions, corev1.PodReady)
+					if condition == nil || condition.Status != corev1.ConditionTrue {
+						return false, nil
+					}
+				}
+				svc.Spec.PublishNotReadyAddresses = false
+				return true, nil
+			}
+		} else {
+			_, condition := util.GetPodConditionFromList(pod.Status.Conditions, corev1.PodReady)
+			if condition == nil || condition.Status != corev1.ConditionTrue {
+				return false, nil
+			}
+			svc.Spec.PublishNotReadyAddresses = false
+			return true, nil
+		}
+	}
+	return false, nil
+}

cloudprovider/utils/service_test.go

@@ -0,0 +1,289 @@
+package utils
+
+import (
+	"context"
+	kruisePub "github.com/openkruise/kruise-api/apps/pub"
+	kruiseV1alpha1 "github.com/openkruise/kruise-api/apps/v1alpha1"
+	kruiseV1beta1 "github.com/openkruise/kruise-api/apps/v1beta1"
+	gamekruiseiov1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/fake"
+	"testing"
+)
+
+var (
+	scheme = runtime.NewScheme()
+)
+
+func init() {
+	utilruntime.Must(gamekruiseiov1alpha1.AddToScheme(scheme))
+	utilruntime.Must(kruiseV1beta1.AddToScheme(scheme))
+	utilruntime.Must(kruiseV1alpha1.AddToScheme(scheme))
+	utilruntime.Must(corev1.AddToScheme(scheme))
+}
+
+func TestAllowNotReadyContainers(t *testing.T) {
+	tests := []struct {
+		// input
+		pod         *corev1.Pod
+		svc         *corev1.Service
+		gss         *gamekruiseiov1alpha1.GameServerSet
+		isSvcShared bool
+		podElse     []*corev1.Pod
+		// output
+		inplaceUpdateNotReadyBlocker string
+		isSvcUpdated                 bool
+	}{
+		// When svc is not shared, pod updated, svc should not publish NotReadyAddresses
+		{
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "xxx",
+					Name:      "case0-0",
+					UID:       "xxx0",
+					Labels: map[string]string{
+						kruisePub.LifecycleStateKey:                string(kruisePub.LifecycleStateUpdating),
+						gamekruiseiov1alpha1.GameServerOwnerGssKey: "case0",
+					},
+				},
+				Status: corev1.PodStatus{
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionTrue,
+						},
+					},
+					ContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name:  "name_A",
+							Image: "v1.0",
+						},
+						{
+							Name:  "name_B",
+							Image: "v1.0",
+						},
+					},
+				},
+			},
+			svc: &corev1.Service{
+				Spec: corev1.ServiceSpec{
+					PublishNotReadyAddresses: true,
+				},
+			},
+			gss: &gamekruiseiov1alpha1.GameServerSet{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "GameServerSet",
+					APIVersion: "game.kruise.io/v1alpha1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "xxx",
+					Name:      "case0",
+					UID:       "xxx0",
+				},
+				Spec: gamekruiseiov1alpha1.GameServerSetSpec{
+					Network: &gamekruiseiov1alpha1.Network{
+						NetworkConf: []gamekruiseiov1alpha1.NetworkConfParams{
+							{
+								Name:  gamekruiseiov1alpha1.AllowNotReadyContainersNetworkConfName,
+								Value: "name_B",
+							},
+						},
+					},
+					GameServerTemplate: gamekruiseiov1alpha1.GameServerTemplate{
+						PodTemplateSpec: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								Containers: []corev1.Container{
+									{
+										Name:  "name_A",
+										Image: "v1.0",
+									},
+									{
+										Name:  "name_B",
+										Image: "v1.0",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			isSvcShared:                  false,
+			inplaceUpdateNotReadyBlocker: "true",
+			isSvcUpdated:                 true,
+		},
+		// When svc is not shared & pod is pre-updating & svc PublishNotReadyAddresses is false, svc should publish NotReadyAddresses
+		{
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "xxx",
+					Name:      "case1-0",
+					UID:       "xxx0",
+					Labels: map[string]string{
+						kruisePub.LifecycleStateKey:                       string(kruisePub.LifecycleStatePreparingUpdate),
+						gamekruiseiov1alpha1.InplaceUpdateNotReadyBlocker: "true",
+						gamekruiseiov1alpha1.GameServerOwnerGssKey:        "case1",
+					},
+				},
+				Status: corev1.PodStatus{
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionTrue,
+						},
+					},
+					ContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name:  "name_A",
+							Image: "v1.0",
+						},
+						{
+							Name:  "name_B",
+							Image: "v1.0",
+						},
+					},
+				},
+			},
+			svc: &corev1.Service{
+				Spec: corev1.ServiceSpec{
+					PublishNotReadyAddresses: false,
+				},
+			},
+			gss: &gamekruiseiov1alpha1.GameServerSet{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "GameServerSet",
+					APIVersion: "game.kruise.io/v1alpha1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "xxx",
+					Name:      "case1",
+					UID:       "xxx0",
+				},
+				Spec: gamekruiseiov1alpha1.GameServerSetSpec{
+					Network: &gamekruiseiov1alpha1.Network{
+						NetworkConf: []gamekruiseiov1alpha1.NetworkConfParams{
+							{
+								Name:  gamekruiseiov1alpha1.AllowNotReadyContainersNetworkConfName,
+								Value: "name_B",
+							},
+						},
+					},
+					GameServerTemplate: gamekruiseiov1alpha1.GameServerTemplate{
+						PodTemplateSpec: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								Containers: []corev1.Container{
+									{
+										Name:  "name_A",
+										Image: "v1.0",
+									},
+									{
+										Name:  "name_B",
+										Image: "v2.0",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			isSvcShared:                  false,
+			inplaceUpdateNotReadyBlocker: "true",
+			isSvcUpdated:                 true,
+		},
+		// When svc is not shared & pod is pre-updating & svc PublishNotReadyAddresses is true, finalizer of pod should be removed to enter next stage
+		{
+			pod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "xxx",
+					Name:      "case2-0",
+					UID:       "xxx0",
+					Labels: map[string]string{
+						kruisePub.LifecycleStateKey:                string(kruisePub.LifecycleStatePreparingUpdate),
+						gamekruiseiov1alpha1.GameServerOwnerGssKey: "case2",
+					},
+				},
+				Status: corev1.PodStatus{
+					Conditions: []corev1.PodCondition{
+						{
+							Type:   corev1.PodReady,
+							Status: corev1.ConditionTrue,
+						},
+					},
+					ContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name:  "name_A",
+							Image: "v1.0",
+						},
+						{
+							Name:  "name_B",
+							Image: "v1.0",
+						},
+					},
+				},
+			},
+			svc: &corev1.Service{
+				Spec: corev1.ServiceSpec{
+					PublishNotReadyAddresses: true,
+				},
+			},
+			gss: &gamekruiseiov1alpha1.GameServerSet{
+				TypeMeta: metav1.TypeMeta{
+					Kind:       "GameServerSet",
+					APIVersion: "game.kruise.io/v1alpha1",
+				},
+				ObjectMeta: metav1.ObjectMeta{
+					Namespace: "xxx",
+					Name:      "case2",
+					UID:       "xxx0",
+				},
+				Spec: gamekruiseiov1alpha1.GameServerSetSpec{
+					Network: &gamekruiseiov1alpha1.Network{
+						NetworkConf: []gamekruiseiov1alpha1.NetworkConfParams{
+							{
+								Name:  gamekruiseiov1alpha1.AllowNotReadyContainersNetworkConfName,
+								Value: "name_B",
+							},
+						},
+					},
+					GameServerTemplate: gamekruiseiov1alpha1.GameServerTemplate{
+						PodTemplateSpec: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								Containers: []corev1.Container{
+									{
+										Name:  "name_A",
+										Image: "v1.0",
+									},
+									{
+										Name:  "name_B",
+										Image: "v1.0",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			isSvcShared:                  false,
+			inplaceUpdateNotReadyBlocker: "false",
+			isSvcUpdated:                 false,
+		},
+	}
+
+	for i, test := range tests {
+		objs := []client.Object{test.gss, test.pod, test.svc}
+		c := fake.NewClientBuilder().WithScheme(scheme).WithObjects(objs...).Build()
+		actual, err := AllowNotReadyContainers(c, context.TODO(), test.pod, test.svc, test.isSvcShared)
+		if err != nil {
+			t.Errorf("case %d: %s", i, err.Error())
+		}
+		if actual != test.isSvcUpdated {
+			t.Errorf("case %d: expect isSvcUpdated is %v but actually got %v", i, test.isSvcUpdated, actual)
+		}
+		if test.pod.GetLabels()[gamekruiseiov1alpha1.InplaceUpdateNotReadyBlocker] != test.inplaceUpdateNotReadyBlocker {
+			t.Errorf("case %d: expect inplaceUpdateNotReadyBlocker is %v but actually got %v", i, test.inplaceUpdateNotReadyBlocker, test.pod.GetLabels()[gamekruiseiov1alpha1.InplaceUpdateNotReadyBlocker])
+		}
+	}
+}

docs/en/user_manuals/network.md

@@ -430,6 +430,12 @@ Fixed
 - Value: false or true.
 - Configuration change supported or not: yes.
 
+AllowNotReadyContainers
+
+- Meaning: the container names that are allowed not ready when inplace updating, when traffic will not be cut.
+- Value: {containerName_0},{containerName_1},... Example：sidecar
+- Configuration change supported or not: It cannot be changed during the in-place updating process.
+
 #### Plugin configuration
 ```
 [alibabacloud]
@@ -473,6 +479,12 @@ PortProtocols
 - Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters.
 - Configuration change supported or not: no. The configuration change can be supported in future.
 
+AllowNotReadyContainers
+
+- Meaning: the container names that are allowed not ready when inplace updating, when traffic will not be cut.
+- Value: {containerName_0},{containerName_1},... Example：sidecar
+- Configuration change supported or not: It cannot be changed during the in-place updating process.
+
 #### Plugin configuration
 
 None
@@ -616,4 +628,107 @@ status:
   status: InUse
 ```
 
-In addition, the generated EIP resource will be named after {pod namespace}/{pod name} in the Alibaba Cloud console, which corresponds to each game server one by one.
+In addition, the generated EIP resource will be named after {pod namespace}/{pod name} in the Alibaba Cloud console, which corresponds to each game server one by one.
+
+---
+
+### AlibabaCloud-NLB-SharedPort
+
+#### Plugin name
+
+`AlibabaCloud-NLB-SharedPort`
+
+#### Cloud Provider
+
+AlibabaCloud
+
+#### Plugin description
+
+- AlibabaCloud-NLB-SharedPort enables game servers to be accessed from the Internet by using Layer 4 NLB of Alibaba Cloud, which is similar to AlibabaCloud-SLB-SharedPort.
+  This network plugin applies to stateless network services, such as proxy or gateway, in gaming scenarios.
+
+- This network plugin supports network isolation.
+
+#### Network parameters
+
+SlbIds
+
+- Meaning: the CLB instance IDs. You can specify multiple NLB instance IDs.
+- Value: an example value can be nlb-9zeo7prq1m25ctpfrw1m7
+- Configuration change supported or not: no.
+
+PortProtocols
+
+- Meaning: the ports in the pod to be exposed and the protocols. You can specify multiple ports and protocols.
+- Value: in the format of port1/protocol1,port2/protocol2,... The protocol names must be in uppercase letters.
+- Configuration change supported or not: no.
+
+AllowNotReadyContainers
+
+- Meaning: the container names that are allowed not ready when inplace updating, when traffic will not be cut.
+- Value: {containerName_0},{containerName_1},... Example：sidecar
+- Configuration change supported or not: It cannot be changed during the in-place updating process.
+
+#### Plugin configuration
+
+None
+
+#### Example
+
+Deploy a GameServerSet with two containers, one named app-2048 and the other named sidecar.
+
+Specify the network parameter AllowNotReadyContainers as sidecar, 
+then the entire pod will still provide services when the sidecar is updated in place.
+
+```yaml
+apiVersion: game.kruise.io/v1alpha1
+kind: GameServerSet
+metadata:
+  name: gss-2048-nlb
+  namespace: default
+spec:
+  replicas: 3
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 100%
+      podUpdatePolicy: InPlaceIfPossible
+  network:
+    networkType: AlibabaCloud-NLB-SharedPort
+    networkConf:
+      - name: NlbIds
+        value: nlb-26jbknebrjlejt5abu
+      - name: PortProtocols
+        value: 80/TCP
+      - name: AllowNotReadyContainers
+        value: sidecar
+  gameServerTemplate:
+    spec:
+      containers:
+        - image: registry.cn-beijing.aliyuncs.com/acs/2048:v1.0
+          name: app-2048
+          volumeMounts:
+            - name: shared-dir
+              mountPath: /var/www/html/js
+        - image: registry.cn-beijing.aliyuncs.com/acs/2048-sidecar:v1.0
+          name: sidecar
+          args:
+            - bash
+            - -c
+            - rsync -aP /app/js/* /app/scripts/ && while true; do echo 11;sleep 2; done
+          volumeMounts:
+            - name: shared-dir
+              mountPath: /app/scripts
+      volumes:
+        - name: shared-dir
+          emptyDir: {}
+```
+
+After successful deployment, update the sidecar image to v2.0 and observe the corresponding endpoint:
+
+```bash
+kubectl get ep -w | grep nlb-26jbknebrjlejt5abu
+nlb-26jbknebrjlejt5abu      192.168.0.8:80,192.168.0.82:80,192.168.63.228:80    10m
+
+```
+
+After waiting for the entire update process to end, you can find that there are no changes in the ep, indicating that no extraction has been performed.

docs/中文/用户手册/网络模型.md

@@ -427,6 +427,12 @@ Fixed
 - 填写格式：false / true
 - 是否支持变更：支持
 
+AllowNotReadyContainers
+
+- 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个
+- 格式：{containerName_0},{containerName_1},... 例如：sidecar
+- 是否支持变更：在原地升级过程中不可变更。
+
 #### 插件配置
 ```
 [alibabacloud]
@@ -470,6 +476,12 @@ PortProtocols
 - 格式：port1/protocol1,port2/protocol2,...（协议需大写）
 - 是否支持变更：暂不支持。未来将支持
 
+AllowNotReadyContainers
+
+- 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个
+- 格式：{containerName_0},{containerName_1},... 例如：sidecar
+- 是否支持变更：在原地升级过程中不可变更。
+
 #### 插件配置
 
 无
@@ -616,6 +628,104 @@ status:
 
 此外，生成的EIP资源在阿里云控制台中会以{pod namespace}/{pod name}命名，与每一个游戏服一一对应。
 
+### AlibabaCloud-NLB-SharedPort
+
+#### 插件名称
+
+`AlibabaCloud-NLB-SharedPort`
+
+#### Cloud Provider
+
+AlibabaCloud
+
+#### 插件说明
+
+- AlibabaCloud-NLB-SharedPort 使用阿里云网络型负载均衡（NLB）作为对外服务的承载实体。其与AlibabaCloud-SLB-SharedPort作用类似。
+  适用于游戏场景下代理（proxy）或网关等无状态网络服务。
+
+- 是否支持网络隔离：是
+
+#### 网络参数
+
+SlbIds
+
+- 含义：填写nlb的id，暂不支持填写多例
+- 填写格式：例如：nlb-9zeo7prq1m25ctpfrw1m7
+- 是否支持变更：暂不支持。
+
+PortProtocols
+
+- 含义：pod暴露的端口及协议，支持填写多个端口/协议
+- 格式：port1/protocol1,port2/protocol2,...（协议需大写）
+- 是否支持变更：暂不支持。
+
+AllowNotReadyContainers
+
+- 含义：在容器原地升级时允许不断流的对应容器名称，可填写多个
+- 格式：{containerName_0},{containerName_1},... 例如：sidecar
+- 是否支持变更：在原地升级过程中不可变更。
+
+#### 插件配置
+
+无
+
+#### 示例说明
+
+部署一个具有两个容器的GameServerSet，一个容器名为app-2048，另一个为sidecar。
+指定网络参数 AllowNotReadyContainers 为 sidecar，则在sidecar原地更新时整个pod依然会提供服务，不会断流。
+
+```yaml
+apiVersion: game.kruise.io/v1alpha1
+kind: GameServerSet
+metadata:
+  name: gss-2048-nlb
+  namespace: default
+spec:
+  replicas: 3
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: 100%
+      podUpdatePolicy: InPlaceIfPossible
+  network:
+    networkType: AlibabaCloud-NLB-SharedPort
+    networkConf:
+      - name: NlbIds
+        value: nlb-26jbknebrjlejt5abu
+      - name: PortProtocols
+        value: 80/TCP
+      - name: AllowNotReadyContainers
+        value: sidecar
+  gameServerTemplate:
+    spec:
+      containers:
+        - image: registry.cn-beijing.aliyuncs.com/acs/2048:v1.0
+          name: app-2048
+          volumeMounts:
+            - name: shared-dir
+              mountPath: /var/www/html/js
+        - image: registry.cn-beijing.aliyuncs.com/acs/2048-sidecar:v1.0
+          name: sidecar
+          args:
+            - bash
+            - -c
+            - rsync -aP /app/js/* /app/scripts/ && while true; do echo 11;sleep 2; done
+          volumeMounts:
+            - name: shared-dir
+              mountPath: /app/scripts
+      volumes:
+        - name: shared-dir
+          emptyDir: {}
+```
+
+部署成功后，将sidecar镜像更新到v2.0版本，同时观察对应endpoint情况:
+```bash
+kubectl get ep -w | grep nlb-26jbknebrjlejt5abu
+nlb-26jbknebrjlejt5abu      192.168.0.8:80,192.168.0.82:80,192.168.63.228:80    10m
+
+```
+
+等待整个更新过程结束，可以发现ep没有任何变化，说明并未进行摘流。
+
 ## 获取网络信息
 
 GameServer Network Status可以通过两种方式获取

pkg/util/gameserver.go

@@ -132,6 +132,16 @@ func GetNewAstsFromGss(gss *gameKruiseV1alpha1.GameServerSet, asts *kruiseV1beta
 	readinessGates = append(readinessGates, corev1.PodReadinessGate{ConditionType: appspub.InPlaceUpdateReady})
 	asts.Spec.Template.Spec.ReadinessGates = readinessGates
 
+	// AllowNotReadyContainers
+	if gss.Spec.Network != nil && IsAllowNotReadyContainers(gss.Spec.Network.NetworkConf) {
+		// set lifecycle
+		asts.Spec.Lifecycle = &appspub.Lifecycle{
+			InPlaceUpdate: &appspub.LifecycleHook{
+				LabelsHandler: map[string]string{gameKruiseV1alpha1.InplaceUpdateNotReadyBlocker: "true"},
+			},
+		}
+	}
+
 	// set VolumeClaimTemplates
 	asts.Spec.VolumeClaimTemplates = gss.Spec.GameServerTemplate.VolumeClaimTemplates
 
@@ -210,3 +220,12 @@ func GetGameServerSetOfPod(pod *corev1.Pod, c client.Client, ctx context.Context
 	}, gss)
 	return gss, err
 }
+
+func IsAllowNotReadyContainers(networkConfParams []gameKruiseV1alpha1.NetworkConfParams) bool {
+	for _, networkConfParam := range networkConfParams {
+		if networkConfParam.Name == gameKruiseV1alpha1.AllowNotReadyContainersNetworkConfName {
+			return true
+		}
+	}
+	return false
+}

pkg/util/gameserver_test.go

@@ -325,3 +325,37 @@ func TestGetGsTemplateMetadataHash(t *testing.T) {
 		}
 	}
 }
+
+func TestIsAllowNotReadyContainers(t *testing.T) {
+	tests := []struct {
+		networkConfParams         []gameKruiseV1alpha1.NetworkConfParams
+		isAllowNotReadyContainers bool
+	}{
+		{
+			networkConfParams: []gameKruiseV1alpha1.NetworkConfParams{
+				{
+					Name:  gameKruiseV1alpha1.AllowNotReadyContainersNetworkConfName,
+					Value: "xxx",
+				},
+			},
+			isAllowNotReadyContainers: true,
+		},
+		{
+			networkConfParams: []gameKruiseV1alpha1.NetworkConfParams{
+				{
+					Name:  "xxx",
+					Value: "xxx",
+				},
+			},
+			isAllowNotReadyContainers: false,
+		},
+	}
+
+	for i, test := range tests {
+		actual := IsAllowNotReadyContainers(test.networkConfParams)
+		expect := test.isAllowNotReadyContainers
+		if actual != expect {
+			t.Errorf("case %d: expect isAllowNotReadyContainers is %v but actually got %v", i, expect, actual)
+		}
+	}
+}

pkg/util/pod.go

@@ -16,7 +16,10 @@ limitations under the License.
 
 package util
 
-import corev1 "k8s.io/api/core/v1"
+import (
+	gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
+	corev1 "k8s.io/api/core/v1"
+)
 
 // GetPodConditionFromList extracts the provided condition from the given list of condition and
 // returns the index of the condition and the condition. Returns -1 and nil if the condition is not present.
@@ -31,3 +34,20 @@ func GetPodConditionFromList(conditions []corev1.PodCondition, conditionType cor
 	}
 	return -1, nil
 }
+
+func IsContainersPreInplaceUpdating(pod *corev1.Pod, gss *gameKruiseV1alpha1.GameServerSet, containerNames []string) bool {
+	var diffNames []string
+	for _, actual := range pod.Status.ContainerStatuses {
+		for _, expect := range gss.Spec.GameServerTemplate.Spec.Containers {
+			if actual.Name == expect.Name && actual.Image != expect.Image {
+				diffNames = append(diffNames, actual.Name)
+			}
+		}
+	}
+	for _, containerName := range containerNames {
+		if IsStringInList(containerName, diffNames) {
+			return true
+		}
+	}
+	return false
+}

pkg/util/pod_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package util
 
 import (
+	gameKruiseV1alpha1 "github.com/openkruise/kruise-game/apis/v1alpha1"
 	corev1 "k8s.io/api/core/v1"
 	"reflect"
 	"testing"
@@ -59,3 +60,135 @@ func TestGetPodConditionFromList(t *testing.T) {
 		}
 	}
 }
+
+func TestIsContainersPreInplaceUpdating(t *testing.T) {
+	tests := []struct {
+		pod            *corev1.Pod
+		gss            *gameKruiseV1alpha1.GameServerSet
+		containerNames []string
+		isUpdating     bool
+	}{
+		{
+			pod: &corev1.Pod{
+				Status: corev1.PodStatus{
+					ContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name:  "name_A",
+							Image: "v1.0",
+						},
+						{
+							Name:  "name_B",
+							Image: "v1.0",
+						},
+					},
+				},
+			},
+			gss: &gameKruiseV1alpha1.GameServerSet{
+				Spec: gameKruiseV1alpha1.GameServerSetSpec{
+					GameServerTemplate: gameKruiseV1alpha1.GameServerTemplate{
+						PodTemplateSpec: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								Containers: []corev1.Container{
+									{
+										Name:  "name_A",
+										Image: "v1.0",
+									},
+									{
+										Name:  "name_B",
+										Image: "v2.0",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			containerNames: []string{"name_B"},
+			isUpdating:     true,
+		},
+		{
+			pod: &corev1.Pod{
+				Status: corev1.PodStatus{
+					ContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name:  "name_A",
+							Image: "v1.0",
+						},
+						{
+							Name:  "name_B",
+							Image: "v1.0",
+						},
+					},
+				},
+			},
+			gss: &gameKruiseV1alpha1.GameServerSet{
+				Spec: gameKruiseV1alpha1.GameServerSetSpec{
+					GameServerTemplate: gameKruiseV1alpha1.GameServerTemplate{
+						PodTemplateSpec: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								Containers: []corev1.Container{
+									{
+										Name:  "name_A",
+										Image: "v1.0",
+									},
+									{
+										Name:  "name_B",
+										Image: "v2.0",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			containerNames: []string{"name_A"},
+			isUpdating:     false,
+		},
+		{
+			pod: &corev1.Pod{
+				Status: corev1.PodStatus{
+					ContainerStatuses: []corev1.ContainerStatus{
+						{
+							Name:  "name_A",
+							Image: "v1.0",
+						},
+						{
+							Name:  "name_B",
+							Image: "v1.0",
+						},
+					},
+				},
+			},
+			gss: &gameKruiseV1alpha1.GameServerSet{
+				Spec: gameKruiseV1alpha1.GameServerSetSpec{
+					GameServerTemplate: gameKruiseV1alpha1.GameServerTemplate{
+						PodTemplateSpec: corev1.PodTemplateSpec{
+							Spec: corev1.PodSpec{
+								Containers: []corev1.Container{
+									{
+										Name:  "name_A",
+										Image: "v1.0",
+									},
+									{
+										Name:  "name_B",
+										Image: "v1.0",
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+			containerNames: []string{"name_B"},
+			isUpdating:     false,
+		},
+	}
+
+	for i, test := range tests {
+		actual := IsContainersPreInplaceUpdating(test.pod, test.gss, test.containerNames)
+		expect := test.isUpdating
+		if actual != expect {
+			t.Errorf("case %d: expect IsContainersPreInplaceUpdating is %v but actually got %v", i, expect, actual)
+		}
+	}
+}