kruise/pkg/controller/sidecarset/sidecarset_processor.go

/*
Copyright 2020 The Kruise Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package sidecarset

import (
	"context"
	"encoding/json"
	"fmt"
	"strings"
	"time"

	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"

	podutil "k8s.io/kubernetes/pkg/api/v1/pod"

	appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
	"github.com/openkruise/kruise/pkg/control/sidecarcontrol"
	controlutil "github.com/openkruise/kruise/pkg/controller/util"
	"github.com/openkruise/kruise/pkg/util"
	utilclient "github.com/openkruise/kruise/pkg/util/client"
	"github.com/openkruise/kruise/pkg/util/fieldindex"
	historyutil "github.com/openkruise/kruise/pkg/util/history"
	webhookutil "github.com/openkruise/kruise/pkg/webhook/util"

	apps "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	"k8s.io/apimachinery/pkg/labels"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/apimachinery/pkg/util/sets"
	"k8s.io/client-go/tools/record"
	"k8s.io/client-go/util/retry"
	"k8s.io/klog/v2"
	"k8s.io/kubernetes/pkg/controller/history"
	"k8s.io/utils/integer"
	"k8s.io/utils/pointer"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/reconcile"
)

type Processor struct {
	Client            client.Client
	recorder          record.EventRecorder
	historyController history.Interface
}

func NewSidecarSetProcessor(cli client.Client, rec record.EventRecorder) *Processor {
	return &Processor{
		Client:            cli,
		recorder:          rec,
		historyController: historyutil.NewHistory(cli),
	}
}

func (p *Processor) UpdateSidecarSet(sidecarSet *appsv1alpha1.SidecarSet) (reconcile.Result, error) {
	control := sidecarcontrol.New(sidecarSet)
	// check whether sidecarSet is active
	if !control.IsActiveSidecarSet() {
		return reconcile.Result{}, nil
	}
	// 1. get matching pods with the sidecarSet
	pods, err := p.getMatchingPods(sidecarSet)
	if err != nil {
		klog.ErrorS(err, "SidecarSet get matching pods error", "sidecarSet", klog.KObj(sidecarSet))
		return reconcile.Result{}, err
	}

	// register new revision if this sidecarSet is the latest;
	// return the latest revision that corresponds to this sidecarSet.
	latestRevision, collisionCount, err := p.registerLatestRevision(sidecarSet, pods)
	if latestRevision == nil {
		klog.ErrorS(err, "SidecarSet register the latest revision error", "sidecarSet", klog.KObj(sidecarSet))
		return reconcile.Result{}, err
	}

	// 2. calculate SidecarSet status based on pod and revision information
	status := calculateStatus(control, pods, latestRevision, collisionCount)
	//update sidecarSet status in store
	if err := p.updateSidecarSetStatus(sidecarSet, status); err != nil {
		return reconcile.Result{}, err
	}
	sidecarSet.Status = *status

	// in case of informer cache latency
	for _, pod := range pods {
		sidecarcontrol.UpdateExpectations.ObserveUpdated(sidecarSet.Name, sidecarcontrol.GetSidecarSetRevision(sidecarSet), pod)
	}
	allUpdated, _, inflightPods := sidecarcontrol.UpdateExpectations.SatisfiedExpectations(sidecarSet.Name, sidecarcontrol.GetSidecarSetRevision(sidecarSet))
	if !allUpdated {
		klog.V(3).InfoS("Sidecarset matched pods has some update in flight, will sync later", "sidecarSet", klog.KObj(sidecarSet), "pods", inflightPods)
		return reconcile.Result{RequeueAfter: time.Second}, nil
	}

	// 3. If sidecar container hot upgrade complete, then set the other one(empty sidecar container) image to HotUpgradeEmptyImage
	if isSidecarSetHasHotUpgradeContainer(sidecarSet) {
		var podsInHotUpgrading []*corev1.Pod
		for _, pod := range pods {
			// flip other hot sidecar container to empty, in the following:
			// 1. the empty sidecar container image isn't equal HotUpgradeEmptyImage
			// 2. all containers with exception of empty sidecar containers is updated and consistent
			// 3. all containers with exception of empty sidecar containers is ready

			// don't contain sidecar empty containers
			sidecarContainers := sidecarcontrol.GetSidecarContainersInPod(sidecarSet)
			for _, sidecarContainer := range sidecarSet.Spec.Containers {
				if sidecarcontrol.IsHotUpgradeContainer(&sidecarContainer) {
					_, emptyContainer := sidecarcontrol.GetPodHotUpgradeContainers(sidecarContainer.Name, pod)
					sidecarContainers.Delete(emptyContainer)
				}
			}
			if isPodSidecarInHotUpgrading(sidecarSet, pod) && control.IsPodStateConsistent(pod, sidecarContainers) &&
				isHotUpgradingReady(sidecarSet, pod) {
				podsInHotUpgrading = append(podsInHotUpgrading, pod)
			}
		}
		if err := p.flipHotUpgradingContainers(control, podsInHotUpgrading); err != nil {
			return reconcile.Result{}, err
		}
	}

	// 4. SidecarSet upgrade strategy type is NotUpdate
	if isSidecarSetNotUpdate(sidecarSet) {
		return reconcile.Result{}, nil
	}

	// 5. sidecarset already updates all matched pods, then return
	if isSidecarSetUpdateFinish(status) {
		klog.V(3).InfoS("SidecarSet matched pods were latest, and don't need update", "sidecarSet", klog.KObj(sidecarSet), "matchedPodCount", len(pods))
		return reconcile.Result{}, nil
	}

	// 6. Paused indicates that the SidecarSet is paused to update matched pods
	if sidecarSet.Spec.UpdateStrategy.Paused {
		klog.V(3).InfoS("SidecarSet was paused", "sidecarSet", klog.KObj(sidecarSet))
		return reconcile.Result{}, nil
	}

	// 7. upgrade pod sidecar
	if err := p.updatePods(control, pods); err != nil {
		return reconcile.Result{}, err
	}
	return reconcile.Result{}, nil
}

func (p *Processor) updatePods(control sidecarcontrol.SidecarControl, pods []*corev1.Pod) error {
	sidecarset := control.GetSidecarset()
	// compute next updated pods based on the sidecarset upgrade strategy
	upgradePods, notUpgradablePods := NewStrategy().GetNextUpgradePods(control, pods)
	for _, pod := range notUpgradablePods {
		if err := p.updatePodSidecarSetUpgradableCondition(sidecarset, pod, false); err != nil {
			klog.ErrorS(err, "Failed to update NotUpgradable PodCondition", "sidecarSet", klog.KObj(sidecarset), "pod", klog.KObj(pod))
			return err
		}
		// Since the pod sidecarSet hash is not updated here, it cannot be called ExpectUpdated
		// TODO: add ResourceVersionExpectation instead of UpdateExpectations
	}
	if len(notUpgradablePods) > 0 {
		p.recorder.Eventf(sidecarset, corev1.EventTypeNormal, "NotUpgradablePods", "SidecarSet in-place update detected %d not upgradable pod(s) in this round, will skip them.", len(notUpgradablePods))
	}

	if len(upgradePods) == 0 {
		klog.V(3).InfoS("SidecarSet next update was nil, skip this round", "sidecarSet", klog.KObj(sidecarset))
		return nil
	}
	// mark upgrade pods list
	podNames := make([]string, 0, len(upgradePods))
	// upgrade pod sidecar
	for _, pod := range upgradePods {
		podNames = append(podNames, pod.Name)
		if err := p.updatePodSidecarAndHash(control, pod); err != nil {
			klog.ErrorS(err, "UpdatePodSidecarAndHash error", "sidecarSet", klog.KObj(sidecarset), "pod", klog.KObj(pod))
			return err
		}
		sidecarcontrol.UpdateExpectations.ExpectUpdated(sidecarset.Name, sidecarcontrol.GetSidecarSetRevision(sidecarset), pod)
	}

	klog.V(3).InfoS("SidecarSet updated pods", "sidecarSet", klog.KObj(sidecarset), "podNames", strings.Join(podNames, ","))
	return nil
}

func (p *Processor) updatePodSidecarAndHash(control sidecarcontrol.SidecarControl, pod *corev1.Pod) error {
	podClone := &corev1.Pod{}
	sidecarSet := control.GetSidecarset()
	err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
		if err := p.Client.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, podClone); err != nil {
			klog.ErrorS(err, "SidecarSet got updated pod from client failed", "sidecarSet", klog.KObj(sidecarSet), "pod", klog.KObj(pod))
		}
		// update pod sidecar container
		updatePodSidecarContainer(control, podClone)
		// older pod don't have SidecarSetListAnnotation
		// which is to improve the performance of the sidecarSet controller
		sidecarSetNames, ok := podClone.Annotations[sidecarcontrol.SidecarSetListAnnotation]
		if !ok || len(sidecarSetNames) == 0 {
			podClone.Annotations[sidecarcontrol.SidecarSetListAnnotation] = p.listMatchedSidecarSets(podClone)
		}
		// patch pod metadata
		_, err := sidecarcontrol.PatchPodMetadata(&podClone.ObjectMeta, sidecarSet.Spec.PatchPodMetadata)
		if err != nil {
			klog.ErrorS(err, "SidecarSet patched pod metadata failed", "sidecarSet", klog.KObj(sidecarSet), "pod", klog.KObj(podClone))
			return err
		}
		// update pod in store
		return p.Client.Update(context.TODO(), podClone)
	})

	if err != nil {
		return err
	}

	// update pod condition of sidecar upgradable
	return p.updatePodSidecarSetUpgradableCondition(sidecarSet, pod, true)
}

func (p *Processor) listMatchedSidecarSets(pod *corev1.Pod) string {
	sidecarSetList := &appsv1alpha1.SidecarSetList{}
	sidecarSetList2 := &appsv1alpha1.SidecarSetList{}
	podNamespace := pod.Namespace
	if podNamespace == "" {
		podNamespace = "default"
	}
	if err := p.Client.List(context.TODO(), sidecarSetList, client.MatchingFields{fieldindex.IndexNameForSidecarSetNamespace: podNamespace}, utilclient.DisableDeepCopy); err != nil {
		klog.ErrorS(err, "Listed SidecarSets failed")
		return ""
	}
	if err := p.Client.List(context.TODO(), sidecarSetList2, client.MatchingFields{fieldindex.IndexNameForSidecarSetNamespace: fieldindex.IndexValueSidecarSetClusterScope}, utilclient.DisableDeepCopy); err != nil {
		klog.ErrorS(err, "Listed SidecarSets failed")
		return ""
	}

	//matched SidecarSet.Name list
	sidecarSetNames := make([]string, 0)
	for _, sidecarSet := range append(sidecarSetList.Items, sidecarSetList2.Items...) {
		if matched, _ := sidecarcontrol.PodMatchedSidecarSet(p.Client, pod, &sidecarSet); matched {
			sidecarSetNames = append(sidecarSetNames, sidecarSet.Name)
		}
	}

	return strings.Join(sidecarSetNames, ",")
}

func (p *Processor) updateSidecarSetStatus(sidecarSet *appsv1alpha1.SidecarSet, status *appsv1alpha1.SidecarSetStatus) error {
	if !inconsistentStatus(sidecarSet, status) {
		return nil
	}

	sidecarSetClone := sidecarSet.DeepCopy()
	if err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
		sidecarSetClone.Status = *status
		sidecarSetClone.Status.ObservedGeneration = sidecarSetClone.Generation

		updateErr := p.Client.Status().Update(context.TODO(), sidecarSetClone)
		if updateErr == nil {
			return nil
		}

		key := types.NamespacedName{
			Name: sidecarSetClone.Name,
		}
		if err := p.Client.Get(context.TODO(), key, sidecarSetClone); err != nil {
			klog.ErrorS(err, "Failed to get updated SidecarSet from client", "sidecarSet", klog.KObj(sidecarSetClone))
		}
		return updateErr
	}); err != nil {
		return err
	}

	klog.V(3).InfoS("SidecarSet updated status success", "sidecarSet", klog.KObj(sidecarSet), "matchedPods", status.MatchedPods,
		"updatedPods", status.UpdatedPods, "readyPods", status.ReadyPods, "updateReadyPods", status.UpdatedReadyPods)
	return nil
}

// If you need update the pod object, you must DeepCopy it
func (p *Processor) getMatchingPods(s *appsv1alpha1.SidecarSet) ([]*corev1.Pod, error) {
	// get more faster selector
	selector, err := util.ValidatedLabelSelectorAsSelector(s.Spec.Selector)
	if err != nil {
		return nil, err
	}
	scopedNamespaces := sets.NewString()
	if s.Spec.Namespace != "" || s.Spec.NamespaceSelector != nil {
		if scopedNamespaces, err = sidecarcontrol.FetchSidecarSetMatchedNamespace(p.Client, s); err != nil {
			return nil, err
		}
		// If sidecarSet.Spec.Namespace is empty, then select in cluster
	} else {
		// when namespace="", client will list pods in all namespaces
		scopedNamespaces.Insert("")
	}
	selectedPods, err := p.getSelectedPods(scopedNamespaces, selector)
	if err != nil {
		return nil, err
	}

	// filter out pods that don't require updated, include the following:
	// 1. inActive pod
	// 2. never be injected sidecar container
	var filteredPods []*corev1.Pod
	for _, pod := range selectedPods {
		if sidecarcontrol.IsActivePod(pod) && sidecarcontrol.IsPodInjectedSidecarSet(pod, s) &&
			sidecarcontrol.IsPodConsistentWithSidecarSet(pod, s) {
			filteredPods = append(filteredPods, pod)
		}
	}
	return filteredPods, nil
}

// get selected pods(DisableDeepCopy:true, indicates must be deep copy before update pod objection)
func (p *Processor) getSelectedPods(namespaces sets.String, selector labels.Selector) (relatedPods []*corev1.Pod, err error) {
	// DisableDeepCopy:true, indicates must be deep copy before update pod objection
	listOpts := &client.ListOptions{LabelSelector: selector}
	for _, ns := range namespaces.List() {
		allPods := &corev1.PodList{}
		listOpts.Namespace = ns
		if listErr := p.Client.List(context.TODO(), allPods, listOpts, utilclient.DisableDeepCopy); listErr != nil {
			err = fmt.Errorf("sidecarSet list pods by ns error, ns[%s], err:%v", ns, listErr)
			return
		}
		for i := range allPods.Items {
			relatedPods = append(relatedPods, &allPods.Items[i])
		}
	}
	return
}

func (p *Processor) registerLatestRevision(set *appsv1alpha1.SidecarSet, pods []*corev1.Pod) (
	latestRevision *apps.ControllerRevision, collisionCount int32, err error,
) {
	sidecarSet := set.DeepCopy()
	// get revision selector of this sidecarSet
	hc := sidecarcontrol.NewHistoryControl(p.Client)
	// list all revisions
	revisions, err := p.historyController.ListControllerRevisions(sidecarcontrol.MockSidecarSetForRevision(set), hc.GetRevisionSelector(sidecarSet))
	if err != nil {
		klog.ErrorS(err, "Failed to list history controllerRevisions", "sidecarSet", klog.KObj(sidecarSet))
		return nil, collisionCount, err
	}

	// sort revisions by increasing .Revision
	history.SortControllerRevisions(revisions)
	revisionCount := len(revisions)

	if sidecarSet.Status.CollisionCount != nil {
		collisionCount = *sidecarSet.Status.CollisionCount
	}

	// build a new revision from the current sidecarSet,
	// the namespace of sidecarset revision must have very strict permissions for average users.
	// Here use the namespace of kruise-manager.
	latestRevision, err = hc.NewRevision(sidecarSet, webhookutil.GetNamespace(), hc.NextRevision(revisions), &collisionCount)
	if err != nil {
		return nil, collisionCount, err
	}

	// find any equivalent revisions
	equalRevisions := history.FindEqualRevisions(revisions, latestRevision)
	equalCount := len(equalRevisions)

	if equalCount > 0 && history.EqualRevision(revisions[revisionCount-1], equalRevisions[equalCount-1]) {
		// if the equivalent revision is immediately prior the update revision has not changed
		// in case of no change
		latestRevision = revisions[revisionCount-1]
	} else if equalCount > 0 {
		// if the equivalent revision is not immediately prior we will roll back by incrementing the
		// Revision of the equivalent revision
		// in case of roll back
		latestRevision, err = p.historyController.UpdateControllerRevision(equalRevisions[equalCount-1], latestRevision.Revision)
		if err != nil {
			return nil, collisionCount, err
		}
		// the updated revision may be deleted if we don't replace this revision.
		replaceRevision(revisions, equalRevisions[equalCount-1], latestRevision)
	} else {
		// if there is no equivalent revision we create a new one
		// in case of the sidecarSet update
		latestRevision, err = hc.CreateControllerRevision(sidecarSet, latestRevision, &collisionCount)
		if err != nil {
			return nil, collisionCount, err
		}
		revisions = append(revisions, latestRevision)
	}

	// update custom revision for the latest controller revision
	if err = p.updateCustomVersionLabel(latestRevision, sidecarSet.Labels[appsv1alpha1.SidecarSetCustomVersionLabel]); err != nil {
		return nil, collisionCount, err
	}

	// only store limited history revisions
	if err = p.truncateHistory(revisions, sidecarSet, pods); err != nil {
		klog.ErrorS(err, "Failed to truncate history for SidecarSet", "sidecarSet", klog.KObj(sidecarSet))
	}

	return latestRevision, collisionCount, nil
}

func (p *Processor) updateCustomVersionLabel(revision *apps.ControllerRevision, customVersion string) error {
	if customVersion != "" && customVersion != revision.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] {
		newRevision := &apps.ControllerRevision{
			ObjectMeta: metav1.ObjectMeta{
				Name:      revision.Name,
				Namespace: revision.Namespace,
			},
		}
		patchBody := fmt.Sprintf(`{"metadata":{"labels":{"%v":"%v"}}}`, appsv1alpha1.SidecarSetCustomVersionLabel, customVersion)
		err := p.Client.Patch(context.TODO(), newRevision, client.RawPatch(types.StrategicMergePatchType, []byte(patchBody)))
		if err != nil {
			klog.ErrorS(err, `Failed to patch custom revision label to latest revision`, "revision", klog.KObj(revision), "customVersion", customVersion)
			return err
		}
	}
	return nil
}

func (p *Processor) truncateHistory(revisions []*apps.ControllerRevision, s *appsv1alpha1.SidecarSet, pods []*corev1.Pod) error {
	// We do not delete the latest revision because we are using it.
	// Thus, we must ensure the limitation is bounded, minimum value is 1.
	limitation := 10
	if s.Spec.RevisionHistoryLimit != nil {
		limitation = integer.IntMax(1, int(*s.Spec.RevisionHistoryLimit))
	}
	// if no need to truncate
	revisionCount := len(revisions)
	if revisionCount <= limitation {
		return nil
	}

	klog.V(3).InfoS("Found revisions more than limitation", "revisionCount", revisionCount, "limitation", limitation, "sidecarSet", klog.KObj(s))

	// the number of revisions need to delete
	deletionCount := revisionCount - limitation
	// only delete the revisions that no pods use.
	activeRevisions := filterActiveRevisions(s, pods, revisions)
	for i := 0; i < revisionCount-1 && deletionCount > 0; i++ {
		if !activeRevisions.Has(revisions[i].Name) {
			if err := p.historyController.DeleteControllerRevision(revisions[i]); err != nil && !errors.IsNotFound(err) {
				return err
			}
			deletionCount--
		}
	}

	// Sometime we cannot ensure the number of stored revisions is within the limitation because of the use by pods.
	if deletionCount > 0 {
		return fmt.Errorf("failed to limit the number of stored revisions, limited: %d, actual: %d, name: %s", limitation, limitation+deletionCount, s.Name)
	}
	return nil
}

func filterActiveRevisions(s *appsv1alpha1.SidecarSet, pods []*corev1.Pod, revisions []*apps.ControllerRevision) sets.String {
	activeRevisions := sets.NewString()
	for _, pod := range pods {
		if revision := sidecarcontrol.GetPodSidecarSetControllerRevision(s.Name, pod); revision != "" {
			activeRevisions.Insert(revision)
		}
	}

	if s.Spec.InjectionStrategy.Revision != nil {
		equalRevisions := make([]*apps.ControllerRevision, 0)
		if s.Spec.InjectionStrategy.Revision.RevisionName != nil {
			activeRevisions.Insert(*s.Spec.InjectionStrategy.Revision.RevisionName)
		}

		if s.Spec.InjectionStrategy.Revision.CustomVersion != nil {
			for i := range revisions {
				revision := revisions[i]
				if revision.Labels[appsv1alpha1.SidecarSetCustomVersionLabel] == *s.Spec.InjectionStrategy.Revision.CustomVersion {
					equalRevisions = append(equalRevisions, revision)
				}
			}
			if len(equalRevisions) > 0 {
				history.SortControllerRevisions(equalRevisions)
				activeRevisions.Insert(equalRevisions[len(equalRevisions)-1].Name)
			}
		}
	}

	return activeRevisions
}

// replaceRevision will remove old from revisions, and add new to the end of revisions.
// This function keeps the order of revisions.
func replaceRevision(revisions []*apps.ControllerRevision, oldOne, newOne *apps.ControllerRevision) {
	revisionCount := len(revisions)
	if revisionCount == 0 || oldOne == nil {
		return
	}
	// remove old revision from revisions
	found := revisions[0] == oldOne
	for i := 0; i < revisionCount-1; i++ {
		if found {
			revisions[i] = revisions[i+1]
		} else if revisions[i+1] == oldOne {
			found = true
		}
	}
	// add this new revision to the end of revisions
	revisions[revisionCount-1] = newOne
}

// calculate SidecarSet status
// MatchedPods: all matched pods number
// UpdatedPods: updated pods number
// ReadyPods: ready pods number
// UpdatedReadyPods: updated and ready pods number
// UnavailablePods: MatchedPods - UpdatedReadyPods
func calculateStatus(control sidecarcontrol.SidecarControl, pods []*corev1.Pod, latestRevision *apps.ControllerRevision, collisionCount int32,
) *appsv1alpha1.SidecarSetStatus {
	sidecarset := control.GetSidecarset()
	var matchedPods, updatedPods, readyPods, updatedAndReady int32
	matchedPods = int32(len(pods))
	for _, pod := range pods {
		updated := sidecarcontrol.IsPodSidecarUpdated(sidecarset, pod)
		if updated {
			updatedPods++
		}
		if control.IsPodStateConsistent(pod, nil) && control.IsPodReady(pod) {
			readyPods++
			if updated {
				updatedAndReady++
			}
		}
	}
	return &appsv1alpha1.SidecarSetStatus{
		ObservedGeneration: sidecarset.Generation,
		MatchedPods:        matchedPods,
		UpdatedPods:        updatedPods,
		ReadyPods:          readyPods,
		UpdatedReadyPods:   updatedAndReady,
		LatestRevision:     latestRevision.Name,
		CollisionCount:     pointer.Int32Ptr(collisionCount),
	}
}

func isSidecarSetNotUpdate(s *appsv1alpha1.SidecarSet) bool {
	if s.Spec.UpdateStrategy.Type == appsv1alpha1.NotUpdateSidecarSetStrategyType {
		klog.V(3).InfoS("SidecarSet spreading RollingUpdate config type", "sidecarSet", klog.KObj(s), "type", s.Spec.UpdateStrategy.Type)
		return true
	}
	return false
}

func updateContainerInPod(container corev1.Container, pod *corev1.Pod) {
	for i := range pod.Spec.Containers {
		if pod.Spec.Containers[i].Name == container.Name {
			pod.Spec.Containers[i] = container
			return
		}
	}
}

func updatePodSidecarContainer(control sidecarcontrol.SidecarControl, pod *corev1.Pod) {
	sidecarSet := control.GetSidecarset()

	// upgrade sidecar containers
	var changedContainers []string
	for _, sidecarContainer := range sidecarSet.Spec.Containers {
		//sidecarContainer := &sidecarset.Spec.Containers[i]
		// volumeMounts that injected into sidecar container
		// when volumeMounts SubPathExpr contains expansions, then need copy container EnvVars(injectEnvs)
		injectedMounts, injectedEnvs := sidecarcontrol.GetInjectedVolumeMountsAndEnvs(control, &sidecarContainer, pod)
		// merge VolumeMounts from sidecar.VolumeMounts and shared VolumeMounts
		sidecarContainer.VolumeMounts = util.MergeVolumeMounts(sidecarContainer.VolumeMounts, injectedMounts)

		// get injected env & mounts explicitly so that can be compared with old ones in pod
		transferEnvs := sidecarcontrol.GetSidecarTransferEnvs(&sidecarContainer, pod)
		// append volumeMounts SubPathExpr environments
		transferEnvs = util.MergeEnvVar(transferEnvs, injectedEnvs)
		// merged Env from sidecar.Env and transfer envs
		sidecarContainer.Env = util.MergeEnvVar(sidecarContainer.Env, transferEnvs)

		// upgrade sidecar container to latest
		newContainer := control.UpgradeSidecarContainer(&sidecarContainer, pod)
		// no change, then continue
		if newContainer == nil {
			continue
		}
		// change, and need to update in pod
		updateContainerInPod(*newContainer, pod)
		changedContainers = append(changedContainers, newContainer.Name)
		// hot upgrade sidecar container
		if sidecarcontrol.IsHotUpgradeContainer(&sidecarContainer) {
			var olderSidecar string
			name1, name2 := sidecarcontrol.GetHotUpgradeContainerName(sidecarContainer.Name)
			if name1 == newContainer.Name {
				olderSidecar = name2
			} else {
				olderSidecar = name1
			}
			// hot upgrade annotations
			// hotUpgradeContainerInfos: sidecarSet.Spec.Container[x].name -> working sidecar container
			// for example: mesh -> mesh-1, envoy -> envoy-2...
			hotUpgradeContainerInfos := sidecarcontrol.GetPodHotUpgradeInfoInAnnotations(pod)
			hotUpgradeContainerInfos[sidecarContainer.Name] = newContainer.Name
			by, _ := json.Marshal(hotUpgradeContainerInfos)
			pod.Annotations[sidecarcontrol.SidecarSetWorkingHotUpgradeContainer] = string(by)
			// update sidecar container resource version in annotations
			pod.Annotations[sidecarcontrol.GetPodSidecarSetVersionAnnotation(newContainer.Name)] = sidecarSet.ResourceVersion
			pod.Annotations[sidecarcontrol.GetPodSidecarSetVersionAltAnnotation(newContainer.Name)] = pod.Annotations[sidecarcontrol.GetPodSidecarSetVersionAnnotation(olderSidecar)]
			pod.Annotations[sidecarcontrol.GetPodSidecarSetVersionAltAnnotation(olderSidecar)] = sidecarSet.ResourceVersion
		}
	}
	// update sidecarSet hash in pod annotations[kruise.io/sidecarset-hash]
	sidecarcontrol.UpdatePodSidecarSetHash(pod, sidecarSet)
	// update pod information in upgrade
	// UpdatePodAnnotationsInUpgrade needs to be called when Update Container, including hot-upgrade reset empty image.
	// However, reset empty image should not update pod sidecarSet hash annotation, so UpdatePodSidecarSetHash needs to be called additionally
	control.UpdatePodAnnotationsInUpgrade(changedContainers, pod)
}

func inconsistentStatus(sidecarSet *appsv1alpha1.SidecarSet, status *appsv1alpha1.SidecarSetStatus) bool {
	return status.ObservedGeneration > sidecarSet.Status.ObservedGeneration ||
		status.MatchedPods != sidecarSet.Status.MatchedPods ||
		status.UpdatedPods != sidecarSet.Status.UpdatedPods ||
		status.ReadyPods != sidecarSet.Status.ReadyPods ||
		status.UpdatedReadyPods != sidecarSet.Status.UpdatedReadyPods ||
		status.LatestRevision != sidecarSet.Status.LatestRevision ||
		!pointer.Int32Equal(sidecarSet.Status.CollisionCount, status.CollisionCount)
}

func isSidecarSetUpdateFinish(status *appsv1alpha1.SidecarSetStatus) bool {
	return status.UpdatedPods >= status.MatchedPods
}

func (p *Processor) updatePodSidecarSetUpgradableCondition(sidecarset *appsv1alpha1.SidecarSet, pod *corev1.Pod, upgradable bool) error {
	podClone := pod.DeepCopy()

	_, oldCondition := podutil.GetPodCondition(&podClone.Status, sidecarcontrol.SidecarSetUpgradable)
	var condition *corev1.PodCondition
	if oldCondition != nil {
		condition = oldCondition.DeepCopy()
	} else {
		condition = &corev1.PodCondition{
			Type: sidecarcontrol.SidecarSetUpgradable,
		}
	}

	// get message kv from condition message
	messageKv, err := controlutil.GetMessageKvFromCondition(condition)
	if err != nil {
		return err
	}
	// mark sidecarset upgradable status
	messageKv[sidecarset.Name] = upgradable
	// update condition message
	allSidecarsetUpgradable := true
	for _, v := range messageKv {
		if v == false {
			allSidecarsetUpgradable = false
			break
		}
	}

	// only update condition status to true when all sidecarset upgradable status is true
	if allSidecarsetUpgradable {
		condition.Status = corev1.ConditionTrue
		condition.Reason = "AllSidecarsetUpgradable"
	} else {
		condition.Status = corev1.ConditionFalse
		condition.Reason = "UpdateImmutableField"
	}

	controlutil.UpdateMessageKvCondition(messageKv, condition)
	// patch SidecarSetUpgradable condition
	if conditionChanged := podutil.UpdatePodCondition(&podClone.Status, condition); !conditionChanged {
		// reduce unnecessary patch.
		return nil
	}

	mergePatch := fmt.Sprintf(`{"status": {"conditions": [%s]}}`, util.DumpJSON(condition))
	err = p.Client.Status().Patch(context.TODO(), podClone, client.RawPatch(types.StrategicMergePatchType, []byte(mergePatch)))
	if err != nil {
		return err
	}
	klog.V(3).InfoS("SidecarSet updated pod condition success", "sidecarSet", klog.KObj(sidecarset), "pod", klog.KObj(pod), "conditionType", sidecarcontrol.SidecarSetUpgradable, "conditionStatus", condition.Status)
	return nil
}