openyurtio
/
kruise
mirror of https://github.com/openkruise/kruise.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Optimize the CloneSetShortHash feature-gate and make it compatible with the existing Pods (#554) 

Signed-off-by: FillZpp <FillZpp.pub@gmail.com>
This commit is contained in:
Siyu Wang 2021-03-12 12:14:03 +08:00 committed by GitHub
parent 6952c9c480
commit 78c2c11d5c
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
23 changed files with 245 additions and 163 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
pkg/control/sidecarcontrol/util.go
View File

@ -54,6 +54,8 @@ var (
SidecarIgnoredNamespaces = []string{"kube-system", "kube-public"}
// SubPathExprEnvReg format: $(ODD_NAME)、$(POD_NAME)...
SubPathExprEnvReg, _ = regexp.Compile(`\$\(([-._a-zA-Z][-._a-zA-Z0-9]*)\)`)
RevisionAdapterImpl = &revisionAdapterImpl{}
)
type SidecarSetUpgradeSpec struct {
@ -93,6 +95,16 @@ func IsActivePod(pod *corev1.Pod) bool {
return true
}
type revisionAdapterImpl struct{}
func (r *revisionAdapterImpl) EqualToRevisionHash(sidecarSetName string, obj metav1.Object, hash string) bool {
return GetPodSidecarSetRevision(sidecarSetName, obj) == hash
}
func (r *revisionAdapterImpl) WriteRevisionHash(obj metav1.Object, hash string) {
// No need to implement yet.
}
func GetSidecarSetRevision(sidecarSet *appsv1alpha1.SidecarSet) string {
return sidecarSet.Annotations[SidecarSetHashAnnotation]
}

29
pkg/controller/cloneset/cloneset_controller.go
View File

@ -242,17 +242,17 @@ func (r *ReconcileCloneSet) doReconcile(request reconcile.Request) (res reconcil
history.SortControllerRevisions(revisions)
// get the current, and update revisions
currentRevision, updateRevision, collisionCount, err := r.getActiveRevisions(instance, revisions, clonesetutils.GetPodsRevisions(filteredPods))
currentRevision, updateRevision, collisionCount, err := r.getActiveRevisions(instance, revisions)
if err != nil {
return reconcile.Result{}, err
}
// Refresh update expectations
for _, pod := range filteredPods {
clonesetutils.UpdateExpectations.ObserveUpdated(request.String(), clonesetutils.GetRevisionLabel(updateRevision), pod)
clonesetutils.UpdateExpectations.ObserveUpdated(request.String(), updateRevision.Name, pod)
}
// If update expectations have not satisfied yet, just skip this reconcile.
if updateSatisfied, unsatisfiedDuration, updateDirtyPods := clonesetutils.UpdateExpectations.SatisfiedExpectations(request.String(), clonesetutils.GetRevisionLabel(updateRevision)); !updateSatisfied {
if updateSatisfied, unsatisfiedDuration, updateDirtyPods := clonesetutils.UpdateExpectations.SatisfiedExpectations(request.String(), updateRevision.Name); !updateSatisfied {
if unsatisfiedDuration >= expectations.ExpectationTimeout {
klog.Warningf("Expectation unsatisfied overtime for %v, updateDirtyPods=%v, timeout=%v", request.String(), updateDirtyPods, unsatisfiedDuration)
return reconcile.Result{}, nil
@ -285,7 +285,7 @@ func (r *ReconcileCloneSet) doReconcile(request reconcile.Request) (res reconcil
delayDuration, syncErr := r.syncCloneSet(instance, &newStatus, currentRevision, updateRevision, revisions, filteredPods, filteredPVCs)
// update new status
if err = r.statusUpdater.UpdateCloneSetStatus(instance, &newStatus, updateRevision, filteredPods); err != nil {
if err = r.statusUpdater.UpdateCloneSetStatus(instance, &newStatus, filteredPods); err != nil {
return reconcile.Result{}, err
}
@ -330,9 +330,7 @@ func (r *ReconcileCloneSet) syncCloneSet(
var podsScaleErr error
var podsUpdateErr error
scaling, podsScaleErr = r.scaleControl.Manage(currentSet, updateSet,
clonesetutils.GetRevisionLabel(currentRevision), clonesetutils.GetRevisionLabel(updateRevision),
filteredPods, filteredPVCs)
scaling, podsScaleErr = r.scaleControl.Manage(currentSet, updateSet, currentRevision.Name, updateRevision.Name, filteredPods, filteredPVCs)
if podsScaleErr != nil {
newStatus.Conditions = append(newStatus.Conditions, appsv1alpha1.CloneSetCondition{
Type: appsv1alpha1.CloneSetConditionFailedScale,
@ -363,7 +361,7 @@ func (r *ReconcileCloneSet) syncCloneSet(
return delayDuration, err
}
func (r *ReconcileCloneSet) getActiveRevisions(cs *appsv1alpha1.CloneSet, revisions []*apps.ControllerRevision, podsRevisions sets.String) (
func (r *ReconcileCloneSet) getActiveRevisions(cs *appsv1alpha1.CloneSet, revisions []*apps.ControllerRevision) (
*apps.ControllerRevision, *apps.ControllerRevision, int32, error,
) {
var currentRevision, updateRevision *apps.ControllerRevision
@ -484,13 +482,20 @@ func (r *ReconcileCloneSet) truncateHistory(
update *apps.ControllerRevision,
) error {
noLiveRevisions := make([]*apps.ControllerRevision, 0, len(revisions))
live := clonesetutils.GetPodsRevisions(pods)
live.Insert(current.Name, update.Name)
// collect live revisions and historic revisions
for i := range revisions {
if !live.Has(revisions[i].Name) {
noLiveRevisions = append(noLiveRevisions, revisions[i])
if revisions[i].Name != current.Name && revisions[i].Name != update.Name {
var found bool
for _, pod := range pods {
if clonesetutils.EqualToRevisionHash("", pod, revisions[i].Name) {
found = true
break
}
}
if !found {
noLiveRevisions = append(noLiveRevisions, revisions[i])
}
}
}
historyLen := len(noLiveRevisions)

13
pkg/controller/cloneset/cloneset_controller_test.go
View File

@ -17,29 +17,31 @@ limitations under the License.
package cloneset
import (
"fmt"
"reflect"
"strings"
"testing"
"time"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/kubernetes/pkg/apis/apps"
"github.com/onsi/gomega"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
clonesetutils "github.com/openkruise/kruise/pkg/controller/cloneset/utils"
"github.com/openkruise/kruise/pkg/features"
"github.com/openkruise/kruise/pkg/util"
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
"github.com/openkruise/kruise/pkg/util/fieldindex"
"golang.org/x/net/context"
v1 "k8s.io/api/core/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/api/resource"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/fields"
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/util/retry"
"k8s.io/kubernetes/pkg/apis/apps"
"sigs.k8s.io/controller-runtime/pkg/client"
"sigs.k8s.io/controller-runtime/pkg/client/fake"
"sigs.k8s.io/controller-runtime/pkg/manager"
@ -132,6 +134,9 @@ func TestReconcile(t *testing.T) {
// Test for pods scale
testScale(g, instance)
// Enable the CloneSetShortHash feature-gate
utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=true", features.CloneSetShortHash))
// Get latest cloneset
err = c.Get(context.TODO(), expectedRequest.NamespacedName, instance)
g.Expect(err).NotTo(gomega.HaveOccurred())

13
pkg/controller/cloneset/cloneset_status.go
View File

@ -22,7 +22,6 @@ import (
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
clonesetcore "github.com/openkruise/kruise/pkg/controller/cloneset/core"
clonesetutils "github.com/openkruise/kruise/pkg/controller/cloneset/utils"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/client-go/util/retry"
@ -32,7 +31,7 @@ import (
// StatusUpdater is interface for updating CloneSet status.
type StatusUpdater interface {
UpdateCloneSetStatus(cs *appsv1alpha1.CloneSet, newStatus *appsv1alpha1.CloneSetStatus, updateRevision *apps.ControllerRevision, pods []*v1.Pod) error
UpdateCloneSetStatus(cs *appsv1alpha1.CloneSet, newStatus *appsv1alpha1.CloneSetStatus, pods []*v1.Pod) error
}
func newStatusUpdater(c client.Client) StatusUpdater {
@ -43,8 +42,8 @@ type realStatusUpdater struct {
client.Client
}
func (r *realStatusUpdater) UpdateCloneSetStatus(cs *appsv1alpha1.CloneSet, newStatus *appsv1alpha1.CloneSetStatus, updateRevision *apps.ControllerRevision, pods []*v1.Pod) error {
r.calculateStatus(cs, newStatus, updateRevision, pods)
func (r *realStatusUpdater) UpdateCloneSetStatus(cs *appsv1alpha1.CloneSet, newStatus *appsv1alpha1.CloneSetStatus, pods []*v1.Pod) error {
r.calculateStatus(cs, newStatus, pods)
if r.inconsistentStatus(cs, newStatus) {
klog.Infof("To update CloneSet status for %s/%s, replicas=%d ready=%d available=%d updated=%d updatedReady=%d, revisions current=%s update=%s",
cs.Namespace, cs.Name, newStatus.Replicas, newStatus.ReadyReplicas, newStatus.AvailableReplicas, newStatus.UpdatedReplicas, newStatus.UpdatedReadyReplicas, newStatus.CurrentRevision, newStatus.UpdateRevision)
@ -81,7 +80,7 @@ func (r *realStatusUpdater) inconsistentStatus(cs *appsv1alpha1.CloneSet, newSta
newStatus.LabelSelector != oldStatus.LabelSelector
}
func (r *realStatusUpdater) calculateStatus(cs *appsv1alpha1.CloneSet, newStatus *appsv1alpha1.CloneSetStatus, updateRevision *apps.ControllerRevision, pods []*v1.Pod) {
func (r *realStatusUpdater) calculateStatus(cs *appsv1alpha1.CloneSet, newStatus *appsv1alpha1.CloneSetStatus, pods []*v1.Pod) {
coreControl := clonesetcore.New(cs)
for _, pod := range pods {
newStatus.Replicas++
@ -91,10 +90,10 @@ func (r *realStatusUpdater) calculateStatus(cs *appsv1alpha1.CloneSet, newStatus
if coreControl.IsPodUpdateReady(pod, cs.Spec.MinReadySeconds) {
newStatus.AvailableReplicas++
}
if clonesetutils.GetPodRevision("", pod) == clonesetutils.GetRevisionLabel(updateRevision) {
if clonesetutils.EqualToRevisionHash("", pod, newStatus.UpdateRevision) {
newStatus.UpdatedReplicas++
}
if clonesetutils.GetPodRevision("", pod) == clonesetutils.GetRevisionLabel(updateRevision) && coreControl.IsPodUpdateReady(pod, 0) {
if clonesetutils.EqualToRevisionHash("", pod, newStatus.UpdateRevision) && coreControl.IsPodUpdateReady(pod, 0) {
newStatus.UpdatedReadyReplicas++
}
}

3
pkg/controller/cloneset/core/cloneset_core.go
View File

@ -25,7 +25,6 @@ import (
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
clonesetutils "github.com/openkruise/kruise/pkg/controller/cloneset/utils"
"github.com/openkruise/kruise/pkg/util/inplaceupdate"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kubecontroller "k8s.io/kubernetes/pkg/controller"
@ -90,7 +89,7 @@ func (c *commonControl) newVersionedPods(cs *appsv1alpha1.CloneSet, revision str
if pod.Labels == nil {
pod.Labels = make(map[string]string)
}
pod.Labels[apps.ControllerRevisionHashLabelKey] = revision
clonesetutils.WriteRevisionHash(pod, revision)
pod.Name = fmt.Sprintf("%s-%s", cs.Name, id)
pod.Namespace = cs.Namespace

3
pkg/controller/cloneset/scale/cloneset_scale.go
View File

@ -13,7 +13,6 @@ import (
"github.com/openkruise/kruise/pkg/util"
"github.com/openkruise/kruise/pkg/util/expectations"
"github.com/openkruise/kruise/pkg/util/lifecycle"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/util/sets"
"k8s.io/client-go/tools/record"
@ -177,7 +176,7 @@ func (r *realControl) createPods(
pod := <-podsCreationChan
cs := updateCS
if pod.Labels[apps.ControllerRevisionHashLabelKey] == currentRevision {
if clonesetutils.EqualToRevisionHash("", pod, currentRevision) {
cs = currentCS
}
lifecycle.SetPodLifecycle(appspub.LifecycleStateNormal)(pod)

10
pkg/controller/cloneset/scale/cloneset_scale_test.go
View File

@ -34,8 +34,8 @@ func TestCreatePods(t *testing.T) {
currentCS := clonesettest.NewCloneSet(3)
updateCS := currentCS.DeepCopy()
updateCS.Spec.Template.Spec.Containers[0].Env = []v1.EnvVar{{Name: "e-key", Value: "e-value"}}
currentRevision := "revision-abc"
updateRevision := "revision-xyz"
currentRevision := "revision_abc"
updateRevision := "revision_xyz"
ctrl := newFakeControl()
created, err := ctrl.createPods(
@ -66,7 +66,7 @@ func TestCreatePods(t *testing.T) {
GenerateName: "foo-",
Labels: map[string]string{
appsv1alpha1.CloneSetInstanceID: "id1",
apps.ControllerRevisionHashLabelKey: "revision-abc",
apps.ControllerRevisionHashLabelKey: "revision_abc",
"foo": "bar",
appspub.LifecycleStateKey: string(appspub.LifecycleStateNormal),
},
@ -122,7 +122,7 @@ func TestCreatePods(t *testing.T) {
GenerateName: "foo-",
Labels: map[string]string{
appsv1alpha1.CloneSetInstanceID: "id3",
apps.ControllerRevisionHashLabelKey: "revision-xyz",
apps.ControllerRevisionHashLabelKey: "revision_xyz",
"foo": "bar",
appspub.LifecycleStateKey: string(appspub.LifecycleStateNormal),
},
@ -179,7 +179,7 @@ func TestCreatePods(t *testing.T) {
GenerateName: "foo-",
Labels: map[string]string{
appsv1alpha1.CloneSetInstanceID: "id4",
apps.ControllerRevisionHashLabelKey: "revision-xyz",
apps.ControllerRevisionHashLabelKey: "revision_xyz",
"foo": "bar",
appspub.LifecycleStateKey: string(appspub.LifecycleStateNormal),
},

12
pkg/controller/cloneset/update/cloneset_update.go
View File

@ -48,7 +48,7 @@ type Interface interface {
func New(c client.Client, recorder record.EventRecorder) Interface {
return &realControl{
inplaceControl: inplaceupdate.New(c, apps.ControllerRevisionHashLabelKey),
inplaceControl: inplaceupdate.New(c, clonesetutils.RevisionAdapterImpl),
lifecycleControl: lifecycle.New(c),
Client: c,
recorder: recorder,
@ -98,7 +98,7 @@ func (c *realControl) Manage(cs *appsv1alpha1.CloneSet,
continue
}
if clonesetutils.GetPodRevision("", pods[i]) != clonesetutils.GetRevisionLabel(updateRevision) {
if !clonesetutils.EqualToRevisionHash("", pods[i], updateRevision.Name) {
switch lifecycle.GetPodLifecycleState(pods[i]) {
case appspub.LifecycleStatePreparingDelete, appspub.LifecycleStateUpdated:
klog.V(3).Infof("CloneSet %s/%s find pod %s in state %s, so skip to update it",
@ -172,8 +172,7 @@ func (c *realControl) refreshPodState(cs *appsv1alpha1.CloneSet, coreControl clo
return false, 0, err
} else if updated {
clonesetutils.ResourceVersionExpectations.Expect(pod)
klog.V(3).Infof("CloneSet %s update pod %s lifecycle to %s",
clonesetutils.GetControllerKey(cs), pod.Name, state)
klog.V(3).Infof("CloneSet %s update pod %s lifecycle to %s", clonesetutils.GetControllerKey(cs), pod.Name, state)
return true, res.DelayDuration, nil
}
}
@ -261,7 +260,7 @@ func (c *realControl) updatePod(cs *appsv1alpha1.CloneSet, coreControl clonesetc
cs.Spec.UpdateStrategy.Type == appsv1alpha1.InPlaceOnlyCloneSetUpdateStrategyType {
var oldRevision *apps.ControllerRevision
for _, r := range revisions {
if clonesetutils.GetRevisionLabel(r) == clonesetutils.GetPodRevision("", pod) {
if clonesetutils.EqualToRevisionHash("", pod, r.Name) {
oldRevision = r
break
}
@ -291,12 +290,11 @@ func (c *realControl) updatePod(cs *appsv1alpha1.CloneSet, coreControl clonesetc
opts := coreControl.GetUpdateOptions()
opts.AdditionalFuncs = append(opts.AdditionalFuncs, lifecycle.SetPodLifecycle(appspub.LifecycleStateUpdating))
opts.GetRevision = clonesetutils.GetRevisionLabel
res := c.inplaceControl.Update(pod, oldRevision, updateRevision, opts)
if res.InPlaceUpdate {
if res.UpdateErr == nil {
c.recorder.Eventf(cs, v1.EventTypeNormal, "SuccessfulUpdatePodInPlace", "successfully update pod %s in-place(revision %v)", pod.Name, updateRevision.Name)
clonesetutils.UpdateExpectations.ExpectUpdated(clonesetutils.GetControllerKey(cs), clonesetutils.GetRevisionLabel(updateRevision), pod)
clonesetutils.UpdateExpectations.ExpectUpdated(clonesetutils.GetControllerKey(cs), updateRevision.Name, pod)
return res.DelayDuration, nil
}

81
pkg/controller/cloneset/update/cloneset_update_test.go
View File

@ -26,6 +26,7 @@ import (
appspub "github.com/openkruise/kruise/apis/apps/pub"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
clonesetcore "github.com/openkruise/kruise/pkg/controller/cloneset/core"
clonesetutils "github.com/openkruise/kruise/pkg/controller/cloneset/utils"
"github.com/openkruise/kruise/pkg/util"
"github.com/openkruise/kruise/pkg/util/inplaceupdate"
"github.com/openkruise/kruise/pkg/util/lifecycle"
@ -79,10 +80,10 @@ func TestMange(t *testing.T) {
{
name: "do nothing",
cs: &appsv1alpha1.CloneSet{Spec: appsv1alpha1.CloneSetSpec{Replicas: getInt32Pointer(1)}},
updateRevision: &apps.ControllerRevision{ObjectMeta: metav1.ObjectMeta{Name: "rev-new"}},
updateRevision: &apps.ControllerRevision{ObjectMeta: metav1.ObjectMeta{Name: "rev_new"}},
pods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev-new"}},
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev_new"}},
Spec: v1.PodSpec{ReadinessGates: []v1.PodReadinessGate{{ConditionType: appspub.InPlaceUpdateReady}}},
Status: v1.PodStatus{Phase: v1.PodRunning, Conditions: []v1.PodCondition{
{Type: v1.PodReady, Status: v1.ConditionTrue},
@ -92,7 +93,7 @@ func TestMange(t *testing.T) {
},
expectedPods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev-new"}},
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev_new"}},
Spec: v1.PodSpec{ReadinessGates: []v1.PodReadinessGate{{ConditionType: appspub.InPlaceUpdateReady}}},
Status: v1.PodStatus{Phase: v1.PodRunning, Conditions: []v1.PodCondition{
{Type: v1.PodReady, Status: v1.ConditionTrue},
@ -104,10 +105,10 @@ func TestMange(t *testing.T) {
{
name: "normal update condition",
cs: &appsv1alpha1.CloneSet{Spec: appsv1alpha1.CloneSetSpec{Replicas: getInt32Pointer(1)}},
updateRevision: &apps.ControllerRevision{ObjectMeta: metav1.ObjectMeta{Name: "rev-new"}},
updateRevision: &apps.ControllerRevision{ObjectMeta: metav1.ObjectMeta{Name: "rev_new"}},
pods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev-new"}},
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev_new"}},
Spec: v1.PodSpec{ReadinessGates: []v1.PodReadinessGate{{ConditionType: appspub.InPlaceUpdateReady}}},
Status: v1.PodStatus{Phase: v1.PodRunning, Conditions: []v1.PodCondition{
{Type: v1.PodReady, Status: v1.ConditionTrue},
@ -116,7 +117,7 @@ func TestMange(t *testing.T) {
},
expectedPods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev-new"}, ResourceVersion: "1"},
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev_new"}, ResourceVersion: "1"},
Spec: v1.PodSpec{ReadinessGates: []v1.PodReadinessGate{{ConditionType: appspub.InPlaceUpdateReady}}},
Status: v1.PodStatus{Phase: v1.PodRunning, Conditions: []v1.PodCondition{
{Type: v1.PodReady, Status: v1.ConditionTrue},
@ -131,11 +132,11 @@ func TestMange(t *testing.T) {
Replicas: getInt32Pointer(1),
UpdateStrategy: appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.RecreateCloneSetUpdateStrategyType},
}},
updateRevision: &apps.ControllerRevision{ObjectMeta: metav1.ObjectMeta{Name: "rev-new"}},
updateRevision: &apps.ControllerRevision{ObjectMeta: metav1.ObjectMeta{Name: "rev_new"}},
pods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{
apps.ControllerRevisionHashLabelKey: "rev-old",
apps.ControllerRevisionHashLabelKey: "rev_old",
appsv1alpha1.CloneSetInstanceID: "id-0",
}},
Spec: v1.PodSpec{ReadinessGates: []v1.PodReadinessGate{{ConditionType: appspub.InPlaceUpdateReady}}},
@ -153,7 +154,7 @@ func TestMange(t *testing.T) {
expectedPods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", ResourceVersion: "1", Labels: map[string]string{
apps.ControllerRevisionHashLabelKey: "rev-old",
apps.ControllerRevisionHashLabelKey: "rev_old",
appsv1alpha1.CloneSetInstanceID: "id-0",
appsv1alpha1.SpecifiedDeleteKey: "true",
}},
@ -177,19 +178,19 @@ func TestMange(t *testing.T) {
UpdateStrategy: appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType},
}},
updateRevision: &apps.ControllerRevision{
ObjectMeta: metav1.ObjectMeta{Name: "rev-new"},
ObjectMeta: metav1.ObjectMeta{Name: "rev_new"},
Data: runtime.RawExtension{Raw: []byte(`{"spec":{"template":{"$patch":"replace","spec":{"containers":[{"name":"c1","image":"foo2","env":["name":"k", "value":"v"]}]}}}}`)},
},
revisions: []*apps.ControllerRevision{
{
ObjectMeta: metav1.ObjectMeta{Name: "rev-old"},
ObjectMeta: metav1.ObjectMeta{Name: "rev_old"},
Data: runtime.RawExtension{Raw: []byte(`{"spec":{"template":{"$patch":"replace","spec":{"containers":[{"name":"c1","image":"foo1"}]}}}}`)},
},
},
pods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{
apps.ControllerRevisionHashLabelKey: "rev-old",
apps.ControllerRevisionHashLabelKey: "rev_old",
appsv1alpha1.CloneSetInstanceID: "id-0",
}},
Spec: v1.PodSpec{
@ -214,7 +215,7 @@ func TestMange(t *testing.T) {
expectedPods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", ResourceVersion: "1", Labels: map[string]string{
apps.ControllerRevisionHashLabelKey: "rev-old",
apps.ControllerRevisionHashLabelKey: "rev_old",
appsv1alpha1.CloneSetInstanceID: "id-0",
appsv1alpha1.SpecifiedDeleteKey: "true",
}},
@ -245,19 +246,19 @@ func TestMange(t *testing.T) {
UpdateStrategy: appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType},
}},
updateRevision: &apps.ControllerRevision{
ObjectMeta: metav1.ObjectMeta{Name: "rev-new"},
ObjectMeta: metav1.ObjectMeta{Name: "rev_new"},
Data: runtime.RawExtension{Raw: []byte(`{"spec":{"template":{"$patch":"replace","spec":{"containers":[{"name":"c1","image":"foo2"}]}}}}`)},
},
revisions: []*apps.ControllerRevision{
{
ObjectMeta: metav1.ObjectMeta{Name: "rev-old"},
ObjectMeta: metav1.ObjectMeta{Name: "rev_old"},
Data: runtime.RawExtension{Raw: []byte(`{"spec":{"template":{"$patch":"replace","spec":{"containers":[{"name":"c1","image":"foo1"}]}}}}`)},
},
},
pods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{
apps.ControllerRevisionHashLabelKey: "rev-old",
apps.ControllerRevisionHashLabelKey: "rev_old",
appsv1alpha1.CloneSetInstanceID: "id-0",
}},
Spec: v1.PodSpec{
@ -283,12 +284,12 @@ func TestMange(t *testing.T) {
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0",
Labels: map[string]string{
apps.ControllerRevisionHashLabelKey: "rev-new",
apps.ControllerRevisionHashLabelKey: "rev_new",
appsv1alpha1.CloneSetInstanceID: "id-0",
appspub.LifecycleStateKey: string(appspub.LifecycleStateUpdating),
},
Annotations: map[string]string{appspub.InPlaceUpdateStateKey: util.DumpJSON(appspub.InPlaceUpdateState{
Revision: "rev-new",
Revision: "rev_new",
UpdateTimestamp: now,
LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "image-id-xyz"}},
})},
@ -321,19 +322,19 @@ func TestMange(t *testing.T) {
UpdateStrategy: appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType, InPlaceUpdateStrategy: &appspub.InPlaceUpdateStrategy{GracePeriodSeconds: 3630}},
}},
updateRevision: &apps.ControllerRevision{
ObjectMeta: metav1.ObjectMeta{Name: "rev-new"},
ObjectMeta: metav1.ObjectMeta{Name: "rev_new"},
Data: runtime.RawExtension{Raw: []byte(`{"spec":{"template":{"$patch":"replace","spec":{"containers":[{"name":"c1","image":"foo2"}]}}}}`)},
},
revisions: []*apps.ControllerRevision{
{
ObjectMeta: metav1.ObjectMeta{Name: "rev-old"},
ObjectMeta: metav1.ObjectMeta{Name: "rev_old"},
Data: runtime.RawExtension{Raw: []byte(`{"spec":{"template":{"$patch":"replace","spec":{"containers":[{"name":"c1","image":"foo1"}]}}}}`)},
},
},
pods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0", Labels: map[string]string{
apps.ControllerRevisionHashLabelKey: "rev-old",
apps.ControllerRevisionHashLabelKey: "rev_old",
appsv1alpha1.CloneSetInstanceID: "id-0",
}},
Spec: v1.PodSpec{
@ -359,17 +360,17 @@ func TestMange(t *testing.T) {
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0",
Labels: map[string]string{
apps.ControllerRevisionHashLabelKey: "rev-new",
apps.ControllerRevisionHashLabelKey: "rev_new",
appsv1alpha1.CloneSetInstanceID: "id-0",
appspub.LifecycleStateKey: string(appspub.LifecycleStateUpdating),
},
Annotations: map[string]string{
appspub.InPlaceUpdateStateKey: util.DumpJSON(appspub.InPlaceUpdateState{
Revision: "rev-new",
Revision: "rev_new",
UpdateTimestamp: now,
LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "image-id-xyz"}},
}),
appspub.InPlaceUpdateGraceKey: `{"revision":"rev-new","containerImages":{"c1":"foo2"},"graceSeconds":3630}`,
appspub.InPlaceUpdateGraceKey: `{"revision":"rev_new","containerImages":{"c1":"foo2"},"graceSeconds":3630}`,
},
ResourceVersion: "2",
},
@ -400,26 +401,26 @@ func TestMange(t *testing.T) {
UpdateStrategy: appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType, InPlaceUpdateStrategy: &appspub.InPlaceUpdateStrategy{GracePeriodSeconds: 3630}},
}},
updateRevision: &apps.ControllerRevision{
ObjectMeta: metav1.ObjectMeta{Name: "rev-new"},
ObjectMeta: metav1.ObjectMeta{Name: "rev_new"},
Data: runtime.RawExtension{Raw: []byte(`{"spec":{"template":{"$patch":"replace","spec":{"containers":[{"name":"c1","image":"foo2"}]}}}}`)},
},
revisions: []*apps.ControllerRevision{
{
ObjectMeta: metav1.ObjectMeta{Name: "rev-old"},
ObjectMeta: metav1.ObjectMeta{Name: "rev_old"},
Data: runtime.RawExtension{Raw: []byte(`{"spec":{"template":{"$patch":"replace","spec":{"containers":[{"name":"c1","image":"foo1"}]}}}}`)},
},
},
pods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0",
Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev-new", appsv1alpha1.CloneSetInstanceID: "id-0"},
Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev_new", appsv1alpha1.CloneSetInstanceID: "id-0"},
Annotations: map[string]string{
appspub.InPlaceUpdateStateKey: util.DumpJSON(appspub.InPlaceUpdateState{
Revision: "rev-new",
Revision: "rev_new",
UpdateTimestamp: metav1.NewTime(now.Add(-time.Second * 10)),
LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "image-id-xyz"}},
}),
appspub.InPlaceUpdateGraceKey: `{"revision":"rev-new","containerImages":{"c1":"foo2"},"graceSeconds":3630}`,
appspub.InPlaceUpdateGraceKey: `{"revision":"rev_new","containerImages":{"c1":"foo2"},"graceSeconds":3630}`,
},
},
Spec: v1.PodSpec{
@ -445,16 +446,16 @@ func TestMange(t *testing.T) {
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0",
Labels: map[string]string{
apps.ControllerRevisionHashLabelKey: "rev-new",
apps.ControllerRevisionHashLabelKey: "rev_new",
appsv1alpha1.CloneSetInstanceID: "id-0",
},
Annotations: map[string]string{
appspub.InPlaceUpdateStateKey: util.DumpJSON(appspub.InPlaceUpdateState{
Revision: "rev-new",
Revision: "rev_new",
UpdateTimestamp: metav1.NewTime(now.Add(-time.Second * 10)),
LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "image-id-xyz"}},
}),
appspub.InPlaceUpdateGraceKey: `{"revision":"rev-new","containerImages":{"c1":"foo2"},"graceSeconds":3630}`,
appspub.InPlaceUpdateGraceKey: `{"revision":"rev_new","containerImages":{"c1":"foo2"},"graceSeconds":3630}`,
},
},
Spec: v1.PodSpec{
@ -484,26 +485,26 @@ func TestMange(t *testing.T) {
UpdateStrategy: appsv1alpha1.CloneSetUpdateStrategy{Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType, InPlaceUpdateStrategy: &appspub.InPlaceUpdateStrategy{GracePeriodSeconds: 3630}},
}},
updateRevision: &apps.ControllerRevision{
ObjectMeta: metav1.ObjectMeta{Name: "rev-new"},
ObjectMeta: metav1.ObjectMeta{Name: "rev_new"},
Data: runtime.RawExtension{Raw: []byte(`{"spec":{"template":{"$patch":"replace","spec":{"containers":[{"name":"c1","image":"foo2"}]}}}}`)},
},
revisions: []*apps.ControllerRevision{
{
ObjectMeta: metav1.ObjectMeta{Name: "rev-old"},
ObjectMeta: metav1.ObjectMeta{Name: "rev_old"},
Data: runtime.RawExtension{Raw: []byte(`{"spec":{"template":{"$patch":"replace","spec":{"containers":[{"name":"c1","image":"foo1"}]}}}}`)},
},
},
pods: []*v1.Pod{
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0",
Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev-new", appsv1alpha1.CloneSetInstanceID: "id-0"},
Labels: map[string]string{apps.ControllerRevisionHashLabelKey: "rev_new", appsv1alpha1.CloneSetInstanceID: "id-0"},
Annotations: map[string]string{
appspub.InPlaceUpdateStateKey: util.DumpJSON(appspub.InPlaceUpdateState{
Revision: "rev-new",
Revision: "rev_new",
UpdateTimestamp: metav1.NewTime(now.Add(-time.Minute)),
LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "image-id-xyz"}},
}),
appspub.InPlaceUpdateGraceKey: `{"revision":"rev-new","containerImages":{"c1":"foo2"},"graceSeconds":3630}`,
appspub.InPlaceUpdateGraceKey: `{"revision":"rev_new","containerImages":{"c1":"foo2"},"graceSeconds":3630}`,
},
},
Spec: v1.PodSpec{
@ -529,12 +530,12 @@ func TestMange(t *testing.T) {
{
ObjectMeta: metav1.ObjectMeta{Name: "pod-0",
Labels: map[string]string{
apps.ControllerRevisionHashLabelKey: "rev-new",
apps.ControllerRevisionHashLabelKey: "rev_new",
appsv1alpha1.CloneSetInstanceID: "id-0",
},
Annotations: map[string]string{
appspub.InPlaceUpdateStateKey: util.DumpJSON(appspub.InPlaceUpdateState{
Revision: "rev-new",
Revision: "rev_new",
UpdateTimestamp: metav1.NewTime(now.Add(-time.Minute)),
LastContainerStatuses: map[string]appspub.InPlaceUpdateContainerStatus{"c1": {ImageID: "image-id-xyz"}},
}),
@ -569,7 +570,7 @@ func TestMange(t *testing.T) {
ctrl := &realControl{
fakeClient,
lifecycle.NewForTest(fakeClient),
inplaceupdate.NewForTest(fakeClient, apps.ControllerRevisionHashLabelKey, func() metav1.Time { return now }),
inplaceupdate.NewForTest(fakeClient, clonesetutils.RevisionAdapterImpl, func() metav1.Time { return now }),
record.NewFakeRecorder(10),
}
if _, err := ctrl.Manage(mc.cs, mc.updateRevision, mc.revisions, mc.pods, mc.pvcs); err != nil {

63
pkg/controller/cloneset/utils/cloneset_utils.go
View File

@ -19,6 +19,7 @@ package utils
import (
"context"
"fmt"
"strings"
"sync"
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
@ -29,23 +30,52 @@ import (
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"k8s.io/apimachinery/pkg/util/sets"
podutil "k8s.io/kubernetes/pkg/api/v1/pod"
kubecontroller "k8s.io/kubernetes/pkg/controller"
"k8s.io/kubernetes/pkg/controller/history"
"k8s.io/utils/integer"
"sigs.k8s.io/controller-runtime/pkg/client"
)
var (
// ControllerKind is GroupVersionKind for CloneSet.
ControllerKind = appsv1alpha1.SchemeGroupVersion.WithKind("CloneSet")
ControllerKind = appsv1alpha1.SchemeGroupVersion.WithKind("CloneSet")
RevisionAdapterImpl = &revisionAdapterImpl{}
EqualToRevisionHash = RevisionAdapterImpl.EqualToRevisionHash
WriteRevisionHash = RevisionAdapterImpl.WriteRevisionHash
ScaleExpectations = expectations.NewScaleExpectations()
UpdateExpectations = expectations.NewUpdateExpectations(GetPodRevision)
UpdateExpectations = expectations.NewUpdateExpectations(RevisionAdapterImpl)
ResourceVersionExpectations = expectations.NewResourceVersionExpectation()
)
type revisionAdapterImpl struct {
}
func (r *revisionAdapterImpl) EqualToRevisionHash(_ string, obj metav1.Object, hash string) bool {
objHash := obj.GetLabels()[apps.ControllerRevisionHashLabelKey]
if objHash == hash {
return true
}
return r.getShortHash(hash) == r.getShortHash(objHash)
}
func (r *revisionAdapterImpl) WriteRevisionHash(obj metav1.Object, hash string) {
if obj.GetLabels() == nil {
obj.SetLabels(make(map[string]string, 1))
}
if utilfeature.DefaultFeatureGate.Enabled(features.CloneSetShortHash) {
hash = r.getShortHash(hash)
}
obj.GetLabels()[apps.ControllerRevisionHashLabelKey] = hash
}
func (r *revisionAdapterImpl) getShortHash(hash string) string {
// This makes sure the real hash must be the last '-' substring of revision name
// vendor/k8s.io/kubernetes/pkg/controller/history/controller_history.go#82
list := strings.Split(hash, "-")
return list[len(list)-1]
}
// GetControllerKey return key of CloneSet.
func GetControllerKey(cs *appsv1alpha1.CloneSet) string {
return types.NamespacedName{Namespace: cs.Namespace, Name: cs.Name}.String()
@ -69,29 +99,6 @@ func GetActivePods(reader client.Reader, opts *client.ListOptions) ([]*v1.Pod, e
return activePods, nil
}
// GetPodRevision returns revision hash of this pod.
func GetPodRevision(controllerKey string, pod metav1.Object) string {
return pod.GetLabels()[apps.ControllerRevisionHashLabelKey]
}
// GetPodsRevisions return revision hash set of these pods.
func GetPodsRevisions(pods []*v1.Pod) sets.String {
revisions := sets.NewString()
for _, p := range pods {
revisions.Insert(GetPodRevision("", p))
}
return revisions
}
// GetRevisionLabel return revision hash label value from revision to create pods.
// https://github.com/openkruise/kruise/issues/531
func GetRevisionLabel(revision *apps.ControllerRevision) string {
if utilfeature.DefaultFeatureGate.Enabled(features.CloneSetHashOnlyRevisionName) {
return revision.Labels[history.ControllerRevisionHashLabel]
}
return revision.Name
}
// NextRevision finds the next valid revision number based on revisions. If the length of revisions
// is 0 this is 1. Otherwise, it is 1 greater than the largest revision's Revision. This method
// assumes that revisions has been sorted by Revision.
@ -116,7 +123,7 @@ func IsRunningAndAvailable(pod *v1.Pod, minReadySeconds int32) bool {
// SplitPodsByRevision returns Pods matched and unmatched the given revision
func SplitPodsByRevision(pods []*v1.Pod, rev string) (matched, unmatched []*v1.Pod) {
for _, p := range pods {
if GetPodRevision("", p) == rev {
if EqualToRevisionHash("", p, rev) {
matched = append(matched, p)
} else {
unmatched = append(unmatched, p)

5
pkg/controller/daemonset/daemonset_controller.go
View File

@ -26,6 +26,7 @@ import (
"time"
utildiscovery "github.com/openkruise/kruise/pkg/util/discovery"
"github.com/openkruise/kruise/pkg/util/revisionadapter"
apps "k8s.io/api/apps/v1"
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/api/errors"
@ -85,7 +86,7 @@ var (
// A TTLCache of pod creates/deletes each ds expects to see
expectations = kubecontroller.NewControllerExpectations()
updateExpectations = kruiseExpectations.NewUpdateExpectations(GetPodRevision)
updateExpectations = kruiseExpectations.NewUpdateExpectations(revisionadapter.NewDefaultImpl())
)
const (
@ -172,7 +173,7 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
nodeLister: nodeLister,
suspendedDaemonPods: map[string]sets.String{},
failedPodsBackoff: failedPodsBackoff,
inplaceControl: inplaceupdate.New(cli, apps.ControllerRevisionHashLabelKey),
inplaceControl: inplaceupdate.New(cli, revisionadapter.NewDefaultImpl()),
updateExp: updateExpectations,
}
dsc.podNodeIndex = podInformer.(cache.SharedIndexInformer).GetIndexer()

2
pkg/controller/sidecarset/sidecarset_controller.go
View File

@ -63,7 +63,7 @@ func Add(mgr manager.Manager) error {
// newReconciler returns a new reconcile.Reconciler
func newReconciler(mgr manager.Manager) reconcile.Reconciler {
expectations := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
expectations := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl)
recorder := mgr.GetEventRecorderFor("sidecarset-controller")
cli := util.NewClientFromManager(mgr, "sidecarset-controller")
return &ReconcileSidecarSet{

12
pkg/controller/sidecarset/sidecarset_controller_test.go
View File

@ -175,10 +175,10 @@ func testUpdateWhenUseNotUpdateStrategy(t *testing.T, sidecarSetInput *appsv1alp
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl)
reconciler := ReconcileSidecarSet{
Client: fakeClient,
updateExpectations: expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision),
updateExpectations: expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl),
processor: NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10)),
}
if _, err := reconciler.Reconcile(request); err != nil {
@ -210,7 +210,7 @@ func testUpdateWhenSidecarSetPaused(t *testing.T, sidecarSetInput *appsv1alpha1.
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl)
reconciler := ReconcileSidecarSet{
Client: fakeClient,
updateExpectations: exps,
@ -245,7 +245,7 @@ func testUpdateWhenMaxUnavailableNotZero(t *testing.T, sidecarSetInput *appsv1al
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl)
reconciler := ReconcileSidecarSet{
Client: fakeClient,
updateExpectations: exps,
@ -281,7 +281,7 @@ func testUpdateWhenPartitionFinished(t *testing.T, sidecarSetInput *appsv1alpha1
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl)
reconciler := ReconcileSidecarSet{
Client: fakeClient,
updateExpectations: exps,
@ -317,7 +317,7 @@ func testRemoveSidecarSet(t *testing.T, sidecarSetInput *appsv1alpha1.SidecarSet
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSetInput, podInput)
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl)
reconciler := ReconcileSidecarSet{
Client: fakeClient,
updateExpectations: exps,

6
pkg/controller/sidecarset/sidecarset_processor_test.go
View File

@ -167,7 +167,7 @@ func testUpdateColdUpgradeSidecar(t *testing.T, podDemo *corev1.Pod, sidecarSetI
expectedStatus: []int32{2, 2, 2, 2},
},
}
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl)
for _, cs := range cases {
t.Run(cs.name, func(t *testing.T) {
pods := cs.getPods()
@ -248,7 +248,7 @@ func TestScopeNamespacePods(t *testing.T) {
}
fakeClient.Create(context.TODO(), pod)
}
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl)
processor := NewSidecarSetProcessor(fakeClient, exps, record.NewFakeRecorder(10))
pods, err := processor.getMatchingPods(sidecarSet)
if err != nil {
@ -270,7 +270,7 @@ func TestCanUpgradePods(t *testing.T) {
}
fakeClient := fake.NewFakeClientWithScheme(scheme, sidecarSet)
pods := factoryPodsCommon(100, 0, sidecarSet)
exps := expectations.NewUpdateExpectations(sidecarcontrol.GetPodSidecarSetRevision)
exps := expectations.NewUpdateExpectations(sidecarcontrol.RevisionAdapterImpl)
for i := range pods {
if i < 50 {
pods[i].Annotations[sidecarcontrol.SidecarSetHashWithoutImageAnnotation] = `{"test-sidecarset":{"hash":"without-aaa"}}`

5
pkg/controller/statefulset/stateful_set_control_test.go
View File

@ -56,6 +56,7 @@ import (
kruiseappslisters "github.com/openkruise/kruise/pkg/client/listers/apps/v1beta1"
"github.com/openkruise/kruise/pkg/util/inplaceupdate"
"github.com/openkruise/kruise/pkg/util/lifecycle"
"github.com/openkruise/kruise/pkg/util/revisionadapter"
)
type invariantFunc func(set *appsv1beta1.StatefulSet, spc *fakeStatefulPodControl) error
@ -66,7 +67,7 @@ func setupController(client clientset.Interface, kruiseClient kruiseclientset.In
spc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), kruiseInformerFactory.Apps().V1beta1().StatefulSets())
ssu := newFakeStatefulSetStatusUpdater(kruiseInformerFactory.Apps().V1beta1().StatefulSets())
recorder := record.NewFakeRecorder(10)
inplaceControl := inplaceupdate.NewForInformer(informerFactory.Core().V1().Pods(), apps.ControllerRevisionHashLabelKey)
inplaceControl := inplaceupdate.NewForInformer(informerFactory.Core().V1().Pods(), revisionadapter.NewDefaultImpl())
lifecycleControl := lifecycle.NewForInformer(informerFactory.Core().V1().Pods())
ssc := NewDefaultStatefulSetControl(spc, inplaceControl, lifecycleControl, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), recorder)
@ -548,7 +549,7 @@ func TestStatefulSetControl_getSetRevisions(t *testing.T) {
spc := newFakeStatefulPodControl(informerFactory.Core().V1().Pods(), kruiseInformerFactory.Apps().V1beta1().StatefulSets())
ssu := newFakeStatefulSetStatusUpdater(kruiseInformerFactory.Apps().V1beta1().StatefulSets())
recorder := record.NewFakeRecorder(10)
inplaceControl := inplaceupdate.NewForInformer(informerFactory.Core().V1().Pods(), apps.ControllerRevisionHashLabelKey)
inplaceControl := inplaceupdate.NewForInformer(informerFactory.Core().V1().Pods(), revisionadapter.NewDefaultImpl())
lifecycleControl := lifecycle.NewForInformer(informerFactory.Core().V1().Pods())
ssc := defaultStatefulSetControl{spc, ssu, history.NewFakeHistory(informerFactory.Apps().V1().ControllerRevisions()), recorder, inplaceControl, lifecycleControl}

8
pkg/controller/statefulset/statefulset_controller.go
View File

@ -33,6 +33,7 @@ import (
"github.com/openkruise/kruise/pkg/util/lifecycle"
"github.com/openkruise/kruise/pkg/util/ratelimiter"
"github.com/openkruise/kruise/pkg/util/requeueduration"
"github.com/openkruise/kruise/pkg/util/revisionadapter"
apps "k8s.io/api/apps/v1"
appsv1 "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
@ -67,10 +68,7 @@ var (
controllerKind = appsv1beta1.SchemeGroupVersion.WithKind("StatefulSet")
concurrentReconciles = 3
updateExpectations = expectations.NewUpdateExpectations(func(controllerKey string, o metav1.Object) string {
p := o.(*v1.Pod)
return getPodRevision(p)
})
updateExpectations = expectations.NewUpdateExpectations(revisionadapter.NewDefaultImpl())
// this is a short cut for any sub-functions to notify the reconcile how long to wait to requeue
durationStore = requeueduration.DurationStore{}
)
@ -127,7 +125,7 @@ func newReconciler(mgr manager.Manager) (reconcile.Reconciler, error) {
podLister,
pvcLister,
recorder),
inplaceupdate.New(util.NewClientFromManager(mgr, "statefulset-controller"), appsv1.ControllerRevisionHashLabelKey),
inplaceupdate.New(util.NewClientFromManager(mgr, "statefulset-controller"), revisionadapter.NewDefaultImpl()),
lifecycle.New(util.NewClientFromManager(mgr, "statefulset-controller")),
NewRealStatefulSetStatusUpdater(genericClient.KruiseClient, statefulSetLister),
history.NewHistory(genericClient.KubeClient, appslisters.NewControllerRevisionLister(revInformer.(toolscache.SharedIndexInformer).GetIndexer())),

6
pkg/controller/statefulset/statefulset_controller_test.go
View File

@ -32,7 +32,7 @@ import (
kruiseappslisters "github.com/openkruise/kruise/pkg/client/listers/apps/v1beta1"
"github.com/openkruise/kruise/pkg/util/inplaceupdate"
"github.com/openkruise/kruise/pkg/util/lifecycle"
apps "k8s.io/api/apps/v1"
"github.com/openkruise/kruise/pkg/util/revisionadapter"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/labels"
@ -634,7 +634,7 @@ func newFakeStatefulSetController(initialObjects ...runtime.Object) (*StatefulSe
ssc.podListerSynced = alwaysReady
ssc.setListerSynced = alwaysReady
recorder := record.NewFakeRecorder(10)
inplaceControl := inplaceupdate.NewForInformer(informerFactory.Core().V1().Pods(), apps.ControllerRevisionHashLabelKey)
inplaceControl := inplaceupdate.NewForInformer(informerFactory.Core().V1().Pods(), revisionadapter.NewDefaultImpl())
lifecycleControl := lifecycle.NewForInformer(informerFactory.Core().V1().Pods())
ssc.control = NewDefaultStatefulSetControl(fpc, inplaceControl, lifecycleControl, ssu, ssh, recorder)
@ -794,7 +794,7 @@ func NewStatefulSetController(
podInformer.Lister(),
pvcInformer.Lister(),
recorder),
inplaceupdate.NewForTypedClient(kubeClient, apps.ControllerRevisionHashLabelKey),
inplaceupdate.NewForTypedClient(kubeClient, revisionadapter.NewDefaultImpl()),
lifecycle.NewForTypedClient(kubeClient),
NewRealStatefulSetStatusUpdater(kruiseClient, setInformer.Lister()),
history.NewHistory(kubeClient, revInformer.Lister()),

10
pkg/features/kruise_features.go
View File

@ -33,14 +33,14 @@ const (
// PodWebhook enables webhook for Pods creations. This is also related to SidecarSet.
PodWebhook featuregate.Feature = "PodWebhook"
// CloneSetHashOnlyRevisionName enables CloneSet controller only set revision hash name to pod.
CloneSetHashOnlyRevisionName featuregate.Feature = "CloneSetHashOnlyRevisionName"
// CloneSetShortHash enables CloneSet controller only set revision hash name to pod label.
CloneSetShortHash featuregate.Feature = "CloneSetShortHash"
)
var defaultFeatureGates = map[featuregate.Feature]featuregate.FeatureSpec{
PodWebhook: {Default: true, PreRelease: featuregate.Beta},
KruiseDaemon: {Default: true, PreRelease: featuregate.Beta},
CloneSetHashOnlyRevisionName: {Default: false, PreRelease: featuregate.Beta},
PodWebhook: {Default: true, PreRelease: featuregate.Beta},
KruiseDaemon: {Default: true, PreRelease: featuregate.Beta},
CloneSetShortHash: {Default: false, PreRelease: featuregate.Alpha},
}
func init() {

11
pkg/util/expectations/update_expectations.go
View File

@ -20,6 +20,7 @@ import (
"sync"
"time"
"github.com/openkruise/kruise/pkg/util/revisionadapter"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/util/sets"
)
@ -34,10 +35,10 @@ type UpdateExpectations interface {
}
// NewUpdateExpectations returns a common UpdateExpectations.
func NewUpdateExpectations(getRevision func(string, metav1.Object) string) UpdateExpectations {
func NewUpdateExpectations(revisionAdapter revisionadapter.Interface) UpdateExpectations {
return &realUpdateExpectations{
controllerCache: make(map[string]*realControllerUpdateExpectations),
getRevision: getRevision,
revisionAdapter: revisionAdapter,
}
}
@ -45,8 +46,8 @@ type realUpdateExpectations struct {
sync.Mutex
// key: parent key, workload namespace/name
controllerCache map[string]*realControllerUpdateExpectations
// how to get pod revision
getRevision func(string, metav1.Object) string
// the impl of interface
revisionAdapter revisionadapter.Interface
}
type realControllerUpdateExpectations struct {
@ -82,7 +83,7 @@ func (r *realUpdateExpectations) ObserveUpdated(controllerKey, revision string,
return
}
if expectations.revision == revision && expectations.objsUpdated.Has(getKey(obj)) && r.getRevision(controllerKey, obj) == revision {
if expectations.revision == revision && expectations.objsUpdated.Has(getKey(obj)) && r.revisionAdapter.EqualToRevisionHash(controllerKey, obj, revision) {
expectations.objsUpdated.Delete(getKey(obj))
}

15
pkg/util/expectations/update_expectations_test.go
View File

@ -23,6 +23,19 @@ import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type fakeRevisionAdapterImpl struct{}
func (r *fakeRevisionAdapterImpl) EqualToRevisionHash(_ string, obj metav1.Object, hash string) bool {
return obj.GetLabels()["revision"] == hash
}
func (r *fakeRevisionAdapterImpl) WriteRevisionHash(obj metav1.Object, hash string) {
if obj.GetLabels() == nil {
obj.SetLabels(make(map[string]string, 1))
}
obj.GetLabels()["revision"] = hash
}
func TestUpdate(t *testing.T) {
controllerKey := "default/controller-test"
revisions := []string{"rev-0", "rev-1"}
@ -35,7 +48,7 @@ func TestUpdate(t *testing.T) {
},
},
}
c := NewUpdateExpectations(func(controllerKey string, p metav1.Object) string { return p.GetLabels()["revision"] })
c := NewUpdateExpectations(&fakeRevisionAdapterImpl{})
// no pod in cache
if satisfied, _, _ := c.SatisfiedExpectations(controllerKey, revisions[0]); !satisfied {

42
pkg/util/inplaceupdate/inplace_utils.go
View File

@ -24,6 +24,8 @@ import (
"time"
appspub "github.com/openkruise/kruise/apis/apps/pub"
"github.com/openkruise/kruise/pkg/util/podadapter"
"github.com/openkruise/kruise/pkg/util/revisionadapter"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -32,8 +34,6 @@ import (
"k8s.io/client-go/util/retry"
"k8s.io/klog"
"sigs.k8s.io/controller-runtime/pkg/client"
"github.com/openkruise/kruise/pkg/util/podadapter"
)
var inPlaceUpdatePatchRexp = regexp.MustCompile("^/spec/containers/([0-9]+)/image$")
@ -70,27 +70,27 @@ type UpdateSpec struct {
}
type realControl struct {
adp podadapter.Adapter
revisionKey string
podAdapter podadapter.Adapter
revisionAdapter revisionadapter.Interface
// just for test
now func() metav1.Time
}
func New(c client.Client, revisionKey string) Interface {
return &realControl{adp: &podadapter.AdapterRuntimeClient{Client: c}, revisionKey: revisionKey, now: metav1.Now}
func New(c client.Client, revisionAdapter revisionadapter.Interface) Interface {
return &realControl{podAdapter: &podadapter.AdapterRuntimeClient{Client: c}, revisionAdapter: revisionAdapter, now: metav1.Now}
}
func NewForTypedClient(c clientset.Interface, revisionKey string) Interface {
return &realControl{adp: &podadapter.AdapterTypedClient{Client: c}, revisionKey: revisionKey, now: metav1.Now}
func NewForTypedClient(c clientset.Interface, revisionAdapter revisionadapter.Interface) Interface {
return &realControl{podAdapter: &podadapter.AdapterTypedClient{Client: c}, revisionAdapter: revisionAdapter, now: metav1.Now}
}
func NewForInformer(informer coreinformers.PodInformer, revisionKey string) Interface {
return &realControl{adp: &podadapter.AdapterInformer{PodInformer: informer}, revisionKey: revisionKey, now: metav1.Now}
func NewForInformer(informer coreinformers.PodInformer, revisionAdapter revisionadapter.Interface) Interface {
return &realControl{podAdapter: &podadapter.AdapterInformer{PodInformer: informer}, revisionAdapter: revisionAdapter, now: metav1.Now}
}
func NewForTest(c client.Client, revisionKey string, now func() metav1.Time) Interface {
return &realControl{adp: &podadapter.AdapterRuntimeClient{Client: c}, revisionKey: revisionKey, now: now}
func NewForTest(c client.Client, revisionAdapter revisionadapter.Interface, now func() metav1.Time) Interface {
return &realControl{podAdapter: &podadapter.AdapterRuntimeClient{Client: c}, revisionAdapter: revisionAdapter, now: now}
}
func (c *realControl) Refresh(pod *v1.Pod, opts *UpdateOptions) RefreshResult {
@ -138,7 +138,7 @@ func (c *realControl) refreshCondition(pod *v1.Pod, opts *UpdateOptions) error {
func (c *realControl) updateCondition(pod *v1.Pod, condition v1.PodCondition) error {
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
clone, err := c.adp.GetPod(pod.Namespace, pod.Name)
clone, err := c.podAdapter.GetPod(pod.Namespace, pod.Name)
if err != nil {
return err
}
@ -148,14 +148,14 @@ func (c *realControl) updateCondition(pod *v1.Pod, condition v1.PodCondition) er
if condition.Status == v1.ConditionFalse {
updatePodReadyCondition(clone)
}
return c.adp.UpdatePodStatus(clone)
return c.podAdapter.UpdatePodStatus(clone)
})
}
func (c *realControl) finishGracePeriod(pod *v1.Pod, opts *UpdateOptions) (time.Duration, error) {
var delayDuration time.Duration
err := retry.RetryOnConflict(retry.DefaultBackoff, func() error {
clone, err := c.adp.GetPod(pod.Namespace, pod.Name)
clone, err := c.podAdapter.GetPod(pod.Namespace, pod.Name)
if err != nil {
return err
}
@ -179,7 +179,7 @@ func (c *realControl) finishGracePeriod(pod *v1.Pod, opts *UpdateOptions) (time.
return nil
}
if clone.Labels[c.revisionKey] != spec.Revision {
if !c.revisionAdapter.EqualToRevisionHash("", clone, spec.Revision) {
// If revision-hash has changed, just drop this GracePeriodSpec and go through the normal update process again.
appspub.RemoveInPlaceUpdateGrace(clone)
} else {
@ -194,7 +194,7 @@ func (c *realControl) finishGracePeriod(pod *v1.Pod, opts *UpdateOptions) (time.
appspub.RemoveInPlaceUpdateGrace(clone)
}
return c.adp.UpdatePod(clone)
return c.podAdapter.UpdatePod(clone)
})
return delayDuration, err
@ -243,15 +243,13 @@ func (c *realControl) Update(pod *v1.Pod, oldRevision, newRevision *apps.Control
func (c *realControl) updatePodInPlace(pod *v1.Pod, spec *UpdateSpec, opts *UpdateOptions) error {
return retry.RetryOnConflict(retry.DefaultBackoff, func() error {
clone, err := c.adp.GetPod(pod.Namespace, pod.Name)
clone, err := c.podAdapter.GetPod(pod.Namespace, pod.Name)
if err != nil {
return err
}
// update new revision
if c.revisionKey != "" {
clone.Labels[c.revisionKey] = spec.Revision
}
c.revisionAdapter.WriteRevisionHash(clone, spec.Revision)
if clone.Annotations == nil {
clone.Annotations = map[string]string{}
}
@ -286,7 +284,7 @@ func (c *realControl) updatePodInPlace(pod *v1.Pod, spec *UpdateSpec, opts *Upda
clone.Annotations[appspub.InPlaceUpdateGraceKey] = string(inPlaceUpdateSpecJSON)
}
return c.adp.UpdatePod(clone)
return c.podAdapter.UpdatePod(clone)
})
}

3
pkg/util/inplaceupdate/inplace_utils_test.go
View File

@ -25,6 +25,7 @@ import (
appspub "github.com/openkruise/kruise/apis/apps/pub"
"github.com/openkruise/kruise/pkg/util"
"github.com/openkruise/kruise/pkg/util/revisionadapter"
apps "k8s.io/api/apps/v1"
v1 "k8s.io/api/core/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@ -433,7 +434,7 @@ func TestRefresh(t *testing.T) {
testCase.expectedPod.Kind = "Pod"
cli := fake.NewFakeClient(testCase.pod)
ctrl := NewForTest(cli, apps.ControllerRevisionHashLabelKey, func() metav1.Time { return aHourAgo })
ctrl := NewForTest(cli, revisionadapter.NewDefaultImpl(), func() metav1.Time { return aHourAgo })
if res := ctrl.Refresh(testCase.pod, nil); res.RefreshErr != nil {
t.Fatalf("failed to update condition: %v", res.RefreshErr)
}

44
pkg/util/revisionadapter/revision_adapter.go Normal file
View File

@ -0,0 +1,44 @@
/*
Copyright 2021 The Kruise Authors.
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package revisionadapter
import (
apps "k8s.io/api/apps/v1"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
type Interface interface {
EqualToRevisionHash(controllerKey string, obj metav1.Object, hash string) bool
WriteRevisionHash(obj metav1.Object, hash string)
}
func NewDefaultImpl() Interface {
return &defaultImpl{}
}
type defaultImpl struct{}
func (r *defaultImpl) EqualToRevisionHash(_ string, obj metav1.Object, hash string) bool {
return obj.GetLabels()[apps.ControllerRevisionHashLabelKey] == hash
}
func (r *defaultImpl) WriteRevisionHash(obj metav1.Object, hash string) {
if obj.GetLabels() == nil {
obj.SetLabels(make(map[string]string, 1))
}
obj.GetLabels()[apps.ControllerRevisionHashLabelKey] = hash
}