mirror of https://github.com/openkruise/kruise.git
836 lines
30 KiB
Go
836 lines
30 KiB
Go
/*
|
|
Copyright 2019 The Kruise Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package cloneset
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
"fmt"
|
|
"reflect"
|
|
"strings"
|
|
"testing"
|
|
"time"
|
|
|
|
"github.com/openkruise/kruise/apis/apps/defaults"
|
|
appsv1alpha1 "github.com/openkruise/kruise/apis/apps/v1alpha1"
|
|
clonesetutils "github.com/openkruise/kruise/pkg/controller/cloneset/utils"
|
|
"github.com/openkruise/kruise/pkg/features"
|
|
"github.com/openkruise/kruise/pkg/util"
|
|
utilclient "github.com/openkruise/kruise/pkg/util/client"
|
|
utilfeature "github.com/openkruise/kruise/pkg/util/feature"
|
|
"github.com/openkruise/kruise/pkg/util/fieldindex"
|
|
historyutil "github.com/openkruise/kruise/pkg/util/history"
|
|
"github.com/openkruise/kruise/pkg/util/volumeclaimtemplate"
|
|
|
|
"github.com/onsi/gomega"
|
|
v1 "k8s.io/api/core/v1"
|
|
apierrors "k8s.io/apimachinery/pkg/api/errors"
|
|
"k8s.io/apimachinery/pkg/api/resource"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/fields"
|
|
"k8s.io/apimachinery/pkg/runtime"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/apimachinery/pkg/util/sets"
|
|
"k8s.io/client-go/util/retry"
|
|
"k8s.io/kubernetes/pkg/apis/apps"
|
|
"k8s.io/kubernetes/pkg/controller/history"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
"sigs.k8s.io/controller-runtime/pkg/client/fake"
|
|
"sigs.k8s.io/controller-runtime/pkg/manager"
|
|
metricsserver "sigs.k8s.io/controller-runtime/pkg/metrics/server"
|
|
"sigs.k8s.io/controller-runtime/pkg/reconcile"
|
|
)
|
|
|
|
var c client.Client
|
|
|
|
var (
|
|
expectedRequest = reconcile.Request{NamespacedName: types.NamespacedName{Name: "foo", Namespace: "default"}}
|
|
images = []string{"nginx:1.9.1", "nginx:1.9.2", "nginx:1.9.3", "nginx:1.9.2-alphine", "nginx:1.9.3-alphine"}
|
|
clonesetUID = "123"
|
|
productionLabel = map[string]string{"type": "production"}
|
|
nilLabel = map[string]string{}
|
|
)
|
|
|
|
//const timeout = time.Second * 5
|
|
|
|
func TestReconcile(t *testing.T) {
|
|
g := gomega.NewGomegaWithT(t)
|
|
instance := &appsv1alpha1.CloneSet{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "foo", Namespace: "default"},
|
|
Spec: appsv1alpha1.CloneSetSpec{
|
|
Replicas: getInt32(1),
|
|
Selector: &metav1.LabelSelector{MatchLabels: map[string]string{"foo": "bar"}},
|
|
Template: v1.PodTemplateSpec{
|
|
ObjectMeta: metav1.ObjectMeta{Labels: map[string]string{"foo": "bar"}},
|
|
Spec: v1.PodSpec{
|
|
Containers: []v1.Container{{Name: "nginx", Image: images[0]}},
|
|
},
|
|
},
|
|
VolumeClaimTemplates: []v1.PersistentVolumeClaim{
|
|
{
|
|
ObjectMeta: metav1.ObjectMeta{Name: "foo-vol"},
|
|
Spec: v1.PersistentVolumeClaimSpec{
|
|
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
|
|
Resources: v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}},
|
|
},
|
|
},
|
|
},
|
|
},
|
|
}
|
|
|
|
// Setup the Manager and Controller. Wrap the Controller Reconcile function so it writes each request to a
|
|
// channel when it is finished.
|
|
mgr, err := manager.New(cfg, manager.Options{Metrics: metricsserver.Options{BindAddress: "0"}})
|
|
_ = fieldindex.RegisterFieldIndexes(mgr.GetCache())
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
c = utilclient.NewClientFromManager(mgr, "test-cloneset-controller")
|
|
|
|
//recFn, requests := SetupTestReconcile(newReconciler(mgr))
|
|
g.Expect(add(mgr, newReconciler(mgr))).NotTo(gomega.HaveOccurred())
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
mgrStopped := StartTestManager(ctx, mgr, g)
|
|
|
|
defer func() {
|
|
cancel()
|
|
mgrStopped.Wait()
|
|
}()
|
|
|
|
// Create an orphan pod
|
|
orphanPod := v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Namespace: "default",
|
|
Name: "foo-abcde",
|
|
Labels: map[string]string{"foo": "bar"},
|
|
},
|
|
Spec: v1.PodSpec{
|
|
Containers: []v1.Container{{Name: "nginx", Image: "nginx:apline"}},
|
|
},
|
|
}
|
|
err = c.Create(context.TODO(), &orphanPod)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
|
|
// Create the CloneSet object and expect the Reconcile
|
|
defaults.SetDefaultsCloneSet(instance, true)
|
|
err = c.Create(context.TODO(), instance)
|
|
// The instance object may not be a valid object because it might be missing some required fields.
|
|
// Please modify the instance object by adding required fields and then remove the following if statement.
|
|
if apierrors.IsInvalid(err) {
|
|
t.Logf("failed to create object, got an invalid object error: %v", err)
|
|
return
|
|
}
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
defer c.Delete(context.TODO(), instance)
|
|
|
|
// Check 1 pod and 1 pvc have been created
|
|
pods, pvcs := checkInstances(g, instance, 1, 1)
|
|
g.Expect(pods[0].Labels[appsv1alpha1.CloneSetInstanceID]).Should(gomega.Equal(pvcs[0].Labels[appsv1alpha1.CloneSetInstanceID]))
|
|
|
|
// Test for pods scale
|
|
testScale(g, instance)
|
|
|
|
// Enable the CloneSetShortHash feature-gate
|
|
utilfeature.DefaultMutableFeatureGate.Set(fmt.Sprintf("%s=true", features.CloneSetShortHash))
|
|
|
|
// Get latest cloneset
|
|
err = c.Get(context.TODO(), expectedRequest.NamespacedName, instance)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
|
|
// Test for pods update
|
|
testUpdate(g, instance)
|
|
|
|
testVCTWhenFG := func(enable bool) {
|
|
defer utilfeature.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecreatePodWhenChangeVCTInCloneSetGate, enable)()
|
|
|
|
// Test for pods update with volume claim changed
|
|
testUpdateVolumeClaimTemplates(t, g, instance, nil)
|
|
|
|
// Clean vct hash before each case => lack vct hash -> keep in-place update
|
|
testUpdateVolumeClaimTemplates(t, g, instance, cleanVCTHashInRevisions)
|
|
}
|
|
testVCTWhenFG(true)
|
|
testVCTWhenFG(false)
|
|
|
|
defer utilfeature.SetFeatureGateDuringTest(t, utilfeature.DefaultFeatureGate, features.RecreatePodWhenChangeVCTInCloneSetGate, true)()
|
|
// Test case with history revision
|
|
testUpdateVolumeClaimTemplates2(t, g, instance)
|
|
}
|
|
|
|
func TestClaimPods(t *testing.T) {
|
|
g := gomega.NewGomegaWithT(t)
|
|
|
|
scheme := runtime.NewScheme()
|
|
appsv1alpha1.AddToScheme(scheme)
|
|
v1.AddToScheme(scheme)
|
|
|
|
instance := &appsv1alpha1.CloneSet{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "foo",
|
|
Namespace: metav1.NamespaceDefault,
|
|
UID: types.UID(clonesetUID),
|
|
},
|
|
Spec: appsv1alpha1.CloneSetSpec{
|
|
Selector: &metav1.LabelSelector{MatchLabels: productionLabel},
|
|
},
|
|
}
|
|
defaults.SetDefaultsCloneSet(instance, true)
|
|
|
|
type test struct {
|
|
name string
|
|
pods []*v1.Pod
|
|
claimed []*v1.Pod
|
|
}
|
|
var tests = []test{
|
|
{
|
|
name: "Controller releases claimed pods when selector doesn't match",
|
|
pods: []*v1.Pod{newPod("pod1", productionLabel, instance), newPod("pod2", nilLabel, instance)},
|
|
claimed: []*v1.Pod{newPod("pod1", productionLabel, instance)},
|
|
},
|
|
{
|
|
name: "Claim pods with correct label",
|
|
pods: []*v1.Pod{newPod("pod3", productionLabel, nil), newPod("pod4", nilLabel, nil)},
|
|
claimed: []*v1.Pod{newPod("pod3", productionLabel, nil)},
|
|
},
|
|
}
|
|
|
|
for _, test := range tests {
|
|
initObjs := []client.Object{instance}
|
|
for i := range test.pods {
|
|
initObjs = append(initObjs, test.pods[i])
|
|
}
|
|
fakeClient := fake.NewClientBuilder().WithScheme(scheme).WithObjects(initObjs...).Build()
|
|
|
|
reconciler := &ReconcileCloneSet{
|
|
Client: fakeClient,
|
|
scheme: scheme,
|
|
}
|
|
|
|
claimed, err := reconciler.claimPods(instance, test.pods)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
if !reflect.DeepEqual(podToStringSlice(test.claimed), podToStringSlice(claimed)) {
|
|
t.Errorf("Test case `%s`, claimed wrong pods. Expected %v, got %v", test.name, podToStringSlice(test.claimed), podToStringSlice(claimed))
|
|
}
|
|
}
|
|
}
|
|
|
|
func newPod(podName string, label map[string]string, owner metav1.Object) *v1.Pod {
|
|
pod := &v1.Pod{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: podName,
|
|
Labels: label,
|
|
Namespace: metav1.NamespaceDefault,
|
|
},
|
|
Spec: v1.PodSpec{
|
|
Containers: []v1.Container{
|
|
{
|
|
Name: "test",
|
|
Image: "foo/bar",
|
|
},
|
|
},
|
|
},
|
|
}
|
|
if owner != nil {
|
|
pod.OwnerReferences = []metav1.OwnerReference{*metav1.NewControllerRef(owner, apps.SchemeGroupVersion.WithKind("Fake"))}
|
|
}
|
|
return pod
|
|
}
|
|
|
|
func podToStringSlice(pods []*v1.Pod) []string {
|
|
var names []string
|
|
for _, pod := range pods {
|
|
names = append(names, pod.Name)
|
|
}
|
|
return names
|
|
}
|
|
|
|
func testScale(g *gomega.GomegaWithT, instance *appsv1alpha1.CloneSet) {
|
|
// Create orphan and owned pvcs
|
|
pvc1 := v1.PersistentVolumeClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "foo-vol-foo-hlfn7",
|
|
Namespace: "default",
|
|
Labels: map[string]string{
|
|
"foo": "bar",
|
|
appsv1alpha1.CloneSetInstanceID: "hlfn7",
|
|
},
|
|
},
|
|
Spec: v1.PersistentVolumeClaimSpec{
|
|
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
|
|
Resources: v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}},
|
|
},
|
|
}
|
|
err := c.Create(context.TODO(), &pvc1)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
pvc2 := v1.PersistentVolumeClaim{
|
|
ObjectMeta: metav1.ObjectMeta{
|
|
Name: "foo-vol-foo-xub0a",
|
|
Namespace: "default",
|
|
Labels: map[string]string{
|
|
"foo": "bar",
|
|
appsv1alpha1.CloneSetInstanceID: "xub0a",
|
|
},
|
|
OwnerReferences: []metav1.OwnerReference{{
|
|
APIVersion: clonesetutils.ControllerKind.GroupVersion().String(),
|
|
Kind: clonesetutils.ControllerKind.Kind,
|
|
Name: instance.Name,
|
|
UID: instance.UID,
|
|
Controller: func() *bool { v := true; return &v }(),
|
|
BlockOwnerDeletion: func() *bool { v := true; return &v }(),
|
|
}},
|
|
},
|
|
Spec: v1.PersistentVolumeClaimSpec{
|
|
AccessModes: []v1.PersistentVolumeAccessMode{v1.ReadWriteOnce},
|
|
Resources: v1.VolumeResourceRequirements{Requests: v1.ResourceList{v1.ResourceStorage: resource.MustParse("1Mi")}},
|
|
},
|
|
}
|
|
err = c.Create(context.TODO(), &pvc2)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
|
|
// Check 1 pod and 2 pvc have been created
|
|
pods, pvcs := checkInstances(g, instance, 1, 2)
|
|
g.Expect([]string{
|
|
pods[0].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pvc2.Labels[appsv1alpha1.CloneSetInstanceID],
|
|
}).Should(gomega.ConsistOf(
|
|
pvcs[0].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pvcs[1].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
))
|
|
|
|
// Add replicas to 5, should reuse
|
|
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
cs := appsv1alpha1.CloneSet{}
|
|
if err := c.Get(context.TODO(), expectedRequest.NamespacedName, &cs); err != nil {
|
|
return err
|
|
}
|
|
cs.Spec.Replicas = getInt32(5)
|
|
return c.Update(context.TODO(), &cs)
|
|
})
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
|
|
// Check 5 pod and 5 pvc have been created
|
|
pods, pvcs = checkInstances(g, instance, 5, 5)
|
|
g.Expect([]string{
|
|
pods[0].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pods[1].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pods[2].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pods[3].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pods[4].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
}).Should(gomega.ConsistOf(
|
|
pvcs[0].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pvcs[1].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pvcs[2].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pvcs[3].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pvcs[4].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
))
|
|
g.Expect(pvc1.Labels[appsv1alpha1.CloneSetInstanceID]).Should(gomega.And(
|
|
gomega.Not(gomega.Equal(pods[0].Labels[appsv1alpha1.CloneSetInstanceID])),
|
|
gomega.Not(gomega.Equal(pods[1].Labels[appsv1alpha1.CloneSetInstanceID])),
|
|
gomega.Not(gomega.Equal(pods[2].Labels[appsv1alpha1.CloneSetInstanceID])),
|
|
gomega.Not(gomega.Equal(pods[3].Labels[appsv1alpha1.CloneSetInstanceID])),
|
|
gomega.Not(gomega.Equal(pods[4].Labels[appsv1alpha1.CloneSetInstanceID])),
|
|
))
|
|
|
|
// Specified delete instance 'xub0a'
|
|
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
cs := appsv1alpha1.CloneSet{}
|
|
if err := c.Get(context.TODO(), expectedRequest.NamespacedName, &cs); err != nil {
|
|
return err
|
|
}
|
|
cs.Spec.Replicas = getInt32(4)
|
|
cs.Spec.ScaleStrategy.PodsToDelete = append(cs.Spec.ScaleStrategy.PodsToDelete, "foo-xub0a")
|
|
return c.Update(context.TODO(), &cs)
|
|
})
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
|
|
// Should not delete yet because of maxUnavailable
|
|
pods, _ = checkInstances(g, instance, 5, 5)
|
|
|
|
// Update 4 Pods to ready
|
|
err = updatePodsStatus(pods[:4], v1.PodStatus{Conditions: []v1.PodCondition{{Type: v1.PodReady, Status: v1.ConditionTrue}}, Phase: v1.PodRunning})
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
|
|
// Then 'xub0a' should be deleted
|
|
pods, pvcs = checkInstances(g, instance, 4, 4)
|
|
g.Expect([]string{
|
|
pods[0].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pods[1].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pods[2].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pods[3].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
}).Should(gomega.ConsistOf(
|
|
pvcs[0].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pvcs[1].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pvcs[2].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
pvcs[3].Labels[appsv1alpha1.CloneSetInstanceID],
|
|
))
|
|
}
|
|
|
|
type testCloneSetHookFn = func(g *gomega.GomegaWithT, instance *appsv1alpha1.CloneSet)
|
|
|
|
func updateCloneSetInTwoStep(description string, t *testing.T, g *gomega.GomegaWithT, instance *appsv1alpha1.CloneSet,
|
|
image, reqStr string, updateType appsv1alpha1.CloneSetUpdateStrategyType,
|
|
partitionInStep1, updatedInStep1, expectedSameInStep1, replica int,
|
|
pods0 []*v1.Pod, pvcs0 []*v1.PersistentVolumeClaim, preHook, postHook testCloneSetHookFn,
|
|
volumeSizeCheck bool, fnsInStep1 ...func(*appsv1alpha1.CloneSet)) ([]*v1.Pod, []*v1.PersistentVolumeClaim) {
|
|
|
|
t.Logf("updateCloneSetInTwoStep with case '%v' start", description)
|
|
defer func() {
|
|
t.Logf("updateCloneSetInTwoStep with case '%v' end", description)
|
|
}()
|
|
if preHook != nil {
|
|
preHook(g, instance)
|
|
}
|
|
maxUnavailable := intstr.FromString("100%")
|
|
|
|
res := resource.MustParse(reqStr)
|
|
//step1
|
|
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
cs := appsv1alpha1.CloneSet{}
|
|
if err := c.Get(context.TODO(), expectedRequest.NamespacedName, &cs); err != nil {
|
|
return err
|
|
}
|
|
cs.Spec.Template.Spec.Containers[0].Image = image
|
|
cs.Spec.VolumeClaimTemplates[0].Spec.Resources.Requests[v1.ResourceStorage] = res
|
|
cs.Spec.UpdateStrategy = appsv1alpha1.CloneSetUpdateStrategy{
|
|
Type: updateType,
|
|
Partition: util.GetIntOrStrPointer(intstr.FromInt(partitionInStep1)),
|
|
MaxUnavailable: &maxUnavailable,
|
|
}
|
|
for _, fn := range fnsInStep1 {
|
|
fn(&cs)
|
|
}
|
|
|
|
return c.Update(context.TODO(), &cs)
|
|
})
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
checkStatus(g, int32(replica), int32(updatedInStep1))
|
|
|
|
pods1, pvcs1 := checkInstances(g, instance, replica, replica)
|
|
samePodNames := getPodNames(pods0).Intersection(getPodNames(pods1))
|
|
samePVCNames := getPVCNames(pvcs0).Intersection(getPVCNames(pvcs1))
|
|
g.Expect(samePodNames.Len()).Should(gomega.Equal(expectedSameInStep1))
|
|
g.Expect(samePVCNames.Len()).Should(gomega.Equal(expectedSameInStep1))
|
|
if expectedSameInStep1 == 1 {
|
|
g.Expect(strings.HasSuffix(samePVCNames.List()[0], samePodNames.List()[0])).Should(gomega.BeTrue())
|
|
}
|
|
if partitionInStep1 == 0 || updatedInStep1 == replica {
|
|
return pods1, pvcs1
|
|
}
|
|
|
|
// wait all instances ok
|
|
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
cs := appsv1alpha1.CloneSet{}
|
|
if err := c.Get(context.TODO(), expectedRequest.NamespacedName, &cs); err != nil {
|
|
return err
|
|
}
|
|
cs.Spec.UpdateStrategy = appsv1alpha1.CloneSetUpdateStrategy{
|
|
Type: updateType,
|
|
MaxUnavailable: &maxUnavailable,
|
|
}
|
|
return c.Update(context.TODO(), &cs)
|
|
})
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
checkStatus(g, int32(replica), int32(replica))
|
|
pods1, pvcs1 = checkInstances(g, instance, replica, replica)
|
|
if volumeSizeCheck {
|
|
for _, pvc := range pvcs1 {
|
|
if !pvc.Spec.Resources.Requests.Storage().Equal(res) {
|
|
t.Errorf("pvc %v's storage is not equal to %v, actual %v", pvc.Name, reqStr, pvc.Spec.Resources.Requests.Storage())
|
|
}
|
|
}
|
|
}
|
|
|
|
if postHook != nil {
|
|
postHook(g, instance)
|
|
}
|
|
return pods1, pvcs1
|
|
}
|
|
|
|
func cleanVCTHashInRevisions(g *gomega.GomegaWithT, instance *appsv1alpha1.CloneSet) {
|
|
controllerHistory := historyutil.NewHistory(c)
|
|
selector, err := util.ValidatedLabelSelectorAsSelector(instance.Spec.Selector)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
|
|
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
revisions, err := controllerHistory.ListControllerRevisions(instance, selector)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
history.SortControllerRevisions(revisions)
|
|
maxRevision := clonesetutils.NextRevision(revisions)
|
|
for i, revision := range revisions {
|
|
if len(revision.Annotations) > 0 {
|
|
revision.Annotations["m-edited"] = ""
|
|
delete(revision.Annotations, volumeclaimtemplate.HashAnnotation)
|
|
}
|
|
revision, err = controllerHistory.UpdateControllerRevision(revision, maxRevision+int64(i))
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
g.Expect("").Should(gomega.Equal(revision.Annotations[volumeclaimtemplate.HashAnnotation]))
|
|
}
|
|
|
|
revisions, err = controllerHistory.ListControllerRevisions(instance, selector)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
history.SortControllerRevisions(revisions)
|
|
for _, revision := range revisions {
|
|
val, exist := revision.Annotations[volumeclaimtemplate.HashAnnotation]
|
|
if val != "" {
|
|
fmt.Printf("unexpected hash value: %v in revision %v(%v)\n", val, revision.Name, revision.Revision)
|
|
}
|
|
if exist {
|
|
return apierrors.NewConflict(schema.GroupResource{
|
|
Group: revision.GroupVersionKind().Group,
|
|
Resource: revision.GroupVersionKind().Kind},
|
|
revision.Name, errors.New("changed by others, retry"))
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
}
|
|
|
|
func testUpdateVolumeClaimTemplates(t *testing.T, g *gomega.GomegaWithT, instance *appsv1alpha1.CloneSet, preHook testCloneSetHookFn) {
|
|
missingVCTHash, volumeSizeCheck := false, true
|
|
replicas := 4
|
|
var postHook testCloneSetHookFn = nil
|
|
if preHook != nil || !utilfeature.DefaultFeatureGate.Enabled(features.RecreatePodWhenChangeVCTInCloneSetGate) {
|
|
// when vct hash missing, recreate pod => in-place update pod
|
|
missingVCTHash = true
|
|
volumeSizeCheck = false
|
|
}
|
|
|
|
// format clone set
|
|
pods0, pvcs0 := checkInstances(g, instance, 4, 4)
|
|
fn := func(cs *appsv1alpha1.CloneSet) {
|
|
// 避免因为回退到历史版本导致的 vcTemplates 被忽略
|
|
cs.Spec.Template.Spec.Containers[0].Env = append(cs.Spec.Template.Spec.Containers[0].Env, v1.EnvVar{
|
|
Name: fmt.Sprintf("test-%v", len(cs.Spec.Template.Spec.Containers[0].Env)),
|
|
Value: "add_test4",
|
|
})
|
|
}
|
|
expectedSameInStep1 := 1
|
|
pods0, pvcs0 = updateCloneSetInTwoStep("format clone set", t, g, instance, images[0], "12Mi", appsv1alpha1.RecreateCloneSetUpdateStrategyType,
|
|
1, 3, expectedSameInStep1, replicas, pods0, pvcs0, preHook, postHook, volumeSizeCheck, fn)
|
|
|
|
// inplace update strategy however change volumeClaimTemplate size and image
|
|
// => recreate
|
|
// when history revision vct hash is empty => inplace update
|
|
expectedSameInStep1 = 3
|
|
if missingVCTHash {
|
|
expectedSameInStep1 = 4
|
|
}
|
|
pods0, pvcs0 = updateCloneSetInTwoStep("inplace update strategy however change volumeClaimTemplate size and image", t, g, instance, images[1], "30Mi", appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType,
|
|
3, 1, expectedSameInStep1, replicas, pods0, pvcs0, preHook, postHook, volumeSizeCheck)
|
|
|
|
// inplace update only change image
|
|
// => inplace update
|
|
expectedSameInStep1 = 4
|
|
pods0, pvcs0 = updateCloneSetInTwoStep("inplace update only change image", t, g, instance, images[2], "30Mi", appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType,
|
|
2, 2, expectedSameInStep1, replicas, pods0, pvcs0, preHook, postHook, volumeSizeCheck)
|
|
|
|
// inplace update strategy however only change volumeClaimTemplate size
|
|
// => nothing changed
|
|
expectedSameInStep1 = 4
|
|
pods0, pvcs0 = updateCloneSetInTwoStep("inplace update strategy however only change volumeClaimTemplate size", t, g, instance, images[2], "50Mi", appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType,
|
|
2, 4, expectedSameInStep1, replicas, pods0, pvcs0, preHook, postHook, volumeSizeCheck)
|
|
|
|
// avoid effect of the previous case
|
|
time.Sleep(1 * time.Second)
|
|
// inplace update only change image however change volume claim template in previous
|
|
// => recreate
|
|
expectedSameInStep1 = 1
|
|
if missingVCTHash {
|
|
expectedSameInStep1 = 4
|
|
}
|
|
_, _ = updateCloneSetInTwoStep("inplace update only change image however change volume claim template in previous", t, g, instance, images[0], "50Mi", appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType,
|
|
1, 3, expectedSameInStep1, replicas, pods0, pvcs0, preHook, postHook, volumeSizeCheck)
|
|
}
|
|
|
|
// case with history revision
|
|
// 1. image 1, vctemplate size of 1MB.
|
|
// 2. Image 2, vcTemplate size of 1MB.
|
|
// 3. Image 3, vcTemplate size of 1MB. (in function testUpdate)
|
|
// 4. Image 4, vcTemplate size of 10MB => recreate.
|
|
// 5. Image 2, vcTemplate size of 10MB => (expected in-place upgrade)
|
|
// --- Although the history revision hash is consistent, the vct hash is not, update history revision id and hash value
|
|
// 6. Image 4, vcTemplate size of 10MB => in-place upgrade --- Update history revision.
|
|
// 7. Image 4, vcTemplate size of 100MB => (temporarily unchanged) --- Do not generate a new revision.
|
|
// 8. Image 5, vcTemplate size of 100MB => recreate.
|
|
func testUpdateVolumeClaimTemplates2(t *testing.T, g *gomega.GomegaWithT, instance *appsv1alpha1.CloneSet) {
|
|
pods0, pvcs0 := checkInstances(g, instance, 4, 4)
|
|
|
|
type args struct {
|
|
image, reqStr string
|
|
updateType appsv1alpha1.CloneSetUpdateStrategyType
|
|
partitionInStep1, updatedInStep1, expectedSameInStep1, replica int
|
|
preHook, postHook testCloneSetHookFn
|
|
volumeSizeCheck bool
|
|
fnsInStep1 []func(*appsv1alpha1.CloneSet)
|
|
}
|
|
|
|
tests := []struct {
|
|
name string
|
|
args args
|
|
}{
|
|
{
|
|
// step 3 ensure is ok
|
|
name: "testUpdateVolumeClaimTemplates2 step 3",
|
|
args: args{
|
|
image: images[2],
|
|
reqStr: "1Mi",
|
|
updateType: appsv1alpha1.RecreateCloneSetUpdateStrategyType,
|
|
partitionInStep1: 1,
|
|
updatedInStep1: 3, expectedSameInStep1: 1, replica: 4,
|
|
preHook: nil,
|
|
postHook: nil,
|
|
volumeSizeCheck: true,
|
|
},
|
|
},
|
|
{
|
|
// step 4 inplace update strategy however change volumeClaimTemplate size and image
|
|
// => recreate
|
|
name: "testUpdateVolumeClaimTemplates2 step 4",
|
|
args: args{
|
|
image: images[3],
|
|
reqStr: "10Mi",
|
|
updateType: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType,
|
|
partitionInStep1: 1,
|
|
updatedInStep1: 3, expectedSameInStep1: 1, replica: 4,
|
|
preHook: nil,
|
|
postHook: nil,
|
|
volumeSizeCheck: true,
|
|
},
|
|
},
|
|
{
|
|
// step 5 inplace update only change image
|
|
// => inplace update
|
|
name: "testUpdateVolumeClaimTemplates2 step 5",
|
|
args: args{
|
|
image: images[1],
|
|
reqStr: "10Mi",
|
|
updateType: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType,
|
|
partitionInStep1: 2, updatedInStep1: 2,
|
|
expectedSameInStep1: 4,
|
|
replica: 4,
|
|
preHook: nil,
|
|
postHook: nil,
|
|
volumeSizeCheck: true,
|
|
},
|
|
},
|
|
{
|
|
// step 6 inplace update strategy history image and vcTemplate
|
|
// => inplace update and update history revision
|
|
name: "testUpdateVolumeClaimTemplates2 step 6",
|
|
args: args{
|
|
image: images[3],
|
|
reqStr: "10Mi",
|
|
updateType: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType,
|
|
partitionInStep1: 2, updatedInStep1: 2,
|
|
expectedSameInStep1: 4,
|
|
replica: 4,
|
|
preHook: nil,
|
|
postHook: nil,
|
|
volumeSizeCheck: true,
|
|
},
|
|
},
|
|
{
|
|
// step 7 inplace update strategy however only change volumeClaimTemplate size
|
|
// => nothing changed
|
|
name: "testUpdateVolumeClaimTemplates2 step 7",
|
|
args: args{
|
|
image: images[3],
|
|
reqStr: "100Mi",
|
|
updateType: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType,
|
|
partitionInStep1: 2, updatedInStep1: 4,
|
|
expectedSameInStep1: 4,
|
|
replica: 4,
|
|
preHook: nil,
|
|
postHook: nil,
|
|
volumeSizeCheck: true,
|
|
},
|
|
},
|
|
{
|
|
// step 7 inplace update strategy however only change volumeClaimTemplate size
|
|
// => nothing changed
|
|
name: "testUpdateVolumeClaimTemplates2 step 8",
|
|
args: args{
|
|
image: images[4],
|
|
reqStr: "100Mi",
|
|
updateType: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType,
|
|
partitionInStep1: 1, updatedInStep1: 3,
|
|
expectedSameInStep1: 1,
|
|
replica: 4,
|
|
preHook: nil,
|
|
postHook: nil,
|
|
volumeSizeCheck: true,
|
|
},
|
|
},
|
|
}
|
|
|
|
for _, tt := range tests {
|
|
t.Run(tt.name, func(t *testing.T) {
|
|
arg := tt.args
|
|
replica := arg.replica
|
|
partitionInStep1 := arg.partitionInStep1
|
|
updatedInStep1 := arg.updatedInStep1
|
|
expectedSameInStep1 := arg.expectedSameInStep1
|
|
pods0, pvcs0 = updateCloneSetInTwoStep(tt.name, t, g, instance, arg.image, arg.reqStr, arg.updateType,
|
|
partitionInStep1, updatedInStep1, expectedSameInStep1, replica, pods0, pvcs0,
|
|
arg.preHook, arg.postHook, arg.volumeSizeCheck, arg.fnsInStep1...)
|
|
})
|
|
}
|
|
}
|
|
|
|
func testUpdate(g *gomega.GomegaWithT, instance *appsv1alpha1.CloneSet) {
|
|
// No way to test maxUnavailable, for this is a k8s cluster with only etcd and kube-apiserver
|
|
maxUnavailable := intstr.FromString("100%")
|
|
pods0, pvcs0 := checkInstances(g, instance, 4, 4)
|
|
|
|
// default to recreate update
|
|
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
cs := appsv1alpha1.CloneSet{}
|
|
if err := c.Get(context.TODO(), expectedRequest.NamespacedName, &cs); err != nil {
|
|
return err
|
|
}
|
|
cs.Spec.Template.Spec.Containers[0].Image = images[1]
|
|
cs.Spec.UpdateStrategy = appsv1alpha1.CloneSetUpdateStrategy{
|
|
Type: appsv1alpha1.RecreateCloneSetUpdateStrategyType,
|
|
Partition: util.GetIntOrStrPointer(intstr.FromInt(1)),
|
|
MaxUnavailable: &maxUnavailable,
|
|
}
|
|
return c.Update(context.TODO(), &cs)
|
|
})
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
|
|
checkStatus(g, 4, 3)
|
|
pods1, pvcs1 := checkInstances(g, instance, 4, 4)
|
|
samePodNames := getPodNames(pods0).Intersection(getPodNames(pods1))
|
|
samePVCNames := getPVCNames(pvcs0).Intersection(getPVCNames(pvcs1))
|
|
g.Expect(samePodNames.Len()).Should(gomega.Equal(1))
|
|
g.Expect(samePVCNames.Len()).Should(gomega.Equal(1))
|
|
g.Expect(strings.HasSuffix(samePVCNames.List()[0], samePodNames.List()[0])).Should(gomega.BeTrue())
|
|
|
|
// inplace update
|
|
err = retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
cs := appsv1alpha1.CloneSet{}
|
|
if err := c.Get(context.TODO(), expectedRequest.NamespacedName, &cs); err != nil {
|
|
return err
|
|
}
|
|
cs.Spec.Template.Spec.Containers[0].Image = images[2]
|
|
cs.Spec.UpdateStrategy = appsv1alpha1.CloneSetUpdateStrategy{
|
|
Type: appsv1alpha1.InPlaceIfPossibleCloneSetUpdateStrategyType,
|
|
Partition: util.GetIntOrStrPointer(intstr.FromInt(2)),
|
|
MaxUnavailable: &maxUnavailable,
|
|
}
|
|
return c.Update(context.TODO(), &cs)
|
|
})
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
|
|
checkStatus(g, 4, 2)
|
|
pods2, pvcs2 := checkInstances(g, instance, 4, 4)
|
|
samePodNames = getPodNames(pods1).Intersection(getPodNames(pods2))
|
|
samePVCNames = getPVCNames(pvcs1).Intersection(getPVCNames(pvcs2))
|
|
g.Expect(samePodNames.Len()).Should(gomega.Equal(4))
|
|
g.Expect(samePVCNames.Len()).Should(gomega.Equal(4))
|
|
}
|
|
|
|
func updatePodsStatus(pods []*v1.Pod, status v1.PodStatus) error {
|
|
for _, pod := range pods {
|
|
err := retry.RetryOnConflict(retry.DefaultRetry, func() error {
|
|
clone := &v1.Pod{}
|
|
if err := c.Get(context.TODO(), types.NamespacedName{Namespace: pod.Namespace, Name: pod.Name}, clone); err != nil {
|
|
return err
|
|
}
|
|
clone.Status = status
|
|
return c.Status().Update(context.TODO(), clone)
|
|
})
|
|
if err != nil {
|
|
if apierrors.IsNotFound(err) {
|
|
return nil
|
|
}
|
|
return err
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
func getInt32(i int32) *int32 {
|
|
return &i
|
|
}
|
|
|
|
func checkInstances(g *gomega.GomegaWithT, cs *appsv1alpha1.CloneSet, podNum int, pvcNum int) ([]*v1.Pod, []*v1.PersistentVolumeClaim) {
|
|
var pods []*v1.Pod
|
|
g.Eventually(func() int {
|
|
var err error
|
|
opts := &client.ListOptions{
|
|
Namespace: "default",
|
|
FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(cs.UID)}),
|
|
}
|
|
pods, _, err = clonesetutils.GetActiveAndInactivePods(c, opts)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
return len(pods)
|
|
}, time.Second*10, time.Millisecond*500).Should(gomega.Equal(podNum))
|
|
|
|
var pvcs []*v1.PersistentVolumeClaim
|
|
g.Eventually(func() int {
|
|
pvcList := v1.PersistentVolumeClaimList{}
|
|
opts := &client.ListOptions{
|
|
Namespace: cs.Namespace,
|
|
FieldSelector: fields.SelectorFromSet(fields.Set{fieldindex.IndexNameForOwnerRefUID: string(cs.UID)}),
|
|
}
|
|
err := c.List(context.TODO(), &pvcList, opts)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
pvcs = []*v1.PersistentVolumeClaim{}
|
|
for i, pvc := range pvcList.Items {
|
|
if pvc.DeletionTimestamp == nil {
|
|
pvcs = append(pvcs, &pvcList.Items[i])
|
|
}
|
|
}
|
|
return len(pvcs)
|
|
}, time.Second*3, time.Millisecond*500).Should(gomega.Equal(pvcNum))
|
|
|
|
return pods, pvcs
|
|
}
|
|
|
|
func checkStatus(g *gomega.GomegaWithT, total, updated int32) {
|
|
g.Eventually(func() []int32 {
|
|
cs := appsv1alpha1.CloneSet{}
|
|
err := c.Get(context.TODO(), expectedRequest.NamespacedName, &cs)
|
|
g.Expect(err).NotTo(gomega.HaveOccurred())
|
|
|
|
// When the status has not been synchronized yet, force to wait
|
|
if cs.Generation != cs.Status.ObservedGeneration {
|
|
return []int32{-1, -1}
|
|
}
|
|
return []int32{cs.Status.Replicas, cs.Status.UpdatedReplicas}
|
|
}, time.Second*10, time.Millisecond*500).Should(gomega.Equal([]int32{total, updated}))
|
|
}
|
|
|
|
func getPodNames(pods []*v1.Pod) sets.String {
|
|
s := sets.NewString()
|
|
for _, p := range pods {
|
|
s.Insert(p.Name)
|
|
}
|
|
return s
|
|
}
|
|
|
|
func getPVCNames(pvcs []*v1.PersistentVolumeClaim) sets.String {
|
|
s := sets.NewString()
|
|
for _, p := range pvcs {
|
|
s.Insert(p.Name)
|
|
}
|
|
return s
|
|
}
|