338 lines
13 KiB
Go
338 lines
13 KiB
Go
/*
|
|
Copyright 2022 The Kruise Authors.
|
|
|
|
Licensed under the Apache License, Version 2.0 (the "License");
|
|
you may not use this file except in compliance with the License.
|
|
You may obtain a copy of the License at
|
|
|
|
http://www.apache.org/licenses/LICENSE-2.0
|
|
|
|
Unless required by applicable law or agreed to in writing, software
|
|
distributed under the License is distributed on an "AS IS" BASIS,
|
|
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
See the License for the specific language governing permissions and
|
|
limitations under the License.
|
|
*/
|
|
|
|
package deployment
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
|
|
apps "k8s.io/api/apps/v1"
|
|
corev1 "k8s.io/api/core/v1"
|
|
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
|
|
"k8s.io/apimachinery/pkg/runtime/schema"
|
|
"k8s.io/apimachinery/pkg/types"
|
|
"k8s.io/apimachinery/pkg/util/intstr"
|
|
"k8s.io/klog/v2"
|
|
utilpointer "k8s.io/utils/pointer"
|
|
"sigs.k8s.io/controller-runtime/pkg/client"
|
|
|
|
"github.com/openkruise/rollouts/api/v1alpha1"
|
|
"github.com/openkruise/rollouts/api/v1beta1"
|
|
batchcontext "github.com/openkruise/rollouts/pkg/controller/batchrelease/context"
|
|
"github.com/openkruise/rollouts/pkg/controller/batchrelease/control"
|
|
"github.com/openkruise/rollouts/pkg/controller/batchrelease/control/bluegreenstyle"
|
|
"github.com/openkruise/rollouts/pkg/controller/batchrelease/control/bluegreenstyle/hpa"
|
|
deploymentutil "github.com/openkruise/rollouts/pkg/controller/deployment/util"
|
|
"github.com/openkruise/rollouts/pkg/util"
|
|
"github.com/openkruise/rollouts/pkg/util/errors"
|
|
"github.com/openkruise/rollouts/pkg/util/patch"
|
|
)
|
|
|
|
type realController struct {
|
|
*util.WorkloadInfo
|
|
client client.Client
|
|
pods []*corev1.Pod
|
|
key types.NamespacedName
|
|
object *apps.Deployment
|
|
finder *util.ControllerFinder
|
|
}
|
|
|
|
func NewController(cli client.Client, key types.NamespacedName, _ schema.GroupVersionKind) bluegreenstyle.Interface {
|
|
return &realController{
|
|
key: key,
|
|
client: cli,
|
|
finder: util.NewControllerFinder(cli),
|
|
}
|
|
}
|
|
|
|
func (rc *realController) GetWorkloadInfo() *util.WorkloadInfo {
|
|
return rc.WorkloadInfo
|
|
}
|
|
|
|
func (rc *realController) BuildController() (bluegreenstyle.Interface, error) {
|
|
if rc.object != nil {
|
|
return rc, nil
|
|
}
|
|
object := &apps.Deployment{}
|
|
if err := rc.client.Get(context.TODO(), rc.key, object); err != nil {
|
|
return rc, err
|
|
}
|
|
rc.object = object
|
|
rc.WorkloadInfo = rc.getWorkloadInfo(object)
|
|
return rc, nil
|
|
}
|
|
|
|
func (rc *realController) ListOwnedPods() ([]*corev1.Pod, error) {
|
|
if rc.pods != nil {
|
|
return rc.pods, nil
|
|
}
|
|
var err error
|
|
rc.pods, err = util.ListOwnedPods(rc.client, rc.object)
|
|
return rc.pods, err
|
|
}
|
|
|
|
// Initialize prepares the Deployment for the BatchRelease process
|
|
func (rc *realController) Initialize(release *v1beta1.BatchRelease) error {
|
|
if rc.object == nil || control.IsControlledByBatchRelease(release, rc.object) {
|
|
return nil
|
|
}
|
|
// Disable the HPA
|
|
if err := hpa.DisableHPA(rc.client, rc.object); err != nil {
|
|
return err
|
|
}
|
|
klog.InfoS("Initialize: disabled HPA for deployment successfully", "deployment", klog.KObj(rc.object))
|
|
|
|
// Patch minReadySeconds for stable ReplicaSet
|
|
if err := rc.patchStableRSMinReadySeconds(v1beta1.MaxReadySeconds); err != nil {
|
|
return err
|
|
}
|
|
klog.InfoS("Initialize: patched minReadySeconds for stable replicaset successfully", "deployment", klog.KObj(rc.object))
|
|
|
|
// Patch Deplopyment
|
|
if err := rc.patchDeployment(release); err != nil {
|
|
return err
|
|
}
|
|
klog.InfoS("Initialize: patched deployment successfully", "deployment", klog.KObj(rc.object))
|
|
return nil
|
|
}
|
|
|
|
func (rc *realController) UpgradeBatch(ctx *batchcontext.BatchContext) error {
|
|
if err := control.ValidateReadyForBlueGreenRelease(rc.object); err != nil {
|
|
return errors.NewBadRequestError(fmt.Errorf("cannot upgrade batch, because deployment %v doesn't satisfy conditions: %s", klog.KObj(rc.object), err.Error()))
|
|
}
|
|
desired, _ := intstr.GetScaledValueFromIntOrPercent(&ctx.DesiredSurge, int(ctx.Replicas), true)
|
|
current, _ := intstr.GetScaledValueFromIntOrPercent(&ctx.CurrentSurge, int(ctx.Replicas), true)
|
|
|
|
if current >= desired {
|
|
klog.Infof("No need to upgrade batch for deployment %v: because current %d >= desired %d", klog.KObj(rc.object), current, desired)
|
|
return nil
|
|
}
|
|
klog.Infof("Ready to upgrade batch for deployment %v: current %d < desired %d", klog.KObj(rc.object), current, desired)
|
|
patchData := patch.NewDeploymentPatch()
|
|
// different with canary release, bluegreen don't need to set pause in the process of rollout
|
|
// because our webhook may pause the Deployment in some situations, we ensure that the Deployment is not paused
|
|
patchData.UpdatePaused(false)
|
|
patchData.UpdateStrategy(apps.DeploymentStrategy{
|
|
Type: apps.RollingUpdateDeploymentStrategyType,
|
|
RollingUpdate: &apps.RollingUpdateDeployment{
|
|
MaxSurge: &ctx.DesiredSurge,
|
|
MaxUnavailable: &intstr.IntOrString{},
|
|
},
|
|
})
|
|
return rc.client.Patch(context.TODO(), util.GetEmptyObjectWithKey(rc.object), patchData)
|
|
}
|
|
|
|
// set pause to false, restore the original setting, delete annotation
|
|
func (rc *realController) Finalize(release *v1beta1.BatchRelease) error {
|
|
if release.Spec.ReleasePlan.BatchPartition != nil {
|
|
// continuous release (not supported yet)
|
|
/*
|
|
patchData := patch.NewDeploymentPatch()
|
|
patchData.DeleteAnnotation(util.BatchReleaseControlAnnotation)
|
|
if err := rc.client.Patch(context.TODO(), d, patchData); err != nil {
|
|
return err
|
|
}
|
|
*/
|
|
klog.Warningf("continuous release is not supported yet for bluegreen style release")
|
|
return nil
|
|
}
|
|
|
|
// restore the original setting and remove annotation
|
|
d := util.GetEmptyObjectWithKey(rc.object)
|
|
if !rc.restored() {
|
|
setting, err := control.GetOriginalSetting(rc.object)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
patchData := patch.NewDeploymentPatch()
|
|
// restore the original setting
|
|
patchData.UpdatePaused(false)
|
|
patchData.UpdateMinReadySeconds(setting.MinReadySeconds)
|
|
patchData.UpdateProgressDeadlineSeconds(setting.ProgressDeadlineSeconds)
|
|
patchData.UpdateMaxSurge(setting.MaxSurge)
|
|
patchData.UpdateMaxUnavailable(setting.MaxUnavailable)
|
|
// restore label and annotation
|
|
patchData.DeleteAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation)
|
|
patchData.DeleteLabel(v1alpha1.DeploymentStableRevisionLabel)
|
|
patchData.DeleteAnnotation(util.BatchReleaseControlAnnotation)
|
|
if err := rc.client.Patch(context.TODO(), d, patchData); err != nil {
|
|
return err
|
|
}
|
|
klog.InfoS("Finalize: deployment bluegreen release: wait all pods updated and ready", "Deployment", klog.KObj(rc.object))
|
|
}
|
|
|
|
// wait all pods updated and ready
|
|
if err := waitAllUpdatedAndReady(d.(*apps.Deployment)); err != nil {
|
|
return errors.NewRetryError(err)
|
|
}
|
|
klog.InfoS("Finalize: All pods updated and ready, then restore hpa", "Deployment", klog.KObj(rc.object))
|
|
|
|
// restore hpa
|
|
return hpa.RestoreHPA(rc.client, rc.object)
|
|
}
|
|
|
|
func (rc *realController) restored() bool {
|
|
if rc.object == nil || rc.object.DeletionTimestamp != nil {
|
|
return true
|
|
}
|
|
if rc.object.Annotations == nil || len(rc.object.Annotations[v1beta1.OriginalDeploymentStrategyAnnotation]) == 0 {
|
|
return true
|
|
}
|
|
return false
|
|
}
|
|
|
|
func (rc *realController) CalculateBatchContext(release *v1beta1.BatchRelease) (*batchcontext.BatchContext, error) {
|
|
currentBatch := release.Status.CanaryStatus.CurrentBatch
|
|
desiredSurge := release.Spec.ReleasePlan.Batches[currentBatch].CanaryReplicas
|
|
PlannedUpdatedReplicas := deploymentutil.NewRSReplicasLimit(desiredSurge, rc.object)
|
|
currentSurge := intstr.FromInt(0)
|
|
if rc.object.Spec.Strategy.RollingUpdate != nil && rc.object.Spec.Strategy.RollingUpdate.MaxSurge != nil {
|
|
currentSurge = *rc.object.Spec.Strategy.RollingUpdate.MaxSurge
|
|
if currentSurge == intstr.FromInt(1) {
|
|
// currentSurge == intstr.FromInt(1) means that currentSurge is the initial value
|
|
// if the value is indeed set by user, setting it to 0 still does no harm
|
|
currentSurge = intstr.FromInt(0)
|
|
}
|
|
}
|
|
return &batchcontext.BatchContext{
|
|
Pods: rc.pods,
|
|
RolloutID: release.Spec.ReleasePlan.RolloutID,
|
|
CurrentBatch: currentBatch,
|
|
CurrentSurge: currentSurge,
|
|
DesiredSurge: desiredSurge,
|
|
UpdateRevision: release.Status.UpdateRevision,
|
|
|
|
Replicas: rc.Replicas,
|
|
UpdatedReplicas: rc.Status.UpdatedReplicas,
|
|
UpdatedReadyReplicas: rc.Status.UpdatedReadyReplicas,
|
|
PlannedUpdatedReplicas: PlannedUpdatedReplicas,
|
|
DesiredUpdatedReplicas: PlannedUpdatedReplicas,
|
|
}, nil
|
|
}
|
|
|
|
func (rc *realController) getWorkloadInfo(d *apps.Deployment) *util.WorkloadInfo {
|
|
workloadInfo := util.ParseWorkload(d)
|
|
workloadInfo.Status.UpdatedReadyReplicas = 0
|
|
if res, err := rc.getUpdatedReadyReplicas(d); err == nil {
|
|
workloadInfo.Status.UpdatedReadyReplicas = res
|
|
}
|
|
workloadInfo.Status.StableRevision = d.Labels[v1alpha1.DeploymentStableRevisionLabel]
|
|
return workloadInfo
|
|
}
|
|
|
|
func (rc *realController) getUpdatedReadyReplicas(d *apps.Deployment) (int32, error) {
|
|
rss := &apps.ReplicaSetList{}
|
|
listOpts := []client.ListOption{
|
|
client.InNamespace(d.Namespace),
|
|
client.MatchingLabels(d.Spec.Selector.MatchLabels),
|
|
client.UnsafeDisableDeepCopy,
|
|
}
|
|
if err := rc.client.List(context.TODO(), rss, listOpts...); err != nil {
|
|
klog.Warningf("getWorkloadInfo failed, because"+"%s", err.Error())
|
|
return -1, err
|
|
}
|
|
allRSs := rss.Items
|
|
// select rs owner by current deployment
|
|
ownedRSs := make([]*apps.ReplicaSet, 0)
|
|
for i := range allRSs {
|
|
rs := &allRSs[i]
|
|
if !rs.DeletionTimestamp.IsZero() {
|
|
continue
|
|
}
|
|
|
|
if metav1.IsControlledBy(rs, d) {
|
|
ownedRSs = append(ownedRSs, rs)
|
|
}
|
|
}
|
|
newRS := deploymentutil.FindNewReplicaSet(d, ownedRSs)
|
|
updatedReadyReplicas := int32(0)
|
|
// if newRS is nil, it means the replicaset hasn't been created (because the deployment is paused)
|
|
// therefore we can return 0 directly
|
|
if newRS != nil {
|
|
updatedReadyReplicas = newRS.Status.ReadyReplicas
|
|
}
|
|
return updatedReadyReplicas, nil
|
|
}
|
|
|
|
func waitAllUpdatedAndReady(deployment *apps.Deployment) error {
|
|
if deployment.Spec.Paused {
|
|
return fmt.Errorf("deployment should not be paused")
|
|
}
|
|
|
|
// ALL pods updated AND ready
|
|
if deployment.Status.ReadyReplicas != deployment.Status.UpdatedReplicas {
|
|
return fmt.Errorf("all ready replicas should be updated, and all updated replicas should be ready")
|
|
}
|
|
|
|
availableReplicas := deployment.Status.AvailableReplicas
|
|
allowedUnavailable := util.DeploymentMaxUnavailable(deployment)
|
|
if allowedUnavailable+availableReplicas < deployment.Status.Replicas {
|
|
return fmt.Errorf("ready replicas should satisfy maxUnavailable")
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Patch minReadySeconds for stable ReplicaSet
|
|
/*
|
|
Here is why:
|
|
For rollback scenario, we should set the stable rs minReadySeconds to infinity to make pods of the stable rs unavailable,
|
|
otherwise Pods in new version would be terminated immediately when rollback happens.
|
|
we want to keep them until traffic is switched to the stable version
|
|
*/
|
|
func (rc *realController) patchStableRSMinReadySeconds(seconds int32) error {
|
|
if stableRS, err := rc.finder.GetDeploymentStableRs(rc.object); err != nil {
|
|
return fmt.Errorf("failed to get stable ReplicaSet: %v", err)
|
|
} else if stableRS == nil {
|
|
klog.Warningf("No stable ReplicaSet found for deployment %s/%s", rc.object.Namespace, rc.object.Name)
|
|
} else {
|
|
body := fmt.Sprintf(`{"spec":{"minReadySeconds":%v}}`, seconds)
|
|
if err = rc.client.Patch(context.TODO(), stableRS, client.RawPatch(types.MergePatchType, []byte(body))); err != nil {
|
|
return fmt.Errorf("failed to patch ReplicaSet %s/%s minReadySeconds to %v: %v", stableRS.Namespace, stableRS.Name, v1beta1.MaxReadySeconds, err)
|
|
}
|
|
}
|
|
return nil
|
|
}
|
|
|
|
// Update deployment strategy: MinReadySeconds, ProgressDeadlineSeconds, MaxSurge, MaxUnavailable
|
|
func (rc *realController) patchDeployment(release *v1beta1.BatchRelease) error {
|
|
setting, err := control.GetOriginalSetting(rc.object)
|
|
if err != nil {
|
|
return errors.NewBadRequestError(fmt.Errorf("cannot get original setting for deployment %v: %s", klog.KObj(rc.object), err.Error()))
|
|
}
|
|
control.InitOriginalSetting(&setting, rc.object)
|
|
patchData := patch.NewDeploymentPatch()
|
|
patchData.InsertAnnotation(v1beta1.OriginalDeploymentStrategyAnnotation, util.DumpJSON(&setting))
|
|
patchData.InsertAnnotation(util.BatchReleaseControlAnnotation, util.DumpJSON(metav1.NewControllerRef(
|
|
release, release.GetObjectKind().GroupVersionKind())))
|
|
|
|
patchData.UpdateStrategy(apps.DeploymentStrategy{
|
|
Type: apps.RollingUpdateDeploymentStrategyType,
|
|
RollingUpdate: &apps.RollingUpdateDeployment{
|
|
MaxSurge: &intstr.IntOrString{Type: intstr.Int, IntVal: 1},
|
|
MaxUnavailable: &intstr.IntOrString{Type: intstr.Int, IntVal: 0},
|
|
},
|
|
})
|
|
patchData.UpdateMinReadySeconds(v1beta1.MaxReadySeconds)
|
|
patchData.UpdateProgressDeadlineSeconds(utilpointer.Int32(v1beta1.MaxProgressSeconds))
|
|
|
|
// Apply the patch to the Deployment
|
|
if err := rc.client.Patch(context.TODO(), util.GetEmptyObjectWithKey(rc.object), patchData); err != nil {
|
|
return fmt.Errorf("failed to patch deployment %v: %v", klog.KObj(rc.object), err)
|
|
}
|
|
return nil
|
|
}
|