rollouts/pkg/controller/batchrelease/control/canarystyle/deployment/canary.go

/*
Copyright 2022 The Kruise Authors.

Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at

    http://www.apache.org/licenses/LICENSE-2.0

Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/

package deployment

import (
	"context"
	"encoding/json"
	"fmt"
	"sort"

	"github.com/openkruise/rollouts/api/v1alpha1"
	batchcontext "github.com/openkruise/rollouts/pkg/controller/batchrelease/context"
	"github.com/openkruise/rollouts/pkg/util"
	utilclient "github.com/openkruise/rollouts/pkg/util/client"
	expectations "github.com/openkruise/rollouts/pkg/util/expectation"
	apps "k8s.io/api/apps/v1"
	corev1 "k8s.io/api/core/v1"
	"k8s.io/apimachinery/pkg/api/errors"
	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
	"k8s.io/apimachinery/pkg/types"
	"k8s.io/klog/v2"
	"k8s.io/utils/pointer"
	"sigs.k8s.io/controller-runtime/pkg/client"
	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
)

type realCanaryController struct {
	canaryInfo   *util.WorkloadInfo
	canaryObject *apps.Deployment
	canaryClient client.Client
	objectKey    types.NamespacedName
}

func newCanary(cli client.Client, key types.NamespacedName) realCanaryController {
	return realCanaryController{canaryClient: cli, objectKey: key}
}

func (r *realCanaryController) GetCanaryInfo() *util.WorkloadInfo {
	return r.canaryInfo
}

// Delete do not delete canary deployments actually, it only removes the finalizers of
// Deployments. These deployments will be cascaded deleted when BatchRelease is deleted.
func (r *realCanaryController) Delete(release *v1alpha1.BatchRelease) error {
	deployments, err := r.listDeployment(release, client.InNamespace(r.objectKey.Namespace), utilclient.DisableDeepCopy)
	if err != nil {
		return err
	}

	for _, d := range deployments {
		if !controllerutil.ContainsFinalizer(d, util.CanaryDeploymentFinalizer) {
			continue
		}
		err = util.UpdateFinalizer(r.canaryClient, d, util.RemoveFinalizerOpType, util.CanaryDeploymentFinalizer)
		if err != nil && !errors.IsNotFound(err) {
			return err
		}
		klog.Infof("Successfully remove finalizers for Deplot %v", klog.KObj(d))
	}
	return nil
}

func (r *realCanaryController) UpgradeBatch(ctx *batchcontext.BatchContext) error {
	// desired replicas for canary deployment
	desired := ctx.DesiredUpdatedReplicas
	deployment := util.GetEmptyObjectWithKey(r.canaryObject)

	if r.canaryInfo.Replicas >= desired {
		return nil
	}

	body := fmt.Sprintf(`{"spec":{"replicas":%d}}`, desired)
	if err := r.canaryClient.Patch(context.TODO(), deployment, client.RawPatch(types.MergePatchType, []byte(body))); err != nil {
		return err
	}
	klog.Infof("Successfully submit rolling replicas %d to Deployment %v", desired, klog.KObj(deployment))
	return nil
}

func (r *realCanaryController) Create(release *v1alpha1.BatchRelease) error {
	if r.canaryObject != nil {
		return nil // Don't re-create if exists
	}

	// check expectation before creating canary deployment to avoid
	// repeatedly create multiple canary deployment incorrectly.
	controllerKey := client.ObjectKeyFromObject(release).String()
	satisfied, timeoutDuration, rest := expectations.ResourceExpectations.SatisfiedExpectations(controllerKey)
	if !satisfied {
		if timeoutDuration >= expectations.ExpectationTimeout {
			klog.Warningf("Unsatisfied time of expectation exceeds %v, delete key and continue, key: %v, rest: %v",
				expectations.ExpectationTimeout, klog.KObj(release), rest)
			expectations.ResourceExpectations.DeleteExpectations(controllerKey)
		} else {
			return fmt.Errorf("expectation is not satisfied, key: %v, rest: %v", klog.KObj(release), rest)
		}
	}

	// fetch the stable deployment as template to create canary deployment.
	stable := &apps.Deployment{}
	if err := r.canaryClient.Get(context.TODO(), r.objectKey, stable); err != nil {
		return err
	}
	return r.create(release, stable)
}
func (r *realCanaryController) create(release *v1alpha1.BatchRelease, template *apps.Deployment) error {
	canary := &apps.Deployment{
		ObjectMeta: metav1.ObjectMeta{
			GenerateName: fmt.Sprintf("%v-", r.objectKey.Name),
			Namespace:    r.objectKey.Namespace,
			Labels:       map[string]string{},
			Annotations:  map[string]string{},
		},
	}

	// metadata
	canary.Finalizers = append(canary.Finalizers, util.CanaryDeploymentFinalizer)
	canary.OwnerReferences = append(canary.OwnerReferences, *metav1.NewControllerRef(release, release.GroupVersionKind()))
	canary.Labels[util.CanaryDeploymentLabel] = template.Name
	ownerInfo, _ := json.Marshal(metav1.NewControllerRef(release, release.GroupVersionKind()))
	canary.Annotations[util.BatchReleaseControlAnnotation] = string(ownerInfo)

	// spec
	canary.Spec = *template.Spec.DeepCopy()
	canary.Spec.Replicas = pointer.Int32Ptr(0)
	canary.Spec.Paused = false

	if err := r.canaryClient.Create(context.TODO(), canary); err != nil {
		klog.Errorf("Failed to create canary Deployment(%v), error: %v", klog.KObj(canary), err)
		return err
	}

	// add expect to avoid to create repeatedly
	controllerKey := client.ObjectKeyFromObject(release).String()
	expectations.ResourceExpectations.Expect(controllerKey, expectations.Create, string(canary.UID))

	canaryInfo, _ := json.Marshal(canary)
	klog.Infof("Create canary Deployment(%v) successfully, details: %s", klog.KObj(canary), string(canaryInfo))
	return fmt.Errorf("created canary deployment %v succeeded, but waiting informer synced", klog.KObj(canary))
}

func (r *realCanaryController) listDeployment(release *v1alpha1.BatchRelease, options ...client.ListOption) ([]*apps.Deployment, error) {
	dList := &apps.DeploymentList{}
	if err := r.canaryClient.List(context.TODO(), dList, options...); err != nil {
		return nil, err
	}

	var ds []*apps.Deployment
	for i := range dList.Items {
		d := &dList.Items[i]
		o := metav1.GetControllerOf(d)
		if o == nil || o.UID != release.UID {
			continue
		}
		ds = append(ds, d)
	}
	return ds, nil
}

//  return the latest deployment with the newer creation time
func filterCanaryDeployment(ds []*apps.Deployment, template *corev1.PodTemplateSpec) *apps.Deployment {
	if len(ds) == 0 {
		return nil
	}
	sort.Slice(ds, func(i, j int) bool {
		return ds[i].CreationTimestamp.After(ds[j].CreationTimestamp.Time)
	})
	if template == nil {
		return ds[0]
	}
	for _, d := range ds {
		if util.EqualIgnoreHash(template, &d.Spec.Template) {
			return d
		}
	}
	return nil
}