From ff58f4aac95041faaee4b878e2a389d00f54264c Mon Sep 17 00:00:00 2001 From: liranp Date: Mon, 15 Jul 2019 19:20:44 +0300 Subject: [PATCH] fix(spotinst): rolling-update workaround --- pkg/resources/spotinst/resources.go | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/pkg/resources/spotinst/resources.go b/pkg/resources/spotinst/resources.go index 4f34017dd2..72edf4c12e 100644 --- a/pkg/resources/spotinst/resources.go +++ b/pkg/resources/spotinst/resources.go @@ -337,6 +337,25 @@ func buildCloudInstanceGroupFromLaunchSpec(cloud Cloud, ig *kops.InstanceGroup, func registerCloudInstanceGroupMembers(instanceGroup *cloudinstances.CloudInstanceGroup, nodeMap map[string]*v1.Node, instances []Instance, currentInstanceGroupName string, instanceGroupUpdatedAt time.Time) error { + // The instance registration below registers all active instances with + // their instance group. In addition, it looks for outdated instances by + // comparing each instance creation timestamp against the modification + // timestamp of its instance group. + // + // In a rolling-update operation, one or more detach operations are + // performed to replace existing instances. This is done by updating the + // instance group and results in updating the modification timestamp to the + // current time. + // + // The update of the modification timestamp occurs only after the detach + // operation is completed, meaning that new instances have already been + // created, so our comparison may be incorrect. + // + // In order to work around this issue, we assume that the detach operation + // will take up to two minutes, and therefore we subtract this duration from + // the modification timestamp of the instance group before the comparison. + instanceGroupUpdatedAt = instanceGroupUpdatedAt.Add(-2 * time.Minute) + for _, instance := range instances { if instance.Id() == "" { klog.Warningf("Ignoring instance with no ID: %v", instance)