fix member update when draining

upup/pkg/fi/cloudup/openstack/instance.go

@@ -254,7 +254,6 @@ func drainSingleLB(c OpenstackCloud, lb loadbalancers.LoadBalancer, instanceName
 	}
 
 	if draining {
-		// TODO: should we do somekind of loop here and check that connections are really drained?
 		time.Sleep(20 * time.Second)
 
 		newStats, err := c.GetLBStats(lb.ID)
@@ -262,7 +261,8 @@ func drainSingleLB(c OpenstackCloud, lb loadbalancers.LoadBalancer, instanceName
 			return err
 		}
 
-		klog.Infof("Loadbalancer %s connections before draining %d and after %d", lb.Name, oldStats.ActiveConnections, newStats.ActiveConnections)
+		// NOTE! this is total loadbalancer connections NOT member connections
+		klog.V(4).Infof("Loadbalancer %s connections before draining %d and after %d", lb.Name, oldStats.ActiveConnections, newStats.ActiveConnections)
 	}
 	return nil
 }

upup/pkg/fi/cloudup/openstack/loadbalancer.go

@@ -19,6 +19,7 @@ package openstack
 import (
 	"fmt"
 
+	"github.com/gophercloud/gophercloud"
 	"github.com/gophercloud/gophercloud/openstack/compute/v2/servers"
 	"github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/listeners"
 	"github.com/gophercloud/gophercloud/openstack/loadbalancer/v2/loadbalancers"
@@ -337,6 +338,17 @@ func updateMemberInPool(c OpenstackCloud, poolID string, memberID string, opts v
 	done, err := vfs.RetryWithBackoff(writeBackoff, func() (bool, error) {
 		association, err = v2pools.UpdateMember(c.LoadBalancerClient(), poolID, memberID, opts).Extract()
 		if err != nil {
+			// member not found anymore
+			if isNotFound(err) {
+				return true, nil
+			}
+			// pool is currently in immutable state, try to retry
+			if errCode, ok := err.(gophercloud.ErrUnexpectedResponseCode); ok {
+				if errCode.Actual == http.StatusConflict {
+					klog.Infof("got error %v retrying...", err)
+					return false, nil
+				}
+			}
 			return false, fmt.Errorf("failed to update pool membership: %v", err)
 		}
 		return true, nil