From 30e0ca00b65b11b7562e00077bab1cab7f9a8146 Mon Sep 17 00:00:00 2001
From: Dave Henderson <Dave.Henderson@ca.ibm.com>
Date: Tue, 10 Mar 2015 22:21:56 -0400
Subject: [PATCH] Wait for any activeTransactions to finish during Create

This adds an additional wait phase to make sure any active Transactions
are completed before Create tries to SSH in to the new host. This is
sometimes necessary because SSH can become available before SoftLayer
is fully done setting up the host and strange things can happen...

Signed-off-by: Dave Henderson <Dave.Henderson@ca.ibm.com>
---
 drivers/softlayer/driver.go    | 39 ++++++++++++++++++++++++++++++++++
 drivers/softlayer/softlayer.go | 33 ++++++++++++++++++++++++++++
 2 files changed, 72 insertions(+)

diff --git a/drivers/softlayer/driver.go b/drivers/softlayer/driver.go
index 155ce74890..f3dfccc250 100644
--- a/drivers/softlayer/driver.go
+++ b/drivers/softlayer/driver.go
@@ -295,6 +295,14 @@ func (d *Driver) GetState() (state.State, error) {
 	return vmState, nil
 }
 
+func (d *Driver) GetActiveTransaction() (string, error) {
+	t, err := d.getClient().VirtualGuest().ActiveTransaction(d.Id)
+	if err != nil {
+		return "", err
+	}
+	return t, nil
+}
+
 func (d *Driver) PreCreateCheck() error {
 	return nil
 }
@@ -305,11 +313,41 @@ func (d *Driver) Create() error {
 		for {
 			s, err := d.GetState()
 			if err != nil {
+				log.Debugf("Failed to GetState - %+v", err)
 				continue
 			}
 
 			if s == state.Running {
 				break
+			} else {
+				log.Debugf("Still waiting - state is %s...", s)
+			}
+			time.Sleep(2 * time.Second)
+		}
+	}
+
+	waitForSetupTransactions := func() {
+		log.Infof("Waiting for host setup transactions to complete")
+		// sometimes we'll hit a case where there's no active transaction, but if
+		// we check again in a few seconds, it moves to the next transaction. We
+		// don't want to get false-positives, so we check a few times in a row to make sure!
+		noActiveCount, maxNoActiveCount := 0, 3
+		for {
+			t, err := d.GetActiveTransaction()
+			if err != nil {
+				noActiveCount = 0
+				log.Debugf("Failed to GetActiveTransaction - %+v", err)
+				continue
+			}
+
+			if t == "" {
+				if noActiveCount == maxNoActiveCount {
+					break
+				}
+				noActiveCount++
+			} else {
+				noActiveCount = 0
+				log.Debugf("Still waiting - active transaction is %s...", t)
 			}
 			time.Sleep(2 * time.Second)
 		}
@@ -357,6 +395,7 @@ func (d *Driver) Create() error {
 	d.Id = id
 	getIp()
 	waitForStart()
+	waitForSetupTransactions()
 	ssh.WaitForTCP(d.IPAddress + ":22")
 
 	cmd, err := drivers.GetSSHCommandFromDriver(d, "sudo apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install -yq curl")
diff --git a/drivers/softlayer/softlayer.go b/drivers/softlayer/softlayer.go
index e094b50459..0f448b77d5 100644
--- a/drivers/softlayer/softlayer.go
+++ b/drivers/softlayer/softlayer.go
@@ -180,6 +180,39 @@ func (c *virtualGuest) PowerState(id int) (string, error) {
 	return s.Name, nil
 }
 
+func (c *virtualGuest) ActiveTransaction(id int) (string, error) {
+	type transactionStatus struct {
+		AverageDuration string `json:"averageDuration"`
+		FriendlyName    string `json:"friendlyName"`
+		Name            string `json:"name"`
+	}
+	type transaction struct {
+		CreateDate        string            `json:"createDate"`
+		ElapsedSeconds    int               `json:"elapsedSeconds"`
+		GuestID           int               `json:"guestId"`
+		HardwareID        int               `json:"hardwareId"`
+		ID                int               `json:"id"`
+		ModifyDate        string            `json:"modifyDate"`
+		StatusChangeDate  string            `json:"statusChangeDate"`
+		TransactionStatus transactionStatus `json:"transactionStatus"`
+	}
+	var (
+		method = "GET"
+		uri    = fmt.Sprintf("%s/%v/getActiveTransaction.json", c.namespace(), id)
+	)
+
+	data, err := c.newRequest(method, uri, nil)
+	if err != nil {
+		return "", err
+	}
+	var t transaction
+	if err := json.Unmarshal(data, &t); err != nil {
+		return "", err
+	}
+
+	return t.TransactionStatus.Name, nil
+}
+
 func (c *virtualGuest) Create(spec *HostSpec) (int, error) {
 	var (
 		method = "POST"