Wait for any activeTransactions to finish during Create

This adds an additional wait phase to make sure any active Transactions
are completed before Create tries to SSH in to the new host. This is
sometimes necessary because SSH can become available before SoftLayer
is fully done setting up the host and strange things can happen...

Signed-off-by: Dave Henderson <Dave.Henderson@ca.ibm.com>

drivers/softlayer/driver.go

@@ -295,6 +295,14 @@ func (d *Driver) GetState() (state.State, error) {
 	return vmState, nil
 }
 
+func (d *Driver) GetActiveTransaction() (string, error) {
+	t, err := d.getClient().VirtualGuest().ActiveTransaction(d.Id)
+	if err != nil {
+		return "", err
+	}
+	return t, nil
+}
+
 func (d *Driver) PreCreateCheck() error {
 	return nil
 }
@@ -305,11 +313,41 @@ func (d *Driver) Create() error {
 		for {
 			s, err := d.GetState()
 			if err != nil {
+				log.Debugf("Failed to GetState - %+v", err)
 				continue
 			}
 
 			if s == state.Running {
 				break
+			} else {
+				log.Debugf("Still waiting - state is %s...", s)
+			}
+			time.Sleep(2 * time.Second)
+		}
+	}
+
+	waitForSetupTransactions := func() {
+		log.Infof("Waiting for host setup transactions to complete")
+		// sometimes we'll hit a case where there's no active transaction, but if
+		// we check again in a few seconds, it moves to the next transaction. We
+		// don't want to get false-positives, so we check a few times in a row to make sure!
+		noActiveCount, maxNoActiveCount := 0, 3
+		for {
+			t, err := d.GetActiveTransaction()
+			if err != nil {
+				noActiveCount = 0
+				log.Debugf("Failed to GetActiveTransaction - %+v", err)
+				continue
+			}
+
+			if t == "" {
+				if noActiveCount == maxNoActiveCount {
+					break
+				}
+				noActiveCount++
+			} else {
+				noActiveCount = 0
+				log.Debugf("Still waiting - active transaction is %s...", t)
 			}
 			time.Sleep(2 * time.Second)
 		}
@@ -357,6 +395,7 @@ func (d *Driver) Create() error {
 	d.Id = id
 	getIp()
 	waitForStart()
+	waitForSetupTransactions()
 	ssh.WaitForTCP(d.IPAddress + ":22")
 
 	cmd, err := drivers.GetSSHCommandFromDriver(d, "sudo apt-get update && DEBIAN_FRONTEND=noninteractive sudo apt-get install -yq curl")

drivers/softlayer/softlayer.go

@@ -180,6 +180,39 @@ func (c *virtualGuest) PowerState(id int) (string, error) {
 	return s.Name, nil
 }
 
+func (c *virtualGuest) ActiveTransaction(id int) (string, error) {
+	type transactionStatus struct {
+		AverageDuration string `json:"averageDuration"`
+		FriendlyName    string `json:"friendlyName"`
+		Name            string `json:"name"`
+	}
+	type transaction struct {
+		CreateDate        string            `json:"createDate"`
+		ElapsedSeconds    int               `json:"elapsedSeconds"`
+		GuestID           int               `json:"guestId"`
+		HardwareID        int               `json:"hardwareId"`
+		ID                int               `json:"id"`
+		ModifyDate        string            `json:"modifyDate"`
+		StatusChangeDate  string            `json:"statusChangeDate"`
+		TransactionStatus transactionStatus `json:"transactionStatus"`
+	}
+	var (
+		method = "GET"
+		uri    = fmt.Sprintf("%s/%v/getActiveTransaction.json", c.namespace(), id)
+	)
+
+	data, err := c.newRequest(method, uri, nil)
+	if err != nil {
+		return "", err
+	}
+	var t transaction
+	if err := json.Unmarshal(data, &t); err != nil {
+		return "", err
+	}
+
+	return t.TransactionStatus.Name, nil
+}
+
 func (c *virtualGuest) Create(spec *HostSpec) (int, error) {
 	var (
 		method = "POST"