diff --git a/Makefile b/Makefile
index d6b7dc4c10..a45271ddda 100644
--- a/Makefile
+++ b/Makefile
@@ -15,7 +15,7 @@ GO_LDFLAGS=-ldflags "-w $(CTIMEVAR)"
 GO_LDFLAGS_STATIC=-ldflags "-w $(CTIMEVAR) -extldflags -static"
 GOOSES = darwin freebsd linux
 GOARCHS = amd64
-NOTARY_BUILDTAGS ?= "pkcs11"
+NOTARY_BUILDTAGS ?= pkcs11
 GO_EXC = go
 NOTARYDIR := /go/src/github.com/docker/notary
 
diff --git a/trustmanager/yubikey/yubikeystore.go b/trustmanager/yubikey/yubikeystore.go
index 47a97dc9bd..5df3f430ed 100644
--- a/trustmanager/yubikey/yubikeystore.go
+++ b/trustmanager/yubikey/yubikeystore.go
@@ -771,7 +771,11 @@ func cleanup(ctx IPKCS11Ctx, session pkcs11.SessionHandle) {
 	if err != nil {
 		logrus.Debugf("Error closing session: %s", err.Error())
 	}
-	err = ctx.Finalize()
+	finalizeAndDestroy(ctx)
+}
+
+func finalizeAndDestroy(ctx IPKCS11Ctx) {
+	err := ctx.Finalize()
 	if err != nil {
 		logrus.Debugf("Error finalizing: %s", err.Error())
 	}
@@ -792,18 +796,18 @@ func SetupHSMEnv(libraryPath string, libLoader pkcs11LibLoader) (
 	}
 
 	if err := p.Initialize(); err != nil {
-		defer cleanup(p, 0)
+		defer finalizeAndDestroy(p)
 		return nil, 0, fmt.Errorf("Initialize error %s", err.Error())
 	}
 
 	slots, err := p.GetSlotList(true)
 	if err != nil {
-		defer cleanup(p, 0)
+		defer finalizeAndDestroy(p)
 		return nil, 0, fmt.Errorf("Failed to list HSM slots %s", err)
 	}
 	// Check to see if we got any slots from the HSM.
 	if len(slots) < 1 {
-		defer cleanup(p, 0)
+		defer finalizeAndDestroy(p)
 		return nil, 0, fmt.Errorf("No HSM Slots found")
 	}
 
@@ -811,7 +815,7 @@ func SetupHSMEnv(libraryPath string, libLoader pkcs11LibLoader) (
 	// CKF_RW_SESSION: TRUE if the session is read/write; FALSE if the session is read-only
 	session, err := p.OpenSession(slots[0], pkcs11.CKF_SERIAL_SESSION|pkcs11.CKF_RW_SESSION)
 	if err != nil {
-		defer cleanup(p, 0)
+		defer cleanup(p, session)
 		return nil, 0, fmt.Errorf("Failed to Start Session with HSM %s", err)
 	}
 
diff --git a/trustmanager/yubikey/yubikeystore_test.go b/trustmanager/yubikey/yubikeystore_test.go
index ab53f2eaea..672a61eb35 100644
--- a/trustmanager/yubikey/yubikeystore_test.go
+++ b/trustmanager/yubikey/yubikeystore_test.go
@@ -64,7 +64,7 @@ func testAddKey(t *testing.T, store trustmanager.KeyStore) (data.PrivateKey, err
 }
 
 func addMaxKeys(t *testing.T, store trustmanager.KeyStore) []string {
-	keys := make([]string, 0, numSlots)
+	var keys []string
 	// create the maximum number of keys
 	for i := 0; i < numSlots; i++ {
 		privKey, err := testAddKey(t, store)
@@ -378,6 +378,7 @@ func TestYubiExportKeyFails(t *testing.T) {
 
 	_, err = store.ExportKey(key.ID())
 	assert.Error(t, err)
+	assert.Equal(t, "Keys cannot be exported from a Yubikey.", err.Error())
 }
 
 // If there are keys in the backup store but no keys in the Yubikey,