diff --git a/pkcs11helpers/helpers.go b/pkcs11helpers/helpers.go
index 37b29d62a..3b4d6b8e0 100644
--- a/pkcs11helpers/helpers.go
+++ b/pkcs11helpers/helpers.go
@@ -64,12 +64,13 @@ var curveOIDs = map[string]asn1.ObjectIdentifier{
 // getPublicKeyID looks up the given public key in the PKCS#11 token, and
 // returns its ID as a []byte, for use in looking up the corresponding private
 // key.
-func (s *Session) getPublicKeyID(publicKey crypto.PublicKey) ([]byte, error) {
+func (s *Session) getPublicKeyID(label string, publicKey crypto.PublicKey) ([]byte, error) {
 	var template []*pkcs11.Attribute
 	switch key := publicKey.(type) {
 	case *rsa.PublicKey:
 		template = []*pkcs11.Attribute{
 			pkcs11.NewAttribute(pkcs11.CKA_CLASS, pkcs11.CKO_PUBLIC_KEY),
+			pkcs11.NewAttribute(pkcs11.CKA_LABEL, []byte(label)),
 			pkcs11.NewAttribute(pkcs11.CKA_KEY_TYPE, pkcs11.CKK_RSA),
 			pkcs11.NewAttribute(pkcs11.CKA_MODULUS, key.N.Bytes()),
 			pkcs11.NewAttribute(pkcs11.CKA_PUBLIC_EXPONENT, big.NewInt(int64(key.E)).Bytes()),
@@ -92,6 +93,7 @@ func (s *Session) getPublicKeyID(publicKey crypto.PublicKey) ([]byte, error) {
 		}
 		template = []*pkcs11.Attribute{
 			pkcs11.NewAttribute(pkcs11.CKA_CLASS, pkcs11.CKO_PUBLIC_KEY),
+			pkcs11.NewAttribute(pkcs11.CKA_LABEL, []byte(label)),
 			pkcs11.NewAttribute(pkcs11.CKA_KEY_TYPE, pkcs11.CKK_EC),
 			pkcs11.NewAttribute(pkcs11.CKA_EC_PARAMS, curveOID),
 			pkcs11.NewAttribute(pkcs11.CKA_EC_POINT, marshalledPoint),
@@ -385,7 +387,7 @@ func (s *Session) NewSigner(label string, publicKey crypto.PublicKey) (crypto.Si
 		return nil, fmt.Errorf("unsupported public key of type %T", publicKey)
 	}
 
-	publicKeyID, err := s.getPublicKeyID(publicKey)
+	publicKeyID, err := s.getPublicKeyID(label, publicKey)
 	if err != nil {
 		return nil, fmt.Errorf("looking up public key: %s", err)
 	}
diff --git a/pkcs11helpers/helpers_test.go b/pkcs11helpers/helpers_test.go
index e33d1e852..f70899645 100644
--- a/pkcs11helpers/helpers_test.go
+++ b/pkcs11helpers/helpers_test.go
@@ -289,6 +289,36 @@ func TestX509Signer(t *testing.T) {
 	test.AssertEquals(t, signer.Public(), tk.Public())
 }
 
+func TestGetKeyWhenLabelIsWrong(t *testing.T) {
+	s, ctx := newSessionWithMock()
+	pubKey := &rsa.PublicKey{N: big.NewInt(1), E: 1}
+	rightLabel := "label"
+	var objectsToReturn []pkcs11.ObjectHandle
+
+	ctx.FindObjectsInitFunc = func(_ pkcs11.SessionHandle, attr []*pkcs11.Attribute) error {
+		objectsToReturn = []pkcs11.ObjectHandle{1}
+		for _, a := range attr {
+			if a.Type == pkcs11.CKA_LABEL && !bytes.Equal(a.Value, []byte(rightLabel)) {
+				objectsToReturn = nil
+			}
+		}
+		return nil
+	}
+	ctx.FindObjectsFunc = func(_ pkcs11.SessionHandle, _ int) ([]pkcs11.ObjectHandle, bool, error) {
+		return objectsToReturn, false, nil
+	}
+	ctx.FindObjectsFinalFunc = func(_ pkcs11.SessionHandle) error {
+		return nil
+	}
+
+	_, err := s.NewSigner("wrong-label", pubKey)
+	test.AssertError(t, err, "newSigner didn't fail when label was a mismatch for public key")
+	expected := "no objects found matching provided template"
+	if !strings.Contains(err.Error(), expected) {
+		t.Errorf("expected error to contain %q but it was %q", expected, err)
+	}
+}
+
 func TestGetKeyWhenGetAttributeValueFails(t *testing.T) {
 	s, ctx := newSessionWithMock()
 	pubKey := &rsa.PublicKey{N: big.NewInt(1), E: 1}
diff --git a/test/cert-ceremonies/generate.go b/test/cert-ceremonies/generate.go
index 6480dbb11..0b20bd951 100644
--- a/test/cert-ceremonies/generate.go
+++ b/test/cert-ceremonies/generate.go
@@ -2,6 +2,7 @@ package main
 
 import (
 	"errors"
+	"fmt"
 	"io/ioutil"
 	"os"
 	"os/exec"
@@ -35,9 +36,9 @@ func genKey(path string, inSlot string) error {
 	if err != nil {
 		return err
 	}
-	_, err = exec.Command("bin/ceremony", "-config", tmpPath).CombinedOutput()
+	output, err := exec.Command("bin/ceremony", "-config", tmpPath).CombinedOutput()
 	if err != nil {
-		return err
+		return fmt.Errorf("error running ceremony for %s: %s:\n%s", tmpPath, err, string(output))
 	}
 	return nil
 }