Implement root certificate rotation in tuf.Repo

Repo.RotateBaseKeys() can now be used to replace the root keypair
(i.e. primarily the expiring certificate), and Repo.SignRoot will make
signatures using the old keypairs if they are available (but not fail if
they are not; only the keypairs listed in the role as trusted are
mandatory).

To do this, we need to keep even the old, possibly currently untrusted,
keypairs around (to allow rollover from clients which trust these old
keys):

The private keys are simply not deleted from the repo's CryptoService;
this means no change for the current setup with a long-term private key
and periodically expiring certificates, but a true rotation of the
private key will eventually require explicit management of the preserved
long-term private keys (if we are keeping several of them around, but
most are either obsolete and non-preferred or possibly even known to be
compromised, we will want to make sure that we always use the
new/preferred private key for new certificate generation).

The public keys are tricker:

1) We need to keep a list of them; the private keys can be looked up
by their IDs, and that allows extracting the public part as well,
but we need a list of the key IDs.  We can't just keep the key IDs
included in the role's list of authorized keys, that would make it
impossible to rotate away from a suspect or known compromsied key.

2) With the X.509 certificate “public keys”, key ID is not actually
sufficient to retrieve the full public key even if we have access to the
private key; we actually need to store the full public key ==
certificate somewhere.  And preferably without having tuf.Repo depend on
a certs.Manager, designed to deal with concepts of trust at a higher
level than TUF cares about.  Actually, to the extent certs.Manager's
purpose is to manage and verify trust, storing old, possibly suspect or
known compromised certificates would be explicitly contrary to its
mission.

So, this patch keeps around full copies of the certificates in the
root.json “keys” map (not the “roles” map of trusted keys). It means
sending to clients a little data which they don't need but it is
otherwise harmless; and keeping the certificates within the
structured and managed tuf.data.Root format could allow us to build nice
UI (e.g. show me all certificates we still carry and keep signing with, let
me drop two of them now that our company has changed a name and does not
want to advertise the history) if we ever needed to something like this

Signed-off-by: Ying Li <ying.li@docker.com>

tuf/tuf.go

@@ -151,13 +151,16 @@ func (tr *Repo) RemoveBaseKeys(role string, keyIDs ...string) error {
 		}
 	}
 
-	// remove keys no longer in use by any roles
+	// Remove keys no longer in use by any roles, except for root keys.
+	// Root private keys must be kept in tr.cryptoService to be able to sign
+	// for rotation, and root certificates must be kept in tr.Root.SignedKeys
+	// because we are not necessarily storing them elsewhere (tuf.Repo does not
+	// depend on certs.Manager, that is an upper layer), and without storing
+	// the certificates in their x509 form we are not able to do the
+	// util.CanonicalKeyID conversion.
+	if role != data.CanonicalRootRole {
 		for k := range toDelete {
 			delete(tr.Root.Signed.Keys, k)
-		// remove the signing key from the cryptoservice if it
-		// isn't a root key. Root keys must be kept for rotation
-		// signing
-		if role != data.CanonicalRootRole {
 			tr.cryptoService.RemoveKey(k)
 		}
 	}
@@ -786,9 +789,18 @@ func (tr *Repo) UpdateTimestamp(s *data.Signed) error {
 	return nil
 }
 
-// SignRoot signs the root
+// SignRoot signs the root, using all keys from the "root" role (i.e. currently trusted)
+// as well as available keys used to sign the previous version, if the public part is
+// carried in tr.Root.Keys and the private key is available (i.e. probably previously
+// trusted keys, to allow rollover).
 func (tr *Repo) SignRoot(expires time.Time) (*data.Signed, error) {
 	logrus.Debug("signing root...")
+
+	oldKeyIDs := make([]string, 0, len(tr.Root.Signatures))
+	for _, oldSig := range tr.Root.Signatures {
+		oldKeyIDs = append(oldKeyIDs, oldSig.KeyID)
+	}
+
 	tr.Root.Signed.Expires = expires
 	tr.Root.Signed.Version++
 	root, err := tr.GetBaseRole(data.CanonicalRootRole)
@@ -799,7 +811,7 @@ func (tr *Repo) SignRoot(expires time.Time) (*data.Signed, error) {
 	if err != nil {
 		return nil, err
 	}
-	signed, err = tr.sign(signed, root)
+	signed, err = tr.sign(signed, root, oldKeyIDs)
 	if err != nil {
 		return nil, err
 	}
@@ -838,7 +850,7 @@ func (tr *Repo) SignTargets(role string, expires time.Time) (*data.Signed, error
 		return nil, err
 	}
 
-	signed, err = tr.sign(signed, targets)
+	signed, err = tr.sign(signed, targets, nil)
 	if err != nil {
 		logrus.Debug("errored signing ", role)
 		return nil, err
@@ -880,7 +892,7 @@ func (tr *Repo) SignSnapshot(expires time.Time) (*data.Signed, error) {
 	if err != nil {
 		return nil, err
 	}
-	signed, err = tr.sign(signed, snapshot)
+	signed, err = tr.sign(signed, snapshot, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -909,7 +921,7 @@ func (tr *Repo) SignTimestamp(expires time.Time) (*data.Signed, error) {
 	if err != nil {
 		return nil, err
 	}
-	signed, err = tr.sign(signed, timestamp)
+	signed, err = tr.sign(signed, timestamp, nil)
 	if err != nil {
 		return nil, err
 	}
@@ -918,8 +930,16 @@ func (tr *Repo) SignTimestamp(expires time.Time) (*data.Signed, error) {
 	return signed, nil
 }
 
-func (tr Repo) sign(signedData *data.Signed, role data.BaseRole) (*data.Signed, error) {
+func (tr Repo) sign(signedData *data.Signed, role data.BaseRole, optionalKeyIDs []string) (*data.Signed, error) {
-	if err := signed.Sign(tr.cryptoService, signedData, role.ListKeys(), role.Threshold, nil); err != nil {
+	optionalKeys := make([]data.PublicKey, 0, len(optionalKeyIDs))
+	for _, kid := range optionalKeyIDs {
+		k, ok := tr.Root.Signed.Keys[kid]
+		if !ok {
+			continue
+		}
+		optionalKeys = append(optionalKeys, k)
+	}
+	if err := signed.Sign(tr.cryptoService, signedData, role.ListKeys(), role.Threshold, optionalKeys); err != nil {
 		return nil, err
 	}
 	return signedData, nil

tuf/tuf_test.go

@@ -8,7 +8,11 @@ import (
 	"path"
 	"path/filepath"
 	"testing"
+	"time"
 
+	"github.com/docker/notary/cryptoservice"
+	"github.com/docker/notary/passphrase"
+	"github.com/docker/notary/trustmanager"
 	"github.com/docker/notary/tuf/data"
 	"github.com/docker/notary/tuf/signed"
 	"github.com/stretchr/testify/require"
@@ -19,6 +23,10 @@ var testGUN = "gun"
 func initRepo(t *testing.T, cryptoService signed.CryptoService) *Repo {
 	rootKey, err := cryptoService.Create("root", testGUN, data.ED25519Key)
 	require.NoError(t, err)
+	return initRepoWithRoot(t, cryptoService, rootKey)
+}
+
+func initRepoWithRoot(t *testing.T, cryptoService signed.CryptoService, rootKey data.PublicKey) *Repo {
 	targetsKey, err := cryptoService.Create("targets", testGUN, data.ED25519Key)
 	require.NoError(t, err)
 	snapshotKey, err := cryptoService.Create("snapshot", testGUN, data.ED25519Key)
@@ -1042,3 +1050,131 @@ func TestGetDelegationRoleKeyMissing(t *testing.T) {
 	require.Error(t, err)
 	require.IsType(t, data.ErrInvalidRole{}, err)
 }
+
+func verifySignatureList(t *testing.T, signed *data.Signed, expectedKeys ...data.PublicKey) {
+	require.Equal(t, len(expectedKeys), len(signed.Signatures))
+	usedKeys := make(map[string]struct{}, len(signed.Signatures))
+	for _, sig := range signed.Signatures {
+		usedKeys[sig.KeyID] = struct{}{}
+	}
+	for _, key := range expectedKeys {
+		_, ok := usedKeys[key.ID()]
+		require.True(t, ok)
+	}
+}
+
+func verifyRootSignatureAgainstKey(t *testing.T, signedRoot *data.Signed, key data.PublicKey) error {
+	roleWithKeys := data.BaseRole{Name: data.CanonicalRootRole, Keys: data.Keys{key.ID(): key}, Threshold: 1}
+	return signed.VerifySignatures(signedRoot, roleWithKeys)
+}
+
+func TestSignRootOldKeyExists(t *testing.T) {
+	gun := "docker/test-sign-root"
+	referenceTime := time.Now()
+
+	cs := cryptoservice.NewCryptoService(trustmanager.NewKeyMemoryStore(
+		passphrase.ConstantRetriever("password")))
+
+	rootPublicKey, err := cs.Create(data.CanonicalRootRole, gun, data.ECDSAKey)
+	require.NoError(t, err)
+	rootPrivateKey, _, err := cs.GetPrivateKey(rootPublicKey.ID())
+	require.NoError(t, err)
+	oldRootCert, err := cryptoservice.GenerateCertificate(rootPrivateKey, gun, referenceTime.AddDate(-9, 0, 0),
+		referenceTime.AddDate(1, 0, 0))
+	require.NoError(t, err)
+	oldRootCertKey := trustmanager.CertToKey(oldRootCert)
+
+	repo := initRepoWithRoot(t, cs, oldRootCertKey)
+
+	// Create a first signature, using the old key.
+	signedRoot, err := repo.SignRoot(data.DefaultExpires(data.CanonicalRootRole))
+	require.NoError(t, err)
+	verifySignatureList(t, signedRoot, oldRootCertKey)
+	err = verifyRootSignatureAgainstKey(t, signedRoot, oldRootCertKey)
+	require.NoError(t, err)
+
+	// Create a new certificate
+	newRootCert, err := cryptoservice.GenerateCertificate(rootPrivateKey, gun, referenceTime, referenceTime.AddDate(10, 0, 0))
+	require.NoError(t, err)
+	newRootCertKey := trustmanager.CertToKey(newRootCert)
+	require.NotEqual(t, oldRootCertKey.ID(), newRootCertKey.ID())
+
+	// Only trust the new certificate
+	err = repo.ReplaceBaseKeys(data.CanonicalRootRole, newRootCertKey)
+	require.NoError(t, err)
+	updatedRootRole, err := repo.GetBaseRole(data.CanonicalRootRole)
+	require.NoError(t, err)
+	updatedRootKeyIDs := updatedRootRole.ListKeyIDs()
+	require.Equal(t, 1, len(updatedRootKeyIDs))
+	require.Equal(t, newRootCertKey.ID(), updatedRootKeyIDs[0])
+
+	// Create a second signature
+	signedRoot, err = repo.SignRoot(data.DefaultExpires(data.CanonicalRootRole))
+	require.NoError(t, err)
+	verifySignatureList(t, signedRoot, oldRootCertKey, newRootCertKey)
+
+	// Verify that the signature can be verified when trusting the old certificate
+	err = verifyRootSignatureAgainstKey(t, signedRoot, oldRootCertKey)
+	require.NoError(t, err)
+	// Verify that the signature can be verified when trusting the new certificate
+	err = verifyRootSignatureAgainstKey(t, signedRoot, newRootCertKey)
+	require.NoError(t, err)
+}
+
+func TestSignRootOldKeyMissing(t *testing.T) {
+	gun := "docker/test-sign-root"
+	referenceTime := time.Now()
+
+	cs := cryptoservice.NewCryptoService(trustmanager.NewKeyMemoryStore(
+		passphrase.ConstantRetriever("password")))
+
+	rootPublicKey, err := cs.Create(data.CanonicalRootRole, gun, data.ECDSAKey)
+	require.NoError(t, err)
+	rootPrivateKey, _, err := cs.GetPrivateKey(rootPublicKey.ID())
+	require.NoError(t, err)
+	oldRootCert, err := cryptoservice.GenerateCertificate(rootPrivateKey, gun, referenceTime.AddDate(-9, 0, 0),
+		referenceTime.AddDate(1, 0, 0))
+	require.NoError(t, err)
+	oldRootCertKey := trustmanager.CertToKey(oldRootCert)
+
+	repo := initRepoWithRoot(t, cs, oldRootCertKey)
+
+	// Create a first signature, using the old key.
+	signedRoot, err := repo.SignRoot(data.DefaultExpires(data.CanonicalRootRole))
+	require.NoError(t, err)
+	verifySignatureList(t, signedRoot, oldRootCertKey)
+	err = verifyRootSignatureAgainstKey(t, signedRoot, oldRootCertKey)
+	require.NoError(t, err)
+
+	// Create a new certificate
+	newRootCert, err := cryptoservice.GenerateCertificate(rootPrivateKey, gun, referenceTime, referenceTime.AddDate(10, 0, 0))
+	require.NoError(t, err)
+	newRootCertKey := trustmanager.CertToKey(newRootCert)
+	require.NotEqual(t, oldRootCertKey.ID(), newRootCertKey.ID())
+
+	// Only trust the new certificate
+	err = repo.ReplaceBaseKeys(data.CanonicalRootRole, newRootCertKey)
+	require.NoError(t, err)
+	updatedRootRole, err := repo.GetBaseRole(data.CanonicalRootRole)
+	require.NoError(t, err)
+	updatedRootKeyIDs := updatedRootRole.ListKeyIDs()
+	require.Equal(t, 1, len(updatedRootKeyIDs))
+	require.Equal(t, newRootCertKey.ID(), updatedRootKeyIDs[0])
+
+	// Now forget all about the old certificate: drop it from the Root carried keys, and set up a new key DB
+	delete(repo.Root.Signed.Keys, oldRootCertKey.ID())
+	repo2 := NewRepo(cs)
+	err = repo2.SetRoot(repo.Root)
+	require.NoError(t, err)
+
+	// Create a second signature
+	signedRoot, err = repo2.SignRoot(data.DefaultExpires(data.CanonicalRootRole))
+	require.NoError(t, err)
+	verifySignatureList(t, signedRoot, newRootCertKey) // Without oldRootCertKey
+
+	// Verify that the signature can be verified when trusting the new certificate
+	err = verifyRootSignatureAgainstKey(t, signedRoot, newRootCertKey)
+	require.NoError(t, err)
+	err = verifyRootSignatureAgainstKey(t, signedRoot, oldRootCertKey)
+	require.Error(t, err)
+}