diff --git a/client/client.go b/client/client.go index 675b575815..12f2af5933 100644 --- a/client/client.go +++ b/client/client.go @@ -663,7 +663,10 @@ func (r *NotaryRepository) bootstrapRepo() error { } } - r.tufRepo = b.GetRepo() + tufRepo, err := b.Finish() + if err == nil { + r.tufRepo = tufRepo + } return nil } @@ -726,22 +729,17 @@ func (r *NotaryRepository) errRepositoryNotExist() error { // Update bootstraps a trust anchor (root.json) before updating all the // metadata from the repo. func (r *NotaryRepository) Update(forWrite bool) error { - b, err := r.bootstrapClient(forWrite) + c, err := r.bootstrapClient(forWrite) if err != nil { if _, ok := err.(store.ErrMetaNotFound); ok { return r.errRepositoryNotExist() } return err } - remote, remoteErr := getRemoteStore(r.baseURL, r.gun, r.roundTrip) - if remoteErr != nil { - logrus.Error(remoteErr) - } - c := tufclient.NewClient(b, remote, r.fileStore) repo, err := c.Update() if err != nil { // notFound.Resource may include a checksum so when the role is root, - // it will be root.json or root..json. Therefore best we can + // it will be root or root.. Therefore best we can // do it match a "root." prefix if notFound, ok := err.(store.ErrMetaNotFound); ok && strings.HasPrefix(notFound.Resource, data.CanonicalRootRole+".") { return r.errRepositoryNotExist() @@ -758,7 +756,7 @@ func (r *NotaryRepository) Update(forWrite bool) error { // is initialized or not. If set to true, we will always attempt to download // and return an error if the remote repository errors. // -// Partially populates r.tufRepo with this root metadata (only; use +// Populates a tuf.RepoBuilder with this root metadata (only use // tufclient.Client.Update to load the rest). // // Fails if the remote server is reachable and does not know the repo @@ -768,28 +766,38 @@ func (r *NotaryRepository) Update(forWrite bool) error { // // Returns a tufclient.Client for the remote server, which may not be actually // operational (if the URL is invalid but a root.json is cached). -func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (tuf.RepoBuilder, error) { +func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (*tufclient.Client, error) { version := 0 - b := tuf.NewRepoBuilder(r.gun, r.CryptoService, r.trustPinning) + oldBuilder := tuf.NewRepoBuilder(r.gun, r.CryptoService, r.trustPinning) + var newBuilder tuf.RepoBuilder // try to read root from cache first. We will trust this root // until we detect a problem during update which will cause // us to download a new root and perform a rotation. - rootJSON, cachedRootErr := r.fileStore.GetMeta(data.CanonicalRootRole, -1) - - if cachedRootErr == nil { + if rootJSON, err := r.fileStore.GetMeta(data.CanonicalRootRole, -1); err == nil { // if we can't load the cached root, fail hard because that is how we pin trust - if err := b.Load(data.CanonicalRootRole, rootJSON, version, false); err != nil { + if err := oldBuilder.Load(data.CanonicalRootRole, rootJSON, version, true); err != nil { return nil, err } + + // use the old builder to bootstrap the new builder - we're just going to + // verify the same data again, but with this time we want to validate the expiry + version = oldBuilder.GetLoadedVersion(data.CanonicalRootRole) + newBuilder = oldBuilder.BootstrapNewBuilder() + // ignore error - if there's an error, the root won't be loaded + newBuilder.Load(data.CanonicalRootRole, rootJSON, version, false) + } + + if newBuilder == nil { + newBuilder = tuf.NewRepoBuilder(r.gun, r.CryptoService, r.trustPinning) } remote, remoteErr := getRemoteStore(r.baseURL, r.gun, r.roundTrip) if remoteErr != nil { logrus.Error(remoteErr) - } else if cachedRootErr != nil || checkInitialized { - // remoteErr was nil and we had a cachedRootErr (or are specifically - // checking for initialization of the repo). + } else if !newBuilder.IsLoaded(data.CanonicalRootRole) || checkInitialized { + // remoteErr was nil and we were not able to load a root from cache or + // are specifically checking for initialization of the repo. // if remote store successfully set up, try and get root from remote // We don't have any local data to determine the size of root, so try the maximum (though it is restricted at 100MB) @@ -799,10 +807,10 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (tuf.RepoBuild // the server. Nothing we can do but error. return nil, err } - if cachedRootErr != nil { - // we always want to use the downloaded root if there was a cache get - // error. - if err := b.Load(data.CanonicalRootRole, tmpJSON, version, false); err != nil { + + if !newBuilder.IsLoaded(data.CanonicalRootRole) { + // we always want to use the downloaded root if we couldn't load from cache + if err := newBuilder.Load(data.CanonicalRootRole, tmpJSON, version, false); err != nil { return nil, err } @@ -814,11 +822,11 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (tuf.RepoBuild } } - if !b.IsLoaded(data.CanonicalRootRole) { + if !newBuilder.IsLoaded(data.CanonicalRootRole) { return nil, ErrRepoNotInitialized{} } - return b, nil + return tufclient.NewClient(oldBuilder, newBuilder, remote, r.fileStore), nil } // RotateKey removes all existing keys associated with the role, and either diff --git a/client/client_update_test.go b/client/client_update_test.go index cd53f50aad..433f8fe3f4 100644 --- a/client/client_update_test.go +++ b/client/client_update_test.go @@ -236,8 +236,11 @@ func TestUpdateReplacesCorruptOrMissingMetadata(t *testing.T) { for _, forWrite := range []bool{true, false} { require.NoError(t, messItUp(repoSwizzler, role), "could not fuzz %s (%s)", role, text) err := repo.Update(forWrite) - // if this is a root role, we should error if it's corrupted or invalid data; missing metadata is ok - if role == data.CanonicalRootRole && expt.desc != "missing metadata" { + // If this is a root role, we should error if it's corrupted or invalid data; + // missing metadata is ok. + if role == data.CanonicalRootRole && expt.desc != "missing metadata" && + expt.desc != "expired metadata" { + require.Error(t, err, "%s for %s: expected to error when bootstrapping root", text, role) // revert our original metadata for role := range origMeta { @@ -1252,9 +1255,9 @@ func testUpdateLocalAndRemoteRootCorrupt(t *testing.T, forWrite bool, localExpt, require.Error(t, err, "expected failure updating when %s", msg) expectedErrs := serverExpt.expectErrs - // if the local root is corrupt or invalid, we won't even try to update and - // will fail with the local metadata error - if localExpt.desc != "missing metadata" { + // If the local root is corrupt or invalid, we won't even try to update and + // will fail with the local metadata error. Missing or expired metadata is ok. + if localExpt.desc != "missing metadata" && localExpt.desc != "expired metadata" { expectedErrs = localExpt.expectErrs } diff --git a/tuf/builder.go b/tuf/builder.go index 4a0c73ab8b..f1ac5cfcef 100644 --- a/tuf/builder.go +++ b/tuf/builder.go @@ -4,10 +4,12 @@ import ( "fmt" "github.com/docker/go/canonical/json" + "github.com/docker/notary" "github.com/docker/notary/trustpinning" "github.com/docker/notary/tuf/data" "github.com/docker/notary/tuf/signed" + "github.com/docker/notary/tuf/utils" ) // ErrBuildDone is returned when any functions are called on RepoBuilder, and it @@ -34,6 +36,35 @@ func (e ErrInvalidBuilderInput) Error() string { return e.msg } +// ConsistentInfo is the consistent name and size of a role, or just the name +// of the role and a -1 if no file metadata for the role is known +type ConsistentInfo struct { + RoleName string + fileMeta data.FileMeta +} + +// ChecksumKnown determines whether or not we know enough to provide a size and +// consistent name +func (c ConsistentInfo) ChecksumKnown() bool { + // empty hash, no size : this is the zero value + return len(c.fileMeta.Hashes) > 0 || c.fileMeta.Length != 0 +} + +// ConsistentName returns the consistent name (rolename.sha256) for the role +// given this consistent information +func (c ConsistentInfo) ConsistentName() string { + return utils.ConsistentName(c.RoleName, c.fileMeta.Hashes[notary.SHA256]) +} + +// Length returns the expected length of the role as per this consistent +// information - if no checksum information is known, the size is -1. +func (c ConsistentInfo) Length() int64 { + if c.ChecksumKnown() { + return c.fileMeta.Length + } + return -1 +} + // RepoBuilder is an interface for an object which builds a tuf.Repo type RepoBuilder interface { Load(roleName string, content []byte, minVersion int, allowExpired bool) error @@ -41,8 +72,11 @@ type RepoBuilder interface { GenerateTimestamp(prev *data.SignedTimestamp) ([]byte, int, error) Finish() (*Repo, error) BootstrapNewBuilder() RepoBuilder + + // informative functions IsLoaded(roleName string) bool - GetRepo() *Repo + GetLoadedVersion(roleName string) int + GetConsistentInfo(roleName string) ConsistentInfo } // NewRepoBuilder is the only way to get a pre-built RepoBuilder @@ -71,11 +105,7 @@ type repoBuilder struct { // needed for bootstrapping a builder to validate a new root prevRoot *data.SignedRoot - rootChecksum *data.Hashes -} - -func (rb *repoBuilder) GetRepo() *Repo { - return rb.repo + rootChecksum *data.FileMeta } func (rb *repoBuilder) Finish() (*Repo, error) { @@ -88,11 +118,11 @@ func (rb *repoBuilder) Finish() (*Repo, error) { } func (rb *repoBuilder) BootstrapNewBuilder() RepoBuilder { - var rootChecksum *data.Hashes + var rootChecksum *data.FileMeta if rb.repo.Snapshot != nil { - hashes := rb.repo.Snapshot.Signed.Meta[data.CanonicalRootRole].Hashes - rootChecksum = &hashes + meta := rb.repo.Snapshot.Signed.Meta[data.CanonicalRootRole] + rootChecksum = &meta } return &repoBuilder{ @@ -164,6 +194,49 @@ func (rb *repoBuilder) IsLoaded(roleName string) bool { } } +// GetLoadedVersion returns the metadata version, if it is loaded, or -1 otherwise +func (rb *repoBuilder) GetLoadedVersion(roleName string) int { + switch { + case roleName == data.CanonicalRootRole && rb.repo.Root != nil: + return rb.repo.Root.Signed.Version + case roleName == data.CanonicalSnapshotRole && rb.repo.Snapshot != nil: + return rb.repo.Snapshot.Signed.Version + case roleName == data.CanonicalTimestampRole && rb.repo.Timestamp != nil: + return rb.repo.Timestamp.Signed.Version + default: + if tgts, ok := rb.repo.Targets[roleName]; ok { + return tgts.Signed.Version + } + } + + return -1 +} + +// GetConsistentInfo returns the consistent name and size of a role, if it is known, +// otherwise just the rolename and a -1 for size +func (rb *repoBuilder) GetConsistentInfo(roleName string) ConsistentInfo { + info := ConsistentInfo{RoleName: roleName} // starts out with unknown filemeta + switch roleName { + case data.CanonicalTimestampRole: + // we do not want to get a consistent timestamp, but we do want to + // limit its size + info.fileMeta.Length = notary.MaxTimestampSize + case data.CanonicalSnapshotRole: + if rb.repo.Timestamp != nil { + info.fileMeta = rb.repo.Timestamp.Signed.Meta[roleName] + } + case data.CanonicalRootRole: + if rb.rootChecksum != nil { + info.fileMeta = *rb.rootChecksum + } + default: + if rb.repo.Snapshot != nil { + info.fileMeta = rb.repo.Snapshot.Signed.Meta[roleName] + } + } + return info +} + func (rb *repoBuilder) GenerateSnapshot(prev *data.SignedSnapshot) ([]byte, int, error) { if rb.IsLoaded(data.CanonicalSnapshotRole) { return nil, 0, ErrInvalidBuilderInput{msg: "snapshot has already been loaded"} @@ -461,7 +534,7 @@ func (rb *repoBuilder) validateCachedSnapshotChecksums(sn *data.SignedSnapshot) func (rb *repoBuilder) validateChecksumFor(content []byte, roleName string) error { // validate the bootstrap checksum for root, if provided if roleName == data.CanonicalRootRole && rb.rootChecksum != nil { - if err := data.CheckHashes(content, roleName, *rb.rootChecksum); err != nil { + if err := data.CheckHashes(content, roleName, rb.rootChecksum.Hashes); err != nil { return err } } @@ -533,7 +606,7 @@ func (rb *repoBuilder) getChecksumsFor(role string) *data.Hashes { hashes = rb.repo.Timestamp.Signed.Meta[data.CanonicalSnapshotRole].Hashes default: if rb.repo.Snapshot == nil { - return rb.rootChecksum + return nil } hashes = rb.repo.Snapshot.Signed.Meta[role].Hashes } diff --git a/tuf/client/client.go b/tuf/client/client.go index 4f8c587fca..337915a8f1 100644 --- a/tuf/client/client.go +++ b/tuf/client/client.go @@ -1,7 +1,6 @@ package client import ( - "encoding/hex" "encoding/json" "github.com/Sirupsen/logrus" @@ -9,22 +8,23 @@ import ( tuf "github.com/docker/notary/tuf" "github.com/docker/notary/tuf/data" "github.com/docker/notary/tuf/store" - "github.com/docker/notary/tuf/utils" ) // Client is a usability wrapper around a raw TUF repo type Client struct { - remote store.RemoteStore - cache store.MetadataStore - builder tuf.RepoBuilder + remote store.RemoteStore + cache store.MetadataStore + oldBuilder tuf.RepoBuilder + newBuilder tuf.RepoBuilder } // NewClient initialized a Client with the given repo, remote source of content, and cache -func NewClient(builder tuf.RepoBuilder, remote store.RemoteStore, cache store.MetadataStore) *Client { +func NewClient(oldBuilder, newBuilder tuf.RepoBuilder, remote store.RemoteStore, cache store.MetadataStore) *Client { return &Client{ - builder: builder, - remote: remote, - cache: cache, + oldBuilder: oldBuilder, + newBuilder: newBuilder, + remote: remote, + cache: cache, } } @@ -44,10 +44,9 @@ func (c *Client) Update() (*tuf.Repo, error) { logrus.Debug("Error occurred. Root will be downloaded and another update attempted") logrus.Debug("Resetting the TUF builder...") - repoSoFar := c.builder.GetRepo() - c.builder = c.builder.BootstrapNewBuilder() + c.newBuilder = c.newBuilder.BootstrapNewBuilder() - if err := c.downloadRoot(repoSoFar.Snapshot); err != nil { + if err := c.downloadRoot(); err != nil { logrus.Debug("Client Update (Root):", err) return nil, err } @@ -58,7 +57,7 @@ func (c *Client) Update() (*tuf.Repo, error) { return nil, err } } - return c.builder.Finish() + return c.newBuilder.Finish() } func (c *Client) update() error { @@ -79,26 +78,23 @@ func (c *Client) update() error { } // downloadRoot is responsible for downloading the root.json -func (c *Client) downloadRoot(prevSnapshot *data.SignedSnapshot) error { +func (c *Client) downloadRoot() error { role := data.CanonicalRootRole + consistentInfo := c.newBuilder.GetConsistentInfo(role) // We can't read an exact size for the root metadata without risking getting stuck in the TUF update cycle // since it's possible that downloading timestamp/snapshot metadata may fail due to a signature mismatch - if prevSnapshot == nil { + if !consistentInfo.ChecksumKnown() { logrus.Debugf("Loading root with no expected checksum") // get the cached root, if it exists, just for version checking cachedRoot, _ := c.cache.GetMeta(role, -1) // prefer to download a new root - _, remoteErr := c.tryLoadRemote(role, -1, nil, cachedRoot) + _, remoteErr := c.tryLoadRemote(consistentInfo, cachedRoot) return remoteErr } - size := prevSnapshot.Signed.Meta[role].Length - expectedSha256 := prevSnapshot.Signed.Meta[role].Hashes["sha256"] - logrus.Debugf("Loading root with expected checksum %s", hex.EncodeToString(expectedSha256)) - - _, err := c.tryLoadCacheThenRemote(role, size, expectedSha256) + _, err := c.tryLoadCacheThenRemote(consistentInfo) return err } @@ -108,11 +104,12 @@ func (c *Client) downloadRoot(prevSnapshot *data.SignedSnapshot) error { func (c *Client) downloadTimestamp() error { logrus.Debug("Loading timestamp...") role := data.CanonicalTimestampRole + consistentInfo := c.newBuilder.GetConsistentInfo(role) // get the cached timestamp, if it exists cachedTS, cachedErr := c.cache.GetMeta(role, notary.MaxTimestampSize) // always get the remote timestamp, since it supercedes the local one - _, remoteErr := c.tryLoadRemote(role, notary.MaxTimestampSize, nil, cachedTS) + _, remoteErr := c.tryLoadRemote(consistentInfo, cachedTS) switch { case remoteErr == nil: @@ -121,7 +118,7 @@ func (c *Client) downloadTimestamp() error { logrus.Debug(remoteErr.Error()) logrus.Warn("Error while downloading remote metadata, using cached timestamp - this might not be the latest version available remotely") - err := c.builder.Load(role, cachedTS, 0, false) + err := c.newBuilder.Load(role, cachedTS, 0, false) if err == nil { logrus.Debug("successfully verified cached timestamp") } @@ -136,13 +133,9 @@ func (c *Client) downloadTimestamp() error { func (c *Client) downloadSnapshot() error { logrus.Debug("Loading snapshot...") role := data.CanonicalSnapshotRole - timestamp := c.builder.GetRepo().Timestamp + consistentInfo := c.newBuilder.GetConsistentInfo(role) - // we're expecting it's previously been vetted - size := timestamp.Signed.Meta[role].Length - expectedSha256 := timestamp.Signed.Meta[role].Hashes["sha256"] - - _, err := c.tryLoadCacheThenRemote(role, size, expectedSha256) + _, err := c.tryLoadCacheThenRemote(consistentInfo) return err } @@ -158,7 +151,13 @@ func (c *Client) downloadTargets() error { role := toDownload[0] toDownload = toDownload[1:] - children, err := c.getTargetsFile(role) + consistentInfo := c.newBuilder.GetConsistentInfo(role.Name) + if !consistentInfo.ChecksumKnown() { + logrus.Debugf("skipping %s because there is no checksum for it", role.Name) + continue + } + + children, err := c.getTargetsFile(role, consistentInfo) if err != nil { if _, ok := err.(data.ErrMissingMeta); ok && role.Name != data.CanonicalTargetsRole { // if the role meta hasn't been published, @@ -173,65 +172,57 @@ func (c *Client) downloadTargets() error { return nil } -func (c Client) getTargetsFile(role data.DelegationRole) ([]data.DelegationRole, error) { +func (c Client) getTargetsFile(role data.DelegationRole, ci tuf.ConsistentInfo) ([]data.DelegationRole, error) { logrus.Debugf("Loading %s...", role.Name) tgs := &data.SignedTargets{} - // we're expecting it's previously been vetted - roleMeta, err := c.builder.GetRepo().Snapshot.GetMeta(role.Name) - if err != nil { - logrus.Debugf("skipping %s because there is no checksum for it") - return nil, err - } - expectedSha256 := roleMeta.Hashes["sha256"] - size := roleMeta.Length - - raw, err := c.tryLoadCacheThenRemote(role.Name, size, expectedSha256) + raw, err := c.tryLoadCacheThenRemote(ci) if err != nil { return nil, err } + // we know it unmarshals fine json.Unmarshal(raw, tgs) return tgs.GetValidDelegations(role), nil } -func (c *Client) tryLoadCacheThenRemote(role string, size int64, expectedSha256 []byte) ([]byte, error) { - cachedTS, err := c.cache.GetMeta(role, size) +func (c *Client) tryLoadCacheThenRemote(consistentInfo tuf.ConsistentInfo) ([]byte, error) { + cachedTS, err := c.cache.GetMeta(consistentInfo.RoleName, consistentInfo.Length()) if err != nil { - logrus.Debugf("no %s in cache, must download", role) - return c.tryLoadRemote(role, size, expectedSha256, nil) + logrus.Debugf("no %s in cache, must download", consistentInfo.RoleName) + return c.tryLoadRemote(consistentInfo, nil) } - if err = c.builder.Load(role, cachedTS, 0, false); err == nil { - logrus.Debugf("successfully verified cached %s", role) + if err = c.newBuilder.Load(consistentInfo.RoleName, cachedTS, 0, false); err == nil { + logrus.Debugf("successfully verified cached %s", consistentInfo.RoleName) return cachedTS, nil } - logrus.Debugf("cached %s is invalid (must download): %s", role, err) - return c.tryLoadRemote(role, size, expectedSha256, cachedTS) + logrus.Debugf("cached %s is invalid (must download): %s", consistentInfo.RoleName, err) + return c.tryLoadRemote(consistentInfo, cachedTS) } -func (c *Client) tryLoadRemote(role string, size int64, expectedSha256, old []byte) ([]byte, error) { - rolePath := utils.ConsistentName(role, expectedSha256) - raw, err := c.remote.GetMeta(rolePath, size) +func (c *Client) tryLoadRemote(consistentInfo tuf.ConsistentInfo, old []byte) ([]byte, error) { + consistentName := consistentInfo.ConsistentName() + raw, err := c.remote.GetMeta(consistentName, consistentInfo.Length()) if err != nil { - logrus.Debugf("error downloading %s: %s", role, err) + logrus.Debugf("error downloading %s: %s", consistentName, err) return old, err } - minVersion := 0 - if old != nil && len(old) > 0 { - oldSignedMeta := &data.SignedMeta{} - if readOldErr := json.Unmarshal(old, oldSignedMeta); readOldErr == nil { - minVersion = oldSignedMeta.Signed.Version - } - } - if err := c.builder.Load(role, raw, minVersion, false); err != nil { - logrus.Debugf("downloaded %s is invalid: %s", role, err) + + // try to load the old data into the old builder - only use it to validate + // versions if it loads successfully. If it errors, then the loaded version + // will be 0 + c.oldBuilder.Load(consistentInfo.RoleName, old, 0, true) + minVersion := c.oldBuilder.GetLoadedVersion(consistentInfo.RoleName) + + if err := c.newBuilder.Load(consistentInfo.RoleName, raw, minVersion, false); err != nil { + logrus.Debugf("downloaded %s is invalid: %s", consistentName, err) return raw, err } - logrus.Debugf("successfully verified downloaded %s", role) - if err := c.cache.SetMeta(role, raw); err != nil { - logrus.Debugf("Unable to write %s to cache: %s", role, err) + logrus.Debugf("successfully verified downloaded %s", consistentName) + if err := c.cache.SetMeta(consistentInfo.RoleName, raw); err != nil { + logrus.Debugf("Unable to write %s to cache: %s", consistentInfo.RoleName, err) } return raw, nil }