docker
/
docs
mirror of https://github.com/docker/docs.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Update the client to have an old builder and a new builder, and to only use 

cached version numbers to check downloaded version numbers of cached data
validates against the old builder.

This also removes the `GetRepo` function of the builder and adds some data
accessors instead that are necessary to do a consistent download and check
versions, that way the downloader doesn't need to fish around in the repo
itself for data in order to figure out what to download.

Signed-off-by: Ying Li <ying.li@docker.com>
This commit is contained in:
Ying Li 2016-04-06 10:33:59 -07:00
parent 04ec865b31
commit 5acab543e4
4 changed files with 180 additions and 105 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

56
client/client.go
View File

@ -663,7 +663,10 @@ func (r *NotaryRepository) bootstrapRepo() error {
}
}
r.tufRepo = b.GetRepo()
tufRepo, err := b.Finish()
if err == nil {
r.tufRepo = tufRepo
}
return nil
}
@ -726,22 +729,17 @@ func (r *NotaryRepository) errRepositoryNotExist() error {
// Update bootstraps a trust anchor (root.json) before updating all the
// metadata from the repo.
func (r *NotaryRepository) Update(forWrite bool) error {
b, err := r.bootstrapClient(forWrite)
c, err := r.bootstrapClient(forWrite)
if err != nil {
if _, ok := err.(store.ErrMetaNotFound); ok {
return r.errRepositoryNotExist()
}
return err
}
remote, remoteErr := getRemoteStore(r.baseURL, r.gun, r.roundTrip)
if remoteErr != nil {
logrus.Error(remoteErr)
}
c := tufclient.NewClient(b, remote, r.fileStore)
repo, err := c.Update()
if err != nil {
// notFound.Resource may include a checksum so when the role is root,
// it will be root.json or root.<checksum>.json. Therefore best we can
// it will be root or root.<checksum>. Therefore best we can
// do it match a "root." prefix
if notFound, ok := err.(store.ErrMetaNotFound); ok && strings.HasPrefix(notFound.Resource, data.CanonicalRootRole+".") {
return r.errRepositoryNotExist()
@ -758,7 +756,7 @@ func (r *NotaryRepository) Update(forWrite bool) error {
// is initialized or not. If set to true, we will always attempt to download
// and return an error if the remote repository errors.
//
// Partially populates r.tufRepo with this root metadata (only; use
// Populates a tuf.RepoBuilder with this root metadata (only use
// tufclient.Client.Update to load the rest).
//
// Fails if the remote server is reachable and does not know the repo
@ -768,28 +766,38 @@ func (r *NotaryRepository) Update(forWrite bool) error {
//
// Returns a tufclient.Client for the remote server, which may not be actually
// operational (if the URL is invalid but a root.json is cached).
func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (tuf.RepoBuilder, error) {
func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (*tufclient.Client, error) {
version := 0
b := tuf.NewRepoBuilder(r.gun, r.CryptoService, r.trustPinning)
oldBuilder := tuf.NewRepoBuilder(r.gun, r.CryptoService, r.trustPinning)
var newBuilder tuf.RepoBuilder
// try to read root from cache first. We will trust this root
// until we detect a problem during update which will cause
// us to download a new root and perform a rotation.
rootJSON, cachedRootErr := r.fileStore.GetMeta(data.CanonicalRootRole, -1)
if cachedRootErr == nil {
if rootJSON, err := r.fileStore.GetMeta(data.CanonicalRootRole, -1); err == nil {
// if we can't load the cached root, fail hard because that is how we pin trust
if err := b.Load(data.CanonicalRootRole, rootJSON, version, false); err != nil {
if err := oldBuilder.Load(data.CanonicalRootRole, rootJSON, version, true); err != nil {
return nil, err
}
// use the old builder to bootstrap the new builder - we're just going to
// verify the same data again, but with this time we want to validate the expiry
version = oldBuilder.GetLoadedVersion(data.CanonicalRootRole)
newBuilder = oldBuilder.BootstrapNewBuilder()
// ignore error - if there's an error, the root won't be loaded
newBuilder.Load(data.CanonicalRootRole, rootJSON, version, false)
}
if newBuilder == nil {
newBuilder = tuf.NewRepoBuilder(r.gun, r.CryptoService, r.trustPinning)
}
remote, remoteErr := getRemoteStore(r.baseURL, r.gun, r.roundTrip)
if remoteErr != nil {
logrus.Error(remoteErr)
} else if cachedRootErr != nil || checkInitialized {
// remoteErr was nil and we had a cachedRootErr (or are specifically
// checking for initialization of the repo).
} else if !newBuilder.IsLoaded(data.CanonicalRootRole) || checkInitialized {
// remoteErr was nil and we were not able to load a root from cache or
// are specifically checking for initialization of the repo.
// if remote store successfully set up, try and get root from remote
// We don't have any local data to determine the size of root, so try the maximum (though it is restricted at 100MB)
@ -799,10 +807,10 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (tuf.RepoBuild
// the server. Nothing we can do but error.
return nil, err
}
if cachedRootErr != nil {
// we always want to use the downloaded root if there was a cache get
// error.
if err := b.Load(data.CanonicalRootRole, tmpJSON, version, false); err != nil {
if !newBuilder.IsLoaded(data.CanonicalRootRole) {
// we always want to use the downloaded root if we couldn't load from cache
if err := newBuilder.Load(data.CanonicalRootRole, tmpJSON, version, false); err != nil {
return nil, err
}
@ -814,11 +822,11 @@ func (r *NotaryRepository) bootstrapClient(checkInitialized bool) (tuf.RepoBuild
}
}
if !b.IsLoaded(data.CanonicalRootRole) {
if !newBuilder.IsLoaded(data.CanonicalRootRole) {
return nil, ErrRepoNotInitialized{}
}
return b, nil
return tufclient.NewClient(oldBuilder, newBuilder, remote, r.fileStore), nil
}
// RotateKey removes all existing keys associated with the role, and either

13
client/client_update_test.go
View File

@ -236,8 +236,11 @@ func TestUpdateReplacesCorruptOrMissingMetadata(t *testing.T) {
for _, forWrite := range []bool{true, false} {
require.NoError(t, messItUp(repoSwizzler, role), "could not fuzz %s (%s)", role, text)
err := repo.Update(forWrite)
// if this is a root role, we should error if it's corrupted or invalid data; missing metadata is ok
if role == data.CanonicalRootRole && expt.desc != "missing metadata" {
// If this is a root role, we should error if it's corrupted or invalid data;
// missing metadata is ok.
if role == data.CanonicalRootRole && expt.desc != "missing metadata" &&
expt.desc != "expired metadata" {
require.Error(t, err, "%s for %s: expected to error when bootstrapping root", text, role)
// revert our original metadata
for role := range origMeta {
@ -1252,9 +1255,9 @@ func testUpdateLocalAndRemoteRootCorrupt(t *testing.T, forWrite bool, localExpt,
require.Error(t, err, "expected failure updating when %s", msg)
expectedErrs := serverExpt.expectErrs
// if the local root is corrupt or invalid, we won't even try to update and
// will fail with the local metadata error
if localExpt.desc != "missing metadata" {
// If the local root is corrupt or invalid, we won't even try to update and
// will fail with the local metadata error. Missing or expired metadata is ok.
if localExpt.desc != "missing metadata" && localExpt.desc != "expired metadata" {
expectedErrs = localExpt.expectErrs
}

95
tuf/builder.go
View File

@ -4,10 +4,12 @@ import (
"fmt"
"github.com/docker/go/canonical/json"
"github.com/docker/notary"
"github.com/docker/notary/trustpinning"
"github.com/docker/notary/tuf/data"
"github.com/docker/notary/tuf/signed"
"github.com/docker/notary/tuf/utils"
)
// ErrBuildDone is returned when any functions are called on RepoBuilder, and it
@ -34,6 +36,35 @@ func (e ErrInvalidBuilderInput) Error() string {
return e.msg
}
// ConsistentInfo is the consistent name and size of a role, or just the name
// of the role and a -1 if no file metadata for the role is known
type ConsistentInfo struct {
RoleName string
fileMeta data.FileMeta
}
// ChecksumKnown determines whether or not we know enough to provide a size and
// consistent name
func (c ConsistentInfo) ChecksumKnown() bool {
// empty hash, no size : this is the zero value
return len(c.fileMeta.Hashes) > 0 || c.fileMeta.Length != 0
}
// ConsistentName returns the consistent name (rolename.sha256) for the role
// given this consistent information
func (c ConsistentInfo) ConsistentName() string {
return utils.ConsistentName(c.RoleName, c.fileMeta.Hashes[notary.SHA256])
}
// Length returns the expected length of the role as per this consistent
// information - if no checksum information is known, the size is -1.
func (c ConsistentInfo) Length() int64 {
if c.ChecksumKnown() {
return c.fileMeta.Length
}
return -1
}
// RepoBuilder is an interface for an object which builds a tuf.Repo
type RepoBuilder interface {
Load(roleName string, content []byte, minVersion int, allowExpired bool) error
@ -41,8 +72,11 @@ type RepoBuilder interface {
GenerateTimestamp(prev *data.SignedTimestamp) ([]byte, int, error)
Finish() (*Repo, error)
BootstrapNewBuilder() RepoBuilder
// informative functions
IsLoaded(roleName string) bool
GetRepo() *Repo
GetLoadedVersion(roleName string) int
GetConsistentInfo(roleName string) ConsistentInfo
}
// NewRepoBuilder is the only way to get a pre-built RepoBuilder
@ -71,11 +105,7 @@ type repoBuilder struct {
// needed for bootstrapping a builder to validate a new root
prevRoot *data.SignedRoot
rootChecksum *data.Hashes
}
func (rb *repoBuilder) GetRepo() *Repo {
return rb.repo
rootChecksum *data.FileMeta
}
func (rb *repoBuilder) Finish() (*Repo, error) {
@ -88,11 +118,11 @@ func (rb *repoBuilder) Finish() (*Repo, error) {
}
func (rb *repoBuilder) BootstrapNewBuilder() RepoBuilder {
var rootChecksum *data.Hashes
var rootChecksum *data.FileMeta
if rb.repo.Snapshot != nil {
hashes := rb.repo.Snapshot.Signed.Meta[data.CanonicalRootRole].Hashes
rootChecksum = &hashes
meta := rb.repo.Snapshot.Signed.Meta[data.CanonicalRootRole]
rootChecksum = &meta
}
return &repoBuilder{
@ -164,6 +194,49 @@ func (rb *repoBuilder) IsLoaded(roleName string) bool {
}
}
// GetLoadedVersion returns the metadata version, if it is loaded, or -1 otherwise
func (rb *repoBuilder) GetLoadedVersion(roleName string) int {
switch {
case roleName == data.CanonicalRootRole && rb.repo.Root != nil:
return rb.repo.Root.Signed.Version
case roleName == data.CanonicalSnapshotRole && rb.repo.Snapshot != nil:
return rb.repo.Snapshot.Signed.Version
case roleName == data.CanonicalTimestampRole && rb.repo.Timestamp != nil:
return rb.repo.Timestamp.Signed.Version
default:
if tgts, ok := rb.repo.Targets[roleName]; ok {
return tgts.Signed.Version
}
}
return -1
}
// GetConsistentInfo returns the consistent name and size of a role, if it is known,
// otherwise just the rolename and a -1 for size
func (rb *repoBuilder) GetConsistentInfo(roleName string) ConsistentInfo {
info := ConsistentInfo{RoleName: roleName} // starts out with unknown filemeta
switch roleName {
case data.CanonicalTimestampRole:
// we do not want to get a consistent timestamp, but we do want to
// limit its size
info.fileMeta.Length = notary.MaxTimestampSize
case data.CanonicalSnapshotRole:
if rb.repo.Timestamp != nil {
info.fileMeta = rb.repo.Timestamp.Signed.Meta[roleName]
}
case data.CanonicalRootRole:
if rb.rootChecksum != nil {
info.fileMeta = *rb.rootChecksum
}
default:
if rb.repo.Snapshot != nil {
info.fileMeta = rb.repo.Snapshot.Signed.Meta[roleName]
}
}
return info
}
func (rb *repoBuilder) GenerateSnapshot(prev *data.SignedSnapshot) ([]byte, int, error) {
if rb.IsLoaded(data.CanonicalSnapshotRole) {
return nil, 0, ErrInvalidBuilderInput{msg: "snapshot has already been loaded"}
@ -461,7 +534,7 @@ func (rb *repoBuilder) validateCachedSnapshotChecksums(sn *data.SignedSnapshot)
func (rb *repoBuilder) validateChecksumFor(content []byte, roleName string) error {
// validate the bootstrap checksum for root, if provided
if roleName == data.CanonicalRootRole && rb.rootChecksum != nil {
if err := data.CheckHashes(content, roleName, *rb.rootChecksum); err != nil {
if err := data.CheckHashes(content, roleName, rb.rootChecksum.Hashes); err != nil {
return err
}
}
@ -533,7 +606,7 @@ func (rb *repoBuilder) getChecksumsFor(role string) *data.Hashes {
hashes = rb.repo.Timestamp.Signed.Meta[data.CanonicalSnapshotRole].Hashes
default:
if rb.repo.Snapshot == nil {
return rb.rootChecksum
return nil
}
hashes = rb.repo.Snapshot.Signed.Meta[role].Hashes
}

121
tuf/client/client.go
View File

@ -1,7 +1,6 @@
package client
import (
"encoding/hex"
"encoding/json"
"github.com/Sirupsen/logrus"
@ -9,22 +8,23 @@ import (
tuf "github.com/docker/notary/tuf"
"github.com/docker/notary/tuf/data"
"github.com/docker/notary/tuf/store"
"github.com/docker/notary/tuf/utils"
)
// Client is a usability wrapper around a raw TUF repo
type Client struct {
remote store.RemoteStore
cache store.MetadataStore
builder tuf.RepoBuilder
remote store.RemoteStore
cache store.MetadataStore
oldBuilder tuf.RepoBuilder
newBuilder tuf.RepoBuilder
}
// NewClient initialized a Client with the given repo, remote source of content, and cache
func NewClient(builder tuf.RepoBuilder, remote store.RemoteStore, cache store.MetadataStore) *Client {
func NewClient(oldBuilder, newBuilder tuf.RepoBuilder, remote store.RemoteStore, cache store.MetadataStore) *Client {
return &Client{
builder: builder,
remote: remote,
cache: cache,
oldBuilder: oldBuilder,
newBuilder: newBuilder,
remote: remote,
cache: cache,
}
}
@ -44,10 +44,9 @@ func (c *Client) Update() (*tuf.Repo, error) {
logrus.Debug("Error occurred. Root will be downloaded and another update attempted")
logrus.Debug("Resetting the TUF builder...")
repoSoFar := c.builder.GetRepo()
c.builder = c.builder.BootstrapNewBuilder()
c.newBuilder = c.newBuilder.BootstrapNewBuilder()
if err := c.downloadRoot(repoSoFar.Snapshot); err != nil {
if err := c.downloadRoot(); err != nil {
logrus.Debug("Client Update (Root):", err)
return nil, err
}
@ -58,7 +57,7 @@ func (c *Client) Update() (*tuf.Repo, error) {
return nil, err
}
}
return c.builder.Finish()
return c.newBuilder.Finish()
}
func (c *Client) update() error {
@ -79,26 +78,23 @@ func (c *Client) update() error {
}
// downloadRoot is responsible for downloading the root.json
func (c *Client) downloadRoot(prevSnapshot *data.SignedSnapshot) error {
func (c *Client) downloadRoot() error {
role := data.CanonicalRootRole
consistentInfo := c.newBuilder.GetConsistentInfo(role)
// We can't read an exact size for the root metadata without risking getting stuck in the TUF update cycle
// since it's possible that downloading timestamp/snapshot metadata may fail due to a signature mismatch
if prevSnapshot == nil {
if !consistentInfo.ChecksumKnown() {
logrus.Debugf("Loading root with no expected checksum")
// get the cached root, if it exists, just for version checking
cachedRoot, _ := c.cache.GetMeta(role, -1)
// prefer to download a new root
_, remoteErr := c.tryLoadRemote(role, -1, nil, cachedRoot)
_, remoteErr := c.tryLoadRemote(consistentInfo, cachedRoot)
return remoteErr
}
size := prevSnapshot.Signed.Meta[role].Length
expectedSha256 := prevSnapshot.Signed.Meta[role].Hashes["sha256"]
logrus.Debugf("Loading root with expected checksum %s", hex.EncodeToString(expectedSha256))
_, err := c.tryLoadCacheThenRemote(role, size, expectedSha256)
_, err := c.tryLoadCacheThenRemote(consistentInfo)
return err
}
@ -108,11 +104,12 @@ func (c *Client) downloadRoot(prevSnapshot *data.SignedSnapshot) error {
func (c *Client) downloadTimestamp() error {
logrus.Debug("Loading timestamp...")
role := data.CanonicalTimestampRole
consistentInfo := c.newBuilder.GetConsistentInfo(role)
// get the cached timestamp, if it exists
cachedTS, cachedErr := c.cache.GetMeta(role, notary.MaxTimestampSize)
// always get the remote timestamp, since it supercedes the local one
_, remoteErr := c.tryLoadRemote(role, notary.MaxTimestampSize, nil, cachedTS)
_, remoteErr := c.tryLoadRemote(consistentInfo, cachedTS)
switch {
case remoteErr == nil:
@ -121,7 +118,7 @@ func (c *Client) downloadTimestamp() error {
logrus.Debug(remoteErr.Error())
logrus.Warn("Error while downloading remote metadata, using cached timestamp - this might not be the latest version available remotely")
err := c.builder.Load(role, cachedTS, 0, false)
err := c.newBuilder.Load(role, cachedTS, 0, false)
if err == nil {
logrus.Debug("successfully verified cached timestamp")
}
@ -136,13 +133,9 @@ func (c *Client) downloadTimestamp() error {
func (c *Client) downloadSnapshot() error {
logrus.Debug("Loading snapshot...")
role := data.CanonicalSnapshotRole
timestamp := c.builder.GetRepo().Timestamp
consistentInfo := c.newBuilder.GetConsistentInfo(role)
// we're expecting it's previously been vetted
size := timestamp.Signed.Meta[role].Length
expectedSha256 := timestamp.Signed.Meta[role].Hashes["sha256"]
_, err := c.tryLoadCacheThenRemote(role, size, expectedSha256)
_, err := c.tryLoadCacheThenRemote(consistentInfo)
return err
}
@ -158,7 +151,13 @@ func (c *Client) downloadTargets() error {
role := toDownload[0]
toDownload = toDownload[1:]
children, err := c.getTargetsFile(role)
consistentInfo := c.newBuilder.GetConsistentInfo(role.Name)
if !consistentInfo.ChecksumKnown() {
logrus.Debugf("skipping %s because there is no checksum for it", role.Name)
continue
}
children, err := c.getTargetsFile(role, consistentInfo)
if err != nil {
if _, ok := err.(data.ErrMissingMeta); ok && role.Name != data.CanonicalTargetsRole {
// if the role meta hasn't been published,
@ -173,65 +172,57 @@ func (c *Client) downloadTargets() error {
return nil
}
func (c Client) getTargetsFile(role data.DelegationRole) ([]data.DelegationRole, error) {
func (c Client) getTargetsFile(role data.DelegationRole, ci tuf.ConsistentInfo) ([]data.DelegationRole, error) {
logrus.Debugf("Loading %s...", role.Name)
tgs := &data.SignedTargets{}
// we're expecting it's previously been vetted
roleMeta, err := c.builder.GetRepo().Snapshot.GetMeta(role.Name)
if err != nil {
logrus.Debugf("skipping %s because there is no checksum for it")
return nil, err
}
expectedSha256 := roleMeta.Hashes["sha256"]
size := roleMeta.Length
raw, err := c.tryLoadCacheThenRemote(role.Name, size, expectedSha256)
raw, err := c.tryLoadCacheThenRemote(ci)
if err != nil {
return nil, err
}
// we know it unmarshals fine
json.Unmarshal(raw, tgs)
return tgs.GetValidDelegations(role), nil
}
func (c *Client) tryLoadCacheThenRemote(role string, size int64, expectedSha256 []byte) ([]byte, error) {
cachedTS, err := c.cache.GetMeta(role, size)
func (c *Client) tryLoadCacheThenRemote(consistentInfo tuf.ConsistentInfo) ([]byte, error) {
cachedTS, err := c.cache.GetMeta(consistentInfo.RoleName, consistentInfo.Length())
if err != nil {
logrus.Debugf("no %s in cache, must download", role)
return c.tryLoadRemote(role, size, expectedSha256, nil)
logrus.Debugf("no %s in cache, must download", consistentInfo.RoleName)
return c.tryLoadRemote(consistentInfo, nil)
}
if err = c.builder.Load(role, cachedTS, 0, false); err == nil {
logrus.Debugf("successfully verified cached %s", role)
if err = c.newBuilder.Load(consistentInfo.RoleName, cachedTS, 0, false); err == nil {
logrus.Debugf("successfully verified cached %s", consistentInfo.RoleName)
return cachedTS, nil
}
logrus.Debugf("cached %s is invalid (must download): %s", role, err)
return c.tryLoadRemote(role, size, expectedSha256, cachedTS)
logrus.Debugf("cached %s is invalid (must download): %s", consistentInfo.RoleName, err)
return c.tryLoadRemote(consistentInfo, cachedTS)
}
func (c *Client) tryLoadRemote(role string, size int64, expectedSha256, old []byte) ([]byte, error) {
rolePath := utils.ConsistentName(role, expectedSha256)
raw, err := c.remote.GetMeta(rolePath, size)
func (c *Client) tryLoadRemote(consistentInfo tuf.ConsistentInfo, old []byte) ([]byte, error) {
consistentName := consistentInfo.ConsistentName()
raw, err := c.remote.GetMeta(consistentName, consistentInfo.Length())
if err != nil {
logrus.Debugf("error downloading %s: %s", role, err)
logrus.Debugf("error downloading %s: %s", consistentName, err)
return old, err
}
minVersion := 0
if old != nil && len(old) > 0 {
oldSignedMeta := &data.SignedMeta{}
if readOldErr := json.Unmarshal(old, oldSignedMeta); readOldErr == nil {
minVersion = oldSignedMeta.Signed.Version
}
}
if err := c.builder.Load(role, raw, minVersion, false); err != nil {
logrus.Debugf("downloaded %s is invalid: %s", role, err)
// try to load the old data into the old builder - only use it to validate
// versions if it loads successfully. If it errors, then the loaded version
// will be 0
c.oldBuilder.Load(consistentInfo.RoleName, old, 0, true)
minVersion := c.oldBuilder.GetLoadedVersion(consistentInfo.RoleName)
if err := c.newBuilder.Load(consistentInfo.RoleName, raw, minVersion, false); err != nil {
logrus.Debugf("downloaded %s is invalid: %s", consistentName, err)
return raw, err
}
logrus.Debugf("successfully verified downloaded %s", role)
if err := c.cache.SetMeta(role, raw); err != nil {
logrus.Debugf("Unable to write %s to cache: %s", role, err)
logrus.Debugf("successfully verified downloaded %s", consistentName)
if err := c.cache.SetMeta(consistentInfo.RoleName, raw); err != nil {
logrus.Debugf("Unable to write %s to cache: %s", consistentInfo.RoleName, err)
}
return raw, nil
}