From 41e20cecb9944137de82ff5fa0898f953aa2bf87 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Thu, 11 Dec 2014 17:55:15 -0800 Subject: [PATCH 01/29] Adds support for v2 registry login summary of changes: registry/auth.go - More logging around the login functions - split Login() out to handle different code paths for v1 (unchanged logic) and v2 (does not currently do account creation) - handling for either basic or token based login attempts registry/authchallenge.go - New File - credit to Brian Bland (github: BrianBland) - handles parsing of WWW-Authenticate response headers registry/endpoint.go - EVEN MOAR LOGGING - Many edits throught to make the coad less dense. Sparse code is more readable code. - slit Ping() out to handle different code paths for v1 (unchanged logic) and v2. - Updated Endpoint struct type to include an entry for authorization challenges discovered during ping of a v2 registry. - If registry endpoint version is unknown, v2 code path is first attempted, then fallback to v1 upon failure. registry/service.go - STILL MOAR LOGGING - simplified the logic around starting the 'auth' job. registry/session.go - updated use of a registry.Endpoint struct field. registry/token.go - New File - Handles getting token from the parameters of a token auth challenge. - Modified from function written by Brian Bland (see above credit). registry/types.go - Removed 'DefaultAPIVersion' in lieu of 'APIVersionUnknown = 0'` Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- registry/auth.go | 114 ++++++++++++++++++++++++++- registry/authchallenge.go | 150 ++++++++++++++++++++++++++++++++++++ registry/endpoint.go | 158 ++++++++++++++++++++++++++++---------- registry/endpoint_test.go | 6 +- registry/service.go | 42 ++++++---- registry/session.go | 2 +- registry/token.go | 70 +++++++++++++++++ registry/types.go | 5 +- 8 files changed, 484 insertions(+), 63 deletions(-) create mode 100644 registry/authchallenge.go create mode 100644 registry/token.go diff --git a/registry/auth.go b/registry/auth.go index 102078d7a2..2044236cfb 100644 --- a/registry/auth.go +++ b/registry/auth.go @@ -11,6 +11,7 @@ import ( "path" "strings" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" ) @@ -144,8 +145,18 @@ func SaveConfig(configFile *ConfigFile) error { return nil } -// try to register/login to the registry server -func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { +// Login tries to register/login to the registry server. +func Login(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { + // Separates the v2 registry login logic from the v1 logic. + if registryEndpoint.Version == APIVersion2 { + return loginV2(authConfig, registryEndpoint, factory) + } + + return loginV1(authConfig, registryEndpoint, factory) +} + +// loginV1 tries to register/login to the v1 registry server. +func loginV1(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { var ( status string reqBody []byte @@ -161,6 +172,8 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e serverAddress = authConfig.ServerAddress ) + log.Debugf("attempting v1 login to registry endpoint %s", registryEndpoint) + if serverAddress == "" { return "", fmt.Errorf("Server Error: Server Address not set.") } @@ -253,6 +266,103 @@ func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, e return status, nil } +// loginV2 tries to login to the v2 registry server. The given registry endpoint has been +// pinged or setup with a list of authorization challenges. Each of these challenges are +// tried until one of them succeeds. Currently supported challenge schemes are: +// HTTP Basic Authorization +// Token Authorization with a separate token issuing server +// NOTE: the v2 logic does not attempt to create a user account if one doesn't exist. For +// now, users should create their account through other means like directly from a web page +// served by the v2 registry service provider. Whether this will be supported in the future +// is to be determined. +func loginV2(authConfig *AuthConfig, registryEndpoint *Endpoint, factory *utils.HTTPRequestFactory) (string, error) { + log.Debugf("attempting v2 login to registry endpoint %s", registryEndpoint) + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } + + var ( + err error + allErrors []error + ) + + for _, challenge := range registryEndpoint.AuthChallenges { + log.Debugf("trying %q auth challenge with params %s", challenge.Scheme, challenge.Parameters) + + switch strings.ToLower(challenge.Scheme) { + case "basic": + err = tryV2BasicAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + case "bearer": + err = tryV2TokenAuthLogin(authConfig, challenge.Parameters, registryEndpoint, client, factory) + default: + // Unsupported challenge types are explicitly skipped. + err = fmt.Errorf("unsupported auth scheme: %q", challenge.Scheme) + } + + if err == nil { + return "Login Succeeded", nil + } + + log.Debugf("error trying auth challenge %q: %s", challenge.Scheme, err) + + allErrors = append(allErrors, err) + } + + return "", fmt.Errorf("no successful auth challenge for %s - errors: %s", registryEndpoint, allErrors) +} + +func tryV2BasicAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { + req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + if err != nil { + return err + } + + req.SetBasicAuth(authConfig.Username, authConfig.Password) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("basic auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + return nil +} + +func tryV2TokenAuthLogin(authConfig *AuthConfig, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) error { + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + if err != nil { + return err + } + + req, err := factory.NewRequest("GET", registryEndpoint.Path(""), nil) + if err != nil { + return err + } + + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + + resp, err := client.Do(req) + if err != nil { + return err + } + defer resp.Body.Close() + + if resp.StatusCode != http.StatusOK { + return fmt.Errorf("token auth attempt to %s realm %q failed with status: %d %s", registryEndpoint, params["realm"], resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + return nil +} + // this method matches a auth configuration to a server address or a url func (config *ConfigFile) ResolveAuthConfig(index *IndexInfo) AuthConfig { configKey := index.GetAuthConfigKey() diff --git a/registry/authchallenge.go b/registry/authchallenge.go new file mode 100644 index 0000000000..e300d82a05 --- /dev/null +++ b/registry/authchallenge.go @@ -0,0 +1,150 @@ +package registry + +import ( + "net/http" + "strings" +) + +// Octet types from RFC 2616. +type octetType byte + +// AuthorizationChallenge carries information +// from a WWW-Authenticate response header. +type AuthorizationChallenge struct { + Scheme string + Parameters map[string]string +} + +var octetTypes [256]octetType + +const ( + isToken octetType = 1 << iota + isSpace +) + +func init() { + // OCTET = + // CHAR = + // CTL = + // CR = + // LF = + // SP = + // HT = + // <"> = + // CRLF = CR LF + // LWS = [CRLF] 1*( SP | HT ) + // TEXT = + // separators = "(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\" | <"> + // | "/" | "[" | "]" | "?" | "=" | "{" | "}" | SP | HT + // token = 1* + // qdtext = > + + for c := 0; c < 256; c++ { + var t octetType + isCtl := c <= 31 || c == 127 + isChar := 0 <= c && c <= 127 + isSeparator := strings.IndexRune(" \t\"(),/:;<=>?@[]\\{}", rune(c)) >= 0 + if strings.IndexRune(" \t\r\n", rune(c)) >= 0 { + t |= isSpace + } + if isChar && !isCtl && !isSeparator { + t |= isToken + } + octetTypes[c] = t + } +} + +func parseAuthHeader(header http.Header) []*AuthorizationChallenge { + var challenges []*AuthorizationChallenge + for _, h := range header[http.CanonicalHeaderKey("WWW-Authenticate")] { + v, p := parseValueAndParams(h) + if v != "" { + challenges = append(challenges, &AuthorizationChallenge{Scheme: v, Parameters: p}) + } + } + return challenges +} + +func parseValueAndParams(header string) (value string, params map[string]string) { + params = make(map[string]string) + value, s := expectToken(header) + if value == "" { + return + } + value = strings.ToLower(value) + s = "," + skipSpace(s) + for strings.HasPrefix(s, ",") { + var pkey string + pkey, s = expectToken(skipSpace(s[1:])) + if pkey == "" { + return + } + if !strings.HasPrefix(s, "=") { + return + } + var pvalue string + pvalue, s = expectTokenOrQuoted(s[1:]) + if pvalue == "" { + return + } + pkey = strings.ToLower(pkey) + params[pkey] = pvalue + s = skipSpace(s) + } + return +} + +func skipSpace(s string) (rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isSpace == 0 { + break + } + } + return s[i:] +} + +func expectToken(s string) (token, rest string) { + i := 0 + for ; i < len(s); i++ { + if octetTypes[s[i]]&isToken == 0 { + break + } + } + return s[:i], s[i:] +} + +func expectTokenOrQuoted(s string) (value string, rest string) { + if !strings.HasPrefix(s, "\"") { + return expectToken(s) + } + s = s[1:] + for i := 0; i < len(s); i++ { + switch s[i] { + case '"': + return s[:i], s[i+1:] + case '\\': + p := make([]byte, len(s)-1) + j := copy(p, s[:i]) + escape := true + for i = i + i; i < len(s); i++ { + b := s[i] + switch { + case escape: + escape = false + p[j] = b + j++ + case b == '\\': + escape = true + case b == '"': + return string(p[:j]), s[i+1:] + default: + p[j] = b + j++ + } + } + return "", "" + } + } + return "", "" +} diff --git a/registry/endpoint.go b/registry/endpoint.go index 95680c5efc..5c5b052000 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -15,28 +15,31 @@ import ( // for mocking in unit tests var lookupIP = net.LookupIP -// scans string for api version in the URL path. returns the trimmed hostname, if version found, string and API version. -func scanForAPIVersion(hostname string) (string, APIVersion) { +// scans string for api version in the URL path. returns the trimmed address, if version found, string and API version. +func scanForAPIVersion(address string) (string, APIVersion) { var ( chunks []string apiVersionStr string ) - if strings.HasSuffix(hostname, "/") { - chunks = strings.Split(hostname[:len(hostname)-1], "/") - apiVersionStr = chunks[len(chunks)-1] - } else { - chunks = strings.Split(hostname, "/") - apiVersionStr = chunks[len(chunks)-1] + + if strings.HasSuffix(address, "/") { + address = address[:len(address)-1] } + + chunks = strings.Split(address, "/") + apiVersionStr = chunks[len(chunks)-1] + for k, v := range apiVersions { if apiVersionStr == v { - hostname = strings.Join(chunks[:len(chunks)-1], "/") - return hostname, k + address = strings.Join(chunks[:len(chunks)-1], "/") + return address, k } } - return hostname, DefaultAPIVersion + + return address, APIVersionUnknown } +// NewEndpoint parses the given address to return a registry endpoint. func NewEndpoint(index *IndexInfo) (*Endpoint, error) { // *TODO: Allow per-registry configuration of endpoints. endpoint, err := newEndpoint(index.GetAuthConfigKey(), index.Secure) @@ -44,81 +47,124 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { return nil, err } + log.Debugf("pinging registry endpoint %s", endpoint) + // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - - //TODO: triggering highland build can be done there without "failing" - if index.Secure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. - return nil, fmt.Errorf("Invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) + return nil, fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } // If registry is insecure and HTTPS failed, fallback to HTTP. log.Debugf("Error from registry %q marked as insecure: %v. Insecurely falling back to HTTP", endpoint, err) endpoint.URL.Scheme = "http" - _, err2 := endpoint.Ping() - if err2 == nil { + + var err2 error + if _, err2 = endpoint.Ping(); err2 == nil { return endpoint, nil } - return nil, fmt.Errorf("Invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) + return nil, fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } return endpoint, nil } -func newEndpoint(hostname string, secure bool) (*Endpoint, error) { + +func newEndpoint(address string, secure bool) (*Endpoint, error) { var ( - endpoint = Endpoint{} - trimmedHostname string - err error + endpoint = new(Endpoint) + trimmedAddress string + err error ) - if !strings.HasPrefix(hostname, "http") { - hostname = "https://" + hostname + + if !strings.HasPrefix(address, "http") { + address = "https://" + address } - trimmedHostname, endpoint.Version = scanForAPIVersion(hostname) - endpoint.URL, err = url.Parse(trimmedHostname) - if err != nil { + + trimmedAddress, endpoint.Version = scanForAPIVersion(address) + + if endpoint.URL, err = url.Parse(trimmedAddress); err != nil { return nil, err } - endpoint.secure = secure - return &endpoint, nil + endpoint.IsSecure = secure + return endpoint, nil } func (repoInfo *RepositoryInfo) GetEndpoint() (*Endpoint, error) { return NewEndpoint(repoInfo.Index) } +// Endpoint stores basic information about a registry endpoint. type Endpoint struct { - URL *url.URL - Version APIVersion - secure bool + URL *url.URL + Version APIVersion + IsSecure bool + AuthChallenges []*AuthorizationChallenge } // Get the formated URL for the root of this registry Endpoint -func (e Endpoint) String() string { - return fmt.Sprintf("%s/v%d/", e.URL.String(), e.Version) +func (e *Endpoint) String() string { + return fmt.Sprintf("%s/v%d/", e.URL, e.Version) } -func (e Endpoint) VersionString(version APIVersion) string { - return fmt.Sprintf("%s/v%d/", e.URL.String(), version) +// VersionString returns a formatted string of this +// endpoint address using the given API Version. +func (e *Endpoint) VersionString(version APIVersion) string { + return fmt.Sprintf("%s/v%d/", e.URL, version) } -func (e Endpoint) Ping() (RegistryInfo, error) { +// Path returns a formatted string for the URL +// of this endpoint with the given path appended. +func (e *Endpoint) Path(path string) string { + return fmt.Sprintf("%s/v%d/%s", e.URL, e.Version, path) +} + +func (e *Endpoint) Ping() (RegistryInfo, error) { + // The ping logic to use is determined by the registry endpoint version. + switch e.Version { + case APIVersion1: + return e.pingV1() + case APIVersion2: + return e.pingV2() + } + + // APIVersionUnknown + // We should try v2 first... + e.Version = APIVersion2 + regInfo, errV2 := e.pingV2() + if errV2 == nil { + return regInfo, nil + } + + // ... then fallback to v1. + e.Version = APIVersion1 + regInfo, errV1 := e.pingV1() + if errV1 == nil { + return regInfo, nil + } + + e.Version = APIVersionUnknown + return RegistryInfo{}, fmt.Errorf("unable to ping registry endpoint %s\nv2 ping attempt failed with error: %s\n v1 ping attempt failed with error: %s", e, errV2, errV1) +} + +func (e *Endpoint) pingV1() (RegistryInfo, error) { + log.Debugf("attempting v1 ping for registry endpoint %s", e) + if e.String() == IndexServerAddress() { - // Skip the check, we now this one is valid + // Skip the check, we know this one is valid // (and we never want to fallback to http in case of error) return RegistryInfo{Standalone: false}, nil } - req, err := http.NewRequest("GET", e.String()+"_ping", nil) + req, err := http.NewRequest("GET", e.Path("_ping"), nil) if err != nil { return RegistryInfo{Standalone: false}, err } - resp, _, err := doRequest(req, nil, ConnectTimeout, e.secure) + resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) if err != nil { return RegistryInfo{Standalone: false}, err } @@ -127,7 +173,7 @@ func (e Endpoint) Ping() (RegistryInfo, error) { jsonString, err := ioutil.ReadAll(resp.Body) if err != nil { - return RegistryInfo{Standalone: false}, fmt.Errorf("Error while reading the http response: %s", err) + return RegistryInfo{Standalone: false}, fmt.Errorf("error while reading the http response: %s", err) } // If the header is absent, we assume true for compatibility with earlier @@ -157,3 +203,33 @@ func (e Endpoint) Ping() (RegistryInfo, error) { log.Debugf("RegistryInfo.Standalone: %t", info.Standalone) return info, nil } + +func (e *Endpoint) pingV2() (RegistryInfo, error) { + log.Debugf("attempting v2 ping for registry endpoint %s", e) + + req, err := http.NewRequest("GET", e.Path(""), nil) + if err != nil { + return RegistryInfo{}, err + } + + resp, _, err := doRequest(req, nil, ConnectTimeout, e.IsSecure) + if err != nil { + return RegistryInfo{}, err + } + defer resp.Body.Close() + + if resp.StatusCode == http.StatusOK { + // It would seem that no authentication/authorization is required. + // So we don't need to parse/add any authorization schemes. + return RegistryInfo{Standalone: true}, nil + } + + if resp.StatusCode == http.StatusUnauthorized { + // Parse the WWW-Authenticate Header and store the challenges + // on this endpoint object. + e.AuthChallenges = parseAuthHeader(resp.Header) + return RegistryInfo{}, nil + } + + return RegistryInfo{}, fmt.Errorf("v2 registry endpoint returned status %d: %q", resp.StatusCode, http.StatusText(resp.StatusCode)) +} diff --git a/registry/endpoint_test.go b/registry/endpoint_test.go index b691a4fb98..f6489034fe 100644 --- a/registry/endpoint_test.go +++ b/registry/endpoint_test.go @@ -8,8 +8,10 @@ func TestEndpointParse(t *testing.T) { expected string }{ {IndexServerAddress(), IndexServerAddress()}, - {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v1/"}, - {"0.0.0.0:5000", "https://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/v1/", "http://0.0.0.0:5000/v1/"}, + {"http://0.0.0.0:5000/v2/", "http://0.0.0.0:5000/v2/"}, + {"http://0.0.0.0:5000", "http://0.0.0.0:5000/v0/"}, + {"0.0.0.0:5000", "https://0.0.0.0:5000/v0/"}, } for _, td := range testData { e, err := newEndpoint(td.str, false) diff --git a/registry/service.go b/registry/service.go index c34e384236..0483402248 100644 --- a/registry/service.go +++ b/registry/service.go @@ -1,6 +1,7 @@ package registry import ( + log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" ) @@ -38,28 +39,39 @@ func (s *Service) Install(eng *engine.Engine) error { // and returns OK if authentication was sucessful. // It can be used to verify the validity of a client's credentials. func (s *Service) Auth(job *engine.Job) engine.Status { - var authConfig = new(AuthConfig) + var ( + authConfig = new(AuthConfig) + endpoint *Endpoint + index *IndexInfo + status string + err error + ) job.GetenvJson("authConfig", authConfig) - if authConfig.ServerAddress != "" { - index, err := ResolveIndexInfo(job, authConfig.ServerAddress) - if err != nil { - return job.Error(err) - } - if !index.Official { - endpoint, err := NewEndpoint(index) - if err != nil { - return job.Error(err) - } - authConfig.ServerAddress = endpoint.String() - } + addr := authConfig.ServerAddress + if addr == "" { + // Use the official registry address if not specified. + addr = IndexServerAddress() } - status, err := Login(authConfig, HTTPRequestFactory(nil)) - if err != nil { + if index, err = ResolveIndexInfo(job, addr); err != nil { return job.Error(err) } + + if endpoint, err = NewEndpoint(index); err != nil { + log.Errorf("unable to get new registry endpoint: %s", err) + return job.Error(err) + } + + authConfig.ServerAddress = endpoint.String() + + if status, err = Login(authConfig, endpoint, HTTPRequestFactory(nil)); err != nil { + log.Errorf("unable to login against registry endpoint %s: %s", endpoint, err) + return job.Error(err) + } + + log.Infof("successful registry login for endpoint %s: %s", endpoint, status) job.Printf("%s\n", status) return engine.StatusOK diff --git a/registry/session.go b/registry/session.go index 781a91b157..b1980e1ae6 100644 --- a/registry/session.go +++ b/registry/session.go @@ -65,7 +65,7 @@ func NewSession(authConfig *AuthConfig, factory *utils.HTTPRequestFactory, endpo } func (r *Session) doRequest(req *http.Request) (*http.Response, *http.Client, error) { - return doRequest(req, r.jar, r.timeout, r.indexEndpoint.secure) + return doRequest(req, r.jar, r.timeout, r.indexEndpoint.IsSecure) } // Retrieve the history of a given image from the Registry. diff --git a/registry/token.go b/registry/token.go new file mode 100644 index 0000000000..0403734f87 --- /dev/null +++ b/registry/token.go @@ -0,0 +1,70 @@ +package registry + +import ( + "errors" + "fmt" + "net/http" + "net/url" + "strings" + + "github.com/docker/docker/utils" +) + +func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) (token string, err error) { + realm, ok := params["realm"] + if !ok { + return "", errors.New("no realm specified for token auth challenge") + } + + realmURL, err := url.Parse(realm) + if err != nil { + return "", fmt.Errorf("invalid token auth challenge realm: %s", err) + } + + if realmURL.Scheme == "" { + if registryEndpoint.IsSecure { + realmURL.Scheme = "https" + } else { + realmURL.Scheme = "http" + } + } + + req, err := factory.NewRequest("GET", realmURL.String(), nil) + if err != nil { + return "", err + } + + reqParams := req.URL.Query() + service := params["service"] + scope := params["scope"] + + if service != "" { + reqParams.Add("service", service) + } + + for _, scopeField := range strings.Fields(scope) { + reqParams.Add("scope", scopeField) + } + + reqParams.Add("account", username) + + req.URL.RawQuery = reqParams.Encode() + req.SetBasicAuth(username, password) + + resp, err := client.Do(req) + if err != nil { + return "", err + } + defer resp.Body.Close() + + if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent) { + return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) + } + + token = resp.Header.Get("X-Auth-Token") + if token == "" { + return "", errors.New("token server did not include a token in the response header") + } + + return token, nil +} diff --git a/registry/types.go b/registry/types.go index fbbc0e7098..bd0bf8b75b 100644 --- a/registry/types.go +++ b/registry/types.go @@ -55,14 +55,15 @@ func (av APIVersion) String() string { return apiVersions[av] } -var DefaultAPIVersion APIVersion = APIVersion1 var apiVersions = map[APIVersion]string{ 1: "v1", 2: "v2", } +// API Version identifiers. const ( - APIVersion1 = iota + 1 + APIVersionUnknown = iota + APIVersion1 APIVersion2 ) From ac8d964b28f23c9790102462a040054e7857cb26 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 22 Oct 2014 11:07:03 -0700 Subject: [PATCH 02/29] Add trust key creation on client Signed-off-by: Derek McGowan (github: dmcgowan) --- docker/docker.go | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index 3137f5c99f..84ffeace9a 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -6,6 +6,7 @@ import ( "fmt" "io/ioutil" "os" + "path" "strings" log "github.com/Sirupsen/logrus" @@ -15,6 +16,7 @@ import ( flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) const ( @@ -77,6 +79,23 @@ func main() { } protoAddrParts := strings.SplitN(flHosts[0], "://", 2) + err := os.MkdirAll(path.Dir(*flTrustKey), 0700) + if err != nil { + log.Fatal(err) + } + trustKey, err := libtrust.LoadKeyFile(*flTrustKey) + if err == libtrust.ErrKeyFileDoesNotExist { + trustKey, err = libtrust.GenerateECP256PrivateKey() + if err != nil { + log.Fatalf("Error generating key: %s", err) + } + if err := libtrust.SaveKey(*flTrustKey, trustKey); err != nil { + log.Fatalf("Error saving key file: %s", err) + } + } else if err != nil { + log.Fatalf("Error loading key file: %s", err) + } + var ( cli *client.DockerCli tlsConfig tls.Config @@ -118,9 +137,9 @@ func main() { } if *flTls || *flTlsVerify { - cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, nil, protoAddrParts[0], protoAddrParts[1], &tlsConfig) + cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, trustKey, protoAddrParts[0], protoAddrParts[1], &tlsConfig) } else { - cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, nil, protoAddrParts[0], protoAddrParts[1], nil) + cli = client.NewDockerCli(os.Stdin, os.Stdout, os.Stderr, trustKey, protoAddrParts[0], protoAddrParts[1], nil) } if err := cli.Cmd(flag.Args()...); err != nil { From 188b56c836e49e3c888e1e27e4e26b5cc0f1caaa Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 30 Sep 2014 17:03:57 -0700 Subject: [PATCH 03/29] Push flow Signed-off-by: Derek McGowan (github: dmcgowan) --- api/client/commands.go | 23 +++++++- api/server/server.go | 19 +++++++ graph/manifest.go | 116 +++++++++++++++++++++++++++++++++++++++++ graph/push.go | 90 ++++++++++++++++++++++++++++++++ graph/service.go | 1 + registry/session_v2.go | 7 ++- 6 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 graph/manifest.go diff --git a/api/client/commands.go b/api/client/commands.go index 06c369958a..2311b898b8 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -43,6 +43,7 @@ import ( "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) const ( @@ -1215,6 +1216,26 @@ func (cli *DockerCli) CmdPush(args ...string) error { v := url.Values{} v.Set("tag", tag) + + body, _, err := readBody(cli.call("GET", "/images/"+remote+"/manifest?"+v.Encode(), nil, false)) + if err != nil { + return err + } + + js, err := libtrust.NewJSONSignature(body) + if err != nil { + return err + } + err = js.Sign(cli.key) + if err != nil { + return err + } + + signedBody, err := js.PrettySignature("signatures") + if err != nil { + return err + } + push := func(authConfig registry.AuthConfig) error { buf, err := json.Marshal(authConfig) if err != nil { @@ -1224,7 +1245,7 @@ func (cli *DockerCli) CmdPush(args ...string) error { base64.URLEncoding.EncodeToString(buf), } - return cli.stream("POST", "/images/"+remote+"/push?"+v.Encode(), nil, cli.out, map[string][]string{ + return cli.stream("POST", "/images/"+remote+"/push?"+v.Encode(), bytes.NewReader(signedBody), cli.out, map[string][]string{ "X-Registry-Auth": registryAuthHeader, }) } diff --git a/api/server/server.go b/api/server/server.go index 4907ff0349..d2715f1bc6 100644 --- a/api/server/server.go +++ b/api/server/server.go @@ -608,6 +608,18 @@ func getImagesSearch(eng *engine.Engine, version version.Version, w http.Respons return job.Run() } +func getImageManifest(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if err := parseForm(r); err != nil { + return err + } + + job := eng.Job("image_manifest", vars["name"]) + job.Setenv("tag", r.Form.Get("tag")) + job.Stdout.Add(utils.NewWriteFlusher(w)) + + return job.Run() +} + func postImagesPush(eng *engine.Engine, version version.Version, w http.ResponseWriter, r *http.Request, vars map[string]string) error { if vars == nil { return fmt.Errorf("Missing parameter") @@ -639,9 +651,15 @@ func postImagesPush(eng *engine.Engine, version version.Version, w http.Response } } + manifest, err := ioutil.ReadAll(r.Body) + if err != nil { + return err + } + job := eng.Job("push", vars["name"]) job.SetenvJson("metaHeaders", metaHeaders) job.SetenvJson("authConfig", authConfig) + job.Setenv("manifest", string(manifest)) job.Setenv("tag", r.Form.Get("tag")) if version.GreaterThan("1.0") { job.SetenvBool("json", true) @@ -1294,6 +1312,7 @@ func createRouter(eng *engine.Engine, logging, enableCors bool, dockerVersion st "/images/viz": getImagesViz, "/images/search": getImagesSearch, "/images/get": getImagesGet, + "/images/{name:.*}/manifest": getImageManifest, "/images/{name:.*}/get": getImagesGet, "/images/{name:.*}/history": getImagesHistory, "/images/{name:.*}/json": getImagesByName, diff --git a/graph/manifest.go b/graph/manifest.go new file mode 100644 index 0000000000..39cabb6e41 --- /dev/null +++ b/graph/manifest.go @@ -0,0 +1,116 @@ +package graph + +import ( + "encoding/json" + "fmt" + "io" + "io/ioutil" + "path" + + "github.com/docker/docker/engine" + "github.com/docker/docker/pkg/tarsum" + "github.com/docker/docker/registry" + "github.com/docker/docker/runconfig" +) + +func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { + if len(job.Args) != 1 { + return job.Errorf("usage: %s NAME", job.Name) + } + name := job.Args[0] + tag := job.Getenv("tag") + if tag == "" { + tag = "latest" + } + + // Resolve the Repository name from fqn to endpoint + name + _, remoteName, err := registry.ResolveRepositoryName(name) + if err != nil { + return job.Error(err) + } + + manifest := ®istry.ManifestData{ + Name: remoteName, + Tag: tag, + SchemaVersion: 1, + } + localRepo, exists := s.Repositories[name] + if !exists { + return job.Errorf("Repo does not exist: %s", name) + } + + layerId, exists := localRepo[tag] + if !exists { + return job.Errorf("Tag does not exist for %s: %s", name, tag) + } + tarsums := make([]string, 0, 4) + layersSeen := make(map[string]bool) + + layer, err := s.graph.Get(layerId) + if err != nil { + return job.Error(err) + } + if layer.Config == nil { + return job.Errorf("Missing layer configuration") + } + manifest.Architecture = layer.Architecture + var metadata runconfig.Config + metadata = *layer.Config + history := make([]string, 0, cap(tarsums)) + + for ; layer != nil; layer, err = layer.GetParent() { + if err != nil { + return job.Error(err) + } + + if layersSeen[layer.ID] { + break + } + if layer.Config != nil && metadata.Image != layer.ID { + err = runconfig.Merge(&metadata, layer.Config) + if err != nil { + return job.Error(err) + } + } + + archive, err := layer.TarLayer() + if err != nil { + return job.Error(err) + } + + tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version0) + if err != nil { + return job.Error(err) + } + if _, err := io.Copy(ioutil.Discard, tarSum); err != nil { + return job.Error(err) + } + + tarId := tarSum.Sum(nil) + // Save tarsum to image json + + tarsums = append(tarsums, tarId) + + layersSeen[layer.ID] = true + jsonData, err := ioutil.ReadFile(path.Join(s.graph.Root, layer.ID, "json")) + if err != nil { + return job.Error(fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)) + } + history = append(history, string(jsonData)) + } + + manifest.BlobSums = tarsums + manifest.History = history + + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return job.Error(err) + } + + _, err = job.Stdout.Write(manifestBytes) + if err != nil { + return job.Error(err) + } + + return engine.StatusOK +} diff --git a/graph/push.go b/graph/push.go index 0ec81a5152..e68886f9a7 100644 --- a/graph/push.go +++ b/graph/push.go @@ -1,15 +1,18 @@ package graph import ( + "bytes" "fmt" "io" "io/ioutil" "os" "path" + "strings" "sync" log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" + "github.com/docker/docker/image" "github.com/docker/docker/pkg/archive" "github.com/docker/docker/registry" "github.com/docker/docker/utils" @@ -267,6 +270,7 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } tag := job.Getenv("tag") + manifestBytes := job.Getenv("manifest") job.GetenvJson("authConfig", authConfig) job.GetenvJson("metaHeaders", &metaHeaders) @@ -286,6 +290,92 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Error(err2) } + var isOfficial bool + if endpoint.String() == registry.IndexServerAddress() { + isOfficial = isOfficialName(remoteName) + if isOfficial && strings.IndexRune(remoteName, '/') == -1 { + remoteName = "library/" + remoteName + } + } + + if len(tag) == 0 { + tag = DEFAULTTAG + } + if isOfficial || endpoint.Version == registry.APIVersion2 { + j := job.Eng.Job("trust_update_base") + if err = j.Run(); err != nil { + return job.Errorf("error updating trust base graph: %s", err) + } + + repoData, err := r.PushImageJSONIndex(remoteName, []*registry.ImgData{}, false, nil) + if err != nil { + return job.Error(err) + } + + // try via manifest + manifest, verified, err := s.verifyManifest(job.Eng, []byte(manifestBytes)) + if err != nil { + return job.Errorf("error verifying manifest: %s", err) + } + + if len(manifest.FSLayers) != len(manifest.History) { + return job.Errorf("length of history not equal to number of layers") + } + + if !verified { + log.Debugf("Pushing unverified image") + } + + for i := len(manifest.FSLayers) - 1; i >= 0; i-- { + var ( + sumStr = manifest.FSLayers[i].BlobSum + imgJSON = []byte(manifest.History[i].V1Compatibility) + ) + + sumParts := strings.SplitN(sumStr, ":", 2) + if len(sumParts) < 2 { + return job.Errorf("Invalid checksum: %s", sumStr) + } + manifestSum := sumParts[1] + + // for each layer, check if it exists ... + // XXX wait this requires having the TarSum of the layer.tar first + // skip this step for now. Just push the layer every time for this naive implementation + //shouldPush, err := r.PostV2ImageMountBlob(imageName, sumType, sum string, token []string) + + img, err := image.NewImgJSON(imgJSON) + if err != nil { + return job.Errorf("Failed to parse json: %s", err) + } + + img, err = s.graph.Get(img.ID) + if err != nil { + return job.Error(err) + } + + arch, err := img.TarLayer() + if err != nil { + return job.Errorf("Could not get tar layer: %s", err) + } + + _, err = r.PutV2ImageBlob(remoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), repoData.Tokens) + if err != nil { + job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) + return job.Error(err) + } + job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) + } + + // push the manifest + err = r.PutV2ImageManifest(remoteName, tag, bytes.NewReader([]byte(manifestBytes)), repoData.Tokens) + if err != nil { + return job.Error(err) + } + + // done, no fallback to V1 + return engine.StatusOK + } + if err != nil { reposLen := 1 if tag == "" { diff --git a/graph/service.go b/graph/service.go index 2858d9b3e6..675e12a1a9 100644 --- a/graph/service.go +++ b/graph/service.go @@ -25,6 +25,7 @@ func (s *TagStore) Install(eng *engine.Engine) error { "import": s.CmdImport, "pull": s.CmdPull, "push": s.CmdPush, + "image_manifest": s.CmdManifest, } { if err := eng.Register(name, handler); err != nil { return fmt.Errorf("Could not register %q: %v", name, err) diff --git a/registry/session_v2.go b/registry/session_v2.go index 20e9e2ee9c..0498bf702e 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -267,7 +267,7 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []s // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { +func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { vars := map[string]string{ "imagename": imageName, "sumtype": sumType, @@ -285,6 +285,7 @@ func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, t return "", err } setTokenAuth(req, token) + req.Header.Set("X-Tarsum", sumStr) res, _, err := r.doRequest(req) if err != nil { return "", err @@ -309,6 +310,10 @@ func (r *Session) PutV2ImageBlob(imageName, sumType string, blobRdr io.Reader, t return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) } + if sumInfo.Checksum != sumStr { + return "", fmt.Errorf("failed checksum comparison. serverChecksum: %q, localChecksum: %q", sumInfo.Checksum, sumStr) + } + // XXX this is a json struct from the registry, with its checksum return sumInfo.Checksum, nil } From bcc0a343bb9c75443238e614e4c2da5f707aef8d Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 9 Oct 2014 17:34:52 -0700 Subject: [PATCH 04/29] Update manifest format for push Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/manifest.go | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/graph/manifest.go b/graph/manifest.go index 39cabb6e41..752b782308 100644 --- a/graph/manifest.go +++ b/graph/manifest.go @@ -43,7 +43,6 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { if !exists { return job.Errorf("Tag does not exist for %s: %s", name, tag) } - tarsums := make([]string, 0, 4) layersSeen := make(map[string]bool) layer, err := s.graph.Get(layerId) @@ -54,9 +53,10 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { return job.Errorf("Missing layer configuration") } manifest.Architecture = layer.Architecture + manifest.FSLayers = make([]*registry.FSLayer, 0, 4) + manifest.History = make([]*registry.ManifestHistory, 0, 4) var metadata runconfig.Config metadata = *layer.Config - history := make([]string, 0, cap(tarsums)) for ; layer != nil; layer, err = layer.GetParent() { if err != nil { @@ -89,19 +89,16 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { tarId := tarSum.Sum(nil) // Save tarsum to image json - tarsums = append(tarsums, tarId) + manifest.FSLayers = append(manifest.FSLayers, ®istry.FSLayer{BlobSum: tarId}) layersSeen[layer.ID] = true jsonData, err := ioutil.ReadFile(path.Join(s.graph.Root, layer.ID, "json")) if err != nil { return job.Error(fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)) } - history = append(history, string(jsonData)) + manifest.History = append(manifest.History, ®istry.ManifestHistory{V1Compatibility: string(jsonData)}) } - manifest.BlobSums = tarsums - manifest.History = history - manifestBytes, err := json.MarshalIndent(manifest, "", " ") if err != nil { return job.Error(err) From 3e4fd005449448ab85c917a5d27ca584b260309c Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 9 Oct 2014 17:32:16 -0700 Subject: [PATCH 05/29] Use tarsum dev version to fix mtime issue Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/manifest.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/graph/manifest.go b/graph/manifest.go index 752b782308..ddcb22b650 100644 --- a/graph/manifest.go +++ b/graph/manifest.go @@ -78,7 +78,7 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { return job.Error(err) } - tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version0) + tarSum, err := tarsum.NewTarSum(archive, true, tarsum.VersionDev) if err != nil { return job.Error(err) } From e9b590d85e9c622316b8be71004737f63e6b9503 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 14 Nov 2014 16:22:06 -0800 Subject: [PATCH 06/29] Update push to use mount blob endpoint Using mount blob prevents repushing images which have already been uploaded Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/push.go | 14 ++++++++++++-- registry/session_v2.go | 4 ++-- 2 files changed, 14 insertions(+), 4 deletions(-) diff --git a/graph/push.go b/graph/push.go index e68886f9a7..64c1f7c616 100644 --- a/graph/push.go +++ b/graph/push.go @@ -358,12 +358,22 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Errorf("Could not get tar layer: %s", err) } - _, err = r.PutV2ImageBlob(remoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), repoData.Tokens) + // Call mount blob + exists, err := r.PostV2ImageMountBlob(remoteName, sumParts[0], manifestSum, repoData.Tokens) if err != nil { job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) return job.Error(err) } - job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) + if !exists { + _, err = r.PutV2ImageBlob(remoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), repoData.Tokens) + if err != nil { + job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) + return job.Error(err) + } + job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) + } else { + job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image already exists", nil)) + } } // push the manifest diff --git a/registry/session_v2.go b/registry/session_v2.go index 0498bf702e..86d0c228a7 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -34,7 +34,7 @@ func newV2RegistryRouter() *mux.Router { v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") // Mounting a blob in an image - v2Router.Path("/mountblob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") + v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") return router } @@ -184,7 +184,7 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []s case 200: // return something indicating no push needed return true, nil - case 300: + case 404: // return something indicating blob push needed return false, nil } From e23362597dcaa8839271210d24bda2ba55f1e12f Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Fri, 12 Dec 2014 13:30:12 -0800 Subject: [PATCH 07/29] Update token response handling Registry authorization token is now taken from the response body rather than the repsonse header. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- registry/token.go | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/registry/token.go b/registry/token.go index 0403734f87..2504863048 100644 --- a/registry/token.go +++ b/registry/token.go @@ -1,6 +1,7 @@ package registry import ( + "encoding/json" "errors" "fmt" "net/http" @@ -10,6 +11,10 @@ import ( "github.com/docker/docker/utils" ) +type tokenResponse struct { + Token string `json:"token"` +} + func getToken(username, password string, params map[string]string, registryEndpoint *Endpoint, client *http.Client, factory *utils.HTTPRequestFactory) (token string, err error) { realm, ok := params["realm"] if !ok { @@ -57,14 +62,20 @@ func getToken(username, password string, params map[string]string, registryEndpo } defer resp.Body.Close() - if !(resp.StatusCode == http.StatusOK || resp.StatusCode == http.StatusNoContent) { + if resp.StatusCode != http.StatusOK { return "", fmt.Errorf("token auth attempt for registry %s: %s request failed with status: %d %s", registryEndpoint, req.URL, resp.StatusCode, http.StatusText(resp.StatusCode)) } - token = resp.Header.Get("X-Auth-Token") - if token == "" { - return "", errors.New("token server did not include a token in the response header") + decoder := json.NewDecoder(resp.Body) + + tr := new(tokenResponse) + if err = decoder.Decode(tr); err != nil { + return "", fmt.Errorf("unable to decode token response: %s", err) } - return token, nil + if tr.Token == "" { + return "", errors.New("authorization server did not include a token in the response") + } + + return tr.Token, nil } From a0f92a26d90e870dfddae9694a5de97e36d3f586 Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Fri, 12 Dec 2014 11:27:22 -0800 Subject: [PATCH 08/29] Registry V2 HTTP route and error code definitions This package, ported from next-generation docker regsitry, includes route and error definitions. These facilitate compliant V2 client implementation. The portions of the HTTP API that are included in this package are considered to be locked down and should only be changed through a careful change proposal. Descriptor definitions package layout may change without affecting API behavior until the exported Go API is ready to be locked down. When the new registry stabilizes and becomes the master branch, this package can be vendored from the registry. Signed-off-by: Stephen J Day --- registry/v2/descriptors.go | 144 +++++++++++++++++++++++++++++ registry/v2/doc.go | 13 +++ registry/v2/errors.go | 185 +++++++++++++++++++++++++++++++++++++ registry/v2/errors_test.go | 165 +++++++++++++++++++++++++++++++++ registry/v2/routes.go | 69 ++++++++++++++ registry/v2/routes_test.go | 184 ++++++++++++++++++++++++++++++++++++ registry/v2/urls.go | 165 +++++++++++++++++++++++++++++++++ registry/v2/urls_test.go | 100 ++++++++++++++++++++ 8 files changed, 1025 insertions(+) create mode 100644 registry/v2/descriptors.go create mode 100644 registry/v2/doc.go create mode 100644 registry/v2/errors.go create mode 100644 registry/v2/errors_test.go create mode 100644 registry/v2/routes.go create mode 100644 registry/v2/routes_test.go create mode 100644 registry/v2/urls.go create mode 100644 registry/v2/urls_test.go diff --git a/registry/v2/descriptors.go b/registry/v2/descriptors.go new file mode 100644 index 0000000000..68d182411d --- /dev/null +++ b/registry/v2/descriptors.go @@ -0,0 +1,144 @@ +package v2 + +import "net/http" + +// TODO(stevvooe): Add route descriptors for each named route, along with +// accepted methods, parameters, returned status codes and error codes. + +// ErrorDescriptor provides relevant information about a given error code. +type ErrorDescriptor struct { + // Code is the error code that this descriptor describes. + Code ErrorCode + + // Value provides a unique, string key, often captilized with + // underscores, to identify the error code. This value is used as the + // keyed value when serializing api errors. + Value string + + // Message is a short, human readable decription of the error condition + // included in API responses. + Message string + + // Description provides a complete account of the errors purpose, suitable + // for use in documentation. + Description string + + // HTTPStatusCodes provides a list of status under which this error + // condition may arise. If it is empty, the error condition may be seen + // for any status code. + HTTPStatusCodes []int +} + +// ErrorDescriptors provides a list of HTTP API Error codes that may be +// encountered when interacting with the registry API. +var ErrorDescriptors = []ErrorDescriptor{ + { + Code: ErrorCodeUnknown, + Value: "UNKNOWN", + Message: "unknown error", + Description: `Generic error returned when the error does not have an + API classification.`, + }, + { + Code: ErrorCodeDigestInvalid, + Value: "DIGEST_INVALID", + Message: "provided digest did not match uploaded content", + Description: `When a blob is uploaded, the registry will check that + the content matches the digest provided by the client. The error may + include a detail structure with the key "digest", including the + invalid digest string. This error may also be returned when a manifest + includes an invalid layer digest.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeSizeInvalid, + Value: "SIZE_INVALID", + Message: "provided length did not match content length", + Description: `When a layer is uploaded, the provided size will be + checked against the uploaded content. If they do not match, this error + will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeNameInvalid, + Value: "NAME_INVALID", + Message: "manifest name did not match URI", + Description: `During a manifest upload, if the name in the manifest + does not match the uri name, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeTagInvalid, + Value: "TAG_INVALID", + Message: "manifest tag did not match URI", + Description: `During a manifest upload, if the tag in the manifest + does not match the uri tag, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + { + Code: ErrorCodeNameUnknown, + Value: "NAME_UNKNOWN", + Message: "repository name not known to registry", + Description: `This is returned if the name used during an operation is + unknown to the registry.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, + { + Code: ErrorCodeManifestUnknown, + Value: "MANIFEST_UNKNOWN", + Message: "manifest unknown", + Description: `This error is returned when the manifest, identified by + name and tag is unknown to the repository.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, + { + Code: ErrorCodeManifestInvalid, + Value: "MANIFEST_INVALID", + Message: "manifest invalid", + Description: `During upload, manifests undergo several checks ensuring + validity. If those checks fail, this error may be returned, unless a + more specific error is included. The detail will contain information + the failed validation.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeManifestUnverified, + Value: "MANIFEST_UNVERIFIED", + Message: "manifest failed signature verification", + Description: `During manifest upload, if the manifest fails signature + verification, this error will be returned.`, + HTTPStatusCodes: []int{http.StatusBadRequest}, + }, + { + Code: ErrorCodeBlobUnknown, + Value: "BLOB_UNKNOWN", + Message: "blob unknown to registry", + Description: `This error may be returned when a blob is unknown to the + registry in a specified repository. This can be returned with a + standard get or if a manifest references an unknown layer during + upload.`, + HTTPStatusCodes: []int{http.StatusBadRequest, http.StatusNotFound}, + }, + + { + Code: ErrorCodeBlobUploadUnknown, + Value: "BLOB_UPLOAD_UNKNOWN", + Message: "blob upload unknown to registry", + Description: `If a blob upload has been cancelled or was never + started, this error code may be returned.`, + HTTPStatusCodes: []int{http.StatusNotFound}, + }, +} + +var errorCodeToDescriptors map[ErrorCode]ErrorDescriptor +var idToDescriptors map[string]ErrorDescriptor + +func init() { + errorCodeToDescriptors = make(map[ErrorCode]ErrorDescriptor, len(ErrorDescriptors)) + idToDescriptors = make(map[string]ErrorDescriptor, len(ErrorDescriptors)) + + for _, descriptor := range ErrorDescriptors { + errorCodeToDescriptors[descriptor.Code] = descriptor + idToDescriptors[descriptor.Value] = descriptor + } +} diff --git a/registry/v2/doc.go b/registry/v2/doc.go new file mode 100644 index 0000000000..30fe2271a1 --- /dev/null +++ b/registry/v2/doc.go @@ -0,0 +1,13 @@ +// Package v2 describes routes, urls and the error codes used in the Docker +// Registry JSON HTTP API V2. In addition to declarations, descriptors are +// provided for routes and error codes that can be used for implementation and +// automatically generating documentation. +// +// Definitions here are considered to be locked down for the V2 registry api. +// Any changes must be considered carefully and should not proceed without a +// change proposal. +// +// Currently, while the HTTP API definitions are considered stable, the Go API +// exports are considered unstable. Go API consumers should take care when +// relying on these definitions until this message is deleted. +package v2 diff --git a/registry/v2/errors.go b/registry/v2/errors.go new file mode 100644 index 0000000000..8c85d3a97f --- /dev/null +++ b/registry/v2/errors.go @@ -0,0 +1,185 @@ +package v2 + +import ( + "fmt" + "strings" +) + +// ErrorCode represents the error type. The errors are serialized via strings +// and the integer format may change and should *never* be exported. +type ErrorCode int + +const ( + // ErrorCodeUnknown is a catch-all for errors not defined below. + ErrorCodeUnknown ErrorCode = iota + + // ErrorCodeDigestInvalid is returned when uploading a blob if the + // provided digest does not match the blob contents. + ErrorCodeDigestInvalid + + // ErrorCodeSizeInvalid is returned when uploading a blob if the provided + // size does not match the content length. + ErrorCodeSizeInvalid + + // ErrorCodeNameInvalid is returned when the name in the manifest does not + // match the provided name. + ErrorCodeNameInvalid + + // ErrorCodeTagInvalid is returned when the tag in the manifest does not + // match the provided tag. + ErrorCodeTagInvalid + + // ErrorCodeNameUnknown when the repository name is not known. + ErrorCodeNameUnknown + + // ErrorCodeManifestUnknown returned when image manifest is unknown. + ErrorCodeManifestUnknown + + // ErrorCodeManifestInvalid returned when an image manifest is invalid, + // typically during a PUT operation. This error encompasses all errors + // encountered during manifest validation that aren't signature errors. + ErrorCodeManifestInvalid + + // ErrorCodeManifestUnverified is returned when the manifest fails + // signature verfication. + ErrorCodeManifestUnverified + + // ErrorCodeBlobUnknown is returned when a blob is unknown to the + // registry. This can happen when the manifest references a nonexistent + // layer or the result is not found by a blob fetch. + ErrorCodeBlobUnknown + + // ErrorCodeBlobUploadUnknown is returned when an upload is unknown. + ErrorCodeBlobUploadUnknown +) + +// ParseErrorCode attempts to parse the error code string, returning +// ErrorCodeUnknown if the error is not known. +func ParseErrorCode(s string) ErrorCode { + desc, ok := idToDescriptors[s] + + if !ok { + return ErrorCodeUnknown + } + + return desc.Code +} + +// Descriptor returns the descriptor for the error code. +func (ec ErrorCode) Descriptor() ErrorDescriptor { + d, ok := errorCodeToDescriptors[ec] + + if !ok { + return ErrorCodeUnknown.Descriptor() + } + + return d +} + +// String returns the canonical identifier for this error code. +func (ec ErrorCode) String() string { + return ec.Descriptor().Value +} + +// Message returned the human-readable error message for this error code. +func (ec ErrorCode) Message() string { + return ec.Descriptor().Message +} + +// MarshalText encodes the receiver into UTF-8-encoded text and returns the +// result. +func (ec ErrorCode) MarshalText() (text []byte, err error) { + return []byte(ec.String()), nil +} + +// UnmarshalText decodes the form generated by MarshalText. +func (ec *ErrorCode) UnmarshalText(text []byte) error { + desc, ok := idToDescriptors[string(text)] + + if !ok { + desc = ErrorCodeUnknown.Descriptor() + } + + *ec = desc.Code + + return nil +} + +// Error provides a wrapper around ErrorCode with extra Details provided. +type Error struct { + Code ErrorCode `json:"code"` + Message string `json:"message,omitempty"` + Detail interface{} `json:"detail,omitempty"` +} + +// Error returns a human readable representation of the error. +func (e Error) Error() string { + return fmt.Sprintf("%s: %s", + strings.ToLower(strings.Replace(e.Code.String(), "_", " ", -1)), + e.Message) +} + +// Errors provides the envelope for multiple errors and a few sugar methods +// for use within the application. +type Errors struct { + Errors []Error `json:"errors,omitempty"` +} + +// Push pushes an error on to the error stack, with the optional detail +// argument. It is a programming error (ie panic) to push more than one +// detail at a time. +func (errs *Errors) Push(code ErrorCode, details ...interface{}) { + if len(details) > 1 { + panic("please specify zero or one detail items for this error") + } + + var detail interface{} + if len(details) > 0 { + detail = details[0] + } + + if err, ok := detail.(error); ok { + detail = err.Error() + } + + errs.PushErr(Error{ + Code: code, + Message: code.Message(), + Detail: detail, + }) +} + +// PushErr pushes an error interface onto the error stack. +func (errs *Errors) PushErr(err error) { + switch err.(type) { + case Error: + errs.Errors = append(errs.Errors, err.(Error)) + default: + errs.Errors = append(errs.Errors, Error{Message: err.Error()}) + } +} + +func (errs *Errors) Error() string { + switch errs.Len() { + case 0: + return "" + case 1: + return errs.Errors[0].Error() + default: + msg := "errors:\n" + for _, err := range errs.Errors { + msg += err.Error() + "\n" + } + return msg + } +} + +// Clear clears the errors. +func (errs *Errors) Clear() { + errs.Errors = errs.Errors[:0] +} + +// Len returns the current number of errors. +func (errs *Errors) Len() int { + return len(errs.Errors) +} diff --git a/registry/v2/errors_test.go b/registry/v2/errors_test.go new file mode 100644 index 0000000000..d2fc091aca --- /dev/null +++ b/registry/v2/errors_test.go @@ -0,0 +1,165 @@ +package v2 + +import ( + "encoding/json" + "reflect" + "testing" + + "github.com/docker/docker-registry/digest" +) + +// TestErrorCodes ensures that error code format, mappings and +// marshaling/unmarshaling. round trips are stable. +func TestErrorCodes(t *testing.T) { + for _, desc := range ErrorDescriptors { + if desc.Code.String() != desc.Value { + t.Fatalf("error code string incorrect: %q != %q", desc.Code.String(), desc.Value) + } + + if desc.Code.Message() != desc.Message { + t.Fatalf("incorrect message for error code %v: %q != %q", desc.Code, desc.Code.Message(), desc.Message) + } + + // Serialize the error code using the json library to ensure that we + // get a string and it works round trip. + p, err := json.Marshal(desc.Code) + + if err != nil { + t.Fatalf("error marshaling error code %v: %v", desc.Code, err) + } + + if len(p) <= 0 { + t.Fatalf("expected content in marshaled before for error code %v", desc.Code) + } + + // First, unmarshal to interface and ensure we have a string. + var ecUnspecified interface{} + if err := json.Unmarshal(p, &ecUnspecified); err != nil { + t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) + } + + if _, ok := ecUnspecified.(string); !ok { + t.Fatalf("expected a string for error code %v on unmarshal got a %T", desc.Code, ecUnspecified) + } + + // Now, unmarshal with the error code type and ensure they are equal + var ecUnmarshaled ErrorCode + if err := json.Unmarshal(p, &ecUnmarshaled); err != nil { + t.Fatalf("error unmarshaling error code %v: %v", desc.Code, err) + } + + if ecUnmarshaled != desc.Code { + t.Fatalf("unexpected error code during error code marshal/unmarshal: %v != %v", ecUnmarshaled, desc.Code) + } + } +} + +// TestErrorsManagement does a quick check of the Errors type to ensure that +// members are properly pushed and marshaled. +func TestErrorsManagement(t *testing.T) { + var errs Errors + + errs.Push(ErrorCodeDigestInvalid) + errs.Push(ErrorCodeBlobUnknown, + map[string]digest.Digest{"digest": "sometestblobsumdoesntmatter"}) + + p, err := json.Marshal(errs) + + if err != nil { + t.Fatalf("error marashaling errors: %v", err) + } + + expectedJSON := "{\"errors\":[{\"code\":\"DIGEST_INVALID\",\"message\":\"provided digest did not match uploaded content\"},{\"code\":\"BLOB_UNKNOWN\",\"message\":\"blob unknown to registry\",\"detail\":{\"digest\":\"sometestblobsumdoesntmatter\"}}]}" + + if string(p) != expectedJSON { + t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) + } + + errs.Clear() + errs.Push(ErrorCodeUnknown) + expectedJSON = "{\"errors\":[{\"code\":\"UNKNOWN\",\"message\":\"unknown error\"}]}" + p, err = json.Marshal(errs) + + if err != nil { + t.Fatalf("error marashaling errors: %v", err) + } + + if string(p) != expectedJSON { + t.Fatalf("unexpected json: %q != %q", string(p), expectedJSON) + } +} + +// TestMarshalUnmarshal ensures that api errors can round trip through json +// without losing information. +func TestMarshalUnmarshal(t *testing.T) { + + var errors Errors + + for _, testcase := range []struct { + description string + err Error + }{ + { + description: "unknown error", + err: Error{ + + Code: ErrorCodeUnknown, + Message: ErrorCodeUnknown.Descriptor().Message, + }, + }, + { + description: "unknown manifest", + err: Error{ + Code: ErrorCodeManifestUnknown, + Message: ErrorCodeManifestUnknown.Descriptor().Message, + }, + }, + { + description: "unknown manifest", + err: Error{ + Code: ErrorCodeBlobUnknown, + Message: ErrorCodeBlobUnknown.Descriptor().Message, + Detail: map[string]interface{}{"digest": "asdfqwerqwerqwerqwer"}, + }, + }, + } { + fatalf := func(format string, args ...interface{}) { + t.Fatalf(testcase.description+": "+format, args...) + } + + unexpectedErr := func(err error) { + fatalf("unexpected error: %v", err) + } + + p, err := json.Marshal(testcase.err) + if err != nil { + unexpectedErr(err) + } + + var unmarshaled Error + if err := json.Unmarshal(p, &unmarshaled); err != nil { + unexpectedErr(err) + } + + if !reflect.DeepEqual(unmarshaled, testcase.err) { + fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, testcase.err) + } + + // Roll everything up into an error response envelope. + errors.PushErr(testcase.err) + } + + p, err := json.Marshal(errors) + if err != nil { + t.Fatalf("unexpected error marshaling error envelope: %v", err) + } + + var unmarshaled Errors + if err := json.Unmarshal(p, &unmarshaled); err != nil { + t.Fatalf("unexpected error unmarshaling error envelope: %v", err) + } + + if !reflect.DeepEqual(unmarshaled, errors) { + t.Fatalf("errors not equal after round trip: %#v != %#v", unmarshaled, errors) + } +} diff --git a/registry/v2/routes.go b/registry/v2/routes.go new file mode 100644 index 0000000000..7ebe61d665 --- /dev/null +++ b/registry/v2/routes.go @@ -0,0 +1,69 @@ +package v2 + +import ( + "github.com/docker/docker-registry/common" + "github.com/gorilla/mux" +) + +// The following are definitions of the name under which all V2 routes are +// registered. These symbols can be used to look up a route based on the name. +const ( + RouteNameBase = "base" + RouteNameManifest = "manifest" + RouteNameTags = "tags" + RouteNameBlob = "blob" + RouteNameBlobUpload = "blob-upload" + RouteNameBlobUploadChunk = "blob-upload-chunk" +) + +var allEndpoints = []string{ + RouteNameManifest, + RouteNameTags, + RouteNameBlob, + RouteNameBlobUpload, + RouteNameBlobUploadChunk, +} + +// Router builds a gorilla router with named routes for the various API +// methods. This can be used directly by both server implementations and +// clients. +func Router() *mux.Router { + router := mux.NewRouter(). + StrictSlash(true) + + // GET /v2/ Check Check that the registry implements API version 2(.1) + router. + Path("/v2/"). + Name(RouteNameBase) + + // GET /v2//manifest/ Image Manifest Fetch the image manifest identified by name and tag. + // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. + // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/manifests/{tag:" + common.TagNameRegexp.String() + "}"). + Name(RouteNameManifest) + + // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/tags/list"). + Name(RouteNameTags) + + // GET /v2//blob/ Layer Fetch the blob identified by digest. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). + Name(RouteNameBlob) + + // POST /v2//blob/upload/ Layer Upload Initiate an upload of the layer identified by tarsum. + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/"). + Name(RouteNameBlobUpload) + + // GET /v2//blob/upload/ Layer Upload Get the status of the upload identified by tarsum and uuid. + // PUT /v2//blob/upload/ Layer Upload Upload all or a chunk of the upload identified by tarsum and uuid. + // DELETE /v2//blob/upload/ Layer Upload Cancel the upload identified by layer and uuid + router. + Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). + Name(RouteNameBlobUploadChunk) + + return router +} diff --git a/registry/v2/routes_test.go b/registry/v2/routes_test.go new file mode 100644 index 0000000000..9969ebcc44 --- /dev/null +++ b/registry/v2/routes_test.go @@ -0,0 +1,184 @@ +package v2 + +import ( + "encoding/json" + "net/http" + "net/http/httptest" + "reflect" + "testing" + + "github.com/gorilla/mux" +) + +type routeTestCase struct { + RequestURI string + Vars map[string]string + RouteName string + StatusCode int +} + +// TestRouter registers a test handler with all the routes and ensures that +// each route returns the expected path variables. Not method verification is +// present. This not meant to be exhaustive but as check to ensure that the +// expected variables are extracted. +// +// This may go away as the application structure comes together. +func TestRouter(t *testing.T) { + + router := Router() + + testHandler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + testCase := routeTestCase{ + RequestURI: r.RequestURI, + Vars: mux.Vars(r), + RouteName: mux.CurrentRoute(r).GetName(), + } + + enc := json.NewEncoder(w) + + if err := enc.Encode(testCase); err != nil { + http.Error(w, err.Error(), http.StatusInternalServerError) + return + } + }) + + // Startup test server + server := httptest.NewServer(router) + + for _, testcase := range []routeTestCase{ + { + RouteName: RouteNameBase, + RequestURI: "/v2/", + Vars: map[string]string{}, + }, + { + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/bar/manifests/tag", + Vars: map[string]string{ + "name": "foo/bar", + "tag": "tag", + }, + }, + { + RouteName: RouteNameTags, + RequestURI: "/v2/foo/bar/tags/list", + Vars: map[string]string{ + "name": "foo/bar", + }, + }, + { + RouteName: RouteNameBlob, + RequestURI: "/v2/foo/bar/blobs/tarsum.dev+foo:abcdef0919234", + Vars: map[string]string{ + "name": "foo/bar", + "digest": "tarsum.dev+foo:abcdef0919234", + }, + }, + { + RouteName: RouteNameBlob, + RequestURI: "/v2/foo/bar/blobs/sha256:abcdef0919234", + Vars: map[string]string{ + "name": "foo/bar", + "digest": "sha256:abcdef0919234", + }, + }, + { + RouteName: RouteNameBlobUpload, + RequestURI: "/v2/foo/bar/blobs/uploads/", + Vars: map[string]string{ + "name": "foo/bar", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/uuid", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "uuid", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/bar/blobs/uploads/RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", + Vars: map[string]string{ + "name": "foo/bar", + "uuid": "RDk1MzA2RkEtRkFEMy00RTM2LThENDEtQ0YxQzkzRUY4Mjg2IA==", + }, + }, + { + // Check ambiguity: ensure we can distinguish between tags for + // "foo/bar/image/image" and image for "foo/bar/image" with tag + // "tags" + RouteName: RouteNameManifest, + RequestURI: "/v2/foo/bar/manifests/manifests/tags", + Vars: map[string]string{ + "name": "foo/bar/manifests", + "tag": "tags", + }, + }, + { + // This case presents an ambiguity between foo/bar with tag="tags" + // and list tags for "foo/bar/manifest" + RouteName: RouteNameTags, + RequestURI: "/v2/foo/bar/manifests/tags/list", + Vars: map[string]string{ + "name": "foo/bar/manifests", + }, + }, + { + RouteName: RouteNameBlobUploadChunk, + RequestURI: "/v2/foo/../../blob/uploads/D95306FA-FAD3-4E36-8D41-CF1C93EF8286", + StatusCode: http.StatusNotFound, + }, + } { + // Register the endpoint + router.GetRoute(testcase.RouteName).Handler(testHandler) + u := server.URL + testcase.RequestURI + + resp, err := http.Get(u) + + if err != nil { + t.Fatalf("error issuing get request: %v", err) + } + + if testcase.StatusCode == 0 { + // Override default, zero-value + testcase.StatusCode = http.StatusOK + } + + if resp.StatusCode != testcase.StatusCode { + t.Fatalf("unexpected status for %s: %v %v", u, resp.Status, resp.StatusCode) + } + + if testcase.StatusCode != http.StatusOK { + // We don't care about json response. + continue + } + + dec := json.NewDecoder(resp.Body) + + var actualRouteInfo routeTestCase + if err := dec.Decode(&actualRouteInfo); err != nil { + t.Fatalf("error reading json response: %v", err) + } + // Needs to be set out of band + actualRouteInfo.StatusCode = resp.StatusCode + + if actualRouteInfo.RouteName != testcase.RouteName { + t.Fatalf("incorrect route %q matched, expected %q", actualRouteInfo.RouteName, testcase.RouteName) + } + + if !reflect.DeepEqual(actualRouteInfo, testcase) { + t.Fatalf("actual does not equal expected: %#v != %#v", actualRouteInfo, testcase) + } + } + +} diff --git a/registry/v2/urls.go b/registry/v2/urls.go new file mode 100644 index 0000000000..72f44299ab --- /dev/null +++ b/registry/v2/urls.go @@ -0,0 +1,165 @@ +package v2 + +import ( + "net/http" + "net/url" + + "github.com/docker/docker-registry/digest" + "github.com/gorilla/mux" +) + +// URLBuilder creates registry API urls from a single base endpoint. It can be +// used to create urls for use in a registry client or server. +// +// All urls will be created from the given base, including the api version. +// For example, if a root of "/foo/" is provided, urls generated will be fall +// under "/foo/v2/...". Most application will only provide a schema, host and +// port, such as "https://localhost:5000/". +type URLBuilder struct { + root *url.URL // url root (ie http://localhost/) + router *mux.Router +} + +// NewURLBuilder creates a URLBuilder with provided root url object. +func NewURLBuilder(root *url.URL) *URLBuilder { + return &URLBuilder{ + root: root, + router: Router(), + } +} + +// NewURLBuilderFromString workes identically to NewURLBuilder except it takes +// a string argument for the root, returning an error if it is not a valid +// url. +func NewURLBuilderFromString(root string) (*URLBuilder, error) { + u, err := url.Parse(root) + if err != nil { + return nil, err + } + + return NewURLBuilder(u), nil +} + +// NewURLBuilderFromRequest uses information from an *http.Request to +// construct the root url. +func NewURLBuilderFromRequest(r *http.Request) *URLBuilder { + u := &url.URL{ + Scheme: r.URL.Scheme, + Host: r.Host, + } + + return NewURLBuilder(u) +} + +// BuildBaseURL constructs a base url for the API, typically just "/v2/". +func (ub *URLBuilder) BuildBaseURL() (string, error) { + route := ub.cloneRoute(RouteNameBase) + + baseURL, err := route.URL() + if err != nil { + return "", err + } + + return baseURL.String(), nil +} + +// BuildTagsURL constructs a url to list the tags in the named repository. +func (ub *URLBuilder) BuildTagsURL(name string) (string, error) { + route := ub.cloneRoute(RouteNameTags) + + tagsURL, err := route.URL("name", name) + if err != nil { + return "", err + } + + return tagsURL.String(), nil +} + +// BuildManifestURL constructs a url for the manifest identified by name and tag. +func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { + route := ub.cloneRoute(RouteNameManifest) + + manifestURL, err := route.URL("name", name, "tag", tag) + if err != nil { + return "", err + } + + return manifestURL.String(), nil +} + +// BuildBlobURL constructs the url for the blob identified by name and dgst. +func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, error) { + route := ub.cloneRoute(RouteNameBlob) + + layerURL, err := route.URL("name", name, "digest", dgst.String()) + if err != nil { + return "", err + } + + return layerURL.String(), nil +} + +// BuildBlobUploadURL constructs a url to begin a blob upload in the +// repository identified by name. +func (ub *URLBuilder) BuildBlobUploadURL(name string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUpload) + + uploadURL, err := route.URL("name", name) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// BuildBlobUploadChunkURL constructs a url for the upload identified by uuid, +// including any url values. This should generally not be used by clients, as +// this url is provided by server implementations during the blob upload +// process. +func (ub *URLBuilder) BuildBlobUploadChunkURL(name, uuid string, values ...url.Values) (string, error) { + route := ub.cloneRoute(RouteNameBlobUploadChunk) + + uploadURL, err := route.URL("name", name, "uuid", uuid) + if err != nil { + return "", err + } + + return appendValuesURL(uploadURL, values...).String(), nil +} + +// clondedRoute returns a clone of the named route from the router. Routes +// must be cloned to avoid modifying them during url generation. +func (ub *URLBuilder) cloneRoute(name string) *mux.Route { + route := new(mux.Route) + *route = *ub.router.GetRoute(name) // clone the route + + return route. + Schemes(ub.root.Scheme). + Host(ub.root.Host) +} + +// appendValuesURL appends the parameters to the url. +func appendValuesURL(u *url.URL, values ...url.Values) *url.URL { + merged := u.Query() + + for _, v := range values { + for k, vv := range v { + merged[k] = append(merged[k], vv...) + } + } + + u.RawQuery = merged.Encode() + return u +} + +// appendValues appends the parameters to the url. Panics if the string is not +// a url. +func appendValues(u string, values ...url.Values) string { + up, err := url.Parse(u) + + if err != nil { + panic(err) // should never happen + } + + return appendValuesURL(up, values...).String() +} diff --git a/registry/v2/urls_test.go b/registry/v2/urls_test.go new file mode 100644 index 0000000000..a9590dba90 --- /dev/null +++ b/registry/v2/urls_test.go @@ -0,0 +1,100 @@ +package v2 + +import ( + "net/url" + "testing" +) + +type urlBuilderTestCase struct { + description string + expected string + build func() (string, error) +} + +// TestURLBuilder tests the various url building functions, ensuring they are +// returning the expected values. +func TestURLBuilder(t *testing.T) { + + root := "http://localhost:5000/" + urlBuilder, err := NewURLBuilderFromString(root) + if err != nil { + t.Fatalf("unexpected error creating urlbuilder: %v", err) + } + + for _, testcase := range []struct { + description string + expected string + build func() (string, error) + }{ + { + description: "test base url", + expected: "http://localhost:5000/v2/", + build: urlBuilder.BuildBaseURL, + }, + { + description: "test tags url", + expected: "http://localhost:5000/v2/foo/bar/tags/list", + build: func() (string, error) { + return urlBuilder.BuildTagsURL("foo/bar") + }, + }, + { + description: "test manifest url", + expected: "http://localhost:5000/v2/foo/bar/manifests/tag", + build: func() (string, error) { + return urlBuilder.BuildManifestURL("foo/bar", "tag") + }, + }, + { + description: "build blob url", + expected: "http://localhost:5000/v2/foo/bar/blobs/tarsum.v1+sha256:abcdef0123456789", + build: func() (string, error) { + return urlBuilder.BuildBlobURL("foo/bar", "tarsum.v1+sha256:abcdef0123456789") + }, + }, + { + description: "build blob upload url", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadURL("foo/bar") + }, + }, + { + description: "build blob upload url with digest and size", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadURL("foo/bar", url.Values{ + "size": []string{"10000"}, + "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, + }) + }, + }, + { + description: "build blob upload chunk url", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part") + }, + }, + { + description: "build blob upload chunk url with digest and size", + expected: "http://localhost:5000/v2/foo/bar/blobs/uploads/uuid-part?digest=tarsum.v1%2Bsha256%3Aabcdef0123456789&size=10000", + build: func() (string, error) { + return urlBuilder.BuildBlobUploadChunkURL("foo/bar", "uuid-part", url.Values{ + "size": []string{"10000"}, + "digest": []string{"tarsum.v1+sha256:abcdef0123456789"}, + }) + }, + }, + } { + u, err := testcase.build() + if err != nil { + t.Fatalf("%s: error building url: %v", testcase.description, err) + } + + if u != testcase.expected { + t.Fatalf("%s: %q != %q", testcase.description, u, testcase.expected) + } + } + +} From dbb4b03bfc82eadefaf68c1a81d215949980550e Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 15 Dec 2014 12:42:52 -0800 Subject: [PATCH 09/29] Remove dependencies on registry packages Because docker core cannot vendor non-master Go dependencies, we need to remove dependencies on registry package. The definition of digest.Digest has been changed to a string and the regular expressions have been ported from docker-registry/common library. We'll likely change this be dependent on the registry in the future when the API stabilizies and use of the master branch becomes the norm. Signed-off-by: Stephen J Day --- registry/v2/errors_test.go | 4 +--- registry/v2/regexp.go | 19 +++++++++++++++++++ registry/v2/routes.go | 15 ++++++--------- registry/v2/urls.go | 5 ++--- 4 files changed, 28 insertions(+), 15 deletions(-) create mode 100644 registry/v2/regexp.go diff --git a/registry/v2/errors_test.go b/registry/v2/errors_test.go index d2fc091aca..4a80cdfe2d 100644 --- a/registry/v2/errors_test.go +++ b/registry/v2/errors_test.go @@ -4,8 +4,6 @@ import ( "encoding/json" "reflect" "testing" - - "github.com/docker/docker-registry/digest" ) // TestErrorCodes ensures that error code format, mappings and @@ -61,7 +59,7 @@ func TestErrorsManagement(t *testing.T) { errs.Push(ErrorCodeDigestInvalid) errs.Push(ErrorCodeBlobUnknown, - map[string]digest.Digest{"digest": "sometestblobsumdoesntmatter"}) + map[string]string{"digest": "sometestblobsumdoesntmatter"}) p, err := json.Marshal(errs) diff --git a/registry/v2/regexp.go b/registry/v2/regexp.go new file mode 100644 index 0000000000..b7e95b9ff3 --- /dev/null +++ b/registry/v2/regexp.go @@ -0,0 +1,19 @@ +package v2 + +import "regexp" + +// This file defines regular expressions for use in route definition. These +// are also defined in the registry code base. Until they are in a common, +// shared location, and exported, they must be repeated here. + +// RepositoryNameComponentRegexp restricts registtry path components names to +// start with at least two letters or numbers, with following parts able to +// separated by one period, dash or underscore. +var RepositoryNameComponentRegexp = regexp.MustCompile(`[a-z0-9]+(?:[._-][a-z0-9]+)*`) + +// RepositoryNameRegexp builds on RepositoryNameComponentRegexp to allow 2 to +// 5 path components, separated by a forward slash. +var RepositoryNameRegexp = regexp.MustCompile(`(?:` + RepositoryNameComponentRegexp.String() + `/){1,4}` + RepositoryNameComponentRegexp.String()) + +// TagNameRegexp matches valid tag names. From docker/docker:graph/tags.go. +var TagNameRegexp = regexp.MustCompile(`[\w][\w.-]{0,127}`) diff --git a/registry/v2/routes.go b/registry/v2/routes.go index 7ebe61d665..08f36e2f71 100644 --- a/registry/v2/routes.go +++ b/registry/v2/routes.go @@ -1,9 +1,6 @@ package v2 -import ( - "github.com/docker/docker-registry/common" - "github.com/gorilla/mux" -) +import "github.com/gorilla/mux" // The following are definitions of the name under which all V2 routes are // registered. These symbols can be used to look up a route based on the name. @@ -40,29 +37,29 @@ func Router() *mux.Router { // PUT /v2//manifest/ Image Manifest Upload the image manifest identified by name and tag. // DELETE /v2//manifest/ Image Manifest Delete the image identified by name and tag. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/manifests/{tag:" + common.TagNameRegexp.String() + "}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/manifests/{tag:" + TagNameRegexp.String() + "}"). Name(RouteNameManifest) // GET /v2//tags/list Tags Fetch the tags under the repository identified by name. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/tags/list"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/tags/list"). Name(RouteNameTags) // GET /v2//blob/ Layer Fetch the blob identified by digest. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/{digest:[a-zA-Z0-9-_+.]+:[a-zA-Z0-9-_+.=]+}"). Name(RouteNameBlob) // POST /v2//blob/upload/ Layer Upload Initiate an upload of the layer identified by tarsum. router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/"). Name(RouteNameBlobUpload) // GET /v2//blob/upload/ Layer Upload Get the status of the upload identified by tarsum and uuid. // PUT /v2//blob/upload/ Layer Upload Upload all or a chunk of the upload identified by tarsum and uuid. // DELETE /v2//blob/upload/ Layer Upload Cancel the upload identified by layer and uuid router. - Path("/v2/{name:" + common.RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). + Path("/v2/{name:" + RepositoryNameRegexp.String() + "}/blobs/uploads/{uuid}"). Name(RouteNameBlobUploadChunk) return router diff --git a/registry/v2/urls.go b/registry/v2/urls.go index 72f44299ab..19ef06fa12 100644 --- a/registry/v2/urls.go +++ b/registry/v2/urls.go @@ -4,7 +4,6 @@ import ( "net/http" "net/url" - "github.com/docker/docker-registry/digest" "github.com/gorilla/mux" ) @@ -88,10 +87,10 @@ func (ub *URLBuilder) BuildManifestURL(name, tag string) (string, error) { } // BuildBlobURL constructs the url for the blob identified by name and dgst. -func (ub *URLBuilder) BuildBlobURL(name string, dgst digest.Digest) (string, error) { +func (ub *URLBuilder) BuildBlobURL(name string, dgst string) (string, error) { route := ub.cloneRoute(RouteNameBlob) - layerURL, err := route.URL("name", name, "digest", dgst.String()) + layerURL, err := route.URL("name", name, "digest", dgst) if err != nil { return "", err } From 0336b0cdaa74ac03003c4a933eb866fb0cec8125 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 16 Dec 2014 16:57:37 -0800 Subject: [PATCH 10/29] Update push and pull to registry 2.1 specification Signed-off-by: Derek McGowan --- graph/manifest.go | 154 +++++++++++---------- graph/pull.go | 21 +-- graph/push.go | 65 +++++---- registry/auth.go | 53 ++++++++ registry/session_v2.go | 298 ++++++++++++++--------------------------- utils/jsonmessage.go | 3 + 6 files changed, 281 insertions(+), 313 deletions(-) diff --git a/graph/manifest.go b/graph/manifest.go index ddcb22b650..54d6083cba 100644 --- a/graph/manifest.go +++ b/graph/manifest.go @@ -2,6 +2,7 @@ package graph import ( "encoding/json" + "errors" "fmt" "io" "io/ioutil" @@ -24,82 +25,12 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { } // Resolve the Repository name from fqn to endpoint + name - _, remoteName, err := registry.ResolveRepositoryName(name) + repoInfo, err := registry.ParseRepositoryInfo(name) if err != nil { return job.Error(err) } - manifest := ®istry.ManifestData{ - Name: remoteName, - Tag: tag, - SchemaVersion: 1, - } - localRepo, exists := s.Repositories[name] - if !exists { - return job.Errorf("Repo does not exist: %s", name) - } - - layerId, exists := localRepo[tag] - if !exists { - return job.Errorf("Tag does not exist for %s: %s", name, tag) - } - layersSeen := make(map[string]bool) - - layer, err := s.graph.Get(layerId) - if err != nil { - return job.Error(err) - } - if layer.Config == nil { - return job.Errorf("Missing layer configuration") - } - manifest.Architecture = layer.Architecture - manifest.FSLayers = make([]*registry.FSLayer, 0, 4) - manifest.History = make([]*registry.ManifestHistory, 0, 4) - var metadata runconfig.Config - metadata = *layer.Config - - for ; layer != nil; layer, err = layer.GetParent() { - if err != nil { - return job.Error(err) - } - - if layersSeen[layer.ID] { - break - } - if layer.Config != nil && metadata.Image != layer.ID { - err = runconfig.Merge(&metadata, layer.Config) - if err != nil { - return job.Error(err) - } - } - - archive, err := layer.TarLayer() - if err != nil { - return job.Error(err) - } - - tarSum, err := tarsum.NewTarSum(archive, true, tarsum.VersionDev) - if err != nil { - return job.Error(err) - } - if _, err := io.Copy(ioutil.Discard, tarSum); err != nil { - return job.Error(err) - } - - tarId := tarSum.Sum(nil) - // Save tarsum to image json - - manifest.FSLayers = append(manifest.FSLayers, ®istry.FSLayer{BlobSum: tarId}) - - layersSeen[layer.ID] = true - jsonData, err := ioutil.ReadFile(path.Join(s.graph.Root, layer.ID, "json")) - if err != nil { - return job.Error(fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err)) - } - manifest.History = append(manifest.History, ®istry.ManifestHistory{V1Compatibility: string(jsonData)}) - } - - manifestBytes, err := json.MarshalIndent(manifest, "", " ") + manifestBytes, err := s.newManifest(name, repoInfo.RemoteName, tag) if err != nil { return job.Error(err) } @@ -111,3 +42,82 @@ func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { return engine.StatusOK } + +func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error) { + manifest := ®istry.ManifestData{ + Name: remoteName, + Tag: tag, + SchemaVersion: 1, + } + localRepo, exists := s.Repositories[localName] + if !exists { + return nil, fmt.Errorf("Repo does not exist: %s", localName) + } + + layerId, exists := localRepo[tag] + if !exists { + return nil, fmt.Errorf("Tag does not exist for %s: %s", localName, tag) + } + layersSeen := make(map[string]bool) + + layer, err := s.graph.Get(layerId) + if err != nil { + return nil, err + } + if layer.Config == nil { + return nil, errors.New("Missing layer configuration") + } + manifest.Architecture = layer.Architecture + manifest.FSLayers = make([]*registry.FSLayer, 0, 4) + manifest.History = make([]*registry.ManifestHistory, 0, 4) + var metadata runconfig.Config + metadata = *layer.Config + + for ; layer != nil; layer, err = layer.GetParent() { + if err != nil { + return nil, err + } + + if layersSeen[layer.ID] { + break + } + if layer.Config != nil && metadata.Image != layer.ID { + err = runconfig.Merge(&metadata, layer.Config) + if err != nil { + return nil, err + } + } + + archive, err := layer.TarLayer() + if err != nil { + return nil, err + } + + tarSum, err := tarsum.NewTarSum(archive, true, tarsum.Version1) + if err != nil { + return nil, err + } + if _, err := io.Copy(ioutil.Discard, tarSum); err != nil { + return nil, err + } + + tarId := tarSum.Sum(nil) + // Save tarsum to image json + + manifest.FSLayers = append(manifest.FSLayers, ®istry.FSLayer{BlobSum: tarId}) + + layersSeen[layer.ID] = true + jsonData, err := ioutil.ReadFile(path.Join(s.graph.Root, layer.ID, "json")) + if err != nil { + return nil, fmt.Errorf("Cannot retrieve the path for {%s}: %s", layer.ID, err) + } + manifest.History = append(manifest.History, ®istry.ManifestHistory{V1Compatibility: string(jsonData)}) + } + + manifestBytes, err := json.MarshalIndent(manifest, "", " ") + if err != nil { + return nil, err + } + + return manifestBytes, nil +} diff --git a/graph/pull.go b/graph/pull.go index 587eb5f500..b138793d1f 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -133,7 +133,12 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { return job.Errorf("error updating trust base graph: %s", err) } - if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err == nil { + auth, err := r.GetV2Authorization(repoInfo.RemoteName, true) + if err != nil { + return job.Errorf("error getting authorization: %s", err) + } + + if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil { if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { log.Errorf("Error logging event 'pull' for %s: %s", logName, err) } @@ -423,23 +428,23 @@ type downloadInfo struct { err chan error } -func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) error { +func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) error { var layersDownloaded bool if tag == "" { log.Debugf("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName) - tags, err := r.GetV2RemoteTags(repoInfo.RemoteName, nil) + tags, err := r.GetV2RemoteTags(repoInfo.RemoteName, auth) if err != nil { return err } for _, t := range tags { - if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, t, sf, parallel); err != nil { + if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, t, sf, parallel, auth); err != nil { return err } else if downloaded { layersDownloaded = true } } } else { - if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, tag, sf, parallel); err != nil { + if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, tag, sf, parallel, auth); err != nil { return err } else if downloaded { layersDownloaded = true @@ -454,9 +459,9 @@ func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out return nil } -func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) (bool, error) { +func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) (bool, error) { log.Debugf("Pulling tag from V2 registry: %q", tag) - manifestBytes, err := r.GetV2ImageManifest(repoInfo.RemoteName, tag, nil) + manifestBytes, err := r.GetV2ImageManifest(repoInfo.RemoteName, tag, auth) if err != nil { return false, err } @@ -525,7 +530,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri return err } - r, l, err := r.GetV2ImageBlobReader(repoInfo.RemoteName, sumType, checksum, nil) + r, l, err := r.GetV2ImageBlobReader(repoInfo.RemoteName, sumType, checksum, auth) if err != nil { return err } diff --git a/graph/push.go b/graph/push.go index 64c1f7c616..0d008b84c4 100644 --- a/graph/push.go +++ b/graph/push.go @@ -290,26 +290,24 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { return job.Error(err2) } - var isOfficial bool - if endpoint.String() == registry.IndexServerAddress() { - isOfficial = isOfficialName(remoteName) - if isOfficial && strings.IndexRune(remoteName, '/') == -1 { - remoteName = "library/" + remoteName - } - } - if len(tag) == 0 { tag = DEFAULTTAG } - if isOfficial || endpoint.Version == registry.APIVersion2 { + + if repoInfo.Official || endpoint.Version == registry.APIVersion2 { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { return job.Errorf("error updating trust base graph: %s", err) } - repoData, err := r.PushImageJSONIndex(remoteName, []*registry.ImgData{}, false, nil) + // Get authentication type + auth, err := r.GetV2Authorization(repoInfo.RemoteName, false) if err != nil { - return job.Error(err) + return job.Errorf("error getting authorization: %s", err) + } + + if len(manifestBytes) == 0 { + // TODO Create manifest and sign } // try via manifest @@ -359,13 +357,13 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } // Call mount blob - exists, err := r.PostV2ImageMountBlob(remoteName, sumParts[0], manifestSum, repoData.Tokens) + exists, err := r.PostV2ImageMountBlob(repoInfo.RemoteName, sumParts[0], manifestSum, auth) if err != nil { job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) return job.Error(err) } if !exists { - _, err = r.PutV2ImageBlob(remoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), repoData.Tokens) + err = r.PutV2ImageBlob(repoInfo.RemoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), auth) if err != nil { job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) return job.Error(err) @@ -377,35 +375,36 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } // push the manifest - err = r.PutV2ImageManifest(remoteName, tag, bytes.NewReader([]byte(manifestBytes)), repoData.Tokens) + err = r.PutV2ImageManifest(repoInfo.RemoteName, tag, bytes.NewReader([]byte(manifestBytes)), auth) if err != nil { return job.Error(err) } // done, no fallback to V1 return engine.StatusOK - } + } else { - if err != nil { - reposLen := 1 - if tag == "" { - reposLen = len(s.Repositories[repoInfo.LocalName]) - } - job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) - // If it fails, try to get the repository - if localRepo, exists := s.Repositories[repoInfo.LocalName]; exists { - if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { - return job.Error(err) + if err != nil { + reposLen := 1 + if tag == "" { + reposLen = len(s.Repositories[repoInfo.LocalName]) } - return engine.StatusOK + job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) + // If it fails, try to get the repository + if localRepo, exists := s.Repositories[repoInfo.LocalName]; exists { + if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { + return job.Error(err) + } + return engine.StatusOK + } + return job.Error(err) } - return job.Error(err) - } - var token []string - job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", repoInfo.CanonicalName)) - if _, err := s.pushImage(r, job.Stdout, img.ID, endpoint.String(), token, sf); err != nil { - return job.Error(err) + var token []string + job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", repoInfo.CanonicalName)) + if _, err := s.pushImage(r, job.Stdout, img.ID, endpoint.String(), token, sf); err != nil { + return job.Error(err) + } + return engine.StatusOK } - return engine.StatusOK } diff --git a/registry/auth.go b/registry/auth.go index 2044236cfb..b138fb530d 100644 --- a/registry/auth.go +++ b/registry/auth.go @@ -37,6 +37,59 @@ type ConfigFile struct { rootPath string } +type RequestAuthorization struct { + Token string + Username string + Password string +} + +func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) (*RequestAuthorization, error) { + var auth RequestAuthorization + + client := &http.Client{ + Transport: &http.Transport{ + DisableKeepAlives: true, + Proxy: http.ProxyFromEnvironment, + }, + CheckRedirect: AddRequiredHeadersToRedirectedRequests, + } + factory := HTTPRequestFactory(nil) + + for _, challenge := range registryEndpoint.AuthChallenges { + log.Debugf("Using %q auth challenge with params %s for %s", challenge.Scheme, challenge.Parameters, authConfig.Username) + + switch strings.ToLower(challenge.Scheme) { + case "basic": + auth.Username = authConfig.Username + auth.Password = authConfig.Password + case "bearer": + params := map[string]string{} + for k, v := range challenge.Parameters { + params[k] = v + } + params["scope"] = fmt.Sprintf("%s:%s:%s", resource, scope, strings.Join(actions, ",")) + token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + if err != nil { + return nil, err + } + + auth.Token = token + default: + log.Infof("Unsupported auth scheme: %q", challenge.Scheme) + } + } + + return &auth, nil +} + +func (auth *RequestAuthorization) Authorize(req *http.Request) { + if auth.Token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", auth.Token)) + } else if auth.Username != "" && auth.Password != "" { + req.SetBasicAuth(auth.Username, auth.Password) + } +} + // create a base64 encoded auth string to store in config func encodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password diff --git a/registry/session_v2.go b/registry/session_v2.go index 86d0c228a7..407c5f3a23 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -9,100 +9,34 @@ import ( "strconv" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/registry/v2" "github.com/docker/docker/utils" - "github.com/gorilla/mux" ) -func newV2RegistryRouter() *mux.Router { - router := mux.NewRouter() +var registryURLBuilder *v2.URLBuilder - v2Router := router.PathPrefix("/v2/").Subrouter() - - // Version Info - v2Router.Path("/version").Name("version") - - // Image Manifests - v2Router.Path("/manifest/{imagename:[a-z0-9-._/]+}/{tagname:[a-zA-Z0-9-._]+}").Name("manifests") - - // List Image Tags - v2Router.Path("/tags/{imagename:[a-z0-9-._/]+}").Name("tags") - - // Download a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("downloadBlob") - - // Upload a blob - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}").Name("uploadBlob") - - // Mounting a blob in an image - v2Router.Path("/blob/{imagename:[a-z0-9-._/]+}/{sumtype:[a-z0-9._+-]+}/{sum:[a-fA-F0-9]{4,}}").Name("mountBlob") - - return router -} - -// APIVersion2 /v2/ -var v2HTTPRoutes = newV2RegistryRouter() - -func getV2URL(e *Endpoint, routeName string, vars map[string]string) (*url.URL, error) { - route := v2HTTPRoutes.Get(routeName) - if route == nil { - return nil, fmt.Errorf("unknown regisry v2 route name: %q", routeName) - } - - varReplace := make([]string, 0, len(vars)*2) - for key, val := range vars { - varReplace = append(varReplace, key, val) - } - - routePath, err := route.URLPath(varReplace...) - if err != nil { - return nil, fmt.Errorf("unable to make registry route %q with vars %v: %s", routeName, vars, err) - } +func init() { u, err := url.Parse(REGISTRYSERVER) if err != nil { - return nil, fmt.Errorf("invalid registry url: %s", err) + panic(fmt.Errorf("invalid registry url: %s", err)) } - - return &url.URL{ - Scheme: u.Scheme, - Host: u.Host, - Path: routePath.Path, - }, nil + registryURLBuilder = v2.NewURLBuilder(u) } -// V2 Provenance POC +func getV2Builder(e *Endpoint) *v2.URLBuilder { + return registryURLBuilder +} -func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { - routeURL, err := getV2URL(r.indexEndpoint, "version", nil) - if err != nil { - return nil, err +// GetV2Authorization gets the authorization needed to the given image +// If readonly access is requested, then only the authorization may +// only be used for Get operations. +func (r *Session) GetV2Authorization(imageName string, readOnly bool) (*RequestAuthorization, error) { + scopes := []string{"pull"} + if !readOnly { + scopes = append(scopes, "push") } - method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) - if err != nil { - return nil, err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d fetching Version", res.StatusCode), res) - } - - decoder := json.NewDecoder(res.Body) - versionInfo := new(RegistryInfo) - - err = decoder.Decode(versionInfo) - if err != nil { - return nil, fmt.Errorf("unable to decode GetV2Version JSON response: %s", err) - } - - return versionInfo, nil + return NewRequestAuthorization(r.GetAuthConfig(true), r.indexEndpoint, "repository", imageName, scopes) } // @@ -112,25 +46,20 @@ func (r *Session) GetV2Version(token []string) (*RegistryInfo, error) { // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) ([]byte, error) { - vars := map[string]string{ - "imagename": imageName, - "tagname": tagName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) +func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) if err != nil { return nil, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, err @@ -155,26 +84,20 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, token []string) // - Succeeded to mount for this image scope // - Failed with no error (So continue to Push the Blob) // - Failed with error -func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []string) (bool, error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "mountBlob", vars) +func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return false, err } - method := "POST" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + method := "HEAD" + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return false, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return false, err @@ -191,25 +114,19 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, token []s return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) } -func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, token []string) error { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) +func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return err @@ -226,25 +143,19 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri return err } -func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []string) (io.ReadCloser, int64, error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, - "sum": sum, - } - - routeURL, err := getV2URL(r.indexEndpoint, "downloadBlob", vars) +func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return nil, 0, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, 0, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, 0, err @@ -267,85 +178,76 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, token []s // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, token []string) (serverChecksum string, err error) { - vars := map[string]string{ - "imagename": imageName, - "sumtype": sumType, +func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobUploadURL(imageName) + if err != nil { + return err } - routeURL, err := getV2URL(r.indexEndpoint, "uploadBlob", vars) + log.Debugf("[registry] Calling %q %s", "POST", routeURL) + req, err := r.reqFactory.NewRequest("POST", routeURL, nil) if err != nil { - return "", err + return err } - method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), blobRdr) - if err != nil { - return "", err - } - setTokenAuth(req, token) - req.Header.Set("X-Tarsum", sumStr) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { - return "", err + return err + } + location := res.Header.Get("Location") + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, location) + req, err = r.reqFactory.NewRequest(method, location, blobRdr) + if err != nil { + return err + } + queryParams := url.Values{} + queryParams.Add("digest", sumType+":"+sumStr) + req.URL.RawQuery = queryParams.Encode() + auth.Authorize(req) + res, _, err = r.doRequest(req) + if err != nil { + return err } defer res.Body.Close() - if res.StatusCode != 201 { - if res.StatusCode == 401 { - return "", errLoginRequired - } - return "", utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) - } - type sumReturn struct { - Checksum string `json:"checksum"` - } - - decoder := json.NewDecoder(res.Body) - var sumInfo sumReturn - - err = decoder.Decode(&sumInfo) - if err != nil { - return "", fmt.Errorf("unable to decode PutV2ImageBlob JSON response: %s", err) - } - - if sumInfo.Checksum != sumStr { - return "", fmt.Errorf("failed checksum comparison. serverChecksum: %q, localChecksum: %q", sumInfo.Checksum, sumStr) - } - - // XXX this is a json struct from the registry, with its checksum - return sumInfo.Checksum, nil -} - -// Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, token []string) error { - vars := map[string]string{ - "imagename": imageName, - "tagname": tagName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "manifests", vars) - if err != nil { - return err - } - - method := "PUT" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), manifestRdr) - if err != nil { - return err - } - setTokenAuth(req, token) - res, _, err := r.doRequest(req) - if err != nil { - return err - } - res.Body.Close() if res.StatusCode != 201 { if res.StatusCode == 401 { return errLoginRequired } + return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s blob", res.StatusCode, imageName), res) + } + + return nil +} + +// Finally Push the (signed) manifest of the blobs we've just pushed +func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) + if err != nil { + return err + } + + method := "PUT" + log.Debugf("[registry] Calling %q %s", method, routeURL) + req, err := r.reqFactory.NewRequest(method, routeURL, manifestRdr) + if err != nil { + return err + } + auth.Authorize(req) + res, _, err := r.doRequest(req) + if err != nil { + return err + } + b, _ := ioutil.ReadAll(res.Body) + res.Body.Close() + if res.StatusCode != 200 { + if res.StatusCode == 401 { + return errLoginRequired + } + log.Debugf("Unexpected response from server: %q %#v", b, res.Header) return utils.NewHTTPRequestError(fmt.Sprintf("Server error: %d trying to push %s:%s manifest", res.StatusCode, imageName, tagName), res) } @@ -353,24 +255,20 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R } // Given a repository name, returns a json array of string tags -func (r *Session) GetV2RemoteTags(imageName string, token []string) ([]string, error) { - vars := map[string]string{ - "imagename": imageName, - } - - routeURL, err := getV2URL(r.indexEndpoint, "tags", vars) +func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { + routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) if err != nil { return nil, err } method := "GET" - log.Debugf("[registry] Calling %q %s", method, routeURL.String()) + log.Debugf("[registry] Calling %q %s", method, routeURL) - req, err := r.reqFactory.NewRequest(method, routeURL.String(), nil) + req, err := r.reqFactory.NewRequest(method, routeURL, nil) if err != nil { return nil, err } - setTokenAuth(req, token) + auth.Authorize(req) res, _, err := r.doRequest(req) if err != nil { return nil, err diff --git a/utils/jsonmessage.go b/utils/jsonmessage.go index a2bbbcf4d4..74d3112719 100644 --- a/utils/jsonmessage.go +++ b/utils/jsonmessage.go @@ -50,6 +50,9 @@ func (p *JSONProgress) String() string { } total := units.HumanSize(float64(p.Total)) percentage := int(float64(p.Current)/float64(p.Total)*100) / 2 + if percentage > 50 { + percentage = 50 + } if width > 110 { // this number can't be negetive gh#7136 numSpaces := 0 From 7d61255f578bae7dc5c2a5d44c50bf32bbc9f568 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 19 Dec 2014 14:44:18 -0800 Subject: [PATCH 11/29] Allow private V2 registry endpoints Signed-off-by: Derek McGowan --- graph/pull.go | 2 +- graph/push.go | 12 ++++++------ registry/config.go | 2 +- registry/endpoint.go | 2 ++ registry/session_v2.go | 32 +++++++++++++++++++------------- 5 files changed, 29 insertions(+), 21 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index b138793d1f..0b75881cde 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -127,7 +127,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { logName += ":" + tag } - if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Official || endpoint.Version == registry.APIVersion2) { + if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { return job.Errorf("error updating trust base graph: %s", err) diff --git a/graph/push.go b/graph/push.go index 0d008b84c4..88b207a458 100644 --- a/graph/push.go +++ b/graph/push.go @@ -294,13 +294,14 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { tag = DEFAULTTAG } - if repoInfo.Official || endpoint.Version == registry.APIVersion2 { - j := job.Eng.Job("trust_update_base") - if err = j.Run(); err != nil { - return job.Errorf("error updating trust base graph: %s", err) + if repoInfo.Index.Official || endpoint.Version == registry.APIVersion2 { + if repoInfo.Official { + j := job.Eng.Job("trust_update_base") + if err = j.Run(); err != nil { + return job.Errorf("error updating trust base graph: %s", err) + } } - // Get authentication type auth, err := r.GetV2Authorization(repoInfo.RemoteName, false) if err != nil { return job.Errorf("error getting authorization: %s", err) @@ -383,7 +384,6 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { // done, no fallback to V1 return engine.StatusOK } else { - if err != nil { reposLen := 1 if tag == "" { diff --git a/registry/config.go b/registry/config.go index b5652b15d8..4d13aaea35 100644 --- a/registry/config.go +++ b/registry/config.go @@ -23,7 +23,7 @@ type Options struct { const ( // Only used for user auth + account creation INDEXSERVER = "https://index.docker.io/v1/" - REGISTRYSERVER = "https://registry-1.docker.io/v1/" + REGISTRYSERVER = "https://registry-1.docker.io/v2/" INDEXNAME = "docker.io" // INDEXSERVER = "https://registry-stage.hub.docker.com/v1/" diff --git a/registry/endpoint.go b/registry/endpoint.go index 5c5b052000..9a783f1f05 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -10,6 +10,7 @@ import ( "strings" log "github.com/Sirupsen/logrus" + "github.com/docker/docker/registry/v2" ) // for mocking in unit tests @@ -103,6 +104,7 @@ type Endpoint struct { Version APIVersion IsSecure bool AuthChallenges []*AuthorizationChallenge + URLBuilder *v2.URLBuilder } // Get the formated URL for the root of this registry Endpoint diff --git a/registry/session_v2.go b/registry/session_v2.go index 407c5f3a23..2304a61344 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -13,30 +13,36 @@ import ( "github.com/docker/docker/utils" ) -var registryURLBuilder *v2.URLBuilder - -func init() { - u, err := url.Parse(REGISTRYSERVER) - if err != nil { - panic(fmt.Errorf("invalid registry url: %s", err)) - } - registryURLBuilder = v2.NewURLBuilder(u) -} - func getV2Builder(e *Endpoint) *v2.URLBuilder { - return registryURLBuilder + if e.URLBuilder == nil { + e.URLBuilder = v2.NewURLBuilder(e.URL) + } + return e.URLBuilder } // GetV2Authorization gets the authorization needed to the given image // If readonly access is requested, then only the authorization may // only be used for Get operations. -func (r *Session) GetV2Authorization(imageName string, readOnly bool) (*RequestAuthorization, error) { +func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *RequestAuthorization, err error) { scopes := []string{"pull"} if !readOnly { scopes = append(scopes, "push") } - return NewRequestAuthorization(r.GetAuthConfig(true), r.indexEndpoint, "repository", imageName, scopes) + var registry *Endpoint + if r.indexEndpoint.URL.Host == IndexServerURL.Host { + registry, err = NewEndpoint(REGISTRYSERVER, nil) + if err != nil { + return + } + } else { + registry = r.indexEndpoint + } + registry.URLBuilder = v2.NewURLBuilder(registry.URL) + r.indexEndpoint = registry + + log.Debugf("Getting authorization for %s %s", imageName, scopes) + return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes) } // From d094eb6f7ffe6b608ecde54297e107e5caa0954d Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 19 Dec 2014 16:14:04 -0800 Subject: [PATCH 12/29] Get token on each request Signed-off-by: Derek McGowan --- registry/auth.go | 60 ++++++++++++++++++++++++++---------------- registry/session_v2.go | 34 +++++++++++++++++------- 2 files changed, 62 insertions(+), 32 deletions(-) diff --git a/registry/auth.go b/registry/auth.go index b138fb530d..1e1c7ddb82 100644 --- a/registry/auth.go +++ b/registry/auth.go @@ -38,56 +38,70 @@ type ConfigFile struct { } type RequestAuthorization struct { - Token string - Username string - Password string + authConfig *AuthConfig + registryEndpoint *Endpoint + resource string + scope string + actions []string } -func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) (*RequestAuthorization, error) { - var auth RequestAuthorization +func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { + return &RequestAuthorization{ + authConfig: authConfig, + registryEndpoint: registryEndpoint, + resource: resource, + scope: scope, + actions: actions, + } +} +func (auth *RequestAuthorization) getToken() (string, error) { + // TODO check if already has token and before expiration client := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, - Proxy: http.ProxyFromEnvironment, - }, + Proxy: http.ProxyFromEnvironment}, CheckRedirect: AddRequiredHeadersToRedirectedRequests, } factory := HTTPRequestFactory(nil) - for _, challenge := range registryEndpoint.AuthChallenges { - log.Debugf("Using %q auth challenge with params %s for %s", challenge.Scheme, challenge.Parameters, authConfig.Username) - + for _, challenge := range auth.registryEndpoint.AuthChallenges { switch strings.ToLower(challenge.Scheme) { case "basic": - auth.Username = authConfig.Username - auth.Password = authConfig.Password + // no token necessary case "bearer": + log.Debugf("Getting bearer token with %s for %s", challenge.Parameters, auth.authConfig.Username) params := map[string]string{} for k, v := range challenge.Parameters { params[k] = v } - params["scope"] = fmt.Sprintf("%s:%s:%s", resource, scope, strings.Join(actions, ",")) - token, err := getToken(authConfig.Username, authConfig.Password, params, registryEndpoint, client, factory) + params["scope"] = fmt.Sprintf("%s:%s:%s", auth.resource, auth.scope, strings.Join(auth.actions, ",")) + token, err := getToken(auth.authConfig.Username, auth.authConfig.Password, params, auth.registryEndpoint, client, factory) if err != nil { - return nil, err + return "", err } + // TODO cache token and set expiration to one minute from now - auth.Token = token + return token, nil default: log.Infof("Unsupported auth scheme: %q", challenge.Scheme) } } - - return &auth, nil + // TODO no expiration, do not reattempt to get a token + return "", nil } -func (auth *RequestAuthorization) Authorize(req *http.Request) { - if auth.Token != "" { - req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", auth.Token)) - } else if auth.Username != "" && auth.Password != "" { - req.SetBasicAuth(auth.Username, auth.Password) +func (auth *RequestAuthorization) Authorize(req *http.Request) error { + token, err := auth.getToken() + if err != nil { + return err } + if token != "" { + req.Header.Set("Authorization", fmt.Sprintf("Bearer %s", token)) + } else if auth.authConfig.Username != "" && auth.authConfig.Password != "" { + req.SetBasicAuth(auth.authConfig.Username, auth.authConfig.Password) + } + return nil } // create a base64 encoded auth string to store in config diff --git a/registry/session_v2.go b/registry/session_v2.go index 2304a61344..491cd2c6e0 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -42,7 +42,7 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req r.indexEndpoint = registry log.Debugf("Getting authorization for %s %s", imageName, scopes) - return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes) + return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes), nil } // @@ -65,7 +65,9 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut if err != nil { return nil, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, err @@ -103,7 +105,9 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req if err != nil { return false, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return false, err @@ -132,7 +136,9 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri if err != nil { return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -161,7 +167,9 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req if err != nil { return nil, 0, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, 0, err @@ -196,7 +204,9 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -212,7 +222,9 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R queryParams := url.Values{} queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err = r.doRequest(req) if err != nil { return err @@ -242,7 +254,9 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R if err != nil { return err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return err @@ -274,7 +288,9 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) if err != nil { return nil, err } - auth.Authorize(req) + if err := auth.Authorize(req) { + return nil, err + } res, _, err := r.doRequest(req) if err != nil { return nil, err From 1b43144ad8597d0d0ca089042c1162ba668259ab Mon Sep 17 00:00:00 2001 From: Stephen J Day Date: Mon, 22 Dec 2014 14:58:08 -0800 Subject: [PATCH 13/29] Correctly check and propagate errors in v2 session Signed-off-by: Stephen J Day --- registry/session_v2.go | 28 ++++++++++++++-------------- 1 file changed, 14 insertions(+), 14 deletions(-) diff --git a/registry/session_v2.go b/registry/session_v2.go index 491cd2c6e0..411df46e35 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -65,7 +65,7 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut if err != nil { return nil, err } - if err := auth.Authorize(req) { + if err := auth.Authorize(req); err != nil { return nil, err } res, _, err := r.doRequest(req) @@ -105,8 +105,8 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req if err != nil { return false, err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return false, err } res, _, err := r.doRequest(req) if err != nil { @@ -136,8 +136,8 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri if err != nil { return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -167,8 +167,8 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req if err != nil { return nil, 0, err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return nil, 0, err } res, _, err := r.doRequest(req) if err != nil { @@ -204,8 +204,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -222,8 +222,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R queryParams := url.Values{} queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err = r.doRequest(req) if err != nil { @@ -254,8 +254,8 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R if err != nil { return err } - if err := auth.Authorize(req) { - return nil, err + if err := auth.Authorize(req); err != nil { + return err } res, _, err := r.doRequest(req) if err != nil { @@ -288,7 +288,7 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) if err != nil { return nil, err } - if err := auth.Authorize(req) { + if err := auth.Authorize(req); err != nil { return nil, err } res, _, err := r.doRequest(req) From 7eeda3f14de744b98b1c5aca4f2ecce87a479baa Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 22 Dec 2014 18:58:01 -0800 Subject: [PATCH 14/29] Fix tests Signed-off-by: Derek McGowan --- utils/jsonmessage_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/utils/jsonmessage_test.go b/utils/jsonmessage_test.go index 0ce9492c98..b9103da1a4 100644 --- a/utils/jsonmessage_test.go +++ b/utils/jsonmessage_test.go @@ -30,7 +30,7 @@ func TestProgress(t *testing.T) { } // this number can't be negetive gh#7136 - expected = "[==============================================================>] 50 B/40 B" + expected = "[==================================================>] 50 B/40 B" jp4 := JSONProgress{Current: 50, Total: 40} if jp4.String() != expected { t.Fatalf("Expected %q, got %q", expected, jp4.String()) From 213e3d116642431adbe634d39740eddc5a81e063 Mon Sep 17 00:00:00 2001 From: Josh Hawn Date: Tue, 23 Dec 2014 13:40:06 -0800 Subject: [PATCH 15/29] Add Tarsum Calculation during v2 Pull operation While the v2 pull operation is writing the body of the layer blob to disk it now computes the tarsum checksum of the archive before extracting it to the backend storage driver. If the checksum does not match that from the image manifest an error is raised. Also adds more debug logging to the pull operation and fixes existing test cases which were failing. Adds a reverse lookup constructor to the tarsum package so that you can get a tarsum object using a checksum label. Docker-DCO-1.1-Signed-off-by: Josh Hawn (github: jlhawn) --- graph/pull.go | 30 ++++++++++++++++++++++++++++-- image/image.go | 39 ++++++++++++++++++++------------------- pkg/tarsum/tarsum.go | 39 +++++++++++++++++++++++++++++++++++++++ pkg/tarsum/versioning.go | 17 ++++++++++++----- registry/endpoint.go | 17 ++++++++++++----- registry/session_v2.go | 8 ++++++-- 6 files changed, 117 insertions(+), 33 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index 0b75881cde..88e939a481 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -15,6 +15,7 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" "github.com/docker/docker/image" + "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/registry" "github.com/docker/docker/utils" "github.com/docker/libtrust" @@ -112,6 +113,8 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { } defer s.poolRemove("pull", repoInfo.LocalName+":"+tag) + + log.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName) endpoint, err := repoInfo.GetEndpoint() if err != nil { return job.Error(err) @@ -127,6 +130,10 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { logName += ":" + tag } + // Calling the v2 code path might change the session + // endpoint value, so save the original one! + originalSession := *r + if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { @@ -138,6 +145,7 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { return job.Errorf("error getting authorization: %s", err) } + log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil { if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { log.Errorf("Error logging event 'pull' for %s: %s", logName, err) @@ -146,8 +154,13 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { } else if err != registry.ErrDoesNotExist { log.Errorf("Error from V2 registry: %s", err) } + + log.Debug("image does not exist on v2 registry, falling back to v1") } + r = &originalSession + + log.Debugf("pulling v1 repository with local name %q", repoInfo.LocalName) if err = s.pullRepository(r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err != nil { return job.Error(err) } @@ -174,7 +187,7 @@ func (s *TagStore) pullRepository(r *registry.Session, out io.Writer, repoInfo * log.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, repoInfo.RemoteName, repoData.Tokens) if err != nil { - log.Errorf("%v", err) + log.Errorf("unable to get remote tags: %s", err) return err } @@ -535,7 +548,20 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri return err } defer r.Close() - io.Copy(tmpFile, utils.ProgressReader(r, int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading")) + + // Wrap the reader with the appropriate TarSum reader. + tarSumReader, err := tarsum.NewTarSumForLabel(r, true, sumType) + if err != nil { + return fmt.Errorf("unable to wrap image blob reader with TarSum: %s", err) + } + + io.Copy(tmpFile, utils.ProgressReader(ioutil.NopCloser(tarSumReader), int(l), out, sf, false, utils.TruncateID(img.ID), "Downloading")) + + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Verifying Checksum", nil)) + + if finalChecksum := tarSumReader.Sum(nil); !strings.EqualFold(finalChecksum, sumStr) { + return fmt.Errorf("image verification failed: checksum mismatch - expected %q but got %q", sumStr, finalChecksum) + } out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Download complete", nil)) diff --git a/image/image.go b/image/image.go index 8cd9aa3755..7664602cd8 100644 --- a/image/image.go +++ b/image/image.go @@ -94,28 +94,29 @@ func StoreImage(img *Image, layerData archive.ArchiveReader, root string) error // If layerData is not nil, unpack it into the new layer if layerData != nil { - layerDataDecompressed, err := archive.DecompressStream(layerData) - if err != nil { + // If the image doesn't have a checksum, we should add it. The layer + // checksums are verified when they are pulled from a remote, but when + // a container is committed it should be added here. + if img.Checksum == "" { + layerDataDecompressed, err := archive.DecompressStream(layerData) + if err != nil { + return err + } + defer layerDataDecompressed.Close() + + if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil { + return err + } + + if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil { + return err + } + + img.Checksum = layerTarSum.Sum(nil) + } else if size, err = driver.ApplyDiff(img.ID, img.Parent, layerData); err != nil { return err } - defer layerDataDecompressed.Close() - - if layerTarSum, err = tarsum.NewTarSum(layerDataDecompressed, true, tarsum.VersionDev); err != nil { - return err - } - - if size, err = driver.ApplyDiff(img.ID, img.Parent, layerTarSum); err != nil { - return err - } - - checksum := layerTarSum.Sum(nil) - - if img.Checksum != "" && img.Checksum != checksum { - log.Warnf("image layer checksum mismatch: computed %q, expected %q", checksum, img.Checksum) - } - - img.Checksum = checksum } img.Size = size diff --git a/pkg/tarsum/tarsum.go b/pkg/tarsum/tarsum.go index c9f1315cf5..c6a7294e74 100644 --- a/pkg/tarsum/tarsum.go +++ b/pkg/tarsum/tarsum.go @@ -3,8 +3,11 @@ package tarsum import ( "bytes" "compress/gzip" + "crypto" "crypto/sha256" "encoding/hex" + "errors" + "fmt" "hash" "io" "strings" @@ -39,6 +42,30 @@ func NewTarSumHash(r io.Reader, dc bool, v Version, tHash THash) (TarSum, error) return ts, err } +// Create a new TarSum using the provided TarSum version+hash label. +func NewTarSumForLabel(r io.Reader, disableCompression bool, label string) (TarSum, error) { + parts := strings.SplitN(label, "+", 2) + if len(parts) != 2 { + return nil, errors.New("tarsum label string should be of the form: {tarsum_version}+{hash_name}") + } + + versionName, hashName := parts[0], parts[1] + + version, ok := tarSumVersionsByName[versionName] + if !ok { + return nil, fmt.Errorf("unknown TarSum version name: %q", versionName) + } + + hashConfig, ok := standardHashConfigs[hashName] + if !ok { + return nil, fmt.Errorf("unknown TarSum hash name: %q", hashName) + } + + tHash := NewTHash(hashConfig.name, hashConfig.hash.New) + + return NewTarSumHash(r, disableCompression, version, tHash) +} + // TarSum is the generic interface for calculating fixed time // checksums of a tar archive type TarSum interface { @@ -89,6 +116,18 @@ func NewTHash(name string, h func() hash.Hash) THash { return simpleTHash{n: name, h: h} } +type tHashConfig struct { + name string + hash crypto.Hash +} + +var ( + standardHashConfigs = map[string]tHashConfig{ + "sha256": {name: "sha256", hash: crypto.SHA256}, + "sha512": {name: "sha512", hash: crypto.SHA512}, + } +) + // TarSum default is "sha256" var DefaultTHash = NewTHash("sha256", sha256.New) diff --git a/pkg/tarsum/versioning.go b/pkg/tarsum/versioning.go index 3a656612ff..be1d07040f 100644 --- a/pkg/tarsum/versioning.go +++ b/pkg/tarsum/versioning.go @@ -31,11 +31,18 @@ func GetVersions() []Version { return v } -var tarSumVersions = map[Version]string{ - Version0: "tarsum", - Version1: "tarsum.v1", - VersionDev: "tarsum.dev", -} +var ( + tarSumVersions = map[Version]string{ + Version0: "tarsum", + Version1: "tarsum.v1", + VersionDev: "tarsum.dev", + } + tarSumVersionsByName = map[string]Version{ + "tarsum": Version0, + "tarsum.v1": Version1, + "tarsum.dev": VersionDev, + } +) func (tsv Version) String() string { return tarSumVersions[tsv] diff --git a/registry/endpoint.go b/registry/endpoint.go index 9a783f1f05..9ca9ed8b9a 100644 --- a/registry/endpoint.go +++ b/registry/endpoint.go @@ -47,16 +47,23 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { if err != nil { return nil, err } + if err := validateEndpoint(endpoint); err != nil { + return nil, err + } + return endpoint, nil +} + +func validateEndpoint(endpoint *Endpoint) error { log.Debugf("pinging registry endpoint %s", endpoint) // Try HTTPS ping to registry endpoint.URL.Scheme = "https" if _, err := endpoint.Ping(); err != nil { - if index.Secure { + if endpoint.IsSecure { // If registry is secure and HTTPS failed, show user the error and tell them about `--insecure-registry` // in case that's what they need. DO NOT accept unknown CA certificates, and DO NOT fallback to HTTP. - return nil, fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) + return fmt.Errorf("invalid registry endpoint %s: %v. If this private registry supports only HTTP or HTTPS with an unknown CA certificate, please add `--insecure-registry %s` to the daemon's arguments. In the case of HTTPS, if you have access to the registry's CA certificate, no need for the flag; simply place the CA certificate at /etc/docker/certs.d/%s/ca.crt", endpoint, err, endpoint.URL.Host, endpoint.URL.Host) } // If registry is insecure and HTTPS failed, fallback to HTTP. @@ -65,13 +72,13 @@ func NewEndpoint(index *IndexInfo) (*Endpoint, error) { var err2 error if _, err2 = endpoint.Ping(); err2 == nil { - return endpoint, nil + return nil } - return nil, fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) + return fmt.Errorf("invalid registry endpoint %q. HTTPS attempt: %v. HTTP attempt: %v", endpoint, err, err2) } - return endpoint, nil + return nil } func newEndpoint(address string, secure bool) (*Endpoint, error) { diff --git a/registry/session_v2.go b/registry/session_v2.go index 411df46e35..031122dcf6 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -30,8 +30,12 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req } var registry *Endpoint - if r.indexEndpoint.URL.Host == IndexServerURL.Host { - registry, err = NewEndpoint(REGISTRYSERVER, nil) + if r.indexEndpoint.String() == IndexServerAddress() { + registry, err = newEndpoint(REGISTRYSERVER, true) + if err != nil { + return + } + err = validateEndpoint(registry) if err != nil { return } From 25945a40c4f352a754cbd8dba9c846c7539fe463 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 2 Jan 2015 11:13:11 -0800 Subject: [PATCH 16/29] Refactor from feedback Signed-off-by: Derek McGowan (github: dmcgowan) --- docker/docker.go | 16 +-------- graph/manifest.go | 79 ++++++++++++++++++++++++++++++++++++++++-- graph/pull.go | 67 ++--------------------------------- graph/push.go | 10 ++---- registry/session_v2.go | 3 +- 5 files changed, 82 insertions(+), 93 deletions(-) diff --git a/docker/docker.go b/docker/docker.go index 84ffeace9a..92f5f14603 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -6,7 +6,6 @@ import ( "fmt" "io/ioutil" "os" - "path" "strings" log "github.com/Sirupsen/logrus" @@ -16,7 +15,6 @@ import ( flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/reexec" "github.com/docker/docker/utils" - "github.com/docker/libtrust" ) const ( @@ -79,22 +77,10 @@ func main() { } protoAddrParts := strings.SplitN(flHosts[0], "://", 2) - err := os.MkdirAll(path.Dir(*flTrustKey), 0700) + trustKey, err := api.LoadOrCreateTrustKey(*flTrustKey) if err != nil { log.Fatal(err) } - trustKey, err := libtrust.LoadKeyFile(*flTrustKey) - if err == libtrust.ErrKeyFileDoesNotExist { - trustKey, err = libtrust.GenerateECP256PrivateKey() - if err != nil { - log.Fatalf("Error generating key: %s", err) - } - if err := libtrust.SaveKey(*flTrustKey, trustKey); err != nil { - log.Fatalf("Error saving key file: %s", err) - } - } else if err != nil { - log.Fatalf("Error loading key file: %s", err) - } var ( cli *client.DockerCli diff --git a/graph/manifest.go b/graph/manifest.go index 54d6083cba..3d4ab1c5de 100644 --- a/graph/manifest.go +++ b/graph/manifest.go @@ -1,6 +1,7 @@ package graph import ( + "bytes" "encoding/json" "errors" "fmt" @@ -8,10 +9,12 @@ import ( "io/ioutil" "path" + log "github.com/Sirupsen/logrus" "github.com/docker/docker/engine" "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/registry" "github.com/docker/docker/runconfig" + "github.com/docker/libtrust" ) func (s *TagStore) CmdManifest(job *engine.Job) engine.Status { @@ -49,11 +52,15 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error Tag: tag, SchemaVersion: 1, } - localRepo, exists := s.Repositories[localName] - if !exists { + localRepo, err := s.Get(localName) + if err != nil { + return nil, err + } + if localRepo == nil { return nil, fmt.Errorf("Repo does not exist: %s", localName) } + // Get the top-most layer id which the tag points to layerId, exists := localRepo[tag] if !exists { return nil, fmt.Errorf("Tag does not exist for %s: %s", localName, tag) @@ -102,7 +109,6 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error } tarId := tarSum.Sum(nil) - // Save tarsum to image json manifest.FSLayers = append(manifest.FSLayers, ®istry.FSLayer{BlobSum: tarId}) @@ -121,3 +127,70 @@ func (s *TagStore) newManifest(localName, remoteName, tag string) ([]byte, error return manifestBytes, nil } + +func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*registry.ManifestData, bool, error) { + sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures") + if err != nil { + return nil, false, fmt.Errorf("error parsing payload: %s", err) + } + + keys, err := sig.Verify() + if err != nil { + return nil, false, fmt.Errorf("error verifying payload: %s", err) + } + + payload, err := sig.Payload() + if err != nil { + return nil, false, fmt.Errorf("error retrieving payload: %s", err) + } + + var manifest registry.ManifestData + if err := json.Unmarshal(payload, &manifest); err != nil { + return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err) + } + if manifest.SchemaVersion != 1 { + return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion) + } + + var verified bool + for _, key := range keys { + job := eng.Job("trust_key_check") + b, err := key.MarshalJSON() + if err != nil { + return nil, false, fmt.Errorf("error marshalling public key: %s", err) + } + namespace := manifest.Name + if namespace[0] != '/' { + namespace = "/" + namespace + } + stdoutBuffer := bytes.NewBuffer(nil) + + job.Args = append(job.Args, namespace) + job.Setenv("PublicKey", string(b)) + // Check key has read/write permission (0x03) + job.SetenvInt("Permission", 0x03) + job.Stdout.Add(stdoutBuffer) + if err = job.Run(); err != nil { + return nil, false, fmt.Errorf("error running key check: %s", err) + } + result := engine.Tail(stdoutBuffer, 1) + log.Debugf("Key check result: %q", result) + if result == "verified" { + verified = true + } + } + + return &manifest, verified, nil +} + +func checkValidManifest(manifest *registry.ManifestData) error { + if len(manifest.FSLayers) != len(manifest.History) { + return fmt.Errorf("length of history not equal to number of layers") + } + + if len(manifest.FSLayers) == 0 { + return fmt.Errorf("no FSLayers in manifest") + } + + return nil +} diff --git a/graph/pull.go b/graph/pull.go index 88e939a481..b2710e9b68 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -1,8 +1,6 @@ package graph import ( - "bytes" - "encoding/json" "fmt" "io" "io/ioutil" @@ -18,63 +16,8 @@ import ( "github.com/docker/docker/pkg/tarsum" "github.com/docker/docker/registry" "github.com/docker/docker/utils" - "github.com/docker/libtrust" ) -func (s *TagStore) verifyManifest(eng *engine.Engine, manifestBytes []byte) (*registry.ManifestData, bool, error) { - sig, err := libtrust.ParsePrettySignature(manifestBytes, "signatures") - if err != nil { - return nil, false, fmt.Errorf("error parsing payload: %s", err) - } - keys, err := sig.Verify() - if err != nil { - return nil, false, fmt.Errorf("error verifying payload: %s", err) - } - - payload, err := sig.Payload() - if err != nil { - return nil, false, fmt.Errorf("error retrieving payload: %s", err) - } - - var manifest registry.ManifestData - if err := json.Unmarshal(payload, &manifest); err != nil { - return nil, false, fmt.Errorf("error unmarshalling manifest: %s", err) - } - if manifest.SchemaVersion != 1 { - return nil, false, fmt.Errorf("unsupported schema version: %d", manifest.SchemaVersion) - } - - var verified bool - for _, key := range keys { - job := eng.Job("trust_key_check") - b, err := key.MarshalJSON() - if err != nil { - return nil, false, fmt.Errorf("error marshalling public key: %s", err) - } - namespace := manifest.Name - if namespace[0] != '/' { - namespace = "/" + namespace - } - stdoutBuffer := bytes.NewBuffer(nil) - - job.Args = append(job.Args, namespace) - job.Setenv("PublicKey", string(b)) - // Check key has read/write permission (0x03) - job.SetenvInt("Permission", 0x03) - job.Stdout.Add(stdoutBuffer) - if err = job.Run(); err != nil { - return nil, false, fmt.Errorf("error running key check: %s", err) - } - result := engine.Tail(stdoutBuffer, 1) - log.Debugf("Key check result: %q", result) - if result == "verified" { - verified = true - } - } - - return &manifest, verified, nil -} - func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if n := len(job.Args); n != 1 && n != 2 { return job.Errorf("Usage: %s IMAGE [TAG]", job.Name) @@ -113,7 +56,6 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { } defer s.poolRemove("pull", repoInfo.LocalName+":"+tag) - log.Debugf("pulling image from host %q with remote name %q", repoInfo.Index.Name, repoInfo.RemoteName) endpoint, err := repoInfo.GetEndpoint() if err != nil { @@ -484,8 +426,8 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri return false, fmt.Errorf("error verifying manifest: %s", err) } - if len(manifest.FSLayers) != len(manifest.History) { - return false, fmt.Errorf("length of history not equal to number of layers") + if err := checkValidManifest(manifest); err != nil { + return false, err } if verified { @@ -493,11 +435,6 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri } else { out.Write(sf.FormatStatus(tag, "Pulling from %s", repoInfo.CanonicalName)) } - - if len(manifest.FSLayers) == 0 { - return false, fmt.Errorf("no blobSums in manifest") - } - downloads := make([]downloadInfo, len(manifest.FSLayers)) for i := len(manifest.FSLayers) - 1; i >= 0; i-- { diff --git a/graph/push.go b/graph/push.go index 88b207a458..8d51e28798 100644 --- a/graph/push.go +++ b/graph/push.go @@ -311,14 +311,13 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { // TODO Create manifest and sign } - // try via manifest manifest, verified, err := s.verifyManifest(job.Eng, []byte(manifestBytes)) if err != nil { return job.Errorf("error verifying manifest: %s", err) } - if len(manifest.FSLayers) != len(manifest.History) { - return job.Errorf("length of history not equal to number of layers") + if err := checkValidManifest(manifest); err != nil { + return job.Errorf("invalid manifest: %s", err) } if !verified { @@ -337,11 +336,6 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } manifestSum := sumParts[1] - // for each layer, check if it exists ... - // XXX wait this requires having the TarSum of the layer.tar first - // skip this step for now. Just push the layer every time for this naive implementation - //shouldPush, err := r.PostV2ImageMountBlob(imageName, sumType, sum string, token []string) - img, err := image.NewImgJSON(imgJSON) if err != nil { return job.Errorf("Failed to parse json: %s", err) diff --git a/registry/session_v2.go b/registry/session_v2.go index 031122dcf6..0e03f4a9ce 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -5,7 +5,6 @@ import ( "fmt" "io" "io/ioutil" - "net/url" "strconv" log "github.com/Sirupsen/logrus" @@ -223,7 +222,7 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R if err != nil { return err } - queryParams := url.Values{} + queryParams := req.URL.Query() queryParams.Add("digest", sumType+":"+sumStr) req.URL.RawQuery = queryParams.Encode() if err := auth.Authorize(req); err != nil { From 8ceb9d20d66097b90ca3a529da258669ef6b8412 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 7 Jan 2015 14:59:12 -0800 Subject: [PATCH 17/29] Update push to sign with the daemon's key when no manifest is given Signed-off-by: Derek McGowan (github: dmcgowan) --- daemon/daemon.go | 12 ++++++------ graph/push.go | 22 +++++++++++++++++++++- graph/tags.go | 5 ++++- graph/tags_unit_test.go | 2 +- 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/daemon/daemon.go b/daemon/daemon.go index 9f5df4c3c5..8a5db74a33 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -895,8 +895,13 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) return nil, err } + trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath) + if err != nil { + return nil, err + } + log.Debugf("Creating repository list") - repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g) + repositories, err := graph.NewTagStore(path.Join(config.Root, "repositories-"+driver.String()), g, trustKey) if err != nil { return nil, fmt.Errorf("Couldn't create Tag store: %s", err) } @@ -961,11 +966,6 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) return nil, err } - trustKey, err := api.LoadOrCreateTrustKey(config.TrustKeyPath) - if err != nil { - return nil, err - } - daemon := &Daemon{ ID: trustKey.PublicKey().KeyID(), repository: daemonRepo, diff --git a/graph/push.go b/graph/push.go index 8d51e28798..4d6b1e0838 100644 --- a/graph/push.go +++ b/graph/push.go @@ -16,6 +16,7 @@ import ( "github.com/docker/docker/pkg/archive" "github.com/docker/docker/registry" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) // Retrieve the all the images to be uploaded in the correct order @@ -308,7 +309,26 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } if len(manifestBytes) == 0 { - // TODO Create manifest and sign + mBytes, err := s.newManifest(repoInfo.LocalName, repoInfo.RemoteName, tag) + if err != nil { + return job.Error(err) + } + js, err := libtrust.NewJSONSignature(mBytes) + if err != nil { + return job.Error(err) + } + + if err = js.Sign(s.trustKey); err != nil { + return job.Error(err) + } + + signedBody, err := js.PrettySignature("signatures") + if err != nil { + return job.Error(err) + } + log.Infof("Signed manifest using daemon's key: %s", s.trustKey.KeyID()) + + manifestBytes = string(signedBody) } manifest, verified, err := s.verifyManifest(job.Eng, []byte(manifestBytes)) diff --git a/graph/tags.go b/graph/tags.go index 998b447e6c..6bdb296cd1 100644 --- a/graph/tags.go +++ b/graph/tags.go @@ -15,6 +15,7 @@ import ( "github.com/docker/docker/pkg/parsers" "github.com/docker/docker/registry" "github.com/docker/docker/utils" + "github.com/docker/libtrust" ) const DEFAULTTAG = "latest" @@ -27,6 +28,7 @@ type TagStore struct { path string graph *Graph Repositories map[string]Repository + trustKey libtrust.PrivateKey sync.Mutex // FIXME: move push/pull-related fields // to a helper type @@ -54,7 +56,7 @@ func (r Repository) Contains(u Repository) bool { return true } -func NewTagStore(path string, graph *Graph) (*TagStore, error) { +func NewTagStore(path string, graph *Graph, key libtrust.PrivateKey) (*TagStore, error) { abspath, err := filepath.Abs(path) if err != nil { return nil, err @@ -63,6 +65,7 @@ func NewTagStore(path string, graph *Graph) (*TagStore, error) { store := &TagStore{ path: abspath, graph: graph, + trustKey: key, Repositories: make(map[string]Repository), pullingPool: make(map[string]chan struct{}), pushingPool: make(map[string]chan struct{}), diff --git a/graph/tags_unit_test.go b/graph/tags_unit_test.go index 45dad62951..58ad8ed878 100644 --- a/graph/tags_unit_test.go +++ b/graph/tags_unit_test.go @@ -57,7 +57,7 @@ func mkTestTagStore(root string, t *testing.T) *TagStore { if err != nil { t.Fatal(err) } - store, err := NewTagStore(path.Join(root, "tags"), graph) + store, err := NewTagStore(path.Join(root, "tags"), graph, nil) if err != nil { t.Fatal(err) } From 1a9cdb13943c6af397472e235708cb10824681cd Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 7 Jan 2015 15:55:29 -0800 Subject: [PATCH 18/29] Fix list tags Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/session_v2.go | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/registry/session_v2.go b/registry/session_v2.go index 0e03f4a9ce..b08f4cf0d8 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -277,6 +277,11 @@ func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.R return nil } +type remoteTags struct { + name string + tags []string +} + // Given a repository name, returns a json array of string tags func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) @@ -309,10 +314,10 @@ func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) } decoder := json.NewDecoder(res.Body) - var tags []string - err = decoder.Decode(&tags) + var remote remoteTags + err = decoder.Decode(&remote) if err != nil { return nil, fmt.Errorf("Error while decoding the http response: %s", err) } - return tags, nil + return remote.tags, nil } From 9a38aa0279ccae5aeded854a9cbbd7e398088ab2 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Mon, 12 Jan 2015 14:17:50 -0800 Subject: [PATCH 19/29] Fix integration test failures Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/pull.go | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index b2710e9b68..1c4bb9d88c 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -79,22 +79,23 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { - return job.Errorf("error updating trust base graph: %s", err) + log.Errorf("error updating trust base graph: %s", err) } auth, err := r.GetV2Authorization(repoInfo.RemoteName, true) if err != nil { - return job.Errorf("error getting authorization: %s", err) - } + log.Errorf("error getting authorization: %s", err) + } else { - log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) - if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil { - if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { - log.Errorf("Error logging event 'pull' for %s: %s", logName, err) + log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) + if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil { + if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { + log.Errorf("Error logging event 'pull' for %s: %s", logName, err) + } + return engine.StatusOK + } else if err != registry.ErrDoesNotExist { + log.Errorf("Error from V2 registry: %s", err) } - return engine.StatusOK - } else if err != registry.ErrDoesNotExist { - log.Errorf("Error from V2 registry: %s", err) } log.Debug("image does not exist on v2 registry, falling back to v1") From ef96c28754706da921644e5cf9202f9cc78d4c7e Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Mon, 12 Jan 2015 11:47:40 -0800 Subject: [PATCH 20/29] Install registry V2 in image Signed-off-by: Alexander Morozov --- Dockerfile | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/Dockerfile b/Dockerfile index 9ef05b5613..6bad987205 100644 --- a/Dockerfile +++ b/Dockerfile @@ -148,6 +148,17 @@ RUN set -x \ && git clone -b v1.2 https://github.com/russross/blackfriday.git /go/src/github.com/russross/blackfriday \ && go install -v github.com/cpuguy83/go-md2man +# Install registry +COPY pkg/tarsum /go/src/github.com/docker/docker/pkg/tarsum +# REGISTRY_COMMIT gives us the repeatability guarantees we need +# (so that we're all testing the same version of the registry) +ENV REGISTRY_COMMIT 21a69f53b5c7986b831f33849d551cd59ec8cbd1 +RUN set -x \ + && git clone https://github.com/docker/distribution.git /go/src/github.com/docker/distribution \ + && (cd /go/src/github.com/docker/distribution && git checkout -q $REGISTRY_COMMIT) \ + && go get -d github.com/docker/distribution/cmd/registry \ + && go build -o /go/bin/registry-v2 github.com/docker/distribution/cmd/registry + # Wrap all commands in the "docker-in-docker" script to allow nested containers ENTRYPOINT ["hack/dind"] From 2fc2862a73dbbc612f59f61f66c465d2e48bcbea Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Mon, 12 Jan 2015 13:26:49 -0800 Subject: [PATCH 21/29] RegistryV2 datastructure for tests Signed-off-by: Alexander Morozov --- integration-cli/registry.go | 60 +++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 integration-cli/registry.go diff --git a/integration-cli/registry.go b/integration-cli/registry.go new file mode 100644 index 0000000000..f0ef05cca1 --- /dev/null +++ b/integration-cli/registry.go @@ -0,0 +1,60 @@ +package main + +import ( + "fmt" + "io/ioutil" + "os" + "os/exec" + "path/filepath" + "testing" +) + +const v2binary = "registry-v2" + +type testRegistryV2 struct { + URL string + cmd *exec.Cmd + dir string +} + +func newTestRegistryV2(t *testing.T) (*testRegistryV2, error) { + template := `version: 0.1 +loglevel: debug +storage: + filesystem: + rootdirectory: %s +http: + addr: :%s` + tmp, err := ioutil.TempDir("", "registry-test-") + if err != nil { + return nil, err + } + confPath := filepath.Join(tmp, "config.yaml") + config, err := os.Create(confPath) + if err != nil { + return nil, err + } + if _, err := fmt.Fprintf(config, template, tmp, "5000"); err != nil { + os.RemoveAll(tmp) + return nil, err + } + + cmd := exec.Command(v2binary, confPath) + if err := cmd.Start(); err != nil { + os.RemoveAll(tmp) + if os.IsNotExist(err) { + t.Skip() + } + return nil, err + } + return &testRegistryV2{ + cmd: cmd, + dir: tmp, + URL: "localhost:5000", + }, nil +} + +func (r *testRegistryV2) Close() { + r.cmd.Process.Kill() + os.RemoveAll(r.dir) +} From f138f7bd50a1c5a435f3146f0b0298a2a4e260ce Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Mon, 12 Jan 2015 14:30:19 -0800 Subject: [PATCH 22/29] Tests for push to registry v2 Signed-off-by: Alexander Morozov --- integration-cli/docker_cli_push_test.go | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/integration-cli/docker_cli_push_test.go b/integration-cli/docker_cli_push_test.go index 0dfd85a9d4..a8c2ccdbc3 100644 --- a/integration-cli/docker_cli_push_test.go +++ b/integration-cli/docker_cli_push_test.go @@ -10,29 +10,27 @@ import ( // pulling an image from the central registry should work func TestPushBusyboxImage(t *testing.T) { - // skip this test until we're able to use a registry - t.Skip() + reg, err := newTestRegistryV2(t) + if err != nil { + t.Fatal(err) + } + defer reg.Close() + repoName := fmt.Sprintf("%v/dockercli/busybox", reg.URL) // tag the image to upload it tot he private registry - repoName := fmt.Sprintf("%v/busybox", privateRegistryURL) tagCmd := exec.Command(dockerBinary, "tag", "busybox", repoName) if out, _, err := runCommandWithOutput(tagCmd); err != nil { t.Fatalf("image tagging failed: %s, %v", out, err) } - + defer deleteImages(repoName) pushCmd := exec.Command(dockerBinary, "push", repoName) if out, _, err := runCommandWithOutput(pushCmd); err != nil { t.Fatalf("pushing the image to the private registry has failed: %s, %v", out, err) } - - deleteImages(repoName) - logDone("push - push busybox to private registry") } // pushing an image without a prefix should throw an error func TestPushUnprefixedRepo(t *testing.T) { - // skip this test until we're able to use a registry - t.Skip() pushCmd := exec.Command(dockerBinary, "push", "busybox") if out, _, err := runCommandWithOutput(pushCmd); err == nil { t.Fatalf("pushing an unprefixed repo didn't result in a non-zero exit status: %s", out) From dbec2317e503b8a0190102332168f9d0256392b7 Mon Sep 17 00:00:00 2001 From: Arnaud Porterie Date: Tue, 13 Jan 2015 10:46:32 -0800 Subject: [PATCH 23/29] Add some push test coverage Signed-off-by: Arnaud Porterie --- integration-cli/docker_cli_push_test.go | 63 +++++++++++++++++++++---- integration-cli/docker_utils.go | 8 ++++ integration-cli/registry.go | 6 +-- 3 files changed, 63 insertions(+), 14 deletions(-) diff --git a/integration-cli/docker_cli_push_test.go b/integration-cli/docker_cli_push_test.go index a8c2ccdbc3..484e5db70b 100644 --- a/integration-cli/docker_cli_push_test.go +++ b/integration-cli/docker_cli_push_test.go @@ -3,30 +3,28 @@ package main import ( "fmt" "os/exec" + "strings" "testing" + "time" ) -// these tests need a freshly started empty private docker registry - // pulling an image from the central registry should work func TestPushBusyboxImage(t *testing.T) { - reg, err := newTestRegistryV2(t) - if err != nil { - t.Fatal(err) - } - defer reg.Close() - repoName := fmt.Sprintf("%v/dockercli/busybox", reg.URL) + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) // tag the image to upload it tot he private registry tagCmd := exec.Command(dockerBinary, "tag", "busybox", repoName) if out, _, err := runCommandWithOutput(tagCmd); err != nil { t.Fatalf("image tagging failed: %s, %v", out, err) } defer deleteImages(repoName) + pushCmd := exec.Command(dockerBinary, "push", repoName) if out, _, err := runCommandWithOutput(pushCmd); err != nil { t.Fatalf("pushing the image to the private registry has failed: %s, %v", out, err) } - logDone("push - push busybox to private registry") + logDone("push - busybox to private registry") } // pushing an image without a prefix should throw an error @@ -35,5 +33,50 @@ func TestPushUnprefixedRepo(t *testing.T) { if out, _, err := runCommandWithOutput(pushCmd); err == nil { t.Fatalf("pushing an unprefixed repo didn't result in a non-zero exit status: %s", out) } - logDone("push - push unprefixed busybox repo --> must fail") + logDone("push - unprefixed busybox repo must fail") +} + +func TestPushUntagged(t *testing.T) { + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + + expected := "does not exist" + pushCmd := exec.Command(dockerBinary, "push", repoName) + if out, _, err := runCommandWithOutput(pushCmd); err == nil { + t.Fatalf("pushing the image to the private registry should have failed: outuput %q", out) + } else if !strings.Contains(out, expected) { + t.Fatalf("pushing the image failed with an unexpected message: expected %q, got %q", expected, out) + } + logDone("push - untagged image") +} + +func TestPushInterrupt(t *testing.T) { + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + // tag the image to upload it tot he private registry + tagCmd := exec.Command(dockerBinary, "tag", "busybox", repoName) + if out, _, err := runCommandWithOutput(tagCmd); err != nil { + t.Fatalf("image tagging failed: %s, %v", out, err) + } + defer deleteImages(repoName) + + pushCmd := exec.Command(dockerBinary, "push", repoName) + if err := pushCmd.Start(); err != nil { + t.Fatalf("Failed to start pushing to private registry: %v", err) + } + + // Interrupt push (yes, we have no idea at what point it will get killed). + time.Sleep(200 * time.Millisecond) + if err := pushCmd.Process.Kill(); err != nil { + t.Fatalf("Failed to kill push process: %v", err) + } + // Try agin + pushCmd = exec.Command(dockerBinary, "push", repoName) + if err := pushCmd.Start(); err != nil { + t.Fatalf("Failed to start pushing to private registry: %v", err) + } + + logDone("push - interrupted") } diff --git a/integration-cli/docker_utils.go b/integration-cli/docker_utils.go index c58bcfbf75..3af6d9a60e 100644 --- a/integration-cli/docker_utils.go +++ b/integration-cli/docker_utils.go @@ -864,3 +864,11 @@ func readContainerFile(containerId, filename string) ([]byte, error) { return content, nil } + +func setupRegistry(t *testing.T) func() { + reg, err := newTestRegistryV2(t) + if err != nil { + t.Fatal(err) + } + return func() { reg.Close() } +} diff --git a/integration-cli/registry.go b/integration-cli/registry.go index f0ef05cca1..00ba3030a9 100644 --- a/integration-cli/registry.go +++ b/integration-cli/registry.go @@ -12,7 +12,6 @@ import ( const v2binary = "registry-v2" type testRegistryV2 struct { - URL string cmd *exec.Cmd dir string } @@ -24,7 +23,7 @@ storage: filesystem: rootdirectory: %s http: - addr: :%s` + addr: %s` tmp, err := ioutil.TempDir("", "registry-test-") if err != nil { return nil, err @@ -34,7 +33,7 @@ http: if err != nil { return nil, err } - if _, err := fmt.Fprintf(config, template, tmp, "5000"); err != nil { + if _, err := fmt.Fprintf(config, template, tmp, privateRegistryURL); err != nil { os.RemoveAll(tmp) return nil, err } @@ -50,7 +49,6 @@ http: return &testRegistryV2{ cmd: cmd, dir: tmp, - URL: "localhost:5000", }, nil } From 92d5eafe03eca8ca931ddca5ef7d6e41ca25caad Mon Sep 17 00:00:00 2001 From: Arnaud Porterie Date: Tue, 13 Jan 2015 15:19:44 -0800 Subject: [PATCH 24/29] Test pulling image with aliases Signed-off-by: Arnaud Porterie --- integration-cli/docker_cli_pull_test.go | 46 ++++++++++++++++++++++++- 1 file changed, 45 insertions(+), 1 deletion(-) diff --git a/integration-cli/docker_cli_pull_test.go b/integration-cli/docker_cli_pull_test.go index bed015be0e..e76f4ee950 100644 --- a/integration-cli/docker_cli_pull_test.go +++ b/integration-cli/docker_cli_pull_test.go @@ -1,12 +1,56 @@ package main import ( + "fmt" "os/exec" "strings" "testing" ) -// FIXME: we need a test for pulling all aliases for an image (issue #8141) +// See issue docker/docker#8141 +func TestPullImageWithAliases(t *testing.T) { + defer setupRegistry(t)() + + repoName := fmt.Sprintf("%v/dockercli/busybox", privateRegistryURL) + defer deleteImages(repoName) + + repos := []string{} + for _, tag := range []string{"recent", "fresh"} { + repos = append(repos, fmt.Sprintf("%v:%v", repoName, tag)) + } + + // Tag and push the same image multiple times. + for _, repo := range repos { + if out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "tag", "busybox", repo)); err != nil { + t.Fatalf("Failed to tag image %v: error %v, output %q", repos, err, out) + } + if out, err := exec.Command(dockerBinary, "push", repo).CombinedOutput(); err != nil { + t.Fatalf("Failed to push image %v: error %v, output %q", err, string(out)) + } + } + + // Clear local images store. + args := append([]string{"rmi"}, repos...) + if out, err := exec.Command(dockerBinary, args...).CombinedOutput(); err != nil { + t.Fatalf("Failed to clean images: error %v, output %q", err, string(out)) + } + + // Pull a single tag and verify it doesn't bring down all aliases. + pullCmd := exec.Command(dockerBinary, "pull", repos[0]) + if out, _, err := runCommandWithOutput(pullCmd); err != nil { + t.Fatalf("Failed to pull %v: error %v, output %q", repoName, err, out) + } + if err := exec.Command(dockerBinary, "inspect", repos[0]).Run(); err != nil { + t.Fatalf("Image %v was not pulled down", repos[0]) + } + for _, repo := range repos[1:] { + if err := exec.Command(dockerBinary, "inspect", repo).Run(); err == nil { + t.Fatalf("Image %v shouldn't have been pulled down", repo) + } + } + + logDone("pull - image with aliases") +} // pulling an image from the central registry should work func TestPullImageFromCentralRegistry(t *testing.T) { From 750b41ced42bda0ccda405c1aa7c43ded5821e40 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Tue, 13 Jan 2015 15:48:49 -0800 Subject: [PATCH 25/29] Refactor push and pull to move code out of cmd function Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/pull.go | 26 +++--- graph/push.go | 249 +++++++++++++++++++++++++------------------------- 2 files changed, 139 insertions(+), 136 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index 1c4bb9d88c..c70b220cc9 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -82,20 +82,14 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { log.Errorf("error updating trust base graph: %s", err) } - auth, err := r.GetV2Authorization(repoInfo.RemoteName, true) - if err != nil { - log.Errorf("error getting authorization: %s", err) - } else { - - log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) - if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel"), auth); err == nil { - if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { - log.Errorf("Error logging event 'pull' for %s: %s", logName, err) - } - return engine.StatusOK - } else if err != registry.ErrDoesNotExist { - log.Errorf("Error from V2 registry: %s", err) + log.Debugf("pulling v2 repository with local name %q", repoInfo.LocalName) + if err := s.pullV2Repository(job.Eng, r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err == nil { + if err = job.Eng.Job("log", "pull", logName, "").Run(); err != nil { + log.Errorf("Error logging event 'pull' for %s: %s", logName, err) } + return engine.StatusOK + } else if err != registry.ErrDoesNotExist { + log.Errorf("Error from V2 registry: %s", err) } log.Debug("image does not exist on v2 registry, falling back to v1") @@ -384,7 +378,11 @@ type downloadInfo struct { err chan error } -func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) error { +func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) error { + auth, err := r.GetV2Authorization(repoInfo.RemoteName, true) + if err != nil { + return fmt.Errorf("error getting authorization: %s", err) + } var layersDownloaded bool if tag == "" { log.Debugf("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName) diff --git a/graph/push.go b/graph/push.go index 4d6b1e0838..5b5011243b 100644 --- a/graph/push.go +++ b/graph/push.go @@ -252,6 +252,105 @@ func (s *TagStore) pushImage(r *registry.Session, out io.Writer, imgID, ep strin return imgData.Checksum, nil } +func (s *TagStore) pushV2Repository(r *registry.Session, eng *engine.Engine, out io.Writer, repoInfo *registry.RepositoryInfo, manifestBytes, tag string, sf *utils.StreamFormatter) error { + if repoInfo.Official { + j := eng.Job("trust_update_base") + if err := j.Run(); err != nil { + log.Errorf("error updating trust base graph: %s", err) + } + } + + auth, err := r.GetV2Authorization(repoInfo.RemoteName, false) + if err != nil { + return fmt.Errorf("error getting authorization: %s", err) + } + + // if no manifest is given, generate and sign with the key associated with the local tag store + if len(manifestBytes) == 0 { + mBytes, err := s.newManifest(repoInfo.LocalName, repoInfo.RemoteName, tag) + if err != nil { + return err + } + js, err := libtrust.NewJSONSignature(mBytes) + if err != nil { + return err + } + + if err = js.Sign(s.trustKey); err != nil { + return err + } + + signedBody, err := js.PrettySignature("signatures") + if err != nil { + return err + } + log.Infof("Signed manifest using daemon's key: %s", s.trustKey.KeyID()) + + manifestBytes = string(signedBody) + } + + manifest, verified, err := s.verifyManifest(eng, []byte(manifestBytes)) + if err != nil { + return fmt.Errorf("error verifying manifest: %s", err) + } + + if err := checkValidManifest(manifest); err != nil { + return fmt.Errorf("invalid manifest: %s", err) + } + + if !verified { + log.Debugf("Pushing unverified image") + } + + for i := len(manifest.FSLayers) - 1; i >= 0; i-- { + var ( + sumStr = manifest.FSLayers[i].BlobSum + imgJSON = []byte(manifest.History[i].V1Compatibility) + ) + + sumParts := strings.SplitN(sumStr, ":", 2) + if len(sumParts) < 2 { + return fmt.Errorf("Invalid checksum: %s", sumStr) + } + manifestSum := sumParts[1] + + img, err := image.NewImgJSON(imgJSON) + if err != nil { + return fmt.Errorf("Failed to parse json: %s", err) + } + + img, err = s.graph.Get(img.ID) + if err != nil { + return err + } + + arch, err := img.TarLayer() + if err != nil { + return fmt.Errorf("Could not get tar layer: %s", err) + } + + // Call mount blob + exists, err := r.PostV2ImageMountBlob(repoInfo.RemoteName, sumParts[0], manifestSum, auth) + if err != nil { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) + return err + } + if !exists { + err = r.PutV2ImageBlob(repoInfo.RemoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), out, sf, false, utils.TruncateID(img.ID), "Pushing"), auth) + if err != nil { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) + return err + } + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) + } else { + out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image already exists", nil)) + } + } + + // push the manifest + return r.PutV2ImageManifest(repoInfo.RemoteName, tag, bytes.NewReader([]byte(manifestBytes)), auth) +} + // FIXME: Allow to interrupt current push when new push of same image is done. func (s *TagStore) CmdPush(job *engine.Job) engine.Status { if n := len(job.Args); n != 1 { @@ -296,129 +395,35 @@ func (s *TagStore) CmdPush(job *engine.Job) engine.Status { } if repoInfo.Index.Official || endpoint.Version == registry.APIVersion2 { - if repoInfo.Official { - j := job.Eng.Job("trust_update_base") - if err = j.Run(); err != nil { - return job.Errorf("error updating trust base graph: %s", err) - } + err := s.pushV2Repository(r, job.Eng, job.Stdout, repoInfo, manifestBytes, tag, sf) + if err == nil { + return engine.StatusOK } - auth, err := r.GetV2Authorization(repoInfo.RemoteName, false) - if err != nil { - return job.Errorf("error getting authorization: %s", err) - } - - if len(manifestBytes) == 0 { - mBytes, err := s.newManifest(repoInfo.LocalName, repoInfo.RemoteName, tag) - if err != nil { - return job.Error(err) - } - js, err := libtrust.NewJSONSignature(mBytes) - if err != nil { - return job.Error(err) - } - - if err = js.Sign(s.trustKey); err != nil { - return job.Error(err) - } - - signedBody, err := js.PrettySignature("signatures") - if err != nil { - return job.Error(err) - } - log.Infof("Signed manifest using daemon's key: %s", s.trustKey.KeyID()) - - manifestBytes = string(signedBody) - } - - manifest, verified, err := s.verifyManifest(job.Eng, []byte(manifestBytes)) - if err != nil { - return job.Errorf("error verifying manifest: %s", err) - } - - if err := checkValidManifest(manifest); err != nil { - return job.Errorf("invalid manifest: %s", err) - } - - if !verified { - log.Debugf("Pushing unverified image") - } - - for i := len(manifest.FSLayers) - 1; i >= 0; i-- { - var ( - sumStr = manifest.FSLayers[i].BlobSum - imgJSON = []byte(manifest.History[i].V1Compatibility) - ) - - sumParts := strings.SplitN(sumStr, ":", 2) - if len(sumParts) < 2 { - return job.Errorf("Invalid checksum: %s", sumStr) - } - manifestSum := sumParts[1] - - img, err := image.NewImgJSON(imgJSON) - if err != nil { - return job.Errorf("Failed to parse json: %s", err) - } - - img, err = s.graph.Get(img.ID) - if err != nil { - return job.Error(err) - } - - arch, err := img.TarLayer() - if err != nil { - return job.Errorf("Could not get tar layer: %s", err) - } - - // Call mount blob - exists, err := r.PostV2ImageMountBlob(repoInfo.RemoteName, sumParts[0], manifestSum, auth) - if err != nil { - job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) - return job.Error(err) - } - if !exists { - err = r.PutV2ImageBlob(repoInfo.RemoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), job.Stdout, sf, false, utils.TruncateID(img.ID), "Pushing"), auth) - if err != nil { - job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) - return job.Error(err) - } - job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image successfully pushed", nil)) - } else { - job.Stdout.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image already exists", nil)) - } - } - - // push the manifest - err = r.PutV2ImageManifest(repoInfo.RemoteName, tag, bytes.NewReader([]byte(manifestBytes)), auth) - if err != nil { - return job.Error(err) - } - - // done, no fallback to V1 - return engine.StatusOK - } else { - if err != nil { - reposLen := 1 - if tag == "" { - reposLen = len(s.Repositories[repoInfo.LocalName]) - } - job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) - // If it fails, try to get the repository - if localRepo, exists := s.Repositories[repoInfo.LocalName]; exists { - if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { - return job.Error(err) - } - return engine.StatusOK - } - return job.Error(err) - } - - var token []string - job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", repoInfo.CanonicalName)) - if _, err := s.pushImage(r, job.Stdout, img.ID, endpoint.String(), token, sf); err != nil { - return job.Error(err) - } - return engine.StatusOK + // error out, no fallback to V1 + return job.Error(err) } + + if err != nil { + reposLen := 1 + if tag == "" { + reposLen = len(s.Repositories[repoInfo.LocalName]) + } + job.Stdout.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", repoInfo.CanonicalName, reposLen)) + // If it fails, try to get the repository + if localRepo, exists := s.Repositories[repoInfo.LocalName]; exists { + if err := s.pushRepository(r, job.Stdout, repoInfo, localRepo, tag, sf); err != nil { + return job.Error(err) + } + return engine.StatusOK + } + return job.Error(err) + } + + var token []string + job.Stdout.Write(sf.FormatStatus("", "The push refers to an image: [%s]", repoInfo.CanonicalName)) + if _, err := s.pushImage(r, job.Stdout, img.ID, endpoint.String(), token, sf); err != nil { + return job.Error(err) + } + return engine.StatusOK } From 9c6f8e14398e794cbe20504556c22a1c83260bd8 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 14 Jan 2015 16:46:31 -0800 Subject: [PATCH 26/29] Cleanup v2 session to require endpoint Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/pull.go | 18 ++++++---- graph/push.go | 12 ++++--- registry/session_v2.go | 76 +++++++++++++++++++++++------------------- 3 files changed, 61 insertions(+), 45 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index c70b220cc9..d0fca38b3c 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -379,26 +379,30 @@ type downloadInfo struct { } func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool) error { - auth, err := r.GetV2Authorization(repoInfo.RemoteName, true) + endpoint, err := r.V2RegistryEndpoint(repoInfo.Index) + if err != nil { + return fmt.Errorf("error getting registry endpoint: %s", err) + } + auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, true) if err != nil { return fmt.Errorf("error getting authorization: %s", err) } var layersDownloaded bool if tag == "" { log.Debugf("Pulling tag list from V2 registry for %s", repoInfo.CanonicalName) - tags, err := r.GetV2RemoteTags(repoInfo.RemoteName, auth) + tags, err := r.GetV2RemoteTags(endpoint, repoInfo.RemoteName, auth) if err != nil { return err } for _, t := range tags { - if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, t, sf, parallel, auth); err != nil { + if downloaded, err := s.pullV2Tag(eng, r, out, endpoint, repoInfo, t, sf, parallel, auth); err != nil { return err } else if downloaded { layersDownloaded = true } } } else { - if downloaded, err := s.pullV2Tag(eng, r, out, repoInfo, tag, sf, parallel, auth); err != nil { + if downloaded, err := s.pullV2Tag(eng, r, out, endpoint, repoInfo, tag, sf, parallel, auth); err != nil { return err } else if downloaded { layersDownloaded = true @@ -413,9 +417,9 @@ func (s *TagStore) pullV2Repository(eng *engine.Engine, r *registry.Session, out return nil } -func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) (bool, error) { +func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Writer, endpoint *registry.Endpoint, repoInfo *registry.RepositoryInfo, tag string, sf *utils.StreamFormatter, parallel bool, auth *registry.RequestAuthorization) (bool, error) { log.Debugf("Pulling tag from V2 registry: %q", tag) - manifestBytes, err := r.GetV2ImageManifest(repoInfo.RemoteName, tag, auth) + manifestBytes, err := r.GetV2ImageManifest(endpoint, repoInfo.RemoteName, tag, auth) if err != nil { return false, err } @@ -479,7 +483,7 @@ func (s *TagStore) pullV2Tag(eng *engine.Engine, r *registry.Session, out io.Wri return err } - r, l, err := r.GetV2ImageBlobReader(repoInfo.RemoteName, sumType, checksum, auth) + r, l, err := r.GetV2ImageBlobReader(endpoint, repoInfo.RemoteName, sumType, checksum, auth) if err != nil { return err } diff --git a/graph/push.go b/graph/push.go index 5b5011243b..46469daead 100644 --- a/graph/push.go +++ b/graph/push.go @@ -260,7 +260,11 @@ func (s *TagStore) pushV2Repository(r *registry.Session, eng *engine.Engine, out } } - auth, err := r.GetV2Authorization(repoInfo.RemoteName, false) + endpoint, err := r.V2RegistryEndpoint(repoInfo.Index) + if err != nil { + return fmt.Errorf("error getting registry endpoint: %s", err) + } + auth, err := r.GetV2Authorization(endpoint, repoInfo.RemoteName, false) if err != nil { return fmt.Errorf("error getting authorization: %s", err) } @@ -330,13 +334,13 @@ func (s *TagStore) pushV2Repository(r *registry.Session, eng *engine.Engine, out } // Call mount blob - exists, err := r.PostV2ImageMountBlob(repoInfo.RemoteName, sumParts[0], manifestSum, auth) + exists, err := r.HeadV2ImageBlob(endpoint, repoInfo.RemoteName, sumParts[0], manifestSum, auth) if err != nil { out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) return err } if !exists { - err = r.PutV2ImageBlob(repoInfo.RemoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), out, sf, false, utils.TruncateID(img.ID), "Pushing"), auth) + err = r.PutV2ImageBlob(endpoint, repoInfo.RemoteName, sumParts[0], manifestSum, utils.ProgressReader(arch, int(img.Size), out, sf, false, utils.TruncateID(img.ID), "Pushing"), auth) if err != nil { out.Write(sf.FormatProgress(utils.TruncateID(img.ID), "Image push failed", nil)) return err @@ -348,7 +352,7 @@ func (s *TagStore) pushV2Repository(r *registry.Session, eng *engine.Engine, out } // push the manifest - return r.PutV2ImageManifest(repoInfo.RemoteName, tag, bytes.NewReader([]byte(manifestBytes)), auth) + return r.PutV2ImageManifest(endpoint, repoInfo.RemoteName, tag, bytes.NewReader([]byte(manifestBytes)), auth) } // FIXME: Allow to interrupt current push when new push of same image is done. diff --git a/registry/session_v2.go b/registry/session_v2.go index b08f4cf0d8..11b96bd65a 100644 --- a/registry/session_v2.go +++ b/registry/session_v2.go @@ -19,33 +19,41 @@ func getV2Builder(e *Endpoint) *v2.URLBuilder { return e.URLBuilder } +func (r *Session) V2RegistryEndpoint(index *IndexInfo) (ep *Endpoint, err error) { + // TODO check if should use Mirror + if index.Official { + ep, err = newEndpoint(REGISTRYSERVER, true) + if err != nil { + return + } + err = validateEndpoint(ep) + if err != nil { + return + } + } else if r.indexEndpoint.String() == index.GetAuthConfigKey() { + ep = r.indexEndpoint + } else { + ep, err = NewEndpoint(index) + if err != nil { + return + } + } + + ep.URLBuilder = v2.NewURLBuilder(ep.URL) + return +} + // GetV2Authorization gets the authorization needed to the given image // If readonly access is requested, then only the authorization may // only be used for Get operations. -func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *RequestAuthorization, err error) { +func (r *Session) GetV2Authorization(ep *Endpoint, imageName string, readOnly bool) (auth *RequestAuthorization, err error) { scopes := []string{"pull"} if !readOnly { scopes = append(scopes, "push") } - var registry *Endpoint - if r.indexEndpoint.String() == IndexServerAddress() { - registry, err = newEndpoint(REGISTRYSERVER, true) - if err != nil { - return - } - err = validateEndpoint(registry) - if err != nil { - return - } - } else { - registry = r.indexEndpoint - } - registry.URLBuilder = v2.NewURLBuilder(registry.URL) - r.indexEndpoint = registry - log.Debugf("Getting authorization for %s %s", imageName, scopes) - return NewRequestAuthorization(r.GetAuthConfig(true), registry, "repository", imageName, scopes), nil + return NewRequestAuthorization(r.GetAuthConfig(true), ep, "repository", imageName, scopes), nil } // @@ -55,8 +63,8 @@ func (r *Session) GetV2Authorization(imageName string, readOnly bool) (auth *Req // 1.c) if anything else, err // 2) PUT the created/signed manifest // -func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) +func (r *Session) GetV2ImageManifest(ep *Endpoint, imageName, tagName string, auth *RequestAuthorization) ([]byte, error) { + routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { return nil, err } @@ -92,11 +100,11 @@ func (r *Session) GetV2ImageManifest(imageName, tagName string, auth *RequestAut return buf, nil } -// - Succeeded to mount for this image scope -// - Failed with no error (So continue to Push the Blob) +// - Succeeded to head image blob (already exists) +// - Failed with no error (continue to Push the Blob) // - Failed with error -func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) HeadV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (bool, error) { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return false, err } @@ -127,8 +135,8 @@ func (r *Session) PostV2ImageMountBlob(imageName, sumType, sum string, auth *Req return false, fmt.Errorf("Failed to mount %q - %s:%s : %d", imageName, sumType, sum, res.StatusCode) } -func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) GetV2ImageBlob(ep *Endpoint, imageName, sumType, sum string, blobWrtr io.Writer, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return err } @@ -158,8 +166,8 @@ func (r *Session) GetV2ImageBlob(imageName, sumType, sum string, blobWrtr io.Wri return err } -func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobURL(imageName, sumType+":"+sum) +func (r *Session) GetV2ImageBlobReader(ep *Endpoint, imageName, sumType, sum string, auth *RequestAuthorization) (io.ReadCloser, int64, error) { + routeURL, err := getV2Builder(ep).BuildBlobURL(imageName, sumType+":"+sum) if err != nil { return nil, 0, err } @@ -195,8 +203,8 @@ func (r *Session) GetV2ImageBlobReader(imageName, sumType, sum string, auth *Req // Push the image to the server for storage. // 'layer' is an uncompressed reader of the blob to be pushed. // The server will generate it's own checksum calculation. -func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(r.indexEndpoint).BuildBlobUploadURL(imageName) +func (r *Session) PutV2ImageBlob(ep *Endpoint, imageName, sumType, sumStr string, blobRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildBlobUploadURL(imageName) if err != nil { return err } @@ -245,8 +253,8 @@ func (r *Session) PutV2ImageBlob(imageName, sumType, sumStr string, blobRdr io.R } // Finally Push the (signed) manifest of the blobs we've just pushed -func (r *Session) PutV2ImageManifest(imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { - routeURL, err := getV2Builder(r.indexEndpoint).BuildManifestURL(imageName, tagName) +func (r *Session) PutV2ImageManifest(ep *Endpoint, imageName, tagName string, manifestRdr io.Reader, auth *RequestAuthorization) error { + routeURL, err := getV2Builder(ep).BuildManifestURL(imageName, tagName) if err != nil { return err } @@ -283,8 +291,8 @@ type remoteTags struct { } // Given a repository name, returns a json array of string tags -func (r *Session) GetV2RemoteTags(imageName string, auth *RequestAuthorization) ([]string, error) { - routeURL, err := getV2Builder(r.indexEndpoint).BuildTagsURL(imageName) +func (r *Session) GetV2RemoteTags(ep *Endpoint, imageName string, auth *RequestAuthorization) ([]string, error) { + routeURL, err := getV2Builder(ep).BuildTagsURL(imageName) if err != nil { return nil, err } From f11f3f6203da596f50eec0edc3c5dfb8c93bc271 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Wed, 14 Jan 2015 17:14:14 -0800 Subject: [PATCH 27/29] Remove session backup The v2 session code will no longer update the indexEndpoint value, therefore it is not necessary to save and restore the value for use with v1. Signed-off-by: Derek McGowan (github: dmcgowan) --- graph/pull.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/graph/pull.go b/graph/pull.go index d0fca38b3c..6129ea39a1 100644 --- a/graph/pull.go +++ b/graph/pull.go @@ -72,10 +72,6 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { logName += ":" + tag } - // Calling the v2 code path might change the session - // endpoint value, so save the original one! - originalSession := *r - if len(repoInfo.Index.Mirrors) == 0 && (repoInfo.Index.Official || endpoint.Version == registry.APIVersion2) { j := job.Eng.Job("trust_update_base") if err = j.Run(); err != nil { @@ -95,8 +91,6 @@ func (s *TagStore) CmdPull(job *engine.Job) engine.Status { log.Debug("image does not exist on v2 registry, falling back to v1") } - r = &originalSession - log.Debugf("pulling v1 repository with local name %q", repoInfo.LocalName) if err = s.pullRepository(r, job.Stdout, repoInfo, tag, sf, job.GetenvBool("parallel")); err != nil { return job.Error(err) From dd914f91d779f64e20ce86767ab4f84f40b9ef6a Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Thu, 15 Jan 2015 13:06:52 -0800 Subject: [PATCH 28/29] Add token cache Token cache prevents the need to get a new token for every registry interaction. Since the tokens are short lived, the cache expires after only a minute. Signed-off-by: Derek McGowan (github: dmcgowan) --- registry/auth.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/registry/auth.go b/registry/auth.go index 1e1c7ddb82..1ce99805f2 100644 --- a/registry/auth.go +++ b/registry/auth.go @@ -10,6 +10,8 @@ import ( "os" "path" "strings" + "sync" + "time" log "github.com/Sirupsen/logrus" "github.com/docker/docker/utils" @@ -43,6 +45,10 @@ type RequestAuthorization struct { resource string scope string actions []string + + tokenLock sync.Mutex + tokenCache string + tokenExpiration time.Time } func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, resource, scope string, actions []string) *RequestAuthorization { @@ -56,7 +62,14 @@ func NewRequestAuthorization(authConfig *AuthConfig, registryEndpoint *Endpoint, } func (auth *RequestAuthorization) getToken() (string, error) { - // TODO check if already has token and before expiration + auth.tokenLock.Lock() + defer auth.tokenLock.Unlock() + now := time.Now() + if now.Before(auth.tokenExpiration) { + log.Debugf("Using cached token for %s", auth.authConfig.Username) + return auth.tokenCache, nil + } + client := &http.Client{ Transport: &http.Transport{ DisableKeepAlives: true, @@ -80,14 +93,18 @@ func (auth *RequestAuthorization) getToken() (string, error) { if err != nil { return "", err } - // TODO cache token and set expiration to one minute from now + auth.tokenCache = token + auth.tokenExpiration = now.Add(time.Minute) return token, nil default: log.Infof("Unsupported auth scheme: %q", challenge.Scheme) } } - // TODO no expiration, do not reattempt to get a token + + // Do not expire cache since there are no challenges which use a token + auth.tokenExpiration = time.Now().Add(time.Hour * 24) + return "", nil } From f29aacbc4804e3aca1c21b9411e960b2a2543da1 Mon Sep 17 00:00:00 2001 From: Derek McGowan Date: Fri, 16 Jan 2015 11:34:45 -0800 Subject: [PATCH 29/29] Fix failing integration tests Signed-off-by: Derek McGowan (github: dmcgowan) --- integration-cli/docker_cli_pull_test.go | 1 + 1 file changed, 1 insertion(+) diff --git a/integration-cli/docker_cli_pull_test.go b/integration-cli/docker_cli_pull_test.go index e76f4ee950..7649688583 100644 --- a/integration-cli/docker_cli_pull_test.go +++ b/integration-cli/docker_cli_pull_test.go @@ -40,6 +40,7 @@ func TestPullImageWithAliases(t *testing.T) { if out, _, err := runCommandWithOutput(pullCmd); err != nil { t.Fatalf("Failed to pull %v: error %v, output %q", repoName, err, out) } + defer deleteImages(repos[0]) if err := exec.Command(dockerBinary, "inspect", repos[0]).Run(); err != nil { t.Fatalf("Image %v was not pulled down", repos[0]) }