From 8d51393181adca0c74a4b08cfb2dc2da390f983b Mon Sep 17 00:00:00 2001 From: Matej Vasek Date: Mon, 20 Dec 2021 23:28:17 +0100 Subject: [PATCH] feat: allow push to cluster internal registries (#718) * feat: allow push to cluster internal registries Signed-off-by: Matej Vasek * fix: NewRoundTripper consults http.DefaultTransport Signed-off-by: Matej Vasek * src: move credential code to sub-package Signed-off-by: Matej Vasek * src: refactor Signed-off-by: Matej Vasek * src: share RoundTripper avoid creating expensive RoundTripper twice Signed-off-by: Matej Vasek * test: added test for pusher Signed-off-by: Matej Vasek * src: disable parallel layer upload it's more reliable Signed-off-by: Matej Vasek * fixup: lint Signed-off-by: Matej Vasek * fixup: lint Signed-off-by: Matej Vasek * fixup: doc, rm commented code Signed-off-by: Matej Vasek --- cmd/build.go | 20 +- cmd/deploy.go | 22 +- docker/credentials_helper.go | 193 --- docker/credentials_helper_test.go | 38 - docker/creds/credentials.go | 451 +++++++ docker/creds/credentials_test.go | 870 +++++++++++++ docker/pusher.go | 390 ++---- docker/pusher_test.go | 1085 ++++++----------- docker/testData/image.tar | Bin 0 -> 53248 bytes http/transport.go | 40 +- .../internal/httptest/httptest.go | 104 ++ .../pkg/registry/blobs.go | 224 ++++ .../pkg/registry/error.go | 46 + .../pkg/registry/manifest.go | 334 +++++ .../pkg/registry/registry.go | 104 ++ .../go-containerregistry/pkg/registry/tls.go | 29 + .../pkg/v1/daemon/README.md | 11 + .../go-containerregistry/pkg/v1/daemon/doc.go | 17 + .../pkg/v1/daemon/image.go | 89 ++ .../pkg/v1/daemon/options.go | 102 ++ .../pkg/v1/daemon/write.go | 61 + vendor/modules.txt | 3 + 22 files changed, 2974 insertions(+), 1259 deletions(-) delete mode 100644 docker/credentials_helper.go delete mode 100644 docker/credentials_helper_test.go create mode 100644 docker/creds/credentials.go create mode 100644 docker/creds/credentials_test.go create mode 100644 docker/testData/image.tar create mode 100644 vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/error.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/registry.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/registry/tls.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/daemon/README.md create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/daemon/doc.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go create mode 100644 vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go diff --git a/cmd/build.go b/cmd/build.go index f4c3687b2..c61a7a755 100644 --- a/cmd/build.go +++ b/cmd/build.go @@ -3,6 +3,9 @@ package cmd import ( "errors" "fmt" + "net/http" + + fnhttp "knative.dev/kn-plugin-func/http" "github.com/AlecAivazis/survey/v2" "github.com/AlecAivazis/survey/v2/terminal" @@ -12,6 +15,7 @@ import ( fn "knative.dev/kn-plugin-func" "knative.dev/kn-plugin-func/buildpacks" "knative.dev/kn-plugin-func/docker" + "knative.dev/kn-plugin-func/docker/creds" "knative.dev/kn-plugin-func/progress" ) @@ -27,14 +31,14 @@ func newBuildClient(cfg buildConfig) (*fn.Client, error) { pusherOption := fn.WithPusher(nil) if cfg.Push { - credentialsProvider := docker.NewCredentialsProvider( - docker.WithPromptForCredentials(newPromptForCredentials()), - docker.WithPromptForCredentialStore(newPromptForCredentialStore()), - ) + credentialsProvider := creds.NewCredentialsProvider( + creds.WithPromptForCredentials(newPromptForCredentials()), + creds.WithPromptForCredentialStore(newPromptForCredentialStore()), + creds.WithTransport(cfg.Transport)) pusher, err := docker.NewPusher( docker.WithCredentialsProvider(credentialsProvider), docker.WithProgressListener(listener), - ) + docker.WithTransport(cfg.Transport)) if err != nil { return nil, err @@ -186,6 +190,10 @@ func runBuild(cmd *cobra.Command, _ []string, clientFn buildClientFn) (err error config.Registry = "" } + rt := fnhttp.NewRoundTripper() + defer rt.Close() + config.Transport = rt + client, err := clientFn(config) if err != nil { return err @@ -221,6 +229,8 @@ type buildConfig struct { // with interactive prompting (only applicable when attached to a TTY). Confirm bool Builder string + + Transport http.RoundTripper } func newBuildConfig() buildConfig { diff --git a/cmd/deploy.go b/cmd/deploy.go index b7cc84e06..6fbb71c2d 100644 --- a/cmd/deploy.go +++ b/cmd/deploy.go @@ -2,8 +2,11 @@ package cmd import ( "fmt" + "net/http" "os" + fnhttp "knative.dev/kn-plugin-func/http" + "github.com/AlecAivazis/survey/v2" "github.com/AlecAivazis/survey/v2/terminal" "github.com/ory/viper" @@ -13,6 +16,7 @@ import ( fn "knative.dev/kn-plugin-func" "knative.dev/kn-plugin-func/buildpacks" "knative.dev/kn-plugin-func/docker" + "knative.dev/kn-plugin-func/docker/creds" "knative.dev/kn-plugin-func/knative" "knative.dev/kn-plugin-func/progress" ) @@ -26,12 +30,14 @@ func newDeployClient(cfg deployConfig) (*fn.Client, error) { builder := buildpacks.NewBuilder() - credentialsProvider := docker.NewCredentialsProvider( - docker.WithPromptForCredentials(newPromptForCredentials()), - docker.WithPromptForCredentialStore(newPromptForCredentialStore())) + credentialsProvider := creds.NewCredentialsProvider( + creds.WithPromptForCredentials(newPromptForCredentials()), + creds.WithPromptForCredentialStore(newPromptForCredentialStore()), + creds.WithTransport(cfg.Transport)) pusher, err := docker.NewPusher( docker.WithCredentialsProvider(credentialsProvider), - docker.WithProgressListener(listener)) + docker.WithProgressListener(listener), + docker.WithTransport(cfg.Transport)) if err != nil { return nil, err } @@ -174,6 +180,10 @@ func runDeploy(cmd *cobra.Command, _ []string, clientFn deployClientFn) (err err config.Registry = "" } + rt := fnhttp.NewRoundTripper() + defer rt.Close() + config.Transport = rt + client, err := clientFn(config) if err != nil { if err == terminal.InterruptErr { @@ -228,7 +238,7 @@ func newPromptForCredentials() func(registry string) (docker.Credentials, error) } } -func newPromptForCredentialStore() docker.ChooseCredentialHelperCallback { +func newPromptForCredentialStore() creds.ChooseCredentialHelperCallback { return func(availableHelpers []string) (string, error) { if len(availableHelpers) < 1 { fmt.Fprintf(os.Stderr, `Credentials will not be saved. @@ -284,6 +294,8 @@ type deployConfig struct { // Envs passed via cmd to removed EnvToRemove []string + + Transport http.RoundTripper } // newDeployConfig creates a buildConfig populated from command flags and diff --git a/docker/credentials_helper.go b/docker/credentials_helper.go deleted file mode 100644 index 37944b04a..000000000 --- a/docker/credentials_helper.go +++ /dev/null @@ -1,193 +0,0 @@ -package docker - -import ( - "encoding/json" - "errors" - "fmt" - "io/ioutil" - "net" - "net/url" - "os" - "path/filepath" - "runtime" - "strings" - - "github.com/docker/docker-credential-helpers/client" - "github.com/docker/docker-credential-helpers/credentials" -) - -var errCredentialsNotFound = errors.New("credentials not found") -var errNoCredentialHelperConfigured = errors.New("no credential helper configure") - -func getCredentialHelperFromConfig(confFilePath string) (string, error) { - data, err := ioutil.ReadFile(confFilePath) - if err != nil { - return "", err - } - - conf := struct { - Store string `json:"credsStore"` - }{} - - err = json.Unmarshal(data, &conf) - if err != nil { - return "", err - } - - return conf.Store, nil -} - -func setCredentialHelperToConfig(confFilePath, helper string) error { - var err error - - configData := make(map[string]interface{}) - - if data, err := ioutil.ReadFile(confFilePath); err == nil { - err = json.Unmarshal(data, &configData) - if err != nil { - return err - } - } - - configData["credsStore"] = helper - - data, err := json.MarshalIndent(&configData, "", " ") - if err != nil { - return err - } - - err = ioutil.WriteFile(confFilePath, data, 0600) - if err != nil { - return err - } - - return nil -} - -func getCredentialsByCredentialHelper(confFilePath, registry string) (Credentials, error) { - result := Credentials{} - - helper, err := getCredentialHelperFromConfig(confFilePath) - if err != nil && !os.IsNotExist(err) { - return result, fmt.Errorf("failed to get helper from config: %w", err) - } - if helper == "" { - return result, errCredentialsNotFound - } - - helperName := fmt.Sprintf("docker-credential-%s", helper) - p := client.NewShellProgramFunc(helperName) - - credentialsMap, err := client.List(p) - if err != nil { - return result, fmt.Errorf("failed to list credentials: %w", err) - } - - for serverUrl := range credentialsMap { - if registryEquals(serverUrl, registry) { - creds, err := client.Get(p, serverUrl) - if err != nil { - return result, fmt.Errorf("failed to get credentials: %w", err) - } - result.Username = creds.Username - result.Password = creds.Secret - return result, nil - } - } - - return result, fmt.Errorf("failed to get credentials from helper specified in ~/.docker/config.json: %w", errCredentialsNotFound) -} - -func setCredentialsByCredentialHelper(confFilePath, registry, username, secret string) error { - helper, err := getCredentialHelperFromConfig(confFilePath) - - if helper == "" || os.IsNotExist(err) { - return errNoCredentialHelperConfigured - } - if err != nil { - return fmt.Errorf("failed to get helper from config: %w", err) - } - - helperName := fmt.Sprintf("docker-credential-%s", helper) - p := client.NewShellProgramFunc(helperName) - - return client.Store(p, &credentials.Credentials{ServerURL: registry, Username: username, Secret: secret}) -} - -func listCredentialHelpers() []string { - path := os.Getenv("PATH") - paths := strings.Split(path, string(os.PathListSeparator)) - - helpers := make(map[string]bool) - for _, p := range paths { - fss, err := ioutil.ReadDir(p) - if err != nil { - continue - } - for _, fi := range fss { - if fi.IsDir() { - continue - } - if !strings.HasPrefix(fi.Name(), "docker-credential-") { - continue - } - if runtime.GOOS == "windows" { - ext := filepath.Ext(fi.Name()) - if ext != "exe" && ext != "bat" { - continue - } - } - helpers[fi.Name()] = true - } - } - result := make([]string, 0, len(helpers)) - for h := range helpers { - result = append(result, h) - } - return result -} - -func hostPort(registry string) (host string, port string) { - host, port = registry, "" - if !strings.Contains(registry, "://") { - h, p, err := net.SplitHostPort(registry) - - if err == nil { - host, port = h, p - return - } - registry = "https://" + registry - } - - u, err := url.Parse(registry) - if err != nil { - panic(err) - } - host = u.Hostname() - port = u.Port() - return -} - -// checks whether registry matches in host and port -// with exception where empty port matches standard ports (80,443) -func registryEquals(regA, regB string) bool { - h1, p1 := hostPort(regA) - h2, p2 := hostPort(regB) - - isStdPort := func(p string) bool { return p == "443" || p == "80" } - - portEq := p1 == p2 || - (p1 == "" && isStdPort(p2)) || - (isStdPort(p1) && p2 == "") - - if h1 == h2 && portEq { - return true - } - - if strings.HasSuffix(h1, "docker.io") && - strings.HasSuffix(h2, "docker.io") { - return true - } - - return false -} diff --git a/docker/credentials_helper_test.go b/docker/credentials_helper_test.go deleted file mode 100644 index 0c536f39e..000000000 --- a/docker/credentials_helper_test.go +++ /dev/null @@ -1,38 +0,0 @@ -//go:build !integration -// +build !integration - -package docker - -import "testing" - -func Test_registryEquals(t *testing.T) { - tests := []struct { - name string - urlA string - urlB string - want bool - }{ - {"no port matching host", "quay.io", "quay.io", true}, - {"non-matching host added sub-domain", "sub.quay.io", "quay.io", false}, - {"non-matching host different sub-domain", "sub.quay.io", "sub3.quay.io", false}, - {"localhost", "localhost", "localhost", true}, - {"localhost with standard ports", "localhost:80", "localhost:443", false}, - {"localhost with matching port", "https://localhost:1234", "http://localhost:1234", true}, - {"localhost with match by default port 80", "http://localhost", "localhost:80", true}, - {"localhost with match by default port 443", "https://localhost", "localhost:443", true}, - {"localhost with mismatch by non-default port 5000", "https://localhost", "localhost:5000", false}, - {"localhost with match by empty ports", "https://localhost", "http://localhost", true}, - {"docker.io matching host https", "https://docker.io", "docker.io", true}, - {"docker.io matching host http", "http://docker.io", "docker.io", true}, - {"docker.io with path", "docker.io/v1/", "docker.io", true}, - {"docker.io with protocol and path", "https://docker.io/v1/", "docker.io", true}, - {"docker.io with subdomain index.", "https://index.docker.io/v1/", "docker.io", true}, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := registryEquals(tt.urlA, tt.urlB); got != tt.want { - t.Errorf("to2ndLevelDomain() = %v, want %v", got, tt.want) - } - }) - } -} diff --git a/docker/creds/credentials.go b/docker/creds/credentials.go new file mode 100644 index 000000000..d96e3537e --- /dev/null +++ b/docker/creds/credentials.go @@ -0,0 +1,451 @@ +package creds + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io/ioutil" + "net" + "net/http" + "net/url" + "os" + "path/filepath" + "runtime" + "strings" + + fn "knative.dev/kn-plugin-func" + "knative.dev/kn-plugin-func/docker" + + "github.com/containers/image/v5/pkg/docker/config" + containersTypes "github.com/containers/image/v5/types" + "github.com/docker/docker-credential-helpers/client" + "github.com/docker/docker-credential-helpers/credentials" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/v1/remote/transport" +) + +type CredentialsCallback func(registry string) (docker.Credentials, error) + +var ErrUnauthorized = errors.New("bad credentials") + +// VerifyCredentialsCallback checks if credentials are accepted by the registry. +// If credentials are incorrect this callback shall return ErrUnauthorized. +type VerifyCredentialsCallback func(ctx context.Context, registry string, credentials docker.Credentials) error + +// CheckAuth verifies that credentials are correct +func CheckAuth(ctx context.Context, registry string, credentials docker.Credentials, trans http.RoundTripper) error { + serverAddress := registry + if !strings.HasPrefix(serverAddress, "https://") && !strings.HasPrefix(serverAddress, "http://") { + serverAddress = "https://" + serverAddress + } + + url := fmt.Sprintf("%s/v2", serverAddress) + + authenticator := &authn.Basic{ + Username: credentials.Username, + Password: credentials.Password, + } + + reg, err := name.NewRegistry(registry) + if err != nil { + return err + } + + tr, err := transport.NewWithContext(ctx, reg, authenticator, trans, nil) + if err != nil { + return err + } + + cli := http.Client{Transport: tr} + + req, err := http.NewRequestWithContext(ctx, "GET", url, nil) + if err != nil { + return err + } + + resp, err := cli.Do(req) + if err != nil { + return fmt.Errorf("failed to verify credentials: %w", err) + } + defer resp.Body.Close() + + switch { + case resp.StatusCode == http.StatusUnauthorized: + return ErrUnauthorized + case resp.StatusCode != http.StatusOK: + return fmt.Errorf("failed to verify credentials: status code: %d", resp.StatusCode) + default: + return nil + } +} + +type ChooseCredentialHelperCallback func(available []string) (string, error) + +type credentialsProvider struct { + promptForCredentials CredentialsCallback + verifyCredentials VerifyCredentialsCallback + promptForCredentialStore ChooseCredentialHelperCallback + credentialLoaders []CredentialsCallback + authFilePath string + transport http.RoundTripper +} + +type Opt func(opts *credentialsProvider) + +// WithPromptForCredentials sets custom callback that is supposed to +// interactively ask for credentials in case the credentials cannot be found in configuration files. +// The callback may be called multiple times in case incorrect credentials were returned before. +func WithPromptForCredentials(cbk CredentialsCallback) Opt { + return func(opts *credentialsProvider) { + opts.promptForCredentials = cbk + } +} + +// WithVerifyCredentials sets custom callback for credentials validation. +func WithVerifyCredentials(cbk VerifyCredentialsCallback) Opt { + return func(opts *credentialsProvider) { + opts.verifyCredentials = cbk + } +} + +// WithPromptForCredentialStore sets custom callback that is supposed to +// interactively ask user which credentials store/helper is used to store credentials obtained +// from user. +func WithPromptForCredentialStore(cbk ChooseCredentialHelperCallback) Opt { + return func(opts *credentialsProvider) { + opts.promptForCredentialStore = cbk + } +} + +func WithTransport(transport http.RoundTripper) Opt { + return func(opts *credentialsProvider) { + opts.transport = transport + } +} + +// WithAdditionalCredentialLoaders adds custom callbacks for credential retrieval. +// The callbacks are supposed to be non-interactive as opposed to WithPromptForCredentials. +// +// This might be useful when credentials are shared with some other service. +// +// Example: OpenShift builtin registry shares credentials with the cluster (k8s) credentials. +func WithAdditionalCredentialLoaders(loaders ...CredentialsCallback) Opt { + return func(opts *credentialsProvider) { + opts.credentialLoaders = append(opts.credentialLoaders, loaders...) + } +} + +// NewCredentialsProvider returns new CredentialsProvider that tries to get credentials from docker/func config files. +// +// In case getting credentials from the config files fails +// the caller provided callback (see WithPromptForCredentials) will be invoked to obtain credentials. +// The callback may be called multiple times in case the returned credentials +// are not correct (see WithVerifyCredentials). +// +// When the callback succeeds the credentials will be saved by using helper defined in the func config. +// If the helper is not defined in the config file +// it may be picked by provided callback (see WithPromptForCredentialStore). +// The picked value will be saved in the func config. +// +// To verify that credentials are correct custom callback can be used (see WithVerifyCredentials). +func NewCredentialsProvider(opts ...Opt) docker.CredentialsProvider { + var c credentialsProvider + + for _, o := range opts { + o(&c) + } + + if c.transport == nil { + c.transport = http.DefaultTransport + } + + if c.verifyCredentials == nil { + c.verifyCredentials = func(ctx context.Context, registry string, credentials docker.Credentials) error { + return CheckAuth(ctx, registry, credentials, c.transport) + } + } + + if c.promptForCredentialStore == nil { + c.promptForCredentialStore = func(available []string) (string, error) { + return "", nil + } + } + + c.authFilePath = filepath.Join(fn.ConfigPath(), "auth.json") + sys := &containersTypes.SystemContext{ + AuthFilePath: c.authFilePath, + } + + home, err := os.UserHomeDir() + if err != nil { + panic(err) + } + dockerConfigPath := filepath.Join(home, ".docker", "config.json") + + var defaultCredentialLoaders = []CredentialsCallback{ + func(registry string) (docker.Credentials, error) { + creds, err := config.GetCredentials(sys, registry) + if err != nil { + return docker.Credentials{}, err + } + return docker.Credentials{ + Username: creds.Username, + Password: creds.Password, + }, nil + }, + func(registry string) (docker.Credentials, error) { + return getCredentialsByCredentialHelper(c.authFilePath, registry) + }, + func(registry string) (docker.Credentials, error) { + return getCredentialsByCredentialHelper(dockerConfigPath, registry) + }, + } + + c.credentialLoaders = append(c.credentialLoaders, defaultCredentialLoaders...) + + return c.getCredentials +} + +func (c *credentialsProvider) getCredentials(ctx context.Context, registry string) (docker.Credentials, error) { + var err error + result := docker.Credentials{} + + for _, load := range c.credentialLoaders { + + result, err = load(registry) + + if err != nil && !errors.Is(err, errCredentialsNotFound) { + return docker.Credentials{}, err + } + + if result != (docker.Credentials{}) { + err = c.verifyCredentials(ctx, registry, result) + if err == nil { + return result, nil + } else { + if !errors.Is(err, ErrUnauthorized) { + return docker.Credentials{}, err + } + } + } + } + + if c.promptForCredentials == nil { + return docker.Credentials{}, errCredentialsNotFound + } + + for { + result, err = c.promptForCredentials(registry) + if err != nil { + return docker.Credentials{}, err + } + + err = c.verifyCredentials(ctx, registry, result) + if err == nil { + err = setCredentialsByCredentialHelper(c.authFilePath, registry, result.Username, result.Password) + if err != nil { + if !errors.Is(err, errNoCredentialHelperConfigured) { + return docker.Credentials{}, err + } + helpers := listCredentialHelpers() + helper, err := c.promptForCredentialStore(helpers) + if err != nil { + return docker.Credentials{}, err + } + helper = strings.TrimPrefix(helper, "docker-credential-") + err = setCredentialHelperToConfig(c.authFilePath, helper) + if err != nil { + return docker.Credentials{}, fmt.Errorf("faild to set the helper to the config: %w", err) + } + err = setCredentialsByCredentialHelper(c.authFilePath, registry, result.Username, result.Password) + if err != nil && !errors.Is(err, errNoCredentialHelperConfigured) { + return docker.Credentials{}, err + } + } + return result, nil + } else { + if errors.Is(err, ErrUnauthorized) { + continue + } + return docker.Credentials{}, err + } + } +} + +var errCredentialsNotFound = errors.New("credentials not found") +var errNoCredentialHelperConfigured = errors.New("no credential helper configure") + +func getCredentialHelperFromConfig(confFilePath string) (string, error) { + data, err := ioutil.ReadFile(confFilePath) + if err != nil { + return "", err + } + + conf := struct { + Store string `json:"credsStore"` + }{} + + err = json.Unmarshal(data, &conf) + if err != nil { + return "", err + } + + return conf.Store, nil +} + +func setCredentialHelperToConfig(confFilePath, helper string) error { + var err error + + configData := make(map[string]interface{}) + + if data, err := ioutil.ReadFile(confFilePath); err == nil { + err = json.Unmarshal(data, &configData) + if err != nil { + return err + } + } + + configData["credsStore"] = helper + + data, err := json.MarshalIndent(&configData, "", " ") + if err != nil { + return err + } + + err = ioutil.WriteFile(confFilePath, data, 0600) + if err != nil { + return err + } + + return nil +} + +func getCredentialsByCredentialHelper(confFilePath, registry string) (docker.Credentials, error) { + result := docker.Credentials{} + + helper, err := getCredentialHelperFromConfig(confFilePath) + if err != nil && !os.IsNotExist(err) { + return result, fmt.Errorf("failed to get helper from config: %w", err) + } + if helper == "" { + return result, errCredentialsNotFound + } + + helperName := fmt.Sprintf("docker-credential-%s", helper) + p := client.NewShellProgramFunc(helperName) + + credentialsMap, err := client.List(p) + if err != nil { + return result, fmt.Errorf("failed to list credentials: %w", err) + } + + for serverUrl := range credentialsMap { + if RegistryEquals(serverUrl, registry) { + creds, err := client.Get(p, serverUrl) + if err != nil { + return result, fmt.Errorf("failed to get credentials: %w", err) + } + result.Username = creds.Username + result.Password = creds.Secret + return result, nil + } + } + + return result, fmt.Errorf("failed to get credentials from helper specified in ~/.docker/config.json: %w", errCredentialsNotFound) +} + +func setCredentialsByCredentialHelper(confFilePath, registry, username, secret string) error { + helper, err := getCredentialHelperFromConfig(confFilePath) + + if helper == "" || os.IsNotExist(err) { + return errNoCredentialHelperConfigured + } + if err != nil { + return fmt.Errorf("failed to get helper from config: %w", err) + } + + helperName := fmt.Sprintf("docker-credential-%s", helper) + p := client.NewShellProgramFunc(helperName) + + return client.Store(p, &credentials.Credentials{ServerURL: registry, Username: username, Secret: secret}) +} + +func listCredentialHelpers() []string { + path := os.Getenv("PATH") + paths := strings.Split(path, string(os.PathListSeparator)) + + helpers := make(map[string]bool) + for _, p := range paths { + fss, err := ioutil.ReadDir(p) + if err != nil { + continue + } + for _, fi := range fss { + if fi.IsDir() { + continue + } + if !strings.HasPrefix(fi.Name(), "docker-credential-") { + continue + } + if runtime.GOOS == "windows" { + ext := filepath.Ext(fi.Name()) + if ext != "exe" && ext != "bat" { + continue + } + } + helpers[fi.Name()] = true + } + } + result := make([]string, 0, len(helpers)) + for h := range helpers { + result = append(result, h) + } + return result +} + +func hostPort(registry string) (host string, port string) { + host, port = registry, "" + if !strings.Contains(registry, "://") { + h, p, err := net.SplitHostPort(registry) + + if err == nil { + host, port = h, p + return + } + registry = "https://" + registry + } + + u, err := url.Parse(registry) + if err != nil { + panic(err) + } + host = u.Hostname() + port = u.Port() + return +} + +// RegistryEquals checks whether registry matches in host and port +// with exception where empty port matches standard ports (80,443) +func RegistryEquals(regA, regB string) bool { + h1, p1 := hostPort(regA) + h2, p2 := hostPort(regB) + + isStdPort := func(p string) bool { return p == "443" || p == "80" } + + portEq := p1 == p2 || + (p1 == "" && isStdPort(p2)) || + (isStdPort(p1) && p2 == "") + + if h1 == h2 && portEq { + return true + } + + if strings.HasSuffix(h1, "docker.io") && + strings.HasSuffix(h2, "docker.io") { + return true + } + + return false +} diff --git a/docker/creds/credentials_test.go b/docker/creds/credentials_test.go new file mode 100644 index 000000000..22de7a8ac --- /dev/null +++ b/docker/creds/credentials_test.go @@ -0,0 +1,870 @@ +//go:build !integration +// +build !integration + +package creds_test + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/base64" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "math/big" + "net" + "net/http" + "os" + "path/filepath" + "runtime" + "strings" + "sync" + "testing" + "time" + + fn "knative.dev/kn-plugin-func" + "knative.dev/kn-plugin-func/docker" + "knative.dev/kn-plugin-func/docker/creds" + . "knative.dev/kn-plugin-func/testing" + + "github.com/docker/docker-credential-helpers/credentials" +) + +func Test_registryEquals(t *testing.T) { + tests := []struct { + name string + urlA string + urlB string + want bool + }{ + {"no port matching host", "quay.io", "quay.io", true}, + {"non-matching host added sub-domain", "sub.quay.io", "quay.io", false}, + {"non-matching host different sub-domain", "sub.quay.io", "sub3.quay.io", false}, + {"localhost", "localhost", "localhost", true}, + {"localhost with standard ports", "localhost:80", "localhost:443", false}, + {"localhost with matching port", "https://localhost:1234", "http://localhost:1234", true}, + {"localhost with match by default port 80", "http://localhost", "localhost:80", true}, + {"localhost with match by default port 443", "https://localhost", "localhost:443", true}, + {"localhost with mismatch by non-default port 5000", "https://localhost", "localhost:5000", false}, + {"localhost with match by empty ports", "https://localhost", "http://localhost", true}, + {"docker.io matching host https", "https://docker.io", "docker.io", true}, + {"docker.io matching host http", "http://docker.io", "docker.io", true}, + {"docker.io with path", "docker.io/v1/", "docker.io", true}, + {"docker.io with protocol and path", "https://docker.io/v1/", "docker.io", true}, + {"docker.io with subdomain index.", "https://index.docker.io/v1/", "docker.io", true}, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := creds.RegistryEquals(tt.urlA, tt.urlB); got != tt.want { + t.Errorf("to2ndLevelDomain() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCheckAuth(t *testing.T) { + localhost, localhostTLS, stopServer := startServer(t) + defer stopServer() + + _, portTLS, err := net.SplitHostPort(localhostTLS) + if err != nil { + t.Fatal(err) + } + + nonLocalhostTLS := "test.io:" + portTLS + + type args struct { + ctx context.Context + username string + password string + registry string + } + tests := []struct { + name string + args args + wantErr bool + }{ + { + name: "correct credentials localhost no-TLS", + args: args{ + ctx: context.Background(), + username: "testuser", + password: "testpwd", + registry: localhost, + }, + wantErr: false, + }, + { + name: "correct credentials localhost", + args: args{ + ctx: context.Background(), + username: "testuser", + password: "testpwd", + registry: localhostTLS, + }, + wantErr: false, + }, + + { + name: "correct credentials non-localhost", + args: args{ + ctx: context.Background(), + username: "testuser", + password: "testpwd", + registry: nonLocalhostTLS, + }, + wantErr: false, + }, + { + name: "incorrect credentials localhost no-TLS", + args: args{ + ctx: context.Background(), + username: "testuser", + password: "badpwd", + registry: localhost, + }, + wantErr: true, + }, + { + name: "incorrect credentials localhost", + args: args{ + ctx: context.Background(), + username: "testuser", + password: "badpwd", + registry: localhostTLS, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + c := docker.Credentials{ + Username: tt.args.username, + Password: tt.args.password, + } + if err := creds.CheckAuth(tt.args.ctx, tt.args.registry, c, http.DefaultTransport); (err != nil) != tt.wantErr { + t.Errorf("CheckAuth() error = %v, wantErr %v", err, tt.wantErr) + } + }) + } +} + +func startServer(t *testing.T) (addr, addrTLS string, stopServer func()) { + listener, err := net.Listen("tcp", "localhost:8080") + if err != nil { + t.Fatal(err) + } + addr = listener.Addr().String() + + listenerTLS, err := net.Listen("tcp", "localhost:4433") + if err != nil { + t.Fatal(err) + } + addrTLS = listenerTLS.Addr().String() + + handler := http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { + resp.Header().Add("WWW-Authenticate", "basic") + if user, pwd, ok := req.BasicAuth(); ok { + if user == "testuser" && pwd == "testpwd" { + resp.WriteHeader(http.StatusOK) + return + } + } + resp.WriteHeader(http.StatusUnauthorized) + }) + + var randReader io.Reader = rand.Reader + + caPublicKey, caPrivateKey, err := ed25519.GenerateKey(randReader) + if err != nil { + t.Fatal(err) + } + + ca := &x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: "localhost", + }, + IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, + DNSNames: []string{"localhost", "test.io"}, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(10, 0, 0), + IsCA: true, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, + ExtraExtensions: []pkix.Extension{}, + KeyUsage: x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + BasicConstraintsValid: true, + } + + caBytes, err := x509.CreateCertificate(randReader, ca, ca, caPublicKey, caPrivateKey) + if err != nil { + t.Fatal(err) + } + + ca, err = x509.ParseCertificate(caBytes) + if err != nil { + t.Fatal(err) + } + + cert := tls.Certificate{ + Certificate: [][]byte{caBytes}, + PrivateKey: caPrivateKey, + Leaf: ca, + } + + server := http.Server{ + Handler: handler, + TLSConfig: &tls.Config{ + ServerName: "localhost", + Certificates: []tls.Certificate{cert}, + }, + } + + go func() { + err := server.ServeTLS(listenerTLS, "", "") + if err != nil && !strings.Contains(err.Error(), "Server closed") { + panic(err) + } + }() + + go func() { + err := server.Serve(listener) + if err != nil && !strings.Contains(err.Error(), "Server closed") { + panic(err) + } + }() + + // make the testing CA trusted by default HTTP transport/client + oldDefaultTransport := http.DefaultTransport + newDefaultTransport := http.DefaultTransport.(*http.Transport).Clone() + http.DefaultTransport = newDefaultTransport + caPool := x509.NewCertPool() + caPool.AddCert(ca) + newDefaultTransport.TLSClientConfig.RootCAs = caPool + dc := newDefaultTransport.DialContext + newDefaultTransport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { + h, p, err := net.SplitHostPort(addr) + if err != nil { + return nil, err + } + if h == "test.io" { + h = "localhost" + } + addr = net.JoinHostPort(h, p) + return dc(ctx, network, addr) + } + + return addr, addrTLS, func() { + err := server.Shutdown(context.Background()) + if err != nil { + t.Fatal(err) + } + http.DefaultTransport = oldDefaultTransport + } +} + +const ( + dockerIoUser = "testUser1" + dockerIoUserPwd = "goodPwd1" + quayIoUser = "testUser2" + quayIoUserPwd = "goodPwd2" +) + +type Credentials = docker.Credentials + +func TestNewCredentialsProvider(t *testing.T) { + defer withCleanHome(t)() + + helperWithQuayIO := newInMemoryHelper() + + err := helperWithQuayIO.Add(&credentials.Credentials{ + ServerURL: "quay.io", + Username: quayIoUser, + Secret: quayIoUserPwd, + }) + if err != nil { + t.Fatal(err) + } + + type args struct { + promptUser creds.CredentialsCallback + verifyCredentials creds.VerifyCredentialsCallback + additionalLoaders []creds.CredentialsCallback + registry string + setUpEnv setUpEnv + } + tests := []struct { + name string + args args + want Credentials + }{ + { + name: "test user callback correct password on first try", + args: args{ + promptUser: correctPwdCallback, + verifyCredentials: correctVerifyCbk, + registry: "docker.io", + }, + want: Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, + }, + { + name: "test user callback correct password on second try", + args: args{ + promptUser: pwdCbkFirstWrongThenCorrect(t), + verifyCredentials: correctVerifyCbk, + registry: "docker.io", + }, + want: Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, + }, + { + name: "get quay-io credentials with func config populated", + args: args{ + promptUser: pwdCbkThatShallNotBeCalled(t), + verifyCredentials: correctVerifyCbk, + registry: "quay.io", + setUpEnv: withPopulatedFuncAuthConfig, + }, + want: Credentials{Username: quayIoUser, Password: quayIoUserPwd}, + }, + { + name: "get docker-io credentials with func config populated", + args: args{ + promptUser: pwdCbkThatShallNotBeCalled(t), + verifyCredentials: correctVerifyCbk, + registry: "docker.io", + setUpEnv: withPopulatedFuncAuthConfig, + }, + want: Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, + }, + { + name: "get quay-io credentials with docker config populated", + args: args{ + promptUser: pwdCbkThatShallNotBeCalled(t), + verifyCredentials: correctVerifyCbk, + registry: "quay.io", + setUpEnv: all( + withPopulatedDockerAuthConfig, + setUpMockHelper("docker-credential-mock", helperWithQuayIO)), + }, + want: Credentials{Username: quayIoUser, Password: quayIoUserPwd}, + }, + { + name: "get docker-io credentials with docker config populated", + args: args{ + promptUser: pwdCbkThatShallNotBeCalled(t), + verifyCredentials: correctVerifyCbk, + registry: "docker.io", + setUpEnv: withPopulatedDockerAuthConfig, + }, + want: Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, + }, + { + name: "get docker-io credentials from custom loader", + args: args{ + promptUser: pwdCbkThatShallNotBeCalled(t), + verifyCredentials: correctVerifyCbk, + registry: "docker.io", + additionalLoaders: []creds.CredentialsCallback{correctPwdCallback}, + }, + want: Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + defer cleanUpConfigs(t) + + if tt.args.setUpEnv != nil { + defer tt.args.setUpEnv(t)() + } + + credentialsProvider := creds.NewCredentialsProvider( + creds.WithPromptForCredentials(tt.args.promptUser), + creds.WithVerifyCredentials(tt.args.verifyCredentials), + creds.WithAdditionalCredentialLoaders(tt.args.additionalLoaders...)) + got, err := credentialsProvider(context.Background(), tt.args.registry) + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + if got != tt.want { + t.Errorf("credentialsProvider() = %v, want %v", got, tt.want) + } + }) + } +} + +func TestCredentialsProviderSavingFromUserInput(t *testing.T) { + defer withCleanHome(t)() + + helper := newInMemoryHelper() + defer setUpMockHelper("docker-credential-mock", helper)(t)() + + var pwdCbkInvocations int + pwdCbk := func(r string) (Credentials, error) { + pwdCbkInvocations++ + return correctPwdCallback(r) + } + + chooseNoStore := func(available []string) (string, error) { + if len(available) < 1 { + t.Errorf("this should have been invoked with non empty list") + } + return "", nil + } + chooseMockStore := func(available []string) (string, error) { + if len(available) < 1 { + t.Errorf("this should have been invoked with non empty list") + } + return "docker-credential-mock", nil + } + shallNotBeInvoked := func(available []string) (string, error) { + t.Fatal("this choose helper callback shall not be invoked") + return "", errors.New("this callback shall not be invoked") + } + + credentialsProvider := creds.NewCredentialsProvider( + creds.WithPromptForCredentials(pwdCbk), + creds.WithVerifyCredentials(correctVerifyCbk), + creds.WithPromptForCredentialStore(chooseNoStore)) + _, err := credentialsProvider(context.Background(), "docker.io") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + + // now credentials should not be saved because no helper was provided + l, err := helper.List() + if err != nil { + t.Fatal(err) + } + credsInStore := len(l) + if credsInStore != 0 { + t.Errorf("expected to have zero credentials in store, but has: %d", credsInStore) + } + credentialsProvider = creds.NewCredentialsProvider( + creds.WithPromptForCredentials(pwdCbk), + creds.WithVerifyCredentials(correctVerifyCbk), + creds.WithPromptForCredentialStore(chooseMockStore)) + _, err = credentialsProvider(context.Background(), "docker.io") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } + if pwdCbkInvocations != 2 { + t.Errorf("the pwd callback should have been invoked exactly twice but was invoked %d time", pwdCbkInvocations) + } + + // now credentials should be saved in the mock secure store + l, err = helper.List() + if err != nil { + t.Fatal(err) + } + credsInStore = len(l) + if len(l) != 1 { + t.Errorf("expected to have exactly one credentials in store, but has: %d", credsInStore) + } + credentialsProvider = creds.NewCredentialsProvider( + creds.WithPromptForCredentials(pwdCbkThatShallNotBeCalled(t)), + creds.WithVerifyCredentials(correctVerifyCbk), + creds.WithPromptForCredentialStore(shallNotBeInvoked)) + _, err = credentialsProvider(context.Background(), "docker.io") + if err != nil { + t.Errorf("unexpected error: %v", err) + return + } +} + +func cleanUpConfigs(t *testing.T) { + home, err := os.Hostname() + if err != nil { + t.Fatal(err) + } + + os.RemoveAll(fn.ConfigPath()) + + os.RemoveAll(filepath.Join(home, ".docker")) +} + +type setUpEnv = func(t *testing.T) func() + +func withPopulatedDockerAuthConfig(t *testing.T) func() { + t.Helper() + home, err := os.UserHomeDir() + if err != nil { + t.Fatal(err) + } + dockerConfigDir := filepath.Join(home, ".docker") + dockerConfigPath := filepath.Join(dockerConfigDir, "config.json") + err = os.MkdirAll(filepath.Dir(dockerConfigPath), 0700) + if err != nil { + t.Fatal(err) + } + + configJSON := `{ + "auths": { + "docker.io": { "auth": "%s" }, + "quay.io": {} + }, + "credsStore": "mock" +}` + configJSON = fmt.Sprintf(configJSON, base64.StdEncoding.EncodeToString([]byte(dockerIoUser+":"+dockerIoUserPwd))) + + err = ioutil.WriteFile(dockerConfigPath, []byte(configJSON), 0600) + if err != nil { + t.Fatal(err) + } + + return func() { + + os.RemoveAll(dockerConfigDir) + } +} + +func withPopulatedFuncAuthConfig(t *testing.T) func() { + t.Helper() + + var err error + + authConfig := filepath.Join(fn.ConfigPath(), "auth.json") + err = os.MkdirAll(filepath.Dir(authConfig), 0700) + if err != nil { + t.Fatal(err) + } + + authJSON := `{ + "auths": { + "docker.io": { "auth": "%s" }, + "quay.io": { "auth": "%s" } + } +}` + authJSON = fmt.Sprintf(authJSON, + base64.StdEncoding.EncodeToString([]byte(dockerIoUser+":"+dockerIoUserPwd)), + base64.StdEncoding.EncodeToString([]byte(quayIoUser+":"+quayIoUserPwd))) + + err = ioutil.WriteFile(authConfig, []byte(authJSON), 0600) + if err != nil { + t.Fatal(err) + } + return func() { + os.RemoveAll(fn.ConfigPath()) + } +} + +func pwdCbkThatShallNotBeCalled(t *testing.T) creds.CredentialsCallback { + t.Helper() + return func(registry string) (Credentials, error) { + return Credentials{}, errors.New("this pwd cbk code shall not be called") + } +} + +func pwdCbkFirstWrongThenCorrect(t *testing.T) func(registry string) (Credentials, error) { + t.Helper() + var firstInvocation bool + return func(registry string) (Credentials, error) { + if registry != "docker.io" && registry != "quay.io" { + return Credentials{}, fmt.Errorf("unexpected registry: %s", registry) + } + if firstInvocation { + firstInvocation = false + return Credentials{Username: dockerIoUser, Password: "badPwd"}, nil + } + return correctPwdCallback(registry) + } +} + +func correctPwdCallback(registry string) (Credentials, error) { + if registry == "docker.io" { + return Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, nil + } + if registry == "quay.io" { + return Credentials{Username: quayIoUser, Password: quayIoUserPwd}, nil + } + return Credentials{}, errors.New("this cbk don't know the pwd") +} + +func correctVerifyCbk(ctx context.Context, registry string, credentials Credentials) error { + username, password := credentials.Username, credentials.Password + if username == dockerIoUser && password == dockerIoUserPwd && registry == "docker.io" { + return nil + } + if username == quayIoUser && password == quayIoUserPwd && registry == "quay.io" { + return nil + } + return creds.ErrUnauthorized +} + +func withCleanHome(t *testing.T) func() { + t.Helper() + homeName := "HOME" + if runtime.GOOS == "windows" { + homeName = "USERPROFILE" + } + tmpHome := t.TempDir() + oldHome, hadHome := os.LookupEnv(homeName) + os.Setenv(homeName, tmpHome) + + oldXDGConfigHome, hadXDGConfigHome := os.LookupEnv("XDG_CONFIG_HOME") + + if runtime.GOOS == "linux" { + os.Setenv("XDG_CONFIG_HOME", filepath.Join(tmpHome, ".config")) + } + + return func() { + if hadHome { + os.Setenv(homeName, oldHome) + } else { + os.Unsetenv(homeName) + } + if hadXDGConfigHome { + os.Setenv("XDG_CONFIG_HOME", oldXDGConfigHome) + } else { + os.Unsetenv("XDG_CONFIG_HOME") + } + } +} + +func handlerForCredHelper(t *testing.T, credHelper credentials.Helper) http.Handler { + return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { + defer request.Body.Close() + + var err error + var outBody interface{} + + uri := strings.Trim(request.RequestURI, "/") + + var serverURL string + if uri == "get" || uri == "erase" { + data, err := ioutil.ReadAll(request.Body) + if err != nil { + writer.WriteHeader(http.StatusInternalServerError) + return + } + serverURL = string(data) + serverURL = strings.Trim(serverURL, "\n\r\t ") + } + + switch uri { + case "list": + var list map[string]string + list, err = credHelper.List() + if err == nil { + outBody = &list + } + case "store": + creds := credentials.Credentials{} + dec := json.NewDecoder(request.Body) + err = dec.Decode(&creds) + if err != nil { + break + } + err = credHelper.Add(&creds) + case "get": + var user, secret string + user, secret, err = credHelper.Get(serverURL) + if err == nil { + outBody = &credentials.Credentials{ServerURL: serverURL, Username: user, Secret: secret} + } + case "erase": + err = credHelper.Delete(serverURL) + default: + writer.WriteHeader(http.StatusNotFound) + return + } + + if err != nil { + if credentials.IsErrCredentialsNotFound(err) { + writer.WriteHeader(http.StatusNotFound) + } else { + writer.WriteHeader(http.StatusInternalServerError) + writer.Header().Add("Content-Type", "text/plain") + fmt.Fprintf(writer, "error: %+v\n", err) + } + return + } + + if outBody != nil { + var data []byte + data, err = json.Marshal(outBody) + if err != nil { + writer.WriteHeader(http.StatusInternalServerError) + return + } + writer.Header().Add("Content-Type", "application/json") + _, err = writer.Write(data) + if err != nil { + t.Fatal(err) + } + } + }) + +} + +// Go source code of mock docker-credential-helper implementation. +// Its storage is backed by inMemoryHelper instantiated in test and exposed via HTTP. +const helperGoSrc = `package main + +import ( + "errors" + "io" + "log" + "net/http" + "os" +) + +func main() { + var ( + baseURL = os.Getenv("HELPER_BASE_URL") + resp *http.Response + err error + ) + cmd := os.Args[1] + switch cmd { + case "list": + resp, err = http.Get(baseURL + "/" + cmd) + if err != nil { + log.Fatal(err) + } + io.Copy(os.Stdout, resp.Body) + case "get", "erase": + resp, err = http.Post(baseURL+ "/" + cmd, "text/plain", os.Stdin) + if err != nil { + log.Fatal(err) + } + io.Copy(os.Stdout, resp.Body) + case "store": + resp, err = http.Post(baseURL+ "/" + cmd, "application/json", os.Stdin) + if err != nil { + log.Fatal(err) + } + default: + log.Fatal(errors.New("unknown cmd: " + cmd)) + } + if resp.StatusCode != http.StatusOK { + log.Fatal(errors.New(resp.Status)) + } + return +} +` + +// Creates executable with name determined by the helperName parameter and puts it on $PATH. +// +// The executable behaves like docker credential helper (https://github.com/docker/docker-credential-helpers). +// +// The content of the store presented by the executable is backed by the helper parameter. +func setUpMockHelper(helperName string, helper credentials.Helper) func(t *testing.T) func() { + var cleanUps []func() + return func(t *testing.T) func() { + + cleanUps = append(cleanUps, WithExecutable(t, helperName, helperGoSrc)) + + listener, err := net.Listen("tcp", "localhost:0") + if err != nil { + t.Fatal(err) + } + + cleanUps = append(cleanUps, func() { + _ = listener.Close() + }) + + baseURL := fmt.Sprintf("http://%s", listener.Addr().String()) + cleanUps = append(cleanUps, WithEnvVar(t, "HELPER_BASE_URL", baseURL)) + + server := http.Server{Handler: handlerForCredHelper(t, helper)} + servErrChan := make(chan error) + go func() { + servErrChan <- server.Serve(listener) + }() + + cleanUps = append(cleanUps, func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + defer cancel() + _ = server.Shutdown(ctx) + e := <-servErrChan + if !errors.Is(e, http.ErrServerClosed) { + t.Fatal(e) + } + }) + + return func() { + for i := len(cleanUps) - 1; i <= 0; i-- { + cleanUps[i]() + } + } + } +} + +// combines multiple setUp routines into one setUp routine +func all(fns ...setUpEnv) setUpEnv { + return func(t *testing.T) func() { + t.Helper() + var cleanUps []func() + for _, fn := range fns { + cleanUps = append(cleanUps, fn(t)) + } + + return func() { + for i := len(cleanUps) - 1; i >= 0; i-- { + cleanUps[i]() + } + } + } +} + +func newInMemoryHelper() *inMemoryHelper { + return &inMemoryHelper{lock: &sync.Mutex{}, credentials: make(map[string]credentials.Credentials)} +} + +type inMemoryHelper struct { + credentials map[string]credentials.Credentials + lock sync.Locker +} + +func (i *inMemoryHelper) Add(credentials *credentials.Credentials) error { + i.lock.Lock() + defer i.lock.Unlock() + + i.credentials[credentials.ServerURL] = *credentials + + return nil +} + +func (i *inMemoryHelper) Get(serverURL string) (string, string, error) { + i.lock.Lock() + defer i.lock.Unlock() + + if result, ok := i.credentials[serverURL]; ok { + return result.Username, result.Secret, nil + } + + return "", "", credentials.NewErrCredentialsNotFound() +} + +func (i *inMemoryHelper) List() (map[string]string, error) { + i.lock.Lock() + defer i.lock.Unlock() + + result := make(map[string]string, len(i.credentials)) + + for k, v := range i.credentials { + result[k] = v.Username + } + + return result, nil +} + +func (i *inMemoryHelper) Delete(serverURL string) error { + i.lock.Lock() + defer i.lock.Unlock() + + if _, ok := i.credentials[serverURL]; ok { + delete(i.credentials, serverURL) + return nil + } + + return credentials.NewErrCredentialsNotFound() +} diff --git a/docker/pusher.go b/docker/pusher.go index ea5b5f1fc..5390e08a7 100644 --- a/docker/pusher.go +++ b/docker/pusher.go @@ -8,24 +8,22 @@ import ( "errors" "fmt" "io" + "net" "net/http" "os" - "path/filepath" "regexp" "strings" - "github.com/containers/image/v5/pkg/docker/config" - - "github.com/google/go-containerregistry/pkg/authn" - "github.com/google/go-containerregistry/pkg/name" - "github.com/google/go-containerregistry/pkg/v1/remote/transport" - "github.com/docker/docker/client" fn "knative.dev/kn-plugin-func" - containersTypes "github.com/containers/image/v5/types" "github.com/docker/docker/api/types" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/daemon" + "github.com/google/go-containerregistry/pkg/v1/remote" ) type Opt func(*Pusher) error @@ -37,235 +35,17 @@ type Credentials struct { type CredentialsProvider func(ctx context.Context, registry string) (Credentials, error) -type CredentialsCallback func(registry string) (Credentials, error) - -var ErrUnauthorized = errors.New("bad credentials") - -// VerifyCredentialsCallback checks if credentials are accepted by the registry. -// If credentials are incorrect this callback shall return ErrUnauthorized. -type VerifyCredentialsCallback func(ctx context.Context, registry string, credentials Credentials) error - -func CheckAuth(ctx context.Context, registry string, credentials Credentials) error { - serverAddress := registry - if !strings.HasPrefix(serverAddress, "https://") && !strings.HasPrefix(serverAddress, "http://") { - serverAddress = "https://" + serverAddress - } - - url := fmt.Sprintf("%s/v2", serverAddress) - - authenticator := &authn.Basic{ - Username: credentials.Username, - Password: credentials.Password, - } - - reg, err := name.NewRegistry(registry) - if err != nil { - return err - } - - tr, err := transport.NewWithContext(ctx, reg, authenticator, http.DefaultTransport, nil) - if err != nil { - return err - } - - cli := http.Client{Transport: tr} - - req, err := http.NewRequestWithContext(ctx, "GET", url, nil) - if err != nil { - return err - } - - resp, err := cli.Do(req) - if err != nil { - return fmt.Errorf("failed to verify credentials: %w", err) - } - defer resp.Body.Close() - - switch { - case resp.StatusCode == http.StatusUnauthorized: - return ErrUnauthorized - case resp.StatusCode != http.StatusOK: - return fmt.Errorf("failed to verify credentials: status code: %d", resp.StatusCode) - default: - return nil - } +// PusherDockerClient is sub-interface of client.CommonAPIClient required by pusher. +type PusherDockerClient interface { + NegotiateAPIVersion(ctx context.Context) + ImageSave(context.Context, []string) (io.ReadCloser, error) + ImageLoad(context.Context, io.Reader, bool) (types.ImageLoadResponse, error) + ImageTag(context.Context, string, string) error + ImagePush(ctx context.Context, ref string, options types.ImagePushOptions) (io.ReadCloser, error) + Close() error } -type ChooseCredentialHelperCallback func(available []string) (string, error) - -type credentialProviderConfig struct { - promptForCredentials CredentialsCallback - verifyCredentials VerifyCredentialsCallback - promptForCredentialStore ChooseCredentialHelperCallback - additionalCredentialLoaders []CredentialsCallback -} - -type CredentialProviderOptions func(opts *credentialProviderConfig) - -// WithPromptForCredentials sets custom callback that is supposed to -// interactively ask for credentials in case the credentials cannot be found in configuration files. -// The callback may be called multiple times in case incorrect credentials were returned before. -func WithPromptForCredentials(cbk CredentialsCallback) CredentialProviderOptions { - return func(opts *credentialProviderConfig) { - opts.promptForCredentials = cbk - } -} - -// WithVerifyCredentials sets custom callback for credentials validation. -func WithVerifyCredentials(cbk VerifyCredentialsCallback) CredentialProviderOptions { - return func(opts *credentialProviderConfig) { - opts.verifyCredentials = cbk - } -} - -// WithPromptForCredentialStore sets custom callback that is supposed to -// interactively ask user which credentials store/helper is used to store credentials obtained -// from user. -func WithPromptForCredentialStore(cbk ChooseCredentialHelperCallback) CredentialProviderOptions { - return func(opts *credentialProviderConfig) { - opts.promptForCredentialStore = cbk - } -} - -// WithAdditionalCredentialLoaders adds custom callbacks for credential retrieval. -// The callbacks are supposed to be non-interactive as opposed to WithPromptForCredentials. -// -// This might be useful when credentials are shared with some other service. -// -// Example: OpenShift builtin registry shares credentials with the cluster (k8s) credentials. -func WithAdditionalCredentialLoaders(loaders ...CredentialsCallback) CredentialProviderOptions { - return func(opts *credentialProviderConfig) { - opts.additionalCredentialLoaders = append(opts.additionalCredentialLoaders, loaders...) - } -} - -// NewCredentialsProvider returns new CredentialsProvider that tires to get credentials from docker/func config files. -// -// In case getting credentials from the config files fails -// the caller provided callback (see WithPromptForCredentials) will be invoked to obtain credentials. -// The callback may be called multiple times in case the returned credentials -// are not correct (see WithVerifyCredentials). -// -// When the callback succeeds the credentials will be saved by using helper defined in the func config. -// If the helper is not defined in the config file -// it may be picked by provided callback (see WithPromptForCredentialStore). -// The picked value will be saved in the func config. -// -// To verify that credentials are correct callback will be used (see WithVerifyCredentials). -// If the callback is not set then CheckAuth will be used as a fallback. -func NewCredentialsProvider(opts ...CredentialProviderOptions) CredentialsProvider { - var conf credentialProviderConfig - - for _, o := range opts { - o(&conf) - } - - askUser, verifyCredentials, chooseCredentialHelper := conf.promptForCredentials, conf.verifyCredentials, conf.promptForCredentialStore - - if verifyCredentials == nil { - verifyCredentials = CheckAuth - } - - if chooseCredentialHelper == nil { - chooseCredentialHelper = func(available []string) (string, error) { - return "", nil - } - } - - authFilePath := filepath.Join(fn.ConfigPath(), "auth.json") - sys := &containersTypes.SystemContext{ - AuthFilePath: authFilePath, - } - - home, err := os.UserHomeDir() - if err != nil { - panic(err) - //return result, fmt.Errorf("failed to determine home directory: %w", err) - } - dockerConfigPath := filepath.Join(home, ".docker", "config.json") - - var authLoaders = []CredentialsCallback{ - func(registry string) (Credentials, error) { - creds, err := config.GetCredentials(sys, registry) - if err != nil { - return Credentials{}, err - } - return Credentials{ - Username: creds.Username, - Password: creds.Password, - }, nil - }, - func(registry string) (Credentials, error) { - return getCredentialsByCredentialHelper(authFilePath, registry) - }, - func(registry string) (Credentials, error) { - return getCredentialsByCredentialHelper(dockerConfigPath, registry) - }, - } - - authLoaders = append(conf.additionalCredentialLoaders, authLoaders...) - - return func(ctx context.Context, registry string) (Credentials, error) { - result := Credentials{} - - for _, load := range authLoaders { - - result, err = load(registry) - - if err != nil && !errors.Is(err, errCredentialsNotFound) { - return Credentials{}, err - } - - if result != (Credentials{}) { - err = verifyCredentials(ctx, registry, result) - if err == nil { - return result, nil - } else { - if !errors.Is(err, ErrUnauthorized) { - return Credentials{}, err - } - } - } - } - - for { - result, err = askUser(registry) - if err != nil { - return Credentials{}, err - } - - err = verifyCredentials(ctx, registry, result) - if err == nil { - err = setCredentialsByCredentialHelper(authFilePath, registry, result.Username, result.Password) - if err != nil { - if !errors.Is(err, errNoCredentialHelperConfigured) { - return Credentials{}, err - } - helpers := listCredentialHelpers() - helper, err := chooseCredentialHelper(helpers) - if err != nil { - return Credentials{}, err - } - helper = strings.TrimPrefix(helper, "docker-credential-") - err = setCredentialHelperToConfig(authFilePath, helper) - if err != nil { - return Credentials{}, fmt.Errorf("faild to set the helper to the config: %w", err) - } - err = setCredentialsByCredentialHelper(authFilePath, registry, result.Username, result.Password) - if err != nil && !errors.Is(err, errNoCredentialHelperConfigured) { - return Credentials{}, err - } - } - return result, nil - } else { - if errors.Is(err, ErrUnauthorized) { - continue - } - return Credentials{}, err - } - } - } -} +type PusherDockerClientFactory func() (PusherDockerClient, error) // Pusher of images from local to remote registry. type Pusher struct { @@ -273,6 +53,8 @@ type Pusher struct { Verbose bool credentialsProvider CredentialsProvider progressListener fn.ProgressListener + transport http.RoundTripper + dockerClientFactory PusherDockerClientFactory } func WithCredentialsProvider(cp CredentialsProvider) Opt { @@ -289,6 +71,20 @@ func WithProgressListener(pl fn.ProgressListener) Opt { } } +func WithTransport(transport http.RoundTripper) Opt { + return func(pusher *Pusher) error { + pusher.transport = transport + return nil + } +} + +func WithPusherDockerClientFactory(dockerClientFactory PusherDockerClientFactory) Opt { + return func(pusher *Pusher) error { + pusher.dockerClientFactory = dockerClientFactory + return nil + } +} + func EmptyCredentialsProvider(ctx context.Context, registry string) (Credentials, error) { return Credentials{}, nil } @@ -299,6 +95,11 @@ func NewPusher(opts ...Opt) (*Pusher, error) { Verbose: false, credentialsProvider: EmptyCredentialsProvider, progressListener: &fn.NoopProgressListener{}, + transport: http.DefaultTransport, + dockerClientFactory: func() (PusherDockerClient, error) { + c, _, err := NewClient(client.DefaultDockerHost) + return c, err + }, } for _, opt := range opts { err := opt(result) @@ -306,10 +107,11 @@ func NewPusher(opts ...Opt) (*Pusher, error) { return nil, err } } + return result, nil } -func getRegistry(image_url string) (string, error) { +func GetRegistry(image_url string) (string, error) { var registry string parts := strings.Split(image_url, "/") switch { @@ -327,21 +129,23 @@ func getRegistry(image_url string) (string, error) { // Push the image of the Function. func (n *Pusher) Push(ctx context.Context, f fn.Function) (digest string, err error) { + var output io.Writer + + if n.Verbose { + output = os.Stderr + } else { + output = io.Discard + } + if f.Image == "" { return "", errors.New("Function has no associated image. Has it been built?") } - registry, err := getRegistry(f.Image) + registry, err := GetRegistry(f.Image) if err != nil { return "", err } - cli, _, err := NewClient(client.DefaultDockerHost) - if err != nil { - return "", fmt.Errorf("failed to create docker api client: %w", err) - } - defer cli.Close() - n.progressListener.Stopping() credentials, err := n.credentialsProvider(ctx, registry) if err != nil { @@ -349,10 +153,25 @@ func (n *Pusher) Push(ctx context.Context, f fn.Function) (digest string, err er } n.progressListener.Increment("Pushing function image to the registry") + // if the registry is not cluster private do push directly from daemon + if _, err = net.DefaultResolver.LookupHost(ctx, registry); err == nil { + return n.daemonPush(ctx, f, credentials, output) + } + + // push with custom transport to be able to push into cluster private registries + return n.push(ctx, f, credentials, output) +} + +func (n *Pusher) daemonPush(ctx context.Context, f fn.Function, credentials Credentials, output io.Writer) (digest string, err error) { + cli, err := n.dockerClientFactory() + if err != nil { + return "", fmt.Errorf("failed to create docker api client: %w", err) + } + defer cli.Close() + authConfig := types.AuthConfig{ - Username: credentials.Username, - Password: credentials.Password, - ServerAddress: registry, + Username: credentials.Username, + Password: credentials.Password, } b, err := json.Marshal(&authConfig) @@ -368,15 +187,8 @@ func (n *Pusher) Push(ctx context.Context, f fn.Function) (digest string, err er } defer r.Close() - var output io.Writer var outBuff bytes.Buffer - - // If verbose logging is enabled, echo chatty stdout. - if n.Verbose { - output = io.MultiWriter(&outBuff, os.Stdout) - } else { - output = &outBuff - } + output = io.MultiWriter(&outBuff, output) decoder := json.NewDecoder(r) li := logItem{} @@ -403,17 +215,17 @@ func (n *Pusher) Push(ctx context.Context, f fn.Function) (digest string, err er fmt.Fprintf(output, "%s (%d%%)\n", li.Status, percent) } - digest = parseDigest(outBuff.String()) + digest = ParseDigest(outBuff.String()) - return + return digest, nil } var digestRE = regexp.MustCompile(`digest:\s+(sha256:\w{64})`) -// parseDigest tries to parse the last line from the output, which holds the pushed image digest +// ParseDigest tries to parse the last line from the output, which holds the pushed image digest // The output should contain line like this: // latest: digest: sha256:a278a91112d17f8bde6b5f802a3317c7c752cf88078dae6f4b5a0784deb81782 size: 2613 -func parseDigest(output string) string { +func ParseDigest(output string) string { match := digestRE.FindStringSubmatch(output) if len(match) >= 2 { return match[1] @@ -438,3 +250,65 @@ type logItem struct { Progress string `json:"progress"` ProgressDetail progressDetail `json:"progressDetail"` } + +func (n *Pusher) push(ctx context.Context, f fn.Function, credentials Credentials, output io.Writer) (digest string, err error) { + auth := &authn.Basic{ + Username: credentials.Username, + Password: credentials.Password, + } + + ref, err := name.ParseReference(f.Image) + if err != nil { + return "", err + } + + dockerClient, err := n.dockerClientFactory() + if err != nil { + return "", fmt.Errorf("failed to create docker api client: %w", err) + } + defer dockerClient.Close() + + img, err := daemon.Image(ref, + daemon.WithContext(ctx), + daemon.WithClient(dockerClient)) + if err != nil { + return "", err + } + + progressChannel := make(chan v1.Update, 1024) + errChan := make(chan error) + go func() { + defer fmt.Fprint(output, "\n") + + for progress := range progressChannel { + if progress.Error != nil { + errChan <- progress.Error + return + } + fmt.Fprintf(output, "\rprogress: %d%%", progress.Complete*100/progress.Total) + } + + errChan <- nil + }() + + err = remote.Write(ref, img, + remote.WithAuth(auth), + remote.WithProgress(progressChannel), + remote.WithTransport(n.transport), + remote.WithJobs(1), + remote.WithContext(ctx)) + if err != nil { + return "", err + } + err = <-errChan + if err != nil { + return "", err + } + + hash, err := img.Digest() + if err != nil { + return "", err + } + + return hash.String(), nil +} diff --git a/docker/pusher_test.go b/docker/pusher_test.go index ed8a11da2..6895b58f8 100644 --- a/docker/pusher_test.go +++ b/docker/pusher_test.go @@ -1,60 +1,42 @@ //go:build !integration // +build !integration -package docker +package docker_test import ( + "bytes" "context" - "crypto/ed25519" + "crypto/ecdsa" + "crypto/elliptic" "crypto/rand" "crypto/tls" "crypto/x509" "crypto/x509/pkix" "encoding/base64" "encoding/json" - "errors" "fmt" "io" - "io/ioutil" + "log" "math/big" "net" "net/http" "os" - "path/filepath" - "runtime" + "reflect" "strings" - "sync" "testing" "time" - fn "knative.dev/kn-plugin-func" - . "knative.dev/kn-plugin-func/testing" + "github.com/docker/docker/api/types" + "github.com/google/go-containerregistry/pkg/authn" + "github.com/google/go-containerregistry/pkg/name" + "github.com/google/go-containerregistry/pkg/registry" + "github.com/google/go-containerregistry/pkg/v1/remote" - "github.com/docker/docker-credential-helpers/credentials" + fn "knative.dev/kn-plugin-func" + "knative.dev/kn-plugin-func/docker" ) -func Test_parseDigest(t *testing.T) { - tests := []struct { - name string - arg string - want string - }{ - { - name: "basic test", - arg: "latest: digest: sha256:a278a91112d17f8bde6b5f802a3317c7c752cf88078dae6f4b5a0784deb81782 size: 2613", - want: "sha256:a278a91112d17f8bde6b5f802a3317c7c752cf88078dae6f4b5a0784deb81782", - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - if got := parseDigest(tt.arg); got != tt.want { - t.Errorf("parseDigest() = %v, want %v", got, tt.want) - } - }) - } -} - -func Test_getRegistry(t *testing.T) { +func TestGetRegistry(t *testing.T) { tests := []struct { name string arg string @@ -78,739 +60,282 @@ func Test_getRegistry(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got, _ := getRegistry(tt.arg); got != tt.want { - t.Errorf("getRegistry() = %v, want %v", got, tt.want) + if got, _ := docker.GetRegistry(tt.arg); got != tt.want { + t.Errorf("GetRegistry() = %v, want %v", got, tt.want) } }) } } const ( - dockerIoUser = "testUser1" - dockerIoUserPwd = "goodPwd1" - quayIoUser = "testUser2" - quayIoUserPwd = "goodPwd2" + testUser = "testuser" + testPwd = "testpwd" + registryHostname = "my.testing.registry" + functionImage = "/testuser/func:latest" + functionImageRemote = registryHostname + functionImage + functionImageLocal = "localhost" + functionImage ) -func TestNewCredentialsProvider(t *testing.T) { - defer withCleanHome(t)() +var testCredProvider = docker.CredentialsProvider(func(ctx context.Context, registry string) (docker.Credentials, error) { + return docker.Credentials{ + Username: testUser, + Password: testPwd, + }, nil +}) - helperWithQuayIO := newInMemoryHelper() +func TestDaemonPush(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + defer cancel() - err := helperWithQuayIO.Add(&credentials.Credentials{ - ServerURL: "quay.io", - Username: quayIoUser, - Secret: quayIoUserPwd, - }) - if err != nil { - t.Fatal(err) - } + var optsPassedToMock types.ImagePushOptions + var imagePassedToMock string + var closeCalledOnMock bool - type args struct { - promptUser CredentialsCallback - verifyCredentials VerifyCredentialsCallback - additionalLoaders []CredentialsCallback - registry string - setUpEnv setUpEnv - } - tests := []struct { - name string - args args - want Credentials - }{ - { - name: "test user callback correct password on first try", - args: args{ - promptUser: correctPwdCallback, - verifyCredentials: correctVerifyCbk, - registry: "docker.io", - }, - want: Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, - }, - { - name: "test user callback correct password on second try", - args: args{ - promptUser: pwdCbkFirstWrongThenCorrect(t), - verifyCredentials: correctVerifyCbk, - registry: "docker.io", - }, - want: Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, - }, - { - name: "get quay-io credentials with func config populated", - args: args{ - promptUser: pwdCbkThatShallNotBeCalled(t), - verifyCredentials: correctVerifyCbk, - registry: "quay.io", - setUpEnv: withPopulatedFuncAuthConfig, - }, - want: Credentials{Username: quayIoUser, Password: quayIoUserPwd}, - }, - { - name: "get docker-io credentials with func config populated", - args: args{ - promptUser: pwdCbkThatShallNotBeCalled(t), - verifyCredentials: correctVerifyCbk, - registry: "docker.io", - setUpEnv: withPopulatedFuncAuthConfig, - }, - want: Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, - }, - { - name: "get quay-io credentials with docker config populated", - args: args{ - promptUser: pwdCbkThatShallNotBeCalled(t), - verifyCredentials: correctVerifyCbk, - registry: "quay.io", - setUpEnv: all( - withPopulatedDockerAuthConfig, - setUpMockHelper("docker-credential-mock", helperWithQuayIO)), - }, - want: Credentials{Username: quayIoUser, Password: quayIoUserPwd}, - }, - { - name: "get docker-io credentials with docker config populated", - args: args{ - promptUser: pwdCbkThatShallNotBeCalled(t), - verifyCredentials: correctVerifyCbk, - registry: "docker.io", - setUpEnv: withPopulatedDockerAuthConfig, - }, - want: Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, - }, - { - name: "get docker-io credentials from custom loader", - args: args{ - promptUser: pwdCbkThatShallNotBeCalled(t), - verifyCredentials: correctVerifyCbk, - registry: "docker.io", - additionalLoaders: []CredentialsCallback{correctPwdCallback}, - }, - want: Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - defer cleanUpConfigs(t) + dockerClient := newMockPusherDockerClient() - if tt.args.setUpEnv != nil { - defer tt.args.setUpEnv(t)() - } - - credentialsProvider := NewCredentialsProvider( - WithPromptForCredentials(tt.args.promptUser), - WithVerifyCredentials(tt.args.verifyCredentials), - WithAdditionalCredentialLoaders(tt.args.additionalLoaders...)) - got, err := credentialsProvider(context.Background(), tt.args.registry) - if err != nil { - t.Errorf("unexpected error: %v", err) - return - } - if got != tt.want { - t.Errorf("credentialsProvider() = %v, want %v", got, tt.want) - } - }) - } + dockerClient.imagePush = func(ctx context.Context, ref string, options types.ImagePushOptions) (io.ReadCloser, error) { + imagePassedToMock = ref + optsPassedToMock = options + return io.NopCloser(strings.NewReader(`{ + "status": "latest: digest: sha256:00af51d125f3092e157a7f8a717029412dc9d266c017e89cecdfeccb4cc3d7a7 size: 2613" } - -func TestCredentialsProviderSavingFromUserInput(t *testing.T) { - defer withCleanHome(t)() - - helper := newInMemoryHelper() - defer setUpMockHelper("docker-credential-mock", helper)(t)() - - var pwdCbkInvocations int - pwdCbk := func(r string) (Credentials, error) { - pwdCbkInvocations++ - return correctPwdCallback(r) +`)), nil } - chooseNoStore := func(available []string) (string, error) { - if len(available) < 1 { - t.Errorf("this should have been invoked with non empty list") - } - return "", nil - } - chooseMockStore := func(available []string) (string, error) { - if len(available) < 1 { - t.Errorf("this should have been invoked with non empty list") - } - return "docker-credential-mock", nil - } - shallNotBeInvoked := func(available []string) (string, error) { - t.Fatal("this choose helper callback shall not be invoked") - return "", errors.New("this callback shall not be invoked") - } - - credentialsProvider := NewCredentialsProvider( - WithPromptForCredentials(pwdCbk), - WithVerifyCredentials(correctVerifyCbk), - WithPromptForCredentialStore(chooseNoStore)) - _, err := credentialsProvider(context.Background(), "docker.io") - if err != nil { - t.Errorf("unexpected error: %v", err) - return - } - - // now credentials should not be saved because no helper was provided - l, err := helper.List() - if err != nil { - t.Fatal(err) - } - credsInStore := len(l) - if credsInStore != 0 { - t.Errorf("expected to have zero credentials in store, but has: %d", credsInStore) - } - credentialsProvider = NewCredentialsProvider( - WithPromptForCredentials(pwdCbk), - WithVerifyCredentials(correctVerifyCbk), - WithPromptForCredentialStore(chooseMockStore)) - _, err = credentialsProvider(context.Background(), "docker.io") - if err != nil { - t.Errorf("unexpected error: %v", err) - return - } - if pwdCbkInvocations != 2 { - t.Errorf("the pwd callback should have been invoked exactly twice but was invoked %d time", pwdCbkInvocations) - } - - // now credentials should be saved in the mock secure store - l, err = helper.List() - if err != nil { - t.Fatal(err) - } - credsInStore = len(l) - if len(l) != 1 { - t.Errorf("expected to have exactly one credentials in store, but has: %d", credsInStore) - } - credentialsProvider = NewCredentialsProvider( - WithPromptForCredentials(pwdCbkThatShallNotBeCalled(t)), - WithVerifyCredentials(correctVerifyCbk), - WithPromptForCredentialStore(shallNotBeInvoked)) - _, err = credentialsProvider(context.Background(), "docker.io") - if err != nil { - t.Errorf("unexpected error: %v", err) - return - } -} - -func cleanUpConfigs(t *testing.T) { - home, err := os.Hostname() - if err != nil { - t.Fatal(err) - } - - os.RemoveAll(fn.ConfigPath()) - - os.RemoveAll(filepath.Join(home, ".docker")) -} - -type setUpEnv = func(t *testing.T) func() - -func withPopulatedDockerAuthConfig(t *testing.T) func() { - t.Helper() - home, err := os.UserHomeDir() - if err != nil { - t.Fatal(err) - } - dockerConfigDir := filepath.Join(home, ".docker") - dockerConfigPath := filepath.Join(dockerConfigDir, "config.json") - err = os.MkdirAll(filepath.Dir(dockerConfigPath), 0700) - if err != nil { - t.Fatal(err) - } - - configJSON := `{ - "auths": { - "docker.io": { "auth": "%s" }, - "quay.io": {} - }, - "credsStore": "mock" -}` - configJSON = fmt.Sprintf(configJSON, base64.StdEncoding.EncodeToString([]byte(dockerIoUser+":"+dockerIoUserPwd))) - - err = ioutil.WriteFile(dockerConfigPath, []byte(configJSON), 0600) - if err != nil { - t.Fatal(err) - } - - return func() { - - os.RemoveAll(dockerConfigDir) - } -} - -func withPopulatedFuncAuthConfig(t *testing.T) func() { - t.Helper() - - var err error - - authConfig := filepath.Join(fn.ConfigPath(), "auth.json") - err = os.MkdirAll(filepath.Dir(authConfig), 0700) - if err != nil { - t.Fatal(err) - } - - authJSON := `{ - "auths": { - "docker.io": { "auth": "%s" }, - "quay.io": { "auth": "%s" } - } -}` - authJSON = fmt.Sprintf(authJSON, - base64.StdEncoding.EncodeToString([]byte(dockerIoUser+":"+dockerIoUserPwd)), - base64.StdEncoding.EncodeToString([]byte(quayIoUser+":"+quayIoUserPwd))) - - err = ioutil.WriteFile(authConfig, []byte(authJSON), 0600) - if err != nil { - t.Fatal(err) - } - return func() { - os.RemoveAll(fn.ConfigPath()) - } -} - -func pwdCbkThatShallNotBeCalled(t *testing.T) CredentialsCallback { - t.Helper() - return func(registry string) (Credentials, error) { - return Credentials{}, errors.New("this pwd cbk code shall not be called") - } -} - -func pwdCbkFirstWrongThenCorrect(t *testing.T) func(registry string) (Credentials, error) { - t.Helper() - var firstInvocation bool - return func(registry string) (Credentials, error) { - if registry != "docker.io" && registry != "quay.io" { - return Credentials{}, fmt.Errorf("unexpected registry: %s", registry) - } - if firstInvocation { - firstInvocation = false - return Credentials{dockerIoUser, "badPwd"}, nil - } - return correctPwdCallback(registry) - } -} - -func correctPwdCallback(registry string) (Credentials, error) { - if registry == "docker.io" { - return Credentials{Username: dockerIoUser, Password: dockerIoUserPwd}, nil - } - if registry == "quay.io" { - return Credentials{Username: quayIoUser, Password: quayIoUserPwd}, nil - } - return Credentials{}, errors.New("this cbk don't know the pwd") -} - -func correctVerifyCbk(ctx context.Context, registry string, creds Credentials) error { - username, password := creds.Username, creds.Password - if username == dockerIoUser && password == dockerIoUserPwd && registry == "docker.io" { + dockerClient.close = func() error { + closeCalledOnMock = true return nil } - if username == quayIoUser && password == quayIoUserPwd && registry == "quay.io" { - return nil + + dockerClientFactory := func() (docker.PusherDockerClient, error) { + return dockerClient, nil } - return ErrUnauthorized -} - -func withCleanHome(t *testing.T) func() { - t.Helper() - homeName := "HOME" - if runtime.GOOS == "windows" { - homeName = "USERPROFILE" - } - tmpHome := t.TempDir() - oldHome, hadHome := os.LookupEnv(homeName) - os.Setenv(homeName, tmpHome) - - oldXDGConfigHome, hadXDGConfigHome := os.LookupEnv("XDG_CONFIG_HOME") - - if runtime.GOOS == "linux" { - os.Setenv("XDG_CONFIG_HOME", filepath.Join(tmpHome, ".config")) + pusher, err := docker.NewPusher( + docker.WithCredentialsProvider(testCredProvider), + docker.WithPusherDockerClientFactory(dockerClientFactory)) + if err != nil { + t.Fatal(err) } - return func() { - if hadHome { - os.Setenv(homeName, oldHome) - } else { - os.Unsetenv(homeName) - } - if hadXDGConfigHome { - os.Setenv("XDG_CONFIG_HOME", oldXDGConfigHome) - } else { - os.Unsetenv("XDG_CONFIG_HOME") - } + f := fn.Function{ + Image: functionImageLocal, + } + + digest, err := pusher.Push(ctx, f) + if err != nil { + t.Fatal(err) + } + + if digest != "sha256:00af51d125f3092e157a7f8a717029412dc9d266c017e89cecdfeccb4cc3d7a7" { + t.Errorf("got bad digest: %q", digest) + } + + authData, err := base64.StdEncoding.DecodeString(optsPassedToMock.RegistryAuth) + if err != nil { + t.Fatal(err) + } + + authStruct := struct { + Username, Password string + }{} + + dec := json.NewDecoder(bytes.NewReader(authData)) + + err = dec.Decode(&authStruct) + if err != nil { + t.Fatal(err) + } + + if imagePassedToMock != functionImageLocal { + t.Errorf("Bad image name passed to the Docker API Client: %q.", imagePassedToMock) + } + + if authStruct.Username != testUser || authStruct.Password != testPwd { + t.Errorf("Bad credentials passed to the Docker API Client: %q:%q", authStruct.Username, authStruct.Password) + } + + if !closeCalledOnMock { + t.Error("The Close() function has not been called on the Docker API Client.") } } -func handlerForCredHelper(t *testing.T, credHelper credentials.Helper) http.Handler { - return http.HandlerFunc(func(writer http.ResponseWriter, request *http.Request) { - defer request.Body.Close() +func TestNonDaemonPush(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*5) + defer cancel() - var err error - var outBody interface{} + // in memory network emulation + connections := conns(make(chan net.Conn)) - uri := strings.Trim(request.RequestURI, "/") + serveRegistry(t, connections) - var serverURL string - if uri == "get" || uri == "erase" { - data, err := ioutil.ReadAll(request.Body) - if err != nil { - writer.WriteHeader(http.StatusInternalServerError) - return - } - serverURL = string(data) - serverURL = strings.Trim(serverURL, "\n\r\t ") - } - - switch uri { - case "list": - var list map[string]string - list, err = credHelper.List() - if err == nil { - outBody = &list - } - case "store": - creds := credentials.Credentials{} - dec := json.NewDecoder(request.Body) - err = dec.Decode(&creds) - if err != nil { - break - } - err = credHelper.Add(&creds) - case "get": - var user, secret string - user, secret, err = credHelper.Get(serverURL) - if err == nil { - outBody = &credentials.Credentials{ServerURL: serverURL, Username: user, Secret: secret} - } - case "erase": - err = credHelper.Delete(serverURL) - default: - writer.WriteHeader(http.StatusNotFound) - return - } - - if err != nil { - if credentials.IsErrCredentialsNotFound(err) { - writer.WriteHeader(http.StatusNotFound) - } else { - writer.WriteHeader(http.StatusInternalServerError) - writer.Header().Add("Content-Type", "text/plain") - fmt.Fprintf(writer, "error: %+v\n", err) - } - return - } - - if outBody != nil { - var data []byte - data, err = json.Marshal(outBody) - if err != nil { - writer.WriteHeader(http.StatusInternalServerError) - return - } - writer.Header().Add("Content-Type", "application/json") - _, err = writer.Write(data) - if err != nil { - t.Fatal(err) - } - } - }) - -} - -// Go source code of mock docker-credential-helper implementation. -// Its storage is backed by inMemoryHelper instantiated in test and exposed via HTTP. -const helperGoSrc = `package main - -import ( - "errors" - "io" - "log" - "net/http" - "os" -) - -func main() { - var ( - baseURL = os.Getenv("HELPER_BASE_URL") - resp *http.Response - err error - ) - cmd := os.Args[1] - switch cmd { - case "list": - resp, err = http.Get(baseURL + "/" + cmd) - if err != nil { - log.Fatal(err) - } - io.Copy(os.Stdout, resp.Body) - case "get", "erase": - resp, err = http.Post(baseURL+ "/" + cmd, "text/plain", os.Stdin) - if err != nil { - log.Fatal(err) - } - io.Copy(os.Stdout, resp.Body) - case "store": - resp, err = http.Post(baseURL+ "/" + cmd, "application/json", os.Stdin) - if err != nil { - log.Fatal(err) - } - default: - log.Fatal(errors.New("unknown cmd: " + cmd)) + transport := http.DefaultTransport.(*http.Transport).Clone() + transport.TLSClientConfig = &tls.Config{ + InsecureSkipVerify: true, } - if resp.StatusCode != http.StatusOK { - log.Fatal(errors.New(resp.Status)) + transport.DialContext = connections.DialContext + + dockerClient := newMockPusherDockerClient() + + var imagesPassedToMock []string + dockerClient.imageSave = func(ctx context.Context, images []string) (io.ReadCloser, error) { + imagesPassedToMock = images + f, err := os.Open("./testData/image.tar") + if err != nil { + return nil, err + } + return f, nil } - return -} -` -// Creates executable with name determined by the helperName parameter and puts it on $PATH. -// -// The executable behaves like docker credential helper (https://github.com/docker/docker-credential-helpers). -// -// The content of the store presented by the executable is backed by the helper parameter. -func setUpMockHelper(helperName string, helper credentials.Helper) func(t *testing.T) func() { - var cleanUps []func() - return func(t *testing.T) func() { + dockerClientFactory := func() (docker.PusherDockerClient, error) { + return dockerClient, nil + } - cleanUps = append(cleanUps, WithExecutable(t, helperName, helperGoSrc)) + pusher, err := docker.NewPusher( + docker.WithTransport(transport), + docker.WithCredentialsProvider(testCredProvider), + docker.WithPusherDockerClientFactory(dockerClientFactory)) + if err != nil { + t.Fatal(err) + } - listener, err := net.Listen("tcp", "localhost:0") + f := fn.Function{ + Image: functionImageRemote, + } + + actualDigest, err := pusher.Push(ctx, f) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(imagesPassedToMock, []string{f.Image}) { + t.Errorf("Bad image name passed to the Docker API Client: %q.", imagesPassedToMock) + } + + r, err := name.NewRegistry(registryHostname) + if err != nil { + t.Fatal(err) + } + + remoteOpts := []remote.Option{ + remote.WithTransport(transport), + remote.WithAuth(&authn.Basic{ + Username: testUser, + Password: testPwd, + }), + } + + c, err := remote.Catalog(ctx, r, remoteOpts...) + if err != nil { + t.Fatal(err) + } + + if !reflect.DeepEqual(c, []string{"testuser/func"}) { + t.Error("unexpected catalog content") + } + + imgRef := name.MustParseReference(functionImageRemote) + + img, err := remote.Image(imgRef, remoteOpts...) + if err != nil { + t.Fatal(err) + } + + expectedDigest, err := img.Digest() + if err != nil { + t.Fatal(err) + } + + if actualDigest != expectedDigest.String() { + t.Error("digest does not match") + } + + layers, err := img.Layers() + if err != nil { + t.Fatal(err) + } + + expectedDiffs := []string{ + "sha256:cba17de713254df68662c399558da08f1cd9abfa4e3b404544757982b4f11597", + "sha256:d5c07940dc570b965530593384a3cf47f47bf07d68eb741d875090392a6331c3", + "sha256:a4dd43077393aff80a0bec5c1bf2db4941d620dcaa545662068476e324efefaa", + "sha256:abe6122e067d0f8bee1a8b8f0c4bb9d2b23cc73617bc8ff4addd6c1329bca23e", + "sha256:515e67f7a08c1798cad8ee4d5f0ce7e606540f5efe5163a967d8dc58994f9641", + "sha256:a5fdabf59fa2a4a0e60d21c1ffc4d482e62debe196bae742a476f4d5b893f0ce", + "sha256:8d6bae166e585b4b503d36a7de0ba749b68ef689884e94dfa0655bbf8ce4d213", + "sha256:b136b7c51981af6493ecbb1136f6ff0f23734f7b9feacb20c8090ac9dec6333d", + "sha256:e9055109e5f7999da5add4b5fff11a34783cddc9aef492d9b790024d1bc1b7d0", + "sha256:5a1ff39e0e0291a43170cbcd70515bfccef4bed4c7e7b97f82d49d3d557fe04b", + } + + actualDiffs := make([]string, 0, len(expectedDiffs)) + + for _, layer := range layers { + diffID, err := layer.DiffID() if err != nil { t.Fatal(err) } + actualDiffs = append(actualDiffs, diffID.String()) + } - cleanUps = append(cleanUps, func() { - _ = listener.Close() - }) - - baseURL := fmt.Sprintf("http://%s", listener.Addr().String()) - cleanUps = append(cleanUps, WithEnvVar(t, "HELPER_BASE_URL", baseURL)) - - server := http.Server{Handler: handlerForCredHelper(t, helper)} - servErrChan := make(chan error) - go func() { - servErrChan <- server.Serve(listener) - }() - - cleanUps = append(cleanUps, func() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - defer cancel() - _ = server.Shutdown(ctx) - e := <-servErrChan - if !errors.Is(e, http.ErrServerClosed) { - t.Fatal(e) - } - }) - - return func() { - for i := len(cleanUps) - 1; i <= 0; i-- { - cleanUps[i]() - } - } + if !reflect.DeepEqual(expectedDiffs, actualDiffs) { + t.Error("layer diffs in tar and from registry differs") } } -// combines multiple setUp routines into one setUp routine -func all(fns ...setUpEnv) setUpEnv { - return func(t *testing.T) func() { - t.Helper() - var cleanUps []func() - for _, fn := range fns { - cleanUps = append(cleanUps, fn(t)) - } - - return func() { - for i := len(cleanUps) - 1; i >= 0; i-- { - cleanUps[i]() - } - } +func newMockPusherDockerClient() *mockPusherDockerClient { + return &mockPusherDockerClient{ + negotiateAPIVersion: func(ctx context.Context) {}, + close: func() error { return nil }, } } -func newInMemoryHelper() *inMemoryHelper { - return &inMemoryHelper{lock: &sync.Mutex{}, credentials: make(map[string]credentials.Credentials)} +type mockPusherDockerClient struct { + negotiateAPIVersion func(ctx context.Context) + imagePush func(ctx context.Context, ref string, options types.ImagePushOptions) (io.ReadCloser, error) + imageSave func(ctx context.Context, strings []string) (io.ReadCloser, error) + close func() error } -type inMemoryHelper struct { - credentials map[string]credentials.Credentials - lock sync.Locker +func (m *mockPusherDockerClient) NegotiateAPIVersion(ctx context.Context) { + m.negotiateAPIVersion(ctx) } -func (i *inMemoryHelper) Add(credentials *credentials.Credentials) error { - i.lock.Lock() - defer i.lock.Unlock() - - i.credentials[credentials.ServerURL] = *credentials - - return nil +func (m *mockPusherDockerClient) ImageSave(ctx context.Context, strings []string) (io.ReadCloser, error) { + return m.imageSave(ctx, strings) } -func (i *inMemoryHelper) Get(serverURL string) (string, string, error) { - i.lock.Lock() - defer i.lock.Unlock() - - if result, ok := i.credentials[serverURL]; ok { - return result.Username, result.Secret, nil - } - - return "", "", credentials.NewErrCredentialsNotFound() +func (m *mockPusherDockerClient) ImageLoad(ctx context.Context, reader io.Reader, b bool) (types.ImageLoadResponse, error) { + panic("implement me") } -func (i *inMemoryHelper) List() (map[string]string, error) { - i.lock.Lock() - defer i.lock.Unlock() - - result := make(map[string]string, len(i.credentials)) - - for k, v := range i.credentials { - result[k] = v.Username - } - - return result, nil +func (m *mockPusherDockerClient) ImageTag(ctx context.Context, s string, s2 string) error { + panic("implement me") } -func (i *inMemoryHelper) Delete(serverURL string) error { - i.lock.Lock() - defer i.lock.Unlock() - - if _, ok := i.credentials[serverURL]; ok { - delete(i.credentials, serverURL) - return nil - } - - return credentials.NewErrCredentialsNotFound() +func (m *mockPusherDockerClient) ImagePush(ctx context.Context, ref string, options types.ImagePushOptions) (io.ReadCloser, error) { + return m.imagePush(ctx, ref, options) } -func TestCheckAuth(t *testing.T) { - localhost, localhostTLS, stopServer := startServer(t) - defer stopServer() +func (m *mockPusherDockerClient) Close() error { + return m.close() +} - _, portTLS, err := net.SplitHostPort(localhostTLS) - if err != nil { - t.Fatal(err) - } - - nonLocalhostTLS := "test.io:" + portTLS - - type args struct { - ctx context.Context - username string - password string - registry string - } - tests := []struct { - name string - args args - wantErr bool - }{ - { - name: "correct credentials localhost no-TLS", - args: args{ - ctx: context.Background(), - username: "testuser", - password: "testpwd", - registry: localhost, - }, - wantErr: false, - }, - { - name: "correct credentials localhost", - args: args{ - ctx: context.Background(), - username: "testuser", - password: "testpwd", - registry: localhostTLS, - }, - wantErr: false, - }, - - { - name: "correct credentials non-localhost", - args: args{ - ctx: context.Background(), - username: "testuser", - password: "testpwd", - registry: nonLocalhostTLS, - }, - wantErr: false, - }, - { - name: "incorrect credentials localhost no-TLS", - args: args{ - ctx: context.Background(), - username: "testuser", - password: "badpwd", - registry: localhost, - }, - wantErr: true, - }, - { - name: "incorrect credentials localhost", - args: args{ - ctx: context.Background(), - username: "testuser", - password: "badpwd", - registry: localhostTLS, - }, - wantErr: true, - }, - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - creds := Credentials{ - Username: tt.args.username, - Password: tt.args.password, - } - if err := CheckAuth(tt.args.ctx, tt.args.registry, creds); (err != nil) != tt.wantErr { - t.Errorf("CheckAuth() error = %v, wantErr %v", err, tt.wantErr) - } - }) - } -} - -func startServer(t *testing.T) (addr, addrTLS string, stopServer func()) { - listener, err := net.Listen("tcp", "localhost:8080") - if err != nil { - t.Fatal(err) - } - addr = listener.Addr().String() - - listenerTLS, err := net.Listen("tcp", "localhost:4433") - if err != nil { - t.Fatal(err) - } - addrTLS = listenerTLS.Addr().String() - - handler := http.HandlerFunc(func(resp http.ResponseWriter, req *http.Request) { - resp.Header().Add("WWW-Authenticate", "basic") - if user, pwd, ok := req.BasicAuth(); ok { - if user == "testuser" && pwd == "testpwd" { - resp.WriteHeader(http.StatusOK) - return - } - } - resp.WriteHeader(http.StatusUnauthorized) - }) - - var randReader io.Reader = rand.Reader - - caPublicKey, caPrivateKey, err := ed25519.GenerateKey(randReader) +func serveRegistry(t *testing.T, l net.Listener) { + + caPrivateKey, err := ecdsa.GenerateKey(elliptic.P256(), rand.Reader) if err != nil { t.Fatal(err) } + caPublicKey := &caPrivateKey.PublicKey ca := &x509.Certificate{ SerialNumber: big.NewInt(1), Subject: pkix.Name{ - CommonName: "localhost", + CommonName: registryHostname, }, - IPAddresses: []net.IP{net.IPv4(127, 0, 0, 1), net.IPv6loopback}, - DNSNames: []string{"localhost", "test.io"}, + DNSNames: []string{registryHostname}, NotBefore: time.Now(), - NotAfter: time.Now().AddDate(10, 0, 0), + NotAfter: time.Now().Add(time.Minute * 10), IsCA: true, ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageClientAuth, x509.ExtKeyUsageServerAuth}, ExtraExtensions: []pkix.Extension{}, @@ -818,7 +343,7 @@ func startServer(t *testing.T) (addr, addrTLS string, stopServer func()) { BasicConstraintsValid: true, } - caBytes, err := x509.CreateCertificate(randReader, ca, ca, caPublicKey, caPrivateKey) + caBytes, err := x509.CreateCertificate(rand.Reader, ca, ca, caPublicKey, caPrivateKey) if err != nil { t.Fatal(err) } @@ -835,52 +360,122 @@ func startServer(t *testing.T) (addr, addrTLS string, stopServer func()) { } server := http.Server{ - Handler: handler, + Handler: withAuth(registry.New( + registry.Logger(log.New(io.Discard, "", 0)))), TLSConfig: &tls.Config{ - ServerName: "localhost", + ServerName: registryHostname, Certificates: []tls.Certificate{cert}, }, + // The line below disables HTTP/2. + // See: https://github.com/google/go-containerregistry/issues/1210 + TLSNextProto: make(map[string]func(*http.Server, *tls.Conn, http.Handler)), } - go func() { - err := server.ServeTLS(listenerTLS, "", "") - if err != nil && !strings.Contains(err.Error(), "Server closed") { - panic(err) - } + _ = server.ServeTLS(l, "", "") }() - - go func() { - err := server.Serve(listener) - if err != nil && !strings.Contains(err.Error(), "Server closed") { - panic(err) - } - }() - - // make the testing CA trusted by default HTTP transport/client - oldDefaultTransport := http.DefaultTransport - newDefaultTransport := http.DefaultTransport.(*http.Transport).Clone() - http.DefaultTransport = newDefaultTransport - caPool := x509.NewCertPool() - caPool.AddCert(ca) - newDefaultTransport.TLSClientConfig.RootCAs = caPool - dc := newDefaultTransport.DialContext - newDefaultTransport.DialContext = func(ctx context.Context, network, addr string) (net.Conn, error) { - h, p, err := net.SplitHostPort(addr) - if err != nil { - return nil, err - } - if h == "test.io" { - h = "localhost" - } - addr = net.JoinHostPort(h, p) - return dc(ctx, network, addr) - } - - return addr, addrTLS, func() { - err := server.Shutdown(context.Background()) - if err != nil { - t.Fatal(err) - } - http.DefaultTransport = oldDefaultTransport - } + t.Cleanup(func() { + server.Close() + }) } + +// middleware for basic auth +func withAuth(h http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + user, pass, ok := r.BasicAuth() + if ok && user == testUser && pass == testPwd { + h.ServeHTTP(w, r) + return + } + w.Header().Add("WWW-Authenticate", "basic") + w.WriteHeader(401) + fmt.Fprintln(w, "Unauthorised.") + }) +} + +type conns chan net.Conn + +func (c conns) DialContext(ctx context.Context, network, addr string) (net.Conn, error) { + if addr == registryHostname+":443" { + + pr0, pw0 := io.Pipe() + pr1, pw1 := io.Pipe() + + c <- conn{pr0, pw1} + + return conn{pr1, pw0}, nil + } + return (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext(ctx, network, addr) +} + +func (c conns) Accept() (net.Conn, error) { + con, ok := <-c + if !ok { + return nil, net.ErrClosed + } + return con, nil +} + +func (c conns) Close() error { + close(c) + return nil +} + +func (c conns) Addr() net.Addr { + return addr{} +} + +type conn struct { + pr *io.PipeReader + pw *io.PipeWriter +} + +type addr struct{} + +func (a addr) Network() string { + return "mock-addr" +} + +func (a addr) String() string { + return "mock-addr" +} + +func (c conn) Read(b []byte) (n int, err error) { + return c.pr.Read(b) +} + +func (c conn) Write(b []byte) (n int, err error) { + return c.pw.Write(b) +} + +func (c conn) Close() error { + var err error + + err = c.pr.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "err: %v\n", err) + } + + err = c.pw.Close() + if err != nil { + fmt.Fprintf(os.Stderr, "err: %v\n", err) + } + + return nil +} + +func (c conn) LocalAddr() net.Addr { + return addr{} +} + +func (c conn) RemoteAddr() net.Addr { + return addr{} +} + +func (c conn) SetDeadline(t time.Time) error { return nil } + +func (c conn) SetReadDeadline(t time.Time) error { return nil } + +func (c conn) SetWriteDeadline(t time.Time) error { return nil } diff --git a/docker/testData/image.tar b/docker/testData/image.tar new file mode 100644 index 0000000000000000000000000000000000000000..ffbd0128e973c3babd5381c20cc08cf60cfa686b GIT binary patch literal 53248 zcmeHQZHpVp5%y16lrIVRwEh0FkPqQFNFd~n%MnNpWBR4qR1 z9%-*<#u~209kSG#p5ChJu4lTdo=P!ro5^x1w93UeV?>hPYwdDiuJDw7h$SeJQ8A^J zvf6qlqAHwgZ~5;d%j%k5GX8RT`|Isk_hg(g^JV?-#KslZmMg)fe8#xsMoz5%&CN}(0FJfq zv_C)nkEQe0Z<9ci!1+rc>%^>AEGKPQ^hRsRw3pJUAX8Bm|HZ;=HjbiIJUgqIXVQyc zq~uACCSLz*RW;M~fA9ar`T75;^?%I2A8Vhf{%g&?UEd_oBrsnI1eLRr%v$MX zD8;eBA|;)8EFwqc;qr|DItW24jlBV8oU(?bP*kXdFp~ebMmPUIg;nkSzdS$xKehgk z`S)Y(&HqnfuI(O80!;z~N+3isTnJ*uW>#Dz;=x5%SW+?iETWJpS!uY9$(5o)&ef$VPE4+Hh91UhqD6by(-=)K92uqI18S5{jXKeOxORt z|JUc||Lw&3Kjz<$wa--lwPxS0ZxUz{n6CtaE*bHC+Ls`L3Je}U3!Zo>NoC~(8IdUx z_a+9i3cs?9DmG|?u_z2l)6Y&i_2--;cG=RR6VR z->z>GXcCyO1jsY3IrF58_1@>85#fhzfKrg(AEdHQrks-xRFoIlN9!3AD)WdG0fY>W z<^RRh>wm2ZXS)9H{l7my|35YUJLcbywa--lwPxS0ZxUz{n6CtMz)`>S#E1#cgOc2` z6jQbgk%6(K1iv3Ct7HkA0Jb=hmCrJ3Z3{6KNAmw%PksJdtHPPC|2zNB+1dI3#{W}m zknQ>=fhK__fqn__5)Cg1w_^-cKg7uyhcrN8O3v0=#tp*!aUg)u02UD)h?Kt)KoCe# zvdjf7|GT|e_upbiiPyvbhmZqvPn*|y(PvC>&RhK75vdN-{C<%(6mBUmt`-7<|Avci zxc!OSE8(vsUm7PhGkE=H@q7_B>GkTC((T=b@cZy48-=fLw)k+pTHpPO59wxIR<9OU z?-$>#e_34py!hcSKmGO3mv`IE<@HSp*O%M4T3 zZQd_dH_Ldpy3XPC@)z1{S2t_ioG*o3^2LV_&llZudfEN*_TAfVl-J=Meut6eRVgo5 z8Ero=wy%TG<|oc8F;yUeeq4Y z_`@I9H*f!R@$L6N{QIJNZn*N;0Hz>#Fu_L^oOLclmO&JQ?PH26rznL`Hs_dw0d*Xl z2*SvML%^AEie0SM7nc`{56}CDrJlANK?9L54)LGdv%mxi9DyiA_>w`CN?rl0d7O99MuCBr#IcrP&g;*LXN_c%mDg;7GiE+QOogNKA@s zB*J(VcsspCPOn7#P8Z_?BpwsXp0iYZK%}k2VjU?ar2RyPA)EW*z4$N;#%#TuuLOiL zQ6(%O(#41wrtzqUwK8|>dEqv3te;lNOacxVC&GE>EkO{EWkX=o_;LrY`*(q1#og=v z8SnG=$8Gz6gITsa{}s#SWi3MQ9~t!WJLi9|*8ji0`|+QD`~Dw&7Tzhf=lLJ4;Cz30 z;rSmPXKMb}zCYlUw!`@E|BzEIIfviC1)jq&+8Dl)1ghtM{r(@GPWb(_>1I(k{|(++ z{f`N30vl?(t-nebuxLDS<)0S^r1 zu-${b6Fj;W?37R{s0c6;{sREmluU%3dpO~h3pl<~ zih=goDbt;Y;V4WBXTbAhGWPu=@X|Q$7Dj!G8!loSOfoR(~^n{yX5mS~GZz|EjTM z+e@r9#?!AS%3Ymr)cqibGT%abt`)dI5YdBZ5X1!Fe^diYG6E%9O^8|2D9;T>a0J_k z$j8BdxCiP#?C@V@3}eH8{kBl8|55NCs)4ulzh7|!wKn`WkRWH;P4)aA1^;oOrq2Jb z)!)4FU#%HD#(&jVvf;m4V?6zOME@z2cSF#=MWkvrgt%lQ5GraAbql0NF))M$DpQ0( z7_dT#7%;DSyatquPTBgeV+ zQa%4i!GFxM#{V^mNgek8YR%v={;S544gXD|`rS=5^3(`(yL|48_6a{jMc70rA9uhtA67#+(FF z)fLD*QUGlNW0rIPg#kE#I$r1^P$b%wE?b0kC&ND3vG5<#`}g>-mgS}G|MlBKwf@J! zf1tEz_^)4c!?ZU1H;f$T+DrBP9|iw$Zrc9;Bqnv(|Eo2F$M~-rOE&yBiRyPZ0m)Y) z#PR72QX??DD3zts3EWf$1X{?$M?MI5sQ7`C-;@w42#zuW5ll3w3W72rCpX zQOnKO@L#_zRqKBo{0G{rhX49CH%x28f5XUeuDw*x|55NCicGioze!B$fd6XE;4%KI z#*z*HO``hUOIAK1m)^WP)} zb)WxR2?HMEziKSm@ZTh=-`&I%(3ks+g1!O;98d{ZC88oZ%Ku;s5F3ClgYCr^Fu&Uv zON^eEpp;KwVYk7L#Q&_Y0k}W^t*wL6@L#_zRqKBo{D%nCw*S|!xnWux{u@S)bM2*i z{ttlvP<-HX{y$XGX!Cyxn|lAHqHbTijvMZO8&+%DEUwmf*Vntka{Dh{-Vg2Y{pt1V z@=f|5ZC-w?8@KOZM}S>B4*-BaDTDWs4YBqCYh*UWH(|Xhw7os0j^l5&$1XIsr~U)i zAp1Rp&vtLmKr)CLSZ)6QR(U_=I#h=H_E52LF-R>?OoSil?afc&)vnOun|I4wnAd9k zYPq3Ts7JSXciDY$x24Txxm%~JYgC`y-tLP$?kY8+JmU~mUoem%$ivai5FC_UHv>=^ zp_?Hx;=9odkXVIohKSEaH$#MZpqn913((CF51lB$_~I!O@hkmygSon0p-SY6wqp&U zThx2j|1Rh7^Y!28#{WK+IzH+5{o?2f-BsN^EYeqtYMk_;?e2Y|fcBwDph=)f;44Ys E|4TqSBme*a literal 0 HcmV?d00001 diff --git a/http/transport.go b/http/transport.go index 7ac5acad0..690003034 100644 --- a/http/transport.go +++ b/http/transport.go @@ -22,14 +22,23 @@ type RoundTripCloser interface { // // This is useful for accessing cluster internal services (pushing a CloudEvent into Knative broker). func NewRoundTripper() RoundTripCloser { - d := &dialer{ - netDialer: net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, - }, - } - return &roundTripCloser{ - Transport: http.Transport{ + result := &roundTripCloser{} + if dt, ok := http.DefaultTransport.(*http.Transport); ok { + d := &dialer{ + defaultDialContext: dt.DialContext, + } + result.d = d + result.Transport = dt.Clone() + result.Transport.DialContext = d.DialContext + } else { + d := &dialer{ + defaultDialContext: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).DialContext, + } + result.d = d + result.Transport = &http.Transport{ Proxy: http.ProxyFromEnvironment, DialContext: d.DialContext, ForceAttemptHTTP2: true, @@ -37,13 +46,14 @@ func NewRoundTripper() RoundTripCloser { IdleConnTimeout: 90 * time.Second, TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, - }, - d: d, + } } + + return result } type roundTripCloser struct { - http.Transport + *http.Transport d *dialer } @@ -52,13 +62,13 @@ func (r *roundTripCloser) Close() error { } type dialer struct { - o sync.Once - netDialer net.Dialer - inClusterDialer k8s.ContextDialer + o sync.Once + defaultDialContext func(ctx context.Context, network, address string) (net.Conn, error) + inClusterDialer k8s.ContextDialer } func (d *dialer) DialContext(ctx context.Context, network, address string) (net.Conn, error) { - conn, err := d.netDialer.DialContext(ctx, network, address) + conn, err := d.defaultDialContext(ctx, network, address) if err == nil { return conn, nil } diff --git a/vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go b/vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go new file mode 100644 index 000000000..85b171907 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/internal/httptest/httptest.go @@ -0,0 +1,104 @@ +// Copyright 2020 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package httptest provides a method for testing a TLS server a la net/http/httptest. +package httptest + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "encoding/pem" + "math/big" + "net" + "net/http" + "net/http/httptest" + "time" +) + +// NewTLSServer returns an httptest server, with an http client that has been configured to +// send all requests to the returned server. The TLS certs are generated for the given domain. +// If you need a transport, Client().Transport is correctly configured. +func NewTLSServer(domain string, handler http.Handler) (*httptest.Server, error) { + s := httptest.NewUnstartedServer(handler) + + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + NotBefore: time.Now().Add(-1 * time.Hour), + NotAfter: time.Now().Add(time.Hour), + IPAddresses: []net.IP{ + net.IPv4(127, 0, 0, 1), + net.IPv6loopback, + }, + DNSNames: []string{domain}, + + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + } + + priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader) + if err != nil { + return nil, err + } + + b, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + return nil, err + } + + pc := &bytes.Buffer{} + if err := pem.Encode(pc, &pem.Block{Type: "CERTIFICATE", Bytes: b}); err != nil { + return nil, err + } + + ek, err := x509.MarshalECPrivateKey(priv) + if err != nil { + return nil, err + } + + pk := &bytes.Buffer{} + if err := pem.Encode(pk, &pem.Block{Type: "EC PRIVATE KEY", Bytes: ek}); err != nil { + return nil, err + } + + c, err := tls.X509KeyPair(pc.Bytes(), pk.Bytes()) + if err != nil { + return nil, err + } + s.TLS = &tls.Config{ + Certificates: []tls.Certificate{c}, + } + s.StartTLS() + + certpool := x509.NewCertPool() + certpool.AddCert(s.Certificate()) + + t := &http.Transport{ + TLSClientConfig: &tls.Config{ + RootCAs: certpool, + }, + DialContext: func(ctx context.Context, network, addr string) (net.Conn, error) { + return net.Dial(s.Listener.Addr().Network(), s.Listener.Addr().String()) + }, + } + s.Client().Transport = t + + return s, nil +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go b/vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go new file mode 100644 index 000000000..978ff4803 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/blobs.go @@ -0,0 +1,224 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "fmt" + "io" + "math/rand" + "net/http" + "path" + "strings" + "sync" +) + +// Returns whether this url should be handled by the blob handler +// This is complicated because blob is indicated by the trailing path, not the leading path. +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pulling-a-layer +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pushing-a-layer +func isBlob(req *http.Request) bool { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + if elem[len(elem)-1] == "" { + elem = elem[:len(elem)-1] + } + if len(elem) < 3 { + return false + } + return elem[len(elem)-2] == "blobs" || (elem[len(elem)-3] == "blobs" && + elem[len(elem)-2] == "uploads") +} + +// blobs +type blobs struct { + // Blobs are content addresses. we store them globally underneath their sha and make no distinctions per image. + contents map[string][]byte + // Each upload gets a unique id that writes occur to until finalized. + uploads map[string][]byte + lock sync.Mutex +} + +func (b *blobs) handle(resp http.ResponseWriter, req *http.Request) *regError { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + if elem[len(elem)-1] == "" { + elem = elem[:len(elem)-1] + } + // Must have a path of form /v2/{name}/blobs/{upload,sha256:} + if len(elem) < 4 { + return ®Error{ + Status: http.StatusBadRequest, + Code: "NAME_INVALID", + Message: "blobs must be attached to a repo", + } + } + target := elem[len(elem)-1] + service := elem[len(elem)-2] + digest := req.URL.Query().Get("digest") + contentRange := req.Header.Get("Content-Range") + + if req.Method == "HEAD" { + b.lock.Lock() + defer b.lock.Unlock() + b, ok := b.contents[target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "BLOB_UNKNOWN", + Message: "Unknown blob", + } + } + + resp.Header().Set("Content-Length", fmt.Sprint(len(b))) + resp.Header().Set("Docker-Content-Digest", target) + resp.WriteHeader(http.StatusOK) + return nil + } + + if req.Method == "GET" { + b.lock.Lock() + defer b.lock.Unlock() + b, ok := b.contents[target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "BLOB_UNKNOWN", + Message: "Unknown blob", + } + } + + resp.Header().Set("Content-Length", fmt.Sprint(len(b))) + resp.Header().Set("Docker-Content-Digest", target) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader(b)) + return nil + } + + if req.Method == "POST" && target == "uploads" && digest != "" { + l := &bytes.Buffer{} + io.Copy(l, req.Body) + rd := sha256.Sum256(l.Bytes()) + d := "sha256:" + hex.EncodeToString(rd[:]) + if d != digest { + return ®Error{ + Status: http.StatusBadRequest, + Code: "DIGEST_INVALID", + Message: "digest does not match contents", + } + } + + b.lock.Lock() + defer b.lock.Unlock() + b.contents[d] = l.Bytes() + resp.Header().Set("Docker-Content-Digest", d) + resp.WriteHeader(http.StatusCreated) + return nil + } + + if req.Method == "POST" && target == "uploads" && digest == "" { + id := fmt.Sprint(rand.Int63()) + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-2]...), "blobs/uploads", id)) + resp.Header().Set("Range", "0-0") + resp.WriteHeader(http.StatusAccepted) + return nil + } + + if req.Method == "PATCH" && service == "uploads" && contentRange != "" { + start, end := 0, 0 + if _, err := fmt.Sscanf(contentRange, "%d-%d", &start, &end); err != nil { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UPLOAD_UNKNOWN", + Message: "We don't understand your Content-Range", + } + } + b.lock.Lock() + defer b.lock.Unlock() + if start != len(b.uploads[target]) { + return ®Error{ + Status: http.StatusRequestedRangeNotSatisfiable, + Code: "BLOB_UPLOAD_UNKNOWN", + Message: "Your content range doesn't match what we have", + } + } + l := bytes.NewBuffer(b.uploads[target]) + io.Copy(l, req.Body) + b.uploads[target] = l.Bytes() + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target)) + resp.Header().Set("Range", fmt.Sprintf("0-%d", len(l.Bytes())-1)) + resp.WriteHeader(http.StatusNoContent) + return nil + } + + if req.Method == "PATCH" && service == "uploads" && contentRange == "" { + b.lock.Lock() + defer b.lock.Unlock() + if _, ok := b.uploads[target]; ok { + return ®Error{ + Status: http.StatusBadRequest, + Code: "BLOB_UPLOAD_INVALID", + Message: "Stream uploads after first write are not allowed", + } + } + + l := &bytes.Buffer{} + io.Copy(l, req.Body) + + b.uploads[target] = l.Bytes() + resp.Header().Set("Location", "/"+path.Join("v2", path.Join(elem[1:len(elem)-3]...), "blobs/uploads", target)) + resp.Header().Set("Range", fmt.Sprintf("0-%d", len(l.Bytes())-1)) + resp.WriteHeader(http.StatusNoContent) + return nil + } + + if req.Method == "PUT" && service == "uploads" && digest == "" { + return ®Error{ + Status: http.StatusBadRequest, + Code: "DIGEST_INVALID", + Message: "digest not specified", + } + } + + if req.Method == "PUT" && service == "uploads" && digest != "" { + b.lock.Lock() + defer b.lock.Unlock() + l := bytes.NewBuffer(b.uploads[target]) + io.Copy(l, req.Body) + rd := sha256.Sum256(l.Bytes()) + d := "sha256:" + hex.EncodeToString(rd[:]) + if d != digest { + return ®Error{ + Status: http.StatusBadRequest, + Code: "DIGEST_INVALID", + Message: "digest does not match contents", + } + } + + b.contents[d] = l.Bytes() + delete(b.uploads, target) + resp.Header().Set("Docker-Content-Digest", d) + resp.WriteHeader(http.StatusCreated) + return nil + } + + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/error.go b/vendor/github.com/google/go-containerregistry/pkg/registry/error.go new file mode 100644 index 000000000..64e98671c --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/error.go @@ -0,0 +1,46 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "encoding/json" + "net/http" +) + +type regError struct { + Status int + Code string + Message string +} + +func (r *regError) Write(resp http.ResponseWriter) error { + resp.WriteHeader(r.Status) + + type err struct { + Code string `json:"code"` + Message string `json:"message"` + } + type wrap struct { + Errors []err `json:"errors"` + } + return json.NewEncoder(resp).Encode(wrap{ + Errors: []err{ + { + Code: r.Code, + Message: r.Message, + }, + }, + }) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go b/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go new file mode 100644 index 000000000..5bb4fe899 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/manifest.go @@ -0,0 +1,334 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "encoding/json" + "fmt" + "io" + "log" + "net/http" + "sort" + "strconv" + "strings" + "sync" + + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/types" +) + +type catalog struct { + Repos []string `json:"repositories"` +} + +type listTags struct { + Name string `json:"name"` + Tags []string `json:"tags"` +} + +type manifest struct { + contentType string + blob []byte +} + +type manifests struct { + // maps repo -> manifest tag/digest -> manifest + manifests map[string]map[string]manifest + lock sync.Mutex + log *log.Logger +} + +func isManifest(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 4 { + return false + } + return elems[len(elems)-2] == "manifests" +} + +func isTags(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 4 { + return false + } + return elems[len(elems)-2] == "tags" +} + +func isCatalog(req *http.Request) bool { + elems := strings.Split(req.URL.Path, "/") + elems = elems[1:] + if len(elems) < 2 { + return false + } + + return elems[len(elems)-1] == "_catalog" +} + +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pulling-an-image-manifest +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#pushing-an-image +func (m *manifests) handle(resp http.ResponseWriter, req *http.Request) *regError { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + target := elem[len(elem)-1] + repo := strings.Join(elem[1:len(elem)-2], "/") + + if req.Method == "GET" { + m.lock.Lock() + defer m.lock.Unlock() + + c, ok := m.manifests[repo] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + m, ok := c[target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: "Unknown manifest", + } + } + rd := sha256.Sum256(m.blob) + d := "sha256:" + hex.EncodeToString(rd[:]) + resp.Header().Set("Docker-Content-Digest", d) + resp.Header().Set("Content-Type", m.contentType) + resp.Header().Set("Content-Length", fmt.Sprint(len(m.blob))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader(m.blob)) + return nil + } + + if req.Method == "HEAD" { + m.lock.Lock() + defer m.lock.Unlock() + if _, ok := m.manifests[repo]; !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + m, ok := m.manifests[repo][target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: "Unknown manifest", + } + } + rd := sha256.Sum256(m.blob) + d := "sha256:" + hex.EncodeToString(rd[:]) + resp.Header().Set("Docker-Content-Digest", d) + resp.Header().Set("Content-Type", m.contentType) + resp.Header().Set("Content-Length", fmt.Sprint(len(m.blob))) + resp.WriteHeader(http.StatusOK) + return nil + } + + if req.Method == "PUT" { + m.lock.Lock() + defer m.lock.Unlock() + if _, ok := m.manifests[repo]; !ok { + m.manifests[repo] = map[string]manifest{} + } + b := &bytes.Buffer{} + io.Copy(b, req.Body) + rd := sha256.Sum256(b.Bytes()) + digest := "sha256:" + hex.EncodeToString(rd[:]) + mf := manifest{ + blob: b.Bytes(), + contentType: req.Header.Get("Content-Type"), + } + + // If the manifest is a manifest list, check that the manifest + // list's constituent manifests are already uploaded. + // This isn't strictly required by the registry API, but some + // registries require this. + if types.MediaType(mf.contentType).IsIndex() { + im, err := v1.ParseIndexManifest(b) + if err != nil { + return ®Error{ + Status: http.StatusBadRequest, + Code: "MANIFEST_INVALID", + Message: err.Error(), + } + } + for _, desc := range im.Manifests { + if !desc.MediaType.IsDistributable() { + continue + } + if desc.MediaType.IsIndex() || desc.MediaType.IsImage() { + if _, found := m.manifests[repo][desc.Digest.String()]; !found { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: fmt.Sprintf("Sub-manifest %q not found", desc.Digest), + } + } + } else { + // TODO: Probably want to do an existence check for blobs. + m.log.Printf("TODO: Check blobs for %q", desc.Digest) + } + } + } + + // Allow future references by target (tag) and immutable digest. + // See https://docs.docker.com/engine/reference/commandline/pull/#pull-an-image-by-digest-immutable-identifier. + m.manifests[repo][target] = mf + m.manifests[repo][digest] = mf + resp.Header().Set("Docker-Content-Digest", digest) + resp.WriteHeader(http.StatusCreated) + return nil + } + + if req.Method == "DELETE" { + m.lock.Lock() + defer m.lock.Unlock() + if _, ok := m.manifests[repo]; !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + + _, ok := m.manifests[repo][target] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "MANIFEST_UNKNOWN", + Message: "Unknown manifest", + } + } + + delete(m.manifests[repo], target) + resp.WriteHeader(http.StatusAccepted) + return nil + } + + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } +} + +func (m *manifests) handleTags(resp http.ResponseWriter, req *http.Request) *regError { + elem := strings.Split(req.URL.Path, "/") + elem = elem[1:] + repo := strings.Join(elem[1:len(elem)-2], "/") + query := req.URL.Query() + nStr := query.Get("n") + n := 1000 + if nStr != "" { + n, _ = strconv.Atoi(nStr) + } + + if req.Method == "GET" { + m.lock.Lock() + defer m.lock.Unlock() + + c, ok := m.manifests[repo] + if !ok { + return ®Error{ + Status: http.StatusNotFound, + Code: "NAME_UNKNOWN", + Message: "Unknown name", + } + } + + var tags []string + countTags := 0 + // TODO: implement pagination https://github.com/opencontainers/distribution-spec/blob/b505e9cc53ec499edbd9c1be32298388921bb705/detail.md#tags-paginated + for tag := range c { + if countTags >= n { + break + } + countTags++ + if !strings.Contains(tag, "sha256:") { + tags = append(tags, tag) + } + } + sort.Strings(tags) + + tagsToList := listTags{ + Name: repo, + Tags: tags, + } + + msg, _ := json.Marshal(tagsToList) + resp.Header().Set("Content-Length", fmt.Sprint(len(msg))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader([]byte(msg))) + return nil + } + + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } +} + +func (m *manifests) handleCatalog(resp http.ResponseWriter, req *http.Request) *regError { + query := req.URL.Query() + nStr := query.Get("n") + n := 10000 + if nStr != "" { + n, _ = strconv.Atoi(nStr) + } + + if req.Method == "GET" { + m.lock.Lock() + defer m.lock.Unlock() + + var repos []string + countRepos := 0 + // TODO: implement pagination + for key := range m.manifests { + if countRepos >= n { + break + } + countRepos++ + + repos = append(repos, key) + } + + repositoriesToList := catalog{ + Repos: repos, + } + + msg, _ := json.Marshal(repositoriesToList) + resp.Header().Set("Content-Length", fmt.Sprint(len(msg))) + resp.WriteHeader(http.StatusOK) + io.Copy(resp, bytes.NewReader([]byte(msg))) + return nil + } + + return ®Error{ + Status: http.StatusBadRequest, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go b/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go new file mode 100644 index 000000000..c56dae26d --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/registry.go @@ -0,0 +1,104 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package registry implements a docker V2 registry and the OCI distribution specification. +// +// It is designed to be used anywhere a low dependency container registry is needed, with an +// initial focus on tests. +// +// Its goal is to be standards compliant and its strictness will increase over time. +// +// This is currently a low flightmiles system. It's likely quite safe to use in tests; If you're using it +// in production, please let us know how and send us CL's for integration tests. +package registry + +import ( + "log" + "net/http" + "os" +) + +type registry struct { + log *log.Logger + blobs blobs + manifests manifests +} + +// https://docs.docker.com/registry/spec/api/#api-version-check +// https://github.com/opencontainers/distribution-spec/blob/master/spec.md#api-version-check +func (r *registry) v2(resp http.ResponseWriter, req *http.Request) *regError { + if isBlob(req) { + return r.blobs.handle(resp, req) + } + if isManifest(req) { + return r.manifests.handle(resp, req) + } + if isTags(req) { + return r.manifests.handleTags(resp, req) + } + if isCatalog(req) { + return r.manifests.handleCatalog(resp, req) + } + resp.Header().Set("Docker-Distribution-API-Version", "registry/2.0") + if req.URL.Path != "/v2/" && req.URL.Path != "/v2" { + return ®Error{ + Status: http.StatusNotFound, + Code: "METHOD_UNKNOWN", + Message: "We don't understand your method + url", + } + } + resp.WriteHeader(200) + return nil +} + +func (r *registry) root(resp http.ResponseWriter, req *http.Request) { + if rerr := r.v2(resp, req); rerr != nil { + r.log.Printf("%s %s %d %s %s", req.Method, req.URL, rerr.Status, rerr.Code, rerr.Message) + rerr.Write(resp) + return + } + r.log.Printf("%s %s", req.Method, req.URL) +} + +// New returns a handler which implements the docker registry protocol. +// It should be registered at the site root. +func New(opts ...Option) http.Handler { + r := ®istry{ + log: log.New(os.Stderr, "", log.LstdFlags), + blobs: blobs{ + contents: map[string][]byte{}, + uploads: map[string][]byte{}, + }, + manifests: manifests{ + manifests: map[string]map[string]manifest{}, + log: log.New(os.Stderr, "", log.LstdFlags), + }, + } + for _, o := range opts { + o(r) + } + return http.HandlerFunc(r.root) +} + +// Option describes the available options +// for creating the registry. +type Option func(r *registry) + +// Logger overrides the logger used to record requests to the registry. +func Logger(l *log.Logger) Option { + return func(r *registry) { + r.log = l + r.manifests.log = l + } +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go b/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go new file mode 100644 index 000000000..cb2644e61 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/registry/tls.go @@ -0,0 +1,29 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package registry + +import ( + "net/http/httptest" + + ggcrtest "github.com/google/go-containerregistry/internal/httptest" +) + +// TLS returns an httptest server, with an http client that has been configured to +// send all requests to the returned server. The TLS certs are generated for the given domain +// which should correspond to the domain the image is stored in. +// If you need a transport, Client().Transport is correctly configured. +func TLS(domain string) (*httptest.Server, error) { + return ggcrtest.NewTLSServer(domain, New()) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/README.md b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/README.md new file mode 100644 index 000000000..74fc3a87c --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/README.md @@ -0,0 +1,11 @@ +# `daemon` + +[![GoDoc](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/daemon?status.svg)](https://godoc.org/github.com/google/go-containerregistry/pkg/v1/daemon) + +The `daemon` package enables reading/writing images from/to the docker daemon. + +It is not fully fleshed out, but is useful for interoperability, see various issues: + +* https://github.com/google/go-containerregistry/issues/205 +* https://github.com/google/go-containerregistry/issues/552 +* https://github.com/google/go-containerregistry/issues/627 diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/doc.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/doc.go new file mode 100644 index 000000000..ac05d9612 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/doc.go @@ -0,0 +1,17 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// Package daemon provides facilities for reading/writing v1.Image from/to +// a running daemon. +package daemon diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go new file mode 100644 index 000000000..5013c3d30 --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/image.go @@ -0,0 +1,89 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "bytes" + "context" + "io" + "io/ioutil" + "sync" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +type imageOpener struct { + ref name.Reference + ctx context.Context + + buffered bool + client Client + + once sync.Once + bytes []byte + err error +} + +func (i *imageOpener) saveImage() (io.ReadCloser, error) { + return i.client.ImageSave(i.ctx, []string{i.ref.Name()}) +} + +func (i *imageOpener) bufferedOpener() (io.ReadCloser, error) { + // Store the tarball in memory and return a new reader into the bytes each time we need to access something. + i.once.Do(func() { + i.bytes, i.err = func() ([]byte, error) { + rc, err := i.saveImage() + if err != nil { + return nil, err + } + defer rc.Close() + + return ioutil.ReadAll(rc) + }() + }) + + // Wrap the bytes in a ReadCloser so it looks like an opened file. + return ioutil.NopCloser(bytes.NewReader(i.bytes)), i.err +} + +func (i *imageOpener) opener() tarball.Opener { + if i.buffered { + return i.bufferedOpener + } + + // To avoid storing the tarball in memory, do a save every time we need to access something. + return i.saveImage +} + +// Image provides access to an image reference from the Docker daemon, +// applying functional options to the underlying imageOpener before +// resolving the reference into a v1.Image. +func Image(ref name.Reference, options ...Option) (v1.Image, error) { + o, err := makeOptions(options...) + if err != nil { + return nil, err + } + + i := &imageOpener{ + ref: ref, + buffered: o.buffered, + client: o.client, + ctx: o.ctx, + } + + return tarball.Image(i.opener(), nil) +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go new file mode 100644 index 000000000..c3a0ac66b --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/options.go @@ -0,0 +1,102 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "context" + "io" + + "github.com/docker/docker/api/types" + "github.com/docker/docker/client" +) + +// ImageOption is an alias for Option. +// Deprecated: Use Option instead. +type ImageOption Option + +// Option is a functional option for daemon operations. +type Option func(*options) + +type options struct { + ctx context.Context + client Client + buffered bool +} + +var defaultClient = func() (Client, error) { + return client.NewClientWithOpts(client.FromEnv) +} + +func makeOptions(opts ...Option) (*options, error) { + o := &options{ + buffered: true, + ctx: context.Background(), + } + for _, opt := range opts { + opt(o) + } + + if o.client == nil { + client, err := defaultClient() + if err != nil { + return nil, err + } + o.client = client + } + o.client.NegotiateAPIVersion(o.ctx) + + return o, nil +} + +// WithBufferedOpener buffers the image. +func WithBufferedOpener() Option { + return func(o *options) { + o.buffered = true + } +} + +// WithUnbufferedOpener streams the image to avoid buffering. +func WithUnbufferedOpener() Option { + return func(o *options) { + o.buffered = false + } +} + +// WithClient is a functional option to allow injecting a docker client. +// +// By default, github.com/docker/docker/client.FromEnv is used. +func WithClient(client Client) Option { + return func(o *options) { + o.client = client + } +} + +// WithContext is a functional option to pass through a context.Context. +// +// By default, context.Background() is used. +func WithContext(ctx context.Context) Option { + return func(o *options) { + o.ctx = ctx + } +} + +// Client represents the subset of a docker client that the daemon +// package uses. +type Client interface { + NegotiateAPIVersion(ctx context.Context) + ImageSave(context.Context, []string) (io.ReadCloser, error) + ImageLoad(context.Context, io.Reader, bool) (types.ImageLoadResponse, error) + ImageTag(context.Context, string, string) error +} diff --git a/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go new file mode 100644 index 000000000..5d400156f --- /dev/null +++ b/vendor/github.com/google/go-containerregistry/pkg/v1/daemon/write.go @@ -0,0 +1,61 @@ +// Copyright 2018 Google LLC All Rights Reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package daemon + +import ( + "fmt" + "io" + "io/ioutil" + + "github.com/google/go-containerregistry/pkg/name" + v1 "github.com/google/go-containerregistry/pkg/v1" + "github.com/google/go-containerregistry/pkg/v1/tarball" +) + +// Tag adds a tag to an already existent image. +func Tag(src, dest name.Tag, options ...Option) error { + o, err := makeOptions(options...) + if err != nil { + return err + } + + return o.client.ImageTag(o.ctx, src.String(), dest.String()) +} + +// Write saves the image into the daemon as the given tag. +func Write(tag name.Tag, img v1.Image, options ...Option) (string, error) { + o, err := makeOptions(options...) + if err != nil { + return "", err + } + + pr, pw := io.Pipe() + go func() { + pw.CloseWithError(tarball.Write(tag, img, pw)) + }() + + // write the image in docker save format first, then load it + resp, err := o.client.ImageLoad(o.ctx, pr, false) + if err != nil { + return "", fmt.Errorf("error loading image: %v", err) + } + defer resp.Body.Close() + b, err := ioutil.ReadAll(resp.Body) + response := string(b) + if err != nil { + return response, fmt.Errorf("error reading load response body: %v", err) + } + return response, nil +} diff --git a/vendor/modules.txt b/vendor/modules.txt index 99cb90e96..50ad8de35 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -400,6 +400,7 @@ github.com/google/go-cmp/cmp/internal/value github.com/google/go-containerregistry/internal/and github.com/google/go-containerregistry/internal/estargz github.com/google/go-containerregistry/internal/gzip +github.com/google/go-containerregistry/internal/httptest github.com/google/go-containerregistry/internal/redact github.com/google/go-containerregistry/internal/retry github.com/google/go-containerregistry/internal/retry/wait @@ -407,7 +408,9 @@ github.com/google/go-containerregistry/internal/verify github.com/google/go-containerregistry/pkg/authn github.com/google/go-containerregistry/pkg/logs github.com/google/go-containerregistry/pkg/name +github.com/google/go-containerregistry/pkg/registry github.com/google/go-containerregistry/pkg/v1 +github.com/google/go-containerregistry/pkg/v1/daemon github.com/google/go-containerregistry/pkg/v1/empty github.com/google/go-containerregistry/pkg/v1/layout github.com/google/go-containerregistry/pkg/v1/match