diff --git a/api.go b/api.go index 8df3291357..3a7f05d7ca 100644 --- a/api.go +++ b/api.go @@ -4,8 +4,8 @@ import ( "encoding/json" "fmt" "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/utils" "github.com/gorilla/mux" - "github.com/shin-/cookiejar" "io" "log" "net/http" @@ -45,11 +45,7 @@ func writeJson(w http.ResponseWriter, b []byte) { } func getAuth(srv *Server, w http.ResponseWriter, r *http.Request, vars map[string]string) error { - config := &auth.AuthConfig{ - Username: srv.runtime.authConfig.Username, - Email: srv.runtime.authConfig.Email, - } - b, err := json.Marshal(config) + b, err := json.Marshal(srv.registry.GetAuthConfig()) if err != nil { return err } @@ -63,18 +59,17 @@ func postAuth(srv *Server, w http.ResponseWriter, r *http.Request, vars map[stri return err } - if config.Username == srv.runtime.authConfig.Username { - config.Password = srv.runtime.authConfig.Password + if config.Username == srv.registry.GetAuthConfig().Username { + config.Password = srv.registry.GetAuthConfig().Password } newAuthConfig := auth.NewAuthConfig(config.Username, config.Password, config.Email, srv.runtime.root) status, err := auth.Login(newAuthConfig) if err != nil { return err - } else { - srv.runtime.graph.getHttpClient().Jar = cookiejar.NewCookieJar() - srv.runtime.authConfig = newAuthConfig } + srv.registry.ResetClient(newAuthConfig) + if status != "" { b, err := json.Marshal(&ApiAuth{Status: status}) if err != nil { @@ -116,7 +111,7 @@ func getContainersExport(srv *Server, w http.ResponseWriter, r *http.Request, va name := vars["name"] if err := srv.ContainerExport(name, w); err != nil { - Debugf("%s", err.Error()) + utils.Debugf("%s", err.Error()) return err } return nil @@ -239,7 +234,7 @@ func postCommit(srv *Server, w http.ResponseWriter, r *http.Request, vars map[st } config := &Config{} if err := json.NewDecoder(r.Body).Decode(config); err != nil { - Debugf("%s", err.Error()) + utils.Debugf("%s", err.Error()) } repo := r.Form.Get("repo") tag := r.Form.Get("tag") @@ -335,7 +330,6 @@ func postImagesPush(srv *Server, w http.ResponseWriter, r *http.Request, vars ma if err := parseForm(r); err != nil { return err } - registry := r.Form.Get("registry") if vars == nil { @@ -602,20 +596,20 @@ func ListenAndServe(addr string, srv *Server, logging bool) error { for method, routes := range m { for route, fct := range routes { - Debugf("Registering %s, %s", method, route) + utils.Debugf("Registering %s, %s", method, route) // NOTE: scope issue, make sure the variables are local and won't be changed localRoute := route localMethod := method localFct := fct r.Path(localRoute).Methods(localMethod).HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - Debugf("Calling %s %s", localMethod, localRoute) + utils.Debugf("Calling %s %s", localMethod, localRoute) if logging { log.Println(r.Method, r.RequestURI) } if strings.Contains(r.Header.Get("User-Agent"), "Docker-Client/") { userAgent := strings.Split(r.Header.Get("User-Agent"), "/") if len(userAgent) == 2 && userAgent[1] != VERSION { - Debugf("Warning: client and server don't have the same version (client: %s, server: %s)", userAgent[1], VERSION) + utils.Debugf("Warning: client and server don't have the same version (client: %s, server: %s)", userAgent[1], VERSION) } } if err := localFct(srv, w, r, mux.Vars(r)); err != nil { diff --git a/api_test.go b/api_test.go index 5096b05212..219c39a18f 100644 --- a/api_test.go +++ b/api_test.go @@ -6,6 +6,8 @@ import ( "bytes" "encoding/json" "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/registry" + "github.com/dotcloud/docker/utils" "io" "net" "net/http" @@ -23,7 +25,10 @@ func TestGetAuth(t *testing.T) { } defer nuke(runtime) - srv := &Server{runtime: runtime} + srv := &Server{ + runtime: runtime, + registry: registry.NewRegistry(runtime.root), + } r := httptest.NewRecorder() @@ -46,13 +51,14 @@ func TestGetAuth(t *testing.T) { if err := postAuth(srv, r, req, nil); err != nil { t.Fatal(err) } + if r.Code != http.StatusOK && r.Code != 0 { t.Fatalf("%d OK or 0 expected, received %d\n", http.StatusOK, r.Code) } - if runtime.authConfig.Username != authConfig.Username || - runtime.authConfig.Password != authConfig.Password || - runtime.authConfig.Email != authConfig.Email { + newAuthConfig := srv.registry.GetAuthConfig() + if newAuthConfig.Username != authConfig.Username || + newAuthConfig.Email != authConfig.Email { t.Fatalf("The auth configuration hasn't been set correctly") } } @@ -222,7 +228,10 @@ func TestGetImagesSearch(t *testing.T) { } defer nuke(runtime) - srv := &Server{runtime: runtime} + srv := &Server{ + runtime: runtime, + registry: registry.NewRegistry(runtime.root), + } r := httptest.NewRecorder() @@ -476,13 +485,16 @@ func TestPostAuth(t *testing.T) { } defer nuke(runtime) - srv := &Server{runtime: runtime} + srv := &Server{ + runtime: runtime, + registry: registry.NewRegistry(runtime.root), + } authConfigOrig := &auth.AuthConfig{ Username: "utest", Email: "utest@yopmail.com", } - runtime.authConfig = authConfigOrig + srv.registry.ResetClient(authConfigOrig) r := httptest.NewRecorder() if err := getAuth(srv, r, nil, nil); err != nil { @@ -811,7 +823,7 @@ func TestPostContainersCreate(t *testing.T) { if _, err := os.Stat(path.Join(container.rwPath(), "test")); err != nil { if os.IsNotExist(err) { - Debugf("Err: %s", err) + utils.Debugf("Err: %s", err) t.Fatalf("The test file has not been created") } t.Fatal(err) diff --git a/auth/auth.go b/auth/auth.go index 5a5987ace8..a851d0d070 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -15,7 +15,7 @@ import ( const CONFIGFILE = ".dockercfg" // the registry server we want to login against -const INDEX_SERVER = "https://index.docker.io" +const INDEX_SERVER = "https://index.docker.io/v1" type AuthConfig struct { Username string `json:"username"` @@ -33,6 +33,13 @@ func NewAuthConfig(username, password, email, rootPath string) *AuthConfig { } } +func IndexServerAddress() string { + if os.Getenv("DOCKER_INDEX_URL") != "" { + return os.Getenv("DOCKER_INDEX_URL") + "/v1" + } + return INDEX_SERVER +} + // create a base64 encoded auth string to store in config func EncodeAuth(authConfig *AuthConfig) string { authStr := authConfig.Username + ":" + authConfig.Password @@ -119,7 +126,7 @@ func Login(authConfig *AuthConfig) (string, error) { // using `bytes.NewReader(jsonBody)` here causes the server to respond with a 411 status. b := strings.NewReader(string(jsonBody)) - req1, err := http.Post(INDEX_SERVER+"/v1/users/", "application/json; charset=utf-8", b) + req1, err := http.Post(IndexServerAddress()+"/users/", "application/json; charset=utf-8", b) if err != nil { return "", fmt.Errorf("Server Error: %s", err) } @@ -139,7 +146,7 @@ func Login(authConfig *AuthConfig) (string, error) { "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { - req, err := http.NewRequest("GET", INDEX_SERVER+"/v1/users/", nil) + req, err := http.NewRequest("GET", IndexServerAddress()+"/users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { diff --git a/auth/auth_test.go b/auth/auth_test.go index ca584f9314..6c8d032cf7 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -1,6 +1,10 @@ package auth import ( + "crypto/rand" + "encoding/hex" + "os" + "strings" "testing" ) @@ -21,3 +25,49 @@ func TestEncodeAuth(t *testing.T) { t.Fatal("AuthString encoding isn't correct.") } } + +func TestLogin(t *testing.T) { + os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") + defer os.Setenv("DOCKER_INDEX_URL", "") + authConfig := NewAuthConfig("unittester", "surlautrerivejetattendrai", "noise+unittester@dotcloud.com", "/tmp") + status, err := Login(authConfig) + if err != nil { + t.Fatal(err) + } + if status != "Login Succeeded\n" { + t.Fatalf("Expected status \"Login Succeeded\", found \"%s\" instead", status) + } +} + +func TestCreateAccount(t *testing.T) { + os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") + defer os.Setenv("DOCKER_INDEX_URL", "") + tokenBuffer := make([]byte, 16) + _, err := rand.Read(tokenBuffer) + if err != nil { + t.Fatal(err) + } + token := hex.EncodeToString(tokenBuffer)[:12] + username := "ut" + token + authConfig := NewAuthConfig(username, "test42", "docker-ut+"+token+"@example.com", "/tmp") + status, err := Login(authConfig) + if err != nil { + t.Fatal(err) + } + expectedStatus := "Account created. Please use the confirmation link we sent" + + " to your e-mail to activate it.\n" + if status != expectedStatus { + t.Fatalf("Expected status: \"%s\", found \"%s\" instead.", expectedStatus, status) + } + + status, err = Login(authConfig) + if err == nil { + t.Fatalf("Expected error but found nil instead") + } + + expectedError := "Login: Account is not Active" + + if !strings.Contains(err.Error(), expectedError) { + t.Fatalf("Expected message \"%s\" but found \"%s\" instead", expectedError, err.Error()) + } +} diff --git a/builder.go b/builder.go index 5c51d62b9e..507c6b3d7e 100644 --- a/builder.go +++ b/builder.go @@ -4,6 +4,7 @@ import ( "bufio" "encoding/json" "fmt" + "github.com/dotcloud/docker/utils" "io" "os" "path" @@ -161,11 +162,11 @@ func (builder *Builder) clearTmp(containers, images map[string]struct{}) { for c := range containers { tmp := builder.runtime.Get(c) builder.runtime.Destroy(tmp) - Debugf("Removing container %s", c) + utils.Debugf("Removing container %s", c) } for i := range images { builder.runtime.graph.Delete(i) - Debugf("Removing image %s", i) + utils.Debugf("Removing image %s", i) } } @@ -234,28 +235,29 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e fmt.Fprintf(stdout, "FROM %s\n", arguments) image, err = builder.runtime.repositories.LookupImage(arguments) if err != nil { - if builder.runtime.graph.IsNotExist(err) { + // if builder.runtime.graph.IsNotExist(err) { - var tag, remote string - if strings.Contains(arguments, ":") { - remoteParts := strings.Split(arguments, ":") - tag = remoteParts[1] - remote = remoteParts[0] - } else { - remote = arguments - } + // var tag, remote string + // if strings.Contains(arguments, ":") { + // remoteParts := strings.Split(arguments, ":") + // tag = remoteParts[1] + // remote = remoteParts[0] + // } else { + // remote = arguments + // } - if err := builder.runtime.graph.PullRepository(stdout, remote, tag, builder.runtime.repositories, builder.runtime.authConfig); err != nil { - return nil, err - } + // panic("TODO: reimplement this") + // // if err := builder.runtime.graph.PullRepository(stdout, remote, tag, builder.runtime.repositories, builder.runtime.authConfig); err != nil { + // // return nil, err + // // } - image, err = builder.runtime.repositories.LookupImage(arguments) - if err != nil { - return nil, err - } - } else { - return nil, err - } + // image, err = builder.runtime.repositories.LookupImage(arguments) + // if err != nil { + // return nil, err + // } + // } else { + return nil, err + // } } config = &Config{} @@ -286,7 +288,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e break } - Debugf("Env -----> %v ------ %v\n", config.Env, env) + utils.Debugf("Env -----> %v ------ %v\n", config.Env, env) // Create the container and start it c, err := builder.Create(config) @@ -410,7 +412,7 @@ func (builder *Builder) Build(dockerfile io.Reader, stdout io.Writer) (*Image, e destPath := strings.Trim(tmp[1], " ") fmt.Fprintf(stdout, "COPY %s to %s in %s\n", sourceUrl, destPath, base.ShortId()) - file, err := Download(sourceUrl, stdout) + file, err := utils.Download(sourceUrl, stdout) if err != nil { return nil, err } diff --git a/builder_test.go b/builder_test.go index 08b7dd58cc..e3a24e86ec 100644 --- a/builder_test.go +++ b/builder_test.go @@ -1,6 +1,7 @@ package docker import ( + "github.com/dotcloud/docker/utils" "strings" "testing" ) @@ -24,7 +25,7 @@ func TestBuild(t *testing.T) { builder := NewBuilder(runtime) - img, err := builder.Build(strings.NewReader(Dockerfile), &nopWriter{}) + img, err := builder.Build(strings.NewReader(Dockerfile), &utils.NopWriter{}) if err != nil { t.Fatal(err) } diff --git a/commands.go b/commands.go index 9e26790f5d..7cb3511cda 100644 --- a/commands.go +++ b/commands.go @@ -7,6 +7,7 @@ import ( "fmt" "github.com/dotcloud/docker/auth" "github.com/dotcloud/docker/term" + "github.com/dotcloud/docker/utils" "io" "io/ioutil" "net" @@ -188,11 +189,11 @@ func CmdLogin(args ...string) error { return readStringOnRawTerminal(stdin, stdout, false) } - oldState, err := SetRawTerminal() + oldState, err := term.SetRawTerminal() if err != nil { return err } else { - defer RestoreTerminal(oldState) + defer term.RestoreTerminal(oldState) } cmd := Subcmd("login", "", "Register or Login to the docker registry server") @@ -252,7 +253,7 @@ func CmdLogin(args ...string) error { return err } if out2.Status != "" { - RestoreTerminal(oldState) + term.RestoreTerminal(oldState) fmt.Print(out2.Status) } return nil @@ -303,7 +304,7 @@ func CmdVersion(args ...string) error { var out ApiVersion err = json.Unmarshal(body, &out) if err != nil { - Debugf("Error unmarshal: body: %s, err: %s\n", body, err) + utils.Debugf("Error unmarshal: body: %s, err: %s\n", body, err) return err } fmt.Println("Version:", out.Version) @@ -519,7 +520,7 @@ func CmdHistory(args ...string) error { fmt.Fprintln(w, "ID\tCREATED\tCREATED BY") for _, out := range outs { - fmt.Fprintf(w, "%s\t%s ago\t%s\n", out.Id, HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.CreatedBy) + fmt.Fprintf(w, "%s\t%s ago\t%s\n", out.Id, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.CreatedBy) } w.Flush() return nil @@ -742,14 +743,14 @@ func CmdImages(args ...string) error { if *noTrunc { fmt.Fprintf(w, "%s\t", out.Id) } else { - fmt.Fprintf(w, "%s\t", TruncateId(out.Id)) + fmt.Fprintf(w, "%s\t", utils.TruncateId(out.Id)) } - fmt.Fprintf(w, "%s ago\n", HumanDuration(time.Now().Sub(time.Unix(out.Created, 0)))) + fmt.Fprintf(w, "%s ago\n", utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0)))) } else { if *noTrunc { fmt.Fprintln(w, out.Id) } else { - fmt.Fprintln(w, TruncateId(out.Id)) + fmt.Fprintln(w, utils.TruncateId(out.Id)) } } } @@ -809,15 +810,15 @@ func CmdPs(args ...string) error { for _, out := range outs { if !*quiet { if *noTrunc { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", out.Id, out.Image, out.Command, out.Status, HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Ports) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", out.Id, out.Image, out.Command, out.Status, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Ports) } else { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", TruncateId(out.Id), out.Image, Trunc(out.Command, 20), out.Status, HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Ports) + fmt.Fprintf(w, "%s\t%s\t%s\t%s\t%s ago\t%s\n", utils.TruncateId(out.Id), out.Image, utils.Trunc(out.Command, 20), out.Status, utils.HumanDuration(time.Now().Sub(time.Unix(out.Created, 0))), out.Ports) } } else { if *noTrunc { fmt.Fprintln(w, out.Id) } else { - fmt.Fprintln(w, TruncateId(out.Id)) + fmt.Fprintln(w, utils.TruncateId(out.Id)) } } } @@ -1244,20 +1245,20 @@ func hijack(method, path string, setRawTerminal bool) error { rwc, br := clientconn.Hijack() defer rwc.Close() - receiveStdout := Go(func() error { + receiveStdout := utils.Go(func() error { _, err := io.Copy(os.Stdout, br) return err }) if setRawTerminal && term.IsTerminal(int(os.Stdin.Fd())) && os.Getenv("NORAW") == "" { - if oldState, err := SetRawTerminal(); err != nil { + if oldState, err := term.SetRawTerminal(); err != nil { return err } else { - defer RestoreTerminal(oldState) + defer term.RestoreTerminal(oldState) } } - sendStdin := Go(func() error { + sendStdin := utils.Go(func() error { _, err := io.Copy(rwc, os.Stdin) if err := rwc.(*net.TCPConn).CloseWrite(); err != nil { fmt.Fprintf(os.Stderr, "Couldn't send EOF: %s\n", err) diff --git a/container.go b/container.go index d4ebc60c8b..a82ce0291a 100644 --- a/container.go +++ b/container.go @@ -4,6 +4,7 @@ import ( "encoding/json" "flag" "fmt" + "github.com/dotcloud/docker/utils" "github.com/kr/pty" "io" "io/ioutil" @@ -39,8 +40,8 @@ type Container struct { ResolvConfPath string cmd *exec.Cmd - stdout *writeBroadcaster - stderr *writeBroadcaster + stdout *utils.WriteBroadcaster + stderr *utils.WriteBroadcaster stdin io.ReadCloser stdinPipe io.WriteCloser ptyMaster io.Closer @@ -251,9 +252,9 @@ func (container *Container) startPty() error { // Copy the PTYs to our broadcasters go func() { defer container.stdout.CloseWriters() - Debugf("[startPty] Begin of stdout pipe") + utils.Debugf("[startPty] Begin of stdout pipe") io.Copy(container.stdout, ptyMaster) - Debugf("[startPty] End of stdout pipe") + utils.Debugf("[startPty] End of stdout pipe") }() // stdin @@ -262,9 +263,9 @@ func (container *Container) startPty() error { container.cmd.SysProcAttr = &syscall.SysProcAttr{Setctty: true, Setsid: true} go func() { defer container.stdin.Close() - Debugf("[startPty] Begin of stdin pipe") + utils.Debugf("[startPty] Begin of stdin pipe") io.Copy(ptyMaster, container.stdin) - Debugf("[startPty] End of stdin pipe") + utils.Debugf("[startPty] End of stdin pipe") }() } if err := container.cmd.Start(); err != nil { @@ -284,9 +285,9 @@ func (container *Container) start() error { } go func() { defer stdin.Close() - Debugf("Begin of stdin pipe [start]") + utils.Debugf("Begin of stdin pipe [start]") io.Copy(stdin, container.stdin) - Debugf("End of stdin pipe [start]") + utils.Debugf("End of stdin pipe [start]") }() } return container.cmd.Start() @@ -303,8 +304,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s errors <- err } else { go func() { - Debugf("[start] attach stdin\n") - defer Debugf("[end] attach stdin\n") + utils.Debugf("[start] attach stdin\n") + defer utils.Debugf("[end] attach stdin\n") // No matter what, when stdin is closed (io.Copy unblock), close stdout and stderr if cStdout != nil { defer cStdout.Close() @@ -316,12 +317,12 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s defer cStdin.Close() } if container.Config.Tty { - _, err = CopyEscapable(cStdin, stdin) + _, err = utils.CopyEscapable(cStdin, stdin) } else { _, err = io.Copy(cStdin, stdin) } if err != nil { - Debugf("[error] attach stdin: %s\n", err) + utils.Debugf("[error] attach stdin: %s\n", err) } // Discard error, expecting pipe error errors <- nil @@ -335,8 +336,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s } else { cStdout = p go func() { - Debugf("[start] attach stdout\n") - defer Debugf("[end] attach stdout\n") + utils.Debugf("[start] attach stdout\n") + defer utils.Debugf("[end] attach stdout\n") // If we are in StdinOnce mode, then close stdin if container.Config.StdinOnce { if stdin != nil { @@ -348,7 +349,7 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s } _, err := io.Copy(stdout, cStdout) if err != nil { - Debugf("[error] attach stdout: %s\n", err) + utils.Debugf("[error] attach stdout: %s\n", err) } errors <- err }() @@ -361,8 +362,8 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s } else { cStderr = p go func() { - Debugf("[start] attach stderr\n") - defer Debugf("[end] attach stderr\n") + utils.Debugf("[start] attach stderr\n") + defer utils.Debugf("[end] attach stderr\n") // If we are in StdinOnce mode, then close stdin if container.Config.StdinOnce { if stdin != nil { @@ -374,13 +375,13 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s } _, err := io.Copy(stderr, cStderr) if err != nil { - Debugf("[error] attach stderr: %s\n", err) + utils.Debugf("[error] attach stderr: %s\n", err) } errors <- err }() } } - return Go(func() error { + return utils.Go(func() error { if cStdout != nil { defer cStdout.Close() } @@ -390,14 +391,14 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s // FIXME: how do clean up the stdin goroutine without the unwanted side effect // of closing the passed stdin? Add an intermediary io.Pipe? for i := 0; i < nJobs; i += 1 { - Debugf("Waiting for job %d/%d\n", i+1, nJobs) + utils.Debugf("Waiting for job %d/%d\n", i+1, nJobs) if err := <-errors; err != nil { - Debugf("Job %d returned error %s. Aborting all jobs\n", i+1, err) + utils.Debugf("Job %d returned error %s. Aborting all jobs\n", i+1, err) return err } - Debugf("Job %d completed successfully\n", i+1) + utils.Debugf("Job %d completed successfully\n", i+1) } - Debugf("All jobs completed successfully\n") + utils.Debugf("All jobs completed successfully\n") return nil }) } @@ -555,13 +556,13 @@ func (container *Container) StdinPipe() (io.WriteCloser, error) { func (container *Container) StdoutPipe() (io.ReadCloser, error) { reader, writer := io.Pipe() container.stdout.AddWriter(writer) - return newBufReader(reader), nil + return utils.NewBufReader(reader), nil } func (container *Container) StderrPipe() (io.ReadCloser, error) { reader, writer := io.Pipe() container.stderr.AddWriter(writer) - return newBufReader(reader), nil + return utils.NewBufReader(reader), nil } func (container *Container) allocateNetwork() error { @@ -609,20 +610,20 @@ func (container *Container) waitLxc() error { func (container *Container) monitor() { // Wait for the program to exit - Debugf("Waiting for process") + utils.Debugf("Waiting for process") // If the command does not exists, try to wait via lxc if container.cmd == nil { if err := container.waitLxc(); err != nil { - Debugf("%s: Process: %s", container.Id, err) + utils.Debugf("%s: Process: %s", container.Id, err) } } else { if err := container.cmd.Wait(); err != nil { // Discard the error as any signals or non 0 returns will generate an error - Debugf("%s: Process: %s", container.Id, err) + utils.Debugf("%s: Process: %s", container.Id, err) } } - Debugf("Process finished") + utils.Debugf("Process finished") var exitCode int = -1 if container.cmd != nil { @@ -633,19 +634,19 @@ func (container *Container) monitor() { container.releaseNetwork() if container.Config.OpenStdin { if err := container.stdin.Close(); err != nil { - Debugf("%s: Error close stdin: %s", container.Id, err) + utils.Debugf("%s: Error close stdin: %s", container.Id, err) } } if err := container.stdout.CloseWriters(); err != nil { - Debugf("%s: Error close stdout: %s", container.Id, err) + utils.Debugf("%s: Error close stdout: %s", container.Id, err) } if err := container.stderr.CloseWriters(); err != nil { - Debugf("%s: Error close stderr: %s", container.Id, err) + utils.Debugf("%s: Error close stderr: %s", container.Id, err) } if container.ptyMaster != nil { if err := container.ptyMaster.Close(); err != nil { - Debugf("%s: Error closing Pty master: %s", container.Id, err) + utils.Debugf("%s: Error closing Pty master: %s", container.Id, err) } } @@ -762,7 +763,7 @@ func (container *Container) RwChecksum() (string, error) { if err != nil { return "", err } - return HashData(rwData) + return utils.HashData(rwData) } func (container *Container) Export() (Archive, error) { @@ -833,7 +834,7 @@ func (container *Container) Unmount() error { // In case of a collision a lookup with Runtime.Get() will fail, and the caller // will need to use a langer prefix, or the full-length container Id. func (container *Container) ShortId() string { - return TruncateId(container.Id) + return utils.TruncateId(container.Id) } func (container *Container) logPath(name string) string { diff --git a/docker/docker.go b/docker/docker.go index 778326a810..c8c1a65603 100644 --- a/docker/docker.go +++ b/docker/docker.go @@ -4,6 +4,7 @@ import ( "flag" "fmt" "github.com/dotcloud/docker" + "github.com/dotcloud/docker/utils" "io/ioutil" "log" "os" @@ -17,7 +18,7 @@ var ( ) func main() { - if docker.SelfPath() == "/sbin/init" { + if utils.SelfPath() == "/sbin/init" { // Running in init mode docker.SysInit() return diff --git a/docs/sources/index/variable.rst b/docs/sources/index/variable.rst new file mode 100644 index 0000000000..efbcfae80c --- /dev/null +++ b/docs/sources/index/variable.rst @@ -0,0 +1,23 @@ +================================= +Docker Index Environment Variable +================================= + +Variable +-------- + +.. code-block:: sh + + DOCKER_INDEX_URL + +Setting this environment variable on the docker server will change the URL docker index. +This address is used in commands such as ``docker login``, ``docker push`` and ``docker pull``. +The docker daemon doesn't need to be restarted for this parameter to take effect. + +Example +------- + +.. code-block:: sh + + docker -d & + export DOCKER_INDEX_URL="https://index.docker.io" + diff --git a/getKernelVersion_darwin.go b/getKernelVersion_darwin.go index be3b733b68..2fce282716 100644 --- a/getKernelVersion_darwin.go +++ b/getKernelVersion_darwin.go @@ -2,8 +2,9 @@ package docker import ( "fmt" + "github.com/dotcloud/docker/utils" ) -func getKernelVersion() (*KernelVersionInfo, error) { +func getKernelVersion() (*utils.KernelVersionInfo, error) { return nil, fmt.Errorf("Kernel version detection is not available on darwin") } diff --git a/getKernelVersion_linux.go b/getKernelVersion_linux.go index 04bb1edcb8..4f9c7db70c 100644 --- a/getKernelVersion_linux.go +++ b/getKernelVersion_linux.go @@ -2,12 +2,14 @@ package docker import ( "bytes" + "github.com/dotcloud/docker/utils" "strconv" "strings" "syscall" ) -func getKernelVersion() (*KernelVersionInfo, error) { +// FIXME: Move this to utils package +func getKernelVersion() (*utils.KernelVersionInfo, error) { var ( uts syscall.Utsname flavor string @@ -60,7 +62,7 @@ func getKernelVersion() (*KernelVersionInfo, error) { flavor = "" } - return &KernelVersionInfo{ + return &utils.KernelVersionInfo{ Kernel: kernel, Major: major, Minor: minor, diff --git a/graph.go b/graph.go index 14731622e6..5bf305fc22 100644 --- a/graph.go +++ b/graph.go @@ -3,9 +3,10 @@ package docker import ( "encoding/json" "fmt" + "github.com/dotcloud/docker/registry" + "github.com/dotcloud/docker/utils" "io" "io/ioutil" - "net/http" "os" "path" "path/filepath" @@ -17,8 +18,7 @@ import ( // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { Root string - idIndex *TruncIndex - httpClient *http.Client + idIndex *utils.TruncIndex checksumLock map[string]*sync.Mutex lockSumFile *sync.Mutex lockSumMap *sync.Mutex @@ -37,7 +37,7 @@ func NewGraph(root string) (*Graph, error) { } graph := &Graph{ Root: abspath, - idIndex: NewTruncIndex(), + idIndex: utils.NewTruncIndex(), checksumLock: make(map[string]*sync.Mutex), lockSumFile: &sync.Mutex{}, lockSumMap: &sync.Mutex{}, @@ -165,7 +165,7 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, output if err != nil { return nil, err } - return NewTempArchive(ProgressReader(ioutil.NopCloser(archive), 0, output, "Buffering to disk %v/%v (%v)"), tmp.Root) + return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, "Buffering to disk %v/%v (%v)"), tmp.Root) } // Mktemp creates a temporary sub-directory inside the graph's filesystem. @@ -324,3 +324,17 @@ func (graph *Graph) storeChecksums(checksums map[string]string) error { } return nil } + +func (graph *Graph) UpdateChecksums(newChecksums map[string]*registry.ImgData) error { + graph.lockSumFile.Lock() + defer graph.lockSumFile.Unlock() + + localChecksums, err := graph.getStoredChecksums() + if err != nil { + return err + } + for id, elem := range newChecksums { + localChecksums[id] = elem.Checksum + } + return graph.storeChecksums(localChecksums) +} diff --git a/graph_test.go b/graph_test.go index 19c6c07cf2..7c1e5c0978 100644 --- a/graph_test.go +++ b/graph_test.go @@ -4,6 +4,7 @@ import ( "archive/tar" "bytes" "errors" + "github.com/dotcloud/docker/utils" "io" "io/ioutil" "os" @@ -155,7 +156,7 @@ func TestDeletePrefix(t *testing.T) { graph := tempGraph(t) defer os.RemoveAll(graph.Root) img := createTestImage(graph, t) - if err := graph.Delete(TruncateId(img.Id)); err != nil { + if err := graph.Delete(utils.TruncateId(img.Id)); err != nil { t.Fatal(err) } assertNImages(graph, t, 0) diff --git a/image.go b/image.go index 413d95673b..342c0c94b7 100644 --- a/image.go +++ b/image.go @@ -6,6 +6,7 @@ import ( "encoding/hex" "encoding/json" "fmt" + "github.com/dotcloud/docker/utils" "io" "io/ioutil" "log" @@ -180,7 +181,7 @@ func (image *Image) Changes(rw string) ([]Change, error) { } func (image *Image) ShortId() string { - return TruncateId(image.Id) + return utils.TruncateId(image.Id) } func ValidateId(id string) error { @@ -359,3 +360,15 @@ func (img *Image) Checksum() (string, error) { return hash, nil } + +// Build an Image object from raw json data +func NewImgJson(src []byte) (*Image, error) { + ret := &Image{} + + utils.Debugf("Json string: {%s}\n", src) + // FIXME: Is there a cleaner way to "purify" the input json? + if err := json.Unmarshal(src, ret); err != nil { + return nil, err + } + return ret, nil +} diff --git a/network.go b/network.go index 373625d59c..77a82ed9f0 100644 --- a/network.go +++ b/network.go @@ -4,6 +4,7 @@ import ( "encoding/binary" "errors" "fmt" + "github.com/dotcloud/docker/utils" "io" "log" "net" @@ -97,7 +98,7 @@ func checkRouteOverlaps(dockerNetwork *net.IPNet) error { if err != nil { return err } - Debugf("Routes:\n\n%s", output) + utils.Debugf("Routes:\n\n%s", output) for _, line := range strings.Split(output, "\n") { if strings.Trim(line, "\r\n\t ") == "" || strings.Contains(line, "default") { continue @@ -126,13 +127,13 @@ func CreateBridgeIface(ifaceName string) error { ifaceAddr = addr break } else { - Debugf("%s: %s", addr, err) + utils.Debugf("%s: %s", addr, err) } } if ifaceAddr == "" { return fmt.Errorf("Could not find a free IP address range for interface '%s'. Please configure its address manually and run 'docker -b %s'", ifaceName, ifaceName) } else { - Debugf("Creating bridge %s with network %s", ifaceName, ifaceAddr) + utils.Debugf("Creating bridge %s with network %s", ifaceName, ifaceAddr) } if output, err := ip("link", "add", ifaceName, "type", "bridge"); err != nil { @@ -239,22 +240,22 @@ func (mapper *PortMapper) Map(port int, dest net.TCPAddr) error { // proxy listens for socket connections on `listener`, and forwards them unmodified // to `proto:address` func proxy(listener net.Listener, proto, address string) error { - Debugf("proxying to %s:%s", proto, address) - defer Debugf("Done proxying to %s:%s", proto, address) + utils.Debugf("proxying to %s:%s", proto, address) + defer utils.Debugf("Done proxying to %s:%s", proto, address) for { - Debugf("Listening on %s", listener) + utils.Debugf("Listening on %s", listener) src, err := listener.Accept() if err != nil { return err } - Debugf("Connecting to %s:%s", proto, address) + utils.Debugf("Connecting to %s:%s", proto, address) dst, err := net.Dial(proto, address) if err != nil { log.Printf("Error connecting to %s:%s: %s", proto, address, err) src.Close() continue } - Debugf("Connected to backend, splicing") + utils.Debugf("Connected to backend, splicing") splice(src, dst) } return nil @@ -317,7 +318,7 @@ func (alloc *PortAllocator) runFountain() { // FIXME: Release can no longer fail, change its prototype to reflect that. func (alloc *PortAllocator) Release(port int) error { - Debugf("Releasing %d", port) + utils.Debugf("Releasing %d", port) alloc.lock.Lock() delete(alloc.inUse, port) alloc.lock.Unlock() @@ -325,7 +326,7 @@ func (alloc *PortAllocator) Release(port int) error { } func (alloc *PortAllocator) Acquire(port int) (int, error) { - Debugf("Acquiring %d", port) + utils.Debugf("Acquiring %d", port) if port == 0 { // Allocate a port from the fountain for port := range alloc.fountain { diff --git a/registry.go b/registry.go deleted file mode 100644 index b8a3e10599..0000000000 --- a/registry.go +++ /dev/null @@ -1,748 +0,0 @@ -package docker - -import ( - "bytes" - "encoding/json" - "fmt" - "github.com/dotcloud/docker/auth" - "github.com/shin-/cookiejar" - "io" - "io/ioutil" - "net/http" - "net/url" - "os" - "path" - "strings" -) - -//FIXME: Set the endpoint in a conf file or via commandline -const INDEX_ENDPOINT = auth.INDEX_SERVER + "/v1" - -// Build an Image object from raw json data -func NewImgJson(src []byte) (*Image, error) { - ret := &Image{} - - Debugf("Json string: {%s}\n", src) - // FIXME: Is there a cleaner way to "purify" the input json? - if err := json.Unmarshal(src, ret); err != nil { - return nil, err - } - return ret, nil -} - -func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { - for _, cookie := range c.Jar.Cookies(req.URL) { - req.AddCookie(cookie) - } - return c.Do(req) -} - -// Retrieve the history of a given image from the Registry. -// Return a list of the parent's json (requested image included) -func (graph *Graph) getRemoteHistory(imgId, registry string, token []string) ([]string, error) { - client := graph.getHttpClient() - - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err := client.Do(req) - if err != nil || res.StatusCode != 200 { - if res != nil { - return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId) - } - return nil, err - } - defer res.Body.Close() - - jsonString, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, fmt.Errorf("Error while reading the http response: %s\n", err) - } - - Debugf("Ancestry: %s", jsonString) - history := new([]string) - if err := json.Unmarshal(jsonString, history); err != nil { - return nil, err - } - return *history, nil -} - -func (graph *Graph) getHttpClient() *http.Client { - if graph.httpClient == nil { - graph.httpClient = &http.Client{} - graph.httpClient.Jar = cookiejar.NewCookieJar() - } - return graph.httpClient -} - -// Check if an image exists in the Registry -func (graph *Graph) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { - rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) - if err != nil { - return false - } - req.SetBasicAuth(authConfig.Username, authConfig.Password) - res, err := rt.RoundTrip(req) - return err == nil && res.StatusCode == 307 -} - -func (graph *Graph) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { - u := INDEX_ENDPOINT + "/repositories/" + repository + "/images" - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, err - } - if authConfig != nil && len(authConfig.Username) > 0 { - req.SetBasicAuth(authConfig.Username, authConfig.Password) - } - res, err := graph.getHttpClient().Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - - // Repository doesn't exist yet - if res.StatusCode == 404 { - return nil, nil - } - - jsonData, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - - imageList := []map[string]string{} - - err = json.Unmarshal(jsonData, &imageList) - if err != nil { - Debugf("Body: %s (%s)\n", res.Body, u) - return nil, err - } - - return imageList, nil -} - -// Retrieve an image from the Registry. -// Returns the Image object as well as the layer as an Archive (io.Reader) -func (graph *Graph) getRemoteImage(stdout io.Writer, imgId, registry string, token []string) (*Image, Archive, error) { - client := graph.getHttpClient() - - fmt.Fprintf(stdout, "Pulling %s metadata\r\n", imgId) - // Get the Json - req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) - if err != nil { - return nil, nil, fmt.Errorf("Failed to download json: %s", err) - } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err := client.Do(req) - if err != nil { - return nil, nil, fmt.Errorf("Failed to download json: %s", err) - } - if res.StatusCode != 200 { - return nil, nil, fmt.Errorf("HTTP code %d", res.StatusCode) - } - defer res.Body.Close() - - jsonString, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, nil, fmt.Errorf("Failed to download json: %s", err) - } - - img, err := NewImgJson(jsonString) - if err != nil { - return nil, nil, fmt.Errorf("Failed to parse json: %s", err) - } - img.Id = imgId - - // Get the layer - fmt.Fprintf(stdout, "Pulling %s fs layer\r\n", imgId) - req, err = http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) - if err != nil { - return nil, nil, fmt.Errorf("Error while getting from the server: %s\n", err) - } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err = client.Do(req) - if err != nil { - return nil, nil, err - } - return img, ProgressReader(res.Body, int(res.ContentLength), stdout, "Downloading %v/%v (%v)"), nil -} - -func (graph *Graph) getRemoteTags(stdout io.Writer, registries []string, repository string, token []string) (map[string]string, error) { - client := graph.getHttpClient() - if strings.Count(repository, "/") == 0 { - // This will be removed once the Registry supports auto-resolution on - // the "library" namespace - repository = "library/" + repository - } - for _, host := range registries { - endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) - req, err := http.NewRequest("GET", endpoint, nil) - if err != nil { - return nil, err - } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err := client.Do(req) - defer res.Body.Close() - Debugf("Got status code %d from %s", res.StatusCode, endpoint) - if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { - continue - } else if res.StatusCode == 404 { - return nil, fmt.Errorf("Repository not found") - } - - result := make(map[string]string) - - rawJson, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - if err = json.Unmarshal(rawJson, &result); err != nil { - return nil, err - } - return result, nil - } - return nil, fmt.Errorf("Could not reach any registry endpoint") -} - -func (graph *Graph) getImageForTag(stdout io.Writer, tag, remote, registry string, token []string) (string, error) { - client := graph.getHttpClient() - - if !strings.Contains(remote, "/") { - remote = "library/" + remote - } - - registryEndpoint := "https://" + registry + "/v1" - repositoryTarget := registryEndpoint + "/repositories/" + remote + "/tags/" + tag - - req, err := http.NewRequest("GET", repositoryTarget, nil) - if err != nil { - return "", err - } - req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - res, err := client.Do(req) - if err != nil { - return "", fmt.Errorf("Error while retrieving repository info: %v", err) - } - defer res.Body.Close() - if res.StatusCode == 403 { - return "", fmt.Errorf("You aren't authorized to access this resource") - } else if res.StatusCode != 200 { - return "", fmt.Errorf("HTTP code: %d", res.StatusCode) - } - - var imgId string - rawJson, err := ioutil.ReadAll(res.Body) - if err != nil { - return "", err - } - if err = json.Unmarshal(rawJson, &imgId); err != nil { - return "", err - } - return imgId, nil -} - -func (graph *Graph) PullImage(stdout io.Writer, imgId, registry string, token []string) error { - history, err := graph.getRemoteHistory(imgId, registry, token) - if err != nil { - return err - } - // FIXME: Try to stream the images? - // FIXME: Launch the getRemoteImage() in goroutines - for _, id := range history { - if !graph.Exists(id) { - img, layer, err := graph.getRemoteImage(stdout, id, registry, token) - if err != nil { - // FIXME: Keep goging in case of error? - return err - } - if err = graph.Register(layer, false, img); err != nil { - return err - } - } - } - return nil -} - -func (graph *Graph) PullRepository(stdout io.Writer, remote, askedTag string, repositories *TagStore, authConfig *auth.AuthConfig) error { - client := graph.getHttpClient() - - fmt.Fprintf(stdout, "Pulling repository %s from %s\r\n", remote, INDEX_ENDPOINT) - repositoryTarget := INDEX_ENDPOINT + "/repositories/" + remote + "/images" - - req, err := http.NewRequest("GET", repositoryTarget, nil) - if err != nil { - return err - } - if authConfig != nil && len(authConfig.Username) > 0 { - req.SetBasicAuth(authConfig.Username, authConfig.Password) - } - req.Header.Set("X-Docker-Token", "true") - - res, err := client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - if res.StatusCode == 401 { - return fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) - } - // TODO: Right now we're ignoring checksums in the response body. - // In the future, we need to use them to check image validity. - if res.StatusCode != 200 { - return fmt.Errorf("HTTP code: %d", res.StatusCode) - } - - var token, endpoints []string - if res.Header.Get("X-Docker-Token") != "" { - token = res.Header["X-Docker-Token"] - } - if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints = res.Header["X-Docker-Endpoints"] - } else { - return fmt.Errorf("Index response didn't contain any endpoints") - } - - checksumsJson, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - - // Reload the json file to make sure not to overwrite faster sums - err = func() error { - localChecksums := make(map[string]string) - remoteChecksums := []ImgListJson{} - checksumDictPth := path.Join(graph.Root, "checksums") - - if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil { - return err - } - - graph.lockSumFile.Lock() - defer graph.lockSumFile.Unlock() - - if checksumDict, err := ioutil.ReadFile(checksumDictPth); err == nil { - if err := json.Unmarshal(checksumDict, &localChecksums); err != nil { - return err - } - } - - for _, elem := range remoteChecksums { - localChecksums[elem.Id] = elem.Checksum - } - - checksumsJson, err = json.Marshal(localChecksums) - if err != nil { - return err - } - if err := ioutil.WriteFile(checksumDictPth, checksumsJson, 0600); err != nil { - return err - } - return nil - }() - if err != nil { - return err - } - - var tagsList map[string]string - if askedTag == "" { - tagsList, err = graph.getRemoteTags(stdout, endpoints, remote, token) - if err != nil { - return err - } - } else { - tagsList = map[string]string{askedTag: ""} - } - - for askedTag, imgId := range tagsList { - fmt.Fprintf(stdout, "Resolving tag \"%s:%s\" from %s\n", remote, askedTag, endpoints) - success := false - for _, registry := range endpoints { - if imgId == "" { - imgId, err = graph.getImageForTag(stdout, askedTag, remote, registry, token) - if err != nil { - fmt.Fprintf(stdout, "Error while retrieving image for tag: %v (%v) ; "+ - "checking next endpoint", askedTag, err) - continue - } - } - - if err := graph.PullImage(stdout, imgId, "https://"+registry+"/v1", token); err != nil { - return err - } - - if err = repositories.Set(remote, askedTag, imgId, true); err != nil { - return err - } - success = true - } - - if !success { - return fmt.Errorf("Could not find repository on any of the indexed registries.") - } - } - - if err = repositories.Save(); err != nil { - return err - } - - return nil -} - -// Push a local image to the registry -func (graph *Graph) PushImage(stdout io.Writer, img *Image, registry string, token []string) error { - registry = "https://" + registry + "/v1" - - client := graph.getHttpClient() - jsonRaw, err := ioutil.ReadFile(path.Join(graph.Root, img.Id, "json")) - if err != nil { - return fmt.Errorf("Error while retreiving the path for {%s}: %s", img.Id, err) - } - - fmt.Fprintf(stdout, "Pushing %s metadata\r\n", img.Id) - - // FIXME: try json with UTF8 - jsonData := strings.NewReader(string(jsonRaw)) - req, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/json", jsonData) - if err != nil { - return err - } - req.Header.Add("Content-type", "application/json") - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - - checksum, err := img.Checksum() - if err != nil { - return fmt.Errorf("Error while retrieving checksum for %s: %v", img.Id, err) - } - req.Header.Set("X-Docker-Checksum", checksum) - Debugf("Setting checksum for %s: %s", img.ShortId(), checksum) - res, err := doWithCookies(client, req) - if err != nil { - return fmt.Errorf("Failed to upload metadata: %s", err) - } - defer res.Body.Close() - if len(res.Cookies()) > 0 { - client.Jar.SetCookies(req.URL, res.Cookies()) - } - if res.StatusCode != 200 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ - " trying to parse response body: %v", res.StatusCode, err) - } - var jsonBody map[string]string - if err := json.Unmarshal(errBody, &jsonBody); err != nil { - errBody = []byte(err.Error()) - } else if jsonBody["error"] == "Image already exists" { - fmt.Fprintf(stdout, "Image %v already uploaded ; skipping\n", img.Id) - return nil - } - return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) - } - - fmt.Fprintf(stdout, "Pushing %s fs layer\r\n", img.Id) - root, err := img.root() - if err != nil { - return err - } - - var layerData *TempArchive - // If the archive exists, use it - file, err := os.Open(layerArchivePath(root)) - if err != nil { - if os.IsNotExist(err) { - // If the archive does not exist, create one from the layer - layerData, err = graph.TempLayerArchive(img.Id, Xz, stdout) - if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) - } - } else { - return err - } - } else { - defer file.Close() - st, err := file.Stat() - if err != nil { - return err - } - layerData = &TempArchive{file, st.Size()} - } - - req3, err := http.NewRequest("PUT", registry+"/images/"+img.Id+"/layer", - ProgressReader(layerData, int(layerData.Size), stdout, "")) - if err != nil { - return err - } - - req3.ContentLength = -1 - req3.TransferEncoding = []string{"chunked"} - req3.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - res3, err := doWithCookies(client, req3) - if err != nil { - return fmt.Errorf("Failed to upload layer: %s", err) - } - defer res3.Body.Close() - - if res3.StatusCode != 200 { - errBody, err := ioutil.ReadAll(res3.Body) - if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when"+ - " trying to parse response body: %v", res.StatusCode, err) - } - return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res3.StatusCode, errBody) - } - return nil -} - -// push a tag on the registry. -// Remote has the format '/ -func (graph *Graph) pushTag(remote, revision, tag, registry string, token []string) error { - // "jsonify" the string - revision = "\"" + revision + "\"" - registry = "https://" + registry + "/v1" - - Debugf("Pushing tags for rev [%s] on {%s}\n", revision, registry+"/users/"+remote+"/"+tag) - - client := graph.getHttpClient() - req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) - if err != nil { - return err - } - req.Header.Add("Content-type", "application/json") - req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - req.ContentLength = int64(len(revision)) - res, err := doWithCookies(client, req) - if err != nil { - return err - } - res.Body.Close() - if res.StatusCode != 200 && res.StatusCode != 201 { - return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) - } - return nil -} - -// FIXME: this should really be PushTag -func (graph *Graph) pushPrimitive(stdout io.Writer, remote, tag, imgId, registry string, token []string) error { - // Check if the local impage exists - img, err := graph.Get(imgId) - if err != nil { - fmt.Fprintf(stdout, "Skipping tag %s:%s: %s does not exist\r\n", remote, tag, imgId) - return nil - } - fmt.Fprintf(stdout, "Pushing image %s:%s\r\n", remote, tag) - // Push the image - if err = graph.PushImage(stdout, img, registry, token); err != nil { - return err - } - fmt.Fprintf(stdout, "Registering tag %s:%s\r\n", remote, tag) - // And then the tag - if err = graph.pushTag(remote, imgId, tag, registry, token); err != nil { - return err - } - return nil -} - -// Retrieve the checksum of an image -// Priority: -// - Check on the stored checksums -// - Check if the archive exists, if it does not, ask the registry -// - If the archive does exists, process the checksum from it -// - If the archive does not exists and not found on registry, process checksum from layer -func (graph *Graph) getChecksum(imageId string) (string, error) { - // FIXME: Use in-memory map instead of reading the file each time - if sums, err := graph.getStoredChecksums(); err != nil { - return "", err - } else if checksum, exists := sums[imageId]; exists { - return checksum, nil - } - - img, err := graph.Get(imageId) - if err != nil { - return "", err - } - - if _, err := os.Stat(layerArchivePath(graph.imageRoot(imageId))); err != nil { - if os.IsNotExist(err) { - // TODO: Ask the registry for the checksum - // As the archive is not there, it is supposed to come from a pull. - } else { - return "", err - } - } - - checksum, err := img.Checksum() - if err != nil { - return "", err - } - return checksum, nil -} - -type ImgListJson struct { - Id string `json:"id"` - Checksum string `json:"checksum,omitempty"` - tag string -} - -// Push a repository to the registry. -// Remote has the format '/ -func (graph *Graph) PushRepository(stdout io.Writer, remote string, localRepo Repository, authConfig *auth.AuthConfig) error { - client := graph.getHttpClient() - // FIXME: Do not reset the cookie each time? (need to reset it in case updating latest of a repo and repushing) - client.Jar = cookiejar.NewCookieJar() - var imgList []*ImgListJson - - fmt.Fprintf(stdout, "Processing checksums\n") - imageSet := make(map[string]struct{}) - - for tag, id := range localRepo { - img, err := graph.Get(id) - if err != nil { - return err - } - img.WalkHistory(func(img *Image) error { - if _, exists := imageSet[img.Id]; exists { - return nil - } - imageSet[img.Id] = struct{}{} - checksum, err := graph.getChecksum(img.Id) - if err != nil { - return err - } - imgList = append([]*ImgListJson{{ - Id: img.Id, - Checksum: checksum, - tag: tag, - }}, imgList...) - return nil - }) - } - - imgListJson, err := json.Marshal(imgList) - if err != nil { - return err - } - - Debugf("json sent: %s\n", imgListJson) - - fmt.Fprintf(stdout, "Sending image list\n") - req, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/", bytes.NewReader(imgListJson)) - if err != nil { - return err - } - req.SetBasicAuth(authConfig.Username, authConfig.Password) - req.ContentLength = int64(len(imgListJson)) - req.Header.Set("X-Docker-Token", "true") - - res, err := client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - - for res.StatusCode >= 300 && res.StatusCode < 400 { - Debugf("Redirected to %s\n", res.Header.Get("Location")) - req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) - if err != nil { - return err - } - req.SetBasicAuth(authConfig.Username, authConfig.Password) - req.ContentLength = int64(len(imgListJson)) - req.Header.Set("X-Docker-Token", "true") - - res, err = client.Do(req) - if err != nil { - return err - } - defer res.Body.Close() - } - - if res.StatusCode != 200 && res.StatusCode != 201 { - errBody, err := ioutil.ReadAll(res.Body) - if err != nil { - return err - } - return fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) - } - - var token, endpoints []string - if res.Header.Get("X-Docker-Token") != "" { - token = res.Header["X-Docker-Token"] - Debugf("Auth token: %v", token) - } else { - return fmt.Errorf("Index response didn't contain an access token") - } - if res.Header.Get("X-Docker-Endpoints") != "" { - endpoints = res.Header["X-Docker-Endpoints"] - } else { - return fmt.Errorf("Index response didn't contain any endpoints") - } - - // FIXME: Send only needed images - for _, registry := range endpoints { - fmt.Fprintf(stdout, "Pushing repository %s to %s (%d tags)\r\n", remote, registry, len(localRepo)) - // For each image within the repo, push them - for _, elem := range imgList { - if err := graph.pushPrimitive(stdout, remote, elem.tag, elem.Id, registry, token); err != nil { - // FIXME: Continue on error? - return err - } - } - } - - req2, err := http.NewRequest("PUT", INDEX_ENDPOINT+"/repositories/"+remote+"/images", bytes.NewReader(imgListJson)) - if err != nil { - return err - } - req2.SetBasicAuth(authConfig.Username, authConfig.Password) - req2.Header["X-Docker-Endpoints"] = endpoints - req2.ContentLength = int64(len(imgListJson)) - res2, err := client.Do(req2) - if err != nil { - return err - } - defer res2.Body.Close() - if res2.StatusCode != 204 { - if errBody, err := ioutil.ReadAll(res2.Body); err != nil { - return err - } else { - return fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res2.StatusCode, remote, errBody) - } - } - - return nil -} - -type SearchResults struct { - Query string `json:"query"` - NumResults int `json:"num_results"` - Results []map[string]string `json:"results"` -} - -func (graph *Graph) SearchRepositories(stdout io.Writer, term string) (*SearchResults, error) { - client := graph.getHttpClient() - u := INDEX_ENDPOINT + "/search?q=" + url.QueryEscape(term) - req, err := http.NewRequest("GET", u, nil) - if err != nil { - return nil, err - } - res, err := client.Do(req) - if err != nil { - return nil, err - } - defer res.Body.Close() - if res.StatusCode != 200 { - return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode) - } - rawData, err := ioutil.ReadAll(res.Body) - if err != nil { - return nil, err - } - result := new(SearchResults) - err = json.Unmarshal(rawData, result) - return result, err -} diff --git a/registry/registry.go b/registry/registry.go new file mode 100644 index 0000000000..e2ffb292c5 --- /dev/null +++ b/registry/registry.go @@ -0,0 +1,472 @@ +package registry + +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/utils" + "github.com/shin-/cookiejar" + "io" + "io/ioutil" + "net/http" + "net/url" + "strings" +) + +var ErrAlreadyExists error = errors.New("Image already exists") + +func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { + for _, cookie := range c.Jar.Cookies(req.URL) { + req.AddCookie(cookie) + } + return c.Do(req) +} + +// Retrieve the history of a given image from the Registry. +// Return a list of the parent's json (requested image included) +func (r *Registry) GetRemoteHistory(imgId, registry string, token []string) ([]string, error) { + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/ancestry", nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := r.client.Do(req) + if err != nil || res.StatusCode != 200 { + if res != nil { + return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgId) + } + return nil, err + } + defer res.Body.Close() + + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Error while reading the http response: %s", err) + } + + utils.Debugf("Ancestry: %s", jsonString) + history := new([]string) + if err := json.Unmarshal(jsonString, history); err != nil { + return nil, err + } + return *history, nil +} + +// Check if an image exists in the Registry +func (r *Registry) LookupRemoteImage(imgId, registry string, authConfig *auth.AuthConfig) bool { + rt := &http.Transport{Proxy: http.ProxyFromEnvironment} + + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + if err != nil { + return false + } + req.SetBasicAuth(authConfig.Username, authConfig.Password) + res, err := rt.RoundTrip(req) + return err == nil && res.StatusCode == 307 +} + +func (r *Registry) getImagesInRepository(repository string, authConfig *auth.AuthConfig) ([]map[string]string, error) { + u := auth.IndexServerAddress() + "/repositories/" + repository + "/images" + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + if authConfig != nil && len(authConfig.Username) > 0 { + req.SetBasicAuth(authConfig.Username, authConfig.Password) + } + res, err := r.client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + // Repository doesn't exist yet + if res.StatusCode == 404 { + return nil, nil + } + + jsonData, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + + imageList := []map[string]string{} + if err := json.Unmarshal(jsonData, &imageList); err != nil { + utils.Debugf("Body: %s (%s)\n", res.Body, u) + return nil, err + } + + return imageList, nil +} + +// Retrieve an image from the Registry. +// Returns the Image object as well as the layer as an Archive (io.Reader) +func (r *Registry) GetRemoteImageJson(imgId, registry string, token []string) ([]byte, error) { + // Get the Json + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/json", nil) + if err != nil { + return nil, fmt.Errorf("Failed to download json: %s", err) + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := r.client.Do(req) + if err != nil { + return nil, fmt.Errorf("Failed to download json: %s", err) + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, fmt.Errorf("HTTP code %d", res.StatusCode) + } + jsonString, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, fmt.Errorf("Failed to parse downloaded json: %s (%s)", err, jsonString) + } + return jsonString, nil +} + +func (r *Registry) GetRemoteImageLayer(imgId, registry string, token []string) (io.ReadCloser, int, error) { + req, err := http.NewRequest("GET", registry+"/images/"+imgId+"/layer", nil) + if err != nil { + return nil, -1, fmt.Errorf("Error while getting from the server: %s\n", err) + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := r.client.Do(req) + if err != nil { + return nil, -1, err + } + return res.Body, int(res.ContentLength), nil +} + +func (r *Registry) GetRemoteTags(registries []string, repository string, token []string) (map[string]string, error) { + if strings.Count(repository, "/") == 0 { + // This will be removed once the Registry supports auto-resolution on + // the "library" namespace + repository = "library/" + repository + } + for _, host := range registries { + endpoint := fmt.Sprintf("https://%s/v1/repositories/%s/tags", host, repository) + req, err := http.NewRequest("GET", endpoint, nil) + if err != nil { + return nil, err + } + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := r.client.Do(req) + defer res.Body.Close() + utils.Debugf("Got status code %d from %s", res.StatusCode, endpoint) + if err != nil || (res.StatusCode != 200 && res.StatusCode != 404) { + continue + } else if res.StatusCode == 404 { + return nil, fmt.Errorf("Repository not found") + } + + result := make(map[string]string) + + rawJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + if err := json.Unmarshal(rawJson, &result); err != nil { + return nil, err + } + return result, nil + } + return nil, fmt.Errorf("Could not reach any registry endpoint") +} + +func (r *Registry) GetRepositoryData(remote string) (*RepositoryData, error) { + utils.Debugf("Pulling repository %s from %s\r\n", remote, auth.IndexServerAddress()) + repositoryTarget := auth.IndexServerAddress() + "/repositories/" + remote + "/images" + + req, err := http.NewRequest("GET", repositoryTarget, nil) + if err != nil { + return nil, err + } + if r.authConfig != nil && len(r.authConfig.Username) > 0 { + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + } + req.Header.Set("X-Docker-Token", "true") + + res, err := r.client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode == 401 { + return nil, fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + } + // TODO: Right now we're ignoring checksums in the response body. + // In the future, we need to use them to check image validity. + if res.StatusCode != 200 { + return nil, fmt.Errorf("HTTP code: %d", res.StatusCode) + } + + var tokens []string + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + } + + var endpoints []string + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return nil, fmt.Errorf("Index response didn't contain any endpoints") + } + + checksumsJson, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + remoteChecksums := []*ImgData{} + if err := json.Unmarshal(checksumsJson, &remoteChecksums); err != nil { + return nil, err + } + + // Forge a better object from the retrieved data + imgsData := make(map[string]*ImgData) + for _, elem := range remoteChecksums { + imgsData[elem.Id] = elem + } + + return &RepositoryData{ + ImgList: imgsData, + Endpoints: endpoints, + Tokens: tokens, + }, nil +} + +// Push a local image to the registry +func (r *Registry) PushImageJsonRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { + registry = "https://" + registry + "/v1" + // FIXME: try json with UTF8 + req, err := http.NewRequest("PUT", registry+"/images/"+imgData.Id+"/json", strings.NewReader(string(jsonRaw))) + if err != nil { + return err + } + req.Header.Add("Content-type", "application/json") + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + req.Header.Set("X-Docker-Checksum", imgData.Checksum) + + utils.Debugf("Setting checksum for %s: %s", imgData.Id, imgData.Checksum) + res, err := doWithCookies(r.client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if len(res.Cookies()) > 0 { + r.client.Jar.SetCookies(req.URL, res.Cookies()) + } + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + return ErrAlreadyExists + } + return fmt.Errorf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody) + } + return nil +} + +func (r *Registry) PushImageLayerRegistry(imgId string, layer io.Reader, registry string, token []string) error { + registry = "https://" + registry + "/v1" + req, err := http.NewRequest("PUT", registry+"/images/"+imgId+"/layer", layer) + if err != nil { + return err + } + req.ContentLength = -1 + req.TransferEncoding = []string{"chunked"} + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + res, err := doWithCookies(r.client, req) + if err != nil { + return fmt.Errorf("Failed to upload layer: %s", err) + } + defer res.Body.Close() + + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + } + return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + } + return nil +} + +// push a tag on the registry. +// Remote has the format '/ +func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token []string) error { + // "jsonify" the string + revision = "\"" + revision + "\"" + registry = "https://" + registry + "/v1" + + req, err := http.NewRequest("PUT", registry+"/repositories/"+remote+"/tags/"+tag, strings.NewReader(revision)) + if err != nil { + return err + } + req.Header.Add("Content-type", "application/json") + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + req.ContentLength = int64(len(revision)) + res, err := doWithCookies(r.client, req) + if err != nil { + return err + } + res.Body.Close() + if res.StatusCode != 200 && res.StatusCode != 201 { + return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) + } + return nil +} + +func (r *Registry) PushImageJsonIndex(remote string, imgList []*ImgData, validate bool) (*RepositoryData, error) { + imgListJson, err := json.Marshal(imgList) + if err != nil { + return nil, err + } + + utils.Debugf("json sent: %s\n", imgListJson) + + req, err := http.NewRequest("PUT", auth.IndexServerAddress()+"/repositories/"+remote+"/", bytes.NewReader(imgListJson)) + if err != nil { + return nil, err + } + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + req.ContentLength = int64(len(imgListJson)) + req.Header.Set("X-Docker-Token", "true") + + res, err := r.client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + + // Redirect if necessary + for res.StatusCode >= 300 && res.StatusCode < 400 { + utils.Debugf("Redirected to %s\n", res.Header.Get("Location")) + req, err = http.NewRequest("PUT", res.Header.Get("Location"), bytes.NewReader(imgListJson)) + if err != nil { + return nil, err + } + req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) + req.ContentLength = int64(len(imgListJson)) + req.Header.Set("X-Docker-Token", "true") + + res, err = r.client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + } + + if res.StatusCode != 200 && res.StatusCode != 201 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) + } + + var tokens []string + if res.Header.Get("X-Docker-Token") != "" { + tokens = res.Header["X-Docker-Token"] + utils.Debugf("Auth token: %v", tokens) + } else { + return nil, fmt.Errorf("Index response didn't contain an access token") + } + + var endpoints []string + if res.Header.Get("X-Docker-Endpoints") != "" { + endpoints = res.Header["X-Docker-Endpoints"] + } else { + return nil, fmt.Errorf("Index response didn't contain any endpoints") + } + + if validate { + if res.StatusCode != 204 { + if errBody, err := ioutil.ReadAll(res.Body); err != nil { + return nil, err + } else { + return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody) + } + } + } + + return &RepositoryData{ + Tokens: tokens, + Endpoints: endpoints, + }, nil +} + +func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { + u := auth.IndexServerAddress() + "/search?q=" + url.QueryEscape(term) + req, err := http.NewRequest("GET", u, nil) + if err != nil { + return nil, err + } + res, err := r.client.Do(req) + if err != nil { + return nil, err + } + defer res.Body.Close() + if res.StatusCode != 200 { + return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode) + } + rawData, err := ioutil.ReadAll(res.Body) + if err != nil { + return nil, err + } + result := new(SearchResults) + err = json.Unmarshal(rawData, result) + return result, err +} + +func (r *Registry) ResetClient(authConfig *auth.AuthConfig) { + r.authConfig = authConfig + r.client.Jar = cookiejar.NewCookieJar() +} + +func (r *Registry) GetAuthConfig() *auth.AuthConfig { + return &auth.AuthConfig{ + Username: r.authConfig.Username, + Email: r.authConfig.Email, + } +} + +type SearchResults struct { + Query string `json:"query"` + NumResults int `json:"num_results"` + Results []map[string]string `json:"results"` +} + +type RepositoryData struct { + ImgList map[string]*ImgData + Endpoints []string + Tokens []string +} + +type ImgData struct { + Id string `json:"id"` + Checksum string `json:"checksum,omitempty"` + Tag string `json:",omitempty"` +} + +type Registry struct { + client *http.Client + authConfig *auth.AuthConfig +} + +func NewRegistry(root string) *Registry { + // If the auth file does not exist, keep going + authConfig, _ := auth.LoadConfig(root) + + r := &Registry{ + authConfig: authConfig, + client: &http.Client{}, + } + r.client.Jar = cookiejar.NewCookieJar() + return r +} diff --git a/registry/registry_test.go b/registry/registry_test.go new file mode 100644 index 0000000000..fd955b7b73 --- /dev/null +++ b/registry/registry_test.go @@ -0,0 +1,168 @@ +package registry + +// import ( +// "crypto/rand" +// "encoding/hex" +// "github.com/dotcloud/docker" +// "github.com/dotcloud/docker/auth" +// "io/ioutil" +// "os" +// "path" +// "testing" +// ) + +// func newTestRuntime() (*Runtime, error) { +// root, err := ioutil.TempDir("", "docker-test") +// if err != nil { +// return nil, err +// } +// if err := os.Remove(root); err != nil { +// return nil, err +// } + +// if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) { +// return nil, err +// } + +// return runtime, nil +// } + +// func TestPull(t *testing.T) { +// os.Setenv("DOCKER_INDEX_URL", "") +// runtime, err := newTestRuntime() +// if err != nil { +// t.Fatal(err) +// } +// defer nuke(runtime) + +// err = runtime.graph.PullRepository(ioutil.Discard, "busybox", "", runtime.repositories, nil) +// if err != nil { +// t.Fatal(err) +// } +// img, err := runtime.repositories.LookupImage("busybox") +// if err != nil { +// t.Fatal(err) +// } + +// // Try to run something on this image to make sure the layer's been downloaded properly. +// config, _, err := docker.ParseRun([]string{img.Id, "echo", "Hello World"}, runtime.capabilities) +// if err != nil { +// t.Fatal(err) +// } + +// b := NewBuilder(runtime) +// container, err := b.Create(config) +// if err != nil { +// t.Fatal(err) +// } +// if err := container.Start(); err != nil { +// t.Fatal(err) +// } + +// if status := container.Wait(); status != 0 { +// t.Fatalf("Expected status code 0, found %d instead", status) +// } +// } + +// func TestPullTag(t *testing.T) { +// os.Setenv("DOCKER_INDEX_URL", "") +// runtime, err := newTestRuntime() +// if err != nil { +// t.Fatal(err) +// } +// defer nuke(runtime) + +// err = runtime.graph.PullRepository(ioutil.Discard, "ubuntu", "12.04", runtime.repositories, nil) +// if err != nil { +// t.Fatal(err) +// } +// _, err = runtime.repositories.LookupImage("ubuntu:12.04") +// if err != nil { +// t.Fatal(err) +// } + +// img2, err := runtime.repositories.LookupImage("ubuntu:12.10") +// if img2 != nil { +// t.Fatalf("Expected nil image but found %v instead", img2.Id) +// } +// } + +// func login(runtime *Runtime) error { +// authConfig := auth.NewAuthConfig("unittester", "surlautrerivejetattendrai", "noise+unittester@dotcloud.com", runtime.root) +// runtime.authConfig = authConfig +// _, err := auth.Login(authConfig) +// return err +// } + +// func TestPush(t *testing.T) { +// os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") +// defer os.Setenv("DOCKER_INDEX_URL", "") +// runtime, err := newTestRuntime() +// if err != nil { +// t.Fatal(err) +// } +// defer nuke(runtime) + +// err = login(runtime) +// if err != nil { +// t.Fatal(err) +// } + +// err = runtime.graph.PullRepository(ioutil.Discard, "joffrey/busybox", "", runtime.repositories, nil) +// if err != nil { +// t.Fatal(err) +// } +// tokenBuffer := make([]byte, 16) +// _, err = rand.Read(tokenBuffer) +// if err != nil { +// t.Fatal(err) +// } +// token := hex.EncodeToString(tokenBuffer)[:29] +// config, _, err := ParseRun([]string{"joffrey/busybox", "touch", "/" + token}, runtime.capabilities) +// if err != nil { +// t.Fatal(err) +// } + +// b := NewBuilder(runtime) +// container, err := b.Create(config) +// if err != nil { +// t.Fatal(err) +// } +// if err := container.Start(); err != nil { +// t.Fatal(err) +// } + +// if status := container.Wait(); status != 0 { +// t.Fatalf("Expected status code 0, found %d instead", status) +// } + +// img, err := b.Commit(container, "unittester/"+token, "", "", "", nil) +// if err != nil { +// t.Fatal(err) +// } + +// repo := runtime.repositories.Repositories["unittester/"+token] +// err = runtime.graph.PushRepository(ioutil.Discard, "unittester/"+token, repo, runtime.authConfig) +// if err != nil { +// t.Fatal(err) +// } + +// // Remove image so we can pull it again +// if err := runtime.graph.Delete(img.Id); err != nil { +// t.Fatal(err) +// } + +// err = runtime.graph.PullRepository(ioutil.Discard, "unittester/"+token, "", runtime.repositories, runtime.authConfig) +// if err != nil { +// t.Fatal(err) +// } + +// layerPath, err := img.layer() +// if err != nil { +// t.Fatal(err) +// } + +// if _, err := os.Stat(path.Join(layerPath, token)); err != nil { +// t.Fatalf("Error while trying to retrieve token file: %v", err) +// } +// } diff --git a/runtime.go b/runtime.go index f2914dba21..3c190e934e 100644 --- a/runtime.go +++ b/runtime.go @@ -3,7 +3,7 @@ package docker import ( "container/list" "fmt" - "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/utils" "io" "io/ioutil" "log" @@ -26,18 +26,18 @@ type Runtime struct { networkManager *NetworkManager graph *Graph repositories *TagStore - authConfig *auth.AuthConfig - idIndex *TruncIndex + idIndex *utils.TruncIndex capabilities *Capabilities - kernelVersion *KernelVersionInfo + kernelVersion *utils.KernelVersionInfo autoRestart bool volumes *Graph + srv *Server } var sysInitPath string func init() { - sysInitPath = SelfPath() + sysInitPath = utils.SelfPath() } func (runtime *Runtime) List() []*Container { @@ -113,13 +113,13 @@ func (runtime *Runtime) Register(container *Container) error { container.runtime = runtime // Attach to stdout and stderr - container.stderr = newWriteBroadcaster() - container.stdout = newWriteBroadcaster() + container.stderr = utils.NewWriteBroadcaster() + container.stdout = utils.NewWriteBroadcaster() // Attach to stdin if container.Config.OpenStdin { container.stdin, container.stdinPipe = io.Pipe() } else { - container.stdinPipe = NopWriteCloser(ioutil.Discard) // Silently drop stdin + container.stdinPipe = utils.NopWriteCloser(ioutil.Discard) // Silently drop stdin } // done runtime.containers.PushBack(container) @@ -137,9 +137,9 @@ func (runtime *Runtime) Register(container *Container) error { return err } else { if !strings.Contains(string(output), "RUNNING") { - Debugf("Container %s was supposed to be running be is not.", container.Id) + utils.Debugf("Container %s was supposed to be running be is not.", container.Id) if runtime.autoRestart { - Debugf("Restarting") + utils.Debugf("Restarting") container.State.Ghost = false container.State.setStopped(0) if err := container.Start(); err != nil { @@ -147,7 +147,7 @@ func (runtime *Runtime) Register(container *Container) error { } nomonitor = true } else { - Debugf("Marking as stopped") + utils.Debugf("Marking as stopped") container.State.setStopped(-127) if err := container.ToDisk(); err != nil { return err @@ -168,7 +168,7 @@ func (runtime *Runtime) Register(container *Container) error { return nil } -func (runtime *Runtime) LogToDisk(src *writeBroadcaster, dst string) error { +func (runtime *Runtime) LogToDisk(src *utils.WriteBroadcaster, dst string) error { log, err := os.OpenFile(dst, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600) if err != nil { return err @@ -215,16 +215,16 @@ func (runtime *Runtime) restore() error { id := v.Name() container, err := runtime.Load(id) if err != nil { - Debugf("Failed to load container %v: %v", id, err) + utils.Debugf("Failed to load container %v: %v", id, err) continue } - Debugf("Loaded container %v", container.Id) + utils.Debugf("Loaded container %v", container.Id) } return nil } func (runtime *Runtime) UpdateCapabilities(quiet bool) { - if cgroupMemoryMountpoint, err := FindCgroupMountpoint("memory"); err != nil { + if cgroupMemoryMountpoint, err := utils.FindCgroupMountpoint("memory"); err != nil { if !quiet { log.Printf("WARNING: %s\n", err) } @@ -251,11 +251,11 @@ func NewRuntime(autoRestart bool) (*Runtime, error) { return nil, err } - if k, err := GetKernelVersion(); err != nil { + if k, err := utils.GetKernelVersion(); err != nil { log.Printf("WARNING: %s\n", err) } else { runtime.kernelVersion = k - if CompareKernelVersion(k, &KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}) < 0 { + if utils.CompareKernelVersion(k, &utils.KernelVersionInfo{Kernel: 3, Major: 8, Minor: 0}) < 0 { log.Printf("WARNING: You are running linux kernel version %s, which might be unstable running docker. Please upgrade your kernel to 3.8.0.", k.String()) } } @@ -289,11 +289,6 @@ func NewRuntimeFromDirectory(root string, autoRestart bool) (*Runtime, error) { if err != nil { return nil, err } - authConfig, err := auth.LoadConfig(root) - if err != nil && authConfig == nil { - // If the auth file does not exist, keep going - return nil, err - } runtime := &Runtime{ root: root, repository: runtimeRepo, @@ -301,8 +296,7 @@ func NewRuntimeFromDirectory(root string, autoRestart bool) (*Runtime, error) { networkManager: netManager, graph: g, repositories: repositories, - authConfig: authConfig, - idIndex: NewTruncIndex(), + idIndex: utils.NewTruncIndex(), capabilities: &Capabilities{}, autoRestart: autoRestart, volumes: volumes, diff --git a/runtime_test.go b/runtime_test.go index 64956baa67..01bd2a0128 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -2,6 +2,8 @@ package docker import ( "fmt" + "github.com/dotcloud/docker/registry" + "github.com/dotcloud/docker/utils" "io" "io/ioutil" "net" @@ -48,7 +50,7 @@ func layerArchive(tarfile string) (io.Reader, error) { func init() { // Hack to run sys init during unit testing - if SelfPath() == "/sbin/init" { + if utils.SelfPath() == "/sbin/init" { SysInit() return } @@ -69,7 +71,8 @@ func init() { // Create the "Server" srv := &Server{ - runtime: runtime, + runtime: runtime, + registry: registry.NewRegistry(runtime.root), } // Retrieve the Image if err := srv.ImagePull(unitTestImageName, "", "", os.Stdout); err != nil { diff --git a/server.go b/server.go index 453574946d..2f45802ca6 100644 --- a/server.go +++ b/server.go @@ -2,11 +2,15 @@ package docker import ( "fmt" + "github.com/dotcloud/docker/registry" + "github.com/dotcloud/docker/utils" "io" + "io/ioutil" "log" "net/http" "net/url" "os" + "path" "runtime" "strings" ) @@ -44,7 +48,7 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error { } func (srv *Server) ImagesSearch(term string) ([]ApiSearch, error) { - results, err := srv.runtime.graph.SearchRepositories(nil, term) + results, err := srv.registry.SearchRepositories(term) if err != nil { return nil, err } @@ -54,7 +58,7 @@ func (srv *Server) ImagesSearch(term string) ([]ApiSearch, error) { var out ApiSearch out.Description = repo["description"] if len(out.Description) > 45 { - out.Description = Trunc(out.Description, 42) + "..." + out.Description = utils.Trunc(out.Description, 42) + "..." } out.Name = repo["name"] outs = append(outs, out) @@ -68,7 +72,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer) error { return err } - file, err := Download(url, out) + file, err := utils.Download(url, out) if err != nil { return err } @@ -85,7 +89,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer) error { return err } - if err := c.Inject(ProgressReader(file.Body, int(file.ContentLength), out, "Downloading %v/%v (%v)"), path); err != nil { + if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, "Downloading %v/%v (%v)"), path); err != nil { return err } // FIXME: Handle custom repo, tag comment, author @@ -124,7 +128,7 @@ func (srv *Server) ImagesViz(out io.Writer) error { for name, repository := range srv.runtime.repositories.Repositories { for tag, id := range repository { - reporefs[TruncateId(id)] = append(reporefs[TruncateId(id)], fmt.Sprintf("%s:%s", name, tag)) + reporefs[utils.TruncateId(id)] = append(reporefs[utils.TruncateId(id)], fmt.Sprintf("%s:%s", name, tag)) } } @@ -193,7 +197,7 @@ func (srv *Server) DockerInfo() ApiInfo { out.GoVersion = runtime.Version() if os.Getenv("DEBUG") != "" { out.Debug = true - out.NFd = getTotalUsedFds() + out.NFd = utils.GetTotalUsedFds() out.NGoroutines = runtime.NumGoroutine() } return out @@ -283,14 +287,272 @@ func (srv *Server) ContainerTag(name, repo, tag string, force bool) error { return nil } +func (srv *Server) pullImage(out io.Writer, imgId, registry string, token []string) error { + history, err := srv.registry.GetRemoteHistory(imgId, registry, token) + if err != nil { + return err + } + + // FIXME: Try to stream the images? + // FIXME: Launch the getRemoteImage() in goroutines + for _, id := range history { + if !srv.runtime.graph.Exists(id) { + fmt.Fprintf(out, "Pulling %s metadata\r\n", id) + imgJson, err := srv.registry.GetRemoteImageJson(id, registry, token) + if err != nil { + // FIXME: Keep goging in case of error? + return err + } + img, err := NewImgJson(imgJson) + if err != nil { + return fmt.Errorf("Failed to parse json: %s", err) + } + + // Get the layer + fmt.Fprintf(out, "Pulling %s fs layer\r\n", img.Id) + layer, contentLength, err := srv.registry.GetRemoteImageLayer(img.Id, registry, token) + if err != nil { + return err + } + if err := srv.runtime.graph.Register(utils.ProgressReader(layer, contentLength, out, "Downloading %v/%v (%v)"), false, img); err != nil { + return err + } + } + } + return nil +} + +func (srv *Server) pullRepository(stdout io.Writer, remote, askedTag string) error { + utils.Debugf("Retrieving repository data") + repoData, err := srv.registry.GetRepositoryData(remote) + if err != nil { + return err + } + + utils.Debugf("Updating checksums") + // Reload the json file to make sure not to overwrite faster sums + if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { + return err + } + + utils.Debugf("Retrieving the tag list") + tagsList, err := srv.registry.GetRemoteTags(repoData.Endpoints, remote, repoData.Tokens) + if err != nil { + return err + } + for tag, id := range tagsList { + repoData.ImgList[id].Tag = tag + } + + for _, img := range repoData.ImgList { + // If we asked for a specific tag, skip all tags expect the wanted one + if askedTag != "" && askedTag != img.Tag { + continue + } + fmt.Fprintf(stdout, "Pulling image %s (%s) from %s\n", img.Id, img.Tag, remote) + success := false + for _, ep := range repoData.Endpoints { + if err := srv.pullImage(stdout, img.Id, "https://"+ep+"/v1", repoData.Tokens); err != nil { + fmt.Fprintf(stdout, "Error while retrieving image for tag: %s (%s); checking next endpoint\n", askedTag, err) + continue + } + if err := srv.runtime.repositories.Set(remote, img.Tag, img.Id, true); err != nil { + return err + } + success = true + delete(tagsList, img.Tag) + break + } + if !success { + return fmt.Errorf("Could not find repository on any of the indexed registries.") + } + } + for tag, id := range tagsList { + if err := srv.runtime.repositories.Set(remote, tag, id, true); err != nil { + return err + } + } + if err := srv.runtime.repositories.Save(); err != nil { + return err + } + + return nil +} + func (srv *Server) ImagePull(name, tag, registry string, out io.Writer) error { if registry != "" { - if err := srv.runtime.graph.PullImage(out, name, registry, nil); err != nil { + if err := srv.pullImage(out, name, registry, nil); err != nil { return err } return nil } - if err := srv.runtime.graph.PullRepository(out, name, tag, srv.runtime.repositories, srv.runtime.authConfig); err != nil { + + if err := srv.pullRepository(out, name, tag); err != nil { + return err + } + + return nil +} + +// Retrieve the checksum of an image +// Priority: +// - Check on the stored checksums +// - Check if the archive exists, if it does not, ask the registry +// - If the archive does exists, process the checksum from it +// - If the archive does not exists and not found on registry, process checksum from layer +func (srv *Server) getChecksum(imageId string) (string, error) { + // FIXME: Use in-memory map instead of reading the file each time + if sums, err := srv.runtime.graph.getStoredChecksums(); err != nil { + return "", err + } else if checksum, exists := sums[imageId]; exists { + return checksum, nil + } + + img, err := srv.runtime.graph.Get(imageId) + if err != nil { + return "", err + } + + if _, err := os.Stat(layerArchivePath(srv.runtime.graph.imageRoot(imageId))); err != nil { + if os.IsNotExist(err) { + // TODO: Ask the registry for the checksum + // As the archive is not there, it is supposed to come from a pull. + } else { + return "", err + } + } + + checksum, err := img.Checksum() + if err != nil { + return "", err + } + return checksum, nil +} + +// Retrieve the all the images to be uploaded in the correct order +// Note: we can't use a map as it is not ordered +func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgData, error) { + var imgList []*registry.ImgData + + imageSet := make(map[string]struct{}) + for tag, id := range localRepo { + img, err := srv.runtime.graph.Get(id) + if err != nil { + return nil, err + } + img.WalkHistory(func(img *Image) error { + if _, exists := imageSet[img.Id]; exists { + return nil + } + imageSet[img.Id] = struct{}{} + checksum, err := srv.getChecksum(img.Id) + if err != nil { + return err + } + imgList = append([]*registry.ImgData{{ + Id: img.Id, + Checksum: checksum, + Tag: tag, + }}, imgList...) + return nil + }) + } + return imgList, nil +} + +func (srv *Server) pushRepository(out io.Writer, name string, localRepo map[string]string) error { + fmt.Fprintf(out, "Processing checksums\n") + imgList, err := srv.getImageList(localRepo) + if err != nil { + return err + } + fmt.Fprintf(out, "Sending image list\n") + + repoData, err := srv.registry.PushImageJsonIndex(name, imgList, false) + if err != nil { + return err + } + + // FIXME: Send only needed images + for _, ep := range repoData.Endpoints { + fmt.Fprintf(out, "Pushing repository %s to %s (%d tags)\r\n", name, ep, len(localRepo)) + // For each image within the repo, push them + for _, elem := range imgList { + if _, exists := repoData.ImgList[elem.Id]; exists { + fmt.Fprintf(out, "Image %s already on registry, skipping\n", name) + continue + } + if err := srv.pushImage(out, name, elem.Id, ep, repoData.Tokens); err != nil { + // FIXME: Continue on error? + return err + } + fmt.Fprintf(out, "Pushing tags for rev [%s] on {%s}\n", elem.Id, ep+"/users/"+name+"/"+elem.Tag) + if err := srv.registry.PushRegistryTag(name, elem.Id, elem.Tag, ep, repoData.Tokens); err != nil { + return err + } + } + } + + if _, err := srv.registry.PushImageJsonIndex(name, imgList, true); err != nil { + return err + } + return nil +} + +func (srv *Server) pushImage(out io.Writer, remote, imgId, ep string, token []string) error { + jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgId, "json")) + if err != nil { + return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgId, err) + } + fmt.Fprintf(out, "Pushing %s\r\n", imgId) + + // Make sure we have the image's checksum + checksum, err := srv.getChecksum(imgId) + if err != nil { + return err + } + imgData := ®istry.ImgData{ + Id: imgId, + Checksum: checksum, + } + + // Send the json + if err := srv.registry.PushImageJsonRegistry(imgData, jsonRaw, ep, token); err != nil { + if err == registry.ErrAlreadyExists { + fmt.Fprintf(out, "Image %s already uploaded ; skipping\n", imgData.Id) + return nil + } + return err + } + + // Retrieve the tarball to be sent + var layerData *TempArchive + // If the archive exists, use it + file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgId))) + if err != nil { + if os.IsNotExist(err) { + // If the archive does not exist, create one from the layer + layerData, err = srv.runtime.graph.TempLayerArchive(imgId, Xz, out) + if err != nil { + return fmt.Errorf("Failed to generate layer archive: %s", err) + } + } else { + return err + } + } else { + defer file.Close() + st, err := file.Stat() + if err != nil { + return err + } + layerData = &TempArchive{ + File: file, + Size: st.Size(), + } + } + + // Send the layer + if err := srv.registry.PushImageLayerRegistry(imgData.Id, utils.ProgressReader(layerData, int(layerData.Size), out, ""), ep, token); err != nil { return err } return nil @@ -299,10 +561,10 @@ func (srv *Server) ImagePull(name, tag, registry string, out io.Writer) error { func (srv *Server) ImagePush(name, registry string, out io.Writer) error { img, err := srv.runtime.graph.Get(name) if err != nil { - Debugf("The push refers to a repository [%s] (len: %d)\n", name, len(srv.runtime.repositories.Repositories[name])) + fmt.Fprintf(out, "The push refers to a repository [%s] (len: %d)\n", name, len(srv.runtime.repositories.Repositories[name])) // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[name]; exists { - if err := srv.runtime.graph.PushRepository(out, name, localRepo, srv.runtime.authConfig); err != nil { + if err := srv.pushRepository(out, name, localRepo); err != nil { return err } return nil @@ -310,8 +572,8 @@ func (srv *Server) ImagePush(name, registry string, out io.Writer) error { return err } - err = srv.runtime.graph.PushImage(out, img, registry, nil) - if err != nil { + fmt.Fprintf(out, "The push refers to an image: [%s]\n", name) + if err := srv.pushImage(out, name, img.Id, registry, nil); err != nil { return err } return nil @@ -336,11 +598,11 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write fmt.Fprintln(out, "Downloading from", u) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() - resp, err = Download(u.String(), out) + resp, err = utils.Download(u.String(), out) if err != nil { return err } - archive = ProgressReader(resp.Body, int(resp.ContentLength), out, "Importing %v/%v (%v)") + archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, "Importing %v/%v (%v)") } img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil) if err != nil { @@ -397,7 +659,6 @@ func (srv *Server) ContainerRestart(name string, t int) error { } func (srv *Server) ContainerDestroy(name string, removeVolume bool) error { - if container := srv.runtime.Get(name); container != nil { volumes := make(map[string]struct{}) // Store all the deleted containers volumes @@ -486,17 +747,17 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std if stdout { cLog, err := container.ReadLog("stdout") if err != nil { - Debugf(err.Error()) + utils.Debugf(err.Error()) } else if _, err := io.Copy(out, cLog); err != nil { - Debugf(err.Error()) + utils.Debugf(err.Error()) } } if stderr { cLog, err := container.ReadLog("stderr") if err != nil { - Debugf(err.Error()) + utils.Debugf(err.Error()) } else if _, err := io.Copy(out, cLog); err != nil { - Debugf(err.Error()) + utils.Debugf(err.Error()) } } } @@ -517,7 +778,7 @@ func (srv *Server) ContainerAttach(name string, logs, stream, stdin, stdout, std r, w := io.Pipe() go func() { defer w.Close() - defer Debugf("Closing buffered stdin pipe") + defer utils.Debugf("Closing buffered stdin pipe") io.Copy(w, in) }() cStdin = r @@ -564,11 +825,14 @@ func NewServer(autoRestart bool) (*Server, error) { return nil, err } srv := &Server{ - runtime: runtime, + runtime: runtime, + registry: registry.NewRegistry(runtime.root), } + runtime.srv = srv return srv, nil } type Server struct { - runtime *Runtime + runtime *Runtime + registry *registry.Registry } diff --git a/state.go b/state.go index f51a06b01a..a972e376a2 100644 --- a/state.go +++ b/state.go @@ -2,6 +2,7 @@ package docker import ( "fmt" + "github.com/dotcloud/docker/utils" "sync" "time" ) @@ -21,7 +22,7 @@ func (s *State) String() string { if s.Ghost { return fmt.Sprintf("Ghost") } - return fmt.Sprintf("Up %s", HumanDuration(time.Now().Sub(s.StartedAt))) + return fmt.Sprintf("Up %s", utils.HumanDuration(time.Now().Sub(s.StartedAt))) } return fmt.Sprintf("Exit %d", s.ExitCode) } diff --git a/tags.go b/tags.go index 1b9cd19c83..5bc2978e09 100644 --- a/tags.go +++ b/tags.go @@ -3,6 +3,7 @@ package docker import ( "encoding/json" "fmt" + "github.com/dotcloud/docker/utils" "io/ioutil" "os" "path/filepath" @@ -106,7 +107,7 @@ func (store *TagStore) ImageName(id string) string { if names, exists := store.ById()[id]; exists && len(names) > 0 { return names[0] } - return TruncateId(id) + return utils.TruncateId(id) } func (store *TagStore) Set(repoName, tag, imageName string, force bool) error { diff --git a/term/term.go b/term/term.go index 39b77dcc59..8c07b93356 100644 --- a/term/term.go +++ b/term/term.go @@ -1,6 +1,8 @@ package term import ( + "os" + "os/signal" "syscall" "unsafe" ) @@ -120,3 +122,22 @@ func Restore(fd int, state *State) error { _, _, err := syscall.Syscall6(syscall.SYS_IOCTL, uintptr(fd), uintptr(setTermios), uintptr(unsafe.Pointer(&state.termios)), 0, 0, 0) return err } + +func SetRawTerminal() (*State, error) { + oldState, err := MakeRaw(int(os.Stdin.Fd())) + if err != nil { + return nil, err + } + c := make(chan os.Signal, 1) + signal.Notify(c, os.Interrupt) + go func() { + _ = <-c + Restore(int(os.Stdin.Fd()), oldState) + os.Exit(0) + }() + return oldState, err +} + +func RestoreTerminal(state *State) { + Restore(int(os.Stdin.Fd()), state) +} diff --git a/utils.go b/utils.go index e7f7d319d9..d67f50e526 100644 --- a/utils.go +++ b/utils.go @@ -1,500 +1,5 @@ package docker -import ( - "bytes" - "crypto/sha256" - "encoding/hex" - "errors" - "fmt" - "github.com/dotcloud/docker/term" - "index/suffixarray" - "io" - "io/ioutil" - "net/http" - "os" - "os/exec" - "os/signal" - "path/filepath" - "runtime" - "strings" - "sync" - "time" -) - -// Go is a basic promise implementation: it wraps calls a function in a goroutine, -// and returns a channel which will later return the function's return value. -func Go(f func() error) chan error { - ch := make(chan error) - go func() { - ch <- f() - }() - return ch -} - -// Request a given URL and return an io.Reader -func Download(url string, stderr io.Writer) (*http.Response, error) { - var resp *http.Response - var err error = nil - if resp, err = http.Get(url); err != nil { - return nil, err - } - if resp.StatusCode >= 400 { - return nil, errors.New("Got HTTP status code >= 400: " + resp.Status) - } - return resp, nil -} - -// Debug function, if the debug flag is set, then display. Do nothing otherwise -// If Docker is in damon mode, also send the debug info on the socket -func Debugf(format string, a ...interface{}) { - if os.Getenv("DEBUG") != "" { - - // Retrieve the stack infos - _, file, line, ok := runtime.Caller(1) - if !ok { - file = "" - line = -1 - } else { - file = file[strings.LastIndex(file, "/")+1:] - } - - fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s:%d %s\n", file, line, format), a...) - } -} - -// Reader with progress bar -type progressReader struct { - reader io.ReadCloser // Stream to read from - output io.Writer // Where to send progress bar to - readTotal int // Expected stream length (bytes) - readProgress int // How much has been read so far (bytes) - lastUpdate int // How many bytes read at least update - template string // Template to print. Default "%v/%v (%v)" -} - -func (r *progressReader) Read(p []byte) (n int, err error) { - read, err := io.ReadCloser(r.reader).Read(p) - r.readProgress += read - - updateEvery := 4096 - if r.readTotal > 0 { - // Only update progress for every 1% read - if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery { - updateEvery = increment - } - } - if r.readProgress-r.lastUpdate > updateEvery || err != nil { - if r.readTotal > 0 { - fmt.Fprintf(r.output, r.template+"\r", r.readProgress, r.readTotal, fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100)) - } else { - fmt.Fprintf(r.output, r.template+"\r", r.readProgress, "?", "n/a") - } - r.lastUpdate = r.readProgress - } - // Send newline when complete - if err != nil { - fmt.Fprintf(r.output, "\n") - } - - return read, err -} -func (r *progressReader) Close() error { - return io.ReadCloser(r.reader).Close() -} -func ProgressReader(r io.ReadCloser, size int, output io.Writer, template string) *progressReader { - if template == "" { - template = "%v/%v (%v)" - } - return &progressReader{r, output, size, 0, 0, template} -} - -// HumanDuration returns a human-readable approximation of a duration -// (eg. "About a minute", "4 hours ago", etc.) -func HumanDuration(d time.Duration) string { - if seconds := int(d.Seconds()); seconds < 1 { - return "Less than a second" - } else if seconds < 60 { - return fmt.Sprintf("%d seconds", seconds) - } else if minutes := int(d.Minutes()); minutes == 1 { - return "About a minute" - } else if minutes < 60 { - return fmt.Sprintf("%d minutes", minutes) - } else if hours := int(d.Hours()); hours == 1 { - return "About an hour" - } else if hours < 48 { - return fmt.Sprintf("%d hours", hours) - } else if hours < 24*7*2 { - return fmt.Sprintf("%d days", hours/24) - } else if hours < 24*30*3 { - return fmt.Sprintf("%d weeks", hours/24/7) - } else if hours < 24*365*2 { - return fmt.Sprintf("%d months", hours/24/30) - } - return fmt.Sprintf("%d years", d.Hours()/24/365) -} - -func Trunc(s string, maxlen int) string { - if len(s) <= maxlen { - return s - } - return s[:maxlen] -} - -// Figure out the absolute path of our own binary -func SelfPath() string { - path, err := exec.LookPath(os.Args[0]) - if err != nil { - panic(err) - } - path, err = filepath.Abs(path) - if err != nil { - panic(err) - } - return path -} - -type nopWriter struct { -} - -func (w *nopWriter) Write(buf []byte) (int, error) { - return len(buf), nil -} - -type nopWriteCloser struct { - io.Writer -} - -func (w *nopWriteCloser) Close() error { return nil } - -func NopWriteCloser(w io.Writer) io.WriteCloser { - return &nopWriteCloser{w} -} - -type bufReader struct { - buf *bytes.Buffer - reader io.Reader - err error - l sync.Mutex - wait sync.Cond -} - -func newBufReader(r io.Reader) *bufReader { - reader := &bufReader{ - buf: &bytes.Buffer{}, - reader: r, - } - reader.wait.L = &reader.l - go reader.drain() - return reader -} - -func (r *bufReader) drain() { - buf := make([]byte, 1024) - for { - n, err := r.reader.Read(buf) - r.l.Lock() - if err != nil { - r.err = err - } else { - r.buf.Write(buf[0:n]) - } - r.wait.Signal() - r.l.Unlock() - if err != nil { - break - } - } -} - -func (r *bufReader) Read(p []byte) (n int, err error) { - r.l.Lock() - defer r.l.Unlock() - for { - n, err = r.buf.Read(p) - if n > 0 { - return n, err - } - if r.err != nil { - return 0, r.err - } - r.wait.Wait() - } - panic("unreachable") -} - -func (r *bufReader) Close() error { - closer, ok := r.reader.(io.ReadCloser) - if !ok { - return nil - } - return closer.Close() -} - -type writeBroadcaster struct { - mu sync.Mutex - writers map[io.WriteCloser]struct{} -} - -func (w *writeBroadcaster) AddWriter(writer io.WriteCloser) { - w.mu.Lock() - w.writers[writer] = struct{}{} - w.mu.Unlock() -} - -// FIXME: Is that function used? -// FIXME: This relies on the concrete writer type used having equality operator -func (w *writeBroadcaster) RemoveWriter(writer io.WriteCloser) { - w.mu.Lock() - delete(w.writers, writer) - w.mu.Unlock() -} - -func (w *writeBroadcaster) Write(p []byte) (n int, err error) { - w.mu.Lock() - defer w.mu.Unlock() - for writer := range w.writers { - if n, err := writer.Write(p); err != nil || n != len(p) { - // On error, evict the writer - delete(w.writers, writer) - } - } - return len(p), nil -} - -func (w *writeBroadcaster) CloseWriters() error { - w.mu.Lock() - defer w.mu.Unlock() - for writer := range w.writers { - writer.Close() - } - w.writers = make(map[io.WriteCloser]struct{}) - return nil -} - -func newWriteBroadcaster() *writeBroadcaster { - return &writeBroadcaster{writers: make(map[io.WriteCloser]struct{})} -} - -func getTotalUsedFds() int { - if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil { - Debugf("Error opening /proc/%d/fd: %s", os.Getpid(), err) - } else { - return len(fds) - } - return -1 -} - -// TruncIndex allows the retrieval of string identifiers by any of their unique prefixes. -// This is used to retrieve image and container IDs by more convenient shorthand prefixes. -type TruncIndex struct { - index *suffixarray.Index - ids map[string]bool - bytes []byte -} - -func NewTruncIndex() *TruncIndex { - return &TruncIndex{ - index: suffixarray.New([]byte{' '}), - ids: make(map[string]bool), - bytes: []byte{' '}, - } -} - -func (idx *TruncIndex) Add(id string) error { - if strings.Contains(id, " ") { - return fmt.Errorf("Illegal character: ' '") - } - if _, exists := idx.ids[id]; exists { - return fmt.Errorf("Id already exists: %s", id) - } - idx.ids[id] = true - idx.bytes = append(idx.bytes, []byte(id+" ")...) - idx.index = suffixarray.New(idx.bytes) - return nil -} - -func (idx *TruncIndex) Delete(id string) error { - if _, exists := idx.ids[id]; !exists { - return fmt.Errorf("No such id: %s", id) - } - before, after, err := idx.lookup(id) - if err != nil { - return err - } - delete(idx.ids, id) - idx.bytes = append(idx.bytes[:before], idx.bytes[after:]...) - idx.index = suffixarray.New(idx.bytes) - return nil -} - -func (idx *TruncIndex) lookup(s string) (int, int, error) { - offsets := idx.index.Lookup([]byte(" "+s), -1) - //log.Printf("lookup(%s): %v (index bytes: '%s')\n", s, offsets, idx.index.Bytes()) - if offsets == nil || len(offsets) == 0 || len(offsets) > 1 { - return -1, -1, fmt.Errorf("No such id: %s", s) - } - offsetBefore := offsets[0] + 1 - offsetAfter := offsetBefore + strings.Index(string(idx.bytes[offsetBefore:]), " ") - return offsetBefore, offsetAfter, nil -} - -func (idx *TruncIndex) Get(s string) (string, error) { - before, after, err := idx.lookup(s) - //log.Printf("Get(%s) bytes=|%s| before=|%d| after=|%d|\n", s, idx.bytes, before, after) - if err != nil { - return "", err - } - return string(idx.bytes[before:after]), err -} - -// TruncateId returns a shorthand version of a string identifier for convenience. -// A collision with other shorthands is very unlikely, but possible. -// In case of a collision a lookup with TruncIndex.Get() will fail, and the caller -// will need to use a langer prefix, or the full-length Id. -func TruncateId(id string) string { - shortLen := 12 - if len(id) < shortLen { - shortLen = len(id) - } - return id[:shortLen] -} - -// Code c/c from io.Copy() modified to handle escape sequence -func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) { - buf := make([]byte, 32*1024) - for { - nr, er := src.Read(buf) - if nr > 0 { - // ---- Docker addition - // char 16 is C-p - if nr == 1 && buf[0] == 16 { - nr, er = src.Read(buf) - // char 17 is C-q - if nr == 1 && buf[0] == 17 { - if err := src.Close(); err != nil { - return 0, err - } - return 0, io.EOF - } - } - // ---- End of docker - nw, ew := dst.Write(buf[0:nr]) - if nw > 0 { - written += int64(nw) - } - if ew != nil { - err = ew - break - } - if nr != nw { - err = io.ErrShortWrite - break - } - } - if er == io.EOF { - break - } - if er != nil { - err = er - break - } - } - return written, err -} - -func SetRawTerminal() (*term.State, error) { - oldState, err := term.MakeRaw(int(os.Stdin.Fd())) - if err != nil { - return nil, err - } - c := make(chan os.Signal, 1) - signal.Notify(c, os.Interrupt) - go func() { - _ = <-c - term.Restore(int(os.Stdin.Fd()), oldState) - os.Exit(0) - }() - return oldState, err -} - -func RestoreTerminal(state *term.State) { - term.Restore(int(os.Stdin.Fd()), state) -} - -func HashData(src io.Reader) (string, error) { - h := sha256.New() - if _, err := io.Copy(h, src); err != nil { - return "", err - } - return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil -} - -type KernelVersionInfo struct { - Kernel int - Major int - Minor int - Flavor string -} - -// FIXME: this doens't build on Darwin -func GetKernelVersion() (*KernelVersionInfo, error) { - return getKernelVersion() -} - -func (k *KernelVersionInfo) String() string { - flavor := "" - if len(k.Flavor) > 0 { - flavor = fmt.Sprintf("-%s", k.Flavor) - } - return fmt.Sprintf("%d.%d.%d%s", k.Kernel, k.Major, k.Minor, flavor) -} - -// Compare two KernelVersionInfo struct. -// Returns -1 if a < b, = if a == b, 1 it a > b -func CompareKernelVersion(a, b *KernelVersionInfo) int { - if a.Kernel < b.Kernel { - return -1 - } else if a.Kernel > b.Kernel { - return 1 - } - - if a.Major < b.Major { - return -1 - } else if a.Major > b.Major { - return 1 - } - - if a.Minor < b.Minor { - return -1 - } else if a.Minor > b.Minor { - return 1 - } - - return 0 -} - -func FindCgroupMountpoint(cgroupType string) (string, error) { - output, err := ioutil.ReadFile("/proc/mounts") - if err != nil { - return "", err - } - - // /proc/mounts has 6 fields per line, one mount per line, e.g. - // cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0 - for _, line := range strings.Split(string(output), "\n") { - parts := strings.Split(line, " ") - if len(parts) == 6 && parts[2] == "cgroup" { - for _, opt := range strings.Split(parts[3], ",") { - if opt == cgroupType { - return parts[1], nil - } - } - } - } - - return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) -} - // Compare two Config struct. Do not compare the "Image" nor "Hostname" fields // If OpenStdin is set, then it differs func CompareConfig(a, b *Config) bool { diff --git a/utils/uname_darwin.go b/utils/uname_darwin.go new file mode 100644 index 0000000000..d799554bbd --- /dev/null +++ b/utils/uname_darwin.go @@ -0,0 +1,10 @@ +package utils + +import ( + "errors" + "syscall" +) + +func uname() (*syscall.Utsname, error) { + return nil, errors.New("Kernel version detection is not available on darwin") +} diff --git a/utils/uname_linux.go b/utils/uname_linux.go new file mode 100644 index 0000000000..675a89b001 --- /dev/null +++ b/utils/uname_linux.go @@ -0,0 +1,15 @@ +package utils + +import ( + "syscall" +) + +// FIXME: Move this to utils package +func uname() (*syscall.Utsname, error) { + uts := &syscall.Utsname{} + + if err := syscall.Uname(uts); err != nil { + return nil, err + } + return uts, nil +} diff --git a/utils/utils.go b/utils/utils.go new file mode 100644 index 0000000000..88d0c87f5c --- /dev/null +++ b/utils/utils.go @@ -0,0 +1,532 @@ +package utils + +import ( + "bytes" + "crypto/sha256" + "encoding/hex" + "errors" + "fmt" + "index/suffixarray" + "io" + "io/ioutil" + "net/http" + "os" + "os/exec" + "path/filepath" + "runtime" + "strconv" + "strings" + "sync" + "time" +) + +// Go is a basic promise implementation: it wraps calls a function in a goroutine, +// and returns a channel which will later return the function's return value. +func Go(f func() error) chan error { + ch := make(chan error) + go func() { + ch <- f() + }() + return ch +} + +// Request a given URL and return an io.Reader +func Download(url string, stderr io.Writer) (*http.Response, error) { + var resp *http.Response + var err error = nil + if resp, err = http.Get(url); err != nil { + return nil, err + } + if resp.StatusCode >= 400 { + return nil, errors.New("Got HTTP status code >= 400: " + resp.Status) + } + return resp, nil +} + +// Debug function, if the debug flag is set, then display. Do nothing otherwise +// If Docker is in damon mode, also send the debug info on the socket +func Debugf(format string, a ...interface{}) { + if os.Getenv("DEBUG") != "" { + + // Retrieve the stack infos + _, file, line, ok := runtime.Caller(1) + if !ok { + file = "" + line = -1 + } else { + file = file[strings.LastIndex(file, "/")+1:] + } + + fmt.Fprintf(os.Stderr, fmt.Sprintf("[debug] %s:%d %s\n", file, line, format), a...) + } +} + +// Reader with progress bar +type progressReader struct { + reader io.ReadCloser // Stream to read from + output io.Writer // Where to send progress bar to + readTotal int // Expected stream length (bytes) + readProgress int // How much has been read so far (bytes) + lastUpdate int // How many bytes read at least update + template string // Template to print. Default "%v/%v (%v)" +} + +func (r *progressReader) Read(p []byte) (n int, err error) { + read, err := io.ReadCloser(r.reader).Read(p) + r.readProgress += read + + updateEvery := 4096 + if r.readTotal > 0 { + // Only update progress for every 1% read + if increment := int(0.01 * float64(r.readTotal)); increment > updateEvery { + updateEvery = increment + } + } + if r.readProgress-r.lastUpdate > updateEvery || err != nil { + if r.readTotal > 0 { + fmt.Fprintf(r.output, r.template+"\r", r.readProgress, r.readTotal, fmt.Sprintf("%.0f%%", float64(r.readProgress)/float64(r.readTotal)*100)) + } else { + fmt.Fprintf(r.output, r.template+"\r", r.readProgress, "?", "n/a") + } + r.lastUpdate = r.readProgress + } + // Send newline when complete + if err != nil { + fmt.Fprintf(r.output, "\n") + } + + return read, err +} +func (r *progressReader) Close() error { + return io.ReadCloser(r.reader).Close() +} +func ProgressReader(r io.ReadCloser, size int, output io.Writer, template string) *progressReader { + if template == "" { + template = "%v/%v (%v)" + } + return &progressReader{r, output, size, 0, 0, template} +} + +// HumanDuration returns a human-readable approximation of a duration +// (eg. "About a minute", "4 hours ago", etc.) +func HumanDuration(d time.Duration) string { + if seconds := int(d.Seconds()); seconds < 1 { + return "Less than a second" + } else if seconds < 60 { + return fmt.Sprintf("%d seconds", seconds) + } else if minutes := int(d.Minutes()); minutes == 1 { + return "About a minute" + } else if minutes < 60 { + return fmt.Sprintf("%d minutes", minutes) + } else if hours := int(d.Hours()); hours == 1 { + return "About an hour" + } else if hours < 48 { + return fmt.Sprintf("%d hours", hours) + } else if hours < 24*7*2 { + return fmt.Sprintf("%d days", hours/24) + } else if hours < 24*30*3 { + return fmt.Sprintf("%d weeks", hours/24/7) + } else if hours < 24*365*2 { + return fmt.Sprintf("%d months", hours/24/30) + } + return fmt.Sprintf("%d years", d.Hours()/24/365) +} + +func Trunc(s string, maxlen int) string { + if len(s) <= maxlen { + return s + } + return s[:maxlen] +} + +// Figure out the absolute path of our own binary +func SelfPath() string { + path, err := exec.LookPath(os.Args[0]) + if err != nil { + panic(err) + } + path, err = filepath.Abs(path) + if err != nil { + panic(err) + } + return path +} + +type NopWriter struct { +} + +func (w *NopWriter) Write(buf []byte) (int, error) { + return len(buf), nil +} + +type nopWriteCloser struct { + io.Writer +} + +func (w *nopWriteCloser) Close() error { return nil } + +func NopWriteCloser(w io.Writer) io.WriteCloser { + return &nopWriteCloser{w} +} + +type bufReader struct { + buf *bytes.Buffer + reader io.Reader + err error + l sync.Mutex + wait sync.Cond +} + +func NewBufReader(r io.Reader) *bufReader { + reader := &bufReader{ + buf: &bytes.Buffer{}, + reader: r, + } + reader.wait.L = &reader.l + go reader.drain() + return reader +} + +func (r *bufReader) drain() { + buf := make([]byte, 1024) + for { + n, err := r.reader.Read(buf) + r.l.Lock() + if err != nil { + r.err = err + } else { + r.buf.Write(buf[0:n]) + } + r.wait.Signal() + r.l.Unlock() + if err != nil { + break + } + } +} + +func (r *bufReader) Read(p []byte) (n int, err error) { + r.l.Lock() + defer r.l.Unlock() + for { + n, err = r.buf.Read(p) + if n > 0 { + return n, err + } + if r.err != nil { + return 0, r.err + } + r.wait.Wait() + } + panic("unreachable") +} + +func (r *bufReader) Close() error { + closer, ok := r.reader.(io.ReadCloser) + if !ok { + return nil + } + return closer.Close() +} + +type WriteBroadcaster struct { + mu sync.Mutex + writers map[io.WriteCloser]struct{} +} + +func (w *WriteBroadcaster) AddWriter(writer io.WriteCloser) { + w.mu.Lock() + w.writers[writer] = struct{}{} + w.mu.Unlock() +} + +// FIXME: Is that function used? +// FIXME: This relies on the concrete writer type used having equality operator +func (w *WriteBroadcaster) RemoveWriter(writer io.WriteCloser) { + w.mu.Lock() + delete(w.writers, writer) + w.mu.Unlock() +} + +func (w *WriteBroadcaster) Write(p []byte) (n int, err error) { + w.mu.Lock() + defer w.mu.Unlock() + for writer := range w.writers { + if n, err := writer.Write(p); err != nil || n != len(p) { + // On error, evict the writer + delete(w.writers, writer) + } + } + return len(p), nil +} + +func (w *WriteBroadcaster) CloseWriters() error { + w.mu.Lock() + defer w.mu.Unlock() + for writer := range w.writers { + writer.Close() + } + w.writers = make(map[io.WriteCloser]struct{}) + return nil +} + +func NewWriteBroadcaster() *WriteBroadcaster { + return &WriteBroadcaster{writers: make(map[io.WriteCloser]struct{})} +} + +func GetTotalUsedFds() int { + if fds, err := ioutil.ReadDir(fmt.Sprintf("/proc/%d/fd", os.Getpid())); err != nil { + Debugf("Error opening /proc/%d/fd: %s", os.Getpid(), err) + } else { + return len(fds) + } + return -1 +} + +// TruncIndex allows the retrieval of string identifiers by any of their unique prefixes. +// This is used to retrieve image and container IDs by more convenient shorthand prefixes. +type TruncIndex struct { + index *suffixarray.Index + ids map[string]bool + bytes []byte +} + +func NewTruncIndex() *TruncIndex { + return &TruncIndex{ + index: suffixarray.New([]byte{' '}), + ids: make(map[string]bool), + bytes: []byte{' '}, + } +} + +func (idx *TruncIndex) Add(id string) error { + if strings.Contains(id, " ") { + return fmt.Errorf("Illegal character: ' '") + } + if _, exists := idx.ids[id]; exists { + return fmt.Errorf("Id already exists: %s", id) + } + idx.ids[id] = true + idx.bytes = append(idx.bytes, []byte(id+" ")...) + idx.index = suffixarray.New(idx.bytes) + return nil +} + +func (idx *TruncIndex) Delete(id string) error { + if _, exists := idx.ids[id]; !exists { + return fmt.Errorf("No such id: %s", id) + } + before, after, err := idx.lookup(id) + if err != nil { + return err + } + delete(idx.ids, id) + idx.bytes = append(idx.bytes[:before], idx.bytes[after:]...) + idx.index = suffixarray.New(idx.bytes) + return nil +} + +func (idx *TruncIndex) lookup(s string) (int, int, error) { + offsets := idx.index.Lookup([]byte(" "+s), -1) + //log.Printf("lookup(%s): %v (index bytes: '%s')\n", s, offsets, idx.index.Bytes()) + if offsets == nil || len(offsets) == 0 || len(offsets) > 1 { + return -1, -1, fmt.Errorf("No such id: %s", s) + } + offsetBefore := offsets[0] + 1 + offsetAfter := offsetBefore + strings.Index(string(idx.bytes[offsetBefore:]), " ") + return offsetBefore, offsetAfter, nil +} + +func (idx *TruncIndex) Get(s string) (string, error) { + before, after, err := idx.lookup(s) + //log.Printf("Get(%s) bytes=|%s| before=|%d| after=|%d|\n", s, idx.bytes, before, after) + if err != nil { + return "", err + } + return string(idx.bytes[before:after]), err +} + +// TruncateId returns a shorthand version of a string identifier for convenience. +// A collision with other shorthands is very unlikely, but possible. +// In case of a collision a lookup with TruncIndex.Get() will fail, and the caller +// will need to use a langer prefix, or the full-length Id. +func TruncateId(id string) string { + shortLen := 12 + if len(id) < shortLen { + shortLen = len(id) + } + return id[:shortLen] +} + +// Code c/c from io.Copy() modified to handle escape sequence +func CopyEscapable(dst io.Writer, src io.ReadCloser) (written int64, err error) { + buf := make([]byte, 32*1024) + for { + nr, er := src.Read(buf) + if nr > 0 { + // ---- Docker addition + // char 16 is C-p + if nr == 1 && buf[0] == 16 { + nr, er = src.Read(buf) + // char 17 is C-q + if nr == 1 && buf[0] == 17 { + if err := src.Close(); err != nil { + return 0, err + } + return 0, io.EOF + } + } + // ---- End of docker + nw, ew := dst.Write(buf[0:nr]) + if nw > 0 { + written += int64(nw) + } + if ew != nil { + err = ew + break + } + if nr != nw { + err = io.ErrShortWrite + break + } + } + if er == io.EOF { + break + } + if er != nil { + err = er + break + } + } + return written, err +} + +func HashData(src io.Reader) (string, error) { + h := sha256.New() + if _, err := io.Copy(h, src); err != nil { + return "", err + } + return "sha256:" + hex.EncodeToString(h.Sum(nil)), nil +} + +type KernelVersionInfo struct { + Kernel int + Major int + Minor int + Flavor string +} + +func (k *KernelVersionInfo) String() string { + flavor := "" + if len(k.Flavor) > 0 { + flavor = fmt.Sprintf("-%s", k.Flavor) + } + return fmt.Sprintf("%d.%d.%d%s", k.Kernel, k.Major, k.Minor, flavor) +} + +// Compare two KernelVersionInfo struct. +// Returns -1 if a < b, = if a == b, 1 it a > b +func CompareKernelVersion(a, b *KernelVersionInfo) int { + if a.Kernel < b.Kernel { + return -1 + } else if a.Kernel > b.Kernel { + return 1 + } + + if a.Major < b.Major { + return -1 + } else if a.Major > b.Major { + return 1 + } + + if a.Minor < b.Minor { + return -1 + } else if a.Minor > b.Minor { + return 1 + } + + return 0 +} + +func FindCgroupMountpoint(cgroupType string) (string, error) { + output, err := ioutil.ReadFile("/proc/mounts") + if err != nil { + return "", err + } + + // /proc/mounts has 6 fields per line, one mount per line, e.g. + // cgroup /sys/fs/cgroup/devices cgroup rw,relatime,devices 0 0 + for _, line := range strings.Split(string(output), "\n") { + parts := strings.Split(line, " ") + if len(parts) == 6 && parts[2] == "cgroup" { + for _, opt := range strings.Split(parts[3], ",") { + if opt == cgroupType { + return parts[1], nil + } + } + } + } + + return "", fmt.Errorf("cgroup mountpoint not found for %s", cgroupType) +} + +func GetKernelVersion() (*KernelVersionInfo, error) { + var ( + flavor string + kernel, major, minor int + err error + ) + + uts, err := uname() + if err != nil { + return nil, err + } + + release := make([]byte, len(uts.Release)) + + i := 0 + for _, c := range uts.Release { + release[i] = byte(c) + i++ + } + + // Remove the \x00 from the release for Atoi to parse correctly + release = release[:bytes.IndexByte(release, 0)] + + tmp := strings.SplitN(string(release), "-", 2) + tmp2 := strings.SplitN(tmp[0], ".", 3) + + if len(tmp2) > 0 { + kernel, err = strconv.Atoi(tmp2[0]) + if err != nil { + return nil, err + } + } + + if len(tmp2) > 1 { + major, err = strconv.Atoi(tmp2[1]) + if err != nil { + return nil, err + } + } + + if len(tmp2) > 2 { + minor, err = strconv.Atoi(tmp2[2]) + if err != nil { + return nil, err + } + } + + if len(tmp) == 2 { + flavor = tmp[1] + } else { + flavor = "" + } + + return &KernelVersionInfo{ + Kernel: kernel, + Major: major, + Minor: minor, + Flavor: flavor, + }, nil +} diff --git a/utils_test.go b/utils/utils_test.go similarity index 98% rename from utils_test.go rename to utils/utils_test.go index aa2a1b9682..4413f44efc 100644 --- a/utils_test.go +++ b/utils/utils_test.go @@ -1,4 +1,4 @@ -package docker +package utils import ( "bytes" @@ -10,7 +10,7 @@ import ( func TestBufReader(t *testing.T) { reader, writer := io.Pipe() - bufreader := newBufReader(reader) + bufreader := NewBufReader(reader) // Write everything down to a Pipe // Usually, a pipe should block but because of the buffered reader, @@ -55,7 +55,7 @@ func (dw *dummyWriter) Close() error { } func TestWriteBroadcaster(t *testing.T) { - writer := newWriteBroadcaster() + writer := NewWriteBroadcaster() // Test 1: Both bufferA and bufferB should contain "foo" bufferA := &dummyWriter{} @@ -137,7 +137,7 @@ func (d devNullCloser) Write(buf []byte) (int, error) { // This test checks for races. It is only useful when run with the race detector. func TestRaceWriteBroadcaster(t *testing.T) { - writer := newWriteBroadcaster() + writer := NewWriteBroadcaster() c := make(chan bool) go func() { writer.AddWriter(devNullCloser(0))