diff --git a/contrib/httpserver/Dockerfile b/contrib/httpserver/Dockerfile new file mode 100644 index 0000000000..747dc91bcf --- /dev/null +++ b/contrib/httpserver/Dockerfile @@ -0,0 +1,4 @@ +FROM busybox +EXPOSE 80/tcp +COPY httpserver . +CMD ["./httpserver"] diff --git a/contrib/httpserver/server.go b/contrib/httpserver/server.go new file mode 100644 index 0000000000..a75d5abb3d --- /dev/null +++ b/contrib/httpserver/server.go @@ -0,0 +1,12 @@ +package main + +import ( + "log" + "net/http" +) + +func main() { + fs := http.FileServer(http.Dir("/static")) + http.Handle("/", fs) + log.Panic(http.ListenAndServe(":80", nil)) +} diff --git a/integration-cli/docker_api_containers_test.go b/integration-cli/docker_api_containers_test.go index c0ea97acb9..77638fa530 100644 --- a/integration-cli/docker_api_containers_test.go +++ b/integration-cli/docker_api_containers_test.go @@ -364,7 +364,7 @@ RUN find /tmp/`, } defer server.Close() - buf, err := sockRequestRaw("POST", "/build?dockerfile=baz&remote="+server.URL+"/testD", nil, "application/json") + buf, err := sockRequestRaw("POST", "/build?dockerfile=baz&remote="+server.URL()+"/testD", nil, "application/json") if err != nil { t.Fatalf("Build failed: %s", err) } diff --git a/integration-cli/docker_cli_build_test.go b/integration-cli/docker_cli_build_test.go index cf74aa1e30..c0a5580bfc 100644 --- a/integration-cli/docker_cli_build_test.go +++ b/integration-cli/docker_cli_build_test.go @@ -8,14 +8,12 @@ import ( "io/ioutil" "os" "os/exec" - "path" "path/filepath" "reflect" "regexp" "runtime" "strconv" "strings" - "syscall" "testing" "text/template" "time" @@ -645,9 +643,10 @@ func TestBuildCacheADD(t *testing.T) { t.Fatal(err) } defer server.Close() + if _, err := buildImage(name, fmt.Sprintf(`FROM scratch - ADD %s/robots.txt /`, server.URL), + ADD %s/robots.txt /`, server.URL()), true); err != nil { t.Fatal(err) } @@ -657,7 +656,7 @@ func TestBuildCacheADD(t *testing.T) { deleteImages(name) _, out, err := buildImageWithOut(name, fmt.Sprintf(`FROM scratch - ADD %s/index.html /`, server.URL), + ADD %s/index.html /`, server.URL()), true) if err != nil { t.Fatal(err) @@ -797,7 +796,7 @@ RUN [ $(ls -l /exists/test_file4 | awk '{print $3":"$4}') = 'root:root' ] RUN [ $(ls -l /exists/robots.txt | awk '{print $3":"$4}') = 'root:root' ] RUN [ $(ls -l /exists/exists_file | awk '{print $3":"$4}') = 'dockerio:dockerio' ] -`, server.URL), +`, server.URL()), map[string]string{ "test_file1": "test1", "test_file2": "test2", @@ -1084,6 +1083,7 @@ func TestBuildCopyWildcard(t *testing.T) { t.Fatal(err) } defer server.Close() + ctx, err := fakeContext(fmt.Sprintf(`FROM busybox COPY file*.txt /tmp/ RUN ls /tmp/file1.txt /tmp/file2.txt @@ -1093,7 +1093,7 @@ func TestBuildCopyWildcard(t *testing.T) { RUN mkdir /tmp2 ADD dir/*dir %s/robots.txt /tmp2/ RUN ls /tmp2/nest_nest_file /tmp2/robots.txt - `, server.URL), + `, server.URL()), map[string]string{ "file1.txt": "test1", "file2.txt": "test2", @@ -2831,10 +2831,11 @@ func TestBuildADDRemoteFileWithCache(t *testing.T) { t.Fatal(err) } defer server.Close() + id1, err := buildImage(name, fmt.Sprintf(`FROM scratch MAINTAINER dockerio - ADD %s/baz /usr/lib/baz/quux`, server.URL), + ADD %s/baz /usr/lib/baz/quux`, server.URL()), true) if err != nil { t.Fatal(err) @@ -2842,7 +2843,7 @@ func TestBuildADDRemoteFileWithCache(t *testing.T) { id2, err := buildImage(name, fmt.Sprintf(`FROM scratch MAINTAINER dockerio - ADD %s/baz /usr/lib/baz/quux`, server.URL), + ADD %s/baz /usr/lib/baz/quux`, server.URL()), true) if err != nil { t.Fatal(err) @@ -2864,10 +2865,11 @@ func TestBuildADDRemoteFileWithoutCache(t *testing.T) { t.Fatal(err) } defer server.Close() + id1, err := buildImage(name, fmt.Sprintf(`FROM scratch MAINTAINER dockerio - ADD %s/baz /usr/lib/baz/quux`, server.URL), + ADD %s/baz /usr/lib/baz/quux`, server.URL()), true) if err != nil { t.Fatal(err) @@ -2875,7 +2877,7 @@ func TestBuildADDRemoteFileWithoutCache(t *testing.T) { id2, err := buildImage(name2, fmt.Sprintf(`FROM scratch MAINTAINER dockerio - ADD %s/baz /usr/lib/baz/quux`, server.URL), + ADD %s/baz /usr/lib/baz/quux`, server.URL()), false) if err != nil { t.Fatal(err) @@ -2894,7 +2896,8 @@ func TestBuildADDRemoteFileMTime(t *testing.T) { defer deleteImages(name, name2, name3, name4) - server, err := fakeStorage(map[string]string{"baz": "hello"}) + files := map[string]string{"baz": "hello"} + server, err := fakeStorage(files) if err != nil { t.Fatal(err) } @@ -2902,7 +2905,7 @@ func TestBuildADDRemoteFileMTime(t *testing.T) { ctx, err := fakeContext(fmt.Sprintf(`FROM scratch MAINTAINER dockerio - ADD %s/baz /usr/lib/baz/quux`, server.URL), nil) + ADD %s/baz /usr/lib/baz/quux`, server.URL()), nil) if err != nil { t.Fatal(err) } @@ -2921,15 +2924,26 @@ func TestBuildADDRemoteFileMTime(t *testing.T) { t.Fatal("The cache should have been used but wasn't - #1") } - // Now set baz's times to anything else and redo the build + // Now create a different server withsame contents (causes different mtim) // This time the cache should not be used - bazPath := path.Join(server.FakeContext.Dir, "baz") - err = syscall.UtimesNano(bazPath, make([]syscall.Timespec, 2)) - if err != nil { - t.Fatalf("Error setting mtime on %q: %v", bazPath, err) - } - id3, err := buildImageFromContext(name3, ctx, true) + // allow some time for clock to pass as mtime precision is only 1s + time.Sleep(2 * time.Second) + + server2, err := fakeStorage(files) + if err != nil { + t.Fatal(err) + } + defer server2.Close() + + ctx2, err := fakeContext(fmt.Sprintf(`FROM scratch + MAINTAINER dockerio + ADD %s/baz /usr/lib/baz/quux`, server2.URL()), nil) + if err != nil { + t.Fatal(err) + } + defer ctx2.Close() + id3, err := buildImageFromContext(name3, ctx2, true) if err != nil { t.Fatal(err) } @@ -2938,7 +2952,7 @@ func TestBuildADDRemoteFileMTime(t *testing.T) { } // And for good measure do it again and make sure cache is used this time - id4, err := buildImageFromContext(name4, ctx, true) + id4, err := buildImageFromContext(name4, ctx2, true) if err != nil { t.Fatal(err) } @@ -2958,10 +2972,11 @@ func TestBuildADDLocalAndRemoteFilesWithCache(t *testing.T) { t.Fatal(err) } defer server.Close() + ctx, err := fakeContext(fmt.Sprintf(`FROM scratch MAINTAINER dockerio ADD foo /usr/lib/bla/bar - ADD %s/baz /usr/lib/baz/quux`, server.URL), + ADD %s/baz /usr/lib/baz/quux`, server.URL()), map[string]string{ "foo": "hello world", }) @@ -3047,10 +3062,11 @@ func TestBuildADDLocalAndRemoteFilesWithoutCache(t *testing.T) { t.Fatal(err) } defer server.Close() + ctx, err := fakeContext(fmt.Sprintf(`FROM scratch MAINTAINER dockerio ADD foo /usr/lib/bla/bar - ADD %s/baz /usr/lib/baz/quux`, server.URL), + ADD %s/baz /usr/lib/baz/quux`, server.URL()), map[string]string{ "foo": "hello world", }) @@ -4773,7 +4789,7 @@ RUN echo from Dockerfile`, // Make sure that -f is ignored and that we don't use the Dockerfile // that's in the current dir - out, _, err := dockerCmdInDir(t, ctx.Dir, "build", "-f", "baz", "-t", "test1", server.URL+"/baz") + out, _, err := dockerCmdInDir(t, ctx.Dir, "build", "-f", "baz", "-t", "test1", server.URL()+"/baz") if err != nil { t.Fatalf("Failed to build: %s\n%s", out, err) } diff --git a/integration-cli/docker_utils.go b/integration-cli/docker_utils.go index d4c930ee1c..445bec58d2 100644 --- a/integration-cli/docker_utils.go +++ b/integration-cli/docker_utils.go @@ -581,17 +581,42 @@ func fakeContext(dockerfile string, files map[string]string) (*FakeContext, erro return ctx, nil } -type FakeStorage struct { +// FakeStorage is a static file server. It might be running locally or remotely +// on test host. +type FakeStorage interface { + Close() error + URL() string + CtxDir() string +} + +// fakeStorage returns either a local or remote (at daemon machine) file server +func fakeStorage(files map[string]string) (FakeStorage, error) { + if isLocalDaemon { + return newLocalFakeStorage(files) + } + return newRemoteFileServer(files) +} + +// localFileStorage is a file storage on the running machine +type localFileStorage struct { *FakeContext *httptest.Server } -func (f *FakeStorage) Close() error { - f.Server.Close() - return f.FakeContext.Close() +func (s *localFileStorage) URL() string { + return s.Server.URL } -func fakeStorage(files map[string]string) (*FakeStorage, error) { +func (s *localFileStorage) CtxDir() string { + return s.FakeContext.Dir +} + +func (s *localFileStorage) Close() error { + defer s.Server.Close() + return s.FakeContext.Close() +} + +func newLocalFakeStorage(files map[string]string) (*localFileStorage, error) { tmp, err := ioutil.TempDir("", "fake-storage") if err != nil { return nil, err @@ -605,40 +630,99 @@ func fakeStorage(files map[string]string) (*FakeStorage, error) { } handler := http.FileServer(http.Dir(ctx.Dir)) server := httptest.NewServer(handler) - return &FakeStorage{ + return &localFileStorage{ FakeContext: ctx, Server: server, }, nil } -func inspectField(name, field string) (string, error) { - format := fmt.Sprintf("{{.%s}}", field) +// remoteFileServer is a containerized static file server started on the remote +// testing machine to be used in URL-accepting docker build functionality. +type remoteFileServer struct { + host string // hostname/port web server is listening to on docker host e.g. 0.0.0.0:43712 + container string + image string + ctx *FakeContext +} + +func (f *remoteFileServer) URL() string { + u := url.URL{ + Scheme: "http", + Host: f.host} + return u.String() +} + +func (f *remoteFileServer) CtxDir() string { + return f.ctx.Dir +} + +func (f *remoteFileServer) Close() error { + defer func() { + if f.ctx != nil { + f.ctx.Close() + } + if f.image != "" { + deleteImages(f.image) + } + }() + if f.container == "" { + return nil + } + return deleteContainer(f.container) +} + +func newRemoteFileServer(files map[string]string) (*remoteFileServer, error) { + var ( + image = fmt.Sprintf("fileserver-img-%s", strings.ToLower(makeRandomString(10))) + container = fmt.Sprintf("fileserver-cnt-%s", strings.ToLower(makeRandomString(10))) + ) + + // Build the image + ctx, err := fakeContext(`FROM httpserver +COPY . /static`, files) + if _, err := buildImageFromContext(image, ctx, false); err != nil { + return nil, fmt.Errorf("failed building file storage container image: %v", err) + } + + // Start the container + runCmd := exec.Command(dockerBinary, "run", "-d", "-P", "--name", container, image) + if out, ec, err := runCommandWithOutput(runCmd); err != nil { + return nil, fmt.Errorf("failed to start file storage container. ec=%v\nout=%s\nerr=%v", ec, out, err) + } + + // Find out the system assigned port + out, _, err := runCommandWithOutput(exec.Command(dockerBinary, "port", container, "80/tcp")) + if err != nil { + return nil, fmt.Errorf("failed to find container port: err=%v\nout=%s", err, out) + } + + return &remoteFileServer{ + container: container, + image: image, + host: strings.Trim(out, "\n"), + ctx: ctx}, nil +} + +func inspectFilter(name, filter string) (string, error) { + format := fmt.Sprintf("{{%s}}", filter) inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name) out, exitCode, err := runCommandWithOutput(inspectCmd) if err != nil || exitCode != 0 { - return "", fmt.Errorf("failed to inspect %s: %s", name, out) + return "", fmt.Errorf("failed to inspect container %s: %s", name, out) } return strings.TrimSpace(out), nil } +func inspectField(name, field string) (string, error) { + return inspectFilter(name, fmt.Sprintf(".%s", field)) +} + func inspectFieldJSON(name, field string) (string, error) { - format := fmt.Sprintf("{{json .%s}}", field) - inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name) - out, exitCode, err := runCommandWithOutput(inspectCmd) - if err != nil || exitCode != 0 { - return "", fmt.Errorf("failed to inspect %s: %s", name, out) - } - return strings.TrimSpace(out), nil + return inspectFilter(name, fmt.Sprintf("json .%s", field)) } func inspectFieldMap(name, path, field string) (string, error) { - format := fmt.Sprintf("{{index .%s %q}}", path, field) - inspectCmd := exec.Command(dockerBinary, "inspect", "-f", format, name) - out, exitCode, err := runCommandWithOutput(inspectCmd) - if err != nil || exitCode != 0 { - return "", fmt.Errorf("failed to inspect %s: %s", name, out) - } - return strings.TrimSpace(out), nil + return inspectFilter(name, fmt.Sprintf("index .%s %q", path, field)) } func getIDByName(name string) (string, error) { diff --git a/project/make/.ensure-httpserver b/project/make/.ensure-httpserver new file mode 100644 index 0000000000..38659edaba --- /dev/null +++ b/project/make/.ensure-httpserver @@ -0,0 +1,15 @@ +#!/bin/bash +set -e + +# Build a Go static web server on top of busybox image +# and compile it for target daemon + +dir="$DEST/httpserver" +mkdir -p "$dir" +( + cd "$dir" + GOOS=linux GOARCH=amd64 go build -o httpserver github.com/docker/docker/contrib/httpserver + cp ../../../../contrib/httpserver/Dockerfile . + docker build -qt httpserver . > /dev/null +) +rm -rf "$dir" diff --git a/project/make/test-integration-cli b/project/make/test-integration-cli index 5ea3be3872..23283968b0 100644 --- a/project/make/test-integration-cli +++ b/project/make/test-integration-cli @@ -17,6 +17,7 @@ bundle_test_integration_cli() { didFail= if ! { source "$(dirname "$BASH_SOURCE")/.ensure-busybox" + source "$(dirname "$BASH_SOURCE")/.ensure-httpserver" source "$(dirname "$BASH_SOURCE")/.ensure-emptyfs" bundle_test_integration_cli