diff --git a/.mailmap b/.mailmap index 452ac41d8f..11ff5357d8 100644 --- a/.mailmap +++ b/.mailmap @@ -23,3 +23,5 @@ Thatcher Peskens Walter Stanish Roberto Hashioka +Konstantin Pelykh +David Sissitka diff --git a/AUTHORS b/AUTHORS index 86d03f6e12..6f6c597722 100644 --- a/AUTHORS +++ b/AUTHORS @@ -4,12 +4,15 @@ # For a list of active project maintainers, see the MAINTAINERS file. # Al Tobey +Alex Gaynor Alexey Shamrin Andrea Luzzardi Andreas Tiefenthaler Andrew Munsell +Andrews Medina Andy Rothfusz Andy Smith +Anthony Bishopric Antony Messerli Barry Allard Brandon Liu @@ -19,21 +22,28 @@ Caleb Spare Calen Pennington Charles Hooper Christopher Currie +Colin Rice Daniel Gasienica Daniel Mizyrycki Daniel Robinson Daniel Von Fange +Daniel YC Lin +David Calavera +David Sissitka Dominik Honnef Don Spaulding Dr Nic Williams Elias Probst Eric Hanchrow -Evan Wies Eric Myhre +Erno Hopearuoho +Evan Wies ezbercih +Fabrizio Regini Flavio Castelli Francisco Souza Frederick F. Kautz IV +Gabriel Monroy Gareth Rushgrove Guillaume J. Charmes Harley Laue @@ -41,6 +51,7 @@ Hunter Blanks Jeff Lindsay Jeremy Grosser Joffrey F +Johan Euphrosine John Costa Jon Wedaman Jonas Pfenniger @@ -48,28 +59,39 @@ Jonathan Rudenberg Joseph Anthony Pasquale Holsten Julien Barbier Jérôme Petazzoni +Karan Lyons +Keli Hu Ken Cochrane Kevin J. Lynagh kim0 +Kimbro Staken Kiran Gangadharan +Konstantin Pelykh Louis Opter +Marco Hennings Marcus Farkas Mark McGranaghan Maxim Treskin meejah Michael Crosby +Mike Gaffney Mikhail Sobolev +Nan Monnand Deng Nate Jones Nelson Chen Niall O'Higgins +Nick Stenning +Nick Stinemates odk- Paul Bowsher Paul Hammond Phil Spitler Piotr Bogdan Renato Riccieri Santos Zannon +Rhys Hiltner Robert Obryk Roberto Hashioka +Ryan Fowler Sam Alba Sam J Sharpe Shawn Siefkas @@ -83,6 +105,8 @@ Thomas Hansen Tianon Gravi Tim Terhorst Tobias Bieniek +Tobias Schwab +Tom Hulihan unclejack Victor Vieux Vivek Agarwal diff --git a/CHANGELOG.md b/CHANGELOG.md index 00e358c136..843f0212d0 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,24 @@ # Changelog +## 0.5.1 (2013-07-30) + + API: Docker client now sets useragent (RFC 2616) + + Runtime: Add `ps` args to `docker top` + + Runtime: Add support for container ID files (pidfile like) + + Runtime: Add container=lxc in default env + + Runtime: Support networkless containers with `docker run -n` and `docker -d -b=none` + + API: Add /events endpoint + + Builder: ADD command now understands URLs + + Builder: CmdAdd and CmdEnv now respect Dockerfile-set ENV variables + * Hack: Simplify unit tests with helpers + * Hack: Improve docker.upstart event + * Hack: Add coverage testing into docker-ci + * Runtime: Stdout/stderr logs are now stored in the same file as JSON + * Runtime: Allocate a /16 IP range by default, with fallback to /24. Try 12 ranges instead of 3. + * Runtime: Change .dockercfg format to json and support multiple auth remote + - Runtime: Do not override volumes from config + - Runtime: Fix issue with EXPOSE override + - Builder: Create directories with 755 instead of 700 within ADD instruction + ## 0.5.0 (2013-07-17) + Runtime: List all processes running inside a container with 'docker top' + Runtime: Host directories can be mounted as volumes with 'docker run -v' diff --git a/MAINTAINERS b/MAINTAINERS index dc0d1a19f6..55cb07b72b 100644 --- a/MAINTAINERS +++ b/MAINTAINERS @@ -1,5 +1,6 @@ Solomon Hykes Guillaume Charmes Victor Vieux +Michael Crosby api.go: Victor Vieux Vagrantfile: Daniel Mizyrycki diff --git a/Makefile b/Makefile index 6050000582..dd365dc30e 100644 --- a/Makefile +++ b/Makefile @@ -50,6 +50,7 @@ release: $(BINRELEASE) s3cmd -P put $(BINRELEASE) s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-$(RELEASE_VERSION).tgz s3cmd -P put docker-latest.tgz s3://get.docker.io/builds/`uname -s`/`uname -m`/docker-latest.tgz s3cmd -P put $(SRCRELEASE)/bin/docker s3://get.docker.io/builds/`uname -s`/`uname -m`/docker + echo $(RELEASE_VERSION) > latest ; s3cmd -P put latest s3://get.docker.io/latest ; rm latest srcrelease: $(SRCRELEASE) deps: $(DOCKER_DIR) @@ -65,7 +66,6 @@ $(BINRELEASE): $(SRCRELEASE) rm -f $(BINRELEASE) cd $(SRCRELEASE); make; cp -R bin docker-$(RELEASE_VERSION); tar -f ../$(BINRELEASE) -zv -c docker-$(RELEASE_VERSION) cd $(SRCRELEASE); cp -R bin docker-latest; tar -f ../docker-latest.tgz -zv -c docker-latest - clean: @rm -rf $(dir $(DOCKER_BIN)) ifeq ($(GOPATH), $(BUILD_DIR)) diff --git a/README.md b/README.md index 823e48496a..5eee1a3db6 100644 --- a/README.md +++ b/README.md @@ -1,80 +1,129 @@ Docker: the Linux container engine ================================== -Docker is an open-source engine which automates the deployment of applications as highly portable, self-sufficient containers. +Docker is an open-source engine which automates the deployment of +applications as highly portable, self-sufficient containers. -Docker containers are both *hardware-agnostic* and *platform-agnostic*. This means that they can run anywhere, from your -laptop to the largest EC2 compute instance and everything in between - and they don't require that you use a particular -language, framework or packaging system. That makes them great building blocks for deploying and scaling web apps, databases -and backend services without depending on a particular stack or provider. +Docker containers are both *hardware-agnostic* and +*platform-agnostic*. This means that they can run anywhere, from your +laptop to the largest EC2 compute instance and everything in between - +and they don't require that you use a particular language, framework +or packaging system. That makes them great building blocks for +deploying and scaling web apps, databases and backend services without +depending on a particular stack or provider. -Docker is an open-source implementation of the deployment engine which powers [dotCloud](http://dotcloud.com), a popular Platform-as-a-Service. -It benefits directly from the experience accumulated over several years of large-scale operation and support of hundreds of thousands -of applications and databases. +Docker is an open-source implementation of the deployment engine which +powers [dotCloud](http://dotcloud.com), a popular +Platform-as-a-Service. It benefits directly from the experience +accumulated over several years of large-scale operation and support of +hundreds of thousands of applications and databases. -![Docker L](docs/sources/concepts/images/lego_docker.jpg "Docker") +![Docker L](docs/sources/concepts/images/dockerlogo-h.png "Docker") ## Better than VMs -A common method for distributing applications and sandbox their execution is to use virtual machines, or VMs. Typical VM formats -are VMWare's vmdk, Oracle Virtualbox's vdi, and Amazon EC2's ami. In theory these formats should allow every developer to -automatically package their application into a "machine" for easy distribution and deployment. In practice, that almost never -happens, for a few reasons: +A common method for distributing applications and sandbox their +execution is to use virtual machines, or VMs. Typical VM formats are +VMWare's vmdk, Oracle Virtualbox's vdi, and Amazon EC2's ami. In +theory these formats should allow every developer to automatically +package their application into a "machine" for easy distribution and +deployment. In practice, that almost never happens, for a few reasons: - * *Size*: VMs are very large which makes them impractical to store and transfer. - * *Performance*: running VMs consumes significant CPU and memory, which makes them impractical in many scenarios, for example local development of multi-tier applications, and - large-scale deployment of cpu and memory-intensive applications on large numbers of machines. - * *Portability*: competing VM environments don't play well with each other. Although conversion tools do exist, they are limited and add even more overhead. - * *Hardware-centric*: VMs were designed with machine operators in mind, not software developers. As a result, they offer very limited tooling for what developers need most: - building, testing and running their software. For example, VMs offer no facilities for application versioning, monitoring, configuration, logging or service discovery. + * *Size*: VMs are very large which makes them impractical to store + and transfer. + * *Performance*: running VMs consumes significant CPU and memory, + which makes them impractical in many scenarios, for example local + development of multi-tier applications, and large-scale deployment + of cpu and memory-intensive applications on large numbers of + machines. + * *Portability*: competing VM environments don't play well with each + other. Although conversion tools do exist, they are limited and + add even more overhead. + * *Hardware-centric*: VMs were designed with machine operators in + mind, not software developers. As a result, they offer very + limited tooling for what developers need most: building, testing + and running their software. For example, VMs offer no facilities + for application versioning, monitoring, configuration, logging or + service discovery. -By contrast, Docker relies on a different sandboxing method known as *containerization*. Unlike traditional virtualization, -containerization takes place at the kernel level. Most modern operating system kernels now support the primitives necessary -for containerization, including Linux with [openvz](http://openvz.org), [vserver](http://linux-vserver.org) and more recently [lxc](http://lxc.sourceforge.net), - Solaris with [zones](http://docs.oracle.com/cd/E26502_01/html/E29024/preface-1.html#scrolltoc) and FreeBSD with [Jails](http://www.freebsd.org/doc/handbook/jails.html). +By contrast, Docker relies on a different sandboxing method known as +*containerization*. Unlike traditional virtualization, +containerization takes place at the kernel level. Most modern +operating system kernels now support the primitives necessary for +containerization, including Linux with [openvz](http://openvz.org), +[vserver](http://linux-vserver.org) and more recently +[lxc](http://lxc.sourceforge.net), Solaris with +[zones](http://docs.oracle.com/cd/E26502_01/html/E29024/preface-1.html#scrolltoc) +and FreeBSD with +[Jails](http://www.freebsd.org/doc/handbook/jails.html). -Docker builds on top of these low-level primitives to offer developers a portable format and runtime environment that solves -all 4 problems. Docker containers are small (and their transfer can be optimized with layers), they have basically zero memory and cpu overhead, -they are completely portable and are designed from the ground up with an application-centric design. +Docker builds on top of these low-level primitives to offer developers +a portable format and runtime environment that solves all 4 +problems. Docker containers are small (and their transfer can be +optimized with layers), they have basically zero memory and cpu +overhead, they are completely portable and are designed from the +ground up with an application-centric design. -The best part: because docker operates at the OS level, it can still be run inside a VM! +The best part: because ``docker`` operates at the OS level, it can +still be run inside a VM! ## Plays well with others -Docker does not require that you buy into a particular programming language, framework, packaging system or configuration language. +Docker does not require that you buy into a particular programming +language, framework, packaging system or configuration language. -Is your application a unix process? Does it use files, tcp connections, environment variables, standard unix streams and command-line -arguments as inputs and outputs? Then docker can run it. +Is your application a Unix process? Does it use files, tcp +connections, environment variables, standard Unix streams and +command-line arguments as inputs and outputs? Then ``docker`` can run +it. -Can your application's build be expressed as a sequence of such commands? Then docker can build it. +Can your application's build be expressed as a sequence of such +commands? Then ``docker`` can build it. ## Escape dependency hell -A common problem for developers is the difficulty of managing all their application's dependencies in a simple and automated way. +A common problem for developers is the difficulty of managing all +their application's dependencies in a simple and automated way. This is usually difficult for several reasons: - * *Cross-platform dependencies*. Modern applications often depend on a combination of system libraries and binaries, language-specific packages, framework-specific modules, - internal components developed for another project, etc. These dependencies live in different "worlds" and require different tools - these tools typically don't work - well with each other, requiring awkward custom integrations. + * *Cross-platform dependencies*. Modern applications often depend on + a combination of system libraries and binaries, language-specific + packages, framework-specific modules, internal components + developed for another project, etc. These dependencies live in + different "worlds" and require different tools - these tools + typically don't work well with each other, requiring awkward + custom integrations. - * Conflicting dependencies. Different applications may depend on different versions of the same dependency. Packaging tools handle these situations with various degrees of ease - - but they all handle them in different and incompatible ways, which again forces the developer to do extra work. + * Conflicting dependencies. Different applications may depend on + different versions of the same dependency. Packaging tools handle + these situations with various degrees of ease - but they all + handle them in different and incompatible ways, which again forces + the developer to do extra work. - * Custom dependencies. A developer may need to prepare a custom version of their application's dependency. Some packaging systems can handle custom versions of a dependency, - others can't - and all of them handle it differently. + * Custom dependencies. A developer may need to prepare a custom + version of their application's dependency. Some packaging systems + can handle custom versions of a dependency, others can't - and all + of them handle it differently. -Docker solves dependency hell by giving the developer a simple way to express *all* their application's dependencies in one place, -and streamline the process of assembling them. If this makes you think of [XKCD 927](http://xkcd.com/927/), don't worry. Docker doesn't -*replace* your favorite packaging systems. It simply orchestrates their use in a simple and repeatable way. How does it do that? With layers. +Docker solves dependency hell by giving the developer a simple way to +express *all* their application's dependencies in one place, and +streamline the process of assembling them. If this makes you think of +[XKCD 927](http://xkcd.com/927/), don't worry. Docker doesn't +*replace* your favorite packaging systems. It simply orchestrates +their use in a simple and repeatable way. How does it do that? With +layers. -Docker defines a build as running a sequence of unix commands, one after the other, in the same container. Build commands modify the contents of the container -(usually by installing new files on the filesystem), the next command modifies it some more, etc. Since each build command inherits the result of the previous -commands, the *order* in which the commands are executed expresses *dependencies*. +Docker defines a build as running a sequence of Unix commands, one +after the other, in the same container. Build commands modify the +contents of the container (usually by installing new files on the +filesystem), the next command modifies it some more, etc. Since each +build command inherits the result of the previous commands, the +*order* in which the commands are executed expresses *dependencies*. -Here's a typical docker build process: +Here's a typical Docker build process: ```bash from ubuntu:12.10 @@ -87,7 +136,8 @@ run curl -L https://github.com/shykes/helloflask/archive/master.tar.gz | tar -xz run cd helloflask-master && pip install -r requirements.txt ``` -Note that Docker doesn't care *how* dependencies are built - as long as they can be built by running a unix command in a container. +Note that Docker doesn't care *how* dependencies are built - as long +as they can be built by running a Unix command in a container. Install instructions @@ -103,8 +153,9 @@ curl get.docker.io | sudo sh -x Binary installs ---------------- -Docker supports the following binary installation methods. -Note that some methods are community contributions and not yet officially supported. +Docker supports the following binary installation methods. Note that +some methods are community contributions and not yet officially +supported. * [Ubuntu 12.04 and 12.10 (officially supported)](http://docs.docker.io/en/latest/installation/ubuntulinux/) * [Arch Linux](http://docs.docker.io/en/latest/installation/archlinux/) @@ -115,15 +166,15 @@ Note that some methods are community contributions and not yet officially suppor Installing from source ---------------------- -1. Make sure you have a [Go language](http://golang.org/doc/install) compiler and [git](http://git-scm.com) installed. - +1. Make sure you have a [Go language](http://golang.org/doc/install) +compiler >= 1.1 and [git](http://git-scm.com) installed. 2. Checkout the source code ```bash git clone http://github.com/dotcloud/docker ``` -3. Build the docker binary +3. Build the ``docker`` binary ```bash cd docker @@ -134,17 +185,20 @@ Installing from source Usage examples ============== -First run the docker daemon ---------------------------- +First run the ``docker`` daemon +------------------------------- -All the examples assume your machine is running the docker daemon. To run the docker daemon in the background, simply type: +All the examples assume your machine is running the ``docker`` +daemon. To run the ``docker`` daemon in the background, simply type: ```bash # On a production system you want this running in an init script sudo docker -d & ``` -Now you can run docker in client mode: all commands will be forwarded to the docker daemon, so the client can run from any account. +Now you can run ``docker`` in client mode: all commands will be +forwarded to the ``docker`` daemon, so the client can run from any +account. ```bash # Now you can run docker commands from any account. @@ -152,7 +206,7 @@ docker help ``` -Throwaway shell in a base ubuntu image +Throwaway shell in a base Ubuntu image -------------------------------------- ```bash @@ -202,7 +256,8 @@ docker commit -m "Installed curl" $CONTAINER $USER/betterbase docker push $USER/betterbase ``` -A list of publicly available images is [available here](https://github.com/dotcloud/docker/wiki/Public-docker-images). +A list of publicly available images is [available +here](https://github.com/dotcloud/docker/wiki/Public-docker-images). Expose a service on a TCP port ------------------------------ @@ -229,32 +284,40 @@ Under the hood Under the hood, Docker is built on the following components: - -* The [cgroup](http://blog.dotcloud.com/kernel-secrets-from-the-paas-garage-part-24-c) and [namespacing](http://blog.dotcloud.com/under-the-hood-linux-kernels-on-dotcloud-part) capabilities of the Linux kernel; - -* [AUFS](http://aufs.sourceforge.net/aufs.html), a powerful union filesystem with copy-on-write capabilities; - +* The + [cgroup](http://blog.dotcloud.com/kernel-secrets-from-the-paas-garage-part-24-c) + and + [namespacing](http://blog.dotcloud.com/under-the-hood-linux-kernels-on-dotcloud-part) + capabilities of the Linux kernel; +* [AUFS](http://aufs.sourceforge.net/aufs.html), a powerful union + filesystem with copy-on-write capabilities; * The [Go](http://golang.org) programming language; - -* [lxc](http://lxc.sourceforge.net/), a set of convenience scripts to simplify the creation of linux containers. +* [lxc](http://lxc.sourceforge.net/), a set of convenience scripts to + simplify the creation of Linux containers. Contributing to Docker ====================== -Want to hack on Docker? Awesome! There are instructions to get you started on the website: http://docs.docker.io/en/latest/contributing/contributing/ +Want to hack on Docker? Awesome! There are instructions to get you +started on the website: +http://docs.docker.io/en/latest/contributing/contributing/ -They are probably not perfect, please let us know if anything feels wrong or incomplete. +They are probably not perfect, please let us know if anything feels +wrong or incomplete. Note ---- -We also keep the documentation in this repository. The website documentation is generated using sphinx using these sources. -Please find it under docs/sources/ and read more about it https://github.com/dotcloud/docker/tree/master/docs/README.md +We also keep the documentation in this repository. The website +documentation is generated using Sphinx using these sources. Please +find it under docs/sources/ and read more about it +https://github.com/dotcloud/docker/tree/master/docs/README.md -Please feel free to fix / update the documentation and send us pull requests. More tutorials are also welcome. +Please feel free to fix / update the documentation and send us pull +requests. More tutorials are also welcome. Setting up a dev environment @@ -289,93 +352,104 @@ Run the `go install` command (above) to recompile docker. What is a Standard Container? ============================= -Docker defines a unit of software delivery called a Standard Container. The goal of a Standard Container is to encapsulate a software component and all its dependencies in -a format that is self-describing and portable, so that any compliant runtime can run it without extra dependencies, regardless of the underlying machine and the contents of the container. +Docker defines a unit of software delivery called a Standard +Container. The goal of a Standard Container is to encapsulate a +software component and all its dependencies in a format that is +self-describing and portable, so that any compliant runtime can run it +without extra dependencies, regardless of the underlying machine and +the contents of the container. -The spec for Standard Containers is currently a work in progress, but it is very straightforward. It mostly defines 1) an image format, 2) a set of standard operations, and 3) an execution environment. +The spec for Standard Containers is currently a work in progress, but +it is very straightforward. It mostly defines 1) an image format, 2) a +set of standard operations, and 3) an execution environment. -A great analogy for this is the shipping container. Just like how Standard Containers are a fundamental unit of software delivery, shipping containers (http://bricks.argz.com/ins/7823-1/12) are a fundamental unit of physical delivery. +A great analogy for this is the shipping container. Just like how +Standard Containers are a fundamental unit of software delivery, +shipping containers are a fundamental unit of physical delivery. ### 1. STANDARD OPERATIONS -Just like shipping containers, Standard Containers define a set of STANDARD OPERATIONS. Shipping containers can be lifted, stacked, locked, loaded, unloaded and labelled. Similarly, standard containers can be started, stopped, copied, snapshotted, downloaded, uploaded and tagged. +Just like shipping containers, Standard Containers define a set of +STANDARD OPERATIONS. Shipping containers can be lifted, stacked, +locked, loaded, unloaded and labelled. Similarly, Standard Containers +can be started, stopped, copied, snapshotted, downloaded, uploaded and +tagged. ### 2. CONTENT-AGNOSTIC -Just like shipping containers, Standard Containers are CONTENT-AGNOSTIC: all standard operations have the same effect regardless of the contents. A shipping container will be stacked in exactly the same way whether it contains Vietnamese powder coffee or spare Maserati parts. Similarly, Standard Containers are started or uploaded in the same way whether they contain a postgres database, a php application with its dependencies and application server, or Java build artifacts. +Just like shipping containers, Standard Containers are +CONTENT-AGNOSTIC: all standard operations have the same effect +regardless of the contents. A shipping container will be stacked in +exactly the same way whether it contains Vietnamese powder coffee or +spare Maserati parts. Similarly, Standard Containers are started or +uploaded in the same way whether they contain a postgres database, a +php application with its dependencies and application server, or Java +build artifacts. ### 3. INFRASTRUCTURE-AGNOSTIC -Both types of containers are INFRASTRUCTURE-AGNOSTIC: they can be transported to thousands of facilities around the world, and manipulated by a wide variety of equipment. A shipping container can be packed in a factory in Ukraine, transported by truck to the nearest routing center, stacked onto a train, loaded into a German boat by an Australian-built crane, stored in a warehouse at a US facility, etc. Similarly, a standard container can be bundled on my laptop, uploaded to S3, downloaded, run and snapshotted by a build server at Equinix in Virginia, uploaded to 10 staging servers in a home-made Openstack cluster, then sent to 30 production instances across 3 EC2 regions. +Both types of containers are INFRASTRUCTURE-AGNOSTIC: they can be +transported to thousands of facilities around the world, and +manipulated by a wide variety of equipment. A shipping container can +be packed in a factory in Ukraine, transported by truck to the nearest +routing center, stacked onto a train, loaded into a German boat by an +Australian-built crane, stored in a warehouse at a US facility, +etc. Similarly, a standard container can be bundled on my laptop, +uploaded to S3, downloaded, run and snapshotted by a build server at +Equinix in Virginia, uploaded to 10 staging servers in a home-made +Openstack cluster, then sent to 30 production instances across 3 EC2 +regions. ### 4. DESIGNED FOR AUTOMATION -Because they offer the same standard operations regardless of content and infrastructure, Standard Containers, just like their physical counterpart, are extremely well-suited for automation. In fact, you could say automation is their secret weapon. +Because they offer the same standard operations regardless of content +and infrastructure, Standard Containers, just like their physical +counterparts, are extremely well-suited for automation. In fact, you +could say automation is their secret weapon. -Many things that once required time-consuming and error-prone human effort can now be programmed. Before shipping containers, a bag of powder coffee was hauled, dragged, dropped, rolled and stacked by 10 different people in 10 different locations by the time it reached its destination. 1 out of 50 disappeared. 1 out of 20 was damaged. The process was slow, inefficient and cost a fortune - and was entirely different depending on the facility and the type of goods. +Many things that once required time-consuming and error-prone human +effort can now be programmed. Before shipping containers, a bag of +powder coffee was hauled, dragged, dropped, rolled and stacked by 10 +different people in 10 different locations by the time it reached its +destination. 1 out of 50 disappeared. 1 out of 20 was damaged. The +process was slow, inefficient and cost a fortune - and was entirely +different depending on the facility and the type of goods. -Similarly, before Standard Containers, by the time a software component ran in production, it had been individually built, configured, bundled, documented, patched, vendored, templated, tweaked and instrumented by 10 different people on 10 different computers. Builds failed, libraries conflicted, mirrors crashed, post-it notes were lost, logs were misplaced, cluster updates were half-broken. The process was slow, inefficient and cost a fortune - and was entirely different depending on the language and infrastructure provider. +Similarly, before Standard Containers, by the time a software +component ran in production, it had been individually built, +configured, bundled, documented, patched, vendored, templated, tweaked +and instrumented by 10 different people on 10 different +computers. Builds failed, libraries conflicted, mirrors crashed, +post-it notes were lost, logs were misplaced, cluster updates were +half-broken. The process was slow, inefficient and cost a fortune - +and was entirely different depending on the language and +infrastructure provider. ### 5. INDUSTRIAL-GRADE DELIVERY -There are 17 million shipping containers in existence, packed with every physical good imaginable. Every single one of them can be loaded onto the same boats, by the same cranes, in the same facilities, and sent anywhere in the World with incredible efficiency. It is embarrassing to think that a 30 ton shipment of coffee can safely travel half-way across the World in *less time* than it takes a software team to deliver its code from one datacenter to another sitting 10 miles away. +There are 17 million shipping containers in existence, packed with +every physical good imaginable. Every single one of them can be loaded +onto the same boats, by the same cranes, in the same facilities, and +sent anywhere in the World with incredible efficiency. It is +embarrassing to think that a 30 ton shipment of coffee can safely +travel half-way across the World in *less time* than it takes a +software team to deliver its code from one datacenter to another +sitting 10 miles away. -With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. +With Standard Containers we can put an end to that embarrassment, by +making INDUSTRIAL-GRADE DELIVERY of software a reality. - - -Standard Container Specification --------------------------------- - -(TODO) - -### Image format - - -### Standard operations - -* Copy -* Run -* Stop -* Wait -* Commit -* Attach standard streams -* List filesystem changes -* ... - -### Execution environment - -#### Root filesystem - -#### Environment variables - -#### Process arguments - -#### Networking - -#### Process namespacing - -#### Resource limits - -#### Process monitoring - -#### Logging - -#### Signals - -#### Pseudo-terminal allocation - -#### Security - ### Legal -Transfers of Docker shall be in accordance with applicable export controls of any country and all other applicable -legal requirements. Docker shall not be distributed or downloaded to or in Cuba, Iran, North Korea, Sudan or Syria -and shall not be distributed or downloaded to any person on the Denied Persons List administered by the U.S. +Transfers of Docker shall be in accordance with applicable export +controls of any country and all other applicable legal requirements. +Docker shall not be distributed or downloaded to or in Cuba, Iran, +North Korea, Sudan or Syria and shall not be distributed or downloaded +to any person on the Denied Persons List administered by the U.S. Department of Commerce. diff --git a/Vagrantfile b/Vagrantfile index aadabb8711..18aa5b5afb 100644 --- a/Vagrantfile +++ b/Vagrantfile @@ -82,15 +82,15 @@ Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| end if !FORWARD_DOCKER_PORTS.nil? - Vagrant::VERSION < "1.1.0" and Vagrant::Config.run do |config| - (49000..49900).each do |port| - config.vm.forward_port port, port - end + Vagrant::VERSION < "1.1.0" and Vagrant::Config.run do |config| + (49000..49900).each do |port| + config.vm.forward_port port, port end + end - Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| - (49000..49900).each do |port| - config.vm.network :forwarded_port, :host => port, :guest => port - end + Vagrant::VERSION >= "1.1.0" and Vagrant.configure("2") do |config| + (49000..49900).each do |port| + config.vm.network :forwarded_port, :host => port, :guest => port end + end end diff --git a/api.go b/api.go index 834c41a68c..8eb892f259 100644 --- a/api.go +++ b/api.go @@ -9,6 +9,7 @@ import ( "io" "io/ioutil" "log" + "mime" "net" "net/http" "os" @@ -17,7 +18,7 @@ import ( "strings" ) -const APIVERSION = 1.3 +const APIVERSION = 1.4 const DEFAULTHTTPHOST string = "127.0.0.1" const DEFAULTHTTPPORT int = 4243 @@ -81,13 +82,21 @@ func getBoolParam(value string) (bool, error) { return ret, nil } +func matchesContentType(contentType, expectedType string) bool { + mimetype, _, err := mime.ParseMediaType(contentType) + if err != nil { + utils.Debugf("Error parsing media type: %s error: %s", contentType, err.Error()) + } + return err == nil && mimetype == expectedType +} + func postAuth(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { authConfig := &auth.AuthConfig{} err := json.NewDecoder(r.Body).Decode(authConfig) if err != nil { return err } - status, err := auth.Login(authConfig) + status, err := auth.Login(authConfig, srv.HTTPRequestFactory()) if err != nil { return err } @@ -271,11 +280,18 @@ func getContainersChanges(srv *Server, version float64, w http.ResponseWriter, r } func getContainersTop(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if version < 1.4 { + return fmt.Errorf("top was improved a lot since 1.3, Please upgrade your docker client.") + } if vars == nil { return fmt.Errorf("Missing parameter") } + if err := parseForm(r); err != nil { + return err + } name := vars["name"] - procsStr, err := srv.ContainerTop(name) + ps_args := r.Form.Get("ps_args") + procsStr, err := srv.ContainerTop(name, ps_args) if err != nil { return err } @@ -379,7 +395,7 @@ func postImagesCreate(srv *Server, version float64, w http.ResponseWriter, r *ht } sf := utils.NewStreamFormatter(version > 1.0) if image != "" { //pull - if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}); err != nil { + if err := srv.ImagePull(image, tag, w, sf, &auth.AuthConfig{}, version > 1.3); err != nil { if sf.Used() { w.Write(sf.FormatError(err)) return nil @@ -481,7 +497,12 @@ func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r return err } - if len(config.Dns) == 0 && len(srv.runtime.Dns) == 0 && utils.CheckLocalDns() { + resolvConf, err := utils.GetResolvConf() + if err != nil { + return err + } + + if len(config.Dns) == 0 && len(srv.runtime.Dns) == 0 && utils.CheckLocalDns(resolvConf) { out.Warnings = append(out.Warnings, fmt.Sprintf("Docker detected local DNS server on resolv.conf. Using default external servers: %v", defaultDns)) config.Dns = defaultDns } @@ -501,6 +522,11 @@ func postContainersCreate(srv *Server, version float64, w http.ResponseWriter, r out.Warnings = append(out.Warnings, "Your kernel does not support memory swap capabilities. Limitation discarded.") } + if !srv.runtime.capabilities.IPv4Forwarding { + log.Println("Warning: IPv4 forwarding is disabled.") + out.Warnings = append(out.Warnings, "IPv4 forwarding is disabled.") + } + b, err := json.Marshal(out) if err != nil { return err @@ -582,7 +608,7 @@ func postContainersStart(srv *Server, version float64, w http.ResponseWriter, r // allow a nil body for backwards compatibility if r.Body != nil { - if r.Header.Get("Content-Type") == "application/json" { + if matchesContentType(r.Header.Get("Content-Type"), "application/json") { if err := json.NewDecoder(r.Body).Decode(hostConfig); err != nil { return err } @@ -786,12 +812,8 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ remoteURL := r.FormValue("remote") repoName := r.FormValue("t") rawSuppressOutput := r.FormValue("q") - tag := "" - if strings.Contains(repoName, ":") { - remoteParts := strings.Split(repoName, ":") - tag = remoteParts[1] - repoName = remoteParts[0] - } + rawNoCache := r.FormValue("nocache") + repoName, tag := utils.ParseRepositoryTag(repoName) var context io.Reader @@ -837,8 +859,12 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ if err != nil { return err } + noCache, err := getBoolParam(rawNoCache) + if err != nil { + return err + } - b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput) + b := NewBuildFile(srv, utils.NewWriteFlusher(w), !suppressOutput, !noCache) id, err := b.Build(context) if err != nil { fmt.Fprintf(w, "Error build: %s\n", err) @@ -850,6 +876,36 @@ func postBuild(srv *Server, version float64, w http.ResponseWriter, r *http.Requ return nil } +func postContainersCopy(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { + if vars == nil { + return fmt.Errorf("Missing parameter") + } + name := vars["name"] + + copyData := &APICopy{} + contentType := r.Header.Get("Content-Type") + if contentType == "application/json" { + if err := json.NewDecoder(r.Body).Decode(copyData); err != nil { + return err + } + } else { + return fmt.Errorf("Content-Type not supported: %s", contentType) + } + + if copyData.Resource == "" { + return fmt.Errorf("Resource cannot be empty") + } + if copyData.Resource[0] == '/' { + copyData.Resource = copyData.Resource[1:] + } + + if err := srv.ContainerCopy(name, copyData.Resource, w); err != nil { + utils.Debugf("%s", err.Error()) + return err + } + return nil +} + func optionsHandler(srv *Server, version float64, w http.ResponseWriter, r *http.Request, vars map[string]string) error { w.WriteHeader(http.StatusOK) return nil @@ -897,6 +953,7 @@ func createRouter(srv *Server, logging bool) (*mux.Router, error) { "/containers/{name:.*}/wait": postContainersWait, "/containers/{name:.*}/resize": postContainersResize, "/containers/{name:.*}/attach": postContainersAttach, + "/containers/{name:.*}/copy": postContainersCopy, }, "DELETE": { "/containers/{name:.*}": deleteContainers, diff --git a/api_params.go b/api_params.go index 26d70711a1..df879f63ee 100644 --- a/api_params.go +++ b/api_params.go @@ -17,21 +17,23 @@ type APIImages struct { } type APIInfo struct { - Debug bool - Containers int - Images int - NFd int `json:",omitempty"` - NGoroutines int `json:",omitempty"` - MemoryLimit bool `json:",omitempty"` - SwapLimit bool `json:",omitempty"` - NEventsListener int `json:",omitempty"` + Debug bool + Containers int + Images int + NFd int `json:",omitempty"` + NGoroutines int `json:",omitempty"` + MemoryLimit bool `json:",omitempty"` + SwapLimit bool `json:",omitempty"` + IPv4Forwarding bool `json:",omitempty"` + LXCVersion string `json:",omitempty"` + NEventsListener int `json:",omitempty"` + KernelVersion string `json:",omitempty"` + IndexServerAddress string `json:",omitempty"` } type APITop struct { - PID string - Tty string - Time string - Cmd string + Titles []string + Processes [][]string } type APIRmi struct { @@ -86,3 +88,8 @@ type APIImageConfig struct { ID string `json:"Id"` *Config } + +type APICopy struct { + Resource string + HostPath string +} diff --git a/api_test.go b/api_test.go index 13731dbf9e..928daf963c 100644 --- a/api_test.go +++ b/api_test.go @@ -482,24 +482,33 @@ func TestGetContainersTop(t *testing.T) { } r := httptest.NewRecorder() - if err := getContainersTop(srv, APIVERSION, r, nil, map[string]string{"name": container.ID}); err != nil { + req, err := http.NewRequest("GET", "/"+container.ID+"/top?ps_args=u", bytes.NewReader([]byte{})) + if err != nil { t.Fatal(err) } - procs := []APITop{} + if err := getContainersTop(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil { + t.Fatal(err) + } + procs := APITop{} if err := json.Unmarshal(r.Body.Bytes(), &procs); err != nil { t.Fatal(err) } - if len(procs) != 2 { - t.Fatalf("Expected 2 processes, found %d.", len(procs)) + if len(procs.Titles) != 11 { + t.Fatalf("Expected 11 titles, found %d.", len(procs.Titles)) + } + if procs.Titles[0] != "USER" || procs.Titles[10] != "COMMAND" { + t.Fatalf("Expected Titles[0] to be USER and Titles[10] to be COMMAND, found %s and %s.", procs.Titles[0], procs.Titles[10]) } - if procs[0].Cmd != "sh" && procs[0].Cmd != "busybox" { - t.Fatalf("Expected `busybox` or `sh`, found %s.", procs[0].Cmd) + if len(procs.Processes) != 2 { + t.Fatalf("Expected 2 processes, found %d.", len(procs.Processes)) } - - if procs[1].Cmd != "sh" && procs[1].Cmd != "busybox" { - t.Fatalf("Expected `busybox` or `sh`, found %s.", procs[1].Cmd) + if procs.Processes[0][10] != "/bin/sh" && procs.Processes[0][10] != "sleep" { + t.Fatalf("Expected `sleep` or `/bin/sh`, found %s.", procs.Processes[0][10]) + } + if procs.Processes[1][10] != "/bin/sh" && procs.Processes[1][10] != "sleep" { + t.Fatalf("Expected `sleep` or `/bin/sh`, found %s.", procs.Processes[1][10]) } } @@ -895,6 +904,12 @@ func TestPostContainersAttach(t *testing.T) { stdin, stdinPipe := io.Pipe() stdout, stdoutPipe := io.Pipe() + // Try to avoid the timeoout in destroy. Best effort, don't check error + defer func() { + closeWrap(stdin, stdinPipe, stdout, stdoutPipe) + container.Kill() + }() + // Attach to it c1 := make(chan struct{}) go func() { @@ -934,7 +949,7 @@ func TestPostContainersAttach(t *testing.T) { } // Wait for attach to finish, the client disconnected, therefore, Attach finished his job - setTimeout(t, "Waiting for CmdAttach timed out", 2*time.Second, func() { + setTimeout(t, "Waiting for CmdAttach timed out", 10*time.Second, func() { <-c1 }) @@ -1127,6 +1142,84 @@ func TestDeleteImages(t *testing.T) { } */ } +func TestJsonContentType(t *testing.T) { + if !matchesContentType("application/json", "application/json") { + t.Fail() + } + + if !matchesContentType("application/json; charset=utf-8", "application/json") { + t.Fail() + } + + if matchesContentType("dockerapplication/json", "application/json") { + t.Fail() + } +} + +func TestPostContainersCopy(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + + srv := &Server{runtime: runtime} + + builder := NewBuilder(runtime) + + // Create a container and remove a file + container, err := builder.Create( + &Config{ + Image: GetTestImage(runtime).ID, + Cmd: []string{"touch", "/test.txt"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) + + if err := container.Run(); err != nil { + t.Fatal(err) + } + + r := httptest.NewRecorder() + copyData := APICopy{HostPath: ".", Resource: "/test.txt"} + + jsonData, err := json.Marshal(copyData) + if err != nil { + t.Fatal(err) + } + + req, err := http.NewRequest("POST", "/containers/"+container.ID+"/copy", bytes.NewReader(jsonData)) + if err != nil { + t.Fatal(err) + } + req.Header.Add("Content-Type", "application/json") + if err = postContainersCopy(srv, APIVERSION, r, req, map[string]string{"name": container.ID}); err != nil { + t.Fatal(err) + } + + if r.Code != http.StatusOK { + t.Fatalf("%d OK expected, received %d\n", http.StatusOK, r.Code) + } + + found := false + for tarReader := tar.NewReader(r.Body); ; { + h, err := tarReader.Next() + if err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + if h.Name == "test.txt" { + found = true + break + } + } + if !found { + t.Fatalf("The created test file has not been found in the copied output") + } +} + // Mocked types for tests type NopConn struct { io.ReadCloser diff --git a/archive.go b/archive.go index 01af86006f..bb79cd34d4 100644 --- a/archive.go +++ b/archive.go @@ -173,7 +173,7 @@ func CopyWithTar(src, dst string) error { } // Create dst, copy src's content into it utils.Debugf("Creating dest directory: %s", dst) - if err := os.MkdirAll(dst, 0700); err != nil && !os.IsExist(err) { + if err := os.MkdirAll(dst, 0755); err != nil && !os.IsExist(err) { return err } utils.Debugf("Calling TarUntar(%s, %s)", src, dst) diff --git a/auth/auth.go b/auth/auth.go index bffed49807..b9e1ee153b 100644 --- a/auth/auth.go +++ b/auth/auth.go @@ -5,6 +5,7 @@ import ( "encoding/json" "errors" "fmt" + "github.com/dotcloud/docker/utils" "io/ioutil" "net/http" "os" @@ -18,7 +19,7 @@ const CONFIGFILE = ".dockercfg" // Only used for user auth + account creation const INDEXSERVER = "https://index.docker.io/v1/" -//const INDEXSERVER = "http://indexstaging-docker.dotcloud.com/" +//const INDEXSERVER = "https://indexstaging-docker.dotcloud.com/v1/" var ( ErrConfigFileMissing = errors.New("The Auth config file is missing") @@ -116,14 +117,19 @@ func SaveConfig(configFile *ConfigFile) error { os.Remove(confFile) return nil } + + configs := make(map[string]AuthConfig, len(configFile.Configs)) for k, authConfig := range configFile.Configs { - authConfig.Auth = encodeAuth(&authConfig) - authConfig.Username = "" - authConfig.Password = "" - configFile.Configs[k] = authConfig + authCopy := authConfig + + authCopy.Auth = encodeAuth(&authCopy) + authCopy.Username = "" + authCopy.Password = "" + + configs[k] = authCopy } - b, err := json.Marshal(configFile.Configs) + b, err := json.Marshal(configs) if err != nil { return err } @@ -135,7 +141,7 @@ func SaveConfig(configFile *ConfigFile) error { } // try to register/login to the registry server -func Login(authConfig *AuthConfig) (string, error) { +func Login(authConfig *AuthConfig, factory *utils.HTTPRequestFactory) (string, error) { client := &http.Client{} reqStatusCode := 0 var status string @@ -166,7 +172,7 @@ func Login(authConfig *AuthConfig) (string, error) { "Please check your e-mail for a confirmation link.") } else if reqStatusCode == 400 { if string(reqBody) == "\"Username or email already exists\"" { - req, err := http.NewRequest("GET", IndexServerAddress()+"users/", nil) + req, err := factory.NewRequest("GET", IndexServerAddress()+"users/", nil) req.SetBasicAuth(authConfig.Username, authConfig.Password) resp, err := client.Do(req) if err != nil { diff --git a/auth/auth_test.go b/auth/auth_test.go index 458a505ea2..24a0666cf7 100644 --- a/auth/auth_test.go +++ b/auth/auth_test.go @@ -3,6 +3,7 @@ package auth import ( "crypto/rand" "encoding/hex" + "io/ioutil" "os" "strings" "testing" @@ -32,7 +33,7 @@ func TestLogin(t *testing.T) { os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") defer os.Setenv("DOCKER_INDEX_URL", "") authConfig := &AuthConfig{Username: "unittester", Password: "surlautrerivejetattendrai", Email: "noise+unittester@dotcloud.com"} - status, err := Login(authConfig) + status, err := Login(authConfig, nil) if err != nil { t.Fatal(err) } @@ -51,8 +52,8 @@ func TestCreateAccount(t *testing.T) { } token := hex.EncodeToString(tokenBuffer)[:12] username := "ut" + token - authConfig := &AuthConfig{Username: username, Password: "test42", Email: "docker-ut+"+token+"@example.com"} - status, err := Login(authConfig) + authConfig := &AuthConfig{Username: username, Password: "test42", Email: "docker-ut+" + token + "@example.com"} + status, err := Login(authConfig, nil) if err != nil { t.Fatal(err) } @@ -62,7 +63,7 @@ func TestCreateAccount(t *testing.T) { t.Fatalf("Expected status: \"%s\", found \"%s\" instead.", expectedStatus, status) } - status, err = Login(authConfig) + status, err = Login(authConfig, nil) if err == nil { t.Fatalf("Expected error but found nil instead") } @@ -73,3 +74,39 @@ func TestCreateAccount(t *testing.T) { t.Fatalf("Expected message \"%s\" but found \"%s\" instead", expectedError, err) } } + +func TestSameAuthDataPostSave(t *testing.T) { + root, err := ioutil.TempDir("", "docker-test") + if err != nil { + t.Fatal(err) + } + configFile := &ConfigFile{ + rootPath: root, + Configs: make(map[string]AuthConfig, 1), + } + + configFile.Configs["testIndex"] = AuthConfig{ + Username: "docker-user", + Password: "docker-pass", + Email: "docker@docker.io", + } + + err = SaveConfig(configFile) + if err != nil { + t.Fatal(err) + } + + authConfig := configFile.Configs["testIndex"] + if authConfig.Username != "docker-user" { + t.Fail() + } + if authConfig.Password != "docker-pass" { + t.Fail() + } + if authConfig.Email != "docker@docker.io" { + t.Fail() + } + if authConfig.Auth != "" { + t.Fail() + } +} diff --git a/builder.go b/builder.go index ab07233cb8..9124f76ac1 100644 --- a/builder.go +++ b/builder.go @@ -38,7 +38,9 @@ func (builder *Builder) Create(config *Config) (*Container, error) { MergeConfig(config, img.Config) } - if config.Cmd == nil || len(config.Cmd) == 0 { + if len(config.Entrypoint) != 0 && config.Cmd == nil { + config.Cmd = []string{} + } else if config.Cmd == nil || len(config.Cmd) == 0 { return nil, fmt.Errorf("No command specified") } @@ -80,7 +82,12 @@ func (builder *Builder) Create(config *Config) (*Container, error) { return nil, err } - if len(config.Dns) == 0 && len(builder.runtime.Dns) == 0 && utils.CheckLocalDns() { + resolvConf, err := utils.GetResolvConf() + if err != nil { + return nil, err + } + + if len(config.Dns) == 0 && len(builder.runtime.Dns) == 0 && utils.CheckLocalDns(resolvConf) { //"WARNING: Docker detected local DNS server on resolv.conf. Using default external servers: %v", defaultDns builder.runtime.Dns = defaultDns } @@ -124,6 +131,10 @@ func (builder *Builder) Create(config *Config) (*Container, error) { func (builder *Builder) Commit(container *Container, repository, tag, comment, author string, config *Config) (*Image, error) { // FIXME: freeze the container before copying it to avoid data corruption? // FIXME: this shouldn't be in commands. + if err := container.EnsureMounted(); err != nil { + return nil, err + } + rwTar, err := container.ExportRw() if err != nil { return nil, err diff --git a/buildfile.go b/buildfile.go index 75ebdd7a7c..b969f758ec 100644 --- a/buildfile.go +++ b/buildfile.go @@ -11,6 +11,7 @@ import ( "os" "path" "reflect" + "regexp" "strings" ) @@ -25,11 +26,12 @@ type buildFile struct { builder *Builder srv *Server - image string - maintainer string - config *Config - context string - verbose bool + image string + maintainer string + config *Config + context string + verbose bool + utilizeCache bool tmpContainers map[string]struct{} tmpImages map[string]struct{} @@ -54,7 +56,7 @@ func (b *buildFile) CmdFrom(name string) error { if err != nil { if b.runtime.graph.IsNotExist(err) { remote, tag := utils.ParseRepositoryTag(name) - if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil); err != nil { + if err := b.srv.ImagePull(remote, tag, b.out, utils.NewStreamFormatter(false), nil, true); err != nil { return err } image, err = b.runtime.repositories.LookupImage(name) @@ -67,6 +69,9 @@ func (b *buildFile) CmdFrom(name string) error { } b.image = image.ID b.config = &Config{} + if b.config.Env == nil || len(b.config.Env) == 0 { + b.config.Env = append(b.config.Env, "HOME=/", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin") + } return nil } @@ -88,17 +93,21 @@ func (b *buildFile) CmdRun(args string) error { b.config.Cmd = nil MergeConfig(b.config, config) + defer func(cmd []string) { b.config.Cmd = cmd }(cmd) + utils.Debugf("Command to be executed: %v", b.config.Cmd) - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return err + } else if cache != nil { + fmt.Fprintf(b.out, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } } cid, err := b.run() @@ -108,10 +117,44 @@ func (b *buildFile) CmdRun(args string) error { if err := b.commit(cid, cmd, "run"); err != nil { return err } - b.config.Cmd = cmd + return nil } +func (b *buildFile) FindEnvKey(key string) int { + for k, envVar := range b.config.Env { + envParts := strings.SplitN(envVar, "=", 2) + if key == envParts[0] { + return k + } + } + return -1 +} + +func (b *buildFile) ReplaceEnvMatches(value string) (string, error) { + exp, err := regexp.Compile("(\\\\\\\\+|[^\\\\]|\\b|\\A)\\$({?)([[:alnum:]_]+)(}?)") + if err != nil { + return value, err + } + matches := exp.FindAllString(value, -1) + for _, match := range matches { + match = match[strings.Index(match, "$"):] + matchKey := strings.Trim(match, "${}") + + for _, envVar := range b.config.Env { + envParts := strings.SplitN(envVar, "=", 2) + envKey := envParts[0] + envValue := envParts[1] + + if envKey == matchKey { + value = strings.Replace(value, match, envValue, -1) + break + } + } + } + return value, nil +} + func (b *buildFile) CmdEnv(args string) error { tmp := strings.SplitN(args, " ", 2) if len(tmp) != 2 { @@ -120,14 +163,19 @@ func (b *buildFile) CmdEnv(args string) error { key := strings.Trim(tmp[0], " \t") value := strings.Trim(tmp[1], " \t") - for i, elem := range b.config.Env { - if strings.HasPrefix(elem, key+"=") { - b.config.Env[i] = key + "=" + value - return nil - } + envKey := b.FindEnvKey(key) + replacedValue, err := b.ReplaceEnvMatches(value) + if err != nil { + return err } - b.config.Env = append(b.config.Env, key+"="+value) - return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s=%s", key, value)) + replacedVar := fmt.Sprintf("%s=%s", key, replacedValue) + + if envKey >= 0 { + b.config.Env[envKey] = replacedVar + } else { + b.config.Env = append(b.config.Env, replacedVar) + } + return b.commit("", b.config.Cmd, fmt.Sprintf("ENV %s", replacedVar)) } func (b *buildFile) CmdCmd(args string) error { @@ -230,6 +278,9 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { if strings.HasSuffix(dest, "/") { destPath = destPath + "/" } + if !strings.HasPrefix(origPath, b.context) { + return fmt.Errorf("Forbidden path: %s", origPath) + } fi, err := os.Stat(origPath) if err != nil { return err @@ -242,7 +293,7 @@ func (b *buildFile) addContext(container *Container, orig, dest string) error { } else if err := UntarPath(origPath, destPath); err != nil { utils.Debugf("Couldn't untar %s to %s: %s", origPath, destPath, err) // If that fails, just copy it as a regular file - if err := os.MkdirAll(path.Dir(destPath), 0700); err != nil { + if err := os.MkdirAll(path.Dir(destPath), 0755); err != nil { return err } if err := CopyWithTar(origPath, destPath); err != nil { @@ -260,8 +311,16 @@ func (b *buildFile) CmdAdd(args string) error { if len(tmp) != 2 { return fmt.Errorf("Invalid ADD format") } - orig := strings.Trim(tmp[0], " \t") - dest := strings.Trim(tmp[1], " \t") + + orig, err := b.ReplaceEnvMatches(strings.Trim(tmp[0], " \t")) + if err != nil { + return err + } + + dest, err := b.ReplaceEnvMatches(strings.Trim(tmp[1], " \t")) + if err != nil { + return err + } cmd := b.config.Cmd b.config.Cmd = []string{"/bin/sh", "-c", fmt.Sprintf("#(nop) ADD %s in %s", orig, dest)} @@ -346,16 +405,19 @@ func (b *buildFile) commit(id string, autoCmd []string, comment string) error { b.config.Cmd = []string{"/bin/sh", "-c", "#(nop) " + comment} defer func(cmd []string) { b.config.Cmd = cmd }(cmd) - if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { - return err - } else if cache != nil { - fmt.Fprintf(b.out, " ---> Using cache\n") - utils.Debugf("[BUILDER] Use cached version") - b.image = cache.ID - return nil - } else { - utils.Debugf("[BUILDER] Cache miss") + if b.utilizeCache { + if cache, err := b.srv.ImageGetCached(b.image, b.config); err != nil { + return err + } else if cache != nil { + fmt.Fprintf(b.out, " ---> Using cache\n") + utils.Debugf("[BUILDER] Use cached version") + b.image = cache.ID + return nil + } else { + utils.Debugf("[BUILDER] Cache miss") + } } + container, err := b.builder.Create(b.config) if err != nil { return err @@ -449,7 +511,7 @@ func (b *buildFile) Build(context io.Reader) (string, error) { return "", fmt.Errorf("An error occured during the build\n") } -func NewBuildFile(srv *Server, out io.Writer, verbose bool) BuildFile { +func NewBuildFile(srv *Server, out io.Writer, verbose, utilizeCache bool) BuildFile { return &buildFile{ builder: NewBuilder(srv.runtime), runtime: srv.runtime, @@ -459,5 +521,6 @@ func NewBuildFile(srv *Server, out io.Writer, verbose bool) BuildFile { tmpContainers: make(map[string]struct{}), tmpImages: make(map[string]struct{}), verbose: verbose, + utilizeCache: utilizeCache, } } diff --git a/buildfile_test.go b/buildfile_test.go index b7eca52336..d89c40d16c 100644 --- a/buildfile_test.go +++ b/buildfile_test.go @@ -129,6 +129,38 @@ CMD Hello world nil, nil, }, + + { + ` +from {IMAGE} +env FOO /foo/baz +env BAR /bar +env BAZ $BAR +env FOOPATH $PATH:$FOO +run [ "$BAR" = "$BAZ" ] +run [ "$FOOPATH" = "$PATH:/foo/baz" ] +`, + nil, + nil, + }, + + { + ` +from {IMAGE} +env FOO /bar +env TEST testdir +env BAZ /foobar +add testfile $BAZ/ +add $TEST $FOO +run [ "$(cat /foobar/testfile)" = "test1" ] +run [ "$(cat /bar/withfile)" = "test2" ] +`, + [][2]string{ + {"testfile", "test1"}, + {"testdir/withfile", "test2"}, + }, + nil, + }, } // FIXME: test building with 2 successive overlapping ADD commands @@ -163,21 +195,23 @@ func mkTestingFileServer(files [][2]string) (*httptest.Server, error) { func TestBuild(t *testing.T) { for _, ctx := range testContexts { - buildImage(ctx, t) + buildImage(ctx, t, nil, true) } } -func buildImage(context testContextTemplate, t *testing.T) *Image { - runtime, err := newTestRuntime() - if err != nil { - t.Fatal(err) - } - defer nuke(runtime) +func buildImage(context testContextTemplate, t *testing.T, srv *Server, useCache bool) *Image { + if srv == nil { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) - srv := &Server{ - runtime: runtime, - pullingPool: make(map[string]struct{}), - pushingPool: make(map[string]struct{}), + srv = &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } } httpServer, err := mkTestingFileServer(context.remoteFiles) @@ -192,10 +226,10 @@ func buildImage(context testContextTemplate, t *testing.T) *Image { } port := httpServer.URL[idx+1:] - ip := runtime.networkManager.bridgeNetwork.IP + ip := srv.runtime.networkManager.bridgeNetwork.IP dockerfile := constructDockerfile(context.dockerfile, ip, port) - buildfile := NewBuildFile(srv, ioutil.Discard, false) + buildfile := NewBuildFile(srv, ioutil.Discard, false, useCache) id, err := buildfile.Build(mkTestContext(dockerfile, context.files, t)) if err != nil { t.Fatal(err) @@ -213,7 +247,7 @@ func TestVolume(t *testing.T) { from {IMAGE} volume /test cmd Hello world - `, nil, nil}, t) + `, nil, nil}, t, nil, true) if len(img.Config.Volumes) == 0 { t.Fail() @@ -229,7 +263,7 @@ func TestBuildMaintainer(t *testing.T) { img := buildImage(testContextTemplate{` from {IMAGE} maintainer dockerio - `, nil, nil}, t) + `, nil, nil}, t, nil, true) if img.Author != "dockerio" { t.Fail() @@ -241,9 +275,15 @@ func TestBuildEnv(t *testing.T) { from {IMAGE} env port 4243 `, - nil, nil}, t) - - if img.Config.Env[0] != "port=4243" { + nil, nil}, t, nil, true) + hasEnv := false + for _, envVar := range img.Config.Env { + if envVar == "port=4243" { + hasEnv = true + break + } + } + if !hasEnv { t.Fail() } } @@ -253,7 +293,7 @@ func TestBuildCmd(t *testing.T) { from {IMAGE} cmd ["/bin/echo", "Hello World"] `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.Cmd[0] != "/bin/echo" { t.Log(img.Config.Cmd[0]) @@ -270,7 +310,7 @@ func TestBuildExpose(t *testing.T) { from {IMAGE} expose 4243 `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.PortSpecs[0] != "4243" { t.Fail() @@ -282,8 +322,153 @@ func TestBuildEntrypoint(t *testing.T) { from {IMAGE} entrypoint ["/bin/echo"] `, - nil, nil}, t) + nil, nil}, t, nil, true) if img.Config.Entrypoint[0] != "/bin/echo" { } } + +// testing #1405 - config.Cmd does not get cleaned up if +// utilizing cache +func TestBuildEntrypointRunCleanup(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + img := buildImage(testContextTemplate{` + from {IMAGE} + run echo "hello" + `, + nil, nil}, t, srv, true) + + img = buildImage(testContextTemplate{` + from {IMAGE} + run echo "hello" + add foo /foo + entrypoint ["/bin/echo"] + `, + [][2]string{{"foo", "HEYO"}}, nil}, t, srv, true) + + if len(img.Config.Cmd) != 0 { + t.Fail() + } +} + +func TestBuildImageWithCache(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + `, + nil, nil} + + img := buildImage(template, t, srv, true) + imageId := img.ID + + img = nil + img = buildImage(template, t, srv, true) + + if imageId != img.ID { + t.Logf("Image ids should match: %s != %s", imageId, img.ID) + t.Fail() + } +} + +func TestBuildImageWithoutCache(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + template := testContextTemplate{` + from {IMAGE} + maintainer dockerio + `, + nil, nil} + + img := buildImage(template, t, srv, true) + imageId := img.ID + + img = nil + img = buildImage(template, t, srv, false) + + if imageId == img.ID { + t.Logf("Image ids should not match: %s == %s", imageId, img.ID) + t.Fail() + } +} + +func TestForbiddenContextPath(t *testing.T) { + runtime, err := newTestRuntime() + if err != nil { + t.Fatal(err) + } + defer nuke(runtime) + + srv := &Server{ + runtime: runtime, + pullingPool: make(map[string]struct{}), + pushingPool: make(map[string]struct{}), + } + + context := testContextTemplate{` + from {IMAGE} + maintainer dockerio + add ../../ test/ + `, + [][2]string{{"test.txt", "test1"}, {"other.txt", "other"}}, nil} + + httpServer, err := mkTestingFileServer(context.remoteFiles) + if err != nil { + t.Fatal(err) + } + defer httpServer.Close() + + idx := strings.LastIndex(httpServer.URL, ":") + if idx < 0 { + t.Fatalf("could not get port from test http server address %s", httpServer.URL) + } + port := httpServer.URL[idx+1:] + + ip := srv.runtime.networkManager.bridgeNetwork.IP + dockerfile := constructDockerfile(context.dockerfile, ip, port) + + buildfile := NewBuildFile(srv, ioutil.Discard, false, true) + _, err = buildfile.Build(mkTestContext(dockerfile, context.files, t)) + + if err == nil { + t.Log("Error should not be nil") + t.Fail() + } + + if err.Error() != "Forbidden path: /" { + t.Logf("Error message is not expected: %s", err.Error()) + t.Fail() + } +} diff --git a/commands.go b/commands.go index 2d8ea4efb5..92e24e89e0 100644 --- a/commands.go +++ b/commands.go @@ -27,7 +27,7 @@ import ( "unicode" ) -const VERSION = "0.5.0-dev" +const VERSION = "0.5.1-dev" var ( GITCOMMIT string @@ -77,6 +77,7 @@ func (cli *DockerCli) CmdHelp(args ...string) error { {"attach", "Attach to a running container"}, {"build", "Build a container from a Dockerfile"}, {"commit", "Create a new image from a container's changes"}, + {"cp", "Copy files/folders from the containers filesystem to the host path"}, {"diff", "Inspect changes on a container's filesystem"}, {"events", "Get real time events from the server"}, {"export", "Stream the contents of a container as a tar archive"}, @@ -158,9 +159,9 @@ func mkBuildContext(dockerfile string, files [][2]string) (Archive, error) { func (cli *DockerCli) CmdBuild(args ...string) error { cmd := Subcmd("build", "[OPTIONS] PATH | URL | -", "Build a new container image from the source code at PATH") - tag := cmd.String("t", "", "Tag to be applied to the resulting image in case of success") + tag := cmd.String("t", "", "Repository name (and optionally a tag) to be applied to the resulting image in case of success") suppressOutput := cmd.Bool("q", false, "Suppress verbose build output") - + noCache := cmd.Bool("no-cache", false, "Do not use cache when building the image") if err := cmd.Parse(args); err != nil { return nil } @@ -196,7 +197,7 @@ func (cli *DockerCli) CmdBuild(args ...string) error { // FIXME: ProgressReader shouldn't be this annoyning to use if context != nil { sf := utils.NewStreamFormatter(false) - body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("Uploading context", "%v bytes%0.0s%0.0s"), sf) + body = utils.ProgressReader(ioutil.NopCloser(context), 0, cli.err, sf.FormatProgress("", "Uploading context", "%v bytes%0.0s%0.0s"), sf, true) } // Upload the build context v := &url.Values{} @@ -208,6 +209,9 @@ func (cli *DockerCli) CmdBuild(args ...string) error { if isRemote { v.Set("remote", cmd.Arg(0)) } + if *noCache { + v.Set("nocache", "1") + } req, err := http.NewRequest("POST", fmt.Sprintf("/v%g/build?%s", APIVERSION, v.Encode()), body) if err != nil { return err @@ -314,13 +318,21 @@ func (cli *DockerCli) CmdLogin(args ...string) error { email string ) + var promptDefault = func(prompt string, configDefault string) { + if configDefault == "" { + fmt.Fprintf(cli.out, "%s: ", prompt) + } else { + fmt.Fprintf(cli.out, "%s (%s): ", prompt, configDefault) + } + } + authconfig, ok := cli.configFile.Configs[auth.IndexServerAddress()] if !ok { authconfig = auth.AuthConfig{} } if *flUsername == "" { - fmt.Fprintf(cli.out, "Username (%s): ", authconfig.Username) + promptDefault("Username", authconfig.Username) username = readAndEchoString(cli.in, cli.out) if username == "" { username = authconfig.Username @@ -340,7 +352,7 @@ func (cli *DockerCli) CmdLogin(args ...string) error { } if *flEmail == "" { - fmt.Fprintf(cli.out, "Email (%s): ", authconfig.Email) + promptDefault("Email", authconfig.Email) email = readAndEchoString(cli.in, cli.out) if email == "" { email = authconfig.Email @@ -440,6 +452,15 @@ func (cli *DockerCli) CmdVersion(args ...string) error { if out.GoVersion != "" { fmt.Fprintf(cli.out, "Go version: %s\n", out.GoVersion) } + + release := utils.GetReleaseVersion() + if release != "" { + fmt.Fprintf(cli.out, "Last stable version: %s", release) + if strings.Trim(VERSION, "-dev") != release || strings.Trim(out.Version, "-dev") != release { + fmt.Fprintf(cli.out, ", please update docker") + } + fmt.Fprintf(cli.out, "\n") + } return nil } @@ -471,7 +492,17 @@ func (cli *DockerCli) CmdInfo(args ...string) error { fmt.Fprintf(cli.out, "Debug mode (client): %v\n", os.Getenv("DEBUG") != "") fmt.Fprintf(cli.out, "Fds: %d\n", out.NFd) fmt.Fprintf(cli.out, "Goroutines: %d\n", out.NGoroutines) + fmt.Fprintf(cli.out, "LXC Version: %s\n", out.LXCVersion) fmt.Fprintf(cli.out, "EventsListeners: %d\n", out.NEventsListener) + fmt.Fprintf(cli.out, "Kernel Version: %s\n", out.KernelVersion) + } + + if len(out.IndexServerAddress) != 0 { + u := cli.configFile.Configs[out.IndexServerAddress].Username + if len(u) > 0 { + fmt.Fprintf(cli.out, "Username: %v\n", u) + fmt.Fprintf(cli.out, "Registry: %v\n", out.IndexServerAddress) + } } if !out.MemoryLimit { fmt.Fprintf(cli.err, "WARNING: No memory limit support\n") @@ -479,6 +510,9 @@ func (cli *DockerCli) CmdInfo(args ...string) error { if !out.SwapLimit { fmt.Fprintf(cli.err, "WARNING: No swap limit support\n") } + if !out.IPv4Forwarding { + fmt.Fprintf(cli.err, "WARNING: IPv4 forwarding is disabled.\n") + } return nil } @@ -594,23 +628,28 @@ func (cli *DockerCli) CmdTop(args ...string) error { if err := cmd.Parse(args); err != nil { return nil } - if cmd.NArg() != 1 { + if cmd.NArg() == 0 { cmd.Usage() return nil } - body, _, err := cli.call("GET", "/containers/"+cmd.Arg(0)+"/top", nil) + val := url.Values{} + if cmd.NArg() > 1 { + val.Set("ps_args", strings.Join(cmd.Args()[1:], " ")) + } + + body, _, err := cli.call("GET", "/containers/"+cmd.Arg(0)+"/top?"+val.Encode(), nil) if err != nil { return err } - var procs []APITop + procs := APITop{} err = json.Unmarshal(body, &procs) if err != nil { return err } w := tabwriter.NewWriter(cli.out, 20, 1, 3, ' ', 0) - fmt.Fprintln(w, "PID\tTTY\tTIME\tCMD") - for _, proc := range procs { - fmt.Fprintf(w, "%s\t%s\t%s\t%s\n", proc.PID, proc.Tty, proc.Time, proc.Cmd) + fmt.Fprintln(w, strings.Join(procs.Titles, "\t")) + for _, proc := range procs.Processes { + fmt.Fprintln(w, strings.Join(proc, "\t")) } w.Flush() return nil @@ -765,7 +804,7 @@ func (cli *DockerCli) CmdKill(args ...string) error { } func (cli *DockerCli) CmdImport(args ...string) error { - cmd := Subcmd("import", "URL|- [REPOSITORY [TAG]]", "Create a new filesystem image from the contents of a tarball") + cmd := Subcmd("import", "URL|- [REPOSITORY [TAG]]", "Create a new filesystem image from the contents of a tarball(.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz).") if err := cmd.Parse(args); err != nil { return nil @@ -799,10 +838,6 @@ func (cli *DockerCli) CmdPush(args ...string) error { return nil } - if err := cli.checkIfLogged("push"); err != nil { - return err - } - // If we're not using a custom registry, we know the restrictions // applied to repository names and can warn the user in advance. // Custom repositories can have different rules, and we must also @@ -811,13 +846,22 @@ func (cli *DockerCli) CmdPush(args ...string) error { return fmt.Errorf("Impossible to push a \"root\" repository. Please rename your repository in / (ex: %s/%s)", cli.configFile.Configs[auth.IndexServerAddress()].Username, name) } - buf, err := json.Marshal(cli.configFile.Configs[auth.IndexServerAddress()]) - if err != nil { - return err + v := url.Values{} + push := func() error { + buf, err := json.Marshal(cli.configFile.Configs[auth.IndexServerAddress()]) + if err != nil { + return err + } + + return cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out) } - v := url.Values{} - if err := cli.stream("POST", "/images/"+name+"/push?"+v.Encode(), bytes.NewBuffer(buf), cli.out); err != nil { + if err := push(); err != nil { + if err == fmt.Errorf("Authentication is required.") { + if err = cli.checkIfLogged("push"); err == nil { + return push() + } + } return err } return nil @@ -1407,7 +1451,7 @@ func (cli *DockerCli) CmdRun(args ...string) error { if config.AttachStdin || config.AttachStdout || config.AttachStderr { if config.Tty { if err := cli.monitorTtySize(runResult.ID); err != nil { - return err + utils.Debugf("Error monitoring TTY size: %s\n", err) } } @@ -1437,6 +1481,37 @@ func (cli *DockerCli) CmdRun(args ...string) error { return nil } +func (cli *DockerCli) CmdCp(args ...string) error { + cmd := Subcmd("cp", "CONTAINER:RESOURCE HOSTPATH", "Copy files/folders from the RESOURCE to the HOSTPATH") + if err := cmd.Parse(args); err != nil { + return nil + } + + if cmd.NArg() != 2 { + cmd.Usage() + return nil + } + + var copyData APICopy + info := strings.Split(cmd.Arg(0), ":") + + copyData.Resource = info[1] + copyData.HostPath = cmd.Arg(1) + + data, statusCode, err := cli.call("POST", "/containers/"+info[0]+"/copy", copyData) + if err != nil { + return err + } + + if statusCode == 200 { + r := bytes.NewReader(data) + if err := Untar(r, copyData.HostPath); err != nil { + return err + } + } + return nil +} + func (cli *DockerCli) checkIfLogged(action string) error { // If condition AND the login failed if cli.configFile.Configs[auth.IndexServerAddress()].Username == "" { @@ -1465,6 +1540,7 @@ func (cli *DockerCli) call(method, path string, data interface{}) ([]byte, int, return nil, -1, err } req.Header.Set("User-Agent", "Docker-Client/"+VERSION) + req.Host = cli.addr if data != nil { req.Header.Set("Content-Type", "application/json") } else if method == "POST" { @@ -1506,6 +1582,7 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e return err } req.Header.Set("User-Agent", "Docker-Client/"+VERSION) + req.Host = cli.addr if method == "POST" { req.Header.Set("Content-Type", "plain/text") } @@ -1535,17 +1612,8 @@ func (cli *DockerCli) stream(method, path string, in io.Reader, out io.Writer) e return fmt.Errorf("Error: %s", body) } - if resp.Header.Get("Content-Type") == "application/json" { - dec := json.NewDecoder(resp.Body) - for { - var jm utils.JSONMessage - if err := dec.Decode(&jm); err == io.EOF { - break - } else if err != nil { - return err - } - jm.Display(out) - } + if matchesContentType(resp.Header.Get("Content-Type"), "application/json") { + return utils.DisplayJSONMessagesStream(resp.Body, out) } else { if _, err := io.Copy(out, resp.Body); err != nil { return err @@ -1562,6 +1630,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea } req.Header.Set("User-Agent", "Docker-Client/"+VERSION) req.Header.Set("Content-Type", "plain/text") + req.Host = cli.addr dial, err := net.Dial(cli.proto, cli.addr) if err != nil { @@ -1578,6 +1647,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea receiveStdout := utils.Go(func() error { _, err := io.Copy(out, br) + utils.Debugf("[hijack] End of stdout") return err }) @@ -1592,6 +1662,7 @@ func (cli *DockerCli) hijack(method, path string, setRawTerminal bool, in io.Rea sendStdin := utils.Go(func() error { if in != nil { io.Copy(rwc, in) + utils.Debugf("[hijack] End of stdin") } if tcpc, ok := rwc.(*net.TCPConn); ok { if err := tcpc.CloseWrite(); err != nil { @@ -1654,13 +1725,12 @@ func (cli *DockerCli) monitorTtySize(id string) error { } cli.resizeTty(id) - c := make(chan os.Signal, 1) - signal.Notify(c, syscall.SIGWINCH) + sigchan := make(chan os.Signal, 1) + signal.Notify(sigchan, syscall.SIGWINCH) go func() { - for sig := range c { - if sig == syscall.SIGWINCH { - cli.resizeTty(id) - } + for { + <-sigchan + cli.resizeTty(id) } }() return nil diff --git a/commands_test.go b/commands_test.go index 030fb29f95..ddbc7f6b40 100644 --- a/commands_test.go +++ b/commands_test.go @@ -73,7 +73,7 @@ func TestRunHostname(t *testing.T) { t.Fatal(err) } }() - utils.Debugf("--") + setTimeout(t, "Reading command output time out", 2*time.Second, func() { cmdOutput, err := bufio.NewReader(stdout).ReadString('\n') if err != nil { @@ -90,6 +90,157 @@ func TestRunHostname(t *testing.T) { } +func TestRunExit(t *testing.T) { + stdin, stdinPipe := io.Pipe() + stdout, stdoutPipe := io.Pipe() + + cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + + c1 := make(chan struct{}) + go func() { + cli.CmdRun("-i", unitTestImageID, "/bin/cat") + close(c1) + }() + + setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + t.Fatal(err) + } + }) + + container := globalRuntime.List()[0] + + // Closing /bin/cat stdin, expect it to exit + if err := stdin.Close(); err != nil { + t.Fatal(err) + } + + // as the process exited, CmdRun must finish and unblock. Wait for it + setTimeout(t, "Waiting for CmdRun timed out", 10*time.Second, func() { + <-c1 + + go func() { + cli.CmdWait(container.ID) + }() + + if _, err := bufio.NewReader(stdout).ReadString('\n'); err != nil { + t.Fatal(err) + } + }) + + // Make sure that the client has been disconnected + setTimeout(t, "The client should have been disconnected once the remote process exited.", 2*time.Second, func() { + // Expecting pipe i/o error, just check that read does not block + stdin.Read([]byte{}) + }) + + // Cleanup pipes + if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil { + t.Fatal(err) + } +} + +// Expected behaviour: the process dies when the client disconnects +func TestRunDisconnect(t *testing.T) { + + stdin, stdinPipe := io.Pipe() + stdout, stdoutPipe := io.Pipe() + + cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + + c1 := make(chan struct{}) + go func() { + // We're simulating a disconnect so the return value doesn't matter. What matters is the + // fact that CmdRun returns. + cli.CmdRun("-i", unitTestImageID, "/bin/cat") + close(c1) + }() + + setTimeout(t, "Read/Write assertion timed out", 2*time.Second, func() { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + t.Fatal(err) + } + }) + + // Close pipes (simulate disconnect) + if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil { + t.Fatal(err) + } + + // as the pipes are close, we expect the process to die, + // therefore CmdRun to unblock. Wait for CmdRun + setTimeout(t, "Waiting for CmdRun timed out", 2*time.Second, func() { + <-c1 + }) + + // Client disconnect after run -i should cause stdin to be closed, which should + // cause /bin/cat to exit. + setTimeout(t, "Waiting for /bin/cat to exit timed out", 2*time.Second, func() { + container := globalRuntime.List()[0] + container.Wait() + if container.State.Running { + t.Fatalf("/bin/cat is still running after closing stdin") + } + }) +} + +// Expected behaviour: the process dies when the client disconnects +func TestRunDisconnectTty(t *testing.T) { + + stdin, stdinPipe := io.Pipe() + stdout, stdoutPipe := io.Pipe() + + cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + + c1 := make(chan struct{}) + go func() { + // We're simulating a disconnect so the return value doesn't matter. What matters is the + // fact that CmdRun returns. + if err := cli.CmdRun("-i", "-t", unitTestImageID, "/bin/cat"); err != nil { + utils.Debugf("Error CmdRun: %s\n", err) + } + + close(c1) + }() + + setTimeout(t, "Waiting for the container to be started timed out", 10*time.Second, func() { + for { + // Client disconnect after run -i should keep stdin out in TTY mode + l := globalRuntime.List() + if len(l) == 1 && l[0].State.Running { + break + } + time.Sleep(10 * time.Millisecond) + } + }) + + // Client disconnect after run -i should keep stdin out in TTY mode + container := globalRuntime.List()[0] + + setTimeout(t, "Read/Write assertion timed out", 2000*time.Second, func() { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + t.Fatal(err) + } + }) + + // Close pipes (simulate disconnect) + if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil { + t.Fatal(err) + } + + // In tty mode, we expect the process to stay alive even after client's stdin closes. + // Do not wait for run to finish + + // Give some time to monitor to do his thing + container.WaitTimeout(500 * time.Millisecond) + if !container.State.Running { + t.Fatalf("/bin/cat should still be running after closing stdin (tty mode)") + } +} + // TestAttachStdin checks attaching to stdin without stdout and stderr. // 'docker run -i -a stdin' should sends the client's stdin to the command, // then detach from it and print the container id. @@ -157,3 +308,73 @@ func TestRunAttachStdin(t *testing.T) { } } } + +// Expected behaviour, the process stays alive when the client disconnects +func TestAttachDisconnect(t *testing.T) { + stdin, stdinPipe := io.Pipe() + stdout, stdoutPipe := io.Pipe() + + cli := NewDockerCli(stdin, stdoutPipe, ioutil.Discard, testDaemonProto, testDaemonAddr) + defer cleanup(globalRuntime) + + go func() { + // Start a process in daemon mode + if err := cli.CmdRun("-d", "-i", unitTestImageID, "/bin/cat"); err != nil { + utils.Debugf("Error CmdRun: %s\n", err) + } + }() + + setTimeout(t, "Waiting for CmdRun timed out", 10*time.Second, func() { + if _, err := bufio.NewReader(stdout).ReadString('\n'); err != nil { + t.Fatal(err) + } + }) + + setTimeout(t, "Waiting for the container to be started timed out", 10*time.Second, func() { + for { + l := globalRuntime.List() + if len(l) == 1 && l[0].State.Running { + break + } + time.Sleep(10 * time.Millisecond) + } + }) + + container := globalRuntime.List()[0] + + // Attach to it + c1 := make(chan struct{}) + go func() { + // We're simulating a disconnect so the return value doesn't matter. What matters is the + // fact that CmdAttach returns. + cli.CmdAttach(container.ID) + close(c1) + }() + + setTimeout(t, "First read/write assertion timed out", 2*time.Second, func() { + if err := assertPipe("hello\n", "hello", stdout, stdinPipe, 15); err != nil { + t.Fatal(err) + } + }) + // Close pipes (client disconnects) + if err := closeWrap(stdin, stdinPipe, stdout, stdoutPipe); err != nil { + t.Fatal(err) + } + + // Wait for attach to finish, the client disconnected, therefore, Attach finished his job + setTimeout(t, "Waiting for CmdAttach timed out", 2*time.Second, func() { + <-c1 + }) + + // We closed stdin, expect /bin/cat to still be running + // Wait a little bit to make sure container.monitor() did his thing + err := container.WaitTimeout(500 * time.Millisecond) + if err == nil || !container.State.Running { + t.Fatalf("/bin/cat is not running after closing stdin") + } + + // Try to avoid the timeoout in destroy. Best effort, don't check error + cStdin, _ := container.StdinPipe() + cStdin.Close() + container.Wait() +} diff --git a/container.go b/container.go index dea81b6def..8721d45a55 100644 --- a/container.go +++ b/container.go @@ -52,7 +52,7 @@ type Container struct { waitLock chan struct{} Volumes map[string]string - // Store rw/ro in a separate structure to preserve reserve-compatibility on-disk. + // Store rw/ro in a separate structure to preserve reverse-compatibility on-disk. // Easier than migrating older container configs :) VolumesRW map[string]bool } @@ -100,7 +100,7 @@ func ParseRun(args []string, capabilities *Capabilities) (*Config, *HostConfig, flHostname := cmd.String("h", "", "Container host name") flUser := cmd.String("u", "", "Username or UID") - flDetach := cmd.Bool("d", false, "Detached mode: leave the container running in the background") + flDetach := cmd.Bool("d", false, "Detached mode: Run container in the background, print new container id") flAttach := NewAttachOpts() cmd.Var(flAttach, "a", "Attach to stdin, stdout or stderr.") flStdin := cmd.Bool("i", false, "Keep stdin open even if not attached") @@ -266,7 +266,8 @@ func (container *Container) FromDisk() error { return err } // Load container settings - if err := json.Unmarshal(data, container); err != nil { + // udp broke compat of docker.PortMapping, but it's not used when loading a container, we can skip it + if err := json.Unmarshal(data, container); err != nil && !strings.Contains(err.Error(), "docker.PortMapping") { return err } return nil @@ -379,14 +380,15 @@ func (container *Container) Attach(stdin io.ReadCloser, stdinCloser io.Closer, s utils.Debugf("[start] attach stdin\n") defer utils.Debugf("[end] attach stdin\n") // No matter what, when stdin is closed (io.Copy unblock), close stdout and stderr - if cStdout != nil { - defer cStdout.Close() - } - if cStderr != nil { - defer cStderr.Close() - } if container.Config.StdinOnce && !container.Config.Tty { defer cStdin.Close() + } else { + if cStdout != nil { + defer cStdout.Close() + } + if cStderr != nil { + defer cStderr.Close() + } } if container.Config.Tty { _, err = utils.CopyEscapable(cStdin, stdin) @@ -532,6 +534,10 @@ func (container *Container) Start(hostConfig *HostConfig) error { container.Config.MemorySwap = -1 } + if !container.runtime.capabilities.IPv4Forwarding { + log.Printf("WARNING: IPv4 forwarding is disabled. Networking will not work") + } + // Create the requested bind mounts binds := make(map[string]BindMap) // Define illegal container destinations @@ -629,7 +635,7 @@ func (container *Container) Start(hostConfig *HostConfig) error { "-n", container.ID, "-f", container.lxcConfigPath(), "--", - "/sbin/init", + "/.dockerinit", } // Networking @@ -651,6 +657,7 @@ func (container *Container) Start(hostConfig *HostConfig) error { "-e", "HOME=/", "-e", "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "-e", "container=lxc", + "-e", "HOSTNAME="+container.Config.Hostname, ) for _, elem := range container.Config.Env { @@ -1086,3 +1093,24 @@ func (container *Container) GetSize() (int64, int64) { } return sizeRw, sizeRootfs } + +func (container *Container) Copy(resource string) (Archive, error) { + if err := container.EnsureMounted(); err != nil { + return nil, err + } + var filter []string + basePath := path.Join(container.RootfsPath(), resource) + stat, err := os.Stat(basePath) + if err != nil { + return nil, err + } + if !stat.IsDir() { + d, f := path.Split(basePath) + basePath = d + filter = []string{f} + } else { + filter = []string{path.Base(basePath)} + basePath = path.Dir(basePath) + } + return TarFilter(basePath, Uncompressed, filter) +} diff --git a/container_test.go b/container_test.go index a1ac0bd33a..1050bfb2ad 100644 --- a/container_test.go +++ b/container_test.go @@ -960,6 +960,7 @@ func TestEnv(t *testing.T) { "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin", "HOME=/", "container=lxc", + "HOSTNAME=" + container.ShortID(), } sort.Strings(goodEnv) if len(goodEnv) != len(actualEnv) { @@ -995,6 +996,28 @@ func TestEntrypoint(t *testing.T) { } } +func TestEntrypointNoCmd(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + container, err := NewBuilder(runtime).Create( + &Config{ + Image: GetTestImage(runtime).ID, + Entrypoint: []string{"/bin/echo", "foobar"}, + }, + ) + if err != nil { + t.Fatal(err) + } + defer runtime.Destroy(container) + output, err := container.Output() + if err != nil { + t.Fatal(err) + } + if strings.Trim(string(output), "\r\n") != "foobar" { + t.Error(string(output)) + } +} + func grepFile(t *testing.T, path string, pattern string) { f, err := os.Open(path) if err != nil { @@ -1283,10 +1306,10 @@ func TestOnlyLoopbackExistsWhenUsingDisableNetworkOption(t *testing.T) { interfaces := regexp.MustCompile(`(?m)^[0-9]+: [a-zA-Z0-9]+`).FindAllString(string(output), -1) if len(interfaces) != 1 { - t.Fatalf("Wrong interface count in test container: expected [1: lo], got [%s]", interfaces) + t.Fatalf("Wrong interface count in test container: expected [*: lo], got %s", interfaces) } - if interfaces[0] != "1: lo" { - t.Fatalf("Wrong interface in test container: expected [1: lo], got [%s]", interfaces) + if !strings.HasSuffix(interfaces[0], ": lo") { + t.Fatalf("Wrong interface in test container: expected [*: lo], got %s", interfaces) } } diff --git a/contrib/install.sh b/contrib/install.sh index cf097da670..e58f054685 100755 --- a/contrib/install.sh +++ b/contrib/install.sh @@ -45,7 +45,13 @@ then echo "Upstart script already exists." else echo "Creating /etc/init/dockerd.conf..." - echo "exec env LANG=\"en_US.UTF-8\" /usr/local/bin/docker -d" > /etc/init/dockerd.conf + cat >/etc/init/dockerd.conf < in your favorite browser. +* To preview what you have generated run `make server` and open http://localhost:8000/ in your favorite browser. Working using GitHub's file editor ---------------------------------- @@ -36,15 +34,11 @@ Images ------ When you need to add images, try to make them as small as possible (e.g. as gif). - Notes ----- -* The index.html and gettingstarted.html files are copied from the source dir to the output dir without modification. -So changes to those pages should be made directly in html * For the template the css is compiled from less. When changes are needed they can be compiled using lessc ``lessc main.less`` or watched using watch-lessc ``watch-lessc -i main.less -o main.css`` - Guides on using sphinx ---------------------- * To make links to certain pages create a link target like so: @@ -75,3 +69,12 @@ Guides on using sphinx * Code examples Start without $, so it's easy to copy and paste. + +Manpages +-------- + +* To make the manpages, simply run 'make man'. Please note there is a bug in spinx 1.1.3 which makes this fail. +Upgrade to the latest version of sphinx. +* Then preview the manpage by running `man _build/man/docker.1`, where _build/man/docker.1 is the path to the generated +manfile +* The manpages are also autogenerated by our hosted readthedocs here: http://docs-docker.dotcloud.com/projects/docker/downloads/ diff --git a/docs/sources/api/docker_remote_api.rst b/docs/sources/api/docker_remote_api.rst index 792d43f501..3be02141c0 100644 --- a/docs/sources/api/docker_remote_api.rst +++ b/docs/sources/api/docker_remote_api.rst @@ -15,7 +15,7 @@ Docker Remote API ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr @@ -26,23 +26,39 @@ Docker Remote API 2. Versions =========== -The current verson of the API is 1.3 +The current verson of the API is 1.4 Calling /images//insert is the same as calling -/v1.3/images//insert +/v1.4/images//insert You can still call an old version of the api using /v1.0/images//insert +:doc:`docker_remote_api_v1.4` +***************************** + +What's new +---------- + +.. http:post:: /images/create + + **New!** When pull a repo, all images are now downloaded in parallel. + +.. http:get:: /containers/(id)/top + + **New!** You can now use ps args with docker top, like `docker top aux` + :doc:`docker_remote_api_v1.3` ***************************** +docker v0.5.0 51f6c4a_ + What's new ---------- .. http:get:: /containers/(id)/top - **New!** List the processes running inside a container. + List the processes running inside a container. .. http:get:: /events: @@ -138,6 +154,7 @@ Initial version .. _a8ae398: https://github.com/dotcloud/docker/commit/a8ae398bf52e97148ee7bd0d5868de2e15bd297f .. _8d73740: https://github.com/dotcloud/docker/commit/8d73740343778651c09160cde9661f5f387b36f4 .. _2e7649b: https://github.com/dotcloud/docker/commit/2e7649beda7c820793bd46766cbc2cfeace7b168 +.. _51f6c4a: https://github.com/dotcloud/docker/commit/51f6c4a7372450d164c61e0054daf0223ddbd909 ================================== Docker Remote API Client Libraries diff --git a/docs/sources/api/docker_remote_api_v1.0.rst b/docs/sources/api/docker_remote_api_v1.0.rst index 5aa98cbe59..3ee3245137 100644 --- a/docs/sources/api/docker_remote_api_v1.0.rst +++ b/docs/sources/api/docker_remote_api_v1.0.rst @@ -17,7 +17,7 @@ Docker Remote API v1.0 ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr 2. Endpoints @@ -832,7 +832,7 @@ Build an image from Dockerfile via stdin {{ STREAM }} - :query t: tag to be applied to the resulting image in case of success + :query t: repository name to be applied to the resulting image in case of success :statuscode 200: no error :statuscode 500: server error diff --git a/docs/sources/api/docker_remote_api_v1.1.rst b/docs/sources/api/docker_remote_api_v1.1.rst index e0159ddb65..3e906ed50f 100644 --- a/docs/sources/api/docker_remote_api_v1.1.rst +++ b/docs/sources/api/docker_remote_api_v1.1.rst @@ -17,7 +17,7 @@ Docker Remote API v1.1 ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr 2. Endpoints diff --git a/docs/sources/api/docker_remote_api_v1.2.rst b/docs/sources/api/docker_remote_api_v1.2.rst index 96ee6bb9bb..b956d1dfe6 100644 --- a/docs/sources/api/docker_remote_api_v1.2.rst +++ b/docs/sources/api/docker_remote_api_v1.2.rst @@ -17,7 +17,7 @@ Docker Remote API v1.2 ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr 2. Endpoints @@ -870,7 +870,7 @@ Build an image from Dockerfile via stdin {{ STREAM }} - :query t: tag to be applied to the resulting image in case of success + :query t: repository name to be applied to the resulting image in case of success :query remote: resource to fetch, as URI :statuscode 200: no error :statuscode 500: server error diff --git a/docs/sources/api/docker_remote_api_v1.3.rst b/docs/sources/api/docker_remote_api_v1.3.rst index 69f480e453..8e5c7b2a3b 100644 --- a/docs/sources/api/docker_remote_api_v1.3.rst +++ b/docs/sources/api/docker_remote_api_v1.3.rst @@ -17,7 +17,7 @@ Docker Remote API v1.3 ===================== - The Remote API is replacing rcli -- Default port in the docker deamon is 4243 +- Default port in the docker daemon is 4243 - The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr 2. Endpoints @@ -925,7 +925,7 @@ Build an image from Dockerfile via stdin The Content-type header should be set to "application/tar". - :query t: tag to be applied to the resulting image in case of success + :query t: repository name (and optionally a tag) to be applied to the resulting image in case of success :query q: suppress verbose build output :statuscode 200: no error :statuscode 500: server error @@ -989,7 +989,10 @@ Display system-wide information "NFd": 11, "NGoroutines":21, "MemoryLimit":true, - "SwapLimit":false + "SwapLimit":false, + "EventsListeners":"0", + "LXCVersion":"0.7.5", + "KernelVersion":"3.8.0-19-generic" } :statuscode 200: no error diff --git a/docs/sources/api/docker_remote_api_v1.4.rst b/docs/sources/api/docker_remote_api_v1.4.rst new file mode 100644 index 0000000000..06e8f46f99 --- /dev/null +++ b/docs/sources/api/docker_remote_api_v1.4.rst @@ -0,0 +1,1130 @@ +:title: Remote API v1.4 +:description: API Documentation for Docker +:keywords: API, Docker, rcli, REST, documentation + +:orphan: + +====================== +Docker Remote API v1.4 +====================== + +.. contents:: Table of Contents + +1. Brief introduction +===================== + +- The Remote API is replacing rcli +- Default port in the docker daemon is 4243 +- The API tends to be REST, but for some complex commands, like attach or pull, the HTTP connection is hijacked to transport stdout stdin and stderr + +2. Endpoints +============ + +2.1 Containers +-------------- + +List containers +*************** + +.. http:get:: /containers/json + + List containers + + **Example request**: + + .. sourcecode:: http + + GET /containers/json?all=1&before=8dfafdbc3a40&size=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id": "8dfafdbc3a40", + "Image": "base:latest", + "Command": "echo 1", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "9cd87474be90", + "Image": "base:latest", + "Command": "echo 222222", + "Created": 1367854155, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "3176a2479c92", + "Image": "base:latest", + "Command": "echo 3333333333333333", + "Created": 1367854154, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + }, + { + "Id": "4cb07b47f9fb", + "Image": "base:latest", + "Command": "echo 444444444444444444444444444444444", + "Created": 1367854152, + "Status": "Exit 0", + "Ports":"", + "SizeRw":12288, + "SizeRootFs":0 + } + ] + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :query limit: Show ``limit`` last created containers, include non-running ones. + :query since: Show only containers created since Id, include non-running ones. + :query before: Show only containers created before Id, include non-running ones. + :query size: 1/True/true or 0/False/false, Show the containers sizes + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create a container +****************** + +.. http:post:: /containers/create + + Create a container + + **Example request**: + + .. sourcecode:: http + + POST /containers/create HTTP/1.1 + Content-Type: application/json + + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":true, + "AttachStderr":true, + "PortSpecs":null, + "Tty":false, + "OpenStdin":false, + "StdinOnce":false, + "Env":null, + "Cmd":[ + "date" + ], + "Dns":null, + "Image":"base", + "Volumes":{}, + "VolumesFrom":"" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/json + + { + "Id":"e90e34656806" + "Warnings":[] + } + + :jsonparam config: the container's configuration + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 406: impossible to attach (container not running) + :statuscode 500: server error + + +Inspect a container +******************* + +.. http:get:: /containers/(id)/json + + Return low-level information on the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Id": "4fa6e0f0c6786287e131c3852c58a2e01cc697a68231826813597e4994f1d6e2", + "Created": "2013-05-07T14:51:42.041847+02:00", + "Path": "date", + "Args": [], + "Config": { + "Hostname": "4fa6e0f0c678", + "User": "", + "Memory": 0, + "MemorySwap": 0, + "AttachStdin": false, + "AttachStdout": true, + "AttachStderr": true, + "PortSpecs": null, + "Tty": false, + "OpenStdin": false, + "StdinOnce": false, + "Env": null, + "Cmd": [ + "date" + ], + "Dns": null, + "Image": "base", + "Volumes": {}, + "VolumesFrom": "" + }, + "State": { + "Running": false, + "Pid": 0, + "ExitCode": 0, + "StartedAt": "2013-05-07T14:51:42.087658+02:01360", + "Ghost": false + }, + "Image": "b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "NetworkSettings": { + "IpAddress": "", + "IpPrefixLen": 0, + "Gateway": "", + "Bridge": "", + "PortMapping": null + }, + "SysInitPath": "/home/kitty/go/src/github.com/dotcloud/docker/bin/docker", + "ResolvConfPath": "/etc/resolv.conf", + "Volumes": {} + } + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +List processes running inside a container +***************************************** + +.. http:get:: /containers/(id)/top + + List processes running inside the container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/top HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Titles":[ + "USER", + "PID", + "%CPU", + "%MEM", + "VSZ", + "RSS", + "TTY", + "STAT", + "START", + "TIME", + "COMMAND" + ], + "Processes":[ + ["root","20147","0.0","0.1","18060","1864","pts/4","S","10:06","0:00","bash"], + ["root","20271","0.0","0.0","4312","352","pts/4","S+","10:07","0:00","sleep","10"] + ] + } + + :query ps_args: ps arguments to use (eg. aux) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Inspect changes on a container's filesystem +******************************************* + +.. http:get:: /containers/(id)/changes + + Inspect changes on container ``id`` 's filesystem + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/changes HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Path":"/dev", + "Kind":0 + }, + { + "Path":"/dev/kmsg", + "Kind":1 + }, + { + "Path":"/test", + "Kind":1 + } + ] + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Export a container +****************** + +.. http:get:: /containers/(id)/export + + Export the contents of container ``id`` + + **Example request**: + + .. sourcecode:: http + + GET /containers/4fa6e0f0c678/export HTTP/1.1 + + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Start a container +***************** + +.. http:post:: /containers/(id)/start + + Start the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/(id)/start HTTP/1.1 + Content-Type: application/json + + { + "Binds":["/tmp:/tmp"] + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 No Content + Content-Type: text/plain + + :jsonparam hostConfig: the container's host configuration (optional) + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Stop a contaier +*************** + +.. http:post:: /containers/(id)/stop + + Stop the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/stop?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Restart a container +******************* + +.. http:post:: /containers/(id)/restart + + Restart the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/restart?t=5 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query t: number of seconds to wait before killing the container + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Kill a container +**************** + +.. http:post:: /containers/(id)/kill + + Kill the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/e90e34656806/kill HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :statuscode 204: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Attach to a container +********************* + +.. http:post:: /containers/(id)/attach + + Attach to the container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/attach?logs=1&stream=0&stdout=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/vnd.docker.raw-stream + + {{ STREAM }} + + :query logs: 1/True/true or 0/False/false, return logs. Default false + :query stream: 1/True/true or 0/False/false, return stream. Default false + :query stdin: 1/True/true or 0/False/false, if stream=true, attach to stdin. Default false + :query stdout: 1/True/true or 0/False/false, if logs=true, return stdout log, if stream=true, attach to stdout. Default false + :query stderr: 1/True/true or 0/False/false, if logs=true, return stderr log, if stream=true, attach to stderr. Default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +Wait a container +**************** + +.. http:post:: /containers/(id)/wait + + Block until container ``id`` stops, then returns the exit code + + **Example request**: + + .. sourcecode:: http + + POST /containers/16253994b7c4/wait HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"StatusCode":0} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +Remove a container +******************* + +.. http:delete:: /containers/(id) + + Remove the container ``id`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /containers/16253994b7c4?v=1 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 204 OK + + :query v: 1/True/true or 0/False/false, Remove the volumes associated to the container. Default false + :statuscode 204: no error + :statuscode 400: bad parameter + :statuscode 404: no such container + :statuscode 500: server error + + +Copy files or folders from a container +************************************** + +.. http:post:: /containers/(id)/copy + + Copy files or folders of container ``id`` + + **Example request**: + + .. sourcecode:: http + + POST /containers/4fa6e0f0c678/copy HTTP/1.1 + Content-Type: application/json + + { + "Resource":"test.txt" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/octet-stream + + {{ STREAM }} + + :statuscode 200: no error + :statuscode 404: no such container + :statuscode 500: server error + + +2.2 Images +---------- + +List Images +*********** + +.. http:get:: /images/(format) + + List images ``format`` could be json or viz (json default) + + **Example request**: + + .. sourcecode:: http + + GET /images/json?all=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Repository":"base", + "Tag":"ubuntu-12.10", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + }, + { + "Repository":"base", + "Tag":"ubuntu-quantal", + "Id":"b750fe79269d", + "Created":1364102658, + "Size":24653, + "VirtualSize":180116135 + } + ] + + + **Example request**: + + .. sourcecode:: http + + GET /images/viz HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: text/plain + + digraph docker { + "d82cbacda43a" -> "074be284591f" + "1496068ca813" -> "08306dc45919" + "08306dc45919" -> "0e7893146ac2" + "b750fe79269d" -> "1496068ca813" + base -> "27cf78414709" [style=invis] + "f71189fff3de" -> "9a33b36209ed" + "27cf78414709" -> "b750fe79269d" + "0e7893146ac2" -> "d6434d954665" + "d6434d954665" -> "d82cbacda43a" + base -> "e9aa60c60128" [style=invis] + "074be284591f" -> "f71189fff3de" + "b750fe79269d" [label="b750fe79269d\nbase",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "e9aa60c60128" [label="e9aa60c60128\nbase2",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + "9a33b36209ed" [label="9a33b36209ed\ntest",shape=box,fillcolor="paleturquoise",style="filled,rounded"]; + base [style=invisible] + } + + :query all: 1/True/true or 0/False/false, Show all containers. Only running containers are shown by default + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 500: server error + + +Create an image +*************** + +.. http:post:: /images/create + + Create an image, either by pull it from the registry or by importing it + + **Example request**: + + .. sourcecode:: http + + POST /images/create?fromImage=base HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pulling..."} + {"status":"Pulling", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query fromImage: name of the image to pull + :query fromSrc: source to import, - means stdin + :query repo: repository + :query tag: tag + :query registry: the registry to pull from + :statuscode 200: no error + :statuscode 500: server error + + +Insert a file in a image +************************ + +.. http:post:: /images/(name)/insert + + Insert a file from ``url`` in the image ``name`` at ``path`` + + **Example request**: + + .. sourcecode:: http + + POST /images/test/insert?path=/usr&url=myurl HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Inserting..."} + {"status":"Inserting", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :statuscode 200: no error + :statuscode 500: server error + + +Inspect an image +**************** + +.. http:get:: /images/(name)/json + + Return low-level information on the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/json HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "id":"b750fe79269d2ec9a3c593ef05b4332b1d1a02a62b4accb2c21d589ff2f5f2dc", + "parent":"27cf784147099545", + "created":"2013-03-23T22:24:18.818426-07:00", + "container":"3d67245a8d72ecf13f33dffac9f79dcdf70f75acb84d308770391510e0c23ad0", + "container_config": + { + "Hostname":"", + "User":"", + "Memory":0, + "MemorySwap":0, + "AttachStdin":false, + "AttachStdout":false, + "AttachStderr":false, + "PortSpecs":null, + "Tty":true, + "OpenStdin":true, + "StdinOnce":false, + "Env":null, + "Cmd": ["/bin/bash"] + ,"Dns":null, + "Image":"base", + "Volumes":null, + "VolumesFrom":"" + }, + "Size": 6824592 + } + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Get the history of an image +*************************** + +.. http:get:: /images/(name)/history + + Return the history of the image ``name`` + + **Example request**: + + .. sourcecode:: http + + GET /images/base/history HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Id":"b750fe79269d", + "Created":1364102658, + "CreatedBy":"/bin/bash" + }, + { + "Id":"27cf78414709", + "Created":1364068391, + "CreatedBy":"" + } + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Push an image on the registry +***************************** + +.. http:post:: /images/(name)/push + + Push the image ``name`` on the registry + + **Example request**: + + .. sourcecode:: http + + POST /images/test/push HTTP/1.1 + {{ authConfig }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + {"status":"Pushing..."} + {"status":"Pushing", "progress":"1/? (n/a)"} + {"error":"Invalid..."} + ... + + :query registry: the registry you wan to push, optional + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 500: server error + + +Tag an image into a repository +****************************** + +.. http:post:: /images/(name)/tag + + Tag the image ``name`` into a repository + + **Example request**: + + .. sourcecode:: http + + POST /images/test/tag?repo=myrepo&force=0 HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :query repo: The repository to tag in + :query force: 1/True/true or 0/False/false, default false + :statuscode 200: no error + :statuscode 400: bad parameter + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Remove an image +*************** + +.. http:delete:: /images/(name) + + Remove the image ``name`` from the filesystem + + **Example request**: + + .. sourcecode:: http + + DELETE /images/test HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-type: application/json + + [ + {"Untagged":"3e2f21a89f"}, + {"Deleted":"3e2f21a89f"}, + {"Deleted":"53b4f83ac9"} + ] + + :statuscode 200: no error + :statuscode 404: no such image + :statuscode 409: conflict + :statuscode 500: server error + + +Search images +************* + +.. http:get:: /images/search + + Search for an image in the docker index + + **Example request**: + + .. sourcecode:: http + + GET /images/search?term=sshd HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + [ + { + "Name":"cespare/sshd", + "Description":"" + }, + { + "Name":"johnfuller/sshd", + "Description":"" + }, + { + "Name":"dhrp/mongodb-sshd", + "Description":"" + } + ] + + :query term: term to search + :statuscode 200: no error + :statuscode 500: server error + + +2.3 Misc +-------- + +Build an image from Dockerfile via stdin +**************************************** + +.. http:post:: /build + + Build an image from Dockerfile via stdin + + **Example request**: + + .. sourcecode:: http + + POST /build HTTP/1.1 + + {{ STREAM }} + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + {{ STREAM }} + + + The stream must be a tar archive compressed with one of the following algorithms: + identity (no compression), gzip, bzip2, xz. The archive must include a file called + `Dockerfile` at its root. It may include any number of other files, which will be + accessible in the build context (See the ADD build command). + + The Content-type header should be set to "application/tar". + + :query t: repository name (and optionally a tag) to be applied to the resulting image in case of success + :query q: suppress verbose build output + :query nocache: do not use the cache when building the image + :statuscode 200: no error + :statuscode 500: server error + + +Check auth configuration +************************ + +.. http:post:: /auth + + Get the default username and email + + **Example request**: + + .. sourcecode:: http + + POST /auth HTTP/1.1 + Content-Type: application/json + + { + "username":"hannibal", + "password:"xxxx", + "email":"hannibal@a-team.com" + } + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + + :statuscode 200: no error + :statuscode 204: no error + :statuscode 500: server error + + +Display system-wide information +******************************* + +.. http:get:: /info + + Display system-wide information + + **Example request**: + + .. sourcecode:: http + + GET /info HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Containers":11, + "Images":16, + "Debug":false, + "NFd": 11, + "NGoroutines":21, + "MemoryLimit":true, + "SwapLimit":false, + "IPv4Forwarding":true + } + + :statuscode 200: no error + :statuscode 500: server error + + +Show the docker version information +*********************************** + +.. http:get:: /version + + Show the docker version information + + **Example request**: + + .. sourcecode:: http + + GET /version HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 200 OK + Content-Type: application/json + + { + "Version":"0.2.2", + "GitCommit":"5a2a5cc+CHANGES", + "GoVersion":"go1.0.3" + } + + :statuscode 200: no error + :statuscode 500: server error + + +Create a new image from a container's changes +********************************************* + +.. http:post:: /commit + + Create a new image from a container's changes + + **Example request**: + + .. sourcecode:: http + + POST /commit?container=44c004db4b17&m=message&repo=myrepo HTTP/1.1 + + **Example response**: + + .. sourcecode:: http + + HTTP/1.1 201 OK + Content-Type: application/vnd.docker.raw-stream + + {"Id":"596069db4bf5"} + + :query container: source container + :query repo: repository + :query tag: tag + :query m: commit message + :query author: author (eg. "John Hannibal Smith ") + :query run: config automatically applied when the image is run. (ex: {"Cmd": ["cat", "/world"], "PortSpecs":["22"]}) + :statuscode 201: no error + :statuscode 404: no such container + :statuscode 500: server error + +3. Going further +================ + +3.1 Inside 'docker run' +----------------------- + +Here are the steps of 'docker run' : + +* Create the container +* If the status code is 404, it means the image doesn't exists: + * Try to pull it + * Then retry to create the container +* Start the container +* If you are not in detached mode: + * Attach to the container, using logs=1 (to have stdout and stderr from the container's start) and stream=1 +* If in detached mode or only stdin is attached: + * Display the container's id + + +3.2 Hijacking +------------- + +In this version of the API, /attach, uses hijacking to transport stdin, stdout and stderr on the same socket. This might change in the future. + +3.3 CORS Requests +----------------- + +To enable cross origin requests to the remote api add the flag "-api-enable-cors" when running docker in daemon mode. + +.. code-block:: bash + + docker -d -H="192.168.1.9:4243" -api-enable-cors + diff --git a/docs/sources/commandline/cli.rst b/docs/sources/commandline/cli.rst index e499b1f096..bdc4827a60 100644 --- a/docs/sources/commandline/cli.rst +++ b/docs/sources/commandline/cli.rst @@ -30,6 +30,7 @@ Available Commands command/attach command/build command/commit + command/cp command/diff command/export command/history diff --git a/docs/sources/commandline/command/build.rst b/docs/sources/commandline/command/build.rst index 45b6d2ec8e..80e5d2c647 100644 --- a/docs/sources/commandline/command/build.rst +++ b/docs/sources/commandline/command/build.rst @@ -10,8 +10,9 @@ Usage: docker build [OPTIONS] PATH | URL | - Build a new container image from the source code at PATH - -t="": Tag to be applied to the resulting image in case of success. + -t="": Repository name (and optionally a tag) to be applied to the resulting image in case of success. -q=false: Suppress verbose build output. + -no-cache: Do not use the cache when building the image. When a single Dockerfile is given as URL, then no context is set. When a git repository is set as URL, the repository is used as context @@ -28,6 +29,13 @@ Examples | If the absolute path is provided instead of '.', only the files and directories required by the ADD commands from the Dockerfile will be added to the context and transferred to the docker daemon. | +.. code-block:: bash + + docker build -t vieux/apache:2.0 . + +| This will build like the preview example, but it will then tag the resulting image, the repository name will be 'vieux/apache' and the tag will be '2.0' + + .. code-block:: bash docker build - < Dockerfile diff --git a/docs/sources/commandline/command/cp.rst b/docs/sources/commandline/command/cp.rst new file mode 100644 index 0000000000..14b5061ef7 --- /dev/null +++ b/docs/sources/commandline/command/cp.rst @@ -0,0 +1,13 @@ +:title: Cp Command +:description: Copy files/folders from the containers filesystem to the host path +:keywords: cp, docker, container, documentation, copy + +=========================================================== +``cp`` -- Copy files/folders from the containers filesystem to the host path +=========================================================== + +:: + + Usage: docker cp CONTAINER:RESOURCE HOSTPATH + + Copy files/folders from the containers filesystem to the host path. Paths are relative to the root of the filesystem. diff --git a/docs/sources/commandline/command/import.rst b/docs/sources/commandline/command/import.rst index 66bcf5de52..0083068e10 100644 --- a/docs/sources/commandline/command/import.rst +++ b/docs/sources/commandline/command/import.rst @@ -12,8 +12,9 @@ Create a new filesystem image from the contents of a tarball -At this time, the URL must start with ``http`` and point to a single file archive (.tar, .tar.gz, .bzip) -containing a root filesystem. If you would like to import from a local directory or archive, +At this time, the URL must start with ``http`` and point to a single file archive +(.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) +containing a root filesystem. If you would like to import from a local directory or archive, you can use the ``-`` parameter to take the data from standard in. Examples @@ -30,7 +31,7 @@ Import from a local file Import to docker via pipe and standard in ``$ cat exampleimage.tgz | docker import - exampleimagelocal`` - + Import from a local directory ............................. diff --git a/docs/sources/commandline/command/run.rst b/docs/sources/commandline/command/run.rst index db67ef0705..c5b363d787 100644 --- a/docs/sources/commandline/command/run.rst +++ b/docs/sources/commandline/command/run.rst @@ -8,14 +8,14 @@ :: - Usage: docker run [OPTIONS] IMAGE [COMMAND] [ARG...] + Usage: docker run [OPTIONS] IMAGE[:TAG] [COMMAND] [ARG...] Run a command in a new container -a=map[]: Attach to stdin, stdout or stderr. -c=0: CPU shares (relative weight) -cidfile="": Write the container ID to the file - -d=false: Detached mode: leave the container running in the background + -d=false: Detached mode: Run container in the background, print new container id -e=[]: Set environment variables -h="": Container host name -i=false: Keep stdin open even if not attached diff --git a/docs/sources/commandline/index.rst b/docs/sources/commandline/index.rst index a7296b27da..5c2b373205 100644 --- a/docs/sources/commandline/index.rst +++ b/docs/sources/commandline/index.rst @@ -15,6 +15,7 @@ Contents: attach build commit + cp diff export history diff --git a/docs/sources/concepts/manifesto.rst b/docs/sources/concepts/manifesto.rst index ae09647094..7dd4b4bdda 100644 --- a/docs/sources/concepts/manifesto.rst +++ b/docs/sources/concepts/manifesto.rst @@ -4,10 +4,6 @@ .. _dockermanifesto: -*(This was our original Welcome page, but it is a bit forward-looking -for docs, and maybe not enough vision for a true manifesto. We'll -reveal more vision in the future to make it more Manifesto-y.)* - Docker Manifesto ---------------- @@ -131,60 +127,3 @@ sitting 10 miles away. With Standard Containers we can put an end to that embarrassment, by making INDUSTRIAL-GRADE DELIVERY of software a reality. - -Standard Container Specification -~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - -(TODO) - -Image format -~~~~~~~~~~~~ - -Standard operations -~~~~~~~~~~~~~~~~~~~ - -- Copy -- Run -- Stop -- Wait -- Commit -- Attach standard streams -- List filesystem changes -- ... - -Execution environment -~~~~~~~~~~~~~~~~~~~~~ - -Root filesystem -^^^^^^^^^^^^^^^ - -Environment variables -^^^^^^^^^^^^^^^^^^^^^ - -Process arguments -^^^^^^^^^^^^^^^^^ - -Networking -^^^^^^^^^^ - -Process namespacing -^^^^^^^^^^^^^^^^^^^ - -Resource limits -^^^^^^^^^^^^^^^ - -Process monitoring -^^^^^^^^^^^^^^^^^^ - -Logging -^^^^^^^ - -Signals -^^^^^^^ - -Pseudo-terminal allocation -^^^^^^^^^^^^^^^^^^^^^^^^^^ - -Security -^^^^^^^^ - diff --git a/docs/sources/conf.py b/docs/sources/conf.py index ea8e1f43f0..b4c23f0c58 100644 --- a/docs/sources/conf.py +++ b/docs/sources/conf.py @@ -203,7 +203,7 @@ latex_elements = { # Grouping the document tree into LaTeX files. List of tuples # (source start file, target name, title, author, documentclass [howto/manual]). latex_documents = [ - ('index', 'Docker.tex', u'Docker Documentation', + ('toctree', 'Docker.tex', u'Docker Documentation', u'Team Docker', 'manual'), ] @@ -233,7 +233,7 @@ latex_documents = [ # One entry per manual page. List of tuples # (source start file, name, description, authors, manual section). man_pages = [ - ('index', 'docker', u'Docker Documentation', + ('toctree', 'docker', u'Docker Documentation', [u'Team Docker'], 1) ] @@ -247,7 +247,7 @@ man_pages = [ # (source start file, target name, title, author, # dir menu entry, description, category) texinfo_documents = [ - ('index', 'Docker', u'Docker Documentation', + ('toctree', 'Docker', u'Docker Documentation', u'Team Docker', 'Docker', 'One line description of project.', 'Miscellaneous'), ] diff --git a/docs/sources/examples/couchdb_data_volumes.rst b/docs/sources/examples/couchdb_data_volumes.rst index d6babe557f..97af733a82 100644 --- a/docs/sources/examples/couchdb_data_volumes.rst +++ b/docs/sources/examples/couchdb_data_volumes.rst @@ -39,7 +39,7 @@ This time, we're requesting shared access to $COUCH1's volumes. .. code-block:: bash - COUCH2=$(docker run -d -volumes-from $COUCH1) shykes/couchdb:2013-05-03) + COUCH2=$(docker run -d -volumes-from $COUCH1 shykes/couchdb:2013-05-03) Browse data on the second database ---------------------------------- @@ -48,6 +48,6 @@ Browse data on the second database HOST=localhost URL="http://$HOST:$(docker port $COUCH2 5984)/_utils/" - echo "Navigate to $URL in your browser. You should see the same data as in the first database!" + echo "Navigate to $URL in your browser. You should see the same data as in the first database"'!' Congratulations, you are running 2 Couchdb containers, completely isolated from each other *except* for their data. diff --git a/docs/sources/examples/hello_world.rst b/docs/sources/examples/hello_world.rst index 1d391f5fb1..448247e572 100644 --- a/docs/sources/examples/hello_world.rst +++ b/docs/sources/examples/hello_world.rst @@ -15,8 +15,8 @@ Download the base container .. code-block:: bash - # Download a base image - docker pull base + # Download an ubuntu image + docker pull ubuntu The *base* image is a minimal *ubuntu* based container, alternatively you can select *busybox*, a bare minimal linux system. The images are retrieved from the docker repository. @@ -47,4 +47,4 @@ See the example in action -Continue to the :ref:`hello_world_daemon` example. \ No newline at end of file +Continue to the :ref:`hello_world_daemon` example. diff --git a/docs/sources/examples/hello_world_daemon.rst b/docs/sources/examples/hello_world_daemon.rst index 7ca251aec8..7d876329dd 100644 --- a/docs/sources/examples/hello_world_daemon.rst +++ b/docs/sources/examples/hello_world_daemon.rst @@ -11,20 +11,20 @@ Hello World Daemon The most boring daemon ever written. -This example assumes you have Docker installed and with the base image already imported ``docker pull base``. -We will use the base image to run a simple hello world daemon that will just print hello world to standard +This example assumes you have Docker installed and with the ubuntu image already imported ``docker pull ubuntu``. +We will use the ubuntu image to run a simple hello world daemon that will just print hello world to standard out every second. It will continue to do this until we stop it. **Steps:** .. code-block:: bash - CONTAINER_ID=$(docker run -d base /bin/sh -c "while true; do echo hello world; sleep 1; done") + CONTAINER_ID=$(docker run -d ubuntu /bin/sh -c "while true; do echo hello world; sleep 1; done") -We are going to run a simple hello world daemon in a new container made from the base image. +We are going to run a simple hello world daemon in a new container made from the ubuntu image. - **"docker run -d "** run a command in a new container. We pass "-d" so it runs as a daemon. -- **"base"** is the image we want to run the command inside of. +- **"ubuntu"** is the image we want to run the command inside of. - **"/bin/sh -c"** is the command we want to run in the container - **"while true; do echo hello world; sleep 1; done"** is the mini script we want to run, that will just print hello world once a second until we stop it. - **$CONTAINER_ID** the output of the run command will return a container id, we can use in future commands to see what is going on with this process. diff --git a/docs/sources/examples/python_web_app.rst b/docs/sources/examples/python_web_app.rst index 8a4ca70750..ed4a490c72 100644 --- a/docs/sources/examples/python_web_app.rst +++ b/docs/sources/examples/python_web_app.rst @@ -42,7 +42,7 @@ We attach to the new container to see what is going on. Ctrl-C to disconnect BUILD_IMG=$(docker commit $BUILD_JOB _/builds/github.com/shykes/helloflask/master) -Save the changed we just made in the container to a new image called "_/builds/github.com/hykes/helloflask/master" and save the image id in the BUILD_IMG variable name. +Save the changes we just made in the container to a new image called "_/builds/github.com/hykes/helloflask/master" and save the image id in the BUILD_IMG variable name. .. code-block:: bash @@ -60,13 +60,13 @@ Use the new image we just created and create a new container with network port 5 docker logs $WEB_WORKER * Running on http://0.0.0.0:5000/ -view the logs for the new container using the WEB_WORKER variable, and if everything worked as planned you should see the line "Running on http://0.0.0.0:5000/" in the log output. +View the logs for the new container using the WEB_WORKER variable, and if everything worked as planned you should see the line "Running on http://0.0.0.0:5000/" in the log output. .. code-block:: bash WEB_PORT=$(docker port $WEB_WORKER 5000) -lookup the public-facing port which is NAT-ed store the private port used by the container and store it inside of the WEB_PORT variable. +Look up the public-facing port which is NAT-ed. Find the private port used by the container and store it inside of the WEB_PORT variable. .. code-block:: bash @@ -74,7 +74,7 @@ lookup the public-facing port which is NAT-ed store the private port used by the curl http://127.0.0.1:$WEB_PORT Hello world! -access the web app using curl. If everything worked as planned you should see the line "Hello world!" inside of your console. +Access the web app using curl. If everything worked as planned you should see the line "Hello world!" inside of your console. **Video:** diff --git a/docs/sources/examples/running_ssh_service.rst b/docs/sources/examples/running_ssh_service.rst index c2f8b86aca..54ca4cd200 100644 --- a/docs/sources/examples/running_ssh_service.rst +++ b/docs/sources/examples/running_ssh_service.rst @@ -20,7 +20,7 @@ minutes and not entirely smooth, but gives you a good idea.
- + You can also get this sshd container by using :: @@ -33,47 +33,57 @@ The password is 'screencast' .. code-block:: bash - # Hello! We are going to try and install openssh on a container and run it as a servic - # let's pull base to get a base ubuntu image. - $ docker pull base - # I had it so it was quick - # now let's connect using -i for interactive and with -t for terminal - # we execute /bin/bash to get a prompt. - $ docker run -i -t base /bin/bash - # now let's commit it - # which container was it? - $ docker ps -a |more - $ docker commit a30a3a2f2b130749995f5902f079dc6ad31ea0621fac595128ec59c6da07feea dhrp/sshd - # I gave the name dhrp/sshd for the container - # now we can run it again - $ docker run -d dhrp/sshd /usr/sbin/sshd -D # D for daemon mode - # is it running? - $ docker ps - # yes! - # let's stop it - $ docker stop 0ebf7cec294755399d063f4b1627980d4cbff7d999f0bc82b59c300f8536a562 - $ docker ps - # and reconnect, but now open a port to it - $ docker run -d -p 22 dhrp/sshd /usr/sbin/sshd -D - $ docker port b2b407cf22cf8e7fa3736fa8852713571074536b1d31def3fdfcd9fa4fd8c8c5 22 - # it has now given us a port to connect to - # we have to connect using a public ip of our host - $ hostname - # *ifconfig* is deprecated, better use *ip addr show* now - $ ifconfig - $ ssh root@192.168.33.10 -p 49153 - # Ah! forgot to set root passwd - $ docker commit b2b407cf22cf8e7fa3736fa8852713571074536b1d31def3fdfcd9fa4fd8c8c5 dhrp/sshd - $ docker ps -a - $ docker run -i -t dhrp/sshd /bin/bash - $ passwd - $ exit - $ docker commit 9e863f0ca0af31c8b951048ba87641d67c382d08d655c2e4879c51410e0fedc1 dhrp/sshd - $ docker run -d -p 22 dhrp/sshd /usr/sbin/sshd -D - $ docker port a0aaa9558c90cf5c7782648df904a82365ebacce523e4acc085ac1213bfe2206 22 - # *ifconfig* is deprecated, better use *ip addr show* now - $ ifconfig - $ ssh root@192.168.33.10 -p 49154 - # Thanks for watching, Thatcher thatcher@dotcloud.com + # Hello! We are going to try and install openssh on a container and run it as a servic + # let's pull ubuntu to get a base ubuntu image. + $ docker pull ubuntu + # I had it so it was quick + # now let's connect using -i for interactive and with -t for terminal + # we execute /bin/bash to get a prompt. + $ docker run -i -t base /bin/bash + # yes! we are in! + # now lets install openssh + $ apt-get update + $ apt-get install openssh-server + # ok. lets see if we can run it. + $ which sshd + # we need to create priviledge separation directory + $ mkdir /var/run/sshd + $ /usr/sbin/sshd + $ exit + # now let's commit it + # which container was it? + $ docker ps -a |more + $ docker commit a30a3a2f2b130749995f5902f079dc6ad31ea0621fac595128ec59c6da07feea dhrp/sshd + # I gave the name dhrp/sshd for the container + # now we can run it again + $ docker run -d dhrp/sshd /usr/sbin/sshd -D # D for daemon mode + # is it running? + $ docker ps + # yes! + # let's stop it + $ docker stop 0ebf7cec294755399d063f4b1627980d4cbff7d999f0bc82b59c300f8536a562 + $ docker ps + # and reconnect, but now open a port to it + $ docker run -d -p 22 dhrp/sshd /usr/sbin/sshd -D + $ docker port b2b407cf22cf8e7fa3736fa8852713571074536b1d31def3fdfcd9fa4fd8c8c5 22 + # it has now given us a port to connect to + # we have to connect using a public ip of our host + $ hostname + # *ifconfig* is deprecated, better use *ip addr show* now + $ ifconfig + $ ssh root@192.168.33.10 -p 49153 + # Ah! forgot to set root passwd + $ docker commit b2b407cf22cf8e7fa3736fa8852713571074536b1d31def3fdfcd9fa4fd8c8c5 dhrp/sshd + $ docker ps -a + $ docker run -i -t dhrp/sshd /bin/bash + $ passwd + $ exit + $ docker commit 9e863f0ca0af31c8b951048ba87641d67c382d08d655c2e4879c51410e0fedc1 dhrp/sshd + $ docker run -d -p 22 dhrp/sshd /usr/sbin/sshd -D + $ docker port a0aaa9558c90cf5c7782648df904a82365ebacce523e4acc085ac1213bfe2206 22 + # *ifconfig* is deprecated, better use *ip addr show* now + $ ifconfig + $ ssh root@192.168.33.10 -p 49154 + # Thanks for watching, Thatcher thatcher@dotcloud.com diff --git a/docs/sources/faq.rst b/docs/sources/faq.rst index 12b9751338..3cc0086c5d 100644 --- a/docs/sources/faq.rst +++ b/docs/sources/faq.rst @@ -47,7 +47,7 @@ Most frequently asked questions. .. _IRC, docker on freenode: irc://chat.freenode.net#docker .. _Github: http://www.github.com/dotcloud/docker .. _Ask questions on Stackoverflow: http://stackoverflow.com/search?q=docker - .. _Join the conversation on Twitter: http://twitter.com/getdocker + .. _Join the conversation on Twitter: http://twitter.com/docker Looking for something else to read? Checkout the :ref:`hello_world` example. diff --git a/docs/sources/installation/amazon.rst b/docs/sources/installation/amazon.rst index 59896bb63c..a537274e97 100644 --- a/docs/sources/installation/amazon.rst +++ b/docs/sources/installation/amazon.rst @@ -1,14 +1,18 @@ -:title: Installation on Amazon EC2 +:title: Installation on Amazon EC2 :description: Docker installation on Amazon EC2 with a single vagrant command. Vagrant 1.1 or higher is required. :keywords: amazon ec2, virtualization, cloud, docker, documentation, installation -Amazon EC2 -========== +Using Vagrant (Amazon EC2) +========================== + +This page explains how to setup and run an Amazon EC2 instance from your local machine. +Vagrant is not necessary to run Docker on EC2. You can follow the :ref:`ubuntu_linux` instructions +installing Docker on any EC2 instance running Ubuntu Please note this is a community contributed installation path. The only 'official' installation is using the :ref:`ubuntu_linux` installation path. This version may sometimes be out of date. - - + + Installation ------------ @@ -89,4 +93,4 @@ Docker can now be installed on Amazon EC2 with a single vagrant command. Vagrant docker -Continue with the :ref:`hello_world` example. \ No newline at end of file +Continue with the :ref:`hello_world` example. diff --git a/docs/sources/installation/kernel.rst b/docs/sources/installation/kernel.rst index 58730f8191..7c5715a62d 100644 --- a/docs/sources/installation/kernel.rst +++ b/docs/sources/installation/kernel.rst @@ -15,12 +15,11 @@ In short, Docker has the following kernel requirements: - Cgroups and namespaces must be enabled. - - The officially supported kernel is the one recommended by the - :ref:`ubuntu_linux` installation path. It is the one that most developers - will use, and the one that receives the most attention from the core - contributors. If you decide to go with a different kernel and hit a bug, - please try to reproduce it with the official kernels first. +The officially supported kernel is the one recommended by the +:ref:`ubuntu_linux` installation path. It is the one that most developers +will use, and the one that receives the most attention from the core +contributors. If you decide to go with a different kernel and hit a bug, +please try to reproduce it with the official kernels first. If you cannot or do not want to use the "official" kernels, here is some technical background about the features (both optional and diff --git a/docs/sources/installation/ubuntulinux.rst b/docs/sources/installation/ubuntulinux.rst index ed592d3a9d..587c8fe31b 100644 --- a/docs/sources/installation/ubuntulinux.rst +++ b/docs/sources/installation/ubuntulinux.rst @@ -19,6 +19,8 @@ Docker has the following dependencies * Linux kernel 3.8 (read more about :ref:`kernel`) * AUFS file system support (we are working on BTRFS support as an alternative) +Please read :ref:`ufw`, if you plan to use `UFW (Uncomplicated Firewall) `_ + .. _ubuntu_precise: Ubuntu Precise 12.04 (LTS) (64-bit) @@ -32,13 +34,20 @@ Dependencies **Linux kernel 3.8** -Due to a bug in LXC docker works best on the 3.8 kernel. Precise comes with a 3.2 kernel, so we need to upgrade it. The kernel we install comes with AUFS built in. +Due to a bug in LXC, docker works best on the 3.8 kernel. Precise +comes with a 3.2 kernel, so we need to upgrade it. The kernel we +install comes with AUFS built in. We also include the generic headers +to enable packages that depend on them, like ZFS and the VirtualBox +guest additions. If you didn't install the headers for your "precise" +kernel, then you can skip these headers for the "raring" kernel. But +it is safer to include them if you're not sure. .. code-block:: bash # install the backported kernel - sudo apt-get update && sudo apt-get install linux-image-generic-lts-raring + sudo apt-get update + sudo apt-get install linux-image-generic-lts-raring linux-headers-generic-lts-raring # reboot sudo reboot @@ -135,3 +144,35 @@ Verify it worked **Done!**, now continue with the :ref:`hello_world` example. + + +.. _ufw: + +Docker and UFW +^^^^^^^^^^^^^^ + +Docker uses a bridge to manage containers networking, by default UFW drop all `forwarding`, a first step is to enable forwarding: + +.. code-block:: bash + + sudo nano /etc/default/ufw + ---- + # Change: + # DEFAULT_FORWARD_POLICY="DROP" + # to + DEFAULT_FORWARD_POLICY="ACCEPT" + +Then reload UFW: + +.. code-block:: bash + + sudo ufw reload + + +UFW's default set of rules denied all `incoming`, so if you want to be able to reach your containers from another host, +you should allow incoming connections on the docker port (default 4243): + +.. code-block:: bash + + sudo ufw allow 4243/tcp + diff --git a/docs/sources/use/basics.rst b/docs/sources/use/basics.rst index 7ce86416dd..a716ab7fa5 100644 --- a/docs/sources/use/basics.rst +++ b/docs/sources/use/basics.rst @@ -26,12 +26,12 @@ Running an interactive shell .. code-block:: bash - # Download a base image - docker pull base + # Download an ubuntu image + docker pull ubuntu - # Run an interactive shell in the base image, + # Run an interactive shell in the ubuntu image, # allocate a tty, attach stdin and stdout - docker run -i -t base /bin/bash + docker run -i -t ubuntu /bin/bash Bind Docker to another host/port or a unix socket ------------------------------------------------- @@ -52,8 +52,8 @@ For example: # Run docker in daemon mode sudo /docker -H 0.0.0.0:5555 -d & - # Download a base image - docker -H :5555 pull base + # Download an ubuntu image + docker -H :5555 pull ubuntu You can use multiple -H, for example, if you want to listen on both tcp and a unix socket @@ -62,10 +62,10 @@ on both tcp and a unix socket # Run docker in daemon mode sudo /docker -H tcp://127.0.0.1:4243 -H unix:///var/run/docker.sock -d & - # Download a base image - docker pull base + # Download an ubuntu image + docker pull ubuntu # OR - docker -H unix:///var/run/docker.sock pull base + docker -H unix:///var/run/docker.sock pull ubuntu Starting a long-running worker process -------------------------------------- @@ -73,7 +73,7 @@ Starting a long-running worker process .. code-block:: bash # Start a very useful long-running process - JOB=$(docker run -d base /bin/sh -c "while true; do echo Hello world; sleep 1; done") + JOB=$(docker run -d ubuntu /bin/sh -c "while true; do echo Hello world; sleep 1; done") # Collect the output of the job so far docker logs $JOB @@ -95,7 +95,7 @@ Expose a service on a TCP port .. code-block:: bash # Expose port 4444 of this container, and tell netcat to listen on it - JOB=$(docker run -d -p 4444 base /bin/nc -l -p 4444) + JOB=$(docker run -d -p 4444 ubuntu /bin/nc -l -p 4444) # Which public port is NATed to my container? PORT=$(docker port $JOB 4444) diff --git a/docs/sources/use/builder.rst b/docs/sources/use/builder.rst index aa2fd6b92a..d111e335ab 100644 --- a/docs/sources/use/builder.rst +++ b/docs/sources/use/builder.rst @@ -107,8 +107,8 @@ the image. This is functionally equivalent to running ``docker commit -run '{"Cmd": }'`` outside the builder. .. note:: - Don't confuse `RUN` with `CMD`. `RUN` actually runs a - command and commits the result; `CMD` does not execute anything at + Don't confuse ``RUN`` with ``CMD``. ``RUN`` actually runs a + command and commits the result; ``CMD`` does not execute anything at build time, but specifies the intended command for the image. 3.5 EXPOSE @@ -182,7 +182,7 @@ The copy obeys the following rules: written at ````. * If ```` doesn't exist, it is created along with all missing directories in its path. All new files and directories are created - with mode 0700, uid and gid 0. + with mode 0755, uid and gid 0. 3.8 ENTRYPOINT -------------- diff --git a/docs/sources/use/puppet.rst b/docs/sources/use/puppet.rst index 5606f2a863..b89d95d8fe 100644 --- a/docs/sources/use/puppet.rst +++ b/docs/sources/use/puppet.rst @@ -53,13 +53,13 @@ defined type which can be used like so: .. code-block:: ruby - docker::image { 'base': } + docker::image { 'ubuntu': } This is equivalent to running: .. code-block:: bash - docker pull base + docker pull ubuntu Note that it will only if the image of that name does not already exist. This is downloading a large binary so on first run can take a while. @@ -68,7 +68,7 @@ for exec. Note that you can also remove images you no longer need with: .. code-block:: ruby - docker::image { 'base': + docker::image { 'ubuntu': ensure => 'absent', } @@ -81,7 +81,7 @@ docker. .. code-block:: ruby docker::run { 'helloworld': - image => 'base', + image => 'ubuntu', command => '/bin/sh -c "while true; do echo hello world; sleep 1; done"', } @@ -89,14 +89,14 @@ This is equivalent to running the following command, but under upstart: .. code-block:: bash - docker run -d base /bin/sh -c "while true; do echo hello world; sleep 1; done" + docker run -d ubuntu /bin/sh -c "while true; do echo hello world; sleep 1; done" Run also contains a number of optional parameters: .. code-block:: ruby docker::run { 'helloworld': - image => 'base', + image => 'ubuntu', command => '/bin/sh -c "while true; do echo hello world; sleep 1; done"', ports => ['4444', '4555'], volumes => ['/var/lib/counchdb', '/var/log'], diff --git a/docs/theme/MAINTAINERS b/docs/theme/MAINTAINERS index 6df367c073..606a1dd746 100644 --- a/docs/theme/MAINTAINERS +++ b/docs/theme/MAINTAINERS @@ -1 +1 @@ -Thatcher Penskens +Thatcher Peskens diff --git a/docs/theme/docker/layout.html b/docs/theme/docker/layout.html index 198cd5d7d8..d6bfff79ba 100755 --- a/docs/theme/docker/layout.html +++ b/docs/theme/docker/layout.html @@ -68,18 +68,18 @@
- +
@@ -130,7 +130,7 @@
@@ -156,11 +156,6 @@ {# {%- endif %}#} -{##} -{#
#} -{# #} -{# GitHub#} -{#
#}
diff --git a/docs/website/MAINTAINERS b/docs/website/MAINTAINERS deleted file mode 100644 index 6df367c073..0000000000 --- a/docs/website/MAINTAINERS +++ /dev/null @@ -1 +0,0 @@ -Thatcher Penskens diff --git a/docs/website/dotcloud.yml b/docs/website/dotcloud.yml deleted file mode 100644 index 5a8f50f9e9..0000000000 --- a/docs/website/dotcloud.yml +++ /dev/null @@ -1,2 +0,0 @@ -www: - type: static \ No newline at end of file diff --git a/docs/website/gettingstarted/index.html b/docs/website/gettingstarted/index.html deleted file mode 100644 index de0cc3512d..0000000000 --- a/docs/website/gettingstarted/index.html +++ /dev/null @@ -1,220 +0,0 @@ - - - - - - - - - - Docker - the Linux container runtime - - - - - - - - - - - - - - - - - - - - -
-
-
- -
- -
- -
- -
-
-
-
- - -
-
- -
- - - -

GETTING STARTED

-
- -
- -
- -
-
- Docker is still under heavy development. It should not yet be used in production. Check the repo for recent progress. -
-
-
-
-

- - Installing on Ubuntu

- -

Requirements

-
    -
  • Ubuntu 12.04 (LTS) (64-bit)
    • -
  • or Ubuntu 12.10 (quantal) (64-bit)
    • -
  • The 3.8 Linux Kernel
    • -
-
    -
  1. -

    Install dependencies

    - The linux-image-extra package is only needed on standard Ubuntu EC2 AMIs in order to install the aufs kernel module. - 
    sudo apt-get install linux-image-extra-`uname -r`
    - - -
    2. -
  2. -

    Install Docker

    -

    Add the Ubuntu PPA (Personal Package Archive) sources to your apt sources list, update and install.

    -

    This may import a new GPG key (key 63561DC6: public key "Launchpad PPA for dotcloud team" imported).

    -
    - 
    sudo apt-get install software-properties-common
    - 
    sudo add-apt-repository ppa:dotcloud/lxc-docker
    - 
    sudo apt-get update
    - 
    sudo apt-get install lxc-docker
    -
    - - -
    3. - -
  3. -

    Run!

    - -
    - 
    docker run -i -t ubuntu /bin/bash
    -
    -
    4. - Continue with the Hello world example.
    - Or check more detailed installation instructions -
-
- -
-

Contributing to Docker

- -

Want to hack on Docker? Awesome! We have some instructions to get you started. They are probably not perfect, please let us know if anything feels wrong or incomplete.

-
- -
-
-
-

Quick install on other operating systems

-

For other operating systems we recommend and provide a streamlined install with virtualbox, - vagrant and an Ubuntu virtual machine.

- - - -
- -
-

Questions? Want to get in touch?

-

There are several ways to get in touch:

-

Join the discussion on IRC. We can be found in the #docker channel on chat.freenode.net

-

Discussions happen on our google group: docker-club at googlegroups.com

-

All our development and decisions are made out in the open on Github github.com/dotcloud/docker

-

Get help on using Docker by asking on Stackoverflow

-

And of course, tweet your tweets to twitter.com/getdocker

-
- - -
-
- Fill out my online form. -
- -
- -
-
-
- - -
- -
- - - - - - - - - - - diff --git a/docs/website/index.html b/docs/website/index.html deleted file mode 100644 index f6f4efbccb..0000000000 --- a/docs/website/index.html +++ /dev/null @@ -1,359 +0,0 @@ - - - - - - - - - - - Docker - the Linux container engine - - - - - - - - - - - - - - - - - - - - - - - -
-
-
- - - - -
- -
-
-
-
- - -
-
- -
-
- -
-
- docker letters - -

The Linux container engine

-
- -
- -
- Docker is an open-source engine which automates the deployment of applications as highly portable, self-sufficient containers which are independent of hardware, language, framework, packaging system and hosting provider. -
- -
- - - - -
- -
-
- -
-
- -
-
-
-
-
- -
-
- -
-
-

Heterogeneous payloads

-

Any combination of binaries, libraries, configuration files, scripts, virtualenvs, jars, gems, tarballs, you name it. No more juggling between domain-specific tools. Docker can deploy and run them all.

-

Any server

-

Docker can run on any x64 machine with a modern linux kernel - whether it's a laptop, a bare metal server or a VM. This makes it perfect for multi-cloud deployments.

-

Isolation

-

Docker isolates processes from each other and from the underlying host, using lightweight containers.

-

Repeatability

-

Because each container is isolated in its own filesystem, they behave the same regardless of where, when, and alongside what they run.

-
-
-
-
- we're hiring -
-
-

Do you think it is cool to hack on docker? Join us!

-
    -
  • Work on open source
    • -
  • Program in Go
    • -
- read more -
-
- -
-
-
-
-

New! Docker Index

- On the Docker Index you can find and explore pre-made container images. It allows you to share your images and download them. - -

- -
- DOCKER index -
- -   - - -
-
-
- Fill out my online form. -
- -
-
-
- -
- -
- -
-
-
- - Mitchell Hashimoto ‏@mitchellh: Docker launched today. It is incredible. They’re also working RIGHT NOW on a Vagrant provider. LXC is COMING!! -
-
-
-
- - Adam Jacob ‏@adamhjk: Docker is clearly the right idea. @solomonstre absolutely killed it. Containerized app deployment is the future, I think. -
-
-
-
-
-
- - Matt Townsend ‏@mtownsend: I have a serious code crush on docker.io - it's Lego for PaaS. Motherfucking awesome Lego. -
-
-
-
- - Rob Harrop ‏@robertharrop: Impressed by @getdocker - it's all kinds of magic. Serious rethink of AWS architecture happening @skillsmatter. -
-
-
-
-
-
- - John Willis @botchagalupe: IMHO docker is to paas what chef was to Iaas 4 years ago -
-
-
-
- - John Feminella ‏@superninjarobot: So, @getdocker is pure excellence. If you've ever wished for arbitrary, PaaS-agnostic, lxc/aufs Linux containers, this is your jam! -
-
-
-
-
-
- - David Romulan ‏@destructuring: I haven't had this much fun since AWS -
-
-
-
- - Ricardo Gladwell ‏@rgladwell: wow @getdocker is either amazing or totally stupid -
-
- -
-
- -
-
-
- -
- -

Notable features

- -
    -
  • Filesystem isolation: each process container runs in a completely separate root filesystem.
    • -
  • Resource isolation: system resources like cpu and memory can be allocated differently to each process container, using cgroups.
    • -
  • Network isolation: each process container runs in its own network namespace, with a virtual interface and IP address of its own.
    • -
  • Copy-on-write: root filesystems are created using copy-on-write, which makes deployment extremely fast, memory-cheap and disk-cheap.
    • -
  • Logging: the standard streams (stdout/stderr/stdin) of each process container is collected and logged for real-time or batch retrieval.
    • -
  • Change management: changes to a container's filesystem can be committed into a new image and re-used to create more containers. No templating or manual configuration required.
    • -
  • Interactive shell: docker can allocate a pseudo-tty and attach to the standard input of any container, for example to run a throwaway interactive shell.
    • -
- -

Under the hood

- -

Under the hood, Docker is built on the following components:

- -
    -
  • The cgroup and namespacing capabilities of the Linux kernel;
    • -
  • AUFS, a powerful union filesystem with copy-on-write capabilities;
    • -
  • The Go programming language;
    • -
  • lxc, a set of convenience scripts to simplify the creation of linux containers.
    • -
- -

Who started it

-

- Docker is an open-source implementation of the deployment engine which powers dotCloud, a popular Platform-as-a-Service.

- -

It benefits directly from the experience accumulated over several years of large-scale operation and support of hundreds of thousands - of applications and databases. -

- -
-
- -
- - -
-

Twitter

- - -
- -
-
- -
- - -
- -
- - - - - - - - - - - - diff --git a/docs/website/nginx.conf b/docs/website/nginx.conf deleted file mode 100644 index 97ffd2c0e5..0000000000 --- a/docs/website/nginx.conf +++ /dev/null @@ -1,6 +0,0 @@ - -# rule to redirect original links created when hosted on github pages -rewrite ^/documentation/(.*).html http://docs.docker.io/en/latest/$1/ permanent; - -# rewrite the stuff which was on the current page -rewrite ^/gettingstarted.html$ /gettingstarted/ permanent; diff --git a/docs/website/static b/docs/website/static deleted file mode 120000 index 95bc97aa10..0000000000 --- a/docs/website/static +++ /dev/null @@ -1 +0,0 @@ -../theme/docker/static \ No newline at end of file diff --git a/graph.go b/graph.go index 42d1bdbd4c..1d2a9f807f 100644 --- a/graph.go +++ b/graph.go @@ -1,9 +1,7 @@ package docker import ( - "encoding/json" "fmt" - "github.com/dotcloud/docker/registry" "github.com/dotcloud/docker/utils" "io" "io/ioutil" @@ -11,17 +9,13 @@ import ( "path" "path/filepath" "strings" - "sync" "time" ) // A Graph is a store for versioned filesystem images and the relationship between them. type Graph struct { - Root string - idIndex *utils.TruncIndex - checksumLock map[string]*sync.Mutex - lockSumFile *sync.Mutex - lockSumMap *sync.Mutex + Root string + idIndex *utils.TruncIndex } // NewGraph instantiates a new graph at the given root path in the filesystem. @@ -36,11 +30,8 @@ func NewGraph(root string) (*Graph, error) { return nil, err } graph := &Graph{ - Root: abspath, - idIndex: utils.NewTruncIndex(), - checksumLock: make(map[string]*sync.Mutex), - lockSumFile: &sync.Mutex{}, - lockSumMap: &sync.Mutex{}, + Root: abspath, + idIndex: utils.NewTruncIndex(), } if err := graph.restore(); err != nil { return nil, err @@ -99,11 +90,6 @@ func (graph *Graph) Get(name string) (*Image, error) { return nil, err } } - graph.lockSumMap.Lock() - defer graph.lockSumMap.Unlock() - if _, exists := graph.checksumLock[img.ID]; !exists { - graph.checksumLock[img.ID] = &sync.Mutex{} - } return img, nil } @@ -123,16 +109,15 @@ func (graph *Graph) Create(layerData Archive, container *Container, comment, aut img.Container = container.ID img.ContainerConfig = *container.Config } - if err := graph.Register(layerData, layerData != nil, img); err != nil { + if err := graph.Register(nil, layerData, img); err != nil { return nil, err } - go img.Checksum() return img, nil } // Register imports a pre-existing image into the graph. // FIXME: pass img as first argument -func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { +func (graph *Graph) Register(jsonData []byte, layerData Archive, img *Image) error { if err := ValidateID(img.ID); err != nil { return err } @@ -145,7 +130,7 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { if err != nil { return fmt.Errorf("Mktemp failed: %s", err) } - if err := StoreImage(img, layerData, tmp, store); err != nil { + if err := StoreImage(img, jsonData, layerData, tmp); err != nil { return err } // Commit @@ -154,7 +139,6 @@ func (graph *Graph) Register(layerData Archive, store bool, img *Image) error { } img.graph = graph graph.idIndex.Add(img.ID) - graph.checksumLock[img.ID] = &sync.Mutex{} return nil } @@ -175,7 +159,7 @@ func (graph *Graph) TempLayerArchive(id string, compression Compression, sf *uti if err != nil { return nil, err } - return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("Buffering to disk", "%v/%v (%v)"), sf), tmp.Root) + return NewTempArchive(utils.ProgressReader(ioutil.NopCloser(archive), 0, output, sf.FormatProgress("", "Buffering to disk", "%v/%v (%v)"), sf, true), tmp.Root) } // Mktemp creates a temporary sub-directory inside the graph's filesystem. @@ -193,8 +177,39 @@ func (graph *Graph) Mktemp(id string) (string, error) { return tmp.imageRoot(id), nil } +// getDockerInitLayer returns the path of a layer containing a mountpoint suitable +// for bind-mounting dockerinit into the container. The mountpoint is simply an +// empty file at /.dockerinit +// +// This extra layer is used by all containers as the top-most ro layer. It protects +// the container from unwanted side-effects on the rw layer. +func (graph *Graph) getDockerInitLayer() (string, error) { + tmp, err := graph.tmp() + if err != nil { + return "", err + } + initLayer := tmp.imageRoot("_dockerinit") + if err := os.Mkdir(initLayer, 0755); err != nil && !os.IsExist(err) { + // If directory already existed, keep going. + // For all other errors, abort. + return "", err + } + // FIXME: how the hell do I break down this line in a way + // that is idiomatic and not ugly as hell? + if f, err := os.OpenFile(path.Join(initLayer, ".dockerinit"), os.O_CREATE|os.O_TRUNC, 0700); err != nil && !os.IsExist(err) { + // If file already existed, keep going. + // For all other errors, abort. + return "", err + } else { + f.Close() + } + // Layer is ready to use, if it wasn't before. + return initLayer, nil +} + func (graph *Graph) tmp() (*Graph, error) { - return NewGraph(path.Join(graph.Root, ":tmp:")) + // Changed to _tmp from :tmp:, because it messed with ":" separators in aufs branch syntax... + return NewGraph(path.Join(graph.Root, "_tmp")) } // Check if given error is "not empty". @@ -311,40 +326,3 @@ func (graph *Graph) Heads() (map[string]*Image, error) { func (graph *Graph) imageRoot(id string) string { return path.Join(graph.Root, id) } - -func (graph *Graph) getStoredChecksums() (map[string]string, error) { - checksums := make(map[string]string) - // FIXME: Store the checksum in memory - - if checksumDict, err := ioutil.ReadFile(path.Join(graph.Root, "checksums")); err == nil { - if err := json.Unmarshal(checksumDict, &checksums); err != nil { - return nil, err - } - } - return checksums, nil -} - -func (graph *Graph) storeChecksums(checksums map[string]string) error { - checksumJSON, err := json.Marshal(checksums) - if err != nil { - return err - } - if err := ioutil.WriteFile(path.Join(graph.Root, "checksums"), checksumJSON, 0600); err != nil { - return err - } - return nil -} - -func (graph *Graph) UpdateChecksums(newChecksums map[string]*registry.ImgData) error { - graph.lockSumFile.Lock() - defer graph.lockSumFile.Unlock() - - localChecksums, err := graph.getStoredChecksums() - if err != nil { - return err - } - for id, elem := range newChecksums { - localChecksums[id] = elem.Checksum - } - return graph.storeChecksums(localChecksums) -} diff --git a/graph_test.go b/graph_test.go index 18682338d9..2898fccf99 100644 --- a/graph_test.go +++ b/graph_test.go @@ -38,7 +38,7 @@ func TestInterruptedRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - go graph.Register(badArchive, false, image) + go graph.Register(nil, badArchive, image) time.Sleep(200 * time.Millisecond) w.CloseWithError(errors.New("But I'm not a tarball!")) // (Nobody's perfect, darling) if _, err := graph.Get(image.ID); err == nil { @@ -49,7 +49,7 @@ func TestInterruptedRegister(t *testing.T) { if err != nil { t.Fatal(err) } - if err := graph.Register(goodArchive, false, image); err != nil { + if err := graph.Register(nil, goodArchive, image); err != nil { t.Fatal(err) } } @@ -95,7 +95,7 @@ func TestRegister(t *testing.T) { Comment: "testing", Created: time.Now(), } - err = graph.Register(archive, false, image) + err = graph.Register(nil, archive, image) if err != nil { t.Fatal(err) } @@ -225,7 +225,7 @@ func TestDelete(t *testing.T) { t.Fatal(err) } // Test delete twice (pull -> rm -> pull -> rm) - if err := graph.Register(archive, false, img1); err != nil { + if err := graph.Register(nil, archive, img1); err != nil { t.Fatal(err) } if err := graph.Delete(img1.ID); err != nil { diff --git a/image.go b/image.go index e1b1ac0418..9f34160b80 100644 --- a/image.go +++ b/image.go @@ -2,7 +2,6 @@ package docker import ( "crypto/rand" - "crypto/sha256" "encoding/hex" "encoding/json" "fmt" @@ -14,6 +13,7 @@ import ( "os/exec" "path" "path/filepath" + "strconv" "strings" "time" ) @@ -47,6 +47,19 @@ func LoadImage(root string) (*Image, error) { if err := ValidateID(img.ID); err != nil { return nil, err } + + if buf, err := ioutil.ReadFile(path.Join(root, "layersize")); err != nil { + if !os.IsNotExist(err) { + return nil, err + } + } else { + if size, err := strconv.Atoi(string(buf)); err != nil { + return nil, err + } else { + img.Size = int64(size) + } + } + // Check that the filesystem layer exists if stat, err := os.Stat(layerPath(root)); err != nil { if os.IsNotExist(err) { @@ -59,7 +72,7 @@ func LoadImage(root string) (*Image, error) { return img, nil } -func StoreImage(img *Image, layerData Archive, root string, store bool) error { +func StoreImage(img *Image, jsonData []byte, layerData Archive, root string) error { // Check that root doesn't already exist if _, err := os.Stat(root); err == nil { return fmt.Errorf("Image %s already exists", img.ID) @@ -68,30 +81,10 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { } // Store the layer layer := layerPath(root) - if err := os.MkdirAll(layer, 0700); err != nil { + if err := os.MkdirAll(layer, 0755); err != nil { return err } - if store { - layerArchive := layerArchivePath(root) - file, err := os.OpenFile(layerArchive, os.O_WRONLY|os.O_CREATE, 0600) - if err != nil { - return err - } - // FIXME: Retrieve the image layer size from here? - if _, err := io.Copy(file, layerData); err != nil { - return err - } - // FIXME: Don't close/open, read/write instead of Copy - file.Close() - - file, err = os.Open(layerArchive) - if err != nil { - return err - } - defer file.Close() - layerData = file - } // If layerData is not nil, unpack it into the new layer if layerData != nil { start := time.Now() @@ -102,25 +95,36 @@ func StoreImage(img *Image, layerData Archive, root string, store bool) error { utils.Debugf("Untar time: %vs\n", time.Now().Sub(start).Seconds()) } + // If raw json is provided, then use it + if jsonData != nil { + return ioutil.WriteFile(jsonPath(root), jsonData, 0600) + } else { // Otherwise, unmarshal the image + jsonData, err := json.Marshal(img) + if err != nil { + return err + } + if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { + return err + } + } + return StoreSize(img, root) } func StoreSize(img *Image, root string) error { layer := layerPath(root) + var totalSize int64 = 0 filepath.Walk(layer, func(path string, fileInfo os.FileInfo, err error) error { - img.Size += fileInfo.Size() + totalSize += fileInfo.Size() return nil }) + img.Size = totalSize - // Store the json ball - jsonData, err := json.Marshal(img) - if err != nil { - return err - } - if err := ioutil.WriteFile(jsonPath(root), jsonData, 0600); err != nil { - return err + if err := ioutil.WriteFile(path.Join(root, "layersize"), []byte(strconv.Itoa(int(totalSize))), 0600); err != nil { + return nil } + return nil } @@ -128,10 +132,6 @@ func layerPath(root string) string { return path.Join(root, "layer") } -func layerArchivePath(root string) string { - return path.Join(root, "layer.tar.xz") -} - func jsonPath(root string) string { return path.Join(root, "json") } @@ -263,6 +263,13 @@ func (img *Image) layers() ([]string, error) { if len(list) == 0 { return nil, fmt.Errorf("No layer found for image %s\n", img.ID) } + + // Inject the dockerinit layer (empty place-holder for mount-binding dockerinit) + if dockerinitLayer, err := img.getDockerInitLayer(); err != nil { + return nil, err + } else { + list = append([]string{dockerinitLayer}, list...) + } return list, nil } @@ -292,6 +299,13 @@ func (img *Image) GetParent() (*Image, error) { return img.graph.Get(img.Parent) } +func (img *Image) getDockerInitLayer() (string, error) { + if img.graph == nil { + return "", fmt.Errorf("Can't lookup dockerinit layer of unregistered image") + } + return img.graph.getDockerInitLayer() +} + func (img *Image) root() (string, error) { if img.graph == nil { return "", fmt.Errorf("Can't lookup root of unregistered image") @@ -308,80 +322,6 @@ func (img *Image) layer() (string, error) { return layerPath(root), nil } -func (img *Image) Checksum() (string, error) { - img.graph.checksumLock[img.ID].Lock() - defer img.graph.checksumLock[img.ID].Unlock() - - root, err := img.root() - if err != nil { - return "", err - } - - checksums, err := img.graph.getStoredChecksums() - if err != nil { - return "", err - } - if checksum, ok := checksums[img.ID]; ok { - return checksum, nil - } - - layer, err := img.layer() - if err != nil { - return "", err - } - jsonData, err := ioutil.ReadFile(jsonPath(root)) - if err != nil { - return "", err - } - - var layerData io.Reader - - if file, err := os.Open(layerArchivePath(root)); err != nil { - if os.IsNotExist(err) { - layerData, err = Tar(layer, Xz) - if err != nil { - return "", err - } - } else { - return "", err - } - } else { - defer file.Close() - layerData = file - } - - h := sha256.New() - if _, err := h.Write(jsonData); err != nil { - return "", err - } - if _, err := h.Write([]byte("\n")); err != nil { - return "", err - } - - if _, err := io.Copy(h, layerData); err != nil { - return "", err - } - hash := "sha256:" + hex.EncodeToString(h.Sum(nil)) - - // Reload the json file to make sure not to overwrite faster sums - img.graph.lockSumFile.Lock() - defer img.graph.lockSumFile.Unlock() - - checksums, err = img.graph.getStoredChecksums() - if err != nil { - return "", err - } - - checksums[img.ID] = hash - - // Dump the checksums to disc - if err := img.graph.storeChecksums(checksums); err != nil { - return hash, err - } - - return hash, nil -} - func (img *Image) getParentsSize(size int64) int64 { parentImage, err := img.GetParent() if err != nil || parentImage == nil { diff --git a/lxc_template.go b/lxc_template.go index d1f67e3376..d49d66b4d9 100644 --- a/lxc_template.go +++ b/lxc_template.go @@ -84,7 +84,7 @@ lxc.mount.entry = devpts {{$ROOTFS}}/dev/pts devpts newinstance,ptmxmode=0666,no #lxc.mount.entry = shm {{$ROOTFS}}/dev/shm tmpfs size=65536k,nosuid,nodev,noexec 0 0 # Inject docker-init -lxc.mount.entry = {{.SysInitPath}} {{$ROOTFS}}/sbin/init none bind,ro 0 0 +lxc.mount.entry = {{.SysInitPath}} {{$ROOTFS}}/.dockerinit none bind,ro 0 0 # In order to get a working DNS environment, mount bind (ro) the host's /etc/resolv.conf into the container lxc.mount.entry = {{.ResolvConfPath}} {{$ROOTFS}}/etc/resolv.conf none bind,ro 0 0 diff --git a/network.go b/network.go index 2e2dc7785c..b289934295 100644 --- a/network.go +++ b/network.go @@ -93,20 +93,29 @@ func iptables(args ...string) error { return nil } -func checkRouteOverlaps(dockerNetwork *net.IPNet) error { - output, err := ip("route") - if err != nil { - return err - } - utils.Debugf("Routes:\n\n%s", output) - for _, line := range strings.Split(output, "\n") { +func checkRouteOverlaps(routes string, dockerNetwork *net.IPNet) error { + utils.Debugf("Routes:\n\n%s", routes) + for _, line := range strings.Split(routes, "\n") { if strings.Trim(line, "\r\n\t ") == "" || strings.Contains(line, "default") { continue } - if _, network, err := net.ParseCIDR(strings.Split(line, " ")[0]); err != nil { - return fmt.Errorf("Unexpected ip route output: %s (%s)", err, line) - } else if networkOverlaps(dockerNetwork, network) { - return fmt.Errorf("Network %s is already routed: '%s'", dockerNetwork.String(), line) + _, network, err := net.ParseCIDR(strings.Split(line, " ")[0]) + if err != nil { + // is this a mask-less IP address? + if ip := net.ParseIP(strings.Split(line, " ")[0]); ip == nil { + // fail only if it's neither a network nor a mask-less IP address + return fmt.Errorf("Unexpected ip route output: %s (%s)", err, line) + } else { + _, network, err = net.ParseCIDR(ip.String() + "/32") + if err != nil { + return err + } + } + } + if err == nil && network != nil { + if networkOverlaps(dockerNetwork, network) { + return fmt.Errorf("Network %s is already routed: '%s'", dockerNetwork, line) + } } } return nil @@ -122,8 +131,8 @@ func CreateBridgeIface(ifaceName string) error { // In theory this shouldn't matter - in practice there's bound to be a few scripts relying // on the internal addressing or other stupid things like that. // The shouldn't, but hey, let's not break them unless we really have to. - "172.16.42.1/16", - "10.0.42.1/16", // Don't even try using the entire /8, that's too intrusive + "172.17.42.1/16", // Don't use 172.16.0.0/16, it conflicts with EC2 DNS 172.16.0.23 + "10.0.42.1/16", // Don't even try using the entire /8, that's too intrusive "10.1.42.1/16", "10.42.42.1/16", "172.16.42.1/24", @@ -142,7 +151,11 @@ func CreateBridgeIface(ifaceName string) error { if err != nil { return err } - if err := checkRouteOverlaps(dockerNetwork); err == nil { + routes, err := ip("route") + if err != nil { + return err + } + if err := checkRouteOverlaps(routes, dockerNetwork); err == nil { ifaceAddr = addr break } else { @@ -240,6 +253,7 @@ func (mapper *PortMapper) setup() error { func (mapper *PortMapper) iptablesForward(rule string, port int, proto string, dest_addr string, dest_port int) error { return iptables("-t", "nat", rule, "DOCKER", "-p", proto, "--dport", strconv.Itoa(port), + "!", "-i", NetworkBridgeIface, "-j", "DNAT", "--to-destination", net.JoinHostPort(dest_addr, strconv.Itoa(dest_port))) } @@ -251,7 +265,7 @@ func (mapper *PortMapper) Map(port int, backendAddr net.Addr) error { return err } mapper.tcpMapping[port] = backendAddr.(*net.TCPAddr) - proxy, err := NewProxy(&net.TCPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}, backendAddr) + proxy, err := NewProxy(&net.TCPAddr{IP: net.IPv4(0, 0, 0, 0), Port: port}, backendAddr) if err != nil { mapper.Unmap(port, "tcp") return err @@ -265,7 +279,7 @@ func (mapper *PortMapper) Map(port int, backendAddr net.Addr) error { return err } mapper.udpMapping[port] = backendAddr.(*net.UDPAddr) - proxy, err := NewProxy(&net.UDPAddr{IP: net.IPv4(127, 0, 0, 1), Port: port}, backendAddr) + proxy, err := NewProxy(&net.UDPAddr{IP: net.IPv4(0, 0, 0, 0), Port: port}, backendAddr) if err != nil { mapper.Unmap(port, "udp") return err diff --git a/network_test.go b/network_test.go index 8e6eaad773..bd3a16a1be 100644 --- a/network_test.go +++ b/network_test.go @@ -383,3 +383,22 @@ func TestNetworkOverlaps(t *testing.T) { //netX starts and ends before netY AssertNoOverlap("172.16.1.1/25", "172.16.2.1/24", t) } + +func TestCheckRouteOverlaps(t *testing.T) { + routes := `default via 10.0.2.2 dev eth0 +10.0.2.0 dev eth0 proto kernel scope link src 10.0.2.15 +10.0.3.0/24 dev lxcbr0 proto kernel scope link src 10.0.3.1 +10.0.42.0/24 dev testdockbr0 proto kernel scope link src 10.0.42.1 +172.16.42.0/24 dev docker0 proto kernel scope link src 172.16.42.1 +192.168.142.0/24 dev eth1 proto kernel scope link src 192.168.142.142` + + _, netX, _ := net.ParseCIDR("172.16.0.1/24") + if err := checkRouteOverlaps(routes, netX); err != nil { + t.Fatal(err) + } + + _, netX, _ = net.ParseCIDR("10.0.2.0/24") + if err := checkRouteOverlaps(routes, netX); err == nil { + t.Fatalf("10.0.2.0/24 and 10.0.2.0 should overlap but it doesn't") + } +} diff --git a/packaging/debian/lxc-docker.1 b/packaging/debian/lxc-docker.1 index 8f98785285..b92853f95b 100644 --- a/packaging/debian/lxc-docker.1 +++ b/packaging/debian/lxc-docker.1 @@ -146,7 +146,7 @@ cd docker\-master .sp .nf .ft C -sudo ./docker run \-i \-t base /bin/bash +sudo ./docker run \-i \-t ubuntu /bin/bash .ft P .fi .sp @@ -496,22 +496,22 @@ This is the most basic example available for using docker .sp This example assumes you have Docker installed. .sp -Download the base container +Download the ubuntu container .sp .nf .ft C -# Download a base image -docker pull base +# Download an ubuntu image +docker pull ubuntu .ft P .fi .sp -The \fIbase\fP image is a minimal \fIubuntu\fP based container, alternatively you can select \fIbusybox\fP, a bare -minimal linux system. The images are retrieved from the docker repository. +Alternatively you can select \fIbusybox\fP, a bare minimal linux system. The +images are retrieved from the docker repository. .sp .nf .ft C #run a simple echo command, that will echo hello world back to the console over standard out. -docker run base /bin/echo hello world +docker run ubuntu /bin/echo hello world .ft P .fi .sp @@ -520,7 +520,7 @@ docker run base /bin/echo hello world .IP \(bu 2 \fB"docker run"\fP run a command in a new container .IP \(bu 2 -\fB"base"\fP is the image we want to run the command inside of. +\fB"ubuntu"\fP is the image we want to run the command inside of. .IP \(bu 2 \fB"/bin/echo"\fP is the command we want to run in the container .IP \(bu 2 @@ -536,15 +536,15 @@ Continue to the \fIhello_world_daemon\fP example. .sp The most boring daemon ever written. .sp -This example assumes you have Docker installed and with the base image already imported \fBdocker pull base\fP. -We will use the base image to run a simple hello world daemon that will just print hello world to standard +This example assumes you have Docker installed and with the ubuntu image already imported \fBdocker pull ubuntu\fP. +We will use the ubuntu image to run a simple hello world daemon that will just print hello world to standard out every second. It will continue to do this until we stop it. .sp \fBSteps:\fP .sp .nf .ft C -$ CONTAINER_ID=$(docker run \-d base /bin/sh \-c "while true; do echo hello world; sleep 1; done") +$ CONTAINER_ID=$(docker run \-d ubuntu /bin/sh \-c "while true; do echo hello world; sleep 1; done") .ft P .fi .sp @@ -553,7 +553,7 @@ We are going to run a simple hello world daemon in a new container made from the .IP \(bu 2 \fB"docker run \-d "\fP run a command in a new container. We pass "\-d" so it runs as a daemon. .IP \(bu 2 -\fB"base"\fP is the image we want to run the command inside of. +\fB"ubuntu"\fP is the image we want to run the command inside of. .IP \(bu 2 \fB"/bin/sh \-c"\fP is the command we want to run in the container .IP \(bu 2 @@ -766,12 +766,12 @@ Contents: .sp .nf .ft C -# Download a base image -docker import base +# Download an ubuntu image +docker import ubuntu -# Run an interactive shell in the base image, +# Run an interactive shell in the ubuntu image, # allocate a tty, attach stdin and stdout -docker run \-a \-i \-t base /bin/bash +docker run \-a \-i \-t ubuntu /bin/bash .ft P .fi .SS Starting a long\-running worker process @@ -782,7 +782,7 @@ docker run \-a \-i \-t base /bin/bash (docker \-d || echo "Docker daemon already running") & # Start a very useful long\-running process -JOB=$(docker run base /bin/sh \-c "while true; do echo Hello world!; sleep 1; done") +JOB=$(docker run ubuntu /bin/sh \-c "while true; do echo Hello world!; sleep 1; done") # Collect the output of the job so far docker logs $JOB @@ -803,7 +803,7 @@ docker ps .nf .ft C # Expose port 4444 of this container, and tell netcat to listen on it -JOB=$(docker run \-p 4444 base /bin/nc \-l \-p 4444) +JOB=$(docker run \-p 4444 ubuntu /bin/nc \-l \-p 4444) # Which public port is NATed to my container? PORT=$(docker port $JOB 4444) @@ -923,6 +923,12 @@ List images Usage: docker import [OPTIONS] URL|\- [REPOSITORY [TAG]] .sp Create a new filesystem image from the contents of a tarball + +At this time, the URL must start with ``http`` and point to a single file archive +(.tar, .tar.gz, .tgz, .bzip, .tar.xz, .txz) +containing a root filesystem. If you would like to import from a local directory or archive, +you can use the ``-`` parameter to take the data from standard in. + .SS info .sp .nf diff --git a/packaging/ubuntu/Makefile b/packaging/ubuntu/Makefile index 7bddf2376f..a1e53b1d5c 100644 --- a/packaging/ubuntu/Makefile +++ b/packaging/ubuntu/Makefile @@ -32,7 +32,7 @@ ubuntu: # Retrieve docker project and its go structure from internet rm -rf ${BUILD_SRC} git clone $(shell git rev-parse --show-toplevel) ${BUILD_SRC}/${GITHUB_PATH} - cd ${BUILD_SRC}/${GITHUB_PATH}; git checkout v${VERSION} && GOPATH=${BUILD_SRC} go get -d + cd ${BUILD_SRC}/${GITHUB_PATH}; git checkout v${VERSION} && GOPATH=${BUILD_SRC} go get -d && cd ${BUILD_SRC}/src/${GITHUB_PATH} && git checkout v${VERSION} # Add debianization mkdir ${BUILD_SRC}/debian cp Makefile ${BUILD_SRC} diff --git a/packaging/ubuntu/docker.upstart b/packaging/ubuntu/docker.upstart index 2bd5565ee7..f4d2fbe922 100644 --- a/packaging/ubuntu/docker.upstart +++ b/packaging/ubuntu/docker.upstart @@ -1,9 +1,8 @@ description "Run docker" -start on runlevel [2345] -stop on starting rc RUNLEVEL=[016] +start on filesystem or runlevel [2345] +stop on runlevel [!2345] + respawn -script - /usr/bin/docker -d -end script +exec /usr/bin/docker -d diff --git a/registry/registry.go b/registry/registry.go index e6f4f592e2..9ed2b486d6 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -9,16 +9,20 @@ import ( "github.com/dotcloud/docker/utils" "io" "io/ioutil" + "net" "net/http" "net/http/cookiejar" "net/url" "regexp" "strconv" "strings" + "time" ) -var ErrAlreadyExists = errors.New("Image already exists") -var ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") +var ( + ErrAlreadyExists = errors.New("Image already exists") + ErrInvalidRepositoryName = errors.New("Invalid repository name (ex: \"registry.domain.tld/myrepos\")") +) func pingRegistryEndpoint(endpoint string) error { if endpoint == auth.IndexServerAddress() { @@ -26,7 +30,19 @@ func pingRegistryEndpoint(endpoint string) error { // (and we never want to fallback to http in case of error) return nil } - resp, err := http.Get(endpoint + "_ping") + httpDial := func(proto string, addr string) (net.Conn, error) { + // Set the connect timeout to 5 seconds + conn, err := net.DialTimeout(proto, addr, time.Duration(5)*time.Second) + if err != nil { + return nil, err + } + // Set the recv timeout to 10 seconds + conn.SetDeadline(time.Now().Add(time.Duration(10) * time.Second)) + return conn, nil + } + httpTransport := &http.Transport{Dial: httpDial} + client := &http.Client{Transport: httpTransport} + resp, err := client.Get(endpoint + "_ping") if err != nil { return err } @@ -67,7 +83,8 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return "", "", ErrInvalidRepositoryName } nameParts := strings.SplitN(reposName, "/", 2) - if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") { + if !strings.Contains(nameParts[0], ".") && !strings.Contains(nameParts[0], ":") && + nameParts[0] != "localhost" { // This is a Docker Index repos (ex: samalba/hipache or ubuntu) err := validateRepositoryName(reposName) return auth.IndexServerAddress(), reposName, err @@ -98,47 +115,32 @@ func ResolveRepositoryName(reposName string) (string, string, error) { return endpoint, reposName, err } -// VersionInfo is used to model entities which has a version. -// It is basically a tupple with name and version. -type VersionInfo interface { - Name() string - Version() string -} - func doWithCookies(c *http.Client, req *http.Request) (*http.Response, error) { for _, cookie := range c.Jar.Cookies(req.URL) { req.AddCookie(cookie) } - return c.Do(req) -} - -// Set the user agent field in the header based on the versions provided -// in NewRegistry() and extra. -func (r *Registry) setUserAgent(req *http.Request, extra ...VersionInfo) { - if len(r.baseVersions)+len(extra) == 0 { - return + res, err := c.Do(req) + if err != nil { + return nil, err } - if len(extra) == 0 { - req.Header.Set("User-Agent", r.baseVersionsStr) - } else { - req.Header.Set("User-Agent", appendVersions(r.baseVersionsStr, extra...)) + if len(res.Cookies()) > 0 { + c.Jar.SetCookies(req.URL, res.Cookies()) } - return + return res, err } // Retrieve the history of a given image from the Registry. // Return a list of the parent's json (requested image included) func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]string, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/ancestry", nil) if err != nil { return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil || res.StatusCode != 200 { if res != nil { - return nil, fmt.Errorf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to fetch remote history for %s", res.StatusCode, imgID), res) } return nil, err } @@ -159,13 +161,14 @@ func (r *Registry) GetRemoteHistory(imgID, registry string, token []string) ([]s // Check if an image exists in the Registry func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) bool { - rt := &http.Transport{Proxy: http.ProxyFromEnvironment} - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return false } - res, err := rt.RoundTrip(req) + req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) + res, err := doWithCookies(r.client, req) if err != nil { return false } @@ -176,19 +179,18 @@ func (r *Registry) LookupRemoteImage(imgID, registry string, token []string) boo // Retrieve an image from the Registry. func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([]byte, int, error) { // Get the JSON - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/json", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/json", nil) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, -1, fmt.Errorf("Failed to download json: %s", err) } defer res.Body.Close() if res.StatusCode != 200 { - return nil, -1, fmt.Errorf("HTTP code %d", res.StatusCode) + return nil, -1, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d", res.StatusCode), res) } imageSize, err := strconv.Atoi(res.Header.Get("X-Docker-Size")) @@ -204,16 +206,19 @@ func (r *Registry) GetRemoteImageJSON(imgID, registry string, token []string) ([ } func (r *Registry) GetRemoteImageLayer(imgID, registry string, token []string) (io.ReadCloser, error) { - req, err := http.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) + req, err := r.reqFactory.NewRequest("GET", registry+"images/"+imgID+"/layer", nil) if err != nil { return nil, fmt.Errorf("Error while getting from the server: %s\n", err) } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, err } + if res.StatusCode != 200 { + return nil, fmt.Errorf("Server error: Status %d while fetching image layer (%s)", + res.StatusCode, imgID) + } return res.Body, nil } @@ -230,8 +235,7 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ return nil, err } req.Header.Set("Authorization", "Token "+strings.Join(token, ", ")) - r.setUserAgent(req) - res, err := r.client.Do(req) + res, err := doWithCookies(r.client, req) if err != nil { return nil, err } @@ -259,8 +263,11 @@ func (r *Registry) GetRemoteTags(registries []string, repository string, token [ } func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, error) { + repositoryTarget := fmt.Sprintf("%srepositories/%s/images", indexEp, remote) + utils.Debugf("[registry] Calling GET %s", repositoryTarget) + req, err := r.opaqueRequest("GET", repositoryTarget, nil) if err != nil { return nil, err @@ -269,7 +276,6 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) } req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) res, err := r.client.Do(req) if err != nil { @@ -277,12 +283,12 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e } defer res.Body.Close() if res.StatusCode == 401 { - return nil, fmt.Errorf("Please login first (HTTP code %d)", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Please login first (HTTP code %d)", res.StatusCode), res) } // TODO: Right now we're ignoring checksums in the response body. // In the future, we need to use them to check image validity. if res.StatusCode != 200 { - return nil, fmt.Errorf("HTTP code: %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("HTTP code: %d", res.StatusCode), res) } var tokens []string @@ -323,19 +329,17 @@ func (r *Registry) GetRepositoryData(indexEp, remote string) (*RepositoryData, e }, nil } -// Push a local image to the registry -func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { - // FIXME: try json with UTF8 - req, err := http.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", strings.NewReader(string(jsonRaw))) +func (r *Registry) PushImageChecksumRegistry(imgData *ImgData, registry string, token []string) error { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/checksum") + + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/checksum", nil) if err != nil { return err } - req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) req.Header.Set("X-Docker-Checksum", imgData.Checksum) - r.setUserAgent(req) - utils.Debugf("Setting checksum for %s: %s", imgData.ID, imgData.Checksum) res, err := doWithCookies(r.client, req) if err != nil { return fmt.Errorf("Failed to upload metadata: %s", err) @@ -360,33 +364,70 @@ func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, regis return nil } -func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string) error { - req, err := http.NewRequest("PUT", registry+"images/"+imgID+"/layer", layer) +// Push a local image to the registry +func (r *Registry) PushImageJSONRegistry(imgData *ImgData, jsonRaw []byte, registry string, token []string) error { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgData.ID+"/json") + + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgData.ID+"/json", bytes.NewReader(jsonRaw)) if err != nil { return err } + req.Header.Add("Content-type", "application/json") + req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) + + res, err := doWithCookies(r.client, req) + if err != nil { + return fmt.Errorf("Failed to upload metadata: %s", err) + } + defer res.Body.Close() + if res.StatusCode != 200 { + errBody, err := ioutil.ReadAll(res.Body) + if err != nil { + return utils.NewHTTPRequestError(fmt.Sprint("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) + } + var jsonBody map[string]string + if err := json.Unmarshal(errBody, &jsonBody); err != nil { + errBody = []byte(err.Error()) + } else if jsonBody["error"] == "Image already exists" { + return ErrAlreadyExists + } + return utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata: %s", res.StatusCode, errBody), res) + } + return nil +} + +func (r *Registry) PushImageLayerRegistry(imgID string, layer io.Reader, registry string, token []string, jsonRaw []byte) (checksum string, err error) { + + utils.Debugf("[registry] Calling PUT %s", registry+"images/"+imgID+"/layer") + + tarsumLayer := &utils.TarSum{Reader: layer} + + req, err := r.reqFactory.NewRequest("PUT", registry+"images/"+imgID+"/layer", tarsumLayer) + if err != nil { + return "", err + } req.ContentLength = -1 req.TransferEncoding = []string{"chunked"} req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) res, err := doWithCookies(r.client, req) if err != nil { - return fmt.Errorf("Failed to upload layer: %s", err) + return "", fmt.Errorf("Failed to upload layer: %s", err) } defer res.Body.Close() if res.StatusCode != 200 { errBody, err := ioutil.ReadAll(res.Body) if err != nil { - return fmt.Errorf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err) + return "", utils.NewHTTPRequestError(fmt.Sprintf("HTTP code %d while uploading metadata and error when trying to parse response body: %s", res.StatusCode, err), res) } - return fmt.Errorf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody) + return "", utils.NewHTTPRequestError(fmt.Sprintf("Received HTTP code %d while uploading layer: %s", res.StatusCode, errBody), res) } - return nil + return tarsumLayer.Sum(jsonRaw), nil } func (r *Registry) opaqueRequest(method, urlStr string, body io.Reader) (*http.Request, error) { - req, err := http.NewRequest(method, urlStr, body) + req, err := r.reqFactory.NewRequest(method, urlStr, body) if err != nil { return nil, err } @@ -406,7 +447,6 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } req.Header.Add("Content-type", "application/json") req.Header.Set("Authorization", "Token "+strings.Join(token, ",")) - r.setUserAgent(req) req.ContentLength = int64(len(revision)) res, err := doWithCookies(r.client, req) if err != nil { @@ -414,13 +454,25 @@ func (r *Registry) PushRegistryTag(remote, revision, tag, registry string, token } res.Body.Close() if res.StatusCode != 200 && res.StatusCode != 201 { - return fmt.Errorf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote) + return utils.NewHTTPRequestError(fmt.Sprintf("Internal server error: %d trying to push tag %s on %s", res.StatusCode, tag, remote), res) } return nil } func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData, validate bool, regs []string) (*RepositoryData, error) { - imgListJSON, err := json.Marshal(imgList) + cleanImgList := []*ImgData{} + + if validate { + for _, elem := range imgList { + if elem.Checksum != "" { + cleanImgList = append(cleanImgList, elem) + } + } + } else { + cleanImgList = imgList + } + + imgListJSON, err := json.Marshal(cleanImgList) if err != nil { return nil, err } @@ -430,7 +482,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData } u := fmt.Sprintf("%srepositories/%s/%s", indexEp, remote, suffix) - utils.Debugf("PUT %s", u) + utils.Debugf("[registry] PUT %s", u) utils.Debugf("Image list pushed to index:\n%s\n", imgListJSON) req, err := r.opaqueRequest("PUT", u, bytes.NewReader(imgListJSON)) if err != nil { @@ -439,7 +491,6 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -460,7 +511,6 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData req.SetBasicAuth(r.authConfig.Username, r.authConfig.Password) req.ContentLength = int64(len(imgListJSON)) req.Header.Set("X-Docker-Token", "true") - r.setUserAgent(req) if validate { req.Header["X-Docker-Endpoints"] = regs } @@ -479,7 +529,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push repository %s: %s", res.StatusCode, remote, errBody), res) } if res.Header.Get("X-Docker-Token") != "" { tokens = res.Header["X-Docker-Token"] @@ -503,7 +553,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData if err != nil { return nil, err } - return nil, fmt.Errorf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Error: Status %d trying to push checksums %s: %s", res.StatusCode, remote, errBody), res) } } @@ -515,7 +565,7 @@ func (r *Registry) PushImageJSONIndex(indexEp, remote string, imgList []*ImgData func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { u := auth.IndexServerAddress() + "search?q=" + url.QueryEscape(term) - req, err := http.NewRequest("GET", u, nil) + req, err := r.reqFactory.NewRequest("GET", u, nil) if err != nil { return nil, err } @@ -525,7 +575,7 @@ func (r *Registry) SearchRepositories(term string) (*SearchResults, error) { } defer res.Body.Close() if res.StatusCode != 200 { - return nil, fmt.Errorf("Unexepected status code %d", res.StatusCode) + return nil, utils.NewHTTPRequestError(fmt.Sprintf("Unexepected status code %d", res.StatusCode), res) } rawData, err := ioutil.ReadAll(res.Body) if err != nil { @@ -567,52 +617,12 @@ type ImgData struct { } type Registry struct { - client *http.Client - authConfig *auth.AuthConfig - baseVersions []VersionInfo - baseVersionsStr string + client *http.Client + authConfig *auth.AuthConfig + reqFactory *utils.HTTPRequestFactory } -func validVersion(version VersionInfo) bool { - stopChars := " \t\r\n/" - if strings.ContainsAny(version.Name(), stopChars) { - return false - } - if strings.ContainsAny(version.Version(), stopChars) { - return false - } - return true -} - -// Convert versions to a string and append the string to the string base. -// -// Each VersionInfo will be converted to a string in the format of -// "product/version", where the "product" is get from the Name() method, while -// version is get from the Version() method. Several pieces of verson information -// will be concatinated and separated by space. -func appendVersions(base string, versions ...VersionInfo) string { - if len(versions) == 0 { - return base - } - - var buf bytes.Buffer - if len(base) > 0 { - buf.Write([]byte(base)) - } - - for _, v := range versions { - if !validVersion(v) { - continue - } - buf.Write([]byte(v.Name())) - buf.Write([]byte("/")) - buf.Write([]byte(v.Version())) - buf.Write([]byte(" ")) - } - return buf.String() -} - -func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...VersionInfo) (r *Registry, err error) { +func NewRegistry(root string, authConfig *auth.AuthConfig, factory *utils.HTTPRequestFactory) (r *Registry, err error) { httpTransport := &http.Transport{ DisableKeepAlives: true, Proxy: http.ProxyFromEnvironment, @@ -628,7 +638,7 @@ func NewRegistry(root string, authConfig *auth.AuthConfig, baseVersions ...Versi if err != nil { return nil, err } - r.baseVersions = baseVersions - r.baseVersionsStr = appendVersions("", baseVersions...) + + r.reqFactory = factory return r, nil } diff --git a/registry/registry_mock_test.go b/registry/registry_mock_test.go new file mode 100644 index 0000000000..e752315510 --- /dev/null +++ b/registry/registry_mock_test.go @@ -0,0 +1,346 @@ +package registry + +import ( + "encoding/json" + "fmt" + "github.com/dotcloud/docker/utils" + "github.com/gorilla/mux" + "io" + "io/ioutil" + "net/http" + "net/http/httptest" + "net/url" + "strconv" + "strings" + "testing" + "time" +) + +var ( + testHttpServer *httptest.Server + testLayers = map[string]map[string]string{ + "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20": { + "json": `{"id":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + "comment":"test base image","created":"2013-03-23T12:53:11.10432-07:00", + "container_config":{"Hostname":"","User":"","Memory":0,"MemorySwap":0, + "CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false, + "PortSpecs":null,"Tty":false,"OpenStdin":false,"StdinOnce":false, + "Env":null,"Cmd":null,"Dns":null,"Image":"","Volumes":null, + "VolumesFrom":"","Entrypoint":null},"Size":424242}`, + "checksum_simple": "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", + "checksum_tarsum": "tarsum+sha256:4409a0685741ca86d38df878ed6f8cbba4c99de5dc73cd71aef04be3bb70be7c", + "ancestry": `["77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20"]`, + "layer": string([]byte{ + 0x1f, 0x8b, 0x08, 0x08, 0x0e, 0xb0, 0xee, 0x51, 0x02, 0x03, 0x6c, 0x61, 0x79, 0x65, + 0x72, 0x2e, 0x74, 0x61, 0x72, 0x00, 0xed, 0xd2, 0x31, 0x0e, 0xc2, 0x30, 0x0c, 0x05, + 0x50, 0xcf, 0x9c, 0xc2, 0x27, 0x48, 0xed, 0x38, 0x4e, 0xce, 0x13, 0x44, 0x2b, 0x66, + 0x62, 0x24, 0x8e, 0x4f, 0xa0, 0x15, 0x63, 0xb6, 0x20, 0x21, 0xfc, 0x96, 0xbf, 0x78, + 0xb0, 0xf5, 0x1d, 0x16, 0x98, 0x8e, 0x88, 0x8a, 0x2a, 0xbe, 0x33, 0xef, 0x49, 0x31, + 0xed, 0x79, 0x40, 0x8e, 0x5c, 0x44, 0x85, 0x88, 0x33, 0x12, 0x73, 0x2c, 0x02, 0xa8, + 0xf0, 0x05, 0xf7, 0x66, 0xf5, 0xd6, 0x57, 0x69, 0xd7, 0x7a, 0x19, 0xcd, 0xf5, 0xb1, + 0x6d, 0x1b, 0x1f, 0xf9, 0xba, 0xe3, 0x93, 0x3f, 0x22, 0x2c, 0xb6, 0x36, 0x0b, 0xf6, + 0xb0, 0xa9, 0xfd, 0xe7, 0x94, 0x46, 0xfd, 0xeb, 0xd1, 0x7f, 0x2c, 0xc4, 0xd2, 0xfb, + 0x97, 0xfe, 0x02, 0x80, 0xe4, 0xfd, 0x4f, 0x77, 0xae, 0x6d, 0x3d, 0x81, 0x73, 0xce, + 0xb9, 0x7f, 0xf3, 0x04, 0x41, 0xc1, 0xab, 0xc6, 0x00, 0x0a, 0x00, 0x00, + }), + }, + "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d": { + "json": `{"id":"42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + "parent":"77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + "comment":"test base image","created":"2013-03-23T12:55:11.10432-07:00", + "container_config":{"Hostname":"","User":"","Memory":0,"MemorySwap":0, + "CpuShares":0,"AttachStdin":false,"AttachStdout":false,"AttachStderr":false, + "PortSpecs":null,"Tty":false,"OpenStdin":false,"StdinOnce":false, + "Env":null,"Cmd":null,"Dns":null,"Image":"","Volumes":null, + "VolumesFrom":"","Entrypoint":null},"Size":424242}`, + "checksum_simple": "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", + "checksum_tarsum": "tarsum+sha256:68fdb56fb364f074eec2c9b3f85ca175329c4dcabc4a6a452b7272aa613a07a2", + "ancestry": `["42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20"]`, + "layer": string([]byte{ + 0x1f, 0x8b, 0x08, 0x08, 0xbd, 0xb3, 0xee, 0x51, 0x02, 0x03, 0x6c, 0x61, 0x79, 0x65, + 0x72, 0x2e, 0x74, 0x61, 0x72, 0x00, 0xed, 0xd1, 0x31, 0x0e, 0xc2, 0x30, 0x0c, 0x05, + 0x50, 0xcf, 0x9c, 0xc2, 0x27, 0x48, 0x9d, 0x38, 0x8e, 0xcf, 0x53, 0x51, 0xaa, 0x56, + 0xea, 0x44, 0x82, 0xc4, 0xf1, 0x09, 0xb4, 0xea, 0x98, 0x2d, 0x48, 0x08, 0xbf, 0xe5, + 0x2f, 0x1e, 0xfc, 0xf5, 0xdd, 0x00, 0xdd, 0x11, 0x91, 0x8a, 0xe0, 0x27, 0xd3, 0x9e, + 0x14, 0xe2, 0x9e, 0x07, 0xf4, 0xc1, 0x2b, 0x0b, 0xfb, 0xa4, 0x82, 0xe4, 0x3d, 0x93, + 0x02, 0x0a, 0x7c, 0xc1, 0x23, 0x97, 0xf1, 0x5e, 0x5f, 0xc9, 0xcb, 0x38, 0xb5, 0xee, + 0xea, 0xd9, 0x3c, 0xb7, 0x4b, 0xbe, 0x7b, 0x9c, 0xf9, 0x23, 0xdc, 0x50, 0x6e, 0xb9, + 0xb8, 0xf2, 0x2c, 0x5d, 0xf7, 0x4f, 0x31, 0xb6, 0xf6, 0x4f, 0xc7, 0xfe, 0x41, 0x55, + 0x63, 0xdd, 0x9f, 0x89, 0x09, 0x90, 0x6c, 0xff, 0xee, 0xae, 0xcb, 0xba, 0x4d, 0x17, + 0x30, 0xc6, 0x18, 0xf3, 0x67, 0x5e, 0xc1, 0xed, 0x21, 0x5d, 0x00, 0x0a, 0x00, 0x00, + }), + }, + } + testRepositories = map[string]map[string]string{ + "foo42/bar": { + "latest": "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + }, + } +) + +func init() { + r := mux.NewRouter() + r.HandleFunc("/v1/_ping", handlerGetPing).Methods("GET") + r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|ancestry}", handlerGetImage).Methods("GET") + r.HandleFunc("/v1/images/{image_id:[^/]+}/{action:json|layer|checksum}", handlerPutImage).Methods("PUT") + r.HandleFunc("/v1/repositories/{repository:.+}/tags", handlerGetDeleteTags).Methods("GET", "DELETE") + r.HandleFunc("/v1/repositories/{repository:.+}/tags/{tag:.+}", handlerGetTag).Methods("GET") + r.HandleFunc("/v1/repositories/{repository:.+}/tags/{tag:.+}", handlerPutTag).Methods("PUT") + r.HandleFunc("/v1/users{null:.*}", handlerUsers).Methods("GET", "POST", "PUT") + r.HandleFunc("/v1/repositories/{repository:.+}{action:/images|/}", handlerImages).Methods("GET", "PUT", "DELETE") + r.HandleFunc("/v1/repositories/{repository:.+}/auth", handlerAuth).Methods("PUT") + r.HandleFunc("/v1/search", handlerSearch).Methods("GET") + testHttpServer = httptest.NewServer(handlerAccessLog(r)) +} + +func handlerAccessLog(handler http.Handler) http.Handler { + logHandler := func(w http.ResponseWriter, r *http.Request) { + utils.Debugf("%s \"%s %s\"", r.RemoteAddr, r.Method, r.URL) + handler.ServeHTTP(w, r) + } + return http.HandlerFunc(logHandler) +} + +func makeURL(req string) string { + return testHttpServer.URL + req +} + +func writeHeaders(w http.ResponseWriter) { + h := w.Header() + h.Add("Server", "docker-tests/mock") + h.Add("Expires", "-1") + h.Add("Content-Type", "application/json") + h.Add("Pragma", "no-cache") + h.Add("Cache-Control", "no-cache") + h.Add("X-Docker-Registry-Version", "0.0.0") + h.Add("X-Docker-Registry-Config", "mock") +} + +func writeResponse(w http.ResponseWriter, message interface{}, code int) { + writeHeaders(w) + w.WriteHeader(code) + body, err := json.Marshal(message) + if err != nil { + io.WriteString(w, err.Error()) + return + } + w.Write(body) +} + +func readJSON(r *http.Request, dest interface{}) error { + body, err := ioutil.ReadAll(r.Body) + if err != nil { + return err + } + return json.Unmarshal(body, dest) +} + +func apiError(w http.ResponseWriter, message string, code int) { + body := map[string]string{ + "error": message, + } + writeResponse(w, body, code) +} + +func assertEqual(t *testing.T, a interface{}, b interface{}, message string) { + if a == b { + return + } + if len(message) == 0 { + message = fmt.Sprintf("%v != %v", a, b) + } + t.Fatal(message) +} + +func requiresAuth(w http.ResponseWriter, r *http.Request) bool { + writeCookie := func() { + value := fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano()) + cookie := &http.Cookie{Name: "session", Value: value, MaxAge: 3600} + http.SetCookie(w, cookie) + //FIXME(sam): this should be sent only on Index routes + value = fmt.Sprintf("FAKE-TOKEN-%d", time.Now().UnixNano()) + w.Header().Add("X-Docker-Token", value) + } + if len(r.Cookies()) > 0 { + writeCookie() + return true + } + if len(r.Header.Get("Authorization")) > 0 { + writeCookie() + return true + } + w.Header().Add("WWW-Authenticate", "token") + apiError(w, "Wrong auth", 401) + return false +} + +func handlerGetPing(w http.ResponseWriter, r *http.Request) { + writeResponse(w, true, 200) +} + +func handlerGetImage(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + layer, exists := testLayers[vars["image_id"]] + if !exists { + http.NotFound(w, r) + return + } + writeHeaders(w) + layer_size := len(layer["layer"]) + w.Header().Add("X-Docker-Size", strconv.Itoa(layer_size)) + io.WriteString(w, layer[vars["action"]]) +} + +func handlerPutImage(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + image_id := vars["image_id"] + action := vars["action"] + layer, exists := testLayers[image_id] + if !exists { + if action != "json" { + http.NotFound(w, r) + return + } + layer = make(map[string]string) + testLayers[image_id] = layer + } + if checksum := r.Header.Get("X-Docker-Checksum"); checksum != "" { + if checksum != layer["checksum_simple"] && checksum != layer["checksum_tarsum"] { + apiError(w, "Wrong checksum", 400) + return + } + } + body, err := ioutil.ReadAll(r.Body) + if err != nil { + apiError(w, fmt.Sprintf("Error: %s", err), 500) + return + } + layer[action] = string(body) + writeResponse(w, true, 200) +} + +func handlerGetDeleteTags(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + repositoryName := mux.Vars(r)["repository"] + tags, exists := testRepositories[repositoryName] + if !exists { + apiError(w, "Repository not found", 404) + } + if r.Method == "DELETE" { + delete(testRepositories, repositoryName) + writeResponse(w, true, 200) + return + } + writeResponse(w, tags, 200) +} + +func handlerGetTag(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + repositoryName := vars["repository"] + tagName := vars["tag"] + tags, exists := testRepositories[repositoryName] + if !exists { + apiError(w, "Repository not found", 404) + } + tag, exists := tags[tagName] + if !exists { + apiError(w, "Tag not found", 404) + } + writeResponse(w, tag, 200) +} + +func handlerPutTag(w http.ResponseWriter, r *http.Request) { + if !requiresAuth(w, r) { + return + } + vars := mux.Vars(r) + repositoryName := vars["repository"] + tagName := vars["tag"] + tags, exists := testRepositories[repositoryName] + if !exists { + tags := make(map[string]string) + testRepositories[repositoryName] = tags + } + tagValue := "" + readJSON(r, tagValue) + tags[tagName] = tagValue + writeResponse(w, true, 200) +} + +func handlerUsers(w http.ResponseWriter, r *http.Request) { + code := 200 + if r.Method == "POST" { + code = 201 + } else if r.Method == "PUT" { + code = 204 + } + writeResponse(w, "", code) +} + +func handlerImages(w http.ResponseWriter, r *http.Request) { + u, _ := url.Parse(testHttpServer.URL) + w.Header().Add("X-Docker-Endpoints", u.Host) + w.Header().Add("X-Docker-Token", fmt.Sprintf("FAKE-SESSION-%d", time.Now().UnixNano())) + if r.Method == "PUT" { + if strings.HasSuffix(r.URL.Path, "images") { + writeResponse(w, "", 204) + return + } + writeResponse(w, "", 200) + return + } + if r.Method == "DELETE" { + writeResponse(w, "", 204) + return + } + images := []map[string]string{} + for image_id, layer := range testLayers { + image := make(map[string]string) + image["id"] = image_id + image["checksum"] = layer["checksum_tarsum"] + image["Tag"] = "latest" + images = append(images, image) + } + writeResponse(w, images, 200) +} + +func handlerAuth(w http.ResponseWriter, r *http.Request) { + writeResponse(w, "OK", 200) +} + +func handlerSearch(w http.ResponseWriter, r *http.Request) { + writeResponse(w, "{}", 200) +} + +func TestPing(t *testing.T) { + res, err := http.Get(makeURL("/v1/_ping")) + if err != nil { + t.Fatal(err) + } + assertEqual(t, res.StatusCode, 200, "") + assertEqual(t, res.Header.Get("X-Docker-Registry-Config"), "mock", + "This is not a Mocked Registry") +} + +/* Uncomment this to test Mocked Registry locally with curl + * WARNING: Don't push on the repos uncommented, it'll block the tests + * +func TestWait(t *testing.T) { + log.Println("Test HTTP server ready and waiting:", testHttpServer.URL) + c := make(chan int) + <-c +} + +//*/ \ No newline at end of file diff --git a/registry/registry_test.go b/registry/registry_test.go index fd955b7b73..a8543f186d 100644 --- a/registry/registry_test.go +++ b/registry/registry_test.go @@ -1,168 +1,198 @@ package registry -// import ( -// "crypto/rand" -// "encoding/hex" -// "github.com/dotcloud/docker" -// "github.com/dotcloud/docker/auth" -// "io/ioutil" -// "os" -// "path" -// "testing" -// ) +import ( + "github.com/dotcloud/docker/auth" + "github.com/dotcloud/docker/utils" + "strings" + "testing" +) -// func newTestRuntime() (*Runtime, error) { -// root, err := ioutil.TempDir("", "docker-test") -// if err != nil { -// return nil, err -// } -// if err := os.Remove(root); err != nil { -// return nil, err -// } +var ( + IMAGE_ID = "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d" + TOKEN = []string{"fake-token"} + REPO = "foo42/bar" +) -// if err := os.MkdirAll(root, 0700); err != nil && !os.IsExist(err) { -// return nil, err -// } +func spawnTestRegistry(t *testing.T) *Registry { + authConfig := &auth.AuthConfig{} + r, err := NewRegistry("", authConfig, utils.NewHTTPRequestFactory()) + if err != nil { + t.Fatal(err) + } + return r +} -// return runtime, nil -// } +func TestPingRegistryEndpoint(t *testing.T) { + err := pingRegistryEndpoint(makeURL("/v1/")) + if err != nil { + t.Fatal(err) + } +} -// func TestPull(t *testing.T) { -// os.Setenv("DOCKER_INDEX_URL", "") -// runtime, err := newTestRuntime() -// if err != nil { -// t.Fatal(err) -// } -// defer nuke(runtime) +func TestGetRemoteHistory(t *testing.T) { + r := spawnTestRegistry(t) + hist, err := r.GetRemoteHistory(IMAGE_ID, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } + assertEqual(t, len(hist), 2, "Expected 2 images in history") + assertEqual(t, hist[0], IMAGE_ID, "Expected "+IMAGE_ID+"as first ancestry") + assertEqual(t, hist[1], "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + "Unexpected second ancestry") +} -// err = runtime.graph.PullRepository(ioutil.Discard, "busybox", "", runtime.repositories, nil) -// if err != nil { -// t.Fatal(err) -// } -// img, err := runtime.repositories.LookupImage("busybox") -// if err != nil { -// t.Fatal(err) -// } +func TestLookupRemoteImage(t *testing.T) { + r := spawnTestRegistry(t) + found := r.LookupRemoteImage(IMAGE_ID, makeURL("/v1/"), TOKEN) + assertEqual(t, found, true, "Expected remote lookup to succeed") + found = r.LookupRemoteImage("abcdef", makeURL("/v1/"), TOKEN) + assertEqual(t, found, false, "Expected remote lookup to fail") +} -// // Try to run something on this image to make sure the layer's been downloaded properly. -// config, _, err := docker.ParseRun([]string{img.Id, "echo", "Hello World"}, runtime.capabilities) -// if err != nil { -// t.Fatal(err) -// } +func TestGetRemoteImageJSON(t *testing.T) { + r := spawnTestRegistry(t) + json, size, err := r.GetRemoteImageJSON(IMAGE_ID, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } + assertEqual(t, size, 154, "Expected size 154") + if len(json) <= 0 { + t.Fatal("Expected non-empty json") + } -// b := NewBuilder(runtime) -// container, err := b.Create(config) -// if err != nil { -// t.Fatal(err) -// } -// if err := container.Start(); err != nil { -// t.Fatal(err) -// } + _, _, err = r.GetRemoteImageJSON("abcdef", makeURL("/v1/"), TOKEN) + if err == nil { + t.Fatal("Expected image not found error") + } +} -// if status := container.Wait(); status != 0 { -// t.Fatalf("Expected status code 0, found %d instead", status) -// } -// } +func TestGetRemoteImageLayer(t *testing.T) { + r := spawnTestRegistry(t) + data, err := r.GetRemoteImageLayer(IMAGE_ID, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } + if data == nil { + t.Fatal("Expected non-nil data result") + } -// func TestPullTag(t *testing.T) { -// os.Setenv("DOCKER_INDEX_URL", "") -// runtime, err := newTestRuntime() -// if err != nil { -// t.Fatal(err) -// } -// defer nuke(runtime) + _, err = r.GetRemoteImageLayer("abcdef", makeURL("/v1/"), TOKEN) + if err == nil { + t.Fatal("Expected image not found error") + } +} -// err = runtime.graph.PullRepository(ioutil.Discard, "ubuntu", "12.04", runtime.repositories, nil) -// if err != nil { -// t.Fatal(err) -// } -// _, err = runtime.repositories.LookupImage("ubuntu:12.04") -// if err != nil { -// t.Fatal(err) -// } +func TestGetRemoteTags(t *testing.T) { + r := spawnTestRegistry(t) + tags, err := r.GetRemoteTags([]string{makeURL("/v1/")}, REPO, TOKEN) + if err != nil { + t.Fatal(err) + } + assertEqual(t, len(tags), 1, "Expected one tag") + assertEqual(t, tags["latest"], IMAGE_ID, "Expected tag latest to map to "+IMAGE_ID) -// img2, err := runtime.repositories.LookupImage("ubuntu:12.10") -// if img2 != nil { -// t.Fatalf("Expected nil image but found %v instead", img2.Id) -// } -// } + _, err = r.GetRemoteTags([]string{makeURL("/v1/")}, "foo42/baz", TOKEN) + if err == nil { + t.Fatal("Expected error when fetching tags for bogus repo") + } +} -// func login(runtime *Runtime) error { -// authConfig := auth.NewAuthConfig("unittester", "surlautrerivejetattendrai", "noise+unittester@dotcloud.com", runtime.root) -// runtime.authConfig = authConfig -// _, err := auth.Login(authConfig) -// return err -// } +func TestGetRepositoryData(t *testing.T) { + r := spawnTestRegistry(t) + data, err := r.GetRepositoryData(makeURL("/v1/"), "foo42/bar") + if err != nil { + t.Fatal(err) + } + assertEqual(t, len(data.ImgList), 2, "Expected 2 images in ImgList") + assertEqual(t, len(data.Endpoints), 1, "Expected one endpoint in Endpoints") +} -// func TestPush(t *testing.T) { -// os.Setenv("DOCKER_INDEX_URL", "https://indexstaging-docker.dotcloud.com") -// defer os.Setenv("DOCKER_INDEX_URL", "") -// runtime, err := newTestRuntime() -// if err != nil { -// t.Fatal(err) -// } -// defer nuke(runtime) +func TestPushImageJSONRegistry(t *testing.T) { + r := spawnTestRegistry(t) + imgData := &ImgData{ + ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", + } -// err = login(runtime) -// if err != nil { -// t.Fatal(err) -// } + err := r.PushImageJSONRegistry(imgData, []byte{0x42, 0xdf, 0x0}, makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } +} -// err = runtime.graph.PullRepository(ioutil.Discard, "joffrey/busybox", "", runtime.repositories, nil) -// if err != nil { -// t.Fatal(err) -// } -// tokenBuffer := make([]byte, 16) -// _, err = rand.Read(tokenBuffer) -// if err != nil { -// t.Fatal(err) -// } -// token := hex.EncodeToString(tokenBuffer)[:29] -// config, _, err := ParseRun([]string{"joffrey/busybox", "touch", "/" + token}, runtime.capabilities) -// if err != nil { -// t.Fatal(err) -// } +func TestPushImageLayerRegistry(t *testing.T) { + r := spawnTestRegistry(t) + layer := strings.NewReader("") + _, err := r.PushImageLayerRegistry(IMAGE_ID, layer, makeURL("/v1/"), TOKEN, []byte{}) + if err != nil { + t.Fatal(err) + } +} -// b := NewBuilder(runtime) -// container, err := b.Create(config) -// if err != nil { -// t.Fatal(err) -// } -// if err := container.Start(); err != nil { -// t.Fatal(err) -// } +func TestResolveRepositoryName(t *testing.T) { + _, _, err := ResolveRepositoryName("https://github.com/dotcloud/docker") + assertEqual(t, err, ErrInvalidRepositoryName, "Expected error invalid repo name") + ep, repo, err := ResolveRepositoryName("fooo/bar") + if err != nil { + t.Fatal(err) + } + assertEqual(t, ep, auth.IndexServerAddress(), "Expected endpoint to be index server address") + assertEqual(t, repo, "fooo/bar", "Expected resolved repo to be foo/bar") -// if status := container.Wait(); status != 0 { -// t.Fatalf("Expected status code 0, found %d instead", status) -// } + u := makeURL("")[7:] + ep, repo, err = ResolveRepositoryName(u + "/private/moonbase") + if err != nil { + t.Fatal(err) + } + assertEqual(t, ep, "http://"+u+"/v1/", "Expected endpoint to be "+u) + assertEqual(t, repo, "private/moonbase", "Expected endpoint to be private/moonbase") +} -// img, err := b.Commit(container, "unittester/"+token, "", "", "", nil) -// if err != nil { -// t.Fatal(err) -// } +func TestPushRegistryTag(t *testing.T) { + r := spawnTestRegistry(t) + err := r.PushRegistryTag("foo42/bar", IMAGE_ID, "stable", makeURL("/v1/"), TOKEN) + if err != nil { + t.Fatal(err) + } +} -// repo := runtime.repositories.Repositories["unittester/"+token] -// err = runtime.graph.PushRepository(ioutil.Discard, "unittester/"+token, repo, runtime.authConfig) -// if err != nil { -// t.Fatal(err) -// } +func TestPushImageJSONIndex(t *testing.T) { + r := spawnTestRegistry(t) + imgData := []*ImgData{ + &ImgData{ + ID: "77dbf71da1d00e3fbddc480176eac8994025630c6590d11cfc8fe1209c2a1d20", + Checksum: "sha256:1ac330d56e05eef6d438586545ceff7550d3bdcb6b19961f12c5ba714ee1bb37", + }, + &ImgData{ + ID: "42d718c941f5c532ac049bf0b0ab53f0062f09a03afd4aa4a02c098e46032b9d", + Checksum: "sha256:bea7bf2e4bacd479344b737328db47b18880d09096e6674165533aa994f5e9f2", + }, + } + ep := makeURL("/v1/") + repoData, err := r.PushImageJSONIndex(ep, "foo42/bar", imgData, false, nil) + if err != nil { + t.Fatal(err) + } + if repoData == nil { + t.Fatal("Expected RepositoryData object") + } + repoData, err = r.PushImageJSONIndex(ep, "foo42/bar", imgData, true, []string{ep}) + if err != nil { + t.Fatal(err) + } + if repoData == nil { + t.Fatal("Expected RepositoryData object") + } +} -// // Remove image so we can pull it again -// if err := runtime.graph.Delete(img.Id); err != nil { -// t.Fatal(err) -// } - -// err = runtime.graph.PullRepository(ioutil.Discard, "unittester/"+token, "", runtime.repositories, runtime.authConfig) -// if err != nil { -// t.Fatal(err) -// } - -// layerPath, err := img.layer() -// if err != nil { -// t.Fatal(err) -// } - -// if _, err := os.Stat(path.Join(layerPath, token)); err != nil { -// t.Fatalf("Error while trying to retrieve token file: %v", err) -// } -// } +func TestSearchRepositories(t *testing.T) { + r := spawnTestRegistry(t) + results, err := r.SearchRepositories("supercalifragilisticepsialidocious") + if err != nil { + t.Fatal(err) + } + if results == nil { + t.Fatal("Expected non-nil SearchResults object") + } + assertEqual(t, results.NumResults, 0, "Expected 0 search results") +} diff --git a/runtime.go b/runtime.go index f4c5b4d380..894028354e 100644 --- a/runtime.go +++ b/runtime.go @@ -15,8 +15,9 @@ import ( ) type Capabilities struct { - MemoryLimit bool - SwapLimit bool + MemoryLimit bool + SwapLimit bool + IPv4Forwarding bool } type Runtime struct { @@ -240,6 +241,12 @@ func (runtime *Runtime) UpdateCapabilities(quiet bool) { if !runtime.capabilities.SwapLimit && !quiet { log.Printf("WARNING: Your kernel does not support cgroup swap limit.") } + + content, err3 := ioutil.ReadFile("/proc/sys/net/ipv4/ip_forward") + runtime.capabilities.IPv4Forwarding = err3 == nil && len(content) > 0 && content[0] == '1' + if !runtime.capabilities.IPv4Forwarding && !quiet { + log.Printf("WARNING: IPv4 forwarding is disabled.") + } } } diff --git a/runtime_test.go b/runtime_test.go index 807097404d..8819df2221 100644 --- a/runtime_test.go +++ b/runtime_test.go @@ -8,6 +8,7 @@ import ( "log" "net" "os" + "runtime" "strconv" "strings" "sync" @@ -25,7 +26,11 @@ const ( testDaemonProto = "tcp" ) -var globalRuntime *Runtime +var ( + globalRuntime *Runtime + startFds int + startGoroutines int +) func nuke(runtime *Runtime) error { var wg sync.WaitGroup @@ -68,7 +73,7 @@ func layerArchive(tarfile string) (io.Reader, error) { func init() { // Hack to run sys init during unit testing - if utils.SelfPath() == "/sbin/init" { + if selfPath := utils.SelfPath(); selfPath == "/sbin/init" || selfPath == "/.dockerinit" { SysInit() return } @@ -80,23 +85,23 @@ func init() { NetworkBridgeIface = unitTestNetworkBridge // Make it our Store root - runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false) - if err != nil { + if runtime, err := NewRuntimeFromDirectory(unitTestStoreBase, false); err != nil { panic(err) + } else { + globalRuntime = runtime } - globalRuntime = runtime // Create the "Server" srv := &Server{ - runtime: runtime, + runtime: globalRuntime, enableCors: false, pullingPool: make(map[string]struct{}), pushingPool: make(map[string]struct{}), } // If the unit test is not found, try to download it. - if img, err := runtime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID { + if img, err := globalRuntime.repositories.LookupImage(unitTestImageName); err != nil || img.ID != unitTestImageID { // Retrieve the Image - if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil); err != nil { + if err := srv.ImagePull(unitTestImageName, "", os.Stdout, utils.NewStreamFormatter(false), nil, true); err != nil { panic(err) } } @@ -109,6 +114,8 @@ func init() { // Give some time to ListenAndServer to actually start time.Sleep(time.Second) + + startFds, startGoroutines = utils.GetTotalUsedFds(), runtime.NumGoroutine() } // FIXME: test that ImagePull(json=true) send correct json output diff --git a/server.go b/server.go index d9399904dd..8b15ecf1b0 100644 --- a/server.go +++ b/server.go @@ -52,9 +52,9 @@ func (v *simpleVersionInfo) Version() string { // docker, go, git-commit (of the docker) and the host's kernel. // // Such information will be used on call to NewRegistry(). -func (srv *Server) versionInfos() []registry.VersionInfo { +func (srv *Server) versionInfos() []utils.VersionInfo { v := srv.DockerVersion() - ret := make([]registry.VersionInfo, 0, 4) + ret := make([]utils.VersionInfo, 0, 4) ret = append(ret, &simpleVersionInfo{"docker", v.Version}) if len(v.GoVersion) > 0 { @@ -102,7 +102,7 @@ func (srv *Server) ContainerExport(name string, out io.Writer) error { } func (srv *Server) ImagesSearch(term string) ([]APISearch, error) { - r, err := registry.NewRegistry(srv.runtime.root, nil, srv.versionInfos()...) + r, err := registry.NewRegistry(srv.runtime.root, nil, srv.HTTPRequestFactory()) if err != nil { return nil, err } @@ -145,7 +145,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. return "", err } - if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), path); err != nil { + if err := c.Inject(utils.ProgressReader(file.Body, int(file.ContentLength), out, sf.FormatProgress("", "Downloading", "%8v/%v (%v)"), sf, true), path); err != nil { return "", err } // FIXME: Handle custom repo, tag comment, author @@ -153,7 +153,7 @@ func (srv *Server) ImageInsert(name, url, path string, out io.Writer, sf *utils. if err != nil { return "", err } - out.Write(sf.FormatStatus(img.ID)) + out.Write(sf.FormatStatus("", img.ID)) return img.ShortID(), nil } @@ -252,15 +252,31 @@ func (srv *Server) DockerInfo() *APIInfo { } else { imgcount = len(images) } + lxcVersion := "" + if output, err := exec.Command("lxc-version").CombinedOutput(); err == nil { + outputStr := string(output) + if len(strings.SplitN(outputStr, ":", 2)) == 2 { + lxcVersion = strings.TrimSpace(strings.SplitN(string(output), ":", 2)[1]) + } + } + kernelVersion := "" + if kv, err := utils.GetKernelVersion(); err == nil { + kernelVersion = kv.String() + } + return &APIInfo{ - Containers: len(srv.runtime.List()), - Images: imgcount, - MemoryLimit: srv.runtime.capabilities.MemoryLimit, - SwapLimit: srv.runtime.capabilities.SwapLimit, - Debug: os.Getenv("DEBUG") != "", - NFd: utils.GetTotalUsedFds(), - NGoroutines: runtime.NumGoroutine(), - NEventsListener: len(srv.events), + Containers: len(srv.runtime.List()), + Images: imgcount, + MemoryLimit: srv.runtime.capabilities.MemoryLimit, + SwapLimit: srv.runtime.capabilities.SwapLimit, + IPv4Forwarding: srv.runtime.capabilities.IPv4Forwarding, + Debug: os.Getenv("DEBUG") != "", + NFd: utils.GetTotalUsedFds(), + NGoroutines: runtime.NumGoroutine(), + LXCVersion: lxcVersion, + NEventsListener: len(srv.events), + KernelVersion: kernelVersion, + IndexServerAddress: auth.IndexServerAddress(), } } @@ -295,35 +311,34 @@ func (srv *Server) ImageHistory(name string) ([]APIHistory, error) { } -func (srv *Server) ContainerTop(name string) ([]APITop, error) { +func (srv *Server) ContainerTop(name, ps_args string) (*APITop, error) { if container := srv.runtime.Get(name); container != nil { - output, err := exec.Command("lxc-ps", "--name", container.ID).CombinedOutput() + output, err := exec.Command("lxc-ps", "--name", container.ID, "--", ps_args).CombinedOutput() if err != nil { return nil, fmt.Errorf("Error trying to use lxc-ps: %s (%s)", err, output) } - var procs []APITop + procs := APITop{} for i, line := range strings.Split(string(output), "\n") { - if i == 0 || len(line) == 0 { + if len(line) == 0 { continue } - proc := APITop{} + words := []string{} scanner := bufio.NewScanner(strings.NewReader(line)) scanner.Split(bufio.ScanWords) if !scanner.Scan() { return nil, fmt.Errorf("Error trying to use lxc-ps") } // no scanner.Text because we skip container id - scanner.Scan() - proc.PID = scanner.Text() - scanner.Scan() - proc.Tty = scanner.Text() - scanner.Scan() - proc.Time = scanner.Text() - scanner.Scan() - proc.Cmd = scanner.Text() - procs = append(procs, proc) + for scanner.Scan() { + words = append(words, scanner.Text()) + } + if i == 0 { + procs.Titles = words + } else { + procs.Processes = append(procs.Processes, words) + } } - return procs, nil + return &procs, nil } return nil, fmt.Errorf("No such container: %s", name) @@ -407,7 +422,7 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin // FIXME: Launch the getRemoteImage() in goroutines for _, id := range history { if !srv.runtime.graph.Exists(id) { - out.Write(sf.FormatStatus("Pulling %s metadata", id)) + out.Write(sf.FormatStatus(utils.TruncateID(id), "Pulling metadata")) imgJSON, imgSize, err := r.GetRemoteImageJSON(id, endpoint, token) if err != nil { // FIXME: Keep goging in case of error? @@ -419,13 +434,13 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin } // Get the layer - out.Write(sf.FormatStatus("Pulling %s fs layer", id)) + out.Write(sf.FormatStatus(utils.TruncateID(id), "Pulling fs layer")) layer, err := r.GetRemoteImageLayer(img.ID, endpoint, token) if err != nil { return err } defer layer.Close() - if err := srv.runtime.graph.Register(utils.ProgressReader(layer, imgSize, out, sf.FormatProgress("Downloading", "%8v/%v (%v)"), sf), false, img); err != nil { + if err := srv.runtime.graph.Register(imgJSON, utils.ProgressReader(layer, imgSize, out, sf.FormatProgress(utils.TruncateID(id), "Downloading", "%8v/%v (%v)"), sf, false), img); err != nil { return err } } @@ -433,20 +448,14 @@ func (srv *Server) pullImage(r *registry.Registry, out io.Writer, imgID, endpoin return nil } -func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter) error { - out.Write(sf.FormatStatus("Pulling repository %s", localName)) +func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName, remoteName, askedTag, indexEp string, sf *utils.StreamFormatter, parallel bool) error { + out.Write(sf.FormatStatus("", "Pulling repository %s", localName)) repoData, err := r.GetRepositoryData(indexEp, remoteName) if err != nil { return err } - utils.Debugf("Updating checksums") - // Reload the json file to make sure not to overwrite faster sums - if err := srv.runtime.graph.UpdateChecksums(repoData.ImgList); err != nil { - return err - } - utils.Debugf("Retrieving the tag list") tagsList, err := r.GetRemoteTags(repoData.Endpoints, remoteName, repoData.Tokens) if err != nil { @@ -477,30 +486,51 @@ func (srv *Server) pullRepository(r *registry.Registry, out io.Writer, localName repoData.ImgList[id].Tag = askedTag } - for _, img := range repoData.ImgList { - if askedTag != "" && img.Tag != askedTag { - utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID) - continue + errors := make(chan error) + for _, image := range repoData.ImgList { + downloadImage := func(img *registry.ImgData) { + if askedTag != "" && img.Tag != askedTag { + utils.Debugf("(%s) does not match %s (id: %s), skipping", img.Tag, askedTag, img.ID) + errors <- nil + return + } + + if img.Tag == "" { + utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) + errors <- nil + return + } + out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Pulling image (%s) from %s", img.Tag, localName)) + success := false + for _, ep := range repoData.Endpoints { + if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { + out.Write(sf.FormatStatus(utils.TruncateID(img.ID), "Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) + continue + } + success = true + break + } + if !success { + errors <- fmt.Errorf("Could not find repository on any of the indexed registries.") + } + errors <- nil } - if img.Tag == "" { - utils.Debugf("Image (id: %s) present in this repository but untagged, skipping", img.ID) - continue - } - out.Write(sf.FormatStatus("Pulling image %s (%s) from %s", img.ID, img.Tag, localName)) - success := false - for _, ep := range repoData.Endpoints { - if err := srv.pullImage(r, out, img.ID, ep, repoData.Tokens, sf); err != nil { - out.Write(sf.FormatStatus("Error while retrieving image for tag: %s (%s); checking next endpoint", askedTag, err)) - continue - } - success = true - break - } - if !success { - return fmt.Errorf("Could not find repository on any of the indexed registries.") + if parallel { + go downloadImage(image) + } else { + downloadImage(image) } } + + if parallel { + for i := 0; i < len(repoData.ImgList); i++ { + if err := <-errors; err != nil { + return err + } + } + } + for tag, id := range tagsList { if askedTag != "" && tag != askedTag { continue @@ -554,8 +584,8 @@ func (srv *Server) poolRemove(kind, key string) error { return nil } -func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig) error { - r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.versionInfos()...) +func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *utils.StreamFormatter, authConfig *auth.AuthConfig, parallel bool) error { + r, err := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory()) if err != nil { return err } @@ -576,7 +606,7 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut } out = utils.NewWriteFlusher(out) - err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf) + err = srv.pullRepository(r, out, localName, remoteName, tag, endpoint, sf, parallel) if err != nil { if err := srv.pullImage(r, out, remoteName, endpoint, nil, sf); err != nil { return err @@ -587,41 +617,6 @@ func (srv *Server) ImagePull(localName string, tag string, out io.Writer, sf *ut return nil } -// Retrieve the checksum of an image -// Priority: -// - Check on the stored checksums -// - Check if the archive exists, if it does not, ask the registry -// - If the archive does exists, process the checksum from it -// - If the archive does not exists and not found on registry, process checksum from layer -func (srv *Server) getChecksum(imageID string) (string, error) { - // FIXME: Use in-memory map instead of reading the file each time - if sums, err := srv.runtime.graph.getStoredChecksums(); err != nil { - return "", err - } else if checksum, exists := sums[imageID]; exists { - return checksum, nil - } - - img, err := srv.runtime.graph.Get(imageID) - if err != nil { - return "", err - } - - if _, err := os.Stat(layerArchivePath(srv.runtime.graph.imageRoot(imageID))); err != nil { - if os.IsNotExist(err) { - // TODO: Ask the registry for the checksum - // As the archive is not there, it is supposed to come from a pull. - } else { - return "", err - } - } - - checksum, err := img.Checksum() - if err != nil { - return "", err - } - return checksum, nil -} - // Retrieve the all the images to be uploaded in the correct order // Note: we can't use a map as it is not ordered func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgData, error) { @@ -638,14 +633,10 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat return nil } imageSet[img.ID] = struct{}{} - checksum, err := srv.getChecksum(img.ID) - if err != nil { - return err - } + imgList = append([]*registry.ImgData{{ - ID: img.ID, - Checksum: checksum, - Tag: tag, + ID: img.ID, + Tag: tag, }}, imgList...) return nil }) @@ -655,12 +646,11 @@ func (srv *Server) getImageList(localRepo map[string]string) ([]*registry.ImgDat func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName, remoteName string, localRepo map[string]string, indexEp string, sf *utils.StreamFormatter) error { out = utils.NewWriteFlusher(out) - out.Write(sf.FormatStatus("Processing checksums")) imgList, err := srv.getImageList(localRepo) if err != nil { return err } - out.Write(sf.FormatStatus("Sending image list")) + out.Write(sf.FormatStatus("", "Sending image list")) var repoData *registry.RepositoryData repoData, err = r.PushImageJSONIndex(indexEp, remoteName, imgList, false, nil) @@ -669,21 +659,23 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName } for _, ep := range repoData.Endpoints { - out.Write(sf.FormatStatus("Pushing repository %s (%d tags)", localName, len(localRepo))) + out.Write(sf.FormatStatus("", "Pushing repository %s (%d tags)", localName, len(localRepo))) // For each image within the repo, push them for _, elem := range imgList { if _, exists := repoData.ImgList[elem.ID]; exists { - out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) + out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID)) continue } else if r.LookupRemoteImage(elem.ID, ep, repoData.Tokens) { - out.Write(sf.FormatStatus("Image %s already pushed, skipping", elem.ID)) + out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", elem.ID)) continue } - if err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { + if checksum, err := srv.pushImage(r, out, remoteName, elem.ID, ep, repoData.Tokens, sf); err != nil { // FIXME: Continue on error? return err + } else { + elem.Checksum = checksum } - out.Write(sf.FormatStatus("Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) + out.Write(sf.FormatStatus("", "Pushing tags for rev [%s] on {%s}", elem.ID, ep+"repositories/"+remoteName+"/tags/"+elem.Tag)) if err := r.PushRegistryTag(remoteName, elem.ID, elem.Tag, ep, repoData.Tokens); err != nil { return err } @@ -697,64 +689,45 @@ func (srv *Server) pushRepository(r *registry.Registry, out io.Writer, localName return nil } -func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) error { +func (srv *Server) pushImage(r *registry.Registry, out io.Writer, remote, imgID, ep string, token []string, sf *utils.StreamFormatter) (checksum string, err error) { out = utils.NewWriteFlusher(out) jsonRaw, err := ioutil.ReadFile(path.Join(srv.runtime.graph.Root, imgID, "json")) if err != nil { - return fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) + return "", fmt.Errorf("Error while retreiving the path for {%s}: %s", imgID, err) } - out.Write(sf.FormatStatus("Pushing %s", imgID)) + out.Write(sf.FormatStatus("", "Pushing %s", imgID)) - // Make sure we have the image's checksum - checksum, err := srv.getChecksum(imgID) - if err != nil { - return err - } imgData := ®istry.ImgData{ - ID: imgID, - Checksum: checksum, + ID: imgID, } // Send the json if err := r.PushImageJSONRegistry(imgData, jsonRaw, ep, token); err != nil { if err == registry.ErrAlreadyExists { - out.Write(sf.FormatStatus("Image %s already pushed, skipping", imgData.ID)) - return nil + out.Write(sf.FormatStatus("", "Image %s already pushed, skipping", imgData.ID)) + return "", nil } - return err + return "", err } - // Retrieve the tarball to be sent - var layerData *TempArchive - // If the archive exists, use it - file, err := os.Open(layerArchivePath(srv.runtime.graph.imageRoot(imgID))) + layerData, err := srv.runtime.graph.TempLayerArchive(imgID, Uncompressed, sf, out) if err != nil { - if os.IsNotExist(err) { - // If the archive does not exist, create one from the layer - layerData, err = srv.runtime.graph.TempLayerArchive(imgID, Xz, sf, out) - if err != nil { - return fmt.Errorf("Failed to generate layer archive: %s", err) - } - } else { - return err - } - } else { - defer file.Close() - st, err := file.Stat() - if err != nil { - return err - } - layerData = &TempArchive{ - File: file, - Size: st.Size(), - } + return "", fmt.Errorf("Failed to generate layer archive: %s", err) } // Send the layer - if err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("Pushing", "%8v/%v (%v)"), sf), ep, token); err != nil { - return err + if checksum, err := r.PushImageLayerRegistry(imgData.ID, utils.ProgressReader(layerData, int(layerData.Size), out, sf.FormatProgress("", "Pushing", "%8v/%v (%v)"), sf, true), ep, token, jsonRaw); err != nil { + return "", err + } else { + imgData.Checksum = checksum } - return nil + + // Send the checksum + if err := r.PushImageChecksumRegistry(imgData, ep, token); err != nil { + return "", err + } + + return imgData.Checksum, nil } // FIXME: Allow to interupt current push when new push of same image is done. @@ -772,14 +745,14 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo out = utils.NewWriteFlusher(out) img, err := srv.runtime.graph.Get(localName) - r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.versionInfos()...) + r, err2 := registry.NewRegistry(srv.runtime.root, authConfig, srv.HTTPRequestFactory()) if err2 != nil { return err2 } if err != nil { reposLen := len(srv.runtime.repositories.Repositories[localName]) - out.Write(sf.FormatStatus("The push refers to a repository [%s] (len: %d)", localName, reposLen)) + out.Write(sf.FormatStatus("", "The push refers to a repository [%s] (len: %d)", localName, reposLen)) // If it fails, try to get the repository if localRepo, exists := srv.runtime.repositories.Repositories[localName]; exists { if err := srv.pushRepository(r, out, localName, remoteName, localRepo, endpoint, sf); err != nil { @@ -791,8 +764,8 @@ func (srv *Server) ImagePush(localName string, out io.Writer, sf *utils.StreamFo } var token []string - out.Write(sf.FormatStatus("The push refers to an image: [%s]", localName)) - if err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { + out.Write(sf.FormatStatus("", "The push refers to an image: [%s]", localName)) + if _, err := srv.pushImage(r, out, remoteName, img.ID, endpoint, token, sf); err != nil { return err } return nil @@ -814,14 +787,14 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write u.Host = src u.Path = "" } - out.Write(sf.FormatStatus("Downloading from %s", u)) + out.Write(sf.FormatStatus("", "Downloading from %s", u)) // Download with curl (pretty progress bar) // If curl is not available, fallback to http.Get() resp, err = utils.Download(u.String(), out) if err != nil { return err } - archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("Importing", "%8v/%v (%v)"), sf) + archive = utils.ProgressReader(resp.Body, int(resp.ContentLength), out, sf.FormatProgress("", "Importing", "%8v/%v (%v)"), sf, true) } img, err := srv.runtime.graph.Create(archive, nil, "Imported from "+src, "", nil) if err != nil { @@ -833,7 +806,7 @@ func (srv *Server) ImageImport(src, repo, tag string, in io.Reader, out io.Write return err } } - out.Write(sf.FormatStatus(img.ShortID())) + out.Write(sf.FormatStatus("", img.ShortID())) return nil } @@ -984,6 +957,9 @@ func (srv *Server) deleteImage(img *Image, repoName, tag string) ([]APIRmi, erro parsedRepo := strings.Split(repoAndTag, ":")[0] if strings.Contains(img.ID, repoName) { repoName = parsedRepo + if len(srv.runtime.repositories.ByID()[img.ID]) == 1 && len(strings.Split(repoAndTag, ":")) > 1 { + tag = strings.Split(repoAndTag, ":")[1] + } } else if repoName != parsedRepo { // the id belongs to multiple repos, like base:latest and user:test, // in that case return conflict @@ -1026,13 +1002,7 @@ func (srv *Server) ImageDelete(name string, autoPrune bool) ([]APIRmi, error) { return nil, nil } - var tag string - if strings.Contains(name, ":") { - nameParts := strings.Split(name, ":") - name = nameParts[0] - tag = nameParts[1] - } - + name, tag := utils.ParseRepositoryTag(name) return srv.deleteImage(img, name, tag) } @@ -1204,6 +1174,23 @@ func (srv *Server) ImageInspect(name string) (*Image, error) { return nil, fmt.Errorf("No such image: %s", name) } +func (srv *Server) ContainerCopy(name string, resource string, out io.Writer) error { + if container := srv.runtime.Get(name); container != nil { + + data, err := container.Copy(resource) + if err != nil { + return err + } + + if _, err := io.Copy(out, data); err != nil { + return err + } + return nil + } + return fmt.Errorf("No such container: %s", name) + +} + func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) (*Server, error) { if runtime.GOARCH != "amd64" { log.Fatalf("The docker runtime currently only supports amd64 (not %s). This will change in the future. Aborting.", runtime.GOARCH) @@ -1219,11 +1206,21 @@ func NewServer(flGraphPath string, autoRestart, enableCors bool, dns ListOpts) ( pushingPool: make(map[string]struct{}), events: make([]utils.JSONMessage, 0, 64), //only keeps the 64 last events listeners: make(map[string]chan utils.JSONMessage), + reqFactory: nil, } runtime.srv = srv return srv, nil } +func (srv *Server) HTTPRequestFactory() *utils.HTTPRequestFactory { + if srv.reqFactory == nil { + ud := utils.NewHTTPUserAgentDecorator(srv.versionInfos()...) + factory := utils.NewHTTPRequestFactory(ud) + srv.reqFactory = factory + } + return srv.reqFactory +} + func (srv *Server) LogEvent(action, id string) { now := time.Now().Unix() jm := utils.JSONMessage{Status: action, ID: id, Time: now} @@ -1244,4 +1241,5 @@ type Server struct { pushingPool map[string]struct{} events []utils.JSONMessage listeners map[string]chan utils.JSONMessage + reqFactory *utils.HTTPRequestFactory } diff --git a/server_test.go b/server_test.go index 58aa66bd20..0e683a4909 100644 --- a/server_test.go +++ b/server_test.go @@ -2,6 +2,7 @@ package docker import ( "github.com/dotcloud/docker/utils" + "strings" "testing" "time" ) @@ -20,16 +21,20 @@ func TestContainerTagImageDelete(t *testing.T) { if err := srv.runtime.repositories.Set("utest", "tag1", unitTestImageName, false); err != nil { t.Fatal(err) } + if err := srv.runtime.repositories.Set("utest/docker", "tag2", unitTestImageName, false); err != nil { t.Fatal(err) } + if err := srv.runtime.repositories.Set("utest:5000/docker", "tag3", unitTestImageName, false); err != nil { + t.Fatal(err) + } images, err := srv.Images(false, "") if err != nil { t.Fatal(err) } - if len(images) != len(initialImages)+2 { + if len(images) != len(initialImages)+3 { t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) } @@ -42,6 +47,19 @@ func TestContainerTagImageDelete(t *testing.T) { t.Fatal(err) } + if len(images) != len(initialImages)+2 { + t.Errorf("Expected %d images, %d found", len(initialImages)+2, len(images)) + } + + if _, err := srv.ImageDelete("utest:5000/docker:tag3", true); err != nil { + t.Fatal(err) + } + + images, err = srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + if len(images) != len(initialImages)+1 { t.Errorf("Expected %d images, %d found", len(initialImages)+1, len(images)) } @@ -90,6 +108,27 @@ func TestCreateRm(t *testing.T) { } +func TestCommit(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + + srv := &Server{runtime: runtime} + + config, _, _, err := ParseRun([]string{GetTestImage(runtime).ID, "/bin/cat"}, nil) + if err != nil { + t.Fatal(err) + } + + id, err := srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + if _, err := srv.ContainerCommit(id, "testrepo", "testtag", "", "", config); err != nil { + t.Fatal(err) + } +} + func TestCreateStartRestartStopStartKillRm(t *testing.T) { runtime := mkRuntime(t) defer nuke(runtime) @@ -289,3 +328,88 @@ func TestLogEvent(t *testing.T) { } }) } + +func TestRmi(t *testing.T) { + runtime := mkRuntime(t) + defer nuke(runtime) + srv := &Server{runtime: runtime} + + initialImages, err := srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + config, hostConfig, _, err := ParseRun([]string{GetTestImage(runtime).ID, "echo test"}, nil) + if err != nil { + t.Fatal(err) + } + + containerID, err := srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + //To remove + err = srv.ContainerStart(containerID, hostConfig) + if err != nil { + t.Fatal(err) + } + + imageID, err := srv.ContainerCommit(containerID, "test", "", "", "", nil) + if err != nil { + t.Fatal(err) + } + + err = srv.ContainerTag(imageID, "test", "0.1", false) + if err != nil { + t.Fatal(err) + } + + containerID, err = srv.ContainerCreate(config) + if err != nil { + t.Fatal(err) + } + + //To remove + err = srv.ContainerStart(containerID, hostConfig) + if err != nil { + t.Fatal(err) + } + + _, err = srv.ContainerCommit(containerID, "test", "", "", "", nil) + if err != nil { + t.Fatal(err) + } + + images, err := srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + if len(images)-len(initialImages) != 2 { + t.Fatalf("Expected 2 new images, found %d.", len(images)-len(initialImages)) + } + + _, err = srv.ImageDelete(imageID, true) + if err != nil { + t.Fatal(err) + } + + images, err = srv.Images(false, "") + if err != nil { + t.Fatal(err) + } + + if len(images)-len(initialImages) != 1 { + t.Fatalf("Expected 1 new image, found %d.", len(images)-len(initialImages)) + } + + for _, image := range images { + if strings.Contains(unitTestImageID, image.ID) { + continue + } + if image.Repository == "" { + t.Fatalf("Expected tagged image, got untagged one.") + } + } +} diff --git a/testing/README.rst b/testing/README.rst index 3b11092f9f..ce5aa837a4 100644 --- a/testing/README.rst +++ b/testing/README.rst @@ -40,6 +40,10 @@ Deployment export SMTP_USER=xxxxxxxxxxxx export SMTP_PWD=xxxxxxxxxxxx + # Define docker registry functional test credentials + export REGISTRY_USER=xxxxxxxxxxxx + export REGISTRY_PWD=xxxxxxxxxxxx + # Checkout docker git clone git://github.com/dotcloud/docker.git diff --git a/testing/Vagrantfile b/testing/Vagrantfile index 47257201dc..e76a951508 100644 --- a/testing/Vagrantfile +++ b/testing/Vagrantfile @@ -29,7 +29,9 @@ Vagrant::Config.run do |config| "chown #{USER}.#{USER} /data; cd /data; " \ "#{CFG_PATH}/setup.sh #{USER} #{CFG_PATH} #{ENV['BUILDBOT_PWD']} " \ "#{ENV['IRC_PWD']} #{ENV['IRC_CHANNEL']} #{ENV['SMTP_USER']} " \ - "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " + "#{ENV['SMTP_PWD']} #{ENV['EMAIL_RCP']}; " \ + "#{CFG_PATH}/setup_credentials.sh #{USER} " \ + "#{ENV['REGISTRY_USER']} #{ENV['REGISTRY_PWD']}; " # Install docker dependencies pkg_cmd << "apt-get install -q -y python-software-properties; " \ "add-apt-repository -y ppa:dotcloud/docker-golang/ubuntu; apt-get update -qq; " \ diff --git a/testing/buildbot/credentials.cfg b/testing/buildbot/credentials.cfg new file mode 100644 index 0000000000..fbdd35d578 --- /dev/null +++ b/testing/buildbot/credentials.cfg @@ -0,0 +1,5 @@ +# Credentials for tests. Buildbot source this file on tests +# when needed. + +# Docker registry credentials. Format: 'username:password' +export DOCKER_CREDS='' diff --git a/testing/buildbot/master.cfg b/testing/buildbot/master.cfg index 61912808ec..29926dbe5f 100644 --- a/testing/buildbot/master.cfg +++ b/testing/buildbot/master.cfg @@ -19,6 +19,7 @@ TEST_USER = 'buildbot' # Credential to authenticate build triggers TEST_PWD = 'docker' # Credential to authenticate build triggers BUILDER_NAME = 'docker' GITHUB_DOCKER = 'github.com/dotcloud/docker' +BUILDBOT_PATH = '/data/buildbot' DOCKER_PATH = '/data/docker' BUILDER_PATH = '/data/buildbot/slave/{0}/build'.format(BUILDER_NAME) DOCKER_BUILD_PATH = BUILDER_PATH + '/src/github.com/dotcloud/docker' @@ -41,16 +42,19 @@ c['db'] = {'db_url':"sqlite:///state.sqlite"} c['slaves'] = [BuildSlave('buildworker', BUILDBOT_PWD)] c['slavePortnum'] = PORT_MASTER + # Schedulers c['schedulers'] = [ForceScheduler(name='trigger', builderNames=[BUILDER_NAME, - 'coverage'])] + 'registry','coverage'])] c['schedulers'] += [SingleBranchScheduler(name="all", change_filter=filter.ChangeFilter(branch='master'), treeStableTimer=None, builderNames=[BUILDER_NAME])] -c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage'], +c['schedulers'] += [Nightly(name='daily', branch=None, builderNames=['coverage','registry'], hour=0, minute=30)] + # Builders +# Docker commit test factory = BuildFactory() factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, command=["sh", "-c", Interpolate("cd ..; rm -rf build; export GOPATH={0}; " @@ -58,6 +62,7 @@ factory.addStep(ShellCommand(description='Docker',logEnviron=False,usePTY=True, "go test -v".format(BUILDER_PATH,GITHUB_DOCKER,DOCKER_BUILD_PATH))])) c['builders'] = [BuilderConfig(name=BUILDER_NAME,slavenames=['buildworker'], factory=factory)] + # Docker coverage test coverage_cmd = ('GOPATH=`pwd` go get -d github.com/dotcloud/docker\n' 'GOPATH=`pwd` go get github.com/axw/gocov/gocov\n' @@ -69,6 +74,17 @@ factory.addStep(ShellCommand(description='Coverage',logEnviron=False,usePTY=True c['builders'] += [BuilderConfig(name='coverage',slavenames=['buildworker'], factory=factory)] +# Registry Functionaltest builder +factory = BuildFactory() +factory.addStep(ShellCommand(description='registry', logEnviron=False, + command='. {0}/master/credentials.cfg; ' + '{1}/testing/functionaltests/test_registry.sh'.format(BUILDBOT_PATH, + DOCKER_PATH), usePTY=True)) + +c['builders'] += [BuilderConfig(name='registry',slavenames=['buildworker'], + factory=factory)] + + # Status authz_cfg = authz.Authz(auth=auth.BasicAuth([(TEST_USER, TEST_PWD)]), forceBuild='auth') diff --git a/testing/buildbot/requirements.txt b/testing/buildbot/requirements.txt index 0e451b017d..4e183ba062 100644 --- a/testing/buildbot/requirements.txt +++ b/testing/buildbot/requirements.txt @@ -4,3 +4,4 @@ buildbot==0.8.7p1 buildbot_slave==0.8.7p1 nose==1.2.1 requests==1.1.0 +flask==0.10.1 diff --git a/testing/buildbot/setup_credentials.sh b/testing/buildbot/setup_credentials.sh new file mode 100755 index 0000000000..f093815d60 --- /dev/null +++ b/testing/buildbot/setup_credentials.sh @@ -0,0 +1,17 @@ +#!/bin/bash + +# Setup of test credentials. Called by Vagrantfile +export PATH="/bin:sbin:/usr/bin:/usr/sbin:/usr/local/bin" + +USER=$1 +REGISTRY_USER=$2 +REGISTRY_PWD=$3 + +BUILDBOT_PATH="/data/buildbot" +DOCKER_PATH="/data/docker" + +function run { su $USER -c "$1"; } + +run "cp $DOCKER_PATH/testing/buildbot/credentials.cfg $BUILDBOT_PATH/master" +cd $BUILDBOT_PATH/master +run "sed -i -E 's#(export DOCKER_CREDS=).+#\1\"$REGISTRY_USER:$REGISTRY_PWD\"#' credentials.cfg" diff --git a/testing/functionaltests/test_registry.sh b/testing/functionaltests/test_registry.sh new file mode 100755 index 0000000000..095a731631 --- /dev/null +++ b/testing/functionaltests/test_registry.sh @@ -0,0 +1,11 @@ +#!/bin/sh + +# Cleanup +rm -rf docker-registry + +# Get latest docker registry +git clone https://github.com/dotcloud/docker-registry.git + +# Configure and run registry tests +cd docker-registry; cp config_sample.yml config.yml +cd test; python -m unittest workflow diff --git a/utils/http.go b/utils/http.go new file mode 100644 index 0000000000..8c1e4b7a79 --- /dev/null +++ b/utils/http.go @@ -0,0 +1,134 @@ +package utils + +import ( + "bytes" + "io" + "net/http" + "strings" +) + +// VersionInfo is used to model entities which has a version. +// It is basically a tupple with name and version. +type VersionInfo interface { + Name() string + Version() string +} + +func validVersion(version VersionInfo) bool { + stopChars := " \t\r\n/" + if strings.ContainsAny(version.Name(), stopChars) { + return false + } + if strings.ContainsAny(version.Version(), stopChars) { + return false + } + return true +} + +// Convert versions to a string and append the string to the string base. +// +// Each VersionInfo will be converted to a string in the format of +// "product/version", where the "product" is get from the Name() method, while +// version is get from the Version() method. Several pieces of verson information +// will be concatinated and separated by space. +func appendVersions(base string, versions ...VersionInfo) string { + if len(versions) == 0 { + return base + } + + var buf bytes.Buffer + if len(base) > 0 { + buf.Write([]byte(base)) + } + + for _, v := range versions { + name := []byte(v.Name()) + version := []byte(v.Version()) + + if len(name) == 0 || len(version) == 0 { + continue + } + if !validVersion(v) { + continue + } + buf.Write([]byte(v.Name())) + buf.Write([]byte("/")) + buf.Write([]byte(v.Version())) + buf.Write([]byte(" ")) + } + return buf.String() +} + +// HTTPRequestDecorator is used to change an instance of +// http.Request. It could be used to add more header fields, +// change body, etc. +type HTTPRequestDecorator interface { + // ChangeRequest() changes the request accordingly. + // The changed request will be returned or err will be non-nil + // if an error occur. + ChangeRequest(req *http.Request) (newReq *http.Request, err error) +} + +// HTTPUserAgentDecorator appends the product/version to the user agent field +// of a request. +type HTTPUserAgentDecorator struct { + versions []VersionInfo +} + +func NewHTTPUserAgentDecorator(versions ...VersionInfo) HTTPRequestDecorator { + ret := new(HTTPUserAgentDecorator) + ret.versions = versions + return ret +} + +func (self *HTTPUserAgentDecorator) ChangeRequest(req *http.Request) (newReq *http.Request, err error) { + if req == nil { + return req, nil + } + + userAgent := appendVersions(req.UserAgent(), self.versions...) + if len(userAgent) > 0 { + req.Header.Set("User-Agent", userAgent) + } + return req, nil +} + +// HTTPRequestFactory creates an HTTP request +// and applies a list of decorators on the request. +type HTTPRequestFactory struct { + decorators []HTTPRequestDecorator +} + +func NewHTTPRequestFactory(d ...HTTPRequestDecorator) *HTTPRequestFactory { + ret := new(HTTPRequestFactory) + ret.decorators = d + return ret +} + +// NewRequest() creates a new *http.Request, +// applies all decorators in the HTTPRequestFactory on the request, +// then applies decorators provided by d on the request. +func (self *HTTPRequestFactory) NewRequest(method, urlStr string, body io.Reader, d ...HTTPRequestDecorator) (*http.Request, error) { + req, err := http.NewRequest(method, urlStr, body) + if err != nil { + return nil, err + } + + // By default, a nil factory should work. + if self == nil { + return req, nil + } + for _, dec := range self.decorators { + req, err = dec.ChangeRequest(req) + if err != nil { + return nil, err + } + } + for _, dec := range d { + req, err = dec.ChangeRequest(req) + if err != nil { + return nil, err + } + } + return req, err +} diff --git a/utils/tarsum.go b/utils/tarsum.go new file mode 100644 index 0000000000..d3e1db61f1 --- /dev/null +++ b/utils/tarsum.go @@ -0,0 +1,158 @@ +package utils + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "crypto/sha256" + "encoding/hex" + "hash" + "io" + "sort" + "strconv" +) + +type verboseHash struct { + hash.Hash +} + +func (h verboseHash) Write(buf []byte) (int, error) { + Debugf("--->%s<---", buf) + return h.Hash.Write(buf) +} + +type TarSum struct { + io.Reader + tarR *tar.Reader + tarW *tar.Writer + gz *gzip.Writer + bufTar *bytes.Buffer + bufGz *bytes.Buffer + h hash.Hash + h2 verboseHash + sums []string + finished bool + first bool +} + +func (ts *TarSum) encodeHeader(h *tar.Header) error { + for _, elem := range [][2]string{ + {"name", h.Name}, + {"mode", strconv.Itoa(int(h.Mode))}, + {"uid", strconv.Itoa(h.Uid)}, + {"gid", strconv.Itoa(h.Gid)}, + {"size", strconv.Itoa(int(h.Size))}, + {"mtime", strconv.Itoa(int(h.ModTime.UTC().Unix()))}, + {"typeflag", string([]byte{h.Typeflag})}, + {"linkname", h.Linkname}, + {"uname", h.Uname}, + {"gname", h.Gname}, + {"devmajor", strconv.Itoa(int(h.Devmajor))}, + {"devminor", strconv.Itoa(int(h.Devminor))}, + // {"atime", strconv.Itoa(int(h.AccessTime.UTC().Unix()))}, + // {"ctime", strconv.Itoa(int(h.ChangeTime.UTC().Unix()))}, + } { + // Debugf("-->%s<-- -->%s<--", elem[0], elem[1]) + if _, err := ts.h.Write([]byte(elem[0] + elem[1])); err != nil { + return err + } + } + return nil +} + +func (ts *TarSum) Read(buf []byte) (int, error) { + if ts.gz == nil { + ts.bufTar = bytes.NewBuffer([]byte{}) + ts.bufGz = bytes.NewBuffer([]byte{}) + ts.tarR = tar.NewReader(ts.Reader) + ts.tarW = tar.NewWriter(ts.bufTar) + ts.gz = gzip.NewWriter(ts.bufGz) + ts.h = sha256.New() + // ts.h = verboseHash{sha256.New()} + ts.h.Reset() + ts.first = true + } + + if ts.finished { + return ts.bufGz.Read(buf) + } + buf2 := make([]byte, len(buf), cap(buf)) + + n, err := ts.tarR.Read(buf2) + if err != nil { + if err == io.EOF { + if _, err := ts.h.Write(buf2[:n]); err != nil { + return 0, err + } + if !ts.first { + ts.sums = append(ts.sums, hex.EncodeToString(ts.h.Sum(nil))) + ts.h.Reset() + } else { + ts.first = false + } + + currentHeader, err := ts.tarR.Next() + if err != nil { + if err == io.EOF { + if err := ts.gz.Close(); err != nil { + return 0, err + } + ts.finished = true + return n, nil + } + return n, err + } + if err := ts.encodeHeader(currentHeader); err != nil { + return 0, err + } + if err := ts.tarW.WriteHeader(currentHeader); err != nil { + return 0, err + } + if _, err := ts.tarW.Write(buf2[:n]); err != nil { + return 0, err + } + ts.tarW.Flush() + if _, err := io.Copy(ts.gz, ts.bufTar); err != nil { + return 0, err + } + ts.gz.Flush() + + return ts.bufGz.Read(buf) + } + return n, err + } + + // Filling the hash buffer + if _, err = ts.h.Write(buf2[:n]); err != nil { + return 0, err + } + + // Filling the tar writter + if _, err = ts.tarW.Write(buf2[:n]); err != nil { + return 0, err + } + ts.tarW.Flush() + + // Filling the gz writter + if _, err = io.Copy(ts.gz, ts.bufTar); err != nil { + return 0, err + } + ts.gz.Flush() + + return ts.bufGz.Read(buf) +} + +func (ts *TarSum) Sum(extra []byte) string { + sort.Strings(ts.sums) + h := sha256.New() + if extra != nil { + h.Write(extra) + } + for _, sum := range ts.sums { + Debugf("-->%s<--", sum) + h.Write([]byte(sum)) + } + checksum := "tarsum+sha256:" + hex.EncodeToString(h.Sum(nil)) + Debugf("checksum processed: %s", checksum) + return checksum +} diff --git a/utils/utils.go b/utils/utils.go index acb015becd..d52dcafe17 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -73,6 +73,7 @@ type progressReader struct { lastUpdate int // How many bytes read at least update template string // Template to print. Default "%v/%v (%v)" sf *StreamFormatter + newLine bool } func (r *progressReader) Read(p []byte) (n int, err error) { @@ -95,21 +96,23 @@ func (r *progressReader) Read(p []byte) (n int, err error) { r.lastUpdate = r.readProgress } // Send newline when complete - if err != nil { - r.output.Write(r.sf.FormatStatus("")) + if r.newLine && err != nil { + r.output.Write(r.sf.FormatStatus("", "")) } - return read, err } func (r *progressReader) Close() error { return io.ReadCloser(r.reader).Close() } -func ProgressReader(r io.ReadCloser, size int, output io.Writer, template []byte, sf *StreamFormatter) *progressReader { - tpl := string(template) - if tpl == "" { - tpl = string(sf.FormatProgress("", "%8v/%v (%v)")) +func ProgressReader(r io.ReadCloser, size int, output io.Writer, tpl []byte, sf *StreamFormatter, newline bool) *progressReader { + return &progressReader{ + reader: r, + output: NewWriteFlusher(output), + readTotal: size, + template: string(tpl), + sf: sf, + newLine: newline, } - return &progressReader{r, NewWriteFlusher(output), size, 0, 0, tpl, sf} } // HumanDuration returns a human-readable approximation of a duration @@ -587,11 +590,14 @@ type NopFlusher struct{} func (f *NopFlusher) Flush() {} type WriteFlusher struct { + sync.Mutex w io.Writer flusher http.Flusher } func (wf *WriteFlusher) Write(b []byte) (n int, err error) { + wf.Lock() + defer wf.Unlock() n, err = wf.w.Write(b) wf.flusher.Flush() return n, err @@ -607,30 +613,89 @@ func NewWriteFlusher(w io.Writer) *WriteFlusher { return &WriteFlusher{w: w, flusher: flusher} } -type JSONMessage struct { - Status string `json:"status,omitempty"` - Progress string `json:"progress,omitempty"` - Error string `json:"error,omitempty"` - ID string `json:"id,omitempty"` - Time int64 `json:"time,omitempty"` +type JSONError struct { + Code int `json:"code,omitempty"` + Message string `json:"message,omitempty"` } -func (jm *JSONMessage) Display(out io.Writer) (error) { +type JSONMessage struct { + Status string `json:"status,omitempty"` + Progress string `json:"progress,omitempty"` + ErrorMessage string `json:"error,omitempty"` //deprecated + ID string `json:"id,omitempty"` + Time int64 `json:"time,omitempty"` + Error *JSONError `json:"errorDetail,omitempty"` +} + +func (e *JSONError) Error() string { + return e.Message +} + +func NewHTTPRequestError(msg string, res *http.Response) error { + return &JSONError{ + Message: msg, + Code: res.StatusCode, + } +} + +func (jm *JSONMessage) Display(out io.Writer) error { + if jm.Error != nil { + if jm.Error.Code == 401 { + return fmt.Errorf("Authentication is required.") + } + return jm.Error + } + fmt.Fprintf(out, "%c[2K", 27) if jm.Time != 0 { fmt.Fprintf(out, "[%s] ", time.Unix(jm.Time, 0)) } + if jm.ID != "" { + fmt.Fprintf(out, "%s: ", jm.ID) + } if jm.Progress != "" { fmt.Fprintf(out, "%s %s\r", jm.Status, jm.Progress) - } else if jm.Error != "" { - return fmt.Errorf(jm.Error) - } else if jm.ID != "" { - fmt.Fprintf(out, "%s: %s\n", jm.ID, jm.Status) } else { - fmt.Fprintf(out, "%s\n", jm.Status) + fmt.Fprintf(out, "%s\r", jm.Status) + } + if jm.ID == "" { + fmt.Fprintf(out, "\n") } return nil } +func DisplayJSONMessagesStream(in io.Reader, out io.Writer) error { + dec := json.NewDecoder(in) + jm := JSONMessage{} + ids := make(map[string]int) + diff := 0 + for { + if err := dec.Decode(&jm); err == io.EOF { + break + } else if err != nil { + return err + } + if jm.ID != "" { + line, ok := ids[jm.ID] + if !ok { + line = len(ids) + ids[jm.ID] = line + fmt.Fprintf(out, "\n") + diff = 0 + } else { + diff = len(ids) - line + } + fmt.Fprintf(out, "%c[%dA", 27, diff) + } + err := jm.Display(out) + if jm.ID != "" { + fmt.Fprintf(out, "%c[%dB", 27, diff) + } + if err != nil { + return err + } + } + return nil +} type StreamFormatter struct { json bool @@ -641,11 +706,11 @@ func NewStreamFormatter(json bool) *StreamFormatter { return &StreamFormatter{json, false} } -func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte { +func (sf *StreamFormatter) FormatStatus(id, format string, a ...interface{}) []byte { sf.used = true str := fmt.Sprintf(format, a...) if sf.json { - b, err := json.Marshal(&JSONMessage{Status: str}) + b, err := json.Marshal(&JSONMessage{ID: id, Status: str}) if err != nil { return sf.FormatError(err) } @@ -657,7 +722,11 @@ func (sf *StreamFormatter) FormatStatus(format string, a ...interface{}) []byte func (sf *StreamFormatter) FormatError(err error) []byte { sf.used = true if sf.json { - if b, err := json.Marshal(&JSONMessage{Error: err.Error()}); err == nil { + jsonError, ok := err.(*JSONError) + if !ok { + jsonError = &JSONError{Message: err.Error()} + } + if b, err := json.Marshal(&JSONMessage{Error: jsonError, ErrorMessage: err.Error()}); err == nil { return b } return []byte("{\"error\":\"format error\"}") @@ -665,16 +734,16 @@ func (sf *StreamFormatter) FormatError(err error) []byte { return []byte("Error: " + err.Error() + "\r\n") } -func (sf *StreamFormatter) FormatProgress(action, str string) []byte { +func (sf *StreamFormatter) FormatProgress(id, action, progress string) []byte { sf.used = true if sf.json { - b, err := json.Marshal(&JSONMessage{Status: action, Progress: str}) + b, err := json.Marshal(&JSONMessage{Status: action, Progress: progress, ID: id}) if err != nil { return nil } return b } - return []byte(action + " " + str + "\r") + return []byte(action + " " + progress + "\r") } func (sf *StreamFormatter) Used() bool { @@ -689,17 +758,29 @@ func IsGIT(str string) bool { return strings.HasPrefix(str, "git://") || strings.HasPrefix(str, "github.com/") } -func CheckLocalDns() bool { +// GetResolvConf opens and read the content of /etc/resolv.conf. +// It returns it as byte slice. +func GetResolvConf() ([]byte, error) { resolv, err := ioutil.ReadFile("/etc/resolv.conf") if err != nil { Debugf("Error openning resolv.conf: %s", err) - return false + return nil, err } - for _, ip := range []string{ - "127.0.0.1", - "127.0.1.1", + return resolv, nil +} + +// CheckLocalDns looks into the /etc/resolv.conf, +// it returns true if there is a local nameserver or if there is no nameserver. +func CheckLocalDns(resolvConf []byte) bool { + if !bytes.Contains(resolvConf, []byte("nameserver")) { + return true + } + + for _, ip := range [][]byte{ + []byte("127.0.0.1"), + []byte("127.0.1.1"), } { - if strings.Contains(string(resolv), ip) { + if bytes.Contains(resolvConf, ip) { return true } } @@ -731,6 +812,22 @@ func ParseHost(host string, port int, addr string) string { return fmt.Sprintf("tcp://%s:%d", host, port) } +func GetReleaseVersion() string { + resp, err := http.Get("http://get.docker.io/latest") + if err != nil { + return "" + } + defer resp.Body.Close() + if resp.ContentLength > 24 || resp.StatusCode != 200 { + return "" + } + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "" + } + return strings.TrimSpace(string(body)) +} + // Get a repos name and returns the right reposName + tag // The tag can be confusing because of a port in a repository name. // Ex: localhost.localdomain:5000/samalba/hipache:latest diff --git a/utils/utils_test.go b/utils/utils_test.go index 5caa809f67..d8ba7f1c31 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -282,3 +282,58 @@ func TestParseHost(t *testing.T) { t.Errorf("unix:///var/run/docker.sock -> expected unix:///var/run/docker.sock, got %s", addr) } } + +func TestParseRepositoryTag(t *testing.T) { + if repo, tag := ParseRepositoryTag("root"); repo != "root" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("root:tag"); repo != "root" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "root", "tag", repo, tag) + } + if repo, tag := ParseRepositoryTag("user/repo"); repo != "user/repo" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("user/repo:tag"); repo != "user/repo" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "user/repo", "tag", repo, tag) + } + if repo, tag := ParseRepositoryTag("url:5000/repo"); repo != "url:5000/repo" || tag != "" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "", repo, tag) + } + if repo, tag := ParseRepositoryTag("url:5000/repo:tag"); repo != "url:5000/repo" || tag != "tag" { + t.Errorf("Expected repo: '%s' and tag: '%s', got '%s' and '%s'", "url:5000/repo", "tag", repo, tag) + } +} + +func TestGetResolvConf(t *testing.T) { + resolvConfUtils, err := GetResolvConf() + if err != nil { + t.Fatal(err) + } + resolvConfSystem, err := ioutil.ReadFile("/etc/resolv.conf") + if err != nil { + t.Fatal(err) + } + if string(resolvConfUtils) != string(resolvConfSystem) { + t.Fatalf("/etc/resolv.conf and GetResolvConf have different content.") + } +} + +func TestCheckLocalDns(t *testing.T) { + for resolv, result := range map[string]bool{`# Dynamic +nameserver 10.0.2.3 +search dotcloud.net`: false, + `# Dynamic +nameserver 127.0.0.1 +search dotcloud.net`: true, + `# Dynamic +nameserver 127.0.1.1 +search dotcloud.net`: true, + `# Dynamic +`: true, + ``: true, + } { + if CheckLocalDns([]byte(resolv)) != result { + t.Fatalf("Wrong local dns detection: {%s} should be %v", resolv, result) + } + } +} diff --git a/utils_test.go b/utils_test.go index d881d713aa..5c37e9e8ec 100644 --- a/utils_test.go +++ b/utils_test.go @@ -249,7 +249,7 @@ func TestMergeConfig(t *testing.T) { if len(configUser.Volumes) != 3 { t.Fatalf("Expected 3 volumes, /test1, /test2 and /test3, found %d", len(configUser.Volumes)) } - for v, _ := range configUser.Volumes { + for v := range configUser.Volumes { if v != "/test1" && v != "/test2" && v != "/test3" { t.Fatalf("Expected /test1 or /test2 or /test3, found %s", v) } diff --git a/z_final_test.go b/z_final_test.go index 78a7acf6e7..08a180baaf 100644 --- a/z_final_test.go +++ b/z_final_test.go @@ -6,7 +6,12 @@ import ( "testing" ) -func TestFinal(t *testing.T) { - cleanup(globalRuntime) +func displayFdGoroutines(t *testing.T) { t.Logf("Fds: %d, Goroutines: %d", utils.GetTotalUsedFds(), runtime.NumGoroutine()) } + +func TestFinal(t *testing.T) { + cleanup(globalRuntime) + t.Logf("Start Fds: %d, Start Goroutines: %d", startFds, startGoroutines) + displayFdGoroutines(t) +}