From 14887e2e1f8d11f30ab8c25beed0ff9e70354005 Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Tue, 3 Feb 2015 15:41:26 -0800 Subject: [PATCH 1/4] Interface Logger for logging drivers Also there is aux datastructure Copier which can copy lines from streams to Loggers Signed-off-by: Alexander Morozov --- daemon/logger/copier.go | 48 +++++++++++++++++ daemon/logger/copier_test.go | 100 +++++++++++++++++++++++++++++++++++ daemon/logger/logger.go | 18 +++++++ 3 files changed, 166 insertions(+) create mode 100644 daemon/logger/copier.go create mode 100644 daemon/logger/copier_test.go create mode 100644 daemon/logger/logger.go diff --git a/daemon/logger/copier.go b/daemon/logger/copier.go new file mode 100644 index 0000000000..5b75c919ef --- /dev/null +++ b/daemon/logger/copier.go @@ -0,0 +1,48 @@ +package logger + +import ( + "bufio" + "io" + "time" + + "github.com/Sirupsen/logrus" +) + +// Copier can copy logs from specified sources to Logger and attach +// ContainerID and Timestamp. +// Writes are concurrent, so you need implement some sync in your logger +type Copier struct { + // cid is container id for which we copying logs + cid string + // srcs is map of name -> reader pairs, for example "stdout", "stderr" + srcs map[string]io.Reader + dst Logger +} + +// NewCopier creates new Copier +func NewCopier(cid string, srcs map[string]io.Reader, dst Logger) (*Copier, error) { + return &Copier{ + cid: cid, + srcs: srcs, + dst: dst, + }, nil +} + +// Run starts logs copying +func (c *Copier) Run() { + for src, w := range c.srcs { + go c.copySrc(src, w) + } +} + +func (c *Copier) copySrc(name string, src io.Reader) { + scanner := bufio.NewScanner(src) + for scanner.Scan() { + if err := c.dst.Log(&Message{ContainerID: c.cid, Line: scanner.Bytes(), Source: name, Timestamp: time.Now().UTC()}); err != nil { + logrus.Errorf("Failed to log msg %q for logger %s: %s", scanner.Bytes(), c.dst.Name(), err) + } + } + if err := scanner.Err(); err != nil { + logrus.Errorf("Error scanning log stream: %s", err) + } +} diff --git a/daemon/logger/copier_test.go b/daemon/logger/copier_test.go new file mode 100644 index 0000000000..0071af3a5f --- /dev/null +++ b/daemon/logger/copier_test.go @@ -0,0 +1,100 @@ +package logger + +import ( + "bytes" + "encoding/json" + "io" + "testing" + "time" +) + +type TestLoggerJSON struct { + *json.Encoder +} + +func (l *TestLoggerJSON) Log(m *Message) error { + return l.Encode(m) +} + +func (l *TestLoggerJSON) Close() error { + return nil +} + +func (l *TestLoggerJSON) Name() string { + return "json" +} + +type TestLoggerText struct { + *bytes.Buffer +} + +func (l *TestLoggerText) Log(m *Message) error { + _, err := l.WriteString(m.ContainerID + " " + m.Source + " " + string(m.Line) + "\n") + return err +} + +func (l *TestLoggerText) Close() error { + return nil +} + +func (l *TestLoggerText) Name() string { + return "text" +} + +func TestCopier(t *testing.T) { + stdoutLine := "Line that thinks that it is log line from docker stdout" + stderrLine := "Line that thinks that it is log line from docker stderr" + var stdout bytes.Buffer + var stderr bytes.Buffer + for i := 0; i < 30; i++ { + if _, err := stdout.WriteString(stdoutLine + "\n"); err != nil { + t.Fatal(err) + } + if _, err := stderr.WriteString(stderrLine + "\n"); err != nil { + t.Fatal(err) + } + } + + var jsonBuf bytes.Buffer + + jsonLog := &TestLoggerJSON{Encoder: json.NewEncoder(&jsonBuf)} + + cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657" + c, err := NewCopier(cid, + map[string]io.Reader{ + "stdout": &stdout, + "stderr": &stderr, + }, + jsonLog) + if err != nil { + t.Fatal(err) + } + c.Run() + time.Sleep(100 * time.Millisecond) + dec := json.NewDecoder(&jsonBuf) + for { + var msg Message + if err := dec.Decode(&msg); err != nil { + if err == io.EOF { + break + } + t.Fatal(err) + } + if msg.Source != "stdout" && msg.Source != "stderr" { + t.Fatalf("Wrong Source: %q, should be %q or %q", msg.Source, "stdout", "stderr") + } + if msg.ContainerID != cid { + t.Fatalf("Wrong ContainerID: %q, expected %q", msg.ContainerID, cid) + } + if msg.Source == "stdout" { + if string(msg.Line) != stdoutLine { + t.Fatalf("Wrong Line: %q, expected %q", msg.Line, stdoutLine) + } + } + if msg.Source == "stderr" { + if string(msg.Line) != stderrLine { + t.Fatalf("Wrong Line: %q, expected %q", msg.Line, stderrLine) + } + } + } +} diff --git a/daemon/logger/logger.go b/daemon/logger/logger.go new file mode 100644 index 0000000000..078e67d8e9 --- /dev/null +++ b/daemon/logger/logger.go @@ -0,0 +1,18 @@ +package logger + +import "time" + +// Message is datastructure that represents record from some container +type Message struct { + ContainerID string + Line []byte + Source string + Timestamp time.Time +} + +// Logger is interface for docker logging drivers +type Logger interface { + Log(*Message) error + Name() string + Close() error +} From 47a6afb93f7f71c7ef7dc692c0f50ac81b5a8c98 Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Wed, 4 Feb 2015 11:04:58 -0800 Subject: [PATCH 2/4] Default 'json-file' logging driver and none logging driver Signed-off-by: Alexander Morozov --- daemon/config.go | 3 + daemon/container.go | 43 +++++++--- daemon/daemon.go | 70 +++++++++-------- daemon/inspect.go | 8 ++ daemon/logger/jsonfilelog/jsonfilelog.go | 49 ++++++++++++ daemon/logger/jsonfilelog/jsonfilelog_test.go | 78 +++++++++++++++++++ daemon/monitor.go | 7 +- .../reference/api/docker_remote_api_v1.18.md | 7 +- docs/sources/reference/commandline/cli.md | 3 + docs/sources/reference/run.md | 12 +++ integration/utils_test.go | 1 + runconfig/hostconfig.go | 9 ++- runconfig/parse.go | 2 + 13 files changed, 242 insertions(+), 50 deletions(-) create mode 100644 daemon/logger/jsonfilelog/jsonfilelog.go create mode 100644 daemon/logger/jsonfilelog/jsonfilelog_test.go diff --git a/daemon/config.go b/daemon/config.go index 94deb3424c..4adc025eef 100644 --- a/daemon/config.go +++ b/daemon/config.go @@ -7,6 +7,7 @@ import ( "github.com/docker/docker/opts" flag "github.com/docker/docker/pkg/mflag" "github.com/docker/docker/pkg/ulimit" + "github.com/docker/docker/runconfig" ) const ( @@ -47,6 +48,7 @@ type Config struct { TrustKeyPath string Labels []string Ulimits map[string]*ulimit.Ulimit + LogConfig runconfig.LogConfig } // InstallFlags adds command-line options to the top-level flag parser for @@ -81,6 +83,7 @@ func (config *Config) InstallFlags() { opts.LabelListVar(&config.Labels, []string{"-label"}, "Set key=value labels to the daemon") config.Ulimits = make(map[string]*ulimit.Ulimit) opts.UlimitMapVar(config.Ulimits, []string{"-default-ulimit"}, "Set default ulimits for containers") + flag.StringVar(&config.LogConfig.Type, []string{"-log-driver"}, "json-file", "Containers logging driver(json-file/none)") } func getDefaultNetworkMtu() int { diff --git a/daemon/container.go b/daemon/container.go index a065d00b57..604fbe882f 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -21,6 +21,8 @@ import ( log "github.com/Sirupsen/logrus" "github.com/docker/docker/daemon/execdriver" + "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/daemon/logger/jsonfilelog" "github.com/docker/docker/engine" "github.com/docker/docker/image" "github.com/docker/docker/links" @@ -98,9 +100,11 @@ type Container struct { VolumesRW map[string]bool hostConfig *runconfig.HostConfig - activeLinks map[string]*links.Link - monitor *containerMonitor - execCommands *execStore + activeLinks map[string]*links.Link + monitor *containerMonitor + execCommands *execStore + // logDriver for closing + logDriver logger.Logger AppliedVolumesFrom map[string]struct{} } @@ -1355,21 +1359,36 @@ func (container *Container) setupWorkingDirectory() error { return nil } -func (container *Container) startLoggingToDisk() error { - // Setup logging of stdout and stderr to disk - logPath, err := container.logPath("json") - if err != nil { - return err +func (container *Container) startLogging() error { + cfg := container.hostConfig.LogConfig + if cfg.Type == "" { + cfg = container.daemon.defaultLogConfig } - container.LogPath = logPath + var l logger.Logger + switch cfg.Type { + case "json-file": + pth, err := container.logPath("json") + if err != nil { + return err + } - if err := container.daemon.LogToDisk(container.stdout, container.LogPath, "stdout"); err != nil { - return err + dl, err := jsonfilelog.New(pth) + if err != nil { + return err + } + l = dl + case "none": + return nil + default: + return fmt.Errorf("Unknown logging driver: %s", cfg.Type) } - if err := container.daemon.LogToDisk(container.stderr, container.LogPath, "stderr"); err != nil { + if copier, err := logger.NewCopier(container.ID, map[string]io.Reader{"stdout": container.StdoutPipe(), "stderr": container.StderrPipe()}, l); err != nil { return err + } else { + copier.Run() } + container.logDriver = l return nil } diff --git a/daemon/daemon.go b/daemon/daemon.go index 7c26de50cc..4d30aa4a6c 100644 --- a/daemon/daemon.go +++ b/daemon/daemon.go @@ -89,23 +89,24 @@ func (c *contStore) List() []*Container { } type Daemon struct { - ID string - repository string - sysInitPath string - containers *contStore - execCommands *execStore - graph *graph.Graph - repositories *graph.TagStore - idIndex *truncindex.TruncIndex - sysInfo *sysinfo.SysInfo - volumes *volumes.Repository - eng *engine.Engine - config *Config - containerGraph *graphdb.Database - driver graphdriver.Driver - execDriver execdriver.Driver - trustStore *trust.TrustStore - statsCollector *statsCollector + ID string + repository string + sysInitPath string + containers *contStore + execCommands *execStore + graph *graph.Graph + repositories *graph.TagStore + idIndex *truncindex.TruncIndex + sysInfo *sysinfo.SysInfo + volumes *volumes.Repository + eng *engine.Engine + config *Config + containerGraph *graphdb.Database + driver graphdriver.Driver + execDriver execdriver.Driver + trustStore *trust.TrustStore + statsCollector *statsCollector + defaultLogConfig runconfig.LogConfig } // Install installs daemon capabilities to eng. @@ -984,23 +985,24 @@ func NewDaemonFromDirectory(config *Config, eng *engine.Engine) (*Daemon, error) } daemon := &Daemon{ - ID: trustKey.PublicKey().KeyID(), - repository: daemonRepo, - containers: &contStore{s: make(map[string]*Container)}, - execCommands: newExecStore(), - graph: g, - repositories: repositories, - idIndex: truncindex.NewTruncIndex([]string{}), - sysInfo: sysInfo, - volumes: volumes, - config: config, - containerGraph: graph, - driver: driver, - sysInitPath: sysInitPath, - execDriver: ed, - eng: eng, - trustStore: t, - statsCollector: newStatsCollector(1 * time.Second), + ID: trustKey.PublicKey().KeyID(), + repository: daemonRepo, + containers: &contStore{s: make(map[string]*Container)}, + execCommands: newExecStore(), + graph: g, + repositories: repositories, + idIndex: truncindex.NewTruncIndex([]string{}), + sysInfo: sysInfo, + volumes: volumes, + config: config, + containerGraph: graph, + driver: driver, + sysInitPath: sysInitPath, + execDriver: ed, + eng: eng, + trustStore: t, + statsCollector: newStatsCollector(1 * time.Second), + defaultLogConfig: config.LogConfig, } // Setup shutdown handlers diff --git a/daemon/inspect.go b/daemon/inspect.go index df68881431..08265795ec 100644 --- a/daemon/inspect.go +++ b/daemon/inspect.go @@ -62,6 +62,14 @@ func (daemon *Daemon) ContainerInspect(job *engine.Job) engine.Status { container.hostConfig.Links = append(container.hostConfig.Links, fmt.Sprintf("%s:%s", child.Name, linkAlias)) } } + // we need this trick to preserve empty log driver, so + // container will use daemon defaults even if daemon change them + if container.hostConfig.LogConfig.Type == "" { + container.hostConfig.LogConfig = daemon.defaultLogConfig + defer func() { + container.hostConfig.LogConfig = runconfig.LogConfig{} + }() + } out.SetJson("HostConfig", container.hostConfig) diff --git a/daemon/logger/jsonfilelog/jsonfilelog.go b/daemon/logger/jsonfilelog/jsonfilelog.go new file mode 100644 index 0000000000..c1d1e0f84e --- /dev/null +++ b/daemon/logger/jsonfilelog/jsonfilelog.go @@ -0,0 +1,49 @@ +package jsonfilelog + +import ( + "bytes" + "os" + + "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/pkg/jsonlog" +) + +// JSONFileLogger is Logger implementation for default docker logging: +// JSON objects to file +type JSONFileLogger struct { + buf *bytes.Buffer + f *os.File // store for closing +} + +// New creates new JSONFileLogger which writes to filename +func New(filename string) (logger.Logger, error) { + log, err := os.OpenFile(filename, os.O_RDWR|os.O_APPEND|os.O_CREATE, 0600) + if err != nil { + return nil, err + } + return &JSONFileLogger{ + f: log, + buf: bytes.NewBuffer(nil), + }, nil +} + +// Log converts logger.Message to jsonlog.JSONLog and serializes it to file +func (l *JSONFileLogger) Log(msg *logger.Message) error { + err := (&jsonlog.JSONLog{Log: string(msg.Line) + "\n", Stream: msg.Source, Created: msg.Timestamp}).MarshalJSONBuf(l.buf) + if err != nil { + return err + } + l.buf.WriteByte('\n') + _, err = l.buf.WriteTo(l.f) + return err +} + +// Close closes underlying file +func (l *JSONFileLogger) Close() error { + return l.f.Close() +} + +// Name returns name of this logger +func (l *JSONFileLogger) Name() string { + return "JSONFile" +} diff --git a/daemon/logger/jsonfilelog/jsonfilelog_test.go b/daemon/logger/jsonfilelog/jsonfilelog_test.go new file mode 100644 index 0000000000..e951c1b869 --- /dev/null +++ b/daemon/logger/jsonfilelog/jsonfilelog_test.go @@ -0,0 +1,78 @@ +package jsonfilelog + +import ( + "io/ioutil" + "os" + "path/filepath" + "testing" + "time" + + "github.com/docker/docker/daemon/logger" + "github.com/docker/docker/pkg/jsonlog" +) + +func TestJSONFileLogger(t *testing.T) { + tmp, err := ioutil.TempDir("", "docker-logger-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmp) + filename := filepath.Join(tmp, "container.log") + l, err := New(filename) + if err != nil { + t.Fatal(err) + } + defer l.Close() + cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657" + if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line1"), Source: "src1"}); err != nil { + t.Fatal(err) + } + if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line2"), Source: "src2"}); err != nil { + t.Fatal(err) + } + if err := l.Log(&logger.Message{ContainerID: cid, Line: []byte("line3"), Source: "src3"}); err != nil { + t.Fatal(err) + } + res, err := ioutil.ReadFile(filename) + if err != nil { + t.Fatal(err) + } + expected := `{"log":"line1\n","stream":"src1","time":"0001-01-01T00:00:00Z"} +{"log":"line2\n","stream":"src2","time":"0001-01-01T00:00:00Z"} +{"log":"line3\n","stream":"src3","time":"0001-01-01T00:00:00Z"} +` + + if string(res) != expected { + t.Fatalf("Wrong log content: %q, expected %q", res, expected) + } +} + +func BenchmarkJSONFileLogger(b *testing.B) { + tmp, err := ioutil.TempDir("", "docker-logger-") + if err != nil { + b.Fatal(err) + } + defer os.RemoveAll(tmp) + filename := filepath.Join(tmp, "container.log") + l, err := New(filename) + if err != nil { + b.Fatal(err) + } + defer l.Close() + cid := "a7317399f3f857173c6179d44823594f8294678dea9999662e5c625b5a1c7657" + testLine := "Line that thinks that it is log line from docker\n" + msg := &logger.Message{ContainerID: cid, Line: []byte(testLine), Source: "stderr", Timestamp: time.Now().UTC()} + jsonlog, err := (&jsonlog.JSONLog{Log: string(msg.Line) + "\n", Stream: msg.Source, Created: msg.Timestamp}).MarshalJSON() + if err != nil { + b.Fatal(err) + } + b.SetBytes(int64(len(jsonlog)+1) * 30) + b.ResetTimer() + for i := 0; i < b.N; i++ { + for j := 0; j < 30; j++ { + if err := l.Log(msg); err != nil { + b.Fatal(err) + } + } + } +} diff --git a/daemon/monitor.go b/daemon/monitor.go index 78d3d226e0..97d451310b 100644 --- a/daemon/monitor.go +++ b/daemon/monitor.go @@ -123,7 +123,7 @@ func (m *containerMonitor) Start() error { for { m.container.RestartCount++ - if err := m.container.startLoggingToDisk(); err != nil { + if err := m.container.startLogging(); err != nil { m.resetContainer(false) return err @@ -302,6 +302,11 @@ func (m *containerMonitor) resetContainer(lock bool) { container.stdin, container.stdinPipe = io.Pipe() } + if container.logDriver != nil { + container.logDriver.Close() + container.logDriver = nil + } + c := container.command.ProcessConfig.Cmd container.command.ProcessConfig.Cmd = exec.Cmd{ diff --git a/docs/sources/reference/api/docker_remote_api_v1.18.md b/docs/sources/reference/api/docker_remote_api_v1.18.md index 46351ed85c..2573f0be49 100644 --- a/docs/sources/reference/api/docker_remote_api_v1.18.md +++ b/docs/sources/reference/api/docker_remote_api_v1.18.md @@ -156,7 +156,8 @@ Create a container "RestartPolicy": { "Name": "", "MaximumRetryCount": 0 }, "NetworkMode": "bridge", "Devices": [], - "Ulimits": [{}] + "Ulimits": [{}], + "LogConfig": { "Type": "json-file", Config: {} } } } @@ -248,6 +249,9 @@ Json Parameters: - **Ulimits** - A list of ulimits to be set in the container, specified as `{ "Name": , "Soft": , "Hard": }`, for example: `Ulimits: { "Name": "nofile", "Soft": 1024, "Hard", 2048 }}` + - **LogConfig** - Logging configuration to container, format + `{ "Type": "", "Config": {"key1": "val1"}} + Available types: `json-file`, `none`. Query Parameters: @@ -340,6 +344,7 @@ Return low-level information on the container `id` "MaximumRetryCount": 2, "Name": "on-failure" }, + "LogConfig": { "Type": "json-file", Config: {} }, "SecurityOpt": null, "VolumesFrom": null, "Ulimits": [{}] diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index 1f61432b5a..f6b7491d06 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -97,6 +97,7 @@ expect an integer, and they can only be specified once. --ipv6=false Enable IPv6 networking -l, --log-level="info" Set the logging level --label=[] Set key=value labels to the daemon + --log-driver="json-file" Container's logging driver (json-file/none) --mtu=0 Set the containers network MTU -p, --pidfile="/var/run/docker.pid" Path to use for daemon PID file --registry-mirror=[] Preferred Docker registry mirror @@ -792,6 +793,7 @@ Creates a new container. -i, --interactive=false Keep STDIN open even if not attached --ipc="" IPC namespace to use --link=[] Add link to another container + --log-driver="" Logging driver for container --lxc-conf=[] Add custom lxc options -m, --memory="" Memory limit --mac-address="" Container MAC address (e.g. 92:d0:c6:0a:29:33) @@ -1660,6 +1662,7 @@ removed before the image is removed. -i, --interactive=false Keep STDIN open even if not attached --ipc="" IPC namespace to use --link=[] Add link to another container + --log-driver="" Logging driver for container --lxc-conf=[] Add custom lxc options -m, --memory="" Memory limit --mac-address="" Container MAC address (e.g. 92:d0:c6:0a:29:33) diff --git a/docs/sources/reference/run.md b/docs/sources/reference/run.md index f06ca968ea..6ccfc5c3fe 100644 --- a/docs/sources/reference/run.md +++ b/docs/sources/reference/run.md @@ -561,6 +561,18 @@ familiar with using LXC directly. > you can use `--lxc-conf` to set a container's IP address, but this will not be > reflected in the `/etc/hosts` file. +## Logging drivers (--log-driver) + +You can specify a different logging driver for the container than for the daemon. + +### Log driver: none + +Disables any logging for the container. + +### Log driver: json-file + +Default logging driver for Docker. Writes JSON messages to file. + ## Overriding Dockerfile image defaults When a developer builds an image from a [*Dockerfile*](/reference/builder) diff --git a/integration/utils_test.go b/integration/utils_test.go index d5a068fe02..2e90e4f515 100644 --- a/integration/utils_test.go +++ b/integration/utils_test.go @@ -191,6 +191,7 @@ func newTestEngine(t Fataler, autorestart bool, root string) *engine.Engine { // otherwise NewDaemon will fail because of conflicting settings. InterContainerCommunication: true, TrustKeyPath: filepath.Join(root, "key.json"), + LogConfig: runconfig.LogConfig{Type: "json-file"}, } d, err := daemon.NewDaemon(cfg, eng) if err != nil { diff --git a/runconfig/hostconfig.go b/runconfig/hostconfig.go index 85db438b7d..3e4cd9624a 100644 --- a/runconfig/hostconfig.go +++ b/runconfig/hostconfig.go @@ -99,6 +99,11 @@ type RestartPolicy struct { MaximumRetryCount int } +type LogConfig struct { + Type string + Config map[string]string +} + type HostConfig struct { Binds []string ContainerIDFile string @@ -121,6 +126,7 @@ type HostConfig struct { SecurityOpt []string ReadonlyRootfs bool Ulimits []*ulimit.Ulimit + LogConfig LogConfig } // This is used by the create command when you want to set both the @@ -158,9 +164,8 @@ func ContainerHostConfigFromJob(job *engine.Job) *HostConfig { job.GetenvJson("PortBindings", &hostConfig.PortBindings) job.GetenvJson("Devices", &hostConfig.Devices) job.GetenvJson("RestartPolicy", &hostConfig.RestartPolicy) - job.GetenvJson("Ulimits", &hostConfig.Ulimits) - + job.GetenvJson("LogConfig", &hostConfig.LogConfig) hostConfig.SecurityOpt = job.GetenvList("SecurityOpt") if Binds := job.GetenvList("Binds"); Binds != nil { hostConfig.Binds = Binds diff --git a/runconfig/parse.go b/runconfig/parse.go index bf87c42f94..a6ab5725fa 100644 --- a/runconfig/parse.go +++ b/runconfig/parse.go @@ -68,6 +68,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe flIpcMode = cmd.String([]string{"-ipc"}, "", "IPC namespace to use") flRestartPolicy = cmd.String([]string{"-restart"}, "no", "Restart policy to apply when a container exits") flReadonlyRootfs = cmd.Bool([]string{"-read-only"}, false, "Mount the container's root filesystem as read only") + flLoggingDriver = cmd.String([]string{"-log-driver"}, "", "Logging driver for container") ) cmd.Var(&flAttach, []string{"a", "-attach"}, "Attach to STDIN, STDOUT or STDERR") @@ -321,6 +322,7 @@ func Parse(cmd *flag.FlagSet, args []string) (*Config, *HostConfig, *flag.FlagSe SecurityOpt: flSecurityOpt.GetAll(), ReadonlyRootfs: *flReadonlyRootfs, Ulimits: flUlimits.GetList(), + LogConfig: LogConfig{Type: *flLoggingDriver}, } // When allocating stdin in attached mode, close stdin at client disconnect From dd6d2cd6603af51a49e41048511038ed8770d32b Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Wed, 4 Feb 2015 13:44:52 -0800 Subject: [PATCH 3/4] Tests for --log-driver options Signed-off-by: Alexander Morozov --- integration-cli/docker_cli_daemon_test.go | 143 ++++++++++++++++++++++ 1 file changed, 143 insertions(+) diff --git a/integration-cli/docker_cli_daemon_test.go b/integration-cli/docker_cli_daemon_test.go index 3cda179a47..aa2b228cd1 100644 --- a/integration-cli/docker_cli_daemon_test.go +++ b/integration-cli/docker_cli_daemon_test.go @@ -11,6 +11,7 @@ import ( "path/filepath" "strings" "testing" + "time" "github.com/docker/libtrust" ) @@ -560,3 +561,145 @@ func TestDaemonRestartRenameContainer(t *testing.T) { logDone("daemon - rename persists through daemon restart") } + +func TestDaemonLoggingDriverDefault(t *testing.T) { + d := NewDaemon(t) + + if err := d.StartWithBusybox(); err != nil { + t.Fatal(err) + } + defer d.Stop() + + out, err := d.Cmd("run", "-d", "busybox", "echo", "testline") + if err != nil { + t.Fatal(out, err) + } + id := strings.TrimSpace(out) + + if out, err := d.Cmd("wait", id); err != nil { + t.Fatal(out, err) + } + logPath := filepath.Join(d.folder, "graph", "containers", id, id+"-json.log") + + if _, err := os.Stat(logPath); err != nil { + t.Fatal(err) + } + f, err := os.Open(logPath) + if err != nil { + t.Fatal(err) + } + var res struct { + Log string `json:log` + Stream string `json:stream` + Time time.Time `json:time` + } + if err := json.NewDecoder(f).Decode(&res); err != nil { + t.Fatal(err) + } + if res.Log != "testline\n" { + t.Fatalf("Unexpected log line: %q, expected: %q", res.Log, "testline\n") + } + if res.Stream != "stdout" { + t.Fatalf("Unexpected stream: %q, expected: %q", res.Stream, "stdout") + } + if !time.Now().After(res.Time) { + t.Fatalf("Log time %v in future", res.Time) + } + logDone("daemon - default 'json-file' logging driver") +} + +func TestDaemonLoggingDriverDefaultOverride(t *testing.T) { + d := NewDaemon(t) + + if err := d.StartWithBusybox(); err != nil { + t.Fatal(err) + } + defer d.Stop() + + out, err := d.Cmd("run", "-d", "--log-driver=none", "busybox", "echo", "testline") + if err != nil { + t.Fatal(out, err) + } + id := strings.TrimSpace(out) + + if out, err := d.Cmd("wait", id); err != nil { + t.Fatal(out, err) + } + logPath := filepath.Join(d.folder, "graph", "containers", id, id+"-json.log") + + if _, err := os.Stat(logPath); err == nil || !os.IsNotExist(err) { + t.Fatalf("%s shouldn't exits, error on Stat: %s", logPath, err) + } + logDone("daemon - default logging driver override in run") +} + +func TestDaemonLoggingDriverNone(t *testing.T) { + d := NewDaemon(t) + + if err := d.StartWithBusybox("--log-driver=none"); err != nil { + t.Fatal(err) + } + defer d.Stop() + + out, err := d.Cmd("run", "-d", "busybox", "echo", "testline") + if err != nil { + t.Fatal(out, err) + } + id := strings.TrimSpace(out) + if out, err := d.Cmd("wait", id); err != nil { + t.Fatal(out, err) + } + + logPath := filepath.Join(d.folder, "graph", "containers", id, id+"-json.log") + + if _, err := os.Stat(logPath); err == nil || !os.IsNotExist(err) { + t.Fatalf("%s shouldn't exits, error on Stat: %s", logPath, err) + } + logDone("daemon - 'none' logging driver") +} + +func TestDaemonLoggingDriverNoneOverride(t *testing.T) { + d := NewDaemon(t) + + if err := d.StartWithBusybox("--log-driver=none"); err != nil { + t.Fatal(err) + } + defer d.Stop() + + out, err := d.Cmd("run", "-d", "--log-driver=json-file", "busybox", "echo", "testline") + if err != nil { + t.Fatal(out, err) + } + id := strings.TrimSpace(out) + + if out, err := d.Cmd("wait", id); err != nil { + t.Fatal(out, err) + } + logPath := filepath.Join(d.folder, "graph", "containers", id, id+"-json.log") + + if _, err := os.Stat(logPath); err != nil { + t.Fatal(err) + } + f, err := os.Open(logPath) + if err != nil { + t.Fatal(err) + } + var res struct { + Log string `json:log` + Stream string `json:stream` + Time time.Time `json:time` + } + if err := json.NewDecoder(f).Decode(&res); err != nil { + t.Fatal(err) + } + if res.Log != "testline\n" { + t.Fatalf("Unexpected log line: %q, expected: %q", res.Log, "testline\n") + } + if res.Stream != "stdout" { + t.Fatalf("Unexpected stream: %q, expected: %q", res.Stream, "stdout") + } + if !time.Now().After(res.Time) { + t.Fatalf("Log time %v in future", res.Time) + } + logDone("daemon - 'none' logging driver override in run") +} From bdf3a0295d401624483d4103fdaacc2bea1c7d46 Mon Sep 17 00:00:00 2001 From: Alexander Morozov Date: Thu, 5 Feb 2015 16:24:47 -0800 Subject: [PATCH 4/4] Fail docker logs on all logging drivers apart from 'json-file' Signed-off-by: Alexander Morozov --- api/client/commands.go | 4 ++++ daemon/container.go | 9 ++++++++ daemon/logs.go | 3 +++ docs/man/docker-create.1.md | 5 ++++ docs/man/docker-logs.1.md | 2 ++ docs/man/docker-run.1.md | 5 ++++ docs/man/docker.1.md | 4 ++++ .../reference/api/docker_remote_api_v1.18.md | 4 ++++ docs/sources/reference/commandline/cli.md | 3 +++ docs/sources/reference/run.md | 8 ++++--- integration-cli/docker_cli_daemon_test.go | 23 +++++++++++++++++++ 11 files changed, 67 insertions(+), 3 deletions(-) diff --git a/api/client/commands.go b/api/client/commands.go index e10bee3b18..8a52b94941 100644 --- a/api/client/commands.go +++ b/api/client/commands.go @@ -1929,6 +1929,10 @@ func (cli *DockerCli) CmdLogs(args ...string) error { return err } + if env.GetSubEnv("HostConfig").GetSubEnv("LogConfig").Get("Type") != "json-file" { + return fmt.Errorf("\"logs\" command is supported only for \"json-file\" logging driver") + } + v := url.Values{} v.Set("stdout", "1") v.Set("stderr", "1") diff --git a/daemon/container.go b/daemon/container.go index 604fbe882f..57c0071a5d 100644 --- a/daemon/container.go +++ b/daemon/container.go @@ -1489,3 +1489,12 @@ func (container *Container) getNetworkedContainer() (*Container, error) { func (container *Container) Stats() (*execdriver.ResourceStats, error) { return container.daemon.Stats(container) } + +func (c *Container) LogDriverType() string { + c.Lock() + defer c.Unlock() + if c.hostConfig.LogConfig.Type == "" { + return c.daemon.defaultLogConfig.Type + } + return c.hostConfig.LogConfig.Type +} diff --git a/daemon/logs.go b/daemon/logs.go index db977ddac1..356d08c5c8 100644 --- a/daemon/logs.go +++ b/daemon/logs.go @@ -44,6 +44,9 @@ func (daemon *Daemon) ContainerLogs(job *engine.Job) engine.Status { if err != nil { return job.Error(err) } + if container.LogDriverType() != "json-file" { + return job.Errorf("\"logs\" endpoint is supported only for \"json-file\" logging driver") + } cLog, err := container.ReadLog("json") if err != nil && os.IsNotExist(err) { // Legacy logs diff --git a/docs/man/docker-create.1.md b/docs/man/docker-create.1.md index d5166ca93a..bf6c1a0002 100644 --- a/docs/man/docker-create.1.md +++ b/docs/man/docker-create.1.md @@ -26,6 +26,7 @@ docker-create - Create a new container [**--ipc**[=*IPC*]] [**--link**[=*[]*]] [**--lxc-conf**[=*[]*]] +[**--log-driver**[=*[]*]] [**-m**|**--memory**[=*MEMORY*]] [**--memory-swap**[=*MEMORY-SWAP*]] [**--mac-address**[=*MAC-ADDRESS*]] @@ -108,6 +109,10 @@ IMAGE [COMMAND] [ARG...] **--lxc-conf**=[] (lxc exec-driver only) Add custom lxc options --lxc-conf="lxc.cgroup.cpuset.cpus = 0,1" +**--log-driver**="|*json-file*|*none*" + Logging driver for container. Default is defined by daemon `--log-driver` flag. + **Warning**: `docker logs` command works only for `json-file` logging driver. + **-m**, **--memory**="" Memory limit (format: , where unit = b, k, m or g) diff --git a/docs/man/docker-logs.1.md b/docs/man/docker-logs.1.md index d55e8d8365..01a15f54dc 100644 --- a/docs/man/docker-logs.1.md +++ b/docs/man/docker-logs.1.md @@ -22,6 +22,8 @@ The **docker logs --follow** command combines commands **docker logs** and **docker attach**. It will first return all logs from the beginning and then continue streaming new output from the container’s stdout and stderr. +**Warning**: This command works only for **json-file** logging driver. + # OPTIONS **--help** Print usage statement diff --git a/docs/man/docker-run.1.md b/docs/man/docker-run.1.md index 587b9a2cd5..36ae9ce422 100644 --- a/docs/man/docker-run.1.md +++ b/docs/man/docker-run.1.md @@ -27,6 +27,7 @@ docker-run - Run a command in a new container [**--ipc**[=*IPC*]] [**--link**[=*[]*]] [**--lxc-conf**[=*[]*]] +[**--log-driver**[=*[]*]] [**-m**|**--memory**[=*MEMORY*]] [**--memory-swap**[=*MEMORY-SWAP]] [**--mac-address**[=*MAC-ADDRESS*]] @@ -209,6 +210,10 @@ which interface and port to use. **--lxc-conf**=[] (lxc exec-driver only) Add custom lxc options --lxc-conf="lxc.cgroup.cpuset.cpus = 0,1" +**--log-driver**="|*json-file*|*none*" + Logging driver for container. Default is defined by daemon `--log-driver` flag. + **Warning**: `docker logs` command works only for `json-file` logging driver. + **-m**, **--memory**="" Memory limit (format: , where unit = b, k, m or g) diff --git a/docs/man/docker.1.md b/docs/man/docker.1.md index bad6cf0494..c322c79ec9 100644 --- a/docs/man/docker.1.md +++ b/docs/man/docker.1.md @@ -82,6 +82,10 @@ unix://[/path/to/socket] to use. **--label**="[]" Set key=value labels to the daemon (displayed in `docker info`) +**--log-driver**="*json-file*|*none*" + Container's logging driver. Default is `default`. + **Warning**: `docker logs` command works only for `json-file` logging driver. + **--mtu**=VALUE Set the containers network mtu. Default is `1500`. diff --git a/docs/sources/reference/api/docker_remote_api_v1.18.md b/docs/sources/reference/api/docker_remote_api_v1.18.md index 2573f0be49..7395e824c7 100644 --- a/docs/sources/reference/api/docker_remote_api_v1.18.md +++ b/docs/sources/reference/api/docker_remote_api_v1.18.md @@ -252,6 +252,7 @@ Json Parameters: - **LogConfig** - Logging configuration to container, format `{ "Type": "", "Config": {"key1": "val1"}} Available types: `json-file`, `none`. + `json-file` logging driver. Query Parameters: @@ -441,6 +442,9 @@ Status Codes: Get stdout and stderr logs from the container ``id`` +> **Note**: +> This endpoint works only for containers with `json-file` logging driver. + **Example request**: GET /containers/4fa6e0f0c678/logs?stderr=1&stdout=1×tamps=1&follow=1&tail=10 HTTP/1.1 diff --git a/docs/sources/reference/commandline/cli.md b/docs/sources/reference/commandline/cli.md index f6b7491d06..da274ca109 100644 --- a/docs/sources/reference/commandline/cli.md +++ b/docs/sources/reference/commandline/cli.md @@ -1405,6 +1405,9 @@ For example: -t, --timestamps=false Show timestamps --tail="all" Number of lines to show from the end of the logs +NOTE: this command is available only for containers with `json-file` logging +driver. + The `docker logs` command batch-retrieves logs present at the time of execution. The `docker logs --follow` command will continue streaming the new output from diff --git a/docs/sources/reference/run.md b/docs/sources/reference/run.md index 6ccfc5c3fe..c578307032 100644 --- a/docs/sources/reference/run.md +++ b/docs/sources/reference/run.md @@ -565,13 +565,15 @@ familiar with using LXC directly. You can specify a different logging driver for the container than for the daemon. -### Log driver: none +### Logging driver: none -Disables any logging for the container. +Disables any logging for the container. `docker logs` won't be available with +this driver. ### Log driver: json-file -Default logging driver for Docker. Writes JSON messages to file. +Default logging driver for Docker. Writes JSON messages to file. `docker logs` +command is available only for this logging driver ## Overriding Dockerfile image defaults diff --git a/integration-cli/docker_cli_daemon_test.go b/integration-cli/docker_cli_daemon_test.go index aa2b228cd1..552342a687 100644 --- a/integration-cli/docker_cli_daemon_test.go +++ b/integration-cli/docker_cli_daemon_test.go @@ -703,3 +703,26 @@ func TestDaemonLoggingDriverNoneOverride(t *testing.T) { } logDone("daemon - 'none' logging driver override in run") } + +func TestDaemonLoggingDriverNoneLogsError(t *testing.T) { + d := NewDaemon(t) + + if err := d.StartWithBusybox("--log-driver=none"); err != nil { + t.Fatal(err) + } + defer d.Stop() + + out, err := d.Cmd("run", "-d", "busybox", "echo", "testline") + if err != nil { + t.Fatal(out, err) + } + id := strings.TrimSpace(out) + out, err = d.Cmd("logs", id) + if err == nil { + t.Fatalf("Logs should fail with \"none\" driver") + } + if !strings.Contains(out, `\"logs\" command is supported only for \"json-file\" logging driver`) { + t.Fatalf("There should be error about non-json-file driver, got %s", out) + } + logDone("daemon - logs not available for non-json-file drivers") +}