daemon/logger: Add logging driver for Google Cloud Logging

Signed-off-by: Mike Danese <mikedanese@google.com>

contrib/completion/bash/docker

@@ -397,6 +397,7 @@ __docker_complete_log_drivers() {
 		awslogs
 		etwlogs
 		fluentd
+		gcplogs
 		gelf
 		journald
 		json-file
@@ -410,13 +411,14 @@ __docker_complete_log_options() {
 	# see docs/reference/logging/index.md
 	local awslogs_options="awslogs-region awslogs-group awslogs-stream"
 	local fluentd_options="env fluentd-address labels tag"
+	local gcplogs_options="env gcp-log-cmd gcp-project labels"
 	local gelf_options="env gelf-address labels tag"
 	local journald_options="env labels tag"
 	local json_file_options="env labels max-file max-size"
 	local syslog_options="syslog-address syslog-tls-ca-cert syslog-tls-cert syslog-tls-key syslog-tls-skip-verify syslog-facility tag"
 	local splunk_options="env labels splunk-caname splunk-capath splunk-index splunk-insecureskipverify splunk-source splunk-sourcetype splunk-token splunk-url tag"
 
-	local all_options="$fluentd_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options"
+	local all_options="$fluentd_options $gcplogs_options $gelf_options $journald_options $json_file_options $syslog_options $splunk_options"
 
 	case $(__docker_value_of_option --log-driver) in
 		'')
@@ -428,6 +430,9 @@ __docker_complete_log_options() {
 		fluentd)
 			COMPREPLY=( $( compgen -W "$fluentd_options" -S = -- "$cur" ) )
 			;;
+		gcplogs)
+			COMPREPLY=( $( compgen -W "$gcplogs_options" -S = -- "$cur" ) )
+			;;
 		gelf)
 			COMPREPLY=( $( compgen -W "$gelf_options" -S = -- "$cur" ) )
 			;;

contrib/completion/zsh/_docker

@@ -201,6 +201,7 @@ __docker_get_log_options() {
 
     awslogs_options=("awslogs-region" "awslogs-group" "awslogs-stream")
     fluentd_options=("env" "fluentd-address" "labels" "tag")
+    gcplogs_options=("env" "gcp-log-cmd" "gcp-project" "labels")
     gelf_options=("env" "gelf-address" "labels" "tag")
     journald_options=("env" "labels")
     json_file_options=("env" "labels" "max-file" "max-size")
@@ -209,6 +210,7 @@ __docker_get_log_options() {
 
     [[ $log_driver = (awslogs|all) ]] && _describe -t awslogs-options "awslogs options" awslogs_options "$@" && ret=0
     [[ $log_driver = (fluentd|all) ]] && _describe -t fluentd-options "fluentd options" fluentd_options "$@" && ret=0
+    [[ $log_driver = (gcplogs|all) ]] && _describe -t gcplogs-options "gcplogs options" gcplogs_options "$@" && ret=0
     [[ $log_driver = (gelf|all) ]] && _describe -t gelf-options "gelf options" gelf_options "$@" && ret=0
     [[ $log_driver = (journald|all) ]] && _describe -t journald-options "journald options" journald_options "$@" && ret=0
     [[ $log_driver = (json-file|all) ]] && _describe -t json-file-options "json-file options" json_file_options "$@" && ret=0

daemon/logdrivers_linux.go

@@ -5,6 +5,7 @@ import (
 	// therefore they register themselves to the logdriver factory.
 	_ "github.com/docker/docker/daemon/logger/awslogs"
 	_ "github.com/docker/docker/daemon/logger/fluentd"
+	_ "github.com/docker/docker/daemon/logger/gcplogs"
 	_ "github.com/docker/docker/daemon/logger/gelf"
 	_ "github.com/docker/docker/daemon/logger/journald"
 	_ "github.com/docker/docker/daemon/logger/jsonfilelog"

daemon/logger/gcplogs/gcplogging.go

@@ -0,0 +1,181 @@
+package gcplogs
+
+import (
+	"fmt"
+	"sync/atomic"
+	"time"
+
+	"github.com/docker/docker/daemon/logger"
+
+	"github.com/Sirupsen/logrus"
+	"golang.org/x/net/context"
+	"google.golang.org/cloud/compute/metadata"
+	"google.golang.org/cloud/logging"
+)
+
+const (
+	name = "gcplogs"
+
+	projectOptKey = "gcp-project"
+	logLabelsKey  = "labels"
+	logEnvKey     = "env"
+	logCmdKey     = "gcp-log-cmd"
+)
+
+var (
+	// The number of logs the gcplogs driver has dropped.
+	droppedLogs uint64
+
+	onGCE = metadata.OnGCE()
+
+	// instance metadata populated from the metadata server if available
+	projectID    string
+	zone         string
+	instanceName string
+	instanceID   string
+)
+
+func init() {
+	if onGCE {
+		// These will fail on instances if the metadata service is
+		// down or the client is compiled with an API version that
+		// has been removed. Since these are not vital, let's ignore
+		// them and make their fields in the dockeLogEntry ,omitempty
+		projectID, _ = metadata.ProjectID()
+		zone, _ = metadata.Zone()
+		instanceName, _ = metadata.InstanceName()
+		instanceID, _ = metadata.InstanceID()
+	}
+
+	if err := logger.RegisterLogDriver(name, New); err != nil {
+		logrus.Fatal(err)
+	}
+
+	if err := logger.RegisterLogOptValidator(name, ValidateLogOpts); err != nil {
+		logrus.Fatal(err)
+	}
+}
+
+type gcplogs struct {
+	client    *logging.Client
+	instance  *instanceInfo
+	container *containerInfo
+}
+
+type dockerLogEntry struct {
+	Instance  *instanceInfo  `json:"instance,omitempty"`
+	Container *containerInfo `json:"container,omitempty"`
+	Data      string         `json:"data,omitempty"`
+}
+
+type instanceInfo struct {
+	Zone string `json:"zone,omitempty"`
+	Name string `json:"name,omitempty"`
+	ID   string `json:"id,omitempty"`
+}
+
+type containerInfo struct {
+	Name      string            `json:"name,omitempty"`
+	ID        string            `json:"id,omitempty"`
+	ImageName string            `json:"imageName,omitempty"`
+	ImageID   string            `json:"imageId,omitempty"`
+	Created   time.Time         `json:"created,omitempty"`
+	Command   string            `json:"command,omitempty"`
+	Metadata  map[string]string `json:"metadata,omitempty"`
+}
+
+// New creates a new logger that logs to Google Cloud Logging using the application
+// default credentials.
+//
+// See https://developers.google.com/identity/protocols/application-default-credentials
+func New(ctx logger.Context) (logger.Logger, error) {
+
+	var project string
+	if projectID != "" {
+		project = projectID
+	}
+	if projectID, found := ctx.Config[projectOptKey]; found {
+		project = projectID
+	}
+	if project == "" {
+		return nil, fmt.Errorf("No project was specified and couldn't read project from the meatadata server. Please specify a project")
+	}
+
+	c, err := logging.NewClient(context.Background(), project, "gcplogs-docker-driver")
+	if err != nil {
+		return nil, err
+	}
+
+	if err := c.Ping(); err != nil {
+		return nil, fmt.Errorf("unable to connect or authenticate with Google Cloud Logging: %v", err)
+	}
+
+	l := &gcplogs{
+		client: c,
+		container: &containerInfo{
+			Name:      ctx.ContainerName,
+			ID:        ctx.ContainerID,
+			ImageName: ctx.ContainerImageName,
+			ImageID:   ctx.ContainerImageID,
+			Created:   ctx.ContainerCreated,
+			Metadata:  ctx.ExtraAttributes(nil),
+		},
+	}
+
+	if ctx.Config[logCmdKey] == "true" {
+		l.container.Command = ctx.Command()
+	}
+
+	if onGCE {
+		l.instance = &instanceInfo{
+			Zone: zone,
+			Name: instanceName,
+			ID:   instanceID,
+		}
+	}
+
+	// The logger "overflows" at a rate of 10,000 logs per second and this
+	// overflow func is called. We want to surface the error to the user
+	// without overly spamming /var/log/docker.log so we log the first time
+	// we overflow and every 1000th time after.
+	c.Overflow = func(_ *logging.Client, _ logging.Entry) error {
+		if i := atomic.AddUint64(&droppedLogs, 1); i%1000 == 1 {
+			logrus.Errorf("gcplogs driver has dropped %v logs", i)
+		}
+		return nil
+	}
+
+	return l, nil
+}
+
+// ValidateLogOpts validates the opts passed to the gcplogs driver. Currently, the gcplogs
+// driver doesn't take any arguments.
+func ValidateLogOpts(cfg map[string]string) error {
+	for k := range cfg {
+		switch k {
+		case projectOptKey, logLabelsKey, logEnvKey, logCmdKey:
+		default:
+			return fmt.Errorf("%q is not a valid option for the gcplogs driver", k)
+		}
+	}
+	return nil
+}
+
+func (l *gcplogs) Log(m *logger.Message) error {
+	return l.client.Log(logging.Entry{
+		Time: m.Timestamp,
+		Payload: &dockerLogEntry{
+			Instance:  l.instance,
+			Container: l.container,
+			Data:      string(m.Line),
+		},
+	})
+}
+
+func (l *gcplogs) Close() error {
+	return l.client.Flush()
+}
+
+func (l *gcplogs) Name() string {
+	return name
+}

docs/admin/logging/gcplogs.md

@@ -0,0 +1,70 @@
+<!--[metadata]>
++++
+title = "Google Cloud Logging driver"
+description = "Describes how to use the Google Cloud Logging driver."
+keywords = ["gcplogs, google, docker, logging, driver"]
+[menu.main]
+parent = "smn_logging"
+weight = 2
++++
+<![end-metadata]-->
+
+# Google Cloud Logging driver
+
+The Google Cloud Logging driver sends container logs to <a href="https://cloud.google.com/logging/docs/" target="_blank">Google Cloud
+Logging</a>.
+
+## Usage
+
+You can configure the default logging driver by passing the `--log-driver`
+option to the Docker daemon:
+
+    docker daemon --log-driver=gcplogs
+
+You can set the logging driver for a specific container by using the
+`--log-driver` option to `docker run`:
+
+    docker run --log-driver=gcplogs ...
+
+This log driver does not implement a reader so it is incompatible with
+`docker logs`.
+
+If Docker detects that it is running in a Google Cloud Project, it will discover configuration
+from the <a href="https://cloud.google.com/compute/docs/metadata" target="_blank">instance metadata service</a>.
+Otherwise, the user must specify which project to log to using the `--gcp-project`
+log option and Docker will attempt to obtain credentials from the
+<a href="https://developers.google.com/identity/protocols/application-default-credentials" target="_blank">Google Application Default Credential</a>.
+The `--gcp-project` takes precedence over information discovered from the metadata server
+so a Docker daemon running in a Google Cloud Project can be overriden to log to a different
+Google Cloud Project using `--gcp-project`.
+
+## gcplogs options
+
+You can use the `--log-opt NAME=VALUE` flag to specify these additional Google
+Cloud Logging driver options:
+
+| Option                      | Required | Description                                                                                                                                 |
+|-----------------------------|----------|---------------------------------------------------------------------------------------------------------------------------------------------|
+| `gcp-project`               | optional | Which GCP project to log to. Defaults to discovering this value from the GCE metadata service.                                              |
+| `gcp-log-cmd`               | optional | Whether to log the command that the container was started with. Defaults to false.                                                          |
+| `labels`                    | optional | Comma-separated list of keys of labels, which should be included in message, if these labels are specified for container.                   |
+| `env`                       | optional | Comma-separated list of keys of environment variables, which should be included in message, if these variables are specified for container. |
+
+If there is collision between `label` and `env` keys, the value of the `env`
+takes precedence. Both options add additional fields to the attributes of a
+logging message.
+
+Below is an example of the logging options required to log to the default
+logging destination which is discovered by querying the GCE metadata server.
+
+    docker run --log-driver=gcplogs \
+        --log-opt labels=location
+        --log-opt env=TEST
+        --log-opt gcp-log-cmd=true
+        --env "TEST=false"
+        --label location=west
+        your/application
+
+This configuration also directs the driver to include in the payload the label
+`location`, the environment variable `ENV`, and the command used to start the
+container.

docs/admin/logging/overview.md

@@ -27,6 +27,7 @@ container's logging driver. The following options are supported:
 | `awslogs`   | Amazon CloudWatch Logs logging driver for Docker. Writes log messages to Amazon CloudWatch Logs.                              |
 | `splunk`    | Splunk logging driver for Docker. Writes log messages to `splunk` using HTTP Event Collector.                                 |
 | `etwlogs`   | ETW logging driver for Docker on Windows. Writes log messages as ETW events.                                                  |
+| `gcplogs`   | Google Cloud Logging driver for Docker. Writes log messages to Google Cloud Logging.                                          |
 
 The `docker logs`command is available only for the `json-file` and `journald`
 logging drivers.
@@ -213,4 +214,14 @@ as an ETW event. An ETW listener can then be created to listen for these events.
 
 For detailed information on working with this logging driver, see [the ETW logging driver](etwlogs.md) reference documentation.
 
+## Google Cloud Logging
 
+The Google Cloud Logging driver supports the following options:
+
+    --log-opt gcp-project=<gcp_projext>
+    --log-opt labels=<label1>,<label2>
+    --log-opt env=<envvar1>,<envvar2>
+    --log-opt log-cmd=true
+
+For detailed information about working with this logging driver, see the [Google Cloud Logging driver](gcplogs.md).
+reference documentation.

man/docker-create.1.md

@@ -214,7 +214,7 @@ millions of trillions.
    Add link to another container in the form of <name or id>:alias or just
    <name or id> in which case the alias will match the name.
 
-**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*none*"
+**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*"
   Logging driver for container. Default is defined by daemon `--log-driver` flag.
   **Warning**: the `docker logs` command works only for the `json-file` and
   `journald` logging drivers.

man/docker-daemon.8.md

@@ -185,7 +185,7 @@ unix://[/path/to/socket] to use.
 **--label**="[]"
   Set key=value labels to the daemon (displayed in `docker info`)
 
-**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*none*"
+**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*"
   Default driver for container logs. Default is `json-file`.
   **Warning**: `docker logs` command works only for `json-file` logging driver.
 

man/docker-run.1.md

@@ -320,7 +320,7 @@ container can access the exposed port via a private networking interface. Docker
 will set some environment variables in the client container to help indicate
 which interface and port to use.
 
-**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*none*"
+**--log-driver**="*json-file*|*syslog*|*journald*|*gelf*|*fluentd*|*awslogs*|*splunk*|*etwlogs*|*gcplogs*|*none*"
   Logging driver for container. Default is defined by daemon `--log-driver` flag.
   **Warning**: the `docker logs` command works only for the `json-file` and
   `journald` logging drivers.