Change Stackdriver resource type for metrics (#210)

* change resource type

* copied PR #188

* add unit tests

* change Fatal back to Error

* solve comments

* change to use map

* modify tests

metrics/config_test.go

@@ -23,7 +23,7 @@ import (
 )
 
 const (
-	servingDomain = "serving.knative.dev"
+	servingDomain = "knative.dev/serving"
 	badDomain     = "test.domain"
 	testComponent = "testComponent"
 	testProj      = "test-project"
@@ -96,16 +96,6 @@ var (
 			cm:        map[string]string{"": ""},
 			domain:    servingDomain,
 			component: testComponent,
-			expectedConfig: metricsConfig{
-				domain:             servingDomain,
-				component:          testComponent,
-				backendDestination: Prometheus,
-				reportingPeriod:    60 * time.Second},
-		}, {
-			name:      "validPrometheus",
-			cm:        map[string]string{"metrics.backend-destination": "prometheus"},
-			domain:    servingDomain,
-			component: testComponent,
 			expectedConfig: metricsConfig{
 				domain:             servingDomain,
 				component:          testComponent,
@@ -123,6 +113,16 @@ var (
 				backendDestination:   Stackdriver,
 				stackdriverProjectID: anotherProj,
 				reportingPeriod:      60 * time.Second},
+		}, {
+			name:      "validPrometheus",
+			cm:        map[string]string{"metrics.backend-destination": "prometheus"},
+			domain:    servingDomain,
+			component: testComponent,
+			expectedConfig: metricsConfig{
+				domain:             servingDomain,
+				component:          testComponent,
+				backendDestination: Prometheus,
+				reportingPeriod:    5 * time.Second},
 		}, {
 			name: "validCapitalStackdriver",
 			cm: map[string]string{"metrics.backend-destination": "Stackdriver",

metrics/exporter.go

@@ -1,12 +1,9 @@
 /*
 Copyright 2018 The Knative Authors
-
 Licensed under the Apache License, Version 2.0 (the "License");
 you may not use this file except in compliance with the License.
 You may obtain a copy of the License at
-
     http://www.apache.org/licenses/LICENSE-2.0
-
 Unless required by applicable law or agreed to in writing, software
 distributed under the License is distributed on an "AS IS" BASIS,
 WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
@@ -22,17 +19,20 @@ import (
 	"sync"
 
 	"contrib.go.opencensus.io/exporter/stackdriver"
+	"contrib.go.opencensus.io/exporter/stackdriver/monitoredresource"
+	"github.com/knative/pkg/metrics/metricskey"
 	"go.opencensus.io/exporter/prometheus"
 	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
 	"go.uber.org/zap"
-	monitoredrespb "google.golang.org/genproto/googleapis/api/monitoredres"
 )
 
 var (
-	curMetricsExporter view.Exporter
-	curMetricsConfig   *metricsConfig
-	curPromSrv         *http.Server
-	metricsMux         sync.Mutex
+	curMetricsExporter       view.Exporter
+	curMetricsConfig         *metricsConfig
+	curPromSrv               *http.Server
+	getMonitoredResourceFunc func(v *view.View, tags []tag.Tag) ([]tag.Tag, monitoredresource.Interface)
+	metricsMux               sync.Mutex
 )
 
 // newMetricsExporter gets a metrics exporter based on the config.
@@ -64,13 +64,60 @@ func newMetricsExporter(config *metricsConfig, logger *zap.SugaredLogger) error
 	return nil
 }
 
+func getKnativeRevisionMonitoredResource(gm *gcpMetadata) func(v *view.View, tags []tag.Tag) ([]tag.Tag, monitoredresource.Interface) {
+	return func(v *view.View, tags []tag.Tag) ([]tag.Tag, monitoredresource.Interface) {
+		tagsMap := getTagsMap(tags)
+		kr := &KnativeRevision{
+			// The first three resource labels are from metadata.
+			Project:     gm.project,
+			Location:    gm.location,
+			ClusterName: gm.cluster,
+			// The rest resource labels are from metrics labels.
+			NamespaceName:     valueOrUnknown(metricskey.LabelNamespaceName, tagsMap),
+			ServiceName:       valueOrUnknown(metricskey.LabelServiceName, tagsMap),
+			ConfigurationName: valueOrUnknown(metricskey.LabelConfigurationName, tagsMap),
+			RevisionName:      valueOrUnknown(metricskey.LabelRevisionName, tagsMap),
+		}
+
+		var newTags []tag.Tag
+		for _, t := range tags {
+			// Keep the metrics labels that are not resource labels
+			if _, ok := metricskey.KnativeRevisionLabels[t.Key.Name()]; !ok {
+				newTags = append(newTags, t)
+			}
+		}
+
+		return newTags, kr
+	}
+}
+
+func getTagsMap(tags []tag.Tag) map[string]string {
+	tagsMap := map[string]string{}
+	for _, t := range tags {
+		tagsMap[t.Key.Name()] = t.Value
+	}
+	return tagsMap
+}
+
+func valueOrUnknown(key string, tagsMap map[string]string) string {
+	if value, ok := tagsMap[key]; ok {
+		return value
+	}
+	return metricskey.ValueUnknown
+}
+
+func getGlobalMonitoredResource() func(v *view.View, tags []tag.Tag) ([]tag.Tag, monitoredresource.Interface) {
+	return func(v *view.View, tags []tag.Tag) ([]tag.Tag, monitoredresource.Interface) {
+		return tags, &Global{}
+	}
+}
+
 func newStackdriverExporter(config *metricsConfig, logger *zap.SugaredLogger) (view.Exporter, error) {
+	setMonitoredResourceFunc(config, logger)
 	e, err := stackdriver.NewExporter(stackdriver.Options{
-		ProjectID:    config.stackdriverProjectID,
-		MetricPrefix: config.domain + "/" + config.component,
-		Resource: &monitoredrespb.MonitoredResource{
-			Type: "global",
-		},
+		ProjectID:               config.stackdriverProjectID,
+		MetricPrefix:            config.domain + "/" + config.component,
+		GetMonitoredResource:    getMonitoredResourceFunc,
 		DefaultMonitoringLabels: &stackdriver.Labels{},
 	})
 	if err != nil {
@@ -111,6 +158,29 @@ func resetCurPromSrv() {
 	}
 }
 
+func resetMonitoredResourceFunc() {
+	metricsMux.Lock()
+	defer metricsMux.Unlock()
+	if getMonitoredResourceFunc != nil {
+		getMonitoredResourceFunc = nil
+	}
+}
+
+func setMonitoredResourceFunc(config *metricsConfig, logger *zap.SugaredLogger) {
+	metricsMux.Lock()
+	defer metricsMux.Unlock()
+	if getMonitoredResourceFunc == nil {
+		gm := retrieveGCPMetadata()
+		metricsPrefix := config.domain + "/" + config.component
+		logger.Infof("metrics prefix", metricsPrefix)
+		if _, ok := metricskey.KnativeRevisionMetricsPrefixes[metricsPrefix]; ok {
+			getMonitoredResourceFunc = getKnativeRevisionMonitoredResource(gm)
+		} else {
+			getMonitoredResourceFunc = getGlobalMonitoredResource()
+		}
+	}
+}
+
 func startNewPromSrv(e *prometheus.Exporter) *http.Server {
 	sm := http.NewServeMux()
 	sm.Handle("/metrics", e)

metrics/exporter_test.go

@@ -13,18 +13,65 @@ limitations under the License.
 package metrics
 
 import (
+	"os"
 	"testing"
 	"time"
 
 	. "github.com/knative/pkg/logging/testing"
+	"github.com/knative/pkg/metrics/metricskey"
+	"go.opencensus.io/stats"
+	"go.opencensus.io/stats/view"
+	"go.opencensus.io/tag"
 )
 
+const (
+	testNS            = "test"
+	testService       = "test-service"
+	testRoute         = "test-route"
+	testConfiguration = "test-configuration"
+	testRevision      = "test-revision"
+)
+
+var (
+	testView = &view.View{
+		Description: "Test View",
+		Measure:     stats.Int64("test", "Test Measure", stats.UnitNone),
+		Aggregation: view.LastValue(),
+		TagKeys:     []tag.Key{},
+	}
+
+	nsKey       = tag.Tag{Key: mustNewTagKey(metricskey.LabelNamespaceName), Value: testNS}
+	serviceKey  = tag.Tag{Key: mustNewTagKey(metricskey.LabelServiceName), Value: testService}
+	routeKey    = tag.Tag{Key: mustNewTagKey(metricskey.LabelRouteName), Value: testRoute}
+	revisionKey = tag.Tag{Key: mustNewTagKey(metricskey.LabelRevisionName), Value: testRevision}
+
+	testTags = []tag.Tag{nsKey, serviceKey, routeKey, revisionKey}
+)
+
+func mustNewTagKey(s string) tag.Key {
+	tagKey, err := tag.NewKey(s)
+	if err != nil {
+		panic(err)
+	}
+	return tagKey
+}
+
+func getResourceLabelValue(key string, tags []tag.Tag) string {
+	for _, t := range tags {
+		if t.Key.Name() == key {
+			return t.Value
+		}
+	}
+	return ""
+}
+
 func TestMain(m *testing.M) {
 	resetCurPromSrv()
-	m.Run()
+	os.Exit(m.Run())
 }
 
-func TestNewStackdriverExporter(t *testing.T) {
+func TestNewStackdriverExporterForGlobal(t *testing.T) {
+	resetMonitoredResourceFunc()
 	// The stackdriver project ID is required for stackdriver exporter.
 	e, err := newStackdriverExporter(&metricsConfig{
 		domain:               servingDomain,
@@ -37,6 +84,58 @@ func TestNewStackdriverExporter(t *testing.T) {
 	if e == nil {
 		t.Error("expected a non-nil metrics exporter")
 	}
+	if getMonitoredResourceFunc == nil {
+		t.Error("expected a non-nil getMonitoredResourceFunc")
+	}
+	newTags, monitoredResource := getMonitoredResourceFunc(testView, testTags)
+	gotResType, labels := monitoredResource.MonitoredResource()
+	wantedResType := "global"
+	if gotResType != wantedResType {
+		t.Errorf("MonitoredResource=%v, got: %v", wantedResType, gotResType)
+	}
+	got := getResourceLabelValue(metricskey.LabelNamespaceName, newTags)
+	if got != testNS {
+		t.Errorf("expected new tag %v with value %v, got: %v", routeKey, testNS, newTags)
+	}
+	if len(labels) != 0 {
+		t.Errorf("expected no label, got: %v", labels)
+	}
+}
+
+func TestNewStackdriverExporterForKnativeRevision(t *testing.T) {
+	resetMonitoredResourceFunc()
+	e, err := newStackdriverExporter(&metricsConfig{
+		domain:               servingDomain,
+		component:            "autoscaler",
+		backendDestination:   Stackdriver,
+		stackdriverProjectID: testProj}, TestLogger(t))
+	if err != nil {
+		t.Error(err)
+	}
+	if e == nil {
+		t.Error("expected a non-nil metrics exporter")
+	}
+	if getMonitoredResourceFunc == nil {
+		t.Error("expected a non-nil getMonitoredResourceFunc")
+	}
+	newTags, monitoredResource := getMonitoredResourceFunc(testView, testTags)
+	gotResType, labels := monitoredResource.MonitoredResource()
+	wantedResType := "knative_revision"
+	if gotResType != wantedResType {
+		t.Errorf("MonitoredResource=%v, got %v", wantedResType, gotResType)
+	}
+	got := getResourceLabelValue(metricskey.LabelRouteName, newTags)
+	if got != testRoute {
+		t.Errorf("expected new tag: %v, got: %v", routeKey, newTags)
+	}
+	got, ok := labels[metricskey.LabelNamespaceName]
+	if !ok || got != testNS {
+		t.Errorf("expected label %v with value %v, got: %v", metricskey.LabelNamespaceName, testNS, got)
+	}
+	got, ok = labels[metricskey.LabelConfigurationName]
+	if !ok || got != metricskey.ValueUnknown {
+		t.Errorf("expected label %v with value %v, got: %v", metricskey.LabelConfigurationName, metricskey.ValueUnknown, got)
+	}
 }
 
 func TestNewPrometheusExporter(t *testing.T) {
@@ -91,8 +190,7 @@ func TestInterlevedExporters(t *testing.T) {
 		domain:               servingDomain,
 		component:            testComponent,
 		backendDestination:   Prometheus,
-		stackdriverProjectID: "",
-	}, TestLogger(t))
+		stackdriverProjectID: ""}, TestLogger(t))
 	if err != nil {
 		t.Error(err)
 	}

metrics/metricskey/constants.go

@@ -32,6 +32,9 @@ const (
 	// LabelServiceName is the label for the deployed service name
 	LabelServiceName = "service_name"
 
+	// LabelRouteName is the label for immutable name of the route that receives the request
+	LabelRouteName = "route_name"
+
 	// LabelConfigurationName is the label for the configuration which created the monitored revision
 	LabelConfigurationName = "configuration_name"
 
@@ -44,19 +47,26 @@ const (
 )
 
 var (
-	// KnativeRevisionLabels stores the set of resource labels for resource type knative_revision
+	// KnativeRevisionLabels stores the set of resource labels for resource type knative_revision.
+	// LabelRouteName is added as extra label since it is optional, not in this map.
 	KnativeRevisionLabels = map[string]struct{}{
-		LabelProject:           {},
-		LabelLocation:          {},
-		LabelClusterName:       {},
-		LabelNamespaceName:     {},
-		LabelServiceName:       {},
-		LabelConfigurationName: {},
-		LabelRevisionName:      {},
+		LabelProject:           struct{}{},
+		LabelLocation:          struct{}{},
+		LabelClusterName:       struct{}{},
+		LabelNamespaceName:     struct{}{},
+		LabelServiceName:       struct{}{},
+		LabelConfigurationName: struct{}{},
+		LabelRevisionName:      struct{}{},
 	}
 
 	// ResourceTypeToLabelsMap maps resource type to the set of resource labels
 	ResourceTypeToLabelsMap = map[string]map[string]struct{}{
 		ResourceTypeKnativeRevision: KnativeRevisionLabels,
 	}
+
+	// KnativeRevisionMetricsPrefixes stores a set of metrics prefixes that belong to resource type knative_revision
+	KnativeRevisionMetricsPrefixes = map[string]struct{}{
+		"knative.dev/serving/autoscaler": struct{}{},
+		"knative.dev/serving/activator":  struct{}{},
+	}
 )

metrics/monitored_resources.go

@@ -0,0 +1,81 @@
+/*
+Copyright 2018 The Knative Authors
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+    http://www.apache.org/licenses/LICENSE-2.0
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package metrics
+
+import (
+	"cloud.google.com/go/compute/metadata"
+	"github.com/knative/pkg/metrics/metricskey"
+)
+
+type gcpMetadata struct {
+	project  string
+	location string
+	cluster  string
+}
+
+func newGcpMetadata() gcpMetadata {
+	gm := gcpMetadata{
+		project:  metricskey.ValueUnknown,
+		location: metricskey.ValueUnknown,
+		cluster:  metricskey.ValueUnknown,
+	}
+	return gm
+}
+
+type KnativeRevision struct {
+	Project           string
+	Location          string
+	ClusterName       string
+	NamespaceName     string
+	ServiceName       string
+	ConfigurationName string
+	RevisionName      string
+}
+
+func (kr *KnativeRevision) MonitoredResource() (resType string, labels map[string]string) {
+	labels = map[string]string{
+		metricskey.LabelProject:           kr.Project,
+		metricskey.LabelLocation:          kr.Location,
+		metricskey.LabelClusterName:       kr.ClusterName,
+		metricskey.LabelNamespaceName:     kr.NamespaceName,
+		metricskey.LabelServiceName:       kr.ServiceName,
+		metricskey.LabelConfigurationName: kr.ConfigurationName,
+		metricskey.LabelRevisionName:      kr.RevisionName,
+	}
+	return "knative_revision", labels
+}
+
+type Global struct {
+}
+
+func (g *Global) MonitoredResource() (resType string, labels map[string]string) {
+	return "global", nil
+}
+
+func retrieveGCPMetadata() *gcpMetadata {
+	gm := newGcpMetadata()
+	project, err := metadata.NumericProjectID()
+	if err == nil && project != "" {
+		gm.project = project
+	}
+	location, err := metadata.Zone()
+	if err == nil && location != "" {
+		gm.location = location
+	}
+	cluster, err := metadata.InstanceAttributeValue("cluster-name")
+	if err == nil && cluster != "" {
+		gm.cluster = cluster
+	}
+	return &gm
+}