StackDriver CustomMetrics with Custom Domain Prefix (#676)

* renaming source_name and source_resource_group to name and resource_group,
as they will be used in other resources

* updating

* updates based on code review

* update

* fix

metrics/config.go

@@ -42,10 +42,11 @@ const (
 	// The following keys are used to configure metrics reporting.
 	// See https://github.com/knative/serving/blob/master/config/config-observability.yaml
 	// for details.
-	AllowStackdriverCustomMetricsKey = "metrics.allow-stackdriver-custom-metrics"
-	BackendDestinationKey            = "metrics.backend-destination"
-	ReportingPeriodKey               = "metrics.reporting-period-seconds"
-	StackdriverProjectIDKey          = "metrics.stackdriver-project-id"
+	AllowStackdriverCustomMetricsKey    = "metrics.allow-stackdriver-custom-metrics"
+	BackendDestinationKey               = "metrics.backend-destination"
+	ReportingPeriodKey                  = "metrics.reporting-period-seconds"
+	StackdriverProjectIDKey             = "metrics.stackdriver-project-id"
+	StackdriverCustomMetricSubDomainKey = "metrics.stackdriver-custom-metrics-subdomain"
 
 	// Stackdriver is used for Stackdriver backend
 	Stackdriver metricsBackend = "stackdriver"
@@ -103,10 +104,14 @@ type metricsConfig struct {
 	stackdriverProjectID string
 	// allowStackdriverCustomMetrics indicates whether it is allowed to send metrics to
 	// Stackdriver using "global" resource type and custom metric type if the
-	// metrics are not supported by "knative_revision" resource type. Setting this
+	// metrics are not supported by the registered monitored resource types. Setting this
 	// flag to "true" could cause extra Stackdriver charge.
 	// If backendDestination is not Stackdriver, this is ignored.
 	allowStackdriverCustomMetrics bool
+	// stackdriverCustomMetricsSubDomain is the subdomain to use when sending custom metrics to StackDriver.
+	// If not specified, the default is `knative.dev`.
+	// If backendDestination is not Stackdriver, this is ignored.
+	stackdriverCustomMetricsSubDomain string
 	// True if backendDestination equals to "stackdriver". Store this in a variable
 	// to reduce string comparison operations.
 	isStackdriverBackend bool
@@ -114,8 +119,8 @@ type metricsConfig struct {
 	// "knative.dev/serving/activator". Store this in a variable to reduce string
 	// join operations.
 	stackdriverMetricTypePrefix string
-	// stackdriverCustomMetricTypePrefix is "custom.googleapis.com/knative.dev" joins
-	// component, e.g. "custom.googleapis.com/knative.dev/serving/activator".
+	// stackdriverCustomMetricTypePrefix is "custom.googleapis.com" joined with the subdomain and component.
+	// E.g., "custom.googleapis.com/<subdomain>/<component>".
 	// Store this in a variable to reduce string join operations.
 	stackdriverCustomMetricTypePrefix string
 }
@@ -173,7 +178,12 @@ func getMetricsConfig(ops ExporterOptions, logger *zap.SugaredLogger) (*metricsC
 		mc.stackdriverProjectID = m[StackdriverProjectIDKey]
 		mc.isStackdriverBackend = true
 		mc.stackdriverMetricTypePrefix = path.Join(mc.domain, mc.component)
-		mc.stackdriverCustomMetricTypePrefix = path.Join(customMetricTypePrefix, mc.component)
+
+		mc.stackdriverCustomMetricsSubDomain = defaultCustomMetricSubDomain
+		if sdcmd, ok := m[StackdriverCustomMetricSubDomainKey]; ok && sdcmd != "" {
+			mc.stackdriverCustomMetricsSubDomain = sdcmd
+		}
+		mc.stackdriverCustomMetricTypePrefix = path.Join(customMetricTypePrefix, mc.stackdriverCustomMetricsSubDomain, mc.component)
 		if ascmStr, ok := m[AllowStackdriverCustomMetricsKey]; ok && ascmStr != "" {
 			ascmBool, err := strconv.ParseBool(ascmStr)
 			if err != nil {

metrics/config_test.go

@@ -31,12 +31,12 @@ import (
 // 	See https://github.com/knative/pkg/issues/608
 
 const (
-	servingDomain  = "knative.dev/serving"
-	eventingDomain = "knative.dev/eventing"
-	badDomain      = "test.domain"
-	testComponent  = "testComponent"
-	testProj       = "test-project"
-	anotherProj    = "another-project"
+	servingDomain   = "knative.dev/serving"
+	eventingDomain  = "knative.dev/eventing"
+	customSubDomain = "test.domain"
+	testComponent   = "testComponent"
+	testProj        = "test-project"
+	anotherProj     = "another-project"
 )
 
 var (
@@ -151,7 +151,8 @@ var (
 				reportingPeriod:                   60 * time.Second,
 				isStackdriverBackend:              true,
 				stackdriverMetricTypePrefix:       path.Join(servingDomain, testComponent),
-				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, testComponent),
+				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, defaultCustomMetricSubDomain, testComponent),
+				stackdriverCustomMetricsSubDomain: defaultCustomMetricSubDomain,
 			},
 			expectedNewExporter: true,
 		}, {
@@ -187,7 +188,8 @@ var (
 				reportingPeriod:                   60 * time.Second,
 				isStackdriverBackend:              true,
 				stackdriverMetricTypePrefix:       path.Join(servingDomain, testComponent),
-				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, testComponent),
+				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, defaultCustomMetricSubDomain, testComponent),
+				stackdriverCustomMetricsSubDomain: defaultCustomMetricSubDomain,
 			},
 			expectedNewExporter: true,
 		}, {
@@ -225,7 +227,8 @@ var (
 				reportingPeriod:                   60 * time.Second,
 				isStackdriverBackend:              true,
 				stackdriverMetricTypePrefix:       path.Join(servingDomain, testComponent),
-				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, testComponent),
+				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, defaultCustomMetricSubDomain, testComponent),
+				stackdriverCustomMetricsSubDomain: defaultCustomMetricSubDomain,
 			},
 			expectedNewExporter: true,
 		}, {
@@ -265,7 +268,8 @@ var (
 				reportingPeriod:                   7 * time.Second,
 				isStackdriverBackend:              true,
 				stackdriverMetricTypePrefix:       path.Join(servingDomain, testComponent),
-				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, testComponent),
+				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, defaultCustomMetricSubDomain, testComponent),
+				stackdriverCustomMetricsSubDomain: defaultCustomMetricSubDomain,
 			},
 			expectedNewExporter: true,
 		}, {
@@ -287,7 +291,8 @@ var (
 				reportingPeriod:                   3 * time.Second,
 				isStackdriverBackend:              true,
 				stackdriverMetricTypePrefix:       path.Join(servingDomain, testComponent),
-				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, testComponent),
+				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, defaultCustomMetricSubDomain, testComponent),
+				stackdriverCustomMetricsSubDomain: defaultCustomMetricSubDomain,
 			},
 		}, {
 			name: "emptyReportingPeriodPrometheus",
@@ -326,7 +331,8 @@ var (
 				reportingPeriod:                   60 * time.Second,
 				isStackdriverBackend:              true,
 				stackdriverMetricTypePrefix:       path.Join(servingDomain, testComponent),
-				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, testComponent),
+				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, defaultCustomMetricSubDomain, testComponent),
+				stackdriverCustomMetricsSubDomain: defaultCustomMetricSubDomain,
 			},
 			expectedNewExporter: true,
 		}, {
@@ -349,8 +355,32 @@ var (
 				reportingPeriod:                   60 * time.Second,
 				isStackdriverBackend:              true,
 				stackdriverMetricTypePrefix:       path.Join(servingDomain, testComponent),
-				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, testComponent),
+				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, defaultCustomMetricSubDomain, testComponent),
 				allowStackdriverCustomMetrics:     true,
+				stackdriverCustomMetricsSubDomain: defaultCustomMetricSubDomain,
+			},
+		}, {
+			name: "allowStackdriverCustomMetric with subdomain",
+			ops: ExporterOptions{
+				ConfigMap: map[string]string{
+					"metrics.backend-destination":                  "stackdriver",
+					"metrics.stackdriver-project-id":               "test2",
+					"metrics.reporting-period-seconds":             "",
+					"metrics.stackdriver-custom-metrics-subdomain": customSubDomain,
+				},
+				Domain:    servingDomain,
+				Component: testComponent,
+			},
+			expectedConfig: metricsConfig{
+				domain:                            servingDomain,
+				component:                         testComponent,
+				backendDestination:                Stackdriver,
+				stackdriverProjectID:              "test2",
+				reportingPeriod:                   60 * time.Second,
+				isStackdriverBackend:              true,
+				stackdriverMetricTypePrefix:       path.Join(servingDomain, testComponent),
+				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, customSubDomain, testComponent),
+				stackdriverCustomMetricsSubDomain: customSubDomain,
 			},
 		}, {
 			name: "overridePrometheusPort",
@@ -390,7 +420,8 @@ var (
 				reportingPeriod:                   60 * time.Second,
 				isStackdriverBackend:              true,
 				stackdriverMetricTypePrefix:       path.Join(servingDomain, testComponent),
-				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, testComponent),
+				stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, defaultCustomMetricSubDomain, testComponent),
+				stackdriverCustomMetricsSubDomain: defaultCustomMetricSubDomain,
 			},
 		}, {
 			name: "validPrometheus",

metrics/exporter_test.go

@@ -193,7 +193,7 @@ func TestFlushExporter(t *testing.T) {
 		reportingPeriod:                   1 * time.Minute,
 		stackdriverProjectID:              "test",
 		stackdriverMetricTypePrefix:       path.Join(servingDomain, testComponent),
-		stackdriverCustomMetricTypePrefix: path.Join(customMetricTypePrefix, testComponent),
+		stackdriverCustomMetricTypePrefix: path.Join(defaultCustomMetricSubDomain, testComponent),
 	}
 	e, err = newMetricsExporter(c, TestLogger(t))
 	if err != nil {

metrics/stackdriver_exporter.go

@@ -27,10 +27,12 @@ import (
 	"knative.dev/pkg/metrics/metricskey"
 )
 
-// customMetricTypePrefix is the metric type prefix for unsupported metrics by
-// resource type knative_revision.
-// See: https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors#MetricDescriptor
-const customMetricTypePrefix = "custom.googleapis.com/knative.dev"
+const (
+	customMetricTypePrefix = "custom.googleapis.com"
+	// defaultCustomMetricSubDomain is the default subdomain to use for unsupported metrics by monitored resource types.
+	// See: https://cloud.google.com/monitoring/api/ref_v3/rest/v3/projects.metricDescriptors#MetricDescriptor
+	defaultCustomMetricSubDomain = "knative.dev"
+)
 
 var (
 	// gcpMetadataFunc is the function used to fetch GCP metadata.

metrics/stackdriver_exporter_test.go

@@ -323,7 +323,7 @@ func TestGetgetMetricTypeFunc_UseKnativeDomain(t *testing.T) {
 		}
 		mtf := getMetricTypeFunc(
 			path.Join(testCase.domain, testCase.component),
-			path.Join(customMetricTypePrefix, testCase.component))
+			path.Join(defaultCustomMetricSubDomain, testCase.component))
 
 		gotMetricType := mtf(testView)
 		wantedMetricType := path.Join(testCase.domain, testCase.component, testView.Measure.Name())
@@ -343,10 +343,10 @@ func TestGetgetMetricTypeFunc_UseCustomDomain(t *testing.T) {
 		}
 		mtf := getMetricTypeFunc(
 			path.Join(testCase.domain, testCase.component),
-			path.Join(customMetricTypePrefix, testCase.component))
+			path.Join(defaultCustomMetricSubDomain, testCase.component))
 
 		gotMetricType := mtf(testView)
-		wantedMetricType := path.Join(customMetricTypePrefix, testCase.component, testView.Measure.Name())
+		wantedMetricType := path.Join(defaultCustomMetricSubDomain, testCase.component, testView.Measure.Name())
 		if gotMetricType != wantedMetricType {
 			t.Fatalf("getMetricType=%v, want %v", gotMetricType, wantedMetricType)
 		}