From 64ed8e4a743bcb176e3d06a071697e7eade6b7b3 Mon Sep 17 00:00:00 2001 From: Andrew Seigner Date: Tue, 23 Jul 2019 17:12:30 -0700 Subject: [PATCH] Introduce Cluster Heartbeat cronjob (#3056) `linkerd check`, the web dashboard, and Grafana all perform version checks to validate Linkerd is up to date. It's common for users to seldom execute these codepaths. This makes it difficult to identify what versions of Linkerd are currently in use and what environments it is being run in, which helps prioritize testing and backports. Introduce a `heartbeat` CronJob to the default Linkerd install. The cronjob executes every 24 hours, starting from 5 minutes after `linkerd install` is run. Example check URL: https://versioncheck.linkerd.io/version.json? install-time=1562761177& k8s-version=v1.15.0& meshed-pods=8& rps=3& source=heartbeat& uuid=cc4bb700-3314-426a-9f0f-ec588b9df020& version=git-b97ee9f7 Fixes #2961 Signed-off-by: Andrew Seigner --- chart/templates/heartbeat-rbac.yaml | 44 +++++ chart/templates/heartbeat.yaml | 42 ++++ chart/templates/psp.yaml | 3 + cli/cmd/install.go | 15 +- cli/cmd/install_test.go | 5 + cli/cmd/testdata/install_config.golden | 45 +++++ cli/cmd/testdata/install_control-plane.golden | 40 ++++ cli/cmd/testdata/install_default.golden | 85 ++++++++ cli/cmd/testdata/install_ha_output.golden | 88 +++++++++ .../install_ha_with_overrides_output.golden | 88 +++++++++ .../testdata/install_no_init_container.golden | 85 ++++++++ cli/cmd/testdata/install_output.golden | 85 ++++++++ cli/cmd/testdata/upgrade_default.golden | 85 ++++++++ cli/cmd/testdata/upgrade_ha.golden | 88 +++++++++ cli/cmd/testdata/upgrade_ha_config.golden | 45 +++++ cli/cmd/upgrade.go | 2 +- cli/cmd/upgrade_test.go | 1 + controller/Dockerfile | 2 + controller/api/public/grpc_server_test.go | 8 +- controller/api/public/stat_summary_test.go | 4 +- controller/api/public/test_helper.go | 67 +++++-- controller/cmd/heartbeat/main.go | 58 ++++++ controller/heartbeat/heartbeat.go | 125 ++++++++++++ controller/heartbeat/heartbeat_test.go | 181 ++++++++++++++++++ pkg/healthcheck/healthcheck.go | 19 +- pkg/healthcheck/healthcheck_test.go | 47 ++++- pkg/version/channels.go | 5 +- test/get/get_test.go | 16 +- 28 files changed, 1339 insertions(+), 39 deletions(-) create mode 100644 chart/templates/heartbeat-rbac.yaml create mode 100644 chart/templates/heartbeat.yaml create mode 100644 controller/cmd/heartbeat/main.go create mode 100644 controller/heartbeat/heartbeat.go create mode 100644 controller/heartbeat/heartbeat_test.go diff --git a/chart/templates/heartbeat-rbac.yaml b/chart/templates/heartbeat-rbac.yaml new file mode 100644 index 000000000..2179e2e5a --- /dev/null +++ b/chart/templates/heartbeat-rbac.yaml @@ -0,0 +1,44 @@ +{{with .Values -}} +--- +### +### Heartbeat RBAC +### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linkerd-heartbeat + namespace: {{.Namespace}} + labels: + {{.ControllerNamespaceLabel}}: {{.Namespace}} +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + resourceNames: ["linkerd-config"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linkerd-heartbeat + namespace: {{.Namespace}} + labels: + {{.ControllerNamespaceLabel}}: {{.Namespace}} +roleRef: + kind: Role + name: linkerd-heartbeat + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: {{.Namespace}} +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: {{.Namespace}} + labels: + {{.ControllerComponentLabel}}: heartbeat + {{.ControllerNamespaceLabel}}: {{.Namespace}} +{{- end}} diff --git a/chart/templates/heartbeat.yaml b/chart/templates/heartbeat.yaml new file mode 100644 index 000000000..ea4591899 --- /dev/null +++ b/chart/templates/heartbeat.yaml @@ -0,0 +1,42 @@ +{{with .Values -}} +--- +### +### Heartbeat +### +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: linkerd-heartbeat + namespace: {{.Namespace}} + labels: + {{.ControllerComponentLabel}}: heartbeat + {{.ControllerNamespaceLabel}}: {{.Namespace}} + annotations: + {{.CreatedByAnnotation}}: {{.CliVersion}} +spec: + schedule: "{{.HeartbeatSchedule}}" + jobTemplate: + spec: + template: + metadata: + labels: + {{.ControllerComponentLabel}}: heartbeat + annotations: + {{.CreatedByAnnotation}}: {{.CliVersion}} + spec: + serviceAccountName: linkerd-heartbeat + restartPolicy: OnFailure + containers: + - name: heartbeat + image: {{.ControllerImage}} + imagePullPolicy: {{.ImagePullPolicy}} + args: + - "heartbeat" + - "-prometheus-url=http://linkerd-prometheus.{{.Namespace}}.svc.cluster.local:9090" + - "-controller-namespace={{.Namespace}}" + - "-log-level={{.ControllerLogLevel}}" + {{- include "resources" .HeartbeatResources | indent 4 | trimPrefix " " }} + securityContext: + runAsUser: {{.ControllerUID}} +{{end -}} diff --git a/chart/templates/psp.yaml b/chart/templates/psp.yaml index 272e1259f..4f13df24c 100644 --- a/chart/templates/psp.yaml +++ b/chart/templates/psp.yaml @@ -77,6 +77,9 @@ subjects: - kind: ServiceAccount name: linkerd-grafana namespace: {{.Namespace}} +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: {{.Namespace}} - kind: ServiceAccount name: linkerd-identity namespace: {{.Namespace}} diff --git a/cli/cmd/install.go b/cli/cmd/install.go index 034a88722..99b867f34 100644 --- a/cli/cmd/install.go +++ b/cli/cmd/install.go @@ -62,11 +62,13 @@ type ( NoInitContainer bool WebhookFailurePolicy string OmitWebhookSideEffects bool + HeartbeatSchedule string Configs configJSONs DestinationResources, GrafanaResources, + HeartbeatResources, IdentityResources, PrometheusResources, ProxyInjectorResources, @@ -140,6 +142,7 @@ type ( // function pointers that can be overridden for tests generateUUID func() string generateWebhookTLS func(webhook string) (*tlsValues, error) + heartbeatSchedule func() string } installIdentityOptions struct { @@ -239,6 +242,11 @@ func newInstallOptionsWithDefaults() *installOptions { CrtPEM: root.Cred.Crt.EncodeCertificatePEM(), }, nil }, + + heartbeatSchedule: func() string { + t := time.Now().Add(5 * time.Minute).UTC() + return fmt.Sprintf("%d %d * * * ", t.Minute(), t.Hour()) + }, } } @@ -615,6 +623,7 @@ func (options *installOptions) buildValuesWithoutIdentity(configs *pb.All) (*ins WebhookFailurePolicy: "Ignore", OmitWebhookSideEffects: options.omitWebhookSideEffects, PrometheusLogLevel: toPromLogLevel(strings.ToLower(options.controllerLogLevel)), + HeartbeatSchedule: options.heartbeatSchedule(), Configs: configJSONs{ Global: globalJSON, @@ -624,6 +633,7 @@ func (options *installOptions) buildValuesWithoutIdentity(configs *pb.All) (*ins DestinationResources: &resources{}, GrafanaResources: &resources{}, + HeartbeatResources: &resources{}, IdentityResources: &resources{}, PrometheusResources: &resources{}, ProxyInjectorResources: &resources{}, @@ -643,6 +653,7 @@ func (options *installOptions) buildValuesWithoutIdentity(configs *pb.All) (*ins // Copy constraints to each so that further modification isn't global. *values.DestinationResources = *defaultConstraints *values.GrafanaResources = *defaultConstraints + *values.HeartbeatResources = *defaultConstraints *values.ProxyInjectorResources = *defaultConstraints *values.PublicAPIResources = *defaultConstraints *values.SPValidatorResources = *defaultConstraints @@ -690,6 +701,7 @@ func (values *installValues) render(w io.Writer, configs *pb.All) error { {Name: "templates/namespace.yaml"}, {Name: "templates/identity-rbac.yaml"}, {Name: "templates/controller-rbac.yaml"}, + {Name: "templates/heartbeat-rbac.yaml"}, {Name: "templates/web-rbac.yaml"}, {Name: "templates/serviceprofile-crd.yaml"}, {Name: "templates/trafficsplit-crd.yaml"}, @@ -709,6 +721,7 @@ func (values *installValues) render(w io.Writer, configs *pb.All) error { {Name: "templates/config.yaml"}, {Name: "templates/identity.yaml"}, {Name: "templates/controller.yaml"}, + {Name: "templates/heartbeat.yaml"}, {Name: "templates/web.yaml"}, {Name: "templates/prometheus.yaml"}, {Name: "templates/grafana.yaml"}, @@ -911,7 +924,7 @@ func errIfLinkerdConfigConfigMapExists() error { return err } - _, err = healthcheck.FetchLinkerdConfigMap(kubeAPI, controlPlaneNamespace) + _, _, err = healthcheck.FetchLinkerdConfigMap(kubeAPI, controlPlaneNamespace) if err != nil { if kerrors.IsNotFound(err) { return nil diff --git a/cli/cmd/install_test.go b/cli/cmd/install_test.go index db6d68c80..dc3944692 100644 --- a/cli/cmd/install_test.go +++ b/cli/cmd/install_test.go @@ -137,6 +137,7 @@ func testInstallOptions() *installOptions { return "deaab91a-f4ab-448a-b7d1-c832a2fa0a60" } o.generateWebhookTLS = fakeGenerateWebhookTLS + o.heartbeatSchedule = fakeHeartbeatSchedule o.identityOptions.crtPEMFile = filepath.Join("testdata", "crt.pem") o.identityOptions.keyPEMFile = filepath.Join("testdata", "key.pem") o.identityOptions.trustPEMFile = filepath.Join("testdata", "trust-anchors.pem") @@ -238,3 +239,7 @@ func fakeGenerateWebhookTLS(webhook string) (*tlsValues, error) { } return nil, nil } + +func fakeHeartbeatSchedule() string { + return "1 2 3 4 5" +} diff --git a/cli/cmd/testdata/install_config.golden b/cli/cmd/testdata/install_config.golden index b845a8294..6a03b3516 100644 --- a/cli/cmd/testdata/install_config.golden +++ b/cli/cmd/testdata/install_config.golden @@ -107,6 +107,48 @@ metadata: linkerd.io/control-plane-ns: linkerd --- ### +### Heartbeat RBAC +### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + resourceNames: ["linkerd-config"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +roleRef: + kind: Role + name: linkerd-heartbeat + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd +--- +### ### Web RBAC ### --- @@ -523,6 +565,9 @@ subjects: - kind: ServiceAccount name: linkerd-grafana namespace: linkerd +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd - kind: ServiceAccount name: linkerd-identity namespace: linkerd diff --git a/cli/cmd/testdata/install_control-plane.golden b/cli/cmd/testdata/install_control-plane.golden index becefc6b5..47e68bb6a 100644 --- a/cli/cmd/testdata/install_control-plane.golden +++ b/cli/cmd/testdata/install_control-plane.golden @@ -498,6 +498,46 @@ spec: status: {} --- ### +### Heartbeat +### +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined +spec: + schedule: "1 2 3 4 5" + jobTemplate: + spec: + template: + metadata: + labels: + linkerd.io/control-plane-component: heartbeat + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined + spec: + serviceAccountName: linkerd-heartbeat + restartPolicy: OnFailure + containers: + - name: heartbeat + image: gcr.io/linkerd-io/controller:install-control-plane-version + imagePullPolicy: IfNotPresent + args: + - "heartbeat" + - "-prometheus-url=http://linkerd-prometheus.linkerd.svc.cluster.local:9090" + - "-controller-namespace=linkerd" + - "-log-level=info" + resources: + securityContext: + runAsUser: 2103 +--- +### ### Web ### --- diff --git a/cli/cmd/testdata/install_default.golden b/cli/cmd/testdata/install_default.golden index 833d1be2c..e6b34fe83 100644 --- a/cli/cmd/testdata/install_default.golden +++ b/cli/cmd/testdata/install_default.golden @@ -107,6 +107,48 @@ metadata: linkerd.io/control-plane-ns: linkerd --- ### +### Heartbeat RBAC +### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + resourceNames: ["linkerd-config"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +roleRef: + kind: Role + name: linkerd-heartbeat + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd +--- +### ### Web RBAC ### --- @@ -523,6 +565,9 @@ subjects: - kind: ServiceAccount name: linkerd-grafana namespace: linkerd +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd - kind: ServiceAccount name: linkerd-identity namespace: linkerd @@ -1041,6 +1086,46 @@ spec: status: {} --- ### +### Heartbeat +### +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined +spec: + schedule: "1 2 3 4 5" + jobTemplate: + spec: + template: + metadata: + labels: + linkerd.io/control-plane-component: heartbeat + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined + spec: + serviceAccountName: linkerd-heartbeat + restartPolicy: OnFailure + containers: + - name: heartbeat + image: gcr.io/linkerd-io/controller:install-control-plane-version + imagePullPolicy: IfNotPresent + args: + - "heartbeat" + - "-prometheus-url=http://linkerd-prometheus.linkerd.svc.cluster.local:9090" + - "-controller-namespace=linkerd" + - "-log-level=info" + resources: + securityContext: + runAsUser: 2103 +--- +### ### Web ### --- diff --git a/cli/cmd/testdata/install_ha_output.golden b/cli/cmd/testdata/install_ha_output.golden index ec2565529..0c7762b4b 100644 --- a/cli/cmd/testdata/install_ha_output.golden +++ b/cli/cmd/testdata/install_ha_output.golden @@ -107,6 +107,48 @@ metadata: linkerd.io/control-plane-ns: linkerd --- ### +### Heartbeat RBAC +### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + resourceNames: ["linkerd-config"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +roleRef: + kind: Role + name: linkerd-heartbeat + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd +--- +### ### Web RBAC ### --- @@ -523,6 +565,9 @@ subjects: - kind: ServiceAccount name: linkerd-grafana namespace: linkerd +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd - kind: ServiceAccount name: linkerd-identity namespace: linkerd @@ -1096,6 +1141,49 @@ spec: status: {} --- ### +### Heartbeat +### +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined +spec: + schedule: "1 2 3 4 5" + jobTemplate: + spec: + template: + metadata: + labels: + linkerd.io/control-plane-component: heartbeat + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined + spec: + serviceAccountName: linkerd-heartbeat + restartPolicy: OnFailure + containers: + - name: heartbeat + image: gcr.io/linkerd-io/controller:install-control-plane-version + imagePullPolicy: IfNotPresent + args: + - "heartbeat" + - "-prometheus-url=http://linkerd-prometheus.linkerd.svc.cluster.local:9090" + - "-controller-namespace=linkerd" + - "-log-level=info" + resources: + requests: + cpu: 100m + memory: 50Mi + securityContext: + runAsUser: 2103 +--- +### ### Web ### --- diff --git a/cli/cmd/testdata/install_ha_with_overrides_output.golden b/cli/cmd/testdata/install_ha_with_overrides_output.golden index cff2a3c27..cd1170916 100644 --- a/cli/cmd/testdata/install_ha_with_overrides_output.golden +++ b/cli/cmd/testdata/install_ha_with_overrides_output.golden @@ -107,6 +107,48 @@ metadata: linkerd.io/control-plane-ns: linkerd --- ### +### Heartbeat RBAC +### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + resourceNames: ["linkerd-config"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +roleRef: + kind: Role + name: linkerd-heartbeat + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd +--- +### ### Web RBAC ### --- @@ -523,6 +565,9 @@ subjects: - kind: ServiceAccount name: linkerd-grafana namespace: linkerd +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd - kind: ServiceAccount name: linkerd-identity namespace: linkerd @@ -1096,6 +1141,49 @@ spec: status: {} --- ### +### Heartbeat +### +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined +spec: + schedule: "1 2 3 4 5" + jobTemplate: + spec: + template: + metadata: + labels: + linkerd.io/control-plane-component: heartbeat + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined + spec: + serviceAccountName: linkerd-heartbeat + restartPolicy: OnFailure + containers: + - name: heartbeat + image: gcr.io/linkerd-io/controller:install-control-plane-version + imagePullPolicy: IfNotPresent + args: + - "heartbeat" + - "-prometheus-url=http://linkerd-prometheus.linkerd.svc.cluster.local:9090" + - "-controller-namespace=linkerd" + - "-log-level=info" + resources: + requests: + cpu: 100m + memory: 50Mi + securityContext: + runAsUser: 2103 +--- +### ### Web ### --- diff --git a/cli/cmd/testdata/install_no_init_container.golden b/cli/cmd/testdata/install_no_init_container.golden index 87edfbe57..6f05c4a33 100644 --- a/cli/cmd/testdata/install_no_init_container.golden +++ b/cli/cmd/testdata/install_no_init_container.golden @@ -107,6 +107,48 @@ metadata: linkerd.io/control-plane-ns: linkerd --- ### +### Heartbeat RBAC +### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + resourceNames: ["linkerd-config"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +roleRef: + kind: Role + name: linkerd-heartbeat + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd +--- +### ### Web RBAC ### --- @@ -520,6 +562,9 @@ subjects: - kind: ServiceAccount name: linkerd-grafana namespace: linkerd +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd - kind: ServiceAccount name: linkerd-identity namespace: linkerd @@ -972,6 +1017,46 @@ spec: status: {} --- ### +### Heartbeat +### +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined +spec: + schedule: "1 2 3 4 5" + jobTemplate: + spec: + template: + metadata: + labels: + linkerd.io/control-plane-component: heartbeat + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined + spec: + serviceAccountName: linkerd-heartbeat + restartPolicy: OnFailure + containers: + - name: heartbeat + image: gcr.io/linkerd-io/controller:install-control-plane-version + imagePullPolicy: IfNotPresent + args: + - "heartbeat" + - "-prometheus-url=http://linkerd-prometheus.linkerd.svc.cluster.local:9090" + - "-controller-namespace=linkerd" + - "-log-level=info" + resources: + securityContext: + runAsUser: 2103 +--- +### ### Web ### --- diff --git a/cli/cmd/testdata/install_output.golden b/cli/cmd/testdata/install_output.golden index 737e88b86..522d20aaa 100644 --- a/cli/cmd/testdata/install_output.golden +++ b/cli/cmd/testdata/install_output.golden @@ -107,6 +107,48 @@ metadata: ControllerNamespaceLabel: Namespace --- ### +### Heartbeat RBAC +### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linkerd-heartbeat + namespace: Namespace + labels: + ControllerNamespaceLabel: Namespace +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + resourceNames: ["linkerd-config"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linkerd-heartbeat + namespace: Namespace + labels: + ControllerNamespaceLabel: Namespace +roleRef: + kind: Role + name: linkerd-heartbeat + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: Namespace +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: Namespace + labels: + ControllerComponentLabel: heartbeat + ControllerNamespaceLabel: Namespace +--- +### ### Web RBAC ### --- @@ -523,6 +565,9 @@ subjects: - kind: ServiceAccount name: linkerd-grafana namespace: Namespace +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: Namespace - kind: ServiceAccount name: linkerd-identity namespace: Namespace @@ -971,6 +1016,46 @@ spec: status: {} --- ### +### Heartbeat +### +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: linkerd-heartbeat + namespace: Namespace + labels: + ControllerComponentLabel: heartbeat + ControllerNamespaceLabel: Namespace + annotations: + CreatedByAnnotation: CliVersion +spec: + schedule: "" + jobTemplate: + spec: + template: + metadata: + labels: + ControllerComponentLabel: heartbeat + annotations: + CreatedByAnnotation: CliVersion + spec: + serviceAccountName: linkerd-heartbeat + restartPolicy: OnFailure + containers: + - name: heartbeat + image: ControllerImage + imagePullPolicy: ImagePullPolicy + args: + - "heartbeat" + - "-prometheus-url=http://linkerd-prometheus.Namespace.svc.cluster.local:9090" + - "-controller-namespace=Namespace" + - "-log-level=ControllerLogLevel" + resources: + securityContext: + runAsUser: 2103 +--- +### ### Web ### --- diff --git a/cli/cmd/testdata/upgrade_default.golden b/cli/cmd/testdata/upgrade_default.golden index f6f551439..9dbb62e3c 100644 --- a/cli/cmd/testdata/upgrade_default.golden +++ b/cli/cmd/testdata/upgrade_default.golden @@ -107,6 +107,48 @@ metadata: linkerd.io/control-plane-ns: linkerd --- ### +### Heartbeat RBAC +### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + resourceNames: ["linkerd-config"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +roleRef: + kind: Role + name: linkerd-heartbeat + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd +--- +### ### Web RBAC ### --- @@ -523,6 +565,9 @@ subjects: - kind: ServiceAccount name: linkerd-grafana namespace: linkerd +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd - kind: ServiceAccount name: linkerd-identity namespace: linkerd @@ -1043,6 +1088,46 @@ spec: status: {} --- ### +### Heartbeat +### +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined +spec: + schedule: "1 2 3 4 5" + jobTemplate: + spec: + template: + metadata: + labels: + linkerd.io/control-plane-component: heartbeat + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined + spec: + serviceAccountName: linkerd-heartbeat + restartPolicy: OnFailure + containers: + - name: heartbeat + image: gcr.io/linkerd-io/controller:UPGRADE-CONTROL-PLANE-VERSION + imagePullPolicy: IfNotPresent + args: + - "heartbeat" + - "-prometheus-url=http://linkerd-prometheus.linkerd.svc.cluster.local:9090" + - "-controller-namespace=linkerd" + - "-log-level=info" + resources: + securityContext: + runAsUser: 2103 +--- +### ### Web ### --- diff --git a/cli/cmd/testdata/upgrade_ha.golden b/cli/cmd/testdata/upgrade_ha.golden index 825d2a327..f863deb8c 100644 --- a/cli/cmd/testdata/upgrade_ha.golden +++ b/cli/cmd/testdata/upgrade_ha.golden @@ -107,6 +107,48 @@ metadata: linkerd.io/control-plane-ns: linkerd --- ### +### Heartbeat RBAC +### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + resourceNames: ["linkerd-config"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +roleRef: + kind: Role + name: linkerd-heartbeat + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd +--- +### ### Web RBAC ### --- @@ -523,6 +565,9 @@ subjects: - kind: ServiceAccount name: linkerd-grafana namespace: linkerd +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd - kind: ServiceAccount name: linkerd-identity namespace: linkerd @@ -1098,6 +1143,49 @@ spec: status: {} --- ### +### Heartbeat +### +--- +apiVersion: batch/v1beta1 +kind: CronJob +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined +spec: + schedule: "1 2 3 4 5" + jobTemplate: + spec: + template: + metadata: + labels: + linkerd.io/control-plane-component: heartbeat + annotations: + linkerd.io/created-by: linkerd/cli dev-undefined + spec: + serviceAccountName: linkerd-heartbeat + restartPolicy: OnFailure + containers: + - name: heartbeat + image: gcr.io/linkerd-io/controller:UPGRADE-CONTROL-PLANE-VERSION + imagePullPolicy: IfNotPresent + args: + - "heartbeat" + - "-prometheus-url=http://linkerd-prometheus.linkerd.svc.cluster.local:9090" + - "-controller-namespace=linkerd" + - "-log-level=info" + resources: + requests: + cpu: 100m + memory: 50Mi + securityContext: + runAsUser: 2103 +--- +### ### Web ### --- diff --git a/cli/cmd/testdata/upgrade_ha_config.golden b/cli/cmd/testdata/upgrade_ha_config.golden index 2a1429b26..eaf926bb9 100644 --- a/cli/cmd/testdata/upgrade_ha_config.golden +++ b/cli/cmd/testdata/upgrade_ha_config.golden @@ -107,6 +107,48 @@ metadata: linkerd.io/control-plane-ns: linkerd --- ### +### Heartbeat RBAC +### +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +rules: +- apiGroups: [""] + resources: ["configmaps"] + verbs: ["get"] + resourceNames: ["linkerd-config"] +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-ns: linkerd +roleRef: + kind: Role + name: linkerd-heartbeat + apiGroup: rbac.authorization.k8s.io +subjects: +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd +--- +kind: ServiceAccount +apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: linkerd + labels: + linkerd.io/control-plane-component: heartbeat + linkerd.io/control-plane-ns: linkerd +--- +### ### Web RBAC ### --- @@ -523,6 +565,9 @@ subjects: - kind: ServiceAccount name: linkerd-grafana namespace: linkerd +- kind: ServiceAccount + name: linkerd-heartbeat + namespace: linkerd - kind: ServiceAccount name: linkerd-identity namespace: linkerd diff --git a/cli/cmd/upgrade.go b/cli/cmd/upgrade.go index 6a3b14a93..ec5a79c9a 100644 --- a/cli/cmd/upgrade.go +++ b/cli/cmd/upgrade.go @@ -190,7 +190,7 @@ func (options *upgradeOptions) validateAndBuild(stage string, k kubernetes.Inter // to upgrade/reinstall the control plane when the API is not available; and // this also serves as a passive check that we have privileges to access this // control plane. - configs, err := healthcheck.FetchLinkerdConfigMap(k, controlPlaneNamespace) + _, configs, err := healthcheck.FetchLinkerdConfigMap(k, controlPlaneNamespace) if err != nil { return nil, nil, fmt.Errorf("could not fetch configs from kubernetes: %s", err) } diff --git a/cli/cmd/upgrade_test.go b/cli/cmd/upgrade_test.go index daf73cbed..5813c13f8 100644 --- a/cli/cmd/upgrade_test.go +++ b/cli/cmd/upgrade_test.go @@ -22,6 +22,7 @@ func testUpgradeOptions() *upgradeOptions { o.controlPlaneVersion = upgradeControlPlaneVersion o.proxyVersion = upgradeProxyVersion o.generateWebhookTLS = fakeGenerateWebhookTLS + o.heartbeatSchedule = fakeHeartbeatSchedule o.verifyTLS = func(tls *tlsValues, service string) error { return nil } diff --git a/controller/Dockerfile b/controller/Dockerfile index 3ff726366..206774b95 100644 --- a/controller/Dockerfile +++ b/controller/Dockerfile @@ -14,6 +14,8 @@ FROM scratch ENV PATH=$PATH:/go/bin COPY LICENSE /linkerd/LICENSE COPY --from=golang /go/bin /go/bin +# for heartbeat (https://versioncheck.linkerd.io/version.json) +COPY --from=golang /etc/ssl/certs/ca-certificates.crt /etc/ssl/certs/ ARG LINKERD_VERSION ENV LINKERD_CONTAINER_VERSION_OVERRIDE=${LINKERD_VERSION} diff --git a/controller/api/public/grpc_server_test.go b/controller/api/public/grpc_server_test.go index f7f3a63f8..93921c0d7 100644 --- a/controller/api/public/grpc_server_test.go +++ b/controller/api/public/grpc_server_test.go @@ -394,7 +394,7 @@ status: t.Fatalf("NewFakeAPI returned an error: %s", err) } - mProm := mockProm{Res: exp.promRes} + mProm := MockProm{Res: exp.promRes} fakeGrpcServer := newGrpcServer( &mProm, @@ -428,7 +428,7 @@ status: } // TODO: consider refactoring with expectedStatRPC.verifyPromQueries -func verifyPromQueries(mProm *mockProm, namespace string) error { +func verifyPromQueries(mProm *MockProm, namespace string) error { namespaceSelector := fmt.Sprintf("namespace=\"%s\"", namespace) for _, element := range mProm.QueriesExecuted { if strings.Contains(element, namespaceSelector) { @@ -500,7 +500,7 @@ metadata: } fakeGrpcServer := newGrpcServer( - &mockProm{}, + &MockProm{}, nil, nil, nil, @@ -531,7 +531,7 @@ func TestConfig(t *testing.T) { } fakeGrpcServer := newGrpcServer( - &mockProm{}, + &MockProm{}, nil, nil, nil, diff --git a/controller/api/public/stat_summary_test.go b/controller/api/public/stat_summary_test.go index afa7ecaa8..04fae7e4a 100644 --- a/controller/api/public/stat_summary_test.go +++ b/controller/api/public/stat_summary_test.go @@ -1293,7 +1293,7 @@ status: for _, exp := range expectations { fakeGrpcServer := newGrpcServer( - &mockProm{Res: exp.mockPromResponse}, + &MockProm{Res: exp.mockPromResponse}, nil, nil, nil, @@ -1319,7 +1319,7 @@ status: t.Fatalf("NewFakeAPI returned an error: %s", err) } fakeGrpcServer := newGrpcServer( - &mockProm{Res: model.Vector{}}, + &MockProm{Res: model.Vector{}}, nil, nil, nil, diff --git a/controller/api/public/test_helper.go b/controller/api/public/test_helper.go index 1aa06dffd..1429341bc 100644 --- a/controller/api/public/test_helper.go +++ b/controller/api/public/test_helper.go @@ -185,7 +185,9 @@ func BuildAddrSet(endpoint AuthorityEndpoints) *destinationPb.WeightedAddrSet { // Prometheus client // -type mockProm struct { +// MockProm satisfies the promv1.API interface for testing. +// TODO: move this into something shared under /controller, or into /pkg +type MockProm struct { Res model.Value QueriesExecuted []string // expose the queries our Mock Prometheus receives, to test query generation rwLock sync.Mutex @@ -201,44 +203,69 @@ type PodCounts struct { Errors map[string]*pb.PodErrors } -func (m *mockProm) Query(ctx context.Context, query string, ts time.Time) (model.Value, error) { - m.rwLock.Lock() - defer m.rwLock.Unlock() - m.QueriesExecuted = append(m.QueriesExecuted, query) - return m.Res, nil -} -func (m *mockProm) QueryRange(ctx context.Context, query string, r promv1.Range) (model.Value, error) { +// Query performs a query for the given time. +func (m *MockProm) Query(ctx context.Context, query string, ts time.Time) (model.Value, error) { m.rwLock.Lock() defer m.rwLock.Unlock() m.QueriesExecuted = append(m.QueriesExecuted, query) return m.Res, nil } -func (m *mockProm) AlertManagers(ctx context.Context) (promv1.AlertManagersResult, error) { +// QueryRange performs a query for the given range. +func (m *MockProm) QueryRange(ctx context.Context, query string, r promv1.Range) (model.Value, error) { + m.rwLock.Lock() + defer m.rwLock.Unlock() + m.QueriesExecuted = append(m.QueriesExecuted, query) + return m.Res, nil +} + +// AlertManagers returns an overview of the current state of the Prometheus alert +// manager discovery. +func (m *MockProm) AlertManagers(ctx context.Context) (promv1.AlertManagersResult, error) { return promv1.AlertManagersResult{}, nil } -func (m *mockProm) CleanTombstones(ctx context.Context) error { + +// CleanTombstones removes the deleted data from disk and cleans up the existing +// tombstones. +func (m *MockProm) CleanTombstones(ctx context.Context) error { return nil } -func (m *mockProm) Config(ctx context.Context) (promv1.ConfigResult, error) { + +// Config returns the current Prometheus configuration. +func (m *MockProm) Config(ctx context.Context) (promv1.ConfigResult, error) { return promv1.ConfigResult{}, nil } -func (m *mockProm) DeleteSeries(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) error { + +// DeleteSeries deletes data for a selection of series in a time range. +func (m *MockProm) DeleteSeries(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) error { return nil } -func (m *mockProm) Flags(ctx context.Context) (promv1.FlagsResult, error) { + +// Flags returns the flag values that Prometheus was launched with. +func (m *MockProm) Flags(ctx context.Context) (promv1.FlagsResult, error) { return promv1.FlagsResult{}, nil } -func (m *mockProm) LabelValues(ctx context.Context, label string) (model.LabelValues, error) { + +// LabelValues performs a query for the values of the given label. +func (m *MockProm) LabelValues(ctx context.Context, label string) (model.LabelValues, error) { return nil, nil } -func (m *mockProm) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, error) { + +// Series finds series by label matchers. +func (m *MockProm) Series(ctx context.Context, matches []string, startTime time.Time, endTime time.Time) ([]model.LabelSet, error) { return nil, nil } -func (m *mockProm) Snapshot(ctx context.Context, skipHead bool) (promv1.SnapshotResult, error) { + +// Snapshot creates a snapshot of all current data into +// snapshots/- under the TSDB's data directory and returns the +// directory as response. +func (m *MockProm) Snapshot(ctx context.Context, skipHead bool) (promv1.SnapshotResult, error) { return promv1.SnapshotResult{}, nil } -func (m *mockProm) Targets(ctx context.Context) (promv1.TargetsResult, error) { + +// Targets returns an overview of the current state of the Prometheus target +// discovery. +func (m *MockProm) Targets(ctx context.Context) (promv1.TargetsResult, error) { return promv1.TargetsResult{}, nil } @@ -398,13 +425,13 @@ type expectedStatRPC struct { expectedPrometheusQueries []string // queries we expect public-api to issue to prometheus } -func newMockGrpcServer(exp expectedStatRPC) (*mockProm, *grpcServer, error) { +func newMockGrpcServer(exp expectedStatRPC) (*MockProm, *grpcServer, error) { k8sAPI, err := k8s.NewFakeAPI(exp.k8sConfigs...) if err != nil { return nil, nil, err } - mockProm := &mockProm{Res: exp.mockPromResponse} + mockProm := &MockProm{Res: exp.mockPromResponse} fakeGrpcServer := newGrpcServer( mockProm, nil, @@ -420,7 +447,7 @@ func newMockGrpcServer(exp expectedStatRPC) (*mockProm, *grpcServer, error) { return mockProm, fakeGrpcServer, nil } -func (exp expectedStatRPC) verifyPromQueries(mockProm *mockProm) error { +func (exp expectedStatRPC) verifyPromQueries(mockProm *MockProm) error { // if exp.expectedPrometheusQueries is an empty slice we still wanna check no queries were executed. if exp.expectedPrometheusQueries != nil { sort.Strings(exp.expectedPrometheusQueries) diff --git a/controller/cmd/heartbeat/main.go b/controller/cmd/heartbeat/main.go new file mode 100644 index 000000000..483393d85 --- /dev/null +++ b/controller/cmd/heartbeat/main.go @@ -0,0 +1,58 @@ +package main + +import ( + "flag" + "net/url" + + "github.com/linkerd/linkerd2/controller/heartbeat" + "github.com/linkerd/linkerd2/pkg/flags" + "github.com/linkerd/linkerd2/pkg/k8s" + "github.com/linkerd/linkerd2/pkg/version" + promApi "github.com/prometheus/client_golang/api" + promv1 "github.com/prometheus/client_golang/api/prometheus/v1" + log "github.com/sirupsen/logrus" +) + +func main() { + kubeConfigPath := flag.String("kubeconfig", "", "path to kube config") + prometheusURL := flag.String("prometheus-url", "http://127.0.0.1:9090", "prometheus url") + controllerNamespace := flag.String("controller-namespace", "linkerd", "namespace in which Linkerd is installed") + flags.ConfigureAndParse() + + // Gather the following fields: + // - version + // - source + // - uuid + // - k8s-version + // - install-time + // - rps + // - meshed-pods + // TODO: + // - k8s-env + // - proxy-injector-injections + v := url.Values{} + v.Set("version", version.Version) + v.Set("source", "heartbeat") + + kubeAPI, err := k8s.NewAPI(*kubeConfigPath, "", 0) + if err != nil { + log.Errorf("Failed to initialize k8s API: %s", err) + } else { + k8sV := heartbeat.K8sValues(kubeAPI, *controllerNamespace) + v = heartbeat.MergeValues(v, k8sV) + } + + prometheusClient, err := promApi.NewClient(promApi.Config{Address: *prometheusURL}) + if err != nil { + log.Errorf("Failed to initialize Prometheus client: %s", err) + } else { + promAPI := promv1.NewAPI(prometheusClient) + promV := heartbeat.PromValues(promAPI) + v = heartbeat.MergeValues(v, promV) + } + + err = heartbeat.Send(v) + if err != nil { + log.Fatalf("Failed to send heartbeat: %s", err) + } +} diff --git a/controller/heartbeat/heartbeat.go b/controller/heartbeat/heartbeat.go new file mode 100644 index 000000000..fa5874554 --- /dev/null +++ b/controller/heartbeat/heartbeat.go @@ -0,0 +1,125 @@ +package heartbeat + +import ( + "context" + "fmt" + "io/ioutil" + "math" + "net/http" + "net/url" + "strconv" + "time" + + "github.com/linkerd/linkerd2/pkg/healthcheck" + "github.com/linkerd/linkerd2/pkg/k8s" + "github.com/linkerd/linkerd2/pkg/version" + promv1 "github.com/prometheus/client_golang/api/prometheus/v1" + "github.com/prometheus/common/model" + log "github.com/sirupsen/logrus" +) + +// K8sValues gathers relevant heartbeat information from Kubernetes +func K8sValues(kubeAPI *k8s.KubernetesAPI, controlPlaneNamespace string) url.Values { + v := url.Values{} + + cm, configPB, err := healthcheck.FetchLinkerdConfigMap(kubeAPI, controlPlaneNamespace) + if err != nil { + log.Errorf("Failed to fetch linkerd-config: %s", err) + } else { + v.Set("uuid", configPB.GetInstall().GetUuid()) + v.Set("install-time", strconv.FormatInt(cm.GetCreationTimestamp().Unix(), 10)) + } + + versionInfo, err := kubeAPI.GetVersionInfo() + if err != nil { + log.Errorf("Failed to fetch Kubernetes version info: %s", err) + } else { + v.Set("k8s-version", versionInfo.String()) + } + + return v +} + +// PromValues gathers relevant heartbeat information from Prometheus +func PromValues(promAPI promv1.API) url.Values { + v := url.Values{} + + value, err := promQuery(promAPI, "sum(irate(request_total{direction=\"inbound\"}[30s]))") + if err != nil { + log.Errorf("Prometheus query failed: %s", err) + } else { + v.Set("total-rps", value) + } + + value, err = promQuery(promAPI, "count(count by (pod) (request_total))") + if err != nil { + log.Errorf("Prometheus query failed: %s", err) + } else { + v.Set("meshed-pods", value) + } + + return v +} + +func promQuery(promAPI promv1.API, query string) (string, error) { + res, err := promAPI.Query(context.Background(), query, time.Time{}) + if err != nil { + return "", err + } + + switch result := res.(type) { + case model.Vector: + if len(result) != 1 { + return "", fmt.Errorf("unexpected result Prometheus result vector length: %d", len(result)) + } + f := float64(result[0].Value) + if math.IsNaN(f) { + return "", fmt.Errorf("unexpected sample value: %v", result[0].Value) + } + + value := int64(math.Round(f)) + return strconv.FormatInt(value, 10), nil + } + + return "", fmt.Errorf("unexpected query result type (expected Vector): %s", res.Type()) +} + +// MergeValues merges two url.Values +func MergeValues(v1, v2 url.Values) url.Values { + v := url.Values{} + for key, val := range v1 { + v[key] = val + } + for key, val := range v2 { + v[key] = val + } + return v +} + +// Send takes a map of url.Values and sends them to versioncheck.linkerd.io +func Send(v url.Values) error { + return send(http.DefaultClient, version.CheckURL, v) +} + +func send(client *http.Client, baseURL string, v url.Values) error { + req, err := http.NewRequest("GET", baseURL, nil) + if err != nil { + return fmt.Errorf("failed to create HTTP request for base URL [%s]: %s", baseURL, err) + } + req.URL.RawQuery = v.Encode() + + log.Infof("Sending heartbeat: %s", req.URL.String()) + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("Check URL [%s] request failed with: %s", req.URL.String(), err) + } + defer resp.Body.Close() + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return fmt.Errorf("failed to read response body: %s", err) + } + + log.Infof("Successfully sent heartbeat: %s", string(body)) + + return nil +} diff --git a/controller/heartbeat/heartbeat_test.go b/controller/heartbeat/heartbeat_test.go new file mode 100644 index 000000000..3248724ee --- /dev/null +++ b/controller/heartbeat/heartbeat_test.go @@ -0,0 +1,181 @@ +package heartbeat + +import ( + "fmt" + "net/http" + "net/http/httptest" + "net/url" + "reflect" + "testing" + + "github.com/linkerd/linkerd2/controller/api/public" + "github.com/linkerd/linkerd2/pkg/k8s" + "github.com/prometheus/common/model" +) + +func TestK8sValues(t *testing.T) { + testCases := []struct { + namespace string + k8sConfigs []string + expected url.Values + }{ + { + "linkerd", + []string{` +kind: ConfigMap +apiVersion: v1 +metadata: + name: linkerd-config + namespace: linkerd + creationTimestamp: 2019-02-15T12:34:56Z +data: + install: | + {"uuid":"fake-uuid"}`, + }, + url.Values{ + "k8s-version": []string{"v0.0.0-master+$Format:%h$"}, + "install-time": []string{"1550234096"}, + "uuid": []string{"fake-uuid"}, + }, + }, + { + "bad-ns", + []string{` +kind: ConfigMap +apiVersion: v1 +metadata: + name: linkerd-config + namespace: linkerd +data: + install: | + {"uuid":"fake-uuid"}`, + }, + url.Values{ + "k8s-version": []string{"v0.0.0-master+$Format:%h$"}, + }, + }, + } + + for i, tc := range testCases { + tc := tc // pin + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + k8sAPI, err := k8s.NewFakeAPI(tc.k8sConfigs...) + if err != nil { + t.Fatalf("NewFakeAPI returned an error: %s", err) + } + + v := K8sValues(k8sAPI, tc.namespace) + if !reflect.DeepEqual(v, tc.expected) { + t.Fatalf("K8sValues returned: %+v, expected: %+v", v, tc.expected) + } + }) + } +} + +func TestPromValues(t *testing.T) { + testCases := []struct { + promRes model.Value + expected url.Values + }{ + { + model.Vector{ + &model.Sample{ + Metric: model.Metric{"pod": "emojivoto-meshed"}, + Value: 100.01, + Timestamp: 456, + }, + }, + url.Values{ + "total-rps": []string{"100"}, + "meshed-pods": []string{"100"}, + }, + }, + { + model.Vector{}, + url.Values{}, + }, + } + + for i, tc := range testCases { + tc := tc // pin + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + v := PromValues(&public.MockProm{Res: tc.promRes}) + if !reflect.DeepEqual(v, tc.expected) { + t.Fatalf("PromValues returned: %+v, expected: %+v", v, tc.expected) + } + }) + } +} + +func TestMergeValues(t *testing.T) { + testCases := []struct { + v1, v2, expected url.Values + }{ + { + url.Values{ + "a": []string{"b"}, + "c": []string{"d"}, + }, + url.Values{ + "e": []string{"f"}, + "g": []string{"h"}, + }, + url.Values{ + "a": []string{"b"}, + "c": []string{"d"}, + "e": []string{"f"}, + "g": []string{"h"}, + }, + }, + { + url.Values{}, + url.Values{}, + url.Values{}, + }, + } + + for i, tc := range testCases { + tc := tc // pin + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + v := MergeValues(tc.v1, tc.v2) + if !reflect.DeepEqual(v, tc.expected) { + t.Fatalf("MergeValues returned: %+v, expected: %+v", v, tc.expected) + } + }) + } +} + +func TestSend(t *testing.T) { + testCases := []struct { + v url.Values + err error + }{ + { + url.Values{ + "a": []string{"b"}, + "c": []string{"d"}, + }, + nil, + }, + } + + for i, tc := range testCases { + tc := tc // pin + t.Run(fmt.Sprintf("%d", i), func(t *testing.T) { + ts := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + if !reflect.DeepEqual(r.URL.Query(), tc.v) { + t.Fatalf("Send queried for: %+v, expected: %+v", r.URL.Query(), tc.v) + } + w.Write([]byte(`{"stable":"stable-a.b.c","edge":"edge-d.e.f"}`)) + }), + ) + defer ts.Close() + + err := send(ts.Client(), ts.URL, tc.v) + if !reflect.DeepEqual(err, tc.err) { + t.Fatalf("Send returned: %+v, expected: %+v", err, tc.err) + } + }) + } +} diff --git a/pkg/healthcheck/healthcheck.go b/pkg/healthcheck/healthcheck.go index 22d492921..c7675d5ee 100644 --- a/pkg/healthcheck/healthcheck.go +++ b/pkg/healthcheck/healthcheck.go @@ -933,7 +933,12 @@ func (hc *HealthChecker) PublicAPIClient() public.APIClient { } func (hc *HealthChecker) checkLinkerdConfigConfigMap() (*configPb.All, error) { - return FetchLinkerdConfigMap(hc.kubeAPI, hc.ControlPlaneNamespace) + _, configPB, err := FetchLinkerdConfigMap(hc.kubeAPI, hc.ControlPlaneNamespace) + if err != nil { + return nil, err + } + + return configPB, nil } // FetchLinkerdConfigMap retrieves the `linkerd-config` ConfigMap from @@ -942,13 +947,18 @@ func (hc *HealthChecker) checkLinkerdConfigConfigMap() (*configPb.All, error) { // healthcheck package because healthcheck depends on it, along with other // packages that also depend on healthcheck. This function depends on both // `pkg/k8s` and `pkg/config`, which do not depend on each other. -func FetchLinkerdConfigMap(k kubernetes.Interface, controlPlaneNamespace string) (*configPb.All, error) { +func FetchLinkerdConfigMap(k kubernetes.Interface, controlPlaneNamespace string) (*corev1.ConfigMap, *configPb.All, error) { cm, err := k.CoreV1().ConfigMaps(controlPlaneNamespace).Get(k8s.ConfigConfigMapName, metav1.GetOptions{}) if err != nil { - return nil, err + return nil, nil, err } - return config.FromConfigMap(cm.Data) + configPB, err := config.FromConfigMap(cm.Data) + if err != nil { + return nil, nil, err + } + + return cm, configPB, nil } // checkNamespace checks whether the given namespace exists, and returns an @@ -982,6 +992,7 @@ func expectedServiceAccountNames() []string { return []string{ "linkerd-controller", "linkerd-grafana", + "linkerd-heartbeat", "linkerd-identity", "linkerd-prometheus", "linkerd-proxy-injector", diff --git a/pkg/healthcheck/healthcheck_test.go b/pkg/healthcheck/healthcheck_test.go index 09f293d78..459bd7172 100644 --- a/pkg/healthcheck/healthcheck_test.go +++ b/pkg/healthcheck/healthcheck_test.go @@ -683,6 +683,15 @@ metadata: ` kind: ServiceAccount apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: test-ns + labels: + linkerd.io/control-plane-ns: test-ns +`, + ` +kind: ServiceAccount +apiVersion: v1 metadata: name: linkerd-web namespace: test-ns @@ -867,6 +876,15 @@ metadata: ` kind: ServiceAccount apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: test-ns + labels: + linkerd.io/control-plane-ns: test-ns +`, + ` +kind: ServiceAccount +apiVersion: v1 metadata: name: linkerd-web namespace: test-ns @@ -1060,6 +1078,15 @@ metadata: ` kind: ServiceAccount apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: test-ns + labels: + linkerd.io/control-plane-ns: test-ns +`, + ` +kind: ServiceAccount +apiVersion: v1 metadata: name: linkerd-web namespace: test-ns @@ -1262,6 +1289,15 @@ metadata: ` kind: ServiceAccount apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: test-ns + labels: + linkerd.io/control-plane-ns: test-ns +`, + ` +kind: ServiceAccount +apiVersion: v1 metadata: name: linkerd-web namespace: test-ns @@ -1473,6 +1509,15 @@ metadata: ` kind: ServiceAccount apiVersion: v1 +metadata: + name: linkerd-heartbeat + namespace: test-ns + labels: + linkerd.io/control-plane-ns: test-ns +`, + ` +kind: ServiceAccount +apiVersion: v1 metadata: name: linkerd-web namespace: test-ns @@ -2093,7 +2138,7 @@ data: t.Fatalf("Unexpected error: %s", err) } - configs, err := FetchLinkerdConfigMap(clientset, "linkerd") + _, configs, err := FetchLinkerdConfigMap(clientset, "linkerd") if !reflect.DeepEqual(err, tc.err) { t.Fatalf("Expected \"%+v\", got \"%+v\"", tc.err, err) } diff --git a/pkg/version/channels.go b/pkg/version/channels.go index f72bfdd7b..49ba52cd2 100644 --- a/pkg/version/channels.go +++ b/pkg/version/channels.go @@ -17,7 +17,8 @@ type Channels struct { } const ( - versionCheckURL = "https://versioncheck.linkerd.io/version.json?version=%s&uuid=%s&source=%s" + // CheckURL provides an online endpoint for Linkerd's version checks + CheckURL = "https://versioncheck.linkerd.io/version.json" ) // NewChannels is used primarily for testing, it returns a Channels struct that @@ -59,7 +60,7 @@ func (c Channels) Match(actualVersion string) error { // GetLatestVersions performs an online request to check for the latest Linkerd // release channels. func GetLatestVersions(ctx context.Context, uuid string, source string) (Channels, error) { - url := fmt.Sprintf(versionCheckURL, Version, uuid, source) + url := fmt.Sprintf("%s?version=%s&uuid=%s&source=%s", CheckURL, Version, uuid, source) return getLatestVersions(ctx, http.DefaultClient, url) } diff --git a/test/get/get_test.go b/test/get/get_test.go index 30b63f184..464cd4667 100644 --- a/test/get/get_test.go +++ b/test/get/get_test.go @@ -88,7 +88,7 @@ func TestCliGet(t *testing.T) { t.Fatalf("Unexpected error: %v output:\n%s", err, out) } - err := checkPodOutput(out, deployReplicas, prefixedNs) + err := checkPodOutput(out, deployReplicas, "", prefixedNs) if err != nil { t.Fatalf("Pod output check failed:\n%s\nCommand output:\n%s", err, out) } @@ -101,14 +101,14 @@ func TestCliGet(t *testing.T) { t.Fatalf("Unexpected error: %v output:\n%s", err, out) } - err := checkPodOutput(out, linkerdPods, TestHelper.GetLinkerdNamespace()) + err := checkPodOutput(out, linkerdPods, "linkerd-heartbeat", TestHelper.GetLinkerdNamespace()) if err != nil { t.Fatalf("Pod output check failed:\n%s\nCommand output:\n%s", err, out) } }) } -func checkPodOutput(cmdOutput string, expectedPodCounts map[string]int, namespace string) error { +func checkPodOutput(cmdOutput string, expectedPodCounts map[string]int, optionalPod string, namespace string) error { expectedPods := []string{} for podName, replicas := range expectedPodCounts { for i := 0; i < replicas; i++ { @@ -146,7 +146,15 @@ func checkPodOutput(cmdOutput string, expectedPodCounts map[string]int, namespac sort.Strings(expectedPods) sort.Strings(actualPods) if !reflect.DeepEqual(expectedPods, actualPods) { - return fmt.Errorf("Expected linkerd get to return:\n%v\nBut got:\n%v", expectedPods, actualPods) + if optionalPod == "" { + return fmt.Errorf("Expected linkerd get to return:\n%v\nBut got:\n%v", expectedPods, actualPods) + } + + expectedPlusOptionalPods := append(expectedPods, optionalPod) + sort.Strings(expectedPlusOptionalPods) + if !reflect.DeepEqual(expectedPlusOptionalPods, actualPods) { + return fmt.Errorf("Expected linkerd get to return:\n%v\nor:\n%v\nBut got:\n%v", expectedPods, expectedPlusOptionalPods, actualPods) + } } return nil