feat: create datadog notification provider

Signed-off-by: Michael Parker <michael@parker.gg>
This commit is contained in:
Michael Parker 2023-08-02 10:44:34 -05:00 committed by Stefan Prodan
parent 049ded50b9
commit 71ed90ee8c
No known key found for this signature in database
GPG Key ID: 3299AEB0E4085BAF
10 changed files with 373 additions and 1 deletions

View File

@ -49,12 +49,13 @@ const (
OpsgenieProvider string = "opsgenie"
AlertManagerProvider string = "alertmanager"
PagerDutyProvider string = "pagerduty"
DataDogProvider string = "datadog"
)
// ProviderSpec defines the desired state of the Provider.
type ProviderSpec struct {
// Type specifies which Provider implementation to use.
// +kubebuilder:validation:Enum=slack;discord;msteams;rocket;generic;generic-hmac;github;gitlab;gitea;bitbucket;azuredevops;googlechat;googlepubsub;webex;sentry;azureeventhub;telegram;lark;matrix;opsgenie;alertmanager;grafana;githubdispatch;pagerduty
// +kubebuilder:validation:Enum=slack;discord;msteams;rocket;generic;generic-hmac;github;gitlab;gitea;bitbucket;azuredevops;googlechat;googlepubsub;webex;sentry;azureeventhub;telegram;lark;matrix;opsgenie;alertmanager;grafana;githubdispatch;pagerduty;datadog
// +required
Type string `json:"type"`

View File

@ -305,6 +305,7 @@ spec:
- grafana
- githubdispatch
- pagerduty
- datadog
type: string
username:
description: Username specifies the name under which events are posted.

View File

@ -109,6 +109,7 @@ The supported alerting providers are:
| [Generic webhook](#generic-webhook) | `generic` |
| [Generic webhook with HMAC](#generic-webhook-with-hmac) | `generic-hmac` |
| [Azure Event Hub](#azure-event-hub) | `azureeventhub` |
| [DataDog](#datadog) | `datadog` |
| [Discord](#discord) | `discord` |
| [GitHub dispatch](#github-dispatch) | `githubdispatch` |
| [Google Chat](#google-chat) | `googlechat` |
@ -405,6 +406,62 @@ stringData:
address: "https://xxx.webhook.office.com/..."
```
##### DataDog
When `.spec.type` is set to `datadog`, the controller will send a payload for
an [Event](events.md#event-structure) to the provided DataDog API [Address](#address).
The Event will be formatted into a [DataDog Event](https://docs.datadoghq.com/api/latest/events/#post-an-event) and sent to the
API endpoint of the provided DataDog [Address](#address).
This Provider type supports the configuration of a [proxy URL](#https-proxy)
and/or [TLS certificates](#tls-certificates).
The metadata of the Event is included in the DataDog event as extra tags.
###### DataDog example
To configure a Provider for DataDog, create a Secret with [the `token`](#token-example)
set to a [DataDog API key](https://docs.datadoghq.com/account_management/api-app-keys/#api-keys)
(not an application key!) and a `datadog` Provider with a [Secret reference](#secret-reference).
```yaml
---
apiVersion: notification.toolkit.fluxcd.io/v1beta2
kind: Provider
metadata:
name: datadog
namespace: default
spec:
type: datadog
address: https://api.datadoghq.com # DataDog Site US1
secretRef:
name: datadog-secret
---
apiVersion: v1
kind: Secret
metadata:
name: datadog-secret
namespace: default
stringData:
token: <DataDog API Key>
---
apiVersion: notification.toolkit.fluxcd.io/v1beta1
kind: Alert
metadata:
name: datadog-info
namespace: default
spec:
eventSeverity: info
eventSources:
- kind: HelmRelease
name: "*"
providerRef:
name: datadog
eventMetadata:
env: my-k8s-cluster # example of adding a custom `env` tag to the event
```
##### Discord
When `.spec.type` is set to `discord`, the controller will send a payload for

3
go.mod
View File

@ -10,6 +10,7 @@ require (
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1
github.com/Azure/azure-amqp-common-go/v4 v4.2.0
github.com/Azure/azure-event-hubs-go/v3 v3.6.1
github.com/DataDog/datadog-api-client-go/v2 v2.15.0
github.com/PagerDuty/go-pagerduty v1.7.0
github.com/containrrr/shoutrrr v0.7.1
github.com/fluxcd/notification-controller/api v1.0.0
@ -62,6 +63,7 @@ require (
github.com/Azure/go-autorest/autorest/validation v0.3.1 // indirect
github.com/Azure/go-autorest/logger v0.2.1 // indirect
github.com/Azure/go-autorest/tracing v0.6.0 // indirect
github.com/DataDog/zstd v1.5.2 // indirect
github.com/MakeNowJust/heredoc v1.0.0 // indirect
github.com/ProtonMail/go-crypto v0.0.0-20230717121422-5aa5874ade95 // indirect
github.com/beorn7/perks v1.0.1 // indirect
@ -83,6 +85,7 @@ require (
github.com/go-openapi/jsonpointer v0.19.6 // indirect
github.com/go-openapi/jsonreference v0.20.2 // indirect
github.com/go-openapi/swag v0.22.3 // indirect
github.com/goccy/go-json v0.10.2 // indirect
github.com/gogo/protobuf v1.3.2 // indirect
github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

6
go.sum
View File

@ -647,7 +647,11 @@ github.com/Azure/go-autorest/tracing v0.6.0 h1:TYi4+3m5t6K48TGI9AUdb+IzbnSxvnvUM
github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
github.com/DataDog/datadog-api-client-go/v2 v2.15.0 h1:5UVON1xs6Lul4d6R5TmLDqqSJxOkunkm/UdM/fjm+zc=
github.com/DataDog/datadog-api-client-go/v2 v2.15.0/go.mod h1:ZG8wS+y2rUmkRDJZQq7Og7EAPFPage+7vXcmuah2I9o=
github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ=
github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8=
github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw=
github.com/JohnCGriffin/overflow v0.0.0-20211019200055-46fa312c352c/go.mod h1:X0CRv0ky0k6m906ixxpzmDRLvX58TFUKS2eePweuyxk=
github.com/MakeNowJust/heredoc v1.0.0 h1:cXCdzVdstXyiTqTvfqk9SDHpKNjxuom+DOlyEeQ4pzQ=
github.com/MakeNowJust/heredoc v1.0.0/go.mod h1:mG5amYoWBHf8vpLOuehzbGGw0EHxpZZ6lCpQ4fNJ8LE=
@ -825,6 +829,8 @@ github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/me
github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE=
github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI=
github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/goccy/go-json v0.10.2 h1:CrxCmQqYDkv1z7lO7Wbh2HN93uovUHgrECaO5ZrCXAU=
github.com/goccy/go-json v0.10.2/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I=
github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q=

View File

@ -0,0 +1,166 @@
/*
Copyright 2023 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package notifier
import (
"context"
"crypto/tls"
"crypto/x509"
"fmt"
"net/http"
"net/url"
"strings"
"github.com/DataDog/datadog-api-client-go/v2/api/datadog"
"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1"
)
type DataDog struct {
apiClient *datadog.APIClient
eventsApi *datadogV1.EventsApi
apiKey string
}
// NewDataDog creates a new DataDog provider by mapping the notification provider API to sensible values for the DataDog API.
// url: The DataDog API endpoint to use. Examples: https://api.datadoghq.com, https://api.datadoghq.eu, etc.
// token: The DataDog API key (not the application key).
// headers: A map of extra tags to add to the event
func NewDataDog(address string, proxyUrl string, certPool *x509.CertPool, token string) (*DataDog, error) {
conf := datadog.NewConfiguration()
if token == "" {
return nil, fmt.Errorf("token cannot be empty")
}
baseUrl, err := url.Parse(address)
if err != nil {
return nil, fmt.Errorf("failed to parse address %q: %w", address, err)
}
conf.Host = baseUrl.Host
conf.Scheme = baseUrl.Scheme
if proxyUrl != "" || certPool != nil {
transport := &http.Transport{}
if proxyUrl != "" {
proxy, err := url.Parse(proxyUrl)
if err != nil {
return nil, fmt.Errorf("failed to parse proxy URL %q: %w", proxyUrl, err)
}
transport.Proxy = http.ProxyURL(proxy)
}
if certPool != nil {
transport.TLSClientConfig = &tls.Config{
RootCAs: certPool,
}
}
conf.HTTPClient = &http.Client{
Transport: transport,
}
}
apiClient := datadog.NewAPIClient(conf)
eventsApi := datadogV1.NewEventsApi(apiClient)
return &DataDog{
apiClient: apiClient,
eventsApi: eventsApi,
apiKey: token,
}, nil
}
func (d *DataDog) Post(ctx context.Context, event eventv1.Event) error {
dataDogEvent := d.toDataDogEvent(&event)
_, _, err := d.eventsApi.CreateEvent(d.dataDogCtx(ctx), dataDogEvent)
if err != nil {
return fmt.Errorf("failed to post event to DataDog: %w", err)
}
return nil
}
// dataDogCtx returns a context with the DataDog API key set.
// This is one way to authenticate with the DataDog API.
func (d *DataDog) dataDogCtx(ctx context.Context) context.Context {
return context.WithValue(ctx, datadog.ContextAPIKeys, map[string]datadog.APIKey{
"apiKeyAuth": {
Key: d.apiKey,
},
})
}
// toDataDogEvent converts an eventv1.Event to a datadogV1.EventCreateRequest.
func (d *DataDog) toDataDogEvent(event *eventv1.Event) datadogV1.EventCreateRequest {
return datadogV1.EventCreateRequest{
// Note: Title's printf format matches other events from datadog's kubernetes integration
Title: fmt.Sprintf("Events from the %s %s/%s", event.InvolvedObject.Kind, event.InvolvedObject.Name, event.InvolvedObject.Namespace),
Text: event.Message,
Tags: d.toDataDogTags(event),
// fluxcd matches the name datadog picked for their flux integration: https://docs.datadoghq.com/integrations/fluxcd/
SourceTypeName: strPtr("fluxcd"),
DateHappened: int64Ptr(event.Timestamp.Unix()),
AlertType: toDataDogAlertType(event),
}
}
// toDataDogTags parses an eventv1.Event to return a slice of tags.
// We set kind, name, and namespace to the appropriate values of the involved object.
func (d *DataDog) toDataDogTags(event *eventv1.Event) []string {
// Note: Datadog's built in kubernetes tagging is documented here: https://docs.datadoghq.com/containers/kubernetes/tag/?tab=containerizedagent#out-of-the-box-tags
tags := []string{
fmt.Sprintf("flux_reporting_controller:%s", event.ReportingController),
fmt.Sprintf("flux_reason:%s", event.Reason),
// Note: DataDog standardizes kubernetes tags as "kube_*": https://github.com/DataDog/datadog-agent/blob/82dc933aa86de037c70fe960384aa06a62e457a8/pkg/collector/corechecks/cluster/kubernetesapiserver/events_common.go#L48
fmt.Sprintf("kube_kind:%s", event.InvolvedObject.Kind),
fmt.Sprintf("kube_name:%s", event.InvolvedObject.Name),
fmt.Sprintf("kube_namespace:%s", event.InvolvedObject.Namespace),
}
// add extra tags from event metadata
for k, v := range event.Metadata {
tags = append(tags, fmt.Sprintf("%s:%s", k, v))
}
// Note: https://docs.datadoghq.com/getting_started/tagging/
// "Tags are converted to lowercase"
// To keep the events consistent, we run toLower on all input strings.
for idx := range tags {
tags[idx] = strings.ToLower(tags[idx])
}
return tags
}
// toDataDogAlertType parses an eventv1.Event to return a datadogV1.EventAlertType.
func toDataDogAlertType(event *eventv1.Event) *datadogV1.EventAlertType {
if event.Severity == eventv1.EventSeverityError {
return dataDogEventAlertTypePtr(datadogV1.EVENTALERTTYPE_ERROR)
}
return dataDogEventAlertTypePtr(datadogV1.EVENTALERTTYPE_INFO)
}
func dataDogEventAlertTypePtr(t datadogV1.EventAlertType) *datadogV1.EventAlertType {
return &t
}

View File

@ -0,0 +1,53 @@
package notifier
import (
"context"
"crypto/x509"
"io"
"net/http"
"net/http/httptest"
"testing"
fuzz "github.com/AdaLogics/go-fuzz-headers"
eventv1 "github.com/fluxcd/pkg/apis/event/v1beta1"
"github.com/stretchr/testify/require"
)
func Fuzz_DataDog(f *testing.F) {
f.Add("token", "error", "", []byte{}, []byte{})
f.Add("token", "info", "", []byte{}, []byte{})
f.Fuzz(func(t *testing.T,
apiKey, severity, message string, seed, response []byte) {
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/events", func(w http.ResponseWriter, r *http.Request) {
_, err := w.Write(response)
require.NoError(t, err)
_, err = io.Copy(io.Discard, r.Body)
require.NoError(t, err)
require.NoError(t, r.Body.Close())
})
ts := httptest.NewServer(mux)
defer ts.Close()
var cert x509.CertPool
_ = fuzz.NewConsumer(seed).GenerateStruct(&cert)
dd, err := NewDataDog(ts.URL, "", &cert, apiKey)
if err != nil {
return
}
event := eventv1.Event{}
_ = fuzz.NewConsumer(seed).GenerateStruct(&event)
if event.Metadata == nil {
event.Metadata = map[string]string{}
}
event.Message = message
event.Severity = severity
_ = dd.Post(context.TODO(), event)
})
}

View File

@ -0,0 +1,75 @@
package notifier
import (
"context"
"encoding/json"
"fmt"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/DataDog/datadog-api-client-go/v2/api/datadogV1"
"github.com/stretchr/testify/require"
)
func TestDataDogPost(t *testing.T) {
thisRun := func(expectedToFail bool) func(t *testing.T) {
return func(t *testing.T) {
ddApiKey := "sdfsdf"
mux := http.NewServeMux()
mux.HandleFunc("/api/v1/events", func(w http.ResponseWriter, r *http.Request) {
b, err := io.ReadAll(r.Body)
require.NoError(t, err)
var payload datadogV1.EventCreateRequest
err = json.Unmarshal(b, &payload)
require.NoError(t, err)
if expectedToFail {
w.WriteHeader(http.StatusForbidden)
}
})
ts := httptest.NewServer(mux)
defer ts.Close()
dd, err := NewDataDog(ts.URL, "", nil, ddApiKey)
require.NoError(t, err)
err = dd.Post(context.Background(), testEvent())
if expectedToFail {
require.Error(t, err)
} else {
require.NoError(t, err)
}
}
}
t.Run("working", thisRun(false))
t.Run("failing", thisRun(true))
}
func TestDataDogProviderErrors(t *testing.T) {
_, err := NewDataDog("https://api.datadoghq.com", "", nil, "")
require.Error(t, err)
require.Equal(t, "token cannot be empty", err.Error())
_, err = NewDataDog("https://bad url :)", "", nil, "token")
require.Error(t, err)
require.Contains(t, err.Error(), "failed to parse address")
}
func TestToDataDogTags(t *testing.T) {
dd, err := NewDataDog("https://api.datadoghq.com", "", nil, "token")
require.NoError(t, err)
event := testEvent()
tags := dd.toDataDogTags(&event)
require.Contains(t, tags, "test:metadata")
require.Contains(t, tags, fmt.Sprintf("kube_kind:%s", strings.ToLower(event.InvolvedObject.Kind)))
require.Contains(t, tags, fmt.Sprintf("kube_namespace:%s", event.InvolvedObject.Namespace))
require.Contains(t, tags, fmt.Sprintf("kube_name:%s", strings.ToLower(event.InvolvedObject.Name)))
require.Contains(t, tags, fmt.Sprintf("flux_reporting_controller:%s", strings.ToLower(event.ReportingController)))
require.Contains(t, tags, fmt.Sprintf("flux_reason:%s", strings.ToLower(event.Reason)))
}

View File

@ -113,6 +113,8 @@ func (f Factory) Notifier(provider string) (Interface, error) {
n, err = NewGrafana(f.URL, f.ProxyURL, f.Token, f.CertPool, f.Username, f.Password)
case apiv1.PagerDutyProvider:
n, err = NewPagerDuty(f.URL, f.ProxyURL, f.CertPool, f.Channel)
case apiv1.DataDogProvider:
n, err = NewDataDog(f.URL, f.ProxyURL, f.CertPool, f.Token)
default:
err = fmt.Errorf("provider %s not supported", provider)
}

View File

@ -125,3 +125,11 @@ func basicAuth(username, password string) string {
auth := username + ":" + password
return base64.StdEncoding.EncodeToString([]byte(auth))
}
func strPtr(s string) *string {
return &s
}
func int64Ptr(i int64) *int64 {
return &i
}