From f1c8807c0d0b11ecbc80cddb6e2aa3b51596b95b Mon Sep 17 00:00:00 2001 From: Thomas Banks Date: Fri, 31 Jan 2025 16:33:15 +0000 Subject: [PATCH] feat: add knative integration Signed-off-by: Sanskar Jaiswal Co-authored-by: Thomas Banks --- .github/workflows/e2e.yaml | 1 + artifacts/flagger/crd.yaml | 1 - charts/flagger/crds/crd.yaml | 1 - charts/flagger/templates/rbac.yaml | 13 ++ cmd/flagger/main.go | 14 +- go.mod | 56 +++--- go.sum | 153 ++++++++++------- kustomize/base/flagger/crd.yaml | 1 - kustomize/base/flagger/rbac.yaml | 14 ++ kustomize/knative/kustomization.yaml | 8 + kustomize/knative/namespace.yaml | 9 + kustomize/knative/patch.yaml | 14 ++ pkg/apis/flagger/v1beta1/canary.go | 8 + pkg/apis/flagger/v1beta1/provider.go | 1 + pkg/canary/factory.go | 19 ++- pkg/canary/knative_controller.go | 161 ++++++++++++++++++ pkg/canary/knative_controller_test.go | 79 +++++++++ pkg/canary/knative_fixture_test.go | 99 +++++++++++ pkg/controller/controller.go | 26 +++ pkg/controller/controller_test.go | 72 ++++++++ pkg/controller/finalizer.go | 2 +- pkg/controller/scheduler.go | 2 +- .../scheduler_daemonset_fixture_test.go | 10 +- .../scheduler_deployment_fixture_test.go | 10 +- pkg/controller/scheduler_metrics.go | 47 ++++- pkg/metrics/observers/factory.go | 4 + pkg/metrics/observers/knative.go | 92 ++++++++++ pkg/metrics/observers/knative_test.go | 102 +++++++++++ pkg/router/factory.go | 8 + pkg/router/knative.go | 127 ++++++++++++++ pkg/router/knative_test.go | 96 +++++++++++ pkg/router/router_test.go | 75 ++++++++ test/knative/init.sh | 41 +++++ test/knative/install.sh | 26 +++ test/knative/run.sh | 11 ++ test/knative/test-canary.sh | 159 +++++++++++++++++ 36 files changed, 1453 insertions(+), 109 deletions(-) create mode 100644 kustomize/knative/kustomization.yaml create mode 100644 kustomize/knative/namespace.yaml create mode 100644 kustomize/knative/patch.yaml create mode 100644 pkg/canary/knative_controller.go create mode 100644 pkg/canary/knative_controller_test.go create mode 100644 pkg/canary/knative_fixture_test.go create mode 100644 pkg/metrics/observers/knative.go create mode 100644 pkg/metrics/observers/knative_test.go create mode 100644 pkg/router/knative.go create mode 100644 pkg/router/knative_test.go create mode 100755 test/knative/init.sh create mode 100755 test/knative/install.sh create mode 100755 test/knative/run.sh create mode 100755 test/knative/test-canary.sh diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 7768764a..f2ee24f3 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -35,6 +35,7 @@ jobs: - gatewayapi - keda - apisix + - knative steps: - name: Checkout uses: actions/checkout@v4 diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml index acc07dca..90482172 100644 --- a/artifacts/flagger/crd.yaml +++ b/artifacts/flagger/crd.yaml @@ -80,7 +80,6 @@ spec: type: object required: - targetRef - - service - analysis properties: provider: diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml index acc07dca..90482172 100644 --- a/charts/flagger/crds/crd.yaml +++ b/charts/flagger/crds/crd.yaml @@ -80,7 +80,6 @@ spec: type: object required: - targetRef - - service - analysis properties: provider: diff --git a/charts/flagger/templates/rbac.yaml b/charts/flagger/templates/rbac.yaml index ae9c7015..cfb07ef2 100644 --- a/charts/flagger/templates/rbac.yaml +++ b/charts/flagger/templates/rbac.yaml @@ -276,6 +276,19 @@ rules: - /version verbs: - get + - apiGroups: + - serving.knative.dev + resources: + - services + verbs: + - get + - update + - apiGroups: + - serving.knative.dev + resources: + - revisions + verbs: + - get --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/cmd/flagger/main.go b/cmd/flagger/main.go index 6438187f..fa7bb1a9 100644 --- a/cmd/flagger/main.go +++ b/cmd/flagger/main.go @@ -51,6 +51,8 @@ import ( "github.com/fluxcd/flagger/pkg/server" "github.com/fluxcd/flagger/pkg/signals" "github.com/fluxcd/flagger/pkg/version" + + knative "knative.dev/serving/pkg/client/clientset/versioned" ) var ( @@ -110,7 +112,7 @@ func init() { flag.BoolVar(&zapReplaceGlobals, "zap-replace-globals", false, "Whether to change the logging level of the global zap logger.") flag.StringVar(&zapEncoding, "zap-encoding", "json", "Zap logger encoding.") flag.StringVar(&namespace, "namespace", "", "Namespace that flagger would watch canary object.") - flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, contour, gloo, nginx, skipper, traefik, apisix, osm or kuma.") + flag.StringVar(&meshProvider, "mesh-provider", "istio", "Service mesh provider, can be istio, linkerd, appmesh, contour, knative, gloo, nginx, skipper, traefik, apisix, osm or kuma.") flag.StringVar(&selectorLabels, "selector-labels", "app,name,app.kubernetes.io/name", "List of pod labels that Flagger uses to create pod selectors.") flag.StringVar(&ingressAnnotationsPrefix, "ingress-annotations-prefix", "nginx.ingress.kubernetes.io", "Annotations prefix for NGINX ingresses.") flag.StringVar(&ingressClass, "ingress-class", "", "Ingress class used for annotating HTTPProxy objects.") @@ -166,6 +168,11 @@ func main() { logger.Fatalf("Error building flagger clientset: %s", err.Error()) } + knativeClient, err := knative.NewForConfig(cfg) + if err != nil { + logger.Fatalf("Error building knative clientset: %s", err.Error()) + } + // use a remote cluster for routing if a service mesh kubeconfig is specified if kubeconfigServiceMesh == "" { kubeconfigServiceMesh = kubeconfig @@ -221,7 +228,7 @@ func main() { setOwnerRefs = false } - routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, ingressAnnotationsPrefix, ingressClass, logger, meshClient, setOwnerRefs) + routerFactory := router.NewFactory(cfg, kubeClient, flaggerClient, knativeClient, ingressAnnotationsPrefix, ingressClass, logger, meshClient, setOwnerRefs) var configTracker canary.Tracker if enableConfigTracking { @@ -236,10 +243,11 @@ func main() { includeLabelPrefixArray := strings.Split(includeLabelPrefix, ",") - canaryFactory := canary.NewFactory(kubeClient, flaggerClient, configTracker, labels, includeLabelPrefixArray, logger) + canaryFactory := canary.NewFactory(kubeClient, flaggerClient, knativeClient, configTracker, labels, includeLabelPrefixArray, logger) c := controller.NewController( kubeClient, + knativeClient, flaggerClient, infos, controlLoopInterval, diff --git a/go.mod b/go.mod index 93a07f9d..4fe0bd8c 100644 --- a/go.mod +++ b/go.mod @@ -21,14 +21,15 @@ require ( golang.org/x/sync v0.10.0 google.golang.org/api v0.211.0 google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38 - google.golang.org/grpc v1.67.1 - google.golang.org/protobuf v1.35.2 + google.golang.org/grpc v1.69.2 + google.golang.org/protobuf v1.36.2 gopkg.in/h2non/gock.v1 v1.1.2 - k8s.io/api v0.31.3 - k8s.io/apimachinery v0.31.3 - k8s.io/client-go v0.31.3 - k8s.io/code-generator v0.31.3 + k8s.io/api v0.31.4 + k8s.io/apimachinery v0.31.4 + k8s.io/client-go v0.31.4 + k8s.io/code-generator v0.31.4 k8s.io/klog/v2 v2.130.1 + knative.dev/serving v0.44.0 ) require ( @@ -37,27 +38,30 @@ require ( cloud.google.com/go/compute/metadata v0.5.2 // indirect github.com/apapsch/go-jsonmerge/v2 v2.0.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/blendle/zapdriver v1.3.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/emicklei/go-restful/v3 v3.11.0 // indirect + github.com/emicklei/go-restful/v3 v3.12.1 // indirect + github.com/evanphx/json-patch/v5 v5.9.0 // indirect github.com/fxamacker/cbor/v2 v2.7.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect - github.com/go-openapi/jsonpointer v0.19.6 // indirect - github.com/go-openapi/jsonreference v0.20.2 // indirect - github.com/go-openapi/swag v0.22.4 // indirect + github.com/go-openapi/jsonpointer v0.21.0 // indirect + github.com/go-openapi/jsonreference v0.21.0 // indirect + github.com/go-openapi/swag v0.23.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.4 // indirect github.com/google/gnostic-models v0.6.8 // indirect + github.com/google/go-containerregistry v0.13.0 // indirect github.com/google/gofuzz v1.2.0 // indirect github.com/google/s2a-go v0.1.8 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.4 // indirect - github.com/gorilla/websocket v1.5.1 // indirect + github.com/gorilla/websocket v1.5.3 // indirect github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect - github.com/imdario/mergo v0.3.15 // indirect + github.com/imdario/mergo v0.3.16 // indirect github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf // indirect - github.com/jmespath/go-jmespath v0.4.0 // indirect + github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect github.com/klauspost/compress v1.17.9 // indirect @@ -66,6 +70,7 @@ require ( github.com/modern-go/reflect2 v1.0.2 // indirect github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oapi-codegen/runtime v1.0.0 // indirect + github.com/opencontainers/go-digest v1.0.0 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect github.com/prometheus/client_model v0.6.1 // indirect @@ -74,28 +79,31 @@ require ( github.com/spf13/pflag v1.0.5 // indirect github.com/x448/float16 v0.8.4 // indirect go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 // indirect - go.opentelemetry.io/otel v1.29.0 // indirect - go.opentelemetry.io/otel/metric v1.29.0 // indirect - go.opentelemetry.io/otel/trace v1.29.0 // indirect + go.opentelemetry.io/otel v1.31.0 // indirect + go.opentelemetry.io/otel/metric v1.31.0 // indirect + go.opentelemetry.io/otel/trace v1.31.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/crypto v0.31.0 // indirect + golang.org/x/crypto v0.32.0 // indirect golang.org/x/mod v0.22.0 // indirect - golang.org/x/net v0.33.0 // indirect + golang.org/x/net v0.34.0 // indirect golang.org/x/oauth2 v0.24.0 // indirect - golang.org/x/sys v0.28.0 // indirect - golang.org/x/term v0.27.0 // indirect + golang.org/x/sys v0.29.0 // indirect + golang.org/x/term v0.28.0 // indirect golang.org/x/text v0.21.0 // indirect golang.org/x/time v0.8.0 // indirect - golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d // indirect + golang.org/x/tools v0.29.0 // indirect + gomodules.xyz/jsonpatch/v2 v2.4.0 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 // indirect gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect gopkg.in/inf.v0 v0.9.1 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 // indirect - k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 // indirect - k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect + k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 // indirect + k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 // indirect + k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 // indirect + knative.dev/networking v0.0.0-20250117155906-67d1c274ba6a // indirect + knative.dev/pkg v0.0.0-20250117084104-c43477f0052b // indirect sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect sigs.k8s.io/structured-merge-diff/v4 v4.4.1 // indirect sigs.k8s.io/yaml v1.4.0 // indirect diff --git a/go.sum b/go.sum index 52212307..05568cdc 100644 --- a/go.sum +++ b/go.sum @@ -6,6 +6,10 @@ cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixA cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k= cloud.google.com/go/monitoring v1.22.0 h1:mQ0040B7dpuRq1+4YiQD43M2vW9HgoVxY98xhqGT+YI= cloud.google.com/go/monitoring v1.22.0/go.mod h1:hS3pXvaG8KgWTSz+dAdyzPrGUYmi2Q+WFX8g2hqVEZU= +contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d h1:LblfooH1lKOpp1hIhukktmSAxFkqMPFk9KR6iZ0MJNI= +contrib.go.opencensus.io/exporter/ocagent v0.7.1-0.20200907061046-05415f1de66d/go.mod h1:IshRmMJBhDfFj5Y67nVhMYTTIze91RUeT73ipWKs/GY= +contrib.go.opencensus.io/exporter/prometheus v0.4.2 h1:sqfsYl5GIY/L570iT+l93ehxaWJs2/OwXtiWwew3oAg= +contrib.go.opencensus.io/exporter/prometheus v0.4.2/go.mod h1:dvEHbiKmgvbr5pjaF9fpw1KeYcjrnC1J8B+JKjsZyRQ= github.com/Masterminds/semver/v3 v3.3.1 h1:QtNSWtVZ3nBfk8mAOu/B6v7FMJ+NHTIgUPi7rj+4nv4= github.com/Masterminds/semver/v3 v3.3.1/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= github.com/RaveNoX/go-jsoncommentstrip v1.0.0/go.mod h1:78ihd09MekBnJnxpICcwzCMzGrKSKYe4AqU6PDYYpjk= @@ -15,22 +19,33 @@ github.com/aws/aws-sdk-go v1.55.5 h1:KKUZBfBoyqy5d3swXyiC7Q76ic40rYcbqH7qjh59kzU github.com/aws/aws-sdk-go v1.55.5/go.mod h1:eRwEWoyTWFMVYVQzKMNHWP5/RV4xIUGMQfXQHfHkpNU= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/blendle/zapdriver v1.3.1 h1:C3dydBOWYRiOk+B8X9IVZ5IOe+7cl+tGOexN4QqHfpE= +github.com/blendle/zapdriver v1.3.1/go.mod h1:mdXfREi6u5MArG4j9fewC+FGnXaBR+T4Ox4J2u4eHCc= github.com/bmatcuk/doublestar v1.1.1/go.mod h1:UD6OnuiIn0yFxxA2le/rnRU1G4RaI4UvFv1sNto9p6w= +github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= +github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs= github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= -github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= -github.com/emicklei/go-restful/v3 v3.11.0 h1:rAQeMHw1c7zTmncogyy8VvRZwtkmkZ4FxERmMY4rD+g= -github.com/emicklei/go-restful/v3 v3.11.0/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/emicklei/go-restful/v3 v3.12.1 h1:PJMDIM/ak7btuL8Ex0iYET9hxM3CI2sjZtzpL63nKAU= +github.com/emicklei/go-restful/v3 v3.12.1/go.mod h1:6n3XBCmQQb25CM2LCACGz8ukIrRry+4bhvbpWn3mrbc= +github.com/evanphx/json-patch v5.9.0+incompatible h1:fBXyNpNMuTTDdquAq/uisOr2lShz4oaXpDTX2bLe7ls= +github.com/evanphx/json-patch v5.9.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= +github.com/evanphx/json-patch/v5 v5.9.0 h1:kcBlZQbplgElYIlo/n1hJbls2z/1awpXxpRi0/FOJfg= +github.com/evanphx/json-patch/v5 v5.9.0/go.mod h1:VNkHZ/282BpEyt/tObQO8s5CMPmYYq14uClGH4abBuQ= github.com/fatih/color v1.16.0 h1:zmkK9Ngbjj+K0yRhTVONQh1p/HknKYSlNT+vZCzyokM= github.com/fatih/color v1.16.0/go.mod h1:fL2Sau1YI5c0pdGEVCbKQbLXB6edEj1ZgiY4NijnWvE= github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fxamacker/cbor/v2 v2.7.0 h1:iM5WgngdRBanHcxugY4JySA0nk1wZorNOpTgCMedv5E= github.com/fxamacker/cbor/v2 v2.7.0/go.mod h1:pxXPTn3joSm21Gbwsv0w9OSA2y1HFR9qXEeXQVeNoDQ= +github.com/go-kit/log v0.2.1 h1:MRVx0/zhvdseW+Gza6N9rVzU/IVzaeE1SFI4raAhmBU= +github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBjv0= +github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= +github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= @@ -38,13 +53,12 @@ github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-logr/zapr v1.3.0 h1:XGdV8XW8zdwFiwOA2Dryh1gj2KRQyOOoNmBy4EplIcQ= github.com/go-logr/zapr v1.3.0/go.mod h1:YKepepNBd1u/oyhd/yQmtjVXmm9uML4IXUgMOwR8/Gg= -github.com/go-openapi/jsonpointer v0.19.6 h1:eCs3fxoIi3Wh6vtgmLTOjdhSpiqphQ+DaPn38N2ZdrE= -github.com/go-openapi/jsonpointer v0.19.6/go.mod h1:osyAmYz/mB/C3I+WsTTSgw1ONzaLJoLCyoi6/zppojs= -github.com/go-openapi/jsonreference v0.20.2 h1:3sVjiK66+uXK/6oQ8xgcRKcFgQ5KXa2KvnJRumpMGbE= -github.com/go-openapi/jsonreference v0.20.2/go.mod h1:Bl1zwGIM8/wsvqjsOQLJ/SH+En5Ap4rVB5KVcIDZG2k= -github.com/go-openapi/swag v0.22.3/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= -github.com/go-openapi/swag v0.22.4 h1:QLMzNJnMGPRNDCbySlcj1x01tzU8/9LTTL9hZZZogBU= -github.com/go-openapi/swag v0.22.4/go.mod h1:UzaqsxGiab7freDnrUUra0MwWfN/q7tE4j+VcZ0yl14= +github.com/go-openapi/jsonpointer v0.21.0 h1:YgdVicSA9vH5RiHs9TZW5oyafXZFc6+2Vc1rr/O9oNQ= +github.com/go-openapi/jsonpointer v0.21.0/go.mod h1:IUyH9l/+uyhIYQ/PXVA41Rexl+kOkAPDdXEYns6fzUY= +github.com/go-openapi/jsonreference v0.21.0 h1:Rs+Y7hSXT83Jacb7kFyjn4ijOuVGSvOdF2+tg1TRrwQ= +github.com/go-openapi/jsonreference v0.21.0/go.mod h1:LmZmgsrTkVg9LG4EaHeY8cBDslNPMo06cago5JNLkm4= +github.com/go-openapi/swag v0.23.0 h1:vsEVJDUo2hPJ2tu0/Xc+4noaxyEffXNIs3cOULZ+GrE= +github.com/go-openapi/swag v0.23.0/go.mod h1:esZ8ITTYEsH1V2trKHjAN8Ai7xHb8RV+YSZ577vPjgQ= github.com/go-task/slim-sprig/v3 v3.0.0 h1:sUs3vkvUymDpBKi3qH1YSqBQk9+9D/8M2mN1vB6EwHI= github.com/go-task/slim-sprig/v3 v3.0.0/go.mod h1:W848ghGpv3Qj3dhTPRyJypKRiqCdHZiAzKg9hl15HA8= github.com/gogo/protobuf v1.3.2 h1:Ov1cvc58UF3b5XjBnZv7+opcTcQFZebYjWzi34vdm4Q= @@ -58,11 +72,13 @@ github.com/google/gnostic-models v0.6.8/go.mod h1:5n7qKqH0f5wFt+aWF8CW6pZLLNOfYu github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-containerregistry v0.13.0 h1:y1C7Z3e149OJbOPDBxLYR8ITPz8dTKqQwjErKVHJC8k= +github.com/google/go-containerregistry v0.13.0/go.mod h1:J9FQ+eSS4a1aC2GNZxvNpbWhgp0487v+cgiilB4FqDo= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af h1:kmjWCqn2qkEml422C2Rrd27c3VGxi6a/6HNq8QmHRKM= -github.com/google/pprof v0.0.0-20240525223248-4bfdf5a9a2af/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 h1:FKHo8hFI3A+7w0aUQuYXQ+6EN5stWmeY/AZqtM8xk9k= +github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8/go.mod h1:K1liHPHnj73Fdn/EKuT8nrFqBihUSKXoLYU0BuatOYo= github.com/google/s2a-go v0.1.8 h1:zZDs9gcbt9ZPLV0ndSyQk6Kacx2g/X+SKYovpnz3SMM= github.com/google/s2a-go v0.1.8/go.mod h1:6iNWHTpQ+nfNRN5E00MSdfDwVesa8hhS32PhPO8deJA= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= @@ -71,8 +87,10 @@ github.com/googleapis/enterprise-certificate-proxy v0.3.4 h1:XYIDZApgAnrN1c855gT github.com/googleapis/enterprise-certificate-proxy v0.3.4/go.mod h1:YKe7cfqYXjKGpGvmSg28/fFvhNzinZQm8DGnaburhGA= github.com/googleapis/gax-go/v2 v2.14.0 h1:f+jMrjBPl+DL9nI4IQzLUxMq7XrAqFYB7hBPqMNIe8o= github.com/googleapis/gax-go/v2 v2.14.0/go.mod h1:lhBCnjdLrWRaPvLWhmc8IS24m9mr07qSYnHncrgo+zk= -github.com/gorilla/websocket v1.5.1 h1:gmztn0JnHVt9JZquRuzLw3g4wouNVzKL15iLr/zn/QY= -github.com/gorilla/websocket v1.5.1/go.mod h1:x3kM2JMyaluk02fnUJpQuwD2dCS5NDG2ZHL0uE0tcaY= +github.com/gorilla/websocket v1.5.3 h1:saDtZ6Pbx/0u+bgYQ3q96pZgCzfhKXGPqt7kZ72aNNg= +github.com/gorilla/websocket v1.5.3/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0 h1:CWyXh/jylQWp2dtiV33mY4iSSp6yf4lmn+c7/tN+ObI= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.21.0/go.mod h1:nCLIt0w3Ept2NwF8ThLmrppXsfT07oC8k0XNDxd8sVU= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542 h1:2VTzZjLZBgl62/EtslCrtky5vbi9dd7HrQPQIx6wqiw= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= @@ -81,14 +99,14 @@ github.com/hashicorp/go-hclog v1.6.3 h1:Qr2kF+eVWjTiYmU7Y31tYlP1h0q/X3Nl3tPGdaB1 github.com/hashicorp/go-hclog v1.6.3/go.mod h1:W4Qnvbt70Wk/zYJryRzDRU/4r0kIg0PVHBcfoyhpF5M= github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= -github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= -github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/influxdata/influxdb-client-go/v2 v2.14.0 h1:AjbBfJuq+QoaXNcrova8smSjwJdUHnwvfjMF71M1iI4= github.com/influxdata/influxdb-client-go/v2 v2.14.0/go.mod h1:Ahpm3QXKMJslpXl3IftVLVezreAUtBOTZssDrjZEFHI= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf h1:7JTmneyiNEwVBOHSjoMxiWAqB992atOeepeFYegn5RU= github.com/influxdata/line-protocol v0.0.0-20210922203350-b1ad95c89adf/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= -github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= -github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= +github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24 h1:liMMTbpW34dhU4az1GN0pTPADwNmvoRSeoZ6PItiqnY= +github.com/jmespath/go-jmespath v0.4.1-0.20220621161143-b0104c826a24/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1 h1:shLQSRRSCCPj3f2gpwzGwWFoC7ycTf1rcQZHOlsJ6N8= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/josharian/intern v1.0.0 h1:vlS4z54oSdjm0bgjRigI+G1HpF+tI+9rE5LLzOg8HmY= @@ -100,11 +118,8 @@ github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.17.9 h1:6KIumPrER1LHsvBVuDa0r5xaG0Es51mhhB9BQB2qeMA= github.com/klauspost/compress v1.17.9/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= -github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= -github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= -github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= @@ -130,6 +145,9 @@ github.com/onsi/ginkgo/v2 v2.19.0 h1:9Cnnf7UHo57Hy3k6/m5k3dRfGTMXGvxhHFvkDTCTpvA github.com/onsi/ginkgo/v2 v2.19.0/go.mod h1:rlwLi9PilAFJ8jCg9UE1QP6VBpd6/xj3SRC0d6TU0To= github.com/onsi/gomega v1.33.1 h1:dsYjIxxSR755MDmKVsaFQTE22ChNBcuuTWgkUDSubOk= github.com/onsi/gomega v1.33.1/go.mod h1:U4R44UsT+9eLIaYRB2a5qajjtQYn0hauxvRm16AVYg0= +github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= @@ -143,6 +161,8 @@ github.com/prometheus/common v0.55.0 h1:KEi6DK7lXW/m7Ig5i47x0vRzuBsHuvJdi5ee6Y3G github.com/prometheus/common v0.55.0/go.mod h1:2SECS4xJG1kd8XF9IcM1gMX6510RAEL65zxzNImwdc8= github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc= github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk= +github.com/prometheus/statsd_exporter v0.22.7 h1:7Pji/i2GuhK6Lu7DHrtTkFmNBCudCPT1pX2CziuyQR0= +github.com/prometheus/statsd_exporter v0.22.7/go.mod h1:N/TevpjkIh9ccs6nuzY3jQn9dFqnUakOjnEuMPJJJnI= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/signalfx/signalflow-client-go v0.1.0 h1:aqyt+st3/y8x8JtuwYRL9pOkOTJb+KeCoRWi0SuY5vw= @@ -153,39 +173,43 @@ github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spkg/bom v0.0.0-20160624110644-59b7046e48ad/go.mod h1:qLr4V1qq6nMqFKkMo8ZTx3f+BZEkzsRUY10Xsm2mwU0= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= -github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/x448/float16 v0.8.4 h1:qLwI1I70+NjRFUR3zs1JPUCgaCXSh3SW62uAKT1mSBM= github.com/x448/float16 v0.8.4/go.mod h1:14CWIYCyZA/cWjXOioeEpHeN/83MdbZDRQHoFcYsOfg= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0 h1:r6I7RJCN86bpD/FQwedZ0vSixDpwuWREjW9oRMsmqDc= go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.54.0/go.mod h1:B9yO6b04uB80CzjedvewuqDhxJxi11s7/GtiGa8bAjI= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0 h1:TT4fX+nBOA/+LUkobKGW1ydGcn+G3vRw9+g5HwCphpk= go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.54.0/go.mod h1:L7UH0GbB0p47T4Rri3uHjbpCFYrVrwc1I25QhNPiGK8= -go.opentelemetry.io/otel v1.29.0 h1:PdomN/Al4q/lN6iBJEN3AwPvUiHPMlt93c8bqTG5Llw= -go.opentelemetry.io/otel v1.29.0/go.mod h1:N/WtXPs1CNCUEx+Agz5uouwCba+i+bJGFicT8SR4NP8= -go.opentelemetry.io/otel/metric v1.29.0 h1:vPf/HFWTNkPu1aYeIsc98l4ktOQaL6LeSoeV2g+8YLc= -go.opentelemetry.io/otel/metric v1.29.0/go.mod h1:auu/QWieFVWx+DmQOUMgj0F8LHWdgalxXqvp7BII/W8= -go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= -go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= +go.opentelemetry.io/otel v1.31.0 h1:NsJcKPIW0D0H3NgzPDHmo0WW6SptzPdqg/L1zsIm2hY= +go.opentelemetry.io/otel v1.31.0/go.mod h1:O0C14Yl9FgkjqcCZAsE053C13OaddMYr/hz6clDkEJE= +go.opentelemetry.io/otel/metric v1.31.0 h1:FSErL0ATQAmYHUIzSezZibnyVlft1ybhy4ozRPcF2fE= +go.opentelemetry.io/otel/metric v1.31.0/go.mod h1:C3dEloVbLuYoX41KpmAhOqNriGbA+qqH6PQ5E5mUfnY= +go.opentelemetry.io/otel/sdk v1.31.0 h1:xLY3abVHYZ5HSfOg3l2E5LUj2Cwva5Y7yGxnSW9H5Gk= +go.opentelemetry.io/otel/sdk v1.31.0/go.mod h1:TfRbMdhvxIIr/B2N2LQW2S5v9m3gOQ/08KsbbO5BPT0= +go.opentelemetry.io/otel/sdk/metric v1.31.0 h1:i9hxxLJF/9kkvfHppyLL55aW7iIJz4JjxTeYusH7zMc= +go.opentelemetry.io/otel/sdk/metric v1.31.0/go.mod h1:CRInTMVvNhUKgSAMbKyTMxqOBC0zgyxzW55lZzX43Y8= +go.opentelemetry.io/otel/trace v1.31.0 h1:ffjsj1aRouKewfr85U2aGagJ46+MvodynlQ1HYdmJys= +go.opentelemetry.io/otel/trace v1.31.0/go.mod h1:TXZkRk7SM2ZQLtR6eoAWQFIHPvzQ06FJAsO1tJg480A= +go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/goleak v1.3.0 h1:2K3zAYmnTNqV73imy9J1T3WC+gmCePx2hEGkimedGto= go.uber.org/goleak v1.3.0/go.mod h1:CoHD4mav9JJNrW/WLlf7HGZPjdw8EucARQHekz1X6bE= +go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= +go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= go.uber.org/zap v1.27.0 h1:aJMhYGrd5QSmlpLMr2MftRKl7t8J8PTZPA732ud/XR8= go.uber.org/zap v1.27.0/go.mod h1:GB2qFLM7cTU87MWRP2mPIjqfIDnGu+VIO4V/SdhGo2E= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= -golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= -golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= +golang.org/x/crypto v0.32.0 h1:euUpcYgM8WcP71gNpTqQCn6rC2t6ULUPiOzfWaXVVfc= +golang.org/x/crypto v0.32.0/go.mod h1:ZnnJkOaASj8g0AjIduWNlq2NRxL0PlBrbKVyZ6V/Ugc= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.22.0 h1:D4nJWe9zXqHOmWqj4VMOJhvzj7bEZg4wEYa759z1pH4= @@ -194,8 +218,8 @@ golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= -golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= -golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= +golang.org/x/net v0.34.0 h1:Mb7Mrk043xzHgnRM88suvJFwzVrRfHEHJEl5/71CKw0= +golang.org/x/net v0.34.0/go.mod h1:di0qlW3YNM5oh6GqDGQr92MyTozJPmybPK4Ev/Gm31k= golang.org/x/oauth2 v0.24.0 h1:KTBBxWqUa0ykRPLtV69rRto9TLXcqYkeswu48x/gvNE= golang.org/x/oauth2 v0.24.0/go.mod h1:XYTD2NtWslqkgxebSiOHnXEap4TF09sJSc7H1sXbhtI= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -206,10 +230,10 @@ golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= -golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= -golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= +golang.org/x/sys v0.29.0 h1:TPYlXGxvx1MGTn2GiZDhnjPA9wZzZeGKHHmKhHYvgaU= +golang.org/x/sys v0.29.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/term v0.28.0 h1:/Ts8HFuMR2E6IP/jlo7QVLZHggjKQbhu/7H0LJFr3Gg= +golang.org/x/term v0.28.0/go.mod h1:Sw/lC2IAUZ92udQNf3WodGtn4k/XoLyZoh8v/8uiwek= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= @@ -220,12 +244,14 @@ golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d h1:vU5i/LfpvrRCpgM/VPfJLg5KjxD3E+hfT1SH+d9zLwg= -golang.org/x/tools v0.21.1-0.20240508182429-e35e4ccd0d2d/go.mod h1:aiJjzUbINMkxbQROHiO6hDPo2LHcIPhhQsa9DLh0yGk= +golang.org/x/tools v0.29.0 h1:Xx0h3TtM9rzQpQuR4dKLrdglAmCEN5Oi+P74JdhdzXE= +golang.org/x/tools v0.29.0/go.mod h1:KMQVMRsVxU6nHCFXrBPhDB8XncLNLM0lIy/F14RP588= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +gomodules.xyz/jsonpatch/v2 v2.4.0 h1:Ci3iUJyx9UeRx7CeFN8ARgGbkESwJK+KB9lLcWxY/Zw= +gomodules.xyz/jsonpatch/v2 v2.4.0/go.mod h1:AH3dM2RI6uoBZxn3LVrfvJ3E0/9dG4cSrbuBJT4moAY= google.golang.org/api v0.211.0 h1:IUpLjq09jxBSV1lACO33CGY3jsRcbctfGzhj+ZSE/Bg= google.golang.org/api v0.211.0/go.mod h1:XOloB4MXFH4UTlQSGuNUxw0UT74qdENK8d6JNsXKLi0= google.golang.org/genproto v0.0.0-20241021214115-324edc3d5d38 h1:Q3nlH8iSQSRUwOskjbcSMcF2jiYMNiQYZ0c2KEJLKKU= @@ -234,10 +260,10 @@ google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697 h1: google.golang.org/genproto/googleapis/api v0.0.0-20241118233622-e639e219e697/go.mod h1:+D9ySVjN8nY8YCVjc5O7PZDIdZporIDY3KaGfJunh88= google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583 h1:IfdSdTcLFy4lqUQrQJLkLt1PB+AsqVz6lwkWPzWEz10= google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583/go.mod h1:5uTbfoYQed2U9p3KIj2/Zzm02PYhndfdmML0qC3q3FU= -google.golang.org/grpc v1.67.1 h1:zWnc1Vrcno+lHZCOofnIMvycFcc0QRGIzm9dhnDX68E= -google.golang.org/grpc v1.67.1/go.mod h1:1gLDyUQU7CTLJI90u3nXZ9ekeghjeM7pTDZlqFNg2AA= -google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io= -google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= +google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= +google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4= +google.golang.org/protobuf v1.36.2 h1:R8FeyR1/eLmkutZOM5CWghmo5itiG9z0ktFlTVLuTmU= +google.golang.org/protobuf v1.36.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -250,25 +276,30 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= -gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -k8s.io/api v0.31.3 h1:umzm5o8lFbdN/hIXbrK9oRpOproJO62CV1zqxXrLgk8= -k8s.io/api v0.31.3/go.mod h1:UJrkIp9pnMOI9K2nlL6vwpxRzzEX5sWgn8kGQe92kCE= -k8s.io/apimachinery v0.31.3 h1:6l0WhcYgasZ/wk9ktLq5vLaoXJJr5ts6lkaQzgeYPq4= -k8s.io/apimachinery v0.31.3/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= -k8s.io/client-go v0.31.3 h1:CAlZuM+PH2cm+86LOBemaJI/lQ5linJ6UFxKX/SoG+4= -k8s.io/client-go v0.31.3/go.mod h1:2CgjPUTpv3fE5dNygAr2NcM8nhHzXvxB8KL5gYc3kJs= -k8s.io/code-generator v0.31.3 h1:Pj0fYOBms+ZrsulLi4DMsCEx1jG8fWKRLy44onHsLBI= -k8s.io/code-generator v0.31.3/go.mod h1:/umCIlT84g1+Yu5ZXtP1KGSRTnGiIzzX5AzUAxsNlts= -k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70 h1:NGrVE502P0s0/1hudf8zjgwki1X/TByhmAoILTarmzo= -k8s.io/gengo/v2 v2.0.0-20240228010128-51d4e06bde70/go.mod h1:VH3AT8AaQOqiGjMF9p0/IM1Dj+82ZwjfxUP1IxaHE+8= +k8s.io/api v0.31.4 h1:I2QNzitPVsPeLQvexMEsj945QumYraqv9m74isPDKhM= +k8s.io/api v0.31.4/go.mod h1:d+7vgXLvmcdT1BCo79VEgJxHHryww3V5np2OYTr6jdw= +k8s.io/apimachinery v0.31.4 h1:8xjE2C4CzhYVm9DGf60yohpNUh5AEBnPxCryPBECmlM= +k8s.io/apimachinery v0.31.4/go.mod h1:rsPdaZJfTfLsNJSQzNHQvYoTmxhoOEofxtOsF3rtsMo= +k8s.io/client-go v0.31.4 h1:t4QEXt4jgHIkKKlx06+W3+1JOwAFU/2OPiOo7H92eRQ= +k8s.io/client-go v0.31.4/go.mod h1:kvuMro4sFYIa8sulL5Gi5GFqUPvfH2O/dXuKstbaaeg= +k8s.io/code-generator v0.31.4 h1:Vu+8fKz+239rKiVDHFVHgjQ162cg5iUQPtTyQbwXeQw= +k8s.io/code-generator v0.31.4/go.mod h1:yMDt13Kn7m4MMZ4LxB1KBzdZjEyxzdT4b4qXq+lnI90= +k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7 h1:cErOOTkQ3JW19o4lo91fFurouhP8NcoBvb7CkvhZZpk= +k8s.io/gengo/v2 v2.0.0-20240826214909-a7b603a56eb7/go.mod h1:EJykeLsmFC60UQbYJezXkEsG2FLrt0GPNkU5iK5GWxU= k8s.io/klog/v2 v2.130.1 h1:n9Xl7H1Xvksem4KFG4PYbdQCQxqc/tTUyrgXaOhHSzk= k8s.io/klog/v2 v2.130.1/go.mod h1:3Jpz1GvMt720eyJH1ckRHK1EDfpxISzJ7I9OYgaDtPE= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340 h1:BZqlfIlq5YbRMFko6/PM7FjZpUb45WallggurYhKGag= -k8s.io/kube-openapi v0.0.0-20240228011516-70dd3763d340/go.mod h1:yD4MZYeKMBwQKVht279WycxKyM84kkAx2DPrTXaeb98= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 h1:pUdcCO1Lk/tbT5ztQWOBi5HBgbBP1J8+AsQnQCKsi8A= -k8s.io/utils v0.0.0-20240711033017-18e509b52bc8/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38 h1:1dWzkmJrrprYvjGwh9kEUxmcUV/CtNU8QM7h1FLWQOo= +k8s.io/kube-openapi v0.0.0-20240903163716-9e1beecbcb38/go.mod h1:coRQXBK9NxO98XUv3ZD6AK3xzHCxV6+b7lrquKwaKzA= +k8s.io/utils v0.0.0-20240921022957-49e7df575cb6 h1:MDF6h2H/h4tbzmtIKTuctcwZmY0tY9mD9fNT47QO6HI= +k8s.io/utils v0.0.0-20240921022957-49e7df575cb6/go.mod h1:OLgZIPagt7ERELqWJFomSt595RzquPNLL48iOWgYOg0= +knative.dev/networking v0.0.0-20250117155906-67d1c274ba6a h1:FaDPXtv42+AkYh/mE269pttPSZ3fDVAjJiEsYUaM4SM= +knative.dev/networking v0.0.0-20250117155906-67d1c274ba6a/go.mod h1:AIKYMfZydhwXR/60c/3KXEnqEnH6aNEEqulifdqJVcQ= +knative.dev/pkg v0.0.0-20250117084104-c43477f0052b h1:a+gP7Yzu5NmoX2w1p8nfTgmSKF+aHLKGzqYT82ijJTw= +knative.dev/pkg v0.0.0-20250117084104-c43477f0052b/go.mod h1:bedSpkdLybR6JhL1J7XDLpd+JMKM/x8M5Apr80i5TeE= +knative.dev/serving v0.44.0 h1:c6TXhoSAI6eXt0/1ET3C69jMWYA4ES9FskSan/fBaac= +knative.dev/serving v0.44.0/go.mod h1:9bFONngDZtkdYZkP5ko9LDS9ZelnFY9SaPoHKG0vFxs= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd h1:EDPBXCAspyGV4jQlpZSudPeMmr1bNJefnuqLsRAsHZo= sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd/go.mod h1:B8JuhiUyNFVKdsE8h686QcCxMaH6HrOAZj4vswFpcB0= sigs.k8s.io/structured-merge-diff/v4 v4.4.1 h1:150L+0vs/8DA78h1u02ooW1/fFq/Lwr+sGiqlzvrtq4= diff --git a/kustomize/base/flagger/crd.yaml b/kustomize/base/flagger/crd.yaml index acc07dca..90482172 100644 --- a/kustomize/base/flagger/crd.yaml +++ b/kustomize/base/flagger/crd.yaml @@ -80,7 +80,6 @@ spec: type: object required: - targetRef - - service - analysis properties: provider: diff --git a/kustomize/base/flagger/rbac.yaml b/kustomize/base/flagger/rbac.yaml index 7e46cd99..59e2fc09 100644 --- a/kustomize/base/flagger/rbac.yaml +++ b/kustomize/base/flagger/rbac.yaml @@ -254,10 +254,24 @@ rules: - update - patch - delete + - apiGroups: + - serving.knative.dev + resources: + - services + verbs: + - get + - update + - apiGroups: + - serving.knative.dev + resources: + - revisions + verbs: + - get - nonResourceURLs: - /version verbs: - get + --- apiVersion: rbac.authorization.k8s.io/v1 kind: ClusterRoleBinding diff --git a/kustomize/knative/kustomization.yaml b/kustomize/knative/kustomization.yaml new file mode 100644 index 00000000..31063f79 --- /dev/null +++ b/kustomize/knative/kustomization.yaml @@ -0,0 +1,8 @@ +namespace: flagger-system +resources: + - namespace.yaml +bases: + - ../base/flagger/ + - ../base/prometheus/ +patchesStrategicMerge: + - patch.yaml diff --git a/kustomize/knative/namespace.yaml b/kustomize/knative/namespace.yaml new file mode 100644 index 00000000..ad234f12 --- /dev/null +++ b/kustomize/knative/namespace.yaml @@ -0,0 +1,9 @@ +apiVersion: v1 +kind: Namespace +metadata: + name: flagger-system + annotations: + linkerd.io/inject: disabled + labels: + istio-injection: disabled + appmesh.k8s.aws/sidecarInjectorWebhook: disabled diff --git a/kustomize/knative/patch.yaml b/kustomize/knative/patch.yaml new file mode 100644 index 00000000..9d95a0fa --- /dev/null +++ b/kustomize/knative/patch.yaml @@ -0,0 +1,14 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: flagger +spec: + template: + spec: + containers: + - name: flagger + args: + - -log-level=info + - -include-label-prefix=app.kubernetes.io + - -mesh-provider=knative + - -metrics-server=http://flagger-prometheus:9090 diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go index 5c92cbac..13efecd5 100644 --- a/pkg/apis/flagger/v1beta1/canary.go +++ b/pkg/apis/flagger/v1beta1/canary.go @@ -463,6 +463,14 @@ type LocalObjectReference struct { Name string `json:"name"` } +func (l *LocalObjectReference) IsKnativeService() bool { + if l.Kind == "Service" && l.APIVersion == "serving.knative.dev/v1" { + return true + } + + return false +} + type AutoscalerRefernce struct { // API version of the scaler // +required diff --git a/pkg/apis/flagger/v1beta1/provider.go b/pkg/apis/flagger/v1beta1/provider.go index bc93c6de..2ddf04c3 100644 --- a/pkg/apis/flagger/v1beta1/provider.go +++ b/pkg/apis/flagger/v1beta1/provider.go @@ -7,6 +7,7 @@ const ( IstioProvider string = "istio" SMIProvider string = "smi" ContourProvider string = "contour" + KnativeProvider string = "knative" GlooProvider string = "gloo" NGINXProvider string = "nginx" KubernetesProvider string = "kubernetes" diff --git a/pkg/canary/factory.go b/pkg/canary/factory.go index ec3924f4..37cde941 100644 --- a/pkg/canary/factory.go +++ b/pkg/canary/factory.go @@ -20,12 +20,15 @@ import ( "go.uber.org/zap" "k8s.io/client-go/kubernetes" + "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned" + knative "knative.dev/serving/pkg/client/clientset/versioned" ) type Factory struct { kubeClient kubernetes.Interface flaggerClient clientset.Interface + knativeClient knative.Interface logger *zap.SugaredLogger configTracker Tracker labels []string @@ -34,6 +37,7 @@ type Factory struct { func NewFactory(kubeClient kubernetes.Interface, flaggerClient clientset.Interface, + knativeClient knative.Interface, configTracker Tracker, labels []string, includeLabelPrefix []string, @@ -41,6 +45,7 @@ func NewFactory(kubeClient kubernetes.Interface, return &Factory{ kubeClient: kubeClient, flaggerClient: flaggerClient, + knativeClient: knativeClient, logger: logger, configTracker: configTracker, labels: labels, @@ -48,7 +53,7 @@ func NewFactory(kubeClient kubernetes.Interface, } } -func (factory *Factory) Controller(kind string) Controller { +func (factory *Factory) Controller(obj v1beta1.LocalObjectReference) Controller { deploymentCtrl := &DeploymentController{ logger: factory.logger, kubeClient: factory.kubeClient, @@ -71,14 +76,22 @@ func (factory *Factory) Controller(kind string) Controller { flaggerClient: factory.flaggerClient, includeLabelPrefix: factory.includeLabelPrefix, } + knativeCtrl := &KnativeController{ + flaggerClient: factory.flaggerClient, + knativeClient: factory.knativeClient, + } - switch kind { + switch obj.Kind { case "DaemonSet": return daemonSetCtrl case "Deployment": return deploymentCtrl case "Service": - return serviceCtrl + if obj.IsKnativeService() { + return knativeCtrl + } else { + return serviceCtrl + } default: return deploymentCtrl } diff --git a/pkg/canary/knative_controller.go b/pkg/canary/knative_controller.go new file mode 100644 index 00000000..a69a33fb --- /dev/null +++ b/pkg/canary/knative_controller.go @@ -0,0 +1,161 @@ +package canary + +import ( + "context" + "fmt" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + serving "knative.dev/serving/pkg/apis/serving/v1" + knative "knative.dev/serving/pkg/client/clientset/versioned" +) + +type KnativeController struct { + flaggerClient clientset.Interface + knativeClient knative.Interface + logger *zap.SugaredLogger +} + +// IsPrimaryReady checks if the primary revision is ready +func (kc *KnativeController) IsPrimaryReady(cd *flaggerv1.Canary) (bool, error) { + service, err := kc.knativeClient.ServingV1().Services(cd.Namespace).Get(context.TODO(), cd.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + return true, fmt.Errorf("Knative Service %s.%s get query error: %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + } + revisionName, exists := service.Annotations["flagger.app/primary-revision"] + if !exists { + return true, fmt.Errorf("Knative Service %s.%s primary revision annotation not found", cd.Spec.TargetRef.Name, cd.Namespace) + } + revision, err := kc.knativeClient.ServingV1().Revisions(cd.Namespace).Get(context.TODO(), revisionName, metav1.GetOptions{}) + if err != nil { + return true, fmt.Errorf("Knative Revision %s.%s get query error: %w", revisionName, cd.Namespace, err) + } + if !revision.IsReady() { + return true, fmt.Errorf("Knative Revision %s.%s is not ready", revision.Name, cd.Namespace) + } + return true, nil +} + +// IsCanaryReady checks if the canary revision is ready +func (kc *KnativeController) IsCanaryReady(cd *flaggerv1.Canary) (bool, error) { + service, err := kc.knativeClient.ServingV1().Services(cd.Namespace).Get(context.TODO(), cd.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + return true, fmt.Errorf("Knative Service %s.%s get query error: %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + } + revision, err := kc.knativeClient.ServingV1().Revisions(cd.Namespace).Get(context.TODO(), service.Status.LatestCreatedRevisionName, metav1.GetOptions{}) + if err != nil { + return true, fmt.Errorf("Knative Revision %s.%s get query error: %w", service.Status.LatestCreatedRevisionName, cd.Namespace, err) + } + if !revision.IsReady() { + return true, fmt.Errorf("Knative Revision %s.%s is not ready", revision.Name, cd.Namespace) + } + return true, nil +} + +func (kc *KnativeController) GetMetadata(canary *flaggerv1.Canary) (string, string, map[string]int32, error) { + return "", "", make(map[string]int32), nil +} + +// SyncStatus encodes list of revisions and updates the canary status +func (kc *KnativeController) SyncStatus(cd *flaggerv1.Canary, status flaggerv1.CanaryStatus) error { + service, err := kc.knativeClient.ServingV1().Services(cd.Namespace).Get(context.TODO(), cd.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("Knative Service %s.%s get query error: %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + } + return syncCanaryStatus(kc.flaggerClient, cd, status, service.Status.LatestCreatedRevisionName, func(copy *flaggerv1.Canary) {}) +} + +// SetStatusFailedChecks updates the canary failed checks counter +func (kc *KnativeController) SetStatusFailedChecks(cd *flaggerv1.Canary, val int) error { + return setStatusFailedChecks(kc.flaggerClient, cd, val) +} + +// SetStatusWeight updates the canary status weight value +func (kc *KnativeController) SetStatusWeight(cd *flaggerv1.Canary, val int) error { + return setStatusWeight(kc.flaggerClient, cd, val) +} + +// SetStatusIterations updates the canary status iterations value +func (kc *KnativeController) SetStatusIterations(cd *flaggerv1.Canary, val int) error { + return setStatusIterations(kc.flaggerClient, cd, val) +} + +// SetStatusPhase updates the canary status phase +func (kc *KnativeController) SetStatusPhase(cd *flaggerv1.Canary, phase flaggerv1.CanaryPhase) error { + return setStatusPhase(kc.flaggerClient, cd, phase) +} + +// Initialize configures the Knative Service to be used for canary rollouts. +func (kc *KnativeController) Initialize(cd *flaggerv1.Canary) (bool, error) { + if cd.Status.Phase == "" || cd.Status.Phase == flaggerv1.CanaryPhaseInitializing { + service, err := kc.knativeClient.ServingV1().Services(cd.Namespace).Get(context.TODO(), cd.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + return true, fmt.Errorf("Knative Service %s.%s get query error: %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + } + + if service.Annotations == nil { + service.Annotations = make(map[string]string, 0) + } + service.Annotations["flagger.app/primary-revision"] = service.Status.LatestCreatedRevisionName + + canaryPercent := int64(0) + primaryPercent := int64(100) + latestRevision := true + traffic := []serving.TrafficTarget{ + { + LatestRevision: &latestRevision, + Percent: &canaryPercent, + }, + { + RevisionName: service.Status.LatestCreatedRevisionName, + Percent: &primaryPercent, + }, + } + service.Spec.Traffic = traffic + + _, err = kc.knativeClient.ServingV1().Services(cd.Namespace).Update(context.TODO(), service, metav1.UpdateOptions{}) + if err != nil { + return true, fmt.Errorf("Knative Service %s.%s update query error %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + } + } + return true, nil +} + +func (kc *KnativeController) Promote(cd *flaggerv1.Canary) error { + service, err := kc.knativeClient.ServingV1().Services(cd.Namespace).Get(context.TODO(), cd.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("Knative Service %s.%s get query error: %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + } + service.Annotations["flagger.app/primary-revision"] = service.Status.LatestCreatedRevisionName + _, err = kc.knativeClient.ServingV1().Services(cd.Namespace).Update(context.TODO(), service, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("Knative Service %s.%s update query error %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + } + return nil +} + +func (kc *KnativeController) HasTargetChanged(cd *flaggerv1.Canary) (bool, error) { + service, err := kc.knativeClient.ServingV1().Services(cd.Namespace).Get(context.TODO(), cd.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + return true, fmt.Errorf("Knative Service %s.%s get query error: %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + } + return hasSpecChanged(cd, service.Status.LatestCreatedRevisionName) +} + +func (kc *KnativeController) HaveDependenciesChanged(canary *flaggerv1.Canary) (bool, error) { + return false, nil +} + +func (kc *KnativeController) ScaleToZero(canary *flaggerv1.Canary) error { + return nil +} + +func (kc *KnativeController) ScaleFromZero(canary *flaggerv1.Canary) error { + return nil +} + +func (kc *KnativeController) Finalize(canary *flaggerv1.Canary) error { + return nil +} diff --git a/pkg/canary/knative_controller_test.go b/pkg/canary/knative_controller_test.go new file mode 100644 index 00000000..f7467c77 --- /dev/null +++ b/pkg/canary/knative_controller_test.go @@ -0,0 +1,79 @@ +package canary + +import ( + "context" + "testing" + + "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +func TestKnativeController_Promote(t *testing.T) { + mocks := newKnativeServiceFixture("podinfo") + _, err := mocks.controller.Initialize(mocks.canary) + require.NoError(t, err) + + service, err := mocks.knativeClient.ServingV1().Services("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + + service.Status.LatestCreatedRevisionName = "latest-revision" + _, err = mocks.knativeClient.ServingV1().Services("default").UpdateStatus(context.TODO(), service, metav1.UpdateOptions{}) + require.NoError(t, err) + + err = mocks.controller.Promote(mocks.canary) + require.NoError(t, err) + + service, err = mocks.knativeClient.ServingV1().Services("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "latest-revision", service.Annotations["flagger.app/primary-revision"]) +} + +func TestKnativeController_Initialize(t *testing.T) { + mocks := newKnativeServiceFixture("podinfo") + + mocks.canary.Status.Phase = v1beta1.CanaryPhasePromoting + ok, err := mocks.controller.Initialize(mocks.canary) + require.NoError(t, err) + assert.Equal(t, true, ok) + + service, err := mocks.knativeClient.ServingV1().Services("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + assert.Len(t, service.Annotations, 0) + assert.Len(t, service.Spec.Traffic, 0) + + mocks.canary.Status.Phase = v1beta1.CanaryPhaseInitializing + + ok, err = mocks.controller.Initialize(mocks.canary) + require.NoError(t, err) + assert.Equal(t, true, ok) + + service, err = mocks.knativeClient.ServingV1().Services("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "podinfo-00001", service.Annotations["flagger.app/primary-revision"]) + assert.Len(t, service.Spec.Traffic, 2) + assert.Equal(t, *service.Spec.Traffic[0].Percent, int64(0)) + assert.True(t, *service.Spec.Traffic[0].LatestRevision) + assert.Equal(t, *service.Spec.Traffic[1].Percent, int64(100)) + assert.Equal(t, service.Spec.Traffic[1].RevisionName, "podinfo-00001") +} + +func TestKnativeController_HasTargetChanged(t *testing.T) { + mocks := newKnativeServiceFixture("podinfo") + _, err := mocks.controller.Initialize(mocks.canary) + require.NoError(t, err) + + service, err := mocks.knativeClient.ServingV1().Services("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + + mocks.canary.Status.LastAppliedSpec = ComputeHash(service.Status.LatestCreatedRevisionName) + + service.Status.LatestCreatedRevisionName = "latest-revision" + _, err = mocks.knativeClient.ServingV1().Services("default").UpdateStatus(context.TODO(), service, metav1.UpdateOptions{}) + require.NoError(t, err) + + ok, err := mocks.controller.HasTargetChanged(mocks.canary) + require.NoError(t, err) + assert.True(t, ok) +} diff --git a/pkg/canary/knative_fixture_test.go b/pkg/canary/knative_fixture_test.go new file mode 100644 index 00000000..9586455f --- /dev/null +++ b/pkg/canary/knative_fixture_test.go @@ -0,0 +1,99 @@ +package canary + +import ( + "go.uber.org/zap" + v1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned" + fakeFlagger "github.com/fluxcd/flagger/pkg/client/clientset/versioned/fake" + "github.com/fluxcd/flagger/pkg/logger" + serving "knative.dev/serving/pkg/apis/serving/v1" + knative "knative.dev/serving/pkg/client/clientset/versioned" + fakeKnative "knative.dev/serving/pkg/client/clientset/versioned/fake" +) + +type knativeControllerFixture struct { + canary *flaggerv1.Canary + flaggerClient clientset.Interface + knativeClient knative.Interface + controller KnativeController + logger *zap.SugaredLogger +} + +func newKnativeServiceFixture(name string) knativeControllerFixture { + canary := newKnativeControllerTestCanary(name) + flaggerClient := fakeFlagger.NewSimpleClientset(canary) + + knativeClient := fakeKnative.NewSimpleClientset(newKnativeControllerTestService(name)) + + logger, _ := logger.NewLogger("debug") + + ctrl := KnativeController{ + flaggerClient: flaggerClient, + knativeClient: knativeClient, + logger: logger, + } + + return knativeControllerFixture{ + canary: canary, + controller: ctrl, + logger: logger, + flaggerClient: flaggerClient, + knativeClient: knativeClient, + } +} + +func newKnativeControllerTestService(name string) *serving.Service { + s := &serving.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: serving.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: "default", + }, + Spec: serving.ServiceSpec{ + ConfigurationSpec: serving.ConfigurationSpec{ + Template: serving.RevisionTemplateSpec{ + Spec: serving.RevisionSpec{ + PodSpec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "podinfo", + Image: "quay.io/stefanprodan/podinfo:1.2.0", + }, + }, + }, + }, + }, + }, + }, + Status: serving.ServiceStatus{ + ConfigurationStatusFields: serving.ConfigurationStatusFields{ + LatestCreatedRevisionName: "podinfo-00001", + }, + }, + } + + return s +} + +func newKnativeControllerTestCanary(name string) *flaggerv1.Canary { + cd := &flaggerv1.Canary{ + TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "podinfo", + }, + Spec: flaggerv1.CanarySpec{ + Provider: "knative", + TargetRef: flaggerv1.LocalObjectReference{ + Name: name, + APIVersion: "serving.knative.dev/v1", + Kind: "Service", + }, + Analysis: &flaggerv1.CanaryAnalysis{}, + }, + } + return cd +} diff --git a/pkg/controller/controller.go b/pkg/controller/controller.go index cabed052..bb7e7249 100644 --- a/pkg/controller/controller.go +++ b/pkg/controller/controller.go @@ -45,6 +45,7 @@ import ( "github.com/fluxcd/flagger/pkg/metrics/observers" "github.com/fluxcd/flagger/pkg/notifier" "github.com/fluxcd/flagger/pkg/router" + knative "knative.dev/serving/pkg/client/clientset/versioned" ) const controllerAgentName = "flagger" @@ -53,6 +54,7 @@ const controllerAgentName = "flagger" type Controller struct { kubeConfig *rest.Config kubeClient kubernetes.Interface + knativeClient knative.Interface flaggerClient clientset.Interface flaggerInformers Informers flaggerSynced cache.InformerSynced @@ -81,6 +83,7 @@ type Informers struct { func NewController( kubeClient kubernetes.Interface, + knativeClient knative.Interface, flaggerClient clientset.Interface, flaggerInformers Informers, flaggerWindow time.Duration, @@ -111,6 +114,7 @@ func NewController( ctrl := &Controller{ kubeConfig: kubeConfig, kubeClient: kubeClient, + knativeClient: knativeClient, flaggerClient: flaggerClient, flaggerInformers: flaggerInformers, flaggerSynced: flaggerInformers.CanaryInformer.Informer().HasSynced, @@ -330,6 +334,10 @@ func (c *Controller) verifyCanary(canary *flaggerv1.Canary) error { return err } } + if err := verifyKnativeCanary(canary); err != nil { + return err + } + return nil } @@ -352,6 +360,24 @@ func verifyNoCrossNamespaceRefs(canary *flaggerv1.Canary) error { return nil } +func verifyKnativeCanary(canary *flaggerv1.Canary) error { + if canary.Spec.TargetRef.IsKnativeService() != (canary.Spec.Provider == flaggerv1.KnativeProvider) { + if canary.Spec.TargetRef.IsKnativeService() { + return fmt.Errorf("can't use %s provider with Knative Service as target", canary.Spec.Provider) + } + return fmt.Errorf("can't use %s/%s as target if provider is set to knative", + canary.Spec.TargetRef.APIVersion, canary.Spec.TargetRef.Kind) + } + + if canary.Spec.TargetRef.IsKnativeService() { + if canary.Spec.AutoscalerRef != nil { + return fmt.Errorf("can't use autoscaler with Knative Service") + } + } + + return nil +} + func checkCustomResourceType(obj interface{}, logger *zap.SugaredLogger) (flaggerv1.Canary, bool) { var roll *flaggerv1.Canary var ok bool diff --git a/pkg/controller/controller_test.go b/pkg/controller/controller_test.go index 8e107d04..9bfc37b3 100644 --- a/pkg/controller/controller_test.go +++ b/pkg/controller/controller_test.go @@ -90,6 +90,78 @@ func TestController_verifyCanary(t *testing.T) { }, wantErr: true, }, + { + name: "knative provider with non-knative service should return an error", + canary: flaggerv1.Canary{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cd-1", + Namespace: "default", + }, + Spec: flaggerv1.CanarySpec{ + Provider: "knative", + TargetRef: flaggerv1.LocalObjectReference{ + Kind: "Deployment", + Name: "podinfo", + }, + }, + }, + wantErr: true, + }, + { + name: "knative service with non-knative provider should return an error", + canary: flaggerv1.Canary{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cd-1", + Namespace: "default", + }, + Spec: flaggerv1.CanarySpec{ + Provider: "istio", + TargetRef: flaggerv1.LocalObjectReference{ + Kind: "Service", + APIVersion: "serving.knative.dev/v1", + Name: "podinfo", + }, + }, + }, + wantErr: true, + }, + { + name: "knative service with autoscaler ref should return an error", + canary: flaggerv1.Canary{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cd-1", + Namespace: "default", + }, + Spec: flaggerv1.CanarySpec{ + Provider: "knative", + AutoscalerRef: &flaggerv1.AutoscalerRefernce{}, + TargetRef: flaggerv1.LocalObjectReference{ + Kind: "Service", + APIVersion: "serving.knative.dev/v1", + Name: "podinfo", + }, + }, + }, + wantErr: true, + }, + { + name: "knative service with knative provider is okay", + canary: flaggerv1.Canary{ + ObjectMeta: metav1.ObjectMeta{ + Name: "cd-1", + Namespace: "default", + }, + Spec: flaggerv1.CanarySpec{ + Provider: "knative", + TargetRef: flaggerv1.LocalObjectReference{ + Kind: "Service", + APIVersion: "serving.knative.dev/v1", + Name: "podinfo", + }, + }, + }, + wantErr: false, + }, } ctrl := &Controller{ diff --git a/pkg/controller/finalizer.go b/pkg/controller/finalizer.go index 57eee9a9..57ab0486 100644 --- a/pkg/controller/finalizer.go +++ b/pkg/controller/finalizer.go @@ -43,7 +43,7 @@ func (c *Controller) finalize(old interface{}) error { } // Retrieve a controller - canaryController := c.canaryFactory.Controller(canary.Spec.TargetRef.Kind) + canaryController := c.canaryFactory.Controller(canary.Spec.TargetRef) // Set the status to terminating if not already in that state if canary.Status.Phase != flaggerv1.CanaryPhaseTerminating { diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go index 6a9a2664..26029c1e 100644 --- a/pkg/controller/scheduler.go +++ b/pkg/controller/scheduler.go @@ -179,7 +179,7 @@ func (c *Controller) advanceCanary(name string, namespace string) { } // init controller based on target kind - canaryController := c.canaryFactory.Controller(cd.Spec.TargetRef.Kind) + canaryController := c.canaryFactory.Controller(cd.Spec.TargetRef) labelSelector, labelValue, ports, err := canaryController.GetMetadata(cd) if err != nil { diff --git a/pkg/controller/scheduler_daemonset_fixture_test.go b/pkg/controller/scheduler_daemonset_fixture_test.go index af7e3965..afbd0881 100644 --- a/pkg/controller/scheduler_daemonset_fixture_test.go +++ b/pkg/controller/scheduler_daemonset_fixture_test.go @@ -92,7 +92,7 @@ func newDaemonSetFixture(c *flaggerv1.Canary) daemonSetFixture { } // init router - rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient, true) + rf := router.NewFactory(nil, kubeClient, flaggerClient, nil, "annotationsPrefix", "", logger, flaggerClient, true) // init observer observerFactory, _ := observers.NewFactory(testMetricsServerURL) @@ -103,7 +103,7 @@ func newDaemonSetFixture(c *flaggerv1.Canary) daemonSetFixture { KubeClient: kubeClient, FlaggerClient: flaggerClient, } - canaryFactory := canary.NewFactory(kubeClient, flaggerClient, configTracker, []string{"app", "name"}, []string{""}, logger) + canaryFactory := canary.NewFactory(kubeClient, flaggerClient, nil, configTracker, []string{"app", "name"}, []string{""}, logger) ctrl := &Controller{ kubeClient: kubeClient, @@ -129,8 +129,10 @@ func newDaemonSetFixture(c *flaggerv1.Canary) daemonSetFixture { meshRouter := rf.MeshRouter("istio", "") return daemonSetFixture{ - canary: c, - deployer: canaryFactory.Controller("DaemonSet"), + canary: c, + deployer: canaryFactory.Controller(flaggerv1.LocalObjectReference{ + Kind: "DaemonSet", + }), logger: logger, flaggerClient: flaggerClient, meshClient: flaggerClient, diff --git a/pkg/controller/scheduler_deployment_fixture_test.go b/pkg/controller/scheduler_deployment_fixture_test.go index 57357567..e18a23cb 100644 --- a/pkg/controller/scheduler_deployment_fixture_test.go +++ b/pkg/controller/scheduler_deployment_fixture_test.go @@ -121,7 +121,7 @@ func newDeploymentFixture(c *flaggerv1.Canary) fixture { } // init router - rf := router.NewFactory(nil, kubeClient, flaggerClient, "annotationsPrefix", "", logger, flaggerClient, true) + rf := router.NewFactory(nil, kubeClient, flaggerClient, nil, "annotationsPrefix", "", logger, flaggerClient, true) // init observer observerFactory, _ := observers.NewFactory(testMetricsServerURL) @@ -132,7 +132,7 @@ func newDeploymentFixture(c *flaggerv1.Canary) fixture { KubeClient: kubeClient, FlaggerClient: flaggerClient, } - canaryFactory := canary.NewFactory(kubeClient, flaggerClient, configTracker, []string{"app", "name"}, []string{""}, logger) + canaryFactory := canary.NewFactory(kubeClient, flaggerClient, nil, configTracker, []string{"app", "name"}, []string{""}, logger) ctrl := &Controller{ kubeClient: kubeClient, @@ -159,8 +159,10 @@ func newDeploymentFixture(c *flaggerv1.Canary) fixture { meshRouter := rf.MeshRouter("istio", "") return fixture{ - canary: c, - deployer: canaryFactory.Controller("Deployment"), + canary: c, + deployer: canaryFactory.Controller(flaggerv1.LocalObjectReference{ + Kind: "Deployment", + }), logger: logger, flaggerClient: flaggerClient, meshClient: flaggerClient, diff --git a/pkg/controller/scheduler_metrics.go b/pkg/controller/scheduler_metrics.go index ec45ef85..b36dfd12 100644 --- a/pkg/controller/scheduler_metrics.go +++ b/pkg/controller/scheduler_metrics.go @@ -28,6 +28,7 @@ import ( flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" "github.com/fluxcd/flagger/pkg/metrics/observers" "github.com/fluxcd/flagger/pkg/metrics/providers" + serving "knative.dev/serving/pkg/apis/serving/v1" ) const ( @@ -110,10 +111,20 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool { } } // set the metrics provider to query Prometheus for the canary Kubernetes service if the canary target is Service - if canary.Spec.TargetRef.Kind == "Service" { + if canary.Spec.TargetRef.Kind == "Service" && !canary.Spec.TargetRef.IsKnativeService() { metricsProvider = metricsProvider + MetricsProviderServiceSuffix } + var knativeService *serving.Service + if canary.Spec.Provider == flaggerv1.KnativeProvider || c.meshProvider == flaggerv1.KnativeProvider { + var err error + knativeService, err = c.knativeClient.ServingV1().Services(canary.Namespace).Get(context.TODO(), canary.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + c.recordEventErrorf(canary, "Error fetching Knative service %s/%s %v", canary.Namespace, canary.Spec.TargetRef.Name, err) + return false + } + } + // create observer based on the mesh provider observerFactory := c.observerFactory @@ -135,7 +146,11 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool { } if metric.Name == "request-success-rate" { - val, err := observer.GetRequestSuccessRate(toMetricModel(canary, metric.Interval, metric.TemplateVariables)) + model := toMetricModel(canary, metric.Interval, metric.TemplateVariables) + if knativeService != nil { + model.Route = knativeService.Status.LatestCreatedRevisionName + } + val, err := observer.GetRequestSuccessRate(model) if err != nil { if errors.Is(err, providers.ErrNoValuesFound) { c.recordEventWarningf(canary, @@ -167,7 +182,11 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool { } if metric.Name == "request-duration" { - val, err := observer.GetRequestDuration(toMetricModel(canary, metric.Interval, metric.TemplateVariables)) + model := toMetricModel(canary, metric.Interval, metric.TemplateVariables) + if knativeService != nil { + model.Route = knativeService.Status.LatestCreatedRevisionName + } + val, err := observer.GetRequestDuration(model) if err != nil { if errors.Is(err, providers.ErrNoValuesFound) { c.recordEventWarningf(canary, "Halt advancement no values found for %s metric %s probably %s.%s is not receiving traffic", @@ -199,7 +218,11 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool { // in-line PromQL if metric.Query != "" { - query, err := observers.RenderQuery(metric.Query, toMetricModel(canary, metric.Interval, metric.TemplateVariables)) + model := toMetricModel(canary, metric.Interval, metric.TemplateVariables) + if knativeService != nil { + model.Route = knativeService.Status.LatestCreatedRevisionName + } + query, err := observers.RenderQuery(metric.Query, model) val, err := observerFactory.Client.RunQuery(query) if err != nil { if errors.Is(err, providers.ErrNoValuesFound) { @@ -235,6 +258,16 @@ func (c *Controller) runBuiltinMetricChecks(canary *flaggerv1.Canary) bool { } func (c *Controller) runMetricChecks(canary *flaggerv1.Canary) bool { + var knativeService *serving.Service + if canary.Spec.Provider == flaggerv1.KnativeProvider || c.meshProvider == flaggerv1.KnativeProvider { + var err error + knativeService, err = c.knativeClient.ServingV1().Services(canary.Namespace).Get(context.TODO(), canary.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + c.recordEventErrorf(canary, "Error fetching Knative service %s/%s %v", canary.Namespace, canary.Spec.TargetRef.Name, err) + return false + } + } + for _, metric := range canary.GetAnalysis().Metrics { if metric.TemplateRef != nil { namespace := canary.Namespace @@ -267,7 +300,11 @@ func (c *Controller) runMetricChecks(canary *flaggerv1.Canary) bool { return false } - query, err := observers.RenderQuery(template.Spec.Query, toMetricModel(canary, metric.Interval, metric.TemplateVariables)) + model := toMetricModel(canary, metric.Interval, metric.TemplateVariables) + if knativeService != nil { + model.Route = knativeService.Status.LatestCreatedRevisionName + } + query, err := observers.RenderQuery(template.Spec.Query, model) c.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, namespace)). Debugf("Metric template %s.%s query: %s", metric.TemplateRef.Name, namespace, query) if err != nil { diff --git a/pkg/metrics/observers/factory.go b/pkg/metrics/observers/factory.go index 044f68f5..404f35ca 100644 --- a/pkg/metrics/observers/factory.go +++ b/pkg/metrics/observers/factory.go @@ -92,6 +92,10 @@ func (factory Factory) Observer(provider string) Interface { return &ApisixObserver{ client: factory.Client, } + case provider == flaggerv1.KnativeProvider: + return &KnativeObserver{ + client: factory.Client, + } default: return &IstioObserver{ client: factory.Client, diff --git a/pkg/metrics/observers/knative.go b/pkg/metrics/observers/knative.go new file mode 100644 index 00000000..6243f733 --- /dev/null +++ b/pkg/metrics/observers/knative.go @@ -0,0 +1,92 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observers + +import ( + "fmt" + "time" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + "github.com/fluxcd/flagger/pkg/metrics/providers" +) + +//envoy_cluster_name="default/hello-00001" + +var knativeQueries = map[string]string{ + "request-success-rate": ` + sum( + rate( + envoy_cluster_upstream_rq{ + envoy_cluster_name=~"{{ namespace }}/{{ route }}", + envoy_response_code!~"5.*" + }[{{ interval }}] + ) + ) + / + sum( + rate( + envoy_cluster_upstream_rq{ + envoy_cluster_name=~"{{ namespace }}/{{ route }}", + }[{{ interval }}] + ) + ) + * 100`, + "request-duration": ` + histogram_quantile( + 0.99, + sum( + rate( + envoy_cluster_upstream_rq_time_bucket{ + envoy_cluster_name=~"{{ namespace }}/{{ route }}", + }[{{ interval }}] + ) + ) by (le) + )`, +} + +type KnativeObserver struct { + client providers.Interface +} + +func (ob *KnativeObserver) GetRequestSuccessRate(model flaggerv1.MetricTemplateModel) (float64, error) { + query, err := RenderQuery(knativeQueries["request-success-rate"], model) + if err != nil { + return 0, fmt.Errorf("rendering query failed: %w", err) + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, fmt.Errorf("running query failed: %w", err) + } + + return value, nil +} + +func (ob *KnativeObserver) GetRequestDuration(model flaggerv1.MetricTemplateModel) (time.Duration, error) { + query, err := RenderQuery(knativeQueries["request-duration"], model) + if err != nil { + return 0, fmt.Errorf("rendering query failed: %w", err) + } + + value, err := ob.client.RunQuery(query) + if err != nil { + return 0, fmt.Errorf("running query failed: %w", err) + } + + ms := time.Duration(int64(value)) * time.Millisecond + return ms, nil +} diff --git a/pkg/metrics/observers/knative_test.go b/pkg/metrics/observers/knative_test.go new file mode 100644 index 00000000..b65210db --- /dev/null +++ b/pkg/metrics/observers/knative_test.go @@ -0,0 +1,102 @@ +/* +Copyright 2024 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package observers + +import ( + "net/http" + "net/http/httptest" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + "github.com/fluxcd/flagger/pkg/metrics/providers" +) + +func TestKnativeObserver_GetRequestSuccessRate(t *testing.T) { + expected := ` sum( rate( envoy_cluster_upstream_rq{ envoy_cluster_name=~"default/podinfo-00001", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ envoy_cluster_name=~"default/podinfo-00001", }[1m] ) ) * 100` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + assert.Equal(t, expected, promql) + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{ + Type: "prometheus", + Address: ts.URL, + SecretRef: nil, + }, nil) + require.NoError(t, err) + + observer := &KnativeObserver{ + client: client, + } + + val, err := observer.GetRequestSuccessRate(flaggerv1.MetricTemplateModel{ + Name: "podinfo", + Namespace: "default", + Target: "podinfo", + Service: "podinfo", + Route: "podinfo-00001", + Interval: "1m", + }) + require.NoError(t, err) + + assert.Equal(t, float64(100), val) +} + +func TestKnativeObserver_GetRequestDuration(t *testing.T) { + expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ envoy_cluster_name=~"default/podinfo-00001", }[1m] ) ) by (le) )` + + ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + promql := r.URL.Query()["query"][0] + assert.Equal(t, expected, promql) + + json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}` + w.Write([]byte(json)) + })) + defer ts.Close() + + client, err := providers.NewPrometheusProvider(flaggerv1.MetricTemplateProvider{ + Type: "prometheus", + Address: ts.URL, + SecretRef: nil, + }, nil) + require.NoError(t, err) + + observer := &KnativeObserver{ + client: client, + } + + val, err := observer.GetRequestDuration(flaggerv1.MetricTemplateModel{ + Name: "podinfo", + Namespace: "default", + Target: "podinfo", + Service: "podinfo", + Route: "podinfo-00001", + Interval: "1m", + }) + require.NoError(t, err) + + assert.Equal(t, 100*time.Millisecond, val) +} diff --git a/pkg/router/factory.go b/pkg/router/factory.go index 0a62a5ef..838cd2bf 100644 --- a/pkg/router/factory.go +++ b/pkg/router/factory.go @@ -25,6 +25,7 @@ import ( flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" clientset "github.com/fluxcd/flagger/pkg/client/clientset/versioned" + knative "knative.dev/serving/pkg/client/clientset/versioned" ) type Factory struct { @@ -32,6 +33,7 @@ type Factory struct { kubeClient kubernetes.Interface meshClient clientset.Interface flaggerClient clientset.Interface + knativeClient knative.Interface ingressAnnotationsPrefix string ingressClass string logger *zap.SugaredLogger @@ -40,6 +42,7 @@ type Factory struct { func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface, flaggerClient clientset.Interface, + knativeClient knative.Interface, ingressAnnotationsPrefix string, ingressClass string, logger *zap.SugaredLogger, @@ -50,6 +53,7 @@ func NewFactory(kubeConfig *restclient.Config, kubeClient kubernetes.Interface, meshClient: meshClient, kubeClient: kubeClient, flaggerClient: flaggerClient, + knativeClient: knativeClient, ingressAnnotationsPrefix: ingressAnnotationsPrefix, ingressClass: ingressClass, logger: logger, @@ -150,6 +154,10 @@ func (factory *Factory) MeshRouter(provider string, labelSelector string) Interf ingressClass: factory.ingressClass, setOwnerRefs: factory.setOwnerRefs, } + case provider == flaggerv1.KnativeProvider: + return &KnativeRouter{ + knativeClient: factory.knativeClient, + } case strings.HasPrefix(provider, flaggerv1.GlooProvider): return &GlooRouter{ logger: factory.logger, diff --git a/pkg/router/knative.go b/pkg/router/knative.go new file mode 100644 index 00000000..e2d35ec6 --- /dev/null +++ b/pkg/router/knative.go @@ -0,0 +1,127 @@ +package router + +import ( + "context" + "fmt" + "slices" + + flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" + "go.uber.org/zap" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + serving "knative.dev/serving/pkg/apis/serving/v1" + knative "knative.dev/serving/pkg/client/clientset/versioned" +) + +type KnativeRouter struct { + knativeClient knative.Interface + logger *zap.SugaredLogger +} + +func (kr *KnativeRouter) Reconcile(canary *flaggerv1.Canary) error { + service, err := kr.knativeClient.ServingV1().Services(canary.Namespace).Get(context.TODO(), canary.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("Knative Service %s.%s get query error: %w", canary.Spec.TargetRef.Name, canary.Namespace, err) + } + + if _, ok := service.Annotations["flagger.app/primary-revision"]; !ok { + if service.Annotations == nil { + service.Annotations = make(map[string]string) + } + service.Annotations["flagger.app/primary-revision"] = service.Status.LatestCreatedRevisionName + _, err = kr.knativeClient.ServingV1().Services(canary.Namespace).Update(context.TODO(), service, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("Knative Service %s.%s update query error: %w", canary.Spec.TargetRef.Name, canary.Namespace, err) + } + kr.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("Knative Service %s.%s updated", service.Name, service.Namespace) + } + + return nil +} + +func (kr *KnativeRouter) SetRoutes(cd *flaggerv1.Canary, primaryWeight int, canaryWeight int, mirrored bool) error { + service, err := kr.knativeClient.ServingV1().Services(cd.Namespace).Get(context.TODO(), cd.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("Knative Service %s.%s get query error: %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + } + + primaryName, exists := service.Annotations["flagger.app/primary-revision"] + if !exists { + return fmt.Errorf("Knative Service %s.%s annotation not found", cd.Spec.TargetRef.Name, cd.Namespace) + } + + canaryPercent := int64(canaryWeight) + primaryPercent := int64(primaryWeight) + latestRevision := true + traffic := []serving.TrafficTarget{ + { + LatestRevision: &latestRevision, + Percent: &canaryPercent, + }, + { + RevisionName: primaryName, + Percent: &primaryPercent, + }, + } + service.Spec.Traffic = traffic + + service, err = kr.knativeClient.ServingV1().Services(cd.Namespace).Update(context.TODO(), service, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("Knative Service %s.%s update query error %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + } + return nil +} + +func (kr *KnativeRouter) GetRoutes(cd *flaggerv1.Canary) (primaryWeight int, canaryWeight int, mirrored bool, error error) { + service, err := kr.knativeClient.ServingV1().Services(cd.Namespace).Get(context.TODO(), cd.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + error = fmt.Errorf("service %s.%s get query error: %w", cd.Spec.TargetRef.Name, cd.Namespace, err) + return + } + primaryName, exists := service.Annotations["flagger.app/primary-revision"] + if !exists { + error = fmt.Errorf("service %s.%s annotation not found", cd.Spec.TargetRef.Name, cd.Namespace) + return + } + + canaryRevisionIdx := slices.IndexFunc(service.Status.Traffic, func(target serving.TrafficTarget) bool { + return *target.LatestRevision + }) + primaryRevisionIdx := slices.IndexFunc(service.Status.Traffic, func(target serving.TrafficTarget) bool { + return target.RevisionName == primaryName + }) + + if canaryRevisionIdx == -1 || primaryRevisionIdx == -1 { + error = fmt.Errorf("Knative Service %s.%s traffic spec invalid", cd.Spec.TargetRef.Name, cd.Namespace) + return + } + if service.Status.Traffic[primaryRevisionIdx].Percent == nil { + error = fmt.Errorf("Knative Service %s.%s primary revision traffic percent does not exist", cd.Spec.TargetRef.Name, cd.Namespace) + return + } + if service.Status.Traffic[canaryRevisionIdx].Percent == nil { + error = fmt.Errorf("Knative Service %s.%s canary revision traffic percent does not exist", cd.Spec.TargetRef.Name, cd.Namespace) + return + } + + return int(*service.Status.Traffic[primaryRevisionIdx].Percent), int(*service.Status.Traffic[canaryRevisionIdx].Percent), false, nil +} + +func (kr *KnativeRouter) Finalize(canary *flaggerv1.Canary) error { + service, err := kr.knativeClient.ServingV1().Services(canary.Namespace).Get(context.TODO(), canary.Spec.TargetRef.Name, metav1.GetOptions{}) + if err != nil { + return fmt.Errorf("Knative Service %s.%s get query error: %w", canary.Spec.TargetRef.Name, canary.Namespace, err) + } + + if _, ok := service.Annotations["flagger.app/primary-revision"]; ok { + delete(service.Annotations, "flagger.app/primary-revision") + _, err = kr.knativeClient.ServingV1().Services(canary.Namespace).Update(context.TODO(), service, metav1.UpdateOptions{}) + if err != nil { + return fmt.Errorf("Knative Service %s.%s update query error: %w", canary.Spec.TargetRef.Name, canary.Namespace, err) + } + kr.logger.With("canary", fmt.Sprintf("%s.%s", canary.Name, canary.Namespace)). + Infof("Knative Service %s.%s updated", service.Name, service.Namespace) + } + + return nil +} diff --git a/pkg/router/knative_test.go b/pkg/router/knative_test.go new file mode 100644 index 00000000..fde4def2 --- /dev/null +++ b/pkg/router/knative_test.go @@ -0,0 +1,96 @@ +package router + +import ( + "context" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + serving "knative.dev/serving/pkg/apis/serving/v1" +) + +func TestKnativeRouter_Reconcile(t *testing.T) { + canary := newTestKnativeCanary() + mocks := newFixture(canary) + + router := &KnativeRouter{ + knativeClient: mocks.knativeClient, + logger: mocks.logger, + } + + err := router.Reconcile(canary) + require.NoError(t, err) + + service, err := mocks.knativeClient.ServingV1().Services("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + assert.Equal(t, "podinfo-00001", service.Annotations["flagger.app/primary-revision"]) +} + +func TestKnativeRouter_SetRoutes(t *testing.T) { + canary := newTestKnativeCanary() + mocks := newFixture(canary) + + router := &KnativeRouter{ + knativeClient: mocks.knativeClient, + logger: mocks.logger, + } + + // error when annotation is not set + err := router.SetRoutes(canary, 10, 90, false) + require.Error(t, err) + + err = router.Reconcile(canary) + require.NoError(t, err) + err = router.SetRoutes(canary, 10, 90, false) + require.NoError(t, err) + + service, err := mocks.knativeClient.ServingV1().Services("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + assert.Len(t, service.Spec.Traffic, 2) + assert.Equal(t, *service.Spec.Traffic[0].LatestRevision, true) + assert.Equal(t, *service.Spec.Traffic[0].Percent, int64(90)) + assert.Equal(t, service.Spec.Traffic[1].RevisionName, "podinfo-00001") + assert.Equal(t, *service.Spec.Traffic[1].Percent, int64(10)) +} + +func TestKnativeRouter_GetRoutes(t *testing.T) { + canary := newTestKnativeCanary() + mocks := newFixture(canary) + + router := &KnativeRouter{ + knativeClient: mocks.knativeClient, + logger: mocks.logger, + } + + // error when annotation is not set + _, _, _, err := router.GetRoutes(canary) + require.Error(t, err) + + err = router.Reconcile(canary) + require.NoError(t, err) + + service, err := mocks.knativeClient.ServingV1().Services("default").Get(context.TODO(), "podinfo", metav1.GetOptions{}) + require.NoError(t, err) + + canaryPercent := int64(90) + primaryPercent := int64(10) + latestRevision := true + service.Status.Traffic = []serving.TrafficTarget{ + { + LatestRevision: &latestRevision, + Percent: &canaryPercent, + }, + { + RevisionName: "podinfo-00001", + Percent: &primaryPercent, + }, + } + _, err = mocks.knativeClient.ServingV1().Services("default").Update(context.TODO(), service, metav1.UpdateOptions{}) + require.NoError(t, err) + + pWeight, cWeight, _, err := router.GetRoutes(canary) + require.NoError(t, err) + assert.Equal(t, pWeight, 10) + assert.Equal(t, cWeight, 90) +} diff --git a/pkg/router/router_test.go b/pkg/router/router_test.go index 094b69ad..e8f09561 100644 --- a/pkg/router/router_test.go +++ b/pkg/router/router_test.go @@ -21,11 +21,15 @@ import ( "go.uber.org/zap" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + v1 "k8s.io/api/core/v1" netv1 "k8s.io/api/networking/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" "k8s.io/client-go/kubernetes" "k8s.io/client-go/kubernetes/fake" + serving "knative.dev/serving/pkg/apis/serving/v1" + knative "knative.dev/serving/pkg/client/clientset/versioned" + fakeKnative "knative.dev/serving/pkg/client/clientset/versioned/fake" appmesh "github.com/fluxcd/flagger/pkg/apis/appmesh" flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1" @@ -43,6 +47,7 @@ type fixture struct { appmeshCanary *flaggerv1.Canary ingressCanary *flaggerv1.Canary kubeClient kubernetes.Interface + knativeClient knative.Interface meshClient clientset.Interface flaggerClient clientset.Interface logger *zap.SugaredLogger @@ -83,6 +88,7 @@ func newFixture(c *flaggerv1.Canary) fixture { kubeClient: kubeClient, meshClient: meshClient, flaggerClient: flaggerClient, + knativeClient: fakeKnative.NewSimpleClientset(newTestKnativeService()), logger: logger, } } @@ -126,6 +132,39 @@ func newTestApisixRoute() *a6v2.ApisixRoute { return ar } +func newTestKnativeService() *serving.Service { + s := &serving.Service{ + TypeMeta: metav1.TypeMeta{APIVersion: serving.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Name: "podinfo", + Namespace: "default", + }, + Spec: serving.ServiceSpec{ + ConfigurationSpec: serving.ConfigurationSpec{ + Template: serving.RevisionTemplateSpec{ + Spec: serving.RevisionSpec{ + PodSpec: v1.PodSpec{ + Containers: []v1.Container{ + { + Name: "podinfo", + Image: "quay.io/stefanprodan/podinfo:1.2.0", + }, + }, + }, + }, + }, + }, + }, + Status: serving.ServiceStatus{ + ConfigurationStatusFields: serving.ConfigurationStatusFields{ + LatestCreatedRevisionName: "podinfo-00001", + }, + }, + } + + return s +} + func newTestCanary() *flaggerv1.Canary { cd := &flaggerv1.Canary{ TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()}, @@ -577,3 +616,39 @@ func newTestGatewayAPICanary() *flaggerv1.Canary { } return cd } + +func newTestKnativeCanary() *flaggerv1.Canary { + cd := &flaggerv1.Canary{ + TypeMeta: metav1.TypeMeta{APIVersion: flaggerv1.SchemeGroupVersion.String()}, + ObjectMeta: metav1.ObjectMeta{ + Namespace: "default", + Name: "podinfo", + }, + Spec: flaggerv1.CanarySpec{ + Provider: "knative", + TargetRef: flaggerv1.LocalObjectReference{ + Name: "podinfo", + APIVersion: "serving.knative.dev/v1", + Kind: "Service", + }, + Analysis: &flaggerv1.CanaryAnalysis{ + Threshold: 10, + StepWeight: 10, + MaxWeight: 50, + Metrics: []flaggerv1.CanaryMetric{ + { + Name: "request-success-rate", + Threshold: 99, + Interval: "1m", + }, + { + Name: "request-duration", + Threshold: 500, + Interval: "1m", + }, + }, + }, + }, + } + return cd +} diff --git a/test/knative/init.sh b/test/knative/init.sh new file mode 100755 index 00000000..ff94bcda --- /dev/null +++ b/test/knative/init.sh @@ -0,0 +1,41 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +echo '>>> Delete test namespace' +kubectl delete namespace test --ignore-not-found=true --wait=true + +echo '>>> Creating test namespace' +kubectl create namespace test + +echo '>>> Installing the load tester' +kubectl apply -k ${REPO_ROOT}/kustomize/tester +kubectl -n test rollout status deployment/flagger-loadtester + +echo '>>> Deploy Knative Service' +cat <>> Installing Knative' +kubectl apply -f https://github.com/knative/serving/releases/download/knative-v${KNATIVE_VER}/serving-crds.yaml +kubectl apply -f https://github.com/knative/serving/releases/download/knative-v${KNATIVE_VER}/serving-core.yaml +kubectl apply -f https://github.com/knative/net-kourier/releases/download/knative-v${KNATIVE_VER}/kourier.yaml +kubectl patch configmap/config-network \ + --namespace knative-serving \ + --type merge \ + --patch '{"data":{"ingress-class":"kourier.ingress.networking.knative.dev"}}' + +kubectl -n knative-serving rollout status deployment +kubectl -n kourier-system rollout status deployment + +echo '>>> Installing Flagger' +kubectl apply -k ${REPO_ROOT}/kustomize/knative + +kubectl -n flagger-system set image deployment/flagger flagger=test/flagger:latest +kubectl -n flagger-system rollout status deployment/flagger diff --git a/test/knative/run.sh b/test/knative/run.sh new file mode 100755 index 00000000..d9eeff80 --- /dev/null +++ b/test/knative/run.sh @@ -0,0 +1,11 @@ +#!/usr/bin/env bash + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) +DIR="$(cd "$(dirname "$0")" && pwd)" + +"$DIR"/install.sh + +"$DIR"/init.sh +"$DIR"/test-canary.sh diff --git a/test/knative/test-canary.sh b/test/knative/test-canary.sh new file mode 100755 index 00000000..f47a66b6 --- /dev/null +++ b/test/knative/test-canary.sh @@ -0,0 +1,159 @@ +#!/usr/bin/env bash + +# This script runs e2e tests for Canary initialization, analysis and promotion +# Prerequisites: Kubernetes Kind and Knative Serving + +set -o errexit + +REPO_ROOT=$(git rev-parse --show-toplevel) + +cat <>> Waiting for primary to be ready' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Initialized' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +kubectl -n test get services.serving podinfo -oyaml | grep 'flagger.app/primary-revision: podinfo-00001' + +echo '✔ Canary initialization test passed' + +echo '>>> Triggering canary deployment' +kubectl -n test patch services.serving podinfo --type=json -p '[{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value": "ghcr.io/stefanprodan/podinfo:6.0.1"}]' + +echo '>>> Waiting for canary promotion' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get services.serving podinfo -oyaml | grep 'flagger.app/primary-revision: podinfo-00002' && ok=true || ok=false + sleep 10 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + kubectl -n test get services.serving podinfo -oyaml + echo "No more retries left" + exit 1 + fi +done + +echo '>>> Waiting for canary finalization' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Succeeded' && ok=true || ok=false + sleep 5 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary promotion test passed' + +cat <>> Triggering canary deployment' +kubectl -n test patch services.serving podinfo --type=json -p '[{"op": "replace", "path": "/spec/template/spec/containers/0/image", "value": "ghcr.io/stefanprodan/podinfo:6.0.2"}]' + +echo '>>> Waiting for canary rollback' +retries=50 +count=0 +ok=false +until ${ok}; do + kubectl -n test get canary/podinfo | grep 'Failed' && ok=true || ok=false + sleep 10 + kubectl -n flagger-system logs deployment/flagger --tail 1 + count=$(($count + 1)) + if [[ ${count} -eq ${retries} ]]; then + kubectl -n flagger-system logs deployment/flagger + echo "No more retries left" + exit 1 + fi +done + +echo '✔ Canary rollback test passed'