fluxcd
/
flagger
mirror of https://github.com/fluxcd/flagger.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Differentiate AppMesh observer vs Crossover observer 

To not break AppMesh integration.
This commit is contained in:
Yusuke Kuoka 2019-12-18 22:03:30 +09:00
parent d75ade5e8c
commit 357ef86c8b
10 changed files with 190 additions and 37 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -41,7 +41,7 @@ Flagger documentation can be found at [docs.flagger.app](https://docs.flagger.ap
* [Istio canary deployments](https://docs.flagger.app/usage/progressive-delivery)
* [Linkerd canary deployments](https://docs.flagger.app/usage/linkerd-progressive-delivery)
* [App Mesh canary deployments](https://docs.flagger.app/usage/appmesh-progressive-delivery)
* [Envoy canary deployments](https://docs.flagger.app/usage/envoy-progressive-delivery)
* [Crossover canary deployments](https://docs.flagger.app/usage/crossover-progressive-delivery)
* [NGINX ingress controller canary deployments](https://docs.flagger.app/usage/nginx-progressive-delivery)
* [Gloo ingress controller canary deployments](https://docs.flagger.app/usage/gloo-progressive-delivery)
* [Blue/Green deployments](https://docs.flagger.app/usage/blue-green)

20
docs/gitbook/usage/envoy-progressive-delivery.md → docs/gitbook/usage/crossover-progressive-delivery.md
View File

@ -1,6 +1,8 @@
# Envoy Canary Deployments
# Envoy/Crossover Canary Deployments
This guide shows you how to use Envoy and Flagger to automate canary deployments.
This guide shows you how to use Envoy, [Crossover](https://github.com/mumoshu/crossover) and Flagger to automate canary deployments.
Crossover is a minimal Envoy xDS implementation supports [Service Mesh Interface](https://smi-spec.io/).
### Prerequisites
@ -12,7 +14,7 @@ Create a test namespace:
kubectl create ns test
```
Install Envoy along with the sidecar with Helm:
Install Envoy along with the Crossover sidecar with Helm:
```bash
helm repo add crossover https://mumoshu.github.io/crossover
@ -23,7 +25,7 @@ helm upgrade --install envoy crossover/envoy \
smi:
apiVersions:
trafficSplits: v1alpha1
services:
upstreams:
podinfo:
smi:
enabled: true
@ -46,7 +48,7 @@ helm repo add flagger https://flagger.app
helm upgrade -i flagger flagger/flagger \
--namespace test \
--set prometheus.install=true \
--set meshProvider=smi:envoy
--set meshProvider=smi:crossover
```
Optionally you can enable Slack notifications:
@ -90,7 +92,7 @@ metadata:
namespace: test
spec:
# specify mesh provider if it isn't the default one
# provider: "smi:envoy"
# provider: "smi:crossover"
# deployment reference
targetRef:
apiVersion: apps/v1
@ -146,7 +148,7 @@ spec:
url: http://flagger-loadtester.test/
timeout: 5s
metadata:
cmd: "hey -z 1m -q 10 -c 2 http://envoy.test:10000/"
cmd: "hey -z 1m -q 10 -c 2 -H 'Host: podinfo.test' http://envoy.test:10000/"
```
Save the above resource as podinfo-canary.yaml and then apply it:
@ -282,13 +284,13 @@ kubectl -n test exec -it deploy/flagger-loadtester bash
Generate HTTP 500 errors:
```bash
hey -z 1m -c 5 -q 5 http://envoy.test:10000/status/500
hey -z 1m -c 5 -q 5 -H 'Host: podinfo.test' http://envoy.test:10000/status/500
```
Generate latency:
```bash
watch -n 1 curl http://envoy.test:10000/delay/1
watch -n 1 curl -H 'Host: podinfo.test' http://envoy.test:10000/delay/1
```
When the number of failed checks reaches the canary analysis threshold, the traffic is routed back to the primary,

8
pkg/controller/scheduler.go
View File

@ -748,10 +748,10 @@ func (c *Controller) analyseCanary(r *flaggerv1.Canary) bool {
// override the global provider if one is specified in the canary spec
var metricsProvider string
// set the metrics provider to Envoy Prometheus when Envoy is the mesh provider
// For example, `envoy` metrics provider should be used for `smi:envoy` mesh provider
if strings.Contains(c.meshProvider, "envoy") {
metricsProvider = "envoy"
// set the metrics provider to Crossover Prometheus when Crossover is the mesh provider
// For example, `crossover` metrics provider should be used for `smi:crossover` mesh provider
if strings.Contains(c.meshProvider, "crossover") {
metricsProvider = "crossover"
} else {
metricsProvider = c.meshProvider
}

73
pkg/metrics/appmesh.go Normal file
View File

@ -0,0 +1,73 @@
package metrics
import (
"time"
)
var appMeshQueries = map[string]string{
"request-success-rate": `
sum(
rate(
envoy_cluster_upstream_rq{
kubernetes_namespace="{{ .Namespace }}",
kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)",
envoy_response_code!~"5.*"
}[{{ .Interval }}]
)
)
/
sum(
rate(
envoy_cluster_upstream_rq{
kubernetes_namespace="{{ .Namespace }}",
kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"
}[{{ .Interval }}]
)
)
* 100`,
"request-duration": `
histogram_quantile(
0.99,
sum(
rate(
envoy_cluster_upstream_rq_time_bucket{
kubernetes_namespace="{{ .Namespace }}",
kubernetes_pod_name=~"{{ .Name }}-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)"
}[{{ .Interval }}]
)
) by (le)
)`,
}
type AppMeshObserver struct {
client *PrometheusClient
}
func (ob *AppMeshObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, appMeshQueries["request-success-rate"])
if err != nil {
return 0, err
}
value, err := ob.client.RunQuery(query)
if err != nil {
return 0, err
}
return value, nil
}
func (ob *AppMeshObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, appMeshQueries["request-duration"])
if err != nil {
return 0, err
}
value, err := ob.client.RunQuery(query)
if err != nil {
return 0, err
}
ms := time.Duration(int64(value)) * time.Millisecond
return ms, nil
}

74
pkg/metrics/appmesh_test.go Normal file
View File

@ -0,0 +1,74 @@
package metrics
import (
"net/http"
"net/http/httptest"
"testing"
"time"
)
func TestAppMeshObserver_GetRequestSuccessRate(t *testing.T) {
expected := ` sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" }[1m] ) ) * 100`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
promql := r.URL.Query()["query"][0]
if promql != expected {
t.Errorf("\nGot %s \nWanted %s", promql, expected)
}
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}`
w.Write([]byte(json))
}))
defer ts.Close()
client, err := NewPrometheusClient(ts.URL, time.Second)
if err != nil {
t.Fatal(err)
}
observer := &AppMeshObserver{
client: client,
}
val, err := observer.GetRequestSuccessRate("podinfo", "default", "1m")
if err != nil {
t.Fatal(err.Error())
}
if val != 100 {
t.Errorf("Got %v wanted %v", val, 100)
}
}
func TestAppMeshObserver_GetRequestDuration(t *testing.T) {
expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="default", kubernetes_pod_name=~"podinfo-[0-9a-zA-Z]+(-[0-9a-zA-Z]+)" }[1m] ) ) by (le) )`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
promql := r.URL.Query()["query"][0]
if promql != expected {
t.Errorf("\nGot %s \nWanted %s", promql, expected)
}
json := `{"status":"success","data":{"resultType":"vector","result":[{"metric":{},"value":[1,"100"]}]}}`
w.Write([]byte(json))
}))
defer ts.Close()
client, err := NewPrometheusClient(ts.URL, time.Second)
if err != nil {
t.Fatal(err)
}
observer := &AppMeshObserver{
client: client,
}
val, err := observer.GetRequestDuration("podinfo", "default", "1m")
if err != nil {
t.Fatal(err.Error())
}
if val != 100*time.Millisecond {
t.Errorf("Got %v wanted %v", val, 100*time.Millisecond)
}
}

12
pkg/metrics/envoy.go → pkg/metrics/crossover.go
View File

@ -4,7 +4,7 @@ import (
"time"
)
var envoyQueries = map[string]string{
var crossoverQueries = map[string]string{
"request-success-rate": `
sum(
rate(
@ -39,12 +39,12 @@ var envoyQueries = map[string]string{
)`,
}
type EnvoyObserver struct {
type CrossoverObserver struct {
client *PrometheusClient
}
func (ob *EnvoyObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, envoyQueries["request-success-rate"])
func (ob *CrossoverObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, crossoverQueries["request-success-rate"])
if err != nil {
return 0, err
}
@ -57,8 +57,8 @@ func (ob *EnvoyObserver) GetRequestSuccessRate(name string, namespace string, in
return value, nil
}
func (ob *EnvoyObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, envoyQueries["request-duration"])
func (ob *CrossoverObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, crossoverQueries["request-duration"])
if err != nil {
return 0, err
}

12
pkg/metrics/envoy_service.go → pkg/metrics/crossover_service.go
View File

@ -4,7 +4,7 @@ import (
"time"
)
var envoyServiceQueries = map[string]string{
var crossoverServiceQueries = map[string]string{
"request-success-rate": `
sum(
rate(
@ -39,12 +39,12 @@ var envoyServiceQueries = map[string]string{
)`,
}
type EnvoyServiceObserver struct {
type CrossoverServiceObserver struct {
client *PrometheusClient
}
func (ob *EnvoyServiceObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, envoyServiceQueries["request-success-rate"])
func (ob *CrossoverServiceObserver) GetRequestSuccessRate(name string, namespace string, interval string) (float64, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, crossoverServiceQueries["request-success-rate"])
if err != nil {
return 0, err
}
@ -57,8 +57,8 @@ func (ob *EnvoyServiceObserver) GetRequestSuccessRate(name string, namespace str
return value, nil
}
func (ob *EnvoyServiceObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, envoyServiceQueries["request-duration"])
func (ob *CrossoverServiceObserver) GetRequestDuration(name string, namespace string, interval string) (time.Duration, error) {
query, err := ob.client.RenderQuery(name, namespace, interval, crossoverServiceQueries["request-duration"])
if err != nil {
return 0, err
}

8
pkg/metrics/envoy_service_test.go → pkg/metrics/crossover_service_test.go
View File

@ -7,7 +7,7 @@ import (
"time"
)
func TestEnvoyServiceObserver_GetRequestSuccessRate(t *testing.T) {
func TestCrossoverServiceObserver_GetRequestSuccessRate(t *testing.T) {
expected := ` sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name="podinfo-canary", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name="podinfo-canary" }[1m] ) ) * 100`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -26,7 +26,7 @@ func TestEnvoyServiceObserver_GetRequestSuccessRate(t *testing.T) {
t.Fatal(err)
}
observer := &EnvoyServiceObserver{
observer := &CrossoverServiceObserver{
client: client,
}
@ -40,7 +40,7 @@ func TestEnvoyServiceObserver_GetRequestSuccessRate(t *testing.T) {
}
}
func TestEnvoyServiceObserver_GetRequestDuration(t *testing.T) {
func TestCrossoverServiceObserver_GetRequestDuration(t *testing.T) {
expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="default", envoy_cluster_name="podinfo-canary" }[1m] ) ) by (le) )`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -59,7 +59,7 @@ func TestEnvoyServiceObserver_GetRequestDuration(t *testing.T) {
t.Fatal(err)
}
observer := &EnvoyServiceObserver{
observer := &CrossoverServiceObserver{
client: client,
}

8
pkg/metrics/envoy_test.go → pkg/metrics/crossover_test.go
View File

@ -7,7 +7,7 @@ import (
"time"
)
func TestEnvoyObserver_GetRequestSuccessRate(t *testing.T) {
func TestCrossoverObserver_GetRequestSuccessRate(t *testing.T) {
expected := ` sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary", envoy_response_code!~"5.*" }[1m] ) ) / sum( rate( envoy_cluster_upstream_rq{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary" }[1m] ) ) * 100`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -26,7 +26,7 @@ func TestEnvoyObserver_GetRequestSuccessRate(t *testing.T) {
t.Fatal(err)
}
observer := &EnvoyObserver{
observer := &CrossoverObserver{
client: client,
}
@ -40,7 +40,7 @@ func TestEnvoyObserver_GetRequestSuccessRate(t *testing.T) {
}
}
func TestEnvoyObserver_GetRequestDuration(t *testing.T) {
func TestCrossoverObserver_GetRequestDuration(t *testing.T) {
expected := ` histogram_quantile( 0.99, sum( rate( envoy_cluster_upstream_rq_time_bucket{ kubernetes_namespace="default", envoy_cluster_name=~"podinfo-canary" }[1m] ) ) by (le) )`
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
@ -59,7 +59,7 @@ func TestEnvoyObserver_GetRequestDuration(t *testing.T) {
t.Fatal(err)
}
observer := &EnvoyObserver{
observer := &CrossoverObserver{
client: client,
}

10
pkg/metrics/factory.go
View File

@ -30,8 +30,12 @@ func (factory Factory) Observer(provider string) Interface {
return &HttpObserver{
client: factory.Client,
}
case provider == "appmesh", provider == "envoy":
return &EnvoyObserver{
case provider == "appmesh":
return &AppMeshObserver{
client: factory.Client,
}
case provider == "crossover":
return &CrossoverObserver{
client: factory.Client,
}
case provider == "nginx":
@ -43,7 +47,7 @@ func (factory Factory) Observer(provider string) Interface {
client: factory.Client,
}
case provider == "appmesh:service", provider == "envoy:service":
return &EnvoyServiceObserver{
return &CrossoverServiceObserver{
client: factory.Client,
}
case provider == "linkerd":