mirror of https://github.com/linkerd/linkerd2.git
Make simulate-proxy match proxy output (#822)
This PR makes two changes to the `simulate-proxy` script:
1. Removed the `protocol={"http", "tcp"}` label from TCP metrics. The proxy no longer adds this label (see https://github.com/runconduit/conduit/pull/785#discussion_r182563499).
2. Fixed failed responses being labeled with `classification="fail"` rather than `classification="failure"` (the label the proxy sets). I noticed that while I was here and decided to fix it as well.
Note that the first change required some minor changes to the `proxyMetricCollectors` struct in `simulate-proxy`; since the label cardinality for TCP open stats decreased by one due to removing the `protocol` label, it's no longer necessary for that struct to `haveCounterVec`/`GaugeVec` pointers for these stats. It now owns the actual `Counter`/`Gauge` instead. This means that the metric vecs that are created to be labeled for `inbound` and `outbound` are now stored as variables in the `newSimulatedProxy` function rather than going in a `proxyMetricCollectors` struct first. This shouldn't impact behaviour at all.
This commit is contained in:
Eliza Weisman
GitHub
parent
ef5fac2109
commit
8147a363e9
|
|
@ -38,11 +38,11 @@ type proxyMetricCollectors struct {
|
||||||
requestDurationMs *prom.HistogramVec
|
requestDurationMs *prom.HistogramVec
|
||||||
responseLatencyMs *prom.HistogramVec
|
responseLatencyMs *prom.HistogramVec
|
||||||
responseDurationMs *prom.HistogramVec
|
responseDurationMs *prom.HistogramVec
|
||||||
tcpAcceptOpenTotal *prom.CounterVec
|
tcpAcceptOpenTotal prom.Counter
|
||||||
tcpAcceptCloseTotal *prom.CounterVec
|
tcpAcceptCloseTotal *prom.CounterVec
|
||||||
tcpConnectOpenTotal *prom.CounterVec
|
tcpConnectOpenTotal prom.Counter
|
||||||
tcpConnectCloseTotal *prom.CounterVec
|
tcpConnectCloseTotal *prom.CounterVec
|
||||||
tcpConnectionsOpen *prom.GaugeVec
|
tcpConnectionsOpen prom.Gauge
|
||||||
tcpConnectionDurationMs *prom.HistogramVec
|
tcpConnectionDurationMs *prom.HistogramVec
|
||||||
receivedBytes *prom.CounterVec
|
receivedBytes *prom.CounterVec
|
||||||
sentBytes *prom.CounterVec
|
sentBytes *prom.CounterVec
|
||||||
|
|
@ -217,15 +217,13 @@ func (p *proxyMetricCollectors) generateTCPStats(randomCount int) {
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
|
||||||
// TODO: throw in some raw TCP connections
|
closeLabels := prom.Labels{"classification": "success"}
|
||||||
openLabels := prom.Labels{"protocol": "http"}
|
failLabels := prom.Labels{"classification": "failure"}
|
||||||
closeLabels := prom.Labels{"protocol": "http", "classification": "success"}
|
|
||||||
failLabels := prom.Labels{"protocol": "http", "classification": "failure"}
|
|
||||||
|
|
||||||
// jitter the accept/connect counts a little bit to simulate connection pooling etc.
|
// jitter the accept/connect counts a little bit to simulate connection pooling etc.
|
||||||
acceptCount := jitter(randomCount, 0.1)
|
acceptCount := jitter(randomCount, 0.1)
|
||||||
|
|
||||||
p.tcpAcceptOpenTotal.With(openLabels).Add(float64(acceptCount))
|
p.tcpAcceptOpenTotal.Add(float64(acceptCount))
|
||||||
|
|
||||||
// up to acceptCount accepted connections remain open...
|
// up to acceptCount accepted connections remain open...
|
||||||
acceptOpenCount := rand.Intn(acceptCount)
|
acceptOpenCount := rand.Intn(acceptCount)
|
||||||
|
|
@ -244,7 +242,7 @@ func (p *proxyMetricCollectors) generateTCPStats(randomCount int) {
|
||||||
|
|
||||||
connectCount := jitter(randomCount, 0.1)
|
connectCount := jitter(randomCount, 0.1)
|
||||||
|
|
||||||
p.tcpConnectOpenTotal.With(openLabels).Add(float64(connectCount))
|
p.tcpConnectOpenTotal.Add(float64(connectCount))
|
||||||
|
|
||||||
connectOpenCount := rand.Intn(connectCount)
|
connectOpenCount := rand.Intn(connectCount)
|
||||||
connectClosedCount := connectCount - connectOpenCount
|
connectClosedCount := connectCount - connectOpenCount
|
||||||
|
|
@ -258,7 +256,7 @@ func (p *proxyMetricCollectors) generateTCPStats(randomCount int) {
|
||||||
|
|
||||||
p.tcpConnectCloseTotal.With(closeLabels).Add(float64(connectClosedCount))
|
p.tcpConnectCloseTotal.With(closeLabels).Add(float64(connectClosedCount))
|
||||||
|
|
||||||
p.tcpConnectionsOpen.With(openLabels).Set(float64(acceptOpenCount + connectOpenCount))
|
p.tcpConnectionsOpen.Set(float64(acceptOpenCount + connectOpenCount))
|
||||||
|
|
||||||
// connect durations + bytes sent/received
|
// connect durations + bytes sent/received
|
||||||
totalClosed := acceptClosedCount + connectClosedCount
|
totalClosed := acceptClosedCount + connectClosedCount
|
||||||
|
|
@ -320,7 +318,7 @@ func randomResponseLabels() prom.Labels {
|
||||||
labelMap["status_code"] = fmt.Sprintf("%d", httpCode)
|
labelMap["status_code"] = fmt.Sprintf("%d", httpCode)
|
||||||
|
|
||||||
if grpcCode != uint32(codes.OK) || httpCode != http.StatusOK {
|
if grpcCode != uint32(codes.OK) || httpCode != http.StatusOK {
|
||||||
labelMap["classification"] = "fail"
|
labelMap["classification"] = "failure"
|
||||||
}
|
}
|
||||||
|
|
||||||
return labelMap
|
return labelMap
|
||||||
|
|
@ -432,7 +430,6 @@ func newSimulatedProxy(pod v1.Pod, deployments []string, replicaSets *k8s.Replic
|
||||||
|
|
||||||
tcpLabels := []string{
|
tcpLabels := []string{
|
||||||
"direction",
|
"direction",
|
||||||
"protocol",
|
|
||||||
}
|
}
|
||||||
|
|
||||||
tcpCloseLabels := append(
|
tcpCloseLabels := append(
|
||||||
|
|
@ -440,90 +437,88 @@ func newSimulatedProxy(pod v1.Pod, deployments []string, replicaSets *k8s.Replic
|
||||||
[]string{"classification"}...,
|
[]string{"classification"}...,
|
||||||
)
|
)
|
||||||
|
|
||||||
proxyMetrics := proxyMetricCollectors{
|
requestTotals := prom.NewCounterVec(
|
||||||
requestTotals: prom.NewCounterVec(
|
prom.CounterOpts{
|
||||||
prom.CounterOpts{
|
Name: "request_total",
|
||||||
Name: "request_total",
|
Help: "A counter of the number of requests the proxy has received",
|
||||||
Help: "A counter of the number of requests the proxy has received",
|
ConstLabels: constLabels,
|
||||||
ConstLabels: constLabels,
|
}, requestLabels)
|
||||||
}, requestLabels),
|
responseTotals := prom.NewCounterVec(
|
||||||
responseTotals: prom.NewCounterVec(
|
prom.CounterOpts{
|
||||||
prom.CounterOpts{
|
Name: "response_total",
|
||||||
Name: "response_total",
|
Help: "A counter of the number of responses the proxy has received",
|
||||||
Help: "A counter of the number of responses the proxy has received",
|
ConstLabels: constLabels,
|
||||||
ConstLabels: constLabels,
|
}, responseLabels)
|
||||||
}, responseLabels),
|
requestDurationMs := prom.NewHistogramVec(
|
||||||
requestDurationMs: prom.NewHistogramVec(
|
prom.HistogramOpts{
|
||||||
prom.HistogramOpts{
|
Name: "request_duration_ms",
|
||||||
Name: "request_duration_ms",
|
Help: "A histogram of the duration of a request",
|
||||||
Help: "A histogram of the duration of a request",
|
ConstLabels: constLabels,
|
||||||
ConstLabels: constLabels,
|
Buckets: latencyBucketBounds,
|
||||||
Buckets: latencyBucketBounds,
|
}, requestLabels)
|
||||||
}, requestLabels),
|
responseLatencyMs := prom.NewHistogramVec(
|
||||||
responseLatencyMs: prom.NewHistogramVec(
|
prom.HistogramOpts{
|
||||||
prom.HistogramOpts{
|
Name: "response_latency_ms",
|
||||||
Name: "response_latency_ms",
|
Help: "A histogram of the total latency of a response",
|
||||||
Help: "A histogram of the total latency of a response",
|
ConstLabels: constLabels,
|
||||||
ConstLabels: constLabels,
|
Buckets: latencyBucketBounds,
|
||||||
Buckets: latencyBucketBounds,
|
}, responseLabels)
|
||||||
}, responseLabels),
|
responseDurationMs := prom.NewHistogramVec(
|
||||||
responseDurationMs: prom.NewHistogramVec(
|
prom.HistogramOpts{
|
||||||
prom.HistogramOpts{
|
Name: "response_duration_ms",
|
||||||
Name: "response_duration_ms",
|
Help: "A histogram of the duration of a response",
|
||||||
Help: "A histogram of the duration of a response",
|
ConstLabels: constLabels,
|
||||||
ConstLabels: constLabels,
|
Buckets: latencyBucketBounds,
|
||||||
Buckets: latencyBucketBounds,
|
}, responseLabels)
|
||||||
}, responseLabels),
|
tcpAcceptOpenTotal := prom.NewCounterVec(
|
||||||
tcpAcceptOpenTotal: prom.NewCounterVec(
|
prom.CounterOpts{
|
||||||
prom.CounterOpts{
|
Name: "tcp_accept_open_total",
|
||||||
Name: "tcp_accept_open_total",
|
Help: "A counter of the total number of transport connections which have been accepted by the proxy.",
|
||||||
Help: "A counter of the total number of transport connections which have been accepted by the proxy.",
|
ConstLabels: constTCPLabels,
|
||||||
ConstLabels: constTCPLabels,
|
}, tcpLabels)
|
||||||
}, tcpLabels),
|
tcpAcceptCloseTotal := prom.NewCounterVec(
|
||||||
tcpAcceptCloseTotal: prom.NewCounterVec(
|
prom.CounterOpts{
|
||||||
prom.CounterOpts{
|
Name: "tcp_accept_close_total",
|
||||||
Name: "tcp_accept_close_total",
|
Help: "A counter of the total number of transport connections accepted by the proxy which have been closed.",
|
||||||
Help: "A counter of the total number of transport connections accepted by the proxy which have been closed.",
|
ConstLabels: constTCPLabels,
|
||||||
ConstLabels: constTCPLabels,
|
}, tcpCloseLabels)
|
||||||
}, tcpCloseLabels),
|
tcpConnectOpenTotal := prom.NewCounterVec(
|
||||||
tcpConnectOpenTotal: prom.NewCounterVec(
|
prom.CounterOpts{
|
||||||
prom.CounterOpts{
|
Name: "tcp_connect_open_total",
|
||||||
Name: "tcp_connect_open_total",
|
Help: "A counter of the total number of transport connections which have been opened by the proxy.",
|
||||||
Help: "A counter of the total number of transport connections which have been opened by the proxy.",
|
ConstLabels: constTCPLabels,
|
||||||
ConstLabels: constTCPLabels,
|
}, tcpLabels)
|
||||||
}, tcpLabels),
|
tcpConnectCloseTotal := prom.NewCounterVec(
|
||||||
tcpConnectCloseTotal: prom.NewCounterVec(
|
prom.CounterOpts{
|
||||||
prom.CounterOpts{
|
Name: "tcp_connect_close_total",
|
||||||
Name: "tcp_connect_close_total",
|
Help: "A counter of the total number of transport connections opened by the proxy which have been closed.",
|
||||||
Help: "A counter of the total number of transport connections opened by the proxy which have been closed.",
|
ConstLabels: constTCPLabels,
|
||||||
ConstLabels: constTCPLabels,
|
}, tcpCloseLabels)
|
||||||
}, tcpCloseLabels),
|
tcpConnectionsOpen := prom.NewGaugeVec(
|
||||||
tcpConnectionsOpen: prom.NewGaugeVec(
|
prom.GaugeOpts{
|
||||||
prom.GaugeOpts{
|
Name: "tcp_connections_open",
|
||||||
Name: "tcp_connections_open",
|
Help: "A gauge of the number of transport connections currently open.",
|
||||||
Help: "A gauge of the number of transport connections currently open.",
|
ConstLabels: constTCPLabels,
|
||||||
ConstLabels: constTCPLabels,
|
}, tcpLabels)
|
||||||
}, tcpLabels),
|
tcpConnectionDurationMs := prom.NewHistogramVec(
|
||||||
tcpConnectionDurationMs: prom.NewHistogramVec(
|
prom.HistogramOpts{
|
||||||
prom.HistogramOpts{
|
Name: "tcp_connection_duration_ms",
|
||||||
Name: "tcp_connection_duration_ms",
|
Help: "A histogram of the duration of the lifetime of a connection, in milliseconds.",
|
||||||
Help: "A histogram of the duration of the lifetime of a connection, in milliseconds.",
|
ConstLabels: constTCPLabels,
|
||||||
ConstLabels: constTCPLabels,
|
Buckets: latencyBucketBounds,
|
||||||
Buckets: latencyBucketBounds,
|
}, tcpCloseLabels)
|
||||||
}, tcpCloseLabels),
|
sentBytes := prom.NewCounterVec(
|
||||||
sentBytes: prom.NewCounterVec(
|
prom.CounterOpts{
|
||||||
prom.CounterOpts{
|
Name: "sent_bytes",
|
||||||
Name: "sent_bytes",
|
Help: "A counter of the total number of sent bytes.",
|
||||||
Help: "A counter of the total number of sent bytes.",
|
ConstLabels: constTCPLabels,
|
||||||
ConstLabels: constTCPLabels,
|
}, tcpCloseLabels)
|
||||||
}, tcpCloseLabels),
|
receivedBytes := prom.NewCounterVec(
|
||||||
receivedBytes: prom.NewCounterVec(
|
prom.CounterOpts{
|
||||||
prom.CounterOpts{
|
Name: "received_bytes",
|
||||||
Name: "received_bytes",
|
Help: "A counter of the total number of recieved bytes.",
|
||||||
Help: "A counter of the total number of recieved bytes.",
|
ConstLabels: constTCPLabels,
|
||||||
ConstLabels: constTCPLabels,
|
}, tcpCloseLabels)
|
||||||
}, tcpCloseLabels),
|
|
||||||
}
|
|
||||||
|
|
||||||
inboundLabels := prom.Labels{
|
inboundLabels := prom.Labels{
|
||||||
"direction": "inbound",
|
"direction": "inbound",
|
||||||
|
|
@ -548,51 +543,51 @@ func newSimulatedProxy(pod v1.Pod, deployments []string, replicaSets *k8s.Replic
|
||||||
deployments: dstDeployments,
|
deployments: dstDeployments,
|
||||||
registerer: prom.NewRegistry(),
|
registerer: prom.NewRegistry(),
|
||||||
inbound: &proxyMetricCollectors{
|
inbound: &proxyMetricCollectors{
|
||||||
requestTotals: proxyMetrics.requestTotals.MustCurryWith(inboundLabels),
|
requestTotals: requestTotals.MustCurryWith(inboundLabels),
|
||||||
responseTotals: proxyMetrics.responseTotals.MustCurryWith(inboundLabels),
|
responseTotals: responseTotals.MustCurryWith(inboundLabels),
|
||||||
requestDurationMs: proxyMetrics.requestDurationMs.MustCurryWith(inboundLabels).(*prom.HistogramVec),
|
requestDurationMs: requestDurationMs.MustCurryWith(inboundLabels).(*prom.HistogramVec),
|
||||||
responseLatencyMs: proxyMetrics.responseLatencyMs.MustCurryWith(inboundLabels).(*prom.HistogramVec),
|
responseLatencyMs: responseLatencyMs.MustCurryWith(inboundLabels).(*prom.HistogramVec),
|
||||||
responseDurationMs: proxyMetrics.responseDurationMs.MustCurryWith(inboundLabels).(*prom.HistogramVec),
|
responseDurationMs: responseDurationMs.MustCurryWith(inboundLabels).(*prom.HistogramVec),
|
||||||
tcpAcceptOpenTotal: proxyMetrics.tcpAcceptOpenTotal.MustCurryWith(inboundTCPLabels),
|
tcpAcceptOpenTotal: tcpAcceptOpenTotal.With(inboundTCPLabels),
|
||||||
tcpAcceptCloseTotal: proxyMetrics.tcpAcceptCloseTotal.MustCurryWith(inboundTCPLabels),
|
tcpAcceptCloseTotal: tcpAcceptCloseTotal.MustCurryWith(inboundTCPLabels),
|
||||||
tcpConnectOpenTotal: proxyMetrics.tcpConnectOpenTotal.MustCurryWith(inboundTCPLabels),
|
tcpConnectOpenTotal: tcpConnectOpenTotal.With(inboundTCPLabels),
|
||||||
tcpConnectCloseTotal: proxyMetrics.tcpConnectCloseTotal.MustCurryWith(inboundTCPLabels),
|
tcpConnectCloseTotal: tcpConnectCloseTotal.MustCurryWith(inboundTCPLabels),
|
||||||
tcpConnectionsOpen: proxyMetrics.tcpConnectionsOpen.MustCurryWith(inboundTCPLabels),
|
tcpConnectionsOpen: tcpConnectionsOpen.With(inboundTCPLabels),
|
||||||
tcpConnectionDurationMs: proxyMetrics.tcpConnectionDurationMs.MustCurryWith(inboundTCPLabels).(*prom.HistogramVec),
|
tcpConnectionDurationMs: tcpConnectionDurationMs.MustCurryWith(inboundTCPLabels).(*prom.HistogramVec),
|
||||||
sentBytes: proxyMetrics.sentBytes.MustCurryWith(inboundTCPLabels),
|
sentBytes: sentBytes.MustCurryWith(inboundTCPLabels),
|
||||||
receivedBytes: proxyMetrics.receivedBytes.MustCurryWith(inboundTCPLabels),
|
receivedBytes: receivedBytes.MustCurryWith(inboundTCPLabels),
|
||||||
},
|
},
|
||||||
outbound: &proxyMetricCollectors{
|
outbound: &proxyMetricCollectors{
|
||||||
requestTotals: proxyMetrics.requestTotals.MustCurryWith(outboundLabels),
|
requestTotals: requestTotals.MustCurryWith(outboundLabels),
|
||||||
responseTotals: proxyMetrics.responseTotals.MustCurryWith(outboundLabels),
|
responseTotals: responseTotals.MustCurryWith(outboundLabels),
|
||||||
requestDurationMs: proxyMetrics.requestDurationMs.MustCurryWith(outboundLabels).(*prom.HistogramVec),
|
requestDurationMs: requestDurationMs.MustCurryWith(outboundLabels).(*prom.HistogramVec),
|
||||||
responseLatencyMs: proxyMetrics.responseLatencyMs.MustCurryWith(outboundLabels).(*prom.HistogramVec),
|
responseLatencyMs: responseLatencyMs.MustCurryWith(outboundLabels).(*prom.HistogramVec),
|
||||||
responseDurationMs: proxyMetrics.responseDurationMs.MustCurryWith(outboundLabels).(*prom.HistogramVec),
|
responseDurationMs: responseDurationMs.MustCurryWith(outboundLabels).(*prom.HistogramVec),
|
||||||
tcpAcceptOpenTotal: proxyMetrics.tcpAcceptOpenTotal.MustCurryWith(outboundLabels),
|
tcpAcceptOpenTotal: tcpAcceptOpenTotal.With(outboundLabels),
|
||||||
tcpAcceptCloseTotal: proxyMetrics.tcpAcceptCloseTotal.MustCurryWith(outboundLabels),
|
tcpAcceptCloseTotal: tcpAcceptCloseTotal.MustCurryWith(outboundLabels),
|
||||||
tcpConnectOpenTotal: proxyMetrics.tcpConnectOpenTotal.MustCurryWith(outboundLabels),
|
tcpConnectOpenTotal: tcpConnectOpenTotal.With(outboundLabels),
|
||||||
tcpConnectCloseTotal: proxyMetrics.tcpConnectCloseTotal.MustCurryWith(outboundLabels),
|
tcpConnectCloseTotal: tcpConnectCloseTotal.MustCurryWith(outboundLabels),
|
||||||
tcpConnectionsOpen: proxyMetrics.tcpConnectionsOpen.MustCurryWith(outboundLabels),
|
tcpConnectionsOpen: tcpConnectionsOpen.With(outboundLabels),
|
||||||
tcpConnectionDurationMs: proxyMetrics.tcpConnectionDurationMs.MustCurryWith(outboundLabels).(*prom.HistogramVec),
|
tcpConnectionDurationMs: tcpConnectionDurationMs.MustCurryWith(outboundLabels).(*prom.HistogramVec),
|
||||||
sentBytes: proxyMetrics.sentBytes.MustCurryWith(outboundLabels),
|
sentBytes: sentBytes.MustCurryWith(outboundLabels),
|
||||||
receivedBytes: proxyMetrics.receivedBytes.MustCurryWith(outboundLabels),
|
receivedBytes: receivedBytes.MustCurryWith(outboundLabels),
|
||||||
},
|
},
|
||||||
}
|
}
|
||||||
|
|
||||||
proxy.registerer.MustRegister(
|
proxy.registerer.MustRegister(
|
||||||
proxyMetrics.requestTotals,
|
requestTotals,
|
||||||
proxyMetrics.responseTotals,
|
responseTotals,
|
||||||
proxyMetrics.requestDurationMs,
|
requestDurationMs,
|
||||||
proxyMetrics.responseLatencyMs,
|
responseLatencyMs,
|
||||||
proxyMetrics.responseDurationMs,
|
responseDurationMs,
|
||||||
proxyMetrics.tcpAcceptOpenTotal,
|
tcpAcceptOpenTotal,
|
||||||
proxyMetrics.tcpAcceptCloseTotal,
|
tcpAcceptCloseTotal,
|
||||||
proxyMetrics.tcpConnectOpenTotal,
|
tcpConnectOpenTotal,
|
||||||
proxyMetrics.tcpConnectCloseTotal,
|
tcpConnectCloseTotal,
|
||||||
proxyMetrics.tcpConnectionsOpen,
|
tcpConnectionsOpen,
|
||||||
proxyMetrics.tcpConnectionDurationMs,
|
tcpConnectionDurationMs,
|
||||||
proxyMetrics.sentBytes,
|
sentBytes,
|
||||||
proxyMetrics.receivedBytes,
|
receivedBytes,
|
||||||
)
|
)
|
||||||
return &proxy
|
return &proxy
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue