mirror of https://github.com/linkerd/linkerd2.git
937 lines
40 KiB
Rust
937 lines
40 KiB
Rust
#[macro_use]
|
|
extern crate log;
|
|
|
|
#[macro_use]
|
|
mod support;
|
|
use self::support::*;
|
|
|
|
#[test]
|
|
fn inbound_sends_telemetry() {
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let srv = server::new().route("/hey", "hello").run();
|
|
|
|
let mut ctrl = controller::new();
|
|
let reports = ctrl.reports();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl.run())
|
|
.inbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.inbound, "tele.test.svc.cluster.local");
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
info!("awaiting report");
|
|
let report = reports.wait().next().unwrap().unwrap();
|
|
// proxy inbound
|
|
assert_eq!(report.proxy, 0);
|
|
// process
|
|
assert_eq!(report.process.as_ref().unwrap().node, "");
|
|
assert_eq!(report.process.as_ref().unwrap().scheduled_instance, "");
|
|
assert_eq!(report.process.as_ref().unwrap().scheduled_namespace, "test");
|
|
// requests
|
|
assert_eq!(report.requests.len(), 1);
|
|
let req = &report.requests[0];
|
|
assert_eq!(req.ctx.as_ref().unwrap().authority, "tele.test.svc.cluster.local");
|
|
//assert_eq!(req.ctx.as_ref().unwrap().method, GET);
|
|
assert_eq!(req.count, 1);
|
|
assert_eq!(req.responses.len(), 1);
|
|
// responses
|
|
let res = &req.responses[0];
|
|
assert_eq!(res.ctx.as_ref().unwrap().http_status_code, 200);
|
|
// response latencies should always have a length equal to the number
|
|
// of latency buckets in the latency histogram.
|
|
assert_eq!(
|
|
res.response_latency_counts.len(),
|
|
report.histogram_bucket_bounds_tenth_ms.len()
|
|
);
|
|
assert_eq!(res.ends.len(), 1);
|
|
// ends
|
|
let ends = &res.ends[0];
|
|
assert_eq!(ends.streams, 1);
|
|
}
|
|
|
|
|
|
#[test]
|
|
fn http1_inbound_sends_telemetry() {
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let srv = server::http1().route("/hey", "hello").run();
|
|
|
|
let mut ctrl = controller::new();
|
|
let reports = ctrl.reports();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl.run())
|
|
.inbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::http1(proxy.inbound, "tele.test.svc.cluster.local");
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
info!("awaiting report");
|
|
let report = reports.wait().next().unwrap().unwrap();
|
|
// proxy inbound
|
|
assert_eq!(report.proxy, 0);
|
|
// requests
|
|
assert_eq!(report.requests.len(), 1);
|
|
let req = &report.requests[0];
|
|
assert_eq!(req.ctx.as_ref().unwrap().authority, "tele.test.svc.cluster.local");
|
|
//assert_eq!(req.ctx.as_ref().unwrap().method, GET);
|
|
assert_eq!(req.count, 1);
|
|
assert_eq!(req.responses.len(), 1);
|
|
// responses
|
|
let res = &req.responses[0];
|
|
assert_eq!(res.ctx.as_ref().unwrap().http_status_code, 200);
|
|
// response latencies should always have a length equal to the number
|
|
// of latency buckets in the latency histogram.
|
|
assert_eq!(
|
|
res.response_latency_counts.len(),
|
|
report.histogram_bucket_bounds_tenth_ms.len()
|
|
);
|
|
assert_eq!(res.ends.len(), 1);
|
|
// ends
|
|
let ends = &res.ends[0];
|
|
assert_eq!(ends.streams, 1);
|
|
}
|
|
|
|
|
|
#[test]
|
|
fn inbound_aggregates_telemetry_over_several_requests() {
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let srv = server::new()
|
|
.route("/hey", "hello")
|
|
.route("/hi", "good morning")
|
|
.run();
|
|
|
|
let mut ctrl = controller::new();
|
|
let reports = ctrl.reports();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl.run())
|
|
.inbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.inbound, "tele.test.svc.cluster.local");
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
info!("client.get(/hi)");
|
|
assert_eq!(client.get("/hi"), "good morning");
|
|
assert_eq!(client.get("/hi"), "good morning");
|
|
|
|
info!("awaiting report");
|
|
let report = reports.wait().next().unwrap().unwrap();
|
|
// proxy inbound
|
|
assert_eq!(report.proxy, 0);
|
|
|
|
// requests -----------------------
|
|
assert_eq!(report.requests.len(), 2);
|
|
|
|
// -- first request -----------------
|
|
let req = &report.requests[0];
|
|
assert_eq!(req.ctx.as_ref().unwrap().authority, "tele.test.svc.cluster.local");
|
|
assert_eq!(req.count, 1);
|
|
assert_eq!(req.responses.len(), 1);
|
|
// ---- response --------------------
|
|
let res = &req.responses[0];
|
|
assert_eq!(res.ctx.as_ref().unwrap().http_status_code, 200);
|
|
// response latencies should always have a length equal to the number
|
|
// of latency buckets in the latency histogram.
|
|
assert_eq!(
|
|
res.response_latency_counts.len(),
|
|
report.histogram_bucket_bounds_tenth_ms.len()
|
|
);
|
|
assert_eq!(res.ends.len(), 1);
|
|
|
|
// ------ ends ----------------------
|
|
let ends = &res.ends[0];
|
|
assert_eq!(ends.streams, 1);
|
|
|
|
// -- second request ----------------
|
|
let req = &report.requests[1];
|
|
assert_eq!(req.ctx.as_ref().unwrap().authority, "tele.test.svc.cluster.local");
|
|
// repeated twice
|
|
assert_eq!(req.count, 2);
|
|
assert_eq!(req.responses.len(), 1);
|
|
// ---- response -------------------
|
|
let res = &req.responses[0];
|
|
assert_eq!(res.ctx.as_ref().unwrap().http_status_code, 200);
|
|
// response latencies should always have a length equal to the number
|
|
// of latency buckets in the latency histogram.
|
|
assert_eq!(
|
|
res.response_latency_counts.len(),
|
|
report.histogram_bucket_bounds_tenth_ms.len()
|
|
);
|
|
assert_eq!(res.ends.len(), 1);
|
|
|
|
// ------ ends ----------------------
|
|
let ends = &res.ends[0];
|
|
assert_eq!(ends.streams, 2);
|
|
|
|
}
|
|
|
|
// Ignore this test on CI, because our method of adding latency to requests
|
|
// (calling `thread::sleep`) is likely to be flakey on Travis.
|
|
// Eventually, we can add some kind of mock timer system for simulating latency
|
|
// more reliably, and re-enable this test.
|
|
#[test]
|
|
#[cfg_attr(not(feature = "flaky_tests"), ignore)]
|
|
fn records_latency_statistics() {
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let srv = server::new()
|
|
.route_with_latency("/hey", "hello", Duration::from_millis(500))
|
|
.route_with_latency("/hi", "good morning", Duration::from_millis(40))
|
|
.run();
|
|
|
|
let mut ctrl = controller::new();
|
|
let reports = ctrl.reports();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl.run())
|
|
.inbound(srv)
|
|
.metrics_flush_interval(Duration::from_secs(5))
|
|
.run();
|
|
let client = client::new(proxy.inbound, "tele.test.svc.cluster.local");
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
info!("client.get(/hi)");
|
|
assert_eq!(client.get("/hi"), "good morning");
|
|
assert_eq!(client.get("/hi"), "good morning");
|
|
|
|
info!("awaiting report");
|
|
let report = reports.wait().next().unwrap().unwrap();
|
|
|
|
// requests -----------------------
|
|
assert_eq!(report.requests.len(), 2);
|
|
// first request
|
|
let req = &report.requests[0];
|
|
assert_eq!(req.ctx.as_ref().unwrap().authority, "tele.test.svc.cluster.local");
|
|
let res = &req.responses[0];
|
|
// response latencies should always have a length equal to the number
|
|
// of latency buckets in the latency histogram.
|
|
assert_eq!(
|
|
res.response_latency_counts.len(),
|
|
report.histogram_bucket_bounds_tenth_ms.len()
|
|
);
|
|
for (idx, bucket) in res.response_latency_counts.iter().enumerate() {
|
|
// 500 ms of extra latency should put us in the 500-1000
|
|
// millisecond bucket (the 15th bucket)
|
|
if idx == 15 {
|
|
assert_eq!(*bucket, 1, "poorly bucketed latencies: {:?}", res.response_latency_counts);
|
|
} else {
|
|
assert_eq!(*bucket, 0, "poorly bucketed latencies: {:?}", res.response_latency_counts);
|
|
}
|
|
}
|
|
|
|
// second request
|
|
let req = &report.requests.get(1).expect("second report");
|
|
assert_eq!(req.ctx.as_ref().unwrap().authority, "tele.test.svc.cluster.local");
|
|
assert_eq!(req.count, 2);
|
|
assert_eq!(req.responses.len(), 1);
|
|
let res = req.responses.get(0).expect("responses[0]");
|
|
// response latencies should always have a length equal to the number
|
|
// of latency buckets in the latency histogram.
|
|
assert_eq!(
|
|
res.response_latency_counts.len(),
|
|
report.histogram_bucket_bounds_tenth_ms.len()
|
|
);
|
|
for (idx, bucket) in res.response_latency_counts.iter().enumerate() {
|
|
// 40 ms of extra latency should put us in the 40-50
|
|
// millisecond bucket (the 10th bucket)
|
|
if idx == 9 {
|
|
assert_eq!(*bucket, 2, "poorly bucketed latencies: {:?}", res.response_latency_counts);
|
|
} else {
|
|
assert_eq!(*bucket, 0, "poorly bucketed latencies: {:?}", res.response_latency_counts);
|
|
}
|
|
}
|
|
|
|
}
|
|
|
|
#[test]
|
|
fn telemetry_report_errors_are_ignored() {}
|
|
|
|
macro_rules! assert_contains {
|
|
($scrape:expr, $contains:expr) => {
|
|
assert_eventually!($scrape.contains($contains), "metrics scrape:\n{:8}\ndid not contain:\n{:8}", $scrape, $contains)
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn metrics_endpoint_inbound_request_count() {
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let srv = server::new().route("/hey", "hello").run();
|
|
|
|
let ctrl = controller::new();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl.run())
|
|
.inbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.inbound, "tele.test.svc.cluster.local");
|
|
let metrics = client::http1(proxy.metrics, "localhost");
|
|
|
|
// prior to seeing any requests, request count should be empty.
|
|
assert!(!metrics.get("/metrics")
|
|
.contains("request_total{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\"}"));
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
// after seeing a request, the request count should be 1.
|
|
assert_contains!(metrics.get("/metrics"), "request_total{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\"} 1");
|
|
|
|
}
|
|
|
|
#[test]
|
|
fn metrics_endpoint_outbound_request_count() {
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let srv = server::new().route("/hey", "hello").run();
|
|
|
|
let ctrl = controller::new()
|
|
.destination("tele.test.svc.cluster.local", srv.addr)
|
|
.run();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl)
|
|
.outbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.outbound, "tele.test.svc.cluster.local");
|
|
let metrics = client::http1(proxy.metrics, "localhost");
|
|
|
|
// prior to seeing any requests, request count should be empty.
|
|
assert!(!metrics.get("/metrics")
|
|
.contains("request_total{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\"}"));
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
// after seeing a request, the request count should be 1.
|
|
assert_contains!(metrics.get("/metrics"), "request_total{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\"} 1");
|
|
|
|
}
|
|
|
|
mod response_classification {
|
|
use super::support::*;
|
|
|
|
const REQ_STATUS_HEADER: &'static str = "x-test-status-requested";
|
|
const REQ_GRPC_STATUS_HEADER: &'static str = "x-test-grpc-status-requested";
|
|
|
|
const STATUSES: [http::StatusCode; 6] = [
|
|
http::StatusCode::OK,
|
|
http::StatusCode::NOT_MODIFIED,
|
|
http::StatusCode::BAD_REQUEST,
|
|
http::StatusCode::IM_A_TEAPOT,
|
|
http::StatusCode::GATEWAY_TIMEOUT,
|
|
http::StatusCode::INTERNAL_SERVER_ERROR,
|
|
];
|
|
|
|
|
|
fn expected_metric(status: &http::StatusCode, direction: &str) -> String {
|
|
format!(
|
|
"response_total{{authority=\"tele.test.svc.cluster.local\",direction=\"{}\",classification=\"{}\",status_code=\"{}\"}} 1",
|
|
direction,
|
|
if status.is_server_error() { "failure" } else { "success" },
|
|
status.as_u16(),
|
|
)
|
|
}
|
|
|
|
fn make_test_server() -> server::Server {
|
|
fn parse_header(headers: &http::HeaderMap, which: &str)
|
|
-> Option<http::StatusCode>
|
|
{
|
|
headers.get(which)
|
|
.map(|val| {
|
|
val.to_str()
|
|
.expect("requested status should be ascii")
|
|
.parse::<http::StatusCode>()
|
|
.expect("requested status should be numbers")
|
|
})
|
|
}
|
|
info!("running test server");
|
|
server::new()
|
|
.route_fn("/", move |req| {
|
|
let headers = req.headers();
|
|
let status = parse_header(headers, REQ_STATUS_HEADER)
|
|
.unwrap_or(http::StatusCode::OK);
|
|
let grpc_status = parse_header(headers, REQ_GRPC_STATUS_HEADER);
|
|
let mut rsp = if let Some(_grpc_status) = grpc_status {
|
|
// TODO: tests for grpc statuses
|
|
unimplemented!()
|
|
} else {
|
|
Response::new("".into())
|
|
};
|
|
*rsp.status_mut() = status;
|
|
rsp
|
|
})
|
|
}
|
|
|
|
#[test]
|
|
fn inbound_http() {
|
|
let _ = env_logger::try_init();
|
|
let srv = make_test_server().run();
|
|
let ctrl = controller::new();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl.run())
|
|
.inbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.inbound, "tele.test.svc.cluster.local");
|
|
let metrics = client::http1(proxy.metrics, "localhost");
|
|
|
|
for (i, status) in STATUSES.iter().enumerate() {
|
|
let request = client.request(
|
|
client.request_builder("/")
|
|
.header(REQ_STATUS_HEADER, status.as_str())
|
|
.method("GET")
|
|
);
|
|
assert_eq!(&request.status(), status);
|
|
|
|
for status in &STATUSES[0..i] {
|
|
// assert that the current status code is incremented, *and* that
|
|
// all previous requests are *not* incremented.
|
|
assert_contains!(metrics.get("/metrics"), &expected_metric(status, "inbound"))
|
|
}
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn outbound_http() {
|
|
let _ = env_logger::try_init();
|
|
let srv = make_test_server().run();
|
|
let ctrl = controller::new()
|
|
.destination("tele.test.svc.cluster.local", srv.addr)
|
|
.run();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl)
|
|
.outbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.outbound, "tele.test.svc.cluster.local");
|
|
let metrics = client::http1(proxy.metrics, "localhost");
|
|
|
|
for (i, status) in STATUSES.iter().enumerate() {
|
|
let request = client.request(
|
|
client.request_builder("/")
|
|
.header(REQ_STATUS_HEADER, status.as_str())
|
|
.method("GET")
|
|
);
|
|
assert_eq!(&request.status(), status);
|
|
|
|
for status in &STATUSES[0..i] {
|
|
// assert that the current status code is incremented, *and* that
|
|
// all previous requests are *not* incremented.
|
|
assert_contains!(metrics.get("/metrics"), &expected_metric(status, "outbound"))
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Ignore this test on CI, because our method of adding latency to requests
|
|
// (calling `thread::sleep`) is likely to be flakey on Travis.
|
|
// Eventually, we can add some kind of mock timer system for simulating latency
|
|
// more reliably, and re-enable this test.
|
|
#[test]
|
|
#[cfg_attr(not(feature = "flaky_tests"), ignore)]
|
|
fn metrics_endpoint_inbound_response_latency() {
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let srv = server::new()
|
|
.route_with_latency("/hey", "hello", Duration::from_millis(500))
|
|
.route_with_latency("/hi", "good morning", Duration::from_millis(40))
|
|
.run();
|
|
|
|
let ctrl = controller::new();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl.run())
|
|
.inbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.inbound, "tele.test.svc.cluster.local");
|
|
let metrics = client::http1(proxy.metrics, "localhost");
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
// assert the >=1000ms bucket is incremented by our request with 500ms
|
|
// extra latency.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\",le=\"1000\"} 1");
|
|
// the histogram's count should be 1.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\"} 1");
|
|
// TODO: we're not going to make any assertions about the
|
|
// response_latency_ms_sum stat, since its granularity depends on the actual
|
|
// observed latencies, which may vary a bit. we could make more reliable
|
|
// assertions about that stat if we were using a mock timer, though, as the
|
|
// observed latency values would be predictable.
|
|
|
|
info!("client.get(/hi)");
|
|
assert_eq!(client.get("/hi"), "good morning");
|
|
|
|
|
|
|
|
// request with 40ms extra latency should fall into the 50ms bucket.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\",le=\"50\"} 1");
|
|
// 1000ms bucket should be incremented as well, since it counts *all*
|
|
// bservations less than or equal to 1000ms, even if they also increment
|
|
// other buckets.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\",le=\"1000\"} 2");
|
|
// the histogram's total count should be 2.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\"} 2");
|
|
|
|
info!("client.get(/hi)");
|
|
assert_eq!(client.get("/hi"), "good morning");
|
|
|
|
// request with 40ms extra latency should fall into the 50ms bucket.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\",le=\"50\"} 2");
|
|
// 1000ms bucket should be incremented as well.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\",le=\"1000\"} 3");
|
|
// the histogram's total count should be 3.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\"} 3");
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
// 50ms bucket should be un-changed by the request with 500ms latency.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\",le=\"50\"} 2");
|
|
// 1000ms bucket should be incremented.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\",le=\"1000\"} 4");
|
|
// the histogram's total count should be 4.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",classification=\"success\",status_code=\"200\"} 4");
|
|
}
|
|
|
|
// Ignore this test on CI, because our method of adding latency to requests
|
|
// (calling `thread::sleep`) is likely to be flakey on Travis.
|
|
// Eventually, we can add some kind of mock timer system for simulating latency
|
|
// more reliably, and re-enable this test.
|
|
#[test]
|
|
#[cfg_attr(not(feature = "flaky_tests"), ignore)]
|
|
fn metrics_endpoint_outbound_response_latency() {
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let srv = server::new()
|
|
.route_with_latency("/hey", "hello", Duration::from_millis(500))
|
|
.route_with_latency("/hi", "good morning", Duration::from_millis(40))
|
|
.run();
|
|
|
|
let ctrl = controller::new()
|
|
.destination("tele.test.svc.cluster.local", srv.addr)
|
|
.run();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl)
|
|
.outbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.outbound, "tele.test.svc.cluster.local");
|
|
let metrics = client::http1(proxy.metrics, "localhost");
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
// assert the >=1000ms bucket is incremented by our request with 500ms
|
|
// extra latency.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\",le=\"1000\"} 1");
|
|
// the histogram's count should be 1.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\"} 1");
|
|
// TODO: we're not going to make any assertions about the
|
|
// response_latency_ms_sum stat, since its granularity depends on the actual
|
|
// observed latencies, which may vary a bit. we could make more reliable
|
|
// assertions about that stat if we were using a mock timer, though, as the
|
|
// observed latency values would be predictable.
|
|
|
|
info!("client.get(/hi)");
|
|
assert_eq!(client.get("/hi"), "good morning");
|
|
|
|
// request with 40ms extra latency should fall into the 50ms bucket.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\",le=\"50\"} 1");
|
|
// 1000ms bucket should be incremented as well, since it counts *all*
|
|
// bservations less than or equal to 1000ms, even if they also increment
|
|
// other buckets.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\",le=\"1000\"} 2");
|
|
// the histogram's total count should be 2.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\"} 2");
|
|
|
|
info!("client.get(/hi)");
|
|
assert_eq!(client.get("/hi"), "good morning");
|
|
|
|
// request with 40ms extra latency should fall into the 50ms bucket.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\",le=\"50\"} 2");
|
|
// 1000ms bucket should be incremented as well.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\",le=\"1000\"} 3");
|
|
// the histogram's total count should be 3.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\"} 3");
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
// 50ms bucket should be un-changed by the request with 500ms latency.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\",le=\"50\"} 2");
|
|
// 1000ms bucket should be incremented.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\",le=\"1000\"} 4");
|
|
// the histogram's total count should be 4.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",classification=\"success\",status_code=\"200\"} 4");
|
|
}
|
|
|
|
// https://github.com/runconduit/conduit/issues/613
|
|
#[test]
|
|
#[cfg_attr(not(feature = "flaky_tests"), ignore)]
|
|
fn metrics_endpoint_inbound_request_duration() {
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let srv = server::new()
|
|
.route("/hey", "hello")
|
|
.run();
|
|
|
|
let ctrl = controller::new();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl.run())
|
|
.inbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.inbound, "tele.test.svc.cluster.local");
|
|
let metrics = client::http1(proxy.metrics, "localhost");
|
|
|
|
// request with body should increment request_duration
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\"} 1");
|
|
|
|
// request without body should also increment request_duration
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\"} 2");
|
|
}
|
|
|
|
// https://github.com/runconduit/conduit/issues/613
|
|
#[test]
|
|
#[cfg_attr(not(feature = "flaky_tests"), ignore)]
|
|
fn metrics_endpoint_outbound_request_duration() {
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let srv = server::new()
|
|
.route("/hey", "hello")
|
|
.run();
|
|
let ctrl = controller::new()
|
|
.destination("tele.test.svc.cluster.local", srv.addr)
|
|
.run();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl)
|
|
.outbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.outbound, "tele.test.svc.cluster.local");
|
|
let metrics = client::http1(proxy.metrics, "localhost");
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\"} 1");
|
|
|
|
info!("client.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\"} 2");
|
|
}
|
|
|
|
// Tests for destination labels provided by control plane service discovery.
|
|
mod outbound_dst_labels {
|
|
use super::support::*;
|
|
|
|
use std::collections::HashMap;
|
|
use std::iter::FromIterator;
|
|
|
|
struct Fixture {
|
|
client: client::Client,
|
|
metrics: client::Client,
|
|
proxy: proxy::Listening,
|
|
}
|
|
|
|
fn fixture<A, B>(addr_labels: A, set_labels: B) -> Fixture
|
|
where
|
|
A: IntoIterator<Item=(String, String)>,
|
|
B: IntoIterator<Item=(String, String)>,
|
|
{
|
|
fixture_with_updates(vec![(addr_labels, set_labels)])
|
|
}
|
|
|
|
fn fixture_with_updates<A, B>(updates: Vec<(A, B)>) -> Fixture
|
|
where
|
|
A: IntoIterator<Item=(String, String)>,
|
|
B: IntoIterator<Item=(String, String)>,
|
|
{
|
|
info!("running test server");
|
|
let srv = server::new()
|
|
.route("/", "hello")
|
|
.run();
|
|
|
|
let mut ctrl = controller::new();
|
|
for (addr_labels, set_labels) in updates {
|
|
ctrl = ctrl.labeled_destination(
|
|
"labeled.test.svc.cluster.local",
|
|
srv.addr,
|
|
HashMap::from_iter(addr_labels),
|
|
HashMap::from_iter(set_labels),
|
|
);
|
|
}
|
|
|
|
let proxy = proxy::new()
|
|
.controller(ctrl.run())
|
|
.outbound(srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let metrics = client::http1(proxy.metrics, "localhost");
|
|
|
|
let client = client::new(
|
|
proxy.outbound,
|
|
"labeled.test.svc.cluster.local"
|
|
);
|
|
Fixture { client, metrics, proxy }
|
|
|
|
}
|
|
|
|
#[test]
|
|
fn multiple_addr_labels() {
|
|
let _ = env_logger::try_init();
|
|
let Fixture { client, metrics, proxy: _proxy } = fixture (
|
|
vec![
|
|
(String::from("addr_label2"), String::from("bar")),
|
|
(String::from("addr_label1"), String::from("foo")),
|
|
],
|
|
Vec::new(),
|
|
);
|
|
|
|
info!("client.get(/)");
|
|
assert_eq!(client.get("/"), "hello");
|
|
// We can't make more specific assertions about the metrics
|
|
// besides asserting that both labels are present somewhere in the
|
|
// scrape, because testing for whole metric lines would depend on
|
|
// the order in which the labels occur, and we can't depend on hash
|
|
// map ordering.
|
|
assert_contains!(metrics.get("/metrics"), "dst_addr_label1=\"foo\"");
|
|
assert_contains!(metrics.get("/metrics"), "dst_addr_label2=\"bar\"");
|
|
}
|
|
|
|
#[test]
|
|
fn multiple_addrset_labels() {
|
|
let _ = env_logger::try_init();
|
|
let Fixture { client, metrics, proxy: _proxy } = fixture (
|
|
Vec::new(),
|
|
vec![
|
|
(String::from("set_label1"), String::from("foo")),
|
|
(String::from("set_label2"), String::from("bar")),
|
|
]
|
|
);
|
|
|
|
info!("client.get(/)");
|
|
assert_eq!(client.get("/"), "hello");
|
|
// We can't make more specific assertions about the metrics
|
|
// besides asserting that both labels are present somewhere in the
|
|
// scrape, because testing for whole metric lines would depend on
|
|
// the order in which the labels occur, and we can't depend on hash
|
|
// map ordering.
|
|
assert_contains!(metrics.get("/metrics"), "dst_set_label1=\"foo\"");
|
|
assert_contains!(metrics.get("/metrics"), "dst_set_label2=\"bar\"");
|
|
}
|
|
|
|
#[test]
|
|
fn labeled_addr_and_addrset() {
|
|
let _ = env_logger::try_init();
|
|
let Fixture { client, metrics, proxy: _proxy } = fixture(
|
|
vec![(String::from("addr_label"), String::from("foo"))],
|
|
vec![(String::from("set_label"), String::from("bar"))],
|
|
);
|
|
|
|
info!("client.get(/)");
|
|
assert_eq!(client.get("/"), "hello");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"bar\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"bar\",classification=\"success\",status_code=\"200\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"bar\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"bar\",classification=\"success\",status_code=\"200\"} 1");
|
|
}
|
|
|
|
#[test]
|
|
fn controller_updates_addr_labels() {
|
|
let _ = env_logger::try_init();
|
|
info!("running test server");
|
|
let Fixture { client, metrics, proxy: _proxy } =
|
|
// the controller will update the value of `addr_label`. the value
|
|
// of `set_label` will remain unchanged throughout the test.
|
|
fixture_with_updates(vec![
|
|
(
|
|
vec![(String::from("addr_label"), String::from("foo"))],
|
|
vec![(String::from("set_label"), String::from("unchanged"))]
|
|
),
|
|
(
|
|
vec![(String::from("addr_label"), String::from("bar"))],
|
|
vec![(String::from("set_label"), String::from("unchanged"))]
|
|
),
|
|
]);
|
|
|
|
info!("client.get(/)");
|
|
assert_eq!(client.get("/"), "hello");
|
|
// the first request should be labeled with `dst_addr_label="foo"`
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\",classification=\"success\",status_code=\"200\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\",classification=\"success\",status_code=\"200\"} 1");
|
|
|
|
info!("client.get(/)");
|
|
assert_eq!(client.get("/"), "hello");
|
|
// the second request should increment stats labeled with `dst_addr_label="bar"`
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"bar\",dst_set_label=\"unchanged\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"bar\",dst_set_label=\"unchanged\",classification=\"success\",status_code=\"200\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"bar\",dst_set_label=\"unchanged\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"bar\",dst_set_label=\"unchanged\",classification=\"success\",status_code=\"200\"} 1");
|
|
// stats recorded from the first request should still be present.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\",classification=\"success\",status_code=\"200\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\",classification=\"success\",status_code=\"200\"} 1");
|
|
}
|
|
|
|
#[test]
|
|
fn controller_updates_set_labels() {
|
|
let _ = env_logger::try_init();
|
|
info!("running test server");
|
|
let Fixture { client, metrics, proxy: _proxy } =
|
|
fixture_with_updates(vec![
|
|
(vec![], vec![(String::from("set_label"), String::from("foo"))]),
|
|
(vec![], vec![(String::from("set_label"), String::from("bar"))]),
|
|
]);
|
|
|
|
info!("client.get(/)");
|
|
assert_eq!(client.get("/"), "hello");
|
|
// the first request should be labeled with `dst_addr_label="foo"`
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\",classification=\"success\",status_code=\"200\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\",classification=\"success\",status_code=\"200\"} 1");
|
|
|
|
info!("client.get(/)");
|
|
assert_eq!(client.get("/"), "hello");
|
|
// the second request should increment stats labeled with `dst_addr_label="bar"`
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"bar\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"bar\",classification=\"success\",status_code=\"200\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"bar\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"bar\",classification=\"success\",status_code=\"200\"} 1");
|
|
// stats recorded from the first request should still be present.
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_duration_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\",classification=\"success\",status_code=\"200\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\"} 1");
|
|
assert_contains!(metrics.get("/metrics"),
|
|
"response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\",classification=\"success\",status_code=\"200\"} 1");
|
|
}
|
|
}
|
|
|
|
#[test]
|
|
fn metrics_have_no_double_commas() {
|
|
// Test for regressions to runconduit/conduit#600.
|
|
let _ = env_logger::try_init();
|
|
|
|
info!("running test server");
|
|
let inbound_srv = server::new().route("/hey", "hello").run();
|
|
let outbound_srv = server::new().route("/hey", "hello").run();
|
|
|
|
let ctrl = controller::new()
|
|
.destination("tele.test.svc.cluster.local", outbound_srv.addr)
|
|
.run();
|
|
let proxy = proxy::new()
|
|
.controller(ctrl)
|
|
.inbound(inbound_srv)
|
|
.outbound(outbound_srv)
|
|
.metrics_flush_interval(Duration::from_millis(500))
|
|
.run();
|
|
let client = client::new(proxy.inbound, "tele.test.svc.cluster.local");
|
|
let metrics = client::http1(proxy.metrics, "localhost");
|
|
|
|
let scrape = metrics.get("/metrics");
|
|
assert!(!scrape.contains(",,"));
|
|
|
|
info!("inbound.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
let scrape = metrics.get("/metrics");
|
|
assert!(!scrape.contains(",,"), "inbound metrics had double comma");
|
|
|
|
let client = client::new(proxy.outbound, "tele.test.svc.cluster.local");
|
|
|
|
info!("outbound.get(/hey)");
|
|
assert_eq!(client.get("/hey"), "hello");
|
|
|
|
let scrape = metrics.get("/metrics");
|
|
assert!(!scrape.contains(",,"), "outbound metrics had double comma");
|
|
|
|
}
|