diff --git a/tests/telemetry.rs b/tests/telemetry.rs index 735f71ed1..8610d2895 100644 --- a/tests/telemetry.rs +++ b/tests/telemetry.rs @@ -10,12 +10,6 @@ use self::support::*; use support::bytes::IntoBuf; use std::io::Read; -macro_rules! assert_contains { - ($scrape:expr, $contains:expr) => { - assert_eventually!($scrape.contains($contains), "metrics scrape:\n{:8}\ndid not contain:\n{:8}", $scrape, $contains) - } -} - struct Fixture { client: client::Client, metrics: client::Client, @@ -125,7 +119,7 @@ fn metrics_endpoint_inbound_request_count() { assert_eq!(client.get("/"), "hello"); // after seeing a request, the request count should be 1. - assert_contains!(metrics.get("/metrics"), "request_total{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\"} 1"); + assert_eventually_contains!(metrics.get("/metrics"), "request_total{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\"} 1"); } @@ -142,7 +136,7 @@ fn metrics_endpoint_outbound_request_count() { assert_eq!(client.get("/"), "hello"); // after seeing a request, the request count should be 1. - assert_contains!(metrics.get("/metrics"), "request_total{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); + assert_eventually_contains!(metrics.get("/metrics"), "request_total{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); } @@ -231,7 +225,7 @@ mod response_classification { for status in &STATUSES[0..i] { // assert that the current status code is incremented, *and* that // all previous requests are *not* incremented. - assert_contains!( + assert_eventually_contains!( metrics.get("/metrics"), &expected_metric(status, "inbound", "disabled", None) ) @@ -256,7 +250,7 @@ mod response_classification { for status in &STATUSES[0..i] { // assert that the current status code is incremented, *and* that // all previous requests are *not* incremented. - assert_contains!( + assert_eventually_contains!( metrics.get("/metrics"), &expected_metric( status, @@ -293,10 +287,10 @@ fn metrics_endpoint_inbound_response_latency() { // assert the >=1000ms bucket is incremented by our request with 500ms // extra latency. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\",le=\"1000\"} 1"); // the histogram's count should be 1. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\"} 1"); // TODO: we're not going to make any assertions about the // response_latency_ms_sum stat, since its granularity depends on the actual @@ -308,41 +302,41 @@ fn metrics_endpoint_inbound_response_latency() { assert_eq!(client.get("/hi"), "good morning"); // request with 40ms extra latency should fall into the 50ms bucket. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\",le=\"50\"} 1"); // 1000ms bucket should be incremented as well, since it counts *all* // observations less than or equal to 1000ms, even if they also increment // other buckets. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\",le=\"1000\"} 2"); // the histogram's total count should be 2. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\"} 2"); info!("client.get(/hi)"); assert_eq!(client.get("/hi"), "good morning"); // request with 40ms extra latency should fall into the 50ms bucket. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\",le=\"50\"} 2"); // 1000ms bucket should be incremented as well. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\",le=\"1000\"} 3"); // the histogram's total count should be 3. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\"} 3"); info!("client.get(/hey)"); assert_eq!(client.get("/hey"), "hello"); // 50ms bucket should be un-changed by the request with 500ms latency. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\",le=\"50\"} 2"); // 1000ms bucket should be incremented. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\",le=\"1000\"} 4"); // the histogram's total count should be 4. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\"} 4"); } @@ -369,10 +363,10 @@ fn metrics_endpoint_outbound_response_latency() { // assert the >=1000ms bucket is incremented by our request with 500ms // extra latency. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\",le=\"1000\"} 1"); // the histogram's count should be 1. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); // TODO: we're not going to make any assertions about the // response_latency_ms_sum stat, since its granularity depends on the actual @@ -384,41 +378,41 @@ fn metrics_endpoint_outbound_response_latency() { assert_eq!(client.get("/hi"), "good morning"); // request with 40ms extra latency should fall into the 50ms bucket. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\",le=\"50\"} 1"); // 1000ms bucket should be incremented as well, since it counts *all* // bservations less than or equal to 1000ms, even if they also increment // other buckets. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\",le=\"1000\"} 2"); // the histogram's total count should be 2. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 2"); info!("client.get(/hi)"); assert_eq!(client.get("/hi"), "good morning"); // request with 40ms extra latency should fall into the 50ms bucket. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\",le=\"50\"} 2"); // 1000ms bucket should be incremented as well. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\",le=\"1000\"} 3"); // the histogram's total count should be 3. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 3"); info!("client.get(/hey)"); assert_eq!(client.get("/hey"), "hello"); // 50ms bucket should be un-changed by the request with 500ms latency. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\",le=\"50\"} 2"); // 1000ms bucket should be incremented. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_bucket{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\",le=\"1000\"} 4"); // the histogram's total count should be 4. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"outbound\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 4"); } @@ -475,8 +469,8 @@ mod outbound_dst_labels { // scrape, because testing for whole metric lines would depend on // the order in which the labels occur, and we can't depend on hash // map ordering. - assert_contains!(metrics.get("/metrics"), "dst_addr_label1=\"foo\""); - assert_contains!(metrics.get("/metrics"), "dst_addr_label2=\"bar\""); + assert_eventually_contains!(metrics.get("/metrics"), "dst_addr_label1=\"foo\""); + assert_eventually_contains!(metrics.get("/metrics"), "dst_addr_label2=\"bar\""); } #[test] @@ -500,8 +494,8 @@ mod outbound_dst_labels { // scrape, because testing for whole metric lines would depend on // the order in which the labels occur, and we can't depend on hash // map ordering. - assert_contains!(metrics.get("/metrics"), "dst_set_label1=\"foo\""); - assert_contains!(metrics.get("/metrics"), "dst_set_label2=\"bar\""); + assert_eventually_contains!(metrics.get("/metrics"), "dst_set_label1=\"foo\""); + assert_eventually_contains!(metrics.get("/metrics"), "dst_set_label2=\"bar\""); } #[test] @@ -520,11 +514,11 @@ mod outbound_dst_labels { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"bar\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"bar\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"bar\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); } @@ -552,11 +546,11 @@ mod outbound_dst_labels { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); // the first request should be labeled with `dst_addr_label="foo"` - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); { @@ -570,18 +564,18 @@ mod outbound_dst_labels { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); // the second request should increment stats labeled with `dst_addr_label="bar"` - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"bar\",dst_set_label=\"unchanged\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"bar\",dst_set_label=\"unchanged\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"bar\",dst_set_label=\"unchanged\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); // stats recorded from the first request should still be present. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_addr_label=\"foo\",dst_set_label=\"unchanged\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); } @@ -607,11 +601,11 @@ mod outbound_dst_labels { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); // the first request should be labeled with `dst_addr_label="foo"` - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); { @@ -624,18 +618,18 @@ mod outbound_dst_labels { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); // the second request should increment stats labeled with `dst_addr_label="bar"` - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"bar\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"bar\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"bar\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); // stats recorded from the first request should still be present. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_latency_ms_count{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "request_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "response_total{authority=\"labeled.test.svc.cluster.local\",direction=\"outbound\",dst_set_label=\"foo\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\",classification=\"success\",status_code=\"200\"} 1"); } } @@ -701,12 +695,12 @@ mod transport { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"inbound\",peer=\"src\",tls=\"disabled\"} 1" ); // drop the client to force the connection to close. drop(client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"inbound\",peer=\"src\",tls=\"disabled\",classification=\"success\"} 1" ); @@ -715,12 +709,12 @@ mod transport { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"inbound\",peer=\"src\",tls=\"disabled\"} 2" ); // drop the client to force the connection to close. drop(client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"inbound\",peer=\"src\",tls=\"disabled\",classification=\"success\"} 2" ); } @@ -733,7 +727,7 @@ mod transport { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"inbound\",peer=\"dst\",tls=\"internal_traffic\"} 1"); // create a new client to force a new connection @@ -742,7 +736,7 @@ mod transport { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); // server connection should be pooled - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"inbound\",peer=\"dst\",tls=\"internal_traffic\"} 1"); } @@ -754,12 +748,12 @@ mod transport { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 1" ); // drop the client to force the connection to close. drop(client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\",classification=\"success\"} 1" ); @@ -768,12 +762,12 @@ mod transport { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 2" ); // drop the client to force the connection to close. drop(client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\",classification=\"success\"} 2" ); } @@ -786,7 +780,7 @@ mod transport { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"outbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); // create a new client to force a new connection @@ -795,7 +789,7 @@ mod transport { info!("client.get(/)"); assert_eq!(client2.get("/"), "hello"); // server connection should be pooled - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"outbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_provided_by_service_discovery\"} 1"); } @@ -810,7 +804,7 @@ mod transport { tcp_client.write(TcpFixture::HELLO_MSG); assert_eq!(tcp_client.read(), TcpFixture::BYE_MSG.as_bytes()); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"inbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_http\"} 1"); } @@ -838,10 +832,10 @@ mod transport { assert_eq!(tcp_client.read(), &[]); // Connection to the server should be a failure with the EXFULL error // code. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"inbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_http\",classification=\"failure\",errno=\"EXFULL\"} 1"); // Connection from the client should have closed cleanly. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"inbound\",peer=\"src\",tls=\"disabled\",classification=\"success\"} 1"); } @@ -869,10 +863,10 @@ mod transport { assert_eq!(tcp_client.read(), &[]); // Connection to the server should be a failure with the EXFULL error // code. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"outbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_http\",classification=\"failure\",errno=\"EXFULL\"} 1"); // Connection from the client should have closed cleanly. - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\",classification=\"success\"} 1"); } @@ -888,11 +882,11 @@ mod transport { tcp_client.write(TcpFixture::HELLO_MSG); assert_eq!(tcp_client.read(), TcpFixture::BYE_MSG.as_bytes()); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"inbound\",peer=\"src\",tls=\"disabled\"} 1"); drop(tcp_client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"inbound\",peer=\"src\",tls=\"disabled\",classification=\"success\"} 1"); let tcp_client = client.connect(); @@ -900,10 +894,10 @@ mod transport { tcp_client.write(TcpFixture::HELLO_MSG); assert_eq!(tcp_client.read(), TcpFixture::BYE_MSG.as_bytes()); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"inbound\",peer=\"src\",tls=\"disabled\"} 2"); drop(tcp_client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"inbound\",peer=\"src\",tls=\"disabled\",classification=\"success\"} 2"); } @@ -922,9 +916,9 @@ mod transport { drop(tcp_client); // TODO: make assertions about buckets let out = metrics.get("/metrics"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"inbound\",peer=\"src\",tls=\"disabled\",classification=\"success\"} 1"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"inbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_http\",classification=\"success\"} 1"); let tcp_client = client.connect(); @@ -932,16 +926,16 @@ mod transport { tcp_client.write(TcpFixture::HELLO_MSG); assert_eq!(tcp_client.read(), TcpFixture::BYE_MSG.as_bytes()); let out = metrics.get("/metrics"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"inbound\",peer=\"src\",tls=\"disabled\",classification=\"success\"} 1"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"inbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_http\",classification=\"success\"} 1"); drop(tcp_client); let out = metrics.get("/metrics"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"inbound\",peer=\"src\",tls=\"disabled\",classification=\"success\"} 2"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"inbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_http\",classification=\"success\"} 2"); } @@ -967,8 +961,8 @@ mod transport { drop(tcp_client); let out = metrics.get("/metrics"); - assert_contains!(out, &src_expected); - assert_contains!(out, &dst_expected); + assert_eventually_contains!(out, &src_expected); + assert_eventually_contains!(out, &dst_expected); } #[test] @@ -993,8 +987,8 @@ mod transport { drop(tcp_client); let out = metrics.get("/metrics"); - assert_contains!(out, &src_expected); - assert_contains!(out, &dst_expected); } + assert_eventually_contains!(out, &src_expected); + assert_eventually_contains!(out, &dst_expected); } #[test] #[cfg_attr(not(feature = "flaky_tests"), ignore)] @@ -1007,7 +1001,7 @@ mod transport { tcp_client.write(TcpFixture::HELLO_MSG); assert_eq!(tcp_client.read(), TcpFixture::BYE_MSG.as_bytes()); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"outbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_http\"} 1"); } @@ -1023,11 +1017,11 @@ mod transport { tcp_client.write(TcpFixture::HELLO_MSG); assert_eq!(tcp_client.read(), TcpFixture::BYE_MSG.as_bytes()); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 1"); drop(tcp_client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\",classification=\"success\"} 1"); let tcp_client = client.connect(); @@ -1035,10 +1029,10 @@ mod transport { tcp_client.write(TcpFixture::HELLO_MSG); assert_eq!(tcp_client.read(), TcpFixture::BYE_MSG.as_bytes()); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_total{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 2"); drop(tcp_client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_close_total{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\",classification=\"success\"} 2"); } @@ -1056,9 +1050,9 @@ mod transport { drop(tcp_client); // TODO: make assertions about buckets let out = metrics.get("/metrics"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\",classification=\"success\"} 1"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"outbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_http\",classification=\"success\"} 1"); let tcp_client = client.connect(); @@ -1066,16 +1060,16 @@ mod transport { tcp_client.write(TcpFixture::HELLO_MSG); assert_eq!(tcp_client.read(), TcpFixture::BYE_MSG.as_bytes()); let out = metrics.get("/metrics"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\",classification=\"success\"} 1"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"outbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_http\",classification=\"success\"} 1"); drop(tcp_client); let out = metrics.get("/metrics"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\",classification=\"success\"} 2"); - assert_contains!(out, + assert_eventually_contains!(out, "tcp_connection_duration_ms_count{direction=\"outbound\",peer=\"dst\",tls=\"no_identity\",no_tls_reason=\"not_http\",classification=\"success\"} 2"); } @@ -1101,8 +1095,8 @@ mod transport { drop(tcp_client); let out = metrics.get("/metrics"); - assert_contains!(out, &src_expected); - assert_contains!(out, &dst_expected); + assert_eventually_contains!(out, &src_expected); + assert_eventually_contains!(out, &dst_expected); } #[test] @@ -1127,8 +1121,8 @@ mod transport { drop(tcp_client); let out = metrics.get("/metrics"); - assert_contains!(out, &src_expected); - assert_contains!(out, &dst_expected); + assert_eventually_contains!(out, &src_expected); + assert_eventually_contains!(out, &dst_expected); } #[test] @@ -1142,20 +1136,20 @@ mod transport { tcp_client.write(TcpFixture::HELLO_MSG); assert_eq!(tcp_client.read(), TcpFixture::BYE_MSG.as_bytes()); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_connections{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 1"); drop(tcp_client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_connections{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 0"); let tcp_client = client.connect(); tcp_client.write(TcpFixture::HELLO_MSG); assert_eq!(tcp_client.read(), TcpFixture::BYE_MSG.as_bytes()); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_connections{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 1"); drop(tcp_client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_connections{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 0"); } @@ -1169,10 +1163,10 @@ mod transport { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_connections{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 1"); drop(client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_connections{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 0"); // create a new client to force a new connection @@ -1180,11 +1174,11 @@ mod transport { info!("client.get(/)"); assert_eq!(client.get("/"), "hello"); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_connections{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 1"); drop(client); - assert_contains!(metrics.get("/metrics"), + assert_eventually_contains!(metrics.get("/metrics"), "tcp_open_connections{direction=\"outbound\",peer=\"src\",tls=\"internal_traffic\"} 0"); } } @@ -1243,7 +1237,7 @@ fn metrics_compression() { assert_eq!(client.get("/"), "hello"); for &encoding in encodings { - assert_contains!(do_scrape(encoding), + assert_eventually_contains!(do_scrape(encoding), "response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\"} 1"); } @@ -1251,7 +1245,7 @@ fn metrics_compression() { assert_eq!(client.get("/"), "hello"); for &encoding in encodings { - assert_contains!(do_scrape(encoding), + assert_eventually_contains!(do_scrape(encoding), "response_latency_ms_count{authority=\"tele.test.svc.cluster.local\",direction=\"inbound\",tls=\"disabled\",classification=\"success\",status_code=\"200\"} 2"); } }