await endpoints ready in more e2e tests to combat flakyness (#12947)

https://github.com/linkerd/linkerd2/pull/11368 added a step to certain e2e integration tests where we await the endpoints becoming ready before attempting to send traffic to them.  This was done to combat flakyness on those tests.

We have observed flakyness in other similar tests, `targets_route` in particular.  We add the same await step to that test and to all other tests in that form.

Given the nature of flaky tests, it's difficult to confirm that this fixes the flakyness.

Signed-off-by: Alex Leong <alex@buoyant.io>

policy-test/tests/e2e_audit.rs

@@ -1,6 +1,9 @@
 use kube::{Client, ResourceExt};
 use linkerd_policy_controller_k8s_api as k8s;
-use linkerd_policy_test::{create, create_ready_pod, curl, web, with_temp_ns, LinkerdInject};
+use linkerd_policy_test::{
+    await_condition, create, create_ready_pod, curl, endpoints_ready, web, with_temp_ns,
+    LinkerdInject,
+};
 
 #[tokio::test(flavor = "current_thread")]
 async fn server_audit() {
@@ -14,6 +17,8 @@ async fn server_audit() {
             create_ready_pod(&client, web::pod(&ns))
         );
 
+        await_condition(&client, &ns, "web", endpoints_ready).await;
+
         // All requests should fail
         let curl = curl::Runner::init(&client, &ns).await;
         let (injected, uninjected) = tokio::join!(
@@ -65,6 +70,8 @@ async fn ns_audit() {
             create_ready_pod(&client, web::pod(&ns))
         );
 
+        await_condition(&client, &ns, "web", endpoints_ready).await;
+
         // Unmeshed requests should fail
         let curl = curl::Runner::init(&client, &ns).await;
         let (injected, uninjected) = tokio::join!(

policy-test/tests/e2e_authorization_policy.rs

@@ -36,6 +36,8 @@ async fn meshtls() {
             create_ready_pod(&client, web::pod(&ns))
         );
 
+        await_condition(&client, &ns, "web", endpoints_ready).await;
+
         let curl = curl::Runner::init(&client, &ns).await;
         let (injected, uninjected) = tokio::join!(
             curl.run("curl-injected", "http://web", LinkerdInject::Enabled),
@@ -89,6 +91,8 @@ async fn targets_route() {
             create_ready_pod(&client, web::pod(&ns))
         );
 
+        await_condition(&client, &ns, "web", endpoints_ready).await;
+
         let curl = curl::Runner::init(&client, &ns).await;
 
         let (allowed, no_route, unauth, no_authz) = tokio::join!(
@@ -195,6 +199,8 @@ async fn targets_namespace() {
             create_ready_pod(&client, web::pod(&ns))
         );
 
+        await_condition(&client, &ns, "web", endpoints_ready).await;
+
         let curl = curl::Runner::init(&client, &ns).await;
         let (injected, uninjected) = tokio::join!(
             curl.run("curl-injected", "http://web", LinkerdInject::Enabled),
@@ -240,6 +246,8 @@ async fn meshtls_namespace() {
             create_ready_pod(&client, web::pod(&ns))
         );
 
+        await_condition(&client, &ns, "web", endpoints_ready).await;
+
         let curl = curl::Runner::init(&client, &ns).await;
         let (injected, uninjected) = tokio::join!(
             curl.run("curl-injected", "http://web", LinkerdInject::Enabled),
@@ -541,6 +549,8 @@ async fn empty_authentications() {
             create_ready_pod(&client, web::pod(&ns))
         );
 
+        await_condition(&client, &ns, "web", endpoints_ready).await;
+
         // All requests should work.
         let curl = curl::Runner::init(&client, &ns).await;
         let (injected, uninjected) = tokio::join!(

policy-test/tests/e2e_http_routing.rs

@@ -1,5 +1,8 @@
 use linkerd_policy_controller_k8s_api as k8s;
-use linkerd_policy_test::{create, create_ready_pod, curl, web, with_temp_ns, LinkerdInject};
+use linkerd_policy_test::{
+    await_condition, create, create_ready_pod, curl, endpoints_ready, web, with_temp_ns,
+    LinkerdInject,
+};
 
 #[tokio::test(flavor = "current_thread")]
 async fn path_based_routing() {
@@ -40,6 +43,8 @@ async fn path_based_routing() {
             create_ready_pod(&client, web::pod(&ns))
         );
 
+        await_condition(&client, &ns, "web", endpoints_ready).await;
+
         let curl = curl::Runner::init(&client, &ns).await;
         let (valid, invalid, notfound) = tokio::join!(
             curl.run("curl-valid", "http://web/valid", LinkerdInject::Enabled),

policy-test/tests/e2e_server_authorization.rs

@@ -34,6 +34,8 @@ async fn meshtls() {
             create_ready_pod(&client, web::pod(&ns))
         );
 
+        await_condition(&client, &ns, "web", endpoints_ready).await;
+
         let curl = curl::Runner::init(&client, &ns).await;
         let (injected, uninjected) = tokio::join!(
             curl.run("curl-injected", "http://web", LinkerdInject::Enabled),