[release-1.27] Multinetwork/Support remote networks for services with waypoints (#1592)

* Multinetwork/Support remote networks for services with waypoints

Currently `build_request` when it sees a service with a waypoint
resolves the waypoint backend and routes request there using regular
HBONE.

In multi network scenario though the waypoint may have workload on a
remote network and to reach it we have to go through E/W gateway and use
double HBONE.

This change enables handling of services with waypoint on a remote
network.

Some of the assumptions that were used when I prepared this change:

1. We assume uniformity of configuration (e.g., if service X in local
   cluster has a waypoint, then service X in remote network also has a
   waypoint, if waypoint is service addressable, then it's using service
   to address waypoint both locally and on remote network)
2  Split-horizon representation of waypoint workloads, just like with
   any regular workloads and services (e.g., in the local cluster
   instead of an actual waypoint workload pointing to a pod on another
   network we will have a "proxy" representation that just has network
   gateway).

Both of those can be in hanled by the controlplane (e.g., controlplane
can generate split-horizon workloads and when configuration is
non-uniform, just filter out remote configs for remote networks), though
we don't yet have a complete implementation.

Signed-off-by: Mikhail Krinkin <mkrinkin@microsoft.com>

* Return an error instead of panicking

Signed-off-by: Mikhail Krinkin <mkrinkin@microsoft.com>

* Update comments in src/proxy/outbound.rs

Co-authored-by: Ian Rudie <ilrudie@gmail.com>

* Update comments in src/proxy/outbound.rs

Signed-off-by: Mikhail Krinkin <mkrinkin@microsoft.com>

* Add a debug assert to provide a bit more context to the error in tests

Signed-off-by: Mikhail Krinkin <mkrinkin@microsoft.com>

* Fix formatting

Signed-off-by: Mikhail Krinkin <mkrinkin@microsoft.com>

* Added a few debug logs to be able to trace when a workload on a remote network is picked

Signed-off-by: Mikhail Krinkin <mkrinkin@microsoft.com>

---------

Signed-off-by: Mikhail Krinkin <mkrinkin@microsoft.com>
Co-authored-by: Mikhail Krinkin <mkrinkin@microsoft.com>
Co-authored-by: Krinkin, Mike <krinkin.m.u@gmail.com>
Co-authored-by: Ian Rudie <ilrudie@gmail.com>

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
   "name": "istio build-tools",
-  "image": "gcr.io/istio-testing/build-tools:master-672e6089ff843019a2b28cf9e87754c7b74358ea",
+  "image": "gcr.io/istio-testing/build-tools:master-9a50b34135f1aa16b04d56dcb89c5b91101a28aa",
   "privileged": true,
   "remoteEnv": {
     "USE_GKE_GCLOUD_AUTH_PLUGIN": "True",

CODEOWNERS

@@ -1,6 +1 @@
-*                @istio/wg-networking-maintainers-ztunnel
+* @istio/release-managers-1-27
-/Makefile*       @istio/wg-test-and-release-maintainers
-/*.md            @istio/wg-test-and-release-maintainers
-/common/         @istio/wg-test-and-release-maintainers
-/common-protos/  @istio/wg-test-and-release-maintainers
-/scripts/        @istio/wg-test-and-release-maintainers

Cargo.lock

@@ -1802,7 +1802,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "fc2f4eb4bc735547cfed7c0a4922cbd04a4655978c09b54f1f7b228750664c34"
 dependencies = [
  "cfg-if",
- "windows-targets 0.52.6",
+ "windows-targets 0.48.5",
 ]
 
 [[package]]
@@ -2777,21 +2777,6 @@ dependencies = [
  "yasna",
 ]
 
-[[package]]
-name = "rcgen"
-version = "0.14.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49bc8ffa8a832eb1d7c8000337f8b0d2f4f2f5ec3cf4ddc26f125e3ad2451824"
-dependencies = [
- "aws-lc-rs",
- "pem",
- "ring",
- "rustls-pki-types",
- "time",
- "x509-parser",
- "yasna",
-]
-
 [[package]]
 name = "redox_syscall"
 version = "0.5.11"
@@ -4470,8 +4455,7 @@ dependencies = [
  "prost-build",
  "prost-types",
  "rand 0.9.0",
- "rcgen 0.13.3",
+ "rcgen",
- "rcgen 0.14.2",
  "ring",
  "rustc_version",
  "rustls",

Cargo.toml

@@ -80,7 +80,7 @@ prometheus-parse = "0.2"
 prost = "0.13"
 prost-types = "0.13"
 rand = { version = "0.9" , features = ["small_rng"]}
-rcgen = { version = "0.14", optional = true, features = ["pem"] }
+rcgen = { version = "0.13", optional = true, features = ["pem"] }
 rustls = { version = "0.23", default-features = false }
 rustls-native-certs = "0.8"
 rustls-pemfile = "2.2"

benches/throughput.rs

@@ -458,10 +458,10 @@ fn hbone_connection_config() -> ztunnel::config::ConfigSource {
             workload: Workload {
                 workload_ips: vec![hbone_connection_ip(i)],
                 protocol: InboundProtocol::HBONE,
-                uid: strng::format!("cluster1//v1/Pod/default/remote{i}"),
+                uid: strng::format!("cluster1//v1/Pod/default/remote{}", i),
-                name: strng::format!("workload-{i}"),
+                name: strng::format!("workload-{}", i),
-                namespace: strng::format!("namespace-{i}"),
+                namespace: strng::format!("namespace-{}", i),
-                service_account: strng::format!("service-account-{i}"),
+                service_account: strng::format!("service-account-{}", i),
                 ..test_helpers::test_default_workload()
             },
             services: Default::default(),

build.rs

@@ -94,6 +94,9 @@ fn main() -> Result<(), anyhow::Error> {
         "cargo:rustc-env=ZTUNNEL_BUILD_RUSTC_VERSION={}",
         rustc_version::version().unwrap()
     );
-    println!("cargo:rustc-env=ZTUNNEL_BUILD_PROFILE_NAME={profile_name}");
+    println!(
+        "cargo:rustc-env=ZTUNNEL_BUILD_PROFILE_NAME={}",
+        profile_name
+    );
     Ok(())
 }

common/.commonfiles.sha

@@ -1 +1 @@
-d235bc9f4a20f3c78c5aacbfa3f24d08a884a82e
+c86fa2ea9bd8350fe4fb040b78201c05a6023ced

common/Makefile.common.mk

@@ -92,7 +92,7 @@ mirror-licenses: mod-download-go
 	@license-lint --mirror
 
 TMP := $(shell mktemp -d -u)
-UPDATE_BRANCH ?= "master"
+UPDATE_BRANCH ?= "release-1.27"
 
 BUILD_TOOLS_ORG ?= "istio"
 

common/config/.golangci.yml

@@ -184,10 +184,6 @@ linters:
       - linters:
           - staticcheck
         text: 'S1007'
-      # TODO: remove once we have updated package names
-      - linters:
-        - revive
-        text: "var-naming: avoid meaningless package names"
     paths:
       - .*\.pb\.go
       - .*\.gen\.go

common/scripts/setup_env.sh

@@ -75,7 +75,7 @@ fi
 TOOLS_REGISTRY_PROVIDER=${TOOLS_REGISTRY_PROVIDER:-gcr.io}
 PROJECT_ID=${PROJECT_ID:-istio-testing}
 if [[ "${IMAGE_VERSION:-}" == "" ]]; then
-  IMAGE_VERSION=master-672e6089ff843019a2b28cf9e87754c7b74358ea
+  IMAGE_VERSION=release-1.27-9a50b34135f1aa16b04d56dcb89c5b91101a28aa
 fi
 if [[ "${IMAGE_NAME:-}" == "" ]]; then
   IMAGE_NAME=build-tools

fuzz/Cargo.lock

@@ -2359,9 +2359,9 @@ dependencies = [
 
 [[package]]
 name = "rcgen"
-version = "0.14.2"
+version = "0.13.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49bc8ffa8a832eb1d7c8000337f8b0d2f4f2f5ec3cf4ddc26f125e3ad2451824"
+checksum = "75e669e5202259b5314d1ea5397316ad400819437857b90861765f24c4cf80a2"
 dependencies = [
  "aws-lc-rs",
  "pem",

src/admin.rs

@@ -390,7 +390,7 @@ fn change_log_level(reset: bool, level: &str) -> Response<Full<Bytes>> {
             // Invalid level provided
             return plaintext_response(
                 hyper::StatusCode::BAD_REQUEST,
-                format!("Invalid level provided: {level}\n{HELP_STRING}"),
+                format!("Invalid level provided: {}\n{}", level, HELP_STRING),
             );
         };
     }
@@ -398,7 +398,7 @@ fn change_log_level(reset: bool, level: &str) -> Response<Full<Bytes>> {
         Ok(_) => list_loggers(),
         Err(e) => plaintext_response(
             hyper::StatusCode::BAD_REQUEST,
-            format!("Failed to set new level: {e}\n{HELP_STRING}"),
+            format!("Failed to set new level: {}\n{}", e, HELP_STRING),
         ),
     }
 }

src/config.rs

@@ -309,8 +309,6 @@ pub struct Config {
     pub ztunnel_identity: Option<identity::Identity>,
 
     pub ztunnel_workload: Option<state::WorkloadInfo>,
-
-    pub ipv6_enabled: bool,
 }
 
 #[derive(serde::Serialize, Clone, Copy, Debug)]
@@ -440,7 +438,7 @@ fn parse_worker_threads(default: usize) -> Result<usize, Error> {
                     Error::EnvVar(
                         ZTUNNEL_WORKER_THREADS.to_string(),
                         value.clone(),
-                        format!("invalid percentage: {e}"),
+                        format!("invalid percentage: {}", e),
                     )
                 })?;
 
@@ -462,7 +460,7 @@ fn parse_worker_threads(default: usize) -> Result<usize, Error> {
                     Error::EnvVar(
                         ZTUNNEL_WORKER_THREADS.to_string(),
                         value,
-                        format!("invalid number: {e}"),
+                        format!("invalid number: {}", e),
                     )
                 })
             }
@@ -599,7 +597,7 @@ pub fn construct_config(pc: ProxyConfig) -> Result<Config, Error> {
     // on a pod-by-pod basis.
     let dns_proxy_addr: Address = match pc.proxy_metadata.get(DNS_PROXY_ADDR_METADATA) {
         Some(dns_addr) => Address::new(ipv6_localhost_enabled, dns_addr)
-            .unwrap_or_else(|_| panic!("failed to parse DNS_PROXY_ADDR: {dns_addr}")),
+            .unwrap_or_else(|_| panic!("failed to parse DNS_PROXY_ADDR: {}", dns_addr)),
         None => Address::Localhost(ipv6_localhost_enabled, DEFAULT_DNS_PORT),
     };
 
@@ -864,7 +862,6 @@ pub fn construct_config(pc: ProxyConfig) -> Result<Config, Error> {
         localhost_app_tunnel: parse_default(LOCALHOST_APP_TUNNEL, true)?,
         ztunnel_identity,
         ztunnel_workload,
-        ipv6_enabled,
     })
 }
 

src/dns/server.rs

@@ -86,7 +86,6 @@ impl Server {
         socket_factory: &(dyn SocketFactory + Send + Sync),
         local_workload_information: Arc<LocalWorkloadFetcher>,
         prefered_service_namespace: Option<String>,
-        ipv6_enabled: bool,
     ) -> Result<Self, Error> {
         // if the address we got from config is supposed to be v6-enabled,
         // actually check if the local pod context our socketfactory operates in supports V6.
@@ -105,7 +104,6 @@ impl Server {
             metrics,
             local_workload_information,
             prefered_service_namespace,
-            ipv6_enabled,
         );
         let store = Arc::new(store);
         let handler = dns::handler::Handler::new(store.clone());
@@ -196,7 +194,6 @@ struct Store {
     metrics: Arc<Metrics>,
     local_workload: Arc<LocalWorkloadFetcher>,
     prefered_service_namespace: Option<String>,
-    ipv6_enabled: bool,
 }
 
 impl Store {
@@ -207,7 +204,6 @@ impl Store {
         metrics: Arc<Metrics>,
         local_workload_information: Arc<LocalWorkloadFetcher>,
         prefered_service_namespace: Option<String>,
-        ipv6_enabled: bool,
     ) -> Self {
         let domain = as_name(domain);
         let svc_domain = append_name(as_name("svc"), &domain);
@@ -220,7 +216,6 @@ impl Store {
             metrics,
             local_workload: local_workload_information,
             prefered_service_namespace,
-            ipv6_enabled,
         }
     }
 
@@ -427,13 +422,6 @@ impl Store {
         None
     }
 
-    fn record_type_enabled(&self, addr: &IpAddr) -> bool {
-        match addr {
-            IpAddr::V4(_) => true,              // IPv4 always
-            IpAddr::V6(_) => self.ipv6_enabled, // IPv6 must be not be disabled in config
-        }
-    }
-
     /// Gets the list of addresses of the requested record type from the server.
     fn get_addresses(
         &self,
@@ -446,7 +434,7 @@ impl Store {
                 .workload_ips
                 .iter()
                 .filter_map(|addr| {
-                    if is_record_type(addr, record_type) && self.record_type_enabled(addr) {
+                    if is_record_type(addr, record_type) {
                         Some(*addr)
                     } else {
                         None
@@ -465,9 +453,10 @@ impl Store {
                                 debug!("failed to fetch workload for {}", ep.workload_uid);
                                 return None;
                             };
-                            wl.workload_ips.iter().copied().find(|addr| {
+                            wl.workload_ips
-                                is_record_type(addr, record_type) && self.record_type_enabled(addr)
+                                .iter()
-                            })
+                                .copied()
+                                .find(|addr| is_record_type(addr, record_type))
                         })
                         .collect()
                 } else {
@@ -479,7 +468,6 @@ impl Store {
                         .filter_map(|vip| {
                             if is_record_type(&vip.address, record_type)
                                 && client.network == vip.network
-                                && self.record_type_enabled(&vip.address)
                             {
                                 Some(vip.address)
                             } else {
@@ -649,7 +637,7 @@ impl Resolver for Store {
         // From this point on, we are the authority for the response.
         let is_authoritative = true;
 
-        if !service_family_allowed(&service_match.server, record_type, self.ipv6_enabled) {
+        if !service_family_allowed(&service_match.server, record_type) {
             access_log(
                 request,
                 Some(&client),
@@ -718,13 +706,7 @@ impl Resolver for Store {
 /// anyway, so would naturally work.
 /// Headless services, however, do not have VIPs, and the Pods behind them can have dual stack IPs even with
 /// the Service being single-stack. In this case, we are NOT supposed to return both IPs.
-/// If IPv6 is globally disabled, AAAA records are not allowed.
+fn service_family_allowed(server: &Address, record_type: RecordType) -> bool {
-fn service_family_allowed(server: &Address, record_type: RecordType, ipv6_enabled: bool) -> bool {
-    // If IPv6 is globally disabled, don't allow AAAA records
-    if !ipv6_enabled && record_type == RecordType::AAAA {
-        return false;
-    }
-
     match server {
         Address::Service(service) => match service.ip_families {
             Some(IpFamily::IPv4) if record_type == RecordType::AAAA => false,
@@ -1105,7 +1087,6 @@ mod tests {
                 metrics: test_metrics(),
                 local_workload,
                 prefered_service_namespace: None,
-                ipv6_enabled: true,
             };
 
             let namespaced_domain = n(format!("{}.svc.cluster.local", c.client_namespace));
@@ -1451,7 +1432,6 @@ mod tests {
             &factory,
             local_workload,
             Some(PREFERRED.to_string()),
-            true, // ipv6_enabled for tests
         )
         .await
         .unwrap();
@@ -1471,8 +1451,8 @@ mod tests {
                 tasks.push(async move {
                     let name = format!("[{protocol}] {}", c.name);
                     let resp = send_request(&mut client, n(c.host), c.query_type).await;
-                    assert_eq!(c.expect_authoritative, resp.authoritative(), "{name}");
+                    assert_eq!(c.expect_authoritative, resp.authoritative(), "{}", name);
-                    assert_eq!(c.expect_code, resp.response_code(), "{name}");
+                    assert_eq!(c.expect_code, resp.response_code(), "{}", name);
 
                     if c.expect_code == ResponseCode::NoError {
                         let mut actual = resp.answers().to_vec();
@@ -1483,7 +1463,7 @@ mod tests {
                         if c.expect_authoritative {
                             sort_records(&mut actual);
                         }
-                        assert_eq!(c.expect_records, actual, "{name}");
+                        assert_eq!(c.expect_records, actual, "{}", name);
                     }
                 });
             }
@@ -1539,7 +1519,6 @@ mod tests {
             &factory,
             local_workload,
             None,
-            true, // ipv6_enabled for tests
         )
         .await
         .unwrap();
@@ -1554,7 +1533,7 @@ mod tests {
             for (protocol, client) in [("tcp", &mut tcp_client), ("udp", &mut udp_client)] {
                 let name = format!("[{protocol}] {}", c.name);
                 let resp = send_request(client, n(c.host), RecordType::A).await;
-                assert_eq!(c.expect_code, resp.response_code(), "{name}");
+                assert_eq!(c.expect_code, resp.response_code(), "{}", name);
                 if c.expect_code == ResponseCode::NoError {
                     assert!(!resp.answers().is_empty());
                 }
@@ -1590,7 +1569,6 @@ mod tests {
                 state.clone(),
             ),
             prefered_service_namespace: None,
-            ipv6_enabled: true,
         };
 
         let ip4n6_client_ip = ip("::ffff:202:202");
@@ -1598,7 +1576,7 @@ mod tests {
         match store.lookup(&req).await {
             Ok(_) => {}
             Err(e) => {
-                panic!("IPv6 encoded IPv4 should work! Error was {e:?}");
+                panic!("IPv6 encoded IPv4 should work! Error was {:?}", e)
             }
         }
     }
@@ -1625,7 +1603,6 @@ mod tests {
             &factory,
             local_workload,
             None,
-            true, // ipv6_enabled for tests
         )
         .await
         .unwrap();

src/inpod/config.rs

@@ -148,7 +148,7 @@ impl crate::proxy::SocketFactory for InPodSocketPortReuseFactory {
         })?;
 
         if let Err(e) = sock.set_reuseport(true) {
-            tracing::warn!("setting set_reuseport failed: {e} addr: {addr}");
+            tracing::warn!("setting set_reuseport failed: {} addr: {}", e, addr);
         }
 
         sock.bind(addr)?;

src/inpod/test_helpers.rs

@@ -37,7 +37,7 @@ use std::os::fd::{AsRawFd, OwnedFd};
 use tracing::debug;
 
 pub fn uid(i: usize) -> crate::inpod::WorkloadUid {
-    crate::inpod::WorkloadUid::new(format!("uid{i}"))
+    crate::inpod::WorkloadUid::new(format!("uid{}", i))
 }
 
 pub struct Fixture {
@@ -138,7 +138,7 @@ pub async fn read_msg(s: &mut UnixStream) -> WorkloadResponse {
     debug!("read {} bytes", read_amount);
 
     let ret = WorkloadResponse::decode(&buf[..read_amount])
-        .unwrap_or_else(|_| panic!("failed to decode. read amount: {read_amount}"));
+        .unwrap_or_else(|_| panic!("failed to decode. read amount: {}", read_amount));
 
     debug!("decoded {:?}", ret);
     ret

src/inpod/workloadmanager.rs

@@ -401,7 +401,7 @@ pub(crate) mod tests {
                 assert!(e.contains("EOF"));
             }
             Ok(()) => {}
-            Err(e) => panic!("expected error due to EOF {e:?}"),
+            Err(e) => panic!("expected error due to EOF {:?}", e),
         }
     }
 

src/proxy.rs

@@ -306,7 +306,7 @@ impl Proxy {
             old_cfg.inbound_addr = inbound.address();
             let mut new_pi = (*pi).clone();
             new_pi.cfg = Arc::new(old_cfg);
-            pi = Arc::new(new_pi);
+            std::mem::swap(&mut pi, &mut Arc::new(new_pi));
             warn!("TEST FAKE: new address is {:?}", pi.cfg.inbound_addr);
         }
 
@@ -373,7 +373,7 @@ impl fmt::Display for AuthorizationRejectionError {
         match self {
             Self::NoWorkload => write!(fmt, "workload not found"),
             Self::WorkloadMismatch => write!(fmt, "workload mismatch"),
-            Self::ExplicitlyDenied(a, b) => write!(fmt, "explicitly denied by: {a}/{b}"),
+            Self::ExplicitlyDenied(a, b) => write!(fmt, "explicitly denied by: {}/{}", a, b),
             Self::NotAllowed => write!(fmt, "allow policies exist, but none allowed"),
         }
     }
@@ -847,8 +847,8 @@ impl HboneAddress {
 impl std::fmt::Display for HboneAddress {
     fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
         match self {
-            HboneAddress::SocketAddr(addr) => write!(f, "{addr}"),
+            HboneAddress::SocketAddr(addr) => write!(f, "{}", addr),
-            HboneAddress::SvcHostname(host, port) => write!(f, "{host}:{port}"),
+            HboneAddress::SvcHostname(host, port) => write!(f, "{}:{}", host, port),
         }
     }
 }

src/proxy/metrics.rs

@@ -391,7 +391,7 @@ pub fn log_early_deny<E: std::error::Error>(
                 "inbound"
             },
 
-            error = format!("{err}"),
+            error = format!("{}", err),
 
             "connection failed"
     );

src/proxy/socks5.rs

@@ -203,7 +203,8 @@ async fn negotiate_socks_connection(
 
     if version != 0x05 {
         return Err(SocksError::invalid_protocol(format!(
-            "unsupported version {version}",
+            "unsupported version {}",
+            version
         )));
     }
 

src/proxyfactory.rs

@@ -113,7 +113,6 @@ impl ProxyFactory {
                 socket_factory.as_ref(),
                 local_workload_information.as_fetcher(),
                 self.config.prefered_service_namespace.clone(),
-                self.config.ipv6_enabled,
             )
             .await?;
             resolver = Some(server.resolver());

src/state.rs

@@ -870,7 +870,7 @@ impl DemandProxyState {
         self.finalize_upstream(source_workload, target_address, res)
             .await?
             .ok_or_else(|| {
-                Error::UnknownNetworkGateway(format!("network gateway {gw_address:?} not found"))
+                Error::UnknownNetworkGateway(format!("network gateway {:?} not found", gw_address))
             })
     }
 
@@ -926,7 +926,7 @@ impl DemandProxyState {
         };
         self.finalize_upstream(source_workload, target_address, res)
             .await?
-            .ok_or_else(|| Error::UnknownWaypoint(format!("waypoint {gw_address:?} not found")))
+            .ok_or_else(|| Error::UnknownWaypoint(format!("waypoint {:?} not found", gw_address)))
     }
 
     pub async fn fetch_service_waypoint(
@@ -1378,17 +1378,17 @@ mod tests {
     fn create_workload(dest_uid: u8) -> Workload {
         Workload {
             name: "test".into(),
-            namespace: format!("ns{dest_uid}").into(),
+            namespace: format!("ns{}", dest_uid).into(),
             trust_domain: "cluster.local".into(),
             service_account: "defaultacct".into(),
             workload_ips: vec![IpAddr::V4(Ipv4Addr::new(192, 168, 0, dest_uid))],
-            uid: format!("{dest_uid}").into(),
+            uid: format!("{}", dest_uid).into(),
             ..test_helpers::test_default_workload()
         }
     }
 
     fn get_workload(state: &DemandProxyState, dest_uid: u8) -> Arc<Workload> {
-        let key: Strng = format!("{dest_uid}").into();
+        let key: Strng = format!("{}", dest_uid).into();
         state.read().workloads.by_uid[&key].clone()
     }
 
@@ -1397,7 +1397,7 @@ mod tests {
         dest_uid: u8,
         src_svc_acct: &str,
     ) -> crate::state::ProxyRbacContext {
-        let key: Strng = format!("{dest_uid}").into();
+        let key: Strng = format!("{}", dest_uid).into();
         let workload = &state.read().workloads.by_uid[&key];
         crate::state::ProxyRbacContext {
             conn: rbac::Connection {

src/state/workload.rs

@@ -1028,8 +1028,8 @@ mod tests {
             },
         )]);
 
-        let uid1 = format!("cluster1//v1/Pod/default/my-pod/{ip1:?}");
+        let uid1 = format!("cluster1//v1/Pod/default/my-pod/{:?}", ip1);
-        let uid2 = format!("cluster1//v1/Pod/default/my-pod/{ip2:?}");
+        let uid2 = format!("cluster1//v1/Pod/default/my-pod/{:?}", ip2);
 
         updater
             .insert_workload(
@@ -1734,7 +1734,7 @@ mod tests {
 
         let xds_ip1 = Bytes::copy_from_slice(&[127, 0, 0, 1]);
         let ip1 = IpAddr::V4(Ipv4Addr::new(127, 0, 0, 1));
-        let uid1 = format!("cluster1//v1/Pod/default/my-pod/{ip1:?}");
+        let uid1 = format!("cluster1//v1/Pod/default/my-pod/{:?}", ip1);
 
         let services = HashMap::from([(
             "ns/svc1.ns.svc.cluster.local".to_string(),

src/telemetry.rs

@@ -170,7 +170,7 @@ impl Visitor<'_> {
         } else {
             " "
         };
-        write!(self.writer, "{padding}{value:?}")
+        write!(self.writer, "{}{:?}", padding, value)
     }
 }
 
@@ -188,9 +188,9 @@ impl field::Visit for Visitor<'_> {
             // Skip fields that are actually log metadata that have already been handled
             name if name.starts_with("log.") => Ok(()),
             // For the message, write out the message and a tab to separate the future fields
-            "message" => write!(self.writer, "{val:?}\t"),
+            "message" => write!(self.writer, "{:?}\t", val),
             // For the rest, k=v.
-            _ => self.write_padded(&format_args!("{}={val:?}", field.name())),
+            _ => self.write_padded(&format_args!("{}={:?}", field.name(), val)),
         }
     }
 }
@@ -234,7 +234,7 @@ where
         let target = meta.target();
         // No need to prefix everything
         let target = target.strip_prefix("ztunnel::").unwrap_or(target);
-        write!(writer, "{target}")?;
+        write!(writer, "{}", target)?;
 
         // Write out span fields. Istio logging outside of Rust doesn't really have this concept
         if let Some(scope) = ctx.event_scope() {
@@ -243,7 +243,7 @@ where
                 let ext = span.extensions();
                 if let Some(fields) = &ext.get::<FormattedFields<N>>() {
                     if !fields.is_empty() {
-                        write!(writer, "{{{fields}}}")?;
+                        write!(writer, "{{{}}}", fields)?;
                     }
                 }
             }
@@ -285,7 +285,7 @@ impl<S: SerializeMap> Visit for JsonVisitory<S> {
         if self.state.is_ok() {
             self.state = self
                 .serializer
-                .serialize_entry(field.name(), &format_args!("{value:?}"))
+                .serialize_entry(field.name(), &format_args!("{:?}", value))
         }
     }
 
@@ -505,7 +505,7 @@ pub mod testing {
             .map(|h| {
                 h.iter()
                     .sorted_by_key(|(k, _)| *k)
-                    .map(|(k, err)| format!("{k}:{err}"))
+                    .map(|(k, err)| format!("{}:{}", k, err))
                     .join("\n")
             })
             .join("\n\n");

src/test_helpers.rs

@@ -169,9 +169,10 @@ pub fn localhost_error_message() -> String {
         TEST_VIP,
     ];
     format!(
-        "These tests use the following loopback addresses: {addrs:?}. \
+        "These tests use the following loopback addresses: {:?}. \
     Your OS may require an explicit alias for each. If so, you'll need to manually \
     configure your system for each IP (e.g. `sudo ifconfig lo0 alias 127.0.0.2 up`).",
+        addrs
     )
 }
 
@@ -238,7 +239,7 @@ fn test_custom_workload(
     hostname_only: bool,
 ) -> anyhow::Result<LocalWorkload> {
     let host = match hostname_only {
-        true => format!("{ip_str}.reflect.internal."),
+        true => format!("{}.reflect.internal.", ip_str),
         false => "".to_string(),
     };
     let wips = match hostname_only {
@@ -249,7 +250,7 @@ fn test_custom_workload(
         workload_ips: wips,
         hostname: host.into(),
         protocol,
-        uid: format!("cluster1//v1/Pod/default/{name}").into(),
+        uid: format!("cluster1//v1/Pod/default/{}", name).into(),
         name: name.into(),
         namespace: "default".into(),
         service_account: "default".into(),
@@ -281,7 +282,7 @@ fn test_custom_svc(
         }],
         ports: HashMap::from([(80u16, echo_port)]),
         endpoints: EndpointSet::from_list([Endpoint {
-            workload_uid: format!("cluster1//v1/Pod/default/{workload_name}").into(),
+            workload_uid: format!("cluster1//v1/Pod/default/{}", workload_name).into(),
             port: HashMap::from([(80u16, echo_port)]),
             status: HealthStatus::Healthy,
         }]),

src/test_helpers/app.rs

@@ -105,7 +105,7 @@ impl TestApp {
         let get_resp = move || async move {
             let req = Request::builder()
                 .method(Method::GET)
-                .uri(format!("http://localhost:{port}/{path}"))
+                .uri(format!("http://localhost:{}/{path}", port))
                 .header("content-type", "application/json")
                 .body(Empty::<Bytes>::new())
                 .unwrap();
@@ -130,7 +130,7 @@ impl TestApp {
         let get_resp = move || async move {
             let req = Request::builder()
                 .method(Method::GET)
-                .uri(format!("http://localhost:{port}/{path}"))
+                .uri(format!("http://localhost:{}/{path}", port))
                 .header("content-type", "application/json")
                 .body(Empty::<Bytes>::new())
                 .unwrap();

src/test_helpers/dns.rs

@@ -299,7 +299,6 @@ pub async fn run_dns(responses: HashMap<Name, Vec<IpAddr>>) -> anyhow::Result<Te
             state.clone(),
         ),
         Some("prefered-namespace".to_string()),
-        true, // ipv6_enabled for tests
     )
     .await?;
 

src/tls/certificate.rs

@@ -428,6 +428,7 @@ mod test {
             SystemTime::now() + Duration::from_secs(60),
             None,
             TEST_ROOT_KEY,
+            TEST_ROOT,
         );
         let cert1 =
             WorkloadCertificate::new(key.as_bytes(), cert.as_bytes(), vec![&joined]).unwrap();
@@ -439,6 +440,7 @@ mod test {
             SystemTime::now() + Duration::from_secs(60),
             None,
             TEST_ROOT2_KEY,
+            TEST_ROOT2,
         );
         let cert2 =
             WorkloadCertificate::new(key.as_bytes(), cert.as_bytes(), vec![&joined]).unwrap();

src/tls/mock.rs

@@ -18,6 +18,7 @@ use std::fmt::{Display, Formatter};
 use rand::RngCore;
 use rand::SeedableRng;
 use rand::rngs::SmallRng;
+use rcgen::{Certificate, CertificateParams, KeyPair};
 use std::net::IpAddr;
 use std::sync::Arc;
 use std::time::{Duration, SystemTime};
@@ -104,7 +105,8 @@ pub fn generate_test_certs_at(
     not_after: SystemTime,
     rng: Option<&mut dyn rand::RngCore>,
 ) -> WorkloadCertificate {
-    let (key, cert) = generate_test_certs_with_root(id, not_before, not_after, rng, TEST_ROOT_KEY);
+    let (key, cert) =
+        generate_test_certs_with_root(id, not_before, not_after, rng, TEST_ROOT_KEY, TEST_ROOT);
     let mut workload =
         WorkloadCertificate::new(key.as_bytes(), cert.as_bytes(), vec![TEST_ROOT]).unwrap();
     // Certificates do not allow sub-millisecond, but we need this for tests.
@@ -119,6 +121,7 @@ pub fn generate_test_certs_with_root(
     not_after: SystemTime,
     rng: Option<&mut dyn rand::RngCore>,
     ca_key: &[u8],
+    ca_cert: &[u8],
 ) -> (String, String) {
     use rcgen::*;
     let serial_number = {
@@ -147,17 +150,15 @@ pub fn generate_test_certs_with_root(
         ExtendedKeyUsagePurpose::ClientAuth,
     ];
     p.subject_alt_names = vec![match id {
-        TestIdentity::Identity(i) => {
+        TestIdentity::Identity(i) => SanType::URI(Ia5String::try_from(i.to_string()).unwrap()),
-            SanType::URI(string::Ia5String::try_from(i.to_string()).unwrap())
-        }
         TestIdentity::Ip(i) => SanType::IpAddress(*i),
     }];
 
     let kp = KeyPair::from_pem(std::str::from_utf8(TEST_PKEY).unwrap()).unwrap();
     let ca_kp = KeyPair::from_pem(std::str::from_utf8(ca_key).unwrap()).unwrap();
     let key = kp.serialize_pem();
-    let issuer = Issuer::from_params(&p, &ca_kp);
+    let ca = test_ca(ca_key, ca_cert);
-    let cert = p.signed_by(&kp, &issuer).unwrap();
+    let cert = p.signed_by(&kp, &ca, &ca_kp).unwrap();
     let cert = cert.pem();
     (key, cert)
 }
@@ -171,6 +172,12 @@ pub fn generate_test_certs(
     generate_test_certs_at(id, not_before, not_before + duration_until_expiry, None)
 }
 
+fn test_ca(key: &[u8], cert: &[u8]) -> Certificate {
+    let key = KeyPair::from_pem(std::str::from_utf8(key).unwrap()).unwrap();
+    let ca_param = CertificateParams::from_ca_cert_pem(std::str::from_utf8(cert).unwrap()).unwrap();
+    ca_param.self_signed(&key).unwrap()
+}
+
 #[derive(Debug, Clone)]
 pub struct MockServerCertProvider(Arc<WorkloadCertificate>);
 

src/xds/client.rs

@@ -876,7 +876,7 @@ mod tests {
 
     fn get_auth(i: usize) -> ProtoResource {
         let addr = XdsAuthorization {
-            name: format!("foo{i}"),
+            name: format!("foo{}", i),
             namespace: "default".to_string(),
             scope: crate::xds::istio::security::Scope::Global as i32,
             action: crate::xds::istio::security::Action::Deny as i32,
@@ -890,7 +890,7 @@ mod tests {
             }],
         };
         ProtoResource {
-            name: format!("foo{i}"),
+            name: format!("foo{}", i),
             aliases: vec![],
             version: "0.0.1".to_string(),
             resource: Some(Any {
@@ -908,8 +908,8 @@ mod tests {
         };
         let addr = XdsAddress {
             r#type: Some(XdsType::Workload(XdsWorkload {
-                name: format!("foo{i}"),
+                name: format!("foo{}", i),
-                uid: format!("default/foo{i}"),
+                uid: format!("default/foo{}", i),
                 namespace: "default".to_string(),
                 addresses: vec![octets.into()],
                 tunnel_protocol: 0,
@@ -924,7 +924,7 @@ mod tests {
         };
 
         ProtoResource {
-            name: format!("foo{i}"),
+            name: format!("foo{}", i),
             aliases: vec![],
             version: "0.0.1".to_string(),
             resource: Some(Any {

tests/direct.rs

@@ -282,7 +282,8 @@ fn on_demand_dns_assertions(metrics: ParsedMetrics) {
         };
         assert!(
             value == expected,
-            "expected metric {metric} to be 1, was {value:?}",
+            "expected metric {metric} to be 1, was {:?}",
+            value
         );
     }
 }
@@ -360,9 +361,9 @@ async fn test_stats_exist() {
         {
             for (name, doc) in metric_info {
                 if stable_metrics.contains(&*name) {
-                    assert!(!doc.contains("unstable"), "{name}: {doc}");
+                    assert!(!doc.contains("unstable"), "{}: {}", name, doc);
                 } else {
-                    assert!(doc.contains("unstable"), "{name}: {doc}");
+                    assert!(doc.contains("unstable"), "{}: {}", name, doc);
                 }
             }
         }

tests/namespaced.rs

@@ -1430,7 +1430,7 @@ mod namespaced {
         // Use the actual metrics address ztunnel is listening on (e.g., [::]:15020)
         // but combine it with the node IP for the client to target.
         let target_metrics_addr = SocketAddr::new(ztunnel_node_ip, zt.metrics_address.port());
-        let target_metrics_url = format!("http://{target_metrics_addr}/metrics");
+        let target_metrics_url = format!("http://{}/metrics", target_metrics_addr);
 
         // Deploy a client workload (simulating Prometheus)
         let client = manager
@@ -1469,7 +1469,8 @@ mod namespaced {
 
                 assert!(
                     response_str.contains("# TYPE"),
-                    "Expected Prometheus metrics (# TYPE) in response, got:\n{response_str}",
+                    "Expected Prometheus metrics (# TYPE) in response, got:\n{}",
+                    response_str
                 );
                 info!("Successfully verified metrics response body");
 
@@ -1486,8 +1487,8 @@ mod namespaced {
         verify_metrics(&zt, &metrics, &destination_labels()).await;
 
         // Verify INBOUND telemetry log for the metrics connection
-        let dst_addr_log = format!("{ztunnel_node_ip}:15008");
+        let dst_addr_log = format!("{}:15008", ztunnel_node_ip);
-        let dst_hbone_addr_log = format!("{target_metrics_addr}");
+        let dst_hbone_addr_log = format!("{}", target_metrics_addr);
 
         // We don't know exact byte counts, so omit them from the check for now
         let want = HashMap::from([