diff --git a/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java b/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java
index 047f8a2e31..9c7d744816 100644
--- a/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java
+++ b/xds/src/main/java/io/grpc/xds/client/ControlPlaneClient.java
@@ -446,6 +446,7 @@ final class ControlPlaneClient {
         // Reset the backoff sequence if had received a response, or backoff sequence
         // has never been initialized.
         retryBackoffPolicy = backoffPolicyProvider.get();
+        stopwatch.reset();
       }
 
       // FakeClock in tests isn't thread-safe. Schedule the retry timer before notifying callbacks
diff --git a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java
index f1c114673b..1b0c363d94 100644
--- a/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java
+++ b/xds/src/test/java/io/grpc/xds/GrpcXdsClientImplTestBase.java
@@ -3524,6 +3524,7 @@ public abstract class GrpcXdsClientImplTestBase {
     call.verifyRequest(EDS, EDS_RESOURCE, "", "", NODE);
 
     // Management server closes the RPC stream with an error.
+    fakeClock.forwardNanos(1000L); // Make sure retry isn't based on stopwatch 0
     call.sendError(Status.UNKNOWN.asException());
     verify(ldsResourceWatcher, Mockito.timeout(1000).times(1))
         .onError(errorCaptor.capture());