diff --git a/xds/src/main/java/io/grpc/xds/AbstractXdsClient.java b/xds/src/main/java/io/grpc/xds/AbstractXdsClient.java
index 61780c60a5..d559c05440 100644
--- a/xds/src/main/java/io/grpc/xds/AbstractXdsClient.java
+++ b/xds/src/main/java/io/grpc/xds/AbstractXdsClient.java
@@ -470,14 +470,9 @@ final class AbstractXdsClient {
         // has never been initialized.
         retryBackoffPolicy = backoffPolicyProvider.get();
       }
-      long delayNanos = 0;
-      if (!responseReceived) {
-        delayNanos =
-            Math.max(
-                0,
-                retryBackoffPolicy.nextBackoffNanos()
-                    - stopwatch.elapsed(TimeUnit.NANOSECONDS));
-      }
+      long delayNanos = Math.max(
+          0,
+          retryBackoffPolicy.nextBackoffNanos() - stopwatch.elapsed(TimeUnit.NANOSECONDS));
       logger.log(XdsLogLevel.INFO, "Retry ADS stream in {0} ns", delayNanos);
       rpcRetryTimer = syncContext.schedule(
           new RpcRetryTask(), delayNanos, TimeUnit.NANOSECONDS, timeService);
diff --git a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java
index 5cedcff17f..4fe067752d 100644
--- a/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java
+++ b/xds/src/test/java/io/grpc/xds/ClientXdsClientTestBase.java
@@ -2526,9 +2526,13 @@ public abstract class ClientXdsClientTestBase {
     verify(edsResourceWatcher, times(3)).onError(errorCaptor.capture());
     assertThat(errorCaptor.getValue().getCode()).isEqualTo(Code.DEADLINE_EXCEEDED);
 
-    // Reset backoff sequence and retry immediately.
+    // Reset backoff sequence and retry after backoff.
     inOrder.verify(backoffPolicyProvider).get();
-    fakeClock.runDueTasks();
+    inOrder.verify(backoffPolicy2).nextBackoffNanos();
+    retryTask =
+        Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER));
+    assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(20L);
+    fakeClock.forwardNanos(20L);
     call = resourceDiscoveryCalls.poll();
     call.verifyRequest(LDS, LDS_RESOURCE, "63", "", NODE);
     call.verifyRequest(RDS, RDS_RESOURCE, "5", "", NODE);
@@ -2550,8 +2554,8 @@ public abstract class ClientXdsClientTestBase {
     inOrder.verify(backoffPolicy2).nextBackoffNanos();
     retryTask =
         Iterables.getOnlyElement(fakeClock.getPendingTasks(RPC_RETRY_TASK_FILTER));
-    assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(20L);
-    fakeClock.forwardNanos(20L);
+    assertThat(retryTask.getDelay(TimeUnit.NANOSECONDS)).isEqualTo(200L);
+    fakeClock.forwardNanos(200L);
     call = resourceDiscoveryCalls.poll();
     call.verifyRequest(LDS, LDS_RESOURCE, "63", "", NODE);
     call.verifyRequest(RDS, RDS_RESOURCE, "5", "", NODE);