diff --git a/xds/src/main/java/io/grpc/xds/internal/sds/ServerWrapperForXds.java b/xds/src/main/java/io/grpc/xds/internal/sds/ServerWrapperForXds.java
index 371557537b..3500bd7bee 100644
--- a/xds/src/main/java/io/grpc/xds/internal/sds/ServerWrapperForXds.java
+++ b/xds/src/main/java/io/grpc/xds/internal/sds/ServerWrapperForXds.java
@@ -56,7 +56,7 @@ public final class ServerWrapperForXds extends Server {
   private XdsServerBuilder.XdsServingStatusListener xdsServingStatusListener;
   @Nullable XdsClientWrapperForServerSds.ServerWatcher serverWatcher;
   private AtomicBoolean started = new AtomicBoolean();
-  private ServingState currentServingState;
+  private volatile ServingState currentServingState;
   private final long delayForRetry;
   private final TimeUnit timeUnitForDelayForRetry;
   private StartRetryTask startRetryTask;
@@ -239,11 +239,13 @@ public final class ServerWrapperForXds extends Server {
       rebuildAndRestartServer();
     }
 
-    private synchronized void cleanUpStartRetryTask() {
-      if (timerService != null) {
-        timerService = SharedResourceHolder.release(GrpcUtil.TIMER_SERVICE, timerService);
+    private void cleanUpStartRetryTask() {
+      synchronized (ServerWrapperForXds.this) {
+        if (timerService != null) {
+          timerService = SharedResourceHolder.release(GrpcUtil.TIMER_SERVICE, timerService);
+        }
+        startRetryTask = null;
       }
-      startRetryTask = null;
     }
 
     public void shutdownNow() {
@@ -290,7 +292,7 @@ public final class ServerWrapperForXds extends Server {
 
   private void cleanupStartRetryTaskAndShutdownDelegateAndXdsClient(boolean shutdownNow) {
     Server delegateCopy = null;
-    synchronized (this) {
+    synchronized (ServerWrapperForXds.this) {
       if (startRetryTask != null) {
         startRetryTask.shutdownNow();
       }
diff --git a/xds/src/test/java/io/grpc/xds/ServerWrapperForXdsTest.java b/xds/src/test/java/io/grpc/xds/ServerWrapperForXdsTest.java
index 1c84a1a071..31ab3aed48 100644
--- a/xds/src/test/java/io/grpc/xds/ServerWrapperForXdsTest.java
+++ b/xds/src/test/java/io/grpc/xds/ServerWrapperForXdsTest.java
@@ -99,7 +99,12 @@ public class ServerWrapperForXdsTest {
       }
     });
     // wait until xdsClientWrapperForServerSds.serverWatchers populated
-    for (int i = 0; i < 10 && xdsClientWrapperForServerSds.serverWatchers.isEmpty(); i++) {
+    for (int i = 0; i < 10; i++) {
+      synchronized (xdsClientWrapperForServerSds.serverWatchers) {
+        if (!xdsClientWrapperForServerSds.serverWatchers.isEmpty()) {
+          break;
+        }
+      }
       Thread.sleep(100L);
     }
     return settableFuture;