diff --git a/packages/grpc-js/src/load-balancer-pick-first.ts b/packages/grpc-js/src/load-balancer-pick-first.ts
index f3888618..c4f289a9 100644
--- a/packages/grpc-js/src/load-balancer-pick-first.ts
+++ b/packages/grpc-js/src/load-balancer-pick-first.ts
@@ -130,7 +130,6 @@ export class PickFirstLoadBalancer implements LoadBalancer {
    *     this load balancer's owner.
    */
   constructor(private channelControlHelper: ChannelControlHelper) {
-    this.updateState(ConnectivityState.IDLE, new QueuePicker(this));
     this.subchannelStateCounts = {
       [ConnectivityState.CONNECTING]: 0,
       [ConnectivityState.IDLE]: 0,
@@ -168,6 +167,8 @@ export class PickFirstLoadBalancer implements LoadBalancer {
            * basic IDLE state where there is no subchannel list to avoid
            * holding unused resources */
           this.resetSubchannelList();
+          this.updateState(ConnectivityState.IDLE, new QueuePicker(this));
+          return;
         }
         if (this.currentPick === null) {
           if (this.triedAllSubchannels) {
diff --git a/packages/grpc-js/src/load-balancer-round-robin.ts b/packages/grpc-js/src/load-balancer-round-robin.ts
index 51244341..9716d7f0 100644
--- a/packages/grpc-js/src/load-balancer-round-robin.ts
+++ b/packages/grpc-js/src/load-balancer-round-robin.ts
@@ -95,7 +95,6 @@ export class RoundRobinLoadBalancer implements LoadBalancer {
   private currentReadyPicker: RoundRobinPicker | null = null;
 
   constructor(private channelControlHelper: ChannelControlHelper) {
-    this.updateState(ConnectivityState.IDLE, new QueuePicker(this));
     this.subchannelStateCounts = {
       [ConnectivityState.CONNECTING]: 0,
       [ConnectivityState.IDLE]: 0,
diff --git a/packages/grpc-js/test/test-client.ts b/packages/grpc-js/test/test-client.ts
index da20c246..0d2878cb 100644
--- a/packages/grpc-js/test/test-client.ts
+++ b/packages/grpc-js/test/test-client.ts
@@ -69,3 +69,26 @@ describe('Client', () => {
     }, deadline - Date.now());
   });
 });
+
+describe('Client without a server', () => {
+  let client: Client;
+  before(() => {
+    // Arbitrary target that should not have a running server
+    client = new Client('localhost:12345', clientInsecureCreds);
+  });
+  after(() => {
+    client.close();
+  });
+  it('should fail multiple calls to the nonexistent server', done => {
+    // Regression test for https://github.com/grpc/grpc-node/issues/1411
+    client.makeUnaryRequest('/service/method', x => x, x => x, Buffer.from([]), (error, value) => {
+      assert(error);
+      assert.strictEqual(error?.code, grpc.status.UNAVAILABLE);
+      client.makeUnaryRequest('/service/method', x => x, x => x, Buffer.from([]), (error, value) => {
+        assert(error);
+        assert.strictEqual(error?.code, grpc.status.UNAVAILABLE);
+        done();
+      });
+    });
+  });
+});
\ No newline at end of file