xds: explicitly set request hash key for the ring hash LB policy

Implements [gRFC A76: explicitly setting the request hash key for the
ring hash LB policy][A76]
* Explictly setting the request hash key is guarded by the
  `GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY` environment
  variable until API stabilized. 

Tested:
* Verified end-to-end by spinning up multiple gRPC servers and a gRPC
  client that injects a custom service (load balancing) config with
  `ring_hash_experimental` and a custom `request_hash_header` (with
  NO associated value in the metadata headers) which generates a random
  hash for each request to the ring hash LB. Verified picks/RPCs are
  split evenly/uniformly across all backends.
* Ran affected unit tests with thread sanitizer and 1000 iterations to
  prevent data races.

[A76]: https://github.com/grpc/proposal/blob/master/A76-ring-hash-improvements.md#explicitly-setting-the-request-hash-key

xds/src/main/java/io/grpc/xds/RingHashLoadBalancer.java

@@ -25,6 +25,8 @@ import static io.grpc.ConnectivityState.READY;
 import static io.grpc.ConnectivityState.SHUTDOWN;
 import static io.grpc.ConnectivityState.TRANSIENT_FAILURE;
 
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Joiner;
 import com.google.common.base.MoreObjects;
 import com.google.common.collect.HashMultiset;
 import com.google.common.collect.Multiset;
@@ -34,9 +36,11 @@ import io.grpc.ConnectivityState;
 import io.grpc.EquivalentAddressGroup;
 import io.grpc.InternalLogId;
 import io.grpc.LoadBalancer;
+import io.grpc.Metadata;
 import io.grpc.Status;
 import io.grpc.SynchronizationContext;
 import io.grpc.util.MultiChildLoadBalancer;
+import io.grpc.xds.ThreadSafeRandom.ThreadSafeRandomImpl;
 import io.grpc.xds.client.XdsLogger;
 import io.grpc.xds.client.XdsLogger.XdsLogLevel;
 import java.net.SocketAddress;
@@ -69,13 +73,21 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
       new LazyLoadBalancer.Factory(pickFirstLbProvider);
   private final XdsLogger logger;
   private final SynchronizationContext syncContext;
+  private final ThreadSafeRandom random;
   private List<RingEntry> ring;
+  @Nullable private Metadata.Key<String> requestHashHeaderKey;
 
   RingHashLoadBalancer(Helper helper) {
+    this(helper, ThreadSafeRandomImpl.instance);
+  }
+
+  @VisibleForTesting
+  RingHashLoadBalancer(Helper helper, ThreadSafeRandom random) {
     super(helper);
     syncContext = checkNotNull(helper.getSynchronizationContext(), "syncContext");
     logger = XdsLogger.withLogId(InternalLogId.allocate("ring_hash_lb", helper.getAuthority()));
     logger.log(XdsLogLevel.INFO, "Created");
+    this.random = checkNotNull(random, "random");
   }
 
   @Override
@@ -92,6 +104,10 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
     if (config == null) {
       throw new IllegalArgumentException("Missing RingHash configuration");
     }
+    requestHashHeaderKey =
+        config.requestHashHeader.isEmpty()
+            ? null
+            : Metadata.Key.of(config.requestHashHeader, Metadata.ASCII_STRING_MARSHALLER);
     Map<EquivalentAddressGroup, Long> serverWeights = new HashMap<>();
     long totalWeight = 0L;
     for (EquivalentAddressGroup eag : addrList) {
@@ -197,7 +213,8 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
       overallState = TRANSIENT_FAILURE;
     }
 
-    RingHashPicker picker = new RingHashPicker(syncContext, ring, getChildLbStates());
+    RingHashPicker picker =
+        new RingHashPicker(syncContext, ring, getChildLbStates(), requestHashHeaderKey, random);
     getHelper().updateBalancingState(overallState, picker);
     this.currentConnectivityState = overallState;
   }
@@ -325,21 +342,32 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
     // TODO(chengyuanzhang): can be more performance-friendly with
     //  IdentityHashMap<Subchannel, ConnectivityStateInfo> and RingEntry contains Subchannel.
     private final Map<Endpoint, SubchannelView> pickableSubchannels;  // read-only
+    @Nullable private final Metadata.Key<String> requestHashHeaderKey;
+    private final ThreadSafeRandom random;
+    private final boolean hasEndpointInConnectingState;
 
     private RingHashPicker(
         SynchronizationContext syncContext, List<RingEntry> ring,
-        Collection<ChildLbState> children) {
+        Collection<ChildLbState> children, Metadata.Key<String> requestHashHeaderKey,
+        ThreadSafeRandom random) {
       this.syncContext = syncContext;
       this.ring = ring;
+      this.requestHashHeaderKey = requestHashHeaderKey;
+      this.random = random;
       pickableSubchannels = new HashMap<>(children.size());
+      boolean hasConnectingState = false;
       for (ChildLbState childLbState : children) {
         pickableSubchannels.put((Endpoint)childLbState.getKey(),
             new SubchannelView(childLbState, childLbState.getCurrentState()));
+        if (childLbState.getCurrentState() == CONNECTING) {
+          hasConnectingState = true;
+        }
       }
+      this.hasEndpointInConnectingState = hasConnectingState;
     }
 
     // Find the ring entry with hash next to (clockwise) the RPC's hash (binary search).
-    private int getTargetIndex(Long requestHash) {
+    private int getTargetIndex(long requestHash) {
       if (ring.size() <= 1) {
         return 0;
       }
@@ -365,39 +393,78 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
 
     @Override
     public PickResult pickSubchannel(PickSubchannelArgs args) {
-      Long requestHash = args.getCallOptions().getOption(XdsNameResolver.RPC_HASH_KEY);
+      // Determine request hash.
-      if (requestHash == null) {
+      boolean usingRandomHash = false;
-        return PickResult.withError(RPC_HASH_NOT_FOUND);
+      long requestHash;
+      if (requestHashHeaderKey == null) {
+        // Set by the xDS config selector.
+        Long rpcHashFromCallOptions = args.getCallOptions().getOption(XdsNameResolver.RPC_HASH_KEY);
+        if (rpcHashFromCallOptions == null) {
+          return PickResult.withError(RPC_HASH_NOT_FOUND);
+        }
+        requestHash = rpcHashFromCallOptions;
+      } else {
+        Iterable<String> headerValues = args.getHeaders().getAll(requestHashHeaderKey);
+        if (headerValues != null) {
+          requestHash = hashFunc.hashAsciiString(Joiner.on(",").join(headerValues));
+        } else {
+          requestHash = random.nextLong();
+          usingRandomHash = true;
+        }
       }
 
       int targetIndex = getTargetIndex(requestHash);
 
-      // Per gRFC A61, because of sticky-TF with PickFirst's auto reconnect on TF, we ignore
+      if (!usingRandomHash) {
-      // all TF subchannels and find the first ring entry in READY, CONNECTING or IDLE.  If
+        // Per gRFC A61, because of sticky-TF with PickFirst's auto reconnect on TF, we ignore
-      // CONNECTING or IDLE we return a pick with no results.  Additionally, if that entry is in
+        // all TF subchannels and find the first ring entry in READY, CONNECTING or IDLE.  If
-      // IDLE, we initiate a connection.
+        // CONNECTING or IDLE we return a pick with no results.  Additionally, if that entry is in
-      for (int i = 0; i < ring.size(); i++) {
+        // IDLE, we initiate a connection.
-        int index = (targetIndex + i) % ring.size();
+        for (int i = 0; i < ring.size(); i++) {
-        SubchannelView subchannelView = pickableSubchannels.get(ring.get(index).addrKey);
+          int index = (targetIndex + i) % ring.size();
-        ChildLbState childLbState = subchannelView.childLbState;
+          SubchannelView subchannelView = pickableSubchannels.get(ring.get(index).addrKey);
+          ChildLbState childLbState = subchannelView.childLbState;
 
-        if (subchannelView.connectivityState  == READY) {
+          if (subchannelView.connectivityState  == READY) {
-          return childLbState.getCurrentPicker().pickSubchannel(args);
+            return childLbState.getCurrentPicker().pickSubchannel(args);
+          }
+
+          // RPCs can be buffered if the next subchannel is pending (per A62). Otherwise, RPCs
+          // are failed unless there is a READY connection.
+          if (subchannelView.connectivityState == CONNECTING) {
+            return PickResult.withNoResult();
+          }
+
+          if (subchannelView.connectivityState == IDLE) {
+            syncContext.execute(() -> {
+              childLbState.getLb().requestConnection();
+            });
+
+            return PickResult.withNoResult(); // Indicates that this should be retried after backoff
+          }
         }
-
+      } else {
-        // RPCs can be buffered if the next subchannel is pending (per A62). Otherwise, RPCs
+        // Using a random hash. Find and use the first READY ring entry, triggering at most one
-        // are failed unless there is a READY connection.
+        // entry to attempt connection.
-        if (subchannelView.connectivityState == CONNECTING) {
+        boolean requestedConnection = hasEndpointInConnectingState;
+        for (int i = 0; i < ring.size(); i++) {
+          int index = (targetIndex + i) % ring.size();
+          SubchannelView subchannelView = pickableSubchannels.get(ring.get(index).addrKey);
+          ChildLbState childLbState = subchannelView.childLbState;
+          if (subchannelView.connectivityState == READY) {
+            return childLbState.getCurrentPicker().pickSubchannel(args);
+          }
+          if (!requestedConnection && subchannelView.connectivityState == IDLE) {
+            syncContext.execute(
+                () -> {
+                  childLbState.getLb().requestConnection();
+                });
+            requestedConnection = true;
+          }
+        }
+        if (requestedConnection) {
           return PickResult.withNoResult();
         }
-
-        if (subchannelView.connectivityState == IDLE) {
-          syncContext.execute(() -> {
-            childLbState.getLb().requestConnection();
-          });
-
-          return PickResult.withNoResult(); // Indicates that this should be retried after backoff
-        }
       }
 
       // return the pick from the original subchannel hit by hash, which is probably an error
@@ -444,13 +511,16 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
   static final class RingHashConfig {
     final long minRingSize;
     final long maxRingSize;
+    final String requestHashHeader;
 
-    RingHashConfig(long minRingSize, long maxRingSize) {
+    RingHashConfig(long minRingSize, long maxRingSize, String requestHashHeader) {
       checkArgument(minRingSize > 0, "minRingSize <= 0");
       checkArgument(maxRingSize > 0, "maxRingSize <= 0");
       checkArgument(minRingSize <= maxRingSize, "minRingSize > maxRingSize");
+      checkNotNull(requestHashHeader);
       this.minRingSize = minRingSize;
       this.maxRingSize = maxRingSize;
+      this.requestHashHeader = requestHashHeader;
     }
 
     @Override
@@ -458,6 +528,7 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
       return MoreObjects.toStringHelper(this)
           .add("minRingSize", minRingSize)
           .add("maxRingSize", maxRingSize)
+          .add("requestHashHeader", requestHashHeader)
           .toString();
     }
   }

xds/src/main/java/io/grpc/xds/RingHashLoadBalancerProvider.java

@@ -24,6 +24,7 @@ import io.grpc.LoadBalancer.Helper;
 import io.grpc.LoadBalancerProvider;
 import io.grpc.NameResolver.ConfigOrError;
 import io.grpc.Status;
+import io.grpc.internal.GrpcUtil;
 import io.grpc.internal.JsonUtil;
 import io.grpc.xds.RingHashLoadBalancer.RingHashConfig;
 import io.grpc.xds.RingHashOptions;
@@ -81,6 +82,10 @@ public final class RingHashLoadBalancerProvider extends LoadBalancerProvider {
       Map<String, ?> rawLoadBalancingPolicyConfig) {
     Long minRingSize = JsonUtil.getNumberAsLong(rawLoadBalancingPolicyConfig, "minRingSize");
     Long maxRingSize = JsonUtil.getNumberAsLong(rawLoadBalancingPolicyConfig, "maxRingSize");
+    String requestHashHeader = "";
+    if (GrpcUtil.getFlag("GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY", false)) {
+      requestHashHeader = JsonUtil.getString(rawLoadBalancingPolicyConfig, "requestHashHeader");
+    }
     long maxRingSizeCap = RingHashOptions.getRingSizeCap();
     if (minRingSize == null) {
       minRingSize = DEFAULT_MIN_RING_SIZE;
@@ -88,6 +93,9 @@ public final class RingHashLoadBalancerProvider extends LoadBalancerProvider {
     if (maxRingSize == null) {
       maxRingSize = DEFAULT_MAX_RING_SIZE;
     }
+    if (requestHashHeader == null) {
+      requestHashHeader = "";
+    }
     if (minRingSize > maxRingSizeCap) {
       minRingSize = maxRingSizeCap;
     }
@@ -98,6 +106,7 @@ public final class RingHashLoadBalancerProvider extends LoadBalancerProvider {
       return ConfigOrError.fromError(Status.UNAVAILABLE.withDescription(
           "Invalid 'mingRingSize'/'maxRingSize'"));
     }
-    return ConfigOrError.fromConfig(new RingHashConfig(minRingSize, maxRingSize));
+    return ConfigOrError.fromConfig(
+        new RingHashConfig(minRingSize, maxRingSize, requestHashHeader));
   }
 }

xds/src/test/java/io/grpc/xds/ClusterResolverLoadBalancerTest.java

@@ -163,7 +163,7 @@ public class ClusterResolverLoadBalancerTest {
           GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig(
               new FakeLoadBalancerProvider("round_robin"), null)));
   private final Object ringHash = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig(
-      new FakeLoadBalancerProvider("ring_hash_experimental"), new RingHashConfig(10L, 100L));
+      new FakeLoadBalancerProvider("ring_hash_experimental"), new RingHashConfig(10L, 100L, ""));
   private final Object leastRequest = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig(
       new FakeLoadBalancerProvider("wrr_locality_experimental"), new WrrLocalityConfig(
           GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig(

xds/src/test/java/io/grpc/xds/RingHashLoadBalancerProviderTest.java

@@ -42,6 +42,8 @@ import org.junit.runners.JUnit4;
 @RunWith(JUnit4.class)
 public class RingHashLoadBalancerProviderTest {
   private static final String AUTHORITY = "foo.googleapis.com";
+  private static final String GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY =
+      "GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY";
 
   private final SynchronizationContext syncContext = new SynchronizationContext(
       new UncaughtExceptionHandler() {
@@ -81,6 +83,7 @@ public class RingHashLoadBalancerProviderTest {
     RingHashConfig config = (RingHashConfig) configOrError.getConfig();
     assertThat(config.minRingSize).isEqualTo(10L);
     assertThat(config.maxRingSize).isEqualTo(100L);
+    assertThat(config.requestHashHeader).isEmpty();
   }
 
   @Test
@@ -92,6 +95,7 @@ public class RingHashLoadBalancerProviderTest {
     RingHashConfig config = (RingHashConfig) configOrError.getConfig();
     assertThat(config.minRingSize).isEqualTo(RingHashLoadBalancerProvider.DEFAULT_MIN_RING_SIZE);
     assertThat(config.maxRingSize).isEqualTo(RingHashLoadBalancerProvider.DEFAULT_MAX_RING_SIZE);
+    assertThat(config.requestHashHeader).isEmpty();
   }
 
   @Test
@@ -127,6 +131,7 @@ public class RingHashLoadBalancerProviderTest {
     RingHashConfig config = (RingHashConfig) configOrError.getConfig();
     assertThat(config.minRingSize).isEqualTo(10);
     assertThat(config.maxRingSize).isEqualTo(RingHashOptions.DEFAULT_RING_SIZE_CAP);
+    assertThat(config.requestHashHeader).isEmpty();
   }
 
   @Test
@@ -142,6 +147,7 @@ public class RingHashLoadBalancerProviderTest {
     RingHashConfig config = (RingHashConfig) configOrError.getConfig();
     assertThat(config.minRingSize).isEqualTo(RingHashOptions.MAX_RING_SIZE_CAP);
     assertThat(config.maxRingSize).isEqualTo(RingHashOptions.MAX_RING_SIZE_CAP);
+    assertThat(config.requestHashHeader).isEmpty();
     // Reset to avoid affecting subsequent test cases
     RingHashOptions.setRingSizeCap(RingHashOptions.DEFAULT_RING_SIZE_CAP);
   }
@@ -159,6 +165,7 @@ public class RingHashLoadBalancerProviderTest {
     RingHashConfig config = (RingHashConfig) configOrError.getConfig();
     assertThat(config.minRingSize).isEqualTo(RingHashOptions.MAX_RING_SIZE_CAP);
     assertThat(config.maxRingSize).isEqualTo(RingHashOptions.MAX_RING_SIZE_CAP);
+    assertThat(config.requestHashHeader).isEmpty();
     // Reset to avoid affecting subsequent test cases
     RingHashOptions.setRingSizeCap(RingHashOptions.DEFAULT_RING_SIZE_CAP);
   }
@@ -176,6 +183,7 @@ public class RingHashLoadBalancerProviderTest {
     RingHashConfig config = (RingHashConfig) configOrError.getConfig();
     assertThat(config.minRingSize).isEqualTo(1);
     assertThat(config.maxRingSize).isEqualTo(1);
+    assertThat(config.requestHashHeader).isEmpty();
     // Reset to avoid affecting subsequent test cases
     RingHashOptions.setRingSizeCap(RingHashOptions.DEFAULT_RING_SIZE_CAP);
   }
@@ -193,6 +201,7 @@ public class RingHashLoadBalancerProviderTest {
     RingHashConfig config = (RingHashConfig) configOrError.getConfig();
     assertThat(config.minRingSize).isEqualTo(1);
     assertThat(config.maxRingSize).isEqualTo(1);
+    assertThat(config.requestHashHeader).isEmpty();
     // Reset to avoid affecting subsequent test cases
     RingHashOptions.setRingSizeCap(RingHashOptions.DEFAULT_RING_SIZE_CAP);
   }
@@ -219,6 +228,59 @@ public class RingHashLoadBalancerProviderTest {
         .isEqualTo("Invalid 'mingRingSize'/'maxRingSize'");
   }
 
+  @Test
+  public void parseLoadBalancingConfig_requestHashHeaderIgnoredWhenEnvVarNotSet()
+      throws IOException {
+    String lbConfig =
+        "{\"minRingSize\" : 10, \"maxRingSize\" : 100, \"requestHashHeader\" : \"dummy-hash\"}";
+    ConfigOrError configOrError =
+        provider.parseLoadBalancingPolicyConfig(parseJsonObject(lbConfig));
+    assertThat(configOrError.getConfig()).isNotNull();
+    RingHashConfig config = (RingHashConfig) configOrError.getConfig();
+    assertThat(config.minRingSize).isEqualTo(10L);
+    assertThat(config.maxRingSize).isEqualTo(100L);
+    assertThat(config.requestHashHeader).isEmpty();
+  }
+
+  @Test
+  public void parseLoadBalancingConfig_requestHashHeaderSetWhenEnvVarSet() throws IOException {
+    System.setProperty(GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY, "true");
+    try {
+      String lbConfig =
+          "{\"minRingSize\" : 10, \"maxRingSize\" : 100, \"requestHashHeader\" : \"dummy-hash\"}";
+      ConfigOrError configOrError =
+          provider.parseLoadBalancingPolicyConfig(parseJsonObject(lbConfig));
+      assertThat(configOrError.getConfig()).isNotNull();
+      RingHashConfig config = (RingHashConfig) configOrError.getConfig();
+      assertThat(config.minRingSize).isEqualTo(10L);
+      assertThat(config.maxRingSize).isEqualTo(100L);
+      assertThat(config.requestHashHeader).isEqualTo("dummy-hash");
+      assertThat(config.toString()).contains("minRingSize=10");
+      assertThat(config.toString()).contains("maxRingSize=100");
+      assertThat(config.toString()).contains("requestHashHeader=dummy-hash");
+    } finally {
+      System.clearProperty(GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY);
+    }
+  }
+
+  @Test
+  public void parseLoadBalancingConfig_requestHashHeaderUnsetWhenEnvVarSet_useDefaults()
+      throws IOException {
+    System.setProperty(GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY, "true");
+    try {
+      String lbConfig = "{\"minRingSize\" : 10, \"maxRingSize\" : 100}";
+      ConfigOrError configOrError =
+          provider.parseLoadBalancingPolicyConfig(parseJsonObject(lbConfig));
+      assertThat(configOrError.getConfig()).isNotNull();
+      RingHashConfig config = (RingHashConfig) configOrError.getConfig();
+      assertThat(config.minRingSize).isEqualTo(10L);
+      assertThat(config.maxRingSize).isEqualTo(100L);
+      assertThat(config.requestHashHeader).isEmpty();
+    } finally {
+      System.clearProperty(GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY);
+    }
+  }
+
   @SuppressWarnings("unchecked")
   private static Map<String, ?> parseJsonObject(String json) throws IOException {
     return (Map<String, ?>) JsonParser.parse(json);

xds/src/test/java/io/grpc/xds/RingHashLoadBalancerTest.java

@@ -98,6 +98,9 @@ import org.mockito.junit.MockitoRule;
 @RunWith(JUnit4.class)
 public class RingHashLoadBalancerTest {
   private static final String AUTHORITY = "foo.googleapis.com";
+  private static final String CUSTOM_REQUEST_HASH_HEADER = "custom-request-hash-header";
+  private static final Metadata.Key<String> CUSTOM_METADATA_KEY =
+      Metadata.Key.of(CUSTOM_REQUEST_HASH_HEADER, Metadata.ASCII_STRING_MARSHALLER);
   private static final Attributes.Key<String> CUSTOM_KEY = Attributes.Key.create("custom-key");
   private static final ConnectivityStateInfo CSI_CONNECTING =
       ConnectivityStateInfo.forNonError(CONNECTING);
@@ -142,7 +145,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void subchannelLazyConnectUntilPicked() {
-    RingHashConfig config = new RingHashConfig(10, 100);
+    RingHashConfig config = new RingHashConfig(10, 100, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1);  // one server
     Status addressesAcceptanceStatus = loadBalancer.acceptResolvedAddresses(
         ResolvedAddresses.newBuilder()
@@ -176,7 +179,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void subchannelNotAutoReconnectAfterReenteringIdle() {
-    RingHashConfig config = new RingHashConfig(10, 100);
+    RingHashConfig config = new RingHashConfig(10, 100, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1);  // one server
     Status addressesAcceptanceStatus = loadBalancer.acceptResolvedAddresses(
         ResolvedAddresses.newBuilder()
@@ -207,7 +210,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void aggregateSubchannelStates_connectingReadyIdleFailure() {
-    RingHashConfig config = new RingHashConfig(10, 100);
+    RingHashConfig config = new RingHashConfig(10, 100, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1);
     InOrder inOrder = Mockito.inOrder(helper);
 
@@ -266,7 +269,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void aggregateSubchannelStates_allSubchannelsInTransientFailure() {
-    RingHashConfig config = new RingHashConfig(10, 100);
+    RingHashConfig config = new RingHashConfig(10, 100, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1, 1);
 
     List<Subchannel> subChannelList = initializeLbSubchannels(config, servers, STAY_IN_CONNECTING);
@@ -324,7 +327,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void ignoreShutdownSubchannelStateChange() {
-    RingHashConfig config = new RingHashConfig(10, 100);
+    RingHashConfig config = new RingHashConfig(10, 100, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
     initializeLbSubchannels(config, servers);
 
@@ -340,7 +343,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void deterministicPickWithHostsPartiallyRemoved() {
-    RingHashConfig config = new RingHashConfig(10, 100);
+    RingHashConfig config = new RingHashConfig(10, 100, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1, 1, 1);
     initializeLbSubchannels(config, servers);
     InOrder inOrder = Mockito.inOrder(helper);
@@ -380,7 +383,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void deterministicPickWithNewHostsAdded() {
-    RingHashConfig config = new RingHashConfig(10, 100);
+    RingHashConfig config = new RingHashConfig(10, 100, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1);  // server0 and server1
     initializeLbSubchannels(config, servers, DO_NOT_VERIFY, DO_NOT_RESET_HELPER);
 
@@ -412,6 +415,139 @@ public class RingHashLoadBalancerTest {
     inOrder.verifyNoMoreInteractions();
   }
 
+  @Test
+  public void deterministicPickWithRequestHashHeader_oneHeaderValue() {
+    // Map each server address to exactly one ring entry.
+    RingHashConfig config = new RingHashConfig(3, 3, CUSTOM_REQUEST_HASH_HEADER);
+    List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
+    initializeLbSubchannels(config, servers);
+    InOrder inOrder = Mockito.inOrder(helper);
+
+    // Bring all subchannels to READY.
+    for (Subchannel subchannel : subchannels.values()) {
+      deliverSubchannelState(subchannel, CSI_READY);
+      inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
+    }
+
+    // Pick subchannel with custom request hash header where the rpc hash hits server1.
+    Metadata headers = new Metadata();
+    headers.put(CUSTOM_METADATA_KEY, "FakeSocketAddress-server1_0");
+    PickSubchannelArgs args =
+        new PickSubchannelArgsImpl(
+            TestMethodDescriptors.voidMethod(),
+            headers,
+            CallOptions.DEFAULT,
+            new PickDetailsConsumer() {});
+    SubchannelPicker picker = pickerCaptor.getValue();
+    PickResult result = picker.pickSubchannel(args);
+    assertThat(result.getStatus().isOk()).isTrue();
+    assertThat(result.getSubchannel().getAddresses()).isEqualTo(servers.get(1));
+  }
+
+  @Test
+  public void deterministicPickWithRequestHashHeader_multipleHeaderValues() {
+    // Map each server address to exactly one ring entry.
+    RingHashConfig config = new RingHashConfig(3, 3, CUSTOM_REQUEST_HASH_HEADER);
+    List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
+    initializeLbSubchannels(config, servers);
+    InOrder inOrder = Mockito.inOrder(helper);
+
+    // Bring all subchannels to READY.
+    for (Subchannel subchannel : subchannels.values()) {
+      deliverSubchannelState(subchannel, CSI_READY);
+      inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
+    }
+
+    // Pick subchannel with custom request hash header with multiple values for the same key where
+    // the rpc hash hits server1.
+    Metadata headers = new Metadata();
+    headers.put(CUSTOM_METADATA_KEY, "FakeSocketAddress-server0_0");
+    headers.put(CUSTOM_METADATA_KEY, "FakeSocketAddress-server1_0");
+    PickSubchannelArgs args =
+        new PickSubchannelArgsImpl(
+            TestMethodDescriptors.voidMethod(),
+            headers,
+            CallOptions.DEFAULT,
+            new PickDetailsConsumer() {});
+    SubchannelPicker picker = pickerCaptor.getValue();
+    PickResult result = picker.pickSubchannel(args);
+    assertThat(result.getStatus().isOk()).isTrue();
+    assertThat(result.getSubchannel().getAddresses()).isEqualTo(servers.get(1));
+  }
+
+  @Test
+  public void pickWithRandomHash_allSubchannelsReady() {
+    loadBalancer = new RingHashLoadBalancer(helper, new FakeRandom());
+    // Map each server address to exactly one ring entry.
+    RingHashConfig config = new RingHashConfig(2, 2, "dummy-random-hash");
+    List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1);
+    initializeLbSubchannels(config, servers);
+    InOrder inOrder = Mockito.inOrder(helper);
+
+    // Bring all subchannels to READY.
+    Map<EquivalentAddressGroup, Integer> pickCounts = new HashMap<>();
+    for (Subchannel subchannel : subchannels.values()) {
+      deliverSubchannelState(subchannel, CSI_READY);
+      pickCounts.put(subchannel.getAddresses(), 0);
+      inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
+    }
+
+    // Pick subchannel 100 times with random hash.
+    SubchannelPicker picker = pickerCaptor.getValue();
+    PickSubchannelArgs args = getDefaultPickSubchannelArgs(hashFunc.hashVoid());
+    for (int i = 0; i < 100; ++i) {
+      Subchannel pickedSubchannel = picker.pickSubchannel(args).getSubchannel();
+      EquivalentAddressGroup addr = pickedSubchannel.getAddresses();
+      pickCounts.put(addr, pickCounts.get(addr) + 1);
+    }
+
+    // Verify the distribution is uniform where server0 and server1 are exactly picked 50 times.
+    assertThat(pickCounts.get(servers.get(0))).isEqualTo(50);
+    assertThat(pickCounts.get(servers.get(1))).isEqualTo(50);
+  }
+
+  @Test
+  public void pickWithRandomHash_atLeastOneSubchannelConnecting() {
+    // Map each server address to exactly one ring entry.
+    RingHashConfig config = new RingHashConfig(3, 3, "dummy-random-hash");
+    List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
+    initializeLbSubchannels(config, servers);
+
+    // Bring one subchannel to CONNECTING.
+    deliverSubchannelState(getSubChannel(servers.get(0)), CSI_CONNECTING);
+    verify(helper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture());
+
+    // Pick subchannel with random hash does not trigger connection.
+    SubchannelPicker picker = pickerCaptor.getValue();
+    PickSubchannelArgs args = getDefaultPickSubchannelArgs(hashFunc.hashVoid());
+    PickResult result = picker.pickSubchannel(args);
+    assertThat(result.getStatus().isOk()).isTrue();
+    assertThat(result.getSubchannel()).isNull(); // buffer request
+    verifyConnection(0);
+  }
+
+  @Test
+  public void pickWithRandomHash_firstSubchannelInTransientFailure_remainingSubchannelsIdle() {
+    // Map each server address to exactly one ring entry.
+    RingHashConfig config = new RingHashConfig(3, 3, "dummy-random-hash");
+    List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
+    initializeLbSubchannels(config, servers);
+
+    // Bring one subchannel to TRANSIENT_FAILURE.
+    deliverSubchannelUnreachable(getSubChannel(servers.get(0)));
+    verify(helper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture());
+    verifyConnection(0);
+
+    // Pick subchannel with random hash does trigger connection by walking the ring
+    // and choosing the first (at most one) IDLE subchannel along the way.
+    SubchannelPicker picker = pickerCaptor.getValue();
+    PickSubchannelArgs args = getDefaultPickSubchannelArgs(hashFunc.hashVoid());
+    PickResult result = picker.pickSubchannel(args);
+    assertThat(result.getStatus().isOk()).isTrue();
+    assertThat(result.getSubchannel()).isNull(); // buffer request
+    verifyConnection(1);
+  }
+
   private Subchannel getSubChannel(EquivalentAddressGroup eag) {
     return subchannels.get(Collections.singletonList(eag));
   }
@@ -419,7 +555,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void skipFailingHosts_pickNextNonFailingHost() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
     Status addressesAcceptanceStatus =
         loadBalancer.acceptResolvedAddresses(
@@ -489,7 +625,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void skipFailingHosts_firstTwoHostsFailed_pickNextFirstReady() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
 
     initializeLbSubchannels(config, servers);
@@ -555,7 +691,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void removingAddressShutdownSubchannel() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> svs1 = createWeightedServerAddrs(1, 1, 1);
     List<Subchannel> subchannels1 = initializeLbSubchannels(config, svs1, STAY_IN_CONNECTING);
 
@@ -572,7 +708,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void allSubchannelsInTransientFailure() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
     initializeLbSubchannels(config, servers);
 
@@ -599,7 +735,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void firstSubchannelIdle() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
     initializeLbSubchannels(config, servers);
 
@@ -620,7 +756,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void firstSubchannelConnecting() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
     initializeLbSubchannels(config, servers);
 
@@ -644,7 +780,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void firstSubchannelFailure() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
 
     List<Subchannel> subchannelList =
@@ -675,7 +811,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void secondSubchannelConnecting() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
 
     initializeLbSubchannels(config, servers);
@@ -706,7 +842,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void secondSubchannelFailure() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
 
     initializeLbSubchannels(config, servers);
@@ -733,7 +869,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void thirdSubchannelConnecting() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
 
     initializeLbSubchannels(config, servers);
@@ -762,7 +898,7 @@ public class RingHashLoadBalancerTest {
   @Test
   public void stickyTransientFailure() {
     // Map each server address to exactly one ring entry.
-    RingHashConfig config = new RingHashConfig(3, 3);
+    RingHashConfig config = new RingHashConfig(3, 3, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
 
     initializeLbSubchannels(config, servers);
@@ -791,7 +927,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void largeWeights() {
-    RingHashConfig config = new RingHashConfig(10000, 100000);  // large ring
+    RingHashConfig config = new RingHashConfig(10000, 100000, "");  // large ring
     List<EquivalentAddressGroup> servers =
         createWeightedServerAddrs(Integer.MAX_VALUE, 10, 100); // MAX:10:100
 
@@ -829,7 +965,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void hostSelectionProportionalToWeights() {
-    RingHashConfig config = new RingHashConfig(10000, 100000);  // large ring
+    RingHashConfig config = new RingHashConfig(10000, 100000, "");  // large ring
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 10, 100); // 1:10:100
 
     initializeLbSubchannels(config, servers);
@@ -872,7 +1008,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void nameResolutionErrorWithActiveSubchannels() {
-    RingHashConfig config = new RingHashConfig(10, 100);
+    RingHashConfig config = new RingHashConfig(10, 100, "");
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1);
 
     initializeLbSubchannels(config, servers, DO_NOT_VERIFY, DO_NOT_RESET_HELPER);
@@ -894,7 +1030,7 @@ public class RingHashLoadBalancerTest {
 
   @Test
   public void duplicateAddresses() {
-    RingHashConfig config = new RingHashConfig(10, 100);
+    RingHashConfig config = new RingHashConfig(10, 100, "");
     List<EquivalentAddressGroup> servers = createRepeatedServerAddrs(1, 2, 3);
 
     initializeLbSubchannels(config, servers, DO_NOT_VERIFY);
@@ -940,7 +1076,7 @@ public class RingHashLoadBalancerTest {
 
     InOrder inOrder = Mockito.inOrder(helper);
     List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1);
-    initializeLbSubchannels(new RingHashConfig(10, 100), servers);
+    initializeLbSubchannels(new RingHashConfig(10, 100, ""), servers);
     Subchannel subchannel0 = subchannels.get(Collections.singletonList(servers.get(0)));
     Subchannel subchannel1 = subchannels.get(Collections.singletonList(servers.get(1)));
 
@@ -1167,6 +1303,30 @@ public class RingHashLoadBalancerTest {
     }
   }
 
+  private static final class FakeRandom implements ThreadSafeRandom {
+    int counter = 0;
+
+    @Override
+    public int nextInt(int bound) {
+      throw new UnsupportedOperationException("Should not be called");
+    }
+
+    @Override
+    public long nextLong() {
+      ++counter;
+      if (counter % 2 == 0) {
+        return XxHash64.INSTANCE.hashAsciiString("FakeSocketAddress-server0_0");
+      } else {
+        return XxHash64.INSTANCE.hashAsciiString("FakeSocketAddress-server1_0");
+      }
+    }
+
+    @Override
+    public long nextLong(long bound) {
+      throw new UnsupportedOperationException("Should not be called");
+    }
+  }
+
   enum InitializationFlags {
     DO_NOT_VERIFY,
     RESET_SUBCHANNEL_MOCKS,