mirror of https://github.com/grpc/grpc-java.git
xds: explicitly set request hash key for the ring hash LB policy
Implements [gRFC A76: explicitly setting the request hash key for the ring hash LB policy][A76] * Explictly setting the request hash key is guarded by the `GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY` environment variable until API stabilized. Tested: * Verified end-to-end by spinning up multiple gRPC servers and a gRPC client that injects a custom service (load balancing) config with `ring_hash_experimental` and a custom `request_hash_header` (with NO associated value in the metadata headers) which generates a random hash for each request to the ring hash LB. Verified picks/RPCs are split evenly/uniformly across all backends. * Ran affected unit tests with thread sanitizer and 1000 iterations to prevent data races. [A76]: https://github.com/grpc/proposal/blob/master/A76-ring-hash-improvements.md#explicitly-setting-the-request-hash-key
This commit is contained in:
Daniel Liu
GitHub
parent
68d79b5130
commit
892144dcac
|
|
@ -25,6 +25,8 @@ import static io.grpc.ConnectivityState.READY;
|
|||
import static io.grpc.ConnectivityState.SHUTDOWN;
|
||||
import static io.grpc.ConnectivityState.TRANSIENT_FAILURE;
|
||||
|
||||
import com.google.common.annotations.VisibleForTesting;
|
||||
import com.google.common.base.Joiner;
|
||||
import com.google.common.base.MoreObjects;
|
||||
import com.google.common.collect.HashMultiset;
|
||||
import com.google.common.collect.Multiset;
|
||||
|
|
@ -34,9 +36,11 @@ import io.grpc.ConnectivityState;
|
|||
import io.grpc.EquivalentAddressGroup;
|
||||
import io.grpc.InternalLogId;
|
||||
import io.grpc.LoadBalancer;
|
||||
import io.grpc.Metadata;
|
||||
import io.grpc.Status;
|
||||
import io.grpc.SynchronizationContext;
|
||||
import io.grpc.util.MultiChildLoadBalancer;
|
||||
import io.grpc.xds.ThreadSafeRandom.ThreadSafeRandomImpl;
|
||||
import io.grpc.xds.client.XdsLogger;
|
||||
import io.grpc.xds.client.XdsLogger.XdsLogLevel;
|
||||
import java.net.SocketAddress;
|
||||
|
|
@ -69,13 +73,21 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
|
|||
new LazyLoadBalancer.Factory(pickFirstLbProvider);
|
||||
private final XdsLogger logger;
|
||||
private final SynchronizationContext syncContext;
|
||||
private final ThreadSafeRandom random;
|
||||
private List<RingEntry> ring;
|
||||
@Nullable private Metadata.Key<String> requestHashHeaderKey;
|
||||
|
||||
RingHashLoadBalancer(Helper helper) {
|
||||
this(helper, ThreadSafeRandomImpl.instance);
|
||||
}
|
||||
|
||||
@VisibleForTesting
|
||||
RingHashLoadBalancer(Helper helper, ThreadSafeRandom random) {
|
||||
super(helper);
|
||||
syncContext = checkNotNull(helper.getSynchronizationContext(), "syncContext");
|
||||
logger = XdsLogger.withLogId(InternalLogId.allocate("ring_hash_lb", helper.getAuthority()));
|
||||
logger.log(XdsLogLevel.INFO, "Created");
|
||||
this.random = checkNotNull(random, "random");
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -92,6 +104,10 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
|
|||
if (config == null) {
|
||||
throw new IllegalArgumentException("Missing RingHash configuration");
|
||||
}
|
||||
requestHashHeaderKey =
|
||||
config.requestHashHeader.isEmpty()
|
||||
? null
|
||||
: Metadata.Key.of(config.requestHashHeader, Metadata.ASCII_STRING_MARSHALLER);
|
||||
Map<EquivalentAddressGroup, Long> serverWeights = new HashMap<>();
|
||||
long totalWeight = 0L;
|
||||
for (EquivalentAddressGroup eag : addrList) {
|
||||
|
|
@ -197,7 +213,8 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
|
|||
overallState = TRANSIENT_FAILURE;
|
||||
}
|
||||
|
||||
RingHashPicker picker = new RingHashPicker(syncContext, ring, getChildLbStates());
|
||||
RingHashPicker picker =
|
||||
new RingHashPicker(syncContext, ring, getChildLbStates(), requestHashHeaderKey, random);
|
||||
getHelper().updateBalancingState(overallState, picker);
|
||||
this.currentConnectivityState = overallState;
|
||||
}
|
||||
|
|
@ -325,21 +342,32 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
|
|||
// TODO(chengyuanzhang): can be more performance-friendly with
|
||||
// IdentityHashMap<Subchannel, ConnectivityStateInfo> and RingEntry contains Subchannel.
|
||||
private final Map<Endpoint, SubchannelView> pickableSubchannels; // read-only
|
||||
@Nullable private final Metadata.Key<String> requestHashHeaderKey;
|
||||
private final ThreadSafeRandom random;
|
||||
private final boolean hasEndpointInConnectingState;
|
||||
|
||||
private RingHashPicker(
|
||||
SynchronizationContext syncContext, List<RingEntry> ring,
|
||||
Collection<ChildLbState> children) {
|
||||
Collection<ChildLbState> children, Metadata.Key<String> requestHashHeaderKey,
|
||||
ThreadSafeRandom random) {
|
||||
this.syncContext = syncContext;
|
||||
this.ring = ring;
|
||||
this.requestHashHeaderKey = requestHashHeaderKey;
|
||||
this.random = random;
|
||||
pickableSubchannels = new HashMap<>(children.size());
|
||||
boolean hasConnectingState = false;
|
||||
for (ChildLbState childLbState : children) {
|
||||
pickableSubchannels.put((Endpoint)childLbState.getKey(),
|
||||
new SubchannelView(childLbState, childLbState.getCurrentState()));
|
||||
if (childLbState.getCurrentState() == CONNECTING) {
|
||||
hasConnectingState = true;
|
||||
}
|
||||
}
|
||||
this.hasEndpointInConnectingState = hasConnectingState;
|
||||
}
|
||||
|
||||
// Find the ring entry with hash next to (clockwise) the RPC's hash (binary search).
|
||||
private int getTargetIndex(Long requestHash) {
|
||||
private int getTargetIndex(long requestHash) {
|
||||
if (ring.size() <= 1) {
|
||||
return 0;
|
||||
}
|
||||
|
|
@ -365,39 +393,78 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
|
|||
|
||||
@Override
|
||||
public PickResult pickSubchannel(PickSubchannelArgs args) {
|
||||
Long requestHash = args.getCallOptions().getOption(XdsNameResolver.RPC_HASH_KEY);
|
||||
if (requestHash == null) {
|
||||
return PickResult.withError(RPC_HASH_NOT_FOUND);
|
||||
// Determine request hash.
|
||||
boolean usingRandomHash = false;
|
||||
long requestHash;
|
||||
if (requestHashHeaderKey == null) {
|
||||
// Set by the xDS config selector.
|
||||
Long rpcHashFromCallOptions = args.getCallOptions().getOption(XdsNameResolver.RPC_HASH_KEY);
|
||||
if (rpcHashFromCallOptions == null) {
|
||||
return PickResult.withError(RPC_HASH_NOT_FOUND);
|
||||
}
|
||||
requestHash = rpcHashFromCallOptions;
|
||||
} else {
|
||||
Iterable<String> headerValues = args.getHeaders().getAll(requestHashHeaderKey);
|
||||
if (headerValues != null) {
|
||||
requestHash = hashFunc.hashAsciiString(Joiner.on(",").join(headerValues));
|
||||
} else {
|
||||
requestHash = random.nextLong();
|
||||
usingRandomHash = true;
|
||||
}
|
||||
}
|
||||
|
||||
int targetIndex = getTargetIndex(requestHash);
|
||||
|
||||
// Per gRFC A61, because of sticky-TF with PickFirst's auto reconnect on TF, we ignore
|
||||
// all TF subchannels and find the first ring entry in READY, CONNECTING or IDLE. If
|
||||
// CONNECTING or IDLE we return a pick with no results. Additionally, if that entry is in
|
||||
// IDLE, we initiate a connection.
|
||||
for (int i = 0; i < ring.size(); i++) {
|
||||
int index = (targetIndex + i) % ring.size();
|
||||
SubchannelView subchannelView = pickableSubchannels.get(ring.get(index).addrKey);
|
||||
ChildLbState childLbState = subchannelView.childLbState;
|
||||
if (!usingRandomHash) {
|
||||
// Per gRFC A61, because of sticky-TF with PickFirst's auto reconnect on TF, we ignore
|
||||
// all TF subchannels and find the first ring entry in READY, CONNECTING or IDLE. If
|
||||
// CONNECTING or IDLE we return a pick with no results. Additionally, if that entry is in
|
||||
// IDLE, we initiate a connection.
|
||||
for (int i = 0; i < ring.size(); i++) {
|
||||
int index = (targetIndex + i) % ring.size();
|
||||
SubchannelView subchannelView = pickableSubchannels.get(ring.get(index).addrKey);
|
||||
ChildLbState childLbState = subchannelView.childLbState;
|
||||
|
||||
if (subchannelView.connectivityState == READY) {
|
||||
return childLbState.getCurrentPicker().pickSubchannel(args);
|
||||
if (subchannelView.connectivityState == READY) {
|
||||
return childLbState.getCurrentPicker().pickSubchannel(args);
|
||||
}
|
||||
|
||||
// RPCs can be buffered if the next subchannel is pending (per A62). Otherwise, RPCs
|
||||
// are failed unless there is a READY connection.
|
||||
if (subchannelView.connectivityState == CONNECTING) {
|
||||
return PickResult.withNoResult();
|
||||
}
|
||||
|
||||
if (subchannelView.connectivityState == IDLE) {
|
||||
syncContext.execute(() -> {
|
||||
childLbState.getLb().requestConnection();
|
||||
});
|
||||
|
||||
return PickResult.withNoResult(); // Indicates that this should be retried after backoff
|
||||
}
|
||||
}
|
||||
|
||||
// RPCs can be buffered if the next subchannel is pending (per A62). Otherwise, RPCs
|
||||
// are failed unless there is a READY connection.
|
||||
if (subchannelView.connectivityState == CONNECTING) {
|
||||
} else {
|
||||
// Using a random hash. Find and use the first READY ring entry, triggering at most one
|
||||
// entry to attempt connection.
|
||||
boolean requestedConnection = hasEndpointInConnectingState;
|
||||
for (int i = 0; i < ring.size(); i++) {
|
||||
int index = (targetIndex + i) % ring.size();
|
||||
SubchannelView subchannelView = pickableSubchannels.get(ring.get(index).addrKey);
|
||||
ChildLbState childLbState = subchannelView.childLbState;
|
||||
if (subchannelView.connectivityState == READY) {
|
||||
return childLbState.getCurrentPicker().pickSubchannel(args);
|
||||
}
|
||||
if (!requestedConnection && subchannelView.connectivityState == IDLE) {
|
||||
syncContext.execute(
|
||||
() -> {
|
||||
childLbState.getLb().requestConnection();
|
||||
});
|
||||
requestedConnection = true;
|
||||
}
|
||||
}
|
||||
if (requestedConnection) {
|
||||
return PickResult.withNoResult();
|
||||
}
|
||||
|
||||
if (subchannelView.connectivityState == IDLE) {
|
||||
syncContext.execute(() -> {
|
||||
childLbState.getLb().requestConnection();
|
||||
});
|
||||
|
||||
return PickResult.withNoResult(); // Indicates that this should be retried after backoff
|
||||
}
|
||||
}
|
||||
|
||||
// return the pick from the original subchannel hit by hash, which is probably an error
|
||||
|
|
@ -444,13 +511,16 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
|
|||
static final class RingHashConfig {
|
||||
final long minRingSize;
|
||||
final long maxRingSize;
|
||||
final String requestHashHeader;
|
||||
|
||||
RingHashConfig(long minRingSize, long maxRingSize) {
|
||||
RingHashConfig(long minRingSize, long maxRingSize, String requestHashHeader) {
|
||||
checkArgument(minRingSize > 0, "minRingSize <= 0");
|
||||
checkArgument(maxRingSize > 0, "maxRingSize <= 0");
|
||||
checkArgument(minRingSize <= maxRingSize, "minRingSize > maxRingSize");
|
||||
checkNotNull(requestHashHeader);
|
||||
this.minRingSize = minRingSize;
|
||||
this.maxRingSize = maxRingSize;
|
||||
this.requestHashHeader = requestHashHeader;
|
||||
}
|
||||
|
||||
@Override
|
||||
|
|
@ -458,6 +528,7 @@ final class RingHashLoadBalancer extends MultiChildLoadBalancer {
|
|||
return MoreObjects.toStringHelper(this)
|
||||
.add("minRingSize", minRingSize)
|
||||
.add("maxRingSize", maxRingSize)
|
||||
.add("requestHashHeader", requestHashHeader)
|
||||
.toString();
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -24,6 +24,7 @@ import io.grpc.LoadBalancer.Helper;
|
|||
import io.grpc.LoadBalancerProvider;
|
||||
import io.grpc.NameResolver.ConfigOrError;
|
||||
import io.grpc.Status;
|
||||
import io.grpc.internal.GrpcUtil;
|
||||
import io.grpc.internal.JsonUtil;
|
||||
import io.grpc.xds.RingHashLoadBalancer.RingHashConfig;
|
||||
import io.grpc.xds.RingHashOptions;
|
||||
|
|
@ -81,6 +82,10 @@ public final class RingHashLoadBalancerProvider extends LoadBalancerProvider {
|
|||
Map<String, ?> rawLoadBalancingPolicyConfig) {
|
||||
Long minRingSize = JsonUtil.getNumberAsLong(rawLoadBalancingPolicyConfig, "minRingSize");
|
||||
Long maxRingSize = JsonUtil.getNumberAsLong(rawLoadBalancingPolicyConfig, "maxRingSize");
|
||||
String requestHashHeader = "";
|
||||
if (GrpcUtil.getFlag("GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY", false)) {
|
||||
requestHashHeader = JsonUtil.getString(rawLoadBalancingPolicyConfig, "requestHashHeader");
|
||||
}
|
||||
long maxRingSizeCap = RingHashOptions.getRingSizeCap();
|
||||
if (minRingSize == null) {
|
||||
minRingSize = DEFAULT_MIN_RING_SIZE;
|
||||
|
|
@ -88,6 +93,9 @@ public final class RingHashLoadBalancerProvider extends LoadBalancerProvider {
|
|||
if (maxRingSize == null) {
|
||||
maxRingSize = DEFAULT_MAX_RING_SIZE;
|
||||
}
|
||||
if (requestHashHeader == null) {
|
||||
requestHashHeader = "";
|
||||
}
|
||||
if (minRingSize > maxRingSizeCap) {
|
||||
minRingSize = maxRingSizeCap;
|
||||
}
|
||||
|
|
@ -98,6 +106,7 @@ public final class RingHashLoadBalancerProvider extends LoadBalancerProvider {
|
|||
return ConfigOrError.fromError(Status.UNAVAILABLE.withDescription(
|
||||
"Invalid 'mingRingSize'/'maxRingSize'"));
|
||||
}
|
||||
return ConfigOrError.fromConfig(new RingHashConfig(minRingSize, maxRingSize));
|
||||
return ConfigOrError.fromConfig(
|
||||
new RingHashConfig(minRingSize, maxRingSize, requestHashHeader));
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -163,7 +163,7 @@ public class ClusterResolverLoadBalancerTest {
|
|||
GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig(
|
||||
new FakeLoadBalancerProvider("round_robin"), null)));
|
||||
private final Object ringHash = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig(
|
||||
new FakeLoadBalancerProvider("ring_hash_experimental"), new RingHashConfig(10L, 100L));
|
||||
new FakeLoadBalancerProvider("ring_hash_experimental"), new RingHashConfig(10L, 100L, ""));
|
||||
private final Object leastRequest = GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig(
|
||||
new FakeLoadBalancerProvider("wrr_locality_experimental"), new WrrLocalityConfig(
|
||||
GracefulSwitchLoadBalancer.createLoadBalancingPolicyConfig(
|
||||
|
|
|
|||
|
|
@ -42,6 +42,8 @@ import org.junit.runners.JUnit4;
|
|||
@RunWith(JUnit4.class)
|
||||
public class RingHashLoadBalancerProviderTest {
|
||||
private static final String AUTHORITY = "foo.googleapis.com";
|
||||
private static final String GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY =
|
||||
"GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY";
|
||||
|
||||
private final SynchronizationContext syncContext = new SynchronizationContext(
|
||||
new UncaughtExceptionHandler() {
|
||||
|
|
@ -81,6 +83,7 @@ public class RingHashLoadBalancerProviderTest {
|
|||
RingHashConfig config = (RingHashConfig) configOrError.getConfig();
|
||||
assertThat(config.minRingSize).isEqualTo(10L);
|
||||
assertThat(config.maxRingSize).isEqualTo(100L);
|
||||
assertThat(config.requestHashHeader).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -92,6 +95,7 @@ public class RingHashLoadBalancerProviderTest {
|
|||
RingHashConfig config = (RingHashConfig) configOrError.getConfig();
|
||||
assertThat(config.minRingSize).isEqualTo(RingHashLoadBalancerProvider.DEFAULT_MIN_RING_SIZE);
|
||||
assertThat(config.maxRingSize).isEqualTo(RingHashLoadBalancerProvider.DEFAULT_MAX_RING_SIZE);
|
||||
assertThat(config.requestHashHeader).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -127,6 +131,7 @@ public class RingHashLoadBalancerProviderTest {
|
|||
RingHashConfig config = (RingHashConfig) configOrError.getConfig();
|
||||
assertThat(config.minRingSize).isEqualTo(10);
|
||||
assertThat(config.maxRingSize).isEqualTo(RingHashOptions.DEFAULT_RING_SIZE_CAP);
|
||||
assertThat(config.requestHashHeader).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
|
|
@ -142,6 +147,7 @@ public class RingHashLoadBalancerProviderTest {
|
|||
RingHashConfig config = (RingHashConfig) configOrError.getConfig();
|
||||
assertThat(config.minRingSize).isEqualTo(RingHashOptions.MAX_RING_SIZE_CAP);
|
||||
assertThat(config.maxRingSize).isEqualTo(RingHashOptions.MAX_RING_SIZE_CAP);
|
||||
assertThat(config.requestHashHeader).isEmpty();
|
||||
// Reset to avoid affecting subsequent test cases
|
||||
RingHashOptions.setRingSizeCap(RingHashOptions.DEFAULT_RING_SIZE_CAP);
|
||||
}
|
||||
|
|
@ -159,6 +165,7 @@ public class RingHashLoadBalancerProviderTest {
|
|||
RingHashConfig config = (RingHashConfig) configOrError.getConfig();
|
||||
assertThat(config.minRingSize).isEqualTo(RingHashOptions.MAX_RING_SIZE_CAP);
|
||||
assertThat(config.maxRingSize).isEqualTo(RingHashOptions.MAX_RING_SIZE_CAP);
|
||||
assertThat(config.requestHashHeader).isEmpty();
|
||||
// Reset to avoid affecting subsequent test cases
|
||||
RingHashOptions.setRingSizeCap(RingHashOptions.DEFAULT_RING_SIZE_CAP);
|
||||
}
|
||||
|
|
@ -176,6 +183,7 @@ public class RingHashLoadBalancerProviderTest {
|
|||
RingHashConfig config = (RingHashConfig) configOrError.getConfig();
|
||||
assertThat(config.minRingSize).isEqualTo(1);
|
||||
assertThat(config.maxRingSize).isEqualTo(1);
|
||||
assertThat(config.requestHashHeader).isEmpty();
|
||||
// Reset to avoid affecting subsequent test cases
|
||||
RingHashOptions.setRingSizeCap(RingHashOptions.DEFAULT_RING_SIZE_CAP);
|
||||
}
|
||||
|
|
@ -193,6 +201,7 @@ public class RingHashLoadBalancerProviderTest {
|
|||
RingHashConfig config = (RingHashConfig) configOrError.getConfig();
|
||||
assertThat(config.minRingSize).isEqualTo(1);
|
||||
assertThat(config.maxRingSize).isEqualTo(1);
|
||||
assertThat(config.requestHashHeader).isEmpty();
|
||||
// Reset to avoid affecting subsequent test cases
|
||||
RingHashOptions.setRingSizeCap(RingHashOptions.DEFAULT_RING_SIZE_CAP);
|
||||
}
|
||||
|
|
@ -219,6 +228,59 @@ public class RingHashLoadBalancerProviderTest {
|
|||
.isEqualTo("Invalid 'mingRingSize'/'maxRingSize'");
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseLoadBalancingConfig_requestHashHeaderIgnoredWhenEnvVarNotSet()
|
||||
throws IOException {
|
||||
String lbConfig =
|
||||
"{\"minRingSize\" : 10, \"maxRingSize\" : 100, \"requestHashHeader\" : \"dummy-hash\"}";
|
||||
ConfigOrError configOrError =
|
||||
provider.parseLoadBalancingPolicyConfig(parseJsonObject(lbConfig));
|
||||
assertThat(configOrError.getConfig()).isNotNull();
|
||||
RingHashConfig config = (RingHashConfig) configOrError.getConfig();
|
||||
assertThat(config.minRingSize).isEqualTo(10L);
|
||||
assertThat(config.maxRingSize).isEqualTo(100L);
|
||||
assertThat(config.requestHashHeader).isEmpty();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseLoadBalancingConfig_requestHashHeaderSetWhenEnvVarSet() throws IOException {
|
||||
System.setProperty(GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY, "true");
|
||||
try {
|
||||
String lbConfig =
|
||||
"{\"minRingSize\" : 10, \"maxRingSize\" : 100, \"requestHashHeader\" : \"dummy-hash\"}";
|
||||
ConfigOrError configOrError =
|
||||
provider.parseLoadBalancingPolicyConfig(parseJsonObject(lbConfig));
|
||||
assertThat(configOrError.getConfig()).isNotNull();
|
||||
RingHashConfig config = (RingHashConfig) configOrError.getConfig();
|
||||
assertThat(config.minRingSize).isEqualTo(10L);
|
||||
assertThat(config.maxRingSize).isEqualTo(100L);
|
||||
assertThat(config.requestHashHeader).isEqualTo("dummy-hash");
|
||||
assertThat(config.toString()).contains("minRingSize=10");
|
||||
assertThat(config.toString()).contains("maxRingSize=100");
|
||||
assertThat(config.toString()).contains("requestHashHeader=dummy-hash");
|
||||
} finally {
|
||||
System.clearProperty(GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY);
|
||||
}
|
||||
}
|
||||
|
||||
@Test
|
||||
public void parseLoadBalancingConfig_requestHashHeaderUnsetWhenEnvVarSet_useDefaults()
|
||||
throws IOException {
|
||||
System.setProperty(GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY, "true");
|
||||
try {
|
||||
String lbConfig = "{\"minRingSize\" : 10, \"maxRingSize\" : 100}";
|
||||
ConfigOrError configOrError =
|
||||
provider.parseLoadBalancingPolicyConfig(parseJsonObject(lbConfig));
|
||||
assertThat(configOrError.getConfig()).isNotNull();
|
||||
RingHashConfig config = (RingHashConfig) configOrError.getConfig();
|
||||
assertThat(config.minRingSize).isEqualTo(10L);
|
||||
assertThat(config.maxRingSize).isEqualTo(100L);
|
||||
assertThat(config.requestHashHeader).isEmpty();
|
||||
} finally {
|
||||
System.clearProperty(GRPC_EXPERIMENTAL_RING_HASH_SET_REQUEST_HASH_KEY);
|
||||
}
|
||||
}
|
||||
|
||||
@SuppressWarnings("unchecked")
|
||||
private static Map<String, ?> parseJsonObject(String json) throws IOException {
|
||||
return (Map<String, ?>) JsonParser.parse(json);
|
||||
|
|
|
|||
|
|
@ -98,6 +98,9 @@ import org.mockito.junit.MockitoRule;
|
|||
@RunWith(JUnit4.class)
|
||||
public class RingHashLoadBalancerTest {
|
||||
private static final String AUTHORITY = "foo.googleapis.com";
|
||||
private static final String CUSTOM_REQUEST_HASH_HEADER = "custom-request-hash-header";
|
||||
private static final Metadata.Key<String> CUSTOM_METADATA_KEY =
|
||||
Metadata.Key.of(CUSTOM_REQUEST_HASH_HEADER, Metadata.ASCII_STRING_MARSHALLER);
|
||||
private static final Attributes.Key<String> CUSTOM_KEY = Attributes.Key.create("custom-key");
|
||||
private static final ConnectivityStateInfo CSI_CONNECTING =
|
||||
ConnectivityStateInfo.forNonError(CONNECTING);
|
||||
|
|
@ -142,7 +145,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void subchannelLazyConnectUntilPicked() {
|
||||
RingHashConfig config = new RingHashConfig(10, 100);
|
||||
RingHashConfig config = new RingHashConfig(10, 100, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1); // one server
|
||||
Status addressesAcceptanceStatus = loadBalancer.acceptResolvedAddresses(
|
||||
ResolvedAddresses.newBuilder()
|
||||
|
|
@ -176,7 +179,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void subchannelNotAutoReconnectAfterReenteringIdle() {
|
||||
RingHashConfig config = new RingHashConfig(10, 100);
|
||||
RingHashConfig config = new RingHashConfig(10, 100, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1); // one server
|
||||
Status addressesAcceptanceStatus = loadBalancer.acceptResolvedAddresses(
|
||||
ResolvedAddresses.newBuilder()
|
||||
|
|
@ -207,7 +210,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void aggregateSubchannelStates_connectingReadyIdleFailure() {
|
||||
RingHashConfig config = new RingHashConfig(10, 100);
|
||||
RingHashConfig config = new RingHashConfig(10, 100, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1);
|
||||
InOrder inOrder = Mockito.inOrder(helper);
|
||||
|
||||
|
|
@ -266,7 +269,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void aggregateSubchannelStates_allSubchannelsInTransientFailure() {
|
||||
RingHashConfig config = new RingHashConfig(10, 100);
|
||||
RingHashConfig config = new RingHashConfig(10, 100, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1, 1);
|
||||
|
||||
List<Subchannel> subChannelList = initializeLbSubchannels(config, servers, STAY_IN_CONNECTING);
|
||||
|
|
@ -324,7 +327,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void ignoreShutdownSubchannelStateChange() {
|
||||
RingHashConfig config = new RingHashConfig(10, 100);
|
||||
RingHashConfig config = new RingHashConfig(10, 100, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
||||
|
|
@ -340,7 +343,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void deterministicPickWithHostsPartiallyRemoved() {
|
||||
RingHashConfig config = new RingHashConfig(10, 100);
|
||||
RingHashConfig config = new RingHashConfig(10, 100, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1, 1, 1);
|
||||
initializeLbSubchannels(config, servers);
|
||||
InOrder inOrder = Mockito.inOrder(helper);
|
||||
|
|
@ -380,7 +383,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void deterministicPickWithNewHostsAdded() {
|
||||
RingHashConfig config = new RingHashConfig(10, 100);
|
||||
RingHashConfig config = new RingHashConfig(10, 100, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1); // server0 and server1
|
||||
initializeLbSubchannels(config, servers, DO_NOT_VERIFY, DO_NOT_RESET_HELPER);
|
||||
|
||||
|
|
@ -412,6 +415,139 @@ public class RingHashLoadBalancerTest {
|
|||
inOrder.verifyNoMoreInteractions();
|
||||
}
|
||||
|
||||
@Test
|
||||
public void deterministicPickWithRequestHashHeader_oneHeaderValue() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3, CUSTOM_REQUEST_HASH_HEADER);
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
initializeLbSubchannels(config, servers);
|
||||
InOrder inOrder = Mockito.inOrder(helper);
|
||||
|
||||
// Bring all subchannels to READY.
|
||||
for (Subchannel subchannel : subchannels.values()) {
|
||||
deliverSubchannelState(subchannel, CSI_READY);
|
||||
inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
|
||||
}
|
||||
|
||||
// Pick subchannel with custom request hash header where the rpc hash hits server1.
|
||||
Metadata headers = new Metadata();
|
||||
headers.put(CUSTOM_METADATA_KEY, "FakeSocketAddress-server1_0");
|
||||
PickSubchannelArgs args =
|
||||
new PickSubchannelArgsImpl(
|
||||
TestMethodDescriptors.voidMethod(),
|
||||
headers,
|
||||
CallOptions.DEFAULT,
|
||||
new PickDetailsConsumer() {});
|
||||
SubchannelPicker picker = pickerCaptor.getValue();
|
||||
PickResult result = picker.pickSubchannel(args);
|
||||
assertThat(result.getStatus().isOk()).isTrue();
|
||||
assertThat(result.getSubchannel().getAddresses()).isEqualTo(servers.get(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void deterministicPickWithRequestHashHeader_multipleHeaderValues() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3, CUSTOM_REQUEST_HASH_HEADER);
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
initializeLbSubchannels(config, servers);
|
||||
InOrder inOrder = Mockito.inOrder(helper);
|
||||
|
||||
// Bring all subchannels to READY.
|
||||
for (Subchannel subchannel : subchannels.values()) {
|
||||
deliverSubchannelState(subchannel, CSI_READY);
|
||||
inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
|
||||
}
|
||||
|
||||
// Pick subchannel with custom request hash header with multiple values for the same key where
|
||||
// the rpc hash hits server1.
|
||||
Metadata headers = new Metadata();
|
||||
headers.put(CUSTOM_METADATA_KEY, "FakeSocketAddress-server0_0");
|
||||
headers.put(CUSTOM_METADATA_KEY, "FakeSocketAddress-server1_0");
|
||||
PickSubchannelArgs args =
|
||||
new PickSubchannelArgsImpl(
|
||||
TestMethodDescriptors.voidMethod(),
|
||||
headers,
|
||||
CallOptions.DEFAULT,
|
||||
new PickDetailsConsumer() {});
|
||||
SubchannelPicker picker = pickerCaptor.getValue();
|
||||
PickResult result = picker.pickSubchannel(args);
|
||||
assertThat(result.getStatus().isOk()).isTrue();
|
||||
assertThat(result.getSubchannel().getAddresses()).isEqualTo(servers.get(1));
|
||||
}
|
||||
|
||||
@Test
|
||||
public void pickWithRandomHash_allSubchannelsReady() {
|
||||
loadBalancer = new RingHashLoadBalancer(helper, new FakeRandom());
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(2, 2, "dummy-random-hash");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1);
|
||||
initializeLbSubchannels(config, servers);
|
||||
InOrder inOrder = Mockito.inOrder(helper);
|
||||
|
||||
// Bring all subchannels to READY.
|
||||
Map<EquivalentAddressGroup, Integer> pickCounts = new HashMap<>();
|
||||
for (Subchannel subchannel : subchannels.values()) {
|
||||
deliverSubchannelState(subchannel, CSI_READY);
|
||||
pickCounts.put(subchannel.getAddresses(), 0);
|
||||
inOrder.verify(helper).updateBalancingState(eq(READY), pickerCaptor.capture());
|
||||
}
|
||||
|
||||
// Pick subchannel 100 times with random hash.
|
||||
SubchannelPicker picker = pickerCaptor.getValue();
|
||||
PickSubchannelArgs args = getDefaultPickSubchannelArgs(hashFunc.hashVoid());
|
||||
for (int i = 0; i < 100; ++i) {
|
||||
Subchannel pickedSubchannel = picker.pickSubchannel(args).getSubchannel();
|
||||
EquivalentAddressGroup addr = pickedSubchannel.getAddresses();
|
||||
pickCounts.put(addr, pickCounts.get(addr) + 1);
|
||||
}
|
||||
|
||||
// Verify the distribution is uniform where server0 and server1 are exactly picked 50 times.
|
||||
assertThat(pickCounts.get(servers.get(0))).isEqualTo(50);
|
||||
assertThat(pickCounts.get(servers.get(1))).isEqualTo(50);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void pickWithRandomHash_atLeastOneSubchannelConnecting() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "dummy-random-hash");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
||||
// Bring one subchannel to CONNECTING.
|
||||
deliverSubchannelState(getSubChannel(servers.get(0)), CSI_CONNECTING);
|
||||
verify(helper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture());
|
||||
|
||||
// Pick subchannel with random hash does not trigger connection.
|
||||
SubchannelPicker picker = pickerCaptor.getValue();
|
||||
PickSubchannelArgs args = getDefaultPickSubchannelArgs(hashFunc.hashVoid());
|
||||
PickResult result = picker.pickSubchannel(args);
|
||||
assertThat(result.getStatus().isOk()).isTrue();
|
||||
assertThat(result.getSubchannel()).isNull(); // buffer request
|
||||
verifyConnection(0);
|
||||
}
|
||||
|
||||
@Test
|
||||
public void pickWithRandomHash_firstSubchannelInTransientFailure_remainingSubchannelsIdle() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "dummy-random-hash");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
||||
// Bring one subchannel to TRANSIENT_FAILURE.
|
||||
deliverSubchannelUnreachable(getSubChannel(servers.get(0)));
|
||||
verify(helper).updateBalancingState(eq(CONNECTING), pickerCaptor.capture());
|
||||
verifyConnection(0);
|
||||
|
||||
// Pick subchannel with random hash does trigger connection by walking the ring
|
||||
// and choosing the first (at most one) IDLE subchannel along the way.
|
||||
SubchannelPicker picker = pickerCaptor.getValue();
|
||||
PickSubchannelArgs args = getDefaultPickSubchannelArgs(hashFunc.hashVoid());
|
||||
PickResult result = picker.pickSubchannel(args);
|
||||
assertThat(result.getStatus().isOk()).isTrue();
|
||||
assertThat(result.getSubchannel()).isNull(); // buffer request
|
||||
verifyConnection(1);
|
||||
}
|
||||
|
||||
private Subchannel getSubChannel(EquivalentAddressGroup eag) {
|
||||
return subchannels.get(Collections.singletonList(eag));
|
||||
}
|
||||
|
|
@ -419,7 +555,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void skipFailingHosts_pickNextNonFailingHost() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
Status addressesAcceptanceStatus =
|
||||
loadBalancer.acceptResolvedAddresses(
|
||||
|
|
@ -489,7 +625,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void skipFailingHosts_firstTwoHostsFailed_pickNextFirstReady() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
|
@ -555,7 +691,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void removingAddressShutdownSubchannel() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> svs1 = createWeightedServerAddrs(1, 1, 1);
|
||||
List<Subchannel> subchannels1 = initializeLbSubchannels(config, svs1, STAY_IN_CONNECTING);
|
||||
|
||||
|
|
@ -572,7 +708,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void allSubchannelsInTransientFailure() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
||||
|
|
@ -599,7 +735,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void firstSubchannelIdle() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
||||
|
|
@ -620,7 +756,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void firstSubchannelConnecting() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
||||
|
|
@ -644,7 +780,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void firstSubchannelFailure() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
|
||||
List<Subchannel> subchannelList =
|
||||
|
|
@ -675,7 +811,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void secondSubchannelConnecting() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
|
@ -706,7 +842,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void secondSubchannelFailure() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
|
@ -733,7 +869,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void thirdSubchannelConnecting() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
|
@ -762,7 +898,7 @@ public class RingHashLoadBalancerTest {
|
|||
@Test
|
||||
public void stickyTransientFailure() {
|
||||
// Map each server address to exactly one ring entry.
|
||||
RingHashConfig config = new RingHashConfig(3, 3);
|
||||
RingHashConfig config = new RingHashConfig(3, 3, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1, 1);
|
||||
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
|
@ -791,7 +927,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void largeWeights() {
|
||||
RingHashConfig config = new RingHashConfig(10000, 100000); // large ring
|
||||
RingHashConfig config = new RingHashConfig(10000, 100000, ""); // large ring
|
||||
List<EquivalentAddressGroup> servers =
|
||||
createWeightedServerAddrs(Integer.MAX_VALUE, 10, 100); // MAX:10:100
|
||||
|
||||
|
|
@ -829,7 +965,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void hostSelectionProportionalToWeights() {
|
||||
RingHashConfig config = new RingHashConfig(10000, 100000); // large ring
|
||||
RingHashConfig config = new RingHashConfig(10000, 100000, ""); // large ring
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 10, 100); // 1:10:100
|
||||
|
||||
initializeLbSubchannels(config, servers);
|
||||
|
|
@ -872,7 +1008,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void nameResolutionErrorWithActiveSubchannels() {
|
||||
RingHashConfig config = new RingHashConfig(10, 100);
|
||||
RingHashConfig config = new RingHashConfig(10, 100, "");
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1);
|
||||
|
||||
initializeLbSubchannels(config, servers, DO_NOT_VERIFY, DO_NOT_RESET_HELPER);
|
||||
|
|
@ -894,7 +1030,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
@Test
|
||||
public void duplicateAddresses() {
|
||||
RingHashConfig config = new RingHashConfig(10, 100);
|
||||
RingHashConfig config = new RingHashConfig(10, 100, "");
|
||||
List<EquivalentAddressGroup> servers = createRepeatedServerAddrs(1, 2, 3);
|
||||
|
||||
initializeLbSubchannels(config, servers, DO_NOT_VERIFY);
|
||||
|
|
@ -940,7 +1076,7 @@ public class RingHashLoadBalancerTest {
|
|||
|
||||
InOrder inOrder = Mockito.inOrder(helper);
|
||||
List<EquivalentAddressGroup> servers = createWeightedServerAddrs(1, 1);
|
||||
initializeLbSubchannels(new RingHashConfig(10, 100), servers);
|
||||
initializeLbSubchannels(new RingHashConfig(10, 100, ""), servers);
|
||||
Subchannel subchannel0 = subchannels.get(Collections.singletonList(servers.get(0)));
|
||||
Subchannel subchannel1 = subchannels.get(Collections.singletonList(servers.get(1)));
|
||||
|
||||
|
|
@ -1167,6 +1303,30 @@ public class RingHashLoadBalancerTest {
|
|||
}
|
||||
}
|
||||
|
||||
private static final class FakeRandom implements ThreadSafeRandom {
|
||||
int counter = 0;
|
||||
|
||||
@Override
|
||||
public int nextInt(int bound) {
|
||||
throw new UnsupportedOperationException("Should not be called");
|
||||
}
|
||||
|
||||
@Override
|
||||
public long nextLong() {
|
||||
++counter;
|
||||
if (counter % 2 == 0) {
|
||||
return XxHash64.INSTANCE.hashAsciiString("FakeSocketAddress-server0_0");
|
||||
} else {
|
||||
return XxHash64.INSTANCE.hashAsciiString("FakeSocketAddress-server1_0");
|
||||
}
|
||||
}
|
||||
|
||||
@Override
|
||||
public long nextLong(long bound) {
|
||||
throw new UnsupportedOperationException("Should not be called");
|
||||
}
|
||||
}
|
||||
|
||||
enum InitializationFlags {
|
||||
DO_NOT_VERIFY,
|
||||
RESET_SUBCHANNEL_MOCKS,
|
||||
|
|
|
|||
Loading…
Reference in New Issue