Fix gRPC forward bug when store is unreachable (#316)

Signed-off-by: birdstorm <samuelwyf@hotmail.com>

Co-authored-by: Little-Wallace <bupt2013211450@gmail.com>
This commit is contained in:
birdstorm 2021-11-08 11:58:22 +08:00 committed by GitHub
parent 64354ac5e2
commit d9479b00e5
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
2 changed files with 20 additions and 10 deletions

View File

@ -109,7 +109,7 @@ public class ConfigUtils {
public static final int DEF_TIKV_RAWKV_SCAN_TIMEOUT_IN_MS = 10000;
public static final int DEF_TIKV_RAWKV_CLEAN_TIMEOUT_IN_MS = 600000;
public static final int DEF_TIKV_BO_REGION_MISS_BASE_IN_MS = 100;
public static final int DEF_TIKV_BO_REGION_MISS_BASE_IN_MS = 20;
public static final String NORMAL_COMMAND_PRIORITY = "NORMAL";
public static final String LOW_COMMAND_PRIORITY = "LOW";

View File

@ -126,8 +126,7 @@ public abstract class AbstractRegionStoreClient
@Override
public boolean onStoreUnreachable() {
if (!targetStore.isValid()) {
logger.warn(
String.format("store [%d] has been invalid", region.getId(), targetStore.getId()));
logger.warn(String.format("store [%d] has been invalid", targetStore.getId()));
targetStore = regionManager.getStoreById(targetStore.getId());
updateClientStub();
return true;
@ -159,7 +158,7 @@ public abstract class AbstractRegionStoreClient
}
logger.warn(
String.format(
"retry time exceed for region[%d], invalid this region[%d]",
"retry time exceed for region[%d], invalid store[%d]",
region.getId(), targetStore.getId()));
regionManager.onRequestFail(region);
return false;
@ -194,11 +193,6 @@ public abstract class AbstractRegionStoreClient
// create a new store object, which is can-forward.
regionManager.updateStore(originStore, targetStore);
} else {
// If we try to forward request to leader by follower failed, it means that the store of old
// leader may be
// unavailable but the new leader has not been report to PD. So we can ban this store for a
// short time to
// avoid too many request try forward rather than try other peer.
originStore.forwardFail();
}
}
@ -260,7 +254,18 @@ public abstract class AbstractRegionStoreClient
}
private boolean retryOtherStoreByProxyForward() {
if (!conf.getEnableGrpcForward() || retryForwardTimes > region.getFollowerList().size()) {
if (!conf.getEnableGrpcForward()) {
return false;
}
if (retryForwardTimes >= region.getFollowerList().size()) {
// If we try to forward request to leader by follower failed, it means that the store of old
// leader may be
// unavailable but the new leader has not been report to PD. So we can ban this store for a
// short time to
// avoid too many request try forward rather than try other peer.
if (originStore != null) {
originStore.forwardFail();
}
return false;
}
TiStore proxyStore = switchProxyStore();
@ -269,6 +274,11 @@ public abstract class AbstractRegionStoreClient
String.format(
"no forward store can be selected for store [%s] and region[%d]",
targetStore.getStore().getAddress(), region.getId()));
if (originStore != null) {
originStore.forwardFail();
} else {
targetStore.forwardFail();
}
return false;
}
if (originStore == null) {