diff --git a/core/src/main/java/io/grpc/internal/ChannelExecutor.java b/core/src/main/java/io/grpc/internal/ChannelExecutor.java index af62cedced..4134df1425 100644 --- a/core/src/main/java/io/grpc/internal/ChannelExecutor.java +++ b/core/src/main/java/io/grpc/internal/ChannelExecutor.java @@ -34,7 +34,7 @@ import javax.annotation.concurrent.ThreadSafe; * order as they are submitted. */ @ThreadSafe -final class ChannelExecutor { +class ChannelExecutor { private static final Logger log = Logger.getLogger(ChannelExecutor.class.getName()); private final Object lock = new Object(); @@ -51,7 +51,7 @@ final class ChannelExecutor { *
Upon returning, it guarantees that all tasks submitted by {@code executeLater()} before it * have been or will eventually be run, while not requiring any more calls to {@code drain()}. */ - void drain() { + final void drain() { boolean drainLeaseAcquired = false; while (true) { Runnable runnable; @@ -72,7 +72,7 @@ final class ChannelExecutor { try { runnable.run(); } catch (Throwable t) { - log.log(Level.WARNING, "Runnable threw exception in ChannelExecutor", t); + handleUncaughtThrowable(t); } } } @@ -82,7 +82,7 @@ final class ChannelExecutor { * * @return this ChannelExecutor */ - ChannelExecutor executeLater(Runnable runnable) { + final ChannelExecutor executeLater(Runnable runnable) { synchronized (lock) { queue.add(checkNotNull(runnable, "runnable is null")); } @@ -90,9 +90,18 @@ final class ChannelExecutor { } @VisibleForTesting - int numPendingTasks() { + final int numPendingTasks() { synchronized (lock) { return queue.size(); } } + + /** + * Handle a throwable from a task. + * + *
The default implementation logs a warning.
+ */
+ void handleUncaughtThrowable(Throwable t) {
+ log.log(Level.WARNING, "Runnable threw exception in ChannelExecutor", t);
+ }
}
diff --git a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java
index ca7a33e474..1bec23ba64 100644
--- a/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java
+++ b/core/src/main/java/io/grpc/internal/ManagedChannelImpl.java
@@ -121,7 +121,13 @@ public final class ManagedChannelImpl extends ManagedChannel implements Instrume
private final ObjectPool extends Executor> executorPool;
private final ObjectPool extends Executor> oobExecutorPool;
- private final ChannelExecutor channelExecutor = new ChannelExecutor();
+ private final ChannelExecutor channelExecutor = new ChannelExecutor() {
+ @Override
+ void handleUncaughtThrowable(Throwable t) {
+ super.handleUncaughtThrowable(t);
+ panic(t);
+ }
+ };
private boolean fullStreamDecompression;
@@ -156,10 +162,14 @@ public final class ManagedChannelImpl extends ManagedChannel implements Instrume
@Nullable
private LbHelperImpl lbHelper;
- // Must be assigned from channelExecutor. null if channel is in idle mode.
+ // Must ONLY be assigned from updateSubchannelPicker(), which is called from channelExecutor.
+ // null if channel is in idle mode.
@Nullable
private volatile SubchannelPicker subchannelPicker;
+ // Must be accessed from the channelExecutor
+ private boolean panicMode;
+
// Must be mutated from channelExecutor
// If any monitoring hook to be added later needs to get a snapshot of this Set, we could
// switch to a ConcurrentHashMap.
@@ -239,16 +249,8 @@ public final class ManagedChannelImpl extends ManagedChannel implements Instrume
public void transportTerminated() {
checkState(shutdown.get(), "Channel must have been shut down");
terminating = true;
- if (lbHelper != null) {
- lbHelper.lb.shutdown();
- lbHelper = null;
- }
- if (nameResolver != null) {
- nameResolver.shutdown();
- nameResolver = null;
- nameResolverStarted = false;
- }
-
+ shutdownNameResolverAndLoadBalancer(false);
+ // No need to call channelStateManager since we are already in SHUTDOWN state.
// Until LoadBalancer is shutdown, it may still create new subchannels. We catch them
// here.
maybeShutdownNowSubchannels();
@@ -324,6 +326,24 @@ public final class ManagedChannelImpl extends ManagedChannel implements Instrume
@Nullable
private IdleModeTimer idleModeTimer;
+ // Must be called from channelExecutor
+ private void shutdownNameResolverAndLoadBalancer(boolean verifyActive) {
+ if (verifyActive) {
+ checkState(nameResolver != null, "nameResolver is null");
+ checkState(lbHelper != null, "lbHelper is null");
+ }
+ if (nameResolver != null) {
+ nameResolver.shutdown();
+ nameResolver = null;
+ nameResolverStarted = false;
+ }
+ if (lbHelper != null) {
+ lbHelper.lb.shutdown();
+ lbHelper = null;
+ }
+ subchannelPicker = null;
+ }
+
/**
* Make the channel exit idle mode, if it's in it.
*
@@ -331,7 +351,7 @@ public final class ManagedChannelImpl extends ManagedChannel implements Instrume
*/
@VisibleForTesting
void exitIdleMode() {
- if (shutdown.get()) {
+ if (shutdown.get() || panicMode) {
return;
}
if (inUseStateAggregator.isInUse()) {
@@ -366,12 +386,8 @@ public final class ManagedChannelImpl extends ManagedChannel implements Instrume
// either the idleModeTimer ran twice without exiting the idle mode, or the task in shutdown()
// did not cancel idleModeTimer, or prepareToLoseNetwork() ran while shutdown or in idle, all of
// which are bugs.
- nameResolver.shutdown();
- nameResolverStarted = false;
+ shutdownNameResolverAndLoadBalancer(true);
nameResolver = getNameResolver(target, nameResolverFactory, nameResolverParams);
- lbHelper.lb.shutdown();
- lbHelper = null;
- subchannelPicker = null;
channelStateManager.gotoState(IDLE);
}
@@ -636,6 +652,35 @@ public final class ManagedChannelImpl extends ManagedChannel implements Instrume
return this;
}
+ // Called from channelExecutor
+ @VisibleForTesting
+ void panic(final Throwable t) {
+ if (panicMode) {
+ // Preserve the first panic information
+ return;
+ }
+ panicMode = true;
+ cancelIdleTimer();
+ shutdownNameResolverAndLoadBalancer(false);
+ SubchannelPicker newPicker = new SubchannelPicker() {
+ final PickResult panicPickResult =
+ PickResult.withDrop(
+ Status.INTERNAL.withDescription("Panic! This is a bug!").withCause(t));
+ @Override
+ public PickResult pickSubchannel(PickSubchannelArgs args) {
+ return panicPickResult;
+ }
+ };
+ updateSubchannelPicker(newPicker);
+ channelStateManager.gotoState(TRANSIENT_FAILURE);
+ }
+
+ // Called from channelExecutor
+ private void updateSubchannelPicker(SubchannelPicker newPicker) {
+ subchannelPicker = newPicker;
+ delayedTransport.reprocess(newPicker);
+ }
+
@Override
public boolean isShutdown() {
return shutdown.get();
@@ -958,8 +1003,7 @@ public final class ManagedChannelImpl extends ManagedChannel implements Instrume
if (LbHelperImpl.this != lbHelper) {
return;
}
- subchannelPicker = newPicker;
- delayedTransport.reprocess(newPicker);
+ updateSubchannelPicker(newPicker);
// It's not appropriate to report SHUTDOWN state from lb.
// Ignore the case of newState == SHUTDOWN for now.
if (newState != SHUTDOWN) {
@@ -1084,16 +1128,7 @@ public final class ManagedChannelImpl extends ManagedChannel implements Instrume
re);
}
- try {
- balancer.handleResolvedAddressGroups(servers, config);
- } catch (Throwable e) {
- logger.log(
- Level.WARNING, "[" + getLogId() + "] Unexpected exception from LoadBalancer", e);
- // It must be a bug! Push the exception back to LoadBalancer in the hope that it may
- // be propagated to the application.
- balancer.handleNameResolutionError(Status.INTERNAL.withCause(e)
- .withDescription("Thrown from handleResolvedAddresses(): " + e));
- }
+ balancer.handleResolvedAddressGroups(servers, config);
}
}
diff --git a/core/src/test/java/io/grpc/internal/ChannelExecutorTest.java b/core/src/test/java/io/grpc/internal/ChannelExecutorTest.java
index 9b0b3c53c1..a3f9c941b3 100644
--- a/core/src/test/java/io/grpc/internal/ChannelExecutorTest.java
+++ b/core/src/test/java/io/grpc/internal/ChannelExecutorTest.java
@@ -16,14 +16,18 @@
package io.grpc.internal;
+import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertSame;
import static org.junit.Assert.assertTrue;
import static org.mockito.Mockito.doAnswer;
import static org.mockito.Mockito.inOrder;
+import java.util.concurrent.BlockingQueue;
import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicReference;
+import org.junit.After;
import org.junit.Before;
import org.junit.Test;
import org.junit.runner.RunWith;
@@ -39,7 +43,13 @@ import org.mockito.stubbing.Answer;
*/
@RunWith(JUnit4.class)
public class ChannelExecutorTest {
- private final ChannelExecutor executor = new ChannelExecutor();
+ private final BlockingQueue