fix TLS reload doesn't work after delete cert file (#609) (#612)

Co-authored-by: Daemonxiao <35677990+Daemonxiao@users.noreply.github.com>

src/main/java/org/tikv/common/util/ChannelFactory.java

@@ -89,9 +89,15 @@ public class ChannelFactory implements AutoCloseable {
           this::tryReload, pollInterval, pollInterval, TimeUnit.SECONDS);
     }
 
+    // If any execution of the task encounters an exception, subsequent executions are suppressed.
     private void tryReload() {
-      if (needReload()) {
+      // Add exception handling to avoid schedule stop.
-        onChange.run();
+      try {
+        if (needReload()) {
+          onChange.run();
+        }
+      } catch (Exception e) {
+        logger.error("Failed to reload cert!", e);
       }
     }
 
@@ -180,11 +186,16 @@ public class ChannelFactory implements AutoCloseable {
     @Override
     public SslContextBuilder createSslContextBuilder() {
       SslContextBuilder builder = GrpcSslContexts.forClient();
-      if (trustPath != null) {
+      try {
-        builder.trustManager(new File(trustPath));
+        if (trustPath != null) {
-      }
+          builder.trustManager(new File(trustPath));
-      if (chainPath != null && keyPath != null) {
+        }
-        builder.keyManager(new File(chainPath), new File(keyPath));
+        if (chainPath != null && keyPath != null) {
+          builder.keyManager(new File(chainPath), new File(keyPath));
+        }
+      } catch (Exception e) {
+        logger.error("Failed to create ssl context builder", e);
+        throw new IllegalArgumentException(e);
       }
       return builder;
     }
@@ -351,7 +362,9 @@ public class ChannelFactory implements AutoCloseable {
 
     if (certContext != null) {
       recycler.shutdown();
-      certWatcher.close();
+      if (certWatcher != null) {
+        certWatcher.close();
+      }
     }
   }
 }

src/test/java/org/tikv/common/ChannelFactoryTest.java

@@ -26,6 +26,7 @@ import java.io.File;
 import java.util.ArrayList;
 import java.util.List;
 import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
 import java.util.concurrent.atomic.AtomicLong;
 import org.junit.Test;
 import org.tikv.common.util.ChannelFactory;
@@ -59,6 +60,22 @@ public class ChannelFactoryTest {
     assertTrue(changed.get());
   }
 
+  @Test
+  public void testCertWatcherWithExceptionTask() throws InterruptedException {
+    AtomicInteger timesOfReloadTask = new AtomicInteger(0);
+    new CertWatcher(
+        1,
+        ImmutableList.of(new File(caPath), new File(clientCertPath), new File(clientKeyPath)),
+        () -> {
+          timesOfReloadTask.getAndIncrement();
+          touchCert();
+          throw new RuntimeException("Mock exception in reload task");
+        });
+
+    Thread.sleep(5000);
+    assertTrue(timesOfReloadTask.get() > 1);
+  }
+
   @Test
   public void testMultiThreadTlsReload() throws InterruptedException {
     ChannelFactory factory = createFactory();