diff --git a/pkg/rpc/client_util.go b/pkg/rpc/client_util.go
index e3fb54638..38f33160a 100644
--- a/pkg/rpc/client_util.go
+++ b/pkg/rpc/client_util.go
@@ -65,7 +65,7 @@ func (conn *Connection) startGC() {
 				serverNode := node.(string)
 				totalNodeSize++
 				atime := accessTime.(time.Time)
-				if time.Since(atime) < conn.connExpireTime {
+				if conn.connExpireTime == 0 || time.Since(atime) < conn.connExpireTime {
 					return true
 				}
 				conn.gcConn(serverNode)