Feature: change log level in-flight (#1023)

* feat: change log level in flight

Signed-off-by: Jim Ma <majinjing3@gmail.com>

docs/en/troubleshooting/README.md

@@ -1,5 +1,27 @@
 # Troubleshooting Guide
 
+## Change log level
+
+Send `SIGUSR1` signal to dragonfly process to change log level
+
+```shell
+kill -s SIGUSR1 <pid of dfdaemon, scheduler, cdn, or manager>
+```
+
+stdout:
+
+```text
+change log level to debug
+change log level to fatal
+change log level to panic
+change log level to dpanic
+change log level to error
+change log level to warn
+change log level to info
+```
+
+> The change log level event will print in stdout and `core.log` file, but if the level is greater than `info`, stdout only.
+
 ## Download slowly than without Dragonfly
 
 1. Confirm limit rate in [dfget.yaml](https://github.com/dragonflyoss/Dragonfly2/blob/main/docs/en/deployment/configuration/dfget.yaml#L65)

docs/zh-CN/troubleshooting/README.md

@@ -1,5 +1,27 @@
 # 问题排查
 
+## 修改日志级别
+
+发送 `SIGUSR1` 信号给蜻蜓进程即可修改日志级别
+
+```shell
+kill -s SIGUSR1 <pid of dfdaemon, scheduler, cdn, or manager>
+```
+
+标准输出：
+
+```text
+change log level to debug
+change log level to fatal
+change log level to panic
+change log level to dpanic
+change log level to error
+change log level to warn
+change log level to info
+```
+
+> 修改日志级别的事件将记录在标准输出和 `core.log` 中，但是如果修改的级别高于 `info` 的话，则仅有标准输出
+
 ## 下载速度比不用蜻蜓的时候慢
 
 1. 确认限速值是否合适 [dfget.yaml](https://github.com/dragonflyoss/Dragonfly2/blob/main/docs/zh-CN/config/dfget.yaml#L61)

internal/dflog/logcore/init.go

@@ -20,9 +20,35 @@ import (
 	"path"
 	"path/filepath"
 
+	"go.uber.org/zap"
+
 	logger "d7y.io/dragonfly/v2/internal/dflog"
 )
 
+type logInitMeta struct {
+	fileName             string
+	setSugaredLoggerFunc func(*zap.SugaredLogger)
+	setLoggerFunc        func(log *zap.Logger)
+}
+
+func createLogger(meta []logInitMeta, logDir string) error {
+	for _, m := range meta {
+		log, level, err := CreateLogger(path.Join(logDir, m.fileName), false, false)
+		if err != nil {
+			return err
+		}
+		if m.setSugaredLoggerFunc != nil {
+			m.setSugaredLoggerFunc(log.Sugar())
+		} else {
+			m.setLoggerFunc(log)
+		}
+
+		levels = append(levels, level)
+	}
+	startLoggerSignalHandler()
+	return nil
+}
+
 func InitManager(console bool, dir string) error {
 	if console {
 		return nil
@@ -30,31 +56,26 @@ func InitManager(console bool, dir string) error {
 
 	logDir := filepath.Join(dir, "manager")
 
-	coreLogger, err := CreateLogger(path.Join(logDir, CoreLogFileName), false, false)
-	if err != nil {
-		return err
+	var meta = []logInitMeta{
+		{
+			fileName:             CoreLogFileName,
+			setSugaredLoggerFunc: logger.SetCoreLogger,
+		},
+		{
+			fileName:             GrpcLogFileName,
+			setSugaredLoggerFunc: logger.SetGrpcLogger,
+		},
+		{
+			fileName:             GCLogFileName,
+			setSugaredLoggerFunc: logger.SetGCLogger,
+		},
+		{
+			fileName:             JobLogFileName,
+			setSugaredLoggerFunc: logger.SetJobLogger,
+		},
 	}
-	logger.SetCoreLogger(coreLogger.Sugar())
 
-	grpcLogger, err := CreateLogger(path.Join(logDir, GrpcLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGrpcLogger(grpcLogger.Sugar())
-
-	gcLogger, err := CreateLogger(path.Join(logDir, GCLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGCLogger(gcLogger.Sugar())
-
-	jobLogger, err := CreateLogger(path.Join(logDir, JobLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetJobLogger(jobLogger.Sugar())
-
-	return nil
+	return createLogger(meta, logDir)
 }
 
 func InitScheduler(console bool, dir string) error {
@@ -64,31 +85,26 @@ func InitScheduler(console bool, dir string) error {
 
 	logDir := filepath.Join(dir, "scheduler")
 
-	coreLogger, err := CreateLogger(path.Join(logDir, CoreLogFileName), false, false)
-	if err != nil {
-		return err
+	var meta = []logInitMeta{
+		{
+			fileName:             CoreLogFileName,
+			setSugaredLoggerFunc: logger.SetCoreLogger,
+		},
+		{
+			fileName:             GrpcLogFileName,
+			setSugaredLoggerFunc: logger.SetGrpcLogger,
+		},
+		{
+			fileName:             GCLogFileName,
+			setSugaredLoggerFunc: logger.SetGCLogger,
+		},
+		{
+			fileName:             JobLogFileName,
+			setSugaredLoggerFunc: logger.SetJobLogger,
+		},
 	}
-	logger.SetCoreLogger(coreLogger.Sugar())
 
-	grpcLogger, err := CreateLogger(path.Join(logDir, GrpcLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGrpcLogger(grpcLogger.Sugar())
-
-	gcLogger, err := CreateLogger(path.Join(logDir, GCLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGCLogger(gcLogger.Sugar())
-
-	jobLogger, err := CreateLogger(path.Join(logDir, JobLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetJobLogger(jobLogger.Sugar())
-
-	return nil
+	return createLogger(meta, logDir)
 }
 
 func InitCdnSystem(console bool, dir string) error {
@@ -97,55 +113,42 @@ func InitCdnSystem(console bool, dir string) error {
 	}
 
 	logDir := filepath.Join(dir, "cdn")
-
-	coreLogger, err := CreateLogger(path.Join(logDir, CoreLogFileName), false, false)
-	if err != nil {
-		return err
+	var meta = []logInitMeta{
+		{
+			fileName:             CoreLogFileName,
+			setSugaredLoggerFunc: logger.SetCoreLogger,
+		},
+		{
+			fileName:             GrpcLogFileName,
+			setSugaredLoggerFunc: logger.SetGrpcLogger,
+		},
+		{
+			fileName:             GCLogFileName,
+			setSugaredLoggerFunc: logger.SetGCLogger,
+		},
+		{
+			fileName:             StorageGCLogFileName,
+			setSugaredLoggerFunc: logger.SetStorageGCLogger,
+		},
+		{
+			fileName:             JobLogFileName,
+			setSugaredLoggerFunc: logger.SetJobLogger,
+		},
+		{
+			fileName:      StatSeedLogFileName,
+			setLoggerFunc: logger.SetStatSeedLogger,
+		},
+		{
+			fileName:      DownloaderLogFileName,
+			setLoggerFunc: logger.SetDownloadLogger,
+		},
+		{
+			fileName:             KeepAliveLogFileName,
+			setSugaredLoggerFunc: logger.SetKeepAliveLogger,
+		},
 	}
-	logger.SetCoreLogger(coreLogger.Sugar())
 
-	grpcLogger, err := CreateLogger(path.Join(logDir, GrpcLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGrpcLogger(grpcLogger.Sugar())
-
-	gcLogger, err := CreateLogger(path.Join(logDir, GCLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGCLogger(gcLogger.Sugar())
-
-	storageGCLogger, err := CreateLogger(path.Join(logDir, StorageGCLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGCLogger(storageGCLogger.Sugar())
-
-	jobLogger, err := CreateLogger(path.Join(logDir, JobLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetJobLogger(jobLogger.Sugar())
-
-	statSeedLogger, err := CreateLogger(path.Join(logDir, StatSeedLogFileName), true, true)
-	if err != nil {
-		return err
-	}
-	logger.SetStatSeedLogger(statSeedLogger)
-
-	downloaderLogger, err := CreateLogger(path.Join(logDir, DownloaderLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetDownloadLogger(downloaderLogger)
-
-	keepAliveLogger, err := CreateLogger(path.Join(logDir, KeepAliveLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetKeepAliveLogger(keepAliveLogger.Sugar())
-	return nil
+	return createLogger(meta, logDir)
 }
 
 func InitDaemon(console bool, dir string) error {
@@ -155,25 +158,22 @@ func InitDaemon(console bool, dir string) error {
 
 	logDir := filepath.Join(dir, "daemon")
 
-	coreLogger, err := CreateLogger(path.Join(logDir, CoreLogFileName), false, false)
-	if err != nil {
-		return err
+	var meta = []logInitMeta{
+		{
+			fileName:             CoreLogFileName,
+			setSugaredLoggerFunc: logger.SetCoreLogger,
+		},
+		{
+			fileName:             GrpcLogFileName,
+			setSugaredLoggerFunc: logger.SetGrpcLogger,
+		},
+		{
+			fileName:             GCLogFileName,
+			setSugaredLoggerFunc: logger.SetGCLogger,
+		},
 	}
-	logger.SetCoreLogger(coreLogger.Sugar())
 
-	grpcLogger, err := CreateLogger(path.Join(logDir, GrpcLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGrpcLogger(grpcLogger.Sugar())
-
-	gcLogger, err := CreateLogger(path.Join(logDir, GCLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGCLogger(gcLogger.Sugar())
-
-	return nil
+	return createLogger(meta, logDir)
 }
 
 func InitDfget(console bool, dir string) error {
@@ -183,17 +183,16 @@ func InitDfget(console bool, dir string) error {
 
 	logDir := filepath.Join(dir, "dfget")
 
-	coreLogger, err := CreateLogger(path.Join(logDir, CoreLogFileName), false, false)
-	if err != nil {
-		return err
+	var meta = []logInitMeta{
+		{
+			fileName:             CoreLogFileName,
+			setSugaredLoggerFunc: logger.SetCoreLogger,
+		},
+		{
+			fileName:             GrpcLogFileName,
+			setSugaredLoggerFunc: logger.SetGrpcLogger,
+		},
 	}
-	logger.SetCoreLogger(coreLogger.Sugar())
 
-	grpcLogger, err := CreateLogger(path.Join(logDir, GrpcLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGrpcLogger(grpcLogger.Sugar())
-
-	return nil
+	return createLogger(meta, logDir)
 }

internal/dflog/logcore/logcore.go

@@ -48,7 +48,7 @@ const (
 var coreLevel = zap.NewAtomicLevelAt(zapcore.InfoLevel)
 var grpcLevel = zap.NewAtomicLevelAt(zapcore.WarnLevel)
 
-func CreateLogger(filePath string, compress bool, stats bool) (*zap.Logger, error) {
+func CreateLogger(filePath string, compress bool, stats bool) (*zap.Logger, zap.AtomicLevel, error) {
 	rotateConfig := &lumberjack.Logger{
 		Filename:   filePath,
 		MaxSize:    defaultRotateMaxSize,
@@ -80,7 +80,7 @@ func CreateLogger(filePath string, compress bool, stats bool) (*zap.Logger, erro
 		opts = append(opts, zap.AddCaller(), zap.AddStacktrace(zap.WarnLevel), zap.AddCallerSkip(1))
 	}
 
-	return zap.New(core, opts...), nil
+	return zap.New(core, opts...), level, nil
 }
 
 func SetCoreLevel(level zapcore.Level) {

internal/dflog/logcore/signal.go

@@ -0,0 +1,58 @@
+/*
+ *     Copyright 2022 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package logcore
+
+import (
+	"fmt"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go.uber.org/zap"
+	"go.uber.org/zap/zapcore"
+
+	logger "d7y.io/dragonfly/v2/internal/dflog"
+)
+
+var (
+	levels []zap.AtomicLevel
+	level  = zapcore.InfoLevel
+)
+
+func startLoggerSignalHandler() {
+	signals := make(chan os.Signal, 1)
+	signal.Notify(signals, syscall.SIGUSR1)
+
+	go func() {
+		for {
+			select {
+			case <-signals:
+				level--
+				if level < zapcore.DebugLevel {
+					level = zapcore.FatalLevel
+				}
+
+				// use fmt.Printf print change log level event when log level is greater than info level
+				fmt.Printf("change log level to %s\n", level.String())
+				logger.Infof("change log level to %s", level.String())
+				for _, l := range levels {
+					l.SetLevel(level)
+				}
+			}
+		}
+	}()
+}

pkg/log/log.go

@@ -17,11 +17,8 @@
 package log
 
 import (
-	"path"
-
 	"go.uber.org/zap/zapcore"
 
-	logger "d7y.io/dragonfly/v2/internal/dflog"
 	"d7y.io/dragonfly/v2/internal/dflog/logcore"
 	"d7y.io/dragonfly/v2/pkg/dfpath"
 )
@@ -52,25 +49,5 @@ func SetupDaemon(logDir string, console bool) error {
 		return err
 	}
 
-	logDir = path.Join(d.LogDir(), "daemon")
-
-	coreLogger, err := logcore.CreateLogger(path.Join(logDir, logcore.CoreLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetCoreLogger(coreLogger.Sugar())
-
-	grpcLogger, err := logcore.CreateLogger(path.Join(logDir, logcore.GrpcLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGrpcLogger(grpcLogger.Sugar())
-
-	gcLogger, err := logcore.CreateLogger(path.Join(logDir, logcore.GCLogFileName), false, false)
-	if err != nil {
-		return err
-	}
-	logger.SetGCLogger(gcLogger.Sugar())
-
-	return nil
+	return logcore.InitDaemon(false, d.LogDir())
 }