Optimized the log creation (#1058)

* optimize logger

Signed-off-by: sunwp <244372610@qq.com>

* resolve rpc server logger panic

Signed-off-by: sunwp <244372610@qq.com>

* resolve rpc server logger panic

Signed-off-by: sunwp <244372610@qq.com>

* reset levels nil to release init method level

Signed-off-by: sunwp <244372610@qq.com>

* golang lint

Signed-off-by: sunwp <244372610@qq.com>

cmd/cdn/cmd/root.go

@@ -56,7 +56,7 @@ from remote source repeatedly.`,
 		}
 
 		// Initialize logger
-		if err := logger.InitCdnSystem(cfg.Console, d.LogDir()); err != nil {
+		if err := logger.InitCdnSystem(cfg.Verbose, cfg.Console, d.LogDir()); err != nil {
 			return errors.Wrap(err, "init cdn system logger")
 		}
 
@@ -104,7 +104,7 @@ func runCdnSystem() error {
 	// cdn system config values
 	logger.Infof("cdn system configuration:\n%s", cfg)
 
-	ff := dependency.InitMonitor(cfg.Verbose, cfg.PProfPort, cfg.Telemetry)
+	ff := dependency.InitMonitor(cfg.PProfPort, cfg.Telemetry)
 	defer ff()
 
 	svr, err := cdn.New(cfg)

cmd/dependency/dependency.go

@@ -40,7 +40,6 @@ import (
 	"go.opentelemetry.io/otel/sdk/resource"
 	sdktrace "go.opentelemetry.io/otel/sdk/trace"
 	semconv "go.opentelemetry.io/otel/semconv/v1.7.0"
-	"go.uber.org/zap/zapcore"
 	"gopkg.in/yaml.v3"
 
 	"d7y.io/dragonfly/v2/client/clientutil"
@@ -89,13 +88,9 @@ func InitCobra(cmd *cobra.Command, useConfigFile bool, config interface{}) {
 }
 
 // InitMonitor initialize monitor and return final handler
-func InitMonitor(verbose bool, pprofPort int, otelOption base.TelemetryOption) func() {
+func InitMonitor(pprofPort int, otelOption base.TelemetryOption) func() {
 	var fc = make(chan func(), 5)
 
-	if verbose {
-		logger.SetLevel(zapcore.DebugLevel)
-	}
-
 	if pprofPort >= 0 {
 		// Enable go pprof and statsview
 		go func() {

cmd/dfget/cmd/daemon.go

@@ -60,7 +60,7 @@ it supports container engine, wget and other downloading tools through proxy fun
 		}
 
 		// Initialize logger
-		if err := logger.InitDaemon(cfg.Console, d.LogDir()); err != nil {
+		if err := logger.InitDaemon(cfg.Verbose, cfg.Console, d.LogDir()); err != nil {
 			return errors.Wrap(err, "init client daemon logger")
 		}
 
@@ -170,7 +170,7 @@ func runDaemon(d dfpath.Dfpath) error {
 	s, _ := yaml.Marshal(cfg)
 	logger.Infof("client daemon configuration:\n%s", string(s))
 
-	ff := dependency.InitMonitor(cfg.Verbose, cfg.PProfPort, cfg.Telemetry)
+	ff := dependency.InitMonitor(cfg.PProfPort, cfg.Telemetry)
 	defer ff()
 
 	svr, err := server.New(cfg, d)

cmd/dfget/cmd/root.go

@@ -76,7 +76,7 @@ var rootCmd = &cobra.Command{
 		}
 
 		// Initialize logger
-		if err := logger.InitDfget(dfgetConfig.Console, d.LogDir()); err != nil {
+		if err := logger.InitDfget(dfgetConfig.Verbose, dfgetConfig.Console, d.LogDir()); err != nil {
 			return errors.Wrap(err, "init client dfget logger")
 		}
 
@@ -215,7 +215,7 @@ func runDfget(dfgetLockPath, daemonSockPath string) error {
 	s, _ := yaml.Marshal(dfgetConfig)
 	logger.Infof("client dfget configuration:\n%s", string(s))
 
-	ff := dependency.InitMonitor(dfgetConfig.Verbose, dfgetConfig.PProfPort, dfgetConfig.Telemetry)
+	ff := dependency.InitMonitor(dfgetConfig.PProfPort, dfgetConfig.Telemetry)
 	defer ff()
 
 	var (

cmd/manager/cmd/root.go

@@ -52,7 +52,7 @@ for managing schedulers and cdns, offering http apis and portal, etc.`,
 		}
 
 		// Initialize logger
-		if err := logger.InitManager(cfg.Console, d.LogDir()); err != nil {
+		if err := logger.InitManager(cfg.Verbose, cfg.Console, d.LogDir()); err != nil {
 			return errors.Wrap(err, "init manager logger")
 		}
 
@@ -102,7 +102,7 @@ func runManager(d dfpath.Dfpath) error {
 
 	logger.Infof("manager configuration:\n%s", string(s))
 
-	ff := dependency.InitMonitor(cfg.Verbose, cfg.PProfPort, cfg.Telemetry)
+	ff := dependency.InitMonitor(cfg.PProfPort, cfg.Telemetry)
 	defer ff()
 
 	svr, err := manager.New(cfg, d)

cmd/scheduler/cmd/root.go

@@ -56,7 +56,7 @@ generate and maintain a P2P network during the download process, and push suitab
 		}
 
 		// Initialize logger
-		if err := logger.InitScheduler(cfg.Console, d.LogDir()); err != nil {
+		if err := logger.InitScheduler(cfg.Verbose, cfg.Console, d.LogDir()); err != nil {
 			return errors.Wrap(err, "init scheduler logger")
 		}
 
@@ -106,7 +106,7 @@ func runScheduler(ctx context.Context, d dfpath.Dfpath) error {
 
 	logger.Infof("scheduler configuration:\n%s", string(s))
 
-	ff := dependency.InitMonitor(cfg.Verbose, cfg.PProfPort, cfg.Telemetry)
+	ff := dependency.InitMonitor(cfg.PProfPort, cfg.Telemetry)
 	defer ff()
 
 	svr, err := scheduler.New(ctx, cfg, d)

internal/dflog/logcore.go

@@ -19,6 +19,7 @@ package logger
 import (
 	"strings"
 
+	"go.uber.org/atomic"
 	"go.uber.org/zap"
 	"go.uber.org/zap/zapcore"
 	"gopkg.in/natefinch/lumberjack.v2"
@@ -46,9 +47,11 @@ const (
 )
 
 var coreLevel = zap.NewAtomicLevelAt(zapcore.InfoLevel)
+var customCoreLevel atomic.Bool
 var grpcLevel = zap.NewAtomicLevelAt(zapcore.WarnLevel)
+var customGrpcLevel atomic.Bool
 
-func CreateLogger(filePath string, compress bool, stats bool) (*zap.Logger, zap.AtomicLevel, error) {
+func CreateLogger(filePath string, compress bool, stats bool, verbose bool) (*zap.Logger, zap.AtomicLevel, error) {
 	rotateConfig := &lumberjack.Logger{
 		Filename:   filePath,
 		MaxSize:    defaultRotateMaxSize,
@@ -61,11 +64,13 @@ func CreateLogger(filePath string, compress bool, stats bool) (*zap.Logger, zap.
 
 	encoderConfig := zap.NewProductionEncoderConfig()
 	encoderConfig.EncodeTime = zapcore.TimeEncoderOfLayout(encodeTimeFormat)
-
-	level := zap.NewAtomicLevel()
-	if strings.HasSuffix(filePath, GrpcLogFileName) {
+	var level = zap.NewAtomicLevel()
+	if verbose {
+		level = zap.NewAtomicLevelAt(zapcore.DebugLevel)
+	}
+	if strings.HasSuffix(filePath, GrpcLogFileName) && customGrpcLevel.Load() {
 		level = grpcLevel
-	} else if strings.HasSuffix(filePath, CoreLogFileName) {
+	} else if strings.HasSuffix(filePath, CoreLogFileName) && customCoreLevel.Load() {
 		level = coreLevel
 	}
 
@@ -84,9 +89,11 @@ func CreateLogger(filePath string, compress bool, stats bool) (*zap.Logger, zap.
 }
 
 func SetCoreLevel(level zapcore.Level) {
+	customCoreLevel.Store(true)
 	coreLevel.SetLevel(level)
 }
 
 func SetGrpcLevel(level zapcore.Level) {
+	customGrpcLevel.Store(true)
 	grpcLevel.SetLevel(level)
 }

internal/dflog/loginit.go

@@ -29,32 +29,11 @@ type logInitMeta struct {
 	setLoggerFunc        func(log *zap.Logger)
 }
 
-func createLogger(console bool, meta []logInitMeta, logDir string) error {
+func InitManager(verbose, console bool, dir string) error {
 	if console {
-		startLoggerSignalHandler()
-		return nil
+		return createConsoleLogger(verbose)
 	}
-	// drop console level
-	levels = nil
 
-	for _, m := range meta {
-		log, level, err := CreateLogger(path.Join(logDir, m.fileName), false, false)
-		if err != nil {
-			return err
-		}
-		if m.setSugaredLoggerFunc != nil {
-			m.setSugaredLoggerFunc(log.Sugar())
-		} else {
-			m.setLoggerFunc(log)
-		}
-
-		levels = append(levels, level)
-	}
-	startLoggerSignalHandler()
-	return nil
-}
-
-func InitManager(console bool, dir string) error {
 	logDir := filepath.Join(dir, "manager")
 
 	var meta = []logInitMeta{
@@ -76,10 +55,14 @@ func InitManager(console bool, dir string) error {
 		},
 	}
 
-	return createLogger(console, meta, logDir)
+	return createFileLogger(verbose, meta, logDir)
 }
 
-func InitScheduler(console bool, dir string) error {
+func InitScheduler(verbose, console bool, dir string) error {
+	if console {
+		return createConsoleLogger(verbose)
+	}
+
 	logDir := filepath.Join(dir, "scheduler")
 
 	var meta = []logInitMeta{
@@ -101,10 +84,13 @@ func InitScheduler(console bool, dir string) error {
 		},
 	}
 
-	return createLogger(console, meta, logDir)
+	return createFileLogger(verbose, meta, logDir)
 }
 
-func InitCdnSystem(console bool, dir string) error {
+func InitCdnSystem(verbose, console bool, dir string) error {
+	if console {
+		return createConsoleLogger(verbose)
+	}
 	logDir := filepath.Join(dir, "cdn")
 	var meta = []logInitMeta{
 		{
@@ -141,10 +127,14 @@ func InitCdnSystem(console bool, dir string) error {
 		},
 	}
 
-	return createLogger(console, meta, logDir)
+	return createFileLogger(verbose, meta, logDir)
 }
 
-func InitDaemon(console bool, dir string) error {
+func InitDaemon(verbose, console bool, dir string) error {
+	if console {
+		return createConsoleLogger(verbose)
+	}
+
 	logDir := filepath.Join(dir, "daemon")
 
 	var meta = []logInitMeta{
@@ -162,10 +152,14 @@ func InitDaemon(console bool, dir string) error {
 		},
 	}
 
-	return createLogger(console, meta, logDir)
+	return createFileLogger(verbose, meta, logDir)
 }
 
-func InitDfget(console bool, dir string) error {
+func InitDfget(verbose, console bool, dir string) error {
+	if console {
+		return createConsoleLogger(verbose)
+	}
+
 	logDir := filepath.Join(dir, "dfget")
 
 	var meta = []logInitMeta{
@@ -179,5 +173,49 @@ func InitDfget(console bool, dir string) error {
 		},
 	}
 
-	return createLogger(console, meta, logDir)
+	return createFileLogger(verbose, meta, logDir)
+}
+
+func createConsoleLogger(verbose bool) error {
+	levels = nil
+	config := zap.NewDevelopmentConfig()
+	config.Level = zap.NewAtomicLevelAt(zap.InfoLevel)
+	if verbose {
+		config.Level = zap.NewAtomicLevelAt(zap.DebugLevel)
+	}
+	log, err := config.Build(zap.AddCaller(), zap.AddStacktrace(zap.WarnLevel), zap.AddCallerSkip(1))
+	if err == nil {
+		sugar := log.Sugar()
+		SetCoreLogger(sugar)
+		SetGrpcLogger(sugar)
+		SetGCLogger(sugar)
+		SetStorageGCLogger(sugar)
+		SetKeepAliveLogger(sugar)
+		SetStatSeedLogger(log)
+		SetDownloadLogger(log)
+		SetJobLogger(sugar)
+	}
+	levels = append(levels, config.Level)
+	startLoggerSignalHandler()
+	return nil
+}
+
+func createFileLogger(verbose bool, meta []logInitMeta, logDir string) error {
+	levels = nil
+
+	for _, m := range meta {
+		log, level, err := CreateLogger(path.Join(logDir, m.fileName), false, false, verbose)
+		if err != nil {
+			return err
+		}
+		if m.setSugaredLoggerFunc != nil {
+			m.setSugaredLoggerFunc(log.Sugar())
+		} else {
+			m.setLoggerFunc(log)
+		}
+
+		levels = append(levels, level)
+	}
+	startLoggerSignalHandler()
+	return nil
 }

pkg/log/log.go

@@ -34,10 +34,7 @@ func SetGrpcLevel(level zapcore.Level) {
 }
 
 // SetupDaemon sets daemon log config: path, console
-func SetupDaemon(logDir string, console bool) error {
-	if console {
-		return nil
-	}
+func SetupDaemon(logDir string, verbose bool, console bool) error {
 
 	var options []dfpath.Option
 	if logDir != "" {
@@ -49,5 +46,5 @@ func SetupDaemon(logDir string, console bool) error {
 		return err
 	}
 
-	return logger.InitDaemon(false, d.LogDir())
+	return logger.InitDaemon(verbose, console, d.LogDir())
 }

pkg/rpc/cdnsystem/server/server.go

@@ -51,7 +51,7 @@ type proxy struct {
 }
 
 func New(seederServer SeederServer, opts ...grpc.ServerOption) *grpc.Server {
-	grpcServer := grpc.NewServer(append(rpc.DefaultServerOptions, opts...)...)
+	grpcServer := grpc.NewServer(append(rpc.DefaultServerOptions(), opts...)...)
 	cdnsystem.RegisterSeederServer(grpcServer, &proxy{server: seederServer})
 	return grpcServer
 }

pkg/rpc/dfdaemon/server/server.go

@@ -51,7 +51,7 @@ type proxy struct {
 }
 
 func New(daemonServer DaemonServer, opts ...grpc.ServerOption) *grpc.Server {
-	grpcServer := grpc.NewServer(append(rpc.DefaultServerOptions, opts...)...)
+	grpcServer := grpc.NewServer(append(rpc.DefaultServerOptions(), opts...)...)
 	dfdaemon.RegisterDaemonServer(grpcServer, &proxy{server: daemonServer})
 	return grpcServer
 }

pkg/rpc/server.go

@@ -35,28 +35,31 @@ import (
 	"d7y.io/dragonfly/v2/pkg/rpc/base/common"
 )
 
-var DefaultServerOptions = []grpc.ServerOption{
-	grpc.ConnectionTimeout(10 * time.Second),
-	grpc.InitialConnWindowSize(8 * 1024 * 1024),
-	grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
-		MinTime: 30 * time.Second,
-	}),
-	grpc.KeepaliveParams(keepalive.ServerParameters{
-		MaxConnectionIdle: 5 * time.Minute,
-	}),
-	grpc.MaxConcurrentStreams(100),
-	grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
-		streamServerInterceptor,
-		grpc_prometheus.StreamServerInterceptor,
-		grpc_zap.StreamServerInterceptor(logger.GrpcLogger.Desugar()),
-		grpc_validator.StreamServerInterceptor(),
-	)),
-	grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
-		unaryServerInterceptor,
-		grpc_prometheus.UnaryServerInterceptor,
-		grpc_zap.UnaryServerInterceptor(logger.GrpcLogger.Desugar()),
-		grpc_validator.UnaryServerInterceptor(),
-	)),
+func DefaultServerOptions() []grpc.ServerOption {
+	return []grpc.ServerOption{
+		grpc.ConnectionTimeout(10 * time.Second),
+		grpc.InitialConnWindowSize(8 * 1024 * 1024),
+		grpc.KeepaliveEnforcementPolicy(keepalive.EnforcementPolicy{
+			MinTime: 1 * time.Minute,
+		}),
+		grpc.KeepaliveParams(keepalive.ServerParameters{
+			MaxConnectionIdle: 5 * time.Minute,
+		}),
+		grpc.MaxConcurrentStreams(100),
+		grpc.StreamInterceptor(grpc_middleware.ChainStreamServer(
+			streamServerInterceptor,
+			grpc_prometheus.StreamServerInterceptor,
+			grpc_zap.StreamServerInterceptor(logger.GrpcLogger.Desugar()),
+			grpc_validator.StreamServerInterceptor(),
+		)),
+		grpc.UnaryInterceptor(grpc_middleware.ChainUnaryServer(
+			unaryServerInterceptor,
+			grpc_prometheus.UnaryServerInterceptor,
+			grpc_zap.UnaryServerInterceptor(logger.GrpcLogger.Desugar()),
+			grpc_validator.UnaryServerInterceptor(),
+		)),
+	}
+
 }
 
 func streamServerInterceptor(srv interface{}, ss grpc.ServerStream, info *grpc.StreamServerInfo, handler grpc.StreamHandler) error {

scheduler/rpcserver/rpcserver.go

@@ -41,7 +41,7 @@ type Server struct {
 // New returns a new transparent scheduler server from the given options
 func New(service *service.Service, opts ...grpc.ServerOption) *grpc.Server {
 	svr := &Server{service: service}
-	grpcServer := grpc.NewServer(append(rpc.DefaultServerOptions, opts...)...)
+	grpcServer := grpc.NewServer(append(rpc.DefaultServerOptions(), opts...)...)
 	scheduler.RegisterSchedulerServer(grpcServer, svr)
 	return grpcServer
 }