diff --git a/cmd/dfbench/cmd/dragonfly.go b/cmd/dfbench/cmd/dragonfly.go index 2a75feb..e449509 100644 --- a/cmd/dfbench/cmd/dragonfly.go +++ b/cmd/dfbench/cmd/dragonfly.go @@ -62,7 +62,7 @@ func init() { // runDragonfly runs the dragonfly benchmark. func runDragonfly(ctx context.Context, cfg *config.Config) error { - stats := stats.New() + stats := stats.New(cfg.Dragonfly.Namespace) fileServer := backend.NewFileServer(cfg.Dragonfly.Namespace) dragonfly := dragonfly.New(cfg.Dragonfly.Namespace, fileServer, stats) @@ -73,7 +73,11 @@ func runDragonfly(ctx context.Context, cfg *config.Config) error { logrus.Errorf("failed to run dragonfly benchmark: %v", err) return err } - stats.PrettyPrint() + + if err := stats.PrettyPrint(); err != nil { + logrus.Errorf("failed to print dragonfly benchmark statistics: %v", err) + return err + } if err := dragonfly.Cleanup(ctx); err != nil { logrus.Errorf("failed to cleanup dragonfly benchmark: %v", err) @@ -88,7 +92,11 @@ func runDragonfly(ctx context.Context, cfg *config.Config) error { logrus.Errorf("failed to run dragonfly benchmark: %v", err) return err } - stats.PrettyPrint() + + if err := stats.PrettyPrint(); err != nil { + logrus.Errorf("failed to print dragonfly benchmark statistics: %v", err) + return err + } if err := dragonfly.Cleanup(ctx); err != nil { logrus.Errorf("failed to cleanup dragonfly benchmark: %v", err) diff --git a/go.mod b/go.mod index 39c9bde..2b00c41 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.23.0 require ( github.com/google/uuid v1.4.0 github.com/olekukonko/tablewriter v0.0.5 + github.com/prometheus/client_model v0.6.1 + github.com/prometheus/common v0.60.1 github.com/sirupsen/logrus v1.9.3 github.com/spf13/cobra v1.8.1 github.com/spf13/viper v1.19.0 @@ -18,6 +20,7 @@ require ( github.com/magiconair/properties v1.8.7 // indirect github.com/mattn/go-runewidth v0.0.9 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/pelletier/go-toml/v2 v2.2.2 // indirect github.com/sagikazarmark/locafero v0.4.0 // indirect github.com/sagikazarmark/slog-shim v0.1.0 // indirect @@ -29,8 +32,9 @@ require ( go.uber.org/atomic v1.9.0 // indirect go.uber.org/multierr v1.9.0 // indirect golang.org/x/exp v0.0.0-20230905200255-921286631fa9 // indirect - golang.org/x/sys v0.18.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sys v0.25.0 // indirect + golang.org/x/text v0.18.0 // indirect + google.golang.org/protobuf v1.34.2 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect ) diff --git a/go.sum b/go.sum index ad0a2e9..9046af2 100644 --- a/go.sum +++ b/go.sum @@ -7,8 +7,8 @@ github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHk github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nosvA= github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.4.0 h1:MtMxsa51/r9yyhkyLsVeVt0B+BGQZzpQiTQ4eHZ8bc4= github.com/google/uuid v1.4.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -25,6 +25,8 @@ github.com/mattn/go-runewidth v0.0.9 h1:Lm995f3rfxdpd6TSmuVCHVb/QhupuXlYr8sCI/Qd github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 h1:C3w9PqII01/Oq1c1nUAm88MOHcQC9l5mIlSMApZMrHA= +github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/pelletier/go-toml/v2 v2.2.2 h1:aYUidT7k73Pcl9nb2gScu7NSrKCSHIDE89b3+6Wq+LM= @@ -32,8 +34,12 @@ github.com/pelletier/go-toml/v2 v2.2.2/go.mod h1:1t835xjRzz80PqgE6HHgN2JOsmgYu/h github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= -github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/prometheus/client_model v0.6.1 h1:ZKSh/rekM+n3CeS952MLRAdFwIKqeY8b62p8ais2e9E= +github.com/prometheus/client_model v0.6.1/go.mod h1:OrxVMOVHjw3lKMa8+x6HeMGkHMQyHDk9E3jmP2AmGiY= +github.com/prometheus/common v0.60.1 h1:FUas6GcOw66yB/73KC+BOZoFJmbo/1pojoILArPAaSc= +github.com/prometheus/common v0.60.1/go.mod h1:h0LYf1R1deLSKtD4Vdg8gy4RuOvENW2J/h19V5NADQw= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sagikazarmark/locafero v0.4.0 h1:HApY1R9zGo4DBgr7dqsTH/JJxLTTsOt7u6keLGt6kNQ= github.com/sagikazarmark/locafero v0.4.0/go.mod h1:Pe1W6UlPYUk/+wc/6KFhbORCfqzgYEpgQ3O5fPuL3H4= @@ -75,13 +81,15 @@ golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqR golang.org/x/sync v0.9.0 h1:fEo0HyrW1GIgZdpbhCRO0PkJajUS5H9IFUztCgEo2jQ= golang.org/x/sync v0.9.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= -golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34= +golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224= +golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= -gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/pkg/backend/file_server.go b/pkg/backend/file_server.go index 82af27d..76e95d8 100644 --- a/pkg/backend/file_server.go +++ b/pkg/backend/file_server.go @@ -47,6 +47,27 @@ func (f FileSizeLevel) String() string { } } +func (f FileSizeLevel) TaskSizeLevel() string { + switch f { + case FileSizeLevelNano: + return "1" + case FileSizeLevelMicro: + return "1" + case FileSizeLevelSmall: + return "2" + case FileSizeLevelMedium: + return "4" + case FileSizeLevelLarge: + return "11" + case FileSizeLevelXLarge: + return "13" + case FileSizeLevelXXLarge: + return "14" + default: + return "unknown" + } +} + const ( FileSizeLevelNano FileSizeLevel = "nano" FileSizeLevelMicro FileSizeLevel = "micro" diff --git a/pkg/dragonfly/dragonfly.go b/pkg/dragonfly/dragonfly.go index 1480f0a..8c76f22 100644 --- a/pkg/dragonfly/dragonfly.go +++ b/pkg/dragonfly/dragonfly.go @@ -21,7 +21,6 @@ import ( "errors" "fmt" "path" - "time" "github.com/dragonflyoss/perf-tests/pkg/backend" "github.com/dragonflyoss/perf-tests/pkg/config" @@ -200,6 +199,11 @@ func (d *dragonfly) DownloadFileByDfget(ctx context.Context, fileSizeLevel backe return err } + if err := d.stats.CollectClientMetrics(ctx, config.DownloaderDfget, fileSizeLevel); err != nil { + logrus.Errorf("failed to collect client metrics: %v", err) + return err + } + return nil } @@ -217,14 +221,12 @@ func (d *dragonfly) downloadFileByDfget(ctx context.Context, podExec *util.PodEx return err } - createdAt := time.Now() output, err := podExec.Command(ctx, "sh", "-c", fmt.Sprintf("dfget '%s' --output %s", downloadURL.String(), outputPath)).CombinedOutput() if err != nil { logrus.Errorf("failed to download file: %v \nmessage: %s", err, string(output)) return err } - d.stats.AddDownload(downloadURL, config.DownloaderDfget, fileSizeLevel, createdAt, time.Now()) logrus.Debugf("dfget output: %s", string(output)) return nil } @@ -254,6 +256,11 @@ func (d *dragonfly) DownloadFileByProxy(ctx context.Context, fileSizeLevel backe return err } + if err := d.stats.CollectClientMetrics(ctx, config.DownloaderProxy, fileSizeLevel); err != nil { + logrus.Errorf("failed to collect client metrics: %v", err) + return err + } + return nil } @@ -271,13 +278,11 @@ func (d *dragonfly) downloadFileByProxy(ctx context.Context, podExec *util.PodEx return err } - createdAt := time.Now() output, err := podExec.Command(ctx, "sh", "-c", fmt.Sprintf("curl -x %s '%s' --output %s", "http://127.0.0.1:4001", downloadURL.String(), outputPath)).CombinedOutput() if err != nil { logrus.Errorf("failed to download file: %v \nmessage: %s", err, string(output)) return err } - d.stats.AddDownload(downloadURL, config.DownloaderProxy, fileSizeLevel, createdAt, time.Now()) logrus.Debugf("curl output: %s", string(output)) return nil @@ -326,6 +331,7 @@ func (d *dragonfly) getClientPods(ctx context.Context) ([]string, error) { logrus.Errorf("no client pod found") return nil, errors.New("no client pod found") } + return pods, nil } diff --git a/pkg/stats/stats.go b/pkg/stats/stats.go index 953e4f4..92d2252 100644 --- a/pkg/stats/stats.go +++ b/pkg/stats/stats.go @@ -17,40 +17,50 @@ package stats import ( + "bytes" + "context" + "errors" "fmt" - "net/url" + "math" "os" "sync" "time" "github.com/dragonflyoss/perf-tests/pkg/backend" "github.com/dragonflyoss/perf-tests/pkg/config" + "github.com/dragonflyoss/perf-tests/pkg/util" "github.com/google/uuid" "github.com/olekukonko/tablewriter" + dto "github.com/prometheus/client_model/go" + "github.com/prometheus/common/expfmt" + "github.com/sirupsen/logrus" ) // Stats represents the statistics of the benchmark. type Stats interface { - // AddDownload adds a download record to the statistics. - AddDownload(*url.URL, string, backend.FileSizeLevel, time.Time, time.Time) - - // GetDownloads returns all download records. + // GetDownloads returns the download statistics. GetDownloads() []*Download + // CollectClientMetrics collects the client metrics and resets the metrics. + CollectClientMetrics(ctx context.Context, downloader string, fileSizeLevel backend.FileSizeLevel) error + // PrettyPrint prints the statistics in a pretty format. - PrettyPrint() + PrettyPrint() error } // stats implements the Stats interface. type stats struct { // downloads stores the download statistics. downloads *sync.Map + + // namespace is the namespace of the benchmark. + namespace string } // Download represents the download statistics. type Download struct { - // url is the URL of the file. - url *url.URL + // podName is the name of the pod. + podName string // downloader is the downloader used to download the file. downloader string @@ -58,35 +68,18 @@ type Download struct { // fileSizeLevel is the file size level of the file. fileSizeLevel backend.FileSizeLevel - // cost is the time cost of downloading the file. - cost time.Duration - - // createdAt is the time when the download started. - createdAt time.Time - - // finishedAt is the time when the download finished. - finishedAt time.Time + // metricFamilies is the metric families of the download. + metricFamilies map[string]*dto.MetricFamily } // New creates a new Stats instance. -func New() Stats { - return &stats{downloads: &sync.Map{}} -} - -// AddDownload adds a download record to the statistics. -func (s *stats) AddDownload(url *url.URL, downloader string, fileSizeLevel backend.FileSizeLevel, createdAt time.Time, finishedAt time.Time) { - s.downloads.Store(uuid.New().String(), &Download{ - url: url, - downloader: downloader, - fileSizeLevel: fileSizeLevel, - cost: finishedAt.Sub(createdAt), - createdAt: createdAt, - finishedAt: finishedAt, - }) +func New(namespace string) Stats { + return &stats{downloads: &sync.Map{}, namespace: namespace} } +// GetDownloads returns the download statistics. func (s *stats) GetDownloads() []*Download { - downloads := make([]*Download, 0) + downloads := []*Download{} s.downloads.Range(func(key, value interface{}) bool { downloads = append(downloads, value.(*Download)) return true @@ -95,13 +88,90 @@ func (s *stats) GetDownloads() []*Download { return downloads } -// PrettyPrint prints the statistics in a pretty format. -func (s *stats) PrettyPrint() { - downloads := s.GetDownloads() +// collectClientMetrics collects the client metrics. +func (s *stats) CollectClientMetrics(ctx context.Context, downloader string, fileSizeLevel backend.FileSizeLevel) error { + clientPods, err := s.getClientPods(ctx) + if err != nil { + logrus.Errorf("failed to get client pods: %v", err) + return err + } + for _, pod := range clientPods { + data, err := s.getClientMetrics(ctx, pod) + if err != nil { + logrus.Errorf("failed to get client metrics: %v", err) + return err + } + reader := bytes.NewReader(data) + + parser := expfmt.TextParser{} + metricFamilies, err := parser.TextToMetricFamilies(reader) + if err != nil { + logrus.Errorf("failed to parse metrics: %v", err) + return err + } + + s.downloads.Store(uuid.New().String(), &Download{ + podName: pod, + downloader: downloader, + fileSizeLevel: fileSizeLevel, + metricFamilies: metricFamilies, + }) + + if err := s.resetClientMetrics(ctx, pod); err != nil { + logrus.Errorf("failed to reset client metrics: %v", err) + return err + } + } + + return nil +} + +// getClientMetrics collects the client metrics by pod name +func (s *stats) getClientMetrics(ctx context.Context, name string) ([]byte, error) { + podExec := util.NewPodExec(s.namespace, name, "client") + output, err := podExec.Command(ctx, "sh", "-c", "curl -s http://127.0.0.1:4002/metrics").CombinedOutput() + if err != nil { + logrus.Errorf("failed to cleanup: %v \nmessage: %s", err, string(output)) + return nil, err + } + + return output, nil +} + +// resetClientMetrics resets the client metrics by pod name +func (s *stats) resetClientMetrics(ctx context.Context, name string) error { + podExec := util.NewPodExec(s.namespace, name, "client") + output, err := podExec.Command(ctx, "sh", "-c", "curl -s -X DELETE http://127.0.0.1:4002/metrics").CombinedOutput() + if err != nil { + logrus.Errorf("failed to cleanup: %v \nmessage: %s", err, string(output)) + return err + } + + return nil +} + +// getClientPods returns the client pods. +func (s *stats) getClientPods(ctx context.Context) ([]string, error) { + pods, err := util.GetPods(ctx, s.namespace, "component=client") + if err != nil { + logrus.Errorf("failed to get pods: %v", err) + return nil, err + } + + if len(pods) == 0 { + logrus.Errorf("no client pod found") + return nil, errors.New("no client pod found") + } + + return pods, nil +} + +// PrettyPrint prints the statistics in a pretty format. +func (s *stats) PrettyPrint() error { + downloads := s.GetDownloads() proxyDownloads := make(map[backend.FileSizeLevel][]*Download) dfgetDownloads := make(map[backend.FileSizeLevel][]*Download) - for _, download := range downloads { switch download.downloader { case config.DownloaderDfget: @@ -112,40 +182,63 @@ func (s *stats) PrettyPrint() { } if len(dfgetDownloads) != 0 { - printTable(dfgetDownloads) + if err := printTable(dfgetDownloads); err != nil { + return err + } } if len(proxyDownloads) != 0 { - printTable(proxyDownloads) + if err := printTable(proxyDownloads); err != nil { + return err + } } + + return nil } // printTable prints the download statistics in a table format. -func printTable(downloads map[backend.FileSizeLevel][]*Download) { +func printTable(downloads map[backend.FileSizeLevel][]*Download) error { table := tablewriter.NewWriter(os.Stdout) table.SetHeader([]string{"File Size Level", "Times", "Min Cost", "Max Cost", "Avg Cost"}) rows := map[backend.FileSizeLevel][]string{} for fileSizeLevel, records := range downloads { - var minCost, maxCost, totalCost time.Duration - if len(records) > 0 { - minCost = records[0].cost - maxCost = records[0].cost - } + maxCost := time.Duration(-math.MaxInt64) + minCost := time.Duration(math.MaxInt64) + var ( + totalCost time.Duration + n uint64 + ) for _, record := range records { - if record.cost < minCost { - minCost = record.cost - } + for name, mf := range record.metricFamilies { + if name == "dragonfly_client_download_task_duration_milliseconds" { + for _, metrics := range mf.GetMetric() { + for _, label := range metrics.GetLabel() { + if *label.Name == "task_size_level" && *label.Value == fileSizeLevel.TaskSizeLevel() { + if metrics.GetHistogram().GetSampleCount() != 1 { + return errors.New("invalid sample count") + } - if record.cost > maxCost { - maxCost = record.cost - } + cost := time.Duration(int64(metrics.GetHistogram().GetSampleSum()) * int64(time.Millisecond)) + totalCost += cost + n += metrics.GetHistogram().GetSampleCount() - totalCost += record.cost + if cost < minCost { + minCost = cost + } + + if cost > maxCost { + maxCost = cost + } + } + } + } + } + } } - avgCost := totalCost / time.Duration(len(records)) + avgCost := totalCost / time.Duration(n) rows[fileSizeLevel] = []string{ fileSizeLevel.String(), fmt.Sprintf("%d", len(records)), @@ -165,6 +258,7 @@ func printTable(downloads map[backend.FileSizeLevel][]*Download) { table.SetAlignment(tablewriter.ALIGN_LEFT) table.SetRowLine(true) table.Render() + return nil } // formatDuration formats the duration to a string.