mirror of https://github.com/etcd-io/dbtester.git
analyze: save aggregated network, disk stats
This commit is contained in:
Gyu-Ho Lee
parent
f435232125
commit
f4cb9603d0
|
|
@ -330,3 +330,12 @@ func (data *analyzeData) save() error {
|
|||
func makeHeader(column string, tag string) string {
|
||||
return fmt.Sprintf("%s-%s", column, tag)
|
||||
}
|
||||
|
||||
func makeTag(legend string) string {
|
||||
legend = strings.ToLower(legend)
|
||||
legend = strings.Replace(legend, "go ", "go", -1)
|
||||
legend = strings.Replace(legend, "java ", "java", -1)
|
||||
legend = strings.Replace(legend, "(", "", -1)
|
||||
legend = strings.Replace(legend, ")", "", -1)
|
||||
return strings.Replace(legend, " ", "-", -1)
|
||||
}
|
||||
|
|
|
|||
|
|
@ -19,7 +19,9 @@ import (
|
|||
"path/filepath"
|
||||
"reflect"
|
||||
"sort"
|
||||
"strings"
|
||||
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/gyuho/dataframe"
|
||||
)
|
||||
|
||||
|
|
@ -71,6 +73,172 @@ func do(configPath string) error {
|
|||
}
|
||||
}
|
||||
|
||||
plog.Println("saving data to %q", cfg.AllAggregatedPath)
|
||||
var (
|
||||
readsCompletedDeltaSumColumns []dataframe.Column
|
||||
sectorsReadDeltaSumColumns []dataframe.Column
|
||||
writesCompletedDeltaSumColumns []dataframe.Column
|
||||
sectorsWrittenDeltaSumColumns []dataframe.Column
|
||||
receiveBytesNumDeltaSumColumns []dataframe.Column
|
||||
receiveBytesColumns []dataframe.Column
|
||||
transmitBytesNumDeltaSumColumns []dataframe.Column
|
||||
transmitBytesColumns []dataframe.Column
|
||||
)
|
||||
for _, ad := range all.data {
|
||||
var (
|
||||
legend string
|
||||
readsCompletedDeltaSum float64
|
||||
sectorsReadDeltaSum float64
|
||||
writesCompletedDeltaSum float64
|
||||
sectorsWrittenDeltaSum float64
|
||||
receiveBytesNumDeltaSum float64
|
||||
transmitBytesNumDeltaSum float64
|
||||
)
|
||||
for _, col := range ad.aggregated.Columns() {
|
||||
hdr := col.Header()
|
||||
legend = all.headerToLegend[hdr]
|
||||
|
||||
switch {
|
||||
case strings.HasPrefix(hdr, "READS-COMPLETED-DELTA-"):
|
||||
cnt := col.Count()
|
||||
for j := 0; j < cnt; j++ {
|
||||
vv, err := col.Value(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fv, _ := vv.Number()
|
||||
readsCompletedDeltaSum += fv
|
||||
}
|
||||
case strings.HasPrefix(hdr, "SECTORS-READS-DELTA-"):
|
||||
cnt := col.Count()
|
||||
for j := 0; j < cnt; j++ {
|
||||
vv, err := col.Value(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fv, _ := vv.Number()
|
||||
sectorsReadDeltaSum += fv
|
||||
}
|
||||
case strings.HasPrefix(hdr, "WRITES-COMPLETED-DELTA-"):
|
||||
cnt := col.Count()
|
||||
for j := 0; j < cnt; j++ {
|
||||
vv, err := col.Value(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fv, _ := vv.Number()
|
||||
writesCompletedDeltaSum += fv
|
||||
}
|
||||
case strings.HasPrefix(hdr, "SECTORS-WRITTEN-DELTA-"):
|
||||
cnt := col.Count()
|
||||
for j := 0; j < cnt; j++ {
|
||||
vv, err := col.Value(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fv, _ := vv.Number()
|
||||
sectorsWrittenDeltaSum += fv
|
||||
}
|
||||
case strings.HasPrefix(hdr, "RECEIVE-BYTES-NUM-DELTA-"):
|
||||
cnt := col.Count()
|
||||
for j := 0; j < cnt; j++ {
|
||||
vv, err := col.Value(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fv, _ := vv.Number()
|
||||
receiveBytesNumDeltaSum += fv
|
||||
}
|
||||
case strings.HasPrefix(hdr, "TRANSMIT-BYTES-NUM-DELTA-"):
|
||||
cnt := col.Count()
|
||||
for j := 0; j < cnt; j++ {
|
||||
vv, err := col.Value(j)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
fv, _ := vv.Number()
|
||||
transmitBytesNumDeltaSum += fv
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
col1 := dataframe.NewColumn("READS-COMPLETED-DELTA-SUM-" + makeTag(legend))
|
||||
col1.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", uint64(readsCompletedDeltaSum))))
|
||||
readsCompletedDeltaSumColumns = append(readsCompletedDeltaSumColumns, col1)
|
||||
|
||||
col2 := dataframe.NewColumn("SECTORS-READS-DELTA-SUM-" + makeTag(legend))
|
||||
col2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", uint64(sectorsReadDeltaSum))))
|
||||
sectorsReadDeltaSumColumns = append(sectorsReadDeltaSumColumns, col2)
|
||||
|
||||
col3 := dataframe.NewColumn("WRITES-COMPLETED-DELTA-SUM-" + makeTag(legend))
|
||||
col3.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", uint64(writesCompletedDeltaSum))))
|
||||
writesCompletedDeltaSumColumns = append(writesCompletedDeltaSumColumns, col3)
|
||||
|
||||
col4 := dataframe.NewColumn("SECTORS-WRITTEN-DELTA-SUM-" + makeTag(legend))
|
||||
col4.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", uint64(sectorsWrittenDeltaSum))))
|
||||
sectorsWrittenDeltaSumColumns = append(sectorsWrittenDeltaSumColumns, col4)
|
||||
|
||||
col5 := dataframe.NewColumn("RECEIVE-BYTES-SUM-" + makeTag(legend))
|
||||
col5.PushBack(dataframe.NewStringValue(humanize.Bytes(uint64(receiveBytesNumDeltaSum))))
|
||||
receiveBytesNumDeltaSumColumns = append(receiveBytesNumDeltaSumColumns, col5)
|
||||
|
||||
col6 := dataframe.NewColumn("RECEIVE-BYTES-NUM-DELTA-SUM-" + makeTag(legend))
|
||||
col6.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", uint64(receiveBytesNumDeltaSum))))
|
||||
receiveBytesColumns = append(receiveBytesColumns, col6)
|
||||
|
||||
col7 := dataframe.NewColumn("TRANSMIT-BYTES-SUM-" + makeTag(legend))
|
||||
col7.PushBack(dataframe.NewStringValue(humanize.Bytes(uint64(transmitBytesNumDeltaSum))))
|
||||
transmitBytesNumDeltaSumColumns = append(transmitBytesNumDeltaSumColumns, col7)
|
||||
|
||||
col8 := dataframe.NewColumn("TRANSMIT-BYTES-NUM-DELTA-SUM-" + makeTag(legend))
|
||||
col8.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", uint64(transmitBytesNumDeltaSum))))
|
||||
transmitBytesColumns = append(transmitBytesColumns, col8)
|
||||
}
|
||||
aggDf := dataframe.New()
|
||||
for _, col := range readsCompletedDeltaSumColumns {
|
||||
if err := aggDf.AddColumn(col); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, col := range sectorsReadDeltaSumColumns {
|
||||
if err := aggDf.AddColumn(col); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, col := range writesCompletedDeltaSumColumns {
|
||||
if err := aggDf.AddColumn(col); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, col := range sectorsWrittenDeltaSumColumns {
|
||||
if err := aggDf.AddColumn(col); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, col := range receiveBytesNumDeltaSumColumns {
|
||||
if err := aggDf.AddColumn(col); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, col := range receiveBytesColumns {
|
||||
if err := aggDf.AddColumn(col); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, col := range transmitBytesNumDeltaSumColumns {
|
||||
if err := aggDf.AddColumn(col); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
for _, col := range transmitBytesColumns {
|
||||
if err := aggDf.AddColumn(col); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
if err := aggDf.CSVHorizontal(cfg.AllAggregatedPath); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
plog.Println("combining data for plotting")
|
||||
for _, plotConfig := range cfg.PlotList {
|
||||
plog.Printf("plotting %q", plotConfig.Column)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
title: Write 2M keys, 1000-client (etcd 100 TCP conns), 8-byte key, 256-byte value
|
||||
work_dir: 2017Q1-02-etcd-zookeeper-consul/01-write-2M-keys
|
||||
all_aggregated_path: 2017Q1-02-etcd-zookeeper-consul/01-write-2M-keys/data-aggregated.csv
|
||||
all_aggregated_path: 2017Q1-02-etcd-zookeeper-consul/01-write-2M-keys/aggregated.csv
|
||||
|
||||
raw_data:
|
||||
- legend: etcd v3.1 (Go 1.7.4)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
title: Write 2M keys, 1000-client (etcd 100 TCP conns), 8-byte key, 256-byte value
|
||||
work_dir: 2017Q1-02-etcd-zookeeper-consul/02-write-2M-keys-rate-limited
|
||||
all_aggregated_path: 2017Q1-02-etcd-zookeeper-consul/02-write-2M-keys-rate-limited/data-aggregated.csv
|
||||
all_aggregated_path: 2017Q1-02-etcd-zookeeper-consul/02-write-2M-keys-rate-limited/aggregated.csv
|
||||
|
||||
raw_data:
|
||||
- legend: etcd v3.1 (Go 1.7.4)
|
||||
|
|
|
|||
|
|
@ -1,6 +1,6 @@
|
|||
title: Write 2M keys, 8-byte key, 256-byte value (clients 1 to 1000)
|
||||
work_dir: 2017Q1-02-etcd-zookeeper-consul/03-write-2M-keys-client-variable
|
||||
all_aggregated_path: 2017Q1-02-etcd-zookeeper-consul/03-write-2M-keys-client-variable/data-aggregated.csv
|
||||
all_aggregated_path: 2017Q1-02-etcd-zookeeper-consul/03-write-2M-keys-client-variable/aggregated.csv
|
||||
|
||||
raw_data:
|
||||
- legend: etcd v3.1 (Go 1.7.4)
|
||||
|
|
|
|||
Loading…
Reference in New Issue