mirror of https://github.com/etcd-io/dbtester.git
Merge pull request #295 from gyuho/update
*: add 'psn' ReadBytesDelta, WriteBytesDelta
This commit is contained in:
Gyu-Ho Lee
GitHub
commit
0b4a6ec96e
|
|
@ -38,6 +38,8 @@ var sysMetricsColumnsToRead = []string{
|
|||
"WRITES-COMPLETED-DELTA",
|
||||
"SECTORS-WRITTEN",
|
||||
"SECTORS-WRITTEN-DELTA",
|
||||
"READ-BYTES-DELTA",
|
||||
"WRITE-BYTES-DELTA",
|
||||
"RECEIVE-BYTES-NUM",
|
||||
"RECEIVE-BYTES-NUM-DELTA",
|
||||
"TRANSMIT-BYTES-NUM",
|
||||
|
|
|
|||
|
|
@ -19,11 +19,12 @@ import (
|
|||
"strings"
|
||||
|
||||
"github.com/coreos/dbtester"
|
||||
humanize "github.com/dustin/go-humanize"
|
||||
"github.com/gyuho/dataframe"
|
||||
)
|
||||
|
||||
// aggregateAll aggregates all system metrics from 3+ nodes.
|
||||
func (data *analyzeData) aggregateAll(memoryByKeyPath string, totalRequests int64) error {
|
||||
func (data *analyzeData) aggregateAll(memoryByKeyPath string, readBytesDeltaByKeyPath string, writeBytesDeltaByKeyPath string, totalRequests int64) error {
|
||||
colSys, err := data.sysAgg.Column("UNIX-SECOND")
|
||||
if err != nil {
|
||||
return err
|
||||
|
|
@ -133,6 +134,8 @@ func (data *analyzeData) aggregateAll(memoryByKeyPath string, totalRequests int6
|
|||
avgWritesCompletedDeltaCol = dataframe.NewColumn("AVG-WRITES-COMPLETED-DELTA") // from WRITES-COMPLETED-DELTA
|
||||
avgSectorsWrittenCol = dataframe.NewColumn("AVG-SECTORS-WRITTEN") // from SECTORS-WRITTEN
|
||||
avgSectorsWrittenDeltaCol = dataframe.NewColumn("AVG-SECTORS-WRITTEN-DELTA") // from SECTORS-WRITTEN-DELTA
|
||||
avgReadBytesNumDeltaCol = dataframe.NewColumn("AVG-READ-BYTES-NUM-DELTA") // from READ-BYTES-DELTA
|
||||
avgWriteBytesNumDeltaCol = dataframe.NewColumn("AVG-WRITE-BYTES-NUM-DELTA") // from WRITE-BYTES-DELTA
|
||||
avgReceiveBytesNumCol = dataframe.NewColumn("AVG-RECEIVE-BYTES-NUM") // from RECEIVE-BYTES-NUM
|
||||
avgReceiveBytesNumDeltaCol = dataframe.NewColumn("AVG-RECEIVE-BYTES-NUM-DELTA") // from RECEIVE-BYTES-NUM-DELTA
|
||||
avgTransmitBytesNumCol = dataframe.NewColumn("AVG-TRANSMIT-BYTES-NUM") // from TRANSMIT-BYTES-NUM
|
||||
|
|
@ -160,6 +163,8 @@ func (data *analyzeData) aggregateAll(memoryByKeyPath string, totalRequests int6
|
|||
writesCompletedDeltaSum float64
|
||||
sectorsWrittenSum float64
|
||||
sectorsWrittenDeltaSum float64
|
||||
readBytesDeltaSum float64
|
||||
writeBytesDeltaSum float64
|
||||
receiveBytesNumSum float64
|
||||
receiveBytesNumDeltaSum float64
|
||||
transmitBytesNumSum float64
|
||||
|
|
@ -230,6 +235,10 @@ func (data *analyzeData) aggregateAll(memoryByKeyPath string, totalRequests int6
|
|||
sectorsWrittenDeltaSum += vv
|
||||
case strings.HasPrefix(hd, "SECTORS-WRITTEN-"):
|
||||
sectorsWrittenSum += vv
|
||||
case strings.HasPrefix(hd, "READ-BYTES-DELTA-"):
|
||||
readBytesDeltaSum += vv
|
||||
case strings.HasPrefix(hd, "WRITE-BYTES-DELTA-"):
|
||||
writeBytesDeltaSum += vv
|
||||
case strings.HasPrefix(hd, "RECEIVE-BYTES-NUM-DELTA-"):
|
||||
receiveBytesNumDeltaSum += vv
|
||||
case strings.HasPrefix(hd, "RECEIVE-BYTES-NUM-"):
|
||||
|
|
@ -255,6 +264,8 @@ func (data *analyzeData) aggregateAll(memoryByKeyPath string, totalRequests int6
|
|||
avgWritesCompletedDeltaCol.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", writesCompletedDeltaSum/sampleSize)))
|
||||
avgSectorsWrittenCol.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", sectorsWrittenSum/sampleSize)))
|
||||
avgSectorsWrittenDeltaCol.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", sectorsWrittenDeltaSum/sampleSize)))
|
||||
avgReadBytesNumDeltaCol.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", readBytesDeltaSum/sampleSize)))
|
||||
avgWriteBytesNumDeltaCol.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", writeBytesDeltaSum/sampleSize)))
|
||||
avgReceiveBytesNumCol.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", receiveBytesNumSum/sampleSize)))
|
||||
avgReceiveBytesNumDeltaCol.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", receiveBytesNumDeltaSum/sampleSize)))
|
||||
avgTransmitBytesNumCol.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", transmitBytesNumSum/sampleSize)))
|
||||
|
|
@ -307,6 +318,12 @@ func (data *analyzeData) aggregateAll(memoryByKeyPath string, totalRequests int6
|
|||
if err = data.aggregated.AddColumn(avgSectorsWrittenDeltaCol); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = data.aggregated.AddColumn(avgReadBytesNumDeltaCol); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = data.aggregated.AddColumn(avgWriteBytesNumDeltaCol); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = data.aggregated.AddColumn(avgReceiveBytesNumCol); err != nil {
|
||||
return err
|
||||
}
|
||||
|
|
@ -366,6 +383,8 @@ func (data *analyzeData) aggregateAll(memoryByKeyPath string, totalRequests int6
|
|||
"AVG-READS-COMPLETED-DELTA",
|
||||
"AVG-SECTORS-READ",
|
||||
"AVG-SECTORS-READ-DELTA",
|
||||
"AVG-READ-BYTES-NUM-DELTA",
|
||||
"AVG-WRITE-BYTES-NUM-DELTA",
|
||||
"AVG-RECEIVE-BYTES-NUM",
|
||||
"AVG-RECEIVE-BYTES-NUM-DELTA",
|
||||
"AVG-TRANSMIT-BYTES-NUM",
|
||||
|
|
@ -398,6 +417,14 @@ func (data *analyzeData) aggregateAll(memoryByKeyPath string, totalRequests int6
|
|||
if err != nil {
|
||||
return err
|
||||
}
|
||||
colReadBytesNumDelta, err := data.aggregated.Column("AVG-READ-BYTES-NUM-DELTA")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
colWriteBytesNumDelta, err := data.aggregated.Column("AVG-WRITE-BYTES-NUM-DELTA")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if colUnixSecond.Count() != colMemoryMB.Count() {
|
||||
return fmt.Errorf("SECOND column count %d, AVG-VMRSS-MB column count %d", colUnixSecond.Count(), colMemoryMB.Count())
|
||||
}
|
||||
|
|
@ -407,8 +434,14 @@ func (data *analyzeData) aggregateAll(memoryByKeyPath string, totalRequests int6
|
|||
if colUnixSecond.Count() != colAvgThroughput.Count() {
|
||||
return fmt.Errorf("SECOND column count %d, AVG-THROUGHPUT column count %d", colUnixSecond.Count(), colAvgThroughput.Count())
|
||||
}
|
||||
if colReadBytesNumDelta.Count() != colAvgThroughput.Count() {
|
||||
return fmt.Errorf("AVG-READ-BYTES-NUM-DELTA column count %d, AVG-THROUGHPUT column count %d", colReadBytesNumDelta.Count(), colAvgThroughput.Count())
|
||||
}
|
||||
if colWriteBytesNumDelta.Count() != colAvgThroughput.Count() {
|
||||
return fmt.Errorf("AVG-WRITE-BYTES-NUM-DELTA column count %d, AVG-THROUGHPUT column count %d", colWriteBytesNumDelta.Count(), colAvgThroughput.Count())
|
||||
}
|
||||
|
||||
var cdata []dbtester.CumulativeKeyNumAndMemory
|
||||
var cdata []dbtester.CumulativeKeyNumAndOtherData
|
||||
for i := 0; i < colUnixSecond.Count(); i++ {
|
||||
vv0, err := colUnixSecond.Value(i)
|
||||
if err != nil {
|
||||
|
|
@ -428,43 +461,98 @@ func (data *analyzeData) aggregateAll(memoryByKeyPath string, totalRequests int6
|
|||
}
|
||||
vf2, _ := vv2.Float64()
|
||||
|
||||
point := dbtester.CumulativeKeyNumAndMemory{
|
||||
vv3, err := colReadBytesNumDelta.Value(i)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vf3, _ := vv3.Float64()
|
||||
|
||||
vv4, err := colWriteBytesNumDelta.Value(i)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
vf4, _ := vv4.Float64()
|
||||
|
||||
point := dbtester.CumulativeKeyNumAndOtherData{
|
||||
UnixSecond: v0,
|
||||
Throughput: int64(vf2),
|
||||
|
||||
MinMemoryMB: sec2minVMRSSMB[v0],
|
||||
AvgMemoryMB: vf1,
|
||||
MaxMemoryMB: sec2maxVMRSSMB[v0],
|
||||
|
||||
AvgReadBytesDelta: vf3,
|
||||
AvgWriteBytesDelta: vf4,
|
||||
}
|
||||
cdata = append(cdata, point)
|
||||
}
|
||||
|
||||
// aggregate memory by number of keys
|
||||
knms := dbtester.FindRangesMemory(cdata, 1000, totalRequests)
|
||||
// aggregate memory,write/read bytes by number of keys
|
||||
knms := dbtester.FindRangesData(cdata, 1000, totalRequests)
|
||||
|
||||
ckk1 := dataframe.NewColumn("KEYS")
|
||||
ckk2 := dataframe.NewColumn("MIN-VMRSS-MB")
|
||||
ckk3 := dataframe.NewColumn("AVG-VMRSS-MB")
|
||||
ckk4 := dataframe.NewColumn("MAX-VMRSS-MB")
|
||||
ckk5 := dataframe.NewColumn("AVG-READ-BYTES-NUM-DELTA")
|
||||
ckk6 := dataframe.NewColumn("AVG-READ-BYTES")
|
||||
ckk7 := dataframe.NewColumn("AVG-WRITE-BYTES-NUM-DELTA")
|
||||
ckk8 := dataframe.NewColumn("AVG-WRITE-BYTES")
|
||||
for i := range knms {
|
||||
ckk1.PushBack(dataframe.NewStringValue(knms[i].CumulativeKeyNum))
|
||||
ckk2.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", knms[i].MinMemoryMB)))
|
||||
ckk3.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", knms[i].AvgMemoryMB)))
|
||||
ckk4.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", knms[i].MaxMemoryMB)))
|
||||
ckk5.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", knms[i].AvgReadBytesDelta)))
|
||||
ckk6.PushBack(dataframe.NewStringValue(humanize.Bytes(uint64(knms[i].AvgReadBytesDelta))))
|
||||
ckk7.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", knms[i].AvgWriteBytesDelta)))
|
||||
ckk8.PushBack(dataframe.NewStringValue(humanize.Bytes(uint64(knms[i].AvgWriteBytesDelta))))
|
||||
}
|
||||
fr := dataframe.New()
|
||||
if err := fr.AddColumn(ckk1); err != nil {
|
||||
|
||||
fr1 := dataframe.New()
|
||||
if err := fr1.AddColumn(ckk1); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
if err := fr.AddColumn(ckk2); err != nil {
|
||||
if err := fr1.AddColumn(ckk2); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
if err := fr.AddColumn(ckk3); err != nil {
|
||||
if err := fr1.AddColumn(ckk3); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
if err := fr.AddColumn(ckk4); err != nil {
|
||||
if err := fr1.AddColumn(ckk4); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
if err := fr.CSV(memoryByKeyPath); err != nil {
|
||||
if err := fr1.CSV(memoryByKeyPath); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
|
||||
// aggregate read bytes by number of keys
|
||||
fr2 := dataframe.New()
|
||||
if err := fr2.AddColumn(ckk1); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
if err := fr2.AddColumn(ckk5); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
if err := fr2.AddColumn(ckk6); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
if err := fr2.CSV(readBytesDeltaByKeyPath); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
|
||||
// aggregate write bytes by number of keys
|
||||
fr3 := dataframe.New()
|
||||
if err := fr3.AddColumn(ckk1); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
if err := fr3.AddColumn(ckk7); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
if err := fr3.AddColumn(ckk8); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
if err := fr3.CSV(writeBytesDeltaByKeyPath); err != nil {
|
||||
plog.Fatal(err)
|
||||
}
|
||||
|
||||
|
|
|
|||
|
|
@ -88,7 +88,7 @@ func do(configPath string) error {
|
|||
if err = ad.importBenchMetrics(testdata.ClientLatencyThroughputTimeseriesPath); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = ad.aggregateAll(testdata.ServerMemoryByKeyNumberPath, testgroup.RequestNumber); err != nil {
|
||||
if err = ad.aggregateAll(testdata.ServerMemoryByKeyNumberPath, testdata.ServerReadBytesDeltaByKeyNumberPath, testdata.ServerWriteBytesDeltaByKeyNumberPath, testgroup.RequestNumber); err != nil {
|
||||
return err
|
||||
}
|
||||
if err = ad.save(); err != nil {
|
||||
|
|
@ -239,7 +239,6 @@ func do(configPath string) error {
|
|||
row28WritesCompletedDeltaSum = append(row28WritesCompletedDeltaSum, humanize.Comma(int64(writesCompletedDeltaSum)))
|
||||
row29SectorsWrittenDeltaSum = append(row29SectorsWrittenDeltaSum, humanize.Comma(int64(sectorsWrittenDeltaSum)))
|
||||
|
||||
// TODO: handle overflowed memory value?
|
||||
sort.Float64s(maxAvgVMRSSMBs)
|
||||
mv := maxAvgVMRSSMBs[len(maxAvgVMRSSMBs)-1]
|
||||
mb := uint64(mv * 1000000)
|
||||
|
|
@ -731,6 +730,80 @@ func do(configPath string) error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
// KEYS, AVG-READ-BYTES-NUM-DELTA, AVG-READ-BYTES
|
||||
plog.Info("combining all server read bytes delta by keys")
|
||||
allReadBytesDeltaFrame := dataframe.New()
|
||||
for _, databaseID := range cfg.AllDatabaseIDList {
|
||||
testdata := cfg.DatabaseIDToTestData[databaseID]
|
||||
|
||||
fr, err := dataframe.NewFromCSV(nil, testdata.ServerReadBytesDeltaByKeyNumberPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
colKeys, err := fr.Column("KEYS")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
colKeys.UpdateHeader(makeHeader("KEYS", testdata.DatabaseTag))
|
||||
if err = allReadBytesDeltaFrame.AddColumn(colKeys); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
col1, err := fr.Column("AVG-READ-BYTES-NUM-DELTA")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
col1.UpdateHeader(makeHeader("AVG-READ-BYTES-NUM-DELTA", testdata.DatabaseTag))
|
||||
if err = allReadBytesDeltaFrame.AddColumn(col1); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
col2, err := fr.Column("AVG-READ-BYTES")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
col2.UpdateHeader(makeHeader("AVG-READ-BYTES", testdata.DatabaseTag))
|
||||
if err = allReadBytesDeltaFrame.AddColumn(col2); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
// KEYS, AVG-WRITE-BYTES-NUM-DELTA, AVG-WRITE-BYTES
|
||||
plog.Info("combining all server write bytes delta by keys")
|
||||
allWriteBytesDeltaFrame := dataframe.New()
|
||||
for _, databaseID := range cfg.AllDatabaseIDList {
|
||||
testdata := cfg.DatabaseIDToTestData[databaseID]
|
||||
|
||||
fr, err := dataframe.NewFromCSV(nil, testdata.ServerWriteBytesDeltaByKeyNumberPath)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
colKeys, err := fr.Column("KEYS")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
colKeys.UpdateHeader(makeHeader("KEYS", testdata.DatabaseTag))
|
||||
if err = allWriteBytesDeltaFrame.AddColumn(colKeys); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
col1, err := fr.Column("AVG-WRITE-BYTES-NUM-DELTA")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
col1.UpdateHeader(makeHeader("AVG-WRITE-BYTES-NUM-DELTA", testdata.DatabaseTag))
|
||||
if err = allWriteBytesDeltaFrame.AddColumn(col1); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
col2, err := fr.Column("AVG-WRITE-BYTES")
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
col2.UpdateHeader(makeHeader("AVG-WRITE-BYTES", testdata.DatabaseTag))
|
||||
if err = allWriteBytesDeltaFrame.AddColumn(col2); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
{
|
||||
allLatencyFrameCfg := dbtester.Plot{
|
||||
|
|
@ -890,6 +963,58 @@ func do(configPath string) error {
|
|||
return err
|
||||
}
|
||||
}
|
||||
{
|
||||
allReadBytesDeltaFrameCfg := dbtester.Plot{
|
||||
Column: "AVG-READ-BYTES-NUM-DELTA",
|
||||
XAxis: "Cumulative Number of Keys",
|
||||
YAxis: "Average Read Bytes Delta by Keys",
|
||||
OutputPathList: make([]string, len(cfg.PlotList[0].OutputPathList)),
|
||||
}
|
||||
allReadBytesDeltaFrameCfg.OutputPathList[0] = filepath.Join(filepath.Dir(cfg.PlotList[0].OutputPathList[0]), "AVG-READ-BYTES-NUM-DELTA-BY-KEY.svg")
|
||||
allReadBytesDeltaFrameCfg.OutputPathList[1] = filepath.Join(filepath.Dir(cfg.PlotList[0].OutputPathList[0]), "AVG-READ-BYTES-NUM-DELTA-BY-KEY.png")
|
||||
plog.Printf("plotting %v", allReadBytesDeltaFrameCfg.OutputPathList)
|
||||
var pairs []pair
|
||||
allCols := allReadBytesDeltaFrame.Columns()
|
||||
for i := 0; i < len(allCols)-2; i += 3 {
|
||||
pairs = append(pairs, pair{
|
||||
x: allCols[i], // x
|
||||
y: allCols[i+1], // avg
|
||||
})
|
||||
}
|
||||
if err = all.drawXY(allReadBytesDeltaFrameCfg, pairs...); err != nil {
|
||||
return err
|
||||
}
|
||||
csvPath := filepath.Join(filepath.Dir(cfg.PlotList[0].OutputPathList[0]), "AVG-READ-BYTES-NUM-DELTA-BY-KEY.csv")
|
||||
if err := allReadBytesDeltaFrame.CSV(csvPath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
{
|
||||
allWriteBytesDeltaFrameCfg := dbtester.Plot{
|
||||
Column: "AVG-WRITE-BYTES-NUM-DELTA",
|
||||
XAxis: "Cumulative Number of Keys",
|
||||
YAxis: "Average Write Bytes Delta by Keys",
|
||||
OutputPathList: make([]string, len(cfg.PlotList[0].OutputPathList)),
|
||||
}
|
||||
allWriteBytesDeltaFrameCfg.OutputPathList[0] = filepath.Join(filepath.Dir(cfg.PlotList[0].OutputPathList[0]), "AVG-WRITE-BYTES-NUM-DELTA-BY-KEY.svg")
|
||||
allWriteBytesDeltaFrameCfg.OutputPathList[1] = filepath.Join(filepath.Dir(cfg.PlotList[0].OutputPathList[0]), "AVG-WRITE-BYTES-NUM-DELTA-BY-KEY.png")
|
||||
plog.Printf("plotting %v", allWriteBytesDeltaFrameCfg.OutputPathList)
|
||||
var pairs []pair
|
||||
allCols := allWriteBytesDeltaFrame.Columns()
|
||||
for i := 0; i < len(allCols)-2; i += 3 {
|
||||
pairs = append(pairs, pair{
|
||||
x: allCols[i], // x
|
||||
y: allCols[i+1], // avg
|
||||
})
|
||||
}
|
||||
if err = all.drawXY(allWriteBytesDeltaFrameCfg, pairs...); err != nil {
|
||||
return err
|
||||
}
|
||||
csvPath := filepath.Join(filepath.Dir(cfg.PlotList[0].OutputPathList[0]), "AVG-WRITE-BYTES-NUM-DELTA-BY-KEY.csv")
|
||||
if err := allWriteBytesDeltaFrame.CSV(csvPath); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
plog.Println("combining data for plotting")
|
||||
for _, plotConfig := range cfg.PlotList {
|
||||
|
|
|
|||
|
|
@ -131,6 +131,8 @@ type TestData struct {
|
|||
ClientLatencyByKeyNumberPath string `yaml:"client_latency_by_key_number_path"`
|
||||
ServerDiskSpaceUsageSummaryPath string `yaml:"server_disk_space_usage_summary_path"`
|
||||
ServerMemoryByKeyNumberPath string `yaml:"server_memory_by_key_number_path"`
|
||||
ServerReadBytesDeltaByKeyNumberPath string `yaml:"server_read_bytes_delta_by_key_number_path"`
|
||||
ServerWriteBytesDeltaByKeyNumberPath string `yaml:"server_write_bytes_delta_by_key_number_path"`
|
||||
ServerSystemMetricsInterpolatedPathList []string `yaml:"server_system_metrics_interpolated_path_list"`
|
||||
AllAggregatedOutputPath string `yaml:"all_aggregated_output_path"`
|
||||
}
|
||||
|
|
@ -214,6 +216,8 @@ func ReadConfig(fpath string, analyze bool) (*Config, error) {
|
|||
testdata.ClientLatencyByKeyNumberPath = testdata.PathPrefix + "-" + testdata.ClientLatencyByKeyNumberPath
|
||||
testdata.ServerDiskSpaceUsageSummaryPath = testdata.PathPrefix + "-" + testdata.ServerDiskSpaceUsageSummaryPath
|
||||
testdata.ServerMemoryByKeyNumberPath = testdata.PathPrefix + "-" + testdata.ServerMemoryByKeyNumberPath
|
||||
testdata.ServerReadBytesDeltaByKeyNumberPath = testdata.PathPrefix + "-" + testdata.ServerReadBytesDeltaByKeyNumberPath
|
||||
testdata.ServerWriteBytesDeltaByKeyNumberPath = testdata.PathPrefix + "-" + testdata.ServerWriteBytesDeltaByKeyNumberPath
|
||||
for i := range testdata.ServerSystemMetricsInterpolatedPathList {
|
||||
testdata.ServerSystemMetricsInterpolatedPathList[i] = testdata.PathPrefix + "-" + testdata.ServerSystemMetricsInterpolatedPathList[i]
|
||||
}
|
||||
|
|
|
|||
|
|
@ -29,7 +29,7 @@ func TestConfig(t *testing.T) {
|
|||
expected := &Config{
|
||||
TestTitle: "Write 1M keys, 256-byte key, 1KB value value, clients 1 to 1,000",
|
||||
TestDescription: `- Google Cloud Compute Engine
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 150 GB SSD (1 for client)
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 300 GB SSD (1 for client)
|
||||
- Ubuntu 16.10
|
||||
- etcd v3.1 (Go 1.7.5)
|
||||
- Zookeeper r3.4.9
|
||||
|
|
@ -171,6 +171,8 @@ func TestConfig(t *testing.T) {
|
|||
ClientLatencyDistributionSummaryPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/etcd-v3.1-go1.7.5-client-latency-distribution-summary.csv",
|
||||
ClientLatencyByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/etcd-v3.1-go1.7.5-client-latency-by-key-number.csv",
|
||||
ServerMemoryByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/etcd-v3.1-go1.7.5-server-memory-by-key-number.csv",
|
||||
ServerReadBytesDeltaByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/etcd-v3.1-go1.7.5-server-read-bytes-delta-by-key-number.csv",
|
||||
ServerWriteBytesDeltaByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/etcd-v3.1-go1.7.5-server-write-bytes-delta-by-key-number.csv",
|
||||
ServerDiskSpaceUsageSummaryPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/etcd-v3.1-go1.7.5-server-disk-space-usage-summary.csv",
|
||||
ServerSystemMetricsInterpolatedPathList: []string{
|
||||
"2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/etcd-v3.1-go1.7.5-1-server-system-metrics-interpolated.csv",
|
||||
|
|
@ -192,6 +194,8 @@ func TestConfig(t *testing.T) {
|
|||
ClientLatencyDistributionSummaryPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/zookeeper-r3.4.9-java8-client-latency-distribution-summary.csv",
|
||||
ClientLatencyByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/zookeeper-r3.4.9-java8-client-latency-by-key-number.csv",
|
||||
ServerMemoryByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/zookeeper-r3.4.9-java8-server-memory-by-key-number.csv",
|
||||
ServerReadBytesDeltaByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/zookeeper-r3.4.9-java8-server-read-bytes-delta-by-key-number.csv",
|
||||
ServerWriteBytesDeltaByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/zookeeper-r3.4.9-java8-server-write-bytes-delta-by-key-number.csv",
|
||||
ServerDiskSpaceUsageSummaryPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/zookeeper-r3.4.9-java8-server-disk-space-usage-summary.csv",
|
||||
ServerSystemMetricsInterpolatedPathList: []string{
|
||||
"2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/zookeeper-r3.4.9-java8-1-server-system-metrics-interpolated.csv",
|
||||
|
|
@ -213,6 +217,8 @@ func TestConfig(t *testing.T) {
|
|||
ClientLatencyDistributionSummaryPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/consul-v0.7.4-go1.7.5-client-latency-distribution-summary.csv",
|
||||
ClientLatencyByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/consul-v0.7.4-go1.7.5-client-latency-by-key-number.csv",
|
||||
ServerMemoryByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/consul-v0.7.4-go1.7.5-server-memory-by-key-number.csv",
|
||||
ServerReadBytesDeltaByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/consul-v0.7.4-go1.7.5-server-read-bytes-delta-by-key-number.csv",
|
||||
ServerWriteBytesDeltaByKeyNumberPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/consul-v0.7.4-go1.7.5-server-write-bytes-delta-by-key-number.csv",
|
||||
ServerDiskSpaceUsageSummaryPath: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/consul-v0.7.4-go1.7.5-server-disk-space-usage-summary.csv",
|
||||
ServerSystemMetricsInterpolatedPathList: []string{
|
||||
"2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/consul-v0.7.4-go1.7.5-1-server-system-metrics-interpolated.csv",
|
||||
|
|
@ -328,6 +334,26 @@ func TestConfig(t *testing.T) {
|
|||
"2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-SECTORS-WRITTEN-DELTA.png",
|
||||
},
|
||||
},
|
||||
{
|
||||
Column: "AVG-READ-BYTES-DELTA",
|
||||
XAxis: "Second",
|
||||
YAxis: "Read Bytes (Delta per Second)",
|
||||
OutputPathCSV: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-READ-BYTES-DELTA.csv",
|
||||
OutputPathList: []string{
|
||||
"2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-READ-BYTES-DELTA.svg",
|
||||
"2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-READ-BYTES-DELTA.png",
|
||||
},
|
||||
},
|
||||
{
|
||||
Column: "AVG-WRITE-BYTES-DELTA",
|
||||
XAxis: "Second",
|
||||
YAxis: "Write Bytes (Delta per Second)",
|
||||
OutputPathCSV: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-WRITE-BYTES-DELTA.csv",
|
||||
OutputPathList: []string{
|
||||
"2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-WRITE-BYTES-DELTA.svg",
|
||||
"2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-WRITE-BYTES-DELTA.png",
|
||||
},
|
||||
},
|
||||
{
|
||||
Column: "AVG-RECEIVE-BYTES-NUM-DELTA",
|
||||
XAxis: "Second",
|
||||
|
|
@ -423,6 +449,16 @@ func TestConfig(t *testing.T) {
|
|||
Path: "https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-SECTORS-WRITTEN-DELTA.svg",
|
||||
Type: "remote",
|
||||
},
|
||||
{
|
||||
Title: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-READ-BYTES-DELTA",
|
||||
Path: "https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-READ-BYTES-DELTA.svg",
|
||||
Type: "remote",
|
||||
},
|
||||
{
|
||||
Title: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-WRITE-BYTES-DELTA",
|
||||
Path: "https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-WRITE-BYTES-DELTA.svg",
|
||||
Type: "remote",
|
||||
},
|
||||
{
|
||||
Title: "2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-RECEIVE-BYTES-NUM-DELTA",
|
||||
Path: "https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-RECEIVE-BYTES-NUM-DELTA.svg",
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
test_title: Write 1M keys, 256-byte key, 1KB value value, clients 1 to 1,000
|
||||
test_description: |
|
||||
- Google Cloud Compute Engine
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 150 GB SSD (1 for client)
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 300 GB SSD (1 for client)
|
||||
- Ubuntu 16.10
|
||||
- etcd v3.1 (Go 1.7.5)
|
||||
- Zookeeper r3.4.9
|
||||
|
|
@ -177,6 +177,8 @@ datatbase_id_to_test_data:
|
|||
client_latency_by_key_number_path: client-latency-by-key-number.csv
|
||||
server_disk_space_usage_summary_path: server-disk-space-usage-summary.csv
|
||||
server_memory_by_key_number_path: server-memory-by-key-number.csv
|
||||
server_read_bytes_delta_by_key_number_path: server-read-bytes-delta-by-key-number.csv
|
||||
server_write_bytes_delta_by_key_number_path: server-write-bytes-delta-by-key-number.csv
|
||||
server_system_metrics_interpolated_path_list:
|
||||
- 1-server-system-metrics-interpolated.csv
|
||||
- 2-server-system-metrics-interpolated.csv
|
||||
|
|
@ -194,6 +196,8 @@ datatbase_id_to_test_data:
|
|||
client_latency_by_key_number_path: client-latency-by-key-number.csv
|
||||
server_disk_space_usage_summary_path: server-disk-space-usage-summary.csv
|
||||
server_memory_by_key_number_path: server-memory-by-key-number.csv
|
||||
server_read_bytes_delta_by_key_number_path: server-read-bytes-delta-by-key-number.csv
|
||||
server_write_bytes_delta_by_key_number_path: server-write-bytes-delta-by-key-number.csv
|
||||
server_system_metrics_interpolated_path_list:
|
||||
- 1-server-system-metrics-interpolated.csv
|
||||
- 2-server-system-metrics-interpolated.csv
|
||||
|
|
@ -211,6 +215,8 @@ datatbase_id_to_test_data:
|
|||
client_latency_by_key_number_path: client-latency-by-key-number.csv
|
||||
server_disk_space_usage_summary_path: server-disk-space-usage-summary.csv
|
||||
server_memory_by_key_number_path: server-memory-by-key-number.csv
|
||||
server_read_bytes_delta_by_key_number_path: server-read-bytes-delta-by-key-number.csv
|
||||
server_write_bytes_delta_by_key_number_path: server-write-bytes-delta-by-key-number.csv
|
||||
server_system_metrics_interpolated_path_list:
|
||||
- 1-server-system-metrics-interpolated.csv
|
||||
- 2-server-system-metrics-interpolated.csv
|
||||
|
|
@ -263,6 +269,14 @@ plot_list:
|
|||
x_axis: Second
|
||||
y_axis: Sectors Written (Delta per Second)
|
||||
|
||||
- column: AVG-READ-BYTES-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Read Bytes (Delta per Second)
|
||||
|
||||
- column: AVG-WRITE-BYTES-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Write Bytes (Delta per Second)
|
||||
|
||||
- column: AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Network Receive(bytes) (Delta per Second)
|
||||
|
|
@ -331,6 +345,14 @@ readme:
|
|||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-SECTORS-WRITTEN-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-READ-BYTES-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-READ-BYTES-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-WRITE-BYTES-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-WRITE-BYTES-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-RECEIVE-BYTES-NUM-DELTA.svg
|
||||
type: remote
|
||||
|
|
|
|||
|
|
@ -117,10 +117,10 @@ func FindRangesLatency(data report.TimeSeries, unit int64, totalRequests int64)
|
|||
return kss
|
||||
}
|
||||
|
||||
// CumulativeKeyNumAndMemory wraps the cumulative number of keys
|
||||
// CumulativeKeyNumAndOtherData wraps the cumulative number of keys
|
||||
// and according memory data. So the higher 'CumulativeKeyNum' is,
|
||||
// the later the data points are in the time series.
|
||||
type CumulativeKeyNumAndMemory struct {
|
||||
type CumulativeKeyNumAndOtherData struct {
|
||||
UnixSecond int64
|
||||
Throughput int64
|
||||
|
||||
|
|
@ -129,39 +129,42 @@ type CumulativeKeyNumAndMemory struct {
|
|||
MinMemoryMB float64
|
||||
AvgMemoryMB float64
|
||||
MaxMemoryMB float64
|
||||
|
||||
AvgReadBytesDelta float64
|
||||
AvgWriteBytesDelta float64
|
||||
}
|
||||
|
||||
// CumulativeKeyNumAndMemorySlice is a slice of CumulativeKeyNumAndMemory to sort by CumulativeKeyNum.
|
||||
type CumulativeKeyNumAndMemorySlice []CumulativeKeyNumAndMemory
|
||||
// CumulativeKeyNumAndOtherDataSlice is a slice of CumulativeKeyNumAndOtherData to sort by CumulativeKeyNum.
|
||||
type CumulativeKeyNumAndOtherDataSlice []CumulativeKeyNumAndOtherData
|
||||
|
||||
func (t CumulativeKeyNumAndMemorySlice) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
|
||||
func (t CumulativeKeyNumAndMemorySlice) Len() int { return len(t) }
|
||||
func (t CumulativeKeyNumAndMemorySlice) Less(i, j int) bool {
|
||||
func (t CumulativeKeyNumAndOtherDataSlice) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
|
||||
func (t CumulativeKeyNumAndOtherDataSlice) Len() int { return len(t) }
|
||||
func (t CumulativeKeyNumAndOtherDataSlice) Less(i, j int) bool {
|
||||
return t[i].CumulativeKeyNum < t[j].CumulativeKeyNum
|
||||
}
|
||||
|
||||
// CumulativeKeyNumAndMemoryByUnixSecond is a slice of CumulativeKeyNumAndMemory to sort by UnixSecond.
|
||||
type CumulativeKeyNumAndMemoryByUnixSecond []CumulativeKeyNumAndMemory
|
||||
// CumulativeKeyNumAndOtherDataByUnixSecond is a slice of CumulativeKeyNumAndOtherData to sort by UnixSecond.
|
||||
type CumulativeKeyNumAndOtherDataByUnixSecond []CumulativeKeyNumAndOtherData
|
||||
|
||||
func (t CumulativeKeyNumAndMemoryByUnixSecond) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
|
||||
func (t CumulativeKeyNumAndMemoryByUnixSecond) Len() int { return len(t) }
|
||||
func (t CumulativeKeyNumAndMemoryByUnixSecond) Less(i, j int) bool {
|
||||
func (t CumulativeKeyNumAndOtherDataByUnixSecond) Swap(i, j int) { t[i], t[j] = t[j], t[i] }
|
||||
func (t CumulativeKeyNumAndOtherDataByUnixSecond) Len() int { return len(t) }
|
||||
func (t CumulativeKeyNumAndOtherDataByUnixSecond) Less(i, j int) bool {
|
||||
return t[i].UnixSecond < t[j].UnixSecond
|
||||
}
|
||||
|
||||
// FindRangesMemory sorts all data points by its timestamp.
|
||||
// FindRangesData sorts all data points by its timestamp.
|
||||
// And then aggregate by the cumulative throughput,
|
||||
// in order to map the number of keys to the average memory usage.
|
||||
func FindRangesMemory(data []CumulativeKeyNumAndMemory, unit int64, totalRequests int64) CumulativeKeyNumAndMemorySlice {
|
||||
func FindRangesData(data []CumulativeKeyNumAndOtherData, unit int64, totalRequests int64) CumulativeKeyNumAndOtherDataSlice {
|
||||
// need to sort by timestamps because we want the 'cumulative'
|
||||
// trends as we write more keys, 'report.TimeSeries' already implements
|
||||
// sort interface, so just sort.Sort(data)
|
||||
sort.Sort(CumulativeKeyNumAndMemoryByUnixSecond(data))
|
||||
sort.Sort(CumulativeKeyNumAndOtherDataByUnixSecond(data))
|
||||
|
||||
cumulKeyN := int64(0)
|
||||
maxKey := int64(0)
|
||||
|
||||
rm := make(map[int64]CumulativeKeyNumAndMemory)
|
||||
rm := make(map[int64]CumulativeKeyNumAndOtherData)
|
||||
|
||||
// this data is aggregated by second
|
||||
// and we want to map number of keys to memory usage
|
||||
|
|
@ -184,27 +187,29 @@ func FindRangesMemory(data []CumulativeKeyNumAndMemory, unit int64, totalRequest
|
|||
// fill-in empty rows
|
||||
for i := maxKey; i < int64(totalRequests); i += unit {
|
||||
if _, ok := rm[i]; !ok {
|
||||
rm[i] = CumulativeKeyNumAndMemory{}
|
||||
rm[i] = CumulativeKeyNumAndOtherData{}
|
||||
}
|
||||
}
|
||||
if _, ok := rm[int64(totalRequests)]; !ok {
|
||||
rm[int64(totalRequests)] = CumulativeKeyNumAndMemory{}
|
||||
rm[int64(totalRequests)] = CumulativeKeyNumAndOtherData{}
|
||||
}
|
||||
|
||||
kss := []CumulativeKeyNumAndMemory{}
|
||||
kss := []CumulativeKeyNumAndOtherData{}
|
||||
delete(rm, 0) // drop data at beginning
|
||||
|
||||
for k, v := range rm {
|
||||
// make sure to use 'k' as keyNum
|
||||
kss = append(kss, CumulativeKeyNumAndMemory{
|
||||
CumulativeKeyNum: k,
|
||||
MinMemoryMB: v.MinMemoryMB,
|
||||
AvgMemoryMB: v.AvgMemoryMB,
|
||||
MaxMemoryMB: v.MaxMemoryMB,
|
||||
kss = append(kss, CumulativeKeyNumAndOtherData{
|
||||
CumulativeKeyNum: k,
|
||||
MinMemoryMB: v.MinMemoryMB,
|
||||
AvgMemoryMB: v.AvgMemoryMB,
|
||||
MaxMemoryMB: v.MaxMemoryMB,
|
||||
AvgReadBytesDelta: v.AvgReadBytesDelta,
|
||||
AvgWriteBytesDelta: v.AvgWriteBytesDelta,
|
||||
})
|
||||
}
|
||||
|
||||
// sort by cumulative throughput (number of keys) in ascending order
|
||||
sort.Sort(CumulativeKeyNumAndMemorySlice(kss))
|
||||
sort.Sort(CumulativeKeyNumAndOtherDataSlice(kss))
|
||||
return kss
|
||||
}
|
||||
|
|
|
|||
|
|
@ -74,10 +74,10 @@ func TestFindRangesLatency(t *testing.T) {
|
|||
}
|
||||
}
|
||||
|
||||
func TestFindRangesMemory(t *testing.T) {
|
||||
var data []CumulativeKeyNumAndMemory
|
||||
func TestFindRangesData(t *testing.T) {
|
||||
var data []CumulativeKeyNumAndOtherData
|
||||
for i := int64(0); i < 10; i++ {
|
||||
dp := CumulativeKeyNumAndMemory{
|
||||
dp := CumulativeKeyNumAndOtherData{
|
||||
UnixSecond: i + 1,
|
||||
Throughput: 50,
|
||||
AvgMemoryMB: float64(i + 1),
|
||||
|
|
@ -85,8 +85,8 @@ func TestFindRangesMemory(t *testing.T) {
|
|||
data = append(data, dp)
|
||||
}
|
||||
|
||||
pss := FindRangesMemory(data, 20, 555)
|
||||
expexcted := []CumulativeKeyNumAndMemory{
|
||||
pss := FindRangesData(data, 20, 555)
|
||||
expexcted := []CumulativeKeyNumAndOtherData{
|
||||
{CumulativeKeyNum: 20, AvgMemoryMB: 1},
|
||||
{CumulativeKeyNum: 40, AvgMemoryMB: 1},
|
||||
{CumulativeKeyNum: 60, AvgMemoryMB: 1},
|
||||
|
|
|
|||
|
|
@ -1,5 +1,5 @@
|
|||
hash: 69c56c66bb6f90f621733ce546bc4fd829abd83c86503dc0b8f68b4c568d62ab
|
||||
updated: 2017-02-10T21:02:38.955731118-08:00
|
||||
hash: 6bd04d9644a95f1caa2c1df864c296078eadff2a349eb724fbc0486aa5a2a72f
|
||||
updated: 2017-02-13T13:49:21.228470942-08:00
|
||||
imports:
|
||||
- name: bitbucket.org/zombiezen/gopdf
|
||||
version: 1c63dc69751bc45441c2ce1f56b631c55294b4d5
|
||||
|
|
@ -100,7 +100,7 @@ imports:
|
|||
- name: github.com/gyuho/dataframe
|
||||
version: 73de2c550b1177c1640f3dacbbc1af00f913fedb
|
||||
- name: github.com/gyuho/psn
|
||||
version: 622e14fd5a06aed2e24fe8b728cddba486e38090
|
||||
version: 0a50f90209cfd81ef2a20cbf01679f118f31f0ad
|
||||
subpackages:
|
||||
- schema
|
||||
- name: github.com/hashicorp/consul
|
||||
|
|
|
|||
|
|
@ -55,7 +55,7 @@ import:
|
|||
- package: github.com/gyuho/dataframe
|
||||
version: 73de2c550b1177c1640f3dacbbc1af00f913fedb
|
||||
- package: github.com/gyuho/psn
|
||||
version: 622e14fd5a06aed2e24fe8b728cddba486e38090
|
||||
version: 0a50f90209cfd81ef2a20cbf01679f118f31f0ad
|
||||
- package: github.com/hashicorp/consul
|
||||
version: 3da73be55c82a7f88f1dfd3ec16d267970ac8ff0
|
||||
subpackages:
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
test_title: Write 1M keys, 256-byte key, 1KB value value, clients 1 to 1,000
|
||||
test_description: |
|
||||
- Google Cloud Compute Engine
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 150 GB SSD (1 for client)
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 300 GB SSD (1 for client)
|
||||
- Ubuntu 16.10
|
||||
- etcd v3.1 (Go 1.7.5)
|
||||
- Zookeeper r3.4.9
|
||||
|
|
@ -176,6 +176,8 @@ datatbase_id_to_test_data:
|
|||
client_latency_by_key_number_path: client-latency-by-key-number.csv
|
||||
server_disk_space_usage_summary_path: server-disk-space-usage-summary.csv
|
||||
server_memory_by_key_number_path: server-memory-by-key-number.csv
|
||||
server_read_bytes_delta_by_key_number_path: server-read-bytes-delta-by-key-number.csv
|
||||
server_write_bytes_delta_by_key_number_path: server-write-bytes-delta-by-key-number.csv
|
||||
server_system_metrics_interpolated_path_list:
|
||||
- 1-server-system-metrics-interpolated.csv
|
||||
- 2-server-system-metrics-interpolated.csv
|
||||
|
|
@ -193,6 +195,8 @@ datatbase_id_to_test_data:
|
|||
client_latency_by_key_number_path: client-latency-by-key-number.csv
|
||||
server_disk_space_usage_summary_path: server-disk-space-usage-summary.csv
|
||||
server_memory_by_key_number_path: server-memory-by-key-number.csv
|
||||
server_read_bytes_delta_by_key_number_path: server-read-bytes-delta-by-key-number.csv
|
||||
server_write_bytes_delta_by_key_number_path: server-write-bytes-delta-by-key-number.csv
|
||||
server_system_metrics_interpolated_path_list:
|
||||
- 1-server-system-metrics-interpolated.csv
|
||||
- 2-server-system-metrics-interpolated.csv
|
||||
|
|
@ -210,6 +214,8 @@ datatbase_id_to_test_data:
|
|||
client_latency_by_key_number_path: client-latency-by-key-number.csv
|
||||
server_disk_space_usage_summary_path: server-disk-space-usage-summary.csv
|
||||
server_memory_by_key_number_path: server-memory-by-key-number.csv
|
||||
server_read_bytes_delta_by_key_number_path: server-read-bytes-delta-by-key-number.csv
|
||||
server_write_bytes_delta_by_key_number_path: server-write-bytes-delta-by-key-number.csv
|
||||
server_system_metrics_interpolated_path_list:
|
||||
- 1-server-system-metrics-interpolated.csv
|
||||
- 2-server-system-metrics-interpolated.csv
|
||||
|
|
@ -262,6 +268,14 @@ plot_list:
|
|||
x_axis: Second
|
||||
y_axis: Sectors Written (Delta per Second)
|
||||
|
||||
- column: AVG-READ-BYTES-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Read Bytes (Delta per Second)
|
||||
|
||||
- column: AVG-WRITE-BYTES-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Write Bytes (Delta per Second)
|
||||
|
||||
- column: AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Network Receive(bytes) (Delta per Second)
|
||||
|
|
@ -330,6 +344,14 @@ readme:
|
|||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-SECTORS-WRITTEN-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-READ-BYTES-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-READ-BYTES-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-WRITE-BYTES-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-WRITE-BYTES-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/01-write-1M-keys-client-variable/AVG-RECEIVE-BYTES-NUM-DELTA.svg
|
||||
type: remote
|
||||
|
|
|
|||
|
|
@ -1,7 +1,7 @@
|
|||
test_title: Write 1M keys, 256-byte key, 1KB value, Best Throughput (etcd 1,000, Zookeeper 500, Consul 500 clients)
|
||||
test_description: |
|
||||
- Google Cloud Compute Engine
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 150 GB SSD (1 for client)
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 300 GB SSD (1 for client)
|
||||
- Ubuntu 16.10
|
||||
- etcd v3.1 (Go 1.7.5)
|
||||
- Zookeeper r3.4.9
|
||||
|
|
@ -262,6 +262,14 @@ plot_list:
|
|||
x_axis: Second
|
||||
y_axis: Sectors Written (Delta per Second)
|
||||
|
||||
- column: AVG-READ-BYTES-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Read Bytes (Delta per Second)
|
||||
|
||||
- column: AVG-WRITE-BYTES-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Write Bytes (Delta per Second)
|
||||
|
||||
- column: AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Network Receive(bytes) (Delta per Second)
|
||||
|
|
@ -330,6 +338,14 @@ readme:
|
|||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/02-write-1M-keys-best-throughput/AVG-SECTORS-WRITTEN-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/02-write-1M-keys-best-throughput/AVG-READ-BYTES-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/02-write-1M-keys-best-throughput/AVG-READ-BYTES-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/02-write-1M-keys-best-throughput/AVG-WRITE-BYTES-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/02-write-1M-keys-best-throughput/AVG-WRITE-BYTES-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/02-write-1M-keys-best-throughput/AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/02-write-1M-keys-best-throughput/AVG-RECEIVE-BYTES-NUM-DELTA.svg
|
||||
type: remote
|
||||
|
|
|
|||
|
|
@ -1,339 +0,0 @@
|
|||
test_title: Write 1M keys, 256-byte key, 1KB value, 1,000 client
|
||||
test_description: |
|
||||
- Google Cloud Compute Engine
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 150 GB SSD (1 for client)
|
||||
- Ubuntu 16.10
|
||||
- etcd v3.1 (Go 1.7.5)
|
||||
- Zookeeper r3.4.9
|
||||
- Java 8
|
||||
- javac 1.8.0_121
|
||||
- Java(TM) SE Runtime Environment (build 1.8.0_121-b13)
|
||||
- Java HotSpot(TM) 64-Bit Server VM (build 25.121-b13, mixed mode)
|
||||
- Consul v0.7.4 (Go 1.7.5)
|
||||
|
||||
# common control options for all client machines
|
||||
control:
|
||||
# if not empty, all test data paths are prefixed
|
||||
path_prefix: /home/gyuho
|
||||
log_path: client-control.log
|
||||
client_system_metrics_path: client-system-metrics.csv
|
||||
client_system_metrics_interpolated_path: client-system-metrics-interpolated.csv
|
||||
client_latency_throughput_timeseries_path: client-latency-throughput-timeseries.csv
|
||||
client_latency_distribution_all_path: client-latency-distribution-all.csv
|
||||
client_latency_distribution_percentile_path: client-latency-distribution-percentile.csv
|
||||
client_latency_distribution_summary_path: client-latency-distribution-summary.csv
|
||||
client_latency_by_key_number_path: client-latency-by-key-number.csv
|
||||
server_disk_space_usage_summary_path: server-disk-space-usage-summary.csv
|
||||
|
||||
# (optional) to automatically upload all files in client machine
|
||||
google_cloud_project_name: etcd-development
|
||||
# set this in 'control' machine, to automate log uploading in remote 'agent' machines
|
||||
google_cloud_storage_key_path: /home/gyuho/gcloud-key.json
|
||||
google_cloud_storage_bucket_name: dbtester-results
|
||||
google_cloud_storage_sub_directory: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client
|
||||
|
||||
all_database_id_list: [etcdv3, zookeeper, consul]
|
||||
|
||||
datatbase_id_to_test_group:
|
||||
etcdv3:
|
||||
database_description: etcd v3.1 (Go 1.7.5)
|
||||
peer_ips:
|
||||
- 10.240.0.20
|
||||
- 10.240.0.21
|
||||
- 10.240.0.22
|
||||
database_port_to_connect: 2379
|
||||
agent_port_to_connect: 3500
|
||||
|
||||
etcdv3:
|
||||
# --snapshot-count
|
||||
snap_count: 100000
|
||||
# --quota-backend-bytes; 8 GB
|
||||
quota_size_bytes: 8000000000
|
||||
|
||||
benchmark_options:
|
||||
type: write
|
||||
request_number: 1000000
|
||||
connection_number: 1000
|
||||
client_number: 1000
|
||||
# if specified, overwrite 'connection_number', 'connection_number'
|
||||
connection_client_numbers: []
|
||||
|
||||
# 0, to not rate limit
|
||||
rate_limit_requests_per_second: 0
|
||||
|
||||
# for 'write', 'read'
|
||||
same_key: false
|
||||
key_size_bytes: 256
|
||||
value_size_bytes: 1024
|
||||
|
||||
stale_read: false
|
||||
|
||||
benchmark_steps:
|
||||
step1_start_database: true
|
||||
step2_stress_database: true
|
||||
step3_stop_database: true
|
||||
step4_upload_logs: true
|
||||
|
||||
zookeeper:
|
||||
database_description: Zookeeper r3.4.9 (Java 8)
|
||||
peer_ips:
|
||||
- 10.240.0.25
|
||||
- 10.240.0.27
|
||||
- 10.240.0.28
|
||||
database_port_to_connect: 2181
|
||||
agent_port_to_connect: 3500
|
||||
|
||||
# http://zookeeper.apache.org/doc/trunk/zookeeperAdmin.html
|
||||
zookeeper:
|
||||
# tickTime; the length of a single tick, which is the basic time unit used by ZooKeeper,
|
||||
# as measured in milliseconds.
|
||||
tick_time: 2000
|
||||
|
||||
# initLimit; Amount of time, in ticks to allow followers to connect and sync to a leader
|
||||
# increased this value as needed, if the amount of data managed by ZooKeeper is large.
|
||||
# (default 5)
|
||||
init_limit: 5
|
||||
|
||||
# syncLimit; Amount of time, in ticks to allow followers to sync with ZooKeeper.
|
||||
# (default 5)
|
||||
sync_limit: 5
|
||||
|
||||
# snapCount; After snapCount transactions are written to a log file a snapshot
|
||||
# is started and a new transaction log file is created. The default snapCount is 100,000.
|
||||
snap_count: 100000
|
||||
|
||||
# maxClientCnxns; Limits the number of concurrent connections (at the socket level)
|
||||
# that a single client, identified by IP address, may make to a single member of the ZooKeeper ensemble.
|
||||
max_client_connections: 5000
|
||||
|
||||
benchmark_options:
|
||||
type: write
|
||||
request_number: 1000000
|
||||
connection_number: 1000
|
||||
client_number: 1000
|
||||
# if specified, overwrite 'connection_number', 'connection_number'
|
||||
connection_client_numbers: []
|
||||
|
||||
# 0, to not rate limit
|
||||
rate_limit_requests_per_second: 0
|
||||
|
||||
# for 'write', 'read'
|
||||
same_key: false
|
||||
key_size_bytes: 256
|
||||
value_size_bytes: 1024
|
||||
|
||||
stale_read: false
|
||||
|
||||
benchmark_steps:
|
||||
step1_start_database: true
|
||||
step2_stress_database: true
|
||||
step3_stop_database: true
|
||||
step4_upload_logs: true
|
||||
|
||||
consul:
|
||||
database_description: Consul v0.7.4 (Go 1.7.5)
|
||||
peer_ips:
|
||||
- 10.240.0.30
|
||||
- 10.240.0.31
|
||||
- 10.240.0.33
|
||||
database_port_to_connect: 8500
|
||||
agent_port_to_connect: 3500
|
||||
|
||||
benchmark_options:
|
||||
type: write
|
||||
request_number: 1000000
|
||||
connection_number: 1000
|
||||
client_number: 1000
|
||||
# if specified, overwrite 'connection_number', 'connection_number'
|
||||
connection_client_numbers: []
|
||||
|
||||
# 0, to not rate limit
|
||||
rate_limit_requests_per_second: 0
|
||||
|
||||
# for 'write', 'read'
|
||||
same_key: false
|
||||
key_size_bytes: 256
|
||||
value_size_bytes: 1024
|
||||
|
||||
stale_read: false
|
||||
|
||||
benchmark_steps:
|
||||
step1_start_database: true
|
||||
step2_stress_database: true
|
||||
step3_stop_database: true
|
||||
step4_upload_logs: true
|
||||
|
||||
|
||||
datatbase_id_to_test_data:
|
||||
etcdv3:
|
||||
# if not empty, all test data paths are prefixed
|
||||
path_prefix: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/etcd-v3.1-go1.7.5
|
||||
client_system_metrics_interpolated_path: client-system-metrics-interpolated.csv
|
||||
client_latency_throughput_timeseries_path: client-latency-throughput-timeseries.csv
|
||||
client_latency_distribution_all_path: client-latency-distribution-all.csv
|
||||
client_latency_distribution_percentile_path: client-latency-distribution-percentile.csv
|
||||
client_latency_distribution_summary_path: client-latency-distribution-summary.csv
|
||||
client_latency_by_key_number_path: client-latency-by-key-number.csv
|
||||
server_disk_space_usage_summary_path: server-disk-space-usage-summary.csv
|
||||
server_memory_by_key_number_path: server-memory-by-key-number.csv
|
||||
server_system_metrics_interpolated_path_list:
|
||||
- 1-server-system-metrics-interpolated.csv
|
||||
- 2-server-system-metrics-interpolated.csv
|
||||
- 3-server-system-metrics-interpolated.csv
|
||||
all_aggregated_output_path: all-aggregated.csv
|
||||
|
||||
zookeeper:
|
||||
# if not empty, all test data paths are prefixed
|
||||
path_prefix: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/zookeeper-r3.4.9-java8
|
||||
client_system_metrics_interpolated_path: client-system-metrics-interpolated.csv
|
||||
client_latency_throughput_timeseries_path: client-latency-throughput-timeseries.csv
|
||||
client_latency_distribution_all_path: client-latency-distribution-all.csv
|
||||
client_latency_distribution_percentile_path: client-latency-distribution-percentile.csv
|
||||
client_latency_distribution_summary_path: client-latency-distribution-summary.csv
|
||||
client_latency_by_key_number_path: client-latency-by-key-number.csv
|
||||
server_disk_space_usage_summary_path: server-disk-space-usage-summary.csv
|
||||
server_memory_by_key_number_path: server-memory-by-key-number.csv
|
||||
server_system_metrics_interpolated_path_list:
|
||||
- 1-server-system-metrics-interpolated.csv
|
||||
- 2-server-system-metrics-interpolated.csv
|
||||
- 3-server-system-metrics-interpolated.csv
|
||||
all_aggregated_output_path: all-aggregated.csv
|
||||
|
||||
consul:
|
||||
# if not empty, all test data paths are prefixed
|
||||
path_prefix: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/consul-v0.7.4-go1.7.5
|
||||
client_system_metrics_interpolated_path: client-system-metrics-interpolated.csv
|
||||
client_latency_throughput_timeseries_path: client-latency-throughput-timeseries.csv
|
||||
client_latency_distribution_all_path: client-latency-distribution-all.csv
|
||||
client_latency_distribution_percentile_path: client-latency-distribution-percentile.csv
|
||||
client_latency_distribution_summary_path: client-latency-distribution-summary.csv
|
||||
client_latency_by_key_number_path: client-latency-by-key-number.csv
|
||||
server_disk_space_usage_summary_path: server-disk-space-usage-summary.csv
|
||||
server_memory_by_key_number_path: server-memory-by-key-number.csv
|
||||
server_system_metrics_interpolated_path_list:
|
||||
- 1-server-system-metrics-interpolated.csv
|
||||
- 2-server-system-metrics-interpolated.csv
|
||||
- 3-server-system-metrics-interpolated.csv
|
||||
all_aggregated_output_path: all-aggregated.csv
|
||||
|
||||
analyze:
|
||||
all_aggregated_output_path_csv: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/all-aggregated.csv
|
||||
all_aggregated_output_path_txt: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/all-aggregated.txt
|
||||
|
||||
plot_path_prefix: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client
|
||||
plot_list:
|
||||
- column: AVG-LATENCY-MS
|
||||
x_axis: Second
|
||||
y_axis: Latency(millisecond)
|
||||
|
||||
- column: AVG-THROUGHPUT
|
||||
x_axis: Second
|
||||
y_axis: Throughput(Requests/Second)
|
||||
|
||||
- column: AVG-VOLUNTARY-CTXT-SWITCHES
|
||||
x_axis: Second
|
||||
y_axis: Voluntary Context Switches
|
||||
|
||||
- column: AVG-NON-VOLUNTARY-CTXT-SWITCHES
|
||||
x_axis: Second
|
||||
y_axis: Non-voluntary Context Switches
|
||||
|
||||
- column: AVG-CPU
|
||||
x_axis: Second
|
||||
y_axis: CPU(%)
|
||||
|
||||
- column: AVG-VMRSS-MB
|
||||
x_axis: Second
|
||||
y_axis: Memory(MB)
|
||||
|
||||
- column: AVG-READS-COMPLETED-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Disk Reads (Delta per Second)
|
||||
|
||||
- column: AVG-SECTORS-READ-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Sectors Read (Delta per Second)
|
||||
|
||||
- column: AVG-WRITES-COMPLETED-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Disk Writes (Delta per Second)
|
||||
|
||||
- column: AVG-SECTORS-WRITTEN-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Sectors Written (Delta per Second)
|
||||
|
||||
- column: AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Network Receive(bytes) (Delta per Second)
|
||||
|
||||
- column: AVG-TRANSMIT-BYTES-NUM-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Network Transmit(bytes) (Delta per Second)
|
||||
|
||||
readme:
|
||||
output_path: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/README.md
|
||||
|
||||
images:
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-LATENCY-MS
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-LATENCY-MS.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-LATENCY-MS-BY-KEY
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-LATENCY-MS-BY-KEY.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-LATENCY-MS-BY-KEY-ERROR-POINTS
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-LATENCY-MS-BY-KEY-ERROR-POINTS.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-THROUGHPUT
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-THROUGHPUT.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-VOLUNTARY-CTXT-SWITCHES
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-VOLUNTARY-CTXT-SWITCHES.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-NON-VOLUNTARY-CTXT-SWITCHES
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-NON-VOLUNTARY-CTXT-SWITCHES.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-CPU
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-CPU.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-VMRSS-MB
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-VMRSS-MB.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-VMRSS-MB-BY-KEY
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-VMRSS-MB-BY-KEY.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-VMRSS-MB-BY-KEY-ERROR-POINTS
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-VMRSS-MB-BY-KEY-ERROR-POINTS.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-READS-COMPLETED-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-READS-COMPLETED-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-SECTORS-READ-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-SECTORS-READ-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-WRITES-COMPLETED-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-WRITES-COMPLETED-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-SECTORS-WRITTEN-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-SECTORS-WRITTEN-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-RECEIVE-BYTES-NUM-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-TRANSMIT-BYTES-NUM-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-1M-keys-1000-client/AVG-TRANSMIT-BYTES-NUM-DELTA.svg
|
||||
type: remote
|
||||
|
|
@ -1,7 +1,7 @@
|
|||
test_title: Write 3-million keys, 256-byte key, 1KB value, Best Throughput (etcd 1,000, Zookeeper 500, Consul 500 clients)
|
||||
test_description: |
|
||||
- Google Cloud Compute Engine
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 150 GB SSD (1 for client)
|
||||
- 4 machines of 16 vCPUs + 30 GB Memory + 300 GB SSD (1 for client)
|
||||
- Ubuntu 16.10
|
||||
- etcd v3.1 (Go 1.7.5)
|
||||
- Zookeeper r3.4.9
|
||||
|
|
@ -30,7 +30,7 @@ control:
|
|||
# set this in 'control' machine, to automate log uploading in remote 'agent' machines
|
||||
google_cloud_storage_key_path: /home/gyuho/gcloud-key.json
|
||||
google_cloud_storage_bucket_name: dbtester-results
|
||||
google_cloud_storage_sub_directory: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys
|
||||
google_cloud_storage_sub_directory: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys
|
||||
|
||||
all_database_id_list: [etcdv3, zookeeper, consul]
|
||||
|
||||
|
|
@ -167,7 +167,7 @@ datatbase_id_to_test_group:
|
|||
datatbase_id_to_test_data:
|
||||
etcdv3:
|
||||
# if not empty, all test data paths are prefixed
|
||||
path_prefix: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/etcd-v3.1-go1.7.5
|
||||
path_prefix: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/etcd-v3.1-go1.7.5
|
||||
client_system_metrics_interpolated_path: client-system-metrics-interpolated.csv
|
||||
client_latency_throughput_timeseries_path: client-latency-throughput-timeseries.csv
|
||||
client_latency_distribution_all_path: client-latency-distribution-all.csv
|
||||
|
|
@ -184,7 +184,7 @@ datatbase_id_to_test_data:
|
|||
|
||||
zookeeper:
|
||||
# if not empty, all test data paths are prefixed
|
||||
path_prefix: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/zookeeper-r3.4.9-java8
|
||||
path_prefix: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/zookeeper-r3.4.9-java8
|
||||
client_system_metrics_interpolated_path: client-system-metrics-interpolated.csv
|
||||
client_latency_throughput_timeseries_path: client-latency-throughput-timeseries.csv
|
||||
client_latency_distribution_all_path: client-latency-distribution-all.csv
|
||||
|
|
@ -201,7 +201,7 @@ datatbase_id_to_test_data:
|
|||
|
||||
consul:
|
||||
# if not empty, all test data paths are prefixed
|
||||
path_prefix: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/consul-v0.7.4-go1.7.5
|
||||
path_prefix: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/consul-v0.7.4-go1.7.5
|
||||
client_system_metrics_interpolated_path: client-system-metrics-interpolated.csv
|
||||
client_latency_throughput_timeseries_path: client-latency-throughput-timeseries.csv
|
||||
client_latency_distribution_all_path: client-latency-distribution-all.csv
|
||||
|
|
@ -217,10 +217,10 @@ datatbase_id_to_test_data:
|
|||
all_aggregated_output_path: all-aggregated.csv
|
||||
|
||||
analyze:
|
||||
all_aggregated_output_path_csv: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/all-aggregated.csv
|
||||
all_aggregated_output_path_txt: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/all-aggregated.txt
|
||||
all_aggregated_output_path_csv: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/all-aggregated.csv
|
||||
all_aggregated_output_path_txt: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/all-aggregated.txt
|
||||
|
||||
plot_path_prefix: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys
|
||||
plot_path_prefix: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys
|
||||
plot_list:
|
||||
- column: AVG-LATENCY-MS
|
||||
x_axis: Second
|
||||
|
|
@ -262,6 +262,14 @@ plot_list:
|
|||
x_axis: Second
|
||||
y_axis: Sectors Written (Delta per Second)
|
||||
|
||||
- column: AVG-READ-BYTES-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Read Bytes (Delta per Second)
|
||||
|
||||
- column: AVG-WRITE-BYTES-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Write Bytes (Delta per Second)
|
||||
|
||||
- column: AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
x_axis: Second
|
||||
y_axis: Network Receive(bytes) (Delta per Second)
|
||||
|
|
@ -271,69 +279,77 @@ plot_list:
|
|||
y_axis: Network Transmit(bytes) (Delta per Second)
|
||||
|
||||
readme:
|
||||
output_path: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/README.md
|
||||
output_path: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/README.md
|
||||
|
||||
images:
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-LATENCY-MS
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-LATENCY-MS.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-LATENCY-MS
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-LATENCY-MS.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-LATENCY-MS-BY-KEY
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-LATENCY-MS-BY-KEY.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-LATENCY-MS-BY-KEY
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-LATENCY-MS-BY-KEY.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-LATENCY-MS-BY-KEY-ERROR-POINTS
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-LATENCY-MS-BY-KEY-ERROR-POINTS.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-LATENCY-MS-BY-KEY-ERROR-POINTS
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-LATENCY-MS-BY-KEY-ERROR-POINTS.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-THROUGHPUT
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-THROUGHPUT.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-THROUGHPUT
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-THROUGHPUT.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-VOLUNTARY-CTXT-SWITCHES
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-VOLUNTARY-CTXT-SWITCHES.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-VOLUNTARY-CTXT-SWITCHES
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-VOLUNTARY-CTXT-SWITCHES.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-NON-VOLUNTARY-CTXT-SWITCHES
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-NON-VOLUNTARY-CTXT-SWITCHES.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-NON-VOLUNTARY-CTXT-SWITCHES
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-NON-VOLUNTARY-CTXT-SWITCHES.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-CPU
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-CPU.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-CPU
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-CPU.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-VMRSS-MB
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-VMRSS-MB.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-VMRSS-MB
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-VMRSS-MB.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-VMRSS-MB-BY-KEY
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-VMRSS-MB-BY-KEY.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-VMRSS-MB-BY-KEY
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-VMRSS-MB-BY-KEY.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-VMRSS-MB-BY-KEY-ERROR-POINTS
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-VMRSS-MB-BY-KEY-ERROR-POINTS.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-VMRSS-MB-BY-KEY-ERROR-POINTS
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-VMRSS-MB-BY-KEY-ERROR-POINTS.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-READS-COMPLETED-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-READS-COMPLETED-DELTA.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-READS-COMPLETED-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-READS-COMPLETED-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-SECTORS-READ-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-SECTORS-READ-DELTA.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-SECTORS-READ-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-SECTORS-READ-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-WRITES-COMPLETED-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-WRITES-COMPLETED-DELTA.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-WRITES-COMPLETED-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-WRITES-COMPLETED-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-SECTORS-WRITTEN-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-SECTORS-WRITTEN-DELTA.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-SECTORS-WRITTEN-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-SECTORS-WRITTEN-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-RECEIVE-BYTES-NUM-DELTA.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-READ-BYTES-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-READ-BYTES-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-TRANSMIT-BYTES-NUM-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/04-write-too-many-keys/AVG-TRANSMIT-BYTES-NUM-DELTA.svg
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-WRITE-BYTES-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-WRITE-BYTES-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-RECEIVE-BYTES-NUM-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-RECEIVE-BYTES-NUM-DELTA.svg
|
||||
type: remote
|
||||
|
||||
- title: 2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-TRANSMIT-BYTES-NUM-DELTA
|
||||
path: https://storage.googleapis.com/dbtester-results/2017Q1-00-etcd-zookeeper-consul/03-write-too-many-keys/AVG-TRANSMIT-BYTES-NUM-DELTA.svg
|
||||
type: remote
|
||||
|
|
@ -24,6 +24,18 @@ type Proc struct {
|
|||
WritesCompletedDelta uint64
|
||||
SectorsWrittenDelta uint64
|
||||
|
||||
// ReadBytesDelta is calculated from SectorsReadDelta
|
||||
// while SECTOR_SIZE is 512 (one sector is 512-byte) in Linux kernel
|
||||
// (http://lkml.iu.edu/hypermail/linux/kernel/1508.2/00431.html).
|
||||
ReadBytesDelta uint64
|
||||
ReadMegabytesDelta uint64
|
||||
|
||||
// WriteBytesDelta is calculated from SectorsWrittenDelta
|
||||
// while SECTOR_SIZE is 512 (one sector is 512-byte) in Linux kernel
|
||||
// (http://lkml.iu.edu/hypermail/linux/kernel/1508.2/00431.html).
|
||||
WriteBytesDelta uint64
|
||||
WriteMegabytesDelta uint64
|
||||
|
||||
NSEntry NSEntry
|
||||
ReceiveBytesDelta string
|
||||
ReceivePacketsDelta uint64
|
||||
|
|
@ -193,6 +205,11 @@ func init() {
|
|||
"WRITES-COMPLETED-DELTA",
|
||||
"SECTORS-WRITTEN-DELTA",
|
||||
|
||||
"READ-BYTES-DELTA",
|
||||
"READ-MEGABYTES-DELTA",
|
||||
"WRITE-BYTES-DELTA",
|
||||
"WRITE-MEGABYTES-DELTA",
|
||||
|
||||
"RECEIVE-BYTES-DELTA",
|
||||
"RECEIVE-PACKETS-DELTA",
|
||||
"TRANSMIT-BYTES-DELTA",
|
||||
|
|
@ -257,14 +274,19 @@ func (p *Proc) ToRow() (row []string) {
|
|||
row[37] = fmt.Sprintf("%d", p.WritesCompletedDelta) // WRITES-COMPLETED-DELTA
|
||||
row[38] = fmt.Sprintf("%d", p.SectorsWrittenDelta) // SECTORS-WRITTEN-DELTA
|
||||
|
||||
row[39] = p.ReceiveBytesDelta // RECEIVE-BYTES-DELTA
|
||||
row[40] = fmt.Sprintf("%d", p.ReceivePacketsDelta) // RECEIVE-PACKETS-DELTA
|
||||
row[41] = p.TransmitBytesDelta // TRANSMIT-BYTES-DELTA
|
||||
row[42] = fmt.Sprintf("%d", p.TransmitPacketsDelta) // TRANSMIT-PACKETS-DELTA
|
||||
row[43] = fmt.Sprintf("%d", p.ReceiveBytesNumDelta) // RECEIVE-BYTES-NUM-DELTA
|
||||
row[44] = fmt.Sprintf("%d", p.TransmitBytesNumDelta) // TRANSMIT-BYTES-NUM-DELTA
|
||||
row[39] = fmt.Sprintf("%d", p.ReadBytesDelta) // READ-BYTES-DELTA
|
||||
row[40] = fmt.Sprintf("%d", p.ReadMegabytesDelta) // READ-MEGABYTES-DELTA
|
||||
row[41] = fmt.Sprintf("%d", p.WriteBytesDelta) // WRITE-BYTES-DELTA
|
||||
row[42] = fmt.Sprintf("%d", p.WriteMegabytesDelta) // WRITE-MEGABYTES-DELTA
|
||||
|
||||
row[45] = string(p.Extra) // EXTRA
|
||||
row[43] = p.ReceiveBytesDelta // RECEIVE-BYTES-DELTA
|
||||
row[44] = fmt.Sprintf("%d", p.ReceivePacketsDelta) // RECEIVE-PACKETS-DELTA
|
||||
row[45] = p.TransmitBytesDelta // TRANSMIT-BYTES-DELTA
|
||||
row[46] = fmt.Sprintf("%d", p.TransmitPacketsDelta) // TRANSMIT-PACKETS-DELTA
|
||||
row[47] = fmt.Sprintf("%d", p.ReceiveBytesNumDelta) // RECEIVE-BYTES-NUM-DELTA
|
||||
row[48] = fmt.Sprintf("%d", p.TransmitBytesNumDelta) // TRANSMIT-BYTES-NUM-DELTA
|
||||
|
||||
row[49] = string(p.Extra) // EXTRA
|
||||
|
||||
return
|
||||
}
|
||||
|
|
|
|||
|
|
@ -103,6 +103,13 @@ func (c *CSV) Add() error {
|
|||
cur.WritesCompletedDelta = cur.DSEntry.WritesCompleted - prev.DSEntry.WritesCompleted
|
||||
cur.SectorsWrittenDelta = cur.DSEntry.SectorsWritten - prev.DSEntry.SectorsWritten
|
||||
|
||||
// SECTOR_SIZE is 512 (one sector is 512-byte) in Linux kernel
|
||||
// (http://lkml.iu.edu/hypermail/linux/kernel/1508.2/00431.html).
|
||||
cur.ReadBytesDelta = cur.SectorsReadDelta * 512
|
||||
cur.ReadMegabytesDelta = cur.SectorsReadDelta * 512 * (1 / 1000000)
|
||||
cur.WriteBytesDelta = cur.SectorsWrittenDelta * 512
|
||||
cur.WriteMegabytesDelta = cur.SectorsWrittenDelta * 512 * (1 / 1000000)
|
||||
|
||||
cur.ReceiveBytesNumDelta = cur.NSEntry.ReceiveBytesNum - prev.NSEntry.ReceiveBytesNum
|
||||
cur.TransmitBytesNumDelta = cur.NSEntry.TransmitBytesNum - prev.NSEntry.TransmitBytesNum
|
||||
cur.ReceivePacketsDelta = cur.NSEntry.ReceivePackets - prev.NSEntry.ReceivePackets
|
||||
|
|
@ -303,6 +310,23 @@ func ReadCSV(fpath string) (*CSV, error) {
|
|||
return nil, err
|
||||
}
|
||||
|
||||
readBytesDelta, err := strconv.ParseUint(row[ProcHeaderIndex["READ-BYTES-DELTA"]], 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
readMegabytesDelta, err := strconv.ParseUint(row[ProcHeaderIndex["READ-MEGABYTES-DELTA"]], 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writeBytesDelta, err := strconv.ParseUint(row[ProcHeaderIndex["WRITE-BYTES-DELTA"]], 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
writeMegabytesDelta, err := strconv.ParseUint(row[ProcHeaderIndex["WRITE-MEGABYTES-DELTA"]], 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
receivePackets, err := strconv.ParseUint(row[ProcHeaderIndex["RECEIVE-PACKETS"]], 10, 64)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
|
|
@ -380,6 +404,11 @@ func ReadCSV(fpath string) (*CSV, error) {
|
|||
WritesCompletedDelta: writesCompletedDelta,
|
||||
SectorsWrittenDelta: sectorsWrittenDelta,
|
||||
|
||||
ReadBytesDelta: readBytesDelta,
|
||||
ReadMegabytesDelta: readMegabytesDelta,
|
||||
WriteBytesDelta: writeBytesDelta,
|
||||
WriteMegabytesDelta: writeMegabytesDelta,
|
||||
|
||||
NSEntry: NSEntry{
|
||||
Interface: row[ProcHeaderIndex["INTERFACE"]],
|
||||
ReceiveBytes: row[ProcHeaderIndex["RECEIVE-BYTES"]],
|
||||
|
|
|
|||
|
|
@ -53,6 +53,10 @@ func Combine(procs ...Proc) Proc {
|
|||
sectorsReadDelta uint64
|
||||
writesCompletedDelta uint64
|
||||
sectorsWrittenDelta uint64
|
||||
readBytesDelta uint64
|
||||
readMegabytesDelta uint64
|
||||
writeBytesDelta uint64
|
||||
writeMegabytesDelta uint64
|
||||
|
||||
// for NSEntry
|
||||
receivePackets uint64
|
||||
|
|
@ -95,6 +99,10 @@ func Combine(procs ...Proc) Proc {
|
|||
sectorsReadDelta += p.SectorsReadDelta
|
||||
writesCompletedDelta += p.WritesCompletedDelta
|
||||
sectorsWrittenDelta += p.SectorsWrittenDelta
|
||||
readBytesDelta += p.ReadBytesDelta
|
||||
readMegabytesDelta += p.ReadMegabytesDelta
|
||||
writeBytesDelta += p.WriteBytesDelta
|
||||
writeMegabytesDelta += p.WriteMegabytesDelta
|
||||
|
||||
// for NSEntry
|
||||
receivePackets += p.NSEntry.ReceivePackets
|
||||
|
|
@ -141,6 +149,10 @@ func Combine(procs ...Proc) Proc {
|
|||
combined.SectorsReadDelta = uint64(sectorsReadDelta) / uint64(pN)
|
||||
combined.WritesCompletedDelta = uint64(writesCompletedDelta) / uint64(pN)
|
||||
combined.SectorsWrittenDelta = uint64(sectorsWrittenDelta) / uint64(pN)
|
||||
combined.ReadBytesDelta = uint64(readBytesDelta) / uint64(pN)
|
||||
combined.ReadMegabytesDelta = uint64(readMegabytesDelta) / uint64(pN)
|
||||
combined.WriteBytesDelta = uint64(writeBytesDelta) / uint64(pN)
|
||||
combined.WriteMegabytesDelta = uint64(writeMegabytesDelta) / uint64(pN)
|
||||
|
||||
// for NSEntry
|
||||
combined.NSEntry.ReceiveBytesNum = uint64(receiveBytesNum) / uint64(pN)
|
||||
|
|
@ -206,6 +218,10 @@ func Interpolate(lower, upper Proc) (procs []Proc, err error) {
|
|||
sectorsReadDelta = int64(upper.SectorsReadDelta-lower.SectorsReadDelta) / (expectedRowN - 1)
|
||||
writesCompletedDelta = int64(upper.WritesCompletedDelta-lower.WritesCompletedDelta) / (expectedRowN - 1)
|
||||
sectorsWrittenDelta = int64(upper.SectorsWrittenDelta-lower.SectorsWrittenDelta) / (expectedRowN - 1)
|
||||
readBytesDelta = int64(upper.ReadBytesDelta-lower.ReadBytesDelta) / (expectedRowN - 1)
|
||||
readMegabytesDelta = int64(upper.ReadMegabytesDelta-lower.ReadMegabytesDelta) / (expectedRowN - 1)
|
||||
writeBytesDelta = int64(upper.WriteBytesDelta-lower.WriteBytesDelta) / (expectedRowN - 1)
|
||||
writeMegabytesDelta = int64(upper.WriteMegabytesDelta-lower.WriteMegabytesDelta) / (expectedRowN - 1)
|
||||
|
||||
// for NSEntry
|
||||
receivePackets = int64(upper.NSEntry.ReceivePackets-lower.NSEntry.ReceivePackets) / (expectedRowN - 1)
|
||||
|
|
@ -257,6 +273,10 @@ func Interpolate(lower, upper Proc) (procs []Proc, err error) {
|
|||
procs[i].SectorsReadDelta = uint64(int64(lower.SectorsReadDelta) + int64(i+1)*sectorsReadDelta)
|
||||
procs[i].WritesCompletedDelta = uint64(int64(lower.WritesCompletedDelta) + int64(i+1)*writesCompletedDelta)
|
||||
procs[i].SectorsWrittenDelta = uint64(int64(lower.SectorsWrittenDelta) + int64(i+1)*sectorsWrittenDelta)
|
||||
procs[i].ReadBytesDelta = uint64(int64(lower.ReadBytesDelta) + int64(i+1)*readBytesDelta)
|
||||
procs[i].ReadMegabytesDelta = uint64(int64(lower.ReadMegabytesDelta) + int64(i+1)*readMegabytesDelta)
|
||||
procs[i].WriteBytesDelta = uint64(int64(lower.WriteBytesDelta) + int64(i+1)*writeBytesDelta)
|
||||
procs[i].WriteMegabytesDelta = uint64(int64(lower.WriteMegabytesDelta) + int64(i+1)*writeMegabytesDelta)
|
||||
|
||||
// for NSEntry
|
||||
procs[i].NSEntry.ReceiveBytesNum = uint64(int64(lower.NSEntry.ReceiveBytesNum) + int64(i+1)*receiveBytesNum)
|
||||
|
|
|
|||
Loading…
Reference in New Issue