etcd-io
/
dbtester
mirror of https://github.com/etcd-io/dbtester.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

analyze: just store raw data for key number, latency, memory data 

analyze: use 'makeTag' for key columns
This commit is contained in:
Gyu-Ho Lee 2017-02-02 10:33:29 -08:00
parent 7adc715ee3
commit 915ee98bdc
2 changed files with 37 additions and 103 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

119
analyze/analyze_data_7_finalize_everything.go
View File

@ -383,14 +383,9 @@ func do(configPath string) error {
return err
}
var legends []string
for _, elem := range cfg.RawData {
legends = append(legends, elem.Legend)
}
// KEYS, AVG-LATENCY-MS
plog.Printf("combining data to %q", cfg.AllLatencyByKey)
allToLatency := make(map[int64]map[string]float64)
allLatencyFrame := dataframe.New()
for _, elem := range cfg.RawData {
fr, err := dataframe.NewFromCSV([]string{"KEYS", "AVG-LATENCY-MS"}, elem.DataBenchmarkLatencyByKey)
if err != nil {
@ -400,55 +395,17 @@ func do(configPath string) error {
if err != nil {
return err
}
colKeys.UpdateHeader(makeHeader("KEYS", makeTag(elem.Legend)))
if err = allLatencyFrame.AddColumn(colKeys); err != nil {
return err
}
colLatency, err := fr.Column("AVG-LATENCY-MS")
if err != nil {
return err
}
for i := 0; i < colKeys.Count(); i++ {
vv1, err := colKeys.Value(i)
if err != nil {
return err
}
kn, _ := vv1.Int64()
vv2, err := colLatency.Value(i)
if err != nil {
return err
}
va, _ := vv2.Float64()
if _, ok := allToLatency[kn]; !ok {
allToLatency[kn] = make(map[string]float64)
}
allToLatency[kn][elem.Legend] = va
}
}
allLatencyKeys := make([]int64, 0, len(allToLatency))
for k := range allToLatency {
allLatencyKeys = append(allLatencyKeys, k)
}
sort.Sort(int64Slice(allLatencyKeys))
cl1 := dataframe.NewColumn("KEYS")
clCols := make([]dataframe.Column, len(legends))
for i, legend := range legends {
clCols[i] = dataframe.NewColumn(makeHeader("AVG-LATENCY-MS", legend))
}
for _, keyNum := range allLatencyKeys {
cl1.PushBack(dataframe.NewStringValue(keyNum))
for i, legend := range legends {
var latV float64
if v, ok := allToLatency[keyNum][legend]; ok {
latV = v
}
clCols[i].PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", latV)))
}
}
allLatencyFrame := dataframe.New()
if err := allLatencyFrame.AddColumn(cl1); err != nil {
return err
}
for _, col := range clCols {
if err := allLatencyFrame.AddColumn(col); err != nil {
colLatency.UpdateHeader(makeHeader("AVG-LATENCY-MS", makeTag(elem.Legend)))
if err = allLatencyFrame.AddColumn(colLatency); err != nil {
return err
}
}
@ -458,7 +415,7 @@ func do(configPath string) error {
// KEYS, AVG-VMRSS-MB
plog.Printf("combining data to %q", cfg.AllMemoryByKey)
allToMemory := make(map[int64]map[string]float64)
allMemoryFrame := dataframe.New()
for _, elem := range cfg.RawData {
fr, err := dataframe.NewFromCSV([]string{"KEYS", "AVG-VMRSS-MB"}, elem.DataBenchmarkMemoryByKey)
if err != nil {
@ -468,55 +425,17 @@ func do(configPath string) error {
if err != nil {
return err
}
colKeys.UpdateHeader(makeHeader("KEYS", makeTag(elem.Legend)))
if err = allMemoryFrame.AddColumn(colKeys); err != nil {
return err
}
colMem, err := fr.Column("AVG-VMRSS-MB")
if err != nil {
return err
}
for i := 0; i < colKeys.Count(); i++ {
vv1, err := colKeys.Value(i)
if err != nil {
return err
}
kn, _ := vv1.Int64()
vv2, err := colMem.Value(i)
if err != nil {
return err
}
va, _ := vv2.Float64()
if _, ok := allToMemory[kn]; !ok {
allToMemory[kn] = make(map[string]float64)
}
allToMemory[kn][elem.Legend] = va
}
}
allMemoryKeys := make([]int64, 0, len(allToMemory))
for k := range allToMemory {
allMemoryKeys = append(allMemoryKeys, k)
}
sort.Sort(int64Slice(allMemoryKeys))
cm1 := dataframe.NewColumn("KEYS")
cmCols := make([]dataframe.Column, len(legends))
for i, legend := range legends {
cmCols[i] = dataframe.NewColumn(makeHeader("AVG-VMRSS-MB", legend))
}
for _, keyNum := range allMemoryKeys {
cm1.PushBack(dataframe.NewStringValue(keyNum))
for i, legend := range legends {
var memoryV float64
if v, ok := allToMemory[keyNum][legend]; ok {
memoryV = v
}
cmCols[i].PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", memoryV)))
}
}
allMemoryFrame := dataframe.New()
if err := allMemoryFrame.AddColumn(cm1); err != nil {
return err
}
for _, col := range cmCols {
if err := allMemoryFrame.AddColumn(col); err != nil {
colMem.UpdateHeader(makeHeader("AVG-LATENCY-MS", makeTag(elem.Legend)))
if err = allMemoryFrame.AddColumn(colMem); err != nil {
return err
}
}
@ -661,9 +580,3 @@ func changeExtToTxt(fpath string) string {
ext := filepath.Ext(fpath)
return strings.Replace(fpath, ext, ".txt", -1)
}
type int64Slice []int64
func (a int64Slice) Len() int { return len(a) }
func (a int64Slice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a int64Slice) Less(i, j int) bool { return a[i] < a[j] }

21
control/util_sort.go Normal file
View File

@ -0,0 +1,21 @@
// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package control
type int64Slice []int64
func (a int64Slice) Len() int { return len(a) }
func (a int64Slice) Swap(i, j int) { a[i], a[j] = a[j], a[i] }
func (a int64Slice) Less(i, j int) bool { return a[i] < a[j] }