analyze: just store raw data for key number, latency, memory data

analyze: use 'makeTag' for key columns

analyze/analyze_data_7_finalize_everything.go

@@ -383,14 +383,9 @@ func do(configPath string) error {
 		return err
 	}
 
-	var legends []string
-	for _, elem := range cfg.RawData {
-		legends = append(legends, elem.Legend)
-	}
-
 	// KEYS, AVG-LATENCY-MS
 	plog.Printf("combining data to %q", cfg.AllLatencyByKey)
-	allToLatency := make(map[int64]map[string]float64)
+	allLatencyFrame := dataframe.New()
 	for _, elem := range cfg.RawData {
 		fr, err := dataframe.NewFromCSV([]string{"KEYS", "AVG-LATENCY-MS"}, elem.DataBenchmarkLatencyByKey)
 		if err != nil {
@@ -400,55 +395,17 @@ func do(configPath string) error {
 		if err != nil {
 			return err
 		}
+		colKeys.UpdateHeader(makeHeader("KEYS", makeTag(elem.Legend)))
+		if err = allLatencyFrame.AddColumn(colKeys); err != nil {
+			return err
+		}
+
 		colLatency, err := fr.Column("AVG-LATENCY-MS")
 		if err != nil {
 			return err
 		}
-		for i := 0; i < colKeys.Count(); i++ {
+		colLatency.UpdateHeader(makeHeader("AVG-LATENCY-MS", makeTag(elem.Legend)))
-			vv1, err := colKeys.Value(i)
+		if err = allLatencyFrame.AddColumn(colLatency); err != nil {
-			if err != nil {
-				return err
-			}
-			kn, _ := vv1.Int64()
-
-			vv2, err := colLatency.Value(i)
-			if err != nil {
-				return err
-			}
-			va, _ := vv2.Float64()
-
-			if _, ok := allToLatency[kn]; !ok {
-				allToLatency[kn] = make(map[string]float64)
-			}
-			allToLatency[kn][elem.Legend] = va
-		}
-	}
-	allLatencyKeys := make([]int64, 0, len(allToLatency))
-	for k := range allToLatency {
-		allLatencyKeys = append(allLatencyKeys, k)
-	}
-	sort.Sort(int64Slice(allLatencyKeys))
-	cl1 := dataframe.NewColumn("KEYS")
-	clCols := make([]dataframe.Column, len(legends))
-	for i, legend := range legends {
-		clCols[i] = dataframe.NewColumn(makeHeader("AVG-LATENCY-MS", legend))
-	}
-	for _, keyNum := range allLatencyKeys {
-		cl1.PushBack(dataframe.NewStringValue(keyNum))
-		for i, legend := range legends {
-			var latV float64
-			if v, ok := allToLatency[keyNum][legend]; ok {
-				latV = v
-			}
-			clCols[i].PushBack(dataframe.NewStringValue(fmt.Sprintf("%f", latV)))
-		}
-	}
-	allLatencyFrame := dataframe.New()
-	if err := allLatencyFrame.AddColumn(cl1); err != nil {
-		return err
-	}
-	for _, col := range clCols {
-		if err := allLatencyFrame.AddColumn(col); err != nil {
 			return err
 		}
 	}
@@ -458,7 +415,7 @@ func do(configPath string) error {
 
 	// KEYS, AVG-VMRSS-MB
 	plog.Printf("combining data to %q", cfg.AllMemoryByKey)
-	allToMemory := make(map[int64]map[string]float64)
+	allMemoryFrame := dataframe.New()
 	for _, elem := range cfg.RawData {
 		fr, err := dataframe.NewFromCSV([]string{"KEYS", "AVG-VMRSS-MB"}, elem.DataBenchmarkMemoryByKey)
 		if err != nil {
@@ -468,55 +425,17 @@ func do(configPath string) error {
 		if err != nil {
 			return err
 		}
+		colKeys.UpdateHeader(makeHeader("KEYS", makeTag(elem.Legend)))
+		if err = allMemoryFrame.AddColumn(colKeys); err != nil {
+			return err
+		}
+
 		colMem, err := fr.Column("AVG-VMRSS-MB")
 		if err != nil {
 			return err
 		}
-		for i := 0; i < colKeys.Count(); i++ {
+		colMem.UpdateHeader(makeHeader("AVG-LATENCY-MS", makeTag(elem.Legend)))
-			vv1, err := colKeys.Value(i)
+		if err = allMemoryFrame.AddColumn(colMem); err != nil {
-			if err != nil {
-				return err
-			}
-			kn, _ := vv1.Int64()
-
-			vv2, err := colMem.Value(i)
-			if err != nil {
-				return err
-			}
-			va, _ := vv2.Float64()
-
-			if _, ok := allToMemory[kn]; !ok {
-				allToMemory[kn] = make(map[string]float64)
-			}
-			allToMemory[kn][elem.Legend] = va
-		}
-	}
-	allMemoryKeys := make([]int64, 0, len(allToMemory))
-	for k := range allToMemory {
-		allMemoryKeys = append(allMemoryKeys, k)
-	}
-	sort.Sort(int64Slice(allMemoryKeys))
-	cm1 := dataframe.NewColumn("KEYS")
-	cmCols := make([]dataframe.Column, len(legends))
-	for i, legend := range legends {
-		cmCols[i] = dataframe.NewColumn(makeHeader("AVG-VMRSS-MB", legend))
-	}
-	for _, keyNum := range allMemoryKeys {
-		cm1.PushBack(dataframe.NewStringValue(keyNum))
-		for i, legend := range legends {
-			var memoryV float64
-			if v, ok := allToMemory[keyNum][legend]; ok {
-				memoryV = v
-			}
-			cmCols[i].PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", memoryV)))
-		}
-	}
-	allMemoryFrame := dataframe.New()
-	if err := allMemoryFrame.AddColumn(cm1); err != nil {
-		return err
-	}
-	for _, col := range cmCols {
-		if err := allMemoryFrame.AddColumn(col); err != nil {
 			return err
 		}
 	}
@@ -661,9 +580,3 @@ func changeExtToTxt(fpath string) string {
 	ext := filepath.Ext(fpath)
 	return strings.Replace(fpath, ext, ".txt", -1)
 }
-
-type int64Slice []int64
-
-func (a int64Slice) Len() int           { return len(a) }
-func (a int64Slice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
-func (a int64Slice) Less(i, j int) bool { return a[i] < a[j] }

control/util_sort.go

@@ -0,0 +1,21 @@
+// Copyright 2017 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package control
+
+type int64Slice []int64
+
+func (a int64Slice) Len() int           { return len(a) }
+func (a int64Slice) Swap(i, j int)      { a[i], a[j] = a[j], a[i] }
+func (a int64Slice) Less(i, j int) bool { return a[i] < a[j] }