From b4250086dda09af489de7bc0ffa362d5cc50cbb1 Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Mon, 23 Jan 2017 20:49:07 -0800 Subject: [PATCH] analyze: clean up rows, update README --- README.md | 48 ++++--- analyze/analyze_data_7_finalize_everything.go | 130 +++++++++--------- 2 files changed, 85 insertions(+), 93 deletions(-) diff --git a/README.md b/README.md index fe0667e3..111707b8 100644 --- a/README.md +++ b/README.md @@ -37,31 +37,29 @@ All logs and results can be found at https://console.cloud.google.com/storage/br ##### Write 1M keys, 1000-client, 256-byte key, 1KB value ``` -+------------------------------+-------------------+------------------------+-----------------------+ -| | etcd-v3.1-go1.7.4 | zookeeper-r3.4.9-java8 | consul-v0.7.2-go1.7.4 | -+------------------------------+-------------------+------------------------+-----------------------+ -| READS-COMPLETED-DELTA | 6 | 6 | 15 | -| SECTORS-READS-DELTA-SUM | 0 | 0 | 0 | -| WRITES-COMPLETED-DELTA-SUM | 96474 | 77628 | 940695 | -| SECTORS-WRITTEN-DELTA-SUM | 542512 | 9387436 | 41272068 | -| RECEIVE-BYTES-SUM | 4.9 GB | 5.1 GB | 7.7 GB | -| RECEIVE-BYTES-NUM-DELTA-SUM | 4879653590 | 5061811231 | 7744263649 | -| TRANSMIT-BYTES-SUM | 3.7 GB | 4.1 GB | 6.5 GB | -| TRANSMIT-BYTES-NUM-DELTA-SUM | 3749247743 | 4050548646 | 6518060316 | -| TOTAL-SECONDS | 36.2024 s | 62.0373 s | 467.9311 s | -| AVG-THROUGHPUT | 27622.4453 | 15951.5555 | 2137.0667 | -| SLOWEST-LATENCY | 246.4560 ms | 6650.0930 ms | 30388.9318 ms | -| FASTEST-LATENCY | 5.3413 ms | 1.7698 ms | 21.5605 ms | -| AVG-LATENCY | 36.1057 ms | 37.7865 ms | 467.4253 ms | -| p10 | 13.712090 ms | 11.923543 ms | 65.910086 ms | -| p25 | 16.625779 ms | 14.581663 ms | 77.221971 ms | -| p50 | 22.306160 ms | 19.217649 ms | 120.663354 ms | -| p75 | 40.376905 ms | 23.642903 ms | 716.373543 ms | -| p90 | 65.849751 ms | 28.756700 ms | 1068.038406 ms | -| p95 | 137.545464 ms | 59.868096 ms | 1080.751412 ms | -| p99 | 177.127309 ms | 544.858078 ms | 2686.919571 ms | -| p99.9 | 198.540415 ms | 2457.827147 ms | 19041.188919 ms | -+------------------------------+-------------------+------------------------+-----------------------+ ++----------------------------+--------------------+------------------------+-----------------------+ +| | etcd-v3.1-go1.7.4 | zookeeper-r3.4.9-java8 | consul-v0.7.2-go1.7.4 | ++----------------------------+--------------------+------------------------+-----------------------+ +| READS-COMPLETED-DELTA | 6 | 6 | 15 | +| SECTORS-READS-DELTA-SUM | 0 | 0 | 0 | +| WRITES-COMPLETED-DELTA-SUM | 96474 | 77628 | 940695 | +| SECTORS-WRITTEN-DELTA-SUM | 542512 | 9387436 | 41272068 | +| RECEIVE-BYTES-SUM | 4.9 GB | 5.1 GB | 7.7 GB | +| TRANSMIT-BYTES-SUM | 3.7 GB | 4.1 GB | 6.5 GB | +| TOTAL-SECONDS | 36.2024 sec | 62.0373 sec | 467.9311 sec | +| AVG-THROUGHPUT | 27622.4453 req/sec | 15951.5555 req/sec | 2137.0667 req/sec | +| SLOWEST-LATENCY | 246.4560 ms | 6650.0930 ms | 30388.9318 ms | +| FASTEST-LATENCY | 5.3413 ms | 1.7698 ms | 21.5605 ms | +| AVG-LATENCY | 36.1057 ms | 37.7865 ms | 467.4253 ms | +| Latency p10 | 13.712090 ms | 11.923543 ms | 65.910086 ms | +| Latency p25 | 16.625779 ms | 14.581663 ms | 77.221971 ms | +| Latency p50 | 22.306160 ms | 19.217649 ms | 120.663354 ms | +| Latency p75 | 40.376905 ms | 23.642903 ms | 716.373543 ms | +| Latency p90 | 65.849751 ms | 28.756700 ms | 1068.038406 ms | +| Latency p95 | 137.545464 ms | 59.868096 ms | 1080.751412 ms | +| Latency p99 | 177.127309 ms | 544.858078 ms | 2686.919571 ms | +| Latency p99.9 | 198.540415 ms | 2457.827147 ms | 19041.188919 ms | ++----------------------------+--------------------+------------------------+-----------------------+ ``` diff --git a/analyze/analyze_data_7_finalize_everything.go b/analyze/analyze_data_7_finalize_everything.go index c0c89fb0..07979d04 100644 --- a/analyze/analyze_data_7_finalize_everything.go +++ b/analyze/analyze_data_7_finalize_everything.go @@ -83,23 +83,21 @@ func do(configPath string) error { // FIRST ROW FOR HEADER: etcd, Zookeeper, Consul, ... // FIRST COLUMN FOR LABELS: READS-COMPLETED-DELTA, ... // SECOND COLUMNS ~ FOR DATA - row1Header := []string{""} // first is empty + row00Header := []string{""} // first is empty for _, ad := range all.data { // per database for _, col := range ad.aggregated.Columns() { legend := all.headerToLegend[col.Header()] - row1Header = append(row1Header, makeTag(legend)) + row00Header = append(row00Header, makeTag(legend)) break } } - row2ReadsCompletedDeltaSum := []string{"READS-COMPLETED-DELTA"} - row3SectorsReadDeltaSum := []string{"SECTORS-READS-DELTA-SUM"} - row4WritesCompletedDeltaSum := []string{"WRITES-COMPLETED-DELTA-SUM"} - row5SectorsWrittenDeltaSum := []string{"SECTORS-WRITTEN-DELTA-SUM"} - row6ReceiveBytesSum := []string{"RECEIVE-BYTES-SUM"} - row7ReceiveBytesNumDeltaSum := []string{"RECEIVE-BYTES-NUM-DELTA-SUM"} - row8TransmitBytesSum := []string{"TRANSMIT-BYTES-SUM"} - row9TransmitBytesNumDeltaSum := []string{"TRANSMIT-BYTES-NUM-DELTA-SUM"} + row01ReadsCompletedDeltaSum := []string{"READS-COMPLETED-DELTA"} + row02SectorsReadDeltaSum := []string{"SECTORS-READS-DELTA-SUM"} + row03WritesCompletedDeltaSum := []string{"WRITES-COMPLETED-DELTA-SUM"} + row04SectorsWrittenDeltaSum := []string{"SECTORS-WRITTEN-DELTA-SUM"} + row05ReceiveBytesSum := []string{"RECEIVE-BYTES-SUM"} + row06TransmitBytesSum := []string{"TRANSMIT-BYTES-SUM"} // iterate each database's all data for _, ad := range all.data { @@ -179,34 +177,32 @@ func do(configPath string) error { } } - row2ReadsCompletedDeltaSum = append(row2ReadsCompletedDeltaSum, fmt.Sprintf("%d", uint64(readsCompletedDeltaSum))) - row3SectorsReadDeltaSum = append(row3SectorsReadDeltaSum, fmt.Sprintf("%d", uint64(sectorsReadDeltaSum))) - row4WritesCompletedDeltaSum = append(row4WritesCompletedDeltaSum, fmt.Sprintf("%d", uint64(writesCompletedDeltaSum))) - row5SectorsWrittenDeltaSum = append(row5SectorsWrittenDeltaSum, fmt.Sprintf("%d", uint64(sectorsWrittenDeltaSum))) - row6ReceiveBytesSum = append(row6ReceiveBytesSum, humanize.Bytes(uint64(receiveBytesNumDeltaSum))) - row7ReceiveBytesNumDeltaSum = append(row7ReceiveBytesNumDeltaSum, fmt.Sprintf("%d", uint64(receiveBytesNumDeltaSum))) - row8TransmitBytesSum = append(row8TransmitBytesSum, humanize.Bytes(uint64(transmitBytesNumDeltaSum))) - row9TransmitBytesNumDeltaSum = append(row9TransmitBytesNumDeltaSum, fmt.Sprintf("%d", uint64(transmitBytesNumDeltaSum))) + row01ReadsCompletedDeltaSum = append(row01ReadsCompletedDeltaSum, fmt.Sprintf("%d", uint64(readsCompletedDeltaSum))) + row02SectorsReadDeltaSum = append(row02SectorsReadDeltaSum, fmt.Sprintf("%d", uint64(sectorsReadDeltaSum))) + row03WritesCompletedDeltaSum = append(row03WritesCompletedDeltaSum, fmt.Sprintf("%d", uint64(writesCompletedDeltaSum))) + row04SectorsWrittenDeltaSum = append(row04SectorsWrittenDeltaSum, fmt.Sprintf("%d", uint64(sectorsWrittenDeltaSum))) + row05ReceiveBytesSum = append(row05ReceiveBytesSum, humanize.Bytes(uint64(receiveBytesNumDeltaSum))) + row06TransmitBytesSum = append(row06TransmitBytesSum, humanize.Bytes(uint64(transmitBytesNumDeltaSum))) } - row10TotalSeconds := []string{"TOTAL-SECONDS"} // TOTAL-SECONDS - row11AverageThroughput := []string{"AVG-THROUGHPUT"} // REQUESTS-PER-SECOND - row12SlowestLatency := []string{"SLOWEST-LATENCY"} // SLOWEST-LATENCY-MS - row13FastestLatency := []string{"FASTEST-LATENCY"} // FASTEST-LATENCY-MS - row14AverageLatency := []string{"AVG-LATENCY"} // AVERAGE-LATENCY-MS - row15p10 := []string{"p10"} // p10 - row16p25 := []string{"p25"} // p25 - row17p50 := []string{"p50"} // p50 - row18p75 := []string{"p75"} // p75 - row19p90 := []string{"p90"} // p90 - row20p95 := []string{"p95"} // p95 - row21p99 := []string{"p99"} // p99 - row22p999 := []string{"p99.9"} // p99.9 + row07TotalSeconds := []string{"TOTAL-SECONDS"} // TOTAL-SECONDS + row08AverageThroughput := []string{"AVG-THROUGHPUT"} // REQUESTS-PER-SECOND + row09SlowestLatency := []string{"SLOWEST-LATENCY"} // SLOWEST-LATENCY-MS + row10FastestLatency := []string{"FASTEST-LATENCY"} // FASTEST-LATENCY-MS + row11AverageLatency := []string{"AVG-LATENCY"} // AVERAGE-LATENCY-MS + row12p10 := []string{"Latency p10"} // p10 + row13p25 := []string{"Latency p25"} // p25 + row14p50 := []string{"Latency p50"} // p50 + row15p75 := []string{"Latency p75"} // p75 + row16p90 := []string{"Latency p90"} // p90 + row17p95 := []string{"Latency p95"} // p95 + row18p99 := []string{"Latency p99"} // p99 + row19p999 := []string{"Latency p99.9"} // p99.9 for i, rcfg := range cfg.RawData { tag := makeTag(rcfg.Legend) - if tag != row1Header[i+1] { - return fmt.Errorf("analyze config has different order; expected %q, got %q", row1Header[i+1], tag) + if tag != row00Header[i+1] { + return fmt.Errorf("analyze config has different order; expected %q, got %q", row00Header[i+1], tag) } { @@ -234,15 +230,15 @@ func do(configPath string) error { for _, row := range rows { switch row[0] { case "TOTAL-SECONDS": - row10TotalSeconds = append(row10TotalSeconds, fmt.Sprintf("%s s", row[1])) + row07TotalSeconds = append(row07TotalSeconds, fmt.Sprintf("%s sec", row[1])) case "REQUESTS-PER-SECOND": - row11AverageThroughput = append(row11AverageThroughput, row[1]) + row08AverageThroughput = append(row08AverageThroughput, fmt.Sprintf("%s req/sec", row[1])) case "SLOWEST-LATENCY-MS": - row12SlowestLatency = append(row12SlowestLatency, fmt.Sprintf("%s ms", row[1])) + row09SlowestLatency = append(row09SlowestLatency, fmt.Sprintf("%s ms", row[1])) case "FASTEST-LATENCY-MS": - row13FastestLatency = append(row13FastestLatency, fmt.Sprintf("%s ms", row[1])) + row10FastestLatency = append(row10FastestLatency, fmt.Sprintf("%s ms", row[1])) case "AVERAGE-LATENCY-MS": - row14AverageLatency = append(row14AverageLatency, fmt.Sprintf("%s ms", row[1])) + row11AverageLatency = append(row11AverageLatency, fmt.Sprintf("%s ms", row[1])) } } } @@ -275,49 +271,47 @@ func do(configPath string) error { } switch row[0] { case "p10": - row15p10 = append(row15p10, fmt.Sprintf("%s ms", row[1])) + row12p10 = append(row12p10, fmt.Sprintf("%s ms", row[1])) case "p25": - row16p25 = append(row16p25, fmt.Sprintf("%s ms", row[1])) + row13p25 = append(row13p25, fmt.Sprintf("%s ms", row[1])) case "p50": - row17p50 = append(row17p50, fmt.Sprintf("%s ms", row[1])) + row14p50 = append(row14p50, fmt.Sprintf("%s ms", row[1])) case "p75": - row18p75 = append(row18p75, fmt.Sprintf("%s ms", row[1])) + row15p75 = append(row15p75, fmt.Sprintf("%s ms", row[1])) case "p90": - row19p90 = append(row19p90, fmt.Sprintf("%s ms", row[1])) + row16p90 = append(row16p90, fmt.Sprintf("%s ms", row[1])) case "p95": - row20p95 = append(row20p95, fmt.Sprintf("%s ms", row[1])) + row17p95 = append(row17p95, fmt.Sprintf("%s ms", row[1])) case "p99": - row21p99 = append(row21p99, fmt.Sprintf("%s ms", row[1])) + row18p99 = append(row18p99, fmt.Sprintf("%s ms", row[1])) case "p99.9": - row22p999 = append(row22p999, fmt.Sprintf("%s ms", row[1])) + row19p999 = append(row19p999, fmt.Sprintf("%s ms", row[1])) } } } } aggRows := [][]string{ - row1Header, - row2ReadsCompletedDeltaSum, - row3SectorsReadDeltaSum, - row4WritesCompletedDeltaSum, - row5SectorsWrittenDeltaSum, - row6ReceiveBytesSum, - row7ReceiveBytesNumDeltaSum, - row8TransmitBytesSum, - row9TransmitBytesNumDeltaSum, - row10TotalSeconds, - row11AverageThroughput, - row12SlowestLatency, - row13FastestLatency, - row14AverageLatency, - row15p10, - row16p25, - row17p50, - row18p75, - row19p90, - row20p95, - row21p99, - row22p999, + row00Header, + row01ReadsCompletedDeltaSum, + row02SectorsReadDeltaSum, + row03WritesCompletedDeltaSum, + row04SectorsWrittenDeltaSum, + row05ReceiveBytesSum, + row06TransmitBytesSum, + row07TotalSeconds, + row08AverageThroughput, + row09SlowestLatency, + row10FastestLatency, + row11AverageLatency, + row12p10, + row13p25, + row14p50, + row15p75, + row16p90, + row17p95, + row18p99, + row19p999, } plog.Printf("saving data to %q", cfg.AllAggregatedPath) file, err := openToOverwrite(cfg.AllAggregatedPath)