*: add configuration 'dynamic client number'

bench-configuration/00-write-2M-keys/01-etcd-v3.1.yaml

@@ -34,10 +34,10 @@ step2:
     skip_stress_database: false
     bench_type: write
     stale_read: true
-    connections: 1000
+    connections: 100
     clients: 1000
-    clients_delta: 0
-    clients_max: 0 # if not 0, increase clients per 100K requests (only for writes)
+    connections_clients_delta: 0
+    connections_clients_max: 0
     key_size: 8
     value_size: 256
     total_requests: 2000000

bench-configuration/00-write-2M-keys/02-zookeeper-r3.4.9.yaml

@@ -36,8 +36,8 @@ step2:
     stale_read: true
     connections: 1000
     clients: 1000
-    clients_delta: 0
-    clients_max: 0 # if not 0, increase clients per 100K requests (only for writes)
+    connections_clients_delta: 0
+    connections_clients_max: 0
     key_size: 8
     value_size: 256
     total_requests: 2000000

bench-configuration/00-write-2M-keys/03-consul-v0.7.2.yaml

@@ -36,8 +36,8 @@ step2:
     stale_read: true
     connections: 1000
     clients: 1000
-    clients_delta: 0
-    clients_max: 0 # if not 0, increase clients per 100K requests (only for writes)
+    connections_clients_delta: 0
+    connections_clients_max: 0
     key_size: 8
     value_size: 256
     total_requests: 2000000

bench-configuration/00-write-2M-keys/analyze.yaml

@@ -1,105 +1,105 @@
-title: Write 2M keys, 1000-client, 8-byte key, 256-byte value
-work_dir: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys
-all_aggregated_path: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/data-aggregated.csv
+title: Write 2M keys, 1000-client (etcd 100 TCP conns), 8-byte key, 256-byte value
+work_dir: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys
+all_aggregated_path: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/data-aggregated.csv
 
 raw_data:
 - legend: etcd v3.1 (Go 1.7.4)
   database_tag: etcd-v3.1-go1.7.4
-  output_path: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-aggregated.csv
+  output_path: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/etcd-v3.1-go1.7.4-aggregated.csv
   source_system_metrics_paths:
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-1-system-metrics.csv
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-2-system-metrics.csv
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-3-system-metrics.csv
-  source_benchmark_metrics_path: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-bench-result-timeseries.csv
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/etcd-v3.1-go1.7.4-1-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/etcd-v3.1-go1.7.4-2-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/etcd-v3.1-go1.7.4-3-system-metrics.csv
+  source_benchmark_metrics_path: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/etcd-v3.1-go1.7.4-bench-result-timeseries.csv
 
 - legend: Zookeeper r3.4.9 (Java 8)
   database_tag: zookeeper-r3.4.9-java8
-  output_path: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/zookeeper-r3.4.9-java8-aggregated.csv
+  output_path: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/zookeeper-r3.4.9-java8-aggregated.csv
   source_system_metrics_paths:
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/zookeeper-r3.4.9-java8-1-system-metrics.csv
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/zookeeper-r3.4.9-java8-2-system-metrics.csv
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/zookeeper-r3.4.9-java8-3-system-metrics.csv
-  source_benchmark_metrics_path: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/zookeeper-r3.4.9-java8-bench-result-timeseries.csv
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/zookeeper-r3.4.9-java8-1-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/zookeeper-r3.4.9-java8-2-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/zookeeper-r3.4.9-java8-3-system-metrics.csv
+  source_benchmark_metrics_path: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/zookeeper-r3.4.9-java8-bench-result-timeseries.csv
 
 - legend: Consul v0.7.2 (Go 1.7.4)
   database_tag: consul-v0.7.2-go1.7.4
-  output_path: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-aggregated.csv
+  output_path: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/consul-v0.7.2-go1.7.4-aggregated.csv
   source_system_metrics_paths:
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-1-system-metrics.csv
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-2-system-metrics.csv
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-3-system-metrics.csv
-  source_benchmark_metrics_path: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-bench-result-timeseries.csv
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/consul-v0.7.2-go1.7.4-1-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/consul-v0.7.2-go1.7.4-2-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/consul-v0.7.2-go1.7.4-3-system-metrics.csv
+  source_benchmark_metrics_path: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/consul-v0.7.2-go1.7.4-bench-result-timeseries.csv
 
 plot_list:
 - column: AVG-LATENCY-MS
   x_axis: Second
   y_axis: Latency(millisecond)
   output_path_list:
-    - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-LATENCY-MS.svg
-    - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-LATENCY-MS.png
+    - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-LATENCY-MS.svg
+    - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-LATENCY-MS.png
 
 - column: AVG-THROUGHPUT
   x_axis: Second
   y_axis: Throughput(Requests/Second)
   output_path_list:
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-THROUGHPUT.svg
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-THROUGHPUT.png
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-THROUGHPUT.svg
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-THROUGHPUT.png
 
 - column: AVG-CPU
   x_axis: Second
   y_axis: CPU(%)
   output_path_list:
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-CPU.svg
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-CPU.png
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-CPU.svg
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-CPU.png
 
 - column: AVG-VMRSS-MB
   x_axis: Second
   y_axis: Memory(MB)
   output_path_list:
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-VMRSS-MB.svg
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-VMRSS-MB.png
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-VMRSS-MB.svg
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-VMRSS-MB.png
 
 - column: AVG-READS-COMPLETED-DELTA
   x_axis: Second
   y_axis: Disk Reads
   output_path_list:
-    - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-READS-COMPLETED-DELTA.svg
-    - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-READS-COMPLETED-DELTA.png
+    - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-READS-COMPLETED-DELTA.svg
+    - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-READS-COMPLETED-DELTA.png
 
 - column: AVG-SECTORS-READ-DELTA
   x_axis: Second
   y_axis: Sectors Read
   output_path_list:
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-SECTORS-READ-DELTA.svg
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-SECTORS-READ-DELTA.png
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-SECTORS-READ-DELTA.svg
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-SECTORS-READ-DELTA.png
 
 - column: AVG-WRITES-COMPLETED-DELTA
   x_axis: Second
   y_axis: Disk Writes
   output_path_list:
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-WRITES-COMPLETED-DELTA.svg
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-WRITES-COMPLETED-DELTA.png
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-WRITES-COMPLETED-DELTA.svg
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-WRITES-COMPLETED-DELTA.png
 
 - column: AVG-SECTORS-WRITTEN-DELTA
   x_axis: Second
   y_axis: Sectors Written
   output_path_list:
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-SECTORS-WRITTEN-DELTA.svg
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-SECTORS-WRITTEN-DELTA.png
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-SECTORS-WRITTEN-DELTA.svg
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-SECTORS-WRITTEN-DELTA.png
 
 - column: AVG-RECEIVE-BYTES-NUM-DELTA
   x_axis: Second
   y_axis: Network Receive(bytes)
   output_path_list:
-    - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-RECEIVE-BYTES-NUM-DELTA.svg
-    - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-RECEIVE-BYTES-NUM-DELTA.png
+    - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-RECEIVE-BYTES-NUM-DELTA.svg
+    - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-RECEIVE-BYTES-NUM-DELTA.png
 
 - column: AVG-TRANSMIT-BYTES-NUM-DELTA
   x_axis: Second
   y_axis: Network Transmit(bytes)
   output_path_list:
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-TRANSMIT-BYTES-NUM-DELTA.svg
-  - 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-TRANSMIT-BYTES-NUM-DELTA.png
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-TRANSMIT-BYTES-NUM-DELTA.svg
+  - 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-TRANSMIT-BYTES-NUM-DELTA.png
 
 readme:
   preface: |
@@ -117,47 +117,47 @@ readme:
     - zetcd v3.1 (Go 1.7.4)
     - cetcd v3.1 (Go 1.7.4)
 
-  output_path: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/README.md
+  output_path: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/README.md
   results:
   - title: Write 2M keys, 1000-client (etcd v3.1 100-conn), 8-byte key, 256-byte value
     images:
 
-    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-LATENCY-MS
-      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-LATENCY-MS.svg
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-LATENCY-MS
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-LATENCY-MS.svg
       image_type: remote
 
-    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-THROUGHPUT
-      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-THROUGHPUT.svg
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-THROUGHPUT
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-THROUGHPUT.svg
       image_type: remote
 
-    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-CPU
-      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-CPU.svg
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-CPU
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-CPU.svg
       image_type: remote
 
-    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-VMRSS-MB
-      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-VMRSS-MB.svg
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-VMRSS-MB
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-VMRSS-MB.svg
       image_type: remote
 
-    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-READS-COMPLETED-DELTA
-      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-READS-COMPLETED-DELTA.svg
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-READS-COMPLETED-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-READS-COMPLETED-DELTA.svg
       image_type: remote
 
-    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-SECTORS-READ-DELTA
-      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-SECTORS-READ-DELTA.svg
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-SECTORS-READ-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-SECTORS-READ-DELTA.svg
       image_type: remote
 
-    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-WRITES-COMPLETED-DELTA
-      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-WRITES-COMPLETED-DELTA.svg
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-WRITES-COMPLETED-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-WRITES-COMPLETED-DELTA.svg
       image_type: remote
 
-    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-SECTORS-WRITTEN-DELTA
-      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-SECTORS-WRITTEN-DELTA.svg
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-SECTORS-WRITTEN-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-SECTORS-WRITTEN-DELTA.svg
       image_type: remote
 
-    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-RECEIVE-BYTES-NUM-DELTA
-      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-RECEIVE-BYTES-NUM-DELTA.svg
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-RECEIVE-BYTES-NUM-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-RECEIVE-BYTES-NUM-DELTA.svg
       image_type: remote
 
-    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-TRANSMIT-BYTES-NUM-DELTA
-      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys/AVG-TRANSMIT-BYTES-NUM-DELTA.svg
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-TRANSMIT-BYTES-NUM-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/00-write-2M-keys/AVG-TRANSMIT-BYTES-NUM-DELTA.svg
       image_type: remote

bench-configuration/01-write/01-etcd-v3.1.yaml

@@ -0,0 +1,49 @@
+database: etcdv3
+test_name: etcd-v3.1-go1.7.4
+
+google_cloud_project_name: etcd-development
+google_cloud_storage_key_path: /home/gyuho/gcloud-key.json # need this only in 'control' side
+google_cloud_storage_bucket_name: dbtester-results
+google_cloud_storage_sub_directory: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys
+# google_cloud_storage_sub_directory: 2017Q1-01-etcd-zookeeper-consul/02-write-2M-keys-1000QPS
+# google_cloud_storage_sub_directory: 2017Q1-01-etcd-zookeeper-consul/03-write-500K-keys-1CONN
+
+peer_ips:
+    - 10.240.0.20
+    - 10.240.0.21
+    - 10.240.0.22
+
+agent_port: 3500
+database_port: 2379
+
+log: control.log
+data_latency_distribution_summary: data-latency-distribution-summary.csv
+data_latency_distribution_percentile: data-latency-distribution-percentile.csv
+data_latency_distribution_all: data-latency-distribution-all.csv
+data_latency_throughput_timeseries: data-latency-throughput-timeseries.csv
+
+# start database by sending RPC calls to agents
+step1:
+    skip_start_database: false
+    etcd_snap_count: 100000
+    zookeeper_snap_count: 100000
+    zookeeper_max_client_connections: 5000
+
+# start benchmark
+step2:
+    skip_stress_database: false
+    bench_type: write
+    stale_read: true
+    connections: 100
+    clients: 100
+    connections_clients_delta: 100
+    connections_clients_max: 2000
+    key_size: 8
+    value_size: 256
+    total_requests: 2000000
+    requests_per_second: 0
+    etcdv3_compaction_cycle: 0
+
+# after benchmark
+step3:
+    action: stop

bench-configuration/01-write/02-zookeeper-r3.4.9.yaml

@@ -0,0 +1,49 @@
+database: zookeeper
+test_name: zookeeper-r3.4.9-java8
+
+google_cloud_project_name: etcd-development
+google_cloud_storage_key_path: /home/gyuho/gcloud-key.json # need this only in 'control' side
+google_cloud_storage_bucket_name: dbtester-results
+google_cloud_storage_sub_directory: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys
+# google_cloud_storage_sub_directory: 2017Q1-01-etcd-zookeeper-consul/02-write-2M-keys-1000QPS
+# google_cloud_storage_sub_directory: 2017Q1-01-etcd-zookeeper-consul/03-write-500K-keys-1CONN
+
+peer_ips:
+    - 10.240.0.25
+    - 10.240.0.27
+    - 10.240.0.28
+
+agent_port: 3500
+database_port: 2181
+
+log: control.log
+data_latency_distribution_summary: data-latency-distribution-summary.csv
+data_latency_distribution_percentile: data-latency-distribution-percentile.csv
+data_latency_distribution_all: data-latency-distribution-all.csv
+data_latency_throughput_timeseries: data-latency-throughput-timeseries.csv
+
+# start database by sending RPC calls to agents
+step1:
+    skip_start_database: false
+    etcd_snap_count: 100000
+    zookeeper_snap_count: 100000
+    zookeeper_max_client_connections: 5000
+
+# start benchmark
+step2:
+    skip_stress_database: false
+    bench_type: write
+    stale_read: true
+    connections: 1000
+    clients: 1000
+    connections_clients_delta: 100
+    connections_clients_max: 2000
+    key_size: 8
+    value_size: 256
+    total_requests: 2000000
+    requests_per_second: 0
+    etcdv3_compaction_cycle: 0
+
+# after benchmark
+step3:
+    action: stop

bench-configuration/01-write/03-consul-v0.7.2.yaml

@@ -0,0 +1,49 @@
+database: consul
+test_name: consul-v0.7.2-go1.7.4
+
+google_cloud_project_name: etcd-development
+google_cloud_storage_key_path: /home/gyuho/gcloud-key.json # need this only in 'control' side
+google_cloud_storage_bucket_name: dbtester-results
+google_cloud_storage_sub_directory: 2017Q1-01-etcd-zookeeper-consul/01-write-2M-keys
+# google_cloud_storage_sub_directory: 2017Q1-01-etcd-zookeeper-consul/02-write-2M-keys-1000QPS
+# google_cloud_storage_sub_directory: 2017Q1-01-etcd-zookeeper-consul/03-write-500K-keys-1CONN
+
+peer_ips:
+    - 10.240.0.30
+    - 10.240.0.31
+    - 10.240.0.35
+
+agent_port: 3500
+database_port: 8500
+
+log: control.log
+data_latency_distribution_summary: data-latency-distribution-summary.csv
+data_latency_distribution_percentile: data-latency-distribution-percentile.csv
+data_latency_distribution_all: data-latency-distribution-all.csv
+data_latency_throughput_timeseries: data-latency-throughput-timeseries.csv
+
+# start database by sending RPC calls to agents
+step1:
+    skip_start_database: false
+    etcd_snap_count: 100000
+    zookeeper_snap_count: 100000
+    zookeeper_max_client_connections: 5000
+
+# start benchmark
+step2:
+    skip_stress_database: false
+    bench_type: write
+    stale_read: true
+    connections: 1000
+    clients: 1000
+    connections_clients_delta: 100
+    connections_clients_max: 2000
+    key_size: 8
+    value_size: 256
+    total_requests: 2000000
+    requests_per_second: 0
+    etcdv3_compaction_cycle: 0
+
+# after benchmark
+step3:
+    action: stop

bench-configuration/01-write/analyze.yaml

@@ -0,0 +1,163 @@
+title: Write 2M keys, 1000-client, 8-byte key, 256-byte value
+work_dir: 2017Q1-01-etcd-zookeeper-consul/01-write
+all_aggregated_path: 2017Q1-01-etcd-zookeeper-consul/01-write/data-aggregated.csv
+
+raw_data:
+- legend: etcd v3.1 (Go 1.7.4)
+  database_tag: etcd-v3.1-go1.7.4
+  output_path: 2017Q1-01-etcd-zookeeper-consul/01-write/etcd-v3.1-go1.7.4-aggregated.csv
+  source_system_metrics_paths:
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/etcd-v3.1-go1.7.4-1-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/etcd-v3.1-go1.7.4-2-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/etcd-v3.1-go1.7.4-3-system-metrics.csv
+  source_benchmark_metrics_path: 2017Q1-01-etcd-zookeeper-consul/01-write/etcd-v3.1-go1.7.4-bench-result-timeseries.csv
+
+- legend: Zookeeper r3.4.9 (Java 8)
+  database_tag: zookeeper-r3.4.9-java8
+  output_path: 2017Q1-01-etcd-zookeeper-consul/01-write/zookeeper-r3.4.9-java8-aggregated.csv
+  source_system_metrics_paths:
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/zookeeper-r3.4.9-java8-1-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/zookeeper-r3.4.9-java8-2-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/zookeeper-r3.4.9-java8-3-system-metrics.csv
+  source_benchmark_metrics_path: 2017Q1-01-etcd-zookeeper-consul/01-write/zookeeper-r3.4.9-java8-bench-result-timeseries.csv
+
+- legend: Consul v0.7.2 (Go 1.7.4)
+  database_tag: consul-v0.7.2-go1.7.4
+  output_path: 2017Q1-01-etcd-zookeeper-consul/01-write/consul-v0.7.2-go1.7.4-aggregated.csv
+  source_system_metrics_paths:
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/consul-v0.7.2-go1.7.4-1-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/consul-v0.7.2-go1.7.4-2-system-metrics.csv
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/consul-v0.7.2-go1.7.4-3-system-metrics.csv
+  source_benchmark_metrics_path: 2017Q1-01-etcd-zookeeper-consul/01-write/consul-v0.7.2-go1.7.4-bench-result-timeseries.csv
+
+plot_list:
+- column: AVG-LATENCY-MS
+  x_axis: Second
+  y_axis: Latency(millisecond)
+  output_path_list:
+    - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-LATENCY-MS.svg
+    - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-LATENCY-MS.png
+
+- column: AVG-THROUGHPUT
+  x_axis: Second
+  y_axis: Throughput(Requests/Second)
+  output_path_list:
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-THROUGHPUT.svg
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-THROUGHPUT.png
+
+- column: AVG-CPU
+  x_axis: Second
+  y_axis: CPU(%)
+  output_path_list:
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-CPU.svg
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-CPU.png
+
+- column: AVG-VMRSS-MB
+  x_axis: Second
+  y_axis: Memory(MB)
+  output_path_list:
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-VMRSS-MB.svg
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-VMRSS-MB.png
+
+- column: AVG-READS-COMPLETED-DELTA
+  x_axis: Second
+  y_axis: Disk Reads
+  output_path_list:
+    - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-READS-COMPLETED-DELTA.svg
+    - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-READS-COMPLETED-DELTA.png
+
+- column: AVG-SECTORS-READ-DELTA
+  x_axis: Second
+  y_axis: Sectors Read
+  output_path_list:
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-SECTORS-READ-DELTA.svg
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-SECTORS-READ-DELTA.png
+
+- column: AVG-WRITES-COMPLETED-DELTA
+  x_axis: Second
+  y_axis: Disk Writes
+  output_path_list:
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-WRITES-COMPLETED-DELTA.svg
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-WRITES-COMPLETED-DELTA.png
+
+- column: AVG-SECTORS-WRITTEN-DELTA
+  x_axis: Second
+  y_axis: Sectors Written
+  output_path_list:
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-SECTORS-WRITTEN-DELTA.svg
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-SECTORS-WRITTEN-DELTA.png
+
+- column: AVG-RECEIVE-BYTES-NUM-DELTA
+  x_axis: Second
+  y_axis: Network Receive(bytes)
+  output_path_list:
+    - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-RECEIVE-BYTES-NUM-DELTA.svg
+    - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-RECEIVE-BYTES-NUM-DELTA.png
+
+- column: AVG-TRANSMIT-BYTES-NUM-DELTA
+  x_axis: Second
+  y_axis: Network Transmit(bytes)
+  output_path_list:
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-TRANSMIT-BYTES-NUM-DELTA.svg
+  - 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-TRANSMIT-BYTES-NUM-DELTA.png
+
+readme:
+  preface: |
+    - Google Cloud Compute Engine
+    - 3 machines of 8 vCPUs + 1 6GB Memory + 50 GB SSD
+    - 1 machine(client) of 16 vCPUs + 30 GB Memory + 50 GB SSD
+    - Ubuntu 16.10
+    - etcd v3.1 (Go 1.7.4)
+    - Zookeepr r3.4.9
+      - Java 8
+      - Java(TM) SE Runtime Environment (build 1.8.0_111-b14)
+      - Java HotSpot(TM) 64-Bit Server VM (build 25.111-b14, mixed mode)
+      - javac 1.8.0_111
+    - Consul v0.7.2 (Go 1.7.4)
+    - zetcd v3.1 (Go 1.7.4)
+    - cetcd v3.1 (Go 1.7.4)
+
+  output_path: 2017Q1-01-etcd-zookeeper-consul/01-write/README.md
+  results:
+  - title: Write 2M keys, 1000-client (etcd v3.1 100-conn), 8-byte key, 256-byte value
+    images:
+
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-LATENCY-MS
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write/AVG-LATENCY-MS.svg
+      image_type: remote
+
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-THROUGHPUT
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write/AVG-THROUGHPUT.svg
+      image_type: remote
+
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-CPU
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write/AVG-CPU.svg
+      image_type: remote
+
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-VMRSS-MB
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write/AVG-VMRSS-MB.svg
+      image_type: remote
+
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-READS-COMPLETED-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write/AVG-READS-COMPLETED-DELTA.svg
+      image_type: remote
+
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-SECTORS-READ-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write/AVG-SECTORS-READ-DELTA.svg
+      image_type: remote
+
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-WRITES-COMPLETED-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write/AVG-WRITES-COMPLETED-DELTA.svg
+      image_type: remote
+
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-SECTORS-WRITTEN-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write/AVG-SECTORS-WRITTEN-DELTA.svg
+      image_type: remote
+
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-RECEIVE-BYTES-NUM-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write/AVG-RECEIVE-BYTES-NUM-DELTA.svg
+      image_type: remote
+
+    - image_title: 2017Q1-01-etcd-zookeeper-consul/01-write/AVG-TRANSMIT-BYTES-NUM-DELTA
+      image_path: https://storage.googleapis.com/dbtester-results/2017Q1-01-etcd-zookeeper-consul/01-write/AVG-TRANSMIT-BYTES-NUM-DELTA.svg
+      image_type: remote

bench-configuration/read-2M-keys-linearizable/01-etcd-v3.1.yaml

@@ -36,8 +36,8 @@ step2:
     stale_read: false
     connections: 1000
     clients: 1000
-    clients_delta: 0
-    clients_max: 0 # if not 0, increase clients per 100K requests (only for writes)
+    connections_clients_delta: 0
+    connections_clients_max: 0
     key_size: 8
     value_size: 256
     total_requests: 2000000

bench-configuration/read-2M-keys-linearizable/02-zookeeper-r3.4.9.yaml

@@ -36,8 +36,8 @@ step2:
     stale_read: false
     connections: 1000
     clients: 1000
-    clients_delta: 0
-    clients_max: 0 # if not 0, increase clients per 100K requests (only for writes)
+    connections_clients_delta: 0
+    connections_clients_max: 0
     key_size: 8
     value_size: 256
     total_requests: 2000000

bench-configuration/read-2M-keys-linearizable/03-consul-v0.7.2.yaml

@@ -36,8 +36,8 @@ step2:
     stale_read: false
     connections: 1000
     clients: 1000
-    clients_delta: 0
-    clients_max: 0 # if not 0, increase clients per 100K requests (only for writes)
+    connections_clients_delta: 0
+    connections_clients_max: 0
     key_size: 8
     value_size: 256
     total_requests: 2000000

control/config.go

@@ -56,18 +56,18 @@ type Config struct {
 	} `yaml:"step1"`
 
 	Step2 struct {
-		SkipStressDatabase bool   `yaml:"skip_stress_database"`
-		BenchType          string `yaml:"bench_type"`
-		StaleRead          bool   `yaml:"stale_read"`
-		Connections        int    `yaml:"connections"`
-		Clients            int    `yaml:"clients"`
-		ClientsDelta       int    `yaml:"clients_delta"`
-		ClientsMax         int    `yaml:"clients_max"`
-		KeySize            int    `yaml:"key_size"`
-		SameKey            bool   `yaml:"same_key"`
-		ValueSize          int    `yaml:"value_size"`
-		TotalRequests      int    `yaml:"total_requests"`
-		RequestsPerSecond  int    `yaml:"requests_per_second"`
+		SkipStressDatabase      bool   `yaml:"skip_stress_database"`
+		BenchType               string `yaml:"bench_type"`
+		StaleRead               bool   `yaml:"stale_read"`
+		Connections             int    `yaml:"connections"`
+		Clients                 int    `yaml:"clients"`
+		ConnectionsClientsDelta int    `yaml:"connections_clients_delta"`
+		ConnectionsClientsMax   int    `yaml:"connections_clients_max"`
+		KeySize                 int    `yaml:"key_size"`
+		SameKey                 bool   `yaml:"same_key"`
+		ValueSize               int    `yaml:"value_size"`
+		TotalRequests           int    `yaml:"total_requests"`
+		RequestsPerSecond       int    `yaml:"requests_per_second"`
 	} `yaml:"step2"`
 
 	Step3 struct {

control/config_test.go

@@ -90,11 +90,11 @@ func TestReadConfig(t *testing.T) {
 	if c.Step2.Clients != 100 {
 		t.Fatalf("unexpected %d", c.Step2.Clients)
 	}
-	if c.Step2.ClientsDelta != 100 {
-		t.Fatalf("unexpected %d", c.Step2.ClientsDelta)
+	if c.Step2.ConnectionsClientsDelta != 100 {
+		t.Fatalf("unexpected %d", c.Step2.ConnectionsClientsDelta)
 	}
-	if c.Step2.ClientsMax != 2000 {
-		t.Fatalf("unexpected %d", c.Step2.ClientsMax)
+	if c.Step2.ConnectionsClientsMax != 2000 {
+		t.Fatalf("unexpected %d", c.Step2.ConnectionsClientsMax)
 	}
 	if c.Step2.KeySize != 8 {
 		t.Fatalf("unexpected %d", c.Step2.KeySize)

control/config_test.yaml

@@ -34,8 +34,8 @@ step2:
     stale_read: true
     connections: 100
     clients: 100
-    clients_delta: 100
-    clients_max: 2000 # if not 0, increase clients per 100K requests (only for writes)
+    connections_clients_delta: 100
+    connections_clients_max: 2000 # if not 0, increase clients per 100K requests (only for writes)
     key_size: 8
     same_key: true
     value_size: 256

control/step2_stress_database.go

@@ -323,7 +323,7 @@ func step2StressDatabase(cfg Config) error {
 	switch cfg.Step2.BenchType {
 	case "write":
 		plog.Println("write generateReport is started...")
-		if cfg.Step2.ClientsMax == 0 {
+		if cfg.Step2.ConnectionsClientsMax == 0 {
 			h, done := newWriteHandlers(cfg)
 			reqGen := func(inflightReqs chan<- request) { generateWrites(cfg, vals, inflightReqs) }
 			generateReport(cfg, h, done, reqGen)
@@ -354,9 +354,9 @@ func step2StressDatabase(cfg Config) error {
 				b.waitRequestsEnd()
 
 				// update request handlers, generator
-				copied.Step2.Clients += copied.Step2.ClientsDelta
-				if copied.Step2.Clients > copied.Step2.ClientsMax {
-					copied.Step2.Clients = copied.Step2.ClientsMax
+				copied.Step2.Clients += copied.Step2.ConnectionsClientsDelta
+				if copied.Step2.Clients > copied.Step2.ConnectionsClientsMax {
+					copied.Step2.Clients = copied.Step2.ConnectionsClientsMax
 				}
 				h, done = newWriteHandlers(copied)
 				reqGen = func(inflightReqs chan<- request) { generateWrites(copied, vals, inflightReqs) }