etcd-io
/
dbtester
mirror of https://github.com/etcd-io/dbtester.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

bench-configuration: run etcd first

This commit is contained in:
Gyu-Ho Lee 2017-01-09 14:45:02 -08:00
parent 27ca7564c9
commit dfc6eb87f8
No known key found for this signature in database
GPG Key ID: 1DDD39C7EB70C24C
10 changed files with 50 additions and 147 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

6
bench-configuration/01-write-2M-keys/02-etcd-v3.1.yaml → bench-configuration/01-write-2M-keys/01-etcd-v3.1.yaml
View File

@ -9,9 +9,9 @@ google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/01-write-2M-keys
# google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/03-write-500K-keys-1CONN
peer_ips:
- 10.240.0.25
- 10.240.0.27
- 10.240.0.28
- 10.240.0.20
- 10.240.0.21
- 10.240.0.22
agent_port: 3500
database_port: 2379

6
bench-configuration/01-write-2M-keys/01-zk-r3.4.9.yaml → bench-configuration/01-write-2M-keys/02-zk-r3.4.9.yaml
View File

@ -9,9 +9,9 @@ google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/01-write-2M-keys
# google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/03-write-500K-keys-1CONN
peer_ips:
- 10.240.0.20
- 10.240.0.21
- 10.240.0.22
- 10.240.0.25
- 10.240.0.27
- 10.240.0.28
agent_port: 3500
database_port: 2181

36
bench-configuration/01-write-2M-keys/analyze.yaml
View File

@ -4,13 +4,6 @@ titles:
# - Write 500K keys, 1-client, 8-byte key, 256-byte value
step1:
- data_path_list:
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-1-monitor.csv
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-2-monitor.csv
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-3-monitor.csv
data_benchmark_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-timeseries.csv
output_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-aggregated.csv
- data_path_list:
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-1-monitor.csv
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-2-monitor.csv
@ -18,6 +11,13 @@ step1:
data_benchmark_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-timeseries.csv
output_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-aggregated.csv
- data_path_list:
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-1-monitor.csv
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-2-monitor.csv
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-3-monitor.csv
data_benchmark_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-timeseries.csv
output_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-aggregated.csv
- data_path_list:
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-1-monitor.csv
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-2-monitor.csv
@ -41,10 +41,10 @@ step1:
step2:
- data_list:
- path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-aggregated.csv
name: zk-r3.4.9-java8
- path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-aggregated.csv
name: etcd-v3.1-go1.7.4
- path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-aggregated.csv
name: zk-r3.4.9-java8
- path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-aggregated.csv
name: consul-v0.7.2-go1.7.4
- path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zetcd-v3.1-go1.7.4-aggregated.csv
@ -58,10 +58,10 @@ step3:
plot_list:
- lines:
- column: avg-latency-ms-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-latency-ms-etcd-v3.1-go1.7.4
legend: etcd v3.1 (Go 1.7.4)
- column: avg-latency-ms-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-latency-ms-consul-v0.7.2-go1.7.4
legend: Consul v0.7.2 (Go 1.7.4)
- column: avg-latency-ms-zetcd-v3.1-go1.7.4
@ -75,10 +75,10 @@ step3:
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/avg-latency-ms.png
- lines:
- column: avg-throughput-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-throughput-etcd-v3.1-go1.7.4
legend: etcd v3.1 (Go 1.7.4)
- column: avg-throughput-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-throughput-consul-v0.7.2-go1.7.4
legend: Consul v0.7.2 (Go 1.7.4)
- column: avg-throughput-zetcd-v3.1-go1.7.4
@ -92,10 +92,10 @@ step3:
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/avg-throughput.png
- lines:
- column: avg-cpu-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-cpu-etcd-v3.1-go1.7.4
legend: etcd v3.1 (Go 1.7.4)
- column: avg-cpu-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-cpu-consul-v0.7.2-go1.7.4
legend: Consul v0.7.2 (Go 1.7.4)
- column: avg-cpu-zetcd-v3.1-go1.7.4
@ -109,10 +109,10 @@ step3:
- 2016Q4-01-etcd-zk-consul/01-write-2M-keys/avg-cpu.png
- lines:
- column: avg-memory-mb-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-memory-mb-etcd-v3.1-go1.7.4
legend: etcd v3.1 (Go 1.7.4)
- column: avg-memory-mb-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-memory-mb-consul-v0.7.2-go1.7.4
legend: Consul v0.7.2 (Go 1.7.4)
- column: avg-memory-mb-zetcd-v3.1-go1.7.4
@ -131,12 +131,12 @@ step4:
- 3 machines of 8 vCPUs + 1 6GB Memory + 50 GB SSD
- 1 machine(client) of 16 vCPUs + 30 GB Memory + 50 GB SSD
- Ubuntu 16.10
- etcd v3.1 (Go 1.7.4)
- Zookeepr r3.4.9
- Java 8
- Java(TM) SE Runtime Environment (build 1.8.0_111-b14)
- Java HotSpot(TM) 64-Bit Server VM (build 25.111-b14, mixed mode)
- javac 1.8.0_111
- etcd v3.1 (Go 1.7.4)
- Consul v0.7.2 (Go 1.7.4)
- zetcd v3.1 (Go 1.7.4)
- cetcd v3.1 (Go 1.7.4)

6
bench-configuration/04-read-2M-keys-linearizable/02-etcd-v3.1.yaml → bench-configuration/04-read-2M-keys-linearizable/01-etcd-v3.1.yaml
View File

@ -9,9 +9,9 @@ google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-lin
# google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/06-read-500K-keys-linearizable-1CONN
peer_ips:
- 10.240.0.25
- 10.240.0.27
- 10.240.0.28
- 10.240.0.20
- 10.240.0.21
- 10.240.0.22
agent_port: 3500
database_port: 2379

6
bench-configuration/04-read-2M-keys-linearizable/01-zk-r3.4.9.yaml → bench-configuration/04-read-2M-keys-linearizable/02-zk-r3.4.9.yaml
View File

@ -9,9 +9,9 @@ google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-lin
# google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/06-read-500K-keys-linearizable-1CONN
peer_ips:
- 10.240.0.20
- 10.240.0.21
- 10.240.0.22
- 10.240.0.25
- 10.240.0.27
- 10.240.0.28
agent_port: 3500
database_port: 2181

36
bench-configuration/04-read-2M-keys-linearizable/analyze.yaml
View File

@ -4,13 +4,6 @@ titles:
# - Linearizable-Read 500K keys, 1-client, 8-byte key, 256-byte value
step1:
- data_path_list:
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-1-monitor.csv
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-2-monitor.csv
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-3-monitor.csv
data_benchmark_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-timeseries.csv
output_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-aggregated.csv
- data_path_list:
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/etcd-v3.1-go1.7.4-1-monitor.csv
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/etcd-v3.1-go1.7.4-2-monitor.csv
@ -18,6 +11,13 @@ step1:
data_benchmark_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/etcd-v3.1-go1.7.4-timeseries.csv
output_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/etcd-v3.1-go1.7.4-aggregated.csv
- data_path_list:
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-1-monitor.csv
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-2-monitor.csv
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-3-monitor.csv
data_benchmark_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-timeseries.csv
output_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-aggregated.csv
- data_path_list:
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/consul-v0.7.2-go1.7.4-1-monitor.csv
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/consul-v0.7.2-go1.7.4-2-monitor.csv
@ -41,10 +41,10 @@ step1:
step2:
- data_list:
- path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-aggregated.csv
name: zk-r3.4.9-java8
- path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/etcd-v3.1-go1.7.4-aggregated.csv
name: etcd-v3.1-go1.7.4
- path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-aggregated.csv
name: zk-r3.4.9-java8
- path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/consul-v0.7.2-go1.7.4-aggregated.csv
name: consul-v0.7.2-go1.7.4
- path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zetcd-v3.1-go1.7.4-aggregated.csv
@ -58,10 +58,10 @@ step3:
plot_list:
- lines:
- column: avg-latency-ms-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-latency-ms-etcd-v3.1-go1.7.4
legend: etcd v3.1 (Go 1.7.4)
- column: avg-latency-ms-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-latency-ms-consul-v0.7.2-go1.7.4
legend: Consul v0.7.2 (Go 1.7.4)
- column: avg-latency-ms-zetcd-v3.1-go1.7.4
@ -75,10 +75,10 @@ step3:
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/avg-latency-ms.png
- lines:
- column: avg-throughput-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-throughput-etcd-v3.1-go1.7.4
legend: etcd v3.1 (Go 1.7.4)
- column: avg-throughput-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-throughput-consul-v0.7.2-go1.7.4
legend: Consul v0.7.2 (Go 1.7.4)
- column: avg-throughput-zetcd-v3.1-go1.7.4
@ -92,10 +92,10 @@ step3:
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/avg-throughput.png
- lines:
- column: avg-cpu-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-cpu-etcd-v3.1-go1.7.4
legend: etcd v3.1 (Go 1.7.4)
- column: avg-cpu-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-cpu-consul-v0.7.2-go1.7.4
legend: Consul v0.7.2 (Go 1.7.4)
- column: avg-cpu-zetcd-v3.1-go1.7.4
@ -109,10 +109,10 @@ step3:
- 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/avg-cpu.png
- lines:
- column: avg-memory-mb-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-memory-mb-etcd-v3.1-go1.7.4
legend: etcd v3.1 (Go 1.7.4)
- column: avg-memory-mb-zk-r3.4.9-java8
legend: Zookeeper r3.4.9 (Java 8)
- column: avg-memory-mb-consul-v0.7.2-go1.7.4
legend: Consul v0.7.2 (Go 1.7.4)
- column: avg-memory-mb-zetcd-v3.1-go1.7.4
@ -131,12 +131,12 @@ step4:
- 3 machines of 8 vCPUs + 1 6GB Memory + 50 GB SSD
- 1 machine(client) of 16 vCPUs + 30 GB Memory + 50 GB SSD
- Ubuntu 16.10
- etcd v3.1 (Go 1.7.4)
- Zookeepr r3.4.9
- Java 8
- Java(TM) SE Runtime Environment (build 1.8.0_111-b14)
- Java HotSpot(TM) 64-Bit Server VM (build 25.111-b14, mixed mode)
- javac 1.8.0_111
- etcd v3.1 (Go 1.7.4)
- Consul v0.7.2 (Go 1.7.4)
- zetcd v3.1 (Go 1.7.4)
- cetcd v3.1 (Go 1.7.4)

6
main.go
View File

@ -1,4 +1,4 @@
// Copyright 2016 CoreOS, Inc.
// Copyright 2017 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
@ -21,7 +21,6 @@
// agent Database agent in remote servers.
// analyze Analyzes test results specific to dbtester.
// control Controls tests.
// upload Uploads to cloud storage.
//
// Flags:
// -h, --help help for dbtester
@ -37,8 +36,6 @@ import (
"github.com/coreos/dbtester/agent"
"github.com/coreos/dbtester/analyze"
"github.com/coreos/dbtester/control"
"github.com/coreos/dbtester/upload"
"github.com/spf13/cobra"
)
@ -58,7 +55,6 @@ func init() {
rootCommand.AddCommand(agent.Command)
rootCommand.AddCommand(analyze.Command)
rootCommand.AddCommand(control.Command)
rootCommand.AddCommand(upload.Command)
}
func main() {

2
scripts/genproto.sh
View File

@ -25,4 +25,4 @@ popd
printf "Generating agent\n"
protoc --gofast_out=plugins=grpc:. \
--proto_path=$GOPATH/src:$GOPATH/src/github.com/gogo/protobuf/protobuf:. \
agent/*.proto;
agent/agentpb/*.proto;

16
upload/doc.go
View File

@ -1,16 +0,0 @@
// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
// Package upload implements storage upload command.
package upload

77
upload/upload.go
View File

@ -1,77 +0,0 @@
// Copyright 2016 CoreOS, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
package upload
import (
"fmt"
"io/ioutil"
"log"
"github.com/coreos/dbtester/remotestorage"
"github.com/spf13/cobra"
)
var (
Command = &cobra.Command{
Use: "upload",
Short: "Uploads to cloud storage.",
RunE: CommandFunc,
}
)
var (
from string
to string
isDirectory bool
googleCloudProjectName string
keyPath string
bucket string
)
func init() {
cobra.EnablePrefixMatching = true
}
func init() {
Command.PersistentFlags().StringVar(&from, "from", "", "file to upload.")
Command.PersistentFlags().StringVar(&to, "to", "", "file path to upload.")
Command.PersistentFlags().BoolVar(&isDirectory, "directory", false, "'true' if uploading directory.")
Command.PersistentFlags().StringVar(&googleCloudProjectName, "google-cloud-project-name", "", "Google cloud project name.")
Command.PersistentFlags().StringVar(&keyPath, "key-path", "", "Path of key file.")
Command.PersistentFlags().StringVar(&bucket, "bucket", "", "Bucket name in cloud storage.")
}
func CommandFunc(cmd *cobra.Command, args []string) error {
log.Println("opening key", keyPath)
kbs, err := ioutil.ReadFile(keyPath)
if err != nil {
return fmt.Errorf("error when opening key %s(%v)", keyPath, err)
}
u, err := remotestorage.NewGoogleCloudStorage(kbs, googleCloudProjectName)
if err != nil {
return fmt.Errorf("error when NewGoogleCloudStorage %s(%v)", googleCloudProjectName, err)
}
if !isDirectory {
if err := u.UploadFile(bucket, from, to); err != nil {
return err
}
} else {
if err := u.UploadDir(bucket, from, to); err != nil {
return err
}
}
return nil
}