diff --git a/bench-configuration/01-write-2M-keys/02-etcd-v3.1.yaml b/bench-configuration/01-write-2M-keys/01-etcd-v3.1.yaml similarity index 95% rename from bench-configuration/01-write-2M-keys/02-etcd-v3.1.yaml rename to bench-configuration/01-write-2M-keys/01-etcd-v3.1.yaml index be01ec5c..30857d5d 100644 --- a/bench-configuration/01-write-2M-keys/02-etcd-v3.1.yaml +++ b/bench-configuration/01-write-2M-keys/01-etcd-v3.1.yaml @@ -9,9 +9,9 @@ google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/01-write-2M-keys # google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/03-write-500K-keys-1CONN peer_ips: - - 10.240.0.25 - - 10.240.0.27 - - 10.240.0.28 + - 10.240.0.20 + - 10.240.0.21 + - 10.240.0.22 agent_port: 3500 database_port: 2379 diff --git a/bench-configuration/01-write-2M-keys/01-zk-r3.4.9.yaml b/bench-configuration/01-write-2M-keys/02-zk-r3.4.9.yaml similarity index 95% rename from bench-configuration/01-write-2M-keys/01-zk-r3.4.9.yaml rename to bench-configuration/01-write-2M-keys/02-zk-r3.4.9.yaml index 33fa0424..04a8fbfd 100644 --- a/bench-configuration/01-write-2M-keys/01-zk-r3.4.9.yaml +++ b/bench-configuration/01-write-2M-keys/02-zk-r3.4.9.yaml @@ -9,9 +9,9 @@ google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/01-write-2M-keys # google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/03-write-500K-keys-1CONN peer_ips: - - 10.240.0.20 - - 10.240.0.21 - - 10.240.0.22 + - 10.240.0.25 + - 10.240.0.27 + - 10.240.0.28 agent_port: 3500 database_port: 2181 diff --git a/bench-configuration/01-write-2M-keys/analyze.yaml b/bench-configuration/01-write-2M-keys/analyze.yaml index 53749714..1d529ab7 100644 --- a/bench-configuration/01-write-2M-keys/analyze.yaml +++ b/bench-configuration/01-write-2M-keys/analyze.yaml @@ -4,13 +4,6 @@ titles: # - Write 500K keys, 1-client, 8-byte key, 256-byte value step1: - - data_path_list: - - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-1-monitor.csv - - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-2-monitor.csv - - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-3-monitor.csv - data_benchmark_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-timeseries.csv - output_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-aggregated.csv - - data_path_list: - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-1-monitor.csv - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-2-monitor.csv @@ -18,6 +11,13 @@ step1: data_benchmark_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-timeseries.csv output_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-aggregated.csv + - data_path_list: + - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-1-monitor.csv + - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-2-monitor.csv + - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-3-monitor.csv + data_benchmark_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-timeseries.csv + output_path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-aggregated.csv + - data_path_list: - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-1-monitor.csv - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-2-monitor.csv @@ -41,10 +41,10 @@ step1: step2: - data_list: - - path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-aggregated.csv - name: zk-r3.4.9-java8 - path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/etcd-v3.1-go1.7.4-aggregated.csv name: etcd-v3.1-go1.7.4 + - path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zk-r3.4.9-java8-aggregated.csv + name: zk-r3.4.9-java8 - path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/consul-v0.7.2-go1.7.4-aggregated.csv name: consul-v0.7.2-go1.7.4 - path: 2016Q4-01-etcd-zk-consul/01-write-2M-keys/zetcd-v3.1-go1.7.4-aggregated.csv @@ -58,10 +58,10 @@ step3: plot_list: - lines: - - column: avg-latency-ms-zk-r3.4.9-java8 - legend: Zookeeper r3.4.9 (Java 8) - column: avg-latency-ms-etcd-v3.1-go1.7.4 legend: etcd v3.1 (Go 1.7.4) + - column: avg-latency-ms-zk-r3.4.9-java8 + legend: Zookeeper r3.4.9 (Java 8) - column: avg-latency-ms-consul-v0.7.2-go1.7.4 legend: Consul v0.7.2 (Go 1.7.4) - column: avg-latency-ms-zetcd-v3.1-go1.7.4 @@ -75,10 +75,10 @@ step3: - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/avg-latency-ms.png - lines: - - column: avg-throughput-zk-r3.4.9-java8 - legend: Zookeeper r3.4.9 (Java 8) - column: avg-throughput-etcd-v3.1-go1.7.4 legend: etcd v3.1 (Go 1.7.4) + - column: avg-throughput-zk-r3.4.9-java8 + legend: Zookeeper r3.4.9 (Java 8) - column: avg-throughput-consul-v0.7.2-go1.7.4 legend: Consul v0.7.2 (Go 1.7.4) - column: avg-throughput-zetcd-v3.1-go1.7.4 @@ -92,10 +92,10 @@ step3: - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/avg-throughput.png - lines: - - column: avg-cpu-zk-r3.4.9-java8 - legend: Zookeeper r3.4.9 (Java 8) - column: avg-cpu-etcd-v3.1-go1.7.4 legend: etcd v3.1 (Go 1.7.4) + - column: avg-cpu-zk-r3.4.9-java8 + legend: Zookeeper r3.4.9 (Java 8) - column: avg-cpu-consul-v0.7.2-go1.7.4 legend: Consul v0.7.2 (Go 1.7.4) - column: avg-cpu-zetcd-v3.1-go1.7.4 @@ -109,10 +109,10 @@ step3: - 2016Q4-01-etcd-zk-consul/01-write-2M-keys/avg-cpu.png - lines: - - column: avg-memory-mb-zk-r3.4.9-java8 - legend: Zookeeper r3.4.9 (Java 8) - column: avg-memory-mb-etcd-v3.1-go1.7.4 legend: etcd v3.1 (Go 1.7.4) + - column: avg-memory-mb-zk-r3.4.9-java8 + legend: Zookeeper r3.4.9 (Java 8) - column: avg-memory-mb-consul-v0.7.2-go1.7.4 legend: Consul v0.7.2 (Go 1.7.4) - column: avg-memory-mb-zetcd-v3.1-go1.7.4 @@ -131,12 +131,12 @@ step4: - 3 machines of 8 vCPUs + 1 6GB Memory + 50 GB SSD - 1 machine(client) of 16 vCPUs + 30 GB Memory + 50 GB SSD - Ubuntu 16.10 + - etcd v3.1 (Go 1.7.4) - Zookeepr r3.4.9 - Java 8 - Java(TM) SE Runtime Environment (build 1.8.0_111-b14) - Java HotSpot(TM) 64-Bit Server VM (build 25.111-b14, mixed mode) - javac 1.8.0_111 - - etcd v3.1 (Go 1.7.4) - Consul v0.7.2 (Go 1.7.4) - zetcd v3.1 (Go 1.7.4) - cetcd v3.1 (Go 1.7.4) diff --git a/bench-configuration/04-read-2M-keys-linearizable/02-etcd-v3.1.yaml b/bench-configuration/04-read-2M-keys-linearizable/01-etcd-v3.1.yaml similarity index 95% rename from bench-configuration/04-read-2M-keys-linearizable/02-etcd-v3.1.yaml rename to bench-configuration/04-read-2M-keys-linearizable/01-etcd-v3.1.yaml index 8101ef79..4ac605bb 100644 --- a/bench-configuration/04-read-2M-keys-linearizable/02-etcd-v3.1.yaml +++ b/bench-configuration/04-read-2M-keys-linearizable/01-etcd-v3.1.yaml @@ -9,9 +9,9 @@ google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-lin # google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/06-read-500K-keys-linearizable-1CONN peer_ips: - - 10.240.0.25 - - 10.240.0.27 - - 10.240.0.28 + - 10.240.0.20 + - 10.240.0.21 + - 10.240.0.22 agent_port: 3500 database_port: 2379 diff --git a/bench-configuration/04-read-2M-keys-linearizable/01-zk-r3.4.9.yaml b/bench-configuration/04-read-2M-keys-linearizable/02-zk-r3.4.9.yaml similarity index 95% rename from bench-configuration/04-read-2M-keys-linearizable/01-zk-r3.4.9.yaml rename to bench-configuration/04-read-2M-keys-linearizable/02-zk-r3.4.9.yaml index 51e749cd..f9e51c90 100644 --- a/bench-configuration/04-read-2M-keys-linearizable/01-zk-r3.4.9.yaml +++ b/bench-configuration/04-read-2M-keys-linearizable/02-zk-r3.4.9.yaml @@ -9,9 +9,9 @@ google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-lin # google_cloud_storage_sub_directory: 2016Q4-01-etcd-zk-consul/06-read-500K-keys-linearizable-1CONN peer_ips: - - 10.240.0.20 - - 10.240.0.21 - - 10.240.0.22 + - 10.240.0.25 + - 10.240.0.27 + - 10.240.0.28 agent_port: 3500 database_port: 2181 diff --git a/bench-configuration/04-read-2M-keys-linearizable/analyze.yaml b/bench-configuration/04-read-2M-keys-linearizable/analyze.yaml index 0d9a3b84..571c2262 100644 --- a/bench-configuration/04-read-2M-keys-linearizable/analyze.yaml +++ b/bench-configuration/04-read-2M-keys-linearizable/analyze.yaml @@ -4,13 +4,6 @@ titles: # - Linearizable-Read 500K keys, 1-client, 8-byte key, 256-byte value step1: - - data_path_list: - - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-1-monitor.csv - - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-2-monitor.csv - - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-3-monitor.csv - data_benchmark_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-timeseries.csv - output_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-aggregated.csv - - data_path_list: - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/etcd-v3.1-go1.7.4-1-monitor.csv - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/etcd-v3.1-go1.7.4-2-monitor.csv @@ -18,6 +11,13 @@ step1: data_benchmark_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/etcd-v3.1-go1.7.4-timeseries.csv output_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/etcd-v3.1-go1.7.4-aggregated.csv + - data_path_list: + - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-1-monitor.csv + - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-2-monitor.csv + - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-3-monitor.csv + data_benchmark_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-timeseries.csv + output_path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-aggregated.csv + - data_path_list: - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/consul-v0.7.2-go1.7.4-1-monitor.csv - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/consul-v0.7.2-go1.7.4-2-monitor.csv @@ -41,10 +41,10 @@ step1: step2: - data_list: - - path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-aggregated.csv - name: zk-r3.4.9-java8 - path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/etcd-v3.1-go1.7.4-aggregated.csv name: etcd-v3.1-go1.7.4 + - path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zk-r3.4.9-java8-aggregated.csv + name: zk-r3.4.9-java8 - path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/consul-v0.7.2-go1.7.4-aggregated.csv name: consul-v0.7.2-go1.7.4 - path: 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/zetcd-v3.1-go1.7.4-aggregated.csv @@ -58,10 +58,10 @@ step3: plot_list: - lines: - - column: avg-latency-ms-zk-r3.4.9-java8 - legend: Zookeeper r3.4.9 (Java 8) - column: avg-latency-ms-etcd-v3.1-go1.7.4 legend: etcd v3.1 (Go 1.7.4) + - column: avg-latency-ms-zk-r3.4.9-java8 + legend: Zookeeper r3.4.9 (Java 8) - column: avg-latency-ms-consul-v0.7.2-go1.7.4 legend: Consul v0.7.2 (Go 1.7.4) - column: avg-latency-ms-zetcd-v3.1-go1.7.4 @@ -75,10 +75,10 @@ step3: - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/avg-latency-ms.png - lines: - - column: avg-throughput-zk-r3.4.9-java8 - legend: Zookeeper r3.4.9 (Java 8) - column: avg-throughput-etcd-v3.1-go1.7.4 legend: etcd v3.1 (Go 1.7.4) + - column: avg-throughput-zk-r3.4.9-java8 + legend: Zookeeper r3.4.9 (Java 8) - column: avg-throughput-consul-v0.7.2-go1.7.4 legend: Consul v0.7.2 (Go 1.7.4) - column: avg-throughput-zetcd-v3.1-go1.7.4 @@ -92,10 +92,10 @@ step3: - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/avg-throughput.png - lines: - - column: avg-cpu-zk-r3.4.9-java8 - legend: Zookeeper r3.4.9 (Java 8) - column: avg-cpu-etcd-v3.1-go1.7.4 legend: etcd v3.1 (Go 1.7.4) + - column: avg-cpu-zk-r3.4.9-java8 + legend: Zookeeper r3.4.9 (Java 8) - column: avg-cpu-consul-v0.7.2-go1.7.4 legend: Consul v0.7.2 (Go 1.7.4) - column: avg-cpu-zetcd-v3.1-go1.7.4 @@ -109,10 +109,10 @@ step3: - 2016Q4-01-etcd-zk-consul/04-read-2M-keys-linearizable/avg-cpu.png - lines: - - column: avg-memory-mb-zk-r3.4.9-java8 - legend: Zookeeper r3.4.9 (Java 8) - column: avg-memory-mb-etcd-v3.1-go1.7.4 legend: etcd v3.1 (Go 1.7.4) + - column: avg-memory-mb-zk-r3.4.9-java8 + legend: Zookeeper r3.4.9 (Java 8) - column: avg-memory-mb-consul-v0.7.2-go1.7.4 legend: Consul v0.7.2 (Go 1.7.4) - column: avg-memory-mb-zetcd-v3.1-go1.7.4 @@ -131,12 +131,12 @@ step4: - 3 machines of 8 vCPUs + 1 6GB Memory + 50 GB SSD - 1 machine(client) of 16 vCPUs + 30 GB Memory + 50 GB SSD - Ubuntu 16.10 + - etcd v3.1 (Go 1.7.4) - Zookeepr r3.4.9 - Java 8 - Java(TM) SE Runtime Environment (build 1.8.0_111-b14) - Java HotSpot(TM) 64-Bit Server VM (build 25.111-b14, mixed mode) - javac 1.8.0_111 - - etcd v3.1 (Go 1.7.4) - Consul v0.7.2 (Go 1.7.4) - zetcd v3.1 (Go 1.7.4) - cetcd v3.1 (Go 1.7.4) diff --git a/main.go b/main.go index 0d028f5a..4108da54 100644 --- a/main.go +++ b/main.go @@ -1,4 +1,4 @@ -// Copyright 2016 CoreOS, Inc. +// Copyright 2017 CoreOS, Inc. // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -21,7 +21,6 @@ // agent Database agent in remote servers. // analyze Analyzes test results specific to dbtester. // control Controls tests. -// upload Uploads to cloud storage. // // Flags: // -h, --help help for dbtester @@ -37,8 +36,6 @@ import ( "github.com/coreos/dbtester/agent" "github.com/coreos/dbtester/analyze" "github.com/coreos/dbtester/control" - "github.com/coreos/dbtester/upload" - "github.com/spf13/cobra" ) @@ -58,7 +55,6 @@ func init() { rootCommand.AddCommand(agent.Command) rootCommand.AddCommand(analyze.Command) rootCommand.AddCommand(control.Command) - rootCommand.AddCommand(upload.Command) } func main() { diff --git a/scripts/genproto.sh b/scripts/genproto.sh index bc8a22c9..2e508001 100755 --- a/scripts/genproto.sh +++ b/scripts/genproto.sh @@ -25,4 +25,4 @@ popd printf "Generating agent\n" protoc --gofast_out=plugins=grpc:. \ --proto_path=$GOPATH/src:$GOPATH/src/github.com/gogo/protobuf/protobuf:. \ - agent/*.proto; + agent/agentpb/*.proto; diff --git a/upload/doc.go b/upload/doc.go deleted file mode 100644 index dc7ea5c6..00000000 --- a/upload/doc.go +++ /dev/null @@ -1,16 +0,0 @@ -// Copyright 2016 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -// Package upload implements storage upload command. -package upload diff --git a/upload/upload.go b/upload/upload.go deleted file mode 100644 index 81c17393..00000000 --- a/upload/upload.go +++ /dev/null @@ -1,77 +0,0 @@ -// Copyright 2016 CoreOS, Inc. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package upload - -import ( - "fmt" - "io/ioutil" - "log" - - "github.com/coreos/dbtester/remotestorage" - "github.com/spf13/cobra" -) - -var ( - Command = &cobra.Command{ - Use: "upload", - Short: "Uploads to cloud storage.", - RunE: CommandFunc, - } -) - -var ( - from string - to string - isDirectory bool - - googleCloudProjectName string - keyPath string - bucket string -) - -func init() { - cobra.EnablePrefixMatching = true -} - -func init() { - Command.PersistentFlags().StringVar(&from, "from", "", "file to upload.") - Command.PersistentFlags().StringVar(&to, "to", "", "file path to upload.") - Command.PersistentFlags().BoolVar(&isDirectory, "directory", false, "'true' if uploading directory.") - Command.PersistentFlags().StringVar(&googleCloudProjectName, "google-cloud-project-name", "", "Google cloud project name.") - Command.PersistentFlags().StringVar(&keyPath, "key-path", "", "Path of key file.") - Command.PersistentFlags().StringVar(&bucket, "bucket", "", "Bucket name in cloud storage.") -} - -func CommandFunc(cmd *cobra.Command, args []string) error { - log.Println("opening key", keyPath) - kbs, err := ioutil.ReadFile(keyPath) - if err != nil { - return fmt.Errorf("error when opening key %s(%v)", keyPath, err) - } - u, err := remotestorage.NewGoogleCloudStorage(kbs, googleCloudProjectName) - if err != nil { - return fmt.Errorf("error when NewGoogleCloudStorage %s(%v)", googleCloudProjectName, err) - } - if !isDirectory { - if err := u.UploadFile(bucket, from, to); err != nil { - return err - } - } else { - if err := u.UploadDir(bucket, from, to); err != nil { - return err - } - } - return nil -}