*: fix zookeeper jvm flag, update configs

This commit is contained in:
Gyu-Ho Lee 2017-03-15 14:31:40 -07:00
parent 6b01a63f54
commit cc73c3e9dc
8 changed files with 112 additions and 64 deletions

View File

@ -167,41 +167,63 @@ func startZookeeper(fs *flags, t *transporterServer) error {
return err
}
args := []string{shell}
var flagString string
switch t.req.DatabaseID {
case dbtesterpb.DatabaseID_zookeeper__r3_4_9:
flagString = JavaClassPathZookeeperr349
if t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer != 0 {
if len(flagString) > 0 {
flagString += " "
}
flagString += fmt.Sprintf("-Djute.maxbuffer=%d", t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer)
}
if t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer != 0 {
if len(flagString) > 0 {
flagString += " "
}
flagString += fmt.Sprintf("-Xms%s", t.req.Flag_Zookeeper_R3_4_9.JavaXms)
}
if t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer != 0 {
if len(flagString) > 0 {
flagString += " "
}
flagString += fmt.Sprintf("-Xmx%s", t.req.Flag_Zookeeper_R3_4_9.JavaXmx)
}
// -Djute.maxbuffer=33554432 -Xms50G -Xmx50G
if t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Djute.maxbuffer=%d", t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer))
}
if t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Xms%s", t.req.Flag_Zookeeper_R3_4_9.JavaXms))
}
if t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Xmx%s", t.req.Flag_Zookeeper_R3_4_9.JavaXmx))
if len(flagString) > 0 {
flagString += " "
}
flagString += JavaClassPathZookeeperr349
case dbtesterpb.DatabaseID_zookeeper__r3_5_2_alpha:
flagString = JavaClassPathZookeeperr352alpha
if t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer != 0 {
if len(flagString) > 0 {
flagString += " "
}
flagString += fmt.Sprintf("-Djute.maxbuffer=%d", t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer)
}
if t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer != 0 {
if len(flagString) > 0 {
flagString += " "
}
flagString += fmt.Sprintf("-Xms%s", t.req.Flag_Zookeeper_R3_5_2Alpha.JavaXms)
}
if t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer != 0 {
if len(flagString) > 0 {
flagString += " "
}
flagString += fmt.Sprintf("-Xmx%s", t.req.Flag_Zookeeper_R3_5_2Alpha.JavaXmx)
}
// -Djute.maxbuffer=33554432 -Xms50G -Xmx50G
if t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Djute.maxbuffer=%d", t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer))
}
if t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Xms%s", t.req.Flag_Zookeeper_R3_5_2Alpha.JavaXms))
}
if t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Xmx%s", t.req.Flag_Zookeeper_R3_5_2Alpha.JavaXmx))
if len(flagString) > 0 {
flagString += " "
}
flagString += JavaClassPathZookeeperr352alpha
default:
return fmt.Errorf("database ID %q is not supported", t.req.DatabaseID)
}
args = append(args, "-c", fs.javaExec+" "+flagString+" "+fs.zkConfig)
args := []string{shell, "-c", fs.javaExec + " " + flagString + " " + fs.zkConfig}
cmd := exec.Command(args[0], args[1:]...)
cmd.Stdout = t.databaseLogFile
cmd.Stderr = t.databaseLogFile

View File

@ -133,7 +133,12 @@ func (t *transporterServer) Transfer(ctx context.Context, req *dbtesterpb.Reques
switch req.Operation {
case dbtesterpb.Operation_Start:
switch t.req.DatabaseID {
case dbtesterpb.DatabaseID_etcd__v2_3, dbtesterpb.DatabaseID_etcd__tip, dbtesterpb.DatabaseID_zetcd__beta, dbtesterpb.DatabaseID_cetcd__beta:
case dbtesterpb.DatabaseID_etcd__v2_3,
dbtesterpb.DatabaseID_etcd__v3_1,
dbtesterpb.DatabaseID_etcd__v3_2,
dbtesterpb.DatabaseID_etcd__tip,
dbtesterpb.DatabaseID_zetcd__beta,
dbtesterpb.DatabaseID_cetcd__beta:
if err := startEtcd(&globalFlags, t); err != nil {
plog.Errorf("startEtcd error %v", err)
return nil, err
@ -166,12 +171,14 @@ func (t *transporterServer) Transfer(ctx context.Context, req *dbtesterpb.Reques
plog.Infof("exiting %q", t.proxyCmd.Path)
}()
}
case dbtesterpb.DatabaseID_zookeeper__r3_4_9:
case dbtesterpb.DatabaseID_zookeeper__r3_4_9,
dbtesterpb.DatabaseID_zookeeper__r3_5_2_alpha:
if err := startZookeeper(&globalFlags, t); err != nil {
plog.Errorf("startZookeeper error %v", err)
return nil, err
}
case dbtesterpb.DatabaseID_consul__v0_7_5:
case dbtesterpb.DatabaseID_consul__v0_7_5,
dbtesterpb.DatabaseID_consul__v0_8_0:
if err := startConsul(&globalFlags, t); err != nil {
plog.Errorf("startConsul error %v", err)
return nil, err

View File

@ -23,6 +23,7 @@ import (
"time"
"github.com/coreos/dbtester/dbtesterpb"
"github.com/coreos/etcd/clientv3"
"github.com/coreos/etcd/pkg/report"
consulapi "github.com/hashicorp/consul/api"
@ -173,14 +174,16 @@ func (cfg *Config) Stress(databaseID string) error {
plog.Println("checking total keys on", gcfg.DatabaseEndpoints)
var totalKeysFunc func([]string) map[string]int64
switch gcfg.DatabaseID {
case "etcdv2":
case "etcd__v2_3":
totalKeysFunc = getTotalKeysEtcdv2
case "etcdv3", "etcdtip":
case "etcd__v3_1", "etcd__v3_2", "etcd__tip":
totalKeysFunc = getTotalKeysEtcdv3
case "zookeeper", "zetcd":
case "zookeeper__r3_4_9", "zookeeper__r3_5_2_alpha", "zetcd__beta":
totalKeysFunc = getTotalKeysZk
case "consul", "cetcd":
case "consul__v0_7_5", "consul__v0_8_0", "cetcd__beta":
totalKeysFunc = getTotalKeysConsul
default:
plog.Panicf("%q is unknown database ID", gcfg.DatabaseID)
}
for k, v := range totalKeysFunc(gcfg.DatabaseEndpoints) {
plog.Infof("expected write total results [expected_total: %d | database: %q | endpoint: %q | number_of_keys: %d]",
@ -191,7 +194,7 @@ func (cfg *Config) Stress(databaseID string) error {
key, value := sameKey(gcfg.ConfigClientMachineBenchmarkOptions.KeySizeBytes), vals.strings[0]
switch gcfg.DatabaseID {
case "etcdv2":
case "etcd__v2_3":
plog.Infof("write started [request: PUT | key: %q | database: %q]", key, gcfg.DatabaseID)
var err error
for i := 0; i < 7; i++ {
@ -208,7 +211,7 @@ func (cfg *Config) Stress(databaseID string) error {
os.Exit(1)
}
case "etcdv3", "etcdtip":
case "etcd__v3_1", "etcd__v3_2", "etcd__tip":
plog.Infof("write started [request: PUT | key: %q | database: %q]", key, gcfg.DatabaseID)
var err error
for i := 0; i < 7; i++ {
@ -228,7 +231,7 @@ func (cfg *Config) Stress(databaseID string) error {
os.Exit(1)
}
case "zookeeper", "zetcd":
case "zookeeper__r3_4_9", "zookeeper__r3_5_2_alpha", "zetcd__beta":
plog.Infof("write started [request: PUT | key: %q | database: %q]", key, gcfg.DatabaseID)
var err error
for i := 0; i < 7; i++ {
@ -248,7 +251,7 @@ func (cfg *Config) Stress(databaseID string) error {
os.Exit(1)
}
case "consul", "cetcd":
case "consul__v0_7_5", "consul__v0_8_0", "cetcd__beta":
plog.Infof("write started [request: PUT | key: %q | database: %q]", key, gcfg.DatabaseID)
var err error
for i := 0; i < 7; i++ {
@ -264,6 +267,9 @@ func (cfg *Config) Stress(databaseID string) error {
plog.Errorf("write done [request: PUT | key: %q | database: %q]", key, gcfg.DatabaseID)
os.Exit(1)
}
default:
plog.Panicf("%q is unknown database ID", gcfg.DatabaseID)
}
h, done := newReadHandlers(gcfg)
@ -276,11 +282,11 @@ func (cfg *Config) Stress(databaseID string) error {
plog.Infof("writing key for read-oneshot [key: %q | database: %q]", key, gcfg.DatabaseID)
var err error
switch gcfg.DatabaseID {
case "etcdv2":
case "etcd__v2_3":
clients := mustCreateClientsEtcdv2(gcfg.DatabaseEndpoints, 1)
_, err = clients[0].Set(context.Background(), key, value, nil)
case "etcdv3", "etcdtip":
case "etcd__v3_1", "etcd__v3_2", "etcd__tip":
clients := mustCreateClientsEtcdv3(gcfg.DatabaseEndpoints, etcdv3ClientCfg{
totalConns: 1,
totalClients: 1,
@ -288,14 +294,17 @@ func (cfg *Config) Stress(databaseID string) error {
_, err = clients[0].Do(context.Background(), clientv3.OpPut(key, value))
clients[0].Close()
case "zookeeper", "zetcd":
case "zookeeper__r3_4_9", "zookeeper__r3_5_2_alpha", "zetcd__beta":
conns := mustCreateConnsZk(gcfg.DatabaseEndpoints, 1)
_, err = conns[0].Create("/"+key, vals.bytes[0], zkCreateFlags, zkCreateACL)
conns[0].Close()
case "consul", "cetcd":
case "consul__v0_7_5", "consul__v0_8_0", "cetcd__beta":
clients := mustCreateConnsConsul(gcfg.DatabaseEndpoints, 1)
_, err = clients[0].Put(&consulapi.KVPair{Key: key, Value: vals.bytes[0]}, nil)
default:
plog.Panicf("%q is unknown database ID", gcfg.DatabaseID)
}
if err != nil {
plog.Errorf("write error on read-oneshot (%v)", err)
@ -314,12 +323,12 @@ func (cfg *Config) Stress(databaseID string) error {
func newReadHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) (rhs []ReqHandler, done func()) {
rhs = make([]ReqHandler, gcfg.ConfigClientMachineBenchmarkOptions.ClientNumber)
switch gcfg.DatabaseID {
case "etcdv2":
case "etcd__v2_3":
conns := mustCreateClientsEtcdv2(gcfg.DatabaseEndpoints, gcfg.ConfigClientMachineBenchmarkOptions.ConnectionNumber)
for i := range conns {
rhs[i] = newGetEtcd2(conns[i])
}
case "etcdv3", "etcdtip":
case "etcd__v3_1", "etcd__v3_2", "etcd__tip":
clients := mustCreateClientsEtcdv3(gcfg.DatabaseEndpoints, etcdv3ClientCfg{
totalConns: gcfg.ConfigClientMachineBenchmarkOptions.ConnectionNumber,
totalClients: gcfg.ConfigClientMachineBenchmarkOptions.ClientNumber,
@ -332,7 +341,7 @@ func newReadHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) (rhs []Req
clients[i].Close()
}
}
case "zookeeper", "zetcd":
case "zookeeper__r3_4_9", "zookeeper__r3_5_2_alpha", "zetcd__beta":
conns := mustCreateConnsZk(gcfg.DatabaseEndpoints, gcfg.ConfigClientMachineBenchmarkOptions.ConnectionNumber)
for i := range conns {
rhs[i] = newGetZK(conns[i])
@ -342,11 +351,13 @@ func newReadHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) (rhs []Req
conns[i].Close()
}
}
case "consul", "cetcd":
case "consul__v0_7_5", "consul__v0_8_0", "cetcd__beta":
conns := mustCreateConnsConsul(gcfg.DatabaseEndpoints, gcfg.ConfigClientMachineBenchmarkOptions.ConnectionNumber)
for i := range conns {
rhs[i] = newGetConsul(conns[i])
}
default:
plog.Panicf("%q is unknown database ID", gcfg.DatabaseID)
}
return rhs, done
}
@ -354,12 +365,12 @@ func newReadHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) (rhs []Req
func newWriteHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) (rhs []ReqHandler, done func()) {
rhs = make([]ReqHandler, gcfg.ConfigClientMachineBenchmarkOptions.ClientNumber)
switch gcfg.DatabaseID {
case "etcdv2":
case "etcd__v2_3":
conns := mustCreateClientsEtcdv2(gcfg.DatabaseEndpoints, gcfg.ConfigClientMachineBenchmarkOptions.ConnectionNumber)
for i := range conns {
rhs[i] = newPutEtcd2(conns[i])
}
case "etcdv3", "etcdtip":
case "etcd__v3_1", "etcd__v3_2", "etcd__tip":
etcdClients := mustCreateClientsEtcdv3(gcfg.DatabaseEndpoints, etcdv3ClientCfg{
totalConns: gcfg.ConfigClientMachineBenchmarkOptions.ConnectionNumber,
totalClients: gcfg.ConfigClientMachineBenchmarkOptions.ClientNumber,
@ -372,7 +383,7 @@ func newWriteHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) (rhs []Re
etcdClients[i].Close()
}
}
case "zookeeper", "zetcd":
case "zookeeper__r3_4_9", "zookeeper__r3_5_2_alpha", "zetcd__beta":
if gcfg.ConfigClientMachineBenchmarkOptions.SameKey {
key := sameKey(gcfg.ConfigClientMachineBenchmarkOptions.KeySizeBytes)
valueBts := randBytes(gcfg.ConfigClientMachineBenchmarkOptions.ValueSizeBytes)
@ -409,11 +420,13 @@ func newWriteHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) (rhs []Re
conns[i].Close()
}
}
case "consul", "cetcd":
case "consul__v0_7_5", "consul__v0_8_0", "cetcd__beta":
conns := mustCreateConnsConsul(gcfg.DatabaseEndpoints, gcfg.ConfigClientMachineBenchmarkOptions.ConnectionNumber)
for i := range conns {
rhs[i] = newPutConsul(conns[i])
}
default:
plog.Panicf("%q is unknown database ID", gcfg.DatabaseID)
}
for k := range rhs {
@ -427,14 +440,14 @@ func newWriteHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) (rhs []Re
func newReadOneshotHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) []ReqHandler {
rhs := make([]ReqHandler, gcfg.ConfigClientMachineBenchmarkOptions.ClientNumber)
switch gcfg.DatabaseID {
case "etcdv2":
case "etcd__v2_3":
for i := range rhs {
rhs[i] = func(ctx context.Context, req *request) error {
conns := mustCreateClientsEtcdv2(gcfg.DatabaseEndpoints, 1)
return newGetEtcd2(conns[0])(ctx, req)
}
}
case "etcdv3", "etcdtip":
case "etcd__v3_1", "etcd__v3_2", "etcd__tip":
for i := range rhs {
rhs[i] = func(ctx context.Context, req *request) error {
conns := mustCreateClientsEtcdv3(gcfg.DatabaseEndpoints, etcdv3ClientCfg{
@ -445,7 +458,7 @@ func newReadOneshotHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) []R
return newGetEtcd3(conns[0])(ctx, req)
}
}
case "zookeeper", "zetcd":
case "zookeeper__r3_4_9", "zookeeper__r3_5_2_alpha", "zetcd__beta":
for i := range rhs {
rhs[i] = func(ctx context.Context, req *request) error {
conns := mustCreateConnsZk(gcfg.DatabaseEndpoints, gcfg.ConfigClientMachineBenchmarkOptions.ConnectionNumber)
@ -453,13 +466,15 @@ func newReadOneshotHandlers(gcfg dbtesterpb.ConfigClientMachineAgentControl) []R
return newGetZK(conns[0])(ctx, req)
}
}
case "consul", "cetcd":
case "consul__v0_7_5", "consul__v0_8_0", "cetcd__beta":
for i := range rhs {
rhs[i] = func(ctx context.Context, req *request) error {
conns := mustCreateConnsConsul(gcfg.DatabaseEndpoints, 1)
return newGetConsul(conns[0])(ctx, req)
}
}
default:
plog.Panicf("%q is unknown database ID", gcfg.DatabaseID)
}
return rhs
}
@ -481,30 +496,32 @@ func generateReads(gcfg dbtesterpb.ConfigClientMachineAgentControl, key string,
}
switch gcfg.DatabaseID {
case "etcdv2":
case "etcd__v2_3":
// serializable read by default
inflightReqs <- request{etcdv2Op: etcdv2Op{key: key}}
case "etcdv3", "etcdtip":
case "etcd__v3_1", "etcd__v3_2", "etcd__tip":
opts := []clientv3.OpOption{clientv3.WithRange("")}
if gcfg.ConfigClientMachineBenchmarkOptions.StaleRead {
opts = append(opts, clientv3.WithSerializable())
}
inflightReqs <- request{etcdv3Op: clientv3.OpGet(key, opts...)}
case "zookeeper", "zetcd":
case "zookeeper__r3_4_9", "zookeeper__r3_5_2_alpha", "zetcd__beta":
op := zkOp{key: key}
if gcfg.ConfigClientMachineBenchmarkOptions.StaleRead {
op.staleRead = true
}
inflightReqs <- request{zkOp: op}
case "consul", "cetcd":
case "consul__v0_7_5", "consul__v0_8_0", "cetcd__beta":
op := consulOp{key: key}
if gcfg.ConfigClientMachineBenchmarkOptions.StaleRead {
op.staleRead = true
}
inflightReqs <- request{consulOp: op}
default:
plog.Panicf("%q is unknown database ID", gcfg.DatabaseID)
}
}
}
@ -538,14 +555,16 @@ func generateWrites(gcfg dbtesterpb.ConfigClientMachineAgentControl, startIdx in
}
switch gcfg.DatabaseID {
case "etcdv2":
case "etcd__v2_3":
inflightReqs <- request{etcdv2Op: etcdv2Op{key: k, value: vs}}
case "etcdv3", "etcdtip":
case "etcd__v3_1", "etcd__v3_2", "etcd__tip":
inflightReqs <- request{etcdv3Op: clientv3.OpPut(k, vs)}
case "zookeeper", "zetcd":
case "zookeeper__r3_4_9", "zookeeper__r3_5_2_alpha", "zetcd__beta":
inflightReqs <- request{zkOp: zkOp{key: "/" + k, value: v}}
case "consul", "cetcd":
case "consul__v0_7_5", "consul__v0_8_0", "cetcd__beta":
inflightReqs <- request{consulOp: consulOp{key: k, value: v}}
default:
plog.Panicf("%q is unknown database ID", gcfg.DatabaseID)
}
}
}

View File

@ -2,7 +2,8 @@ test_title: Write 1M keys, 256-byte key, 1KB value value, clients 1 to 1,000
test_description: |
- Google Cloud Compute Engine
- 4 machines of 16 vCPUs + 60 GB Memory + 300 GB SSD (1 for client)
- Ubuntu 16.10
- Ubuntu 16.10 (GNU/Linux kernel 4.8.0-42-generic)
- `ulimit -n` is 120000
- etcd tip (Go 1.8.0, git SHA 5856c8bce9778a12d79038fdb1f3fba9416bd297)
- Zookeeper r3.5.2-alpha
- Java 8

View File

@ -2,7 +2,8 @@ test_title: Write 1M keys, 256-byte key, 1KB value, Best Throughput (etcd 1,000,
test_description: |
- Google Cloud Compute Engine
- 4 machines of 16 vCPUs + 60 GB Memory + 300 GB SSD (1 for client)
- Ubuntu 16.10
- Ubuntu 16.10 (GNU/Linux kernel 4.8.0-42-generic)
- `ulimit -n` is 120000
- etcd tip (Go 1.8.0, git SHA 5856c8bce9778a12d79038fdb1f3fba9416bd297)
- Zookeeper r3.5.2-alpha
- Java 8

View File

@ -2,7 +2,8 @@ test_title: Write 3-million keys, 256-byte key, 1KB value, Best Throughput (etcd
test_description: |
- Google Cloud Compute Engine
- 4 machines of 16 vCPUs + 60 GB Memory + 300 GB SSD (1 for client)
- Ubuntu 16.10
- Ubuntu 16.10 (GNU/Linux kernel 4.8.0-42-generic)
- `ulimit -n` is 120000
- etcd tip (Go 1.8.0, git SHA 5856c8bce9778a12d79038fdb1f3fba9416bd297)
- Zookeeper r3.5.2-alpha
- Java 8

View File

@ -35,9 +35,6 @@ cd ${GOPATH}/src/${GIT_PATH}
git reset --hard 5856c8bce9778a12d79038fdb1f3fba9416bd297
./build
# FAILPOINTS=1 ./build
# https://github.com/coreos/etcd/commits/master?after=N38ZsAMfnAqv4q7Ci2%2BQGTEfUvkrMTExOQ%3D%3D
${GOPATH}/src/${GIT_PATH}/bin/etcd --version
${GOPATH}/src/${GIT_PATH}/bin/etcdctl --version

View File

@ -1,7 +1,7 @@
#!/usr/bin/env bash
set -e
GO_VERSION=1.7.5
GO_VERSION=1.8
sudo rm -f /usr/local/go/bin/go && sudo rm -rf /usr/local/go && sudo rm -f /bin/go
@ -10,7 +10,7 @@ DOWNLOAD_URL=${GOOGLE_URL}
sudo curl -s ${DOWNLOAD_URL}/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz
if grep -q GOPATH "$(echo $HOME)/.bashrc"; then
if grep -q GOPATH "$(echo $HOME)/.bashrc"; then
echo "bashrc already has GOPATH";
else
echo "adding GOPATH to bashrc";