etcd-io
/
dbtester
mirror of https://github.com/etcd-io/dbtester.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

*: configure 'java' heap flags, Jute max buffer

This commit is contained in:
Gyu-Ho Lee 2017-02-23 10:09:53 -08:00
parent 520f664534
commit 580c46f4ef
No known key found for this signature in database
GPG Key ID: 1DDD39C7EB70C24C
3 changed files with 105 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

118
agent/agent_zookeeper.go
View File

@ -22,6 +22,8 @@ import (
"os/exec" "os/exec"
"path/filepath" "path/filepath"
"strings" "strings"
"github.com/coreos/dbtester/dbtesterpb"
) )
var ( var (
@ -64,6 +66,19 @@ func init() {
} }
} }
// https://zookeeper.apache.org/doc/trunk/zookeeperAdmin.html
const (
// JavaClassPathZookeeperr349 is the Java class paths of Zookeeper r3.4.9.
// CHANGE THIS FOR DIFFERENT ZOOKEEPER RELEASE!
// THIS IS ONLY VALID FOR Zookeeper r3.4.9.
JavaClassPathZookeeperr349 = `-cp zookeeper-3.4.9.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.16.jar:conf org.apache.zookeeper.server.quorum.QuorumPeerMain`
// JavaClassPathZookeeperr352alpha is the Java class paths of Zookeeper r3.5.2-alpha.
// CHANGE THIS FOR DIFFERENT ZOOKEEPER RELEASE!
// THIS IS ONLY VALID FOR Zookeeper r3.5.2-alpha.
JavaClassPathZookeeperr352alpha = `-cp zookeeper-3.4.9.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.16.jar:conf org.apache.zookeeper.server.quorum.QuorumPeerMain`
)
// startZookeeper starts Zookeeper. // startZookeeper starts Zookeeper.
func startZookeeper(fs *flags, t *transporterServer) error { func startZookeeper(fs *flags, t *transporterServer) error {
if !exist(fs.javaExec) { if !exist(fs.javaExec) {
@ -83,46 +98,105 @@ func startZookeeper(fs *flags, t *transporterServer) error {
return err return err
} }
// TODO: support r3.5.2
ipath := filepath.Join(fs.zkDataDir, "myid") ipath := filepath.Join(fs.zkDataDir, "myid")
plog.Infof("writing Zookeeper myid file %d to %s", t.req.Flag_Zookeeper_R3_4_9.MyID, ipath) switch t.req.DatabaseID {
if err := toFile(fmt.Sprintf("%d", t.req.Flag_Zookeeper_R3_4_9.MyID), ipath); err != nil { case dbtesterpb.DatabaseID_zookeeper__r3_4_9:
return err if t.req.Flag_Zookeeper_R3_4_9 == nil {
return fmt.Errorf("request 'Flag_Zookeeper_R3_4_9' is nil")
}
plog.Infof("writing Zookeeper myid file %d to %s", t.req.Flag_Zookeeper_R3_4_9.MyID, ipath)
if err := toFile(fmt.Sprintf("%d", t.req.Flag_Zookeeper_R3_4_9.MyID), ipath); err != nil {
return err
}
case dbtesterpb.DatabaseID_zookeeper__r3_5_2_alpha:
if t.req.Flag_Zookeeper_R3_5_2Alpha == nil {
return fmt.Errorf("request 'Flag_Zookeeper_R3_5_2Alpha' is nil")
}
plog.Infof("writing Zookeeper myid file %d to %s", t.req.Flag_Zookeeper_R3_5_2Alpha.MyID, ipath)
if err := toFile(fmt.Sprintf("%d", t.req.Flag_Zookeeper_R3_5_2Alpha.MyID), ipath); err != nil {
return err
}
default:
return fmt.Errorf("database ID %q is not supported", t.req.DatabaseID)
} }
var cfg ZookeeperConfig
peerIPs := strings.Split(t.req.PeerIPsString, "___") peerIPs := strings.Split(t.req.PeerIPsString, "___")
peers := []ZookeeperPeer{} peers := []ZookeeperPeer{}
for i := range peerIPs { for i := range peerIPs {
peers = append(peers, ZookeeperPeer{MyID: i + 1, IP: peerIPs[i]}) peers = append(peers, ZookeeperPeer{MyID: i + 1, IP: peerIPs[i]})
} }
cfg := ZookeeperConfig{ switch t.req.DatabaseID {
TickTime: t.req.Flag_Zookeeper_R3_4_9.TickTime, case dbtesterpb.DatabaseID_zookeeper__r3_4_9:
DataDir: fs.zkDataDir, cfg = ZookeeperConfig{
ClientPort: t.req.Flag_Zookeeper_R3_4_9.ClientPort, TickTime: t.req.Flag_Zookeeper_R3_4_9.TickTime,
InitLimit: t.req.Flag_Zookeeper_R3_4_9.InitLimit, DataDir: fs.zkDataDir,
SyncLimit: t.req.Flag_Zookeeper_R3_4_9.SyncLimit, ClientPort: t.req.Flag_Zookeeper_R3_4_9.ClientPort,
MaxClientConnections: t.req.Flag_Zookeeper_R3_4_9.MaxClientConnections, InitLimit: t.req.Flag_Zookeeper_R3_4_9.InitLimit,
Peers: peers, SyncLimit: t.req.Flag_Zookeeper_R3_4_9.SyncLimit,
SnapCount: t.req.Flag_Zookeeper_R3_4_9.SnapCount, MaxClientConnections: t.req.Flag_Zookeeper_R3_4_9.MaxClientConnections,
Peers: peers,
SnapCount: t.req.Flag_Zookeeper_R3_4_9.SnapCount,
}
case dbtesterpb.DatabaseID_zookeeper__r3_5_2_alpha:
cfg = ZookeeperConfig{
TickTime: t.req.Flag_Zookeeper_R3_5_2Alpha.TickTime,
DataDir: fs.zkDataDir,
ClientPort: t.req.Flag_Zookeeper_R3_5_2Alpha.ClientPort,
InitLimit: t.req.Flag_Zookeeper_R3_5_2Alpha.InitLimit,
SyncLimit: t.req.Flag_Zookeeper_R3_5_2Alpha.SyncLimit,
MaxClientConnections: t.req.Flag_Zookeeper_R3_5_2Alpha.MaxClientConnections,
Peers: peers,
SnapCount: t.req.Flag_Zookeeper_R3_5_2Alpha.SnapCount,
}
default:
return fmt.Errorf("database ID %q is not supported", t.req.DatabaseID)
} }
tpl := template.Must(template.New("zkTemplate").Parse(zkTemplate)) tpl := template.Must(template.New("zkTemplate").Parse(zkTemplate))
buf := new(bytes.Buffer) buf := new(bytes.Buffer)
if err := tpl.Execute(buf, cfg); err != nil { if err := tpl.Execute(buf, cfg); err != nil {
return err return err
} }
zc := buf.String() zctxt := buf.String()
plog.Infof("writing Zookeeper config file %q (config %q)", fs.zkConfig, zctxt)
plog.Infof("writing Zookeeper config file %q (config %q)", fs.zkConfig, zc) if err := toFile(zctxt, fs.zkConfig); err != nil {
if err := toFile(zc, fs.zkConfig); err != nil {
return err return err
} }
// CHANGE THIS FOR DIFFERENT ZOOKEEPER RELEASE args := []string{shell}
// https://zookeeper.apache.org/doc/trunk/zookeeperAdmin.html var flagString string
// THIS IS ONLY VALID FOR Zookeeper r3.4.9 switch t.req.DatabaseID {
flagString := `-cp zookeeper-3.4.9.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.16.jar:conf org.apache.zookeeper.server.quorum.QuorumPeerMain` case dbtesterpb.DatabaseID_zookeeper__r3_4_9:
flagString = JavaClassPathZookeeperr349
// -Djute.maxbuffer=33554432 -Xms50G -Xmx50G
if t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Djute.maxbuffer=%d", t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer))
}
if t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Xms%s", t.req.Flag_Zookeeper_R3_4_9.JavaXms))
}
if t.req.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Xmx%s", t.req.Flag_Zookeeper_R3_4_9.JavaXmx))
}
args := []string{shell, "-c", fs.javaExec + " " + flagString + " " + fs.zkConfig} case dbtesterpb.DatabaseID_zookeeper__r3_5_2_alpha:
flagString = JavaClassPathZookeeperr352alpha
// -Djute.maxbuffer=33554432 -Xms50G -Xmx50G
if t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Djute.maxbuffer=%d", t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer))
}
if t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Xms%s", t.req.Flag_Zookeeper_R3_5_2Alpha.JavaXms))
}
if t.req.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer != 0 {
args = append(args, fmt.Sprintf("-Xmx%s", t.req.Flag_Zookeeper_R3_5_2Alpha.JavaXmx))
}
default:
return fmt.Errorf("database ID %q is not supported", t.req.DatabaseID)
}
args = append(args, "-c", fs.javaExec+" "+flagString+" "+fs.zkConfig)
cmd := exec.Command(args[0], args[1:]...) cmd := exec.Command(args[0], args[1:]...)
cmd.Stdout = t.databaseLogFile cmd.Stdout = t.databaseLogFile

7
config_dbtester.go
View File

@ -341,9 +341,11 @@ func (cfg *Config) ToRequest(databaseID string, op dbtesterpb.Operation, idx int
QuotaSizeBytes: gcfg.Flag_Etcd_Tip.QuotaSizeBytes, QuotaSizeBytes: gcfg.Flag_Etcd_Tip.QuotaSizeBytes,
} }
// TODO: jvm flags
case dbtesterpb.DatabaseID_zookeeper__r3_4_9: case dbtesterpb.DatabaseID_zookeeper__r3_4_9:
req.Flag_Zookeeper_R3_4_9 = &dbtesterpb.Flag_Zookeeper_R3_4_9{ req.Flag_Zookeeper_R3_4_9 = &dbtesterpb.Flag_Zookeeper_R3_4_9{
JavaDJuteMaxBuffer: gcfg.Flag_Zookeeper_R3_4_9.JavaDJuteMaxBuffer,
JavaXms: gcfg.Flag_Zookeeper_R3_4_9.JavaXms,
JavaXmx: gcfg.Flag_Zookeeper_R3_4_9.JavaXmx,
MyID: uint32(idx + 1), MyID: uint32(idx + 1),
ClientPort: gcfg.Flag_Zookeeper_R3_4_9.ClientPort, ClientPort: gcfg.Flag_Zookeeper_R3_4_9.ClientPort,
TickTime: gcfg.Flag_Zookeeper_R3_4_9.TickTime, TickTime: gcfg.Flag_Zookeeper_R3_4_9.TickTime,
@ -354,6 +356,9 @@ func (cfg *Config) ToRequest(databaseID string, op dbtesterpb.Operation, idx int
} }
case dbtesterpb.DatabaseID_zookeeper__r3_5_2_alpha: case dbtesterpb.DatabaseID_zookeeper__r3_5_2_alpha:
req.Flag_Zookeeper_R3_5_2Alpha = &dbtesterpb.Flag_Zookeeper_R3_5_2Alpha{ req.Flag_Zookeeper_R3_5_2Alpha = &dbtesterpb.Flag_Zookeeper_R3_5_2Alpha{
JavaDJuteMaxBuffer: gcfg.Flag_Zookeeper_R3_5_2Alpha.JavaDJuteMaxBuffer,
JavaXms: gcfg.Flag_Zookeeper_R3_5_2Alpha.JavaXms,
JavaXmx: gcfg.Flag_Zookeeper_R3_5_2Alpha.JavaXmx,
MyID: uint32(idx + 1), MyID: uint32(idx + 1),
ClientPort: gcfg.Flag_Zookeeper_R3_5_2Alpha.ClientPort, ClientPort: gcfg.Flag_Zookeeper_R3_5_2Alpha.ClientPort,
TickTime: gcfg.Flag_Zookeeper_R3_5_2Alpha.TickTime, TickTime: gcfg.Flag_Zookeeper_R3_5_2Alpha.TickTime,

3
config_dbtester_test.go
View File

@ -542,6 +542,9 @@ func TestConfig(t *testing.T) {
GoogleCloudStorageSubDirectory: "2017Q1-01-etcd-zookeeper-consul/01-write-1M-keys-client-variable", GoogleCloudStorageSubDirectory: "2017Q1-01-etcd-zookeeper-consul/01-write-1M-keys-client-variable",
}, },
Flag_Zookeeper_R3_5_2Alpha: &dbtesterpb.Flag_Zookeeper_R3_5_2Alpha{ Flag_Zookeeper_R3_5_2Alpha: &dbtesterpb.Flag_Zookeeper_R3_5_2Alpha{
JavaDJuteMaxBuffer: 33554432,
JavaXms: "50G",
JavaXmx: "50G",
MyID: 3, MyID: 3,
ClientPort: 2181, ClientPort: 2181,
TickTime: 2000, TickTime: 2000,