etcd-io
/
dbtester
mirror of https://github.com/etcd-io/dbtester.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

agent: upload save separate, interpolated system metrics CSV

This commit is contained in:
Gyu-Ho Lee 2017-02-03 00:50:59 -08:00
parent d552c2c8a1
commit 0fcc31fc02
No known key found for this signature in database
GPG Key ID: 1DDD39C7EB70C24C
3 changed files with 105 additions and 61 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
agent/command.go
View File

@ -71,7 +71,8 @@ func init() {
Command.PersistentFlags().StringVar(&globalFlags.agentLog, "agent-log", filepath.Join(homeDir(), "agent.log"), "agent log path.")
Command.PersistentFlags().StringVar(&globalFlags.databaseLog, "database-log", filepath.Join(homeDir(), "database.log"), "Database log path.")
Command.PersistentFlags().StringVar(&globalFlags.systemMetricsCSV, "system-metrics-csv", filepath.Join(homeDir(), "system-metrics.csv"), "System metrics log path.")
Command.PersistentFlags().StringVar(&globalFlags.systemMetricsCSV, "system-metrics-csv", filepath.Join(homeDir(), "system-metrics.csv"), "Raw system metrics data path.")
Command.PersistentFlags().StringVar(&globalFlags.systemMetricsCSVInterpolated, "system-metrics-csv-interpolated", filepath.Join(homeDir(), "system-metrics-interpolated.csv"), "Interpolated system metrics data path.")
Command.PersistentFlags().StringVar(&globalFlags.javaExec, "java-exec", "/usr/bin/java", "Java executable binary path (needed for Zookeeper).")
Command.PersistentFlags().StringVar(&globalFlags.etcdExec, "etcd-exec", filepath.Join(os.Getenv("GOPATH"), "bin/etcd"), "etcd executable binary path.")

15
agent/system_metrics.go → agent/server_system_metrics.go
View File

@ -52,11 +52,24 @@ func startMetrics(fs *flags, t *transporterServer) error {
case <-t.uploadSig:
plog.Infof("upload signal received; saving CSV at %q", t.metricsCSV.FilePath)
if err := t.metricsCSV.Save(); err != nil {
plog.Errorf("psn.CSV.Save error %v", err)
plog.Errorf("psn.CSV.Save(%q) error %v", t.metricsCSV.FilePath, err)
} else {
plog.Infof("CSV saved at %q", t.metricsCSV.FilePath)
}
interpolated, err := t.metricsCSV.Interpolate()
if err != nil {
plog.Fatalf("psn.CSV.Interpolate(%q) failed with %v", t.metricsCSV.FilePath, err)
}
interpolated.FilePath = fs.systemMetricsCSVInterpolated
if err := interpolated.Save(); err != nil {
plog.Errorf("psn.CSV.Save(%q) error %v", interpolated.FilePath, err)
} else {
plog.Infof("CSV saved at %q", interpolated.FilePath)
}
close(t.csvReady)
return

56
agent/upload_log.go
View File

@ -32,6 +32,9 @@ func uploadLog(fs *flags, t *transporterServer) error {
return err
}
var uerr error
{
srcDatabaseLogPath := fs.databaseLog
dstDatabaseLogPath := filepath.Base(fs.databaseLog)
if !strings.HasPrefix(filepath.Base(fs.databaseLog), t.req.TestName) {
@ -39,10 +42,9 @@ func uploadLog(fs *flags, t *transporterServer) error {
}
dstDatabaseLogPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstDatabaseLogPath)
plog.Infof("uploading database log [%q -> %q]", srcDatabaseLogPath, dstDatabaseLogPath)
var uerr error
for k := 0; k < 30; k++ {
if uerr = u.UploadFile(t.req.GoogleCloudStorageBucketName, srcDatabaseLogPath, dstDatabaseLogPath); uerr != nil {
plog.Errorf("UploadFile error... sleep and retry... (%v)", uerr)
plog.Warningf("UploadFile error... sleep and retry... (%v)", uerr)
time.Sleep(2 * time.Second)
continue
} else {
@ -52,7 +54,9 @@ func uploadLog(fs *flags, t *transporterServer) error {
if uerr != nil {
return uerr
}
}
{
if t.req.Database == agentpb.Request_zetcd || t.req.Database == agentpb.Request_cetcd {
dpath := fs.databaseLog + "-" + t.req.Database.String()
srcDatabaseLogPath2 := dpath
@ -62,10 +66,9 @@ func uploadLog(fs *flags, t *transporterServer) error {
}
dstDatabaseLogPath2 = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstDatabaseLogPath2)
plog.Infof("uploading proxy-database log [%q -> %q]", srcDatabaseLogPath2, dstDatabaseLogPath2)
var uerr error
for k := 0; k < 30; k++ {
if uerr = u.UploadFile(t.req.GoogleCloudStorageBucketName, srcDatabaseLogPath2, dstDatabaseLogPath2); uerr != nil {
plog.Errorf("UploadFile error... sleep and retry... (%v)", uerr)
plog.Warningf("UploadFile error... sleep and retry... (%v)", uerr)
time.Sleep(2 * time.Second)
continue
} else {
@ -76,17 +79,19 @@ func uploadLog(fs *flags, t *transporterServer) error {
return uerr
}
}
}
srcMonitorResultPath := fs.systemMetricsCSV
dstMonitorResultPath := filepath.Base(fs.systemMetricsCSV)
{
srcSysMetricsDataPath := fs.systemMetricsCSV
dstSysMetricsDataPath := filepath.Base(fs.systemMetricsCSV)
if !strings.HasPrefix(filepath.Base(fs.systemMetricsCSV), t.req.TestName) {
dstMonitorResultPath = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(fs.systemMetricsCSV))
dstSysMetricsDataPath = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(fs.systemMetricsCSV))
}
dstMonitorResultPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstMonitorResultPath)
plog.Infof("uploading monitor results [%q -> %q]", srcMonitorResultPath, dstMonitorResultPath)
dstSysMetricsDataPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstSysMetricsDataPath)
plog.Infof("uploading system metrics data [%q -> %q]", srcSysMetricsDataPath, dstSysMetricsDataPath)
for k := 0; k < 30; k++ {
if uerr = u.UploadFile(t.req.GoogleCloudStorageBucketName, srcMonitorResultPath, dstMonitorResultPath); uerr != nil {
plog.Errorf("u.UploadFile error... sleep and retry... (%v)", uerr)
if uerr := u.UploadFile(t.req.GoogleCloudStorageBucketName, srcSysMetricsDataPath, dstSysMetricsDataPath); uerr != nil {
plog.Warningf("u.UploadFile error... sleep and retry... (%v)", uerr)
time.Sleep(2 * time.Second)
continue
} else {
@ -96,7 +101,31 @@ func uploadLog(fs *flags, t *transporterServer) error {
if uerr != nil {
return uerr
}
}
{
srcSysMetricsInterpolatedDataPath := fs.systemMetricsCSVInterpolated
dstSysMetricsInterpolatedDataPath := filepath.Base(fs.systemMetricsCSVInterpolated)
if !strings.HasPrefix(filepath.Base(fs.systemMetricsCSVInterpolated), t.req.TestName) {
dstSysMetricsInterpolatedDataPath = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(fs.systemMetricsCSVInterpolated))
}
dstSysMetricsInterpolatedDataPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstSysMetricsInterpolatedDataPath)
plog.Infof("uploading system metrics interpolated data [%q -> %q]", srcSysMetricsInterpolatedDataPath, dstSysMetricsInterpolatedDataPath)
for k := 0; k < 30; k++ {
if uerr := u.UploadFile(t.req.GoogleCloudStorageBucketName, srcSysMetricsInterpolatedDataPath, dstSysMetricsInterpolatedDataPath); uerr != nil {
plog.Warningf("u.UploadFile error... sleep and retry... (%v)", uerr)
time.Sleep(2 * time.Second)
continue
} else {
break
}
}
if uerr != nil {
return uerr
}
}
{
srcAgentLogPath := fs.agentLog
dstAgentLogPath := filepath.Base(fs.agentLog)
if !strings.HasPrefix(filepath.Base(fs.agentLog), t.req.TestName) {
@ -105,14 +134,15 @@ func uploadLog(fs *flags, t *transporterServer) error {
dstAgentLogPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstAgentLogPath)
plog.Infof("uploading agent logs [%q -> %q]", srcAgentLogPath, dstAgentLogPath)
for k := 0; k < 30; k++ {
if uerr = u.UploadFile(t.req.GoogleCloudStorageBucketName, srcAgentLogPath, dstAgentLogPath); uerr != nil {
plog.Errorf("UploadFile error... sleep and retry... (%v)", uerr)
if uerr := u.UploadFile(t.req.GoogleCloudStorageBucketName, srcAgentLogPath, dstAgentLogPath); uerr != nil {
plog.Warningf("UploadFile error... sleep and retry... (%v)", uerr)
time.Sleep(2 * time.Second)
continue
} else {
break
}
}
}
return uerr
}