From 0fcc31fc02558346124b4f7cbb1990dc00c2ca5f Mon Sep 17 00:00:00 2001 From: Gyu-Ho Lee Date: Fri, 3 Feb 2017 00:50:59 -0800 Subject: [PATCH] agent: upload save separate, interpolated system metrics CSV --- agent/command.go | 3 +- ...em_metrics.go => server_system_metrics.go} | 15 +- agent/upload_log.go | 148 +++++++++++------- 3 files changed, 105 insertions(+), 61 deletions(-) rename agent/{system_metrics.go => server_system_metrics.go} (79%) diff --git a/agent/command.go b/agent/command.go index 8854802b..46460d30 100644 --- a/agent/command.go +++ b/agent/command.go @@ -71,7 +71,8 @@ func init() { Command.PersistentFlags().StringVar(&globalFlags.agentLog, "agent-log", filepath.Join(homeDir(), "agent.log"), "agent log path.") Command.PersistentFlags().StringVar(&globalFlags.databaseLog, "database-log", filepath.Join(homeDir(), "database.log"), "Database log path.") - Command.PersistentFlags().StringVar(&globalFlags.systemMetricsCSV, "system-metrics-csv", filepath.Join(homeDir(), "system-metrics.csv"), "System metrics log path.") + Command.PersistentFlags().StringVar(&globalFlags.systemMetricsCSV, "system-metrics-csv", filepath.Join(homeDir(), "system-metrics.csv"), "Raw system metrics data path.") + Command.PersistentFlags().StringVar(&globalFlags.systemMetricsCSVInterpolated, "system-metrics-csv-interpolated", filepath.Join(homeDir(), "system-metrics-interpolated.csv"), "Interpolated system metrics data path.") Command.PersistentFlags().StringVar(&globalFlags.javaExec, "java-exec", "/usr/bin/java", "Java executable binary path (needed for Zookeeper).") Command.PersistentFlags().StringVar(&globalFlags.etcdExec, "etcd-exec", filepath.Join(os.Getenv("GOPATH"), "bin/etcd"), "etcd executable binary path.") diff --git a/agent/system_metrics.go b/agent/server_system_metrics.go similarity index 79% rename from agent/system_metrics.go rename to agent/server_system_metrics.go index 443cefba..139fbbc9 100644 --- a/agent/system_metrics.go +++ b/agent/server_system_metrics.go @@ -52,11 +52,24 @@ func startMetrics(fs *flags, t *transporterServer) error { case <-t.uploadSig: plog.Infof("upload signal received; saving CSV at %q", t.metricsCSV.FilePath) + if err := t.metricsCSV.Save(); err != nil { - plog.Errorf("psn.CSV.Save error %v", err) + plog.Errorf("psn.CSV.Save(%q) error %v", t.metricsCSV.FilePath, err) } else { plog.Infof("CSV saved at %q", t.metricsCSV.FilePath) } + + interpolated, err := t.metricsCSV.Interpolate() + if err != nil { + plog.Fatalf("psn.CSV.Interpolate(%q) failed with %v", t.metricsCSV.FilePath, err) + } + interpolated.FilePath = fs.systemMetricsCSVInterpolated + if err := interpolated.Save(); err != nil { + plog.Errorf("psn.CSV.Save(%q) error %v", interpolated.FilePath, err) + } else { + plog.Infof("CSV saved at %q", interpolated.FilePath) + } + close(t.csvReady) return diff --git a/agent/upload_log.go b/agent/upload_log.go index d3193800..ea96b343 100644 --- a/agent/upload_log.go +++ b/agent/upload_log.go @@ -32,40 +32,19 @@ func uploadLog(fs *flags, t *transporterServer) error { return err } - srcDatabaseLogPath := fs.databaseLog - dstDatabaseLogPath := filepath.Base(fs.databaseLog) - if !strings.HasPrefix(filepath.Base(fs.databaseLog), t.req.TestName) { - dstDatabaseLogPath = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(fs.databaseLog)) - } - dstDatabaseLogPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstDatabaseLogPath) - plog.Infof("uploading database log [%q -> %q]", srcDatabaseLogPath, dstDatabaseLogPath) var uerr error - for k := 0; k < 30; k++ { - if uerr = u.UploadFile(t.req.GoogleCloudStorageBucketName, srcDatabaseLogPath, dstDatabaseLogPath); uerr != nil { - plog.Errorf("UploadFile error... sleep and retry... (%v)", uerr) - time.Sleep(2 * time.Second) - continue - } else { - break - } - } - if uerr != nil { - return uerr - } - if t.req.Database == agentpb.Request_zetcd || t.req.Database == agentpb.Request_cetcd { - dpath := fs.databaseLog + "-" + t.req.Database.String() - srcDatabaseLogPath2 := dpath - dstDatabaseLogPath2 := filepath.Base(dpath) - if !strings.HasPrefix(filepath.Base(dpath), t.req.TestName) { - dstDatabaseLogPath2 = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(dpath)) + { + srcDatabaseLogPath := fs.databaseLog + dstDatabaseLogPath := filepath.Base(fs.databaseLog) + if !strings.HasPrefix(filepath.Base(fs.databaseLog), t.req.TestName) { + dstDatabaseLogPath = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(fs.databaseLog)) } - dstDatabaseLogPath2 = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstDatabaseLogPath2) - plog.Infof("uploading proxy-database log [%q -> %q]", srcDatabaseLogPath2, dstDatabaseLogPath2) - var uerr error + dstDatabaseLogPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstDatabaseLogPath) + plog.Infof("uploading database log [%q -> %q]", srcDatabaseLogPath, dstDatabaseLogPath) for k := 0; k < 30; k++ { - if uerr = u.UploadFile(t.req.GoogleCloudStorageBucketName, srcDatabaseLogPath2, dstDatabaseLogPath2); uerr != nil { - plog.Errorf("UploadFile error... sleep and retry... (%v)", uerr) + if uerr = u.UploadFile(t.req.GoogleCloudStorageBucketName, srcDatabaseLogPath, dstDatabaseLogPath); uerr != nil { + plog.Warningf("UploadFile error... sleep and retry... (%v)", uerr) time.Sleep(2 * time.Second) continue } else { @@ -77,40 +56,91 @@ func uploadLog(fs *flags, t *transporterServer) error { } } - srcMonitorResultPath := fs.systemMetricsCSV - dstMonitorResultPath := filepath.Base(fs.systemMetricsCSV) - if !strings.HasPrefix(filepath.Base(fs.systemMetricsCSV), t.req.TestName) { - dstMonitorResultPath = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(fs.systemMetricsCSV)) - } - dstMonitorResultPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstMonitorResultPath) - plog.Infof("uploading monitor results [%q -> %q]", srcMonitorResultPath, dstMonitorResultPath) - for k := 0; k < 30; k++ { - if uerr = u.UploadFile(t.req.GoogleCloudStorageBucketName, srcMonitorResultPath, dstMonitorResultPath); uerr != nil { - plog.Errorf("u.UploadFile error... sleep and retry... (%v)", uerr) - time.Sleep(2 * time.Second) - continue - } else { - break + { + if t.req.Database == agentpb.Request_zetcd || t.req.Database == agentpb.Request_cetcd { + dpath := fs.databaseLog + "-" + t.req.Database.String() + srcDatabaseLogPath2 := dpath + dstDatabaseLogPath2 := filepath.Base(dpath) + if !strings.HasPrefix(filepath.Base(dpath), t.req.TestName) { + dstDatabaseLogPath2 = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(dpath)) + } + dstDatabaseLogPath2 = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstDatabaseLogPath2) + plog.Infof("uploading proxy-database log [%q -> %q]", srcDatabaseLogPath2, dstDatabaseLogPath2) + for k := 0; k < 30; k++ { + if uerr = u.UploadFile(t.req.GoogleCloudStorageBucketName, srcDatabaseLogPath2, dstDatabaseLogPath2); uerr != nil { + plog.Warningf("UploadFile error... sleep and retry... (%v)", uerr) + time.Sleep(2 * time.Second) + continue + } else { + break + } + } + if uerr != nil { + return uerr + } } } - if uerr != nil { - return uerr + + { + srcSysMetricsDataPath := fs.systemMetricsCSV + dstSysMetricsDataPath := filepath.Base(fs.systemMetricsCSV) + if !strings.HasPrefix(filepath.Base(fs.systemMetricsCSV), t.req.TestName) { + dstSysMetricsDataPath = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(fs.systemMetricsCSV)) + } + dstSysMetricsDataPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstSysMetricsDataPath) + plog.Infof("uploading system metrics data [%q -> %q]", srcSysMetricsDataPath, dstSysMetricsDataPath) + for k := 0; k < 30; k++ { + if uerr := u.UploadFile(t.req.GoogleCloudStorageBucketName, srcSysMetricsDataPath, dstSysMetricsDataPath); uerr != nil { + plog.Warningf("u.UploadFile error... sleep and retry... (%v)", uerr) + time.Sleep(2 * time.Second) + continue + } else { + break + } + } + if uerr != nil { + return uerr + } } - srcAgentLogPath := fs.agentLog - dstAgentLogPath := filepath.Base(fs.agentLog) - if !strings.HasPrefix(filepath.Base(fs.agentLog), t.req.TestName) { - dstAgentLogPath = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(fs.agentLog)) + { + srcSysMetricsInterpolatedDataPath := fs.systemMetricsCSVInterpolated + dstSysMetricsInterpolatedDataPath := filepath.Base(fs.systemMetricsCSVInterpolated) + if !strings.HasPrefix(filepath.Base(fs.systemMetricsCSVInterpolated), t.req.TestName) { + dstSysMetricsInterpolatedDataPath = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(fs.systemMetricsCSVInterpolated)) + } + dstSysMetricsInterpolatedDataPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstSysMetricsInterpolatedDataPath) + plog.Infof("uploading system metrics interpolated data [%q -> %q]", srcSysMetricsInterpolatedDataPath, dstSysMetricsInterpolatedDataPath) + for k := 0; k < 30; k++ { + if uerr := u.UploadFile(t.req.GoogleCloudStorageBucketName, srcSysMetricsInterpolatedDataPath, dstSysMetricsInterpolatedDataPath); uerr != nil { + plog.Warningf("u.UploadFile error... sleep and retry... (%v)", uerr) + time.Sleep(2 * time.Second) + continue + } else { + break + } + } + if uerr != nil { + return uerr + } } - dstAgentLogPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstAgentLogPath) - plog.Infof("uploading agent logs [%q -> %q]", srcAgentLogPath, dstAgentLogPath) - for k := 0; k < 30; k++ { - if uerr = u.UploadFile(t.req.GoogleCloudStorageBucketName, srcAgentLogPath, dstAgentLogPath); uerr != nil { - plog.Errorf("UploadFile error... sleep and retry... (%v)", uerr) - time.Sleep(2 * time.Second) - continue - } else { - break + + { + srcAgentLogPath := fs.agentLog + dstAgentLogPath := filepath.Base(fs.agentLog) + if !strings.HasPrefix(filepath.Base(fs.agentLog), t.req.TestName) { + dstAgentLogPath = fmt.Sprintf("%s-%d-%s", t.req.TestName, t.req.ServerIndex+1, filepath.Base(fs.agentLog)) + } + dstAgentLogPath = filepath.Join(t.req.GoogleCloudStorageSubDirectory, dstAgentLogPath) + plog.Infof("uploading agent logs [%q -> %q]", srcAgentLogPath, dstAgentLogPath) + for k := 0; k < 30; k++ { + if uerr := u.UploadFile(t.req.GoogleCloudStorageBucketName, srcAgentLogPath, dstAgentLogPath); uerr != nil { + plog.Warningf("UploadFile error... sleep and retry... (%v)", uerr) + time.Sleep(2 * time.Second) + continue + } else { + break + } } }