Merge pull request #15 from coreos/readme

README
This commit is contained in:
Gyu-Ho Lee 2016-03-25 13:50:30 -07:00
commit a95e2d81e4
313 changed files with 438992 additions and 16149 deletions

2
Godeps/Godeps.json generated
View File

@ -151,7 +151,7 @@
},
{
"ImportPath": "github.com/gyuho/dataframe",
"Rev": "a0065d25cf8a77ca8f7fe8e0876c727aac3b0525"
"Rev": "740858301d887e27281c083a4f8119944a7a50d3"
},
{
"ImportPath": "github.com/gyuho/psn/ps",

View File

@ -1,38 +0,0 @@
#!/usr/bin/env bash
dbtester analyze \
--output=testdata/bench-01-consul-aggregated.csv \
--bench-file-path=testdata/bench-01-consul-timeseries.csv \
--monitor-data-file-paths=testdata/bench-01-consul-1-monitor.csv,testdata/bench-01-consul-2-monitor.csv,testdata/bench-01-consul-3-monitor.csv
dbtester analyze \
--output=testdata/bench-01-etcd-aggregated.csv \
--bench-file-path=testdata/bench-01-etcd-timeseries.csv \
--monitor-data-file-paths=testdata/bench-01-etcd-1-monitor.csv,testdata/bench-01-etcd-2-monitor.csv,testdata/bench-01-etcd-3-monitor.csv
dbtester analyze \
--output=testdata/bench-01-etcd2-aggregated.csv \
--bench-file-path=testdata/bench-01-etcd2-timeseries.csv \
--monitor-data-file-paths=testdata/bench-01-etcd2-1-monitor.csv,testdata/bench-01-etcd2-2-monitor.csv,testdata/bench-01-etcd2-3-monitor.csv
dbtester analyze \
--output=testdata/bench-01-zk-aggregated.csv \
--bench-file-path=testdata/bench-01-zk-timeseries.csv \
--monitor-data-file-paths=testdata/bench-01-zk-1-monitor.csv,testdata/bench-01-zk-2-monitor.csv,testdata/bench-01-zk-3-monitor.csv
dbtester analyze \
--output=testdata/bench-01-all-aggregated.csv \
--aggregated-file-paths=testdata/bench-01-consul-aggregated.csv,testdata/bench-01-etcd-aggregated.csv,testdata/bench-01-etcd2-aggregated.csv,testdata/bench-01-zk-aggregated.csv
dbtester analyze \
--output=testdata/bench-01-plot \
--image-format=png \
--file-to-plot=testdata/bench-01-all-aggregated.csv
dbtester analyze \
--output=testdata/bench-01-plot \
--image-format=svg \
--file-to-plot=testdata/bench-01-all-aggregated.csv

View File

@ -36,6 +36,7 @@ type (
AggAggFilePath string
ImageFormat string
ImageTitle string
MultiTagTitle string
}
)
@ -60,6 +61,7 @@ func init() {
Command.PersistentFlags().StringVarP(&globalFlags.AggAggFilePath, "file-to-plot", "p", "", "Aggregated CSV file path to plot.")
Command.PersistentFlags().StringVarP(&globalFlags.ImageFormat, "image-format", "f", "png", "Image format (png, svg).")
Command.PersistentFlags().StringVarP(&globalFlags.ImageTitle, "image-title", "t", "", "Image title.")
Command.PersistentFlags().StringVarP(&globalFlags.MultiTagTitle, "multi-tag-title", "g", "", "Special title for *multi test.")
}
func CommandFunc(cmd *cobra.Command, args []string) error {
@ -83,7 +85,7 @@ func CommandFunc(cmd *cobra.Command, args []string) error {
}
case len(globalFlags.AggAggFilePath) > 0:
if err := plotAggAgg(globalFlags.AggAggFilePath, globalFlags.OutputPath, globalFlags.ImageFormat, globalFlags.ImageTitle); err != nil {
if err := plotAggAgg(globalFlags.AggAggFilePath, globalFlags.OutputPath, globalFlags.ImageFormat, globalFlags.ImageTitle, globalFlags.MultiTagTitle); err != nil {
return err
}
}
@ -233,101 +235,89 @@ func aggMonitor(monitorPaths ...string) (dataframe.Frame, error) {
// aggBenchAndMonitor combines benchmark latency results and monitor CSV files.
func aggBenchAndMonitor(benchPath string, monitorPaths ...string) (dataframe.Frame, error) {
fr2, err := aggMonitor(monitorPaths...)
frMonitor, err := aggMonitor(monitorPaths...)
if err != nil {
return nil, err
}
col2, err := fr2.GetColumn("unix_ts")
colMonitorUnixTs, err := frMonitor.GetColumn("unix_ts")
if err != nil {
return nil, err
}
// need to combine fr2 to fr1
fr1, err := dataframe.NewFromCSV(nil, benchPath)
// need to combine frMonitor to frBench
frBench, err := dataframe.NewFromCSV(nil, benchPath)
if err != nil {
return nil, err
}
col1, err := fr1.GetColumn("unix_ts")
colBenchUnixTs, err := frBench.GetColumn("unix_ts")
if err != nil {
return nil, err
}
fv, ok := col1.FrontNonNil()
fv, ok := colBenchUnixTs.FrontNonNil()
if !ok {
return nil, fmt.Errorf("FrontNonNil %s has empty Unix time %v", benchPath, fv)
}
startIdx, ok := col2.FindValue(fv)
startRowMonitor, ok := colMonitorUnixTs.FindValue(fv)
if !ok {
return nil, fmt.Errorf("%v is not found in benchmark result %s", fv, benchPath)
return nil, fmt.Errorf("%v is not found in monitor results %q", fv, monitorPaths)
}
bv, ok := col1.BackNonNil()
bv, ok := colBenchUnixTs.BackNonNil()
if !ok {
return nil, fmt.Errorf("BackNonNil %s has empty Unix time %v", benchPath, bv)
}
endIdx, ok := col2.FindValue(bv)
if !ok {
return nil, fmt.Errorf("%v is not found in benchmark result %s", bv, benchPath)
endRowMonitor, ok := colMonitorUnixTs.FindValue(bv)
if !ok { // monitor short of rows
endRowMonitor = colMonitorUnixTs.RowNumber() - 1
}
var minLen int
for i, hd := range fr1.GetHeader() {
var col dataframe.Column
col, err = fr1.GetColumn(hd)
if err != nil {
return nil, err
var benchLastIdx int
for _, col := range frBench.GetColumns() {
if benchLastIdx == 0 {
benchLastIdx = col.RowNumber()
}
if i == 0 {
minLen = col.RowNumber()
}
if minLen < col.RowNumber() {
minLen = col.RowNumber()
if benchLastIdx > col.RowNumber() {
benchLastIdx = col.RowNumber()
}
}
var delta int
if minLen > endIdx+1-startIdx { // short of rows
delta = minLen - (endIdx + 1 - startIdx)
benchLastIdx--
if benchLastIdx+1 < endRowMonitor-startRowMonitor+1 { // benchmark is short of rows
endRowMonitor = startRowMonitor + benchLastIdx
} else { // monitor is short of rows
benchLastIdx = endRowMonitor - startRowMonitor
}
for _, hd := range fr2.GetHeader() {
for _, hd := range frMonitor.GetHeader() {
if hd == "unix_ts" {
continue
}
var col dataframe.Column
col, err = fr2.GetColumn(hd)
col, err = frMonitor.GetColumn(hd)
if err != nil {
return nil, err
}
if err = col.KeepRows(startIdx, endIdx+1+delta); err != nil {
if err = col.KeepRows(startRowMonitor, endRowMonitor); err != nil {
return nil, err
}
if err = fr1.AddColumn(col); err != nil {
if err = frBench.AddColumn(col); err != nil {
return nil, err
}
}
// get average value
uc, err := fr1.GetColumn("unix_ts")
if err != nil {
return nil, err
}
var (
rowNum = uc.RowNumber()
sampleSize = float64(len(monitorPaths))
cumulativeThroughputCol = dataframe.NewColumn("cumulative_throughput")
totalThrougput int
avgCpuCol = dataframe.NewColumn("avg_cpu")
avgMemCol = dataframe.NewColumn("avg_memory_mb")
)
for i := 0; i < rowNum; i++ {
for i := 0; i < benchLastIdx; i++ {
var (
cpuTotal float64
memoryTotal float64
)
for _, hd := range fr1.GetHeader() {
var col dataframe.Column
col, err = fr1.GetColumn(hd)
if err != nil {
return nil, err
}
for _, col := range frBench.GetColumns() {
var rv dataframe.Value
rv, err = col.GetValue(i)
if err != nil {
@ -335,11 +325,11 @@ func aggBenchAndMonitor(benchPath string, monitorPaths ...string) (dataframe.Fra
}
fv, _ := rv.ToNumber()
switch {
case strings.HasPrefix(hd, "cpu_"):
case strings.HasPrefix(col.GetHeader(), "cpu_"):
cpuTotal += fv
case strings.HasPrefix(hd, "memory_"):
case strings.HasPrefix(col.GetHeader(), "memory_"):
memoryTotal += fv
case hd == "throughput":
case col.GetHeader() == "throughput":
fv, _ := rv.ToNumber()
totalThrougput += int(fv)
cumulativeThroughputCol.PushBack(dataframe.NewStringValue(fmt.Sprintf("%d", totalThrougput)))
@ -349,15 +339,15 @@ func aggBenchAndMonitor(benchPath string, monitorPaths ...string) (dataframe.Fra
avgMemCol.PushBack(dataframe.NewStringValue(fmt.Sprintf("%.2f", memoryTotal/sampleSize)))
}
unixTsCol, err := fr1.GetColumn("unix_ts")
unixTsCol, err := frBench.GetColumn("unix_ts")
if err != nil {
return nil, err
}
latencyCol, err := fr1.GetColumn("avg_latency_ms")
latencyCol, err := frBench.GetColumn("avg_latency_ms")
if err != nil {
return nil, err
}
throughputCol, err := fr1.GetColumn("throughput")
throughputCol, err := frBench.GetColumn("throughput")
if err != nil {
return nil, err
}
@ -368,8 +358,8 @@ func aggBenchAndMonitor(benchPath string, monitorPaths ...string) (dataframe.Fra
nf.AddColumn(throughputCol)
nf.AddColumn(cumulativeThroughputCol)
for _, hd := range fr1.GetHeader() {
col, err := fr1.GetColumn(hd)
for _, hd := range frBench.GetHeader() {
col, err := frBench.GetColumn(hd)
if err != nil {
return nil, err
}
@ -443,6 +433,8 @@ func aggAgg(fpaths ...string) (dataframe.Frame, error) {
dbID = "consul"
case strings.Contains(fpaths[i], "etcd2"):
dbID = "etcd2"
case strings.Contains(fpaths[i], "etcdmulti"):
dbID = "etcd3multi"
case strings.Contains(fpaths[i], "etcd"):
dbID = "etcd3"
case strings.Contains(fpaths[i], "zk"):
@ -457,17 +449,18 @@ func aggAgg(fpaths ...string) (dataframe.Frame, error) {
return nf, nil
}
func plotAggAgg(fpath, outputPath, imageFormat, imageTitle string) error {
func plotAggAgg(fpath, outputPath, imageFormat, imageTitle, multiTagTitle string) error {
fr, err := dataframe.NewFromCSV(nil, fpath)
if err != nil {
return err
}
plot.DefaultFont = "Helvetica"
plotter.DefaultLineStyle.Width /= 2
plotter.DefaultLineStyle.Width = vg.Points(1.5)
plotter.DefaultGlyphStyle.Radius = vg.Points(2.0)
var (
defaultSize = 5 * vg.Inch
defaultPlotWidth = 12 * vg.Inch
defaultPlotHeight = 8 * vg.Inch
avgLatencyPath = outputPath + fmt.Sprintf("-avg-latency-ms.%s", imageFormat)
throughputPath = outputPath + fmt.Sprintf("-throughput.%s", imageFormat)
avgCpuPath = outputPath + fmt.Sprintf("-avg-cpu.%s", imageFormat)
@ -513,7 +506,29 @@ func plotAggAgg(fpath, outputPath, imageFormat, imageTitle string) error {
plotAvgLatency.Title.Text = fmt.Sprintf("%s, Latency", imageTitle)
plotAvgLatency.X.Label.Text = "second"
plotAvgLatency.Y.Label.Text = "latency(ms)"
if err = plotutil.AddLinePoints(
plotAvgLatency.Legend.Top = true
plotAvgLatencyEtcd3Multi, err := fr.GetColumn("avg_latency_ms_etcd3multi")
if err == nil {
var plotAvgLatencyEtcd3MultiPoints plotter.XYs
plotAvgLatencyEtcd3MultiPoints, err = points(plotAvgLatencyEtcd3Multi)
if err != nil {
return err
}
if err = plotutil.AddLines(
plotAvgLatency,
"consul", plotAvgLatencyConsulPoints,
"etcd3", plotAvgLatencyEtcd3Points,
strings.Replace("etcd3multi", "multi", "-"+multiTagTitle, -1), plotAvgLatencyEtcd3MultiPoints,
"etcd2", plotAvgLatencyEtcd2Points,
"zk", plotAvgLatencyZkPoints,
); err != nil {
return err
}
if err = plotAvgLatency.Save(defaultPlotWidth, defaultPlotHeight, avgLatencyPath); err != nil {
return err
}
} else {
if err = plotutil.AddLines(
plotAvgLatency,
"consul", plotAvgLatencyConsulPoints,
"etcd3", plotAvgLatencyEtcd3Points,
@ -522,9 +537,10 @@ func plotAggAgg(fpath, outputPath, imageFormat, imageTitle string) error {
); err != nil {
return err
}
if err = plotAvgLatency.Save(defaultSize, defaultSize, avgLatencyPath); err != nil {
if err = plotAvgLatency.Save(defaultPlotWidth, defaultPlotHeight, avgLatencyPath); err != nil {
return err
}
}
plotThroughputConsul, err := fr.GetColumn("throughput_consul")
if err != nil {
@ -565,7 +581,29 @@ func plotAggAgg(fpath, outputPath, imageFormat, imageTitle string) error {
plotThroughput.Title.Text = fmt.Sprintf("%s, Throughput", imageTitle)
plotThroughput.X.Label.Text = "second"
plotThroughput.Y.Label.Text = "Throughput"
if err = plotutil.AddLinePoints(
plotThroughput.Legend.Top = true
plotThroughputEtcd3Multi, err := fr.GetColumn("throughput_etcd3multi")
if err == nil {
var plotThroughputEtcd3MultiPoints plotter.XYs
plotThroughputEtcd3MultiPoints, err = points(plotThroughputEtcd3Multi)
if err != nil {
return err
}
if err = plotutil.AddLines(
plotThroughput,
"consul", plotThroughputConsulPoints,
"etcd3", plotThroughputEtcd3Points,
strings.Replace("etcd3multi", "multi", "-"+multiTagTitle, -1), plotThroughputEtcd3MultiPoints,
"etcd2", plotThroughputEtcd2Points,
"zk", plotThroughputZkPoints,
); err != nil {
return err
}
if err = plotThroughput.Save(defaultPlotWidth, defaultPlotHeight, throughputPath); err != nil {
return err
}
} else {
if err = plotutil.AddLines(
plotThroughput,
"consul", plotThroughputConsulPoints,
"etcd3", plotThroughputEtcd3Points,
@ -574,9 +612,10 @@ func plotAggAgg(fpath, outputPath, imageFormat, imageTitle string) error {
); err != nil {
return err
}
if err = plotThroughput.Save(defaultSize, defaultSize, throughputPath); err != nil {
if err = plotThroughput.Save(defaultPlotWidth, defaultPlotHeight, throughputPath); err != nil {
return err
}
}
plotAvgCpuConsul, err := fr.GetColumn("avg_cpu_consul")
if err != nil {
@ -617,7 +656,29 @@ func plotAggAgg(fpath, outputPath, imageFormat, imageTitle string) error {
plotAvgCpu.Title.Text = fmt.Sprintf("%s, CPU", imageTitle)
plotAvgCpu.X.Label.Text = "second"
plotAvgCpu.Y.Label.Text = "CPU"
if err = plotutil.AddLinePoints(
plotAvgCpu.Legend.Top = true
plotAvgCpuEtcd3Multi, err := fr.GetColumn("avg_cpu_etcd3multi")
if err == nil {
var plotAvgCpuEtcd3MultiPoints plotter.XYs
plotAvgCpuEtcd3MultiPoints, err = points(plotAvgCpuEtcd3Multi)
if err != nil {
return err
}
if err = plotutil.AddLines(
plotAvgCpu,
"consul", plotAvgCpuConsulPoints,
"etcd3", plotAvgCpuEtcd3Points,
strings.Replace("etcd3multi", "multi", "-"+multiTagTitle, -1), plotAvgCpuEtcd3MultiPoints,
"etcd2", plotAvgCpuEtcd2Points,
"zk", plotAvgCpuZkPoints,
); err != nil {
return err
}
if err = plotAvgCpu.Save(defaultPlotWidth, defaultPlotHeight, avgCpuPath); err != nil {
return err
}
} else {
if err = plotutil.AddLines(
plotAvgCpu,
"consul", plotAvgCpuConsulPoints,
"etcd3", plotAvgCpuEtcd3Points,
@ -626,9 +687,10 @@ func plotAggAgg(fpath, outputPath, imageFormat, imageTitle string) error {
); err != nil {
return err
}
if err = plotAvgCpu.Save(defaultSize, defaultSize, avgCpuPath); err != nil {
if err = plotAvgCpu.Save(defaultPlotWidth, defaultPlotHeight, avgCpuPath); err != nil {
return err
}
}
plotAvgMemConsul, err := fr.GetColumn("avg_memory_mb_consul")
if err != nil {
@ -669,7 +731,29 @@ func plotAggAgg(fpath, outputPath, imageFormat, imageTitle string) error {
plotAvgMem.Title.Text = fmt.Sprintf("%s, Memory", imageTitle)
plotAvgMem.X.Label.Text = "second"
plotAvgMem.Y.Label.Text = "Memory(MB)"
if err = plotutil.AddLinePoints(
plotAvgMem.Legend.Top = true
plotAvgMemEtcd3Multi, err := fr.GetColumn("avg_memory_mb_etcd3multi")
if err == nil {
var plotAvgMemEtcd3MultiPoints plotter.XYs
plotAvgMemEtcd3MultiPoints, err = points(plotAvgMemEtcd3Multi)
if err != nil {
return err
}
if err = plotutil.AddLines(
plotAvgMem,
"consul", plotAvgMemConsulPoints,
"etcd3", plotAvgMemEtcd3Points,
strings.Replace("etcd3multi", "multi", "-"+multiTagTitle, -1), plotAvgMemEtcd3MultiPoints,
"etcd2", plotAvgMemEtcd2Points,
"zk", plotAvgMemZkPoints,
); err != nil {
return err
}
if err = plotAvgMem.Save(defaultPlotWidth, defaultPlotHeight, avgMemPath); err != nil {
return err
}
} else {
if err = plotutil.AddLines(
plotAvgMem,
"consul", plotAvgMemConsulPoints,
"etcd3", plotAvgMemEtcd3Points,
@ -678,9 +762,10 @@ func plotAggAgg(fpath, outputPath, imageFormat, imageTitle string) error {
); err != nil {
return err
}
if err = plotAvgMem.Save(defaultSize, defaultSize, avgMemPath); err != nil {
if err = plotAvgMem.Save(defaultPlotWidth, defaultPlotHeight, avgMemPath); err != nil {
return err
}
}
return nil
}

View File

@ -1,6 +1,16 @@
package analyze
import "github.com/spf13/cobra"
import (
"bytes"
"fmt"
"io/ioutil"
"os"
"path/filepath"
"sort"
"strings"
"github.com/spf13/cobra"
)
var (
ReadmeCommand = &cobra.Command{
@ -10,12 +20,81 @@ var (
}
readmeDir string
readmePrefacePath string
)
func init() {
ReadmeCommand.PersistentFlags().StringVarP(&readmeDir, "readme-dir", "d", "", "Directory path to generate README.")
ReadmeCommand.PersistentFlags().StringVarP(&readmePrefacePath, "readme-preface", "t", "", "README template file to preface.")
}
func ReadmeCommandFunc(cmd *cobra.Command, args []string) error {
bts, err := ioutil.ReadFile(readmePrefacePath)
if err != nil {
return err
}
buf := new(bytes.Buffer)
buf.WriteString(string(bts))
buf.WriteString("\n\n\n")
buf.WriteString("<br><br><hr>\n##### Results")
buf.WriteString("\n\n")
paths, err := walkExt(readmeDir, ".svg")
if err != nil {
return err
}
for _, path := range paths {
buf.WriteString(fmt.Sprintf("![%s](./%s)\n\n", filepath.Base(path), filepath.Base(path)))
}
return toFile(buf.String(), filepath.Join(readmeDir, "README.md"))
}
// walkExt returns all FileInfos with specific extension.
func walkExt(targetDir, ext string) ([]string, error) {
rmap := make(map[string]struct{})
visit := func(path string, f os.FileInfo, err error) error {
if f != nil {
if !f.IsDir() {
if filepath.Ext(path) == ext {
if !filepath.HasPrefix(path, ".") && !strings.Contains(path, "/.") {
wd, err := os.Getwd()
if err != nil {
return err
}
thepath := filepath.Join(wd, strings.Replace(path, wd, "", -1))
rmap[thepath] = struct{}{}
}
}
}
}
return nil
}
err := filepath.Walk(targetDir, visit)
if err != nil {
return nil, err
}
rs := []string{}
for k := range rmap {
rs = append(rs, k)
}
sort.Strings(rs)
return rs, nil
}
func toFile(txt, fpath string) error {
f, err := os.OpenFile(fpath, os.O_RDWR|os.O_TRUNC, 0777)
if err != nil {
f, err = os.Create(fpath)
if err != nil {
return err
}
}
defer f.Close()
if _, err := f.WriteString(txt); err != nil {
return err
}
return nil
}

View File

@ -0,0 +1,10 @@
#### Testing environment
- Google Cloud Compute Engine
- 8 vCPUs + 16GB Memory + 375GB local SSD (SCSI)
- Ubuntu 15.10
- Go 1.6 with etcd master branch as of testing date
- Java 8 with Zookeeper 3.4.8(current)
- Java(TM) SE Runtime Environment (build 1.8.0_74-b02)
- Java HotSpot(TM) 64-Bit Server VM (build 25.74-b02, mixed mode)
- Serialized read

View File

@ -0,0 +1,89 @@
#!/usr/bin/env bash
gsutil -m cp -R gs://bench-20160325/ .
#######################################################################
# create aggregated csv files
dbtester analyze --output=bench-20160325/bench-01-consul-aggregated.csv --bench-file-path=bench-20160325/bench-01-consul-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-01-consul-1-monitor.csv,bench-20160325/bench-01-consul-2-monitor.csv,bench-20160325/bench-01-consul-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-01-etcd-aggregated.csv --bench-file-path=bench-20160325/bench-01-etcd-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-01-etcd-1-monitor.csv,bench-20160325/bench-01-etcd-2-monitor.csv,bench-20160325/bench-01-etcd-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-01-etcd2-aggregated.csv --bench-file-path=bench-20160325/bench-01-etcd2-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-01-etcd2-1-monitor.csv,bench-20160325/bench-01-etcd2-2-monitor.csv,bench-20160325/bench-01-etcd2-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-01-zk-aggregated.csv --bench-file-path=bench-20160325/bench-01-zk-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-01-zk-1-monitor.csv,bench-20160325/bench-01-zk-2-monitor.csv,bench-20160325/bench-01-zk-3-monitor.csv
# create agg/agg
dbtester analyze --output=bench-20160325/bench-01-all-aggregated.csv --aggregated-file-paths=bench-20160325/bench-01-consul-aggregated.csv,bench-20160325/bench-01-etcd-aggregated.csv,bench-20160325/bench-01-etcd2-aggregated.csv,bench-20160325/bench-01-zk-aggregated.csv
# plot
IMAGE_TITLE="Write 300K keys, 1 client, key 64 bytes, value 256 bytes"
dbtester analyze --output=bench-20160325/bench-01-plot --image-format=png --file-to-plot=bench-20160325/bench-01-all-aggregated.csv --image-title="$(echo $IMAGE_TITLE)"
dbtester analyze --output=bench-20160325/bench-01-plot --image-format=svg --file-to-plot=bench-20160325/bench-01-all-aggregated.csv --image-title="$(echo $IMAGE_TITLE)"
#######################################################################
# create aggregated csv files
dbtester analyze --output=bench-20160325/bench-02-consul-aggregated.csv --bench-file-path=bench-20160325/bench-02-consul-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-02-consul-1-monitor.csv,bench-20160325/bench-02-consul-2-monitor.csv,bench-20160325/bench-02-consul-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-02-etcd-aggregated.csv --bench-file-path=bench-20160325/bench-02-etcd-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-02-etcd-1-monitor.csv,bench-20160325/bench-02-etcd-2-monitor.csv,bench-20160325/bench-02-etcd-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-02-etcd2-aggregated.csv --bench-file-path=bench-20160325/bench-02-etcd2-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-02-etcd2-1-monitor.csv,bench-20160325/bench-02-etcd2-2-monitor.csv,bench-20160325/bench-02-etcd2-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-02-zk-aggregated.csv --bench-file-path=bench-20160325/bench-02-zk-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-02-zk-1-monitor.csv,bench-20160325/bench-02-zk-2-monitor.csv,bench-20160325/bench-02-zk-3-monitor.csv
# create agg/agg
dbtester analyze --output=bench-20160325/bench-02-all-aggregated.csv --aggregated-file-paths=bench-20160325/bench-02-consul-aggregated.csv,bench-20160325/bench-02-etcd-aggregated.csv,bench-20160325/bench-02-etcd2-aggregated.csv,bench-20160325/bench-02-zk-aggregated.csv
# plot
IMAGE_TITLE="Write 1M keys, 10 clients, key 64 bytes, value 256 bytes"
dbtester analyze --output=bench-20160325/bench-02-plot --image-format=png --file-to-plot=bench-20160325/bench-02-all-aggregated.csv --image-title="$(echo $IMAGE_TITLE)"
dbtester analyze --output=bench-20160325/bench-02-plot --image-format=svg --file-to-plot=bench-20160325/bench-02-all-aggregated.csv --image-title="$(echo $IMAGE_TITLE)"
#######################################################################
# create aggregated csv files
dbtester analyze --output=bench-20160325/bench-03-consul-aggregated.csv --bench-file-path=bench-20160325/bench-03-consul-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-03-consul-1-monitor.csv,bench-20160325/bench-03-consul-2-monitor.csv,bench-20160325/bench-03-consul-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-03-etcd-aggregated.csv --bench-file-path=bench-20160325/bench-03-etcd-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-03-etcd-1-monitor.csv,bench-20160325/bench-03-etcd-2-monitor.csv,bench-20160325/bench-03-etcd-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-03-etcd2-aggregated.csv --bench-file-path=bench-20160325/bench-03-etcd2-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-03-etcd2-1-monitor.csv,bench-20160325/bench-03-etcd2-2-monitor.csv,bench-20160325/bench-03-etcd2-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-03-zk-aggregated.csv --bench-file-path=bench-20160325/bench-03-zk-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-03-zk-1-monitor.csv,bench-20160325/bench-03-zk-2-monitor.csv,bench-20160325/bench-03-zk-3-monitor.csv
# create agg/agg
dbtester analyze --output=bench-20160325/bench-03-all-aggregated.csv --aggregated-file-paths=bench-20160325/bench-03-consul-aggregated.csv,bench-20160325/bench-03-etcd-aggregated.csv,bench-20160325/bench-03-etcd2-aggregated.csv,bench-20160325/bench-03-zk-aggregated.csv
# plot
IMAGE_TITLE="Write 3M keys, 500 clients, key 64 bytes, value 256 bytes"
dbtester analyze --output=bench-20160325/bench-03-plot --image-format=png --file-to-plot=bench-20160325/bench-03-all-aggregated.csv --image-title="$(echo $IMAGE_TITLE)"
dbtester analyze --output=bench-20160325/bench-03-plot --image-format=svg --file-to-plot=bench-20160325/bench-03-all-aggregated.csv --image-title="$(echo $IMAGE_TITLE)"
#######################################################################
# create aggregated csv files
dbtester analyze --output=bench-20160325/bench-04-consul-aggregated.csv --bench-file-path=bench-20160325/bench-04-consul-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-04-consul-1-monitor.csv,bench-20160325/bench-04-consul-2-monitor.csv,bench-20160325/bench-04-consul-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-04-etcd-aggregated.csv --bench-file-path=bench-20160325/bench-04-etcd-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-04-etcd-1-monitor.csv,bench-20160325/bench-04-etcd-2-monitor.csv,bench-20160325/bench-04-etcd-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-04-etcdmulti-aggregated.csv --bench-file-path=bench-20160325/bench-04-etcdmulti-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-04-etcdmulti-1-monitor.csv,bench-20160325/bench-04-etcdmulti-2-monitor.csv,bench-20160325/bench-04-etcdmulti-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-04-etcd2-aggregated.csv --bench-file-path=bench-20160325/bench-04-etcd2-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-04-etcd2-1-monitor.csv,bench-20160325/bench-04-etcd2-2-monitor.csv,bench-20160325/bench-04-etcd2-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-04-zk-aggregated.csv --bench-file-path=bench-20160325/bench-04-zk-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-04-zk-1-monitor.csv,bench-20160325/bench-04-zk-2-monitor.csv,bench-20160325/bench-04-zk-3-monitor.csv
# create agg/agg
dbtester analyze --output=bench-20160325/bench-04-all-aggregated.csv --aggregated-file-paths=bench-20160325/bench-04-consul-aggregated.csv,bench-20160325/bench-04-etcd-aggregated.csv,bench-20160325/bench-04-etcdmulti-aggregated.csv,bench-20160325/bench-04-etcd2-aggregated.csv,bench-20160325/bench-04-zk-aggregated.csv
# plot
IMAGE_TITLE="Write 3M keys, 1000 clients, key 64 bytes, value 256 bytes"
MULTI_TAG_TITLE="100-conns-1k-clients"
dbtester analyze --output=bench-20160325/bench-04-plot --image-format=png --file-to-plot=bench-20160325/bench-04-all-aggregated.csv --image-title="$(echo $IMAGE_TITLE)" --multi-tag-title="$(echo $MULTI_TAG_TITLE)"
dbtester analyze --output=bench-20160325/bench-04-plot --image-format=svg --file-to-plot=bench-20160325/bench-04-all-aggregated.csv --image-title="$(echo $IMAGE_TITLE)" --multi-tag-title="$(echo $MULTI_TAG_TITLE)"
#######################################################################
# create aggregated csv files
dbtester analyze --output=bench-20160325/bench-05-consul-aggregated.csv --bench-file-path=bench-20160325/bench-05-consul-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-05-consul-1-monitor.csv,bench-20160325/bench-05-consul-2-monitor.csv,bench-20160325/bench-05-consul-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-05-etcd-aggregated.csv --bench-file-path=bench-20160325/bench-05-etcd-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-05-etcd-1-monitor.csv,bench-20160325/bench-05-etcd-2-monitor.csv,bench-20160325/bench-05-etcd-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-05-etcd2-aggregated.csv --bench-file-path=bench-20160325/bench-05-etcd2-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-05-etcd2-1-monitor.csv,bench-20160325/bench-05-etcd2-2-monitor.csv,bench-20160325/bench-05-etcd2-3-monitor.csv
dbtester analyze --output=bench-20160325/bench-05-zk-aggregated.csv --bench-file-path=bench-20160325/bench-05-zk-timeseries.csv --monitor-data-file-paths=bench-20160325/bench-05-zk-1-monitor.csv,bench-20160325/bench-05-zk-2-monitor.csv,bench-20160325/bench-05-zk-3-monitor.csv
# create agg/agg
dbtester analyze --output=bench-20160325/bench-05-all-aggregated.csv --aggregated-file-paths=bench-20160325/bench-05-consul-aggregated.csv,bench-20160325/bench-05-etcd-aggregated.csv,bench-20160325/bench-05-etcd2-aggregated.csv,bench-20160325/bench-05-zk-aggregated.csv
# plot
IMAGE_TITLE="Read 1M keys, 1 client, key 64 bytes, value 1 kb"
dbtester analyze --output=bench-20160325/bench-05-plot --image-format=png --file-to-plot=bench-20160325/bench-05-all-aggregated.csv --image-title="$(echo $IMAGE_TITLE)"
dbtester analyze --output=bench-20160325/bench-05-plot --image-format=svg --file-to-plot=bench-20160325/bench-05-all-aggregated.csv --image-title="$(echo $IMAGE_TITLE)"
#######################################################################
# generate README
dbtester readme --readme-dir=bench-20160325 --readme-preface=README_template

View File

@ -0,0 +1,56 @@
#### Testing environment
- Google Cloud Compute Engine
- 8 vCPUs + 16GB Memory + 375GB local SSD (SCSI)
- Ubuntu 15.10
- Go 1.6 with etcd master branch as of testing date
- Java 8 with Zookeeper 3.4.8(current)
- Java(TM) SE Runtime Environment (build 1.8.0_74-b02)
- Java HotSpot(TM) 64-Bit Server VM (build 25.74-b02, mixed mode)
- Serialized read
<br><br><hr>
##### Results
![bench-01-plot-avg-cpu.svg](./bench-01-plot-avg-cpu.svg)
![bench-01-plot-avg-latency-ms.svg](./bench-01-plot-avg-latency-ms.svg)
![bench-01-plot-avg-mem.svg](./bench-01-plot-avg-mem.svg)
![bench-01-plot-throughput.svg](./bench-01-plot-throughput.svg)
![bench-02-plot-avg-cpu.svg](./bench-02-plot-avg-cpu.svg)
![bench-02-plot-avg-latency-ms.svg](./bench-02-plot-avg-latency-ms.svg)
![bench-02-plot-avg-mem.svg](./bench-02-plot-avg-mem.svg)
![bench-02-plot-throughput.svg](./bench-02-plot-throughput.svg)
![bench-03-plot-avg-cpu.svg](./bench-03-plot-avg-cpu.svg)
![bench-03-plot-avg-latency-ms.svg](./bench-03-plot-avg-latency-ms.svg)
![bench-03-plot-avg-mem.svg](./bench-03-plot-avg-mem.svg)
![bench-03-plot-throughput.svg](./bench-03-plot-throughput.svg)
![bench-04-plot-avg-cpu.svg](./bench-04-plot-avg-cpu.svg)
![bench-04-plot-avg-latency-ms.svg](./bench-04-plot-avg-latency-ms.svg)
![bench-04-plot-avg-mem.svg](./bench-04-plot-avg-mem.svg)
![bench-04-plot-throughput.svg](./bench-04-plot-throughput.svg)
![bench-05-plot-avg-cpu.svg](./bench-05-plot-avg-cpu.svg)
![bench-05-plot-avg-latency-ms.svg](./bench-05-plot-avg-latency-ms.svg)
![bench-05-plot-avg-mem.svg](./bench-05-plot-avg-mem.svg)
![bench-05-plot-throughput.svg](./bench-05-plot-throughput.svg)

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T17:42:01Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/consul agent -server -data-dir /mnt/ssd0/data.consul -bind 10.240.0.12 -client 10.240.0.12 -bootstrap-expect 3","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/consul [PID: 23459]","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"Consul monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T17:43:42Z"}
{"level":"info","msg":"Stopping Consul [PID: 23459]","time":"2016-03-25T19:22:49Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/consul [PID: 23459]","time":"2016-03-25T19:22:49Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T19:22:49Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T19:22:49Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-consul-1-database.log","time":"2016-03-25T19:22:49Z"}
{"level":"info","msg":"/home/gyuho/go/bin/consul cmd.Wait returned exit status 1","time":"2016-03-25T19:22:49Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-consul-1-monitor.csv","time":"2016-03-25T19:22:51Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-consul-1-agent.log","time":"2016-03-25T19:22:51Z"}

View File

@ -0,0 +1,190 @@
==> WARNING: Expect Mode enabled, expecting 3 servers
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Consul agent running!
Node name: 'bench-01-consul-1'
Datacenter: 'dc1'
Server: true (bootstrap: false)
Client Addr: 10.240.0.12 (HTTP: 8500, HTTPS: -1, DNS: 8600, RPC: 8400)
Cluster Addr: 10.240.0.12 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
Atlas: <disabled>
==> Log data will now stream in as it occurs:
2016/03/25 17:43:42 [INFO] raft: Node at 10.240.0.12:8300 [Follower] entering Follower state
2016/03/25 17:43:42 [INFO] serf: EventMemberJoin: bench-01-consul-1 10.240.0.12
2016/03/25 17:43:42 [INFO] serf: EventMemberJoin: bench-01-consul-1.dc1 10.240.0.12
2016/03/25 17:43:42 [INFO] consul: adding LAN server bench-01-consul-1 (Addr: 10.240.0.12:8300) (DC: dc1)
2016/03/25 17:43:42 [INFO] consul: adding WAN server bench-01-consul-1.dc1 (Addr: 10.240.0.12:8300) (DC: dc1)
2016/03/25 17:43:42 [ERR] agent: failed to sync remote state: No cluster leader
2016/03/25 17:43:43 [INFO] serf: EventMemberJoin: bench-01-consul-2 10.240.0.13
2016/03/25 17:43:43 [INFO] consul: adding LAN server bench-01-consul-2 (Addr: 10.240.0.13:8300) (DC: dc1)
2016/03/25 17:43:44 [WARN] raft: EnableSingleNode disabled, and no known peers. Aborting election.
2016/03/25 17:43:44 [INFO] serf: EventMemberJoin: bench-01-consul-3 10.240.0.14
2016/03/25 17:43:44 [INFO] consul: adding LAN server bench-01-consul-3 (Addr: 10.240.0.14:8300) (DC: dc1)
2016/03/25 17:43:44 [INFO] consul: Attempting bootstrap with nodes: [10.240.0.12:8300 10.240.0.13:8300 10.240.0.14:8300]
2016/03/25 17:43:45 [WARN] raft: Heartbeat timeout reached, starting election
2016/03/25 17:43:45 [INFO] raft: Node at 10.240.0.12:8300 [Candidate] entering Candidate state
2016/03/25 17:43:45 [WARN] raft: Remote peer 10.240.0.14:8300 does not have local node 10.240.0.12:8300 as a peer
2016/03/25 17:43:45 [INFO] raft: Election won. Tally: 2
2016/03/25 17:43:45 [INFO] raft: Node at 10.240.0.12:8300 [Leader] entering Leader state
2016/03/25 17:43:45 [INFO] consul: cluster leadership acquired
2016/03/25 17:43:45 [INFO] consul: New leader elected: bench-01-consul-1
2016/03/25 17:43:45 [INFO] raft: pipelining replication to peer 10.240.0.14:8300
2016/03/25 17:43:45 [WARN] raft: Remote peer 10.240.0.13:8300 does not have local node 10.240.0.12:8300 as a peer
2016/03/25 17:43:45 [INFO] raft: pipelining replication to peer 10.240.0.13:8300
2016/03/25 17:43:45 [INFO] consul: member 'bench-01-consul-1' joined, marking health alive
2016/03/25 17:43:45 [INFO] consul: member 'bench-01-consul-2' joined, marking health alive
2016/03/25 17:43:45 [INFO] consul: member 'bench-01-consul-3' joined, marking health alive
2016/03/25 17:43:47 [INFO] agent: Synced service 'consul'
2016/03/25 17:46:15 [INFO] consul.fsm: snapshot created in 23.619µs
2016/03/25 17:46:15 [INFO] raft: Starting snapshot up to 8814
2016/03/25 17:46:15 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-8814-1458927975942.tmp
2016/03/25 17:46:15 [INFO] raft: Snapshot to 8814 complete
2016/03/25 17:49:40 [INFO] consul.fsm: snapshot created in 17.078µs
2016/03/25 17:49:40 [INFO] raft: Starting snapshot up to 22078
2016/03/25 17:49:40 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-22078-1458928180678.tmp
2016/03/25 17:49:40 [INFO] raft: Compacting logs from 1 to 11848
2016/03/25 17:49:40 [INFO] raft: Snapshot to 22078 complete
2016/03/25 17:53:36 [INFO] consul.fsm: snapshot created in 41.226µs
2016/03/25 17:53:36 [INFO] raft: Starting snapshot up to 38046
2016/03/25 17:53:36 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-38046-1458928416512.tmp
2016/03/25 17:53:36 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-8814-1458927975942
2016/03/25 17:53:36 [INFO] raft: Compacting logs from 11849 to 27821
2016/03/25 17:53:36 [INFO] raft: Snapshot to 38046 complete
2016/03/25 17:56:58 [INFO] consul.fsm: snapshot created in 17.613µs
2016/03/25 17:56:58 [INFO] raft: Starting snapshot up to 51431
2016/03/25 17:56:58 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-51431-1458928618427.tmp
2016/03/25 17:56:58 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-22078-1458928180678
2016/03/25 17:56:58 [INFO] raft: Compacting logs from 27822 to 41209
2016/03/25 17:56:58 [INFO] raft: Snapshot to 51431 complete
2016/03/25 17:59:28 [INFO] consul.fsm: snapshot created in 21.607µs
2016/03/25 17:59:28 [INFO] raft: Starting snapshot up to 61210
2016/03/25 17:59:28 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-61210-1458928768890.tmp
2016/03/25 17:59:29 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-38046-1458928416512
2016/03/25 17:59:29 [INFO] raft: Compacting logs from 41210 to 50992
2016/03/25 17:59:29 [INFO] raft: Snapshot to 61210 complete
2016/03/25 18:03:09 [INFO] consul.fsm: snapshot created in 22.683µs
2016/03/25 18:03:09 [INFO] raft: Starting snapshot up to 75675
2016/03/25 18:03:09 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-75675-1458928989669.tmp
2016/03/25 18:03:10 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-51431-1458928618427
2016/03/25 18:03:10 [INFO] raft: Compacting logs from 50993 to 65469
2016/03/25 18:03:10 [INFO] raft: Snapshot to 75675 complete
2016/03/25 18:06:48 [INFO] consul.fsm: snapshot created in 18.494µs
2016/03/25 18:06:48 [INFO] raft: Starting snapshot up to 90211
2016/03/25 18:06:48 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-90211-1458929208380.tmp
2016/03/25 18:06:48 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-61210-1458928768890
2016/03/25 18:06:48 [INFO] raft: Compacting logs from 65470 to 80005
2016/03/25 18:06:48 [INFO] raft: Snapshot to 90211 complete
2016/03/25 18:10:26 [INFO] consul.fsm: snapshot created in 32.052µs
2016/03/25 18:10:26 [INFO] raft: Starting snapshot up to 104555
2016/03/25 18:10:26 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-104555-1458929426672.tmp
2016/03/25 18:10:27 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-75675-1458928989669
2016/03/25 18:10:27 [INFO] raft: Compacting logs from 80006 to 94355
2016/03/25 18:10:27 [INFO] raft: Snapshot to 104555 complete
2016/03/25 18:14:00 [INFO] consul.fsm: snapshot created in 18.347µs
2016/03/25 18:14:00 [INFO] raft: Starting snapshot up to 118476
2016/03/25 18:14:00 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-118476-1458929640937.tmp
2016/03/25 18:14:01 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-90211-1458929208380
2016/03/25 18:14:01 [INFO] raft: Compacting logs from 94356 to 108281
2016/03/25 18:14:01 [INFO] raft: Snapshot to 118476 complete
2016/03/25 18:16:28 [INFO] consul.fsm: snapshot created in 28.66µs
2016/03/25 18:16:28 [INFO] raft: Starting snapshot up to 127981
2016/03/25 18:16:28 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-127981-1458929788127.tmp
2016/03/25 18:16:28 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-104555-1458929426672
2016/03/25 18:16:28 [INFO] raft: Compacting logs from 108282 to 117787
2016/03/25 18:16:28 [INFO] raft: Snapshot to 127981 complete
2016/03/25 18:20:16 [INFO] consul.fsm: snapshot created in 17.943µs
2016/03/25 18:20:16 [INFO] raft: Starting snapshot up to 143013
2016/03/25 18:20:16 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-143013-1458930016746.tmp
2016/03/25 18:20:17 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-118476-1458929640937
2016/03/25 18:20:17 [INFO] raft: Compacting logs from 117788 to 132828
2016/03/25 18:20:17 [INFO] raft: Snapshot to 143013 complete
2016/03/25 18:22:58 [INFO] consul.fsm: snapshot created in 17.323µs
2016/03/25 18:22:58 [INFO] raft: Starting snapshot up to 153440
2016/03/25 18:22:58 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-153440-1458930178268.tmp
2016/03/25 18:22:59 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-127981-1458929788127
2016/03/25 18:22:59 [INFO] raft: Compacting logs from 132829 to 143257
2016/03/25 18:22:59 [INFO] raft: Snapshot to 153440 complete
2016/03/25 18:26:27 [INFO] consul.fsm: snapshot created in 17.479µs
2016/03/25 18:26:27 [INFO] raft: Starting snapshot up to 167089
2016/03/25 18:26:27 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-167089-1458930387414.tmp
2016/03/25 18:26:28 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-143013-1458930016746
2016/03/25 18:26:28 [INFO] raft: Compacting logs from 143258 to 156914
2016/03/25 18:26:28 [INFO] raft: Snapshot to 167089 complete
2016/03/25 18:29:44 [INFO] consul.fsm: snapshot created in 17.314µs
2016/03/25 18:29:44 [INFO] raft: Starting snapshot up to 179994
2016/03/25 18:29:44 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-179994-1458930584868.tmp
2016/03/25 18:29:45 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-153440-1458930178268
2016/03/25 18:29:45 [INFO] raft: Compacting logs from 156915 to 169822
2016/03/25 18:29:45 [INFO] raft: Snapshot to 179994 complete
2016/03/25 18:33:40 [INFO] consul.fsm: snapshot created in 21.059µs
2016/03/25 18:33:40 [INFO] raft: Starting snapshot up to 195561
2016/03/25 18:33:40 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-195561-1458930820845.tmp
2016/03/25 18:33:41 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-167089-1458930387414
2016/03/25 18:33:41 [INFO] raft: Compacting logs from 169823 to 185396
2016/03/25 18:33:42 [INFO] raft: Snapshot to 195561 complete
2016/03/25 18:36:23 [INFO] consul.fsm: snapshot created in 19.747µs
2016/03/25 18:36:23 [INFO] raft: Starting snapshot up to 206040
2016/03/25 18:36:23 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-206040-1458930983115.tmp
2016/03/25 18:36:24 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-179994-1458930584868
2016/03/25 18:36:24 [INFO] raft: Compacting logs from 185397 to 195877
2016/03/25 18:36:24 [INFO] raft: Snapshot to 206040 complete
2016/03/25 18:38:52 [INFO] consul.fsm: snapshot created in 17.311µs
2016/03/25 18:38:52 [INFO] raft: Starting snapshot up to 215800
2016/03/25 18:38:52 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-215800-1458931132284.tmp
2016/03/25 18:38:53 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-195561-1458930820845
2016/03/25 18:38:53 [INFO] raft: Compacting logs from 195878 to 205638
2016/03/25 18:38:53 [INFO] raft: Snapshot to 215800 complete
2016/03/25 18:42:51 [INFO] consul.fsm: snapshot created in 18.154µs
2016/03/25 18:42:51 [INFO] raft: Starting snapshot up to 231359
2016/03/25 18:42:51 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-231359-1458931371008.tmp
2016/03/25 18:42:52 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-206040-1458930983115
2016/03/25 18:42:52 [INFO] raft: Compacting logs from 205639 to 221205
2016/03/25 18:42:52 [INFO] raft: Snapshot to 231359 complete
2016/03/25 18:46:13 [INFO] consul.fsm: snapshot created in 17.695µs
2016/03/25 18:46:13 [INFO] raft: Starting snapshot up to 244683
2016/03/25 18:46:13 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-244683-1458931573085.tmp
2016/03/25 18:46:14 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-215800-1458931132284
2016/03/25 18:46:14 [INFO] raft: Compacting logs from 221206 to 234533
2016/03/25 18:46:14 [INFO] raft: Snapshot to 244683 complete
2016/03/25 18:48:46 [INFO] consul.fsm: snapshot created in 22.513µs
2016/03/25 18:48:46 [INFO] raft: Starting snapshot up to 254602
2016/03/25 18:48:46 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-254602-1458931726332.tmp
2016/03/25 18:48:47 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-231359-1458931371008
2016/03/25 18:48:47 [INFO] raft: Compacting logs from 234534 to 244457
2016/03/25 18:48:47 [INFO] raft: Snapshot to 254602 complete
2016/03/25 18:51:10 [INFO] consul.fsm: snapshot created in 18.457µs
2016/03/25 18:51:10 [INFO] raft: Starting snapshot up to 264024
2016/03/25 18:51:10 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-264024-1458931870373.tmp
2016/03/25 18:51:11 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-244683-1458931573085
2016/03/25 18:51:11 [INFO] raft: Compacting logs from 244458 to 253880
2016/03/25 18:51:11 [INFO] raft: Snapshot to 264024 complete
2016/03/25 18:54:18 [INFO] consul.fsm: snapshot created in 18.41µs
2016/03/25 18:54:18 [INFO] raft: Starting snapshot up to 276248
2016/03/25 18:54:18 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-276248-1458932058771.tmp
2016/03/25 18:54:20 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-254602-1458931726332
2016/03/25 18:54:20 [INFO] raft: Compacting logs from 253881 to 266114
2016/03/25 18:54:20 [INFO] raft: Snapshot to 276248 complete
2016/03/25 18:57:36 [INFO] consul.fsm: snapshot created in 18.282µs
2016/03/25 18:57:36 [INFO] raft: Starting snapshot up to 289292
2016/03/25 18:57:36 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-289292-1458932256951.tmp
2016/03/25 18:57:38 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-264024-1458931870373
2016/03/25 18:57:38 [INFO] raft: Compacting logs from 266115 to 279157
2016/03/25 18:57:38 [INFO] raft: Snapshot to 289292 complete
2016/03/25 19:00:36 [INFO] consul.fsm: snapshot created in 26.587µs
2016/03/25 19:00:36 [INFO] raft: Starting snapshot up to 300556
2016/03/25 19:00:36 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-300556-1458932436626.tmp
2016/03/25 19:00:38 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-276248-1458932058771
2016/03/25 19:00:38 [INFO] raft: Compacting logs from 279158 to 290317
2016/03/25 19:00:38 [INFO] raft: Snapshot to 300556 complete
==> Caught signal: terminated
2016/03/25 19:22:49 [INFO] agent: requesting shutdown
2016/03/25 19:22:49 [INFO] consul: shutting down server
2016/03/25 19:22:49 [WARN] serf: Shutdown without a Leave
2016/03/25 19:22:49 [ERR] dns: error starting tcp server: accept tcp 10.240.0.12:8600: use of closed network connection
2016/03/25 19:22:49 [WARN] serf: Shutdown without a Leave
2016/03/25 19:22:49 [INFO] raft: aborting pipeline replication to peer 10.240.0.14:8300
2016/03/25 19:22:49 [INFO] raft: aborting pipeline replication to peer 10.240.0.13:8300
2016/03/25 19:22:49 [INFO] agent: shutdown complete

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T17:42:30Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/consul agent -server -data-dir /mnt/ssd0/data.consul -bind 10.240.0.13 -client 10.240.0.13 -join 10.240.0.12","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/consul [PID: 23436]","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"Consul monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T17:43:43Z"}
{"level":"info","msg":"Stopping Consul [PID: 23436]","time":"2016-03-25T19:22:50Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/consul [PID: 23436]","time":"2016-03-25T19:22:50Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T19:22:50Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T19:22:50Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-consul-2-database.log","time":"2016-03-25T19:22:50Z"}
{"level":"info","msg":"/home/gyuho/go/bin/consul cmd.Wait returned exit status 1","time":"2016-03-25T19:22:50Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-consul-2-monitor.csv","time":"2016-03-25T19:22:51Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-consul-2-agent.log","time":"2016-03-25T19:22:52Z"}

View File

@ -0,0 +1,173 @@
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Joining cluster...
Join completed. Synced with 1 initial agents
==> Consul agent running!
Node name: 'bench-01-consul-2'
Datacenter: 'dc1'
Server: true (bootstrap: false)
Client Addr: 10.240.0.13 (HTTP: 8500, HTTPS: -1, DNS: 8600, RPC: 8400)
Cluster Addr: 10.240.0.13 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
Atlas: <disabled>
==> Log data will now stream in as it occurs:
2016/03/25 17:43:43 [INFO] raft: Node at 10.240.0.13:8300 [Follower] entering Follower state
2016/03/25 17:43:43 [INFO] serf: EventMemberJoin: bench-01-consul-2 10.240.0.13
2016/03/25 17:43:43 [INFO] serf: EventMemberJoin: bench-01-consul-2.dc1 10.240.0.13
2016/03/25 17:43:43 [INFO] consul: adding LAN server bench-01-consul-2 (Addr: 10.240.0.13:8300) (DC: dc1)
2016/03/25 17:43:43 [INFO] consul: adding WAN server bench-01-consul-2.dc1 (Addr: 10.240.0.13:8300) (DC: dc1)
2016/03/25 17:43:43 [INFO] agent: (LAN) joining: [10.240.0.12]
2016/03/25 17:43:43 [INFO] serf: EventMemberJoin: bench-01-consul-1 10.240.0.12
2016/03/25 17:43:43 [INFO] agent: (LAN) joined: 1 Err: <nil>
2016/03/25 17:43:43 [INFO] consul: adding LAN server bench-01-consul-1 (Addr: 10.240.0.12:8300) (DC: dc1)
2016/03/25 17:43:43 [ERR] agent: failed to sync remote state: No cluster leader
2016/03/25 17:43:45 [INFO] serf: EventMemberJoin: bench-01-consul-3 10.240.0.14
2016/03/25 17:43:45 [INFO] consul: adding LAN server bench-01-consul-3 (Addr: 10.240.0.14:8300) (DC: dc1)
2016/03/25 17:43:45 [WARN] raft: EnableSingleNode disabled, and no known peers. Aborting election.
2016/03/25 17:43:45 [INFO] consul: New leader elected: bench-01-consul-1
2016/03/25 17:43:47 [INFO] agent: Synced service 'consul'
2016/03/25 17:49:13 [INFO] consul.fsm: snapshot created in 21.991µs
2016/03/25 17:49:13 [INFO] raft: Starting snapshot up to 20274
2016/03/25 17:49:13 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-20274-1458928153304.tmp
2016/03/25 17:49:13 [INFO] raft: Compacting logs from 1 to 10043
2016/03/25 17:49:13 [INFO] raft: Snapshot to 20274 complete
2016/03/25 17:51:45 [INFO] consul.fsm: snapshot created in 17.185µs
2016/03/25 17:51:45 [INFO] raft: Starting snapshot up to 30569
2016/03/25 17:51:45 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-30569-1458928305425.tmp
2016/03/25 17:51:45 [INFO] raft: Compacting logs from 10044 to 20344
2016/03/25 17:51:45 [INFO] raft: Snapshot to 30569 complete
2016/03/25 17:55:36 [INFO] consul.fsm: snapshot created in 19.449µs
2016/03/25 17:55:36 [INFO] raft: Starting snapshot up to 46035
2016/03/25 17:55:36 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-46035-1458928536512.tmp
2016/03/25 17:55:36 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-20274-1458928153304
2016/03/25 17:55:36 [INFO] raft: Compacting logs from 20345 to 35814
2016/03/25 17:55:36 [INFO] raft: Snapshot to 46035 complete
2016/03/25 17:58:16 [INFO] consul.fsm: snapshot created in 24.393µs
2016/03/25 17:58:16 [INFO] raft: Starting snapshot up to 56470
2016/03/25 17:58:16 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-56470-1458928696351.tmp
2016/03/25 17:58:16 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-30569-1458928305425
2016/03/25 17:58:16 [INFO] raft: Compacting logs from 35815 to 46252
2016/03/25 17:58:16 [INFO] raft: Snapshot to 56470 complete
2016/03/25 18:01:08 [INFO] consul.fsm: snapshot created in 20.948µs
2016/03/25 18:01:08 [INFO] raft: Starting snapshot up to 67660
2016/03/25 18:01:08 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-67660-1458928868040.tmp
2016/03/25 18:01:08 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-46035-1458928536512
2016/03/25 18:01:08 [INFO] raft: Compacting logs from 46253 to 57447
2016/03/25 18:01:08 [INFO] raft: Snapshot to 67660 complete
2016/03/25 18:04:46 [INFO] consul.fsm: snapshot created in 18.38µs
2016/03/25 18:04:46 [INFO] raft: Starting snapshot up to 82139
2016/03/25 18:04:46 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-82139-1458929086719.tmp
2016/03/25 18:04:47 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-56470-1458928696351
2016/03/25 18:04:47 [INFO] raft: Compacting logs from 57448 to 71933
2016/03/25 18:04:47 [INFO] raft: Snapshot to 82139 complete
2016/03/25 18:07:21 [INFO] consul.fsm: snapshot created in 26.809µs
2016/03/25 18:07:21 [INFO] raft: Starting snapshot up to 92340
2016/03/25 18:07:21 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-92340-1458929241135.tmp
2016/03/25 18:07:21 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-67660-1458928868040
2016/03/25 18:07:21 [INFO] raft: Compacting logs from 71934 to 82136
2016/03/25 18:07:21 [INFO] raft: Snapshot to 92340 complete
2016/03/25 18:10:33 [INFO] consul.fsm: snapshot created in 17.155µs
2016/03/25 18:10:33 [INFO] raft: Starting snapshot up to 104983
2016/03/25 18:10:33 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-104983-1458929433371.tmp
2016/03/25 18:10:33 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-82139-1458929086719
2016/03/25 18:10:33 [INFO] raft: Compacting logs from 82137 to 94782
2016/03/25 18:10:33 [INFO] raft: Snapshot to 104983 complete
2016/03/25 18:14:54 [INFO] consul.fsm: snapshot created in 16.751µs
2016/03/25 18:14:54 [INFO] raft: Starting snapshot up to 121944
2016/03/25 18:14:54 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-121944-1458929694574.tmp
2016/03/25 18:14:55 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-92340-1458929241135
2016/03/25 18:14:55 [INFO] raft: Compacting logs from 94783 to 111754
2016/03/25 18:14:55 [INFO] raft: Snapshot to 121944 complete
2016/03/25 18:18:11 [INFO] consul.fsm: snapshot created in 21.156µs
2016/03/25 18:18:11 [INFO] raft: Starting snapshot up to 134719
2016/03/25 18:18:11 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-134719-1458929891422.tmp
2016/03/25 18:18:12 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-104983-1458929433371
2016/03/25 18:18:12 [INFO] raft: Compacting logs from 111755 to 124533
2016/03/25 18:18:12 [INFO] raft: Snapshot to 134719 complete
2016/03/25 18:20:51 [INFO] consul.fsm: snapshot created in 22.135µs
2016/03/25 18:20:51 [INFO] raft: Starting snapshot up to 145266
2016/03/25 18:20:51 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-145266-1458930051753.tmp
2016/03/25 18:20:52 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-121944-1458929694574
2016/03/25 18:20:52 [INFO] raft: Compacting logs from 124534 to 135083
2016/03/25 18:20:52 [INFO] raft: Snapshot to 145266 complete
2016/03/25 18:24:37 [INFO] consul.fsm: snapshot created in 19.343µs
2016/03/25 18:24:37 [INFO] raft: Starting snapshot up to 159871
2016/03/25 18:24:37 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-159871-1458930277901.tmp
2016/03/25 18:24:38 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-134719-1458929891422
2016/03/25 18:24:38 [INFO] raft: Compacting logs from 135084 to 149691
2016/03/25 18:24:38 [INFO] raft: Snapshot to 159871 complete
2016/03/25 18:27:43 [INFO] consul.fsm: snapshot created in 20.68µs
2016/03/25 18:27:43 [INFO] raft: Starting snapshot up to 171983
2016/03/25 18:27:43 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-171983-1458930463054.tmp
2016/03/25 18:27:44 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-145266-1458930051753
2016/03/25 18:27:44 [INFO] raft: Compacting logs from 149692 to 161807
2016/03/25 18:27:44 [INFO] raft: Snapshot to 171983 complete
2016/03/25 18:30:45 [INFO] consul.fsm: snapshot created in 21.208µs
2016/03/25 18:30:45 [INFO] raft: Starting snapshot up to 183922
2016/03/25 18:30:45 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-183922-1458930645873.tmp
2016/03/25 18:30:46 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-159871-1458930277901
2016/03/25 18:30:46 [INFO] raft: Compacting logs from 161808 to 173754
2016/03/25 18:30:46 [INFO] raft: Snapshot to 183922 complete
2016/03/25 18:33:39 [INFO] consul.fsm: snapshot created in 15.571µs
2016/03/25 18:33:39 [INFO] raft: Starting snapshot up to 195434
2016/03/25 18:33:39 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-195434-1458930819013.tmp
2016/03/25 18:33:40 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-171983-1458930463054
2016/03/25 18:33:40 [INFO] raft: Compacting logs from 173755 to 185271
2016/03/25 18:33:40 [INFO] raft: Snapshot to 195434 complete
2016/03/25 18:37:17 [INFO] consul.fsm: snapshot created in 18.6µs
2016/03/25 18:37:17 [INFO] raft: Starting snapshot up to 209619
2016/03/25 18:37:17 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-209619-1458931037739.tmp
2016/03/25 18:37:18 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-183922-1458930645873
2016/03/25 18:37:18 [INFO] raft: Compacting logs from 185272 to 199459
2016/03/25 18:37:18 [INFO] raft: Snapshot to 209619 complete
2016/03/25 18:39:56 [INFO] consul.fsm: snapshot created in 24.723µs
2016/03/25 18:39:56 [INFO] raft: Starting snapshot up to 219938
2016/03/25 18:39:56 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-219938-1458931196807.tmp
2016/03/25 18:39:58 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-195434-1458930819013
2016/03/25 18:39:58 [INFO] raft: Compacting logs from 199460 to 209781
2016/03/25 18:39:58 [INFO] raft: Snapshot to 219938 complete
2016/03/25 18:42:04 [INFO] consul.fsm: snapshot created in 18.157µs
2016/03/25 18:42:04 [INFO] raft: Starting snapshot up to 228252
2016/03/25 18:42:04 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-228252-1458931324439.tmp
2016/03/25 18:42:05 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-209619-1458931037739
2016/03/25 18:42:05 [INFO] raft: Compacting logs from 209782 to 218104
2016/03/25 18:42:05 [INFO] raft: Snapshot to 228252 complete
2016/03/25 18:46:03 [INFO] consul.fsm: snapshot created in 19.678µs
2016/03/25 18:46:03 [INFO] raft: Starting snapshot up to 244019
2016/03/25 18:46:03 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-244019-1458931563014.tmp
2016/03/25 18:46:04 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-219938-1458931196807
2016/03/25 18:46:04 [INFO] raft: Compacting logs from 218105 to 233877
2016/03/25 18:46:04 [INFO] raft: Snapshot to 244019 complete
2016/03/25 18:49:23 [INFO] consul.fsm: snapshot created in 18.131µs
2016/03/25 18:49:23 [INFO] raft: Starting snapshot up to 256999
2016/03/25 18:49:23 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-256999-1458931763509.tmp
2016/03/25 18:49:24 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-228252-1458931324439
2016/03/25 18:49:25 [INFO] raft: Compacting logs from 233878 to 246857
2016/03/25 18:49:25 [INFO] raft: Snapshot to 256999 complete
2016/03/25 18:52:20 [INFO] consul.fsm: snapshot created in 19.746µs
2016/03/25 18:52:20 [INFO] raft: Starting snapshot up to 268463
2016/03/25 18:52:20 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-268463-1458931940089.tmp
2016/03/25 18:52:21 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-244019-1458931563014
2016/03/25 18:52:21 [INFO] raft: Compacting logs from 246858 to 258326
2016/03/25 18:52:21 [INFO] raft: Snapshot to 268463 complete
2016/03/25 18:58:23 [INFO] consul.fsm: snapshot created in 16.625µs
2016/03/25 18:58:23 [INFO] raft: Starting snapshot up to 292253
2016/03/25 18:58:23 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-292253-1458932303329.tmp
2016/03/25 18:58:25 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-256999-1458931763509
2016/03/25 18:58:25 [INFO] raft: Compacting logs from 258327 to 282123
2016/03/25 18:58:25 [INFO] raft: Snapshot to 292253 complete
2016/03/25 19:01:11 [INFO] consul.fsm: snapshot created in 28.138µs
2016/03/25 19:01:11 [INFO] raft: Starting snapshot up to 300561
2016/03/25 19:01:11 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-300561-1458932471623.tmp
2016/03/25 19:01:13 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-268463-1458931940089
2016/03/25 19:01:13 [INFO] raft: Compacting logs from 282124 to 290321
2016/03/25 19:01:13 [INFO] raft: Snapshot to 300561 complete
==> Caught signal: terminated
2016/03/25 19:22:50 [INFO] agent: requesting shutdown
2016/03/25 19:22:50 [INFO] consul: shutting down server
2016/03/25 19:22:50 [WARN] serf: Shutdown without a Leave
2016/03/25 19:22:50 [ERR] dns: error starting tcp server: accept tcp 10.240.0.13:8600: use of closed network connection
2016/03/25 19:22:50 [WARN] serf: Shutdown without a Leave
2016/03/25 19:22:50 [INFO] agent: shutdown complete

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T17:43:11Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/consul agent -server -data-dir /mnt/ssd0/data.consul -bind 10.240.0.14 -client 10.240.0.14 -join 10.240.0.12","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/consul [PID: 23396]","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"Consul monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T17:43:44Z"}
{"level":"info","msg":"Stopping Consul [PID: 23396]","time":"2016-03-25T19:22:51Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/consul [PID: 23396]","time":"2016-03-25T19:22:51Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T19:22:51Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T19:22:51Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-consul-3-database.log","time":"2016-03-25T19:22:51Z"}
{"level":"info","msg":"/home/gyuho/go/bin/consul cmd.Wait returned exit status 1","time":"2016-03-25T19:22:51Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-consul-3-monitor.csv","time":"2016-03-25T19:22:52Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-consul-3-agent.log","time":"2016-03-25T19:22:54Z"}

View File

@ -0,0 +1,188 @@
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Joining cluster...
Join completed. Synced with 1 initial agents
==> Consul agent running!
Node name: 'bench-01-consul-3'
Datacenter: 'dc1'
Server: true (bootstrap: false)
Client Addr: 10.240.0.14 (HTTP: 8500, HTTPS: -1, DNS: 8600, RPC: 8400)
Cluster Addr: 10.240.0.14 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
Atlas: <disabled>
==> Log data will now stream in as it occurs:
2016/03/25 17:43:44 [INFO] raft: Node at 10.240.0.14:8300 [Follower] entering Follower state
2016/03/25 17:43:44 [INFO] serf: EventMemberJoin: bench-01-consul-3 10.240.0.14
2016/03/25 17:43:44 [INFO] consul: adding LAN server bench-01-consul-3 (Addr: 10.240.0.14:8300) (DC: dc1)
2016/03/25 17:43:44 [INFO] serf: EventMemberJoin: bench-01-consul-3.dc1 10.240.0.14
2016/03/25 17:43:44 [INFO] consul: adding WAN server bench-01-consul-3.dc1 (Addr: 10.240.0.14:8300) (DC: dc1)
2016/03/25 17:43:44 [INFO] agent: (LAN) joining: [10.240.0.12]
2016/03/25 17:43:44 [INFO] serf: EventMemberJoin: bench-01-consul-1 10.240.0.12
2016/03/25 17:43:44 [INFO] serf: EventMemberJoin: bench-01-consul-2 10.240.0.13
2016/03/25 17:43:44 [INFO] agent: (LAN) joined: 1 Err: <nil>
2016/03/25 17:43:44 [INFO] consul: adding LAN server bench-01-consul-1 (Addr: 10.240.0.12:8300) (DC: dc1)
2016/03/25 17:43:44 [INFO] consul: adding LAN server bench-01-consul-2 (Addr: 10.240.0.13:8300) (DC: dc1)
2016/03/25 17:43:44 [ERR] agent: failed to sync remote state: No cluster leader
2016/03/25 17:43:45 [INFO] consul: New leader elected: bench-01-consul-1
2016/03/25 17:43:46 [INFO] agent: Synced service 'consul'
2016/03/25 17:46:10 [INFO] consul.fsm: snapshot created in 21.45µs
2016/03/25 17:46:10 [INFO] raft: Starting snapshot up to 8481
2016/03/25 17:46:10 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-8481-1458927970784.tmp
2016/03/25 17:46:10 [INFO] raft: Snapshot to 8481 complete
2016/03/25 17:50:22 [INFO] consul.fsm: snapshot created in 19.191µs
2016/03/25 17:50:22 [INFO] raft: Starting snapshot up to 24923
2016/03/25 17:50:22 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-24923-1458928222842.tmp
2016/03/25 17:50:22 [INFO] raft: Compacting logs from 1 to 14693
2016/03/25 17:50:23 [INFO] raft: Snapshot to 24923 complete
2016/03/25 17:54:02 [INFO] consul.fsm: snapshot created in 19.501µs
2016/03/25 17:54:02 [INFO] raft: Starting snapshot up to 39786
2016/03/25 17:54:02 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-39786-1458928442060.tmp
2016/03/25 17:54:02 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-8481-1458927970784
2016/03/25 17:54:02 [INFO] raft: Compacting logs from 14694 to 29563
2016/03/25 17:54:02 [INFO] raft: Snapshot to 39786 complete
2016/03/25 17:56:47 [INFO] consul.fsm: snapshot created in 25.32µs
2016/03/25 17:56:47 [INFO] raft: Starting snapshot up to 50745
2016/03/25 17:56:47 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-50745-1458928607866.tmp
2016/03/25 17:56:48 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-24923-1458928222842
2016/03/25 17:56:48 [INFO] raft: Compacting logs from 29564 to 40528
2016/03/25 17:56:48 [INFO] raft: Snapshot to 50745 complete
2016/03/25 17:59:02 [INFO] consul.fsm: snapshot created in 17.381µs
2016/03/25 17:59:02 [INFO] raft: Starting snapshot up to 59475
2016/03/25 17:59:02 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-59475-1458928742343.tmp
2016/03/25 17:59:02 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-39786-1458928442060
2016/03/25 17:59:02 [INFO] raft: Compacting logs from 40529 to 49258
2016/03/25 17:59:02 [INFO] raft: Snapshot to 59475 complete
2016/03/25 18:01:34 [INFO] consul.fsm: snapshot created in 22.77µs
2016/03/25 18:01:34 [INFO] raft: Starting snapshot up to 69374
2016/03/25 18:01:34 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-69374-1458928894546.tmp
2016/03/25 18:01:34 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-50745-1458928607866
2016/03/25 18:01:34 [INFO] raft: Compacting logs from 49259 to 59163
2016/03/25 18:01:34 [INFO] raft: Snapshot to 69374 complete
2016/03/25 18:05:02 [INFO] consul.fsm: snapshot created in 28.492µs
2016/03/25 18:05:02 [INFO] raft: Starting snapshot up to 83176
2016/03/25 18:05:02 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-83176-1458929102217.tmp
2016/03/25 18:05:02 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-59475-1458928742343
2016/03/25 18:05:02 [INFO] raft: Compacting logs from 59164 to 72969
2016/03/25 18:05:02 [INFO] raft: Snapshot to 83176 complete
2016/03/25 18:08:10 [INFO] consul.fsm: snapshot created in 20.281µs
2016/03/25 18:08:10 [INFO] raft: Starting snapshot up to 95525
2016/03/25 18:08:10 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-95525-1458929290316.tmp
2016/03/25 18:08:10 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-69374-1458928894546
2016/03/25 18:08:10 [INFO] raft: Compacting logs from 72970 to 85324
2016/03/25 18:08:10 [INFO] raft: Snapshot to 95525 complete
2016/03/25 18:10:29 [INFO] consul.fsm: snapshot created in 24.723µs
2016/03/25 18:10:29 [INFO] raft: Starting snapshot up to 104742
2016/03/25 18:10:29 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-104742-1458929429587.tmp
2016/03/25 18:10:30 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-83176-1458929102217
2016/03/25 18:10:30 [INFO] raft: Compacting logs from 85325 to 94541
2016/03/25 18:10:30 [INFO] raft: Snapshot to 104742 complete
2016/03/25 18:12:42 [INFO] consul.fsm: snapshot created in 26.96µs
2016/03/25 18:12:42 [INFO] raft: Starting snapshot up to 113301
2016/03/25 18:12:42 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-113301-1458929562615.tmp
2016/03/25 18:12:43 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-95525-1458929290316
2016/03/25 18:12:43 [INFO] raft: Compacting logs from 94542 to 103103
2016/03/25 18:12:43 [INFO] raft: Snapshot to 113301 complete
2016/03/25 18:15:56 [INFO] consul.fsm: snapshot created in 18.749µs
2016/03/25 18:15:56 [INFO] raft: Starting snapshot up to 125954
2016/03/25 18:15:56 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-125954-1458929756676.tmp
2016/03/25 18:15:57 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-104742-1458929429587
2016/03/25 18:15:57 [INFO] raft: Compacting logs from 103104 to 115762
2016/03/25 18:15:57 [INFO] raft: Snapshot to 125954 complete
2016/03/25 18:19:07 [INFO] consul.fsm: snapshot created in 18.282µs
2016/03/25 18:19:07 [INFO] raft: Starting snapshot up to 138400
2016/03/25 18:19:07 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-138400-1458929947703.tmp
2016/03/25 18:19:08 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-113301-1458929562615
2016/03/25 18:19:08 [INFO] raft: Compacting logs from 115763 to 128218
2016/03/25 18:19:08 [INFO] raft: Snapshot to 138400 complete
2016/03/25 18:22:13 [INFO] consul.fsm: snapshot created in 37.51µs
2016/03/25 18:22:13 [INFO] raft: Starting snapshot up to 150514
2016/03/25 18:22:13 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-150514-1458930133066.tmp
2016/03/25 18:22:13 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-125954-1458929756676
2016/03/25 18:22:13 [INFO] raft: Compacting logs from 128219 to 140332
2016/03/25 18:22:13 [INFO] raft: Snapshot to 150514 complete
2016/03/25 18:24:20 [INFO] consul.fsm: snapshot created in 24.117µs
2016/03/25 18:24:20 [INFO] raft: Starting snapshot up to 158757
2016/03/25 18:24:20 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-158757-1458930260922.tmp
2016/03/25 18:24:21 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-138400-1458929947703
2016/03/25 18:24:21 [INFO] raft: Compacting logs from 140333 to 148581
2016/03/25 18:24:21 [INFO] raft: Snapshot to 158757 complete
2016/03/25 18:26:55 [INFO] consul.fsm: snapshot created in 29.453µs
2016/03/25 18:26:55 [INFO] raft: Starting snapshot up to 168924
2016/03/25 18:26:55 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-168924-1458930415676.tmp
2016/03/25 18:26:56 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-150514-1458930133066
2016/03/25 18:26:56 [INFO] raft: Compacting logs from 148582 to 158747
2016/03/25 18:26:56 [INFO] raft: Snapshot to 168924 complete
2016/03/25 18:29:50 [INFO] consul.fsm: snapshot created in 25.029µs
2016/03/25 18:29:50 [INFO] raft: Starting snapshot up to 180362
2016/03/25 18:29:50 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-180362-1458930590483.tmp
2016/03/25 18:29:51 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-158757-1458930260922
2016/03/25 18:29:51 [INFO] raft: Compacting logs from 158748 to 170189
2016/03/25 18:29:51 [INFO] raft: Snapshot to 180362 complete
2016/03/25 18:33:47 [INFO] consul.fsm: snapshot created in 22.149µs
2016/03/25 18:33:47 [INFO] raft: Starting snapshot up to 195950
2016/03/25 18:33:47 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-195950-1458930827019.tmp
2016/03/25 18:33:48 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-168924-1458930415676
2016/03/25 18:33:48 [INFO] raft: Compacting logs from 170190 to 185786
2016/03/25 18:33:48 [INFO] raft: Snapshot to 195950 complete
2016/03/25 18:37:34 [INFO] consul.fsm: snapshot created in 24.626µs
2016/03/25 18:37:34 [INFO] raft: Starting snapshot up to 210725
2016/03/25 18:37:34 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-210725-1458931054499.tmp
2016/03/25 18:37:35 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-180362-1458930590483
2016/03/25 18:37:35 [INFO] raft: Compacting logs from 185787 to 200569
2016/03/25 18:37:35 [INFO] raft: Snapshot to 210725 complete
2016/03/25 18:40:34 [INFO] consul.fsm: snapshot created in 32.3µs
2016/03/25 18:40:34 [INFO] raft: Starting snapshot up to 222390
2016/03/25 18:40:34 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-222390-1458931234451.tmp
2016/03/25 18:40:35 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-195950-1458930827019
2016/03/25 18:40:35 [INFO] raft: Compacting logs from 200570 to 212230
2016/03/25 18:40:35 [INFO] raft: Snapshot to 222390 complete
2016/03/25 18:44:24 [INFO] consul.fsm: snapshot created in 27.431µs
2016/03/25 18:44:24 [INFO] raft: Starting snapshot up to 237483
2016/03/25 18:44:24 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-237483-1458931464884.tmp
2016/03/25 18:44:26 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-210725-1458931054499
2016/03/25 18:44:26 [INFO] raft: Compacting logs from 212231 to 227338
2016/03/25 18:44:26 [INFO] raft: Snapshot to 237483 complete
2016/03/25 18:47:25 [INFO] consul.fsm: snapshot created in 18.528µs
2016/03/25 18:47:25 [INFO] raft: Starting snapshot up to 249361
2016/03/25 18:47:25 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-249361-1458931645565.tmp
2016/03/25 18:47:27 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-222390-1458931234451
2016/03/25 18:47:27 [INFO] raft: Compacting logs from 227339 to 239221
2016/03/25 18:47:27 [INFO] raft: Snapshot to 249361 complete
2016/03/25 18:51:20 [INFO] consul.fsm: snapshot created in 29.914µs
2016/03/25 18:51:20 [INFO] raft: Starting snapshot up to 264675
2016/03/25 18:51:20 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-264675-1458931880488.tmp
2016/03/25 18:51:22 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-237483-1458931464884
2016/03/25 18:51:22 [INFO] raft: Compacting logs from 239222 to 254537
2016/03/25 18:51:22 [INFO] raft: Snapshot to 264675 complete
2016/03/25 18:54:22 [INFO] consul.fsm: snapshot created in 27.444µs
2016/03/25 18:54:22 [INFO] raft: Starting snapshot up to 276493
2016/03/25 18:54:22 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-276493-1458932062580.tmp
2016/03/25 18:54:24 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-249361-1458931645565
2016/03/25 18:54:24 [INFO] raft: Compacting logs from 254538 to 266355
2016/03/25 18:54:24 [INFO] raft: Snapshot to 276493 complete
2016/03/25 18:56:52 [INFO] consul.fsm: snapshot created in 34.832µs
2016/03/25 18:56:52 [INFO] raft: Starting snapshot up to 286357
2016/03/25 18:56:52 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-286357-1458932212719.tmp
2016/03/25 18:56:54 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-264675-1458931880488
2016/03/25 18:56:54 [INFO] raft: Compacting logs from 266356 to 276234
2016/03/25 18:56:54 [INFO] raft: Snapshot to 286357 complete
2016/03/25 18:59:22 [INFO] consul.fsm: snapshot created in 24.225µs
2016/03/25 18:59:22 [INFO] raft: Starting snapshot up to 296074
2016/03/25 18:59:22 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-296074-1458932362942.tmp
2016/03/25 18:59:24 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-276493-1458932062580
2016/03/25 18:59:24 [INFO] raft: Compacting logs from 276235 to 285946
2016/03/25 18:59:24 [INFO] raft: Snapshot to 296074 complete
2016/03/25 19:22:50 [INFO] memberlist: Suspect bench-01-consul-1 has failed, no acks received
2016/03/25 19:22:51 [WARN] raft: Heartbeat timeout reached, starting election
2016/03/25 19:22:51 [INFO] raft: Node at 10.240.0.14:8300 [Candidate] entering Candidate state
2016/03/25 19:22:51 [ERR] raft: Failed to make RequestVote RPC to 10.240.0.12:8300: dial tcp 10.240.0.12:8300: getsockopt: connection refused
2016/03/25 19:22:51 [ERR] raft: Failed to make RequestVote RPC to 10.240.0.13:8300: dial tcp 10.240.0.13:8300: getsockopt: connection refused
==> Caught signal: terminated
2016/03/25 19:22:51 [INFO] agent: requesting shutdown
2016/03/25 19:22:51 [INFO] consul: shutting down server
2016/03/25 19:22:51 [WARN] serf: Shutdown without a Leave
2016/03/25 19:22:51 [ERR] dns: error starting tcp server: accept tcp 10.240.0.14:8600: use of closed network connection
2016/03/25 19:22:51 [WARN] serf: Shutdown without a Leave
2016/03/25 19:22:51 [INFO] agent: shutdown complete

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:05:42Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-1 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.20:2379 --advertise-client-urls http://10.240.0.20:2379 --listen-peer-urls http://10.240.0.20:2380 --initial-advertise-peer-urls http://10.240.0.20:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.20:2380,etcd-2=http://10.240.0.21:2380,etcd-3=http://10.240.0.22:2380 --initial-cluster-state new","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23289]","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"etcd monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:08:46Z"}
{"level":"info","msg":"Stopping etcd [PID: 23289]","time":"2016-03-25T14:52:59Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23289]","time":"2016-03-25T14:52:59Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:52:59Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:52:59Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-etcd-1-database.log","time":"2016-03-25T14:52:59Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:52:59Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-etcd-1-monitor.csv","time":"2016-03-25T14:53:00Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-etcd-1-agent.log","time":"2016-03-25T14:53:01Z"}

View File

@ -0,0 +1,168 @@
2016-03-25 14:08:46.363298 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:08:46.363353 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:08:46.363360 I | etcdmain: Go Version: go1.6
2016-03-25 14:08:46.363366 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:08:46.363375 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:08:46.363427 I | etcdmain: listening for peers on http://10.240.0.20:2380
2016-03-25 14:08:46.363454 I | etcdmain: listening for client requests on 10.240.0.20:2379
2016-03-25 14:08:46.366250 I | etcdserver: name = etcd-1
2016-03-25 14:08:46.366265 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:08:46.366281 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:08:46.366288 I | etcdserver: heartbeat = 100ms
2016-03-25 14:08:46.366293 I | etcdserver: election = 1000ms
2016-03-25 14:08:46.366299 I | etcdserver: snapshot count = 10000
2016-03-25 14:08:46.366308 I | etcdserver: advertise client URLs = http://10.240.0.20:2379
2016-03-25 14:08:46.366316 I | etcdserver: initial advertise peer URLs = http://10.240.0.20:2380
2016-03-25 14:08:46.366330 I | etcdserver: initial cluster = etcd-1=http://10.240.0.20:2380,etcd-2=http://10.240.0.21:2380,etcd-3=http://10.240.0.22:2380
2016-03-25 14:08:46.377330 I | etcdserver: starting member 20722e46d7802cab in cluster 4ce5e214d26bfb9e
2016-03-25 14:08:46.377379 I | raft: 20722e46d7802cab became follower at term 0
2016-03-25 14:08:46.377393 I | raft: newRaft 20722e46d7802cab [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:08:46.377399 I | raft: 20722e46d7802cab became follower at term 1
2016-03-25 14:08:46.421888 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:08:46.422709 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:08:46.422728 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:08:46.422862 N | etcdmain: serving insecure client requests on 10.240.0.20:2379, this is strongly discouraged!
2016-03-25 14:08:46.422890 N | etcdserver: added local member 20722e46d7802cab [http://10.240.0.20:2380] to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:46.423432 N | etcdserver: added member 2087a91b2ccc6d93 [http://10.240.0.21:2380] to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:46.423525 N | etcdserver: added member 21d3c625100d5241 [http://10.240.0.22:2380] to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:46.677682 I | raft: 20722e46d7802cab is starting a new election at term 1
2016-03-25 14:08:46.677785 I | raft: 20722e46d7802cab became candidate at term 2
2016-03-25 14:08:46.677795 I | raft: 20722e46d7802cab received vote from 20722e46d7802cab at term 2
2016-03-25 14:08:46.677807 I | raft: 20722e46d7802cab [logterm: 1, index: 3] sent vote request to 2087a91b2ccc6d93 at term 2
2016-03-25 14:08:46.677816 I | raft: 20722e46d7802cab [logterm: 1, index: 3] sent vote request to 21d3c625100d5241 at term 2
2016-03-25 14:08:47.418152 I | rafthttp: the connection with 2087a91b2ccc6d93 became active
2016-03-25 14:08:47.678843 I | raft: 20722e46d7802cab [logterm: 1, index: 3, vote: 20722e46d7802cab] rejected vote from 2087a91b2ccc6d93 [logterm: 1, index: 3] at term 2
2016-03-25 14:08:48.077762 I | raft: 20722e46d7802cab is starting a new election at term 2
2016-03-25 14:08:48.077841 I | raft: 20722e46d7802cab became candidate at term 3
2016-03-25 14:08:48.077851 I | raft: 20722e46d7802cab received vote from 20722e46d7802cab at term 3
2016-03-25 14:08:48.077862 I | raft: 20722e46d7802cab [logterm: 1, index: 3] sent vote request to 2087a91b2ccc6d93 at term 3
2016-03-25 14:08:48.077872 I | raft: 20722e46d7802cab [logterm: 1, index: 3] sent vote request to 21d3c625100d5241 at term 3
2016-03-25 14:08:48.083151 I | raft: 20722e46d7802cab received vote from 2087a91b2ccc6d93 at term 3
2016-03-25 14:08:48.083170 I | raft: 20722e46d7802cab [quorum:2] has received 2 votes and 0 vote rejections
2016-03-25 14:08:48.083216 I | raft: 20722e46d7802cab became leader at term 3
2016-03-25 14:08:48.083244 I | raft: raft.node: 20722e46d7802cab elected leader 20722e46d7802cab at term 3
2016-03-25 14:08:48.091005 W | etcdserver: failed to reach the peerURL(http://10.240.0.22:2380) of member 21d3c625100d5241 (Get http://10.240.0.22:2380/version: dial tcp 10.240.0.22:2380: getsockopt: connection refused)
2016-03-25 14:08:48.091025 W | etcdserver: cannot get the version of member 21d3c625100d5241 (Get http://10.240.0.22:2380/version: dial tcp 10.240.0.22:2380: getsockopt: connection refused)
2016-03-25 14:08:48.091063 I | etcdserver: setting up the initial cluster version to 2.2
2016-03-25 14:08:48.094578 I | etcdserver: published {Name:etcd-1 ClientURLs:[http://10.240.0.20:2379]} to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:48.098130 N | etcdserver: set the initial cluster version to 2.2
2016-03-25 14:08:48.408724 I | rafthttp: the connection with 21d3c625100d5241 became active
2016-03-25 14:08:52.093002 I | etcdserver: updating the cluster version from 2.2 to 2.3
2016-03-25 14:08:52.095625 N | etcdserver: updated the cluster version from 2.2 to 2.3
2016-03-25 14:09:43.290668 I | etcdserver: start to snapshot (applied: 10001, lastsnap: 0)
2016-03-25 14:09:43.301779 I | etcdserver: saved snapshot at index 10001
2016-03-25 14:09:43.302012 I | etcdserver: compacted raft log at 5001
2016-03-25 14:10:30.196684 I | etcdserver: start to snapshot (applied: 20002, lastsnap: 10001)
2016-03-25 14:10:30.208800 I | etcdserver: saved snapshot at index 20002
2016-03-25 14:10:30.209070 I | etcdserver: compacted raft log at 15002
2016-03-25 14:11:16.726614 I | etcdserver: start to snapshot (applied: 30003, lastsnap: 20002)
2016-03-25 14:11:16.743370 I | etcdserver: saved snapshot at index 30003
2016-03-25 14:11:16.743559 I | etcdserver: compacted raft log at 25003
2016-03-25 14:12:04.482795 I | etcdserver: start to snapshot (applied: 40004, lastsnap: 30003)
2016-03-25 14:12:04.500995 I | etcdserver: saved snapshot at index 40004
2016-03-25 14:12:04.501234 I | etcdserver: compacted raft log at 35004
2016-03-25 14:12:51.885032 I | etcdserver: start to snapshot (applied: 50005, lastsnap: 40004)
2016-03-25 14:12:51.900861 I | etcdserver: saved snapshot at index 50005
2016-03-25 14:12:51.901054 I | etcdserver: compacted raft log at 45005
2016-03-25 14:13:39.583220 I | etcdserver: start to snapshot (applied: 60006, lastsnap: 50005)
2016-03-25 14:13:39.598199 I | etcdserver: saved snapshot at index 60006
2016-03-25 14:13:39.598439 I | etcdserver: compacted raft log at 55006
2016-03-25 14:13:46.424007 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002711.snap successfully
2016-03-25 14:14:26.151096 I | etcdserver: start to snapshot (applied: 70007, lastsnap: 60006)
2016-03-25 14:14:26.169801 I | etcdserver: saved snapshot at index 70007
2016-03-25 14:14:26.169995 I | etcdserver: compacted raft log at 65007
2016-03-25 14:14:46.424331 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e22.snap successfully
2016-03-25 14:15:13.905429 I | etcdserver: start to snapshot (applied: 80008, lastsnap: 70007)
2016-03-25 14:15:13.919325 I | etcdserver: saved snapshot at index 80008
2016-03-25 14:15:13.919568 I | etcdserver: compacted raft log at 75008
2016-03-25 14:15:16.424563 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007533.snap successfully
2016-03-25 14:16:01.256180 I | etcdserver: start to snapshot (applied: 90009, lastsnap: 80008)
2016-03-25 14:16:01.278521 I | etcdserver: saved snapshot at index 90009
2016-03-25 14:16:01.278721 I | etcdserver: compacted raft log at 85009
2016-03-25 14:16:16.424906 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c44.snap successfully
2016-03-25 14:16:49.658879 I | etcdserver: start to snapshot (applied: 100010, lastsnap: 90009)
2016-03-25 14:16:49.675249 I | etcdserver: saved snapshot at index 100010
2016-03-25 14:16:49.675485 I | etcdserver: compacted raft log at 95010
2016-03-25 14:17:16.425239 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c355.snap successfully
2016-03-25 14:17:37.734056 I | etcdserver: start to snapshot (applied: 110011, lastsnap: 100010)
2016-03-25 14:17:37.752075 I | etcdserver: saved snapshot at index 110011
2016-03-25 14:17:37.752319 I | etcdserver: compacted raft log at 105011
2016-03-25 14:17:46.425485 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea66.snap successfully
2016-03-25 14:18:24.509639 I | etcdserver: start to snapshot (applied: 120012, lastsnap: 110011)
2016-03-25 14:18:24.526638 I | etcdserver: saved snapshot at index 120012
2016-03-25 14:18:24.526838 I | etcdserver: compacted raft log at 115012
2016-03-25 14:18:46.425874 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000011177.snap successfully
2016-03-25 14:19:11.407271 I | etcdserver: start to snapshot (applied: 130013, lastsnap: 120012)
2016-03-25 14:19:11.421822 I | etcdserver: saved snapshot at index 130013
2016-03-25 14:19:11.421983 I | etcdserver: compacted raft log at 125013
2016-03-25 14:19:16.426139 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000013888.snap successfully
2016-03-25 14:19:59.171404 I | etcdserver: start to snapshot (applied: 140014, lastsnap: 130013)
2016-03-25 14:19:59.189512 I | etcdserver: saved snapshot at index 140014
2016-03-25 14:19:59.189654 I | etcdserver: compacted raft log at 135014
2016-03-25 14:20:16.426535 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015f99.snap successfully
2016-03-25 14:20:46.889295 I | etcdserver: start to snapshot (applied: 150015, lastsnap: 140014)
2016-03-25 14:20:46.904711 I | etcdserver: saved snapshot at index 150015
2016-03-25 14:20:46.904959 I | etcdserver: compacted raft log at 145015
2016-03-25 14:21:16.426923 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186aa.snap successfully
2016-03-25 14:21:34.661144 I | etcdserver: start to snapshot (applied: 160016, lastsnap: 150015)
2016-03-25 14:21:34.680098 I | etcdserver: saved snapshot at index 160016
2016-03-25 14:21:34.680288 I | etcdserver: compacted raft log at 155016
2016-03-25 14:21:36.787424 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-00000000000272da.wal is created
2016-03-25 14:21:46.427117 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001adbb.snap successfully
2016-03-25 14:22:22.543450 I | etcdserver: start to snapshot (applied: 170017, lastsnap: 160016)
2016-03-25 14:22:22.561416 I | etcdserver: saved snapshot at index 170017
2016-03-25 14:22:22.561536 I | etcdserver: compacted raft log at 165017
2016-03-25 14:22:46.427599 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4cc.snap successfully
2016-03-25 14:23:10.217398 I | etcdserver: start to snapshot (applied: 180018, lastsnap: 170017)
2016-03-25 14:23:10.229966 I | etcdserver: saved snapshot at index 180018
2016-03-25 14:23:10.230110 I | etcdserver: compacted raft log at 175018
2016-03-25 14:23:16.427833 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbdd.snap successfully
2016-03-25 14:23:58.351540 I | etcdserver: start to snapshot (applied: 190019, lastsnap: 180018)
2016-03-25 14:23:58.367266 I | etcdserver: saved snapshot at index 190019
2016-03-25 14:23:58.367383 I | etcdserver: compacted raft log at 185019
2016-03-25 14:24:16.428672 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000222ee.snap successfully
2016-03-25 14:24:46.637430 I | etcdserver: start to snapshot (applied: 200020, lastsnap: 190019)
2016-03-25 14:24:46.649265 I | etcdserver: saved snapshot at index 200020
2016-03-25 14:24:46.649468 I | etcdserver: compacted raft log at 195020
2016-03-25 14:25:16.429160 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000249ff.snap successfully
2016-03-25 14:25:35.374463 I | etcdserver: start to snapshot (applied: 210021, lastsnap: 200020)
2016-03-25 14:25:35.396919 I | etcdserver: saved snapshot at index 210021
2016-03-25 14:25:35.397071 I | etcdserver: compacted raft log at 205021
2016-03-25 14:25:46.429393 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027110.snap successfully
2016-03-25 14:26:23.715599 I | etcdserver: start to snapshot (applied: 220022, lastsnap: 210021)
2016-03-25 14:26:23.723575 I | etcdserver: saved snapshot at index 220022
2016-03-25 14:26:23.723709 I | etcdserver: compacted raft log at 215022
2016-03-25 14:26:46.429884 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000029821.snap successfully
2016-03-25 14:27:10.670028 I | etcdserver: start to snapshot (applied: 230023, lastsnap: 220022)
2016-03-25 14:27:10.683988 I | etcdserver: saved snapshot at index 230023
2016-03-25 14:27:10.684198 I | etcdserver: compacted raft log at 225023
2016-03-25 14:27:16.430123 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf32.snap successfully
2016-03-25 14:27:58.505964 I | etcdserver: start to snapshot (applied: 240024, lastsnap: 230023)
2016-03-25 14:27:58.521950 I | etcdserver: saved snapshot at index 240024
2016-03-25 14:27:58.522092 I | etcdserver: compacted raft log at 235024
2016-03-25 14:28:16.430668 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e643.snap successfully
2016-03-25 14:28:45.211905 I | etcdserver: start to snapshot (applied: 250025, lastsnap: 240024)
2016-03-25 14:28:45.227536 I | etcdserver: saved snapshot at index 250025
2016-03-25 14:28:45.227790 I | etcdserver: compacted raft log at 245025
2016-03-25 14:28:46.430933 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d54.snap successfully
2016-03-25 14:29:32.974523 I | etcdserver: start to snapshot (applied: 260026, lastsnap: 250025)
2016-03-25 14:29:32.984704 I | etcdserver: saved snapshot at index 260026
2016-03-25 14:29:32.984828 I | etcdserver: compacted raft log at 255026
2016-03-25 14:29:46.431385 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000033465.snap successfully
2016-03-25 14:30:20.702580 I | etcdserver: start to snapshot (applied: 270027, lastsnap: 260026)
2016-03-25 14:30:20.720025 I | etcdserver: saved snapshot at index 270027
2016-03-25 14:30:20.720250 I | etcdserver: compacted raft log at 265027
2016-03-25 14:30:46.431899 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035b76.snap successfully
2016-03-25 14:31:07.781463 I | etcdserver: start to snapshot (applied: 280028, lastsnap: 270027)
2016-03-25 14:31:07.797040 I | etcdserver: saved snapshot at index 280028
2016-03-25 14:31:07.797197 I | etcdserver: compacted raft log at 275028
2016-03-25 14:31:16.432138 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000038287.snap successfully
2016-03-25 14:31:55.186708 I | etcdserver: start to snapshot (applied: 290029, lastsnap: 280028)
2016-03-25 14:31:55.205775 I | etcdserver: saved snapshot at index 290029
2016-03-25 14:31:55.205895 I | etcdserver: compacted raft log at 285029
2016-03-25 14:32:16.432560 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a998.snap successfully
2016-03-25 14:32:43.093686 I | etcdserver: start to snapshot (applied: 300030, lastsnap: 290029)
2016-03-25 14:32:43.111792 I | etcdserver: saved snapshot at index 300030
2016-03-25 14:32:43.111912 I | etcdserver: compacted raft log at 295030
2016-03-25 14:32:46.432822 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0a9.snap successfully
2016-03-25 14:52:59.794891 N | osutil: received terminated signal, shutting down...

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:05:32Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-2 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.21:2379 --advertise-client-urls http://10.240.0.21:2379 --listen-peer-urls http://10.240.0.21:2380 --initial-advertise-peer-urls http://10.240.0.21:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.20:2380,etcd-2=http://10.240.0.21:2380,etcd-3=http://10.240.0.22:2380 --initial-cluster-state new","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23544]","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"etcd monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:08:47Z"}
{"level":"info","msg":"Stopping etcd [PID: 23544]","time":"2016-03-25T14:53:00Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23544]","time":"2016-03-25T14:53:00Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:53:00Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:53:00Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-etcd-2-database.log","time":"2016-03-25T14:53:00Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:53:00Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-etcd-2-monitor.csv","time":"2016-03-25T14:53:02Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-etcd-2-agent.log","time":"2016-03-25T14:53:03Z"}

View File

@ -0,0 +1,164 @@
2016-03-25 14:08:47.361630 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:08:47.361686 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:08:47.361693 I | etcdmain: Go Version: go1.6
2016-03-25 14:08:47.361699 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:08:47.361708 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:08:47.361766 I | etcdmain: listening for peers on http://10.240.0.21:2380
2016-03-25 14:08:47.361792 I | etcdmain: listening for client requests on 10.240.0.21:2379
2016-03-25 14:08:47.363517 I | etcdserver: name = etcd-2
2016-03-25 14:08:47.363531 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:08:47.363538 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:08:47.363544 I | etcdserver: heartbeat = 100ms
2016-03-25 14:08:47.363550 I | etcdserver: election = 1000ms
2016-03-25 14:08:47.363555 I | etcdserver: snapshot count = 10000
2016-03-25 14:08:47.363564 I | etcdserver: advertise client URLs = http://10.240.0.21:2379
2016-03-25 14:08:47.363571 I | etcdserver: initial advertise peer URLs = http://10.240.0.21:2380
2016-03-25 14:08:47.363584 I | etcdserver: initial cluster = etcd-1=http://10.240.0.20:2380,etcd-2=http://10.240.0.21:2380,etcd-3=http://10.240.0.22:2380
2016-03-25 14:08:47.370891 I | etcdserver: starting member 2087a91b2ccc6d93 in cluster 4ce5e214d26bfb9e
2016-03-25 14:08:47.370935 I | raft: 2087a91b2ccc6d93 became follower at term 0
2016-03-25 14:08:47.370948 I | raft: newRaft 2087a91b2ccc6d93 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:08:47.370954 I | raft: 2087a91b2ccc6d93 became follower at term 1
2016-03-25 14:08:47.415797 I | rafthttp: the connection with 20722e46d7802cab became active
2016-03-25 14:08:47.417320 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:08:47.418387 N | etcdserver: added member 20722e46d7802cab [http://10.240.0.20:2380] to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:47.418595 N | etcdserver: added local member 2087a91b2ccc6d93 [http://10.240.0.21:2380] to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:47.418725 N | etcdmain: serving insecure client requests on 10.240.0.21:2379, this is strongly discouraged!
2016-03-25 14:08:47.418769 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:08:47.418791 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:08:47.418910 N | etcdserver: added member 21d3c625100d5241 [http://10.240.0.22:2380] to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:47.671283 I | raft: 2087a91b2ccc6d93 is starting a new election at term 1
2016-03-25 14:08:47.671395 I | raft: 2087a91b2ccc6d93 became candidate at term 2
2016-03-25 14:08:47.671406 I | raft: 2087a91b2ccc6d93 received vote from 2087a91b2ccc6d93 at term 2
2016-03-25 14:08:47.671418 I | raft: 2087a91b2ccc6d93 [logterm: 1, index: 3] sent vote request to 20722e46d7802cab at term 2
2016-03-25 14:08:47.671428 I | raft: 2087a91b2ccc6d93 [logterm: 1, index: 3] sent vote request to 21d3c625100d5241 at term 2
2016-03-25 14:08:47.676444 I | raft: 2087a91b2ccc6d93 received vote rejection from 20722e46d7802cab at term 2
2016-03-25 14:08:47.676465 I | raft: 2087a91b2ccc6d93 [quorum:2] has received 1 votes and 1 vote rejections
2016-03-25 14:08:48.077923 I | raft: 2087a91b2ccc6d93 [term: 2] received a MsgVote message with higher term from 20722e46d7802cab [term: 3]
2016-03-25 14:08:48.077982 I | raft: 2087a91b2ccc6d93 became follower at term 3
2016-03-25 14:08:48.078002 I | raft: 2087a91b2ccc6d93 [logterm: 1, index: 3, vote: 0] voted for 20722e46d7802cab [logterm: 1, index: 3] at term 3
2016-03-25 14:08:48.080853 I | raft: raft.node: 2087a91b2ccc6d93 elected leader 20722e46d7802cab at term 3
2016-03-25 14:08:48.095720 N | etcdserver: set the initial cluster version to 2.2
2016-03-25 14:08:48.095806 I | etcdserver: published {Name:etcd-2 ClientURLs:[http://10.240.0.21:2379]} to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:48.409756 I | rafthttp: the connection with 21d3c625100d5241 became active
2016-03-25 14:08:52.093190 N | etcdserver: updated the cluster version from 2.2 to 2.3
2016-03-25 14:09:43.277716 I | etcdserver: start to snapshot (applied: 10001, lastsnap: 0)
2016-03-25 14:09:43.296834 I | etcdserver: saved snapshot at index 10001
2016-03-25 14:09:43.297126 I | etcdserver: compacted raft log at 5001
2016-03-25 14:10:30.193465 I | etcdserver: start to snapshot (applied: 20002, lastsnap: 10001)
2016-03-25 14:10:30.209388 I | etcdserver: saved snapshot at index 20002
2016-03-25 14:10:30.209606 I | etcdserver: compacted raft log at 15002
2016-03-25 14:11:16.723085 I | etcdserver: start to snapshot (applied: 30003, lastsnap: 20002)
2016-03-25 14:11:16.736362 I | etcdserver: saved snapshot at index 30003
2016-03-25 14:11:16.736598 I | etcdserver: compacted raft log at 25003
2016-03-25 14:12:04.479059 I | etcdserver: start to snapshot (applied: 40004, lastsnap: 30003)
2016-03-25 14:12:04.494489 I | etcdserver: saved snapshot at index 40004
2016-03-25 14:12:04.494616 I | etcdserver: compacted raft log at 35004
2016-03-25 14:12:51.880934 I | etcdserver: start to snapshot (applied: 50005, lastsnap: 40004)
2016-03-25 14:12:51.896270 I | etcdserver: saved snapshot at index 50005
2016-03-25 14:12:51.896541 I | etcdserver: compacted raft log at 45005
2016-03-25 14:13:39.584669 I | etcdserver: start to snapshot (applied: 60006, lastsnap: 50005)
2016-03-25 14:13:39.601645 I | etcdserver: saved snapshot at index 60006
2016-03-25 14:13:39.601858 I | etcdserver: compacted raft log at 55006
2016-03-25 14:13:47.419652 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002711.snap successfully
2016-03-25 14:14:26.146314 I | etcdserver: start to snapshot (applied: 70007, lastsnap: 60006)
2016-03-25 14:14:26.158263 I | etcdserver: saved snapshot at index 70007
2016-03-25 14:14:26.158459 I | etcdserver: compacted raft log at 65007
2016-03-25 14:14:47.419987 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e22.snap successfully
2016-03-25 14:15:13.895146 I | etcdserver: start to snapshot (applied: 80008, lastsnap: 70007)
2016-03-25 14:15:13.913869 I | etcdserver: saved snapshot at index 80008
2016-03-25 14:15:13.914111 I | etcdserver: compacted raft log at 75008
2016-03-25 14:15:17.420210 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007533.snap successfully
2016-03-25 14:16:01.250920 I | etcdserver: start to snapshot (applied: 90009, lastsnap: 80008)
2016-03-25 14:16:01.266672 I | etcdserver: saved snapshot at index 90009
2016-03-25 14:16:01.266811 I | etcdserver: compacted raft log at 85009
2016-03-25 14:16:17.420546 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c44.snap successfully
2016-03-25 14:16:49.653147 I | etcdserver: start to snapshot (applied: 100010, lastsnap: 90009)
2016-03-25 14:16:49.672658 I | etcdserver: saved snapshot at index 100010
2016-03-25 14:16:49.672941 I | etcdserver: compacted raft log at 95010
2016-03-25 14:17:17.421075 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c355.snap successfully
2016-03-25 14:17:37.728084 I | etcdserver: start to snapshot (applied: 110011, lastsnap: 100010)
2016-03-25 14:17:37.744980 I | etcdserver: saved snapshot at index 110011
2016-03-25 14:17:37.745232 I | etcdserver: compacted raft log at 105011
2016-03-25 14:17:47.421374 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea66.snap successfully
2016-03-25 14:18:24.503729 I | etcdserver: start to snapshot (applied: 120012, lastsnap: 110011)
2016-03-25 14:18:24.518115 I | etcdserver: saved snapshot at index 120012
2016-03-25 14:18:24.518332 I | etcdserver: compacted raft log at 115012
2016-03-25 14:18:47.421772 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000011177.snap successfully
2016-03-25 14:19:11.402119 I | etcdserver: start to snapshot (applied: 130013, lastsnap: 120012)
2016-03-25 14:19:11.418133 I | etcdserver: saved snapshot at index 130013
2016-03-25 14:19:11.418367 I | etcdserver: compacted raft log at 125013
2016-03-25 14:19:17.422051 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000013888.snap successfully
2016-03-25 14:19:59.167057 I | etcdserver: start to snapshot (applied: 140014, lastsnap: 130013)
2016-03-25 14:19:59.180767 I | etcdserver: saved snapshot at index 140014
2016-03-25 14:19:59.180978 I | etcdserver: compacted raft log at 135014
2016-03-25 14:20:17.422524 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015f99.snap successfully
2016-03-25 14:20:46.895223 I | etcdserver: start to snapshot (applied: 150015, lastsnap: 140014)
2016-03-25 14:20:46.910725 I | etcdserver: saved snapshot at index 150015
2016-03-25 14:20:46.910923 I | etcdserver: compacted raft log at 145015
2016-03-25 14:20:47.422772 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186aa.snap successfully
2016-03-25 14:21:34.657970 I | etcdserver: start to snapshot (applied: 160016, lastsnap: 150015)
2016-03-25 14:21:34.670341 I | etcdserver: saved snapshot at index 160016
2016-03-25 14:21:34.670472 I | etcdserver: compacted raft log at 155016
2016-03-25 14:21:38.432665 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-0000000000027447.wal is created
2016-03-25 14:21:47.423233 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001adbb.snap successfully
2016-03-25 14:22:22.540661 I | etcdserver: start to snapshot (applied: 170017, lastsnap: 160016)
2016-03-25 14:22:22.551495 I | etcdserver: saved snapshot at index 170017
2016-03-25 14:22:22.551651 I | etcdserver: compacted raft log at 165017
2016-03-25 14:22:47.423699 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4cc.snap successfully
2016-03-25 14:23:10.214995 I | etcdserver: start to snapshot (applied: 180018, lastsnap: 170017)
2016-03-25 14:23:10.230487 I | etcdserver: saved snapshot at index 180018
2016-03-25 14:23:10.230611 I | etcdserver: compacted raft log at 175018
2016-03-25 14:23:17.423926 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbdd.snap successfully
2016-03-25 14:23:58.349483 I | etcdserver: start to snapshot (applied: 190019, lastsnap: 180018)
2016-03-25 14:23:58.359091 I | etcdserver: saved snapshot at index 190019
2016-03-25 14:23:58.359305 I | etcdserver: compacted raft log at 185019
2016-03-25 14:24:17.424321 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000222ee.snap successfully
2016-03-25 14:24:46.635524 I | etcdserver: start to snapshot (applied: 200020, lastsnap: 190019)
2016-03-25 14:24:46.650474 I | etcdserver: saved snapshot at index 200020
2016-03-25 14:24:46.650731 I | etcdserver: compacted raft log at 195020
2016-03-25 14:24:47.424621 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000249ff.snap successfully
2016-03-25 14:25:35.373032 I | etcdserver: start to snapshot (applied: 210021, lastsnap: 200020)
2016-03-25 14:25:35.388946 I | etcdserver: saved snapshot at index 210021
2016-03-25 14:25:35.389074 I | etcdserver: compacted raft log at 205021
2016-03-25 14:25:47.425080 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027110.snap successfully
2016-03-25 14:26:23.714378 I | etcdserver: start to snapshot (applied: 220022, lastsnap: 210021)
2016-03-25 14:26:23.731611 I | etcdserver: saved snapshot at index 220022
2016-03-25 14:26:23.731769 I | etcdserver: compacted raft log at 215022
2016-03-25 14:26:47.425521 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000029821.snap successfully
2016-03-25 14:27:10.669163 I | etcdserver: start to snapshot (applied: 230023, lastsnap: 220022)
2016-03-25 14:27:10.681194 I | etcdserver: saved snapshot at index 230023
2016-03-25 14:27:10.681355 I | etcdserver: compacted raft log at 225023
2016-03-25 14:27:17.425815 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf32.snap successfully
2016-03-25 14:27:58.505497 I | etcdserver: start to snapshot (applied: 240024, lastsnap: 230023)
2016-03-25 14:27:58.520760 I | etcdserver: saved snapshot at index 240024
2016-03-25 14:27:58.520890 I | etcdserver: compacted raft log at 235024
2016-03-25 14:28:17.426238 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e643.snap successfully
2016-03-25 14:28:45.211550 I | etcdserver: start to snapshot (applied: 250025, lastsnap: 240024)
2016-03-25 14:28:45.225346 I | etcdserver: saved snapshot at index 250025
2016-03-25 14:28:45.225470 I | etcdserver: compacted raft log at 245025
2016-03-25 14:28:47.426448 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d54.snap successfully
2016-03-25 14:29:32.974230 I | etcdserver: start to snapshot (applied: 260026, lastsnap: 250025)
2016-03-25 14:29:32.984761 I | etcdserver: saved snapshot at index 260026
2016-03-25 14:29:32.984888 I | etcdserver: compacted raft log at 255026
2016-03-25 14:29:47.426871 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000033465.snap successfully
2016-03-25 14:30:20.693191 I | etcdserver: start to snapshot (applied: 270027, lastsnap: 260026)
2016-03-25 14:30:20.707086 I | etcdserver: saved snapshot at index 270027
2016-03-25 14:30:20.707278 I | etcdserver: compacted raft log at 265027
2016-03-25 14:30:47.428157 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035b76.snap successfully
2016-03-25 14:31:07.781429 I | etcdserver: start to snapshot (applied: 280028, lastsnap: 270027)
2016-03-25 14:31:07.800430 I | etcdserver: saved snapshot at index 280028
2016-03-25 14:31:07.800624 I | etcdserver: compacted raft log at 275028
2016-03-25 14:31:17.428405 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000038287.snap successfully
2016-03-25 14:31:55.188210 I | etcdserver: start to snapshot (applied: 290029, lastsnap: 280028)
2016-03-25 14:31:55.197378 I | etcdserver: saved snapshot at index 290029
2016-03-25 14:31:55.197618 I | etcdserver: compacted raft log at 285029
2016-03-25 14:32:17.428854 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a998.snap successfully
2016-03-25 14:32:43.093806 I | etcdserver: start to snapshot (applied: 300030, lastsnap: 290029)
2016-03-25 14:32:43.111344 I | etcdserver: saved snapshot at index 300030
2016-03-25 14:32:43.111464 I | etcdserver: compacted raft log at 295030
2016-03-25 14:32:47.429127 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0a9.snap successfully
2016-03-25 14:52:59.896730 E | rafthttp: failed to dial 20722e46d7802cab on stream MsgApp v2 (dial tcp 10.240.0.20:2380: getsockopt: connection refused)
2016-03-25 14:52:59.896781 I | rafthttp: the connection with 20722e46d7802cab became inactive
2016-03-25 14:53:00.794942 N | osutil: received terminated signal, shutting down...
2016-03-25 14:53:00.796709 E | rafthttp: failed to read 21d3c625100d5241 on stream MsgApp v2 (net/http: request canceled)
2016-03-25 14:53:00.796726 I | rafthttp: the connection with 21d3c625100d5241 became inactive

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:05:52Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-3 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.22:2379 --advertise-client-urls http://10.240.0.22:2379 --listen-peer-urls http://10.240.0.22:2380 --initial-advertise-peer-urls http://10.240.0.22:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.20:2380,etcd-2=http://10.240.0.21:2380,etcd-3=http://10.240.0.22:2380 --initial-cluster-state new","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23460]","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"etcd monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:08:48Z"}
{"level":"info","msg":"Stopping etcd [PID: 23460]","time":"2016-03-25T14:53:01Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23460]","time":"2016-03-25T14:53:01Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:53:01Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:53:01Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-etcd-3-database.log","time":"2016-03-25T14:53:01Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:53:01Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-etcd-3-monitor.csv","time":"2016-03-25T14:53:03Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-etcd-3-agent.log","time":"2016-03-25T14:53:05Z"}

View File

@ -0,0 +1,162 @@
2016-03-25 14:08:48.346719 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:08:48.346792 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:08:48.346800 I | etcdmain: Go Version: go1.6
2016-03-25 14:08:48.346806 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:08:48.346816 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:08:48.346918 I | etcdmain: listening for peers on http://10.240.0.22:2380
2016-03-25 14:08:48.346949 I | etcdmain: listening for client requests on 10.240.0.22:2379
2016-03-25 14:08:48.348558 I | etcdserver: name = etcd-3
2016-03-25 14:08:48.348576 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:08:48.348587 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:08:48.348595 I | etcdserver: heartbeat = 100ms
2016-03-25 14:08:48.348603 I | etcdserver: election = 1000ms
2016-03-25 14:08:48.348611 I | etcdserver: snapshot count = 10000
2016-03-25 14:08:48.348624 I | etcdserver: advertise client URLs = http://10.240.0.22:2379
2016-03-25 14:08:48.348634 I | etcdserver: initial advertise peer URLs = http://10.240.0.22:2380
2016-03-25 14:08:48.348653 I | etcdserver: initial cluster = etcd-1=http://10.240.0.20:2380,etcd-2=http://10.240.0.21:2380,etcd-3=http://10.240.0.22:2380
2016-03-25 14:08:48.357837 I | etcdserver: starting member 21d3c625100d5241 in cluster 4ce5e214d26bfb9e
2016-03-25 14:08:48.357895 I | raft: 21d3c625100d5241 became follower at term 0
2016-03-25 14:08:48.357918 I | raft: newRaft 21d3c625100d5241 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:08:48.357925 I | raft: 21d3c625100d5241 became follower at term 1
2016-03-25 14:08:48.390524 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:08:48.393870 I | rafthttp: the connection with 20722e46d7802cab became active
2016-03-25 14:08:48.393977 N | etcdserver: added member 20722e46d7802cab [http://10.240.0.20:2380] to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:48.394077 N | etcdserver: added member 2087a91b2ccc6d93 [http://10.240.0.21:2380] to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:48.394126 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:08:48.394140 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:08:48.394159 N | etcdserver: added local member 21d3c625100d5241 [http://10.240.0.22:2380] to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:48.394220 N | etcdmain: serving insecure client requests on 10.240.0.22:2379, this is strongly discouraged!
2016-03-25 14:08:48.394356 I | raft: 21d3c625100d5241 [term: 1] received a MsgHeartbeat message with higher term from 20722e46d7802cab [term: 3]
2016-03-25 14:08:48.394398 I | raft: 21d3c625100d5241 became follower at term 3
2016-03-25 14:08:48.394425 I | raft: raft.node: 21d3c625100d5241 elected leader 20722e46d7802cab at term 3
2016-03-25 14:08:48.394605 I | rafthttp: the connection with 2087a91b2ccc6d93 became active
2016-03-25 14:08:48.394770 N | etcdserver: set the initial cluster version to 2.2
2016-03-25 14:08:48.401626 I | etcdserver: published {Name:etcd-3 ClientURLs:[http://10.240.0.22:2379]} to cluster 4ce5e214d26bfb9e
2016-03-25 14:08:52.077760 N | etcdserver: updated the cluster version from 2.2 to 2.3
2016-03-25 14:09:43.260261 I | etcdserver: start to snapshot (applied: 10001, lastsnap: 0)
2016-03-25 14:09:43.279299 I | etcdserver: saved snapshot at index 10001
2016-03-25 14:09:43.279578 I | etcdserver: compacted raft log at 5001
2016-03-25 14:10:30.178812 I | etcdserver: start to snapshot (applied: 20002, lastsnap: 10001)
2016-03-25 14:10:30.199641 I | etcdserver: saved snapshot at index 20002
2016-03-25 14:10:30.199828 I | etcdserver: compacted raft log at 15002
2016-03-25 14:11:16.702261 I | etcdserver: start to snapshot (applied: 30003, lastsnap: 20002)
2016-03-25 14:11:16.717924 I | etcdserver: saved snapshot at index 30003
2016-03-25 14:11:16.718126 I | etcdserver: compacted raft log at 25003
2016-03-25 14:12:04.460328 I | etcdserver: start to snapshot (applied: 40004, lastsnap: 30003)
2016-03-25 14:12:04.472285 I | etcdserver: saved snapshot at index 40004
2016-03-25 14:12:04.472521 I | etcdserver: compacted raft log at 35004
2016-03-25 14:12:51.856260 I | etcdserver: start to snapshot (applied: 50005, lastsnap: 40004)
2016-03-25 14:12:51.868455 I | etcdserver: saved snapshot at index 50005
2016-03-25 14:12:51.868696 I | etcdserver: compacted raft log at 45005
2016-03-25 14:13:39.552355 I | etcdserver: start to snapshot (applied: 60006, lastsnap: 50005)
2016-03-25 14:13:39.569647 I | etcdserver: saved snapshot at index 60006
2016-03-25 14:13:39.569888 I | etcdserver: compacted raft log at 55006
2016-03-25 14:13:48.395118 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002711.snap successfully
2016-03-25 14:14:26.117959 I | etcdserver: start to snapshot (applied: 70007, lastsnap: 60006)
2016-03-25 14:14:26.136027 I | etcdserver: saved snapshot at index 70007
2016-03-25 14:14:26.136252 I | etcdserver: compacted raft log at 65007
2016-03-25 14:14:48.395521 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e22.snap successfully
2016-03-25 14:15:13.863309 I | etcdserver: start to snapshot (applied: 80008, lastsnap: 70007)
2016-03-25 14:15:13.869775 I | etcdserver: saved snapshot at index 80008
2016-03-25 14:15:13.869925 I | etcdserver: compacted raft log at 75008
2016-03-25 14:15:18.395729 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007533.snap successfully
2016-03-25 14:16:01.225768 I | etcdserver: start to snapshot (applied: 90009, lastsnap: 80008)
2016-03-25 14:16:01.238283 I | etcdserver: saved snapshot at index 90009
2016-03-25 14:16:01.238500 I | etcdserver: compacted raft log at 85009
2016-03-25 14:16:18.396094 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c44.snap successfully
2016-03-25 14:16:49.619294 I | etcdserver: start to snapshot (applied: 100010, lastsnap: 90009)
2016-03-25 14:16:49.635177 I | etcdserver: saved snapshot at index 100010
2016-03-25 14:16:49.635426 I | etcdserver: compacted raft log at 95010
2016-03-25 14:17:18.396411 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c355.snap successfully
2016-03-25 14:17:37.693453 I | etcdserver: start to snapshot (applied: 110011, lastsnap: 100010)
2016-03-25 14:17:37.717232 I | etcdserver: saved snapshot at index 110011
2016-03-25 14:17:37.717363 I | etcdserver: compacted raft log at 105011
2016-03-25 14:17:48.396616 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea66.snap successfully
2016-03-25 14:18:24.468704 I | etcdserver: start to snapshot (applied: 120012, lastsnap: 110011)
2016-03-25 14:18:24.483022 I | etcdserver: saved snapshot at index 120012
2016-03-25 14:18:24.483149 I | etcdserver: compacted raft log at 115012
2016-03-25 14:18:48.396924 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000011177.snap successfully
2016-03-25 14:19:11.373116 I | etcdserver: start to snapshot (applied: 130013, lastsnap: 120012)
2016-03-25 14:19:11.398609 I | etcdserver: saved snapshot at index 130013
2016-03-25 14:19:11.398795 I | etcdserver: compacted raft log at 125013
2016-03-25 14:19:18.397135 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000013888.snap successfully
2016-03-25 14:19:59.130401 I | etcdserver: start to snapshot (applied: 140014, lastsnap: 130013)
2016-03-25 14:19:59.146545 I | etcdserver: saved snapshot at index 140014
2016-03-25 14:19:59.146666 I | etcdserver: compacted raft log at 135014
2016-03-25 14:20:18.397595 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015f99.snap successfully
2016-03-25 14:20:46.848366 I | etcdserver: start to snapshot (applied: 150015, lastsnap: 140014)
2016-03-25 14:20:46.861057 I | etcdserver: saved snapshot at index 150015
2016-03-25 14:20:46.861268 I | etcdserver: compacted raft log at 145015
2016-03-25 14:20:48.397928 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186aa.snap successfully
2016-03-25 14:21:34.630223 I | etcdserver: start to snapshot (applied: 160016, lastsnap: 150015)
2016-03-25 14:21:34.649805 I | etcdserver: saved snapshot at index 160016
2016-03-25 14:21:34.649946 I | etcdserver: compacted raft log at 155016
2016-03-25 14:21:48.398493 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001adbb.snap successfully
2016-03-25 14:21:54.251617 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-0000000000028146.wal is created
2016-03-25 14:22:22.511767 I | etcdserver: start to snapshot (applied: 170017, lastsnap: 160016)
2016-03-25 14:22:22.524754 I | etcdserver: saved snapshot at index 170017
2016-03-25 14:22:22.524909 I | etcdserver: compacted raft log at 165017
2016-03-25 14:22:48.399029 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4cc.snap successfully
2016-03-25 14:23:10.178079 I | etcdserver: start to snapshot (applied: 180018, lastsnap: 170017)
2016-03-25 14:23:10.203892 I | etcdserver: saved snapshot at index 180018
2016-03-25 14:23:10.204108 I | etcdserver: compacted raft log at 175018
2016-03-25 14:23:18.399307 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbdd.snap successfully
2016-03-25 14:23:58.325654 I | etcdserver: start to snapshot (applied: 190019, lastsnap: 180018)
2016-03-25 14:23:58.337354 I | etcdserver: saved snapshot at index 190019
2016-03-25 14:23:58.337624 I | etcdserver: compacted raft log at 185019
2016-03-25 14:24:18.399879 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000222ee.snap successfully
2016-03-25 14:24:46.614580 I | etcdserver: start to snapshot (applied: 200020, lastsnap: 190019)
2016-03-25 14:24:46.623118 I | etcdserver: saved snapshot at index 200020
2016-03-25 14:24:46.623233 I | etcdserver: compacted raft log at 195020
2016-03-25 14:24:48.400189 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000249ff.snap successfully
2016-03-25 14:25:35.358976 I | etcdserver: start to snapshot (applied: 210021, lastsnap: 200020)
2016-03-25 14:25:35.369402 I | etcdserver: saved snapshot at index 210021
2016-03-25 14:25:35.369685 I | etcdserver: compacted raft log at 205021
2016-03-25 14:25:48.400638 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027110.snap successfully
2016-03-25 14:26:23.699455 I | etcdserver: start to snapshot (applied: 220022, lastsnap: 210021)
2016-03-25 14:26:23.718983 I | etcdserver: saved snapshot at index 220022
2016-03-25 14:26:23.719101 I | etcdserver: compacted raft log at 215022
2016-03-25 14:26:48.401027 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000029821.snap successfully
2016-03-25 14:27:10.657533 I | etcdserver: start to snapshot (applied: 230023, lastsnap: 220022)
2016-03-25 14:27:10.668191 I | etcdserver: saved snapshot at index 230023
2016-03-25 14:27:10.668310 I | etcdserver: compacted raft log at 225023
2016-03-25 14:27:18.401288 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf32.snap successfully
2016-03-25 14:27:58.503558 I | etcdserver: start to snapshot (applied: 240024, lastsnap: 230023)
2016-03-25 14:27:58.517831 I | etcdserver: saved snapshot at index 240024
2016-03-25 14:27:58.517944 I | etcdserver: compacted raft log at 235024
2016-03-25 14:28:18.401710 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e643.snap successfully
2016-03-25 14:28:45.209882 I | etcdserver: start to snapshot (applied: 250025, lastsnap: 240024)
2016-03-25 14:28:45.270342 I | etcdserver: saved snapshot at index 250025
2016-03-25 14:28:45.270470 I | etcdserver: compacted raft log at 245025
2016-03-25 14:28:48.401998 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d54.snap successfully
2016-03-25 14:29:32.979446 I | etcdserver: start to snapshot (applied: 260026, lastsnap: 250025)
2016-03-25 14:29:33.002304 I | etcdserver: saved snapshot at index 260026
2016-03-25 14:29:33.002425 I | etcdserver: compacted raft log at 255026
2016-03-25 14:29:48.402459 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000033465.snap successfully
2016-03-25 14:30:20.694808 I | etcdserver: start to snapshot (applied: 270027, lastsnap: 260026)
2016-03-25 14:30:20.712597 I | etcdserver: saved snapshot at index 270027
2016-03-25 14:30:20.712722 I | etcdserver: compacted raft log at 265027
2016-03-25 14:30:48.402948 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035b76.snap successfully
2016-03-25 14:31:07.780922 I | etcdserver: start to snapshot (applied: 280028, lastsnap: 270027)
2016-03-25 14:31:07.796841 I | etcdserver: saved snapshot at index 280028
2016-03-25 14:31:07.797057 I | etcdserver: compacted raft log at 275028
2016-03-25 14:31:18.403209 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000038287.snap successfully
2016-03-25 14:31:55.187981 I | etcdserver: start to snapshot (applied: 290029, lastsnap: 280028)
2016-03-25 14:31:55.203191 I | etcdserver: saved snapshot at index 290029
2016-03-25 14:31:55.203342 I | etcdserver: compacted raft log at 285029
2016-03-25 14:32:18.403653 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a998.snap successfully
2016-03-25 14:32:43.106451 I | etcdserver: start to snapshot (applied: 300030, lastsnap: 290029)
2016-03-25 14:32:43.113258 I | etcdserver: saved snapshot at index 300030
2016-03-25 14:32:43.113547 I | etcdserver: compacted raft log at 295030
2016-03-25 14:32:48.403898 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0a9.snap successfully
2016-03-25 14:52:59.912730 E | rafthttp: failed to dial 20722e46d7802cab on stream MsgApp v2 (dial tcp 10.240.0.20:2380: getsockopt: connection refused)
2016-03-25 14:52:59.912774 I | rafthttp: the connection with 20722e46d7802cab became inactive
2016-03-25 14:53:00.912250 E | rafthttp: failed to dial 2087a91b2ccc6d93 on stream MsgApp v2 (dial tcp 10.240.0.21:2380: getsockopt: connection refused)
2016-03-25 14:53:00.912315 I | rafthttp: the connection with 2087a91b2ccc6d93 became inactive
2016-03-25 14:53:01.158306 I | raft: 21d3c625100d5241 is starting a new election at term 3
2016-03-25 14:53:01.158337 I | raft: 21d3c625100d5241 became candidate at term 4
2016-03-25 14:53:01.158346 I | raft: 21d3c625100d5241 received vote from 21d3c625100d5241 at term 4
2016-03-25 14:53:01.158357 I | raft: 21d3c625100d5241 [logterm: 3, index: 305313] sent vote request to 20722e46d7802cab at term 4
2016-03-25 14:53:01.158366 I | raft: 21d3c625100d5241 [logterm: 3, index: 305313] sent vote request to 2087a91b2ccc6d93 at term 4
2016-03-25 14:53:01.158376 I | raft: raft.node: 21d3c625100d5241 lost leader 20722e46d7802cab at term 4
2016-03-25 14:53:01.810627 N | osutil: received terminated signal, shutting down...

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T13:59:41Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-1 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.12:2379 --advertise-client-urls http://10.240.0.12:2379 --listen-peer-urls http://10.240.0.12:2380 --initial-advertise-peer-urls http://10.240.0.12:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.12:2380,etcd-2=http://10.240.0.13:2380,etcd-3=http://10.240.0.14:2380 --initial-cluster-state new","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23281]","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"etcd2 monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:00:01Z"}
{"level":"info","msg":"Stopping etcd2 [PID: 23281]","time":"2016-03-25T14:53:18Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23281]","time":"2016-03-25T14:53:18Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:53:18Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:53:18Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-etcd2-1-database.log","time":"2016-03-25T14:53:18Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:53:18Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-etcd2-1-monitor.csv","time":"2016-03-25T14:53:19Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-etcd2-1-agent.log","time":"2016-03-25T14:53:21Z"}

View File

@ -0,0 +1,195 @@
2016-03-25 14:00:01.886650 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:00:01.886724 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:00:01.886731 I | etcdmain: Go Version: go1.6
2016-03-25 14:00:01.886737 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:00:01.886747 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:00:01.886814 I | etcdmain: listening for peers on http://10.240.0.12:2380
2016-03-25 14:00:01.886849 I | etcdmain: listening for client requests on 10.240.0.12:2379
2016-03-25 14:00:01.889298 I | etcdserver: name = etcd-1
2016-03-25 14:00:01.889314 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:00:01.889322 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:00:01.889327 I | etcdserver: heartbeat = 100ms
2016-03-25 14:00:01.889333 I | etcdserver: election = 1000ms
2016-03-25 14:00:01.889338 I | etcdserver: snapshot count = 10000
2016-03-25 14:00:01.889347 I | etcdserver: advertise client URLs = http://10.240.0.12:2379
2016-03-25 14:00:01.889355 I | etcdserver: initial advertise peer URLs = http://10.240.0.12:2380
2016-03-25 14:00:01.889369 I | etcdserver: initial cluster = etcd-1=http://10.240.0.12:2380,etcd-2=http://10.240.0.13:2380,etcd-3=http://10.240.0.14:2380
2016-03-25 14:00:01.898438 I | etcdserver: starting member f8ba34e834cb847d in cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:01.898480 I | raft: f8ba34e834cb847d became follower at term 0
2016-03-25 14:00:01.898493 I | raft: newRaft f8ba34e834cb847d [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:00:01.898500 I | raft: f8ba34e834cb847d became follower at term 1
2016-03-25 14:00:01.950657 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:00:01.951526 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:00:01.951546 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:00:01.951648 N | etcdmain: serving insecure client requests on 10.240.0.12:2379, this is strongly discouraged!
2016-03-25 14:00:01.952463 N | etcdserver: added member cfc6e2e1de0e7922 [http://10.240.0.13:2380] to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:01.952545 N | etcdserver: added local member f8ba34e834cb847d [http://10.240.0.12:2380] to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:01.952608 N | etcdserver: added member f9e96ef8c0d0e08c [http://10.240.0.14:2380] to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:02.498798 I | raft: f8ba34e834cb847d is starting a new election at term 1
2016-03-25 14:00:02.498881 I | raft: f8ba34e834cb847d became candidate at term 2
2016-03-25 14:00:02.498891 I | raft: f8ba34e834cb847d received vote from f8ba34e834cb847d at term 2
2016-03-25 14:00:02.498901 I | raft: f8ba34e834cb847d [logterm: 1, index: 3] sent vote request to cfc6e2e1de0e7922 at term 2
2016-03-25 14:00:02.498911 I | raft: f8ba34e834cb847d [logterm: 1, index: 3] sent vote request to f9e96ef8c0d0e08c at term 2
2016-03-25 14:00:02.931134 I | rafthttp: the connection with cfc6e2e1de0e7922 became active
2016-03-25 14:00:03.401265 I | raft: f8ba34e834cb847d [logterm: 1, index: 3, vote: f8ba34e834cb847d] rejected vote from cfc6e2e1de0e7922 [logterm: 1, index: 3] at term 2
2016-03-25 14:00:03.598832 I | raft: f8ba34e834cb847d is starting a new election at term 2
2016-03-25 14:00:03.598905 I | raft: f8ba34e834cb847d became candidate at term 3
2016-03-25 14:00:03.598914 I | raft: f8ba34e834cb847d received vote from f8ba34e834cb847d at term 3
2016-03-25 14:00:03.598926 I | raft: f8ba34e834cb847d [logterm: 1, index: 3] sent vote request to cfc6e2e1de0e7922 at term 3
2016-03-25 14:00:03.598935 I | raft: f8ba34e834cb847d [logterm: 1, index: 3] sent vote request to f9e96ef8c0d0e08c at term 3
2016-03-25 14:00:03.603615 I | raft: f8ba34e834cb847d received vote from cfc6e2e1de0e7922 at term 3
2016-03-25 14:00:03.603632 I | raft: f8ba34e834cb847d [quorum:2] has received 2 votes and 0 vote rejections
2016-03-25 14:00:03.603659 I | raft: f8ba34e834cb847d became leader at term 3
2016-03-25 14:00:03.603693 I | raft: raft.node: f8ba34e834cb847d elected leader f8ba34e834cb847d at term 3
2016-03-25 14:00:03.610873 W | etcdserver: failed to reach the peerURL(http://10.240.0.14:2380) of member f9e96ef8c0d0e08c (Get http://10.240.0.14:2380/version: dial tcp 10.240.0.14:2380: getsockopt: connection refused)
2016-03-25 14:00:03.610893 W | etcdserver: cannot get the version of member f9e96ef8c0d0e08c (Get http://10.240.0.14:2380/version: dial tcp 10.240.0.14:2380: getsockopt: connection refused)
2016-03-25 14:00:03.610931 I | etcdserver: setting up the initial cluster version to 2.2
2016-03-25 14:00:03.621042 I | etcdserver: published {Name:etcd-1 ClientURLs:[http://10.240.0.12:2379]} to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:03.622853 N | etcdserver: set the initial cluster version to 2.2
2016-03-25 14:00:03.960257 I | rafthttp: the connection with f9e96ef8c0d0e08c became active
2016-03-25 14:00:07.613460 I | etcdserver: updating the cluster version from 2.2 to 2.3
2016-03-25 14:00:07.620980 N | etcdserver: updated the cluster version from 2.2 to 2.3
2016-03-25 14:01:06.318333 I | etcdserver: start to snapshot (applied: 10001, lastsnap: 0)
2016-03-25 14:01:06.469371 I | etcdserver: saved snapshot at index 10001
2016-03-25 14:01:06.469515 I | etcdserver: compacted raft log at 5001
2016-03-25 14:01:49.824480 I | etcdserver: start to snapshot (applied: 20002, lastsnap: 10001)
2016-03-25 14:01:50.123963 I | etcdserver: saved snapshot at index 20002
2016-03-25 14:01:50.124103 I | etcdserver: compacted raft log at 15002
2016-03-25 14:02:33.688585 I | etcdserver: start to snapshot (applied: 30003, lastsnap: 20002)
2016-03-25 14:02:34.139992 I | etcdserver: saved snapshot at index 30003
2016-03-25 14:02:34.140120 I | etcdserver: compacted raft log at 25003
2016-03-25 14:03:17.413180 I | etcdserver: start to snapshot (applied: 40004, lastsnap: 30003)
2016-03-25 14:03:18.028317 I | etcdserver: saved snapshot at index 40004
2016-03-25 14:03:18.028452 I | etcdserver: compacted raft log at 35004
2016-03-25 14:04:01.225703 I | etcdserver: start to snapshot (applied: 50005, lastsnap: 40004)
2016-03-25 14:04:02.011966 I | etcdserver: saved snapshot at index 50005
2016-03-25 14:04:02.012101 I | etcdserver: compacted raft log at 45005
2016-03-25 14:04:45.186488 I | etcdserver: start to snapshot (applied: 60006, lastsnap: 50005)
2016-03-25 14:04:46.085803 I | etcdserver: saved snapshot at index 60006
2016-03-25 14:04:46.085958 I | etcdserver: compacted raft log at 55006
2016-03-25 14:05:01.954191 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002711.snap successfully
2016-03-25 14:05:28.913281 I | etcdserver: start to snapshot (applied: 70007, lastsnap: 60006)
2016-03-25 14:05:30.038093 I | etcdserver: saved snapshot at index 70007
2016-03-25 14:05:30.038255 I | etcdserver: compacted raft log at 65007
2016-03-25 14:05:31.957370 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e22.snap successfully
2016-03-25 14:06:04.649761 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 46.445138ms)
2016-03-25 14:06:04.649797 W | etcdserver: server is likely overloaded
2016-03-25 14:06:04.649806 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 46.552044ms)
2016-03-25 14:06:04.649812 W | etcdserver: server is likely overloaded
2016-03-25 14:06:13.080289 I | etcdserver: start to snapshot (applied: 80008, lastsnap: 70007)
2016-03-25 14:06:14.415517 I | etcdserver: saved snapshot at index 80008
2016-03-25 14:06:14.415653 I | etcdserver: compacted raft log at 75008
2016-03-25 14:06:31.961727 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007533.snap successfully
2016-03-25 14:06:44.835546 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 36.469241ms)
2016-03-25 14:06:44.835584 W | etcdserver: server is likely overloaded
2016-03-25 14:06:44.835595 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 36.532088ms)
2016-03-25 14:06:44.835613 W | etcdserver: server is likely overloaded
2016-03-25 14:06:57.714226 I | etcdserver: start to snapshot (applied: 90009, lastsnap: 80008)
2016-03-25 14:06:59.055572 I | etcdserver: saved snapshot at index 90009
2016-03-25 14:06:59.055737 I | etcdserver: compacted raft log at 85009
2016-03-25 14:07:01.966959 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c44.snap successfully
2016-03-25 14:07:42.213998 I | etcdserver: start to snapshot (applied: 100010, lastsnap: 90009)
2016-03-25 14:07:43.676509 I | etcdserver: saved snapshot at index 100010
2016-03-25 14:07:43.676656 I | etcdserver: compacted raft log at 95010
2016-03-25 14:08:01.973541 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c355.snap successfully
2016-03-25 14:08:26.763681 I | etcdserver: start to snapshot (applied: 110011, lastsnap: 100010)
2016-03-25 14:08:28.484284 I | etcdserver: saved snapshot at index 110011
2016-03-25 14:08:28.484458 I | etcdserver: compacted raft log at 105011
2016-03-25 14:08:31.983796 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea66.snap successfully
2016-03-25 14:09:11.245455 I | etcdserver: start to snapshot (applied: 120012, lastsnap: 110011)
2016-03-25 14:09:12.961771 I | etcdserver: saved snapshot at index 120012
2016-03-25 14:09:12.961939 I | etcdserver: compacted raft log at 115012
2016-03-25 14:09:31.994338 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000011177.snap successfully
2016-03-25 14:09:56.076652 I | etcdserver: start to snapshot (applied: 130013, lastsnap: 120012)
2016-03-25 14:09:58.213269 I | etcdserver: saved snapshot at index 130013
2016-03-25 14:09:58.213429 I | etcdserver: compacted raft log at 125013
2016-03-25 14:10:02.004908 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000013888.snap successfully
2016-03-25 14:10:40.829434 I | etcdserver: start to snapshot (applied: 140014, lastsnap: 130013)
2016-03-25 14:10:43.040604 I | etcdserver: saved snapshot at index 140014
2016-03-25 14:10:43.040793 I | etcdserver: compacted raft log at 135014
2016-03-25 14:11:02.017051 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015f99.snap successfully
2016-03-25 14:11:22.295620 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-0000000000024730.wal is created
2016-03-25 14:11:25.422907 I | etcdserver: start to snapshot (applied: 150015, lastsnap: 140014)
2016-03-25 14:11:27.662184 I | etcdserver: saved snapshot at index 150015
2016-03-25 14:11:27.662347 I | etcdserver: compacted raft log at 145015
2016-03-25 14:11:32.029428 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186aa.snap successfully
2016-03-25 14:12:10.307400 I | etcdserver: start to snapshot (applied: 160016, lastsnap: 150015)
2016-03-25 14:12:12.907709 I | etcdserver: saved snapshot at index 160016
2016-03-25 14:12:12.907860 I | etcdserver: compacted raft log at 155016
2016-03-25 14:12:32.044696 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001adbb.snap successfully
2016-03-25 14:12:55.515128 I | etcdserver: start to snapshot (applied: 170017, lastsnap: 160016)
2016-03-25 14:12:58.129228 I | etcdserver: saved snapshot at index 170017
2016-03-25 14:12:58.129380 I | etcdserver: compacted raft log at 165017
2016-03-25 14:13:02.059722 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4cc.snap successfully
2016-03-25 14:13:40.324459 I | etcdserver: start to snapshot (applied: 180018, lastsnap: 170017)
2016-03-25 14:13:43.000947 I | etcdserver: saved snapshot at index 180018
2016-03-25 14:13:43.001089 I | etcdserver: compacted raft log at 175018
2016-03-25 14:14:02.077246 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbdd.snap successfully
2016-03-25 14:14:17.964883 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 66.114533ms)
2016-03-25 14:14:17.965017 W | etcdserver: server is likely overloaded
2016-03-25 14:14:17.965036 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 66.289306ms)
2016-03-25 14:14:17.965042 W | etcdserver: server is likely overloaded
2016-03-25 14:14:25.600470 I | etcdserver: start to snapshot (applied: 190019, lastsnap: 180018)
2016-03-25 14:14:28.510176 I | etcdserver: saved snapshot at index 190019
2016-03-25 14:14:28.510343 I | etcdserver: compacted raft log at 185019
2016-03-25 14:14:32.094704 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000222ee.snap successfully
2016-03-25 14:15:10.793741 I | etcdserver: start to snapshot (applied: 200020, lastsnap: 190019)
2016-03-25 14:15:13.767715 I | etcdserver: saved snapshot at index 200020
2016-03-25 14:15:13.767869 I | etcdserver: compacted raft log at 195020
2016-03-25 14:15:32.115006 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000249ff.snap successfully
2016-03-25 14:15:43.927250 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 28.475813ms)
2016-03-25 14:15:43.927297 W | etcdserver: server is likely overloaded
2016-03-25 14:15:43.927306 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 28.539117ms)
2016-03-25 14:15:43.927312 W | etcdserver: server is likely overloaded
2016-03-25 14:15:56.119158 I | etcdserver: start to snapshot (applied: 210021, lastsnap: 200020)
2016-03-25 14:15:59.240381 I | etcdserver: saved snapshot at index 210021
2016-03-25 14:15:59.240542 I | etcdserver: compacted raft log at 205021
2016-03-25 14:16:02.136215 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027110.snap successfully
2016-03-25 14:16:41.124488 I | etcdserver: start to snapshot (applied: 220022, lastsnap: 210021)
2016-03-25 14:16:44.463691 I | etcdserver: saved snapshot at index 220022
2016-03-25 14:16:44.463842 I | etcdserver: compacted raft log at 215022
2016-03-25 14:17:02.157464 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000029821.snap successfully
2016-03-25 14:17:27.065313 I | etcdserver: start to snapshot (applied: 230023, lastsnap: 220022)
2016-03-25 14:17:30.545362 I | etcdserver: saved snapshot at index 230023
2016-03-25 14:17:30.547893 I | etcdserver: compacted raft log at 225023
2016-03-25 14:17:32.181011 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf32.snap successfully
2016-03-25 14:18:00.697542 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 97.421506ms)
2016-03-25 14:18:00.697578 W | etcdserver: server is likely overloaded
2016-03-25 14:18:00.697587 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 97.482543ms)
2016-03-25 14:18:00.697593 W | etcdserver: server is likely overloaded
2016-03-25 14:18:12.649009 I | etcdserver: start to snapshot (applied: 240024, lastsnap: 230023)
2016-03-25 14:18:16.221393 I | etcdserver: saved snapshot at index 240024
2016-03-25 14:18:16.221555 I | etcdserver: compacted raft log at 235024
2016-03-25 14:18:32.205335 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e643.snap successfully
2016-03-25 14:18:58.524231 I | etcdserver: start to snapshot (applied: 250025, lastsnap: 240024)
2016-03-25 14:19:02.419135 I | etcdserver: saved snapshot at index 250025
2016-03-25 14:19:02.419284 I | etcdserver: compacted raft log at 245025
2016-03-25 14:19:32.231949 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d54.snap successfully
2016-03-25 14:19:44.179788 I | etcdserver: start to snapshot (applied: 260026, lastsnap: 250025)
2016-03-25 14:19:48.185002 I | etcdserver: saved snapshot at index 260026
2016-03-25 14:19:48.185175 I | etcdserver: compacted raft log at 255026
2016-03-25 14:20:02.258700 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000033465.snap successfully
2016-03-25 14:20:30.520213 I | etcdserver: start to snapshot (applied: 270027, lastsnap: 260026)
2016-03-25 14:20:34.761112 I | etcdserver: saved snapshot at index 270027
2016-03-25 14:20:34.761262 I | etcdserver: compacted raft log at 265027
2016-03-25 14:21:02.286078 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035b76.snap successfully
2016-03-25 14:21:17.091817 I | etcdserver: start to snapshot (applied: 280028, lastsnap: 270027)
2016-03-25 14:21:21.429966 I | etcdserver: saved snapshot at index 280028
2016-03-25 14:21:21.430120 I | etcdserver: compacted raft log at 275028
2016-03-25 14:21:32.319437 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000038287.snap successfully
2016-03-25 14:22:03.408030 I | etcdserver: start to snapshot (applied: 290029, lastsnap: 280028)
2016-03-25 14:22:07.818255 I | etcdserver: saved snapshot at index 290029
2016-03-25 14:22:07.818436 I | etcdserver: compacted raft log at 285029
2016-03-25 14:22:32.349923 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a998.snap successfully
2016-03-25 14:22:39.789601 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 89.367591ms)
2016-03-25 14:22:39.789644 W | etcdserver: server is likely overloaded
2016-03-25 14:22:39.789653 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 89.435709ms)
2016-03-25 14:22:39.789659 W | etcdserver: server is likely overloaded
2016-03-25 14:22:43.177434 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000002-0000000000048e15.wal is created
2016-03-25 14:22:49.967490 I | etcdserver: start to snapshot (applied: 300030, lastsnap: 290029)
2016-03-25 14:22:54.455239 I | etcdserver: saved snapshot at index 300030
2016-03-25 14:22:54.455411 I | etcdserver: compacted raft log at 295030
2016-03-25 14:23:02.381477 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0a9.snap successfully
2016-03-25 14:53:18.495442 N | osutil: received terminated signal, shutting down...
2016-03-25 14:53:18.496622 E | rafthttp: failed to read cfc6e2e1de0e7922 on stream MsgApp v2 (net/http: request canceled)
2016-03-25 14:53:18.496678 I | rafthttp: the connection with cfc6e2e1de0e7922 became inactive

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T13:59:45Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-2 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.13:2379 --advertise-client-urls http://10.240.0.13:2379 --listen-peer-urls http://10.240.0.13:2380 --initial-advertise-peer-urls http://10.240.0.13:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.12:2380,etcd-2=http://10.240.0.13:2380,etcd-3=http://10.240.0.14:2380 --initial-cluster-state new","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23313]","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"etcd2 monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:00:02Z"}
{"level":"info","msg":"Stopping etcd2 [PID: 23313]","time":"2016-03-25T14:53:19Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23313]","time":"2016-03-25T14:53:19Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:53:19Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:53:19Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-etcd2-2-database.log","time":"2016-03-25T14:53:19Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:53:19Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-etcd2-2-monitor.csv","time":"2016-03-25T14:53:21Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-etcd2-2-agent.log","time":"2016-03-25T14:53:22Z"}

View File

@ -0,0 +1,163 @@
2016-03-25 14:00:02.886227 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:00:02.886274 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:00:02.886281 I | etcdmain: Go Version: go1.6
2016-03-25 14:00:02.886288 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:00:02.886296 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:00:02.886350 I | etcdmain: listening for peers on http://10.240.0.13:2380
2016-03-25 14:00:02.886377 I | etcdmain: listening for client requests on 10.240.0.13:2379
2016-03-25 14:00:02.888029 I | etcdserver: name = etcd-2
2016-03-25 14:00:02.888043 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:00:02.888051 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:00:02.888057 I | etcdserver: heartbeat = 100ms
2016-03-25 14:00:02.888068 I | etcdserver: election = 1000ms
2016-03-25 14:00:02.888074 I | etcdserver: snapshot count = 10000
2016-03-25 14:00:02.888082 I | etcdserver: advertise client URLs = http://10.240.0.13:2379
2016-03-25 14:00:02.888090 I | etcdserver: initial advertise peer URLs = http://10.240.0.13:2380
2016-03-25 14:00:02.888103 I | etcdserver: initial cluster = etcd-1=http://10.240.0.12:2380,etcd-2=http://10.240.0.13:2380,etcd-3=http://10.240.0.14:2380
2016-03-25 14:00:02.898219 I | etcdserver: starting member cfc6e2e1de0e7922 in cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:02.898262 I | raft: cfc6e2e1de0e7922 became follower at term 0
2016-03-25 14:00:02.898275 I | raft: newRaft cfc6e2e1de0e7922 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:00:02.898281 I | raft: cfc6e2e1de0e7922 became follower at term 1
2016-03-25 14:00:02.931500 I | rafthttp: the connection with f8ba34e834cb847d became active
2016-03-25 14:00:02.934215 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:00:02.936305 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:00:02.936325 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:00:02.936362 N | etcdserver: added local member cfc6e2e1de0e7922 [http://10.240.0.13:2380] to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:02.936433 N | etcdserver: added member f8ba34e834cb847d [http://10.240.0.12:2380] to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:02.936496 N | etcdserver: added member f9e96ef8c0d0e08c [http://10.240.0.14:2380] to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:02.936525 N | etcdmain: serving insecure client requests on 10.240.0.13:2379, this is strongly discouraged!
2016-03-25 14:00:03.398622 I | raft: cfc6e2e1de0e7922 is starting a new election at term 1
2016-03-25 14:00:03.398706 I | raft: cfc6e2e1de0e7922 became candidate at term 2
2016-03-25 14:00:03.398715 I | raft: cfc6e2e1de0e7922 received vote from cfc6e2e1de0e7922 at term 2
2016-03-25 14:00:03.398725 I | raft: cfc6e2e1de0e7922 [logterm: 1, index: 3] sent vote request to f8ba34e834cb847d at term 2
2016-03-25 14:00:03.398734 I | raft: cfc6e2e1de0e7922 [logterm: 1, index: 3] sent vote request to f9e96ef8c0d0e08c at term 2
2016-03-25 14:00:03.402067 I | raft: cfc6e2e1de0e7922 received vote rejection from f8ba34e834cb847d at term 2
2016-03-25 14:00:03.402089 I | raft: cfc6e2e1de0e7922 [quorum:2] has received 1 votes and 1 vote rejections
2016-03-25 14:00:03.601566 I | raft: cfc6e2e1de0e7922 [term: 2] received a MsgVote message with higher term from f8ba34e834cb847d [term: 3]
2016-03-25 14:00:03.601635 I | raft: cfc6e2e1de0e7922 became follower at term 3
2016-03-25 14:00:03.601649 I | raft: cfc6e2e1de0e7922 [logterm: 1, index: 3, vote: 0] voted for f8ba34e834cb847d [logterm: 1, index: 3] at term 3
2016-03-25 14:00:03.604356 I | raft: raft.node: cfc6e2e1de0e7922 elected leader f8ba34e834cb847d at term 3
2016-03-25 14:00:03.623087 I | etcdserver: published {Name:etcd-2 ClientURLs:[http://10.240.0.13:2379]} to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:03.623348 N | etcdserver: set the initial cluster version to 2.2
2016-03-25 14:00:03.956951 I | rafthttp: the connection with f9e96ef8c0d0e08c became active
2016-03-25 14:00:07.622144 N | etcdserver: updated the cluster version from 2.2 to 2.3
2016-03-25 14:01:06.318887 I | etcdserver: start to snapshot (applied: 10001, lastsnap: 0)
2016-03-25 14:01:06.460973 I | etcdserver: saved snapshot at index 10001
2016-03-25 14:01:06.461291 I | etcdserver: compacted raft log at 5001
2016-03-25 14:01:49.826396 I | etcdserver: start to snapshot (applied: 20002, lastsnap: 10001)
2016-03-25 14:01:50.127251 I | etcdserver: saved snapshot at index 20002
2016-03-25 14:01:50.127383 I | etcdserver: compacted raft log at 15002
2016-03-25 14:02:33.689319 I | etcdserver: start to snapshot (applied: 30003, lastsnap: 20002)
2016-03-25 14:02:34.119262 I | etcdserver: saved snapshot at index 30003
2016-03-25 14:02:34.119443 I | etcdserver: compacted raft log at 25003
2016-03-25 14:03:17.415955 I | etcdserver: start to snapshot (applied: 40004, lastsnap: 30003)
2016-03-25 14:03:18.014772 I | etcdserver: saved snapshot at index 40004
2016-03-25 14:03:18.014930 I | etcdserver: compacted raft log at 35004
2016-03-25 14:04:01.226646 I | etcdserver: start to snapshot (applied: 50005, lastsnap: 40004)
2016-03-25 14:04:01.905423 I | etcdserver: saved snapshot at index 50005
2016-03-25 14:04:01.905589 I | etcdserver: compacted raft log at 45005
2016-03-25 14:04:45.187491 I | etcdserver: start to snapshot (applied: 60006, lastsnap: 50005)
2016-03-25 14:04:46.034750 I | etcdserver: saved snapshot at index 60006
2016-03-25 14:04:46.034895 I | etcdserver: compacted raft log at 55006
2016-03-25 14:05:02.938268 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002711.snap successfully
2016-03-25 14:05:28.914580 I | etcdserver: start to snapshot (applied: 70007, lastsnap: 60006)
2016-03-25 14:05:29.951803 I | etcdserver: saved snapshot at index 70007
2016-03-25 14:05:29.952018 I | etcdserver: compacted raft log at 65007
2016-03-25 14:05:32.941283 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e22.snap successfully
2016-03-25 14:06:13.081473 I | etcdserver: start to snapshot (applied: 80008, lastsnap: 70007)
2016-03-25 14:06:14.287697 I | etcdserver: saved snapshot at index 80008
2016-03-25 14:06:14.287834 I | etcdserver: compacted raft log at 75008
2016-03-25 14:06:32.945704 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007533.snap successfully
2016-03-25 14:06:57.715576 I | etcdserver: start to snapshot (applied: 90009, lastsnap: 80008)
2016-03-25 14:06:59.061884 I | etcdserver: saved snapshot at index 90009
2016-03-25 14:06:59.062069 I | etcdserver: compacted raft log at 85009
2016-03-25 14:07:02.950930 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c44.snap successfully
2016-03-25 14:07:42.215445 I | etcdserver: start to snapshot (applied: 100010, lastsnap: 90009)
2016-03-25 14:07:43.732387 I | etcdserver: saved snapshot at index 100010
2016-03-25 14:07:43.732533 I | etcdserver: compacted raft log at 95010
2016-03-25 14:08:02.957592 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c355.snap successfully
2016-03-25 14:08:26.765192 I | etcdserver: start to snapshot (applied: 110011, lastsnap: 100010)
2016-03-25 14:08:28.360792 I | etcdserver: saved snapshot at index 110011
2016-03-25 14:08:28.360949 I | etcdserver: compacted raft log at 105011
2016-03-25 14:08:32.965257 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea66.snap successfully
2016-03-25 14:09:11.247114 I | etcdserver: start to snapshot (applied: 120012, lastsnap: 110011)
2016-03-25 14:09:13.024928 I | etcdserver: saved snapshot at index 120012
2016-03-25 14:09:13.025080 I | etcdserver: compacted raft log at 115012
2016-03-25 14:09:32.974309 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000011177.snap successfully
2016-03-25 14:09:56.079403 I | etcdserver: start to snapshot (applied: 130013, lastsnap: 120012)
2016-03-25 14:09:58.067916 I | etcdserver: saved snapshot at index 130013
2016-03-25 14:09:58.068098 I | etcdserver: compacted raft log at 125013
2016-03-25 14:10:02.984650 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000013888.snap successfully
2016-03-25 14:10:40.831328 I | etcdserver: start to snapshot (applied: 140014, lastsnap: 130013)
2016-03-25 14:10:43.039619 I | etcdserver: saved snapshot at index 140014
2016-03-25 14:10:43.039784 I | etcdserver: compacted raft log at 135014
2016-03-25 14:11:02.996100 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015f99.snap successfully
2016-03-25 14:11:23.298173 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-0000000000024813.wal is created
2016-03-25 14:11:25.425043 I | etcdserver: start to snapshot (applied: 150015, lastsnap: 140014)
2016-03-25 14:11:27.700592 I | etcdserver: saved snapshot at index 150015
2016-03-25 14:11:27.700834 I | etcdserver: compacted raft log at 145015
2016-03-25 14:11:33.009402 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186aa.snap successfully
2016-03-25 14:12:10.309337 I | etcdserver: start to snapshot (applied: 160016, lastsnap: 150015)
2016-03-25 14:12:12.671244 I | etcdserver: saved snapshot at index 160016
2016-03-25 14:12:12.671414 I | etcdserver: compacted raft log at 155016
2016-03-25 14:12:33.023485 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001adbb.snap successfully
2016-03-25 14:12:55.517181 I | etcdserver: start to snapshot (applied: 170017, lastsnap: 160016)
2016-03-25 14:12:57.885398 I | etcdserver: saved snapshot at index 170017
2016-03-25 14:12:57.885552 I | etcdserver: compacted raft log at 165017
2016-03-25 14:13:03.040780 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4cc.snap successfully
2016-03-25 14:13:40.326493 I | etcdserver: start to snapshot (applied: 180018, lastsnap: 170017)
2016-03-25 14:13:42.991302 I | etcdserver: saved snapshot at index 180018
2016-03-25 14:13:42.991490 I | etcdserver: compacted raft log at 175018
2016-03-25 14:14:03.057386 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbdd.snap successfully
2016-03-25 14:14:25.601828 I | etcdserver: start to snapshot (applied: 190019, lastsnap: 180018)
2016-03-25 14:14:28.510766 I | etcdserver: saved snapshot at index 190019
2016-03-25 14:14:28.510915 I | etcdserver: compacted raft log at 185019
2016-03-25 14:14:33.075619 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000222ee.snap successfully
2016-03-25 14:15:10.793795 I | etcdserver: start to snapshot (applied: 200020, lastsnap: 190019)
2016-03-25 14:15:13.621854 I | etcdserver: saved snapshot at index 200020
2016-03-25 14:15:13.622012 I | etcdserver: compacted raft log at 195020
2016-03-25 14:15:33.094559 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000249ff.snap successfully
2016-03-25 14:15:56.118309 I | etcdserver: start to snapshot (applied: 210021, lastsnap: 200020)
2016-03-25 14:15:59.175668 I | etcdserver: saved snapshot at index 210021
2016-03-25 14:15:59.175846 I | etcdserver: compacted raft log at 205021
2016-03-25 14:16:03.114548 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027110.snap successfully
2016-03-25 14:16:41.123962 I | etcdserver: start to snapshot (applied: 220022, lastsnap: 210021)
2016-03-25 14:16:44.584332 I | etcdserver: saved snapshot at index 220022
2016-03-25 14:16:44.584477 I | etcdserver: compacted raft log at 215022
2016-03-25 14:17:03.136313 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000029821.snap successfully
2016-03-25 14:17:27.062844 I | etcdserver: start to snapshot (applied: 230023, lastsnap: 220022)
2016-03-25 14:17:30.419862 I | etcdserver: saved snapshot at index 230023
2016-03-25 14:17:30.420015 I | etcdserver: compacted raft log at 225023
2016-03-25 14:17:33.158982 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf32.snap successfully
2016-03-25 14:18:12.645834 I | etcdserver: start to snapshot (applied: 240024, lastsnap: 230023)
2016-03-25 14:18:16.111739 I | etcdserver: saved snapshot at index 240024
2016-03-25 14:18:16.111901 I | etcdserver: compacted raft log at 235024
2016-03-25 14:18:33.182568 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e643.snap successfully
2016-03-25 14:18:58.520576 I | etcdserver: start to snapshot (applied: 250025, lastsnap: 240024)
2016-03-25 14:19:02.071397 I | etcdserver: saved snapshot at index 250025
2016-03-25 14:19:02.071565 I | etcdserver: compacted raft log at 245025
2016-03-25 14:19:03.206804 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d54.snap successfully
2016-03-25 14:19:44.175694 I | etcdserver: start to snapshot (applied: 260026, lastsnap: 250025)
2016-03-25 14:19:48.412136 I | etcdserver: saved snapshot at index 260026
2016-03-25 14:19:48.412281 I | etcdserver: compacted raft log at 255026
2016-03-25 14:20:03.234021 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000033465.snap successfully
2016-03-25 14:20:30.516627 I | etcdserver: start to snapshot (applied: 270027, lastsnap: 260026)
2016-03-25 14:20:34.781654 I | etcdserver: saved snapshot at index 270027
2016-03-25 14:20:34.781932 I | etcdserver: compacted raft log at 265027
2016-03-25 14:21:03.263286 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035b76.snap successfully
2016-03-25 14:21:17.087979 I | etcdserver: start to snapshot (applied: 280028, lastsnap: 270027)
2016-03-25 14:21:21.472758 I | etcdserver: saved snapshot at index 280028
2016-03-25 14:21:21.472943 I | etcdserver: compacted raft log at 275028
2016-03-25 14:21:33.292582 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000038287.snap successfully
2016-03-25 14:22:03.404864 I | etcdserver: start to snapshot (applied: 290029, lastsnap: 280028)
2016-03-25 14:22:07.949976 I | etcdserver: saved snapshot at index 290029
2016-03-25 14:22:07.950142 I | etcdserver: compacted raft log at 285029
2016-03-25 14:22:33.323166 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a998.snap successfully
2016-03-25 14:22:45.310534 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000002-0000000000048fed.wal is created
2016-03-25 14:22:49.964884 I | etcdserver: start to snapshot (applied: 300030, lastsnap: 290029)
2016-03-25 14:22:54.490826 I | etcdserver: saved snapshot at index 300030
2016-03-25 14:22:54.490983 I | etcdserver: compacted raft log at 295030
2016-03-25 14:23:03.353695 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0a9.snap successfully
2016-03-25 14:53:18.597294 E | rafthttp: failed to dial f8ba34e834cb847d on stream MsgApp v2 (dial tcp 10.240.0.12:2380: getsockopt: connection refused)
2016-03-25 14:53:18.597348 I | rafthttp: the connection with f8ba34e834cb847d became inactive
2016-03-25 14:53:19.496891 N | osutil: received terminated signal, shutting down...

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T13:59:52Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-3 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.14:2379 --advertise-client-urls http://10.240.0.14:2379 --listen-peer-urls http://10.240.0.14:2380 --initial-advertise-peer-urls http://10.240.0.14:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.12:2380,etcd-2=http://10.240.0.13:2380,etcd-3=http://10.240.0.14:2380 --initial-cluster-state new","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23479]","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"etcd2 monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:00:03Z"}
{"level":"info","msg":"Stopping etcd2 [PID: 23479]","time":"2016-03-25T14:53:20Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23479]","time":"2016-03-25T14:53:20Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:53:20Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:53:20Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-etcd2-3-database.log","time":"2016-03-25T14:53:20Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:53:20Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-etcd2-3-monitor.csv","time":"2016-03-25T14:53:22Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-etcd2-3-agent.log","time":"2016-03-25T14:53:23Z"}

View File

@ -0,0 +1,162 @@
2016-03-25 14:00:03.877036 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:00:03.877089 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:00:03.877096 I | etcdmain: Go Version: go1.6
2016-03-25 14:00:03.877102 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:00:03.877116 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:00:03.877171 I | etcdmain: listening for peers on http://10.240.0.14:2380
2016-03-25 14:00:03.877199 I | etcdmain: listening for client requests on 10.240.0.14:2379
2016-03-25 14:00:03.878859 I | etcdserver: name = etcd-3
2016-03-25 14:00:03.878873 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:00:03.878881 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:00:03.878887 I | etcdserver: heartbeat = 100ms
2016-03-25 14:00:03.878892 I | etcdserver: election = 1000ms
2016-03-25 14:00:03.878898 I | etcdserver: snapshot count = 10000
2016-03-25 14:00:03.878907 I | etcdserver: advertise client URLs = http://10.240.0.14:2379
2016-03-25 14:00:03.878914 I | etcdserver: initial advertise peer URLs = http://10.240.0.14:2380
2016-03-25 14:00:03.878935 I | etcdserver: initial cluster = etcd-1=http://10.240.0.12:2380,etcd-2=http://10.240.0.13:2380,etcd-3=http://10.240.0.14:2380
2016-03-25 14:00:03.888139 I | etcdserver: starting member f9e96ef8c0d0e08c in cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:03.888202 I | raft: f9e96ef8c0d0e08c became follower at term 0
2016-03-25 14:00:03.888216 I | raft: newRaft f9e96ef8c0d0e08c [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:00:03.888223 I | raft: f9e96ef8c0d0e08c became follower at term 1
2016-03-25 14:00:03.947740 I | rafthttp: the connection with cfc6e2e1de0e7922 became active
2016-03-25 14:00:03.949749 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:00:03.950843 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:00:03.950857 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:00:03.950991 N | etcdmain: serving insecure client requests on 10.240.0.14:2379, this is strongly discouraged!
2016-03-25 14:00:03.951032 I | raft: f9e96ef8c0d0e08c [term: 1] received a MsgApp message with higher term from f8ba34e834cb847d [term: 3]
2016-03-25 14:00:03.951051 I | raft: f9e96ef8c0d0e08c became follower at term 3
2016-03-25 14:00:03.951058 I | rafthttp: the connection with f8ba34e834cb847d became active
2016-03-25 14:00:03.951074 I | raft: raft.node: f9e96ef8c0d0e08c elected leader f8ba34e834cb847d at term 3
2016-03-25 14:00:03.951217 N | etcdserver: added member cfc6e2e1de0e7922 [http://10.240.0.13:2380] to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:03.951316 N | etcdserver: added member f8ba34e834cb847d [http://10.240.0.12:2380] to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:03.951394 N | etcdserver: added local member f9e96ef8c0d0e08c [http://10.240.0.14:2380] to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:03.952864 N | etcdserver: set the initial cluster version to 2.2
2016-03-25 14:00:03.960004 I | etcdserver: published {Name:etcd-3 ClientURLs:[http://10.240.0.14:2379]} to cluster 1f5ca66cfce5f0e7
2016-03-25 14:00:07.611792 N | etcdserver: updated the cluster version from 2.2 to 2.3
2016-03-25 14:01:06.306374 I | etcdserver: start to snapshot (applied: 10001, lastsnap: 0)
2016-03-25 14:01:06.457239 I | etcdserver: saved snapshot at index 10001
2016-03-25 14:01:06.457553 I | etcdserver: compacted raft log at 5001
2016-03-25 14:01:49.810750 I | etcdserver: start to snapshot (applied: 20002, lastsnap: 10001)
2016-03-25 14:01:50.105663 I | etcdserver: saved snapshot at index 20002
2016-03-25 14:01:50.105858 I | etcdserver: compacted raft log at 15002
2016-03-25 14:02:33.672653 I | etcdserver: start to snapshot (applied: 30003, lastsnap: 20002)
2016-03-25 14:02:34.129004 I | etcdserver: saved snapshot at index 30003
2016-03-25 14:02:34.129158 I | etcdserver: compacted raft log at 25003
2016-03-25 14:03:17.395353 I | etcdserver: start to snapshot (applied: 40004, lastsnap: 30003)
2016-03-25 14:03:18.021260 I | etcdserver: saved snapshot at index 40004
2016-03-25 14:03:18.021411 I | etcdserver: compacted raft log at 35004
2016-03-25 14:04:01.205778 I | etcdserver: start to snapshot (applied: 50005, lastsnap: 40004)
2016-03-25 14:04:01.950288 I | etcdserver: saved snapshot at index 50005
2016-03-25 14:04:01.950468 I | etcdserver: compacted raft log at 45005
2016-03-25 14:04:45.164527 I | etcdserver: start to snapshot (applied: 60006, lastsnap: 50005)
2016-03-25 14:04:46.042278 I | etcdserver: saved snapshot at index 60006
2016-03-25 14:04:46.042413 I | etcdserver: compacted raft log at 55006
2016-03-25 14:05:03.953207 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002711.snap successfully
2016-03-25 14:05:28.890923 I | etcdserver: start to snapshot (applied: 70007, lastsnap: 60006)
2016-03-25 14:05:29.946267 I | etcdserver: saved snapshot at index 70007
2016-03-25 14:05:29.946455 I | etcdserver: compacted raft log at 65007
2016-03-25 14:05:33.955960 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e22.snap successfully
2016-03-25 14:06:13.059825 I | etcdserver: start to snapshot (applied: 80008, lastsnap: 70007)
2016-03-25 14:06:14.255383 I | etcdserver: saved snapshot at index 80008
2016-03-25 14:06:14.255569 I | etcdserver: compacted raft log at 75008
2016-03-25 14:06:33.960259 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007533.snap successfully
2016-03-25 14:06:57.686348 I | etcdserver: start to snapshot (applied: 90009, lastsnap: 80008)
2016-03-25 14:06:58.989453 I | etcdserver: saved snapshot at index 90009
2016-03-25 14:06:58.989602 I | etcdserver: compacted raft log at 85009
2016-03-25 14:07:03.965195 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c44.snap successfully
2016-03-25 14:07:42.185964 I | etcdserver: start to snapshot (applied: 100010, lastsnap: 90009)
2016-03-25 14:07:43.633313 I | etcdserver: saved snapshot at index 100010
2016-03-25 14:07:43.633476 I | etcdserver: compacted raft log at 95010
2016-03-25 14:08:03.971926 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c355.snap successfully
2016-03-25 14:08:26.732181 I | etcdserver: start to snapshot (applied: 110011, lastsnap: 100010)
2016-03-25 14:08:28.275414 I | etcdserver: saved snapshot at index 110011
2016-03-25 14:08:28.275582 I | etcdserver: compacted raft log at 105011
2016-03-25 14:08:33.979777 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea66.snap successfully
2016-03-25 14:09:11.215262 I | etcdserver: start to snapshot (applied: 120012, lastsnap: 110011)
2016-03-25 14:09:12.944497 I | etcdserver: saved snapshot at index 120012
2016-03-25 14:09:12.944651 I | etcdserver: compacted raft log at 115012
2016-03-25 14:09:33.989794 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000011177.snap successfully
2016-03-25 14:09:56.040627 I | etcdserver: start to snapshot (applied: 130013, lastsnap: 120012)
2016-03-25 14:09:58.010612 I | etcdserver: saved snapshot at index 130013
2016-03-25 14:09:58.010779 I | etcdserver: compacted raft log at 125013
2016-03-25 14:10:04.000007 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000013888.snap successfully
2016-03-25 14:10:40.791315 I | etcdserver: start to snapshot (applied: 140014, lastsnap: 130013)
2016-03-25 14:10:42.912260 I | etcdserver: saved snapshot at index 140014
2016-03-25 14:10:42.912448 I | etcdserver: compacted raft log at 135014
2016-03-25 14:11:04.014709 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015f99.snap successfully
2016-03-25 14:11:25.382726 I | etcdserver: start to snapshot (applied: 150015, lastsnap: 140014)
2016-03-25 14:11:27.655465 I | etcdserver: saved snapshot at index 150015
2016-03-25 14:11:27.655677 I | etcdserver: compacted raft log at 145015
2016-03-25 14:11:34.027261 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186aa.snap successfully
2016-03-25 14:11:37.699240 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-00000000000254ba.wal is created
2016-03-25 14:12:10.268222 I | etcdserver: start to snapshot (applied: 160016, lastsnap: 150015)
2016-03-25 14:12:12.642044 I | etcdserver: saved snapshot at index 160016
2016-03-25 14:12:12.642203 I | etcdserver: compacted raft log at 155016
2016-03-25 14:12:34.042903 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001adbb.snap successfully
2016-03-25 14:12:55.475271 I | etcdserver: start to snapshot (applied: 170017, lastsnap: 160016)
2016-03-25 14:12:57.977917 I | etcdserver: saved snapshot at index 170017
2016-03-25 14:12:57.978082 I | etcdserver: compacted raft log at 165017
2016-03-25 14:13:04.058284 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4cc.snap successfully
2016-03-25 14:13:40.283345 I | etcdserver: start to snapshot (applied: 180018, lastsnap: 170017)
2016-03-25 14:13:42.978658 I | etcdserver: saved snapshot at index 180018
2016-03-25 14:13:42.978818 I | etcdserver: compacted raft log at 175018
2016-03-25 14:14:04.074695 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbdd.snap successfully
2016-03-25 14:14:25.559393 I | etcdserver: start to snapshot (applied: 190019, lastsnap: 180018)
2016-03-25 14:14:28.326322 I | etcdserver: saved snapshot at index 190019
2016-03-25 14:14:28.326473 I | etcdserver: compacted raft log at 185019
2016-03-25 14:14:34.092300 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000222ee.snap successfully
2016-03-25 14:15:10.752190 I | etcdserver: start to snapshot (applied: 200020, lastsnap: 190019)
2016-03-25 14:15:13.797678 I | etcdserver: saved snapshot at index 200020
2016-03-25 14:15:13.797826 I | etcdserver: compacted raft log at 195020
2016-03-25 14:15:34.111579 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000249ff.snap successfully
2016-03-25 14:15:56.074936 I | etcdserver: start to snapshot (applied: 210021, lastsnap: 200020)
2016-03-25 14:15:59.138534 I | etcdserver: saved snapshot at index 210021
2016-03-25 14:15:59.138696 I | etcdserver: compacted raft log at 205021
2016-03-25 14:16:04.131605 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027110.snap successfully
2016-03-25 14:16:41.079403 I | etcdserver: start to snapshot (applied: 220022, lastsnap: 210021)
2016-03-25 14:16:44.484288 I | etcdserver: saved snapshot at index 220022
2016-03-25 14:16:44.484441 I | etcdserver: compacted raft log at 215022
2016-03-25 14:17:04.153004 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000029821.snap successfully
2016-03-25 14:17:27.019515 I | etcdserver: start to snapshot (applied: 230023, lastsnap: 220022)
2016-03-25 14:17:30.329485 I | etcdserver: saved snapshot at index 230023
2016-03-25 14:17:30.329645 I | etcdserver: compacted raft log at 225023
2016-03-25 14:17:34.175347 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf32.snap successfully
2016-03-25 14:18:12.603804 I | etcdserver: start to snapshot (applied: 240024, lastsnap: 230023)
2016-03-25 14:18:16.219954 I | etcdserver: saved snapshot at index 240024
2016-03-25 14:18:16.220118 I | etcdserver: compacted raft log at 235024
2016-03-25 14:18:34.199390 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e643.snap successfully
2016-03-25 14:18:58.479338 I | etcdserver: start to snapshot (applied: 250025, lastsnap: 240024)
2016-03-25 14:19:02.248212 I | etcdserver: saved snapshot at index 250025
2016-03-25 14:19:02.248396 I | etcdserver: compacted raft log at 245025
2016-03-25 14:19:04.223099 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d54.snap successfully
2016-03-25 14:19:44.132804 I | etcdserver: start to snapshot (applied: 260026, lastsnap: 250025)
2016-03-25 14:19:48.069087 I | etcdserver: saved snapshot at index 260026
2016-03-25 14:19:48.069267 I | etcdserver: compacted raft log at 255026
2016-03-25 14:20:04.250602 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000033465.snap successfully
2016-03-25 14:20:30.481883 I | etcdserver: start to snapshot (applied: 270027, lastsnap: 260026)
2016-03-25 14:20:34.646995 I | etcdserver: saved snapshot at index 270027
2016-03-25 14:20:34.647165 I | etcdserver: compacted raft log at 265027
2016-03-25 14:21:04.277743 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035b76.snap successfully
2016-03-25 14:21:17.065735 I | etcdserver: start to snapshot (applied: 280028, lastsnap: 270027)
2016-03-25 14:21:21.334198 I | etcdserver: saved snapshot at index 280028
2016-03-25 14:21:21.334413 I | etcdserver: compacted raft log at 275028
2016-03-25 14:21:34.305737 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000038287.snap successfully
2016-03-25 14:22:03.383589 I | etcdserver: start to snapshot (applied: 290029, lastsnap: 280028)
2016-03-25 14:22:07.901496 I | etcdserver: saved snapshot at index 290029
2016-03-25 14:22:07.901691 I | etcdserver: compacted raft log at 285029
2016-03-25 14:22:34.335440 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a998.snap successfully
2016-03-25 14:22:49.948776 I | etcdserver: start to snapshot (applied: 300030, lastsnap: 290029)
2016-03-25 14:22:54.511082 I | etcdserver: saved snapshot at index 300030
2016-03-25 14:22:54.511311 I | etcdserver: compacted raft log at 295030
2016-03-25 14:23:04.367523 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0a9.snap successfully
2016-03-25 14:53:18.623384 E | rafthttp: failed to dial f8ba34e834cb847d on stream MsgApp v2 (dial tcp 10.240.0.12:2380: getsockopt: connection refused)
2016-03-25 14:53:18.623456 I | rafthttp: the connection with f8ba34e834cb847d became inactive
2016-03-25 14:53:19.624505 E | rafthttp: failed to dial cfc6e2e1de0e7922 on stream MsgApp v2 (dial tcp 10.240.0.13:2380: getsockopt: connection refused)
2016-03-25 14:53:19.624576 I | rafthttp: the connection with cfc6e2e1de0e7922 became inactive
2016-03-25 14:53:19.888600 I | raft: f9e96ef8c0d0e08c is starting a new election at term 3
2016-03-25 14:53:19.888652 I | raft: f9e96ef8c0d0e08c became candidate at term 4
2016-03-25 14:53:19.888660 I | raft: f9e96ef8c0d0e08c received vote from f9e96ef8c0d0e08c at term 4
2016-03-25 14:53:19.888671 I | raft: f9e96ef8c0d0e08c [logterm: 3, index: 306400] sent vote request to f8ba34e834cb847d at term 4
2016-03-25 14:53:19.888680 I | raft: f9e96ef8c0d0e08c [logterm: 3, index: 306400] sent vote request to cfc6e2e1de0e7922 at term 4
2016-03-25 14:53:19.888690 I | raft: raft.node: f9e96ef8c0d0e08c lost leader f8ba34e834cb847d at term 4
2016-03-25 14:53:20.521292 N | osutil: received terminated signal, shutting down...

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

Binary file not shown.

After

Width:  |  Height:  |  Size: 44 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 103 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 125 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 123 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 42 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 120 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 165 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 120 KiB

View File

@ -0,0 +1,24 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:08:31Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"os.Chdir: /mnt/ssd0/zookeeper","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"os.MkdirAll: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Writing 1 to /mnt/ssd0/zookeeper/data.zk/myid","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Writing \"tickTime=2000\\ndataDir=/mnt/ssd0/zookeeper/data.zk\\nclientPort=2181\\ninitLimit=5\\nsyncLimit=5\\npreAllocSize=67108864\\nmaxClientCnxns=5000\\nserver.1=10.240.0.24:2888:3888\\nserver.2=10.240.0.25:2888:3888\\nserver.3=10.240.0.26:2888:3888\\n\\n\" to /mnt/ssd0/zookeeper/zookeeper.config","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Starting: /bin/bash -c /usr/bin/java -cp zookeeper-3.4.8.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.16.jar:conf org.apache.zookeeper.server.quorum.QuorumPeerMain /mnt/ssd0/zookeeper/zookeeper.config","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Started: /bin/bash [PID: 23480]","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"ZooKeeper monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:10:12Z"}
{"level":"info","msg":"Stopping ZooKeeper [PID: 23480]","time":"2016-03-25T14:52:42Z"}
{"level":"info","msg":"Stopped: /bin/bash [PID: 23480]","time":"2016-03-25T14:52:42Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:52:42Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:52:42Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-zk-1-database.log","time":"2016-03-25T14:52:42Z"}
{"level":"info","msg":"/bin/bash cmd.Wait returned exit status 143","time":"2016-03-25T14:52:42Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-zk-1-monitor.csv","time":"2016-03-25T14:52:43Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-zk-1-agent.log","time":"2016-03-25T14:52:44Z"}

View File

@ -0,0 +1,163 @@
2016-03-25 14:10:12,250 [myid:] - INFO [main:QuorumPeerConfig@103] - Reading configuration from: /mnt/ssd0/zookeeper/zookeeper.config
2016-03-25 14:10:12,265 [myid:] - INFO [main:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.26 to address: /10.240.0.26
2016-03-25 14:10:12,265 [myid:] - INFO [main:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.25 to address: /10.240.0.25
2016-03-25 14:10:12,267 [myid:] - INFO [main:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.24 to address: /10.240.0.24
2016-03-25 14:10:12,267 [myid:] - INFO [main:QuorumPeerConfig@331] - Defaulting to majority quorums
2016-03-25 14:10:12,270 [myid:1] - INFO [main:DatadirCleanupManager@78] - autopurge.snapRetainCount set to 3
2016-03-25 14:10:12,270 [myid:1] - INFO [main:DatadirCleanupManager@79] - autopurge.purgeInterval set to 0
2016-03-25 14:10:12,270 [myid:1] - INFO [main:DatadirCleanupManager@101] - Purge task is not scheduled.
2016-03-25 14:10:12,334 [myid:1] - INFO [main:QuorumPeerMain@127] - Starting quorum peer
2016-03-25 14:10:12,345 [myid:1] - INFO [main:NIOServerCnxnFactory@89] - binding to port 0.0.0.0/0.0.0.0:2181
2016-03-25 14:10:12,352 [myid:1] - INFO [main:QuorumPeer@1019] - tickTime set to 2000
2016-03-25 14:10:12,352 [myid:1] - INFO [main:QuorumPeer@1039] - minSessionTimeout set to -1
2016-03-25 14:10:12,352 [myid:1] - INFO [main:QuorumPeer@1050] - maxSessionTimeout set to -1
2016-03-25 14:10:12,352 [myid:1] - INFO [main:QuorumPeer@1065] - initLimit set to 5
2016-03-25 14:10:12,361 [myid:1] - INFO [main:QuorumPeer@533] - currentEpoch not found! Creating with a reasonable default of 0. This should only happen when you are upgrading your installation
2016-03-25 14:10:12,371 [myid:1] - INFO [main:QuorumPeer@548] - acceptedEpoch not found! Creating with a reasonable default of 0. This should only happen when you are upgrading your installation
2016-03-25 14:10:12,383 [myid:1] - INFO [ListenerThread:QuorumCnxManager$Listener@534] - My election bind port: /10.240.0.24:3888
2016-03-25 14:10:12,390 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumPeer@774] - LOOKING
2016-03-25 14:10:12,391 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:FastLeaderElection@818] - New election. My id = 1, proposed zxid=0x0
2016-03-25 14:10:12,392 [myid:1] - INFO [WorkerReceiver[myid=1]:FastLeaderElection@600] - Notification: 1 (message format version), 1 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 1 (n.sid), 0x0 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:12,396 [myid:1] - WARN [WorkerSender[myid=1]:QuorumCnxManager@400] - Cannot open channel to 2 at election address /10.240.0.25:3888
java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:381)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.toSend(QuorumCnxManager.java:354)
at org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.process(FastLeaderElection.java:452)
at org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.run(FastLeaderElection.java:433)
at java.lang.Thread.run(Thread.java:745)
2016-03-25 14:10:12,398 [myid:1] - INFO [WorkerSender[myid=1]:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.25 to address: /10.240.0.25
2016-03-25 14:10:12,399 [myid:1] - WARN [WorkerSender[myid=1]:QuorumCnxManager@400] - Cannot open channel to 3 at election address /10.240.0.26:3888
java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:381)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.toSend(QuorumCnxManager.java:354)
at org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.process(FastLeaderElection.java:452)
at org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.run(FastLeaderElection.java:433)
at java.lang.Thread.run(Thread.java:745)
2016-03-25 14:10:12,399 [myid:1] - INFO [WorkerSender[myid=1]:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.26 to address: /10.240.0.26
2016-03-25 14:10:12,595 [myid:1] - WARN [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumCnxManager@400] - Cannot open channel to 2 at election address /10.240.0.25:3888
java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:381)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectAll(QuorumCnxManager.java:426)
at org.apache.zookeeper.server.quorum.FastLeaderElection.lookForLeader(FastLeaderElection.java:843)
at org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:822)
2016-03-25 14:10:12,596 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.25 to address: /10.240.0.25
2016-03-25 14:10:12,596 [myid:1] - WARN [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumCnxManager@400] - Cannot open channel to 3 at election address /10.240.0.26:3888
java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:381)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectAll(QuorumCnxManager.java:426)
at org.apache.zookeeper.server.quorum.FastLeaderElection.lookForLeader(FastLeaderElection.java:843)
at org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:822)
2016-03-25 14:10:12,597 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.26 to address: /10.240.0.26
2016-03-25 14:10:12,597 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:FastLeaderElection@852] - Notification time out: 400
2016-03-25 14:10:12,998 [myid:1] - WARN [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumCnxManager@400] - Cannot open channel to 2 at election address /10.240.0.25:3888
java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:381)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectAll(QuorumCnxManager.java:426)
at org.apache.zookeeper.server.quorum.FastLeaderElection.lookForLeader(FastLeaderElection.java:843)
at org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:822)
2016-03-25 14:10:12,999 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.25 to address: /10.240.0.25
2016-03-25 14:10:13,000 [myid:1] - WARN [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumCnxManager@400] - Cannot open channel to 3 at election address /10.240.0.26:3888
java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:381)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectAll(QuorumCnxManager.java:426)
at org.apache.zookeeper.server.quorum.FastLeaderElection.lookForLeader(FastLeaderElection.java:843)
at org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:822)
2016-03-25 14:10:13,000 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.26 to address: /10.240.0.26
2016-03-25 14:10:13,000 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:FastLeaderElection@852] - Notification time out: 800
2016-03-25 14:10:13,390 [myid:1] - INFO [/10.240.0.24:3888:QuorumCnxManager$Listener@541] - Received connection request /10.240.0.25:57350
2016-03-25 14:10:13,393 [myid:1] - INFO [WorkerReceiver[myid=1]:FastLeaderElection@600] - Notification: 1 (message format version), 2 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 2 (n.sid), 0x0 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:13,393 [myid:1] - INFO [WorkerReceiver[myid=1]:FastLeaderElection@600] - Notification: 1 (message format version), 2 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 1 (n.sid), 0x0 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:13,394 [myid:1] - WARN [WorkerSender[myid=1]:QuorumCnxManager@400] - Cannot open channel to 3 at election address /10.240.0.26:3888
java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:381)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.toSend(QuorumCnxManager.java:354)
at org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.process(FastLeaderElection.java:452)
at org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.run(FastLeaderElection.java:433)
at java.lang.Thread.run(Thread.java:745)
2016-03-25 14:10:13,394 [myid:1] - INFO [WorkerSender[myid=1]:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.26 to address: /10.240.0.26
2016-03-25 14:10:13,594 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumPeer@844] - FOLLOWING
2016-03-25 14:10:13,598 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Learner@86] - TCP NoDelay set to: true
2016-03-25 14:10:13,607 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:zookeeper.version=3.4.8--1, built on 02/06/2016 03:18 GMT
2016-03-25 14:10:13,607 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:host.name=bench-01-zk-1.c.etcd-development.internal
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:java.version=1.8.0_74
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:java.vendor=Oracle Corporation
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:java.home=/usr/lib/jvm/java-8-oracle/jre
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:java.class.path=zookeeper-3.4.8.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.16.jar:conf
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:java.io.tmpdir=/tmp
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:java.compiler=<NA>
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:os.name=Linux
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:os.arch=amd64
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:os.version=4.2.0-34-generic
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:user.name=gyuho
2016-03-25 14:10:13,608 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:user.home=/home/gyuho
2016-03-25 14:10:13,609 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Environment@100] - Server environment:user.dir=/mnt/ssd0/zookeeper
2016-03-25 14:10:13,610 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:ZooKeeperServer@170] - Created server with tickTime 2000 minSessionTimeout 4000 maxSessionTimeout 40000 datadir /mnt/ssd0/zookeeper/data.zk/version-2 snapdir /mnt/ssd0/zookeeper/data.zk/version-2
2016-03-25 14:10:13,611 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Follower@63] - FOLLOWING - LEADER ELECTION TOOK - 1220
2016-03-25 14:10:13,613 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.25 to address: /10.240.0.25
2016-03-25 14:10:13,635 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:Learner@326] - Getting a diff from the leader 0x0
2016-03-25 14:10:13,637 [myid:1] - INFO [QuorumPeer[myid=1]/0.0.0.0:2181:FileTxnSnapLog@240] - Snapshotting: 0x0 to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.0
2016-03-25 14:10:14,391 [myid:1] - INFO [/10.240.0.24:3888:QuorumCnxManager$Listener@541] - Received connection request /10.240.0.26:36658
2016-03-25 14:10:14,394 [myid:1] - INFO [WorkerReceiver[myid=1]:FastLeaderElection@600] - Notification: 1 (message format version), 3 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 3 (n.sid), 0x0 (n.peerEpoch) FOLLOWING (my state)
2016-03-25 14:10:29,950 [myid:1] - INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:NIOServerCnxnFactory@192] - Accepted socket connection from /10.240.0.27:52852
2016-03-25 14:10:30,004 [myid:1] - WARN [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:ZooKeeperServer@854] - Connection request from old client /10.240.0.27:52852; will be dropped if server is in r-o mode
2016-03-25 14:10:30,004 [myid:1] - INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:ZooKeeperServer@900] - Client attempting to establish new session at /10.240.0.27:52852
2016-03-25 14:10:30,008 [myid:1] - WARN [QuorumPeer[myid=1]/0.0.0.0:2181:Follower@118] - Got zxid 0x100000001 expected 0x1
2016-03-25 14:10:30,008 [myid:1] - INFO [SyncThread:1:FileTxnLog@199] - Creating new log file: log.100000001
2016-03-25 14:10:30,024 [myid:1] - INFO [CommitProcessor:1:ZooKeeperServer@645] - Established session 0x153ae1c74050000 with negotiated timeout 4000 for client /10.240.0.27:52852
2016-03-25 14:16:49,810 [myid:1] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x100016113 to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.100016113
2016-03-25 14:16:49,821 [myid:1] - INFO [SyncThread:1:FileTxnLog@199] - Creating new log file: log.100016115
2016-03-25 14:21:41,687 [myid:1] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x100026f0f to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.100026f0f
2016-03-25 14:21:41,707 [myid:1] - INFO [SyncThread:1:FileTxnLog@199] - Creating new log file: log.100026f11
2016-03-25 14:27:14,648 [myid:1] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x100039d2c to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.100039d2c
2016-03-25 14:27:14,675 [myid:1] - INFO [SyncThread:1:FileTxnLog@199] - Creating new log file: log.100039d2e
2016-03-25 14:31:26,291 [myid:1] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x1000481ab to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.1000481ab
2016-03-25 14:31:26,321 [myid:1] - INFO [SyncThread:1:FileTxnLog@199] - Creating new log file: log.1000481ad
2016-03-25 14:31:46,261 [myid:1] - WARN [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:NIOServerCnxn@357] - caught end of stream exception
EndOfStreamException: Unable to read additional data from client sessionid 0x153ae1c74050000, likely client has closed socket
at org.apache.zookeeper.server.NIOServerCnxn.doIO(NIOServerCnxn.java:230)
at org.apache.zookeeper.server.NIOServerCnxnFactory.run(NIOServerCnxnFactory.java:203)
at java.lang.Thread.run(Thread.java:745)
2016-03-25 14:31:46,262 [myid:1] - INFO [NIOServerCxn.Factory:0.0.0.0/0.0.0.0:2181:NIOServerCnxn@1008] - Closed socket connection for client /10.240.0.27:52852 which had sessionid 0x153ae1c74050000

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,24 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:08:15Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"os.Chdir: /mnt/ssd0/zookeeper","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"os.MkdirAll: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Writing 2 to /mnt/ssd0/zookeeper/data.zk/myid","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Writing \"tickTime=2000\\ndataDir=/mnt/ssd0/zookeeper/data.zk\\nclientPort=2181\\ninitLimit=5\\nsyncLimit=5\\npreAllocSize=67108864\\nmaxClientCnxns=5000\\nserver.1=10.240.0.24:2888:3888\\nserver.2=10.240.0.25:2888:3888\\nserver.3=10.240.0.26:2888:3888\\n\\n\" to /mnt/ssd0/zookeeper/zookeeper.config","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Starting: /bin/bash -c /usr/bin/java -cp zookeeper-3.4.8.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.16.jar:conf org.apache.zookeeper.server.quorum.QuorumPeerMain /mnt/ssd0/zookeeper/zookeeper.config","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Started: /bin/bash [PID: 23326]","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"ZooKeeper monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:10:13Z"}
{"level":"info","msg":"Stopping ZooKeeper [PID: 23326]","time":"2016-03-25T14:52:43Z"}
{"level":"info","msg":"Stopped: /bin/bash [PID: 23326]","time":"2016-03-25T14:52:43Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:52:43Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:52:43Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-zk-2-database.log","time":"2016-03-25T14:52:43Z"}
{"level":"info","msg":"/bin/bash cmd.Wait returned exit status 143","time":"2016-03-25T14:52:43Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-zk-2-monitor.csv","time":"2016-03-25T14:52:44Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-zk-2-agent.log","time":"2016-03-25T14:52:45Z"}

View File

@ -0,0 +1,101 @@
2016-03-25 14:10:13,247 [myid:] - INFO [main:QuorumPeerConfig@103] - Reading configuration from: /mnt/ssd0/zookeeper/zookeeper.config
2016-03-25 14:10:13,261 [myid:] - INFO [main:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.26 to address: /10.240.0.26
2016-03-25 14:10:13,262 [myid:] - INFO [main:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.25 to address: /10.240.0.25
2016-03-25 14:10:13,264 [myid:] - INFO [main:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.24 to address: /10.240.0.24
2016-03-25 14:10:13,264 [myid:] - INFO [main:QuorumPeerConfig@331] - Defaulting to majority quorums
2016-03-25 14:10:13,267 [myid:2] - INFO [main:DatadirCleanupManager@78] - autopurge.snapRetainCount set to 3
2016-03-25 14:10:13,267 [myid:2] - INFO [main:DatadirCleanupManager@79] - autopurge.purgeInterval set to 0
2016-03-25 14:10:13,267 [myid:2] - INFO [main:DatadirCleanupManager@101] - Purge task is not scheduled.
2016-03-25 14:10:13,331 [myid:2] - INFO [main:QuorumPeerMain@127] - Starting quorum peer
2016-03-25 14:10:13,343 [myid:2] - INFO [main:NIOServerCnxnFactory@89] - binding to port 0.0.0.0/0.0.0.0:2181
2016-03-25 14:10:13,349 [myid:2] - INFO [main:QuorumPeer@1019] - tickTime set to 2000
2016-03-25 14:10:13,349 [myid:2] - INFO [main:QuorumPeer@1039] - minSessionTimeout set to -1
2016-03-25 14:10:13,349 [myid:2] - INFO [main:QuorumPeer@1050] - maxSessionTimeout set to -1
2016-03-25 14:10:13,349 [myid:2] - INFO [main:QuorumPeer@1065] - initLimit set to 5
2016-03-25 14:10:13,359 [myid:2] - INFO [main:QuorumPeer@533] - currentEpoch not found! Creating with a reasonable default of 0. This should only happen when you are upgrading your installation
2016-03-25 14:10:13,367 [myid:2] - INFO [main:QuorumPeer@548] - acceptedEpoch not found! Creating with a reasonable default of 0. This should only happen when you are upgrading your installation
2016-03-25 14:10:13,378 [myid:2] - INFO [ListenerThread:QuorumCnxManager$Listener@534] - My election bind port: /10.240.0.25:3888
2016-03-25 14:10:13,385 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:QuorumPeer@774] - LOOKING
2016-03-25 14:10:13,386 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:FastLeaderElection@818] - New election. My id = 2, proposed zxid=0x0
2016-03-25 14:10:13,393 [myid:2] - INFO [WorkerReceiver[myid=2]:FastLeaderElection@600] - Notification: 1 (message format version), 2 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 2 (n.sid), 0x0 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:13,393 [myid:2] - INFO [WorkerReceiver[myid=2]:FastLeaderElection@600] - Notification: 1 (message format version), 1 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 1 (n.sid), 0x0 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:13,394 [myid:2] - INFO [WorkerReceiver[myid=2]:FastLeaderElection@600] - Notification: 1 (message format version), 2 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 1 (n.sid), 0x0 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:13,394 [myid:2] - WARN [WorkerSender[myid=2]:QuorumCnxManager@400] - Cannot open channel to 3 at election address /10.240.0.26:3888
java.net.ConnectException: Connection refused
at java.net.PlainSocketImpl.socketConnect(Native Method)
at java.net.AbstractPlainSocketImpl.doConnect(AbstractPlainSocketImpl.java:350)
at java.net.AbstractPlainSocketImpl.connectToAddress(AbstractPlainSocketImpl.java:206)
at java.net.AbstractPlainSocketImpl.connect(AbstractPlainSocketImpl.java:188)
at java.net.SocksSocketImpl.connect(SocksSocketImpl.java:392)
at java.net.Socket.connect(Socket.java:589)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.connectOne(QuorumCnxManager.java:381)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.toSend(QuorumCnxManager.java:354)
at org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.process(FastLeaderElection.java:452)
at org.apache.zookeeper.server.quorum.FastLeaderElection$Messenger$WorkerSender.run(FastLeaderElection.java:433)
at java.lang.Thread.run(Thread.java:745)
2016-03-25 14:10:13,396 [myid:2] - INFO [WorkerSender[myid=2]:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.26 to address: /10.240.0.26
2016-03-25 14:10:13,595 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:QuorumPeer@856] - LEADING
2016-03-25 14:10:13,597 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Leader@59] - TCP NoDelay set to: true
2016-03-25 14:10:13,607 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:zookeeper.version=3.4.8--1, built on 02/06/2016 03:18 GMT
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:host.name=bench-01-zk-2.c.etcd-development.internal
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:java.version=1.8.0_74
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:java.vendor=Oracle Corporation
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:java.home=/usr/lib/jvm/java-8-oracle/jre
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:java.class.path=zookeeper-3.4.8.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.16.jar:conf
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:java.io.tmpdir=/tmp
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:java.compiler=<NA>
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:os.name=Linux
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:os.arch=amd64
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:os.version=4.2.0-34-generic
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:user.name=gyuho
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:user.home=/home/gyuho
2016-03-25 14:10:13,608 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Environment@100] - Server environment:user.dir=/mnt/ssd0/zookeeper
2016-03-25 14:10:13,610 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:ZooKeeperServer@170] - Created server with tickTime 2000 minSessionTimeout 4000 maxSessionTimeout 40000 datadir /mnt/ssd0/zookeeper/data.zk/version-2 snapdir /mnt/ssd0/zookeeper/data.zk/version-2
2016-03-25 14:10:13,611 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Leader@361] - LEADING - LEADER ELECTION TOOK - 225
2016-03-25 14:10:13,620 [myid:2] - INFO [LearnerHandler-/10.240.0.24:42770:LearnerHandler@329] - Follower sid: 1 : info : org.apache.zookeeper.server.quorum.QuorumPeer$QuorumServer@5b4d514b
2016-03-25 14:10:13,635 [myid:2] - INFO [LearnerHandler-/10.240.0.24:42770:LearnerHandler@384] - Synchronizing with Follower sid: 1 maxCommittedLog=0x0 minCommittedLog=0x0 peerLastZxid=0x0
2016-03-25 14:10:13,635 [myid:2] - INFO [LearnerHandler-/10.240.0.24:42770:LearnerHandler@393] - leader and follower are in sync, zxid=0x0
2016-03-25 14:10:13,635 [myid:2] - INFO [LearnerHandler-/10.240.0.24:42770:LearnerHandler@458] - Sending DIFF
2016-03-25 14:10:13,646 [myid:2] - INFO [LearnerHandler-/10.240.0.24:42770:LearnerHandler@518] - Received NEWLEADER-ACK message from 1
2016-03-25 14:10:13,646 [myid:2] - INFO [QuorumPeer[myid=2]/0.0.0.0:2181:Leader@946] - Have quorum of supporters, sids: [ 1,2 ]; starting up and setting last processed zxid: 0x100000000
2016-03-25 14:10:14,396 [myid:2] - INFO [/10.240.0.25:3888:QuorumCnxManager$Listener@541] - Received connection request /10.240.0.26:46596
2016-03-25 14:10:14,397 [myid:2] - INFO [WorkerReceiver[myid=2]:FastLeaderElection@600] - Notification: 1 (message format version), 3 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 3 (n.sid), 0x0 (n.peerEpoch) LEADING (my state)
2016-03-25 14:10:14,419 [myid:2] - INFO [LearnerHandler-/10.240.0.26:39382:LearnerHandler@329] - Follower sid: 3 : info : org.apache.zookeeper.server.quorum.QuorumPeer$QuorumServer@443079ed
2016-03-25 14:10:14,425 [myid:2] - INFO [LearnerHandler-/10.240.0.26:39382:LearnerHandler@384] - Synchronizing with Follower sid: 3 maxCommittedLog=0x0 minCommittedLog=0x0 peerLastZxid=0x0
2016-03-25 14:10:14,425 [myid:2] - INFO [LearnerHandler-/10.240.0.26:39382:LearnerHandler@458] - Sending SNAP
2016-03-25 14:10:14,425 [myid:2] - INFO [LearnerHandler-/10.240.0.26:39382:LearnerHandler@482] - Sending snapshot last zxid of peer is 0x0 zxid of leader is 0x100000000sent zxid of db as 0x100000000
2016-03-25 14:10:14,440 [myid:2] - INFO [LearnerHandler-/10.240.0.26:39382:LearnerHandler@518] - Received NEWLEADER-ACK message from 3
2016-03-25 14:10:30,008 [myid:2] - INFO [SyncThread:2:FileTxnLog@199] - Creating new log file: log.100000001
2016-03-25 14:14:52,207 [myid:2] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x10000f2c3 to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.10000f2c3
2016-03-25 14:14:52,211 [myid:2] - INFO [SyncThread:2:FileTxnLog@199] - Creating new log file: log.10000f2c4
2016-03-25 14:20:47,197 [myid:2] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x100023cb1 to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.100023cb1
2016-03-25 14:20:47,211 [myid:2] - INFO [SyncThread:2:FileTxnLog@199] - Creating new log file: log.100023cb3
2016-03-25 14:26:14,808 [myid:2] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x100036665 to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.100036665
2016-03-25 14:26:14,831 [myid:2] - INFO [SyncThread:2:FileTxnLog@199] - Creating new log file: log.100036667
2016-03-25 14:29:56,220 [myid:2] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x100043103 to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.100043103
2016-03-25 14:29:56,248 [myid:2] - INFO [SyncThread:2:FileTxnLog@199] - Creating new log file: log.100043105
2016-03-25 14:31:52,000 [myid:2] - INFO [SessionTracker:ZooKeeperServer@355] - Expiring session 0x153ae1c74050000, timeout of 4000ms exceeded
2016-03-25 14:31:52,001 [myid:2] - INFO [ProcessThread(sid:2 cport:-1)::PrepRequestProcessor@489] - Processed session termination for sessionid: 0x153ae1c74050000
2016-03-25 14:52:42,885 [myid:2] - ERROR [LearnerHandler-/10.240.0.24:42770:LearnerHandler@631] - Unexpected exception causing shutdown while sock still open
java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:392)
at org.apache.jute.BinaryInputArchive.readInt(BinaryInputArchive.java:63)
at org.apache.zookeeper.server.quorum.QuorumPacket.deserialize(QuorumPacket.java:83)
at org.apache.jute.BinaryInputArchive.readRecord(BinaryInputArchive.java:99)
at org.apache.zookeeper.server.quorum.LearnerHandler.run(LearnerHandler.java:542)
2016-03-25 14:52:42,885 [myid:2] - WARN [RecvWorker:1:QuorumCnxManager$RecvWorker@810] - Connection broken for id 1, my id = 2, error =
java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:392)
at org.apache.zookeeper.server.quorum.QuorumCnxManager$RecvWorker.run(QuorumCnxManager.java:795)
2016-03-25 14:52:42,886 [myid:2] - WARN [RecvWorker:1:QuorumCnxManager$RecvWorker@813] - Interrupting SendWorker
2016-03-25 14:52:42,886 [myid:2] - WARN [LearnerHandler-/10.240.0.24:42770:LearnerHandler@644] - ******* GOODBYE /10.240.0.24:42770 ********
2016-03-25 14:52:42,887 [myid:2] - WARN [SendWorker:1:QuorumCnxManager$SendWorker@727] - Interrupted while waiting for message on queue
java.lang.InterruptedException
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2088)
at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.pollSendQueue(QuorumCnxManager.java:879)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.access$500(QuorumCnxManager.java:65)
at org.apache.zookeeper.server.quorum.QuorumCnxManager$SendWorker.run(QuorumCnxManager.java:715)
2016-03-25 14:52:42,887 [myid:2] - WARN [SendWorker:1:QuorumCnxManager$SendWorker@736] - Send worker leaving thread

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,24 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:08:33Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"os.Chdir: /mnt/ssd0/zookeeper","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"os.MkdirAll: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Writing 3 to /mnt/ssd0/zookeeper/data.zk/myid","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Writing \"tickTime=2000\\ndataDir=/mnt/ssd0/zookeeper/data.zk\\nclientPort=2181\\ninitLimit=5\\nsyncLimit=5\\npreAllocSize=67108864\\nmaxClientCnxns=5000\\nserver.1=10.240.0.24:2888:3888\\nserver.2=10.240.0.25:2888:3888\\nserver.3=10.240.0.26:2888:3888\\n\\n\" to /mnt/ssd0/zookeeper/zookeeper.config","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Starting: /bin/bash -c /usr/bin/java -cp zookeeper-3.4.8.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.16.jar:conf org.apache.zookeeper.server.quorum.QuorumPeerMain /mnt/ssd0/zookeeper/zookeeper.config","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Started: /bin/bash [PID: 23245]","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"ZooKeeper monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:10:14Z"}
{"level":"info","msg":"Stopping ZooKeeper [PID: 23245]","time":"2016-03-25T14:52:44Z"}
{"level":"info","msg":"Stopped: /bin/bash [PID: 23245]","time":"2016-03-25T14:52:44Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:52:44Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:52:44Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-01-zk-3-database.log","time":"2016-03-25T14:52:44Z"}
{"level":"info","msg":"/bin/bash cmd.Wait returned exit status 143","time":"2016-03-25T14:52:44Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-01-zk-3-monitor.csv","time":"2016-03-25T14:52:45Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-01-zk-3-agent.log","time":"2016-03-25T14:52:46Z"}

View File

@ -0,0 +1,108 @@
2016-03-25 14:10:14,245 [myid:] - INFO [main:QuorumPeerConfig@103] - Reading configuration from: /mnt/ssd0/zookeeper/zookeeper.config
2016-03-25 14:10:14,260 [myid:] - INFO [main:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.26 to address: /10.240.0.26
2016-03-25 14:10:14,261 [myid:] - INFO [main:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.25 to address: /10.240.0.25
2016-03-25 14:10:14,263 [myid:] - INFO [main:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.24 to address: /10.240.0.24
2016-03-25 14:10:14,263 [myid:] - INFO [main:QuorumPeerConfig@331] - Defaulting to majority quorums
2016-03-25 14:10:14,265 [myid:3] - INFO [main:DatadirCleanupManager@78] - autopurge.snapRetainCount set to 3
2016-03-25 14:10:14,266 [myid:3] - INFO [main:DatadirCleanupManager@79] - autopurge.purgeInterval set to 0
2016-03-25 14:10:14,266 [myid:3] - INFO [main:DatadirCleanupManager@101] - Purge task is not scheduled.
2016-03-25 14:10:14,330 [myid:3] - INFO [main:QuorumPeerMain@127] - Starting quorum peer
2016-03-25 14:10:14,342 [myid:3] - INFO [main:NIOServerCnxnFactory@89] - binding to port 0.0.0.0/0.0.0.0:2181
2016-03-25 14:10:14,349 [myid:3] - INFO [main:QuorumPeer@1019] - tickTime set to 2000
2016-03-25 14:10:14,349 [myid:3] - INFO [main:QuorumPeer@1039] - minSessionTimeout set to -1
2016-03-25 14:10:14,349 [myid:3] - INFO [main:QuorumPeer@1050] - maxSessionTimeout set to -1
2016-03-25 14:10:14,349 [myid:3] - INFO [main:QuorumPeer@1065] - initLimit set to 5
2016-03-25 14:10:14,359 [myid:3] - INFO [main:QuorumPeer@533] - currentEpoch not found! Creating with a reasonable default of 0. This should only happen when you are upgrading your installation
2016-03-25 14:10:14,368 [myid:3] - INFO [main:QuorumPeer@548] - acceptedEpoch not found! Creating with a reasonable default of 0. This should only happen when you are upgrading your installation
2016-03-25 14:10:14,378 [myid:3] - INFO [ListenerThread:QuorumCnxManager$Listener@534] - My election bind port: /10.240.0.26:3888
2016-03-25 14:10:14,385 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:QuorumPeer@774] - LOOKING
2016-03-25 14:10:14,386 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:FastLeaderElection@818] - New election. My id = 3, proposed zxid=0x0
2016-03-25 14:10:14,394 [myid:3] - INFO [WorkerReceiver[myid=3]:FastLeaderElection@600] - Notification: 1 (message format version), 2 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 1 (n.sid), 0x0 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:14,395 [myid:3] - INFO [WorkerReceiver[myid=3]:FastLeaderElection@600] - Notification: 1 (message format version), 2 (n.leader), 0x0 (n.zxid), 0x1 (n.round), FOLLOWING (n.state), 1 (n.sid), 0x1 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:14,395 [myid:3] - INFO [WorkerReceiver[myid=3]:FastLeaderElection@600] - Notification: 1 (message format version), 3 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 3 (n.sid), 0x0 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:14,396 [myid:3] - INFO [WorkerReceiver[myid=3]:FastLeaderElection@600] - Notification: 1 (message format version), 2 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LOOKING (n.state), 2 (n.sid), 0x0 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:14,396 [myid:3] - INFO [WorkerReceiver[myid=3]:FastLeaderElection@600] - Notification: 1 (message format version), 2 (n.leader), 0x0 (n.zxid), 0x1 (n.round), LEADING (n.state), 2 (n.sid), 0x1 (n.peerEpoch) LOOKING (my state)
2016-03-25 14:10:14,397 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:QuorumPeer@844] - FOLLOWING
2016-03-25 14:10:14,400 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Learner@86] - TCP NoDelay set to: true
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:zookeeper.version=3.4.8--1, built on 02/06/2016 03:18 GMT
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:host.name=bench-01-zk-3.c.etcd-development.internal
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:java.version=1.8.0_74
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:java.vendor=Oracle Corporation
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:java.home=/usr/lib/jvm/java-8-oracle/jre
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:java.class.path=zookeeper-3.4.8.jar:lib/slf4j-api-1.6.1.jar:lib/slf4j-log4j12-1.6.1.jar:lib/log4j-1.2.16.jar:conf
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:java.library.path=/usr/java/packages/lib/amd64:/usr/lib64:/lib64:/lib:/usr/lib
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:java.io.tmpdir=/tmp
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:java.compiler=<NA>
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:os.name=Linux
2016-03-25 14:10:14,409 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:os.arch=amd64
2016-03-25 14:10:14,410 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:os.version=4.2.0-34-generic
2016-03-25 14:10:14,410 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:user.name=gyuho
2016-03-25 14:10:14,410 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:user.home=/home/gyuho
2016-03-25 14:10:14,410 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Environment@100] - Server environment:user.dir=/mnt/ssd0/zookeeper
2016-03-25 14:10:14,411 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:ZooKeeperServer@170] - Created server with tickTime 2000 minSessionTimeout 4000 maxSessionTimeout 40000 datadir /mnt/ssd0/zookeeper/data.zk/version-2 snapdir /mnt/ssd0/zookeeper/data.zk/version-2
2016-03-25 14:10:14,412 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Follower@63] - FOLLOWING - LEADER ELECTION TOOK - 26
2016-03-25 14:10:14,413 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:QuorumPeer$QuorumServer@149] - Resolved hostname: 10.240.0.25 to address: /10.240.0.25
2016-03-25 14:10:14,424 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Learner@329] - Getting a snapshot from leader
2016-03-25 14:10:14,428 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:FileTxnSnapLog@240] - Snapshotting: 0x100000000 to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.100000000
2016-03-25 14:10:30,007 [myid:3] - WARN [QuorumPeer[myid=3]/0.0.0.0:2181:Follower@118] - Got zxid 0x100000001 expected 0x1
2016-03-25 14:10:30,008 [myid:3] - INFO [SyncThread:3:FileTxnLog@199] - Creating new log file: log.100000001
2016-03-25 14:14:19,856 [myid:3] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x10000d48b to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.10000d48b
2016-03-25 14:14:19,861 [myid:3] - INFO [SyncThread:3:FileTxnLog@199] - Creating new log file: log.10000d48d
2016-03-25 14:18:50,203 [myid:3] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x10001d0d7 to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.10001d0d7
2016-03-25 14:18:50,211 [myid:3] - INFO [SyncThread:3:FileTxnLog@199] - Creating new log file: log.10001d0d9
2016-03-25 14:24:30,659 [myid:3] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x100030803 to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.100030803
2016-03-25 14:24:30,663 [myid:3] - INFO [SyncThread:3:FileTxnLog@199] - Creating new log file: log.100030805
2016-03-25 14:31:34,729 [myid:3] - INFO [Snapshot Thread:FileTxnSnapLog@240] - Snapshotting: 0x10004895e to /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.10004895e
2016-03-25 14:31:34,734 [myid:3] - INFO [SyncThread:3:FileTxnLog@199] - Creating new log file: log.100048960
2016-03-25 14:52:42,888 [myid:3] - WARN [RecvWorker:1:QuorumCnxManager$RecvWorker@810] - Connection broken for id 1, my id = 3, error =
java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:392)
at org.apache.zookeeper.server.quorum.QuorumCnxManager$RecvWorker.run(QuorumCnxManager.java:795)
2016-03-25 14:52:42,890 [myid:3] - WARN [RecvWorker:1:QuorumCnxManager$RecvWorker@813] - Interrupting SendWorker
2016-03-25 14:52:42,891 [myid:3] - WARN [SendWorker:1:QuorumCnxManager$SendWorker@727] - Interrupted while waiting for message on queue
java.lang.InterruptedException
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2088)
at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.pollSendQueue(QuorumCnxManager.java:879)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.access$500(QuorumCnxManager.java:65)
at org.apache.zookeeper.server.quorum.QuorumCnxManager$SendWorker.run(QuorumCnxManager.java:715)
2016-03-25 14:52:42,891 [myid:3] - WARN [SendWorker:1:QuorumCnxManager$SendWorker@736] - Send worker leaving thread
2016-03-25 14:52:43,888 [myid:3] - WARN [RecvWorker:2:QuorumCnxManager$RecvWorker@810] - Connection broken for id 2, my id = 3, error =
java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:392)
at org.apache.zookeeper.server.quorum.QuorumCnxManager$RecvWorker.run(QuorumCnxManager.java:795)
2016-03-25 14:52:43,889 [myid:3] - WARN [RecvWorker:2:QuorumCnxManager$RecvWorker@813] - Interrupting SendWorker
2016-03-25 14:52:43,888 [myid:3] - WARN [QuorumPeer[myid=3]/0.0.0.0:2181:Follower@89] - Exception when following the leader
java.io.EOFException
at java.io.DataInputStream.readInt(DataInputStream.java:392)
at org.apache.jute.BinaryInputArchive.readInt(BinaryInputArchive.java:63)
at org.apache.zookeeper.server.quorum.QuorumPacket.deserialize(QuorumPacket.java:83)
at org.apache.jute.BinaryInputArchive.readRecord(BinaryInputArchive.java:99)
at org.apache.zookeeper.server.quorum.Learner.readPacket(Learner.java:153)
at org.apache.zookeeper.server.quorum.Follower.followLeader(Follower.java:85)
at org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:846)
2016-03-25 14:52:43,889 [myid:3] - WARN [SendWorker:2:QuorumCnxManager$SendWorker@727] - Interrupted while waiting for message on queue
java.lang.InterruptedException
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.reportInterruptAfterWait(AbstractQueuedSynchronizer.java:2014)
at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:2088)
at java.util.concurrent.ArrayBlockingQueue.poll(ArrayBlockingQueue.java:418)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.pollSendQueue(QuorumCnxManager.java:879)
at org.apache.zookeeper.server.quorum.QuorumCnxManager.access$500(QuorumCnxManager.java:65)
at org.apache.zookeeper.server.quorum.QuorumCnxManager$SendWorker.run(QuorumCnxManager.java:715)
2016-03-25 14:52:43,890 [myid:3] - WARN [SendWorker:2:QuorumCnxManager$SendWorker@736] - Send worker leaving thread
2016-03-25 14:52:43,890 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:Follower@166] - shutdown called
java.lang.Exception: shutdown Follower
at org.apache.zookeeper.server.quorum.Follower.shutdown(Follower.java:166)
at org.apache.zookeeper.server.quorum.QuorumPeer.run(QuorumPeer.java:850)
2016-03-25 14:52:43,890 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:FollowerZooKeeperServer@140] - Shutting down
2016-03-25 14:52:43,890 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:ZooKeeperServer@469] - shutting down
2016-03-25 14:52:43,890 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:FollowerRequestProcessor@107] - Shutting down
2016-03-25 14:52:43,890 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:CommitProcessor@184] - Shutting down
2016-03-25 14:52:43,890 [myid:3] - INFO [FollowerRequestProcessor:3:FollowerRequestProcessor@97] - FollowerRequestProcessor exited loop!
2016-03-25 14:52:43,891 [myid:3] - INFO [CommitProcessor:3:CommitProcessor@153] - CommitProcessor exited loop!
2016-03-25 14:52:43,890 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:FinalRequestProcessor@415] - shutdown of request processor complete
2016-03-25 14:52:43,891 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:SyncRequestProcessor@209] - Shutting down
2016-03-25 14:52:43,891 [myid:3] - INFO [SyncThread:3:SyncRequestProcessor@187] - SyncRequestProcessor exited!
2016-03-25 14:52:43,891 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:QuorumPeer@774] - LOOKING
2016-03-25 14:52:43,894 [myid:3] - INFO [QuorumPeer[myid=3]/0.0.0.0:2181:FileSnap@83] - Reading snapshot /mnt/ssd0/zookeeper/data.zk/version-2/snapshot.10004895e

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:17:05Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/consul agent -server -data-dir /mnt/ssd0/data.consul -bind 10.240.0.28 -client 10.240.0.28 -bootstrap-expect 3","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/consul [PID: 23483]","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"Consul monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:57Z"}
{"level":"info","msg":"Stopping Consul [PID: 23483]","time":"2016-03-25T16:12:34Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/consul [PID: 23483]","time":"2016-03-25T16:12:34Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T16:12:34Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T16:12:34Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-02-consul-1-database.log","time":"2016-03-25T16:12:34Z"}
{"level":"info","msg":"/home/gyuho/go/bin/consul cmd.Wait returned exit status 1","time":"2016-03-25T16:12:34Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-02-consul-1-monitor.csv","time":"2016-03-25T16:12:35Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-02-consul-1-agent.log","time":"2016-03-25T16:12:37Z"}

View File

@ -0,0 +1,191 @@
==> WARNING: Expect Mode enabled, expecting 3 servers
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Consul agent running!
Node name: 'bench-02-consul-1'
Datacenter: 'dc1'
Server: true (bootstrap: false)
Client Addr: 10.240.0.28 (HTTP: 8500, HTTPS: -1, DNS: 8600, RPC: 8400)
Cluster Addr: 10.240.0.28 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
Atlas: <disabled>
==> Log data will now stream in as it occurs:
2016/03/25 14:22:57 [INFO] raft: Node at 10.240.0.28:8300 [Follower] entering Follower state
2016/03/25 14:22:57 [INFO] serf: EventMemberJoin: bench-02-consul-1 10.240.0.28
2016/03/25 14:22:57 [INFO] consul: adding LAN server bench-02-consul-1 (Addr: 10.240.0.28:8300) (DC: dc1)
2016/03/25 14:22:57 [INFO] serf: EventMemberJoin: bench-02-consul-1.dc1 10.240.0.28
2016/03/25 14:22:57 [INFO] consul: adding WAN server bench-02-consul-1.dc1 (Addr: 10.240.0.28:8300) (DC: dc1)
2016/03/25 14:22:57 [ERR] agent: failed to sync remote state: No cluster leader
2016/03/25 14:22:58 [INFO] serf: EventMemberJoin: bench-02-consul-2 10.240.0.29
2016/03/25 14:22:58 [INFO] consul: adding LAN server bench-02-consul-2 (Addr: 10.240.0.29:8300) (DC: dc1)
2016/03/25 14:22:58 [WARN] raft: EnableSingleNode disabled, and no known peers. Aborting election.
2016/03/25 14:22:59 [INFO] serf: EventMemberJoin: bench-02-consul-3 10.240.0.30
2016/03/25 14:22:59 [INFO] consul: adding LAN server bench-02-consul-3 (Addr: 10.240.0.30:8300) (DC: dc1)
2016/03/25 14:22:59 [INFO] consul: Attempting bootstrap with nodes: [10.240.0.28:8300 10.240.0.29:8300 10.240.0.30:8300]
2016/03/25 14:22:59 [WARN] raft: Heartbeat timeout reached, starting election
2016/03/25 14:22:59 [INFO] raft: Node at 10.240.0.28:8300 [Candidate] entering Candidate state
2016/03/25 14:22:59 [WARN] raft: Remote peer 10.240.0.29:8300 does not have local node 10.240.0.28:8300 as a peer
2016/03/25 14:22:59 [INFO] raft: Election won. Tally: 2
2016/03/25 14:22:59 [INFO] raft: Node at 10.240.0.28:8300 [Leader] entering Leader state
2016/03/25 14:22:59 [INFO] consul: cluster leadership acquired
2016/03/25 14:22:59 [INFO] consul: New leader elected: bench-02-consul-1
2016/03/25 14:22:59 [INFO] raft: pipelining replication to peer 10.240.0.29:8300
2016/03/25 14:22:59 [INFO] raft: pipelining replication to peer 10.240.0.30:8300
2016/03/25 14:22:59 [WARN] raft: Remote peer 10.240.0.30:8300 does not have local node 10.240.0.28:8300 as a peer
2016/03/25 14:22:59 [INFO] consul: member 'bench-02-consul-1' joined, marking health alive
2016/03/25 14:22:59 [INFO] consul: member 'bench-02-consul-2' joined, marking health alive
2016/03/25 14:22:59 [INFO] consul: member 'bench-02-consul-3' joined, marking health alive
2016/03/25 14:23:02 [INFO] agent: Synced service 'consul'
2016/03/25 14:26:26 [INFO] consul.fsm: snapshot created in 19.087µs
2016/03/25 14:26:26 [INFO] raft: Starting snapshot up to 45503
2016/03/25 14:26:26 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-45503-1458915986781.tmp
2016/03/25 14:26:27 [INFO] raft: Compacting logs from 1 to 35353
2016/03/25 14:26:27 [INFO] raft: Snapshot to 45503 complete
2016/03/25 14:29:57 [INFO] consul.fsm: snapshot created in 16.529µs
2016/03/25 14:29:57 [INFO] raft: Starting snapshot up to 90355
2016/03/25 14:29:57 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-90355-1458916197462.tmp
2016/03/25 14:29:57 [INFO] raft: Compacting logs from 35354 to 80262
2016/03/25 14:29:58 [INFO] raft: Snapshot to 90355 complete
2016/03/25 14:33:55 [INFO] consul.fsm: snapshot created in 16.555µs
2016/03/25 14:33:55 [INFO] raft: Starting snapshot up to 149877
2016/03/25 14:33:55 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-149877-1458916435977.tmp
2016/03/25 14:33:56 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-45503-1458915986781
2016/03/25 14:33:56 [INFO] raft: Compacting logs from 80263 to 139901
2016/03/25 14:33:56 [INFO] raft: Snapshot to 149877 complete
2016/03/25 14:37:44 [INFO] consul.fsm: snapshot created in 18.405µs
2016/03/25 14:37:44 [INFO] raft: Starting snapshot up to 208647
2016/03/25 14:37:44 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-208647-1458916664702.tmp
2016/03/25 14:37:45 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-90355-1458916197462
2016/03/25 14:37:45 [INFO] raft: Compacting logs from 139902 to 198534
2016/03/25 14:37:45 [INFO] raft: Snapshot to 208647 complete
2016/03/25 14:41:02 [INFO] consul.fsm: snapshot created in 17.26µs
2016/03/25 14:41:02 [INFO] raft: Starting snapshot up to 263638
2016/03/25 14:41:02 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-263638-1458916862080.tmp
2016/03/25 14:41:03 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-149877-1458916435977
2016/03/25 14:41:03 [INFO] raft: Compacting logs from 198535 to 253678
2016/03/25 14:41:03 [INFO] raft: Snapshot to 263638 complete
2016/03/25 14:43:17 [INFO] consul.fsm: snapshot created in 20.332µs
2016/03/25 14:43:17 [INFO] raft: Starting snapshot up to 294599
2016/03/25 14:43:17 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-294599-1458916997129.tmp
2016/03/25 14:43:18 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-208647-1458916664702
2016/03/25 14:43:18 [INFO] raft: Compacting logs from 253679 to 284561
2016/03/25 14:43:18 [INFO] raft: Snapshot to 294599 complete
2016/03/25 14:45:26 [INFO] consul.fsm: snapshot created in 17.7µs
2016/03/25 14:45:26 [INFO] raft: Starting snapshot up to 321951
2016/03/25 14:45:26 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-321951-1458917126877.tmp
2016/03/25 14:45:28 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-263638-1458916862080
2016/03/25 14:45:28 [INFO] raft: Compacting logs from 284562 to 311933
2016/03/25 14:45:28 [INFO] raft: Snapshot to 321951 complete
2016/03/25 14:48:22 [INFO] consul.fsm: snapshot created in 19.147µs
2016/03/25 14:48:22 [INFO] raft: Starting snapshot up to 366774
2016/03/25 14:48:22 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-366774-1458917302085.tmp
2016/03/25 14:48:24 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-294599-1458916997129
2016/03/25 14:48:24 [INFO] raft: Compacting logs from 311934 to 356817
2016/03/25 14:48:24 [INFO] raft: Snapshot to 366774 complete
2016/03/25 14:52:13 [INFO] consul.fsm: snapshot created in 16.836µs
2016/03/25 14:52:13 [INFO] raft: Starting snapshot up to 424884
2016/03/25 14:52:13 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-424884-1458917533314.tmp
2016/03/25 14:52:15 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-321951-1458917126877
2016/03/25 14:52:15 [INFO] raft: Compacting logs from 356818 to 414885
2016/03/25 14:52:15 [INFO] raft: Snapshot to 424884 complete
2016/03/25 14:54:24 [INFO] consul.fsm: snapshot created in 26.19µs
2016/03/25 14:54:24 [INFO] raft: Starting snapshot up to 455144
2016/03/25 14:54:24 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-455144-1458917664226.tmp
2016/03/25 14:54:27 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-366774-1458917302085
2016/03/25 14:54:27 [INFO] raft: Compacting logs from 414886 to 445246
2016/03/25 14:54:27 [INFO] raft: Snapshot to 455144 complete
2016/03/25 14:57:06 [INFO] consul.fsm: snapshot created in 20.436µs
2016/03/25 14:57:06 [INFO] raft: Starting snapshot up to 487897
2016/03/25 14:57:06 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-487897-1458917826590.tmp
2016/03/25 14:57:09 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-424884-1458917533314
2016/03/25 14:57:09 [INFO] raft: Compacting logs from 445247 to 478052
2016/03/25 14:57:09 [INFO] raft: Snapshot to 487897 complete
2016/03/25 15:00:45 [INFO] consul.fsm: snapshot created in 20.879µs
2016/03/25 15:00:45 [INFO] raft: Starting snapshot up to 531135
2016/03/25 15:00:45 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-531135-1458918045930.tmp
2016/03/25 15:00:49 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-455144-1458917664226
2016/03/25 15:00:49 [INFO] raft: Compacting logs from 478053 to 521362
2016/03/25 15:00:49 [INFO] raft: Snapshot to 531135 complete
2016/03/25 15:04:38 [INFO] consul.fsm: snapshot created in 23.522µs
2016/03/25 15:04:38 [INFO] raft: Starting snapshot up to 582066
2016/03/25 15:04:38 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-582066-1458918278627.tmp
2016/03/25 15:04:42 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-487897-1458917826590
2016/03/25 15:04:42 [INFO] raft: Compacting logs from 521363 to 572224
2016/03/25 15:04:42 [INFO] raft: Snapshot to 582066 complete
2016/03/25 15:08:09 [INFO] consul.fsm: snapshot created in 24.211µs
2016/03/25 15:08:09 [INFO] raft: Starting snapshot up to 637073
2016/03/25 15:08:09 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-637073-1458918489829.tmp
2016/03/25 15:08:13 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-531135-1458918045930
2016/03/25 15:08:13 [INFO] raft: Compacting logs from 572225 to 627574
2016/03/25 15:08:13 [INFO] raft: Snapshot to 637073 complete
2016/03/25 15:10:21 [INFO] consul.fsm: snapshot created in 17.237µs
2016/03/25 15:10:21 [INFO] raft: Starting snapshot up to 669624
2016/03/25 15:10:21 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-669624-1458918621568.tmp
2016/03/25 15:10:25 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-582066-1458918278627
2016/03/25 15:10:25 [INFO] raft: Compacting logs from 627575 to 660456
2016/03/25 15:10:25 [INFO] raft: Snapshot to 669624 complete
2016/03/25 15:13:22 [INFO] consul.fsm: snapshot created in 19.144µs
2016/03/25 15:13:22 [INFO] raft: Starting snapshot up to 705703
2016/03/25 15:13:22 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-705703-1458918802235.tmp
2016/03/25 15:13:26 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-637073-1458918489829
2016/03/25 15:13:26 [INFO] raft: Compacting logs from 660457 to 696033
2016/03/25 15:13:26 [INFO] raft: Snapshot to 705703 complete
2016/03/25 15:16:28 [INFO] consul.fsm: snapshot created in 22.182µs
2016/03/25 15:16:28 [INFO] raft: Starting snapshot up to 748482
2016/03/25 15:16:28 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-748482-1458918988709.tmp
2016/03/25 15:16:33 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-669624-1458918621568
2016/03/25 15:16:33 [INFO] raft: Compacting logs from 696034 to 738686
2016/03/25 15:16:33 [INFO] raft: Snapshot to 748482 complete
2016/03/25 15:19:08 [INFO] consul.fsm: snapshot created in 16.99µs
2016/03/25 15:19:08 [INFO] raft: Starting snapshot up to 787153
2016/03/25 15:19:08 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-787153-1458919148423.tmp
2016/03/25 15:19:13 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-705703-1458918802235
2016/03/25 15:19:13 [INFO] raft: Compacting logs from 738687 to 778106
2016/03/25 15:19:13 [INFO] raft: Snapshot to 787153 complete
2016/03/25 15:21:53 [INFO] consul.fsm: snapshot created in 31.17µs
2016/03/25 15:21:53 [INFO] raft: Starting snapshot up to 823867
2016/03/25 15:21:53 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-823867-1458919313693.tmp
2016/03/25 15:21:58 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-748482-1458918988709
2016/03/25 15:21:58 [INFO] raft: Compacting logs from 778107 to 814340
2016/03/25 15:21:58 [INFO] raft: Snapshot to 823867 complete
2016/03/25 15:25:00 [INFO] consul.fsm: snapshot created in 24.096µs
2016/03/25 15:25:00 [INFO] raft: Starting snapshot up to 866877
2016/03/25 15:25:00 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-866877-1458919500050.tmp
2016/03/25 15:25:05 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-787153-1458919148423
2016/03/25 15:25:05 [INFO] raft: Compacting logs from 814341 to 857177
2016/03/25 15:25:05 [INFO] raft: Snapshot to 866877 complete
2016/03/25 15:27:18 [INFO] consul.fsm: snapshot created in 18.804µs
2016/03/25 15:27:18 [INFO] raft: Starting snapshot up to 897465
2016/03/25 15:27:18 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-897465-1458919638205.tmp
2016/03/25 15:27:23 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-823867-1458919313693
2016/03/25 15:27:23 [INFO] raft: Compacting logs from 857178 to 887890
2016/03/25 15:27:23 [INFO] raft: Snapshot to 897465 complete
2016/03/25 15:30:15 [INFO] consul.fsm: snapshot created in 19.802µs
2016/03/25 15:30:15 [INFO] raft: Starting snapshot up to 943569
2016/03/25 15:30:15 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-943569-1458919815210.tmp
2016/03/25 15:30:20 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-866877-1458919500050
2016/03/25 15:30:20 [INFO] raft: Compacting logs from 887891 to 934935
2016/03/25 15:30:20 [INFO] raft: Snapshot to 943569 complete
2016/03/25 15:32:36 [INFO] consul.fsm: snapshot created in 292.985µs
2016/03/25 15:32:36 [INFO] raft: Starting snapshot up to 974137
2016/03/25 15:32:36 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-974137-1458919956943.tmp
2016/03/25 15:32:42 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-897465-1458919638205
2016/03/25 15:32:42 [INFO] raft: Compacting logs from 934936 to 964708
2016/03/25 15:32:42 [INFO] raft: Snapshot to 974137 complete
2016/03/25 15:35:58 [INFO] consul.fsm: snapshot created in 30.32µs
2016/03/25 15:35:58 [INFO] raft: Starting snapshot up to 1000544
2016/03/25 15:35:58 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-1000544-1458920158409.tmp
2016/03/25 15:36:03 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-943569-1458919815210
2016/03/25 15:36:04 [INFO] raft: Compacting logs from 964709 to 990307
2016/03/25 15:36:04 [INFO] raft: Snapshot to 1000544 complete
==> Caught signal: terminated
2016/03/25 16:12:34 [INFO] agent: requesting shutdown
2016/03/25 16:12:34 [INFO] consul: shutting down server
2016/03/25 16:12:34 [WARN] serf: Shutdown without a Leave
2016/03/25 16:12:34 [ERR] dns: error starting tcp server: accept tcp 10.240.0.28:8600: use of closed network connection
2016/03/25 16:12:34 [WARN] serf: Shutdown without a Leave
2016/03/25 16:12:34 [INFO] raft: aborting pipeline replication to peer 10.240.0.30:8300
2016/03/25 16:12:34 [INFO] raft: aborting pipeline replication to peer 10.240.0.29:8300
2016/03/25 16:12:34 [INFO] agent: shutdown complete

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:17:05Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/consul agent -server -data-dir /mnt/ssd0/data.consul -bind 10.240.0.29 -client 10.240.0.29 -join 10.240.0.28","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/consul [PID: 23586]","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"Consul monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:58Z"}
{"level":"info","msg":"Stopping Consul [PID: 23586]","time":"2016-03-25T16:12:35Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/consul [PID: 23586]","time":"2016-03-25T16:12:35Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T16:12:35Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T16:12:35Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-02-consul-2-database.log","time":"2016-03-25T16:12:35Z"}
{"level":"info","msg":"/home/gyuho/go/bin/consul cmd.Wait returned exit status 1","time":"2016-03-25T16:12:35Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-02-consul-2-monitor.csv","time":"2016-03-25T16:12:37Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-02-consul-2-agent.log","time":"2016-03-25T16:12:38Z"}

View File

@ -0,0 +1,172 @@
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Joining cluster...
Join completed. Synced with 1 initial agents
==> Consul agent running!
Node name: 'bench-02-consul-2'
Datacenter: 'dc1'
Server: true (bootstrap: false)
Client Addr: 10.240.0.29 (HTTP: 8500, HTTPS: -1, DNS: 8600, RPC: 8400)
Cluster Addr: 10.240.0.29 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
Atlas: <disabled>
==> Log data will now stream in as it occurs:
2016/03/25 14:22:58 [INFO] raft: Node at 10.240.0.29:8300 [Follower] entering Follower state
2016/03/25 14:22:58 [INFO] serf: EventMemberJoin: bench-02-consul-2 10.240.0.29
2016/03/25 14:22:58 [INFO] consul: adding LAN server bench-02-consul-2 (Addr: 10.240.0.29:8300) (DC: dc1)
2016/03/25 14:22:58 [INFO] serf: EventMemberJoin: bench-02-consul-2.dc1 10.240.0.29
2016/03/25 14:22:58 [INFO] consul: adding WAN server bench-02-consul-2.dc1 (Addr: 10.240.0.29:8300) (DC: dc1)
2016/03/25 14:22:58 [INFO] agent: (LAN) joining: [10.240.0.28]
2016/03/25 14:22:58 [INFO] serf: EventMemberJoin: bench-02-consul-1 10.240.0.28
2016/03/25 14:22:58 [INFO] agent: (LAN) joined: 1 Err: <nil>
2016/03/25 14:22:58 [INFO] consul: adding LAN server bench-02-consul-1 (Addr: 10.240.0.28:8300) (DC: dc1)
2016/03/25 14:22:58 [ERR] agent: failed to sync remote state: No cluster leader
2016/03/25 14:22:59 [INFO] serf: EventMemberJoin: bench-02-consul-3 10.240.0.30
2016/03/25 14:22:59 [INFO] consul: adding LAN server bench-02-consul-3 (Addr: 10.240.0.30:8300) (DC: dc1)
2016/03/25 14:23:00 [INFO] consul: New leader elected: bench-02-consul-1
2016/03/25 14:23:00 [INFO] agent: Synced service 'consul'
2016/03/25 14:25:39 [INFO] consul.fsm: snapshot created in 36.062µs
2016/03/25 14:25:39 [INFO] raft: Starting snapshot up to 33849
2016/03/25 14:25:39 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-33849-1458915939678.tmp
2016/03/25 14:25:39 [INFO] raft: Compacting logs from 1 to 23650
2016/03/25 14:25:39 [INFO] raft: Snapshot to 33849 complete
2016/03/25 14:28:07 [INFO] consul.fsm: snapshot created in 19.59µs
2016/03/25 14:28:07 [INFO] raft: Starting snapshot up to 71912
2016/03/25 14:28:07 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-71912-1458916087679.tmp
2016/03/25 14:28:08 [INFO] raft: Compacting logs from 23651 to 61744
2016/03/25 14:28:08 [INFO] raft: Snapshot to 71912 complete
2016/03/25 14:31:31 [INFO] consul.fsm: snapshot created in 18.759µs
2016/03/25 14:31:31 [INFO] raft: Starting snapshot up to 118378
2016/03/25 14:31:31 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-118378-1458916291844.tmp
2016/03/25 14:31:32 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-33849-1458915939678
2016/03/25 14:31:32 [INFO] raft: Compacting logs from 61745 to 108282
2016/03/25 14:31:32 [INFO] raft: Snapshot to 118378 complete
2016/03/25 14:34:01 [INFO] consul.fsm: snapshot created in 18.495µs
2016/03/25 14:34:01 [INFO] raft: Starting snapshot up to 151478
2016/03/25 14:34:01 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-151478-1458916441765.tmp
2016/03/25 14:34:02 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-71912-1458916087679
2016/03/25 14:34:02 [INFO] raft: Compacting logs from 108283 to 141491
2016/03/25 14:34:02 [INFO] raft: Snapshot to 151478 complete
2016/03/25 14:37:31 [INFO] consul.fsm: snapshot created in 21.644µs
2016/03/25 14:37:31 [INFO] raft: Starting snapshot up to 206485
2016/03/25 14:37:31 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-206485-1458916651486.tmp
2016/03/25 14:37:32 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-118378-1458916291844
2016/03/25 14:37:32 [INFO] raft: Compacting logs from 141492 to 196573
2016/03/25 14:37:32 [INFO] raft: Snapshot to 206485 complete
2016/03/25 14:41:31 [INFO] consul.fsm: snapshot created in 23.028µs
2016/03/25 14:41:31 [INFO] raft: Starting snapshot up to 271595
2016/03/25 14:41:31 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-271595-1458916891781.tmp
2016/03/25 14:41:33 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-151478-1458916441765
2016/03/25 14:41:33 [INFO] raft: Compacting logs from 196574 to 261742
2016/03/25 14:41:33 [INFO] raft: Snapshot to 271595 complete
2016/03/25 14:44:05 [INFO] consul.fsm: snapshot created in 21.193µs
2016/03/25 14:44:05 [INFO] raft: Starting snapshot up to 307150
2016/03/25 14:44:05 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-307150-1458917045348.tmp
2016/03/25 14:44:07 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-206485-1458916651486
2016/03/25 14:44:07 [INFO] raft: Compacting logs from 261743 to 297421
2016/03/25 14:44:07 [INFO] raft: Snapshot to 307150 complete
2016/03/25 14:47:53 [INFO] consul.fsm: snapshot created in 31.834µs
2016/03/25 14:47:53 [INFO] raft: Starting snapshot up to 363538
2016/03/25 14:47:53 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-363538-1458917273240.tmp
2016/03/25 14:47:55 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-271595-1458916891781
2016/03/25 14:47:55 [INFO] raft: Compacting logs from 297422 to 353921
2016/03/25 14:47:55 [INFO] raft: Snapshot to 363538 complete
2016/03/25 14:51:42 [INFO] consul.fsm: snapshot created in 24.632µs
2016/03/25 14:51:42 [INFO] raft: Starting snapshot up to 421415
2016/03/25 14:51:42 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-421415-1458917502201.tmp
2016/03/25 14:51:44 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-307150-1458917045348
2016/03/25 14:51:44 [INFO] raft: Compacting logs from 353922 to 411954
2016/03/25 14:51:44 [INFO] raft: Snapshot to 421415 complete
2016/03/25 14:54:14 [INFO] consul.fsm: snapshot created in 25.589µs
2016/03/25 14:54:14 [INFO] raft: Starting snapshot up to 452794
2016/03/25 14:54:14 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-452794-1458917654998.tmp
2016/03/25 14:54:17 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-363538-1458917273240
2016/03/25 14:54:17 [INFO] raft: Compacting logs from 411955 to 443131
2016/03/25 14:54:17 [INFO] raft: Snapshot to 452794 complete
2016/03/25 14:56:33 [INFO] consul.fsm: snapshot created in 18.78µs
2016/03/25 14:56:33 [INFO] raft: Starting snapshot up to 481535
2016/03/25 14:56:33 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-481535-1458917793328.tmp
2016/03/25 14:56:36 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-421415-1458917502201
2016/03/25 14:56:36 [INFO] raft: Compacting logs from 443132 to 471884
2016/03/25 14:56:36 [INFO] raft: Snapshot to 481535 complete
2016/03/25 14:59:26 [INFO] consul.fsm: snapshot created in 20.418µs
2016/03/25 14:59:26 [INFO] raft: Starting snapshot up to 519531
2016/03/25 14:59:26 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-519531-1458917966046.tmp
2016/03/25 14:59:29 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-452794-1458917654998
2016/03/25 14:59:29 [INFO] raft: Compacting logs from 471885 to 510025
2016/03/25 14:59:29 [INFO] raft: Snapshot to 519531 complete
2016/03/25 15:03:17 [INFO] consul.fsm: snapshot created in 22.6µs
2016/03/25 15:03:17 [INFO] raft: Starting snapshot up to 569912
2016/03/25 15:03:17 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-569912-1458918197868.tmp
2016/03/25 15:03:21 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-481535-1458917793328
2016/03/25 15:03:21 [INFO] raft: Compacting logs from 510026 to 560618
2016/03/25 15:03:21 [INFO] raft: Snapshot to 569912 complete
2016/03/25 15:06:38 [INFO] consul.fsm: snapshot created in 31.669µs
2016/03/25 15:06:38 [INFO] raft: Starting snapshot up to 612880
2016/03/25 15:06:38 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-612880-1458918398876.tmp
2016/03/25 15:06:42 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-519531-1458917966046
2016/03/25 15:06:42 [INFO] raft: Compacting logs from 560619 to 603662
2016/03/25 15:06:42 [INFO] raft: Snapshot to 612880 complete
2016/03/25 15:09:08 [INFO] consul.fsm: snapshot created in 44.97µs
2016/03/25 15:09:08 [INFO] raft: Starting snapshot up to 651667
2016/03/25 15:09:08 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-651667-1458918548250.tmp
2016/03/25 15:09:12 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-569912-1458918197868
2016/03/25 15:09:12 [INFO] raft: Compacting logs from 603663 to 642409
2016/03/25 15:09:12 [INFO] raft: Snapshot to 651667 complete
2016/03/25 15:11:51 [INFO] consul.fsm: snapshot created in 19.131µs
2016/03/25 15:11:51 [INFO] raft: Starting snapshot up to 695045
2016/03/25 15:11:51 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-695045-1458918711174.tmp
2016/03/25 15:11:55 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-612880-1458918398876
2016/03/25 15:11:55 [INFO] raft: Compacting logs from 642410 to 685979
2016/03/25 15:11:55 [INFO] raft: Snapshot to 695045 complete
2016/03/25 15:15:42 [INFO] consul.fsm: snapshot created in 19.228µs
2016/03/25 15:15:42 [INFO] raft: Starting snapshot up to 742267
2016/03/25 15:15:42 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-742267-1458918942658.tmp
2016/03/25 15:15:47 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-651667-1458918548250
2016/03/25 15:15:47 [INFO] raft: Compacting logs from 685980 to 733265
2016/03/25 15:15:47 [INFO] raft: Snapshot to 742267 complete
2016/03/25 15:18:18 [INFO] consul.fsm: snapshot created in 24.827µs
2016/03/25 15:18:18 [INFO] raft: Starting snapshot up to 774088
2016/03/25 15:18:18 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-774088-1458919098543.tmp
2016/03/25 15:18:23 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-695045-1458918711174
2016/03/25 15:18:23 [INFO] raft: Compacting logs from 733266 to 765166
2016/03/25 15:18:23 [INFO] raft: Snapshot to 774088 complete
2016/03/25 15:20:59 [INFO] consul.fsm: snapshot created in 20.12µs
2016/03/25 15:20:59 [INFO] raft: Starting snapshot up to 816947
2016/03/25 15:20:59 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-816947-1458919259343.tmp
2016/03/25 15:21:04 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-742267-1458918942658
2016/03/25 15:21:04 [INFO] raft: Compacting logs from 765167 to 807959
2016/03/25 15:21:04 [INFO] raft: Snapshot to 816947 complete
2016/03/25 15:24:34 [INFO] consul.fsm: snapshot created in 21.258µs
2016/03/25 15:24:34 [INFO] raft: Starting snapshot up to 863543
2016/03/25 15:24:34 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-863543-1458919474216.tmp
2016/03/25 15:24:39 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-774088-1458919098543
2016/03/25 15:24:39 [INFO] raft: Compacting logs from 807960 to 854519
2016/03/25 15:24:39 [INFO] raft: Snapshot to 863543 complete
2016/03/25 15:26:54 [INFO] consul.fsm: snapshot created in 19.3µs
2016/03/25 15:26:54 [INFO] raft: Starting snapshot up to 894514
2016/03/25 15:26:54 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-894514-1458919614328.tmp
2016/03/25 15:26:59 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-816947-1458919259343
2016/03/25 15:26:59 [INFO] raft: Compacting logs from 854520 to 885486
2016/03/25 15:26:59 [INFO] raft: Snapshot to 894514 complete
2016/03/25 15:30:39 [INFO] consul.fsm: snapshot created in 21.891µs
2016/03/25 15:30:39 [INFO] raft: Starting snapshot up to 950166
2016/03/25 15:30:39 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-950166-1458919839092.tmp
2016/03/25 15:30:44 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-863543-1458919474216
2016/03/25 15:30:44 [INFO] raft: Compacting logs from 885487 to 941487
2016/03/25 15:30:45 [INFO] raft: Snapshot to 950166 complete
2016/03/25 15:34:04 [INFO] consul.fsm: snapshot created in 18.744µs
2016/03/25 15:34:04 [INFO] raft: Starting snapshot up to 996848
2016/03/25 15:34:04 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-996848-1458920044286.tmp
2016/03/25 15:34:10 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-894514-1458919614328
2016/03/25 15:34:10 [INFO] raft: Compacting logs from 941488 to 988399
2016/03/25 15:34:10 [INFO] raft: Snapshot to 996848 complete
==> Caught signal: terminated
2016/03/25 16:12:35 [INFO] agent: requesting shutdown
2016/03/25 16:12:35 [INFO] consul: shutting down server
2016/03/25 16:12:35 [WARN] serf: Shutdown without a Leave
2016/03/25 16:12:35 [ERR] dns: error starting tcp server: accept tcp 10.240.0.29:8600: use of closed network connection
2016/03/25 16:12:35 [WARN] serf: Shutdown without a Leave
2016/03/25 16:12:35 [INFO] agent: shutdown complete

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:17:03Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/consul agent -server -data-dir /mnt/ssd0/data.consul -bind 10.240.0.30 -client 10.240.0.30 -join 10.240.0.28","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/consul [PID: 23394]","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"Consul monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:59Z"}
{"level":"info","msg":"Stopping Consul [PID: 23394]","time":"2016-03-25T16:12:36Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/consul [PID: 23394]","time":"2016-03-25T16:12:36Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T16:12:36Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T16:12:36Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-02-consul-3-database.log","time":"2016-03-25T16:12:36Z"}
{"level":"info","msg":"/home/gyuho/go/bin/consul cmd.Wait returned exit status 1","time":"2016-03-25T16:12:36Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-02-consul-3-monitor.csv","time":"2016-03-25T16:12:37Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-02-consul-3-agent.log","time":"2016-03-25T16:12:38Z"}

View File

@ -0,0 +1,182 @@
==> Starting Consul agent...
==> Starting Consul agent RPC...
==> Joining cluster...
Join completed. Synced with 1 initial agents
==> Consul agent running!
Node name: 'bench-02-consul-3'
Datacenter: 'dc1'
Server: true (bootstrap: false)
Client Addr: 10.240.0.30 (HTTP: 8500, HTTPS: -1, DNS: 8600, RPC: 8400)
Cluster Addr: 10.240.0.30 (LAN: 8301, WAN: 8302)
Gossip encrypt: false, RPC-TLS: false, TLS-Incoming: false
Atlas: <disabled>
==> Log data will now stream in as it occurs:
2016/03/25 14:22:59 [INFO] raft: Node at 10.240.0.30:8300 [Follower] entering Follower state
2016/03/25 14:22:59 [INFO] serf: EventMemberJoin: bench-02-consul-3 10.240.0.30
2016/03/25 14:22:59 [INFO] consul: adding LAN server bench-02-consul-3 (Addr: 10.240.0.30:8300) (DC: dc1)
2016/03/25 14:22:59 [INFO] serf: EventMemberJoin: bench-02-consul-3.dc1 10.240.0.30
2016/03/25 14:22:59 [INFO] consul: adding WAN server bench-02-consul-3.dc1 (Addr: 10.240.0.30:8300) (DC: dc1)
2016/03/25 14:22:59 [INFO] agent: (LAN) joining: [10.240.0.28]
2016/03/25 14:22:59 [INFO] serf: EventMemberJoin: bench-02-consul-1 10.240.0.28
2016/03/25 14:22:59 [INFO] serf: EventMemberJoin: bench-02-consul-2 10.240.0.29
2016/03/25 14:22:59 [INFO] agent: (LAN) joined: 1 Err: <nil>
2016/03/25 14:22:59 [INFO] consul: adding LAN server bench-02-consul-1 (Addr: 10.240.0.28:8300) (DC: dc1)
2016/03/25 14:22:59 [INFO] consul: adding LAN server bench-02-consul-2 (Addr: 10.240.0.29:8300) (DC: dc1)
2016/03/25 14:22:59 [ERR] agent: failed to sync remote state: No cluster leader
2016/03/25 14:23:00 [INFO] consul: New leader elected: bench-02-consul-1
2016/03/25 14:23:00 [INFO] agent: Synced service 'consul'
2016/03/25 14:25:38 [INFO] consul.fsm: snapshot created in 23.637µs
2016/03/25 14:25:38 [INFO] raft: Starting snapshot up to 33540
2016/03/25 14:25:38 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-33540-1458915938194.tmp
2016/03/25 14:25:38 [INFO] raft: Compacting logs from 1 to 23360
2016/03/25 14:25:38 [INFO] raft: Snapshot to 33540 complete
2016/03/25 14:29:32 [INFO] consul.fsm: snapshot created in 18.054µs
2016/03/25 14:29:32 [INFO] raft: Starting snapshot up to 84714
2016/03/25 14:29:32 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-84714-1458916172601.tmp
2016/03/25 14:29:33 [INFO] raft: Compacting logs from 23361 to 74588
2016/03/25 14:29:33 [INFO] raft: Snapshot to 84714 complete
2016/03/25 14:32:18 [INFO] consul.fsm: snapshot created in 17.195µs
2016/03/25 14:32:18 [INFO] raft: Starting snapshot up to 131829
2016/03/25 14:32:18 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-131829-1458916338216.tmp
2016/03/25 14:32:18 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-33540-1458915938194
2016/03/25 14:32:18 [INFO] raft: Compacting logs from 74589 to 121831
2016/03/25 14:32:18 [INFO] raft: Snapshot to 131829 complete
2016/03/25 14:34:52 [INFO] consul.fsm: snapshot created in 18.665µs
2016/03/25 14:34:52 [INFO] raft: Starting snapshot up to 165872
2016/03/25 14:34:52 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-165872-1458916492230.tmp
2016/03/25 14:34:53 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-84714-1458916172601
2016/03/25 14:34:53 [INFO] raft: Compacting logs from 121832 to 155930
2016/03/25 14:34:53 [INFO] raft: Snapshot to 165872 complete
2016/03/25 14:37:34 [INFO] consul.fsm: snapshot created in 19.747µs
2016/03/25 14:37:34 [INFO] raft: Starting snapshot up to 207464
2016/03/25 14:37:34 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-207464-1458916654105.tmp
2016/03/25 14:37:35 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-131829-1458916338216
2016/03/25 14:37:35 [INFO] raft: Compacting logs from 155931 to 197487
2016/03/25 14:37:35 [INFO] raft: Snapshot to 207464 complete
2016/03/25 14:39:54 [INFO] consul.fsm: snapshot created in 23.499µs
2016/03/25 14:39:54 [INFO] raft: Starting snapshot up to 244644
2016/03/25 14:39:54 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-244644-1458916794841.tmp
2016/03/25 14:39:56 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-165872-1458916492230
2016/03/25 14:39:56 [INFO] raft: Compacting logs from 197488 to 234839
2016/03/25 14:39:56 [INFO] raft: Snapshot to 244644 complete
2016/03/25 14:42:32 [INFO] consul.fsm: snapshot created in 30.791µs
2016/03/25 14:42:32 [INFO] raft: Starting snapshot up to 289225
2016/03/25 14:42:32 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-289225-1458916952460.tmp
2016/03/25 14:42:34 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-207464-1458916654105
2016/03/25 14:42:34 [INFO] raft: Compacting logs from 234840 to 279497
2016/03/25 14:42:34 [INFO] raft: Snapshot to 289225 complete
2016/03/25 14:44:39 [INFO] consul.fsm: snapshot created in 20.161µs
2016/03/25 14:44:39 [INFO] raft: Starting snapshot up to 316496
2016/03/25 14:44:39 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-316496-1458917079936.tmp
2016/03/25 14:44:41 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-244644-1458916794841
2016/03/25 14:44:41 [INFO] raft: Compacting logs from 279498 to 306737
2016/03/25 14:44:41 [INFO] raft: Snapshot to 316496 complete
2016/03/25 14:47:45 [INFO] consul.fsm: snapshot created in 17.68µs
2016/03/25 14:47:45 [INFO] raft: Starting snapshot up to 361017
2016/03/25 14:47:45 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-361017-1458917265424.tmp
2016/03/25 14:47:47 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-289225-1458916952460
2016/03/25 14:47:47 [INFO] raft: Compacting logs from 306738 to 351508
2016/03/25 14:47:47 [INFO] raft: Snapshot to 361017 complete
2016/03/25 14:51:39 [INFO] consul.fsm: snapshot created in 34.595µs
2016/03/25 14:51:39 [INFO] raft: Starting snapshot up to 420286
2016/03/25 14:51:39 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-420286-1458917499264.tmp
2016/03/25 14:51:41 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-316496-1458917079936
2016/03/25 14:51:41 [INFO] raft: Compacting logs from 351509 to 410767
2016/03/25 14:51:41 [INFO] raft: Snapshot to 420286 complete
2016/03/25 14:54:31 [INFO] consul.fsm: snapshot created in 18.201µs
2016/03/25 14:54:31 [INFO] raft: Starting snapshot up to 456657
2016/03/25 14:54:31 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-456657-1458917671542.tmp
2016/03/25 14:54:34 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-361017-1458917265424
2016/03/25 14:54:34 [INFO] raft: Compacting logs from 410768 to 447156
2016/03/25 14:54:34 [INFO] raft: Snapshot to 456657 complete
2016/03/25 14:57:40 [INFO] consul.fsm: snapshot created in 21.1µs
2016/03/25 14:57:40 [INFO] raft: Starting snapshot up to 496793
2016/03/25 14:57:40 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-496793-1458917860944.tmp
2016/03/25 14:57:44 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-420286-1458917499264
2016/03/25 14:57:44 [INFO] raft: Compacting logs from 447157 to 487348
2016/03/25 14:57:44 [INFO] raft: Snapshot to 496793 complete
2016/03/25 15:01:01 [INFO] consul.fsm: snapshot created in 20.261µs
2016/03/25 15:01:01 [INFO] raft: Starting snapshot up to 535079
2016/03/25 15:01:01 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-535079-1458918061972.tmp
2016/03/25 15:01:05 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-456657-1458917671542
2016/03/25 15:01:05 [INFO] raft: Compacting logs from 487349 to 525758
2016/03/25 15:01:05 [INFO] raft: Snapshot to 535079 complete
2016/03/25 15:04:16 [INFO] consul.fsm: snapshot created in 19.455µs
2016/03/25 15:04:16 [INFO] raft: Starting snapshot up to 579855
2016/03/25 15:04:16 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-579855-1458918256629.tmp
2016/03/25 15:04:20 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-496793-1458917860944
2016/03/25 15:04:20 [INFO] raft: Compacting logs from 525759 to 570089
2016/03/25 15:04:20 [INFO] raft: Snapshot to 579855 complete
2016/03/25 15:08:07 [INFO] consul.fsm: snapshot created in 19.434µs
2016/03/25 15:08:07 [INFO] raft: Starting snapshot up to 636478
2016/03/25 15:08:07 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-636478-1458918487860.tmp
2016/03/25 15:08:11 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-535079-1458918061972
2016/03/25 15:08:11 [INFO] raft: Compacting logs from 570090 to 627400
2016/03/25 15:08:11 [INFO] raft: Snapshot to 636478 complete
2016/03/25 15:11:19 [INFO] consul.fsm: snapshot created in 17.25µs
2016/03/25 15:11:19 [INFO] raft: Starting snapshot up to 686041
2016/03/25 15:11:19 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-686041-1458918679867.tmp
2016/03/25 15:11:23 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-579855-1458918256629
2016/03/25 15:11:24 [INFO] raft: Compacting logs from 627401 to 676991
2016/03/25 15:11:24 [INFO] raft: Snapshot to 686041 complete
2016/03/25 15:14:06 [INFO] consul.fsm: snapshot created in 20.433µs
2016/03/25 15:14:06 [INFO] raft: Starting snapshot up to 716882
2016/03/25 15:14:06 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-716882-1458918846860.tmp
2016/03/25 15:14:11 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-636478-1458918487860
2016/03/25 15:14:11 [INFO] raft: Compacting logs from 676992 to 707888
2016/03/25 15:14:11 [INFO] raft: Snapshot to 716882 complete
2016/03/25 15:16:24 [INFO] consul.fsm: snapshot created in 20.533µs
2016/03/25 15:16:24 [INFO] raft: Starting snapshot up to 747909
2016/03/25 15:16:24 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-747909-1458918984327.tmp
2016/03/25 15:16:28 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-686041-1458918679867
2016/03/25 15:16:28 [INFO] raft: Compacting logs from 707889 to 738261
2016/03/25 15:16:28 [INFO] raft: Snapshot to 747909 complete
2016/03/25 15:19:41 [INFO] consul.fsm: snapshot created in 36.913µs
2016/03/25 15:19:41 [INFO] raft: Starting snapshot up to 796207
2016/03/25 15:19:41 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-796207-1458919181651.tmp
2016/03/25 15:19:46 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-716882-1458918846860
2016/03/25 15:19:46 [INFO] raft: Compacting logs from 738262 to 787151
2016/03/25 15:19:46 [INFO] raft: Snapshot to 796207 complete
2016/03/25 15:23:17 [INFO] consul.fsm: snapshot created in 21.82µs
2016/03/25 15:23:17 [INFO] raft: Starting snapshot up to 846505
2016/03/25 15:23:17 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-846505-1458919397533.tmp
2016/03/25 15:23:22 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-747909-1458918984327
2016/03/25 15:23:22 [INFO] raft: Compacting logs from 787152 to 837791
2016/03/25 15:23:22 [INFO] raft: Snapshot to 846505 complete
2016/03/25 15:26:11 [INFO] consul.fsm: snapshot created in 21.151µs
2016/03/25 15:26:11 [INFO] raft: Starting snapshot up to 885149
2016/03/25 15:26:11 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-885149-1458919571732.tmp
2016/03/25 15:26:17 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-796207-1458919181651
2016/03/25 15:26:17 [INFO] raft: Compacting logs from 837792 to 876426
2016/03/25 15:26:17 [INFO] raft: Snapshot to 885149 complete
2016/03/25 15:29:44 [INFO] consul.fsm: snapshot created in 17.369µs
2016/03/25 15:29:44 [INFO] raft: Starting snapshot up to 936547
2016/03/25 15:29:44 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-936547-1458919784291.tmp
2016/03/25 15:29:49 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-846505-1458919397533
2016/03/25 15:29:49 [INFO] raft: Compacting logs from 876427 to 928071
2016/03/25 15:29:49 [INFO] raft: Snapshot to 936547 complete
2016/03/25 15:31:54 [INFO] consul.fsm: snapshot created in 26.469µs
2016/03/25 15:31:54 [INFO] raft: Starting snapshot up to 968868
2016/03/25 15:31:54 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-968868-1458919914232.tmp
2016/03/25 15:32:00 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-885149-1458919571732
2016/03/25 15:32:00 [INFO] raft: Compacting logs from 928072 to 960007
2016/03/25 15:32:00 [INFO] raft: Snapshot to 968868 complete
2016/03/25 15:34:16 [INFO] consul.fsm: snapshot created in 21.449µs
2016/03/25 15:34:16 [INFO] raft: Starting snapshot up to 1000350
2016/03/25 15:34:16 [INFO] snapshot: Creating new snapshot at /mnt/ssd0/data.consul/raft/snapshots/1-1000350-1458920056458.tmp
2016/03/25 15:34:22 [INFO] snapshot: reaping snapshot /mnt/ssd0/data.consul/raft/snapshots/1-936547-1458919784291
2016/03/25 15:34:22 [INFO] raft: Compacting logs from 960008 to 990293
2016/03/25 15:34:22 [INFO] raft: Snapshot to 1000350 complete
2016/03/25 16:12:35 [WARN] raft: Heartbeat timeout reached, starting election
2016/03/25 16:12:35 [INFO] raft: Node at 10.240.0.30:8300 [Candidate] entering Candidate state
2016/03/25 16:12:35 [ERR] raft: Failed to make RequestVote RPC to 10.240.0.28:8300: dial tcp 10.240.0.28:8300: getsockopt: connection refused
2016/03/25 16:12:35 [ERR] raft: Failed to make RequestVote RPC to 10.240.0.29:8300: dial tcp 10.240.0.29:8300: getsockopt: connection refused
==> Caught signal: terminated
2016/03/25 16:12:36 [INFO] agent: requesting shutdown
2016/03/25 16:12:36 [INFO] consul: shutting down server
2016/03/25 16:12:36 [WARN] serf: Shutdown without a Leave
2016/03/25 16:12:36 [ERR] dns: error starting tcp server: accept tcp 10.240.0.30:8600: use of closed network connection
2016/03/25 16:12:36 [WARN] serf: Shutdown without a Leave
2016/03/25 16:12:36 [INFO] agent: shutdown complete

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:17:43Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-1 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.32:2379 --advertise-client-urls http://10.240.0.32:2379 --listen-peer-urls http://10.240.0.32:2380 --initial-advertise-peer-urls http://10.240.0.32:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.32:2380,etcd-2=http://10.240.0.33:2380,etcd-3=http://10.240.0.34:2380 --initial-cluster-state new","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23389]","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"etcd monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:27Z"}
{"level":"info","msg":"Stopping etcd [PID: 23389]","time":"2016-03-25T14:53:54Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23389]","time":"2016-03-25T14:53:54Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:53:54Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:53:54Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-02-etcd-1-database.log","time":"2016-03-25T14:53:54Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:53:54Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-02-etcd-1-monitor.csv","time":"2016-03-25T14:53:55Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-02-etcd-1-agent.log","time":"2016-03-25T14:53:55Z"}

View File

@ -0,0 +1,457 @@
2016-03-25 14:22:27.777636 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:22:27.777700 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:22:27.777708 I | etcdmain: Go Version: go1.6
2016-03-25 14:22:27.777714 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:22:27.777723 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:22:27.777777 I | etcdmain: listening for peers on http://10.240.0.32:2380
2016-03-25 14:22:27.777803 I | etcdmain: listening for client requests on 10.240.0.32:2379
2016-03-25 14:22:27.781460 I | etcdserver: name = etcd-1
2016-03-25 14:22:27.781476 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:22:27.781483 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:22:27.781490 I | etcdserver: heartbeat = 100ms
2016-03-25 14:22:27.781495 I | etcdserver: election = 1000ms
2016-03-25 14:22:27.781501 I | etcdserver: snapshot count = 10000
2016-03-25 14:22:27.781510 I | etcdserver: advertise client URLs = http://10.240.0.32:2379
2016-03-25 14:22:27.781517 I | etcdserver: initial advertise peer URLs = http://10.240.0.32:2380
2016-03-25 14:22:27.781531 I | etcdserver: initial cluster = etcd-1=http://10.240.0.32:2380,etcd-2=http://10.240.0.33:2380,etcd-3=http://10.240.0.34:2380
2016-03-25 14:22:27.789130 I | etcdserver: starting member 185c55dd02f81a06 in cluster c5560b3524bdd10e
2016-03-25 14:22:27.789180 I | raft: 185c55dd02f81a06 became follower at term 0
2016-03-25 14:22:27.789201 I | raft: newRaft 185c55dd02f81a06 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:22:27.789209 I | raft: 185c55dd02f81a06 became follower at term 1
2016-03-25 14:22:27.829065 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:22:27.830400 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:22:27.830421 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:22:27.830516 N | etcdmain: serving insecure client requests on 10.240.0.32:2379, this is strongly discouraged!
2016-03-25 14:22:27.830788 N | etcdserver: added local member 185c55dd02f81a06 [http://10.240.0.32:2380] to cluster c5560b3524bdd10e
2016-03-25 14:22:27.830885 N | etcdserver: added member a55bd2c806d46279 [http://10.240.0.34:2380] to cluster c5560b3524bdd10e
2016-03-25 14:22:27.830949 N | etcdserver: added member faf98ebdef076a19 [http://10.240.0.33:2380] to cluster c5560b3524bdd10e
2016-03-25 14:22:28.489562 I | raft: 185c55dd02f81a06 is starting a new election at term 1
2016-03-25 14:22:28.489656 I | raft: 185c55dd02f81a06 became candidate at term 2
2016-03-25 14:22:28.489665 I | raft: 185c55dd02f81a06 received vote from 185c55dd02f81a06 at term 2
2016-03-25 14:22:28.489675 I | raft: 185c55dd02f81a06 [logterm: 1, index: 3] sent vote request to a55bd2c806d46279 at term 2
2016-03-25 14:22:28.489685 I | raft: 185c55dd02f81a06 [logterm: 1, index: 3] sent vote request to faf98ebdef076a19 at term 2
2016-03-25 14:22:28.814374 I | rafthttp: the connection with faf98ebdef076a19 became active
2016-03-25 14:22:28.986624 I | raft: 185c55dd02f81a06 [logterm: 1, index: 3, vote: 185c55dd02f81a06] rejected vote from faf98ebdef076a19 [logterm: 1, index: 3] at term 2
2016-03-25 14:22:29.789566 I | raft: 185c55dd02f81a06 is starting a new election at term 2
2016-03-25 14:22:29.789613 I | raft: 185c55dd02f81a06 became candidate at term 3
2016-03-25 14:22:29.789622 I | raft: 185c55dd02f81a06 received vote from 185c55dd02f81a06 at term 3
2016-03-25 14:22:29.789632 I | raft: 185c55dd02f81a06 [logterm: 1, index: 3] sent vote request to faf98ebdef076a19 at term 3
2016-03-25 14:22:29.789642 I | raft: 185c55dd02f81a06 [logterm: 1, index: 3] sent vote request to a55bd2c806d46279 at term 3
2016-03-25 14:22:29.796732 I | raft: 185c55dd02f81a06 received vote from faf98ebdef076a19 at term 3
2016-03-25 14:22:29.796762 I | raft: 185c55dd02f81a06 [quorum:2] has received 2 votes and 0 vote rejections
2016-03-25 14:22:29.796850 I | raft: 185c55dd02f81a06 became leader at term 3
2016-03-25 14:22:29.796889 I | raft: raft.node: 185c55dd02f81a06 elected leader 185c55dd02f81a06 at term 3
2016-03-25 14:22:29.805281 I | etcdserver: published {Name:etcd-1 ClientURLs:[http://10.240.0.32:2379]} to cluster c5560b3524bdd10e
2016-03-25 14:22:29.827782 I | rafthttp: the connection with a55bd2c806d46279 became active
2016-03-25 14:22:29.828746 I | etcdserver: setting up the initial cluster version to 2.3
2016-03-25 14:22:29.832739 N | etcdserver: set the initial cluster version to 2.3
2016-03-25 14:23:19.728939 I | etcdserver: start to snapshot (applied: 10001, lastsnap: 0)
2016-03-25 14:23:19.743219 I | etcdserver: saved snapshot at index 10001
2016-03-25 14:23:19.743448 I | etcdserver: compacted raft log at 5001
2016-03-25 14:23:35.003995 I | etcdserver: start to snapshot (applied: 20003, lastsnap: 10001)
2016-03-25 14:23:35.042878 I | etcdserver: saved snapshot at index 20003
2016-03-25 14:23:35.043173 I | etcdserver: compacted raft log at 15003
2016-03-25 14:23:50.477880 I | etcdserver: start to snapshot (applied: 30005, lastsnap: 20003)
2016-03-25 14:23:50.496091 I | etcdserver: saved snapshot at index 30005
2016-03-25 14:23:50.496305 I | etcdserver: compacted raft log at 25005
2016-03-25 14:24:06.011196 I | etcdserver: start to snapshot (applied: 40008, lastsnap: 30005)
2016-03-25 14:24:06.030885 I | etcdserver: saved snapshot at index 40008
2016-03-25 14:24:06.031126 I | etcdserver: compacted raft log at 35008
2016-03-25 14:24:21.173456 I | etcdserver: start to snapshot (applied: 50014, lastsnap: 40008)
2016-03-25 14:24:21.194188 I | etcdserver: saved snapshot at index 50014
2016-03-25 14:24:21.194387 I | etcdserver: compacted raft log at 45014
2016-03-25 14:24:36.494716 I | etcdserver: start to snapshot (applied: 60023, lastsnap: 50014)
2016-03-25 14:24:36.522715 I | etcdserver: saved snapshot at index 60023
2016-03-25 14:24:36.522951 I | etcdserver: compacted raft log at 55023
2016-03-25 14:24:51.908053 I | etcdserver: start to snapshot (applied: 70024, lastsnap: 60023)
2016-03-25 14:24:51.929914 I | etcdserver: saved snapshot at index 70024
2016-03-25 14:24:51.930039 I | etcdserver: compacted raft log at 65024
2016-03-25 14:24:57.831129 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002711.snap successfully
2016-03-25 14:24:57.831171 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e23.snap successfully
2016-03-25 14:25:07.152015 I | etcdserver: start to snapshot (applied: 80027, lastsnap: 70024)
2016-03-25 14:25:07.169815 I | etcdserver: saved snapshot at index 80027
2016-03-25 14:25:07.170033 I | etcdserver: compacted raft log at 75027
2016-03-25 14:25:22.544613 I | etcdserver: start to snapshot (applied: 90029, lastsnap: 80027)
2016-03-25 14:25:22.556980 I | etcdserver: saved snapshot at index 90029
2016-03-25 14:25:22.557240 I | etcdserver: compacted raft log at 85029
2016-03-25 14:25:27.831387 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007535.snap successfully
2016-03-25 14:25:27.831442 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c48.snap successfully
2016-03-25 14:25:38.558476 I | etcdserver: start to snapshot (applied: 100033, lastsnap: 90029)
2016-03-25 14:25:38.588608 I | etcdserver: saved snapshot at index 100033
2016-03-25 14:25:38.588795 I | etcdserver: compacted raft log at 95033
2016-03-25 14:25:47.878787 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 87.66627ms)
2016-03-25 14:25:47.878815 W | etcdserver: server is likely overloaded
2016-03-25 14:25:47.878824 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 87.718331ms)
2016-03-25 14:25:47.878829 W | etcdserver: server is likely overloaded
2016-03-25 14:25:54.327350 I | etcdserver: start to snapshot (applied: 110034, lastsnap: 100033)
2016-03-25 14:25:54.348363 I | etcdserver: saved snapshot at index 110034
2016-03-25 14:25:54.348555 I | etcdserver: compacted raft log at 105034
2016-03-25 14:25:57.831715 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c35e.snap successfully
2016-03-25 14:25:57.831770 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea77.snap successfully
2016-03-25 14:26:09.856253 I | etcdserver: start to snapshot (applied: 120037, lastsnap: 110034)
2016-03-25 14:26:09.882760 I | etcdserver: saved snapshot at index 120037
2016-03-25 14:26:09.882915 I | etcdserver: compacted raft log at 115037
2016-03-25 14:26:25.330293 I | etcdserver: start to snapshot (applied: 130038, lastsnap: 120037)
2016-03-25 14:26:25.343261 I | etcdserver: saved snapshot at index 130038
2016-03-25 14:26:25.343512 I | etcdserver: compacted raft log at 125038
2016-03-25 14:26:27.832017 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000011188.snap successfully
2016-03-25 14:26:27.832069 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001389b.snap successfully
2016-03-25 14:26:40.562370 I | etcdserver: start to snapshot (applied: 140044, lastsnap: 130038)
2016-03-25 14:26:40.573784 I | etcdserver: saved snapshot at index 140044
2016-03-25 14:26:40.574003 I | etcdserver: compacted raft log at 135044
2016-03-25 14:26:56.000420 I | etcdserver: start to snapshot (applied: 150048, lastsnap: 140044)
2016-03-25 14:26:56.014995 I | etcdserver: saved snapshot at index 150048
2016-03-25 14:26:56.015112 I | etcdserver: compacted raft log at 145048
2016-03-25 14:26:57.832305 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015fad.snap successfully
2016-03-25 14:26:57.832354 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186c1.snap successfully
2016-03-25 14:27:11.492315 I | etcdserver: start to snapshot (applied: 160049, lastsnap: 150048)
2016-03-25 14:27:11.508171 I | etcdserver: saved snapshot at index 160049
2016-03-25 14:27:11.508288 I | etcdserver: compacted raft log at 155049
2016-03-25 14:27:25.252725 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-000000000002940d.wal is created
2016-03-25 14:27:26.902279 I | etcdserver: start to snapshot (applied: 170055, lastsnap: 160049)
2016-03-25 14:27:26.924401 I | etcdserver: saved snapshot at index 170055
2016-03-25 14:27:26.924540 I | etcdserver: compacted raft log at 165055
2016-03-25 14:27:27.832613 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001add2.snap successfully
2016-03-25 14:27:27.832657 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4e5.snap successfully
2016-03-25 14:27:42.598988 I | etcdserver: start to snapshot (applied: 180062, lastsnap: 170055)
2016-03-25 14:27:42.624543 I | etcdserver: saved snapshot at index 180062
2016-03-25 14:27:42.624792 I | etcdserver: compacted raft log at 175062
2016-03-25 14:27:57.832897 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbf6.snap successfully
2016-03-25 14:27:58.133858 I | etcdserver: start to snapshot (applied: 190063, lastsnap: 180062)
2016-03-25 14:27:58.160326 I | etcdserver: saved snapshot at index 190063
2016-03-25 14:27:58.160502 I | etcdserver: compacted raft log at 185063
2016-03-25 14:28:13.763775 I | etcdserver: start to snapshot (applied: 200069, lastsnap: 190063)
2016-03-25 14:28:13.775381 I | etcdserver: saved snapshot at index 200069
2016-03-25 14:28:13.775597 I | etcdserver: compacted raft log at 195069
2016-03-25 14:28:27.833208 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002230c.snap successfully
2016-03-25 14:28:27.833263 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000024a20.snap successfully
2016-03-25 14:28:29.320040 I | etcdserver: start to snapshot (applied: 210070, lastsnap: 200069)
2016-03-25 14:28:29.333943 I | etcdserver: saved snapshot at index 210070
2016-03-25 14:28:29.334085 I | etcdserver: compacted raft log at 205070
2016-03-25 14:28:44.728700 I | etcdserver: start to snapshot (applied: 220071, lastsnap: 210070)
2016-03-25 14:28:44.748259 I | etcdserver: saved snapshot at index 220071
2016-03-25 14:28:44.748414 I | etcdserver: compacted raft log at 215071
2016-03-25 14:28:57.833519 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027131.snap successfully
2016-03-25 14:28:57.833561 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000029847.snap successfully
2016-03-25 14:29:00.274623 I | etcdserver: start to snapshot (applied: 230072, lastsnap: 220071)
2016-03-25 14:29:00.298138 I | etcdserver: saved snapshot at index 230072
2016-03-25 14:29:00.298341 I | etcdserver: compacted raft log at 225072
2016-03-25 14:29:15.657531 I | etcdserver: start to snapshot (applied: 240075, lastsnap: 230072)
2016-03-25 14:29:15.679531 I | etcdserver: saved snapshot at index 240075
2016-03-25 14:29:15.679723 I | etcdserver: compacted raft log at 235075
2016-03-25 14:29:27.833791 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf5e.snap successfully
2016-03-25 14:29:27.833834 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e66f.snap successfully
2016-03-25 14:29:31.177601 I | etcdserver: start to snapshot (applied: 250078, lastsnap: 240075)
2016-03-25 14:29:31.189037 I | etcdserver: saved snapshot at index 250078
2016-03-25 14:29:31.189265 I | etcdserver: compacted raft log at 245078
2016-03-25 14:29:46.368493 I | etcdserver: start to snapshot (applied: 260079, lastsnap: 250078)
2016-03-25 14:29:46.392820 I | etcdserver: saved snapshot at index 260079
2016-03-25 14:29:46.392957 I | etcdserver: compacted raft log at 255079
2016-03-25 14:29:57.834078 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d85.snap successfully
2016-03-25 14:29:57.834141 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000033496.snap successfully
2016-03-25 14:30:01.741659 I | etcdserver: start to snapshot (applied: 270080, lastsnap: 260079)
2016-03-25 14:30:01.762593 I | etcdserver: saved snapshot at index 270080
2016-03-25 14:30:01.762718 I | etcdserver: compacted raft log at 265080
2016-03-25 14:30:17.230801 I | etcdserver: start to snapshot (applied: 280083, lastsnap: 270080)
2016-03-25 14:30:17.252717 I | etcdserver: saved snapshot at index 280083
2016-03-25 14:30:17.252832 I | etcdserver: compacted raft log at 275083
2016-03-25 14:30:27.834353 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035ba7.snap successfully
2016-03-25 14:30:27.834402 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000382b8.snap successfully
2016-03-25 14:30:33.024160 I | etcdserver: start to snapshot (applied: 290084, lastsnap: 280083)
2016-03-25 14:30:33.042500 I | etcdserver: saved snapshot at index 290084
2016-03-25 14:30:33.042648 I | etcdserver: compacted raft log at 285084
2016-03-25 14:30:48.562708 I | etcdserver: start to snapshot (applied: 300086, lastsnap: 290084)
2016-03-25 14:30:48.584859 I | etcdserver: saved snapshot at index 300086
2016-03-25 14:30:48.584984 I | etcdserver: compacted raft log at 295086
2016-03-25 14:30:57.834635 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a9cb.snap successfully
2016-03-25 14:30:57.834677 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0de.snap successfully
2016-03-25 14:31:04.111557 I | etcdserver: start to snapshot (applied: 310088, lastsnap: 300086)
2016-03-25 14:31:04.148125 I | etcdserver: saved snapshot at index 310088
2016-03-25 14:31:04.148333 I | etcdserver: compacted raft log at 305088
2016-03-25 14:31:19.705710 I | etcdserver: start to snapshot (applied: 320090, lastsnap: 310088)
2016-03-25 14:31:19.726352 I | etcdserver: saved snapshot at index 320090
2016-03-25 14:31:19.726634 I | etcdserver: compacted raft log at 315090
2016-03-25 14:31:27.834976 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003f7ef.snap successfully
2016-03-25 14:31:27.835032 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000041f00.snap successfully
2016-03-25 14:31:35.510011 I | etcdserver: start to snapshot (applied: 330094, lastsnap: 320090)
2016-03-25 14:31:35.530651 I | etcdserver: saved snapshot at index 330094
2016-03-25 14:31:35.530832 I | etcdserver: compacted raft log at 325094
2016-03-25 14:31:47.568388 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000002-00000000000527b4.wal is created
2016-03-25 14:31:51.176681 I | etcdserver: start to snapshot (applied: 340096, lastsnap: 330094)
2016-03-25 14:31:51.191419 I | etcdserver: saved snapshot at index 340096
2016-03-25 14:31:51.191574 I | etcdserver: compacted raft log at 335096
2016-03-25 14:31:57.835335 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000044613.snap successfully
2016-03-25 14:31:57.835380 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000046d24.snap successfully
2016-03-25 14:32:06.796227 I | etcdserver: start to snapshot (applied: 350098, lastsnap: 340096)
2016-03-25 14:32:06.821783 I | etcdserver: saved snapshot at index 350098
2016-03-25 14:32:06.821995 I | etcdserver: compacted raft log at 345098
2016-03-25 14:32:22.152742 I | etcdserver: start to snapshot (applied: 360099, lastsnap: 350098)
2016-03-25 14:32:22.179143 I | etcdserver: saved snapshot at index 360099
2016-03-25 14:32:22.179281 I | etcdserver: compacted raft log at 355099
2016-03-25 14:32:27.835635 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000049436.snap successfully
2016-03-25 14:32:27.835682 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004bb48.snap successfully
2016-03-25 14:32:37.868926 I | etcdserver: start to snapshot (applied: 370100, lastsnap: 360099)
2016-03-25 14:32:37.885909 I | etcdserver: saved snapshot at index 370100
2016-03-25 14:32:37.886035 I | etcdserver: compacted raft log at 365100
2016-03-25 14:32:53.550019 I | etcdserver: start to snapshot (applied: 380104, lastsnap: 370100)
2016-03-25 14:32:53.572408 I | etcdserver: saved snapshot at index 380104
2016-03-25 14:32:53.572552 I | etcdserver: compacted raft log at 375104
2016-03-25 14:32:57.835933 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004e25a.snap successfully
2016-03-25 14:32:57.835978 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005096e.snap successfully
2016-03-25 14:33:09.230557 I | etcdserver: start to snapshot (applied: 390107, lastsnap: 380104)
2016-03-25 14:33:09.259367 I | etcdserver: saved snapshot at index 390107
2016-03-25 14:33:09.260216 I | etcdserver: compacted raft log at 385107
2016-03-25 14:33:24.905928 I | etcdserver: start to snapshot (applied: 400109, lastsnap: 390107)
2016-03-25 14:33:24.923374 I | etcdserver: saved snapshot at index 400109
2016-03-25 14:33:24.923537 I | etcdserver: compacted raft log at 395109
2016-03-25 14:33:27.836312 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000053080.snap successfully
2016-03-25 14:33:27.836402 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000055792.snap successfully
2016-03-25 14:33:40.483566 I | etcdserver: start to snapshot (applied: 410110, lastsnap: 400109)
2016-03-25 14:33:40.502031 I | etcdserver: saved snapshot at index 410110
2016-03-25 14:33:40.502198 I | etcdserver: compacted raft log at 405110
2016-03-25 14:33:56.229515 I | etcdserver: start to snapshot (applied: 420114, lastsnap: 410110)
2016-03-25 14:33:56.248919 I | etcdserver: saved snapshot at index 420114
2016-03-25 14:33:56.249045 I | etcdserver: compacted raft log at 415114
2016-03-25 14:33:57.836712 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000057ea3.snap successfully
2016-03-25 14:33:57.836762 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005a5b4.snap successfully
2016-03-25 14:34:11.887576 I | etcdserver: start to snapshot (applied: 430119, lastsnap: 420114)
2016-03-25 14:34:11.906063 I | etcdserver: saved snapshot at index 430119
2016-03-25 14:34:11.906194 I | etcdserver: compacted raft log at 425119
2016-03-25 14:34:27.798026 I | etcdserver: start to snapshot (applied: 440120, lastsnap: 430119)
2016-03-25 14:34:27.832126 I | etcdserver: saved snapshot at index 440120
2016-03-25 14:34:27.832340 I | etcdserver: compacted raft log at 435120
2016-03-25 14:34:27.836981 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005ccc8.snap successfully
2016-03-25 14:34:27.837021 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005f3db.snap successfully
2016-03-25 14:34:43.634363 I | etcdserver: start to snapshot (applied: 450121, lastsnap: 440120)
2016-03-25 14:34:43.652166 I | etcdserver: saved snapshot at index 450121
2016-03-25 14:34:43.652290 I | etcdserver: compacted raft log at 445121
2016-03-25 14:34:57.837329 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000061aed.snap successfully
2016-03-25 14:34:59.357595 I | etcdserver: start to snapshot (applied: 460122, lastsnap: 450121)
2016-03-25 14:34:59.375645 I | etcdserver: saved snapshot at index 460122
2016-03-25 14:34:59.375791 I | etcdserver: compacted raft log at 455122
2016-03-25 14:35:14.735284 I | etcdserver: start to snapshot (applied: 470123, lastsnap: 460122)
2016-03-25 14:35:14.772439 I | etcdserver: saved snapshot at index 470123
2016-03-25 14:35:14.772579 I | etcdserver: compacted raft log at 465123
2016-03-25 14:35:27.837699 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000641fe.snap successfully
2016-03-25 14:35:27.837749 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000066912.snap successfully
2016-03-25 14:35:30.189678 I | etcdserver: start to snapshot (applied: 480125, lastsnap: 470123)
2016-03-25 14:35:30.214112 I | etcdserver: saved snapshot at index 480125
2016-03-25 14:35:30.214254 I | etcdserver: compacted raft log at 475125
2016-03-25 14:35:45.774435 I | etcdserver: start to snapshot (applied: 490127, lastsnap: 480125)
2016-03-25 14:35:45.793996 I | etcdserver: saved snapshot at index 490127
2016-03-25 14:35:45.794136 I | etcdserver: compacted raft log at 485127
2016-03-25 14:35:57.838005 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000069027.snap successfully
2016-03-25 14:35:57.838052 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006b738.snap successfully
2016-03-25 14:36:01.340747 I | etcdserver: start to snapshot (applied: 500128, lastsnap: 490127)
2016-03-25 14:36:01.355334 I | etcdserver: saved snapshot at index 500128
2016-03-25 14:36:01.355633 I | etcdserver: compacted raft log at 495128
2016-03-25 14:36:11.584855 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000003-000000000007bb44.wal is created
2016-03-25 14:36:16.793671 I | etcdserver: start to snapshot (applied: 510129, lastsnap: 500128)
2016-03-25 14:36:16.814494 I | etcdserver: saved snapshot at index 510129
2016-03-25 14:36:16.814628 I | etcdserver: compacted raft log at 505129
2016-03-25 14:36:27.838318 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006de49.snap successfully
2016-03-25 14:36:27.838370 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007055a.snap successfully
2016-03-25 14:36:32.414511 I | etcdserver: start to snapshot (applied: 520131, lastsnap: 510129)
2016-03-25 14:36:32.435639 I | etcdserver: saved snapshot at index 520131
2016-03-25 14:36:32.435792 I | etcdserver: compacted raft log at 515131
2016-03-25 14:36:48.220244 I | etcdserver: start to snapshot (applied: 530135, lastsnap: 520131)
2016-03-25 14:36:48.243318 I | etcdserver: saved snapshot at index 530135
2016-03-25 14:36:48.243473 I | etcdserver: compacted raft log at 525135
2016-03-25 14:36:57.838598 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000072c6b.snap successfully
2016-03-25 14:36:57.838653 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007537d.snap successfully
2016-03-25 14:37:03.761185 I | etcdserver: start to snapshot (applied: 540139, lastsnap: 530135)
2016-03-25 14:37:03.780896 I | etcdserver: saved snapshot at index 540139
2016-03-25 14:37:03.781040 I | etcdserver: compacted raft log at 535139
2016-03-25 14:37:19.246721 I | etcdserver: start to snapshot (applied: 550140, lastsnap: 540139)
2016-03-25 14:37:19.263235 I | etcdserver: saved snapshot at index 550140
2016-03-25 14:37:19.263456 I | etcdserver: compacted raft log at 545140
2016-03-25 14:37:27.838895 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000077a8f.snap successfully
2016-03-25 14:37:27.838939 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007a1a0.snap successfully
2016-03-25 14:37:34.667006 I | etcdserver: start to snapshot (applied: 560146, lastsnap: 550140)
2016-03-25 14:37:34.690490 I | etcdserver: saved snapshot at index 560146
2016-03-25 14:37:34.690622 I | etcdserver: compacted raft log at 555146
2016-03-25 14:37:50.164349 I | etcdserver: start to snapshot (applied: 570150, lastsnap: 560146)
2016-03-25 14:37:50.192405 I | etcdserver: saved snapshot at index 570150
2016-03-25 14:37:50.192569 I | etcdserver: compacted raft log at 565150
2016-03-25 14:37:57.839207 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007c8b1.snap successfully
2016-03-25 14:37:57.839251 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007efc3.snap successfully
2016-03-25 14:38:05.522628 I | etcdserver: start to snapshot (applied: 580154, lastsnap: 570150)
2016-03-25 14:38:05.545700 I | etcdserver: saved snapshot at index 580154
2016-03-25 14:38:05.545834 I | etcdserver: compacted raft log at 575154
2016-03-25 14:38:21.032319 I | etcdserver: start to snapshot (applied: 590157, lastsnap: 580154)
2016-03-25 14:38:21.045590 I | etcdserver: saved snapshot at index 590157
2016-03-25 14:38:21.045717 I | etcdserver: compacted raft log at 585157
2016-03-25 14:38:27.839517 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000816d7.snap successfully
2016-03-25 14:38:27.839583 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000083deb.snap successfully
2016-03-25 14:38:36.569259 I | etcdserver: start to snapshot (applied: 600159, lastsnap: 590157)
2016-03-25 14:38:36.588938 I | etcdserver: saved snapshot at index 600159
2016-03-25 14:38:36.589055 I | etcdserver: compacted raft log at 595159
2016-03-25 14:38:51.973793 I | etcdserver: start to snapshot (applied: 610169, lastsnap: 600159)
2016-03-25 14:38:51.987739 I | etcdserver: saved snapshot at index 610169
2016-03-25 14:38:51.987875 I | etcdserver: compacted raft log at 605169
2016-03-25 14:38:57.839849 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000864fc.snap successfully
2016-03-25 14:38:57.839915 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000088c12.snap successfully
2016-03-25 14:39:07.407649 I | etcdserver: start to snapshot (applied: 620171, lastsnap: 610169)
2016-03-25 14:39:07.430793 I | etcdserver: saved snapshot at index 620171
2016-03-25 14:39:07.430917 I | etcdserver: compacted raft log at 615171
2016-03-25 14:39:23.067848 I | etcdserver: start to snapshot (applied: 630173, lastsnap: 620171)
2016-03-25 14:39:23.089412 I | etcdserver: saved snapshot at index 630173
2016-03-25 14:39:23.089557 I | etcdserver: compacted raft log at 625173
2016-03-25 14:39:27.840257 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008b326.snap successfully
2016-03-25 14:39:27.840309 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008da3a.snap successfully
2016-03-25 14:39:38.539353 I | etcdserver: start to snapshot (applied: 640175, lastsnap: 630173)
2016-03-25 14:39:38.559938 I | etcdserver: saved snapshot at index 640175
2016-03-25 14:39:38.560067 I | etcdserver: compacted raft log at 635175
2016-03-25 14:39:54.284044 I | etcdserver: start to snapshot (applied: 650179, lastsnap: 640175)
2016-03-25 14:39:54.306974 I | etcdserver: saved snapshot at index 650179
2016-03-25 14:39:54.307131 I | etcdserver: compacted raft log at 645179
2016-03-25 14:39:57.840584 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009014d.snap successfully
2016-03-25 14:39:57.840626 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009285f.snap successfully
2016-03-25 14:40:09.608879 I | etcdserver: start to snapshot (applied: 660182, lastsnap: 650179)
2016-03-25 14:40:09.621915 I | etcdserver: saved snapshot at index 660182
2016-03-25 14:40:09.622039 I | etcdserver: compacted raft log at 655182
2016-03-25 14:40:25.401761 I | etcdserver: start to snapshot (applied: 670185, lastsnap: 660182)
2016-03-25 14:40:25.425094 I | etcdserver: saved snapshot at index 670185
2016-03-25 14:40:25.425242 I | etcdserver: compacted raft log at 665185
2016-03-25 14:40:27.840891 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000094f79.snap successfully
2016-03-25 14:40:27.840939 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009768b.snap successfully
2016-03-25 14:40:33.735729 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000004-00000000000a4edb.wal is created
2016-03-25 14:40:40.924950 I | etcdserver: start to snapshot (applied: 680186, lastsnap: 670185)
2016-03-25 14:40:40.950488 I | etcdserver: saved snapshot at index 680186
2016-03-25 14:40:40.950609 I | etcdserver: compacted raft log at 675186
2016-03-25 14:40:56.648438 I | etcdserver: start to snapshot (applied: 690187, lastsnap: 680186)
2016-03-25 14:40:56.672846 I | etcdserver: saved snapshot at index 690187
2016-03-25 14:40:56.672987 I | etcdserver: compacted raft log at 685187
2016-03-25 14:40:57.841219 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000099d9d.snap successfully
2016-03-25 14:40:57.841266 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009c4af.snap successfully
2016-03-25 14:41:12.028795 I | etcdserver: start to snapshot (applied: 700191, lastsnap: 690187)
2016-03-25 14:41:12.057368 I | etcdserver: saved snapshot at index 700191
2016-03-25 14:41:12.057499 I | etcdserver: compacted raft log at 695191
2016-03-25 14:41:27.841554 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009ebc3.snap successfully
2016-03-25 14:41:27.895268 I | etcdserver: start to snapshot (applied: 710197, lastsnap: 700191)
2016-03-25 14:41:27.914992 I | etcdserver: saved snapshot at index 710197
2016-03-25 14:41:27.915107 I | etcdserver: compacted raft log at 705197
2016-03-25 14:41:43.542049 I | etcdserver: start to snapshot (applied: 720201, lastsnap: 710197)
2016-03-25 14:41:43.562322 I | etcdserver: saved snapshot at index 720201
2016-03-25 14:41:43.562476 I | etcdserver: compacted raft log at 715201
2016-03-25 14:41:57.841796 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a12d6.snap successfully
2016-03-25 14:41:57.841840 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a39e9.snap successfully
2016-03-25 14:41:59.245813 I | etcdserver: start to snapshot (applied: 730205, lastsnap: 720201)
2016-03-25 14:41:59.261616 I | etcdserver: saved snapshot at index 730205
2016-03-25 14:41:59.261754 I | etcdserver: compacted raft log at 725205
2016-03-25 14:42:14.875760 I | etcdserver: start to snapshot (applied: 740206, lastsnap: 730205)
2016-03-25 14:42:14.894829 I | etcdserver: saved snapshot at index 740206
2016-03-25 14:42:14.894984 I | etcdserver: compacted raft log at 735206
2016-03-25 14:42:27.842110 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a60fa.snap successfully
2016-03-25 14:42:27.842162 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a880b.snap successfully
2016-03-25 14:42:30.525058 I | etcdserver: start to snapshot (applied: 750210, lastsnap: 740206)
2016-03-25 14:42:30.540153 I | etcdserver: saved snapshot at index 750210
2016-03-25 14:42:30.540294 I | etcdserver: compacted raft log at 745210
2016-03-25 14:42:46.262372 I | etcdserver: start to snapshot (applied: 760211, lastsnap: 750210)
2016-03-25 14:42:46.276444 I | etcdserver: saved snapshot at index 760211
2016-03-25 14:42:46.276601 I | etcdserver: compacted raft log at 755211
2016-03-25 14:42:57.842412 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000aaf1f.snap successfully
2016-03-25 14:42:57.842467 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000ad635.snap successfully
2016-03-25 14:43:01.759392 I | etcdserver: start to snapshot (applied: 770214, lastsnap: 760211)
2016-03-25 14:43:01.779620 I | etcdserver: saved snapshot at index 770214
2016-03-25 14:43:01.779742 I | etcdserver: compacted raft log at 765214
2016-03-25 14:43:17.617729 I | etcdserver: start to snapshot (applied: 780219, lastsnap: 770214)
2016-03-25 14:43:17.641424 I | etcdserver: saved snapshot at index 780219
2016-03-25 14:43:17.641562 I | etcdserver: compacted raft log at 775219
2016-03-25 14:43:27.842694 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000afd49.snap successfully
2016-03-25 14:43:27.842740 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b245d.snap successfully
2016-03-25 14:43:33.122739 I | etcdserver: start to snapshot (applied: 790222, lastsnap: 780219)
2016-03-25 14:43:33.143036 I | etcdserver: saved snapshot at index 790222
2016-03-25 14:43:33.143180 I | etcdserver: compacted raft log at 785222
2016-03-25 14:43:49.001942 I | etcdserver: start to snapshot (applied: 800223, lastsnap: 790222)
2016-03-25 14:43:49.017663 I | etcdserver: saved snapshot at index 800223
2016-03-25 14:43:49.017780 I | etcdserver: compacted raft log at 795223
2016-03-25 14:43:57.842963 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b4b6e.snap successfully
2016-03-25 14:43:57.843006 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b7282.snap successfully
2016-03-25 14:44:04.805673 I | etcdserver: start to snapshot (applied: 810225, lastsnap: 800223)
2016-03-25 14:44:04.826540 I | etcdserver: saved snapshot at index 810225
2016-03-25 14:44:04.826665 I | etcdserver: compacted raft log at 805225
2016-03-25 14:44:20.381516 I | etcdserver: start to snapshot (applied: 820228, lastsnap: 810225)
2016-03-25 14:44:20.416258 I | etcdserver: saved snapshot at index 820228
2016-03-25 14:44:20.416376 I | etcdserver: compacted raft log at 815228
2016-03-25 14:44:27.843291 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b9993.snap successfully
2016-03-25 14:44:27.843400 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000bc0a6.snap successfully
2016-03-25 14:44:36.203869 I | etcdserver: start to snapshot (applied: 830230, lastsnap: 820228)
2016-03-25 14:44:36.221242 I | etcdserver: saved snapshot at index 830230
2016-03-25 14:44:36.221374 I | etcdserver: compacted raft log at 825230
2016-03-25 14:44:51.591984 I | etcdserver: start to snapshot (applied: 840236, lastsnap: 830230)
2016-03-25 14:44:51.611239 I | etcdserver: saved snapshot at index 840236
2016-03-25 14:44:51.611365 I | etcdserver: compacted raft log at 835236
2016-03-25 14:44:57.843601 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000be7bb.snap successfully
2016-03-25 14:44:57.843644 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c0ece.snap successfully
2016-03-25 14:44:58.110487 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000005-00000000000ce280.wal is created
2016-03-25 14:45:07.105867 I | etcdserver: start to snapshot (applied: 850237, lastsnap: 840236)
2016-03-25 14:45:07.127200 I | etcdserver: saved snapshot at index 850237
2016-03-25 14:45:07.127477 I | etcdserver: compacted raft log at 845237
2016-03-25 14:45:23.096726 I | etcdserver: start to snapshot (applied: 860239, lastsnap: 850237)
2016-03-25 14:45:23.112642 I | etcdserver: saved snapshot at index 860239
2016-03-25 14:45:23.112762 I | etcdserver: compacted raft log at 855239
2016-03-25 14:45:27.843913 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c35df.snap successfully
2016-03-25 14:45:27.843975 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c5cf1.snap successfully
2016-03-25 14:45:27.845753 I | fileutil: purged file /mnt/ssd0/data.etcd/member/wal/0000000000000000-0000000000000000.wal successfully
2016-03-25 14:45:38.774030 I | etcdserver: start to snapshot (applied: 870240, lastsnap: 860239)
2016-03-25 14:45:38.805113 I | etcdserver: saved snapshot at index 870240
2016-03-25 14:45:38.805250 I | etcdserver: compacted raft log at 865240
2016-03-25 14:45:54.247291 I | etcdserver: start to snapshot (applied: 880241, lastsnap: 870240)
2016-03-25 14:45:54.279902 I | etcdserver: saved snapshot at index 880241
2016-03-25 14:45:54.280037 I | etcdserver: compacted raft log at 875241
2016-03-25 14:45:57.844206 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c8404.snap successfully
2016-03-25 14:45:57.844250 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cab16.snap successfully
2016-03-25 14:46:10.190873 I | etcdserver: start to snapshot (applied: 890244, lastsnap: 880241)
2016-03-25 14:46:10.209365 I | etcdserver: saved snapshot at index 890244
2016-03-25 14:46:10.209513 I | etcdserver: compacted raft log at 885244
2016-03-25 14:46:26.013598 I | etcdserver: start to snapshot (applied: 900245, lastsnap: 890244)
2016-03-25 14:46:26.031688 I | etcdserver: saved snapshot at index 900245
2016-03-25 14:46:26.031822 I | etcdserver: compacted raft log at 895245
2016-03-25 14:46:27.844516 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cd22c.snap successfully
2016-03-25 14:46:27.844565 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cf93d.snap successfully
2016-03-25 14:46:41.941215 I | etcdserver: start to snapshot (applied: 910247, lastsnap: 900245)
2016-03-25 14:46:41.952000 I | etcdserver: saved snapshot at index 910247
2016-03-25 14:46:41.952129 I | etcdserver: compacted raft log at 905247
2016-03-25 14:46:57.599460 I | etcdserver: start to snapshot (applied: 920252, lastsnap: 910247)
2016-03-25 14:46:57.623503 I | etcdserver: saved snapshot at index 920252
2016-03-25 14:46:57.623628 I | etcdserver: compacted raft log at 915252
2016-03-25 14:46:57.844850 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d204f.snap successfully
2016-03-25 14:46:57.844918 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d4760.snap successfully
2016-03-25 14:47:13.294063 I | etcdserver: start to snapshot (applied: 930254, lastsnap: 920252)
2016-03-25 14:47:13.326566 I | etcdserver: saved snapshot at index 930254
2016-03-25 14:47:13.326693 I | etcdserver: compacted raft log at 925254
2016-03-25 14:47:27.845190 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d6e71.snap successfully
2016-03-25 14:47:29.211663 I | etcdserver: start to snapshot (applied: 940257, lastsnap: 930254)
2016-03-25 14:47:29.250649 I | etcdserver: saved snapshot at index 940257
2016-03-25 14:47:29.250794 I | etcdserver: compacted raft log at 935257
2016-03-25 14:47:44.879522 I | etcdserver: start to snapshot (applied: 950258, lastsnap: 940257)
2016-03-25 14:47:44.894697 I | etcdserver: saved snapshot at index 950258
2016-03-25 14:47:44.894846 I | etcdserver: compacted raft log at 945258
2016-03-25 14:47:57.845437 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d9584.snap successfully
2016-03-25 14:47:57.845488 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000dbc95.snap successfully
2016-03-25 14:48:00.414113 I | etcdserver: start to snapshot (applied: 960259, lastsnap: 950258)
2016-03-25 14:48:00.433682 I | etcdserver: saved snapshot at index 960259
2016-03-25 14:48:00.433827 I | etcdserver: compacted raft log at 955259
2016-03-25 14:48:15.982457 I | etcdserver: start to snapshot (applied: 970260, lastsnap: 960259)
2016-03-25 14:48:15.999364 I | etcdserver: saved snapshot at index 970260
2016-03-25 14:48:15.999515 I | etcdserver: compacted raft log at 965260
2016-03-25 14:48:27.845824 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000de3a7.snap successfully
2016-03-25 14:48:27.845877 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e0abc.snap successfully
2016-03-25 14:48:31.712752 I | etcdserver: start to snapshot (applied: 980261, lastsnap: 970260)
2016-03-25 14:48:31.740772 I | etcdserver: saved snapshot at index 980261
2016-03-25 14:48:31.740918 I | etcdserver: compacted raft log at 975261
2016-03-25 14:48:47.382419 I | etcdserver: start to snapshot (applied: 990264, lastsnap: 980261)
2016-03-25 14:48:47.401451 I | etcdserver: saved snapshot at index 990264
2016-03-25 14:48:47.401576 I | etcdserver: compacted raft log at 985264
2016-03-25 14:48:57.846173 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e31ce.snap successfully
2016-03-25 14:48:57.846223 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e58e1.snap successfully
2016-03-25 14:49:02.633130 I | etcdserver: start to snapshot (applied: 1000267, lastsnap: 990264)
2016-03-25 14:49:02.656997 I | etcdserver: saved snapshot at index 1000267
2016-03-25 14:49:02.657154 I | etcdserver: compacted raft log at 995267
2016-03-25 14:49:27.846621 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e7ff2.snap successfully
2016-03-25 14:53:54.416475 N | osutil: received terminated signal, shutting down...
2016-03-25 14:53:54.417578 E | rafthttp: failed to read a55bd2c806d46279 on stream MsgApp v2 (net/http: request canceled)
2016-03-25 14:53:54.417592 I | rafthttp: the connection with a55bd2c806d46279 became inactive
2016-03-25 14:53:54.418824 E | rafthttp: failed to read faf98ebdef076a19 on stream MsgApp v2 (net/http: request canceled)
2016-03-25 14:53:54.418852 I | rafthttp: the connection with faf98ebdef076a19 became inactive

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:17:44Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-2 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.33:2379 --advertise-client-urls http://10.240.0.33:2379 --listen-peer-urls http://10.240.0.33:2380 --initial-advertise-peer-urls http://10.240.0.33:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.32:2380,etcd-2=http://10.240.0.33:2380,etcd-3=http://10.240.0.34:2380 --initial-cluster-state new","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23373]","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"etcd monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:28Z"}
{"level":"info","msg":"Stopping etcd [PID: 23373]","time":"2016-03-25T14:53:55Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23373]","time":"2016-03-25T14:53:55Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:53:55Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:53:55Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-02-etcd-2-database.log","time":"2016-03-25T14:53:55Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:53:55Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-02-etcd-2-monitor.csv","time":"2016-03-25T14:53:56Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-02-etcd-2-agent.log","time":"2016-03-25T14:53:56Z"}

View File

@ -0,0 +1,448 @@
2016-03-25 14:22:28.782840 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:22:28.782891 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:22:28.782898 I | etcdmain: Go Version: go1.6
2016-03-25 14:22:28.782912 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:22:28.782920 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:22:28.782988 I | etcdmain: listening for peers on http://10.240.0.33:2380
2016-03-25 14:22:28.783015 I | etcdmain: listening for client requests on 10.240.0.33:2379
2016-03-25 14:22:28.784727 I | etcdserver: name = etcd-2
2016-03-25 14:22:28.784752 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:22:28.784759 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:22:28.784766 I | etcdserver: heartbeat = 100ms
2016-03-25 14:22:28.784771 I | etcdserver: election = 1000ms
2016-03-25 14:22:28.784777 I | etcdserver: snapshot count = 10000
2016-03-25 14:22:28.784786 I | etcdserver: advertise client URLs = http://10.240.0.33:2379
2016-03-25 14:22:28.784793 I | etcdserver: initial advertise peer URLs = http://10.240.0.33:2380
2016-03-25 14:22:28.784807 I | etcdserver: initial cluster = etcd-1=http://10.240.0.32:2380,etcd-2=http://10.240.0.33:2380,etcd-3=http://10.240.0.34:2380
2016-03-25 14:22:28.792034 I | etcdserver: starting member faf98ebdef076a19 in cluster c5560b3524bdd10e
2016-03-25 14:22:28.792083 I | raft: faf98ebdef076a19 became follower at term 0
2016-03-25 14:22:28.792097 I | raft: newRaft faf98ebdef076a19 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:22:28.792103 I | raft: faf98ebdef076a19 became follower at term 1
2016-03-25 14:22:28.823586 I | rafthttp: the connection with 185c55dd02f81a06 became active
2016-03-25 14:22:28.824341 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:22:28.825306 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:22:28.825322 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:22:28.825426 N | etcdmain: serving insecure client requests on 10.240.0.33:2379, this is strongly discouraged!
2016-03-25 14:22:28.825541 N | etcdserver: added member 185c55dd02f81a06 [http://10.240.0.32:2380] to cluster c5560b3524bdd10e
2016-03-25 14:22:28.825629 N | etcdserver: added member a55bd2c806d46279 [http://10.240.0.34:2380] to cluster c5560b3524bdd10e
2016-03-25 14:22:28.825705 N | etcdserver: added local member faf98ebdef076a19 [http://10.240.0.33:2380] to cluster c5560b3524bdd10e
2016-03-25 14:22:28.992403 I | raft: faf98ebdef076a19 is starting a new election at term 1
2016-03-25 14:22:28.992488 I | raft: faf98ebdef076a19 became candidate at term 2
2016-03-25 14:22:28.992498 I | raft: faf98ebdef076a19 received vote from faf98ebdef076a19 at term 2
2016-03-25 14:22:28.992515 I | raft: faf98ebdef076a19 [logterm: 1, index: 3] sent vote request to 185c55dd02f81a06 at term 2
2016-03-25 14:22:28.992525 I | raft: faf98ebdef076a19 [logterm: 1, index: 3] sent vote request to a55bd2c806d46279 at term 2
2016-03-25 14:22:28.995557 I | raft: faf98ebdef076a19 received vote rejection from 185c55dd02f81a06 at term 2
2016-03-25 14:22:28.995577 I | raft: faf98ebdef076a19 [quorum:2] has received 1 votes and 1 vote rejections
2016-03-25 14:22:29.802776 I | raft: faf98ebdef076a19 [term: 2] received a MsgVote message with higher term from 185c55dd02f81a06 [term: 3]
2016-03-25 14:22:29.802823 I | raft: faf98ebdef076a19 became follower at term 3
2016-03-25 14:22:29.802837 I | raft: faf98ebdef076a19 [logterm: 1, index: 3, vote: 0] voted for 185c55dd02f81a06 [logterm: 1, index: 3] at term 3
2016-03-25 14:22:29.805987 I | raft: raft.node: faf98ebdef076a19 elected leader 185c55dd02f81a06 at term 3
2016-03-25 14:22:29.814240 I | etcdserver: published {Name:etcd-2 ClientURLs:[http://10.240.0.33:2379]} to cluster c5560b3524bdd10e
2016-03-25 14:22:29.836408 I | rafthttp: the connection with a55bd2c806d46279 became active
2016-03-25 14:22:29.841622 N | etcdserver: set the initial cluster version to 2.3
2016-03-25 14:23:19.750896 I | etcdserver: start to snapshot (applied: 10001, lastsnap: 0)
2016-03-25 14:23:19.789993 I | etcdserver: saved snapshot at index 10001
2016-03-25 14:23:19.790231 I | etcdserver: compacted raft log at 5001
2016-03-25 14:23:35.010277 I | etcdserver: start to snapshot (applied: 20003, lastsnap: 10001)
2016-03-25 14:23:35.034514 I | etcdserver: saved snapshot at index 20003
2016-03-25 14:23:35.034695 I | etcdserver: compacted raft log at 15003
2016-03-25 14:23:50.493114 I | etcdserver: start to snapshot (applied: 30005, lastsnap: 20003)
2016-03-25 14:23:50.516633 I | etcdserver: saved snapshot at index 30005
2016-03-25 14:23:50.516882 I | etcdserver: compacted raft log at 25005
2016-03-25 14:24:06.022493 I | etcdserver: start to snapshot (applied: 40008, lastsnap: 30005)
2016-03-25 14:24:06.042230 I | etcdserver: saved snapshot at index 40008
2016-03-25 14:24:06.042482 I | etcdserver: compacted raft log at 35008
2016-03-25 14:24:21.188745 I | etcdserver: start to snapshot (applied: 50015, lastsnap: 40008)
2016-03-25 14:24:21.219864 I | etcdserver: saved snapshot at index 50015
2016-03-25 14:24:21.220008 I | etcdserver: compacted raft log at 45015
2016-03-25 14:24:36.505414 I | etcdserver: start to snapshot (applied: 60023, lastsnap: 50015)
2016-03-25 14:24:36.519778 I | etcdserver: saved snapshot at index 60023
2016-03-25 14:24:36.520015 I | etcdserver: compacted raft log at 55023
2016-03-25 14:24:51.922101 I | etcdserver: start to snapshot (applied: 70027, lastsnap: 60023)
2016-03-25 14:24:51.938082 I | etcdserver: saved snapshot at index 70027
2016-03-25 14:24:51.938301 I | etcdserver: compacted raft log at 65027
2016-03-25 14:24:58.825604 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002711.snap successfully
2016-03-25 14:24:58.825650 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e23.snap successfully
2016-03-25 14:25:07.172512 I | etcdserver: start to snapshot (applied: 80028, lastsnap: 70027)
2016-03-25 14:25:07.187112 I | etcdserver: saved snapshot at index 80028
2016-03-25 14:25:07.187367 I | etcdserver: compacted raft log at 75028
2016-03-25 14:25:22.553084 I | etcdserver: start to snapshot (applied: 90029, lastsnap: 80028)
2016-03-25 14:25:22.566944 I | etcdserver: saved snapshot at index 90029
2016-03-25 14:25:22.567083 I | etcdserver: compacted raft log at 85029
2016-03-25 14:25:28.825857 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007535.snap successfully
2016-03-25 14:25:28.825906 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c48.snap successfully
2016-03-25 14:25:38.572870 I | etcdserver: start to snapshot (applied: 100033, lastsnap: 90029)
2016-03-25 14:25:38.590335 I | etcdserver: saved snapshot at index 100033
2016-03-25 14:25:38.590584 I | etcdserver: compacted raft log at 95033
2016-03-25 14:25:54.340471 I | etcdserver: start to snapshot (applied: 110034, lastsnap: 100033)
2016-03-25 14:25:54.361501 I | etcdserver: saved snapshot at index 110034
2016-03-25 14:25:54.361648 I | etcdserver: compacted raft log at 105034
2016-03-25 14:25:58.826127 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c35f.snap successfully
2016-03-25 14:25:58.826171 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea77.snap successfully
2016-03-25 14:26:09.871074 I | etcdserver: start to snapshot (applied: 120037, lastsnap: 110034)
2016-03-25 14:26:09.887071 I | etcdserver: saved snapshot at index 120037
2016-03-25 14:26:09.887256 I | etcdserver: compacted raft log at 115037
2016-03-25 14:26:25.343036 I | etcdserver: start to snapshot (applied: 130038, lastsnap: 120037)
2016-03-25 14:26:25.372747 I | etcdserver: saved snapshot at index 130038
2016-03-25 14:26:25.372966 I | etcdserver: compacted raft log at 125038
2016-03-25 14:26:28.826455 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001118b.snap successfully
2016-03-25 14:26:28.826553 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001389c.snap successfully
2016-03-25 14:26:40.579662 I | etcdserver: start to snapshot (applied: 140044, lastsnap: 130038)
2016-03-25 14:26:40.605129 I | etcdserver: saved snapshot at index 140044
2016-03-25 14:26:40.605268 I | etcdserver: compacted raft log at 135044
2016-03-25 14:26:56.020183 I | etcdserver: start to snapshot (applied: 150048, lastsnap: 140044)
2016-03-25 14:26:56.034995 I | etcdserver: saved snapshot at index 150048
2016-03-25 14:26:56.035117 I | etcdserver: compacted raft log at 145048
2016-03-25 14:26:58.826768 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015fad.snap successfully
2016-03-25 14:26:58.826817 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186c1.snap successfully
2016-03-25 14:27:11.505993 I | etcdserver: start to snapshot (applied: 160049, lastsnap: 150048)
2016-03-25 14:27:11.515547 I | etcdserver: saved snapshot at index 160049
2016-03-25 14:27:11.515749 I | etcdserver: compacted raft log at 155049
2016-03-25 14:27:26.102888 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-0000000000029633.wal is created
2016-03-25 14:27:26.919250 I | etcdserver: start to snapshot (applied: 170056, lastsnap: 160049)
2016-03-25 14:27:26.942762 I | etcdserver: saved snapshot at index 170056
2016-03-25 14:27:26.942890 I | etcdserver: compacted raft log at 165056
2016-03-25 14:27:28.827066 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001add2.snap successfully
2016-03-25 14:27:28.827119 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4e5.snap successfully
2016-03-25 14:27:42.616940 I | etcdserver: start to snapshot (applied: 180062, lastsnap: 170056)
2016-03-25 14:27:42.636610 I | etcdserver: saved snapshot at index 180062
2016-03-25 14:27:42.636856 I | etcdserver: compacted raft log at 175062
2016-03-25 14:27:58.155285 I | etcdserver: start to snapshot (applied: 190065, lastsnap: 180062)
2016-03-25 14:27:58.168969 I | etcdserver: saved snapshot at index 190065
2016-03-25 14:27:58.169155 I | etcdserver: compacted raft log at 185065
2016-03-25 14:27:58.827314 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbf6.snap successfully
2016-03-25 14:27:58.827368 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002230c.snap successfully
2016-03-25 14:28:13.788926 I | etcdserver: start to snapshot (applied: 200071, lastsnap: 190065)
2016-03-25 14:28:13.811833 I | etcdserver: saved snapshot at index 200071
2016-03-25 14:28:13.811979 I | etcdserver: compacted raft log at 195071
2016-03-25 14:28:28.827597 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000024a20.snap successfully
2016-03-25 14:28:29.342335 I | etcdserver: start to snapshot (applied: 210073, lastsnap: 200071)
2016-03-25 14:28:29.358560 I | etcdserver: saved snapshot at index 210073
2016-03-25 14:28:29.358780 I | etcdserver: compacted raft log at 205073
2016-03-25 14:28:44.756771 I | etcdserver: start to snapshot (applied: 220074, lastsnap: 210073)
2016-03-25 14:28:44.796353 I | etcdserver: saved snapshot at index 220074
2016-03-25 14:28:44.796472 I | etcdserver: compacted raft log at 215074
2016-03-25 14:28:58.827823 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027131.snap successfully
2016-03-25 14:28:58.827868 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000029848.snap successfully
2016-03-25 14:29:00.278158 I | etcdserver: start to snapshot (applied: 230075, lastsnap: 220074)
2016-03-25 14:29:00.299526 I | etcdserver: saved snapshot at index 230075
2016-03-25 14:29:00.299667 I | etcdserver: compacted raft log at 225075
2016-03-25 14:29:15.679751 I | etcdserver: start to snapshot (applied: 240077, lastsnap: 230075)
2016-03-25 14:29:15.706200 I | etcdserver: saved snapshot at index 240077
2016-03-25 14:29:15.706378 I | etcdserver: compacted raft log at 235077
2016-03-25 14:29:28.828157 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf5e.snap successfully
2016-03-25 14:29:28.828204 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e671.snap successfully
2016-03-25 14:29:31.192918 I | etcdserver: start to snapshot (applied: 250079, lastsnap: 240077)
2016-03-25 14:29:31.210979 I | etcdserver: saved snapshot at index 250079
2016-03-25 14:29:31.211180 I | etcdserver: compacted raft log at 245079
2016-03-25 14:29:46.393377 I | etcdserver: start to snapshot (applied: 260085, lastsnap: 250079)
2016-03-25 14:29:46.414637 I | etcdserver: saved snapshot at index 260085
2016-03-25 14:29:46.414854 I | etcdserver: compacted raft log at 255085
2016-03-25 14:29:58.828455 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d87.snap successfully
2016-03-25 14:29:58.828505 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000033499.snap successfully
2016-03-25 14:30:01.757544 I | etcdserver: start to snapshot (applied: 270086, lastsnap: 260085)
2016-03-25 14:30:01.774454 I | etcdserver: saved snapshot at index 270086
2016-03-25 14:30:01.774573 I | etcdserver: compacted raft log at 265086
2016-03-25 14:30:17.255430 I | etcdserver: start to snapshot (applied: 280088, lastsnap: 270086)
2016-03-25 14:30:17.264546 I | etcdserver: saved snapshot at index 280088
2016-03-25 14:30:17.264706 I | etcdserver: compacted raft log at 275088
2016-03-25 14:30:28.828761 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035baa.snap successfully
2016-03-25 14:30:28.828809 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000382bb.snap successfully
2016-03-25 14:30:33.053761 I | etcdserver: start to snapshot (applied: 290093, lastsnap: 280088)
2016-03-25 14:30:33.119482 I | etcdserver: saved snapshot at index 290093
2016-03-25 14:30:33.119627 I | etcdserver: compacted raft log at 285093
2016-03-25 14:30:48.583630 I | etcdserver: start to snapshot (applied: 300095, lastsnap: 290093)
2016-03-25 14:30:48.608024 I | etcdserver: saved snapshot at index 300095
2016-03-25 14:30:48.608269 I | etcdserver: compacted raft log at 295095
2016-03-25 14:30:58.829183 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a9cd.snap successfully
2016-03-25 14:30:58.829240 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0df.snap successfully
2016-03-25 14:31:04.137404 I | etcdserver: start to snapshot (applied: 310096, lastsnap: 300095)
2016-03-25 14:31:04.154661 I | etcdserver: saved snapshot at index 310096
2016-03-25 14:31:04.154818 I | etcdserver: compacted raft log at 305096
2016-03-25 14:31:19.723660 I | etcdserver: start to snapshot (applied: 320099, lastsnap: 310096)
2016-03-25 14:31:19.738921 I | etcdserver: saved snapshot at index 320099
2016-03-25 14:31:19.739154 I | etcdserver: compacted raft log at 315099
2016-03-25 14:31:28.829489 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003f7f5.snap successfully
2016-03-25 14:31:28.829543 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000041f06.snap successfully
2016-03-25 14:31:35.540695 I | etcdserver: start to snapshot (applied: 330101, lastsnap: 320099)
2016-03-25 14:31:35.589672 I | etcdserver: saved snapshot at index 330101
2016-03-25 14:31:35.589839 I | etcdserver: compacted raft log at 325101
2016-03-25 14:31:49.321200 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000002-0000000000052bcd.wal is created
2016-03-25 14:31:51.196989 I | etcdserver: start to snapshot (applied: 340104, lastsnap: 330101)
2016-03-25 14:31:51.212352 I | etcdserver: saved snapshot at index 340104
2016-03-25 14:31:51.212484 I | etcdserver: compacted raft log at 335104
2016-03-25 14:31:58.829796 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000044618.snap successfully
2016-03-25 14:31:58.829851 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000046d2d.snap successfully
2016-03-25 14:32:06.820055 I | etcdserver: start to snapshot (applied: 350106, lastsnap: 340104)
2016-03-25 14:32:06.847345 I | etcdserver: saved snapshot at index 350106
2016-03-25 14:32:06.847494 I | etcdserver: compacted raft log at 345106
2016-03-25 14:32:22.174096 I | etcdserver: start to snapshot (applied: 360108, lastsnap: 350106)
2016-03-25 14:32:22.203336 I | etcdserver: saved snapshot at index 360108
2016-03-25 14:32:22.203470 I | etcdserver: compacted raft log at 355108
2016-03-25 14:32:28.830104 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004943f.snap successfully
2016-03-25 14:32:28.830144 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004bb50.snap successfully
2016-03-25 14:32:37.886000 I | etcdserver: start to snapshot (applied: 370110, lastsnap: 360108)
2016-03-25 14:32:37.957080 I | etcdserver: saved snapshot at index 370110
2016-03-25 14:32:37.957237 I | etcdserver: compacted raft log at 365110
2016-03-25 14:32:53.576627 I | etcdserver: start to snapshot (applied: 380114, lastsnap: 370110)
2016-03-25 14:32:53.596457 I | etcdserver: saved snapshot at index 380114
2016-03-25 14:32:53.596592 I | etcdserver: compacted raft log at 375114
2016-03-25 14:32:58.830465 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004e263.snap successfully
2016-03-25 14:32:58.830521 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000050975.snap successfully
2016-03-25 14:33:09.255513 I | etcdserver: start to snapshot (applied: 390115, lastsnap: 380114)
2016-03-25 14:33:09.272646 I | etcdserver: saved snapshot at index 390115
2016-03-25 14:33:09.272836 I | etcdserver: compacted raft log at 385115
2016-03-25 14:33:24.940095 I | etcdserver: start to snapshot (applied: 400121, lastsnap: 390115)
2016-03-25 14:33:24.961757 I | etcdserver: saved snapshot at index 400121
2016-03-25 14:33:24.961901 I | etcdserver: compacted raft log at 395121
2016-03-25 14:33:28.830746 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000053088.snap successfully
2016-03-25 14:33:28.830796 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005579a.snap successfully
2016-03-25 14:33:40.512907 I | etcdserver: start to snapshot (applied: 410123, lastsnap: 400121)
2016-03-25 14:33:40.535519 I | etcdserver: saved snapshot at index 410123
2016-03-25 14:33:40.535678 I | etcdserver: compacted raft log at 405123
2016-03-25 14:33:56.264593 I | etcdserver: start to snapshot (applied: 420124, lastsnap: 410123)
2016-03-25 14:33:56.295611 I | etcdserver: saved snapshot at index 420124
2016-03-25 14:33:56.295756 I | etcdserver: compacted raft log at 415124
2016-03-25 14:33:58.831058 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000057eac.snap successfully
2016-03-25 14:33:58.831104 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005a5be.snap successfully
2016-03-25 14:34:11.909938 I | etcdserver: start to snapshot (applied: 430125, lastsnap: 420124)
2016-03-25 14:34:11.973907 I | etcdserver: saved snapshot at index 430125
2016-03-25 14:34:11.974148 I | etcdserver: compacted raft log at 425125
2016-03-25 14:34:27.812222 I | etcdserver: start to snapshot (applied: 440126, lastsnap: 430125)
2016-03-25 14:34:27.834640 I | etcdserver: saved snapshot at index 440126
2016-03-25 14:34:27.834837 I | etcdserver: compacted raft log at 435126
2016-03-25 14:34:28.831383 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005ccd2.snap successfully
2016-03-25 14:34:28.831440 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005f3e3.snap successfully
2016-03-25 14:34:43.666296 I | etcdserver: start to snapshot (applied: 450131, lastsnap: 440126)
2016-03-25 14:34:43.682012 I | etcdserver: saved snapshot at index 450131
2016-03-25 14:34:43.682273 I | etcdserver: compacted raft log at 445131
2016-03-25 14:34:58.831656 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000061af9.snap successfully
2016-03-25 14:34:59.382508 I | etcdserver: start to snapshot (applied: 460133, lastsnap: 450131)
2016-03-25 14:34:59.418387 I | etcdserver: saved snapshot at index 460133
2016-03-25 14:34:59.418516 I | etcdserver: compacted raft log at 455133
2016-03-25 14:35:14.782186 I | etcdserver: start to snapshot (applied: 470135, lastsnap: 460133)
2016-03-25 14:35:14.812258 I | etcdserver: saved snapshot at index 470135
2016-03-25 14:35:14.812401 I | etcdserver: compacted raft log at 465135
2016-03-25 14:35:28.831907 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006420b.snap successfully
2016-03-25 14:35:28.831959 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006691c.snap successfully
2016-03-25 14:35:30.222956 I | etcdserver: start to snapshot (applied: 480137, lastsnap: 470135)
2016-03-25 14:35:30.237325 I | etcdserver: saved snapshot at index 480137
2016-03-25 14:35:30.237473 I | etcdserver: compacted raft log at 475137
2016-03-25 14:35:45.811282 I | etcdserver: start to snapshot (applied: 490141, lastsnap: 480137)
2016-03-25 14:35:45.826578 I | etcdserver: saved snapshot at index 490141
2016-03-25 14:35:45.826753 I | etcdserver: compacted raft log at 485141
2016-03-25 14:35:58.832211 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006902d.snap successfully
2016-03-25 14:35:58.832267 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006b73e.snap successfully
2016-03-25 14:36:01.387132 I | etcdserver: start to snapshot (applied: 500142, lastsnap: 490141)
2016-03-25 14:36:01.400613 I | etcdserver: saved snapshot at index 500142
2016-03-25 14:36:01.400752 I | etcdserver: compacted raft log at 495142
2016-03-25 14:36:14.001192 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000003-000000000007c178.wal is created
2016-03-25 14:36:16.825770 I | etcdserver: start to snapshot (applied: 510144, lastsnap: 500142)
2016-03-25 14:36:16.838174 I | etcdserver: saved snapshot at index 510144
2016-03-25 14:36:16.838313 I | etcdserver: compacted raft log at 505144
2016-03-25 14:36:28.853006 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006de53.snap successfully
2016-03-25 14:36:28.853076 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000070565.snap successfully
2016-03-25 14:36:32.454740 I | etcdserver: start to snapshot (applied: 520148, lastsnap: 510144)
2016-03-25 14:36:32.471750 I | etcdserver: saved snapshot at index 520148
2016-03-25 14:36:32.471897 I | etcdserver: compacted raft log at 515148
2016-03-25 14:36:48.252052 I | etcdserver: start to snapshot (applied: 530150, lastsnap: 520148)
2016-03-25 14:36:48.268795 I | etcdserver: saved snapshot at index 530150
2016-03-25 14:36:48.268939 I | etcdserver: compacted raft log at 525150
2016-03-25 14:36:58.853371 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000072c77.snap successfully
2016-03-25 14:36:58.853427 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000075389.snap successfully
2016-03-25 14:37:03.801995 I | etcdserver: start to snapshot (applied: 540156, lastsnap: 530150)
2016-03-25 14:37:03.837005 I | etcdserver: saved snapshot at index 540156
2016-03-25 14:37:03.837355 I | etcdserver: compacted raft log at 535156
2016-03-25 14:37:19.283895 I | etcdserver: start to snapshot (applied: 550160, lastsnap: 540156)
2016-03-25 14:37:19.298853 I | etcdserver: saved snapshot at index 550160
2016-03-25 14:37:19.299100 I | etcdserver: compacted raft log at 545160
2016-03-25 14:37:28.853774 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000077a9d.snap successfully
2016-03-25 14:37:28.853829 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007a1ae.snap successfully
2016-03-25 14:37:34.702430 I | etcdserver: start to snapshot (applied: 560161, lastsnap: 550160)
2016-03-25 14:37:34.725480 I | etcdserver: saved snapshot at index 560161
2016-03-25 14:37:34.725613 I | etcdserver: compacted raft log at 555161
2016-03-25 14:37:50.194201 I | etcdserver: start to snapshot (applied: 570162, lastsnap: 560161)
2016-03-25 14:37:50.224076 I | etcdserver: saved snapshot at index 570162
2016-03-25 14:37:50.224218 I | etcdserver: compacted raft log at 565162
2016-03-25 14:37:58.854174 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007c8c0.snap successfully
2016-03-25 14:37:58.854224 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007efd4.snap successfully
2016-03-25 14:38:05.553915 I | etcdserver: start to snapshot (applied: 580165, lastsnap: 570162)
2016-03-25 14:38:05.620672 I | etcdserver: saved snapshot at index 580165
2016-03-25 14:38:05.620816 I | etcdserver: compacted raft log at 575165
2016-03-25 14:38:21.067950 I | etcdserver: start to snapshot (applied: 590168, lastsnap: 580165)
2016-03-25 14:38:21.100025 I | etcdserver: saved snapshot at index 590168
2016-03-25 14:38:21.100165 I | etcdserver: compacted raft log at 585168
2016-03-25 14:38:28.854478 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000816e6.snap successfully
2016-03-25 14:38:28.854527 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000083dfc.snap successfully
2016-03-25 14:38:36.588257 I | etcdserver: start to snapshot (applied: 600171, lastsnap: 590168)
2016-03-25 14:38:36.611917 I | etcdserver: saved snapshot at index 600171
2016-03-25 14:38:36.612044 I | etcdserver: compacted raft log at 595171
2016-03-25 14:38:52.000513 I | etcdserver: start to snapshot (applied: 610180, lastsnap: 600171)
2016-03-25 14:38:52.026958 I | etcdserver: saved snapshot at index 610180
2016-03-25 14:38:52.027115 I | etcdserver: compacted raft log at 605180
2016-03-25 14:38:58.854783 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000086510.snap successfully
2016-03-25 14:38:58.854831 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000088c21.snap successfully
2016-03-25 14:39:07.425083 I | etcdserver: start to snapshot (applied: 620183, lastsnap: 610180)
2016-03-25 14:39:07.480968 I | etcdserver: saved snapshot at index 620183
2016-03-25 14:39:07.481103 I | etcdserver: compacted raft log at 615183
2016-03-25 14:39:23.098769 I | etcdserver: start to snapshot (applied: 630186, lastsnap: 620183)
2016-03-25 14:39:23.146063 I | etcdserver: saved snapshot at index 630186
2016-03-25 14:39:23.146183 I | etcdserver: compacted raft log at 625186
2016-03-25 14:39:28.855182 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008b332.snap successfully
2016-03-25 14:39:28.855232 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008da45.snap successfully
2016-03-25 14:39:38.575254 I | etcdserver: start to snapshot (applied: 640188, lastsnap: 630186)
2016-03-25 14:39:38.609384 I | etcdserver: saved snapshot at index 640188
2016-03-25 14:39:38.609529 I | etcdserver: compacted raft log at 635188
2016-03-25 14:39:54.322319 I | etcdserver: start to snapshot (applied: 650195, lastsnap: 640188)
2016-03-25 14:39:54.376153 I | etcdserver: saved snapshot at index 650195
2016-03-25 14:39:54.376303 I | etcdserver: compacted raft log at 645195
2016-03-25 14:39:58.855607 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000090158.snap successfully
2016-03-25 14:39:58.855651 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009286b.snap successfully
2016-03-25 14:40:09.642115 I | etcdserver: start to snapshot (applied: 660197, lastsnap: 650195)
2016-03-25 14:40:09.666118 I | etcdserver: saved snapshot at index 660197
2016-03-25 14:40:09.666263 I | etcdserver: compacted raft log at 655197
2016-03-25 14:40:25.441510 I | etcdserver: start to snapshot (applied: 670199, lastsnap: 660197)
2016-03-25 14:40:25.455018 I | etcdserver: saved snapshot at index 670199
2016-03-25 14:40:25.455150 I | etcdserver: compacted raft log at 665199
2016-03-25 14:40:28.855926 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000094f84.snap successfully
2016-03-25 14:40:28.855972 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000097697.snap successfully
2016-03-25 14:40:36.971218 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000004-00000000000a5723.wal is created
2016-03-25 14:40:40.964918 I | etcdserver: start to snapshot (applied: 680201, lastsnap: 670199)
2016-03-25 14:40:41.003018 I | etcdserver: saved snapshot at index 680201
2016-03-25 14:40:41.003148 I | etcdserver: compacted raft log at 675201
2016-03-25 14:40:56.681154 I | etcdserver: start to snapshot (applied: 690209, lastsnap: 680201)
2016-03-25 14:40:56.697113 I | etcdserver: saved snapshot at index 690209
2016-03-25 14:40:56.697257 I | etcdserver: compacted raft log at 685209
2016-03-25 14:40:58.856240 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000099daa.snap successfully
2016-03-25 14:40:58.856289 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009c4bc.snap successfully
2016-03-25 14:41:12.099162 I | etcdserver: start to snapshot (applied: 700216, lastsnap: 690209)
2016-03-25 14:41:12.126054 I | etcdserver: saved snapshot at index 700216
2016-03-25 14:41:12.126228 I | etcdserver: compacted raft log at 695216
2016-03-25 14:41:27.928252 I | etcdserver: start to snapshot (applied: 710220, lastsnap: 700216)
2016-03-25 14:41:27.946284 I | etcdserver: saved snapshot at index 710220
2016-03-25 14:41:27.946515 I | etcdserver: compacted raft log at 705220
2016-03-25 14:41:28.856564 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009ebd3.snap successfully
2016-03-25 14:41:28.856627 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a12e5.snap successfully
2016-03-25 14:41:43.583224 I | etcdserver: start to snapshot (applied: 720225, lastsnap: 710220)
2016-03-25 14:41:43.620011 I | etcdserver: saved snapshot at index 720225
2016-03-25 14:41:43.620188 I | etcdserver: compacted raft log at 715225
2016-03-25 14:41:58.857174 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a39f7.snap successfully
2016-03-25 14:41:59.289034 I | etcdserver: start to snapshot (applied: 730232, lastsnap: 720225)
2016-03-25 14:41:59.307845 I | etcdserver: saved snapshot at index 730232
2016-03-25 14:41:59.307976 I | etcdserver: compacted raft log at 725232
2016-03-25 14:42:14.920023 I | etcdserver: start to snapshot (applied: 740233, lastsnap: 730232)
2016-03-25 14:42:14.958223 I | etcdserver: saved snapshot at index 740233
2016-03-25 14:42:14.958389 I | etcdserver: compacted raft log at 735233
2016-03-25 14:42:28.857448 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a6109.snap successfully
2016-03-25 14:42:28.857499 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a8821.snap successfully
2016-03-25 14:42:30.569868 I | etcdserver: start to snapshot (applied: 750236, lastsnap: 740233)
2016-03-25 14:42:30.593082 I | etcdserver: saved snapshot at index 750236
2016-03-25 14:42:30.593227 I | etcdserver: compacted raft log at 745236
2016-03-25 14:42:46.310298 I | etcdserver: start to snapshot (applied: 760240, lastsnap: 750236)
2016-03-25 14:42:46.348680 I | etcdserver: saved snapshot at index 760240
2016-03-25 14:42:46.348828 I | etcdserver: compacted raft log at 755240
2016-03-25 14:42:58.857790 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000aaf38.snap successfully
2016-03-25 14:42:58.857854 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000ad64c.snap successfully
2016-03-25 14:43:01.804507 I | etcdserver: start to snapshot (applied: 770241, lastsnap: 760240)
2016-03-25 14:43:01.821513 I | etcdserver: saved snapshot at index 770241
2016-03-25 14:43:01.821641 I | etcdserver: compacted raft log at 765241
2016-03-25 14:43:17.693276 I | etcdserver: start to snapshot (applied: 780252, lastsnap: 770241)
2016-03-25 14:43:17.737177 I | etcdserver: saved snapshot at index 780252
2016-03-25 14:43:17.737307 I | etcdserver: compacted raft log at 775252
2016-03-25 14:43:28.858071 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000afd61.snap successfully
2016-03-25 14:43:28.858132 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b2478.snap successfully
2016-03-25 14:43:33.191597 I | etcdserver: start to snapshot (applied: 790259, lastsnap: 780252)
2016-03-25 14:43:33.218818 I | etcdserver: saved snapshot at index 790259
2016-03-25 14:43:33.218959 I | etcdserver: compacted raft log at 785259
2016-03-25 14:43:49.062619 I | etcdserver: start to snapshot (applied: 800261, lastsnap: 790259)
2016-03-25 14:43:49.081322 I | etcdserver: saved snapshot at index 800261
2016-03-25 14:43:49.081582 I | etcdserver: compacted raft log at 795261
2016-03-25 14:43:58.858650 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b4b89.snap successfully
2016-03-25 14:43:58.858693 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b729c.snap successfully
2016-03-25 14:44:04.855754 I | etcdserver: start to snapshot (applied: 810265, lastsnap: 800261)
2016-03-25 14:44:04.885561 I | etcdserver: saved snapshot at index 810265
2016-03-25 14:44:04.885691 I | etcdserver: compacted raft log at 805265
2016-03-25 14:44:20.445435 I | etcdserver: start to snapshot (applied: 820270, lastsnap: 810265)
2016-03-25 14:44:20.478602 I | etcdserver: saved snapshot at index 820270
2016-03-25 14:44:20.478744 I | etcdserver: compacted raft log at 815270
2016-03-25 14:44:28.859028 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b99b0.snap successfully
2016-03-25 14:44:28.859078 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000bc0c1.snap successfully
2016-03-25 14:44:36.266880 I | etcdserver: start to snapshot (applied: 830272, lastsnap: 820270)
2016-03-25 14:44:36.302533 I | etcdserver: saved snapshot at index 830272
2016-03-25 14:44:36.302683 I | etcdserver: compacted raft log at 825272
2016-03-25 14:44:51.656421 I | etcdserver: start to snapshot (applied: 840275, lastsnap: 830272)
2016-03-25 14:44:51.707351 I | etcdserver: saved snapshot at index 840275
2016-03-25 14:44:51.707465 I | etcdserver: compacted raft log at 835275
2016-03-25 14:44:58.859315 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000be7dc.snap successfully
2016-03-25 14:44:58.859363 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c0ef3.snap successfully
2016-03-25 14:45:02.198170 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000005-00000000000cecd4.wal is created
2016-03-25 14:45:07.182303 I | etcdserver: start to snapshot (applied: 850276, lastsnap: 840275)
2016-03-25 14:45:07.196800 I | etcdserver: saved snapshot at index 850276
2016-03-25 14:45:07.196968 I | etcdserver: compacted raft log at 845276
2016-03-25 14:45:23.159250 I | etcdserver: start to snapshot (applied: 860278, lastsnap: 850276)
2016-03-25 14:45:23.186531 I | etcdserver: saved snapshot at index 860278
2016-03-25 14:45:23.186665 I | etcdserver: compacted raft log at 855278
2016-03-25 14:45:28.859658 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c3605.snap successfully
2016-03-25 14:45:28.859709 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c5d19.snap successfully
2016-03-25 14:45:28.868727 I | fileutil: purged file /mnt/ssd0/data.etcd/member/wal/0000000000000000-0000000000000000.wal successfully
2016-03-25 14:45:38.848554 I | etcdserver: start to snapshot (applied: 870280, lastsnap: 860278)
2016-03-25 14:45:38.860744 I | etcdserver: saved snapshot at index 870280
2016-03-25 14:45:38.860878 I | etcdserver: compacted raft log at 865280
2016-03-25 14:45:54.318372 I | etcdserver: start to snapshot (applied: 880281, lastsnap: 870280)
2016-03-25 14:45:54.344953 I | etcdserver: saved snapshot at index 880281
2016-03-25 14:45:54.345080 I | etcdserver: compacted raft log at 875281
2016-03-25 14:45:58.859972 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c842e.snap successfully
2016-03-25 14:45:58.860019 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cab40.snap successfully
2016-03-25 14:46:10.264280 I | etcdserver: start to snapshot (applied: 890284, lastsnap: 880281)
2016-03-25 14:46:10.307519 I | etcdserver: saved snapshot at index 890284
2016-03-25 14:46:10.307655 I | etcdserver: compacted raft log at 885284
2016-03-25 14:46:26.089157 I | etcdserver: start to snapshot (applied: 900288, lastsnap: 890284)
2016-03-25 14:46:26.116185 I | etcdserver: saved snapshot at index 900288
2016-03-25 14:46:26.116334 I | etcdserver: compacted raft log at 895288
2016-03-25 14:46:28.860317 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cd253.snap successfully
2016-03-25 14:46:28.860360 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cf964.snap successfully
2016-03-25 14:46:42.016854 I | etcdserver: start to snapshot (applied: 910289, lastsnap: 900288)
2016-03-25 14:46:42.037290 I | etcdserver: saved snapshot at index 910289
2016-03-25 14:46:42.037423 I | etcdserver: compacted raft log at 905289
2016-03-25 14:46:57.682060 I | etcdserver: start to snapshot (applied: 920293, lastsnap: 910289)
2016-03-25 14:46:57.703756 I | etcdserver: saved snapshot at index 920293
2016-03-25 14:46:57.703907 I | etcdserver: compacted raft log at 915293
2016-03-25 14:46:58.860605 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d2076.snap successfully
2016-03-25 14:46:58.860653 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d4788.snap successfully
2016-03-25 14:47:13.360049 I | etcdserver: start to snapshot (applied: 930294, lastsnap: 920293)
2016-03-25 14:47:13.377688 I | etcdserver: saved snapshot at index 930294
2016-03-25 14:47:13.377832 I | etcdserver: compacted raft log at 925294
2016-03-25 14:47:28.860977 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d6e99.snap successfully
2016-03-25 14:47:29.282792 I | etcdserver: start to snapshot (applied: 940296, lastsnap: 930294)
2016-03-25 14:47:29.302137 I | etcdserver: saved snapshot at index 940296
2016-03-25 14:47:29.302261 I | etcdserver: compacted raft log at 935296
2016-03-25 14:47:44.937085 I | etcdserver: start to snapshot (applied: 950297, lastsnap: 940296)
2016-03-25 14:47:44.959040 I | etcdserver: saved snapshot at index 950297
2016-03-25 14:47:44.959197 I | etcdserver: compacted raft log at 945297
2016-03-25 14:47:58.861266 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d95ac.snap successfully
2016-03-25 14:47:58.861320 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000dbcc0.snap successfully
2016-03-25 14:48:00.477942 I | etcdserver: start to snapshot (applied: 960298, lastsnap: 950297)
2016-03-25 14:48:00.502751 I | etcdserver: saved snapshot at index 960298
2016-03-25 14:48:00.502962 I | etcdserver: compacted raft log at 955298
2016-03-25 14:48:16.057203 I | etcdserver: start to snapshot (applied: 970299, lastsnap: 960298)
2016-03-25 14:48:16.070722 I | etcdserver: saved snapshot at index 970299
2016-03-25 14:48:16.070860 I | etcdserver: compacted raft log at 965299
2016-03-25 14:48:28.861733 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000de3d1.snap successfully
2016-03-25 14:48:28.861802 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e0ae5.snap successfully
2016-03-25 14:48:31.787692 I | etcdserver: start to snapshot (applied: 980304, lastsnap: 970299)
2016-03-25 14:48:31.810030 I | etcdserver: saved snapshot at index 980304
2016-03-25 14:48:31.810160 I | etcdserver: compacted raft log at 975304
2016-03-25 14:48:47.458962 I | etcdserver: start to snapshot (applied: 990305, lastsnap: 980304)
2016-03-25 14:48:47.483406 I | etcdserver: saved snapshot at index 990305
2016-03-25 14:48:47.483795 I | etcdserver: compacted raft log at 985305
2016-03-25 14:48:58.862081 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e31f6.snap successfully
2016-03-25 14:48:58.862131 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e5908.snap successfully
2016-03-25 14:49:02.688545 I | etcdserver: start to snapshot (applied: 1000306, lastsnap: 990305)
2016-03-25 14:49:02.714184 I | etcdserver: saved snapshot at index 1000306
2016-03-25 14:49:02.714319 I | etcdserver: compacted raft log at 995306
2016-03-25 14:49:28.862494 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e8019.snap successfully
2016-03-25 14:53:54.517274 E | rafthttp: failed to dial 185c55dd02f81a06 on stream MsgApp v2 (dial tcp 10.240.0.32:2380: getsockopt: connection refused)
2016-03-25 14:53:54.517314 I | rafthttp: the connection with 185c55dd02f81a06 became inactive
2016-03-25 14:53:55.415048 N | osutil: received terminated signal, shutting down...
2016-03-25 14:53:55.417679 E | rafthttp: failed to read a55bd2c806d46279 on stream Message (net/http: request canceled)
2016-03-25 14:53:55.417697 I | rafthttp: the connection with a55bd2c806d46279 became inactive

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:17:53Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-3 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.34:2379 --advertise-client-urls http://10.240.0.34:2379 --listen-peer-urls http://10.240.0.34:2380 --initial-advertise-peer-urls http://10.240.0.34:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.32:2380,etcd-2=http://10.240.0.33:2380,etcd-3=http://10.240.0.34:2380 --initial-cluster-state new","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23529]","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"etcd monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:22:29Z"}
{"level":"info","msg":"Stopping etcd [PID: 23529]","time":"2016-03-25T14:53:56Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23529]","time":"2016-03-25T14:53:56Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:53:56Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:53:56Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-02-etcd-3-database.log","time":"2016-03-25T14:53:56Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:53:56Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-02-etcd-3-monitor.csv","time":"2016-03-25T14:53:57Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-02-etcd-3-agent.log","time":"2016-03-25T14:53:58Z"}

View File

@ -0,0 +1,447 @@
2016-03-25 14:22:29.781834 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:22:29.781890 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:22:29.781898 I | etcdmain: Go Version: go1.6
2016-03-25 14:22:29.781904 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:22:29.781913 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:22:29.781967 I | etcdmain: listening for peers on http://10.240.0.34:2380
2016-03-25 14:22:29.781993 I | etcdmain: listening for client requests on 10.240.0.34:2379
2016-03-25 14:22:29.784168 I | etcdserver: name = etcd-3
2016-03-25 14:22:29.784184 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:22:29.784192 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:22:29.784198 I | etcdserver: heartbeat = 100ms
2016-03-25 14:22:29.784204 I | etcdserver: election = 1000ms
2016-03-25 14:22:29.784209 I | etcdserver: snapshot count = 10000
2016-03-25 14:22:29.784219 I | etcdserver: advertise client URLs = http://10.240.0.34:2379
2016-03-25 14:22:29.784226 I | etcdserver: initial advertise peer URLs = http://10.240.0.34:2380
2016-03-25 14:22:29.784249 I | etcdserver: initial cluster = etcd-1=http://10.240.0.32:2380,etcd-2=http://10.240.0.33:2380,etcd-3=http://10.240.0.34:2380
2016-03-25 14:22:29.791440 I | etcdserver: starting member a55bd2c806d46279 in cluster c5560b3524bdd10e
2016-03-25 14:22:29.791491 I | raft: a55bd2c806d46279 became follower at term 0
2016-03-25 14:22:29.791504 I | raft: newRaft a55bd2c806d46279 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:22:29.791510 I | raft: a55bd2c806d46279 became follower at term 1
2016-03-25 14:22:29.831191 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:22:29.831694 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:22:29.831719 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:22:29.834521 N | etcdmain: serving insecure client requests on 10.240.0.34:2379, this is strongly discouraged!
2016-03-25 14:22:29.834649 I | raft: a55bd2c806d46279 [term: 1] received a MsgVote message with higher term from 185c55dd02f81a06 [term: 3]
2016-03-25 14:22:29.834681 I | raft: a55bd2c806d46279 became follower at term 3
2016-03-25 14:22:29.834716 I | raft: a55bd2c806d46279 [logterm: 1, index: 3, vote: 0] voted for 185c55dd02f81a06 [logterm: 1, index: 3] at term 3
2016-03-25 14:22:29.834932 I | raft: raft.node: a55bd2c806d46279 elected leader 185c55dd02f81a06 at term 3
2016-03-25 14:22:29.836211 I | rafthttp: the connection with faf98ebdef076a19 became active
2016-03-25 14:22:29.838343 N | etcdserver: added member 185c55dd02f81a06 [http://10.240.0.32:2380] to cluster c5560b3524bdd10e
2016-03-25 14:22:29.838450 N | etcdserver: added local member a55bd2c806d46279 [http://10.240.0.34:2380] to cluster c5560b3524bdd10e
2016-03-25 14:22:29.838572 I | rafthttp: the connection with 185c55dd02f81a06 became active
2016-03-25 14:22:29.838604 N | etcdserver: added member faf98ebdef076a19 [http://10.240.0.33:2380] to cluster c5560b3524bdd10e
2016-03-25 14:22:29.847775 N | etcdserver: set the initial cluster version to 2.3
2016-03-25 14:22:29.850886 I | etcdserver: published {Name:etcd-3 ClientURLs:[http://10.240.0.34:2379]} to cluster c5560b3524bdd10e
2016-03-25 14:23:19.751196 I | etcdserver: start to snapshot (applied: 10001, lastsnap: 0)
2016-03-25 14:23:19.775986 I | etcdserver: saved snapshot at index 10001
2016-03-25 14:23:19.776246 I | etcdserver: compacted raft log at 5001
2016-03-25 14:23:35.006379 I | etcdserver: start to snapshot (applied: 20003, lastsnap: 10001)
2016-03-25 14:23:35.027435 I | etcdserver: saved snapshot at index 20003
2016-03-25 14:23:35.027652 I | etcdserver: compacted raft log at 15003
2016-03-25 14:23:50.486178 I | etcdserver: start to snapshot (applied: 30005, lastsnap: 20003)
2016-03-25 14:23:50.519928 I | etcdserver: saved snapshot at index 30005
2016-03-25 14:23:50.520176 I | etcdserver: compacted raft log at 25005
2016-03-25 14:24:06.023333 I | etcdserver: start to snapshot (applied: 40008, lastsnap: 30005)
2016-03-25 14:24:06.061930 I | etcdserver: saved snapshot at index 40008
2016-03-25 14:24:06.062153 I | etcdserver: compacted raft log at 35008
2016-03-25 14:24:21.182111 I | etcdserver: start to snapshot (applied: 50014, lastsnap: 40008)
2016-03-25 14:24:21.203075 I | etcdserver: saved snapshot at index 50014
2016-03-25 14:24:21.211156 I | etcdserver: compacted raft log at 45014
2016-03-25 14:24:36.503648 I | etcdserver: start to snapshot (applied: 60023, lastsnap: 50014)
2016-03-25 14:24:36.526581 I | etcdserver: saved snapshot at index 60023
2016-03-25 14:24:36.526712 I | etcdserver: compacted raft log at 55023
2016-03-25 14:24:51.920325 I | etcdserver: start to snapshot (applied: 70027, lastsnap: 60023)
2016-03-25 14:24:51.941487 I | etcdserver: saved snapshot at index 70027
2016-03-25 14:24:51.941729 I | etcdserver: compacted raft log at 65027
2016-03-25 14:24:59.835340 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002711.snap successfully
2016-03-25 14:24:59.835390 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e23.snap successfully
2016-03-25 14:25:07.171484 I | etcdserver: start to snapshot (applied: 80028, lastsnap: 70027)
2016-03-25 14:25:07.192209 I | etcdserver: saved snapshot at index 80028
2016-03-25 14:25:07.192489 I | etcdserver: compacted raft log at 75028
2016-03-25 14:25:22.547113 I | etcdserver: start to snapshot (applied: 90029, lastsnap: 80028)
2016-03-25 14:25:22.565789 I | etcdserver: saved snapshot at index 90029
2016-03-25 14:25:22.565920 I | etcdserver: compacted raft log at 85029
2016-03-25 14:25:29.835602 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007535.snap successfully
2016-03-25 14:25:29.835660 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c48.snap successfully
2016-03-25 14:25:38.576363 I | etcdserver: start to snapshot (applied: 100033, lastsnap: 90029)
2016-03-25 14:25:38.597778 I | etcdserver: saved snapshot at index 100033
2016-03-25 14:25:38.597982 I | etcdserver: compacted raft log at 95033
2016-03-25 14:25:54.337854 I | etcdserver: start to snapshot (applied: 110034, lastsnap: 100033)
2016-03-25 14:25:54.359742 I | etcdserver: saved snapshot at index 110034
2016-03-25 14:25:54.360151 I | etcdserver: compacted raft log at 105034
2016-03-25 14:25:59.835897 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c35e.snap successfully
2016-03-25 14:25:59.835961 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea77.snap successfully
2016-03-25 14:26:09.867041 I | etcdserver: start to snapshot (applied: 120037, lastsnap: 110034)
2016-03-25 14:26:09.890145 I | etcdserver: saved snapshot at index 120037
2016-03-25 14:26:09.890351 I | etcdserver: compacted raft log at 115037
2016-03-25 14:26:25.353634 I | etcdserver: start to snapshot (applied: 130038, lastsnap: 120037)
2016-03-25 14:26:25.373901 I | etcdserver: saved snapshot at index 130038
2016-03-25 14:26:25.374131 I | etcdserver: compacted raft log at 125038
2016-03-25 14:26:29.836196 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001118b.snap successfully
2016-03-25 14:26:29.836245 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001389c.snap successfully
2016-03-25 14:26:40.578322 I | etcdserver: start to snapshot (applied: 140044, lastsnap: 130038)
2016-03-25 14:26:40.604123 I | etcdserver: saved snapshot at index 140044
2016-03-25 14:26:40.604240 I | etcdserver: compacted raft log at 135044
2016-03-25 14:26:56.016984 I | etcdserver: start to snapshot (applied: 150048, lastsnap: 140044)
2016-03-25 14:26:56.043273 I | etcdserver: saved snapshot at index 150048
2016-03-25 14:26:56.043423 I | etcdserver: compacted raft log at 145048
2016-03-25 14:26:59.836532 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015fad.snap successfully
2016-03-25 14:26:59.836590 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186c1.snap successfully
2016-03-25 14:27:11.505223 I | etcdserver: start to snapshot (applied: 160049, lastsnap: 150048)
2016-03-25 14:27:11.528402 I | etcdserver: saved snapshot at index 160049
2016-03-25 14:27:11.528548 I | etcdserver: compacted raft log at 155049
2016-03-25 14:27:26.214157 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-0000000000029674.wal is created
2016-03-25 14:27:26.913548 I | etcdserver: start to snapshot (applied: 170055, lastsnap: 160049)
2016-03-25 14:27:26.974747 I | etcdserver: saved snapshot at index 170055
2016-03-25 14:27:26.975078 I | etcdserver: compacted raft log at 165055
2016-03-25 14:27:29.836835 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001add2.snap successfully
2016-03-25 14:27:29.836892 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4e5.snap successfully
2016-03-25 14:27:42.614441 I | etcdserver: start to snapshot (applied: 180062, lastsnap: 170055)
2016-03-25 14:27:42.634508 I | etcdserver: saved snapshot at index 180062
2016-03-25 14:27:42.634655 I | etcdserver: compacted raft log at 175062
2016-03-25 14:27:58.144307 I | etcdserver: start to snapshot (applied: 190063, lastsnap: 180062)
2016-03-25 14:27:58.164523 I | etcdserver: saved snapshot at index 190063
2016-03-25 14:27:58.164673 I | etcdserver: compacted raft log at 185063
2016-03-25 14:27:59.837193 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbf6.snap successfully
2016-03-25 14:27:59.837246 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002230c.snap successfully
2016-03-25 14:28:13.783424 I | etcdserver: start to snapshot (applied: 200069, lastsnap: 190063)
2016-03-25 14:28:13.818180 I | etcdserver: saved snapshot at index 200069
2016-03-25 14:28:13.818318 I | etcdserver: compacted raft log at 195069
2016-03-25 14:28:29.332969 I | etcdserver: start to snapshot (applied: 210070, lastsnap: 200069)
2016-03-25 14:28:29.354132 I | etcdserver: saved snapshot at index 210070
2016-03-25 14:28:29.354368 I | etcdserver: compacted raft log at 205070
2016-03-25 14:28:29.837538 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000024a20.snap successfully
2016-03-25 14:28:29.837586 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027131.snap successfully
2016-03-25 14:28:44.754985 I | etcdserver: start to snapshot (applied: 220071, lastsnap: 210070)
2016-03-25 14:28:44.816883 I | etcdserver: saved snapshot at index 220071
2016-03-25 14:28:44.817153 I | etcdserver: compacted raft log at 215071
2016-03-25 14:28:59.837833 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000029847.snap successfully
2016-03-25 14:29:00.281784 I | etcdserver: start to snapshot (applied: 230073, lastsnap: 220071)
2016-03-25 14:29:00.309912 I | etcdserver: saved snapshot at index 230073
2016-03-25 14:29:00.310085 I | etcdserver: compacted raft log at 225073
2016-03-25 14:29:15.669319 I | etcdserver: start to snapshot (applied: 240075, lastsnap: 230073)
2016-03-25 14:29:15.688961 I | etcdserver: saved snapshot at index 240075
2016-03-25 14:29:15.689111 I | etcdserver: compacted raft log at 235075
2016-03-25 14:29:29.847432 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf5e.snap successfully
2016-03-25 14:29:29.847500 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e66f.snap successfully
2016-03-25 14:29:31.203110 I | etcdserver: start to snapshot (applied: 250079, lastsnap: 240075)
2016-03-25 14:29:31.224841 I | etcdserver: saved snapshot at index 250079
2016-03-25 14:29:31.224968 I | etcdserver: compacted raft log at 245079
2016-03-25 14:29:46.393202 I | etcdserver: start to snapshot (applied: 260085, lastsnap: 250079)
2016-03-25 14:29:46.415703 I | etcdserver: saved snapshot at index 260085
2016-03-25 14:29:46.415848 I | etcdserver: compacted raft log at 255085
2016-03-25 14:29:59.847762 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d85.snap successfully
2016-03-25 14:29:59.847805 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000033496.snap successfully
2016-03-25 14:30:01.757678 I | etcdserver: start to snapshot (applied: 270086, lastsnap: 260085)
2016-03-25 14:30:01.774212 I | etcdserver: saved snapshot at index 270086
2016-03-25 14:30:01.774448 I | etcdserver: compacted raft log at 265086
2016-03-25 14:30:17.249545 I | etcdserver: start to snapshot (applied: 280088, lastsnap: 270086)
2016-03-25 14:30:17.286166 I | etcdserver: saved snapshot at index 280088
2016-03-25 14:30:17.286293 I | etcdserver: compacted raft log at 275088
2016-03-25 14:30:29.848146 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035ba7.snap successfully
2016-03-25 14:30:29.848199 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000382b9.snap successfully
2016-03-25 14:30:33.043741 I | etcdserver: start to snapshot (applied: 290091, lastsnap: 280088)
2016-03-25 14:30:33.073498 I | etcdserver: saved snapshot at index 290091
2016-03-25 14:30:33.073635 I | etcdserver: compacted raft log at 285091
2016-03-25 14:30:48.581977 I | etcdserver: start to snapshot (applied: 300092, lastsnap: 290091)
2016-03-25 14:30:48.597971 I | etcdserver: saved snapshot at index 300092
2016-03-25 14:30:48.598189 I | etcdserver: compacted raft log at 295092
2016-03-25 14:30:59.848524 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a9cb.snap successfully
2016-03-25 14:30:59.848570 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0df.snap successfully
2016-03-25 14:31:04.110472 I | etcdserver: start to snapshot (applied: 310093, lastsnap: 300092)
2016-03-25 14:31:04.129654 I | etcdserver: saved snapshot at index 310093
2016-03-25 14:31:04.129784 I | etcdserver: compacted raft log at 305093
2016-03-25 14:31:19.722409 I | etcdserver: start to snapshot (applied: 320099, lastsnap: 310093)
2016-03-25 14:31:19.742612 I | etcdserver: saved snapshot at index 320099
2016-03-25 14:31:19.742763 I | etcdserver: compacted raft log at 315099
2016-03-25 14:31:29.848873 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003f7f5.snap successfully
2016-03-25 14:31:29.848917 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000041f06.snap successfully
2016-03-25 14:31:35.534050 I | etcdserver: start to snapshot (applied: 330101, lastsnap: 320099)
2016-03-25 14:31:35.574367 I | etcdserver: saved snapshot at index 330101
2016-03-25 14:31:35.574515 I | etcdserver: compacted raft log at 325101
2016-03-25 14:31:49.639489 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000002-0000000000052ca0.wal is created
2016-03-25 14:31:51.194361 I | etcdserver: start to snapshot (applied: 340104, lastsnap: 330101)
2016-03-25 14:31:51.216892 I | etcdserver: saved snapshot at index 340104
2016-03-25 14:31:51.217022 I | etcdserver: compacted raft log at 335104
2016-03-25 14:31:59.849239 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000044618.snap successfully
2016-03-25 14:31:59.849288 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000046d2b.snap successfully
2016-03-25 14:32:06.822577 I | etcdserver: start to snapshot (applied: 350106, lastsnap: 340104)
2016-03-25 14:32:06.848554 I | etcdserver: saved snapshot at index 350106
2016-03-25 14:32:06.848675 I | etcdserver: compacted raft log at 345106
2016-03-25 14:32:22.176927 I | etcdserver: start to snapshot (applied: 360108, lastsnap: 350106)
2016-03-25 14:32:22.195737 I | etcdserver: saved snapshot at index 360108
2016-03-25 14:32:22.195891 I | etcdserver: compacted raft log at 355108
2016-03-25 14:32:29.849522 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004943c.snap successfully
2016-03-25 14:32:29.849576 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004bb4d.snap successfully
2016-03-25 14:32:37.883186 I | etcdserver: start to snapshot (applied: 370109, lastsnap: 360108)
2016-03-25 14:32:37.899822 I | etcdserver: saved snapshot at index 370109
2016-03-25 14:32:37.899976 I | etcdserver: compacted raft log at 365109
2016-03-25 14:32:53.578250 I | etcdserver: start to snapshot (applied: 380110, lastsnap: 370109)
2016-03-25 14:32:53.591698 I | etcdserver: saved snapshot at index 380110
2016-03-25 14:32:53.591816 I | etcdserver: compacted raft log at 375110
2016-03-25 14:32:59.849907 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004e263.snap successfully
2016-03-25 14:32:59.849959 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000050975.snap successfully
2016-03-25 14:33:09.258079 I | etcdserver: start to snapshot (applied: 390118, lastsnap: 380110)
2016-03-25 14:33:09.278484 I | etcdserver: saved snapshot at index 390118
2016-03-25 14:33:09.278740 I | etcdserver: compacted raft log at 385118
2016-03-25 14:33:24.943308 I | etcdserver: start to snapshot (applied: 400125, lastsnap: 390118)
2016-03-25 14:33:24.962574 I | etcdserver: saved snapshot at index 400125
2016-03-25 14:33:24.962718 I | etcdserver: compacted raft log at 395125
2016-03-25 14:33:29.850244 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000053088.snap successfully
2016-03-25 14:33:29.850289 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005579a.snap successfully
2016-03-25 14:33:40.513449 I | etcdserver: start to snapshot (applied: 410127, lastsnap: 400125)
2016-03-25 14:33:40.537315 I | etcdserver: saved snapshot at index 410127
2016-03-25 14:33:40.537451 I | etcdserver: compacted raft log at 405127
2016-03-25 14:33:56.276186 I | etcdserver: start to snapshot (applied: 420131, lastsnap: 410127)
2016-03-25 14:33:56.297866 I | etcdserver: saved snapshot at index 420131
2016-03-25 14:33:56.298009 I | etcdserver: compacted raft log at 415131
2016-03-25 14:33:59.850592 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000057eac.snap successfully
2016-03-25 14:33:59.850638 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005a5bd.snap successfully
2016-03-25 14:34:11.932088 I | etcdserver: start to snapshot (applied: 430138, lastsnap: 420131)
2016-03-25 14:34:11.943572 I | etcdserver: saved snapshot at index 430138
2016-03-25 14:34:11.943688 I | etcdserver: compacted raft log at 425138
2016-03-25 14:34:27.840565 I | etcdserver: start to snapshot (applied: 440139, lastsnap: 430138)
2016-03-25 14:34:27.864647 I | etcdserver: saved snapshot at index 440139
2016-03-25 14:34:27.864786 I | etcdserver: compacted raft log at 435139
2016-03-25 14:34:29.850856 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005ccce.snap successfully
2016-03-25 14:34:29.850904 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005f3e6.snap successfully
2016-03-25 14:34:43.673306 I | etcdserver: start to snapshot (applied: 450141, lastsnap: 440139)
2016-03-25 14:34:43.695587 I | etcdserver: saved snapshot at index 450141
2016-03-25 14:34:43.695700 I | etcdserver: compacted raft log at 445141
2016-03-25 14:34:59.406998 I | etcdserver: start to snapshot (applied: 460149, lastsnap: 450141)
2016-03-25 14:34:59.428725 I | etcdserver: saved snapshot at index 460149
2016-03-25 14:34:59.428849 I | etcdserver: compacted raft log at 455149
2016-03-25 14:34:59.851227 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000061afd.snap successfully
2016-03-25 14:34:59.851280 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006420f.snap successfully
2016-03-25 14:35:14.795780 I | etcdserver: start to snapshot (applied: 470151, lastsnap: 460149)
2016-03-25 14:35:14.822177 I | etcdserver: saved snapshot at index 470151
2016-03-25 14:35:14.822327 I | etcdserver: compacted raft log at 465151
2016-03-25 14:35:29.851620 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000066923.snap successfully
2016-03-25 14:35:30.250380 I | etcdserver: start to snapshot (applied: 480153, lastsnap: 470151)
2016-03-25 14:35:30.288633 I | etcdserver: saved snapshot at index 480153
2016-03-25 14:35:30.288760 I | etcdserver: compacted raft log at 475153
2016-03-25 14:35:45.832760 I | etcdserver: start to snapshot (applied: 490157, lastsnap: 480153)
2016-03-25 14:35:45.875470 I | etcdserver: saved snapshot at index 490157
2016-03-25 14:35:45.875634 I | etcdserver: compacted raft log at 485157
2016-03-25 14:35:59.851960 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006903a.snap successfully
2016-03-25 14:35:59.852032 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006b74b.snap successfully
2016-03-25 14:36:01.410270 I | etcdserver: start to snapshot (applied: 500161, lastsnap: 490157)
2016-03-25 14:36:01.441339 I | etcdserver: saved snapshot at index 500161
2016-03-25 14:36:01.441480 I | etcdserver: compacted raft log at 495161
2016-03-25 14:36:14.497633 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000003-000000000007c2c1.wal is created
2016-03-25 14:36:16.857501 I | etcdserver: start to snapshot (applied: 510163, lastsnap: 500161)
2016-03-25 14:36:16.882555 I | etcdserver: saved snapshot at index 510163
2016-03-25 14:36:16.882711 I | etcdserver: compacted raft log at 505163
2016-03-25 14:36:29.881880 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006de5d.snap successfully
2016-03-25 14:36:29.881989 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000070575.snap successfully
2016-03-25 14:36:32.485451 I | etcdserver: start to snapshot (applied: 520172, lastsnap: 510163)
2016-03-25 14:36:32.522036 I | etcdserver: saved snapshot at index 520172
2016-03-25 14:36:32.522177 I | etcdserver: compacted raft log at 515172
2016-03-25 14:36:48.291290 I | etcdserver: start to snapshot (applied: 530176, lastsnap: 520172)
2016-03-25 14:36:48.308065 I | etcdserver: saved snapshot at index 530176
2016-03-25 14:36:48.308213 I | etcdserver: compacted raft log at 525176
2016-03-25 14:36:59.882514 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000072c87.snap successfully
2016-03-25 14:36:59.882568 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000075399.snap successfully
2016-03-25 14:37:03.846336 I | etcdserver: start to snapshot (applied: 540182, lastsnap: 530176)
2016-03-25 14:37:03.891386 I | etcdserver: saved snapshot at index 540182
2016-03-25 14:37:03.891512 I | etcdserver: compacted raft log at 535182
2016-03-25 14:37:19.308823 I | etcdserver: start to snapshot (applied: 550183, lastsnap: 540182)
2016-03-25 14:37:19.328699 I | etcdserver: saved snapshot at index 550183
2016-03-25 14:37:19.328839 I | etcdserver: compacted raft log at 545183
2016-03-25 14:37:29.882835 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000077aad.snap successfully
2016-03-25 14:37:29.882881 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007a1c1.snap successfully
2016-03-25 14:37:34.739233 I | etcdserver: start to snapshot (applied: 560186, lastsnap: 550183)
2016-03-25 14:37:34.754272 I | etcdserver: saved snapshot at index 560186
2016-03-25 14:37:34.754443 I | etcdserver: compacted raft log at 555186
2016-03-25 14:37:50.257536 I | etcdserver: start to snapshot (applied: 570191, lastsnap: 560186)
2016-03-25 14:37:50.285802 I | etcdserver: saved snapshot at index 570191
2016-03-25 14:37:50.285929 I | etcdserver: compacted raft log at 565191
2016-03-25 14:37:59.883192 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007c8d3.snap successfully
2016-03-25 14:37:59.883259 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007efec.snap successfully
2016-03-25 14:38:05.596442 I | etcdserver: start to snapshot (applied: 580194, lastsnap: 570191)
2016-03-25 14:38:05.623631 I | etcdserver: saved snapshot at index 580194
2016-03-25 14:38:05.623782 I | etcdserver: compacted raft log at 575194
2016-03-25 14:38:21.107502 I | etcdserver: start to snapshot (applied: 590195, lastsnap: 580194)
2016-03-25 14:38:21.118143 I | etcdserver: saved snapshot at index 590195
2016-03-25 14:38:21.118288 I | etcdserver: compacted raft log at 585195
2016-03-25 14:38:29.883498 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000081700.snap successfully
2016-03-25 14:38:29.883542 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000083e16.snap successfully
2016-03-25 14:38:36.633039 I | etcdserver: start to snapshot (applied: 600196, lastsnap: 590195)
2016-03-25 14:38:36.654970 I | etcdserver: saved snapshot at index 600196
2016-03-25 14:38:36.655122 I | etcdserver: compacted raft log at 595196
2016-03-25 14:38:52.033570 I | etcdserver: start to snapshot (applied: 610197, lastsnap: 600196)
2016-03-25 14:38:52.063941 I | etcdserver: saved snapshot at index 610197
2016-03-25 14:38:52.064113 I | etcdserver: compacted raft log at 605197
2016-03-25 14:38:59.883874 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000086527.snap successfully
2016-03-25 14:38:59.883919 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000088c3a.snap successfully
2016-03-25 14:39:07.454839 I | etcdserver: start to snapshot (applied: 620200, lastsnap: 610197)
2016-03-25 14:39:07.483867 I | etcdserver: saved snapshot at index 620200
2016-03-25 14:39:07.484048 I | etcdserver: compacted raft log at 615200
2016-03-25 14:39:23.138298 I | etcdserver: start to snapshot (applied: 630202, lastsnap: 620200)
2016-03-25 14:39:23.169134 I | etcdserver: saved snapshot at index 630202
2016-03-25 14:39:23.169260 I | etcdserver: compacted raft log at 625202
2016-03-25 14:39:29.884241 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008b34f.snap successfully
2016-03-25 14:39:29.884291 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008da62.snap successfully
2016-03-25 14:39:38.593586 I | etcdserver: start to snapshot (applied: 640203, lastsnap: 630202)
2016-03-25 14:39:38.615278 I | etcdserver: saved snapshot at index 640203
2016-03-25 14:39:38.615400 I | etcdserver: compacted raft log at 635203
2016-03-25 14:39:54.338903 I | etcdserver: start to snapshot (applied: 650209, lastsnap: 640203)
2016-03-25 14:39:54.356130 I | etcdserver: saved snapshot at index 650209
2016-03-25 14:39:54.356279 I | etcdserver: compacted raft log at 645209
2016-03-25 14:39:59.884556 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000090173.snap successfully
2016-03-25 14:39:59.884608 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000092884.snap successfully
2016-03-25 14:40:09.652605 I | etcdserver: start to snapshot (applied: 660210, lastsnap: 650209)
2016-03-25 14:40:09.690604 I | etcdserver: saved snapshot at index 660210
2016-03-25 14:40:09.690768 I | etcdserver: compacted raft log at 655210
2016-03-25 14:40:25.465262 I | etcdserver: start to snapshot (applied: 670218, lastsnap: 660210)
2016-03-25 14:40:25.493344 I | etcdserver: saved snapshot at index 670218
2016-03-25 14:40:25.493489 I | etcdserver: compacted raft log at 665218
2016-03-25 14:40:29.884909 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000094f95.snap successfully
2016-03-25 14:40:29.884960 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000976a8.snap successfully
2016-03-25 14:40:37.672866 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000004-00000000000a58db.wal is created
2016-03-25 14:40:40.986470 I | etcdserver: start to snapshot (applied: 680222, lastsnap: 670218)
2016-03-25 14:40:41.014820 I | etcdserver: saved snapshot at index 680222
2016-03-25 14:40:41.014959 I | etcdserver: compacted raft log at 675222
2016-03-25 14:40:56.703856 I | etcdserver: start to snapshot (applied: 690224, lastsnap: 680222)
2016-03-25 14:40:56.723044 I | etcdserver: saved snapshot at index 690224
2016-03-25 14:40:56.723165 I | etcdserver: compacted raft log at 685224
2016-03-25 14:40:59.885267 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000099dba.snap successfully
2016-03-25 14:40:59.885322 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009c4cb.snap successfully
2016-03-25 14:41:12.120228 I | etcdserver: start to snapshot (applied: 700231, lastsnap: 690224)
2016-03-25 14:41:12.134376 I | etcdserver: saved snapshot at index 700231
2016-03-25 14:41:12.134496 I | etcdserver: compacted raft log at 695231
2016-03-25 14:41:27.958164 I | etcdserver: start to snapshot (applied: 710237, lastsnap: 700231)
2016-03-25 14:41:27.983025 I | etcdserver: saved snapshot at index 710237
2016-03-25 14:41:27.983166 I | etcdserver: compacted raft log at 705237
2016-03-25 14:41:29.885595 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009ebe1.snap successfully
2016-03-25 14:41:29.885646 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a12f2.snap successfully
2016-03-25 14:41:43.603742 I | etcdserver: start to snapshot (applied: 720239, lastsnap: 710237)
2016-03-25 14:41:43.653079 I | etcdserver: saved snapshot at index 720239
2016-03-25 14:41:43.653206 I | etcdserver: compacted raft log at 715239
2016-03-25 14:41:59.299475 I | etcdserver: start to snapshot (applied: 730241, lastsnap: 720239)
2016-03-25 14:41:59.325248 I | etcdserver: saved snapshot at index 730241
2016-03-25 14:41:59.325372 I | etcdserver: compacted raft log at 725241
2016-03-25 14:41:59.895974 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a3a0a.snap successfully
2016-03-25 14:41:59.896047 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a611e.snap successfully
2016-03-25 14:42:14.944890 I | etcdserver: start to snapshot (applied: 740242, lastsnap: 730241)
2016-03-25 14:42:14.966619 I | etcdserver: saved snapshot at index 740242
2016-03-25 14:42:14.966764 I | etcdserver: compacted raft log at 735242
2016-03-25 14:42:29.896309 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a8830.snap successfully
2016-03-25 14:42:30.582149 I | etcdserver: start to snapshot (applied: 750243, lastsnap: 740242)
2016-03-25 14:42:30.604649 I | etcdserver: saved snapshot at index 750243
2016-03-25 14:42:30.604816 I | etcdserver: compacted raft log at 745243
2016-03-25 14:42:46.314241 I | etcdserver: start to snapshot (applied: 760245, lastsnap: 750243)
2016-03-25 14:42:46.346538 I | etcdserver: saved snapshot at index 760245
2016-03-25 14:42:46.346670 I | etcdserver: compacted raft log at 755245
2016-03-25 14:42:59.896672 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000aaf47.snap successfully
2016-03-25 14:42:59.896726 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000ad65d.snap successfully
2016-03-25 14:43:01.821179 I | etcdserver: start to snapshot (applied: 770248, lastsnap: 760245)
2016-03-25 14:43:01.844352 I | etcdserver: saved snapshot at index 770248
2016-03-25 14:43:01.844485 I | etcdserver: compacted raft log at 765248
2016-03-25 14:43:17.688752 I | etcdserver: start to snapshot (applied: 780250, lastsnap: 770248)
2016-03-25 14:43:17.718106 I | etcdserver: saved snapshot at index 780250
2016-03-25 14:43:17.718233 I | etcdserver: compacted raft log at 775250
2016-03-25 14:43:29.896934 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000afd6f.snap successfully
2016-03-25 14:43:29.896999 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b2481.snap successfully
2016-03-25 14:43:33.175521 I | etcdserver: start to snapshot (applied: 790252, lastsnap: 780250)
2016-03-25 14:43:33.212980 I | etcdserver: saved snapshot at index 790252
2016-03-25 14:43:33.213123 I | etcdserver: compacted raft log at 785252
2016-03-25 14:43:49.058412 I | etcdserver: start to snapshot (applied: 800253, lastsnap: 790252)
2016-03-25 14:43:49.082692 I | etcdserver: saved snapshot at index 800253
2016-03-25 14:43:49.082828 I | etcdserver: compacted raft log at 795253
2016-03-25 14:43:59.897313 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b4b92.snap successfully
2016-03-25 14:43:59.897371 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b72a3.snap successfully
2016-03-25 14:44:04.843581 I | etcdserver: start to snapshot (applied: 810256, lastsnap: 800253)
2016-03-25 14:44:04.870480 I | etcdserver: saved snapshot at index 810256
2016-03-25 14:44:04.870632 I | etcdserver: compacted raft log at 805256
2016-03-25 14:44:20.435282 I | etcdserver: start to snapshot (applied: 820260, lastsnap: 810256)
2016-03-25 14:44:20.489358 I | etcdserver: saved snapshot at index 820260
2016-03-25 14:44:20.489498 I | etcdserver: compacted raft log at 815260
2016-03-25 14:44:29.897590 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b99b5.snap successfully
2016-03-25 14:44:29.897635 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000bc0c8.snap successfully
2016-03-25 14:44:36.257697 I | etcdserver: start to snapshot (applied: 830269, lastsnap: 820260)
2016-03-25 14:44:36.287547 I | etcdserver: saved snapshot at index 830269
2016-03-25 14:44:36.287683 I | etcdserver: compacted raft log at 825269
2016-03-25 14:44:51.650461 I | etcdserver: start to snapshot (applied: 840270, lastsnap: 830269)
2016-03-25 14:44:51.673312 I | etcdserver: saved snapshot at index 840270
2016-03-25 14:44:51.673441 I | etcdserver: compacted raft log at 835270
2016-03-25 14:44:59.897922 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000be7da.snap successfully
2016-03-25 14:44:59.897984 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c0eec.snap successfully
2016-03-25 14:45:03.098932 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000005-00000000000ceeff.wal is created
2016-03-25 14:45:07.173058 I | etcdserver: start to snapshot (applied: 850273, lastsnap: 840270)
2016-03-25 14:45:07.189733 I | etcdserver: saved snapshot at index 850273
2016-03-25 14:45:07.190033 I | etcdserver: compacted raft log at 845273
2016-03-25 14:45:23.157439 I | etcdserver: start to snapshot (applied: 860275, lastsnap: 850273)
2016-03-25 14:45:23.183338 I | etcdserver: saved snapshot at index 860275
2016-03-25 14:45:23.183477 I | etcdserver: compacted raft log at 855275
2016-03-25 14:45:29.898329 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c35fd.snap successfully
2016-03-25 14:45:29.898374 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c5d10.snap successfully
2016-03-25 14:45:29.915352 I | fileutil: purged file /mnt/ssd0/data.etcd/member/wal/0000000000000000-0000000000000000.wal successfully
2016-03-25 14:45:38.839805 I | etcdserver: start to snapshot (applied: 870276, lastsnap: 860275)
2016-03-25 14:45:38.881976 I | etcdserver: saved snapshot at index 870276
2016-03-25 14:45:38.882137 I | etcdserver: compacted raft log at 865276
2016-03-25 14:45:54.319222 I | etcdserver: start to snapshot (applied: 880281, lastsnap: 870276)
2016-03-25 14:45:54.351250 I | etcdserver: saved snapshot at index 880281
2016-03-25 14:45:54.351384 I | etcdserver: compacted raft log at 875281
2016-03-25 14:45:59.898651 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c8424.snap successfully
2016-03-25 14:45:59.898713 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cab3d.snap successfully
2016-03-25 14:46:10.264300 I | etcdserver: start to snapshot (applied: 890284, lastsnap: 880281)
2016-03-25 14:46:10.278633 I | etcdserver: saved snapshot at index 890284
2016-03-25 14:46:10.278803 I | etcdserver: compacted raft log at 885284
2016-03-25 14:46:26.090929 I | etcdserver: start to snapshot (applied: 900289, lastsnap: 890284)
2016-03-25 14:46:26.114874 I | etcdserver: saved snapshot at index 900289
2016-03-25 14:46:26.114999 I | etcdserver: compacted raft log at 895289
2016-03-25 14:46:29.899012 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cd24e.snap successfully
2016-03-25 14:46:29.899071 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cf961.snap successfully
2016-03-25 14:46:42.022530 I | etcdserver: start to snapshot (applied: 910290, lastsnap: 900289)
2016-03-25 14:46:42.047272 I | etcdserver: saved snapshot at index 910290
2016-03-25 14:46:42.047547 I | etcdserver: compacted raft log at 905290
2016-03-25 14:46:57.690513 I | etcdserver: start to snapshot (applied: 920293, lastsnap: 910290)
2016-03-25 14:46:57.723529 I | etcdserver: saved snapshot at index 920293
2016-03-25 14:46:57.723676 I | etcdserver: compacted raft log at 915293
2016-03-25 14:46:59.899372 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d2073.snap successfully
2016-03-25 14:46:59.899429 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d4784.snap successfully
2016-03-25 14:47:13.364156 I | etcdserver: start to snapshot (applied: 930295, lastsnap: 920293)
2016-03-25 14:47:13.386624 I | etcdserver: saved snapshot at index 930295
2016-03-25 14:47:13.386860 I | etcdserver: compacted raft log at 925295
2016-03-25 14:47:29.288993 I | etcdserver: start to snapshot (applied: 940296, lastsnap: 930295)
2016-03-25 14:47:29.320219 I | etcdserver: saved snapshot at index 940296
2016-03-25 14:47:29.320343 I | etcdserver: compacted raft log at 935296
2016-03-25 14:47:29.899728 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d6e99.snap successfully
2016-03-25 14:47:29.899813 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d95ac.snap successfully
2016-03-25 14:47:44.939878 I | etcdserver: start to snapshot (applied: 950297, lastsnap: 940296)
2016-03-25 14:47:44.957404 I | etcdserver: saved snapshot at index 950297
2016-03-25 14:47:44.957575 I | etcdserver: compacted raft log at 945297
2016-03-25 14:47:59.900081 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000dbcc1.snap successfully
2016-03-25 14:48:00.480900 I | etcdserver: start to snapshot (applied: 960298, lastsnap: 950297)
2016-03-25 14:48:00.504034 I | etcdserver: saved snapshot at index 960298
2016-03-25 14:48:00.504174 I | etcdserver: compacted raft log at 955298
2016-03-25 14:48:16.053662 I | etcdserver: start to snapshot (applied: 970299, lastsnap: 960298)
2016-03-25 14:48:16.074699 I | etcdserver: saved snapshot at index 970299
2016-03-25 14:48:16.074827 I | etcdserver: compacted raft log at 965299
2016-03-25 14:48:29.900315 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000de3d2.snap successfully
2016-03-25 14:48:29.900374 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e0ae5.snap successfully
2016-03-25 14:48:31.782513 I | etcdserver: start to snapshot (applied: 980300, lastsnap: 970299)
2016-03-25 14:48:31.816722 I | etcdserver: saved snapshot at index 980300
2016-03-25 14:48:31.816859 I | etcdserver: compacted raft log at 975300
2016-03-25 14:48:47.457344 I | etcdserver: start to snapshot (applied: 990305, lastsnap: 980300)
2016-03-25 14:48:47.477528 I | etcdserver: saved snapshot at index 990305
2016-03-25 14:48:47.477663 I | etcdserver: compacted raft log at 985305
2016-03-25 14:48:59.900785 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e31f7.snap successfully
2016-03-25 14:48:59.900861 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e5908.snap successfully
2016-03-25 14:49:02.693018 I | etcdserver: start to snapshot (applied: 1000306, lastsnap: 990305)
2016-03-25 14:49:02.714357 I | etcdserver: saved snapshot at index 1000306
2016-03-25 14:49:02.714497 I | etcdserver: compacted raft log at 995306
2016-03-25 14:49:29.901197 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000e8019.snap successfully
2016-03-25 14:53:54.517373 E | rafthttp: failed to dial 185c55dd02f81a06 on stream Message (dial tcp 10.240.0.32:2380: getsockopt: connection refused)
2016-03-25 14:53:54.517437 I | rafthttp: the connection with 185c55dd02f81a06 became inactive
2016-03-25 14:53:55.518558 E | rafthttp: failed to dial faf98ebdef076a19 on stream MsgApp v2 (dial tcp 10.240.0.33:2380: getsockopt: connection refused)
2016-03-25 14:53:55.518640 I | rafthttp: the connection with faf98ebdef076a19 became inactive
2016-03-25 14:53:55.591919 I | raft: a55bd2c806d46279 is starting a new election at term 3
2016-03-25 14:53:55.591960 I | raft: a55bd2c806d46279 became candidate at term 4
2016-03-25 14:53:55.591968 I | raft: a55bd2c806d46279 received vote from a55bd2c806d46279 at term 4
2016-03-25 14:53:55.591980 I | raft: a55bd2c806d46279 [logterm: 3, index: 1003778] sent vote request to 185c55dd02f81a06 at term 4
2016-03-25 14:53:55.591990 I | raft: a55bd2c806d46279 [logterm: 3, index: 1003778] sent vote request to faf98ebdef076a19 at term 4
2016-03-25 14:53:55.592027 I | raft: raft.node: a55bd2c806d46279 lost leader 185c55dd02f81a06 at term 4
2016-03-25 14:53:56.415882 N | osutil: received terminated signal, shutting down...

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:17:52Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-1 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.36:2379 --advertise-client-urls http://10.240.0.36:2379 --listen-peer-urls http://10.240.0.36:2380 --initial-advertise-peer-urls http://10.240.0.36:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.36:2380,etcd-2=http://10.240.0.37:2380,etcd-3=http://10.240.0.38:2380 --initial-cluster-state new","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23354]","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"etcd2 monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:21:48Z"}
{"level":"info","msg":"Stopping etcd2 [PID: 23354]","time":"2016-03-25T14:54:09Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23354]","time":"2016-03-25T14:54:10Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:54:10Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:54:10Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-02-etcd2-1-database.log","time":"2016-03-25T14:54:10Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:54:10Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-02-etcd2-1-monitor.csv","time":"2016-03-25T14:54:11Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-02-etcd2-1-agent.log","time":"2016-03-25T14:54:12Z"}

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:18:14Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-2 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.37:2379 --advertise-client-urls http://10.240.0.37:2379 --listen-peer-urls http://10.240.0.37:2380 --initial-advertise-peer-urls http://10.240.0.37:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.36:2380,etcd-2=http://10.240.0.37:2380,etcd-3=http://10.240.0.38:2380 --initial-cluster-state new","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23566]","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"etcd2 monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:21:49Z"}
{"level":"info","msg":"Stopping etcd2 [PID: 23566]","time":"2016-03-25T14:54:11Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23566]","time":"2016-03-25T14:54:11Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:54:11Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:54:11Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-02-etcd2-2-database.log","time":"2016-03-25T14:54:11Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:54:11Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-02-etcd2-2-monitor.csv","time":"2016-03-25T14:54:12Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-02-etcd2-2-agent.log","time":"2016-03-25T14:54:13Z"}

View File

@ -0,0 +1,640 @@
2016-03-25 14:21:49.354501 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:21:49.354550 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:21:49.354557 I | etcdmain: Go Version: go1.6
2016-03-25 14:21:49.354564 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:21:49.354572 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:21:49.354631 I | etcdmain: listening for peers on http://10.240.0.37:2380
2016-03-25 14:21:49.354659 I | etcdmain: listening for client requests on 10.240.0.37:2379
2016-03-25 14:21:49.356718 I | etcdserver: name = etcd-2
2016-03-25 14:21:49.356735 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:21:49.356743 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:21:49.356749 I | etcdserver: heartbeat = 100ms
2016-03-25 14:21:49.356755 I | etcdserver: election = 1000ms
2016-03-25 14:21:49.356760 I | etcdserver: snapshot count = 10000
2016-03-25 14:21:49.356770 I | etcdserver: advertise client URLs = http://10.240.0.37:2379
2016-03-25 14:21:49.356777 I | etcdserver: initial advertise peer URLs = http://10.240.0.37:2380
2016-03-25 14:21:49.356796 I | etcdserver: initial cluster = etcd-1=http://10.240.0.36:2380,etcd-2=http://10.240.0.37:2380,etcd-3=http://10.240.0.38:2380
2016-03-25 14:21:49.365921 I | etcdserver: starting member 208de484d8b2cb22 in cluster b73c1f84aa49a9b0
2016-03-25 14:21:49.365988 I | raft: 208de484d8b2cb22 became follower at term 0
2016-03-25 14:21:49.366003 I | raft: newRaft 208de484d8b2cb22 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:21:49.366010 I | raft: 208de484d8b2cb22 became follower at term 1
2016-03-25 14:21:49.412856 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:21:49.413455 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:21:49.413484 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:21:49.413618 N | etcdmain: serving insecure client requests on 10.240.0.37:2379, this is strongly discouraged!
2016-03-25 14:21:49.413880 I | rafthttp: the connection with b0978d17c833d412 became active
2016-03-25 14:21:49.415310 N | etcdserver: added local member 208de484d8b2cb22 [http://10.240.0.37:2380] to cluster b73c1f84aa49a9b0
2016-03-25 14:21:49.415418 N | etcdserver: added member b0978d17c833d412 [http://10.240.0.36:2380] to cluster b73c1f84aa49a9b0
2016-03-25 14:21:49.415489 N | etcdserver: added member ef3ac5ee77284022 [http://10.240.0.38:2380] to cluster b73c1f84aa49a9b0
2016-03-25 14:21:49.766410 I | raft: 208de484d8b2cb22 is starting a new election at term 1
2016-03-25 14:21:49.766532 I | raft: 208de484d8b2cb22 became candidate at term 2
2016-03-25 14:21:49.766542 I | raft: 208de484d8b2cb22 received vote from 208de484d8b2cb22 at term 2
2016-03-25 14:21:49.766565 I | raft: 208de484d8b2cb22 [logterm: 1, index: 3] sent vote request to b0978d17c833d412 at term 2
2016-03-25 14:21:49.766574 I | raft: 208de484d8b2cb22 [logterm: 1, index: 3] sent vote request to ef3ac5ee77284022 at term 2
2016-03-25 14:21:49.774819 I | raft: 208de484d8b2cb22 [term: 2] received a MsgVote message with higher term from b0978d17c833d412 [term: 3]
2016-03-25 14:21:49.774864 I | raft: 208de484d8b2cb22 became follower at term 3
2016-03-25 14:21:49.774877 I | raft: 208de484d8b2cb22 [logterm: 1, index: 3, vote: 0] voted for b0978d17c833d412 [logterm: 1, index: 3] at term 3
2016-03-25 14:21:49.780134 I | raft: raft.node: 208de484d8b2cb22 elected leader b0978d17c833d412 at term 3
2016-03-25 14:21:49.796367 N | etcdserver: set the initial cluster version to 2.2
2016-03-25 14:21:49.796435 I | etcdserver: published {Name:etcd-2 ClientURLs:[http://10.240.0.37:2379]} to cluster b73c1f84aa49a9b0
2016-03-25 14:21:50.402995 I | rafthttp: the connection with ef3ac5ee77284022 became active
2016-03-25 14:21:53.796957 N | etcdserver: updated the cluster version from 2.2 to 2.3
2016-03-25 14:22:53.430637 I | etcdserver: start to snapshot (applied: 10002, lastsnap: 0)
2016-03-25 14:22:53.596518 I | etcdserver: saved snapshot at index 10002
2016-03-25 14:22:53.596739 I | etcdserver: compacted raft log at 5002
2016-03-25 14:23:08.028708 I | etcdserver: start to snapshot (applied: 20004, lastsnap: 10002)
2016-03-25 14:23:08.350645 I | etcdserver: saved snapshot at index 20004
2016-03-25 14:23:08.350806 I | etcdserver: compacted raft log at 15004
2016-03-25 14:23:22.403227 I | etcdserver: start to snapshot (applied: 30008, lastsnap: 20004)
2016-03-25 14:23:22.826092 I | etcdserver: saved snapshot at index 30008
2016-03-25 14:23:22.826243 I | etcdserver: compacted raft log at 25008
2016-03-25 14:23:36.894973 I | etcdserver: start to snapshot (applied: 40009, lastsnap: 30008)
2016-03-25 14:23:37.528235 I | etcdserver: saved snapshot at index 40009
2016-03-25 14:23:37.528389 I | etcdserver: compacted raft log at 35009
2016-03-25 14:23:51.685802 I | etcdserver: start to snapshot (applied: 50010, lastsnap: 40009)
2016-03-25 14:23:52.498840 I | etcdserver: saved snapshot at index 50010
2016-03-25 14:23:52.499077 I | etcdserver: compacted raft log at 45010
2016-03-25 14:24:06.620632 I | etcdserver: start to snapshot (applied: 60017, lastsnap: 50010)
2016-03-25 14:24:07.577455 I | etcdserver: saved snapshot at index 60017
2016-03-25 14:24:07.577605 I | etcdserver: compacted raft log at 55017
2016-03-25 14:24:19.416427 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002712.snap successfully
2016-03-25 14:24:21.453003 I | etcdserver: start to snapshot (applied: 70018, lastsnap: 60017)
2016-03-25 14:24:22.560648 I | etcdserver: saved snapshot at index 70018
2016-03-25 14:24:22.560804 I | etcdserver: compacted raft log at 65018
2016-03-25 14:24:36.717223 I | etcdserver: start to snapshot (applied: 80021, lastsnap: 70018)
2016-03-25 14:24:37.982352 I | etcdserver: saved snapshot at index 80021
2016-03-25 14:24:37.983229 I | etcdserver: compacted raft log at 75021
2016-03-25 14:24:49.419635 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e24.snap successfully
2016-03-25 14:24:49.423689 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007538.snap successfully
2016-03-25 14:24:51.955885 I | etcdserver: start to snapshot (applied: 90024, lastsnap: 80021)
2016-03-25 14:24:53.260924 I | etcdserver: saved snapshot at index 90024
2016-03-25 14:24:53.261095 I | etcdserver: compacted raft log at 85024
2016-03-25 14:25:07.074505 I | etcdserver: start to snapshot (applied: 100028, lastsnap: 90024)
2016-03-25 14:25:08.533474 I | etcdserver: saved snapshot at index 100028
2016-03-25 14:25:08.533620 I | etcdserver: compacted raft log at 95028
2016-03-25 14:25:19.429318 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c49.snap successfully
2016-03-25 14:25:19.436319 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c35a.snap successfully
2016-03-25 14:25:22.457856 I | etcdserver: start to snapshot (applied: 110032, lastsnap: 100028)
2016-03-25 14:25:24.115885 I | etcdserver: saved snapshot at index 110032
2016-03-25 14:25:24.116036 I | etcdserver: compacted raft log at 105032
2016-03-25 14:25:37.833744 I | etcdserver: start to snapshot (applied: 120033, lastsnap: 110032)
2016-03-25 14:25:39.627782 I | etcdserver: saved snapshot at index 120033
2016-03-25 14:25:39.627954 I | etcdserver: compacted raft log at 115033
2016-03-25 14:25:49.444379 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea71.snap successfully
2016-03-25 14:25:49.453659 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000011182.snap successfully
2016-03-25 14:25:53.303102 I | etcdserver: start to snapshot (applied: 130036, lastsnap: 120033)
2016-03-25 14:25:55.332376 I | etcdserver: saved snapshot at index 130036
2016-03-25 14:25:55.332568 I | etcdserver: compacted raft log at 125036
2016-03-25 14:26:08.827245 I | etcdserver: start to snapshot (applied: 140037, lastsnap: 130036)
2016-03-25 14:26:10.998734 I | etcdserver: saved snapshot at index 140037
2016-03-25 14:26:10.998923 I | etcdserver: compacted raft log at 135037
2016-03-25 14:26:19.464013 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000013895.snap successfully
2016-03-25 14:26:19.476062 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015fa8.snap successfully
2016-03-25 14:26:24.583917 I | etcdserver: start to snapshot (applied: 150038, lastsnap: 140037)
2016-03-25 14:26:26.865510 I | etcdserver: saved snapshot at index 150038
2016-03-25 14:26:26.865678 I | etcdserver: compacted raft log at 145038
2016-03-25 14:26:36.572292 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-000000000002673d.wal is created
2016-03-25 14:26:40.410996 I | etcdserver: start to snapshot (applied: 160041, lastsnap: 150038)
2016-03-25 14:26:42.789313 I | etcdserver: saved snapshot at index 160041
2016-03-25 14:26:42.789543 I | etcdserver: compacted raft log at 155041
2016-03-25 14:26:49.489780 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186bc.snap successfully
2016-03-25 14:26:49.504605 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001add0.snap successfully
2016-03-25 14:26:56.121281 I | etcdserver: start to snapshot (applied: 170042, lastsnap: 160041)
2016-03-25 14:26:58.696937 I | etcdserver: saved snapshot at index 170042
2016-03-25 14:26:58.697129 I | etcdserver: compacted raft log at 165042
2016-03-25 14:27:12.057270 I | etcdserver: start to snapshot (applied: 180043, lastsnap: 170042)
2016-03-25 14:27:14.791396 I | etcdserver: saved snapshot at index 180043
2016-03-25 14:27:14.791576 I | etcdserver: compacted raft log at 175043
2016-03-25 14:27:19.520460 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4e1.snap successfully
2016-03-25 14:27:19.537156 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbf4.snap successfully
2016-03-25 14:27:28.089331 I | etcdserver: start to snapshot (applied: 190045, lastsnap: 180043)
2016-03-25 14:27:30.827836 I | etcdserver: saved snapshot at index 190045
2016-03-25 14:27:30.828020 I | etcdserver: compacted raft log at 185045
2016-03-25 14:27:43.697863 I | etcdserver: start to snapshot (applied: 200048, lastsnap: 190045)
2016-03-25 14:27:46.586413 I | etcdserver: saved snapshot at index 200048
2016-03-25 14:27:46.586620 I | etcdserver: compacted raft log at 195048
2016-03-25 14:27:49.555432 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000022305.snap successfully
2016-03-25 14:27:49.575052 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000024a16.snap successfully
2016-03-25 14:27:59.946729 I | etcdserver: start to snapshot (applied: 210050, lastsnap: 200048)
2016-03-25 14:28:03.121728 I | etcdserver: saved snapshot at index 210050
2016-03-25 14:28:03.121953 I | etcdserver: compacted raft log at 205050
2016-03-25 14:28:16.052978 I | etcdserver: start to snapshot (applied: 220056, lastsnap: 210050)
2016-03-25 14:28:19.625598 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027129.snap successfully
2016-03-25 14:28:19.649392 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002983a.snap successfully
2016-03-25 14:28:19.691685 I | etcdserver: saved snapshot at index 220056
2016-03-25 14:28:19.691872 I | etcdserver: compacted raft log at 215056
2016-03-25 14:28:32.344829 I | etcdserver: start to snapshot (applied: 230058, lastsnap: 220056)
2016-03-25 14:28:35.908317 I | etcdserver: saved snapshot at index 230058
2016-03-25 14:28:35.908495 I | etcdserver: compacted raft log at 225058
2016-03-25 14:28:48.632511 I | etcdserver: start to snapshot (applied: 240059, lastsnap: 230058)
2016-03-25 14:28:49.673032 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf4b.snap successfully
2016-03-25 14:28:52.268466 I | etcdserver: saved snapshot at index 240059
2016-03-25 14:28:52.268660 I | etcdserver: compacted raft log at 235059
2016-03-25 14:29:04.972152 I | etcdserver: start to snapshot (applied: 250063, lastsnap: 240059)
2016-03-25 14:29:09.091490 I | etcdserver: saved snapshot at index 250063
2016-03-25 14:29:09.091756 I | etcdserver: compacted raft log at 245063
2016-03-25 14:29:19.698167 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e65d.snap successfully
2016-03-25 14:29:19.723543 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d70.snap successfully
2016-03-25 14:29:21.747140 I | etcdserver: start to snapshot (applied: 260067, lastsnap: 250063)
2016-03-25 14:29:26.116317 I | etcdserver: saved snapshot at index 260067
2016-03-25 14:29:26.116536 I | etcdserver: compacted raft log at 255067
2016-03-25 14:29:37.988865 I | etcdserver: start to snapshot (applied: 270069, lastsnap: 260067)
2016-03-25 14:29:42.185967 I | etcdserver: saved snapshot at index 270069
2016-03-25 14:29:42.186156 I | etcdserver: compacted raft log at 265069
2016-03-25 14:29:49.753029 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000033482.snap successfully
2016-03-25 14:29:49.781755 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035b98.snap successfully
2016-03-25 14:29:54.478160 I | etcdserver: start to snapshot (applied: 280075, lastsnap: 270069)
2016-03-25 14:29:58.997351 I | etcdserver: saved snapshot at index 280075
2016-03-25 14:29:58.997549 I | etcdserver: compacted raft log at 275075
2016-03-25 14:30:11.148398 I | etcdserver: start to snapshot (applied: 290078, lastsnap: 280075)
2016-03-25 14:30:15.756426 I | etcdserver: saved snapshot at index 290078
2016-03-25 14:30:15.756749 I | etcdserver: compacted raft log at 285078
2016-03-25 14:30:19.812905 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000382aa.snap successfully
2016-03-25 14:30:19.845172 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a9bb.snap successfully
2016-03-25 14:30:27.383815 I | etcdserver: start to snapshot (applied: 300081, lastsnap: 290078)
2016-03-25 14:30:31.956100 I | etcdserver: saved snapshot at index 300081
2016-03-25 14:30:31.956370 I | etcdserver: compacted raft log at 295081
2016-03-25 14:30:44.063911 I | etcdserver: start to snapshot (applied: 310088, lastsnap: 300081)
2016-03-25 14:30:48.984193 I | etcdserver: saved snapshot at index 310088
2016-03-25 14:30:48.984405 I | etcdserver: compacted raft log at 305088
2016-03-25 14:30:49.877354 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0cf.snap successfully
2016-03-25 14:30:49.909886 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003f7e3.snap successfully
2016-03-25 14:30:53.455035 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000002-000000000004ce5c.wal is created
2016-03-25 14:31:01.166855 I | etcdserver: start to snapshot (applied: 320090, lastsnap: 310088)
2016-03-25 14:31:05.926669 I | etcdserver: saved snapshot at index 320090
2016-03-25 14:31:05.926864 I | etcdserver: compacted raft log at 315090
2016-03-25 14:31:17.925065 I | etcdserver: start to snapshot (applied: 330094, lastsnap: 320090)
2016-03-25 14:31:19.947898 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000041ef5.snap successfully
2016-03-25 14:31:23.493448 I | etcdserver: saved snapshot at index 330094
2016-03-25 14:31:23.493653 I | etcdserver: compacted raft log at 325094
2016-03-25 14:31:34.895343 I | etcdserver: start to snapshot (applied: 340096, lastsnap: 330094)
2016-03-25 14:31:40.065006 I | etcdserver: saved snapshot at index 340096
2016-03-25 14:31:40.065205 I | etcdserver: compacted raft log at 335096
2016-03-25 14:31:49.987095 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004460b.snap successfully
2016-03-25 14:31:50.022529 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000046d1e.snap successfully
2016-03-25 14:31:51.943076 I | etcdserver: start to snapshot (applied: 350100, lastsnap: 340096)
2016-03-25 14:31:57.282174 I | etcdserver: saved snapshot at index 350100
2016-03-25 14:31:57.282581 I | etcdserver: compacted raft log at 345100
2016-03-25 14:32:11.274871 I | etcdserver: start to snapshot (applied: 360101, lastsnap: 350100)
2016-03-25 14:32:16.716824 I | etcdserver: saved snapshot at index 360101
2016-03-25 14:32:16.717026 I | etcdserver: compacted raft log at 355101
2016-03-25 14:32:20.062158 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000049431.snap successfully
2016-03-25 14:32:20.102554 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004bb48.snap successfully
2016-03-25 14:32:27.248635 I | etcdserver: start to snapshot (applied: 370105, lastsnap: 360101)
2016-03-25 14:32:32.784547 I | etcdserver: saved snapshot at index 370105
2016-03-25 14:32:32.784812 I | etcdserver: compacted raft log at 365105
2016-03-25 14:32:43.375914 I | etcdserver: start to snapshot (applied: 380110, lastsnap: 370105)
2016-03-25 14:32:49.178908 I | etcdserver: saved snapshot at index 380110
2016-03-25 14:32:49.179138 I | etcdserver: compacted raft log at 375110
2016-03-25 14:32:50.144670 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004e25a.snap successfully
2016-03-25 14:32:50.188139 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005096e.snap successfully
2016-03-25 14:33:03.354190 I | etcdserver: start to snapshot (applied: 390118, lastsnap: 380110)
2016-03-25 14:33:09.306823 I | etcdserver: saved snapshot at index 390118
2016-03-25 14:33:09.307090 I | etcdserver: compacted raft log at 385118
2016-03-25 14:33:19.128236 I | etcdserver: start to snapshot (applied: 400119, lastsnap: 390118)
2016-03-25 14:33:20.231104 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000053080.snap successfully
2016-03-25 14:33:25.479340 I | etcdserver: saved snapshot at index 400119
2016-03-25 14:33:25.479611 I | etcdserver: compacted raft log at 395119
2016-03-25 14:33:35.175998 I | etcdserver: start to snapshot (applied: 410121, lastsnap: 400119)
2016-03-25 14:33:41.830578 I | etcdserver: saved snapshot at index 410121
2016-03-25 14:33:41.830797 I | etcdserver: compacted raft log at 405121
2016-03-25 14:33:50.275867 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000055794.snap successfully
2016-03-25 14:33:50.321439 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000057ea5.snap successfully
2016-03-25 14:33:55.054806 I | etcdserver: start to snapshot (applied: 420122, lastsnap: 410121)
2016-03-25 14:34:01.414761 I | etcdserver: saved snapshot at index 420122
2016-03-25 14:34:01.414965 I | etcdserver: compacted raft log at 415122
2016-03-25 14:34:11.629005 I | etcdserver: start to snapshot (applied: 430125, lastsnap: 420122)
2016-03-25 14:34:18.655685 I | etcdserver: saved snapshot at index 430125
2016-03-25 14:34:18.655909 I | etcdserver: compacted raft log at 425125
2016-03-25 14:34:20.369642 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005a5b9.snap successfully
2016-03-25 14:34:20.418930 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005ccce.snap successfully
2016-03-25 14:34:28.349612 I | etcdserver: start to snapshot (applied: 440127, lastsnap: 430125)
2016-03-25 14:34:35.346940 I | etcdserver: saved snapshot at index 440127
2016-03-25 14:34:35.347147 I | etcdserver: compacted raft log at 435127
2016-03-25 14:34:46.091980 I | etcdserver: start to snapshot (applied: 450128, lastsnap: 440127)
2016-03-25 14:34:50.511875 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005f3e6.snap successfully
2016-03-25 14:34:53.095996 I | etcdserver: saved snapshot at index 450128
2016-03-25 14:34:53.096297 I | etcdserver: compacted raft log at 445128
2016-03-25 14:35:04.534914 I | etcdserver: start to snapshot (applied: 460130, lastsnap: 450128)
2016-03-25 14:35:11.903381 I | etcdserver: saved snapshot at index 460130
2016-03-25 14:35:11.903593 I | etcdserver: compacted raft log at 455130
2016-03-25 14:35:20.562658 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000061af7.snap successfully
2016-03-25 14:35:20.613983 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000064209.snap successfully
2016-03-25 14:35:22.673395 I | etcdserver: start to snapshot (applied: 470136, lastsnap: 460130)
2016-03-25 14:35:28.049412 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000003-0000000000073593.wal is created
2016-03-25 14:35:29.919376 I | etcdserver: saved snapshot at index 470136
2016-03-25 14:35:29.919593 I | etcdserver: compacted raft log at 465136
2016-03-25 14:35:43.353411 I | etcdserver: start to snapshot (applied: 480144, lastsnap: 470136)
2016-03-25 14:35:50.668294 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006691a.snap successfully
2016-03-25 14:35:50.972120 I | etcdserver: saved snapshot at index 480144
2016-03-25 14:35:50.972460 I | etcdserver: compacted raft log at 475144
2016-03-25 14:36:00.074216 I | etcdserver: start to snapshot (applied: 490147, lastsnap: 480144)
2016-03-25 14:36:08.124368 I | etcdserver: saved snapshot at index 490147
2016-03-25 14:36:08.124616 I | etcdserver: compacted raft log at 485147
2016-03-25 14:36:19.356254 I | etcdserver: start to snapshot (applied: 500152, lastsnap: 490147)
2016-03-25 14:36:20.722626 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006902d.snap successfully
2016-03-25 14:36:20.776952 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006b73f.snap successfully
2016-03-25 14:36:27.314754 I | etcdserver: saved snapshot at index 500152
2016-03-25 14:36:27.315079 I | etcdserver: compacted raft log at 495152
2016-03-25 14:36:36.842128 I | etcdserver: start to snapshot (applied: 510154, lastsnap: 500152)
2016-03-25 14:36:45.217723 I | etcdserver: saved snapshot at index 510154
2016-03-25 14:36:45.218020 I | etcdserver: compacted raft log at 505154
2016-03-25 14:36:50.837513 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006de50.snap successfully
2016-03-25 14:36:50.898167 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000070562.snap successfully
2016-03-25 14:36:55.877744 I | etcdserver: start to snapshot (applied: 520161, lastsnap: 510154)
2016-03-25 14:37:04.148108 I | etcdserver: saved snapshot at index 520161
2016-03-25 14:37:04.148394 I | etcdserver: compacted raft log at 515161
2016-03-25 14:37:12.863755 I | etcdserver: start to snapshot (applied: 530167, lastsnap: 520161)
2016-03-25 14:37:20.961445 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000072c78.snap successfully
2016-03-25 14:37:21.026413 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000075390.snap successfully
2016-03-25 14:37:21.795908 I | etcdserver: saved snapshot at index 530167
2016-03-25 14:37:21.796174 I | etcdserver: compacted raft log at 525167
2016-03-25 14:37:32.122563 I | etcdserver: start to snapshot (applied: 540168, lastsnap: 530167)
2016-03-25 14:37:40.678189 I | etcdserver: saved snapshot at index 540168
2016-03-25 14:37:40.678471 I | etcdserver: compacted raft log at 535168
2016-03-25 14:37:49.002147 I | etcdserver: start to snapshot (applied: 550174, lastsnap: 540168)
2016-03-25 14:37:51.088293 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000077aa3.snap successfully
2016-03-25 14:37:58.147213 I | etcdserver: saved snapshot at index 550174
2016-03-25 14:37:58.147563 I | etcdserver: compacted raft log at 545174
2016-03-25 14:38:08.588083 I | etcdserver: start to snapshot (applied: 560175, lastsnap: 550174)
2016-03-25 14:38:17.778197 I | etcdserver: saved snapshot at index 560175
2016-03-25 14:38:17.778509 I | etcdserver: compacted raft log at 555175
2016-03-25 14:38:21.153361 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007a1b8.snap successfully
2016-03-25 14:38:21.218360 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007c8ca.snap successfully
2016-03-25 14:38:25.263806 I | etcdserver: start to snapshot (applied: 570176, lastsnap: 560175)
2016-03-25 14:38:34.274498 I | etcdserver: saved snapshot at index 570176
2016-03-25 14:38:34.274792 I | etcdserver: compacted raft log at 565176
2016-03-25 14:38:44.822181 I | etcdserver: start to snapshot (applied: 580180, lastsnap: 570176)
2016-03-25 14:38:51.285167 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007efe1.snap successfully
2016-03-25 14:38:54.097123 I | etcdserver: saved snapshot at index 580180
2016-03-25 14:38:54.097586 I | etcdserver: compacted raft log at 575180
2016-03-25 14:39:01.611216 I | etcdserver: start to snapshot (applied: 590183, lastsnap: 580180)
2016-03-25 14:39:11.221055 I | etcdserver: saved snapshot at index 590183
2016-03-25 14:39:11.221487 I | etcdserver: compacted raft log at 585183
2016-03-25 14:39:21.260877 I | etcdserver: start to snapshot (applied: 600186, lastsnap: 590183)
2016-03-25 14:39:21.355481 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000816f7.snap successfully
2016-03-25 14:39:21.425098 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000083e08.snap successfully
2016-03-25 14:39:30.683143 I | etcdserver: saved snapshot at index 600186
2016-03-25 14:39:30.683528 I | etcdserver: compacted raft log at 595186
2016-03-25 14:39:38.528198 I | etcdserver: start to snapshot (applied: 610188, lastsnap: 600186)
2016-03-25 14:39:49.001700 I | etcdserver: saved snapshot at index 610188
2016-03-25 14:39:49.002200 I | etcdserver: compacted raft log at 605188
2016-03-25 14:39:51.495751 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008651e.snap successfully
2016-03-25 14:39:51.566893 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000088c2f.snap successfully
2016-03-25 14:39:58.523839 I | etcdserver: start to snapshot (applied: 620192, lastsnap: 610188)
2016-03-25 14:40:08.589023 I | etcdserver: saved snapshot at index 620192
2016-03-25 14:40:08.589393 I | etcdserver: compacted raft log at 615192
2016-03-25 14:40:14.793976 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000004-0000000000099c40.wal is created
2016-03-25 14:40:15.502705 I | etcdserver: start to snapshot (applied: 630193, lastsnap: 620192)
2016-03-25 14:40:21.640806 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008b340.snap successfully
2016-03-25 14:40:25.807418 I | etcdserver: saved snapshot at index 630193
2016-03-25 14:40:25.807704 I | etcdserver: compacted raft log at 625193
2016-03-25 14:40:35.499906 I | etcdserver: start to snapshot (applied: 640194, lastsnap: 630193)
2016-03-25 14:40:45.988254 I | etcdserver: saved snapshot at index 640194
2016-03-25 14:40:45.988728 I | etcdserver: compacted raft log at 635194
2016-03-25 14:40:51.712047 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008da54.snap successfully
2016-03-25 14:40:51.783824 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000090167.snap successfully
2016-03-25 14:40:54.053456 I | etcdserver: start to snapshot (applied: 650196, lastsnap: 640194)
2016-03-25 14:41:04.350123 I | etcdserver: saved snapshot at index 650196
2016-03-25 14:41:04.350442 I | etcdserver: compacted raft log at 645196
2016-03-25 14:41:13.128538 I | etcdserver: start to snapshot (applied: 660198, lastsnap: 650196)
2016-03-25 14:41:21.860655 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009287a.snap successfully
2016-03-25 14:41:23.708936 I | etcdserver: saved snapshot at index 660198
2016-03-25 14:41:23.709258 I | etcdserver: compacted raft log at 655198
2016-03-25 14:41:31.842627 I | etcdserver: start to snapshot (applied: 670199, lastsnap: 660198)
2016-03-25 14:41:42.457034 I | etcdserver: saved snapshot at index 670199
2016-03-25 14:41:42.457380 I | etcdserver: compacted raft log at 665199
2016-03-25 14:41:50.255723 I | etcdserver: start to snapshot (applied: 680202, lastsnap: 670199)
2016-03-25 14:41:51.936439 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000094f8c.snap successfully
2016-03-25 14:41:52.014300 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000976a0.snap successfully
2016-03-25 14:42:01.355250 I | etcdserver: saved snapshot at index 680202
2016-03-25 14:42:01.355646 I | etcdserver: compacted raft log at 675202
2016-03-25 14:42:08.956342 I | etcdserver: start to snapshot (applied: 690204, lastsnap: 680202)
2016-03-25 14:42:20.220704 I | etcdserver: saved snapshot at index 690204
2016-03-25 14:42:20.221058 I | etcdserver: compacted raft log at 685204
2016-03-25 14:42:22.095693 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000099db1.snap successfully
2016-03-25 14:42:22.177185 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009c4c2.snap successfully
2016-03-25 14:42:27.748787 I | etcdserver: start to snapshot (applied: 700209, lastsnap: 690204)
2016-03-25 14:42:39.442632 I | etcdserver: saved snapshot at index 700209
2016-03-25 14:42:39.442956 I | etcdserver: compacted raft log at 695209
2016-03-25 14:42:46.306948 I | etcdserver: start to snapshot (applied: 710212, lastsnap: 700209)
2016-03-25 14:42:52.202893 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009ebd4.snap successfully
2016-03-25 14:42:58.719675 I | etcdserver: saved snapshot at index 710212
2016-03-25 14:42:58.719991 I | etcdserver: compacted raft log at 705212
2016-03-25 14:43:05.649623 I | etcdserver: start to snapshot (applied: 720213, lastsnap: 710212)
2016-03-25 14:43:16.996982 I | etcdserver: saved snapshot at index 720213
2016-03-25 14:43:16.997345 I | etcdserver: compacted raft log at 715213
2016-03-25 14:43:22.216088 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a12e6.snap successfully
2016-03-25 14:43:22.293721 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a39f7.snap successfully
2016-03-25 14:43:25.074337 I | etcdserver: start to snapshot (applied: 730214, lastsnap: 720213)
2016-03-25 14:43:37.174219 I | etcdserver: saved snapshot at index 730214
2016-03-25 14:43:37.174637 I | etcdserver: compacted raft log at 725214
2016-03-25 14:43:44.194089 I | etcdserver: start to snapshot (applied: 740215, lastsnap: 730214)
2016-03-25 14:43:52.367907 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a610a.snap successfully
2016-03-25 14:43:56.186871 I | etcdserver: saved snapshot at index 740215
2016-03-25 14:43:56.187207 I | etcdserver: compacted raft log at 735215
2016-03-25 14:44:03.352554 I | etcdserver: start to snapshot (applied: 750217, lastsnap: 740215)
2016-03-25 14:44:15.182805 I | etcdserver: saved snapshot at index 750217
2016-03-25 14:44:15.183089 I | etcdserver: compacted raft log at 745217
2016-03-25 14:44:20.442713 I | etcdserver: start to snapshot (applied: 760218, lastsnap: 750217)
2016-03-25 14:44:22.375682 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a881c.snap successfully
2016-03-25 14:44:22.439444 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000aaf31.snap successfully
2016-03-25 14:44:33.199052 I | etcdserver: saved snapshot at index 760218
2016-03-25 14:44:33.199413 I | etcdserver: compacted raft log at 755218
2016-03-25 14:44:40.610602 I | etcdserver: start to snapshot (applied: 770219, lastsnap: 760218)
2016-03-25 14:44:52.488034 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000ad644.snap successfully
2016-03-25 14:44:52.580659 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000afd55.snap successfully
2016-03-25 14:44:53.437946 I | etcdserver: saved snapshot at index 770219
2016-03-25 14:44:53.438269 I | etcdserver: compacted raft log at 765219
2016-03-25 14:45:00.388362 I | etcdserver: start to snapshot (applied: 780221, lastsnap: 770219)
2016-03-25 14:45:12.489792 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000005-00000000000c0366.wal is created
2016-03-25 14:45:13.249507 I | etcdserver: saved snapshot at index 780221
2016-03-25 14:45:13.249873 I | etcdserver: compacted raft log at 775221
2016-03-25 14:45:17.769904 I | etcdserver: start to snapshot (applied: 790226, lastsnap: 780221)
2016-03-25 14:45:19.488496 I | fileutil: purged file /mnt/ssd0/data.etcd/member/wal/0000000000000000-0000000000000000.wal successfully
2016-03-25 14:45:22.624964 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b2466.snap successfully
2016-03-25 14:45:31.128138 I | etcdserver: saved snapshot at index 790226
2016-03-25 14:45:31.128492 I | etcdserver: compacted raft log at 785226
2016-03-25 14:45:38.854139 I | etcdserver: start to snapshot (applied: 800232, lastsnap: 790226)
2016-03-25 14:45:51.880987 I | etcdserver: saved snapshot at index 800232
2016-03-25 14:45:51.881304 I | etcdserver: compacted raft log at 795232
2016-03-25 14:45:52.633777 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b4b77.snap successfully
2016-03-25 14:45:52.674735 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b7289.snap successfully
2016-03-25 14:45:56.490755 I | etcdserver: start to snapshot (applied: 810233, lastsnap: 800232)
2016-03-25 14:46:09.924852 I | etcdserver: saved snapshot at index 810233
2016-03-25 14:46:09.925413 I | etcdserver: compacted raft log at 805233
2016-03-25 14:46:17.429820 I | etcdserver: start to snapshot (applied: 820234, lastsnap: 810233)
2016-03-25 14:46:22.675067 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b999a.snap successfully
2016-03-25 14:46:30.473895 I | etcdserver: saved snapshot at index 820234
2016-03-25 14:46:30.474238 I | etcdserver: compacted raft log at 815234
2016-03-25 14:46:34.908555 I | etcdserver: start to snapshot (applied: 830236, lastsnap: 820234)
2016-03-25 14:46:48.073128 I | etcdserver: saved snapshot at index 830236
2016-03-25 14:46:48.073779 I | etcdserver: compacted raft log at 825236
2016-03-25 14:46:52.675413 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000bc0ab.snap successfully
2016-03-25 14:46:52.675532 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000be7bd.snap successfully
2016-03-25 14:46:56.591116 I | etcdserver: start to snapshot (applied: 840239, lastsnap: 830236)
2016-03-25 14:47:10.412942 I | etcdserver: saved snapshot at index 840239
2016-03-25 14:47:10.413349 I | etcdserver: compacted raft log at 835239
2016-03-25 14:47:14.262521 I | etcdserver: start to snapshot (applied: 850240, lastsnap: 840239)
2016-03-25 14:47:22.675862 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c0ed2.snap successfully
2016-03-25 14:47:28.313706 I | etcdserver: saved snapshot at index 850240
2016-03-25 14:47:28.314469 I | etcdserver: compacted raft log at 845240
2016-03-25 14:47:36.219802 I | etcdserver: start to snapshot (applied: 860242, lastsnap: 850240)
2016-03-25 14:47:50.931988 I | etcdserver: saved snapshot at index 860242
2016-03-25 14:47:50.932652 I | etcdserver: compacted raft log at 855242
2016-03-25 14:47:52.676222 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c35e8.snap successfully
2016-03-25 14:47:52.739201 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c5cf9.snap successfully
2016-03-25 14:47:55.036154 I | etcdserver: start to snapshot (applied: 870246, lastsnap: 860242)
2016-03-25 14:48:09.154449 I | etcdserver: saved snapshot at index 870246
2016-03-25 14:48:09.154788 I | etcdserver: compacted raft log at 865246
2016-03-25 14:48:15.972112 I | etcdserver: start to snapshot (applied: 880264, lastsnap: 870246)
2016-03-25 14:48:22.785144 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c840a.snap successfully
2016-03-25 14:48:30.882346 I | etcdserver: saved snapshot at index 880264
2016-03-25 14:48:30.882698 I | etcdserver: compacted raft log at 875264
2016-03-25 14:48:35.816736 I | etcdserver: start to snapshot (applied: 890266, lastsnap: 880264)
2016-03-25 14:48:51.187147 I | etcdserver: saved snapshot at index 890266
2016-03-25 14:48:51.187518 I | etcdserver: compacted raft log at 885266
2016-03-25 14:48:52.798949 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cab1c.snap successfully
2016-03-25 14:48:52.814222 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cd22f.snap successfully
2016-03-25 14:48:59.441595 I | etcdserver: start to snapshot (applied: 900270, lastsnap: 890266)
2016-03-25 14:49:15.059823 I | etcdserver: saved snapshot at index 900270
2016-03-25 14:49:15.060278 I | etcdserver: compacted raft log at 895270
2016-03-25 14:49:18.541197 I | etcdserver: start to snapshot (applied: 910271, lastsnap: 900270)
2016-03-25 14:49:22.825938 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cf940.snap successfully
2016-03-25 14:49:33.236970 I | raft: 208de484d8b2cb22 [term: 3] received a MsgVote message with higher term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:33.237026 I | raft: 208de484d8b2cb22 became follower at term 4
2016-03-25 14:49:33.237040 I | raft: 208de484d8b2cb22 [logterm: 3, index: 917652, vote: 0] voted for ef3ac5ee77284022 [logterm: 3, index: 917652] at term 4
2016-03-25 14:49:33.237052 I | raft: raft.node: 208de484d8b2cb22 lost leader b0978d17c833d412 at term 4
2016-03-25 14:49:33.321577 I | raft: raft.node: 208de484d8b2cb22 elected leader ef3ac5ee77284022 at term 4
2016-03-25 14:49:33.591441 I | etcdserver: saved snapshot at index 910271
2016-03-25 14:49:33.591751 I | etcdserver: compacted raft log at 905271
2016-03-25 14:49:33.867177 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.867234 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.867252 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.867261 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.867275 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.867284 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.867298 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.867307 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.867332 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.867341 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.867352 I | raft: 208de484d8b2cb22 [term: 4] ignored a MsgHeartbeat message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:39.054279 I | etcdserver: start to snapshot (applied: 920273, lastsnap: 910271)
2016-03-25 14:49:52.835265 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d2052.snap successfully
2016-03-25 14:49:53.766455 I | raft: 208de484d8b2cb22 is starting a new election at term 4
2016-03-25 14:49:53.766530 I | raft: 208de484d8b2cb22 became candidate at term 5
2016-03-25 14:49:53.766557 I | raft: 208de484d8b2cb22 received vote from 208de484d8b2cb22 at term 5
2016-03-25 14:49:53.766572 I | raft: 208de484d8b2cb22 [logterm: 4, index: 927549] sent vote request to b0978d17c833d412 at term 5
2016-03-25 14:49:53.766584 I | raft: 208de484d8b2cb22 [logterm: 4, index: 927549] sent vote request to ef3ac5ee77284022 at term 5
2016-03-25 14:49:53.766593 I | raft: raft.node: 208de484d8b2cb22 lost leader ef3ac5ee77284022 at term 5
2016-03-25 14:49:53.847363 I | raft: 208de484d8b2cb22 received vote from b0978d17c833d412 at term 5
2016-03-25 14:49:53.847404 I | raft: 208de484d8b2cb22 [quorum:2] has received 2 votes and 0 vote rejections
2016-03-25 14:49:53.847436 I | raft: 208de484d8b2cb22 became leader at term 5
2016-03-25 14:49:53.847452 I | raft: raft.node: 208de484d8b2cb22 elected leader 208de484d8b2cb22 at term 5
2016-03-25 14:49:53.939695 I | etcdserver: saved snapshot at index 920273
2016-03-25 14:49:53.940084 I | etcdserver: compacted raft log at 915273
2016-03-25 14:49:54.763626 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.763667 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.763695 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776573 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776601 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776619 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776635 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776651 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776660 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776675 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776683 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776697 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776706 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776721 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776730 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776744 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776752 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776770 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776779 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776801 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776817 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776832 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:54.776853 I | raft: 208de484d8b2cb22 [term: 5] ignored a MsgApp message with lower term from ef3ac5ee77284022 [term: 4]
2016-03-25 14:49:55.175465 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 455.2449ms)
2016-03-25 14:49:55.175617 W | etcdserver: server is likely overloaded
2016-03-25 14:49:55.175627 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 455.419034ms)
2016-03-25 14:49:55.175633 W | etcdserver: server is likely overloaded
2016-03-25 14:49:59.439465 E | etcdhttp: etcdserver: request timed out, possibly due to previous leader failure
2016-03-25 14:50:01.182992 I | etcdserver: start to snapshot (applied: 930274, lastsnap: 920273)
2016-03-25 14:50:01.354345 E | etcdhttp: etcdserver: request timed out, possibly due to previous leader failure [merged 4 repeated lines in 1.91s]
2016-03-25 14:50:10.397420 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 31.091385ms)
2016-03-25 14:50:10.397471 W | etcdserver: server is likely overloaded
2016-03-25 14:50:10.397484 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 31.16841ms)
2016-03-25 14:50:10.397492 W | etcdserver: server is likely overloaded
2016-03-25 14:50:10.828453 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 45.725994ms)
2016-03-25 14:50:10.828502 W | etcdserver: server is likely overloaded
2016-03-25 14:50:10.828514 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 45.81266ms)
2016-03-25 14:50:10.828522 W | etcdserver: server is likely overloaded
2016-03-25 14:50:17.237455 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 34.237272ms)
2016-03-25 14:50:17.237502 W | etcdserver: server is likely overloaded
2016-03-25 14:50:17.237511 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 34.308066ms)
2016-03-25 14:50:17.237516 W | etcdserver: server is likely overloaded
2016-03-25 14:50:17.237730 I | etcdserver: saved snapshot at index 930274
2016-03-25 14:50:17.238050 I | etcdserver: compacted raft log at 925274
2016-03-25 14:50:17.639479 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 66.432546ms)
2016-03-25 14:50:17.639531 W | etcdserver: server is likely overloaded
2016-03-25 14:50:17.639540 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 66.511733ms)
2016-03-25 14:50:17.639545 W | etcdserver: server is likely overloaded
2016-03-25 14:50:18.225648 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 57.656131ms)
2016-03-25 14:50:18.225735 W | etcdserver: server is likely overloaded
2016-03-25 14:50:18.225745 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 57.777677ms)
2016-03-25 14:50:18.225750 W | etcdserver: server is likely overloaded
2016-03-25 14:50:21.674997 I | etcdserver: start to snapshot (applied: 940281, lastsnap: 930274)
2016-03-25 14:50:22.850567 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d4766.snap successfully
2016-03-25 14:50:22.861538 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d6e88.snap successfully
2016-03-25 14:50:23.626631 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 60.18795ms)
2016-03-25 14:50:23.626674 W | etcdserver: server is likely overloaded
2016-03-25 14:50:23.626683 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 60.251244ms)
2016-03-25 14:50:23.626689 W | etcdserver: server is likely overloaded
2016-03-25 14:50:31.491326 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000006-00000000000e6a51.wal is created
2016-03-25 14:50:36.929554 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 461.782005ms)
2016-03-25 14:50:36.929605 W | etcdserver: server is likely overloaded
2016-03-25 14:50:36.929627 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 461.883658ms)
2016-03-25 14:50:36.929640 W | etcdserver: server is likely overloaded
2016-03-25 14:50:37.832425 I | etcdserver: saved snapshot at index 940281
2016-03-25 14:50:37.832711 I | etcdserver: compacted raft log at 935281
2016-03-25 14:50:44.801541 I | etcdserver: start to snapshot (applied: 950284, lastsnap: 940281)
2016-03-25 14:50:49.490932 I | fileutil: purged file /mnt/ssd0/data.etcd/member/wal/0000000000000001-000000000002673d.wal successfully
2016-03-25 14:50:52.874623 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d959a.snap successfully
2016-03-25 14:50:57.401139 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 32.886407ms)
2016-03-25 14:50:57.401184 W | etcdserver: server is likely overloaded
2016-03-25 14:50:57.401193 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 32.95014ms)
2016-03-25 14:50:57.401199 W | etcdserver: server is likely overloaded
2016-03-25 14:51:00.820543 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 47.454451ms)
2016-03-25 14:51:00.820575 W | etcdserver: server is likely overloaded
2016-03-25 14:51:00.820590 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 47.519697ms)
2016-03-25 14:51:00.820598 W | etcdserver: server is likely overloaded
2016-03-25 14:51:01.306934 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 125.859307ms)
2016-03-25 14:51:01.306993 W | etcdserver: server is likely overloaded
2016-03-25 14:51:01.307020 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 125.956223ms)
2016-03-25 14:51:01.307030 W | etcdserver: server is likely overloaded
2016-03-25 14:51:01.307090 I | etcdserver: saved snapshot at index 950284
2016-03-25 14:51:01.307499 I | etcdserver: compacted raft log at 945284
2016-03-25 14:51:01.827034 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 49.452736ms)
2016-03-25 14:51:01.827072 W | etcdserver: server is likely overloaded
2016-03-25 14:51:01.827080 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 49.515823ms)
2016-03-25 14:51:01.827086 W | etcdserver: server is likely overloaded
2016-03-25 14:51:05.867820 I | etcdserver: start to snapshot (applied: 960285, lastsnap: 950284)
2016-03-25 14:51:07.836330 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 57.277873ms)
2016-03-25 14:51:07.836390 W | etcdserver: server is likely overloaded
2016-03-25 14:51:07.836403 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 57.367343ms)
2016-03-25 14:51:07.836411 W | etcdserver: server is likely overloaded
2016-03-25 14:51:20.204063 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 16.121304ms)
2016-03-25 14:51:20.204120 W | etcdserver: server is likely overloaded
2016-03-25 14:51:20.204136 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 36.340824ms)
2016-03-25 14:51:20.204144 W | etcdserver: server is likely overloaded
2016-03-25 14:51:22.142041 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 69.789444ms)
2016-03-25 14:51:22.142081 W | etcdserver: server is likely overloaded
2016-03-25 14:51:22.142090 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 69.854926ms)
2016-03-25 14:51:22.142097 W | etcdserver: server is likely overloaded
2016-03-25 14:51:22.661543 I | etcdserver: saved snapshot at index 960285
2016-03-25 14:51:22.661961 I | etcdserver: compacted raft log at 955285
2016-03-25 14:51:22.874937 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000dbcae.snap successfully
2016-03-25 14:51:22.875025 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000de3bf.snap successfully
2016-03-25 14:51:23.594118 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 599.875828ms)
2016-03-25 14:51:23.594175 W | etcdserver: server is likely overloaded
2016-03-25 14:51:23.594183 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 599.956765ms)
2016-03-25 14:51:23.594205 W | etcdserver: server is likely overloaded
2016-03-25 14:51:27.351692 I | etcdserver: start to snapshot (applied: 970286, lastsnap: 960285)
2016-03-25 14:51:29.236673 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 50.438097ms)
2016-03-25 14:51:29.236706 W | etcdserver: server is likely overloaded
2016-03-25 14:51:29.236715 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 50.492521ms)
2016-03-25 14:51:29.236720 W | etcdserver: server is likely overloaded
2016-03-25 14:51:34.490566 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 319.922263ms)
2016-03-25 14:51:34.490625 W | etcdserver: server is likely overloaded
2016-03-25 14:51:34.490638 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 320.012956ms)
2016-03-25 14:51:34.490646 W | etcdserver: server is likely overloaded
2016-03-25 14:51:43.073564 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 705.922926ms)
2016-03-25 14:51:43.073622 W | etcdserver: server is likely overloaded
2016-03-25 14:51:43.073853 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 706.22831ms)
2016-03-25 14:51:43.073887 W | etcdserver: server is likely overloaded
2016-03-25 14:51:44.625304 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 58.826353ms)
2016-03-25 14:51:44.625374 W | etcdserver: server is likely overloaded
2016-03-25 14:51:44.625383 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 58.922689ms)
2016-03-25 14:51:44.625388 W | etcdserver: server is likely overloaded
2016-03-25 14:51:44.904660 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 251.1µs)
2016-03-25 14:51:44.904693 W | etcdserver: server is likely overloaded
2016-03-25 14:51:44.904702 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 312.274µs)
2016-03-25 14:51:44.904707 W | etcdserver: server is likely overloaded
2016-03-25 14:51:45.060602 I | etcdserver: saved snapshot at index 970286
2016-03-25 14:51:45.061051 I | etcdserver: compacted raft log at 965286
2016-03-25 14:51:45.733583 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 55.194279ms)
2016-03-25 14:51:45.733639 W | etcdserver: server is likely overloaded
2016-03-25 14:51:45.733673 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 55.310224ms)
2016-03-25 14:51:45.733691 W | etcdserver: server is likely overloaded
2016-03-25 14:51:49.159577 I | etcdserver: start to snapshot (applied: 980287, lastsnap: 970286)
2016-03-25 14:51:51.619986 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 35.070597ms)
2016-03-25 14:51:51.620039 W | etcdserver: server is likely overloaded
2016-03-25 14:51:51.620048 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 35.14755ms)
2016-03-25 14:51:51.620054 W | etcdserver: server is likely overloaded
2016-03-25 14:51:52.875379 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000004-00000000000e0ad1.snap successfully
2016-03-25 14:52:03.989836 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 50.294074ms)
2016-03-25 14:52:03.989920 W | etcdserver: server is likely overloaded
2016-03-25 14:52:03.989947 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 116.752529ms)
2016-03-25 14:52:03.989965 W | etcdserver: server is likely overloaded
2016-03-25 14:52:04.420800 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 45.286129ms)
2016-03-25 14:52:04.420859 W | etcdserver: server is likely overloaded
2016-03-25 14:52:04.420880 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 45.377025ms)
2016-03-25 14:52:04.420901 W | etcdserver: server is likely overloaded
2016-03-25 14:52:06.139987 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 38.962615ms)
2016-03-25 14:52:06.140031 W | etcdserver: server is likely overloaded
2016-03-25 14:52:06.140048 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 39.04144ms)
2016-03-25 14:52:06.140054 W | etcdserver: server is likely overloaded
2016-03-25 14:52:06.608267 I | etcdserver: saved snapshot at index 980287
2016-03-25 14:52:06.608678 I | etcdserver: compacted raft log at 975287
2016-03-25 14:52:07.121647 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 42.478757ms)
2016-03-25 14:52:07.121710 W | etcdserver: server is likely overloaded
2016-03-25 14:52:07.121723 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 42.573609ms)
2016-03-25 14:52:07.121733 W | etcdserver: server is likely overloaded
2016-03-25 14:52:11.113608 I | etcdserver: start to snapshot (applied: 990288, lastsnap: 980287)
2016-03-25 14:52:12.597686 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 26.470246ms)
2016-03-25 14:52:12.597735 W | etcdserver: server is likely overloaded
2016-03-25 14:52:12.597749 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 26.547716ms)
2016-03-25 14:52:12.597755 W | etcdserver: server is likely overloaded
2016-03-25 14:52:13.108095 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 39.581133ms)
2016-03-25 14:52:13.108234 W | etcdserver: server is likely overloaded
2016-03-25 14:52:13.108252 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 39.753683ms)
2016-03-25 14:52:13.108268 W | etcdserver: server is likely overloaded
2016-03-25 14:52:22.888974 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000005-00000000000e31e2.snap successfully
2016-03-25 14:52:26.556574 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 489.339063ms)
2016-03-25 14:52:26.556642 W | etcdserver: server is likely overloaded
2016-03-25 14:52:26.556664 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 489.448896ms)
2016-03-25 14:52:26.556677 W | etcdserver: server is likely overloaded
2016-03-25 14:52:28.101855 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 335.405677ms)
2016-03-25 14:52:28.101914 W | etcdserver: server is likely overloaded
2016-03-25 14:52:28.101934 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 335.503182ms)
2016-03-25 14:52:28.101946 W | etcdserver: server is likely overloaded
2016-03-25 14:52:29.601883 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 33.957836ms)
2016-03-25 14:52:29.601949 W | etcdserver: server is likely overloaded
2016-03-25 14:52:29.601968 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 34.076385ms)
2016-03-25 14:52:29.601987 W | etcdserver: server is likely overloaded
2016-03-25 14:52:30.027667 I | etcdserver: saved snapshot at index 990288
2016-03-25 14:52:30.027722 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 58.230753ms)
2016-03-25 14:52:30.027752 W | etcdserver: server is likely overloaded
2016-03-25 14:52:30.027774 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 58.291585ms)
2016-03-25 14:52:30.027790 W | etcdserver: server is likely overloaded
2016-03-25 14:52:30.028071 I | etcdserver: compacted raft log at 985288
2016-03-25 14:52:34.465095 I | etcdserver: start to snapshot (applied: 1000291, lastsnap: 990288)
2016-03-25 14:52:51.101850 I | raft: 208de484d8b2cb22 [term: 5] received a MsgApp message with higher term from b0978d17c833d412 [term: 6]
2016-03-25 14:52:51.103782 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 1.337271746s)
2016-03-25 14:52:51.103799 W | etcdserver: server is likely overloaded
2016-03-25 14:52:51.103812 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 1.337420326s)
2016-03-25 14:52:51.103820 W | etcdserver: server is likely overloaded
2016-03-25 14:52:51.106990 I | raft: 208de484d8b2cb22 became follower at term 6
2016-03-25 14:52:51.107062 I | raft: raft.node: 208de484d8b2cb22 changed leader from 208de484d8b2cb22 to b0978d17c833d412 at term 6
2016-03-25 14:52:51.107223 I | raft: 208de484d8b2cb22 [logterm: 6, index: 1003710, vote: 0] rejected vote from b0978d17c833d412 [logterm: 5, index: 1003707] at term 6
2016-03-25 14:52:53.153259 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000005-00000000000e58f9.snap successfully
2016-03-25 14:52:53.164546 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000005-00000000000e800c.snap successfully
2016-03-25 14:52:54.207527 I | etcdserver: saved snapshot at index 1000291
2016-03-25 14:52:54.207756 I | etcdserver: compacted raft log at 995291
2016-03-25 14:54:10.284194 E | rafthttp: failed to dial b0978d17c833d412 on stream MsgApp v2 (read tcp 10.240.0.37:50824->10.240.0.36:2380: read: connection reset by peer)
2016-03-25 14:54:10.284244 I | rafthttp: the connection with b0978d17c833d412 became inactive
2016-03-25 14:54:11.026399 N | osutil: received terminated signal, shutting down...
2016-03-25 14:54:11.027420 E | rafthttp: failed to read ef3ac5ee77284022 on stream MsgApp v2 (net/http: request canceled)
2016-03-25 14:54:11.027476 I | rafthttp: the connection with ef3ac5ee77284022 became inactive

File diff suppressed because it is too large Load Diff

View File

@ -0,0 +1,20 @@
{"level":"info","msg":"gRPC serving :3500","time":"2016-03-25T14:17:39Z"}
{"level":"info","msg":"Working directory: /mnt/ssd0","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"etcd data directory: /mnt/ssd0/data.etcd","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"Consul data directory: /mnt/ssd0/data.consul","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"Zookeeper working directory: /mnt/ssd0/zookeeper","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"Zookeeper data directory: /mnt/ssd0/zookeeper/data.zk","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"Database log path: /mnt/ssd0/database.log","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"Monitor result path: /mnt/ssd0/monitor.csv","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"Starting: /home/gyuho/go/bin/etcd --name etcd-3 --data-dir /mnt/ssd0/data.etcd --listen-client-urls http://10.240.0.38:2379 --advertise-client-urls http://10.240.0.38:2379 --listen-peer-urls http://10.240.0.38:2380 --initial-advertise-peer-urls http://10.240.0.38:2380 --initial-cluster-token etcd_token --initial-cluster etcd-1=http://10.240.0.36:2380,etcd-2=http://10.240.0.37:2380,etcd-3=http://10.240.0.38:2380 --initial-cluster-state new","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"Started: /home/gyuho/go/bin/etcd [PID: 23587]","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"etcd2 monitor saved at /mnt/ssd0/monitor.csv","time":"2016-03-25T14:21:50Z"}
{"level":"info","msg":"Stopping etcd2 [PID: 23587]","time":"2016-03-25T14:54:12Z"}
{"level":"info","msg":"Stopped: /home/gyuho/go/bin/etcd [PID: 23587]","time":"2016-03-25T14:54:12Z"}
{"level":"info","msg":"Transfer success!","time":"2016-03-25T14:54:12Z"}
{"level":"info","msg":"Monitoring stopped. Uploading data to cloud storage...","time":"2016-03-25T14:54:12Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/database.log to bench-02-etcd2-3-database.log","time":"2016-03-25T14:54:12Z"}
{"level":"info","msg":"/home/gyuho/go/bin/etcd cmd.Wait returned signal: terminated","time":"2016-03-25T14:54:12Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/monitor.csv to bench-02-etcd2-3-monitor.csv","time":"2016-03-25T14:54:13Z"}
{"level":"info","msg":"Uploading /mnt/ssd0/agent.log to bench-02-etcd2-3-agent.log","time":"2016-03-25T14:54:13Z"}

View File

@ -0,0 +1,488 @@
2016-03-25 14:21:50.367419 I | etcdmain: etcd Version: 2.3.0+git
2016-03-25 14:21:50.367468 I | etcdmain: Git SHA: Not provided (use ./build instead of go build)
2016-03-25 14:21:50.367475 I | etcdmain: Go Version: go1.6
2016-03-25 14:21:50.367481 I | etcdmain: Go OS/Arch: linux/amd64
2016-03-25 14:21:50.367490 I | etcdmain: setting maximum number of CPUs to 8, total number of available CPUs is 8
2016-03-25 14:21:50.367548 I | etcdmain: listening for peers on http://10.240.0.38:2380
2016-03-25 14:21:50.367575 I | etcdmain: listening for client requests on 10.240.0.38:2379
2016-03-25 14:21:50.369611 I | etcdserver: name = etcd-3
2016-03-25 14:21:50.369626 I | etcdserver: data dir = /mnt/ssd0/data.etcd
2016-03-25 14:21:50.369634 I | etcdserver: member dir = /mnt/ssd0/data.etcd/member
2016-03-25 14:21:50.369640 I | etcdserver: heartbeat = 100ms
2016-03-25 14:21:50.369650 I | etcdserver: election = 1000ms
2016-03-25 14:21:50.369656 I | etcdserver: snapshot count = 10000
2016-03-25 14:21:50.369665 I | etcdserver: advertise client URLs = http://10.240.0.38:2379
2016-03-25 14:21:50.369672 I | etcdserver: initial advertise peer URLs = http://10.240.0.38:2380
2016-03-25 14:21:50.369685 I | etcdserver: initial cluster = etcd-1=http://10.240.0.36:2380,etcd-2=http://10.240.0.37:2380,etcd-3=http://10.240.0.38:2380
2016-03-25 14:21:50.377452 I | etcdserver: starting member ef3ac5ee77284022 in cluster b73c1f84aa49a9b0
2016-03-25 14:21:50.377505 I | raft: ef3ac5ee77284022 became follower at term 0
2016-03-25 14:21:50.377519 I | raft: newRaft ef3ac5ee77284022 [peers: [], term: 0, commit: 0, applied: 0, lastindex: 0, lastterm: 0]
2016-03-25 14:21:50.377526 I | raft: ef3ac5ee77284022 became follower at term 1
2016-03-25 14:21:50.416040 I | rafthttp: the connection with 208de484d8b2cb22 became active
2016-03-25 14:21:50.416790 I | etcdserver: starting server... [version: 2.3.0+git, cluster version: to_be_decided]
2016-03-25 14:21:50.417692 E | etcdmain: failed to notify systemd for readiness: No socket
2016-03-25 14:21:50.417706 E | etcdmain: forgot to set Type=notify in systemd service file?
2016-03-25 14:21:50.417713 N | etcdserver: added member 208de484d8b2cb22 [http://10.240.0.37:2380] to cluster b73c1f84aa49a9b0
2016-03-25 14:21:50.417780 N | etcdmain: serving insecure client requests on 10.240.0.38:2379, this is strongly discouraged!
2016-03-25 14:21:50.417831 N | etcdserver: added member b0978d17c833d412 [http://10.240.0.36:2380] to cluster b73c1f84aa49a9b0
2016-03-25 14:21:50.417916 I | raft: ef3ac5ee77284022 [term: 1] received a MsgApp message with higher term from b0978d17c833d412 [term: 3]
2016-03-25 14:21:50.417941 I | raft: ef3ac5ee77284022 became follower at term 3
2016-03-25 14:21:50.417977 I | raft: raft.node: ef3ac5ee77284022 elected leader b0978d17c833d412 at term 3
2016-03-25 14:21:50.418030 N | etcdserver: added local member ef3ac5ee77284022 [http://10.240.0.38:2380] to cluster b73c1f84aa49a9b0
2016-03-25 14:21:50.418181 I | rafthttp: the connection with b0978d17c833d412 became active
2016-03-25 14:21:50.421857 N | etcdserver: set the initial cluster version to 2.2
2016-03-25 14:21:50.431135 I | etcdserver: published {Name:etcd-3 ClientURLs:[http://10.240.0.38:2379]} to cluster b73c1f84aa49a9b0
2016-03-25 14:21:53.809765 N | etcdserver: updated the cluster version from 2.2 to 2.3
2016-03-25 14:22:53.448566 I | etcdserver: start to snapshot (applied: 10001, lastsnap: 0)
2016-03-25 14:22:53.599808 I | etcdserver: saved snapshot at index 10001
2016-03-25 14:22:53.599950 I | etcdserver: compacted raft log at 5001
2016-03-25 14:23:08.050472 I | etcdserver: start to snapshot (applied: 20003, lastsnap: 10001)
2016-03-25 14:23:08.349060 I | etcdserver: saved snapshot at index 20003
2016-03-25 14:23:08.349230 I | etcdserver: compacted raft log at 15003
2016-03-25 14:23:22.425938 I | etcdserver: start to snapshot (applied: 30008, lastsnap: 20003)
2016-03-25 14:23:22.914911 I | etcdserver: saved snapshot at index 30008
2016-03-25 14:23:22.915062 I | etcdserver: compacted raft log at 25008
2016-03-25 14:23:36.921396 I | etcdserver: start to snapshot (applied: 40009, lastsnap: 30008)
2016-03-25 14:23:37.528815 I | etcdserver: saved snapshot at index 40009
2016-03-25 14:23:37.528989 I | etcdserver: compacted raft log at 35009
2016-03-25 14:23:51.715888 I | etcdserver: start to snapshot (applied: 50012, lastsnap: 40009)
2016-03-25 14:23:52.440034 I | etcdserver: saved snapshot at index 50012
2016-03-25 14:23:52.440175 I | etcdserver: compacted raft log at 45012
2016-03-25 14:24:06.641565 I | etcdserver: start to snapshot (applied: 60014, lastsnap: 50012)
2016-03-25 14:24:07.548645 I | etcdserver: saved snapshot at index 60014
2016-03-25 14:24:07.548790 I | etcdserver: compacted raft log at 55014
2016-03-25 14:24:20.419828 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000002711.snap successfully
2016-03-25 14:24:21.469557 I | etcdserver: start to snapshot (applied: 70017, lastsnap: 60014)
2016-03-25 14:24:22.487486 I | etcdserver: saved snapshot at index 70017
2016-03-25 14:24:22.487639 I | etcdserver: compacted raft log at 65017
2016-03-25 14:24:36.738818 I | etcdserver: start to snapshot (applied: 80021, lastsnap: 70017)
2016-03-25 14:24:37.972095 I | etcdserver: saved snapshot at index 80021
2016-03-25 14:24:37.972332 I | etcdserver: compacted raft log at 75021
2016-03-25 14:24:50.422870 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000004e23.snap successfully
2016-03-25 14:24:50.426687 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000007538.snap successfully
2016-03-25 14:24:51.984576 I | etcdserver: start to snapshot (applied: 90024, lastsnap: 80021)
2016-03-25 14:24:53.348165 I | etcdserver: saved snapshot at index 90024
2016-03-25 14:24:53.348361 I | etcdserver: compacted raft log at 85024
2016-03-25 14:25:07.097131 I | etcdserver: start to snapshot (applied: 100028, lastsnap: 90024)
2016-03-25 14:25:08.580006 I | etcdserver: saved snapshot at index 100028
2016-03-25 14:25:08.580207 I | etcdserver: compacted raft log at 95028
2016-03-25 14:25:20.432126 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000009c49.snap successfully
2016-03-25 14:25:20.438537 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000c35c.snap successfully
2016-03-25 14:25:22.484129 I | etcdserver: start to snapshot (applied: 110032, lastsnap: 100028)
2016-03-25 14:25:24.095134 I | etcdserver: saved snapshot at index 110032
2016-03-25 14:25:24.095323 I | etcdserver: compacted raft log at 105032
2016-03-25 14:25:37.857131 I | etcdserver: start to snapshot (applied: 120033, lastsnap: 110032)
2016-03-25 14:25:39.710728 I | etcdserver: saved snapshot at index 120033
2016-03-25 14:25:39.710905 I | etcdserver: compacted raft log at 115033
2016-03-25 14:25:50.446631 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000000ea6e.snap successfully
2016-03-25 14:25:50.455627 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000011181.snap successfully
2016-03-25 14:25:53.329695 I | etcdserver: start to snapshot (applied: 130036, lastsnap: 120033)
2016-03-25 14:25:55.262393 I | etcdserver: saved snapshot at index 130036
2016-03-25 14:25:55.262569 I | etcdserver: compacted raft log at 125036
2016-03-25 14:26:08.856625 I | etcdserver: start to snapshot (applied: 140037, lastsnap: 130036)
2016-03-25 14:26:11.074897 I | etcdserver: saved snapshot at index 140037
2016-03-25 14:26:11.075069 I | etcdserver: compacted raft log at 135037
2016-03-25 14:26:20.466557 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000013895.snap successfully
2016-03-25 14:26:20.478649 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000015fa8.snap successfully
2016-03-25 14:26:24.615582 I | etcdserver: start to snapshot (applied: 150038, lastsnap: 140037)
2016-03-25 14:26:26.868616 I | etcdserver: saved snapshot at index 150038
2016-03-25 14:26:26.868767 I | etcdserver: compacted raft log at 145038
2016-03-25 14:26:38.346182 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000001-0000000000026bbb.wal is created
2016-03-25 14:26:40.445028 I | etcdserver: start to snapshot (applied: 160041, lastsnap: 150038)
2016-03-25 14:26:42.984706 I | etcdserver: saved snapshot at index 160041
2016-03-25 14:26:42.984944 I | etcdserver: compacted raft log at 155041
2016-03-25 14:26:50.491589 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000186bc.snap successfully
2016-03-25 14:26:50.505856 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001add0.snap successfully
2016-03-25 14:26:56.158396 I | etcdserver: start to snapshot (applied: 170048, lastsnap: 160041)
2016-03-25 14:26:58.731304 I | etcdserver: saved snapshot at index 170048
2016-03-25 14:26:58.731490 I | etcdserver: compacted raft log at 165048
2016-03-25 14:27:12.098147 I | etcdserver: start to snapshot (applied: 180049, lastsnap: 170048)
2016-03-25 14:27:14.888393 I | etcdserver: saved snapshot at index 180049
2016-03-25 14:27:14.888575 I | etcdserver: compacted raft log at 175049
2016-03-25 14:27:20.522143 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001d4e1.snap successfully
2016-03-25 14:27:20.539195 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000001fbf4.snap successfully
2016-03-25 14:27:28.129897 I | etcdserver: start to snapshot (applied: 190052, lastsnap: 180049)
2016-03-25 14:27:31.006125 I | etcdserver: saved snapshot at index 190052
2016-03-25 14:27:31.006352 I | etcdserver: compacted raft log at 185052
2016-03-25 14:27:43.746989 I | etcdserver: start to snapshot (applied: 200058, lastsnap: 190052)
2016-03-25 14:27:46.832485 I | etcdserver: saved snapshot at index 200058
2016-03-25 14:27:46.832700 I | etcdserver: compacted raft log at 195058
2016-03-25 14:27:50.557878 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000022305.snap successfully
2016-03-25 14:27:50.577520 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000024a16.snap successfully
2016-03-25 14:27:59.994418 I | etcdserver: start to snapshot (applied: 210060, lastsnap: 200058)
2016-03-25 14:28:03.098872 I | etcdserver: saved snapshot at index 210060
2016-03-25 14:28:03.099066 I | etcdserver: compacted raft log at 205060
2016-03-25 14:28:16.097008 I | etcdserver: start to snapshot (applied: 220061, lastsnap: 210060)
2016-03-25 14:28:19.394665 I | etcdserver: saved snapshot at index 220061
2016-03-25 14:28:19.394871 I | etcdserver: compacted raft log at 215061
2016-03-25 14:28:20.611589 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000027129.snap successfully
2016-03-25 14:28:20.633769 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000029840.snap successfully
2016-03-25 14:28:32.391321 I | etcdserver: start to snapshot (applied: 230064, lastsnap: 220061)
2016-03-25 14:28:36.005001 I | etcdserver: saved snapshot at index 230064
2016-03-25 14:28:36.005268 I | etcdserver: compacted raft log at 225064
2016-03-25 14:28:48.711314 I | etcdserver: start to snapshot (applied: 240066, lastsnap: 230064)
2016-03-25 14:28:50.656712 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002bf51.snap successfully
2016-03-25 14:28:52.451505 I | etcdserver: saved snapshot at index 240066
2016-03-25 14:28:52.451729 I | etcdserver: compacted raft log at 235066
2016-03-25 14:29:05.023813 I | etcdserver: start to snapshot (applied: 250067, lastsnap: 240066)
2016-03-25 14:29:08.941258 I | etcdserver: saved snapshot at index 250067
2016-03-25 14:29:08.941420 I | etcdserver: compacted raft log at 245067
2016-03-25 14:29:20.682104 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000002e664.snap successfully
2016-03-25 14:29:20.708452 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000030d7a.snap successfully
2016-03-25 14:29:21.789732 I | etcdserver: start to snapshot (applied: 260068, lastsnap: 250067)
2016-03-25 14:29:25.891727 I | etcdserver: saved snapshot at index 260068
2016-03-25 14:29:25.891998 I | etcdserver: compacted raft log at 255068
2016-03-25 14:29:38.025554 I | etcdserver: start to snapshot (applied: 270069, lastsnap: 260068)
2016-03-25 14:29:42.349685 I | etcdserver: saved snapshot at index 270069
2016-03-25 14:29:42.350158 I | etcdserver: compacted raft log at 265069
2016-03-25 14:29:50.735638 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003348c.snap successfully
2016-03-25 14:29:50.763797 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000035b9d.snap successfully
2016-03-25 14:29:54.516908 I | etcdserver: start to snapshot (applied: 280070, lastsnap: 270069)
2016-03-25 14:29:58.752081 I | etcdserver: saved snapshot at index 280070
2016-03-25 14:29:58.752247 I | etcdserver: compacted raft log at 275070
2016-03-25 14:30:11.182730 I | etcdserver: start to snapshot (applied: 290073, lastsnap: 280070)
2016-03-25 14:30:16.132546 I | etcdserver: saved snapshot at index 290073
2016-03-25 14:30:16.132749 I | etcdserver: compacted raft log at 285073
2016-03-25 14:30:20.791975 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000382b0.snap successfully
2016-03-25 14:30:20.822355 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003a9c2.snap successfully
2016-03-25 14:30:27.412167 I | etcdserver: start to snapshot (applied: 300074, lastsnap: 290073)
2016-03-25 14:30:32.068498 I | etcdserver: saved snapshot at index 300074
2016-03-25 14:30:32.068692 I | etcdserver: compacted raft log at 295074
2016-03-25 14:30:44.085566 I | etcdserver: start to snapshot (applied: 310077, lastsnap: 300074)
2016-03-25 14:30:49.018469 I | etcdserver: saved snapshot at index 310077
2016-03-25 14:30:49.018692 I | etcdserver: compacted raft log at 305077
2016-03-25 14:30:50.854842 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003d0d3.snap successfully
2016-03-25 14:30:50.888191 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000003f7e4.snap successfully
2016-03-25 14:30:56.867711 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000002-000000000004d722.wal is created
2016-03-25 14:31:01.193437 I | etcdserver: start to snapshot (applied: 320078, lastsnap: 310077)
2016-03-25 14:31:06.030048 I | etcdserver: saved snapshot at index 320078
2016-03-25 14:31:06.030237 I | etcdserver: compacted raft log at 315078
2016-03-25 14:31:17.953558 I | etcdserver: start to snapshot (applied: 330084, lastsnap: 320078)
2016-03-25 14:31:20.923987 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000041ef5.snap successfully
2016-03-25 14:31:23.422213 I | etcdserver: saved snapshot at index 330084
2016-03-25 14:31:23.422472 I | etcdserver: compacted raft log at 325084
2016-03-25 14:31:34.929580 I | etcdserver: start to snapshot (applied: 340086, lastsnap: 330084)
2016-03-25 14:31:40.446704 I | etcdserver: saved snapshot at index 340086
2016-03-25 14:31:40.446916 I | etcdserver: compacted raft log at 335086
2016-03-25 14:31:50.960440 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000044606.snap successfully
2016-03-25 14:31:50.997866 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000046d19.snap successfully
2016-03-25 14:31:51.956598 I | etcdserver: start to snapshot (applied: 350087, lastsnap: 340086)
2016-03-25 14:31:57.313392 I | etcdserver: saved snapshot at index 350087
2016-03-25 14:31:57.313614 I | etcdserver: compacted raft log at 345087
2016-03-25 14:32:11.531395 I | etcdserver: start to snapshot (applied: 360105, lastsnap: 350087)
2016-03-25 14:32:17.076321 I | etcdserver: saved snapshot at index 360105
2016-03-25 14:32:17.076547 I | etcdserver: compacted raft log at 355105
2016-03-25 14:32:21.036608 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004942a.snap successfully
2016-03-25 14:32:21.076800 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004bb3d.snap successfully
2016-03-25 14:32:27.296047 I | etcdserver: start to snapshot (applied: 370108, lastsnap: 360105)
2016-03-25 14:32:33.278243 I | etcdserver: saved snapshot at index 370108
2016-03-25 14:32:33.278451 I | etcdserver: compacted raft log at 365108
2016-03-25 14:32:43.421456 I | etcdserver: start to snapshot (applied: 380110, lastsnap: 370108)
2016-03-25 14:32:49.381602 I | etcdserver: saved snapshot at index 380110
2016-03-25 14:32:49.382174 I | etcdserver: compacted raft log at 375110
2016-03-25 14:32:51.117732 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000004e24e.snap successfully
2016-03-25 14:32:51.159036 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000050964.snap successfully
2016-03-25 14:33:03.042536 I | etcdserver: start to snapshot (applied: 390115, lastsnap: 380110)
2016-03-25 14:33:09.110456 I | etcdserver: saved snapshot at index 390115
2016-03-25 14:33:09.110673 I | etcdserver: compacted raft log at 385115
2016-03-25 14:33:19.159225 I | etcdserver: start to snapshot (applied: 400116, lastsnap: 390115)
2016-03-25 14:33:21.200789 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000053076.snap successfully
2016-03-25 14:33:25.346440 I | etcdserver: saved snapshot at index 400116
2016-03-25 14:33:25.346705 I | etcdserver: compacted raft log at 395116
2016-03-25 14:33:35.204944 I | etcdserver: start to snapshot (applied: 410117, lastsnap: 400116)
2016-03-25 14:33:41.699149 I | etcdserver: saved snapshot at index 410117
2016-03-25 14:33:41.699398 I | etcdserver: compacted raft log at 405117
2016-03-25 14:33:51.244953 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000055787.snap successfully
2016-03-25 14:33:51.290849 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000057ea9.snap successfully
2016-03-25 14:33:55.087119 I | etcdserver: start to snapshot (applied: 420119, lastsnap: 410117)
2016-03-25 14:34:01.450654 I | etcdserver: saved snapshot at index 420119
2016-03-25 14:34:01.450905 I | etcdserver: compacted raft log at 415119
2016-03-25 14:34:11.660940 I | etcdserver: start to snapshot (applied: 430121, lastsnap: 420119)
2016-03-25 14:34:18.551146 I | etcdserver: saved snapshot at index 430121
2016-03-25 14:34:18.551373 I | etcdserver: compacted raft log at 425121
2016-03-25 14:34:21.340558 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005a5bc.snap successfully
2016-03-25 14:34:21.389558 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005ccce.snap successfully
2016-03-25 14:34:28.385108 I | etcdserver: start to snapshot (applied: 440123, lastsnap: 430121)
2016-03-25 14:34:35.845087 I | etcdserver: saved snapshot at index 440123
2016-03-25 14:34:35.845296 I | etcdserver: compacted raft log at 435123
2016-03-25 14:34:46.138101 I | etcdserver: start to snapshot (applied: 450128, lastsnap: 440123)
2016-03-25 14:34:51.466745 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000005f3e3.snap successfully
2016-03-25 14:34:53.180643 I | etcdserver: saved snapshot at index 450128
2016-03-25 14:34:53.180835 I | etcdserver: compacted raft log at 445128
2016-03-25 14:35:04.575450 I | etcdserver: start to snapshot (applied: 460129, lastsnap: 450128)
2016-03-25 14:35:11.907327 I | etcdserver: saved snapshot at index 460129
2016-03-25 14:35:11.907552 I | etcdserver: compacted raft log at 455129
2016-03-25 14:35:21.518258 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000061af4.snap successfully
2016-03-25 14:35:21.570835 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000064205.snap successfully
2016-03-25 14:35:22.713055 I | etcdserver: start to snapshot (applied: 470136, lastsnap: 460129)
2016-03-25 14:35:30.208035 I | etcdserver: saved snapshot at index 470136
2016-03-25 14:35:30.208259 I | etcdserver: compacted raft log at 465136
2016-03-25 14:35:33.740005 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000003-00000000000742b8.wal is created
2016-03-25 14:35:43.390737 I | etcdserver: start to snapshot (applied: 480141, lastsnap: 470136)
2016-03-25 14:35:51.413112 I | etcdserver: saved snapshot at index 480141
2016-03-25 14:35:51.413368 I | etcdserver: compacted raft log at 475141
2016-03-25 14:35:51.625414 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000066917.snap successfully
2016-03-25 14:35:51.681224 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000069029.snap successfully
2016-03-25 14:36:00.111592 I | etcdserver: start to snapshot (applied: 490147, lastsnap: 480141)
2016-03-25 14:36:07.891657 I | etcdserver: saved snapshot at index 490147
2016-03-25 14:36:07.891894 I | etcdserver: compacted raft log at 485147
2016-03-25 14:36:19.398399 I | etcdserver: start to snapshot (applied: 500152, lastsnap: 490147)
2016-03-25 14:36:21.738021 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006b73b.snap successfully
2016-03-25 14:36:28.099654 I | etcdserver: saved snapshot at index 500152
2016-03-25 14:36:28.099915 I | etcdserver: compacted raft log at 495152
2016-03-25 14:36:36.888058 I | etcdserver: start to snapshot (applied: 510154, lastsnap: 500152)
2016-03-25 14:36:45.091889 I | etcdserver: saved snapshot at index 510154
2016-03-25 14:36:45.092163 I | etcdserver: compacted raft log at 505154
2016-03-25 14:36:51.794119 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000006de50.snap successfully
2016-03-25 14:36:51.851075 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000070561.snap successfully
2016-03-25 14:36:55.912576 I | etcdserver: start to snapshot (applied: 520161, lastsnap: 510154)
2016-03-25 14:37:04.716632 I | etcdserver: saved snapshot at index 520161
2016-03-25 14:37:04.716955 I | etcdserver: compacted raft log at 515161
2016-03-25 14:37:12.899695 I | etcdserver: start to snapshot (applied: 530167, lastsnap: 520161)
2016-03-25 14:37:21.576471 I | etcdserver: saved snapshot at index 530167
2016-03-25 14:37:21.576805 I | etcdserver: compacted raft log at 525167
2016-03-25 14:37:21.909856 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000072c78.snap successfully
2016-03-25 14:37:21.969462 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007538d.snap successfully
2016-03-25 14:37:32.158978 I | etcdserver: start to snapshot (applied: 540168, lastsnap: 530167)
2016-03-25 14:37:41.798275 I | etcdserver: saved snapshot at index 540168
2016-03-25 14:37:41.798611 I | etcdserver: compacted raft log at 535168
2016-03-25 14:37:49.038920 I | etcdserver: start to snapshot (applied: 550173, lastsnap: 540168)
2016-03-25 14:37:52.033790 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000077aa3.snap successfully
2016-03-25 14:37:57.740455 I | etcdserver: saved snapshot at index 550173
2016-03-25 14:37:57.740677 I | etcdserver: compacted raft log at 545173
2016-03-25 14:38:08.630648 I | etcdserver: start to snapshot (applied: 560175, lastsnap: 550173)
2016-03-25 14:38:18.057536 I | etcdserver: saved snapshot at index 560175
2016-03-25 14:38:18.057829 I | etcdserver: compacted raft log at 555175
2016-03-25 14:38:22.097671 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007a1b8.snap successfully
2016-03-25 14:38:22.162550 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007c8ca.snap successfully
2016-03-25 14:38:25.665719 I | etcdserver: start to snapshot (applied: 570186, lastsnap: 560175)
2016-03-25 14:38:34.769796 I | etcdserver: saved snapshot at index 570186
2016-03-25 14:38:34.770060 I | etcdserver: compacted raft log at 565186
2016-03-25 14:38:44.882023 I | etcdserver: start to snapshot (applied: 580188, lastsnap: 570186)
2016-03-25 14:38:52.231654 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000007efe1.snap successfully
2016-03-25 14:38:54.270790 I | etcdserver: saved snapshot at index 580188
2016-03-25 14:38:54.271069 I | etcdserver: compacted raft log at 575188
2016-03-25 14:39:02.054695 I | etcdserver: start to snapshot (applied: 590191, lastsnap: 580188)
2016-03-25 14:39:11.534307 I | etcdserver: saved snapshot at index 590191
2016-03-25 14:39:11.534566 I | etcdserver: compacted raft log at 585191
2016-03-25 14:39:21.298115 I | etcdserver: start to snapshot (applied: 600192, lastsnap: 590191)
2016-03-25 14:39:22.297600 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000816f7.snap successfully
2016-03-25 14:39:22.366586 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000083e08.snap successfully
2016-03-25 14:39:31.109491 I | etcdserver: saved snapshot at index 600192
2016-03-25 14:39:31.110236 I | etcdserver: compacted raft log at 595192
2016-03-25 14:39:38.838849 I | etcdserver: start to snapshot (applied: 610195, lastsnap: 600192)
2016-03-25 14:39:48.440026 I | etcdserver: saved snapshot at index 610195
2016-03-25 14:39:48.440296 I | etcdserver: compacted raft log at 605195
2016-03-25 14:39:52.437777 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008651d.snap successfully
2016-03-25 14:39:52.508890 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000088c2f.snap successfully
2016-03-25 14:39:58.568518 I | etcdserver: start to snapshot (applied: 620199, lastsnap: 610195)
2016-03-25 14:40:08.907156 I | etcdserver: saved snapshot at index 620199
2016-03-25 14:40:08.907470 I | etcdserver: compacted raft log at 615199
2016-03-25 14:40:15.864422 I | etcdserver: start to snapshot (applied: 630200, lastsnap: 620199)
2016-03-25 14:40:22.581291 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008b34a.snap successfully
2016-03-25 14:40:22.950326 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000004-000000000009aeed.wal is created
2016-03-25 14:40:25.445398 I | etcdserver: saved snapshot at index 630200
2016-03-25 14:40:25.445761 I | etcdserver: compacted raft log at 625200
2016-03-25 14:40:35.540629 I | etcdserver: start to snapshot (applied: 640202, lastsnap: 630200)
2016-03-25 14:40:46.437486 I | etcdserver: saved snapshot at index 640202
2016-03-25 14:40:46.437740 I | etcdserver: compacted raft log at 635202
2016-03-25 14:40:52.652407 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000008da5c.snap successfully
2016-03-25 14:40:52.727243 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009016f.snap successfully
2016-03-25 14:40:54.083834 I | etcdserver: start to snapshot (applied: 650203, lastsnap: 640202)
2016-03-25 14:41:05.028421 I | etcdserver: saved snapshot at index 650203
2016-03-25 14:41:05.028771 I | etcdserver: compacted raft log at 645203
2016-03-25 14:41:13.148443 I | etcdserver: start to snapshot (applied: 660205, lastsnap: 650203)
2016-03-25 14:41:22.804718 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000092880.snap successfully
2016-03-25 14:41:23.783332 I | etcdserver: saved snapshot at index 660205
2016-03-25 14:41:23.783647 I | etcdserver: compacted raft log at 655205
2016-03-25 14:41:31.870056 I | etcdserver: start to snapshot (applied: 670209, lastsnap: 660205)
2016-03-25 14:41:42.593468 I | etcdserver: saved snapshot at index 670209
2016-03-25 14:41:42.593760 I | etcdserver: compacted raft log at 665209
2016-03-25 14:41:50.278373 I | etcdserver: start to snapshot (applied: 680212, lastsnap: 670209)
2016-03-25 14:41:52.883951 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000094f93.snap successfully
2016-03-25 14:41:52.962251 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000976a7.snap successfully
2016-03-25 14:42:01.351470 I | etcdserver: saved snapshot at index 680212
2016-03-25 14:42:01.351794 I | etcdserver: compacted raft log at 675212
2016-03-25 14:42:08.988019 I | etcdserver: start to snapshot (applied: 690218, lastsnap: 680212)
2016-03-25 14:42:20.157491 I | etcdserver: saved snapshot at index 690218
2016-03-25 14:42:20.157790 I | etcdserver: compacted raft log at 685218
2016-03-25 14:42:23.044086 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-0000000000099db8.snap successfully
2016-03-25 14:42:23.126165 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009c4ca.snap successfully
2016-03-25 14:42:27.779216 I | etcdserver: start to snapshot (applied: 700220, lastsnap: 690218)
2016-03-25 14:42:38.958521 I | etcdserver: saved snapshot at index 700220
2016-03-25 14:42:38.958844 I | etcdserver: compacted raft log at 695220
2016-03-25 14:42:46.333774 I | etcdserver: start to snapshot (applied: 710222, lastsnap: 700220)
2016-03-25 14:42:53.213329 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-000000000009ebdb.snap successfully
2016-03-25 14:42:58.023001 I | etcdserver: saved snapshot at index 710222
2016-03-25 14:42:58.023311 I | etcdserver: compacted raft log at 705222
2016-03-25 14:43:05.672696 I | etcdserver: start to snapshot (applied: 720223, lastsnap: 710222)
2016-03-25 14:43:17.167128 I | etcdserver: saved snapshot at index 720223
2016-03-25 14:43:17.167495 I | etcdserver: compacted raft log at 715223
2016-03-25 14:43:23.292067 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a12ed.snap successfully
2016-03-25 14:43:23.382411 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a3a01.snap successfully
2016-03-25 14:43:25.096661 I | etcdserver: start to snapshot (applied: 730224, lastsnap: 720223)
2016-03-25 14:43:37.113047 I | etcdserver: saved snapshot at index 730224
2016-03-25 14:43:37.113393 I | etcdserver: compacted raft log at 725224
2016-03-25 14:43:44.207764 I | etcdserver: start to snapshot (applied: 740228, lastsnap: 730224)
2016-03-25 14:43:53.469612 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a6114.snap successfully
2016-03-25 14:43:56.038877 I | etcdserver: saved snapshot at index 740228
2016-03-25 14:43:56.039203 I | etcdserver: compacted raft log at 735228
2016-03-25 14:44:03.373552 I | etcdserver: start to snapshot (applied: 750229, lastsnap: 740228)
2016-03-25 14:44:15.118649 I | etcdserver: saved snapshot at index 750229
2016-03-25 14:44:15.119005 I | etcdserver: compacted raft log at 745229
2016-03-25 14:44:20.650991 I | etcdserver: start to snapshot (applied: 760230, lastsnap: 750229)
2016-03-25 14:44:23.525462 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000a882a.snap successfully
2016-03-25 14:44:23.611509 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000aaf3c.snap successfully
2016-03-25 14:44:32.867769 I | etcdserver: saved snapshot at index 760230
2016-03-25 14:44:32.868067 I | etcdserver: compacted raft log at 755230
2016-03-25 14:44:40.631701 I | etcdserver: start to snapshot (applied: 770233, lastsnap: 760230)
2016-03-25 14:44:53.292009 I | etcdserver: saved snapshot at index 770233
2016-03-25 14:44:53.292422 I | etcdserver: compacted raft log at 765233
2016-03-25 14:44:53.649808 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000ad64e.snap successfully
2016-03-25 14:44:53.735724 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000afd5f.snap successfully
2016-03-25 14:45:00.423594 I | etcdserver: start to snapshot (applied: 780235, lastsnap: 770233)
2016-03-25 14:45:13.222479 I | etcdserver: saved snapshot at index 780235
2016-03-25 14:45:13.222837 I | etcdserver: compacted raft log at 775235
2016-03-25 14:45:17.792234 I | etcdserver: start to snapshot (applied: 790241, lastsnap: 780235)
2016-03-25 14:45:23.835513 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b2470.snap successfully
2016-03-25 14:45:24.532450 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000005-00000000000c1a98.wal is created
2016-03-25 14:45:30.647895 I | etcdserver: saved snapshot at index 790241
2016-03-25 14:45:30.648189 I | etcdserver: compacted raft log at 785241
2016-03-25 14:45:38.869400 I | etcdserver: start to snapshot (applied: 800242, lastsnap: 790241)
2016-03-25 14:45:50.434649 I | fileutil: purged file /mnt/ssd0/data.etcd/member/wal/0000000000000000-0000000000000000.wal successfully
2016-03-25 14:45:52.016955 I | etcdserver: saved snapshot at index 800242
2016-03-25 14:45:52.017444 I | etcdserver: compacted raft log at 795242
2016-03-25 14:45:53.850978 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b4b84.snap successfully
2016-03-25 14:45:53.870405 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b7295.snap successfully
2016-03-25 14:45:56.499971 I | etcdserver: start to snapshot (applied: 810243, lastsnap: 800242)
2016-03-25 14:46:09.919867 I | etcdserver: saved snapshot at index 810243
2016-03-25 14:46:09.920215 I | etcdserver: compacted raft log at 805243
2016-03-25 14:46:17.449733 I | etcdserver: start to snapshot (applied: 820248, lastsnap: 810243)
2016-03-25 14:46:23.881460 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000b99a6.snap successfully
2016-03-25 14:46:30.875637 I | etcdserver: saved snapshot at index 820248
2016-03-25 14:46:30.876008 I | etcdserver: compacted raft log at 815248
2016-03-25 14:46:34.932234 I | etcdserver: start to snapshot (applied: 830249, lastsnap: 820248)
2016-03-25 14:46:48.296952 I | etcdserver: saved snapshot at index 830249
2016-03-25 14:46:48.297309 I | etcdserver: compacted raft log at 825249
2016-03-25 14:46:53.891415 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000bc0b9.snap successfully
2016-03-25 14:46:53.902230 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000be7cb.snap successfully
2016-03-25 14:46:56.603969 I | etcdserver: start to snapshot (applied: 840251, lastsnap: 830249)
2016-03-25 14:47:09.865629 I | etcdserver: saved snapshot at index 840251
2016-03-25 14:47:09.866010 I | etcdserver: compacted raft log at 835251
2016-03-25 14:47:14.274850 I | etcdserver: start to snapshot (applied: 850257, lastsnap: 840251)
2016-03-25 14:47:23.940880 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c0ee1.snap successfully
2016-03-25 14:47:28.249930 I | etcdserver: saved snapshot at index 850257
2016-03-25 14:47:28.250257 I | etcdserver: compacted raft log at 845257
2016-03-25 14:47:36.243331 I | etcdserver: start to snapshot (applied: 860259, lastsnap: 850257)
2016-03-25 14:47:50.847238 I | etcdserver: saved snapshot at index 860259
2016-03-25 14:47:50.847574 I | etcdserver: compacted raft log at 855259
2016-03-25 14:47:53.950930 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c35f2.snap successfully
2016-03-25 14:47:53.972684 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c5d03.snap successfully
2016-03-25 14:47:55.048026 I | etcdserver: start to snapshot (applied: 870260, lastsnap: 860259)
2016-03-25 14:48:10.053955 I | etcdserver: saved snapshot at index 870260
2016-03-25 14:48:10.054246 I | etcdserver: compacted raft log at 865260
2016-03-25 14:48:15.729045 I | etcdserver: start to snapshot (applied: 880261, lastsnap: 870260)
2016-03-25 14:48:23.982370 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000c8418.snap successfully
2016-03-25 14:48:30.002872 I | etcdserver: saved snapshot at index 880261
2016-03-25 14:48:30.003237 I | etcdserver: compacted raft log at 875261
2016-03-25 14:48:35.811083 I | etcdserver: start to snapshot (applied: 890266, lastsnap: 880261)
2016-03-25 14:48:50.405500 I | etcdserver: saved snapshot at index 890266
2016-03-25 14:48:50.405837 I | etcdserver: compacted raft log at 885266
2016-03-25 14:48:53.993386 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cab29.snap successfully
2016-03-25 14:48:54.003726 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cd23b.snap successfully
2016-03-25 14:48:59.427864 I | etcdserver: start to snapshot (applied: 900267, lastsnap: 890266)
2016-03-25 14:49:15.548553 I | etcdserver: saved snapshot at index 900267
2016-03-25 14:49:15.548908 I | etcdserver: compacted raft log at 895267
2016-03-25 14:49:18.528015 I | etcdserver: start to snapshot (applied: 910269, lastsnap: 900267)
2016-03-25 14:49:24.014162 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000cf951.snap successfully
2016-03-25 14:49:33.177969 I | raft: ef3ac5ee77284022 is starting a new election at term 3
2016-03-25 14:49:33.178023 I | raft: ef3ac5ee77284022 became candidate at term 4
2016-03-25 14:49:33.178032 I | raft: ef3ac5ee77284022 received vote from ef3ac5ee77284022 at term 4
2016-03-25 14:49:33.178044 I | raft: ef3ac5ee77284022 [logterm: 3, index: 917652] sent vote request to 208de484d8b2cb22 at term 4
2016-03-25 14:49:33.178053 I | raft: ef3ac5ee77284022 [logterm: 3, index: 917652] sent vote request to b0978d17c833d412 at term 4
2016-03-25 14:49:33.178063 I | raft: raft.node: ef3ac5ee77284022 lost leader b0978d17c833d412 at term 4
2016-03-25 14:49:33.309425 I | raft: ef3ac5ee77284022 received vote from 208de484d8b2cb22 at term 4
2016-03-25 14:49:33.309467 I | raft: ef3ac5ee77284022 [quorum:2] has received 2 votes and 0 vote rejections
2016-03-25 14:49:33.309499 I | raft: ef3ac5ee77284022 became leader at term 4
2016-03-25 14:49:33.309517 I | raft: raft.node: ef3ac5ee77284022 elected leader ef3ac5ee77284022 at term 4
2016-03-25 14:49:33.649082 I | etcdserver: saved snapshot at index 910269
2016-03-25 14:49:33.649415 I | etcdserver: compacted raft log at 905269
2016-03-25 14:49:33.855916 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.855937 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.855970 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.855979 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.855994 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.856002 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.856016 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.856033 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.856046 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.856055 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgApp message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:33.969130 I | raft: ef3ac5ee77284022 [term: 4] ignored a MsgHeartbeat message with lower term from b0978d17c833d412 [term: 3]
2016-03-25 14:49:39.040352 I | etcdserver: start to snapshot (applied: 920273, lastsnap: 910269)
2016-03-25 14:49:40.592019 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 2.662785ms)
2016-03-25 14:49:40.592061 W | etcdserver: server is likely overloaded
2016-03-25 14:49:40.592070 W | etcdserver: failed to send out heartbeat on time (deadline exceeded for 2.730308ms)
2016-03-25 14:49:40.592075 W | etcdserver: server is likely overloaded
2016-03-25 14:49:54.044478 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d2063.snap successfully
2016-03-25 14:49:54.750768 I | raft: ef3ac5ee77284022 [term: 4] received a MsgApp message with higher term from 208de484d8b2cb22 [term: 5]
2016-03-25 14:49:54.750838 I | raft: ef3ac5ee77284022 became follower at term 5
2016-03-25 14:49:54.750853 I | raft: found conflict at index 927550 [existing term: 4, conflicting term: 5]
2016-03-25 14:49:54.750860 I | raft: replace the unstable entries from index 927550
2016-03-25 14:49:54.750871 I | raft: raft.node: ef3ac5ee77284022 changed leader from ef3ac5ee77284022 to 208de484d8b2cb22 at term 5
2016-03-25 14:49:54.750908 I | raft: ef3ac5ee77284022 [logterm: 5, index: 927551, vote: 0] rejected vote from 208de484d8b2cb22 [logterm: 4, index: 927549] at term 5
2016-03-25 14:49:56.693233 I | etcdserver: saved snapshot at index 920273
2016-03-25 14:49:56.693583 I | etcdserver: compacted raft log at 915273
2016-03-25 14:50:01.176278 I | etcdserver: start to snapshot (applied: 930274, lastsnap: 920273)
2016-03-25 14:50:01.451682 E | etcdhttp: etcdserver: request timed out, possibly due to previous leader failure
2016-03-25 14:50:03.367236 E | etcdhttp: etcdserver: request timed out, possibly due to previous leader failure [merged 5 repeated lines in 1.92s]
2016-03-25 14:50:17.000815 I | etcdserver: saved snapshot at index 930274
2016-03-25 14:50:17.001323 I | etcdserver: compacted raft log at 925274
2016-03-25 14:50:21.663985 I | etcdserver: start to snapshot (applied: 940284, lastsnap: 930274)
2016-03-25 14:50:24.055919 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d4774.snap successfully
2016-03-25 14:50:24.067752 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d6e85.snap successfully
2016-03-25 14:50:37.746879 I | etcdserver: saved snapshot at index 940284
2016-03-25 14:50:37.747272 I | etcdserver: compacted raft log at 935284
2016-03-25 14:50:44.799738 I | etcdserver: start to snapshot (applied: 950294, lastsnap: 940284)
2016-03-25 14:50:48.286842 I | wal: segmented wal file /mnt/ssd0/data.etcd/member/wal/0000000000000006-00000000000e85c9.wal is created
2016-03-25 14:50:50.443885 I | fileutil: purged file /mnt/ssd0/data.etcd/member/wal/0000000000000001-0000000000026bbb.wal successfully
2016-03-25 14:50:54.079408 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000d959a.snap successfully
2016-03-25 14:51:01.217459 I | etcdserver: saved snapshot at index 950294
2016-03-25 14:51:01.217821 I | etcdserver: compacted raft log at 945294
2016-03-25 14:51:05.872553 I | etcdserver: start to snapshot (applied: 960296, lastsnap: 950294)
2016-03-25 14:51:22.681744 I | etcdserver: saved snapshot at index 960296
2016-03-25 14:51:22.682152 I | etcdserver: compacted raft log at 955296
2016-03-25 14:51:24.091468 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000dbcab.snap successfully
2016-03-25 14:51:24.103331 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000003-00000000000de3bd.snap successfully
2016-03-25 14:51:27.360280 I | etcdserver: start to snapshot (applied: 970297, lastsnap: 960296)
2016-03-25 14:51:44.480494 I | etcdserver: saved snapshot at index 970297
2016-03-25 14:51:44.480941 I | etcdserver: compacted raft log at 965297
2016-03-25 14:51:49.172920 I | etcdserver: start to snapshot (applied: 980298, lastsnap: 970297)
2016-03-25 14:51:54.115537 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000004-00000000000e0ad1.snap successfully
2016-03-25 14:52:05.712700 I | etcdserver: saved snapshot at index 980298
2016-03-25 14:52:05.713035 I | etcdserver: compacted raft log at 975298
2016-03-25 14:52:11.117554 I | etcdserver: start to snapshot (applied: 990300, lastsnap: 980298)
2016-03-25 14:52:24.127715 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000005-00000000000e31e2.snap successfully
2016-03-25 14:52:28.581862 I | etcdserver: saved snapshot at index 990300
2016-03-25 14:52:28.582248 I | etcdserver: compacted raft log at 985300
2016-03-25 14:52:34.467827 I | etcdserver: start to snapshot (applied: 1000301, lastsnap: 990300)
2016-03-25 14:52:50.576156 I | raft: ef3ac5ee77284022 [term: 5] received a MsgVote message with higher term from b0978d17c833d412 [term: 6]
2016-03-25 14:52:50.576227 I | raft: ef3ac5ee77284022 became follower at term 6
2016-03-25 14:52:50.576242 I | raft: ef3ac5ee77284022 [logterm: 5, index: 1003707, vote: 0] voted for b0978d17c833d412 [logterm: 5, index: 1003707] at term 6
2016-03-25 14:52:50.576253 I | raft: raft.node: ef3ac5ee77284022 lost leader 208de484d8b2cb22 at term 6
2016-03-25 14:52:50.581454 I | raft: raft.node: ef3ac5ee77284022 elected leader b0978d17c833d412 at term 6
2016-03-25 14:52:51.088178 I | raft: ef3ac5ee77284022 [term: 6] ignored a MsgHeartbeat message with lower term from 208de484d8b2cb22 [term: 5]
2016-03-25 14:52:54.495227 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000005-00000000000e58fc.snap successfully
2016-03-25 14:52:54.762402 I | etcdserver: saved snapshot at index 1000301
2016-03-25 14:52:54.762638 I | etcdserver: compacted raft log at 995301
2016-03-25 14:52:54.856589 I | fileutil: purged file /mnt/ssd0/data.etcd/member/snap/0000000000000005-00000000000e8016.snap successfully
2016-03-25 14:54:10.268183 E | rafthttp: failed to dial b0978d17c833d412 on stream Message (read tcp 10.240.0.38:33850->10.240.0.36:2380: read: connection reset by peer)
2016-03-25 14:54:10.268247 I | rafthttp: the connection with b0978d17c833d412 became inactive
2016-03-25 14:54:11.131490 E | rafthttp: failed to dial 208de484d8b2cb22 on stream MsgApp v2 (read tcp 10.240.0.38:60412->10.240.0.37:2380: read: connection reset by peer)
2016-03-25 14:54:11.131533 I | rafthttp: the connection with 208de484d8b2cb22 became inactive
2016-03-25 14:54:11.377880 I | raft: ef3ac5ee77284022 is starting a new election at term 6
2016-03-25 14:54:11.377922 I | raft: ef3ac5ee77284022 became candidate at term 7
2016-03-25 14:54:11.377932 I | raft: ef3ac5ee77284022 received vote from ef3ac5ee77284022 at term 7
2016-03-25 14:54:11.377944 I | raft: ef3ac5ee77284022 [logterm: 6, index: 1003869] sent vote request to 208de484d8b2cb22 at term 7
2016-03-25 14:54:11.377961 I | raft: ef3ac5ee77284022 [logterm: 6, index: 1003869] sent vote request to b0978d17c833d412 at term 7
2016-03-25 14:54:11.377970 I | raft: raft.node: ef3ac5ee77284022 lost leader b0978d17c833d412 at term 7
2016-03-25 14:54:12.014118 N | osutil: received terminated signal, shutting down...

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

File diff suppressed because it is too large Load Diff

Binary file not shown.

After

Width:  |  Height:  |  Size: 47 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 118 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 243 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 120 KiB

Binary file not shown.

After

Width:  |  Height:  |  Size: 40 KiB

File diff suppressed because one or more lines are too long

After

Width:  |  Height:  |  Size: 119 KiB

Some files were not shown because too many files have changed in this diff Show More