diff --git a/README.md b/README.md index e64853d2..bf4ce99e 100644 --- a/README.md +++ b/README.md @@ -10,15 +10,15 @@ Run [`install.sh`](install.sh): go get github.com/coreos/dbtester # For each machine -dbtester agent --working-directory $WORKING_DIR --monitor -dbtester agent --working-directory $WORKING_DIR --monitor -dbtester agent --working-directory $WORKING_DIR --monitor -dbtester agent --working-directory $WORKING_DIR --monitor -dbtester agent --working-directory $WORKING_DIR --monitor +dbtester agent +dbtester agent +dbtester agent +dbtester agent +dbtester agent # Client machine -dbtester start --database="etcd" --agent-endpoints="$(echo $ETCD_RPC_ENDPOINTS)" -dbtester start --database="zk" --zk-max-client-conns=5000 --agent-endpoints="$(echo $ZK_RPC_ENDPOINTS)" +dbtester start --agent-endpoints="$(echo $ETCD_RPC_ENDPOINTS)" --database="etcd" +dbtester start --agent-endpoints="$(echo $ZK_RPC_ENDPOINTS)" --database="zk" --zk-max-client-conns=5000 ``` [cistat]: https://travis-ci.org/coreos/dbtester diff --git a/agent/agent.go b/agent/agent.go index 0964d78a..c3136639 100644 --- a/agent/agent.go +++ b/agent/agent.go @@ -123,8 +123,8 @@ func CommandFunc(cmd *cobra.Command, args []string) { } defer f.Close() log.SetOutput(f) - log.Printf("gRPC serving: %s\n", globalFlags.GRPCPort) + log.Printf("gRPC serving: %s", globalFlags.GRPCPort) var ( grpcServer = grpc.NewServer() sender = NewTransporterServer() @@ -187,12 +187,12 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response r.MonitorResultPath = filepath.Join(r.WorkingDirectory, r.MonitorResultPath) } } - log.Printf("Working directory: %s\n", r.WorkingDirectory) - log.Printf("etcd data directory: %s\n", etcdDataDir) - log.Printf("Zookeeper working directory: %s\n", zkWorkingDir) - log.Printf("Zookeeper data directory: %s\n", zkDataDir) - log.Printf("Database log path: %s\n", r.DatabaseLogPath) - log.Printf("Monitor result path: %s\n", r.MonitorResultPath) + log.Printf("Working directory: %s", r.WorkingDirectory) + log.Printf("etcd data directory: %s", etcdDataDir) + log.Printf("Zookeeper working directory: %s", zkWorkingDir) + log.Printf("Zookeeper data directory: %s", zkDataDir) + log.Printf("Database log path: %s", r.DatabaseLogPath) + log.Printf("Monitor result path: %s", r.MonitorResultPath) } t.req = *r @@ -253,7 +253,7 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response cmd := exec.Command(etcdBinaryPath, flags...) cmd.Stdout = f cmd.Stderr = f - log.Printf("Starting: %s %s\n", cmd.Path, flagString) + log.Printf("Starting: %s %s", cmd.Path, flagString) if err := cmd.Start(); err != nil { return nil, err } @@ -263,10 +263,10 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response processPID = t.pid go func() { if err := cmd.Wait(); err != nil { - log.Printf("Start(%s) cmd.Wait returned %v\n", cmd.Path, err) + log.Printf("Start(%s) cmd.Wait returned %v", cmd.Path, err) return } - log.Printf("Exiting %s\n", cmd.Path) + log.Printf("Exiting %s", cmd.Path) }() case Request_ZooKeeper: @@ -275,18 +275,18 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response return nil, err } - log.Printf("os.Chdir: %s\n", zkWorkingDir) + log.Printf("os.Chdir: %s", zkWorkingDir) if err := os.Chdir(zkWorkingDir); err != nil { return nil, err } - log.Printf("os.MkdirAll: %s\n", zkDataDir) + log.Printf("os.MkdirAll: %s", zkDataDir) if err := os.MkdirAll(zkDataDir, 0777); err != nil { return nil, err } idFilePath := filepath.Join(zkDataDir, "myid") - log.Printf("Writing %d to %s\n", r.ZookeeperMyID, idFilePath) + log.Printf("Writing %d to %s", r.ZookeeperMyID, idFilePath) if err := toFile(fmt.Sprintf("%d", r.ZookeeperMyID), idFilePath); err != nil { return nil, err } @@ -309,7 +309,7 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response zc := buf.String() configFilePath := filepath.Join(zkWorkingDir, "zookeeper.config") - log.Printf("Writing %q to %s\n", zc, configFilePath) + log.Printf("Writing %q to %s", zc, configFilePath) if err := toFile(zc, configFilePath); err != nil { return nil, err } @@ -327,7 +327,7 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response cmd := exec.Command(args[0], args[1:]...) cmd.Stdout = f cmd.Stderr = f - log.Printf("Starting: %s %s\n", cmd.Path, strings.Join(args[1:], " ")) + log.Printf("Starting: %s %s", cmd.Path, strings.Join(args[1:], " ")) if err := cmd.Start(); err != nil { return nil, err } @@ -337,10 +337,10 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response processPID = t.pid go func() { if err := cmd.Wait(); err != nil { - log.Printf("Start(%s) cmd.Wait returned %v\n", cmd.Path, err) + log.Printf("Start(%s) cmd.Wait returned %v", cmd.Path, err) return } - log.Printf("Exiting %s\n", cmd.Path) + log.Printf("Exiting %s", cmd.Path) }() default: @@ -352,7 +352,7 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response return nil, fmt.Errorf("nil command") } - log.Printf("Restarting %s\n", t.req.Database) + log.Printf("Restarting %s", t.req.Database) if r.Database == Request_ZooKeeper { if err := os.Chdir(zkWorkingDir); err != nil { return nil, err @@ -368,7 +368,7 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response cmd := exec.Command(t.cmd.Path, t.cmd.Args[1:]...) cmd.Stdout = f cmd.Stderr = f - log.Printf("Restarting: %s\n", strings.Join(t.cmd.Args, " ")) + log.Printf("Restarting: %s", strings.Join(t.cmd.Args, " ")) if err := cmd.Start(); err != nil { return nil, err } @@ -378,22 +378,22 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response processPID = t.pid go func() { if err := cmd.Wait(); err != nil { - log.Printf("Restart(%s) cmd.Wait returned %v\n", cmd.Path, err) + log.Printf("Restart(%s) cmd.Wait returned %v", cmd.Path, err) return } - log.Printf("Exiting %s\n", cmd.Path) + log.Printf("Exiting %s", cmd.Path) }() case Request_Stop: if t.cmd == nil { return nil, fmt.Errorf("nil command") } - log.Printf("Stopping %s [PID: %d]\n", t.req.Database, t.pid) + log.Printf("Stopping %s [PID: %d]", t.req.Database, t.pid) ps.Kill(os.Stdout, false, ps.Process{Stat: ps.Stat{Pid: int64(t.pid)}}) if t.logfile != nil { t.logfile.Close() } - log.Printf("Stopped: %s [PID: %d\n]", t.cmd.Path, t.pid) + log.Printf("Stopped: %s [PID: %d]", t.cmd.Path, t.pid) processPID = t.pid databaseStopped <- struct{}{} @@ -421,7 +421,7 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response return ps.WriteToCSV(f, pss...) } - log.Printf("%s monitor saved at %s\n", r.Database, r.MonitorResultPath) + log.Printf("%s monitor saved at %s", r.Database, r.MonitorResultPath) var err error if err = rFunc(); err != nil { log.Warningln("error:", err) @@ -433,11 +433,11 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response select { case <-time.After(time.Second): if err = rFunc(); err != nil { - log.Warnf("Monitoring error %v\n", err) + log.Warnf("Monitoring error %v", err) break escape } case sig := <-notifier: - log.Printf("Received %v\n", sig) + log.Printf("Received %v", sig) return case <-databaseStopped: log.Println("Monitoring stopped. Uploading data to cloud storage...") @@ -448,26 +448,26 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response storage.ScopeFullControl, ) if err != nil { - log.Warnf("error (%v) with\n\n%q\n\n", err, r.GoogleCloudStorageJSONKey) + log.Warnf("error (%v) with %q", err, r.GoogleCloudStorageJSONKey) return } ctx := context.Background() aclient, err := storage.NewAdminClient(ctx, r.GoogleCloudProjectName, cloud.WithTokenSource(conf.TokenSource(ctx))) if err != nil { - log.Warnf("error (%v) with %q\n", err, r.GoogleCloudProjectName) + log.Warnf("error (%v) with %q", err, r.GoogleCloudProjectName) } defer aclient.Close() if err := aclient.CreateBucket(context.Background(), r.GoogleCloudStorageBucketName, nil); err != nil { if !strings.Contains(err.Error(), "You already own this bucket. Please select another name") { - log.Warnf("error (%v) with %q\n", err, r.GoogleCloudStorageBucketName) + log.Warnf("error (%v) with %q", err, r.GoogleCloudStorageBucketName) } } sctx := context.Background() sclient, err := storage.NewClient(sctx, cloud.WithTokenSource(conf.TokenSource(sctx))) if err != nil { - log.Warnf("error (%v)\n", err) + log.Warnf("error (%v)", err) return } defer sclient.Close() @@ -491,54 +491,54 @@ func (t *transporterServer) Transfer(ctx context.Context, r *Request) (*Response dstAgentLogPath = r.LogPrefix + fmt.Sprintf("_%d_", r.EtcdServerIndex) + filepath.Base(agentLogPath) } - log.Printf("Uploading %s\n", srcDatabaseLogPath) + log.Printf("Uploading %s", srcDatabaseLogPath) wc1 := sclient.Bucket(r.GoogleCloudStorageBucketName).Object(dstDatabaseLogPath).NewWriter(context.Background()) wc1.ContentType = "text/plain" bts1, err := ioutil.ReadFile(srcDatabaseLogPath) if err != nil { - log.Warnf("error (%v)\n", err) + log.Warnf("error (%v)", err) return } if _, err := wc1.Write(bts1); err != nil { - log.Warnf("error (%v)\n", err) + log.Warnf("error (%v)", err) return } if err := wc1.Close(); err != nil { - log.Warnf("error (%v)\n", err) + log.Warnf("error (%v)", err) return } - log.Printf("Uploading %s\n", srcMonitorResultPath) + log.Printf("Uploading %s", srcMonitorResultPath) wc2 := sclient.Bucket(r.GoogleCloudStorageBucketName).Object(dstMonitorResultPath).NewWriter(context.Background()) wc2.ContentType = "text/plain" bts2, err := ioutil.ReadFile(srcMonitorResultPath) if err != nil { - log.Warnf("error (%v)\n", err) + log.Warnf("error (%v)", err) return } if _, err := wc2.Write(bts2); err != nil { - log.Warnf("error (%v)\n", err) + log.Warnf("error (%v)", err) return } if err := wc2.Close(); err != nil { - log.Warnf("error (%v)\n", err) + log.Warnf("error (%v)", err) return } - log.Printf("Uploading %s\n", srcAgentLogPath) + log.Printf("Uploading %s", srcAgentLogPath) wc3 := sclient.Bucket(r.GoogleCloudStorageBucketName).Object(dstAgentLogPath).NewWriter(context.Background()) wc3.ContentType = "text/plain" bts3, err := ioutil.ReadFile(srcAgentLogPath) if err != nil { - log.Warnf("error (%v)\n", err) + log.Warnf("error (%v)", err) return } if _, err := wc3.Write(bts3); err != nil { - log.Warnf("error (%v)\n", err) + log.Warnf("error (%v)", err) return } if err := wc3.Close(); err != nil { - log.Warnf("error (%v)\n", err) + log.Warnf("error (%v)", err) return } return diff --git a/install.sh b/install.sh index 4e0bbca6..beb25228 100644 --- a/install.sh +++ b/install.sh @@ -1,23 +1,88 @@ #!/usr/bin/env bash set -e +< /proc/sys/vm/drop_caches" | sudo sh + +if [ "$1" = "f" ]; then + echo "Formatting following https://cloud.google.com/compute/docs/disks/local-ssd" + sudo mkdir -p /mnt/ssd0 + ls /dev/disk/by-id/ + sudo mkfs.ext4 -F /dev/disk/by-id/google-local-ssd-0 + sudo mount -o discard,defaults /dev/disk/by-id/google-local-ssd-0 /mnt/ssd0 + sudo chmod a+w /mnt/ssd0 + echo '/dev/disk/by-id/google-local-ssd-0 /mnt/ssd0 ext4 defaults 1 1' | sudo tee -a /etc/fstab + echo 'Hello, World!' > /mnt/ssd0/hello.txt + cat /mnt/ssd0/hello.txt + df -h + rm -f /mnt/ssd0/hello.txt + ls /mnt/ssd0/ +else + echo "No formatting" +fi; + WORKING_DIR=$HOME -if [ -n "$1" ]; then - WORKING_DIR=$1 +if [ -n "$2" ]; then + WORKING_DIR=$2 fi; echo Setting working directory to $WORKING_DIR cd $WORKING_DIR +sleep 3s +############################# +echo Installing Go programs # +############################# +GO_VERSION="1.6" && cd /usr/local && sudo rm -rf ./go && sudo curl -s https://storage.googleapis.com/golang/go$GO_VERSION.linux-amd64.tar.gz | sudo tar -v -C /usr/local/ -xz && cd $HOME; +echo "export GOPATH=$(echo $HOME)/go" >> $HOME/.bashrc +PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin" +echo "export PATH=$(echo $PATH_VAR)" >> $HOME/.bashrc +export GOPATH=$(echo $HOME)/go +PATH_VAR=$PATH":/usr/local/go/bin:$(echo $HOME)/go/bin" +export PATH=$(echo $PATH_VAR) -###################### -echo Installing etcd # -###################### -go get github.com/coreos/etcd +go get -v -u -f github.com/gyuho/psn +psn ps-kill --force --clean-up +psn ps-kill --force --program etcd +psn ps-kill --force --program dbtester +psn ps-kill --force --program java +psn ss-kill --local-port 2181 +psn ss-kill --local-port 2378 +psn ss-kill --local-port 2379 +psn ss-kill --local-port 2380 +psn ss-kill --local-port 2888 +psn ss-kill --local-port 3500 +psn ss-kill --local-port 3888 +rm -rf $WORKING_DIR/*.etcd +rm -rf $WORKING_DIR/*.log +rm -rf $WORKING_DIR/failure_archive +go get -v -u -f github.com/coreos/etcd +go get -v -u -f github.com/coreos/dbtester/bench +go get -v -u -f github.com/coreos/dbtester +if [ -n "$3" ]; then + echo "Running node_exporter..." + psn ps-kill --force --program node_exporter + psn ss-kill --local-port 9100 + go get -v -u -f github.com/prometheus/node_exporter + nohup node_exporter -web.listen-address=":9100" > $WORKING_DIR/node_exporter.log 2>&1 & + sleep 5s + psn ps --program node_exporter + psn ss --program node_exporter +else + echo "No node_exporter..." +fi; ######################### echo Installing Ansible # ######################### +sudo apt-get -y update +sudo apt-get -y upgrade +sudo apt-get -y autoremove +sudo apt-get -y autoclean sudo apt-get install -y software-properties-common sudo apt-add-repository -y ppa:ansible/ansible sudo apt-get -y update @@ -69,9 +134,8 @@ echo "--- register: result - debug: var: result.stderr -" > $WORKING_DIR/dbtest_install.yml -ansible-playbook $WORKING_DIR/dbtest_install.yml - +" > $WORKING_DIR/dbtester_install.yml +ansible-playbook $WORKING_DIR/dbtester_install.yml ########################### echo Installing Zookeeper #