add local and remote mode for patroni attack

Signed-off-by: Nikita Savchenko <nikisavchenko@ozon.ru>

cmd/attack/attack.go

@@ -42,6 +42,7 @@ func NewAttackCommand() *cobra.Command {
 		NewHTTPAttackCommand(&uid),
 		NewVMAttackCommand(&uid),
 		NewUserDefinedCommand(&uid),
+		NewPatroniAttackCommand(&uid),
 	)
 
 	return cmd

cmd/attack/patroni.go

@@ -48,6 +48,9 @@ func NewPatroniAttackCommand(uid *string) *cobra.Command {
 
 	cmd.PersistentFlags().StringVarP(&options.User, "user", "u", "patroni", "patroni cluster user")
 	cmd.PersistentFlags().StringVar(&options.Password, "password", "p", "patroni cluster password")
+	cmd.PersistentFlags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts")
+	cmd.PersistentFlags().BoolVarP(&options.LocalMode, "local-mode", "l", false, "execute patronictl on host with chaosd. User with privileges required.")
+	cmd.PersistentFlags().BoolVarP(&options.RemoteMode, "remote-mode", "r", false, "execute patroni command by REST API")
 
 	return cmd
 }
@@ -55,15 +58,15 @@ func NewPatroniAttackCommand(uid *string) *cobra.Command {
 func NewPatroniSwitchoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "switchover",
-		Short: "exec switchover, default without another attack. Warning! Command is not recover!",
+		Short: "exec switchover command. Warning! Command is not recover!",
 		Run: func(*cobra.Command, []string) {
 			options.Action = core.SwitchoverAction
 			utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run()
 		},
 	}
-	cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts")
 	cmd.Flags().StringVarP(&options.Candidate, "candidate", "c", "", "switchover candidate, default random unit for replicas")
-	cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprintln(time.Now().Add(time.Second*60).Format(time.RFC3339)), "scheduled switchover, default now()+1 minute")
+	cmd.Flags().StringVarP(&options.Scheduled_at, "scheduled_at", "d", fmt.Sprint(time.Now().Add(time.Second*60).Format(time.RFC3339)), `scheduled switchover, 
+	default now()+1 minute by remote mode`)
 
 	return cmd
 }
@@ -71,14 +74,13 @@ func NewPatroniSwitchoverCommand(dep fx.Option, options *core.PatroniCommand) *c
 func NewPatroniFailoverCommand(dep fx.Option, options *core.PatroniCommand) *cobra.Command {
 	cmd := &cobra.Command{
 		Use:   "failover",
-		Short: "exec failover, default without another attack",
+		Short: "Exec failover command. Warning! Command is not recover!",
 		Run: func(*cobra.Command, []string) {
 			options.Action = core.FailoverAction
 			utils.FxNewAppWithoutLog(dep, fx.Invoke(PatroniAttackF)).Run()
 		},
 	}
 
-	cmd.Flags().StringVarP(&options.Address, "address", "a", "", "patroni cluster address, any of available hosts")
 	cmd.Flags().StringVarP(&options.Candidate, "leader", "c", "", "failover new leader, default random unit for replicas")
 	return cmd
 }

pkg/core/experiment.go

@@ -129,6 +129,8 @@ func GetAttackByKind(kind string) *AttackConfig {
 		attackConfig = &VMOption{}
 	case UserDefinedAttack:
 		attackConfig = &UserDefinedOption{}
+	case PatroniAttack:
+		attackConfig = &PatroniCommand{}
 	default:
 		return nil
 	}

pkg/core/patroni.go

@@ -35,6 +35,8 @@ type PatroniCommand struct {
 	User         string `json:"user,omitempty"`
 	Password     string `json:"password,omitempty"`
 	Scheduled_at string `json:"scheduled_at,omitempty"`
+	LocalMode    bool   `json:"local_mode,omitempty"`
+	RemoteMode   bool   `json:"remote_mode,omitempty"`
 	RecoverCmd   string `json:"recoverCmd,omitempty"`
 }
 
@@ -42,16 +44,24 @@ func (p *PatroniCommand) Validate() error {
 	if err := p.CommonAttackConfig.Validate(); err != nil {
 		return err
 	}
-	if len(p.Address) == 0 {
-		return errors.New("address not provided")
+
+	if !p.RemoteMode && !p.LocalMode {
+		return errors.New("local or remote mode required")
 	}
 
-	if len(p.User) == 0 {
-		return errors.New("patroni user not provided")
-	}
+	if p.RemoteMode {
 
-	if len(p.Password) == 0 {
-		return errors.New("patroni password not provided")
+		if len(p.Address) == 0 {
+			return errors.New("address not provided")
+		}
+
+		if len(p.User) == 0 {
+			return errors.New("patroni user not provided")
+		}
+
+		if len(p.Password) == 0 {
+			return errors.New("patroni password not provided")
+		}
 	}
 
 	return nil

pkg/server/chaosd/patroni.go

@@ -14,17 +14,18 @@
 package chaosd
 
 import (
-	"bytes"
 	"encoding/json"
 	"fmt"
-	"io"
 	"math/rand"
 	"net/http"
+	"os/exec"
+	"strings"
+
+	"github.com/pingcap/errors"
+	"github.com/pingcap/log"
 
 	"github.com/chaos-mesh/chaosd/pkg/core"
 	"github.com/chaos-mesh/chaosd/pkg/server/utils"
-	"github.com/pingcap/errors"
-	"github.com/pingcap/log"
 )
 
 type patroniAttack struct{}
@@ -34,96 +35,157 @@ var PatroniAttack AttackType = patroniAttack{}
 func (patroniAttack) Attack(options core.AttackConfig, _ Environment) error {
 	attack := options.(*core.PatroniCommand)
 
-	candidate := attack.Candidate
+	var responce []byte
 
-	leader := attack.Leader
+	var address string
 
-	var scheduled_at string
-
-	var url string
+	var err error
 
 	values := make(map[string]string)
 
-	patroniInfo, err := utils.GetPatroniInfo(attack.Address)
+	if attack.RemoteMode {
+		address = attack.Address
+	} else if attack.LocalMode {
+		address, err = utils.GetLocalHostname()
+		if err != nil {
+			return errors.WithStack(err)
+		}
+	}
+
+	patroniInfo, err := utils.GetPatroniInfo(address)
 	if err != nil {
 		err = errors.Errorf("failed to get patroni info for %v: %v", options.String(), err)
 		return errors.WithStack(err)
 	}
 
-	if len(patroniInfo.Replicas) == 0 {
-		err = errors.Errorf("failed to get available replicas. Please, check your cluster")
+	if len(patroniInfo.Replicas) == 0 && len(patroniInfo.SyncStandby) == 0 {
+		err = errors.Errorf("failed to get available candidates. Please, check your cluster")
 		return errors.WithStack(err)
 	}
 
-	if candidate == "" {
-		candidate = patroniInfo.Replicas[rand.Intn(len(patroniInfo.Replicas))]
+	sync_mode_check, err := isSynchronousClusterMode(address, attack.User, attack.Password)
+	if err != nil {
+		err = errors.Errorf("failed to check cluster synchronous mode for %v: %v", options.String(), err)
+		return errors.WithStack(err)
 	}
 
-	if leader == "" {
-		leader = patroniInfo.Master
+	if attack.Candidate == "" {
+		if sync_mode_check {
+			values["candidate"] = patroniInfo.SyncStandby[rand.Intn(len(patroniInfo.SyncStandby))]
+		} else {
+			values["candidate"] = patroniInfo.Replicas[rand.Intn(len(patroniInfo.Replicas))]
+		}
+
 	}
 
-	switch options.String() {
+	if attack.Leader == "" {
+		values["leader"] = patroniInfo.Master
+	}
+
+	values["scheduled_at"] = attack.Scheduled_at
+
+	cmd := options.String()
+
+	switch cmd {
 	case "switchover":
 
-		scheduled_at = attack.Scheduled_at
-
-		values = map[string]string{"leader": leader, "scheduled_at": scheduled_at}
-
-		log.Info(fmt.Sprintf("Switchover will be done from %v to another available replica in %v", patroniInfo.Master, scheduled_at))
+		log.Info(fmt.Sprintf("Switchover will be done from %v to %v in %v", values["leader"], values["candidate"], values["scheduled_at"]))
 
 	case "failover":
 
-		values = map[string]string{"candidate": candidate}
-
-		log.Info(fmt.Sprintf("Failover will be done from %v to %v", patroniInfo.Master, candidate))
+		log.Info(fmt.Sprintf("Failover will be done from %v to %v", values["leader"], values["candidate"]))
 
 	}
 
-	patroniAddr := attack.Address
+	if attack.RemoteMode {
+		responce, err = execPatroniAttackByRemoteMode(address, attack.User, attack.Password, cmd, values)
+		if err != nil {
+			return err
+		}
+	} else if attack.LocalMode {
+		responce, err = execPatroniAttackByLocalMode(cmd, values)
+		if err != nil {
+			return err
+		}
+	}
 
-	cmd := options.String()
+	if attack.RemoteMode {
+		log.S().Infof("Execute %v successfully: %v", cmd, string(responce))
+	}
+
+	if attack.LocalMode {
+		log.S().Infof("Execute %v successfully", cmd)
+		fmt.Println(string(responce))
+	}
+
+	return nil
+}
+
+func execPatroniAttackByRemoteMode(patroniAddr string, user string, password string, cmd string, values map[string]string) ([]byte, error) {
 
 	data, err := json.Marshal(values)
 	if err != nil {
 		err = errors.Errorf("failed to marshal data: %v", values)
-		return errors.WithStack(err)
+		return nil, errors.WithStack(err)
 	}
 
-	url = fmt.Sprintf("http://%v:8008/%v", patroniAddr, cmd)
-
-	request, err := http.NewRequest("POST", url, bytes.NewBuffer(data))
+	buf, err := utils.MakeHTTPRequest(http.MethodPost, patroniAddr, 8008, cmd, data, user, password)
 	if err != nil {
-		err = errors.Errorf("failed to %v: %v", cmd, err)
-		return errors.WithStack(err)
+		return nil, errors.WithStack(err)
 	}
 
-	request.Header.Set("Content-Type", "application/json")
-	request.SetBasicAuth(attack.User, attack.Password)
+	return buf, nil
+}
 
-	client := &http.Client{}
-	resp, error := client.Do(request)
-	if error != nil {
-		err = errors.Errorf("failed to %v: %v", cmd, err)
-		return errors.WithStack(err)
+func execPatroniAttackByLocalMode(cmd string, values map[string]string) ([]byte, error) {
+	var cmdTemplate string
+
+	if cmd == "failover" {
+		cmdTemplate = fmt.Sprintf("patronictl %v --master %v --candidate %v --force", cmd, values["leader"], values["candidate"])
+	} else if cmd == "switchover" {
+		cmdTemplate = fmt.Sprintf("patronictl %v --master %v --candidate %v --scheduled %v --force", cmd, values["leader"], values["candidate"], values["scheduled_at"])
 	}
 
-	defer resp.Body.Close()
-
-	buf, err := io.ReadAll(resp.Body)
+	execCmd := exec.Command("bash", "-c", cmdTemplate)
+	output, err := execCmd.CombinedOutput()
 	if err != nil {
-		err = errors.Errorf("failed to read %v responce: %v", cmd, err)
-		return errors.WithStack(err)
+		log.S().Errorf(fmt.Sprintf("failed to %v: %v", cmdTemplate, string(output)))
+		return nil, err
 	}
 
-	if resp.StatusCode != 200 && resp.StatusCode != 202 {
-		err = errors.Errorf("failed to %v: status code %v, responce %v", cmd, resp.StatusCode, string(buf))
-		return errors.WithStack(err)
+	if strings.Contains(string(output), "failed") {
+		err = errors.New(string(output))
+		return nil, err
 	}
 
-	log.S().Infof("Execute %v successfully: %v", cmd, string(buf))
+	return output, nil
+}
+
+func isSynchronousClusterMode(patroniAddr string, user string, password string) (bool, error) {
+
+	buf, err := utils.MakeHTTPRequest(http.MethodGet, patroniAddr, 8008, "config", []byte{}, user, password)
+	if err != nil {
+		return false, err
+	}
+
+	patroni_responce := make(map[string]interface{})
+
+	err = json.Unmarshal(buf, &patroni_responce)
+	if err != nil {
+		return false, fmt.Errorf("bad request %v %v", err.Error(), http.StatusBadRequest)
+	}
+
+	mode_check, ok := patroni_responce["synchronous_mode"].(bool)
+	if !ok {
+		return false, fmt.Errorf("failed to cast synchronous_mode field from patroni responce")
+	}
+
+	if mode_check {
+		return true, nil
+	}
+
+	return false, nil
 
-	return nil
 }
 
 func (patroniAttack) Recover(exp core.Experiment, _ Environment) error {

pkg/server/chaosd/recover.go

@@ -79,6 +79,8 @@ func (s *Server) RecoverAttack(uid string) error {
 			attackType = VMAttack
 		case core.UserDefinedAttack:
 			attackType = UserDefinedAttack
+		case core.PatroniAttack:
+			attackType = PatroniAttack
 		default:
 			return perr.Errorf("chaos experiment kind %s not found", exp.Kind)
 		}

pkg/server/utils/http_requests.go

@@ -0,0 +1,135 @@
+// Copyright 2023 Chaos Mesh Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"bytes"
+	"fmt"
+	"io"
+	"net/http"
+
+	"github.com/pingcap/log"
+	"github.com/pkg/errors"
+	"github.com/tidwall/gjson"
+)
+
+type PatroniInfo struct {
+	Master      string
+	Replicas    []string
+	SyncStandby []string
+	Status      []string
+}
+
+func GetPatroniInfo(address string) (PatroniInfo, error) {
+	res, err := http.Get(fmt.Sprintf("http://%v:8008/cluster", address))
+	if err != nil {
+		err = errors.Errorf("failed to get patroni status: %v", err)
+		return PatroniInfo{}, errors.WithStack(err)
+	}
+
+	defer res.Body.Close()
+
+	buf, err := io.ReadAll(res.Body)
+	if err != nil {
+		err = errors.Errorf("failed to read responce: %v", err)
+		return PatroniInfo{}, errors.WithStack(err)
+	}
+
+	data := string(buf)
+
+	patroniInfo := PatroniInfo{}
+
+	members := gjson.Get(data, "members")
+
+	for _, member := range members.Array() {
+		switch member.Get("role").Str {
+		case "leader":
+			patroniInfo.Master = member.Get("name").Str
+			patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str)
+		case "replica":
+			patroniInfo.Replicas = append(patroniInfo.Replicas, member.Get("name").Str)
+			patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str)
+		case "sync_standby":
+			patroniInfo.SyncStandby = append(patroniInfo.SyncStandby, member.Get("name").Str)
+			patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str)
+
+		}
+	}
+
+	log.Info(fmt.Sprintf("patroni info: master %v, replicas %v, sync_standy %s, statuses %v\n", patroniInfo.Master, patroniInfo.Replicas,
+		patroniInfo.SyncStandby, patroniInfo.Status))
+
+	return patroniInfo, nil
+
+}
+
+func MakeHTTPRequest(method string, address string, port int64, path string, body []byte, user string, password string) ([]byte, error) {
+
+	url := fmt.Sprintf("http://%v:%v/%v", address, port, path)
+
+	var request *http.Request
+
+	var resp *http.Response
+
+	var err error
+
+	switch method {
+	case http.MethodPost:
+		request, err = http.NewRequest("POST", url, bytes.NewBuffer(body))
+		if err != nil {
+			err = errors.Errorf("failed to post request %v: %v", url, err)
+			return nil, errors.WithStack(err)
+		}
+
+	case http.MethodGet:
+		request, err = http.NewRequest("GET", url, nil)
+		if err != nil {
+			err = errors.Errorf("failed to get request %v: %v", url, err)
+			return nil, errors.WithStack(err)
+		}
+	}
+
+	if user != "" && password != "" {
+		request.Header.Set("Content-Type", "application/json")
+		request.SetBasicAuth(user, password)
+	}
+
+	client := &http.Client{}
+	resp, err = client.Do(request)
+	if err != nil {
+		err = errors.Errorf("failed to exec %v request %v: %v", method, url, err)
+		return nil, errors.WithStack(err)
+	}
+
+	defer resp.Body.Close()
+
+	if resp.StatusCode != 200 && resp.StatusCode != 202 {
+		//to simplify diagnostics
+		buf, err := io.ReadAll(resp.Body)
+		if err != nil {
+			err = errors.Errorf("failed to read from %s responce: status code %v, responce %v, error %v", path, resp.StatusCode, resp.Body, err)
+			return nil, err
+		}
+		err = errors.Errorf("failed to exec %v request: status code %v, responce %v", path, resp.StatusCode, buf)
+		return nil, errors.WithStack(err)
+	}
+
+	buf, err := io.ReadAll(resp.Body)
+	if err != nil {
+		err = errors.Errorf("failed to read %v from %s responce: %v", resp.Body, path, err)
+		return nil, errors.WithStack(err)
+	}
+
+	return buf, nil
+}

pkg/server/utils/status.go

@@ -1,66 +0,0 @@
-// Copyright 2023 Chaos Mesh Authors.
-//
-// Licensed under the Apache License, Version 2.0 (the "License");
-// you may not use this file except in compliance with the License.
-// You may obtain a copy of the License at
-//
-//	http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing, software
-// distributed under the License is distributed on an "AS IS" BASIS,
-// See the License for the specific language governing permissions and
-// limitations under the License.
-package utils
-
-import (
-	"fmt"
-	"io"
-	"net/http"
-
-	"github.com/pingcap/log"
-	"github.com/pkg/errors"
-	"github.com/tidwall/gjson"
-)
-
-type PatroniInfo struct {
-	Master   string
-	Replicas []string
-	Status   []string
-}
-
-func GetPatroniInfo(address string) (PatroniInfo, error) {
-	res, err := http.Get(fmt.Sprintf("http://%v:8008/cluster", address))
-	if err != nil {
-		err = errors.Errorf("failed to get patroni status: %v", err)
-		return PatroniInfo{}, errors.WithStack(err)
-	}
-
-	defer res.Body.Close()
-
-	buf, err := io.ReadAll(res.Body)
-	if err != nil {
-		err = errors.Errorf("failed to read responce: %v", err)
-		return PatroniInfo{}, errors.WithStack(err)
-	}
-
-	data := string(buf)
-
-	patroniInfo := PatroniInfo{}
-
-	members := gjson.Get(data, "members")
-
-	for _, member := range members.Array() {
-		if member.Get("role").Str == "leader" {
-			patroniInfo.Master = member.Get("name").Str
-			patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str)
-		} else if member.Get("role").Str == "replica" || member.Get("role").Str == "sync_standby" {
-			patroniInfo.Replicas = append(patroniInfo.Replicas, member.Get("name").Str)
-			patroniInfo.Status = append(patroniInfo.Status, member.Get("state").Str)
-		}
-	}
-
-	log.Info(fmt.Sprintf("patroni info: master %v, replicas %v, statuses %v\n", patroniInfo.Master, patroniInfo.Replicas, patroniInfo.Status))
-
-	return patroniInfo, nil
-
-}

pkg/server/utils/system_unix.go

@@ -0,0 +1,27 @@
+// Copyright 2023 Chaos Mesh Authors.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package utils
+
+import (
+	"os"
+)
+
+func GetLocalHostname() (string, error) {
+	hostname, err := os.Hostname()
+	if err != nil {
+		return "", err
+	}
+
+	return hostname, nil
+}