chaos-mesh
/
chaosd
mirror of https://github.com/chaos-mesh/chaosd.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Add clock attack support. (#90) 

Signed-off-by: andrewmatilde <davis6813585853062@outlook.com>
This commit is contained in:
Andrewmatilde 2021-09-30 16:22:20 +08:00 committed by GitHub
parent c54f9a5d4d
commit c1b722e87e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 509 additions and 11 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
cmd/attack/attack.go
View File

@ -35,6 +35,7 @@ func NewAttackCommand() *cobra.Command {
NewDiskAttackCommand(&uid),
NewHostAttackCommand(&uid),
NewJVMAttackCommand(&uid),
NewClockAttackCommand(&uid),
)
return cmd

68
cmd/attack/clock.go Normal file
View File

@ -0,0 +1,68 @@
// Copyright 2021 Chaos Mesh Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package attack
import (
"fmt"
"github.com/spf13/cobra"
"go.uber.org/fx"
"github.com/chaos-mesh/chaosd/cmd/server"
"github.com/chaos-mesh/chaosd/pkg/core"
"github.com/chaos-mesh/chaosd/pkg/server/chaosd"
"github.com/chaos-mesh/chaosd/pkg/utils"
)
func NewClockAttackCommand(uid *string) *cobra.Command {
options := core.NewClockOption()
dep := fx.Options(
server.Module,
fx.Provide(func() *core.ClockOption {
options.UID = *uid
return options
}),
)
cmd := &cobra.Command{
Use: "clock attack",
Short: "clock skew",
Run: func(*cobra.Command, []string) {
options.Action = "Attack"
utils.FxNewAppWithoutLog(dep, fx.Invoke(processClockAttack)).Run()
},
}
cmd.Flags().IntVarP(&options.Pid, "pid", "p", 0, "Pid of target program.")
cmd.Flags().StringVarP(&options.TimeOffset, "time-offset", "t", "", "Specifies the length of time offset.")
cmd.Flags().StringVarP(&options.ClockIdsSlice, "clock-ids-slice", "c", "CLOCK_REALTIME",
"The identifier of the particular clock on which to act."+
"More clock description in linux kernel can be found in man page of clock_getres, clock_gettime, clock_settime."+
"Muti clock ids should be split with \",\"")
return cmd
}
func processClockAttack(options *core.ClockOption, chaos *chaosd.Server) {
err := options.PreProcess()
if err != nil {
utils.ExitWithError(utils.ExitBadArgs, err)
}
uid, err := chaos.ExecuteAttack(chaosd.ClockAttack, options, core.CommandMode)
if err != nil {
utils.ExitWithError(utils.ExitError, err)
}
utils.NormalExit(fmt.Sprintf("Clock attack %v successfully, uid: %s", options, uid))
}

112
pkg/core/clock.go Normal file
View File

@ -0,0 +1,112 @@
// Copyright 2021 Chaos Mesh Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package core
import (
"encoding/json"
"fmt"
"os"
"strings"
"syscall"
"time"
"github.com/pingcap/log"
"go.uber.org/zap"
"github.com/chaos-mesh/chaos-mesh/pkg/time/utils"
)
type ClockOption struct {
CommonAttackConfig
Pid int
TimeOffset string
SecDelta int64
NsecDelta int64
ClockIdsSlice string
Store ClockFuncStore
ClockIdsMask uint64
}
type ClockFuncStore struct {
CodeOfGetClockFunc []byte
OriginAddress uint64
}
func NewClockOption() *ClockOption {
return &ClockOption{
CommonAttackConfig: CommonAttackConfig{
Kind: ClockAttack,
},
}
}
func (opt *ClockOption) PreProcess() error {
clkIds := strings.Split(opt.ClockIdsSlice, ",")
offset, err := time.ParseDuration(opt.TimeOffset)
if err != nil {
return err
}
opt.SecDelta = int64(offset / time.Second)
opt.NsecDelta = int64(offset % time.Second)
clockIdsMask, err := utils.EncodeClkIds(clkIds)
if err != nil {
log.Error("error while converting clock ids to mask", zap.Error(err))
return err
}
if clockIdsMask == 0 {
log.Error("clock ids must not be empty")
return fmt.Errorf("clock ids must not be empty")
}
opt.ClockIdsMask = clockIdsMask
if uint64(opt.SecDelta) > 1<<31 {
log.Warn("Monotonic clock will be broken when sec delta is too large or too small.")
if uint64(opt.SecDelta) > 1<<56 {
log.Warn("Time zone info will be broken when sec delta is too large or too small.")
}
}
if uint64(opt.NsecDelta) > 1<<56 {
log.Warn("Time will be broken when nanosecond delta is too large or too small")
}
// Since os.FindProcess in unix systems will always succeed
// regardless of whether the process exists (https://pkg.go.dev/os#FindProcess),
// we need to use process.Signal to check if pid is accessible.
process, err := os.FindProcess(opt.Pid)
if err != nil {
log.Error("failed to find process", zap.Error(err))
return err
}
err = process.Signal(syscall.Signal(0))
if err != nil {
log.Error("pid may not be accessible", zap.Error(err))
return err
}
return nil
}
func (opt ClockOption) RecoverData() string {
data, _ := json.Marshal(opt)
return string(data)
}

3
pkg/core/experiment.go
View File

@ -35,6 +35,7 @@ const (
NetworkAttack = "network"
StressAttack = "stress"
DiskAttack = "disk"
ClockAttack = "clock"
HostAttack = "host"
JVMAttack = "jvm"
)
@ -104,6 +105,8 @@ func GetAttackByKind(kind string) *AttackConfig {
attackConfig = &DiskAttackConfig{}
case JVMAttack:
attackConfig = &JVMCommand{}
case ClockAttack:
attackConfig = &ClockOption{}
default:
return nil
}

14
pkg/server/chaosd/attack.go
View File

@ -30,7 +30,14 @@ type Environment struct {
}
type AttackType interface {
// Attack execute attack with options and env.
// ExecuteAttack will store the options ahead of Attack be executed
// and will store options again after Attack be executed.
// We can also use env.Chaos.expStore to touch the storage of chaosd.
// But do not update it with your own uid ,
// because it will be covered after Attack executed with options.
Attack(options core.AttackConfig, env Environment) error
// Recover can get marshaled options data from experiment and recover it.
Recover(experiment core.Experiment, env Environment) error
}
@ -59,25 +66,26 @@ func (s *Server) ExecuteAttack(attackType AttackType, options core.AttackConfig,
RecoverCommand: options.RecoverData(),
LaunchMode: launchMode,
}
if err = s.exp.Set(context.Background(), exp); err != nil {
if err = s.expStore.Set(context.Background(), exp); err != nil {
err = perr.WithStack(err)
return
}
defer func() {
if err != nil {
if err := s.exp.Update(context.Background(), uid, core.Error, err.Error(), options.RecoverData()); err != nil {
if err := s.expStore.Update(context.Background(), uid, core.Error, err.Error(), options.RecoverData()); err != nil {
log.Error("failed to update experiment", zap.Error(err))
}
return
}
var newStatus string
if len(options.Cron()) > 0 {
newStatus = core.Scheduled
} else {
newStatus = core.Success
}
if err := s.exp.Update(context.Background(), uid, newStatus, "", options.RecoverData()); err != nil {
if err := s.expStore.Update(context.Background(), uid, newStatus, "", options.RecoverData()); err != nil {
log.Error("failed to update experiment", zap.Error(err))
}
}()

273
pkg/server/chaosd/clock.go Normal file
View File

@ -0,0 +1,273 @@
// Copyright 2021 Chaos Mesh Authors.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// See the License for the specific language governing permissions and
// limitations under the License.
package chaosd
import (
"bytes"
"debug/elf"
"fmt"
"runtime"
"github.com/chaos-mesh/chaos-mesh/pkg/mapreader"
"github.com/chaos-mesh/chaos-mesh/pkg/ptrace"
"github.com/pingcap/log"
"go.uber.org/zap"
"github.com/chaos-mesh/chaosd/pkg/core"
)
type clockAttack struct{}
var ClockAttack AttackType = clockAttack{}
// Copied from chaos-mesh/pkg/time/time_linux_amd64,
// I will move the recover part into it just future.
var fakeImage = []byte{
0xb8, 0xe4, 0x00, 0x00, 0x00, //mov $0xe4,%eax
0x0f, 0x05, //syscall
0xba, 0x01, 0x00, 0x00, 0x00, //mov $0x1,%edx
0x89, 0xf9, //mov %edi,%ecx
0xd3, 0xe2, //shl %cl,%edx
0x48, 0x8d, 0x0d, 0x74, 0x00, 0x00, 0x00, //lea 0x74(%rip),%rcx # <CLOCK_IDS_MASK>
0x48, 0x63, 0xd2, //movslq %edx,%rdx
0x48, 0x85, 0x11, //test %rdx,(%rcx)
0x74, 0x6b, //je 108a <clock_gettime+0x8a>
0x48, 0x8d, 0x15, 0x6d, 0x00, 0x00, 0x00, //lea 0x6d(%rip),%rdx # <TV_SEC_DELTA>
0x4c, 0x8b, 0x46, 0x08, //mov 0x8(%rsi),%r8
0x48, 0x8b, 0x0a, //mov (%rdx),%rcx
0x48, 0x8d, 0x15, 0x67, 0x00, 0x00, 0x00, //lea 0x67(%rip),%rdx # <TV_NSEC_DELTA>
0x48, 0x8b, 0x3a, //mov (%rdx),%rdi
0x4a, 0x8d, 0x14, 0x07, //lea (%rdi,%r8,1),%rdx
0x48, 0x81, 0xfa, 0x00, 0xca, 0x9a, 0x3b, //cmp $0x3b9aca00,%rdx
0x7e, 0x1c, //jle <clock_gettime+0x60>
0x0f, 0x1f, 0x40, 0x00, //nopl 0x0(%rax)
0x48, 0x81, 0xef, 0x00, 0xca, 0x9a, 0x3b, //sub $0x3b9aca00,%rdi
0x48, 0x83, 0xc1, 0x01, //add $0x1,%rcx
0x49, 0x8d, 0x14, 0x38, //lea (%r8,%rdi,1),%rdx
0x48, 0x81, 0xfa, 0x00, 0xca, 0x9a, 0x3b, //cmp $0x3b9aca00,%rdx
0x7f, 0xe8, //jg <clock_gettime+0x48>
0x48, 0x85, 0xd2, //test %rdx,%rdx
0x79, 0x1e, //jns <clock_gettime+0x83>
0x4a, 0x8d, 0xbc, 0x07, 0x00, 0xca, 0x9a, //lea 0x3b9aca00(%rdi,%r8,1),%rdi
0x3b, //
0x0f, 0x1f, 0x00, //nopl (%rax)
0x48, 0x89, 0xfa, //mov %rdi,%rdx
0x48, 0x83, 0xe9, 0x01, //sub $0x1,%rcx
0x48, 0x81, 0xc7, 0x00, 0xca, 0x9a, 0x3b, //add $0x3b9aca00,%rdi
0x48, 0x85, 0xd2, //test %rdx,%rdx
0x78, 0xed, //js <clock_gettime+0x70>
0x48, 0x01, 0x0e, //add %rcx,(%rsi)
0x48, 0x89, 0x56, 0x08, //mov %rdx,0x8(%rsi)
0xc3, //retq
// constant
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, //CLOCK_IDS_MASK
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, //TV_SEC_DELTA
0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, //TV_NSEC_DELTA
}
func (c clockAttack) Attack(options core.AttackConfig, env Environment) error {
var opt *core.ClockOption
var ok bool
if opt, ok = options.(*core.ClockOption); !ok {
return fmt.Errorf("AttackConfig -> *ClockOption meet error")
}
runtime.LockOSThread()
defer func() {
runtime.UnlockOSThread()
}()
program, err := ptrace.Trace(opt.Pid)
if err != nil {
return err
}
defer func() {
err = program.Detach()
if err != nil {
log.Error("fail to detach program", zap.Error(err), zap.Int("pid", opt.Pid))
}
}()
var vdsoEntry *mapreader.Entry
for index := range program.Entries {
// reverse loop is faster
e := program.Entries[len(program.Entries)-index-1]
if e.Path == "[vdso]" {
vdsoEntry = &e
break
}
}
if vdsoEntry == nil {
return fmt.Errorf("cannot find [vdso] entry")
}
// minus tailing variable part
// 24 = 3 * 8 because we have three variables
constImageLen := len(fakeImage) - 24
var fakeEntry *mapreader.Entry
// find injected image to avoid redundant inject (which will lead to memory leak)
for _, e := range program.Entries {
e := e
image, err := program.ReadSlice(e.StartAddress, uint64(constImageLen))
if err != nil {
continue
}
if bytes.Equal(*image, fakeImage[0:constImageLen]) {
fakeEntry = &e
log.Warn("found injected image", zap.Uint64("addr", fakeEntry.StartAddress))
}
}
if fakeEntry == nil {
fakeEntry, err = program.MmapSlice(fakeImage)
if err != nil {
return err
}
}
fakeAddr := fakeEntry.StartAddress
// 139 is the index of CLOCK_IDS_MASK in fakeImage
err = program.WriteUint64ToAddr(fakeAddr+139, opt.ClockIdsMask)
if err != nil {
return err
}
// 147 is the index of TV_SEC_DELTA in fakeImage
err = program.WriteUint64ToAddr(fakeAddr+147, uint64(opt.SecDelta))
if err != nil {
return err
}
// 155 is the index of TV_NSEC_DELTA in fakeImage
err = program.WriteUint64ToAddr(fakeAddr+155, uint64(opt.NsecDelta))
if err != nil {
return err
}
originAddr, size, err := FindSymbolInEntry(*program, "clock_gettime", vdsoEntry)
if err != nil {
return err
}
funcBytes, err := program.ReadSlice(originAddr, size)
exps, err := env.Chaos.Search(&core.SearchCommand{
Status: core.Success,
Kind: core.ClockAttack,
})
if err != nil {
return err
}
for _, exp := range exps {
if exp.Kind == core.ClockAttack {
lastOptions, err := exp.GetRequestCommand()
if err != nil {
return err
}
var lastOpt *core.ClockOption
var ok bool
if lastOpt, ok = lastOptions.(*core.ClockOption); !ok {
log.Warn("AttackConfig -> *ClockOption meet error")
continue
}
if lastOpt.Pid == opt.Pid {
return fmt.Errorf("plz recover the last clock attack on pid : %d first \n"+
"chaosd recover %s", opt.Pid, exp.Uid)
}
}
}
opt.Store = core.ClockFuncStore{
CodeOfGetClockFunc: *funcBytes,
OriginAddress: originAddr,
}
if err != nil {
return err
}
err = program.JumpToFakeFunc(originAddr, fakeAddr)
return err
}
func (c clockAttack) Recover(exp core.Experiment, env Environment) error {
options, err := exp.GetRequestCommand()
if err != nil {
return err
}
var opt *core.ClockOption
var ok bool
if opt, ok = options.(*core.ClockOption); !ok {
return fmt.Errorf("AttackConfig -> *ClockOption meet error")
}
runtime.LockOSThread()
defer func() {
runtime.UnlockOSThread()
}()
program, err := ptrace.Trace(opt.Pid)
if err != nil {
return err
}
defer func() {
err = program.Detach()
if err != nil {
log.Error("fail to detach program", zap.Error(err), zap.Int("pid", opt.Pid))
}
}()
err = program.PtraceWriteSlice(opt.Store.OriginAddress, opt.Store.CodeOfGetClockFunc)
return err
}
// FindSymbolInEntry finds symbol in entry through parsing elf
func FindSymbolInEntry(p ptrace.TracedProgram, symbolName string, entry *mapreader.Entry) (uint64, uint64, error) {
libBuffer, err := p.GetLibBuffer(entry)
if err != nil {
return 0, 0, err
}
reader := bytes.NewReader(*libBuffer)
vdsoElf, err := elf.NewFile(reader)
if err != nil {
return 0, 0, err
}
loadOffset := uint64(0)
for _, prog := range vdsoElf.Progs {
if prog.Type == elf.PT_LOAD {
loadOffset = prog.Vaddr - prog.Off
// break here is enough for vdso
break
}
}
symbols, err := vdsoElf.DynamicSymbols()
if err != nil {
return 0, 0, err
}
for _, symbol := range symbols {
if symbol.Name == symbolName {
offset := symbol.Value
return entry.StartAddress + (offset - loadOffset), symbol.Size, nil
}
}
return 0, 0, fmt.Errorf("cannot find symbol")
}

6
pkg/server/chaosd/recover.go
View File

@ -25,7 +25,7 @@ import (
)
func (s *Server) RecoverAttack(uid string) error {
exp, err := s.exp.FindByUid(context.Background(), uid)
exp, err := s.expStore.FindByUid(context.Background(), uid)
if err != nil {
return err
}
@ -63,6 +63,8 @@ func (s *Server) RecoverAttack(uid string) error {
attackType = DiskAttack
case core.JVMAttack:
attackType = JVMAttack
case core.ClockAttack:
attackType = ClockAttack
default:
return perr.Errorf("chaos experiment kind %s not found", exp.Kind)
}
@ -77,7 +79,7 @@ func (s *Server) RecoverAttack(uid string) error {
}
}
if err := s.exp.Update(context.Background(), uid, core.Destroyed, "", exp.RecoverCommand); err != nil {
if err := s.expStore.Update(context.Background(), uid, core.Destroyed, "", exp.RecoverCommand); err != nil {
return perr.WithStack(err)
}
return nil

4
pkg/server/chaosd/search.go
View File

@ -23,7 +23,7 @@ import (
func (s *Server) Search(conds *core.SearchCommand) ([]*core.Experiment, error) {
if len(conds.UID) > 0 {
exp, err := s.exp.FindByUid(context.Background(), conds.UID)
exp, err := s.expStore.FindByUid(context.Background(), conds.UID)
if err != nil {
return nil, errors.WithStack(err)
}
@ -31,7 +31,7 @@ func (s *Server) Search(conds *core.SearchCommand) ([]*core.Experiment, error) {
return []*core.Experiment{exp}, nil
}
exps, err := s.exp.ListByConditions(context.Background(), conds)
exps, err := s.expStore.ListByConditions(context.Background(), conds)
if err != nil {
return nil, errors.WithStack(err)
}

4
pkg/server/chaosd/server.go
View File

@ -22,7 +22,7 @@ import (
)
type Server struct {
exp core.ExperimentStore
expStore core.ExperimentStore
ExpRun core.ExperimentRunStore
Cron scheduler.Scheduler
ipsetRule core.IPSetRuleStore
@ -44,7 +44,7 @@ func NewServer(
) *Server {
return &Server{
conf: conf,
exp: exp,
expStore: exp,
Cron: cron,
ExpRun: expRun,
ipsetRule: ipset,

32
pkg/server/httpserver/server.go
View File

@ -83,6 +83,7 @@ func handler(s *httpServer) {
attack.POST("/stress", s.createStressAttack)
attack.POST("/network", s.createNetworkAttack)
attack.POST("/disk", s.createDiskAttack)
attack.POST("/clock", s.createClockAttack)
attack.DELETE("/:uid", s.recoverAttack)
}
@ -225,6 +226,37 @@ func (s *httpServer) createDiskAttack(c *gin.Context) {
c.JSON(http.StatusOK, utils.AttackSuccessResponse(uid))
}
// @Summary Create clock attack.
// @Description Create clock attack.
// @Tags attack
// @Produce json
// @Param request body core.ClockOption true "Request body"
// @Success 200 {object} utils.Response
// @Failure 400 {object} utils.APIError
// @Failure 500 {object} utils.APIError
// @Router /api/attack/clock [post]
func (s *httpServer) createClockAttack(c *gin.Context) {
options := core.NewClockOption()
if err := c.ShouldBindJSON(options); err != nil {
c.AbortWithError(http.StatusBadRequest, utils.ErrInternalServer.WrapWithNoMessage(err))
return
}
err := options.PreProcess()
if err != nil {
err = core.ErrAttackConfigValidation.Wrap(err, "attack config validation failed")
handleError(c, err)
return
}
uid, err := s.chaos.ExecuteAttack(chaosd.ClockAttack, options, core.CommandMode)
if err != nil {
handleError(c, err)
return
}
c.JSON(http.StatusOK, utils.AttackSuccessResponse(uid))
}
// @Summary Create recover attack.
// @Description Create recover attack.
// @Tags attack

3
pkg/store/experiment/experiment.go
View File

@ -17,9 +17,8 @@ import (
"context"
"errors"
"gorm.io/gorm"
perr "github.com/pkg/errors"
"gorm.io/gorm"
"github.com/chaos-mesh/chaosd/pkg/core"
"github.com/chaos-mesh/chaosd/pkg/store/dbstore"