dragonflyoss
/
dragonfly
mirror of https://github.com/dragonflyoss/dragonfly.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

feat: Update machinery to v1.10.13 and optimize Redis connection pool for weak network conditions (#4072) 

Signed-off-by: Gaius <gaius.qi@gmail.com>
This commit is contained in:
Gaius 2025-05-16 17:22:44 +08:00 committed by GitHub
parent 89ac602d08
commit 63b3ccacba
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 24 additions and 18 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -23,7 +23,7 @@ require (
github.com/docker/docker v28.1.1+incompatible github.com/docker/docker v28.1.1+incompatible
github.com/docker/go-connections v0.5.0 github.com/docker/go-connections v0.5.0
github.com/docker/go-units v0.4.0 github.com/docker/go-units v0.4.0
github.com/dragonflyoss/machinery v1.10.10 github.com/dragonflyoss/machinery v1.10.13
github.com/elastic/go-freelru v0.16.0 github.com/elastic/go-freelru v0.16.0
github.com/fsouza/fake-gcs-server v1.52.2 github.com/fsouza/fake-gcs-server v1.52.2
github.com/gaius-qi/ping v1.0.0 github.com/gaius-qi/ping v1.0.0

4
go.sum
View File

@ -441,8 +441,8 @@ github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1 h1:ZClxb8laGDf5arX
github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE= github.com/docker/libtrust v0.0.0-20150114040149-fa567046d9b1/go.mod h1:cyGadeNEkKy96OOhEzfZl+yxihPEzKnqJwvfuSUqbZE=
github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM= github.com/docker/spdystream v0.0.0-20160310174837-449fdfce4d96/go.mod h1:Qh8CwZgvJUkLughtfhJv5dyTYa91l1fOUCrgjqmcifM=
github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
github.com/dragonflyoss/machinery v1.10.10 h1:vvyPikg8W6Og6jyyUU5VVG+Lzjm+55v9xfVkqKaf8o8= github.com/dragonflyoss/machinery v1.10.13 h1:0iixzz4rn+oDIDHLz8sj5sQ5veTVg+Z1TOVKm2nnWv8=
github.com/dragonflyoss/machinery v1.10.10/go.mod h1:YUhavio0FVIsY9e3mVrj7weroc08gWm1hiauPDu1S28= github.com/dragonflyoss/machinery v1.10.13/go.mod h1:YUhavio0FVIsY9e3mVrj7weroc08gWm1hiauPDu1S28=
github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk=
github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=

15
internal/job/constants.go
View File

@ -39,10 +39,13 @@ const (
// Machinery server configuration. // Machinery server configuration.
const ( const (
DefaultResultsExpireIn = 86400 DefaultResultsExpireIn = 86400
DefaultRedisMaxIdle = 10 DefaultRedisMaxIdle = 0
DefaultRedisIdleTimeout = 300 DefaultRedisMaxActive = 300
DefaultRedisReadTimeout = 60 DefaultRedisIdleTimeout = 30
DefaultRedisWriteTimeout = 60 DefaultRedisReadTimeout = 60
DefaultRedisConnectTimeout = 60 DefaultRedisWriteTimeout = 60
DefaultRedisConnectTimeout = 60
DefaultRedisNormalTasksPollPeriod = 2000
DefaultRedisDelayedTasksPollPeriod = 500
) )

15
internal/job/job.go
View File

@ -88,12 +88,15 @@ func New(cfg *Config, queue Queue) (*Job, error) {
ResultBackend: backend, ResultBackend: backend,
ResultsExpireIn: DefaultResultsExpireIn, ResultsExpireIn: DefaultResultsExpireIn,
Redis: &machineryv1config.RedisConfig{ Redis: &machineryv1config.RedisConfig{
MasterName: cfg.MasterName, MasterName: cfg.MasterName,
MaxIdle: DefaultRedisMaxIdle, MaxIdle: DefaultRedisMaxIdle,
IdleTimeout: DefaultRedisIdleTimeout, MaxActive: DefaultRedisMaxActive,
ReadTimeout: DefaultRedisReadTimeout, IdleTimeout: DefaultRedisIdleTimeout,
WriteTimeout: DefaultRedisWriteTimeout, ReadTimeout: DefaultRedisReadTimeout,
ConnectTimeout: DefaultRedisConnectTimeout, WriteTimeout: DefaultRedisWriteTimeout,
ConnectTimeout: DefaultRedisConnectTimeout,
NormalTasksPollPeriod: DefaultRedisNormalTasksPollPeriod,
DelayedTasksPollPeriod: DefaultRedisDelayedTasksPollPeriod,
}, },
}) })
if err != nil { if err != nil {

2
manager/job/preheat.go
View File

@ -199,7 +199,7 @@ func (p *preheat) createGroupJob(ctx context.Context, files []internaljob.Prehea
} }
logger.Infof("[preheat]: create preheat group %s in queues %v, tasks: %#v", group.GroupUUID, queues, tasks) logger.Infof("[preheat]: create preheat group %s in queues %v, tasks: %#v", group.GroupUUID, queues, tasks)
if _, err := p.job.Server.SendGroupWithContext(ctx, group, 0); err != nil { if _, err := p.job.Server.SendGroupWithContext(ctx, group, 50); err != nil {
logger.Errorf("[preheat]: create preheat group %s failed", group.GroupUUID, err) logger.Errorf("[preheat]: create preheat group %s failed", group.GroupUUID, err)
return nil, err return nil, err
} }

4
manager/job/task.go
View File

@ -102,7 +102,7 @@ func (t *task) CreateGetTask(ctx context.Context, schedulers []models.Scheduler,
} }
logger.Infof("create task group %s in queues %v, tasks: %#v", group.GroupUUID, queues, tasks) logger.Infof("create task group %s in queues %v, tasks: %#v", group.GroupUUID, queues, tasks)
if _, err := t.job.Server.SendGroupWithContext(ctx, group, 0); err != nil { if _, err := t.job.Server.SendGroupWithContext(ctx, group, 50); err != nil {
logger.Errorf("create task group %s failed", group.GroupUUID, err) logger.Errorf("create task group %s failed", group.GroupUUID, err)
return nil, err return nil, err
} }
@ -163,7 +163,7 @@ func (t *task) CreateDeleteTask(ctx context.Context, schedulers []models.Schedul
} }
logger.Infof("create task group %s in queues %v, tasks: %#v", group.GroupUUID, queues, tasks) logger.Infof("create task group %s in queues %v, tasks: %#v", group.GroupUUID, queues, tasks)
if _, err := t.job.Server.SendGroupWithContext(ctx, group, 0); err != nil { if _, err := t.job.Server.SendGroupWithContext(ctx, group, 50); err != nil {
logger.Errorf("create preheat group %s failed", group.GroupUUID, err) logger.Errorf("create preheat group %s failed", group.GroupUUID, err)
return nil, err return nil, err
} }