363 lines
9.8 KiB
Go
363 lines
9.8 KiB
Go
/*
|
|
* Copyright 2020 The Dragonfly Authors
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
//go:generate mockgen -destination mocks/dynconfig_mock.go -source dynconfig.go -package mocks
|
|
|
|
package config
|
|
|
|
import (
|
|
"context"
|
|
"encoding/json"
|
|
"fmt"
|
|
"os"
|
|
"path/filepath"
|
|
"time"
|
|
|
|
"google.golang.org/grpc/resolver"
|
|
|
|
managerv1 "d7y.io/api/pkg/apis/manager/v1"
|
|
|
|
logger "d7y.io/dragonfly/v2/internal/dflog"
|
|
dc "d7y.io/dragonfly/v2/internal/dynconfig"
|
|
"d7y.io/dragonfly/v2/manager/types"
|
|
"d7y.io/dragonfly/v2/pkg/reachable"
|
|
managerclient "d7y.io/dragonfly/v2/pkg/rpc/manager/client"
|
|
)
|
|
|
|
var (
|
|
// Cache filename.
|
|
cacheFileName = "scheduler"
|
|
|
|
// Notify observer interval.
|
|
watchInterval = 10 * time.Second
|
|
)
|
|
|
|
type DynconfigData struct {
|
|
ID uint64 `yaml:"id" mapstructure:"id" json:"id"`
|
|
Hostname string `yaml:"hostname" mapstructure:"hostname" json:"host_name"`
|
|
Idc string `yaml:"idc" mapstructure:"idc" json:"idc"`
|
|
Location string `yaml:"location" mapstructure:"location" json:"location"`
|
|
NetTopology string `yaml:"netTopology" mapstructure:"netTopology" json:"net_topology"`
|
|
IP string `yaml:"ip" mapstructure:"ip" json:"ip"`
|
|
Port int32 `yaml:"port" mapstructure:"port" json:"port"`
|
|
State string `yaml:"state" mapstructure:"state" json:"state"`
|
|
SeedPeers []*SeedPeer `yaml:"seedPeers" mapstructure:"seedPeers" json:"seed_peers"`
|
|
SchedulerCluster *SchedulerCluster `yaml:"schedulerCluster" mapstructure:"schedulerCluster" json:"scheduler_cluster"`
|
|
}
|
|
|
|
type SeedPeer struct {
|
|
ID uint `yaml:"id" mapstructure:"id" json:"id"`
|
|
Hostname string `yaml:"hostname" mapstructure:"hostname" json:"host_name"`
|
|
Type string `yaml:"type" mapstructure:"type" json:"type"`
|
|
IDC string `yaml:"idc" mapstructure:"idc" json:"idc"`
|
|
NetTopology string `yaml:"netTopology" mapstructure:"netTopology" json:"net_topology"`
|
|
Location string `yaml:"location" mapstructure:"location" json:"location"`
|
|
IP string `yaml:"ip" mapstructure:"ip" json:"ip"`
|
|
Port int32 `yaml:"port" mapstructure:"port" json:"port"`
|
|
DownloadPort int32 `yaml:"downloadPort" mapstructure:"downloadPort" json:"download_port"`
|
|
SeedPeerCluster *SeedPeerCluster `yaml:"seedPeerCluster" mapstructure:"seedPeerCluster" json:"seed_peer_cluster"`
|
|
}
|
|
|
|
func (c *SeedPeer) GetSeedPeerClusterConfig() (types.SeedPeerClusterConfig, bool) {
|
|
if c.SeedPeerCluster == nil {
|
|
return types.SeedPeerClusterConfig{}, false
|
|
}
|
|
|
|
var config types.SeedPeerClusterConfig
|
|
if err := json.Unmarshal(c.SeedPeerCluster.Config, &config); err != nil {
|
|
return types.SeedPeerClusterConfig{}, false
|
|
}
|
|
|
|
return config, true
|
|
}
|
|
|
|
type SeedPeerCluster struct {
|
|
ID uint64 `yaml:"id" mapstructure:"id" json:"id"`
|
|
Name string `yaml:"name" mapstructure:"name" json:"name"`
|
|
Config []byte `yaml:"config" mapstructure:"config" json:"config"`
|
|
}
|
|
|
|
type SchedulerCluster struct {
|
|
ID uint64 `yaml:"id" mapstructure:"id" json:"id"`
|
|
Name string `yaml:"name" mapstructure:"name" json:"name"`
|
|
Config []byte `yaml:"config" mapstructure:"config" json:"config"`
|
|
ClientConfig []byte `yaml:"clientConfig" mapstructure:"clientConfig" json:"client_config"`
|
|
}
|
|
|
|
type DynconfigInterface interface {
|
|
// Get the dynamic schedulers resolve addrs.
|
|
GetResolveSeedPeerAddrs() ([]resolver.Address, error)
|
|
|
|
// Get the dynamic seed peers config from manager.
|
|
GetSeedPeers() ([]*SeedPeer, error)
|
|
|
|
// Get the scheduler cluster config.
|
|
GetSchedulerClusterConfig() (types.SchedulerClusterConfig, bool)
|
|
|
|
// Get the client config.
|
|
GetSchedulerClusterClientConfig() (types.SchedulerClusterClientConfig, bool)
|
|
|
|
// Get the dynamic config from manager.
|
|
Get() (*DynconfigData, error)
|
|
|
|
// Refresh refreshes dynconfig in cache.
|
|
Refresh() error
|
|
|
|
// Register allows an instance to register itself to listen/observe events.
|
|
Register(Observer)
|
|
|
|
// Deregister allows an instance to remove itself from the collection of observers/listeners.
|
|
Deregister(Observer)
|
|
|
|
// Notify publishes new events to listeners.
|
|
Notify() error
|
|
|
|
// Serve the dynconfig listening service.
|
|
Serve() error
|
|
|
|
// Stop the dynconfig listening service.
|
|
Stop() error
|
|
}
|
|
|
|
type Observer interface {
|
|
// OnNotify allows an event to be published to interface implementations.
|
|
OnNotify(*DynconfigData)
|
|
}
|
|
|
|
type dynconfig struct {
|
|
dc.Dynconfig
|
|
observers map[Observer]struct{}
|
|
done chan bool
|
|
cachePath string
|
|
}
|
|
|
|
// NewDynconfig returns a new dynconfig instence.
|
|
func NewDynconfig(rawManagerClient managerclient.Client, cacheDir string, cfg *Config) (DynconfigInterface, error) {
|
|
cachePath := filepath.Join(cacheDir, cacheFileName)
|
|
d := &dynconfig{
|
|
observers: map[Observer]struct{}{},
|
|
done: make(chan bool),
|
|
cachePath: cachePath,
|
|
}
|
|
|
|
if rawManagerClient != nil {
|
|
client, err := dc.New(
|
|
newManagerClient(rawManagerClient, cfg),
|
|
cachePath,
|
|
cfg.DynConfig.RefreshInterval,
|
|
)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
d.Dynconfig = client
|
|
}
|
|
|
|
return d, nil
|
|
}
|
|
|
|
// Get the dynamic schedulers resolve addrs.
|
|
func (d *dynconfig) GetResolveSeedPeerAddrs() ([]resolver.Address, error) {
|
|
seedPeers, err := d.GetSeedPeers()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
var (
|
|
addrs = map[string]bool{}
|
|
resolveAddrs = []resolver.Address{}
|
|
)
|
|
for _, seedPeer := range seedPeers {
|
|
addr := fmt.Sprintf("%s:%d", seedPeer.IP, seedPeer.Port)
|
|
r := reachable.New(&reachable.Config{Address: addr})
|
|
if err := r.Check(); err != nil {
|
|
logger.Warnf("seed peer address %s is unreachable", addr)
|
|
continue
|
|
}
|
|
|
|
if addrs[addr] {
|
|
continue
|
|
}
|
|
|
|
resolveAddrs = append(resolveAddrs, resolver.Address{
|
|
ServerName: seedPeer.IP,
|
|
Addr: addr,
|
|
})
|
|
addrs[addr] = true
|
|
}
|
|
|
|
return resolveAddrs, nil
|
|
}
|
|
|
|
// Get the dynamic seed peers config from manager.
|
|
func (d *dynconfig) GetSeedPeers() ([]*SeedPeer, error) {
|
|
data, err := d.Get()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return data.SeedPeers, nil
|
|
}
|
|
|
|
// Get the scheduler cluster config.
|
|
func (d *dynconfig) GetSchedulerClusterConfig() (types.SchedulerClusterConfig, bool) {
|
|
data, err := d.Get()
|
|
if err != nil {
|
|
return types.SchedulerClusterConfig{}, false
|
|
}
|
|
|
|
if data.SchedulerCluster == nil {
|
|
return types.SchedulerClusterConfig{}, false
|
|
}
|
|
|
|
var config types.SchedulerClusterConfig
|
|
if err := json.Unmarshal(data.SchedulerCluster.Config, &config); err != nil {
|
|
return types.SchedulerClusterConfig{}, false
|
|
}
|
|
|
|
return config, true
|
|
}
|
|
|
|
// Get the client config.
|
|
func (d *dynconfig) GetSchedulerClusterClientConfig() (types.SchedulerClusterClientConfig, bool) {
|
|
data, err := d.Get()
|
|
if err != nil {
|
|
return types.SchedulerClusterClientConfig{}, false
|
|
}
|
|
|
|
if data.SchedulerCluster == nil {
|
|
return types.SchedulerClusterClientConfig{}, false
|
|
}
|
|
|
|
var config types.SchedulerClusterClientConfig
|
|
if err := json.Unmarshal(data.SchedulerCluster.ClientConfig, &config); err != nil {
|
|
return types.SchedulerClusterClientConfig{}, false
|
|
}
|
|
|
|
return config, true
|
|
}
|
|
|
|
// Get the dynamic config from manager.
|
|
func (d *dynconfig) Get() (*DynconfigData, error) {
|
|
var config DynconfigData
|
|
if err := d.Unmarshal(&config); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return &config, nil
|
|
}
|
|
|
|
// Refresh refreshes dynconfig in cache.
|
|
func (d *dynconfig) Refresh() error {
|
|
if err := d.Dynconfig.Refresh(); err != nil {
|
|
return err
|
|
}
|
|
|
|
if err := d.Notify(); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Register allows an instance to register itself to listen/observe events.
|
|
func (d *dynconfig) Register(l Observer) {
|
|
d.observers[l] = struct{}{}
|
|
}
|
|
|
|
// Deregister allows an instance to remove itself from the collection of observers/listeners.
|
|
func (d *dynconfig) Deregister(l Observer) {
|
|
delete(d.observers, l)
|
|
}
|
|
|
|
// Notify publishes new events to listeners.
|
|
func (d *dynconfig) Notify() error {
|
|
config, err := d.Get()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
for o := range d.observers {
|
|
o.OnNotify(config)
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Serve the dynconfig listening service.
|
|
func (d *dynconfig) Serve() error {
|
|
if err := d.Notify(); err != nil {
|
|
return err
|
|
}
|
|
|
|
go d.watch()
|
|
return nil
|
|
}
|
|
|
|
// watch the dynconfig events.
|
|
func (d *dynconfig) watch() {
|
|
tick := time.NewTicker(watchInterval)
|
|
|
|
for {
|
|
select {
|
|
case <-tick.C:
|
|
if err := d.Notify(); err != nil {
|
|
logger.Error("dynconfig notify failed", err)
|
|
}
|
|
case <-d.done:
|
|
return
|
|
}
|
|
}
|
|
}
|
|
|
|
// Stop the dynconfig listening service.
|
|
func (d *dynconfig) Stop() error {
|
|
close(d.done)
|
|
if err := os.Remove(d.cachePath); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
// Manager client for dynconfig.
|
|
type managerClient struct {
|
|
managerclient.Client
|
|
config *Config
|
|
}
|
|
|
|
// New the manager client used by dynconfig.
|
|
func newManagerClient(client managerclient.Client, cfg *Config) dc.ManagerClient {
|
|
return &managerClient{
|
|
Client: client,
|
|
config: cfg,
|
|
}
|
|
}
|
|
|
|
func (mc *managerClient) Get() (any, error) {
|
|
scheduler, err := mc.GetScheduler(context.Background(), &managerv1.GetSchedulerRequest{
|
|
SourceType: managerv1.SourceType_SCHEDULER_SOURCE,
|
|
HostName: mc.config.Server.Host,
|
|
Ip: mc.config.Server.IP,
|
|
SchedulerClusterId: uint64(mc.config.Manager.SchedulerClusterID),
|
|
})
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
return scheduler, nil
|
|
}
|