dragonfly/scheduler/config/dynconfig.go

/*
 *     Copyright 2020 The Dragonfly Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

//go:generate mockgen -destination mocks/dynconfig_mock.go -source dynconfig.go -package mocks

package config

import (
	"context"
	"encoding/json"
	"errors"
	"fmt"
	"net"
	"os"
	"path/filepath"
	"time"

	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/credentials"
	"google.golang.org/grpc/credentials/insecure"
	"google.golang.org/grpc/resolver"
	"google.golang.org/grpc/status"

	managerv2 "d7y.io/api/pkg/apis/manager/v2"

	logger "d7y.io/dragonfly/v2/internal/dflog"
	dc "d7y.io/dragonfly/v2/internal/dynconfig"
	"d7y.io/dragonfly/v2/manager/types"
	"d7y.io/dragonfly/v2/pkg/net/ip"
	healthclient "d7y.io/dragonfly/v2/pkg/rpc/health/client"
	managerclient "d7y.io/dragonfly/v2/pkg/rpc/manager/client"
	"d7y.io/dragonfly/v2/pkg/slices"
)

var (
	// Cache filename.
	cacheFileName = "scheduler"

	// Notify observer interval.
	watchInterval = 10 * time.Second
)

type DynconfigData struct {
	Scheduler    *managerv2.Scheduler
	Applications []*managerv2.Application
}

type DynconfigInterface interface {
	// GetResolveSeedPeerAddrs returns the dynamic schedulers resolve addrs.
	GetResolveSeedPeerAddrs() ([]resolver.Address, error)

	// GetScheduler returns the scheduler config from manager.
	GetScheduler() (*managerv2.Scheduler, error)

	// GetApplications returns the applications config from manager.
	GetApplications() ([]*managerv2.Application, error)

	// GetSeedPeers returns the dynamic seed peers config from manager.
	GetSeedPeers() ([]*managerv2.SeedPeer, error)

	// GetSchedulerCluster returns the the scheduler cluster config from manager.
	GetSchedulerCluster() (*managerv2.SchedulerCluster, error)

	// GetSchedulerClusterConfig returns the scheduler cluster config.
	GetSchedulerClusterConfig() (types.SchedulerClusterConfig, error)

	// GetSchedulerClusterClientConfig returns the client config.
	GetSchedulerClusterClientConfig() (types.SchedulerClusterClientConfig, error)

	// Get returns the dynamic config from manager.
	Get() (*DynconfigData, error)

	// Refresh refreshes dynconfig in cache.
	Refresh() error

	// Register allows an instance to register itself to listen/observe events.
	Register(Observer)

	// Deregister allows an instance to remove itself from the collection of observers/listeners.
	Deregister(Observer)

	// Notify publishes new events to listeners.
	Notify() error

	// Serve the dynconfig listening service.
	Serve() error

	// Stop the dynconfig listening service.
	Stop() error
}

type Observer interface {
	// OnNotify allows an event to be published to interface implementations.
	OnNotify(*DynconfigData)
}

type dynconfig struct {
	dc.Dynconfig[DynconfigData]
	observers            map[Observer]struct{}
	done                 chan struct{}
	cachePath            string
	transportCredentials credentials.TransportCredentials
}

// DynconfigOption is a functional option for configuring the dynconfig.
type DynconfigOption func(d *dynconfig) error

// WithTransportCredentials returns a DialOption which configures a connection
// level security credentials (e.g., TLS/SSL).
func WithTransportCredentials(creds credentials.TransportCredentials) DynconfigOption {
	return func(d *dynconfig) error {
		d.transportCredentials = creds
		return nil
	}
}

// NewDynconfig returns a new dynconfig instence.
func NewDynconfig(rawManagerClient managerclient.V2, cacheDir string, cfg *Config, options ...DynconfigOption) (DynconfigInterface, error) {
	cachePath := filepath.Join(cacheDir, cacheFileName)
	d := &dynconfig{
		observers: map[Observer]struct{}{},
		done:      make(chan struct{}),
		cachePath: cachePath,
	}

	for _, opt := range options {
		if err := opt(d); err != nil {
			return nil, err
		}
	}

	if rawManagerClient != nil {
		client, err := dc.New[DynconfigData](
			newManagerClient(rawManagerClient, cfg),
			cachePath,
			cfg.DynConfig.RefreshInterval,
		)
		if err != nil {
			return nil, err
		}

		d.Dynconfig = client
	}

	return d, nil
}

// GetResolveSeedPeerAddrs returns the the schedulers resolve addrs.
func (d *dynconfig) GetResolveSeedPeerAddrs() ([]resolver.Address, error) {
	seedPeers, err := d.GetSeedPeers()
	if err != nil {
		return nil, err
	}

	var (
		addrs        = map[string]bool{}
		resolveAddrs []resolver.Address
	)
	for _, seedPeer := range seedPeers {
		dialOptions := []grpc.DialOption{}
		if d.transportCredentials != nil {
			dialOptions = append(dialOptions, grpc.WithTransportCredentials(d.transportCredentials))
		} else {
			dialOptions = append(dialOptions, grpc.WithTransportCredentials(insecure.NewCredentials()))
		}

		var addr string
		if ip, ok := ip.FormatIP(seedPeer.GetIp()); ok {
			// Check health with ip address.
			target := fmt.Sprintf("%s:%d", ip, seedPeer.GetPort())
			if err := healthclient.Check(context.Background(), target, dialOptions...); err != nil {
				logger.Warnf("seed peer ip address %s is unreachable: %s", addr, err.Error())

				// Check health with host address.
				target = fmt.Sprintf("%s:%d", seedPeer.GetHostname(), seedPeer.GetPort())
				if err := healthclient.Check(context.Background(), target, dialOptions...); err != nil {
					logger.Warnf("seed peer host address %s is unreachable: %s", addr, err.Error())
				} else {
					addr = target
				}
			} else {
				addr = target
			}
		}

		if addr == "" {
			logger.Warnf("seed peer %s %s %s has not reachable addresses",
				seedPeer.GetIp(), seedPeer.GetHostname(), seedPeer.GetPort())
			continue
		}

		if addrs[addr] {
			continue
		}

		host, _, err := net.SplitHostPort(addr)
		if err != nil {
			continue
		}

		resolveAddrs = append(resolveAddrs, resolver.Address{
			ServerName: host,
			Addr:       addr,
		})
		addrs[addr] = true
	}

	if len(resolveAddrs) == 0 {
		return nil, errors.New("available seed peer not found")
	}

	return resolveAddrs, nil
}

// GetScheduler returns the scheduler config from manager.
func (d *dynconfig) GetScheduler() (*managerv2.Scheduler, error) {
	data, err := d.Get()
	if err != nil {
		return nil, err
	}

	if data.Scheduler == nil {
		return nil, errors.New("invalid scheduler")
	}

	return data.Scheduler, nil
}

// GetApplications returns the applications config from manager.
func (d *dynconfig) GetApplications() ([]*managerv2.Application, error) {
	data, err := d.Get()
	if err != nil {
		return nil, err
	}

	if data.Applications == nil {
		return nil, errors.New("invalid applications")
	}

	if len(data.Applications) == 0 {
		return nil, errors.New("application not found")
	}

	return data.Applications, nil
}

// GetSeedPeers returns the the seed peers config from manager.
func (d *dynconfig) GetSeedPeers() ([]*managerv2.SeedPeer, error) {
	scheduler, err := d.GetScheduler()
	if err != nil {
		return nil, err
	}

	if scheduler.SeedPeers == nil {
		return nil, errors.New("invalid seed peers")
	}

	if len(scheduler.SeedPeers) == 0 {
		return nil, errors.New("seed peer not found ")
	}

	return scheduler.SeedPeers, nil
}

// GetSchedulerCluster returns the the scheduler cluster config from manager.
func (d *dynconfig) GetSchedulerCluster() (*managerv2.SchedulerCluster, error) {
	scheduler, err := d.GetScheduler()
	if err != nil {
		return nil, err
	}

	if scheduler.SchedulerCluster == nil {
		return nil, errors.New("invalid scheduler cluster")
	}

	return scheduler.SchedulerCluster, nil
}

// GetSchedulerClusterConfig returns the scheduler cluster config.
func (d *dynconfig) GetSchedulerClusterConfig() (types.SchedulerClusterConfig, error) {
	schedulerCluster, err := d.GetSchedulerCluster()
	if err != nil {
		return types.SchedulerClusterConfig{}, err
	}

	var config types.SchedulerClusterConfig
	if err := json.Unmarshal(schedulerCluster.Config, &config); err != nil {
		return types.SchedulerClusterConfig{}, err
	}

	return config, nil
}

// GetSchedulerClusterClientConfig returns the client config.
func (d *dynconfig) GetSchedulerClusterClientConfig() (types.SchedulerClusterClientConfig, error) {
	schedulerCluster, err := d.GetSchedulerCluster()
	if err != nil {
		return types.SchedulerClusterClientConfig{}, err
	}

	var config types.SchedulerClusterClientConfig
	if err := json.Unmarshal(schedulerCluster.ClientConfig, &config); err != nil {
		return types.SchedulerClusterClientConfig{}, err
	}

	return config, nil
}

// Refresh refreshes dynconfig in cache.
func (d *dynconfig) Refresh() error {
	if err := d.Dynconfig.Refresh(); err != nil {
		return err
	}

	if err := d.Notify(); err != nil {
		return err
	}

	return nil
}

// Register allows an instance to register itself to listen/observe events.
func (d *dynconfig) Register(l Observer) {
	d.observers[l] = struct{}{}
}

// Deregister allows an instance to remove itself from the collection of observers/listeners.
func (d *dynconfig) Deregister(l Observer) {
	delete(d.observers, l)
}

// Notify publishes new events to listeners.
func (d *dynconfig) Notify() error {
	config, err := d.Get()
	if err != nil {
		return err
	}

	for o := range d.observers {
		o.OnNotify(config)
	}

	return nil
}

// Serve the dynconfig listening service.
func (d *dynconfig) Serve() error {
	if err := d.Notify(); err != nil {
		return err
	}

	tick := time.NewTicker(watchInterval)
	for {
		select {
		case <-tick.C:
			if err := d.Notify(); err != nil {
				logger.Error("dynconfig notify failed", err)
			}
		case <-d.done:
			return nil
		}
	}
}

// Stop the dynconfig listening service.
func (d *dynconfig) Stop() error {
	close(d.done)
	if err := os.Remove(d.cachePath); err != nil {
		return err
	}

	return nil
}

// Manager client for dynconfig.
type managerClient struct {
	managerClient managerclient.V2
	config        *Config
}

// New the manager client used by dynconfig.
func newManagerClient(client managerclient.V2, cfg *Config) dc.ManagerClient {
	return &managerClient{
		managerClient: client,
		config:        cfg,
	}
}

func (mc *managerClient) Get() (any, error) {
	getSchedulerResp, err := mc.managerClient.GetScheduler(context.Background(), &managerv2.GetSchedulerRequest{
		SourceType:         managerv2.SourceType_SCHEDULER_SOURCE,
		Hostname:           mc.config.Server.Host,
		Ip:                 mc.config.Server.AdvertiseIP.String(),
		SchedulerClusterId: uint64(mc.config.Manager.SchedulerClusterID),
	})
	if err != nil {
		return nil, err
	}

	listApplicationsResp, err := mc.managerClient.ListApplications(context.Background(), &managerv2.ListApplicationsRequest{
		SourceType: managerv2.SourceType_SCHEDULER_SOURCE,
		Hostname:   mc.config.Server.Host,
		Ip:         mc.config.Server.AdvertiseIP.String(),
	})
	if err != nil {
		if s, ok := status.FromError(err); ok {
			// TODO Compatible with old version manager.
			if slices.Contains([]codes.Code{codes.Unimplemented, codes.NotFound}, s.Code()) {
				return DynconfigData{
					Scheduler:    getSchedulerResp,
					Applications: nil,
				}, nil
			}
		}

		return nil, err
	}

	return DynconfigData{
		Scheduler:    getSchedulerResp,
		Applications: listApplicationsResp.Applications,
	}, nil
}

// GetSeedPeerClusterConfigBySeedPeer returns the seed peer cluster config by seed peer.
func GetSeedPeerClusterConfigBySeedPeer(seedPeer *managerv2.SeedPeer) (types.SeedPeerClusterConfig, error) {
	if seedPeer == nil {
		return types.SeedPeerClusterConfig{}, errors.New("invalid seed peer")
	}

	if seedPeer.SeedPeerCluster == nil {
		return types.SeedPeerClusterConfig{}, errors.New("invalid seed peer cluster")
	}

	var config types.SeedPeerClusterConfig
	if err := json.Unmarshal(seedPeer.SeedPeerCluster.Config, &config); err != nil {
		return types.SeedPeerClusterConfig{}, err
	}

	return config, nil
}