dragonfly/scheduler/core/monitor.go

/*
 *     Copyright 2020 The Dragonfly Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package core

import (
	"bytes"
	"fmt"
	"strconv"
	"strings"
	"time"

	"d7y.io/dragonfly/v2/scheduler/daemon"
	"d7y.io/dragonfly/v2/scheduler/types"
	"github.com/olekukonko/tablewriter"
	"go.uber.org/zap"
	"k8s.io/client-go/util/workqueue"
)

const (
	PeerGoneTimeout      = time.Second * 10
	PeerForceGoneTimeout = time.Minute * 2
)

type monitor struct {
	downloadMonitorQueue workqueue.DelayingInterface
	peerManager          daemon.PeerMgr
	done                 chan struct{}
	log                  *zap.SugaredLogger
}

func newMonitor(peerManager daemon.PeerMgr) *monitor {
	config := zap.NewDevelopmentConfig()
	logger, _ := config.Build()
	return &monitor{
		downloadMonitorQueue: workqueue.NewDelayingQueue(),
		peerManager:          peerManager,
		log:                  logger.Sugar(),
	}
}

func (m *monitor) start() {
	ticker := time.NewTicker(time.Second * 10)
	for {
		select {
		case <-ticker.C:
			m.log.Info(m.printDebugInfo())
		case <-m.done:
			return
		}
	}
}

func (m *monitor) stop() {
	close(m.done)
}

func (m *monitor) printDebugInfo() string {
	var roots []*types.Peer

	buffer := bytes.NewBuffer([]byte{})
	table := tablewriter.NewWriter(buffer)
	table.SetHeader([]string{"PeerID", "URL", "parent node", "status", "start time", "Finished Piece Num", "Finished", "Free Load"})
	m.peerManager.ListPeers().Range(func(key interface{}, value interface{}) (ok bool) {
		ok = true
		peer := value.(*types.Peer)
		if peer == nil {
			return
		}
		parentNode := ""
		if peer.GetParent() == nil {
			roots = append(roots, peer)
		} else {
			parentNode = peer.GetParent().PeerID
		}

		table.Append([]string{peer.PeerID, peer.Task.URL[len(peer.Task.URL)-15 : len(peer.Task.URL)], parentNode, peer.GetStatus().String(),
			peer.CreateTime.String(),
			strconv.Itoa(int(peer.GetFinishedNum())),
			strconv.FormatBool(peer.IsSuccess()), strconv.Itoa(peer.Host.GetFreeUploadLoad())})
		return
	})
	table.Render()

	var msgs []string
	msgs = append(msgs, buffer.String())

	var printTree func(node *types.Peer, path []string)
	printTree = func(node *types.Peer, path []string) {
		if node == nil {
			return
		}
		nPath := append(path, fmt.Sprintf("%s(%d)", node.PeerID, node.GetWholeTreeNode()))
		if len(path) >= 1 {
			msgs = append(msgs, node.PeerID+" || "+strings.Join(nPath, "-"))
		}
		node.GetChildren().Range(func(key, value interface{}) bool {
			child := (value).(*types.Peer)
			if child == nil {
				return true
			}
			printTree(child, nPath)
			return true
		})
	}

	for _, root := range roots {
		printTree(root, nil)
	}

	msg := "============\n" + strings.Join(append(msgs, strconv.Itoa(table.NumLines())), "\n") + "\n==============="
	return msg
}

func (m *monitor) RefreshDownloadMonitor(peer *types.Peer) {
	m.log.With("taskID", peer.Task.TaskID, "peerID", peer.PeerID).Debugf("downloadMonitorWorkingLoop refresh ")
	if !peer.IsRunning() {
		m.downloadMonitorQueue.AddAfter(peer, time.Second*2)
	} else if peer.IsBlocking() {
		m.downloadMonitorQueue.AddAfter(peer, time.Second*2)
	} else {
		delay := time.Millisecond * time.Duration(peer.GetCost()*10)
		if delay < time.Millisecond*20 {
			delay = time.Millisecond * 20
		}
		m.downloadMonitorQueue.AddAfter(peer, delay)
	}
}

// downloadMonitorWorkingLoop monitor peers download
func (m *monitor) downloadMonitorWorkingLoop() {
	for {
		v, shutdown := m.downloadMonitorQueue.Get()
		if shutdown {
			m.log.Infof("download monitor working loop closed")
			break
		}
		//if m.downloadMonitorCallBack != nil {
		peer := v.(*types.Peer)
		if peer != nil {
			m.log.With("taskID", peer.Task.TaskID, "peerID", peer.PeerID).Debugf("downloadMonitorWorkingLoop status[%d]", peer.GetStatus())
			if peer.IsSuccess() || peer.Host.CDN {
				// clear from monitor
			} else {
				if !peer.IsRunning() {
					// peer do not report for a long time, peer gone
					if time.Now().After(peer.GetLastAccessTime().Add(PeerGoneTimeout)) {
						peer.MarkLeave()
						//pt.SendError(dferrors.New(dfcodes.SchedPeerGone, "report time out"))
					}
					//m.downloadMonitorCallBack(peer)
				} else if !peer.IsBlocking() {
					//m.downloadMonitorCallBack(peer)
				} else {
					if time.Now().After(peer.GetLastAccessTime().Add(PeerForceGoneTimeout)) {
						peer.MarkLeave()
						//pt.SendError(dferrors.New(dfcodes.SchedPeerGone, "report fource time out"))
					}
					//m.downloadMonitorCallBack(peer)
				}
				//_, ok := m.Get(pt.GetPeerID())
				//status := pt.Status
				//if ok && !pt.Success && status != types.PeerStatusNodeGone && status != types.PeerStatusLeaveNode {
				//	m.RefreshDownloadMonitor(pt)
				//}
			}
		}
		//}

		m.downloadMonitorQueue.Done(v)
	}
}