dragonfly/pkg/rpc/dfdaemon/client/down_result_stream.go

/*
 *     Copyright 2020 The Dragonfly Authors
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package client

import (
	"context"

	"github.com/pkg/errors"
	"google.golang.org/grpc"
	"google.golang.org/grpc/codes"
	"google.golang.org/grpc/status"

	"d7y.io/dragonfly/v2/internal/dferrors"
	logger "d7y.io/dragonfly/v2/internal/dflog"
	"d7y.io/dragonfly/v2/pkg/rpc"
	"d7y.io/dragonfly/v2/pkg/rpc/dfdaemon"
)

type DownResultStream struct {
	dc      *daemonClient
	ctx     context.Context
	hashKey string
	req     *dfdaemon.DownRequest
	opts    []grpc.CallOption
	// stream for one client
	stream        dfdaemon.Daemon_DownloadClient
	failedServers []string
	rpc.RetryMeta
}

func newDownResultStream(ctx context.Context, dc *daemonClient, hashKey string, req *dfdaemon.DownRequest, opts []grpc.CallOption) (*DownResultStream, error) {
	drs := &DownResultStream{
		dc:      dc,
		ctx:     ctx,
		hashKey: hashKey,
		req:     req,
		opts:    opts,

		RetryMeta: rpc.RetryMeta{
			MaxAttempts: 3,
			MaxBackOff:  2.0,
			InitBackoff: 0.2,
		},
	}

	if err := drs.initStream(); err != nil {
		return nil, err
	}
	return drs, nil
}

func (drs *DownResultStream) initStream() error {
	var target string
	stream, err := rpc.ExecuteWithRetry(func() (interface{}, error) {
		var client dfdaemon.DaemonClient
		var err error
		client, target, err = drs.dc.getDaemonClient(drs.hashKey, false)
		if err != nil {
			return nil, err
		}
		return client.Download(drs.ctx, drs.req, drs.opts...)
	}, drs.InitBackoff, drs.MaxBackOff, drs.MaxAttempts, nil)
	if err != nil {
		if errors.Cause(err) == dferrors.ErrNoCandidateNode {
			return errors.Wrapf(err, "get grpc server instance failed")
		}
		logger.WithTaskID(drs.hashKey).Infof("initStream: invoke daemon node %s Download failed: %v", target, err)
		return drs.replaceClient(err)
	}
	drs.stream = stream.(dfdaemon.Daemon_DownloadClient)
	drs.StreamTimes = 1
	return nil
}

func (drs *DownResultStream) Recv() (dr *dfdaemon.DownResult, err error) {
	defer func() {
		if dr != nil {
			if dr.TaskId != drs.hashKey {
				logger.WithTaskAndPeerID(dr.TaskId, dr.PeerId).Warnf("down result stream correct taskId from %s to %s", drs.hashKey, dr.TaskId)
				drs.dc.Connection.CorrectKey2NodeRelation(drs.hashKey, dr.TaskId)
				drs.hashKey = dr.TaskId
			}
		}
	}()
	drs.dc.UpdateAccessNodeMapByHashKey(drs.hashKey)
	return drs.stream.Recv()
}

func (drs *DownResultStream) retryRecv(cause error) (*dfdaemon.DownResult, error) {
	if status.Code(cause) == codes.DeadlineExceeded || status.Code(cause) == codes.Canceled {
		return nil, cause
	}

	if err := drs.replaceStream(cause); err != nil {
		return nil, err
	}

	return drs.Recv()
}

func (drs *DownResultStream) replaceStream(cause error) error {
	if drs.StreamTimes >= drs.MaxAttempts {
		logger.WithTaskID(drs.hashKey).Info("replace stream reach max attempt")
		return cause
	}
	var target string
	stream, err := rpc.ExecuteWithRetry(func() (interface{}, error) {
		var client dfdaemon.DaemonClient
		var err error
		client, target, err = drs.dc.getDaemonClient(drs.hashKey, true)
		if err != nil {
			return nil, err
		}
		return client.Download(drs.ctx, drs.req, drs.opts...)
	}, drs.InitBackoff, drs.MaxBackOff, drs.MaxAttempts, cause)
	if err != nil {
		logger.WithTaskID(drs.hashKey).Infof("replaceStream: invoke daemon node %s Download failed: %v", target, err)
		return drs.replaceClient(cause)
	}
	drs.stream = stream.(dfdaemon.Daemon_DownloadClient)
	drs.StreamTimes++
	return nil
}

func (drs *DownResultStream) replaceClient(cause error) error {
	preNode, err := drs.dc.TryMigrate(drs.hashKey, cause, drs.failedServers)
	if err != nil {
		logger.WithTaskID(drs.hashKey).Infof("replaceClient: tryMigrate daemon node failed: %v", err)
		return cause
	}
	drs.failedServers = append(drs.failedServers, preNode)

	var target string
	stream, err := rpc.ExecuteWithRetry(func() (interface{}, error) {
		var client dfdaemon.DaemonClient
		var err error
		client, target, err = drs.dc.getDaemonClient(drs.hashKey, true)
		if err != nil {
			return nil, err
		}
		return client.Download(drs.ctx, drs.req, drs.opts...)
	}, drs.InitBackoff, drs.MaxBackOff, drs.MaxAttempts, cause)
	if err != nil {
		logger.WithTaskID(drs.hashKey).Infof("replaceClient: invoke daemon node %s Download failed: %v", target, err)
		return drs.replaceClient(cause)
	}
	drs.stream = stream.(dfdaemon.Daemon_DownloadClient)
	drs.StreamTimes = 1
	return nil
}