209 lines
6.9 KiB
Go
209 lines
6.9 KiB
Go
/*
|
|
* Copyright 2020 The Dragonfly Authors
|
|
*
|
|
* Licensed under the Apache License, Version 2.0 (the "License");
|
|
* you may not use this file except in compliance with the License.
|
|
* You may obtain a copy of the License at
|
|
*
|
|
* http://www.apache.org/licenses/LICENSE-2.0
|
|
*
|
|
* Unless required by applicable law or agreed to in writing, software
|
|
* distributed under the License is distributed on an "AS IS" BASIS,
|
|
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
* See the License for the specific language governing permissions and
|
|
* limitations under the License.
|
|
*/
|
|
|
|
package task
|
|
|
|
import (
|
|
"context"
|
|
"fmt"
|
|
"reflect"
|
|
"time"
|
|
|
|
"github.com/pkg/errors"
|
|
"go.opentelemetry.io/otel/trace"
|
|
|
|
"d7y.io/dragonfly/v2/cdn/cdnutil"
|
|
"d7y.io/dragonfly/v2/cdn/config"
|
|
cdnerrors "d7y.io/dragonfly/v2/cdn/errors"
|
|
"d7y.io/dragonfly/v2/cdn/types"
|
|
logger "d7y.io/dragonfly/v2/internal/dflog"
|
|
"d7y.io/dragonfly/v2/pkg/source"
|
|
"d7y.io/dragonfly/v2/pkg/synclock"
|
|
"d7y.io/dragonfly/v2/pkg/util/net/urlutils"
|
|
"d7y.io/dragonfly/v2/pkg/util/stringutils"
|
|
)
|
|
|
|
// addOrUpdateTask add a new task or update exist task
|
|
func (tm *Manager) addOrUpdateTask(ctx context.Context, request *types.TaskRegisterRequest) (*types.SeedTask, error) {
|
|
var span trace.Span
|
|
ctx, span = tracer.Start(ctx, config.SpanAndOrUpdateTask)
|
|
defer span.End()
|
|
taskURL := request.URL
|
|
if request.Filter != nil {
|
|
taskURL = urlutils.FilterURLParam(request.URL, request.Filter)
|
|
}
|
|
span.SetAttributes(config.AttributeTaskURL.String(taskURL))
|
|
taskID := request.TaskID
|
|
synclock.Lock(taskID, false)
|
|
defer synclock.UnLock(taskID, false)
|
|
if key, err := tm.taskURLUnReachableStore.Get(taskID); err == nil {
|
|
if unReachableStartTime, ok := key.(time.Time); ok && time.Since(unReachableStartTime) < tm.cfg.FailAccessInterval {
|
|
existTask, err := tm.taskStore.Get(taskID)
|
|
if err != nil || reflect.DeepEqual(request.Header, existTask.(*types.SeedTask).Header) {
|
|
span.AddEvent(config.EventHitUnReachableURL)
|
|
return nil, errors.Wrapf(cdnerrors.ErrURLNotReachable{URL: request.URL}, "task hit unReachable cache and interval less than %d, "+
|
|
"url: %s", tm.cfg.FailAccessInterval, request.URL)
|
|
}
|
|
}
|
|
span.AddEvent(config.EventDeleteUnReachableTask)
|
|
tm.taskURLUnReachableStore.Delete(taskID)
|
|
logger.Debugf("delete taskID: %s from url unReachable store", taskID)
|
|
}
|
|
|
|
var task *types.SeedTask
|
|
newTask := types.NewSeedTask(taskID, request.Header, request.Digest, request.URL, taskURL)
|
|
// using the existing task if it already exists corresponding to taskID
|
|
if v, err := tm.taskStore.Get(taskID); err == nil {
|
|
span.SetAttributes(config.AttributeIfReuseTask.Bool(true))
|
|
existTask := v.(*types.SeedTask)
|
|
if !isSameTask(existTask, newTask) {
|
|
span.RecordError(fmt.Errorf("newTask: %+v, existTask: %+v", newTask, existTask))
|
|
return nil, cdnerrors.ErrTaskIDDuplicate{TaskID: taskID, Cause: fmt.Errorf("newTask: %+v, existTask: %+v", newTask, existTask)}
|
|
}
|
|
task = existTask
|
|
logger.Debugf("get exist task for taskID: %s", taskID)
|
|
} else {
|
|
span.SetAttributes(config.AttributeIfReuseTask.Bool(false))
|
|
logger.Debugf("get new task for taskID: %s", taskID)
|
|
task = newTask
|
|
}
|
|
|
|
if task.SourceFileLength != types.IllegalSourceFileLen {
|
|
return task, nil
|
|
}
|
|
|
|
// get sourceContentLength with req.Header
|
|
ctx, cancel := context.WithTimeout(ctx, 4*time.Second)
|
|
defer cancel()
|
|
span.AddEvent(config.EventRequestSourceFileLength)
|
|
sourceFileLength, err := source.GetContentLength(ctx, task.URL, request.Header)
|
|
if err != nil {
|
|
task.Log().Errorf("failed to get url (%s) content length: %v", task.URL, err)
|
|
if cdnerrors.IsURLNotReachable(err) {
|
|
if err := tm.taskURLUnReachableStore.Add(taskID, time.Now()); err != nil {
|
|
task.Log().Errorf("failed to add url (%s) to unreachable store: %v", task.URL, err)
|
|
}
|
|
return nil, err
|
|
}
|
|
}
|
|
// if not support file length header request ,return -1
|
|
task.SourceFileLength = sourceFileLength
|
|
logger.WithTaskID(taskID).Debugf("get file content length: %d", sourceFileLength)
|
|
if task.SourceFileLength > 0 {
|
|
ok, err := tm.cdnMgr.TryFreeSpace(task.SourceFileLength)
|
|
if err != nil {
|
|
logger.Errorf("failed to try free space: %v", err)
|
|
} else if !ok {
|
|
return nil, cdnerrors.ErrResourcesLacked
|
|
}
|
|
}
|
|
|
|
// if success to get the information successfully with the req.Header then update the task.Header to req.Header.
|
|
if request.Header != nil {
|
|
task.Header = request.Header
|
|
}
|
|
|
|
// calculate piece size and update the PieceSize and PieceTotal
|
|
if task.PieceSize <= 0 {
|
|
pieceSize := cdnutil.ComputePieceSize(task.SourceFileLength)
|
|
task.PieceSize = pieceSize
|
|
}
|
|
if err := tm.taskStore.Add(task.TaskID, task); err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
logger.Debugf("success add task: %+v into taskStore", task)
|
|
return task, nil
|
|
}
|
|
|
|
// updateTask
|
|
func (tm *Manager) updateTask(taskID string, updateTaskInfo *types.SeedTask) (*types.SeedTask, error) {
|
|
if stringutils.IsBlank(taskID) {
|
|
return nil, errors.Wrap(cdnerrors.ErrInvalidValue, "taskID is empty")
|
|
}
|
|
|
|
if updateTaskInfo == nil {
|
|
return nil, errors.Wrap(cdnerrors.ErrInvalidValue, "updateTaskInfo is nil")
|
|
}
|
|
|
|
if stringutils.IsBlank(updateTaskInfo.CdnStatus) {
|
|
return nil, errors.Wrap(cdnerrors.ErrInvalidValue, "status of task is empty")
|
|
}
|
|
// get origin task
|
|
task, err := tm.getTask(taskID)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
if !updateTaskInfo.IsSuccess() {
|
|
// when the origin CDNStatus equals success, do not update it to unsuccessful
|
|
if task.IsSuccess() {
|
|
return task, nil
|
|
}
|
|
|
|
// only update the task CdnStatus when the new task CDNStatus and
|
|
// the origin CDNStatus both not equals success
|
|
task.CdnStatus = updateTaskInfo.CdnStatus
|
|
return task, nil
|
|
}
|
|
|
|
// only update the task info when the new CDNStatus equals success
|
|
// and the origin CDNStatus not equals success.
|
|
if updateTaskInfo.CdnFileLength != 0 {
|
|
task.CdnFileLength = updateTaskInfo.CdnFileLength
|
|
}
|
|
|
|
if !stringutils.IsBlank(updateTaskInfo.SourceRealDigest) {
|
|
task.SourceRealDigest = updateTaskInfo.SourceRealDigest
|
|
}
|
|
|
|
if !stringutils.IsBlank(updateTaskInfo.PieceMd5Sign) {
|
|
task.PieceMd5Sign = updateTaskInfo.PieceMd5Sign
|
|
}
|
|
var pieceTotal int32
|
|
if updateTaskInfo.SourceFileLength > 0 {
|
|
pieceTotal = int32((updateTaskInfo.SourceFileLength + int64(task.PieceSize-1)) / int64(task.PieceSize))
|
|
task.SourceFileLength = updateTaskInfo.SourceFileLength
|
|
}
|
|
if pieceTotal != 0 {
|
|
task.PieceTotal = pieceTotal
|
|
}
|
|
task.CdnStatus = updateTaskInfo.CdnStatus
|
|
return task, nil
|
|
}
|
|
|
|
// isSameTask check whether the two task provided are the same
|
|
func isSameTask(task1, task2 *types.SeedTask) bool {
|
|
if task1 == task2 {
|
|
return true
|
|
}
|
|
if task1.TaskURL != task2.TaskURL {
|
|
return false
|
|
}
|
|
|
|
if !stringutils.IsBlank(task1.RequestDigest) && !stringutils.IsBlank(task2.RequestDigest) {
|
|
if task1.RequestDigest != task2.RequestDigest {
|
|
return false
|
|
}
|
|
}
|
|
|
|
if !stringutils.IsBlank(task1.RequestDigest) && !stringutils.IsBlank(task2.SourceRealDigest) {
|
|
return task1.SourceRealDigest == task2.RequestDigest
|
|
}
|
|
|
|
return true
|
|
}
|