chore: update UnknownSourceFileLen (#888)

Signed-off-by: Jim Ma <majinjing3@gmail.com>
This commit is contained in:
Jim Ma 2021-12-07 18:53:06 +08:00 committed by Gaius
parent 18e7822b51
commit ff3a542fde
No known key found for this signature in database
GPG Key ID: 8B4E5D1290FA2FFB
8 changed files with 17 additions and 17 deletions

View File

@ -202,14 +202,14 @@ func (tm *Manager) AddOrUpdate(registerTask *types.SeedTask) (seedTask *types.Se
synclock.UnLock(registerTask.TaskID, true)
return nil, errors.Wrapf(errTaskIDConflict, "register task %#v is conflict with exist task %#v", registerTask, seedTask)
}
if seedTask.SourceFileLength != source.UnKnownSourceFileLen {
if seedTask.SourceFileLength != source.UnknownSourceFileLen {
synclock.UnLock(registerTask.TaskID, true)
return seedTask, nil
}
synclock.UnLock(registerTask.TaskID, true)
synclock.Lock(registerTask.TaskID, false)
defer synclock.UnLock(registerTask.TaskID, false)
if seedTask.SourceFileLength != source.UnKnownSourceFileLen {
if seedTask.SourceFileLength != source.UnknownSourceFileLen {
return seedTask, nil
}
// get sourceContentLength with req.Header

View File

@ -56,7 +56,7 @@ func NewSeedTask(taskID string, rawURL string, urlMeta *base.UrlMeta) *SeedTask
RequestDigest: urlMeta.Digest,
URL: rawURL,
TaskURL: urlutils.FilterURLParam(rawURL, strings.Split(urlMeta.Filter, "&")),
SourceFileLength: source.UnKnownSourceFileLen,
SourceFileLength: source.UnknownSourceFileLen,
CdnFileLength: 0,
PieceSize: 0,
Range: urlMeta.Range,

View File

@ -355,7 +355,7 @@ func (pm *pieceManager) DownloadSource(ctx context.Context, pt Task, request *sc
if err != nil {
log.Warnf("get content length error: %s for %s", err, request.Url)
}
if contentLength == -1 {
if contentLength < 0 {
log.Warnf("can not get content length for %s", request.Url)
} else {
err = pm.storageManager.UpdateTask(ctx,
@ -391,7 +391,7 @@ func (pm *pieceManager) DownloadSource(ctx context.Context, pt Task, request *sc
// 2. save to storage
pieceSize := pm.computePieceSize(contentLength)
// handle resource which content length is unknown
if contentLength == -1 {
if contentLength < 0 {
var n int64
for pieceNum := int32(0); ; pieceNum++ {
size := pieceSize

View File

@ -80,11 +80,11 @@ type HDFSSourceClientOption func(p *hdfsSourceClient)
func (h *hdfsSourceClient) GetContentLength(request *source.Request) (int64, error) {
hdfsClient, path, err := h.getHDFSClientAndPath(request.URL)
if err != nil {
return source.UnKnownSourceFileLen, err
return source.UnknownSourceFileLen, err
}
info, err := hdfsClient.Stat(path)
if err != nil {
return source.UnKnownSourceFileLen, err
return source.UnknownSourceFileLen, err
}
return info.Size(), nil
}

View File

@ -51,7 +51,7 @@ const (
const (
hdfsNotExistFileURL = "hdfs://127.0.0.1:9000/user/root/input/f3.txt"
hdfsNotExistFileContentLength int64 = source.UnKnownSourceFileLen
hdfsNotExistFileContentLength int64 = source.UnknownSourceFileLen
)
var fakeHDFSClient = &hdfs.Client{}

View File

@ -127,12 +127,12 @@ func WithHTTPClient(client *http.Client) HTTPSourceClientOption {
func (client *httpSourceClient) GetContentLength(request *source.Request) (int64, error) {
resp, err := client.doRequest(http.MethodGet, request)
if err != nil {
return source.UnKnownSourceFileLen, err
return source.UnknownSourceFileLen, err
}
defer resp.Body.Close()
err = source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent})
if err != nil {
return source.UnKnownSourceFileLen, err
return source.UnknownSourceFileLen, err
}
return resp.ContentLength, nil
}

View File

@ -92,19 +92,19 @@ type ossSourceClient struct {
func (osc *ossSourceClient) GetContentLength(request *source.Request) (int64, error) {
client, err := osc.getClient(request.Header)
if err != nil {
return source.UnKnownSourceFileLen, err
return source.UnknownSourceFileLen, err
}
bucket, err := client.Bucket(request.URL.Host)
if err != nil {
return source.UnKnownSourceFileLen, errors.Wrapf(err, "get oss bucket: %s", request.URL.Host)
return source.UnknownSourceFileLen, errors.Wrapf(err, "get oss bucket: %s", request.URL.Host)
}
header, err := bucket.GetObjectMeta(request.URL.Path, getOptions(request.Header)...)
if err != nil {
return source.UnKnownSourceFileLen, errors.Wrapf(err, "get oss object %s meta", request.URL.Path)
return source.UnknownSourceFileLen, errors.Wrapf(err, "get oss object %s meta", request.URL.Path)
}
contentLen, err := strconv.ParseInt(header.Get(oss.HTTPHeaderContentLength), 10, 64)
if err != nil {
return source.UnKnownSourceFileLen, errors.Wrapf(err, "parse content-length str to int64")
return source.UnknownSourceFileLen, errors.Wrapf(err, "parse content-length str to int64")
}
return contentLen, nil
}

View File

@ -82,14 +82,14 @@ func IsNoClientFoundError(err error) bool {
}
const (
UnKnownSourceFileLen = -2
UnknownSourceFileLen = -2
)
// ResourceClient defines the API interface to interact with source.
type ResourceClient interface {
// GetContentLength get length of resource content
// return source.UnKnownSourceFileLen if response status is not StatusOK and StatusPartialContent
// return source.UnknownSourceFileLen if response status is not StatusOK and StatusPartialContent
GetContentLength(request *Request) (int64, error)
// IsSupportRange checks if resource supports breakpoint continuation
@ -218,7 +218,7 @@ func (c *clientWrapper) GetLastModified(request *Request) (int64, error) {
func GetContentLength(request *Request) (int64, error) {
client, ok := _defaultManager.GetClient(request.URL.Scheme)
if !ok {
return UnKnownSourceFileLen, errors.Wrapf(ErrNoClientFound, "scheme: %s", request.URL.Scheme)
return UnknownSourceFileLen, errors.Wrapf(ErrNoClientFound, "scheme: %s", request.URL.Scheme)
}
if _, ok := request.Context().Deadline(); !ok {
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)