dragonflyoss
/
dragonfly
mirror of https://github.com/dragonflyoss/dragonfly.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

chore: update UnknownSourceFileLen (#888) 

Signed-off-by: Jim Ma <majinjing3@gmail.com>
This commit is contained in:
Jim Ma 2021-12-07 18:53:06 +08:00 committed by Gaius
parent 18e7822b51
commit ff3a542fde
No known key found for this signature in database
GPG Key ID: 8B4E5D1290FA2FFB
8 changed files with 17 additions and 17 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
cdn/supervisor/task/manager.go
View File

@ -202,14 +202,14 @@ func (tm *Manager) AddOrUpdate(registerTask *types.SeedTask) (seedTask *types.Se
synclock.UnLock(registerTask.TaskID, true) synclock.UnLock(registerTask.TaskID, true)
return nil, errors.Wrapf(errTaskIDConflict, "register task %#v is conflict with exist task %#v", registerTask, seedTask) return nil, errors.Wrapf(errTaskIDConflict, "register task %#v is conflict with exist task %#v", registerTask, seedTask)
} }
if seedTask.SourceFileLength != source.UnKnownSourceFileLen { if seedTask.SourceFileLength != source.UnknownSourceFileLen {
synclock.UnLock(registerTask.TaskID, true) synclock.UnLock(registerTask.TaskID, true)
return seedTask, nil return seedTask, nil
} }
synclock.UnLock(registerTask.TaskID, true) synclock.UnLock(registerTask.TaskID, true)
synclock.Lock(registerTask.TaskID, false) synclock.Lock(registerTask.TaskID, false)
defer synclock.UnLock(registerTask.TaskID, false) defer synclock.UnLock(registerTask.TaskID, false)
if seedTask.SourceFileLength != source.UnKnownSourceFileLen { if seedTask.SourceFileLength != source.UnknownSourceFileLen {
return seedTask, nil return seedTask, nil
} }
// get sourceContentLength with req.Header // get sourceContentLength with req.Header

2
cdn/types/seed_task_info.go
View File

@ -56,7 +56,7 @@ func NewSeedTask(taskID string, rawURL string, urlMeta *base.UrlMeta) *SeedTask
RequestDigest: urlMeta.Digest, RequestDigest: urlMeta.Digest,
URL: rawURL, URL: rawURL,
TaskURL: urlutils.FilterURLParam(rawURL, strings.Split(urlMeta.Filter, "&")), TaskURL: urlutils.FilterURLParam(rawURL, strings.Split(urlMeta.Filter, "&")),
SourceFileLength: source.UnKnownSourceFileLen, SourceFileLength: source.UnknownSourceFileLen,
CdnFileLength: 0, CdnFileLength: 0,
PieceSize: 0, PieceSize: 0,
Range: urlMeta.Range, Range: urlMeta.Range,

4
client/daemon/peer/piece_manager.go
View File

@ -355,7 +355,7 @@ func (pm *pieceManager) DownloadSource(ctx context.Context, pt Task, request *sc
if err != nil { if err != nil {
log.Warnf("get content length error: %s for %s", err, request.Url) log.Warnf("get content length error: %s for %s", err, request.Url)
} }
if contentLength == -1 { if contentLength < 0 {
log.Warnf("can not get content length for %s", request.Url) log.Warnf("can not get content length for %s", request.Url)
} else { } else {
err = pm.storageManager.UpdateTask(ctx, err = pm.storageManager.UpdateTask(ctx,
@ -391,7 +391,7 @@ func (pm *pieceManager) DownloadSource(ctx context.Context, pt Task, request *sc
// 2. save to storage // 2. save to storage
pieceSize := pm.computePieceSize(contentLength) pieceSize := pm.computePieceSize(contentLength)
// handle resource which content length is unknown // handle resource which content length is unknown
if contentLength == -1 { if contentLength < 0 {
var n int64 var n int64
for pieceNum := int32(0); ; pieceNum++ { for pieceNum := int32(0); ; pieceNum++ {
size := pieceSize size := pieceSize

4
pkg/source/hdfsprotocol/hdfs_source_client.go
View File

@ -80,11 +80,11 @@ type HDFSSourceClientOption func(p *hdfsSourceClient)
func (h *hdfsSourceClient) GetContentLength(request *source.Request) (int64, error) { func (h *hdfsSourceClient) GetContentLength(request *source.Request) (int64, error) {
hdfsClient, path, err := h.getHDFSClientAndPath(request.URL) hdfsClient, path, err := h.getHDFSClientAndPath(request.URL)
if err != nil { if err != nil {
return source.UnKnownSourceFileLen, err return source.UnknownSourceFileLen, err
} }
info, err := hdfsClient.Stat(path) info, err := hdfsClient.Stat(path)
if err != nil { if err != nil {
return source.UnKnownSourceFileLen, err return source.UnknownSourceFileLen, err
} }
return info.Size(), nil return info.Size(), nil
} }

2
pkg/source/hdfsprotocol/hdfs_source_client_test.go
View File

@ -51,7 +51,7 @@ const (
const ( const (
hdfsNotExistFileURL = "hdfs://127.0.0.1:9000/user/root/input/f3.txt" hdfsNotExistFileURL = "hdfs://127.0.0.1:9000/user/root/input/f3.txt"
hdfsNotExistFileContentLength int64 = source.UnKnownSourceFileLen hdfsNotExistFileContentLength int64 = source.UnknownSourceFileLen
) )
var fakeHDFSClient = &hdfs.Client{} var fakeHDFSClient = &hdfs.Client{}

4
pkg/source/httpprotocol/http_source_client.go
View File

@ -127,12 +127,12 @@ func WithHTTPClient(client *http.Client) HTTPSourceClientOption {
func (client *httpSourceClient) GetContentLength(request *source.Request) (int64, error) { func (client *httpSourceClient) GetContentLength(request *source.Request) (int64, error) {
resp, err := client.doRequest(http.MethodGet, request) resp, err := client.doRequest(http.MethodGet, request)
if err != nil { if err != nil {
return source.UnKnownSourceFileLen, err return source.UnknownSourceFileLen, err
} }
defer resp.Body.Close() defer resp.Body.Close()
err = source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent}) err = source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent})
if err != nil { if err != nil {
return source.UnKnownSourceFileLen, err return source.UnknownSourceFileLen, err
} }
return resp.ContentLength, nil return resp.ContentLength, nil
} }

8
pkg/source/ossprotocol/oss_source_client.go
View File

@ -92,19 +92,19 @@ type ossSourceClient struct {
func (osc *ossSourceClient) GetContentLength(request *source.Request) (int64, error) { func (osc *ossSourceClient) GetContentLength(request *source.Request) (int64, error) {
client, err := osc.getClient(request.Header) client, err := osc.getClient(request.Header)
if err != nil { if err != nil {
return source.UnKnownSourceFileLen, err return source.UnknownSourceFileLen, err
} }
bucket, err := client.Bucket(request.URL.Host) bucket, err := client.Bucket(request.URL.Host)
if err != nil { if err != nil {
return source.UnKnownSourceFileLen, errors.Wrapf(err, "get oss bucket: %s", request.URL.Host) return source.UnknownSourceFileLen, errors.Wrapf(err, "get oss bucket: %s", request.URL.Host)
} }
header, err := bucket.GetObjectMeta(request.URL.Path, getOptions(request.Header)...) header, err := bucket.GetObjectMeta(request.URL.Path, getOptions(request.Header)...)
if err != nil { if err != nil {
return source.UnKnownSourceFileLen, errors.Wrapf(err, "get oss object %s meta", request.URL.Path) return source.UnknownSourceFileLen, errors.Wrapf(err, "get oss object %s meta", request.URL.Path)
} }
contentLen, err := strconv.ParseInt(header.Get(oss.HTTPHeaderContentLength), 10, 64) contentLen, err := strconv.ParseInt(header.Get(oss.HTTPHeaderContentLength), 10, 64)
if err != nil { if err != nil {
return source.UnKnownSourceFileLen, errors.Wrapf(err, "parse content-length str to int64") return source.UnknownSourceFileLen, errors.Wrapf(err, "parse content-length str to int64")
} }
return contentLen, nil return contentLen, nil
} }

6
pkg/source/source_client.go
View File

@ -82,14 +82,14 @@ func IsNoClientFoundError(err error) bool {
} }
const ( const (
UnKnownSourceFileLen = -2 UnknownSourceFileLen = -2
) )
// ResourceClient defines the API interface to interact with source. // ResourceClient defines the API interface to interact with source.
type ResourceClient interface { type ResourceClient interface {
// GetContentLength get length of resource content // GetContentLength get length of resource content
// return source.UnKnownSourceFileLen if response status is not StatusOK and StatusPartialContent // return source.UnknownSourceFileLen if response status is not StatusOK and StatusPartialContent
GetContentLength(request *Request) (int64, error) GetContentLength(request *Request) (int64, error)
// IsSupportRange checks if resource supports breakpoint continuation // IsSupportRange checks if resource supports breakpoint continuation
@ -218,7 +218,7 @@ func (c *clientWrapper) GetLastModified(request *Request) (int64, error) {
func GetContentLength(request *Request) (int64, error) { func GetContentLength(request *Request) (int64, error) {
client, ok := _defaultManager.GetClient(request.URL.Scheme) client, ok := _defaultManager.GetClient(request.URL.Scheme)
if !ok { if !ok {
return UnKnownSourceFileLen, errors.Wrapf(ErrNoClientFound, "scheme: %s", request.URL.Scheme) return UnknownSourceFileLen, errors.Wrapf(ErrNoClientFound, "scheme: %s", request.URL.Scheme)
} }
if _, ok := request.Context().Deadline(); !ok { if _, ok := request.Context().Deadline(); !ok {
ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second) ctx, cancel := context.WithTimeout(context.Background(), 4*time.Second)