feat: update source temporary error logic (#1739)

Signed-off-by: Jim Ma <majinjing3@gmail.com>
This commit is contained in:
Jim Ma 2022-10-10 20:19:24 +08:00 committed by Gaius
parent ef61df0027
commit 2d46f6424a
No known key found for this signature in database
GPG Key ID: 8B4E5D1290FA2FFB
5 changed files with 24 additions and 28 deletions

View File

@ -385,7 +385,7 @@ singleDownload:
}
}
srcErr := &errordetailsv1.SourceError{
Temporary: response.Temporary == nil || response.Temporary(),
Temporary: response.Temporary,
Metadata: &commonv1.ExtendAttribute{
Header: hdr,
StatusCode: int32(response.StatusCode),

View File

@ -52,7 +52,12 @@ var (
contentRangeRegexp = regexp.MustCompile(`bytes (?P<Start>\d+)-(?P<End>\d+)/(?P<Length>(\d*|\*))`)
contentRangeRegexpLengthIndex = contentRangeRegexp.SubexpIndex("Length")
notTemporaryStatusCode = []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusProxyAuthRequired}
notTemporaryStatusCode = []int{
http.StatusUnauthorized,
http.StatusForbidden,
http.StatusNotFound,
http.StatusProxyAuthRequired,
}
)
func init() {
@ -218,14 +223,7 @@ func (client *httpSourceClient) GetMetadata(request *source.Request) (*source.Me
Validate: func() error {
return source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent})
},
Temporary: func() bool {
for _, code := range notTemporaryStatusCode {
if code == resp.StatusCode {
return false
}
}
return false
},
Temporary: detectTemporary(resp.StatusCode),
}, nil
}
@ -261,14 +259,7 @@ func (client *httpSourceClient) Download(request *source.Request) (*source.Respo
source.WithValidate(func() error {
return source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent})
}),
source.WithTemporary(func() bool {
for _, code := range notTemporaryStatusCode {
if code == resp.StatusCode {
return false
}
}
return true
}),
source.WithTemporary(detectTemporary(resp.StatusCode)),
source.WithHeader(exportPassThroughHeader(resp.Header)),
source.WithExpireInfo(
source.ExpireInfo{
@ -336,3 +327,12 @@ func exportPassThroughHeader(header http.Header) map[string]string {
}
return ph
}
func detectTemporary(statusCode int) bool {
for _, code := range notTemporaryStatusCode {
if code == statusCode {
return false
}
}
return true
}

View File

@ -173,9 +173,7 @@ func (osc *ossSourceClient) GetMetadata(request *source.Request) (*source.Metada
Validate: func() error {
return nil
},
Temporary: func() bool {
return true
},
Temporary: true,
}, nil
}

View File

@ -31,5 +31,5 @@ type Metadata struct {
TotalContentLength int64
Validate func() error
Temporary func() bool
Temporary bool
}

View File

@ -35,8 +35,8 @@ type Response struct {
// Validate this response is okay to transfer in p2p network, like status 200 or 206 in http is valid to do this,
// otherwise return status code to original client
Validate func() error
// Temporary check the error whether the error is temporary, if is true, we can retry it later
Temporary func() bool
// Temporary indecates the error whether the error is temporary, if is true, we can retry it later
Temporary bool
}
func NewResponse(rc io.ReadCloser, opts ...func(*Response)) *Response {
@ -54,9 +54,7 @@ func NewResponse(rc io.ReadCloser, opts ...func(*Response)) *Response {
Validate: func() error {
return nil
},
Temporary: func() bool {
return true
},
Temporary: true,
}
for _, opt := range opts {
@ -103,7 +101,7 @@ func WithValidate(validate func() error) func(*Response) {
}
}
func WithTemporary(temporary func() bool) func(*Response) {
func WithTemporary(temporary bool) func(*Response) {
return func(resp *Response) {
resp.Temporary = temporary
}