feat: update source temporary error logic (#1739)
Signed-off-by: Jim Ma <majinjing3@gmail.com>
This commit is contained in:
parent
ef61df0027
commit
2d46f6424a
|
|
@ -385,7 +385,7 @@ singleDownload:
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
srcErr := &errordetailsv1.SourceError{
|
srcErr := &errordetailsv1.SourceError{
|
||||||
Temporary: response.Temporary == nil || response.Temporary(),
|
Temporary: response.Temporary,
|
||||||
Metadata: &commonv1.ExtendAttribute{
|
Metadata: &commonv1.ExtendAttribute{
|
||||||
Header: hdr,
|
Header: hdr,
|
||||||
StatusCode: int32(response.StatusCode),
|
StatusCode: int32(response.StatusCode),
|
||||||
|
|
|
||||||
|
|
@ -52,7 +52,12 @@ var (
|
||||||
contentRangeRegexp = regexp.MustCompile(`bytes (?P<Start>\d+)-(?P<End>\d+)/(?P<Length>(\d*|\*))`)
|
contentRangeRegexp = regexp.MustCompile(`bytes (?P<Start>\d+)-(?P<End>\d+)/(?P<Length>(\d*|\*))`)
|
||||||
contentRangeRegexpLengthIndex = contentRangeRegexp.SubexpIndex("Length")
|
contentRangeRegexpLengthIndex = contentRangeRegexp.SubexpIndex("Length")
|
||||||
|
|
||||||
notTemporaryStatusCode = []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusProxyAuthRequired}
|
notTemporaryStatusCode = []int{
|
||||||
|
http.StatusUnauthorized,
|
||||||
|
http.StatusForbidden,
|
||||||
|
http.StatusNotFound,
|
||||||
|
http.StatusProxyAuthRequired,
|
||||||
|
}
|
||||||
)
|
)
|
||||||
|
|
||||||
func init() {
|
func init() {
|
||||||
|
|
@ -218,14 +223,7 @@ func (client *httpSourceClient) GetMetadata(request *source.Request) (*source.Me
|
||||||
Validate: func() error {
|
Validate: func() error {
|
||||||
return source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent})
|
return source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent})
|
||||||
},
|
},
|
||||||
Temporary: func() bool {
|
Temporary: detectTemporary(resp.StatusCode),
|
||||||
for _, code := range notTemporaryStatusCode {
|
|
||||||
if code == resp.StatusCode {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return false
|
|
||||||
},
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -261,14 +259,7 @@ func (client *httpSourceClient) Download(request *source.Request) (*source.Respo
|
||||||
source.WithValidate(func() error {
|
source.WithValidate(func() error {
|
||||||
return source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent})
|
return source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent})
|
||||||
}),
|
}),
|
||||||
source.WithTemporary(func() bool {
|
source.WithTemporary(detectTemporary(resp.StatusCode)),
|
||||||
for _, code := range notTemporaryStatusCode {
|
|
||||||
if code == resp.StatusCode {
|
|
||||||
return false
|
|
||||||
}
|
|
||||||
}
|
|
||||||
return true
|
|
||||||
}),
|
|
||||||
source.WithHeader(exportPassThroughHeader(resp.Header)),
|
source.WithHeader(exportPassThroughHeader(resp.Header)),
|
||||||
source.WithExpireInfo(
|
source.WithExpireInfo(
|
||||||
source.ExpireInfo{
|
source.ExpireInfo{
|
||||||
|
|
@ -336,3 +327,12 @@ func exportPassThroughHeader(header http.Header) map[string]string {
|
||||||
}
|
}
|
||||||
return ph
|
return ph
|
||||||
}
|
}
|
||||||
|
|
||||||
|
func detectTemporary(statusCode int) bool {
|
||||||
|
for _, code := range notTemporaryStatusCode {
|
||||||
|
if code == statusCode {
|
||||||
|
return false
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return true
|
||||||
|
}
|
||||||
|
|
|
||||||
|
|
@ -173,9 +173,7 @@ func (osc *ossSourceClient) GetMetadata(request *source.Request) (*source.Metada
|
||||||
Validate: func() error {
|
Validate: func() error {
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
Temporary: func() bool {
|
Temporary: true,
|
||||||
return true
|
|
||||||
},
|
|
||||||
}, nil
|
}, nil
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
|
||||||
|
|
@ -31,5 +31,5 @@ type Metadata struct {
|
||||||
TotalContentLength int64
|
TotalContentLength int64
|
||||||
|
|
||||||
Validate func() error
|
Validate func() error
|
||||||
Temporary func() bool
|
Temporary bool
|
||||||
}
|
}
|
||||||
|
|
|
||||||
|
|
@ -35,8 +35,8 @@ type Response struct {
|
||||||
// Validate this response is okay to transfer in p2p network, like status 200 or 206 in http is valid to do this,
|
// Validate this response is okay to transfer in p2p network, like status 200 or 206 in http is valid to do this,
|
||||||
// otherwise return status code to original client
|
// otherwise return status code to original client
|
||||||
Validate func() error
|
Validate func() error
|
||||||
// Temporary check the error whether the error is temporary, if is true, we can retry it later
|
// Temporary indecates the error whether the error is temporary, if is true, we can retry it later
|
||||||
Temporary func() bool
|
Temporary bool
|
||||||
}
|
}
|
||||||
|
|
||||||
func NewResponse(rc io.ReadCloser, opts ...func(*Response)) *Response {
|
func NewResponse(rc io.ReadCloser, opts ...func(*Response)) *Response {
|
||||||
|
|
@ -54,9 +54,7 @@ func NewResponse(rc io.ReadCloser, opts ...func(*Response)) *Response {
|
||||||
Validate: func() error {
|
Validate: func() error {
|
||||||
return nil
|
return nil
|
||||||
},
|
},
|
||||||
Temporary: func() bool {
|
Temporary: true,
|
||||||
return true
|
|
||||||
},
|
|
||||||
}
|
}
|
||||||
|
|
||||||
for _, opt := range opts {
|
for _, opt := range opts {
|
||||||
|
|
@ -103,7 +101,7 @@ func WithValidate(validate func() error) func(*Response) {
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func WithTemporary(temporary func() bool) func(*Response) {
|
func WithTemporary(temporary bool) func(*Response) {
|
||||||
return func(resp *Response) {
|
return func(resp *Response) {
|
||||||
resp.Temporary = temporary
|
resp.Temporary = temporary
|
||||||
}
|
}
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue