feat: update source temporary error logic (#1739)

Signed-off-by: Jim Ma <majinjing3@gmail.com>
This commit is contained in:
Jim Ma 2022-10-10 20:19:24 +08:00 committed by Gaius
parent ef61df0027
commit 2d46f6424a
No known key found for this signature in database
GPG Key ID: 8B4E5D1290FA2FFB
5 changed files with 24 additions and 28 deletions

View File

@ -385,7 +385,7 @@ singleDownload:
} }
} }
srcErr := &errordetailsv1.SourceError{ srcErr := &errordetailsv1.SourceError{
Temporary: response.Temporary == nil || response.Temporary(), Temporary: response.Temporary,
Metadata: &commonv1.ExtendAttribute{ Metadata: &commonv1.ExtendAttribute{
Header: hdr, Header: hdr,
StatusCode: int32(response.StatusCode), StatusCode: int32(response.StatusCode),

View File

@ -52,7 +52,12 @@ var (
contentRangeRegexp = regexp.MustCompile(`bytes (?P<Start>\d+)-(?P<End>\d+)/(?P<Length>(\d*|\*))`) contentRangeRegexp = regexp.MustCompile(`bytes (?P<Start>\d+)-(?P<End>\d+)/(?P<Length>(\d*|\*))`)
contentRangeRegexpLengthIndex = contentRangeRegexp.SubexpIndex("Length") contentRangeRegexpLengthIndex = contentRangeRegexp.SubexpIndex("Length")
notTemporaryStatusCode = []int{http.StatusUnauthorized, http.StatusForbidden, http.StatusNotFound, http.StatusProxyAuthRequired} notTemporaryStatusCode = []int{
http.StatusUnauthorized,
http.StatusForbidden,
http.StatusNotFound,
http.StatusProxyAuthRequired,
}
) )
func init() { func init() {
@ -218,14 +223,7 @@ func (client *httpSourceClient) GetMetadata(request *source.Request) (*source.Me
Validate: func() error { Validate: func() error {
return source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent}) return source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent})
}, },
Temporary: func() bool { Temporary: detectTemporary(resp.StatusCode),
for _, code := range notTemporaryStatusCode {
if code == resp.StatusCode {
return false
}
}
return false
},
}, nil }, nil
} }
@ -261,14 +259,7 @@ func (client *httpSourceClient) Download(request *source.Request) (*source.Respo
source.WithValidate(func() error { source.WithValidate(func() error {
return source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent}) return source.CheckResponseCode(resp.StatusCode, []int{http.StatusOK, http.StatusPartialContent})
}), }),
source.WithTemporary(func() bool { source.WithTemporary(detectTemporary(resp.StatusCode)),
for _, code := range notTemporaryStatusCode {
if code == resp.StatusCode {
return false
}
}
return true
}),
source.WithHeader(exportPassThroughHeader(resp.Header)), source.WithHeader(exportPassThroughHeader(resp.Header)),
source.WithExpireInfo( source.WithExpireInfo(
source.ExpireInfo{ source.ExpireInfo{
@ -336,3 +327,12 @@ func exportPassThroughHeader(header http.Header) map[string]string {
} }
return ph return ph
} }
func detectTemporary(statusCode int) bool {
for _, code := range notTemporaryStatusCode {
if code == statusCode {
return false
}
}
return true
}

View File

@ -173,9 +173,7 @@ func (osc *ossSourceClient) GetMetadata(request *source.Request) (*source.Metada
Validate: func() error { Validate: func() error {
return nil return nil
}, },
Temporary: func() bool { Temporary: true,
return true
},
}, nil }, nil
} }

View File

@ -31,5 +31,5 @@ type Metadata struct {
TotalContentLength int64 TotalContentLength int64
Validate func() error Validate func() error
Temporary func() bool Temporary bool
} }

View File

@ -35,8 +35,8 @@ type Response struct {
// Validate this response is okay to transfer in p2p network, like status 200 or 206 in http is valid to do this, // Validate this response is okay to transfer in p2p network, like status 200 or 206 in http is valid to do this,
// otherwise return status code to original client // otherwise return status code to original client
Validate func() error Validate func() error
// Temporary check the error whether the error is temporary, if is true, we can retry it later // Temporary indecates the error whether the error is temporary, if is true, we can retry it later
Temporary func() bool Temporary bool
} }
func NewResponse(rc io.ReadCloser, opts ...func(*Response)) *Response { func NewResponse(rc io.ReadCloser, opts ...func(*Response)) *Response {
@ -54,9 +54,7 @@ func NewResponse(rc io.ReadCloser, opts ...func(*Response)) *Response {
Validate: func() error { Validate: func() error {
return nil return nil
}, },
Temporary: func() bool { Temporary: true,
return true
},
} }
for _, opt := range opts { for _, opt := range opts {
@ -103,7 +101,7 @@ func WithValidate(validate func() error) func(*Response) {
} }
} }
func WithTemporary(temporary func() bool) func(*Response) { func WithTemporary(temporary bool) func(*Response) {
return func(resp *Response) { return func(resp *Response) {
resp.Temporary = temporary resp.Temporary = temporary
} }