/* * Copyright 2022 The Dragonfly Authors * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. * You may obtain a copy of the License at * * http://www.apache.org/licenses/LICENSE-2.0 * * Unless required by applicable law or agreed to in writing, software * distributed under the License is distributed on an "AS IS" BASIS, * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. * See the License for the specific language governing permissions and * limitations under the License. */ //go:generate mockgen -destination mocks/dfstore_mock.go -source dfstore.go -package mocks package dfstore import ( "bytes" "context" "errors" "fmt" "io" "mime/multipart" "net/http" "net/url" "path/filepath" "strconv" "github.com/go-http-utils/headers" "d7y.io/dragonfly/v2/client/config" "d7y.io/dragonfly/v2/client/daemon/objectstorage" pkgobjectstorage "d7y.io/dragonfly/v2/pkg/objectstorage" ) // Dfstore is the interface used for object storage. type Dfstore interface { // GetObjectMetadataRequestWithContext returns *http.Request of getting object metadata. GetObjectMetadataRequestWithContext(ctx context.Context, input *GetObjectMetadataInput) (*http.Request, error) // GetObjectMetadataWithContext returns matedata of object. GetObjectMetadataWithContext(ctx context.Context, input *GetObjectMetadataInput) (*pkgobjectstorage.ObjectMetadata, error) // GetObjectRequestWithContext returns *http.Request of getting object. GetObjectRequestWithContext(ctx context.Context, input *GetObjectInput) (*http.Request, error) // GetObjectWithContext returns data of object. GetObjectWithContext(ctx context.Context, input *GetObjectInput) (io.ReadCloser, error) // PutObjectRequestWithContext returns *http.Request of putting object. PutObjectRequestWithContext(ctx context.Context, input *PutObjectInput) (*http.Request, error) // PutObjectWithContext puts data of object. PutObjectWithContext(ctx context.Context, input *PutObjectInput) error // DeleteObjectRequestWithContext returns *http.Request of deleting object. DeleteObjectRequestWithContext(ctx context.Context, input *DeleteObjectInput) (*http.Request, error) // DeleteObjectWithContext deletes data of object. DeleteObjectWithContext(ctx context.Context, input *DeleteObjectInput) error // IsObjectExistRequestWithContext returns *http.Request of heading object. IsObjectExistRequestWithContext(ctx context.Context, input *IsObjectExistInput) (*http.Request, error) // IsObjectExistWithContext returns whether the object exists. IsObjectExistWithContext(ctx context.Context, input *IsObjectExistInput) (bool, error) } // dfstore provides object storage function. type dfstore struct { endpoint string httpClient *http.Client } // Option is a functional option for configuring the dfstore. type Option func(dfs *dfstore) // WithHTTPClient set http client for dfstore. func WithHTTPClient(client *http.Client) Option { return func(dfs *dfstore) { dfs.httpClient = client } } // New dfstore instance. func New(endpoint string, options ...Option) Dfstore { dfs := &dfstore{ endpoint: endpoint, httpClient: http.DefaultClient, } for _, opt := range options { opt(dfs) } return dfs } // GetObjectMetadataInput is used to construct request of getting object metadata. type GetObjectMetadataInput struct { // BucketName is bucket name. BucketName string // ObjectKey is object key. ObjectKey string } // Validate validates GetObjectMetadataInput fields. func (i *GetObjectMetadataInput) Validate() error { if i.BucketName == "" { return errors.New("invalid BucketName") } if i.ObjectKey == "" { return errors.New("invalid ObjectKey") } return nil } // GetObjectMetadataRequestWithContext returns *http.Request of getting object metadata. func (dfs *dfstore) GetObjectMetadataRequestWithContext(ctx context.Context, input *GetObjectMetadataInput) (*http.Request, error) { if err := input.Validate(); err != nil { return nil, err } u, err := url.Parse(dfs.endpoint) if err != nil { return nil, err } u.Path = filepath.Join("buckets", input.BucketName, "objects", input.ObjectKey) req, err := http.NewRequestWithContext(ctx, http.MethodHead, u.String(), nil) if err != nil { return nil, err } return req, nil } // GetObjectMetadataWithContext returns metadata of object. func (dfs *dfstore) GetObjectMetadataWithContext(ctx context.Context, input *GetObjectMetadataInput) (*pkgobjectstorage.ObjectMetadata, error) { req, err := dfs.GetObjectMetadataRequestWithContext(ctx, input) if err != nil { return nil, err } resp, err := dfs.httpClient.Do(req) if err != nil { return nil, err } defer resp.Body.Close() if resp.StatusCode/100 != 2 { return nil, fmt.Errorf("bad response status %s", resp.Status) } contentLength, err := strconv.ParseInt(resp.Header.Get(headers.ContentLength), 10, 64) if err != nil { return nil, err } return &pkgobjectstorage.ObjectMetadata{ ContentDisposition: resp.Header.Get(headers.ContentDisposition), ContentEncoding: resp.Header.Get(headers.ContentEncoding), ContentLanguage: resp.Header.Get(headers.ContentLanguage), ContentLength: int64(contentLength), ContentType: resp.Header.Get(headers.ContentType), ETag: resp.Header.Get(headers.ContentType), Digest: resp.Header.Get(config.HeaderDragonflyObjectMetaDigest), }, nil } // GetObjectInput is used to construct request of getting object. type GetObjectInput struct { // BucketName is bucket name. BucketName string // ObjectKey is object key. ObjectKey string // Filter is used to generate a unique Task ID by // filtering unnecessary query params in the URL, // it is separated by & character. Filter string // Range is the HTTP range header. Range string } // Validate validates GetObjectInput fields. func (i *GetObjectInput) Validate() error { if i.BucketName == "" { return errors.New("invalid BucketName") } if i.ObjectKey == "" { return errors.New("invalid ObjectKey") } return nil } // GetObjectRequestWithContext returns *http.Request of getting object. func (dfs *dfstore) GetObjectRequestWithContext(ctx context.Context, input *GetObjectInput) (*http.Request, error) { if err := input.Validate(); err != nil { return nil, err } u, err := url.Parse(dfs.endpoint) if err != nil { return nil, err } u.Path = filepath.Join("buckets", input.BucketName, "objects", input.ObjectKey) query := u.Query() if input.Filter != "" { query.Set("filter", input.Filter) } u.RawQuery = query.Encode() req, err := http.NewRequestWithContext(ctx, http.MethodGet, u.String(), nil) if err != nil { return nil, err } if input.Range != "" { req.Header.Set(headers.Range, input.Range) } return req, nil } // GetObjectWithContext returns data of object. func (dfs *dfstore) GetObjectWithContext(ctx context.Context, input *GetObjectInput) (io.ReadCloser, error) { req, err := dfs.GetObjectRequestWithContext(ctx, input) if err != nil { return nil, err } resp, err := dfs.httpClient.Do(req) if err != nil { return nil, err } if resp.StatusCode/100 != 2 { return nil, fmt.Errorf("bad response status %s", resp.Status) } return resp.Body, nil } // PutObjectInput is used to construct request of putting object. type PutObjectInput struct { // BucketName is bucket name. BucketName string // ObjectKey is object key. ObjectKey string // Filter is used to generate a unique Task ID by // filtering unnecessary query params in the URL, // it is separated by & character. Filter string // Mode is the mode in which the backend is written, // including WriteBack and AsyncWriteBack. Mode int // MaxReplicas is the maximum number of // replicas of an object cache in seed peers. MaxReplicas int // Reader is reader of object. Reader io.Reader } // Validate validates PutObjectInput fields. func (i *PutObjectInput) Validate() error { if i.BucketName == "" { return errors.New("invalid BucketName") } if i.ObjectKey == "" { return errors.New("invalid ObjectKey") } if i.Mode != objectstorage.WriteBack && i.Mode != objectstorage.AsyncWriteBack { return errors.New("invalid Mode") } if i.MaxReplicas < 0 || i.MaxReplicas > 100 { return errors.New("invalid MaxReplicas") } return nil } // PutObjectRequestWithContext returns *http.Request of putting object. func (dfs *dfstore) PutObjectRequestWithContext(ctx context.Context, input *PutObjectInput) (*http.Request, error) { if err := input.Validate(); err != nil { return nil, err } body := &bytes.Buffer{} writer := multipart.NewWriter(body) // AsyncWriteBack mode is used by default. if err := writer.WriteField("mode", fmt.Sprint(input.Mode)); err != nil { return nil, err } if input.Filter != "" { if err := writer.WriteField("filter", input.Filter); err != nil { return nil, err } } if input.MaxReplicas > 0 { if err := writer.WriteField("maxReplicas", fmt.Sprint(input.MaxReplicas)); err != nil { return nil, err } } part, err := writer.CreateFormFile("file", filepath.Base(input.ObjectKey)) if err != nil { return nil, err } if _, err := io.Copy(part, input.Reader); err != nil { return nil, err } if err := writer.Close(); err != nil { return nil, err } u, err := url.Parse(dfs.endpoint) if err != nil { return nil, err } u.Path = filepath.Join("buckets", input.BucketName, "objects", input.ObjectKey) req, err := http.NewRequestWithContext(ctx, http.MethodPut, u.String(), body) if err != nil { return nil, err } req.Header.Add(headers.ContentType, writer.FormDataContentType()) return req, nil } // PutObjectWithContext puts data of object. func (dfs *dfstore) PutObjectWithContext(ctx context.Context, input *PutObjectInput) error { req, err := dfs.PutObjectRequestWithContext(ctx, input) if err != nil { return err } resp, err := http.DefaultClient.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode/100 != 2 { return fmt.Errorf("bad response status %s", resp.Status) } return nil } // DeleteObjectInput is used to construct request of deleting object. type DeleteObjectInput struct { // BucketName is bucket name. BucketName string // ObjectKey is object key. ObjectKey string } // Validate validates DeleteObjectInput fields. func (i *DeleteObjectInput) Validate() error { if i.BucketName == "" { return errors.New("invalid BucketName") } if i.ObjectKey == "" { return errors.New("invalid ObjectKey") } return nil } // DeleteObjectRequestWithContext returns *http.Request of deleting object. func (dfs *dfstore) DeleteObjectRequestWithContext(ctx context.Context, input *DeleteObjectInput) (*http.Request, error) { if err := input.Validate(); err != nil { return nil, err } u, err := url.Parse(dfs.endpoint) if err != nil { return nil, err } u.Path = filepath.Join("buckets", input.BucketName, "objects", input.ObjectKey) return http.NewRequestWithContext(ctx, http.MethodDelete, u.String(), nil) } // DeleteObjectWithContext deletes data of object. func (dfs *dfstore) DeleteObjectWithContext(ctx context.Context, input *DeleteObjectInput) error { req, err := dfs.DeleteObjectRequestWithContext(ctx, input) if err != nil { return err } resp, err := http.DefaultClient.Do(req) if err != nil { return err } defer resp.Body.Close() if resp.StatusCode/100 != 2 { return fmt.Errorf("bad response status %s", resp.Status) } return nil } // IsObjectExistInput is used to construct request of heading object. type IsObjectExistInput struct { // BucketName is bucket name. BucketName string // ObjectKey is object key. ObjectKey string } // Validate validates IsObjectExistInput fields. func (i *IsObjectExistInput) Validate() error { if i.BucketName == "" { return errors.New("invalid BucketName") } if i.ObjectKey == "" { return errors.New("invalid ObjectKey") } return nil } // IsObjectExistRequestWithContext returns *http.Request of heading object. func (dfs *dfstore) IsObjectExistRequestWithContext(ctx context.Context, input *IsObjectExistInput) (*http.Request, error) { if err := input.Validate(); err != nil { return nil, err } u, err := url.Parse(dfs.endpoint) if err != nil { return nil, err } u.Path = filepath.Join("buckets", input.BucketName, "objects", input.ObjectKey) return http.NewRequestWithContext(ctx, http.MethodHead, u.String(), nil) } // IsObjectExistWithContext returns whether the object exists. func (dfs *dfstore) IsObjectExistWithContext(ctx context.Context, input *IsObjectExistInput) (bool, error) { req, err := dfs.IsObjectExistRequestWithContext(ctx, input) if err != nil { return false, err } resp, err := http.DefaultClient.Do(req) if err != nil { return false, err } defer resp.Body.Close() if resp.StatusCode == http.StatusNotFound { return false, nil } if resp.StatusCode/100 != 2 { return false, fmt.Errorf("bad response status %s", resp.Status) } return true, nil }