diff --git a/api/v1beta1/bucket_types.go b/api/v1beta1/bucket_types.go index 1dc68851..e046eaa8 100644 --- a/api/v1beta1/bucket_types.go +++ b/api/v1beta1/bucket_types.go @@ -30,7 +30,7 @@ const ( // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws + // +kubebuilder:validation:Enum=generic;aws;gcp // +kubebuilder:default:=generic // +optional Provider string `json:"provider,omitempty"` diff --git a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml index 5905c1d7..a64e98b4 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml @@ -66,6 +66,7 @@ spec: enum: - generic - aws + - gcp type: string region: description: The bucket region. diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 3ff17f22..9e4eee73 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -177,16 +177,21 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, error) { - var tempDir string var err error var sourceBucket sourcev1.Bucket + tempDir, err := os.MkdirTemp("", bucket.Name) + if err != nil { + err = fmt.Errorf("tmp dir error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err + } + defer os.RemoveAll(tempDir) if bucket.Spec.Provider == sourcev1.GoogleBucketProvider { - sourceBucket, tempDir, err = r.reconcileWithGCP(ctx, bucket) + sourceBucket, err = r.reconcileWithGCP(ctx, bucket, tempDir) if err != nil { return sourceBucket, err } } else { - sourceBucket, tempDir, err = r.reconcileWithMinio(ctx, bucket) + sourceBucket, err = r.reconcileWithMinio(ctx, bucket, tempDir) if err != nil { return sourceBucket, err } @@ -261,41 +266,36 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1. // reconcileWithGCP handles getting objects from a Google Cloud Platform bucket // using a gcp client -func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { +func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) { gcpClient, err := r.authGCP(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err } defer gcpClient.Client.Close() - // create tmp dir - tempDir, err := os.MkdirTemp("", bucket.Name) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err - } - defer os.RemoveAll(tempDir) ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) defer cancel() exists, err := gcpClient.BucketExists(ctxTimeout, bucket.Spec.BucketName) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if !exists { err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // Look for file with ignore rules first. path := filepath.Join(tempDir, sourceignore.IgnoreFile) if err := gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path); err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + if err == gcp.ErrorObjectDoesNotExist && sourceignore.IgnoreFile != ".sourceignore" { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err + } } ps, err := sourceignore.ReadIgnoreFile(path, nil) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // In-spec patterns take precedence if bucket.Spec.Ignore != nil { @@ -311,7 +311,7 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1 } if err != nil { err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if strings.HasSuffix(object.Name, "/") || object.Name == sourceignore.IgnoreFile { @@ -323,42 +323,33 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1 } localPath := filepath.Join(tempDir, object.Name) - // FGetObject - get and download bucket object if err = gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Name, localPath); err != nil { err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } } - return sourcev1.Bucket{}, tempDir, nil + return sourcev1.Bucket{}, nil } // reconcileWithMinio handles getting objects from an S3 compatible bucket // using a minio client -func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { +func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) { s3Client, err := r.authMinio(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err } - // create tmp dir - tempDir, err := os.MkdirTemp("", bucket.Name) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err - } - defer os.RemoveAll(tempDir) - ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) defer cancel() exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if !exists { err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // Look for file with ignore rules first @@ -367,12 +358,12 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source path := filepath.Join(tempDir, sourceignore.IgnoreFile) if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil { if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } } ps, err := sourceignore.ReadIgnoreFile(path, nil) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // In-spec patterns take precedence if bucket.Spec.Ignore != nil { @@ -387,7 +378,7 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source }) { if object.Err != nil { err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile { @@ -402,20 +393,43 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{}) if err != nil { err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } } - return sourcev1.Bucket{}, tempDir, nil + return sourcev1.Bucket{}, nil } // authGCP creates a new Google Cloud Platform storage client -// to interact with the Storage service. +// to interact with the storage service. func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) (*gcp.GCPClient, error) { - client, err := gcp.NewClient(ctx) - if err != nil { - return nil, err + var client *gcp.GCPClient + var err error + if bucket.Spec.SecretRef != nil { + secretName := types.NamespacedName{ + Namespace: bucket.GetNamespace(), + Name: bucket.Spec.SecretRef.Name, + } + + var secret corev1.Secret + if err := r.Get(ctx, secretName, &secret); err != nil { + return nil, fmt.Errorf("credentials secret error: %w", err) + } + if err := gcp.ValidateSecret(secret.Data, secret.Name); err != nil { + return nil, err + } + serviceAccount := gcp.InitCredentialsWithSecret(secret.Data) + client, err = gcp.NewClientWithSAKey(ctx, serviceAccount) + if err != nil { + return nil, err + } + } else { + client, err = gcp.NewClient(ctx) + if err != nil { + return nil, err + } } return client, nil + } // authMinio creates a new Minio client to interact with S3 diff --git a/go.mod b/go.mod index 49ebac77..1d60520c 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/mock v1.6.0 // indirect github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/libgit2/git2go/v31 v31.4.14 github.com/minio/minio-go/v7 v7.0.10 diff --git a/go.sum b/go.sum index a1ea7ce9..be1c5759 100644 --- a/go.sum +++ b/go.sum @@ -415,6 +415,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 18ea53fe..8f2f8811 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -18,56 +18,171 @@ package gcp import ( "context" + "encoding/json" "errors" + "fmt" "io" "os" "path/filepath" gcpStorage "cloud.google.com/go/storage" interator "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +const ( + ServiceAccount = "service_account" + AuthUri = "https://accounts.google.com/o/oauth2/auth" + TokenUri = "https://oauth2.googleapis.com/token" + AuthProviderX509CertUrl = "https://www.googleapis.com/oauth2/v1/certs" ) var ( // IteratorDone is returned when the looping of objects/content // has reached the end of the iteration. IteratorDone = interator.Done - // DirectoryExists is an error returned when the filename provided + // ErrorDirectoryExists is an error returned when the filename provided // is a directory. - DirectoryExists = errors.New("filename is a directory") - // ObjectDoesNotExist is an error returned when the object whose name + ErrorDirectoryExists = errors.New("filename is a directory") + // ErrorObjectDoesNotExist is an error returned when the object whose name // is provided does not exist. - ObjectDoesNotExist = errors.New("object does not exist") + ErrorObjectDoesNotExist = errors.New("object does not exist") ) +type Client interface { + Bucket(string) *gcpStorage.BucketHandle + Close() error +} + +type BucketHandle interface { + Create(context.Context, string, *gcpStorage.BucketAttrs) error + Delete(context.Context) error + Attrs(context.Context) (*gcpStorage.BucketAttrs, error) + Object(string) *gcpStorage.ObjectHandle + Objects(context.Context, *gcpStorage.Query) *gcpStorage.ObjectIterator +} + +type ObjectHandle interface { + Attrs(context.Context) (*gcpStorage.ObjectAttrs, error) + NewRangeReader(context.Context, int64, int64) (*gcpStorage.Reader, error) +} type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. - Client *gcpStorage.Client + Client Client // startRange is the starting read value for // reading the object from bucket. - startRange int64 + StartRange int64 // endRange is the ending read value for // reading the object from bucket. - endRange int64 + EndRange int64 +} + +// CredentialsFile struct representing the GCP Service Account +// JSON file. +type CredentialsFile struct { + Type string `json:"type"` + ProjectID string `json:"project_id"` + PrivateKeyID string `json:"private_key_id"` + PrivateKey string `json:"private_key"` + ClientEmail string `json:"client_email"` + ClientID string `json:"client_id"` + AuthUri string `json:"auth_uri"` + TokenUri string `json:"token_uri"` + AuthProviderX509CertUrl string `json:"auth_provider_x509_cert_url"` + ClientX509CertUrl string `json:"client_x509_cert_url"` } // NewClient creates a new GCP storage client // The Google Storage Client will automatically // look for the Google Application Credential environment variable -// or look for the Google Application Credential file +// or look for the Google Application Credential file. func NewClient(ctx context.Context) (*GCPClient, error) { client, err := gcpStorage.NewClient(ctx) if err != nil { return nil, err } - return &GCPClient{Client: client, startRange: 0, endRange: -1}, nil + + return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil +} + +// NewClientWithSAKey creates a new GCP storage client +// It uses the provided JSON file with service account details +// To authenticate. +func NewClientWithSAKey(ctx context.Context, credentials *CredentialsFile) (*GCPClient, error) { + saAccount, err := credentials.credentailsToJSON() + if err != nil { + return nil, err + } + + client, err := gcpStorage.NewClient(ctx, option.WithCredentialsJSON(saAccount)) + if err != nil { + return nil, err + } + + return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil +} + +// credentailsToJSON converts GCP service account credentials struct to JSON. +func (credentials *CredentialsFile) credentailsToJSON() ([]byte, error) { + credentialsJSON, err := json.Marshal(credentials) + if err != nil { + return nil, err + } + + return credentialsJSON, nil +} + +// InitCredentialsWithSecret creates a new credential +// by initializing a new CredentialsFile struct +func InitCredentialsWithSecret(secret map[string][]byte) *CredentialsFile { + return &CredentialsFile{ + Type: ServiceAccount, + ProjectID: string(secret["projectid"]), + PrivateKeyID: string(secret["privatekeyid"]), + PrivateKey: string(secret["privatekey"]), + ClientEmail: string(secret["clientemail"]), + ClientID: string(secret["clientid"]), + AuthUri: AuthUri, + TokenUri: TokenUri, + AuthProviderX509CertUrl: AuthProviderX509CertUrl, + ClientX509CertUrl: string(secret["certurl"]), + } +} + +// ValidateSecret validates the credential secrets +// It ensures that needed secret fields are not missing. +func ValidateSecret(secret map[string][]byte, name string) error { + if _, exists := secret["projectid"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'projectid'", name) + } + if _, exists := secret["privatekeyid"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'privatekeyid'", name) + } + if _, exists := secret["privatekey"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'privatekey'", name) + } + if _, exists := secret["clientemail"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) + } + if _, exists := secret["clientemail"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) + } + if _, exists := secret["clientid"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'clientid'", name) + } + if _, exists := secret["certurl"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'certurl'", name) + } + + return nil } // SetRange sets the startRange and endRange used to read the Object from // the bucket. It is a helper method for resumable downloads. func (c *GCPClient) SetRange(start, end int64) { - c.startRange = start - c.endRange = end + c.StartRange = start + c.EndRange = end } // BucketExists checks if the bucket with the provided name exists. @@ -82,15 +197,18 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, return true, nil } -// ObjectExists checks if the object with the provided name exists. +// ObjectAttributes checks if the object with the provided name exists. // If it exists the Object attributes are returned. -func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) { +func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) { attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) // ErrObjectNotExist is returned if the object does not exist + if err == gcpStorage.ErrObjectNotExist { + return false, nil, err + } if err != nil { return false, nil, err } - return true, attrs, err + return true, attrs, nil } // FGetObject gets the object from the bucket and downloads the object locally @@ -101,7 +219,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca if err == nil { // If the destination exists and is a directory. if dirStatus.IsDir() { - return DirectoryExists + return ErrorDirectoryExists } } @@ -124,17 +242,16 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca // ObjectExists verifies if object exists and you have permission to access. // Check if the object exists and if you have permission to access it // The Object attributes are returned if the Object exists. - exists, attrs, err := c.ObjectExists(ctx, bucketName, objectName) + exists, attrs, err := c.ObjectAttributes(ctx, bucketName, objectName) if err != nil { return err } if !exists { - return ObjectDoesNotExist + return ErrorObjectDoesNotExist } // Write to a temporary file "filename.part.gcp" before saving. - filePartPath := localPath + attrs.Etag + ".part.gcp" - + filePartPath := localPath + ".part.gcp" // If exists, open in append mode. If not create it as a part file. filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) if err != nil { @@ -165,25 +282,25 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca } // Get Object from GCP Bucket - objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.startRange, c.endRange) + objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.StartRange, c.EndRange) if err != nil { return err } defer objectReader.Close() // Write to the part file. - if _, err = io.CopyN(filePart, objectReader, attrs.Size); err != nil { + if _, err := io.CopyN(filePart, objectReader, attrs.Size); err != nil { return err } // Close the file before rename, this is specifically needed for Windows users. closeAndRemove = false - if err = filePart.Close(); err != nil { + if err := filePart.Close(); err != nil { return err } // Safely completed. Now commit by renaming to actual filename. - if err = os.Rename(filePartPath, localPath); err != nil { + if err := os.Rename(filePartPath, localPath); err != nil { return err } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 459e691a..f30774ac 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -14,128 +14,119 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gcp +package gcp_test import ( "context" "os" "path/filepath" "testing" + "time" - "gotest.tools/assert" + gcpStorage "cloud.google.com/go/storage" + "github.com/fluxcd/source-controller/pkg/gcp" + "github.com/fluxcd/source-controller/pkg/gcp/mocks" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) -func TestNewClient(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - client, err := NewClient(context.Background()) - assert.NilError(t, err) - assert.Assert(t, client.Client != nil) +var ( + MockCtrl *gomock.Controller + MockClient *mocks.MockClient + MockBucketHandle *mocks.MockBucketHandle + MockObjectHandle *mocks.MockObjectHandle + bucketName string = "test-bucket" + objectName string = "test.yaml" + localPath string +) + +// mockgen -destination=mocks/mock_gcp_storage.go -package=mocks -source=gcp.go GCPStorageService +func TestGCPProvider(t *testing.T) { + MockCtrl = gomock.NewController(GinkgoT()) + RegisterFailHandler(Fail) + RunSpecs(t, "Test GCP Storage Provider Suite") } -func TestSetRange(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - client, err := NewClient(context.Background()) - assert.NilError(t, err) - testCases := []struct { - title string - start int64 - end int64 - }{ - { - title: "Test Case 1", - start: 1, - end: 5, - }, - { - title: "Test Case 2", - start: 3, - end: 6, - }, - { - title: "Test Case 3", - start: 4, - end: 5, - }, - { - title: "Test Case 4", - start: 2, - end: 7, - }, - } - for _, tt := range testCases { - t.Run(tt.title, func(t *testing.T) { - client.SetRange(tt.start, tt.end) - assert.Equal(t, tt.start, client.startRange) - assert.Equal(t, tt.end, client.endRange) - }) - } -} - -func TestBucketExists(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - bucketName := "" - client, err := NewClient(ctx) - assert.NilError(t, err) - exists, err := client.BucketExists(ctx, bucketName) - assert.NilError(t, err) - assert.Assert(t, exists) -} - -func TestObjectExists(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - // bucketName is the name of the bucket which contains the object - bucketName := "" - // objectName is the path to the object within the bucket - objectName := "" - client, err := NewClient(ctx) - assert.NilError(t, err) - exists, attrs, err := client.ObjectExists(ctx, bucketName, objectName) - assert.NilError(t, err) - assert.Assert(t, exists) - assert.Assert(t, attrs != nil) -} - -func TestListObjects(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - // bucketName is the name of the bucket which contains the object - bucketName := "" - client, err := NewClient(ctx) - assert.NilError(t, err) - objects := client.ListObjects(ctx, bucketName, nil) - assert.NilError(t, err) - assert.Assert(t, objects != nil) - for { - object, err := objects.Next() - if err == IteratorDone { - break - } - assert.Assert(t, object != nil) - } -} - -func TestFGetObject(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - // bucketName is the name of the bucket which contains the object - bucketName := "" - // objectName is the path to the object within the bucket - objectName := "" +var _ = BeforeSuite(func() { + MockClient = mocks.NewMockClient(MockCtrl) + MockBucketHandle = mocks.NewMockBucketHandle(MockCtrl) + MockObjectHandle = mocks.NewMockObjectHandle(MockCtrl) tempDir, err := os.MkdirTemp("", bucketName) if err != nil { - assert.NilError(t, err) + Expect(err).ToNot(HaveOccurred()) } - localPath := filepath.Join(tempDir, objectName) - client, err := NewClient(ctx) - assert.NilError(t, err) - objErr := client.FGetObject(ctx, bucketName, objectName, localPath) - assert.NilError(t, objErr) -} + localPath = filepath.Join(tempDir, objectName) + MockClient.EXPECT().Bucket(bucketName).Return(MockBucketHandle).AnyTimes() + MockBucketHandle.EXPECT().Object(objectName).Return(&gcpStorage.ObjectHandle{}).AnyTimes() + MockBucketHandle.EXPECT().Attrs(context.Background()).Return(&gcpStorage.BucketAttrs{ + Name: bucketName, + Created: time.Now(), + Etag: "test-etag", + }, nil).AnyTimes() + MockBucketHandle.EXPECT().Objects(gomock.Any(), nil).Return(&gcpStorage.ObjectIterator{}).AnyTimes() + MockObjectHandle.EXPECT().Attrs(gomock.Any()).Return(&gcpStorage.ObjectAttrs{ + Bucket: bucketName, + Name: objectName, + ContentType: "text/x-yaml", + Etag: "test-etag", + Size: 125, + Created: time.Now(), + }, nil).AnyTimes() + MockObjectHandle.EXPECT().NewRangeReader(gomock.Any(), 10, 125).Return(&gcpStorage.Reader{}, nil).AnyTimes() +}) + +var _ = Describe("GCP Storage Provider", func() { + Describe("Get GCP Storage Provider client from gcp", func() { + + Context("Gcp storage Bucket - BucketExists", func() { + It("should not return an error when fetching gcp storage bucket", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + exists, err := gcpClient.BucketExists(context.Background(), bucketName) + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + }) + Context("Gcp storage Bucket - FGetObject", func() { + It("should get the object from the bucket and download the object locally", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + err := gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath) + Expect(err).ToNot(HaveOccurred()) + }) + }) + Context("Gcp storage Bucket - ObjectAttributes", func() { + It("should get the object attributes", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + exists, attrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + Expect(attrs).ToNot(BeNil()) + }) + + Context("Gcp storage Bucket - SetRange", func() { + It("should set the range of the io reader seeker for the file download", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + gcpClient.SetRange(2, 5) + Expect(gcpClient.StartRange).To(Equal(int64(2))) + Expect(gcpClient.EndRange).To(Equal(int64(5))) + }) + }) + }) + }) +}) diff --git a/pkg/gcp/mocks/mock_gcp_storage.go b/pkg/gcp/mocks/mock_gcp_storage.go new file mode 100644 index 00000000..54b78be1 --- /dev/null +++ b/pkg/gcp/mocks/mock_gcp_storage.go @@ -0,0 +1,211 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: gcp.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + storage "cloud.google.com/go/storage" + gomock "github.com/golang/mock/gomock" +) + +// MockClient is a mock of Client interface. +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder +} + +// MockClientMockRecorder is the mock recorder for MockClient. +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance. +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// Bucket mocks base method. +func (m *MockClient) Bucket(arg0 string) *storage.BucketHandle { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Bucket", arg0) + ret0, _ := ret[0].(*storage.BucketHandle) + return ret0 +} + +// Bucket indicates an expected call of Bucket. +func (mr *MockClientMockRecorder) Bucket(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockClient)(nil).Bucket), arg0) +} + +// Close mocks base method. +func (m *MockClient) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockClientMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClient)(nil).Close)) +} + +// MockBucketHandle is a mock of BucketHandle interface. +type MockBucketHandle struct { + ctrl *gomock.Controller + recorder *MockBucketHandleMockRecorder +} + +// MockBucketHandleMockRecorder is the mock recorder for MockBucketHandle. +type MockBucketHandleMockRecorder struct { + mock *MockBucketHandle +} + +// NewMockBucketHandle creates a new mock instance. +func NewMockBucketHandle(ctrl *gomock.Controller) *MockBucketHandle { + mock := &MockBucketHandle{ctrl: ctrl} + mock.recorder = &MockBucketHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBucketHandle) EXPECT() *MockBucketHandleMockRecorder { + return m.recorder +} + +// Attrs mocks base method. +func (m *MockBucketHandle) Attrs(arg0 context.Context) (*storage.BucketAttrs, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Attrs", arg0) + ret0, _ := ret[0].(*storage.BucketAttrs) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Attrs indicates an expected call of Attrs. +func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0) +} + +// Create mocks base method. +func (m *MockBucketHandle) Create(arg0 context.Context, arg1 string, arg2 *storage.BucketAttrs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockBucketHandleMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockBucketHandle)(nil).Create), arg0, arg1, arg2) +} + +// Delete mocks base method. +func (m *MockBucketHandle) Delete(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockBucketHandleMockRecorder) Delete(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBucketHandle)(nil).Delete), arg0) +} + +// Object mocks base method. +func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Object", arg0) + ret0, _ := ret[0].(*storage.ObjectHandle) + return ret0 +} + +// Object indicates an expected call of Object. +func (mr *MockBucketHandleMockRecorder) Object(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Object", reflect.TypeOf((*MockBucketHandle)(nil).Object), arg0) +} + +// Objects mocks base method. +func (m *MockBucketHandle) Objects(arg0 context.Context, arg1 *storage.Query) *storage.ObjectIterator { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Objects", arg0, arg1) + ret0, _ := ret[0].(*storage.ObjectIterator) + return ret0 +} + +// Objects indicates an expected call of Objects. +func (mr *MockBucketHandleMockRecorder) Objects(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Objects", reflect.TypeOf((*MockBucketHandle)(nil).Objects), arg0, arg1) +} + +// MockObjectHandle is a mock of ObjectHandle interface. +type MockObjectHandle struct { + ctrl *gomock.Controller + recorder *MockObjectHandleMockRecorder +} + +// MockObjectHandleMockRecorder is the mock recorder for MockObjectHandle. +type MockObjectHandleMockRecorder struct { + mock *MockObjectHandle +} + +// NewMockObjectHandle creates a new mock instance. +func NewMockObjectHandle(ctrl *gomock.Controller) *MockObjectHandle { + mock := &MockObjectHandle{ctrl: ctrl} + mock.recorder = &MockObjectHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockObjectHandle) EXPECT() *MockObjectHandleMockRecorder { + return m.recorder +} + +// Attrs mocks base method. +func (m *MockObjectHandle) Attrs(arg0 context.Context) (*storage.ObjectAttrs, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Attrs", arg0) + ret0, _ := ret[0].(*storage.ObjectAttrs) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Attrs indicates an expected call of Attrs. +func (mr *MockObjectHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockObjectHandle)(nil).Attrs), arg0) +} + +// NewRangeReader mocks base method. +func (m *MockObjectHandle) NewRangeReader(arg0 context.Context, arg1, arg2 int64) (*storage.Reader, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewRangeReader", arg0, arg1, arg2) + ret0, _ := ret[0].(*storage.Reader) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NewRangeReader indicates an expected call of NewRangeReader. +func (mr *MockObjectHandleMockRecorder) NewRangeReader(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewRangeReader", reflect.TypeOf((*MockObjectHandle)(nil).NewRangeReader), arg0, arg1, arg2) +}