Service Account Key Authentication to GCP Provider

Signed-off-by: pa250194 <pa250194@ncr.com>

api/v1beta1/bucket_types.go

@@ -30,7 +30,7 @@ const (
 // BucketSpec defines the desired state of an S3 compatible bucket
 type BucketSpec struct {
 	// The S3 compatible storage provider name, default ('generic').
-	// +kubebuilder:validation:Enum=generic;aws
+	// +kubebuilder:validation:Enum=generic;aws;gcp
 	// +kubebuilder:default:=generic
 	// +optional
 	Provider string `json:"provider,omitempty"`

config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml

@@ -66,6 +66,7 @@ spec:
                 enum:
                 - generic
                 - aws
+                - gcp
                 type: string
               region:
                 description: The bucket region.

controllers/bucket_controller.go

@@ -177,16 +177,21 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
 }
 
 func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, error) {
-	var tempDir string
 	var err error
 	var sourceBucket sourcev1.Bucket
+	tempDir, err := os.MkdirTemp("", bucket.Name)
+	if err != nil {
+		err = fmt.Errorf("tmp dir error: %w", err)
+		return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err
+	}
+	defer os.RemoveAll(tempDir)
 	if bucket.Spec.Provider == sourcev1.GoogleBucketProvider {
-		sourceBucket, tempDir, err = r.reconcileWithGCP(ctx, bucket)
+		sourceBucket, err = r.reconcileWithGCP(ctx, bucket, tempDir)
 		if err != nil {
 			return sourceBucket, err
 		}
 	} else {
-		sourceBucket, tempDir, err = r.reconcileWithMinio(ctx, bucket)
+		sourceBucket, err = r.reconcileWithMinio(ctx, bucket, tempDir)
 		if err != nil {
 			return sourceBucket, err
 		}
@@ -261,41 +266,36 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1.
 
 // reconcileWithGCP handles getting objects from a Google Cloud Platform bucket
 // using a gcp client
-func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) {
+func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) {
 	gcpClient, err := r.authGCP(ctx, bucket)
 	if err != nil {
 		err = fmt.Errorf("auth error: %w", err)
-		return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err
+		return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err
 	}
 	defer gcpClient.Client.Close()
-	// create tmp dir
-	tempDir, err := os.MkdirTemp("", bucket.Name)
-	if err != nil {
-		err = fmt.Errorf("tmp dir error: %w", err)
-		return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err
-	}
-	defer os.RemoveAll(tempDir)
 
 	ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration)
 	defer cancel()
 
 	exists, err := gcpClient.BucketExists(ctxTimeout, bucket.Spec.BucketName)
 	if err != nil {
-		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 	}
 	if !exists {
 		err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName)
-		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 	}
 
 	// Look for file with ignore rules first.
 	path := filepath.Join(tempDir, sourceignore.IgnoreFile)
 	if err := gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path); err != nil {
-		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+		if err == gcp.ErrorObjectDoesNotExist && sourceignore.IgnoreFile != ".sourceignore" {
+			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
+		}
 	}
 	ps, err := sourceignore.ReadIgnoreFile(path, nil)
 	if err != nil {
-		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 	}
 	// In-spec patterns take precedence
 	if bucket.Spec.Ignore != nil {
@@ -311,7 +311,7 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1
 		}
 		if err != nil {
 			err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, err)
-			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 		}
 
 		if strings.HasSuffix(object.Name, "/") || object.Name == sourceignore.IgnoreFile {
@@ -323,42 +323,33 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1
 		}
 
 		localPath := filepath.Join(tempDir, object.Name)
-		// FGetObject - get and download bucket object
 		if err = gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Name, localPath); err != nil {
 			err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err)
-			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 		}
 	}
-	return sourcev1.Bucket{}, tempDir, nil
+	return sourcev1.Bucket{}, nil
 }
 
 // reconcileWithMinio handles getting objects from an S3 compatible bucket
 // using a minio client
-func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) {
+func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) {
 	s3Client, err := r.authMinio(ctx, bucket)
 	if err != nil {
 		err = fmt.Errorf("auth error: %w", err)
-		return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err
+		return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err
 	}
 
-	// create tmp dir
-	tempDir, err := os.MkdirTemp("", bucket.Name)
-	if err != nil {
-		err = fmt.Errorf("tmp dir error: %w", err)
-		return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err
-	}
-	defer os.RemoveAll(tempDir)
-
 	ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration)
 	defer cancel()
 
 	exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName)
 	if err != nil {
-		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 	}
 	if !exists {
 		err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName)
-		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 	}
 
 	// Look for file with ignore rules first
@@ -367,12 +358,12 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source
 	path := filepath.Join(tempDir, sourceignore.IgnoreFile)
 	if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil {
 		if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" {
-			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 		}
 	}
 	ps, err := sourceignore.ReadIgnoreFile(path, nil)
 	if err != nil {
-		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 	}
 	// In-spec patterns take precedence
 	if bucket.Spec.Ignore != nil {
@@ -387,7 +378,7 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source
 	}) {
 		if object.Err != nil {
 			err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err)
-			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 		}
 
 		if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile {
@@ -402,20 +393,43 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source
 		err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{})
 		if err != nil {
 			err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err)
-			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
+			return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 		}
 	}
-	return sourcev1.Bucket{}, tempDir, nil
+	return sourcev1.Bucket{}, nil
 }
 
 // authGCP creates a new Google Cloud Platform storage client
-// to interact with the Storage service.
+// to interact with the storage service.
 func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) (*gcp.GCPClient, error) {
-	client, err := gcp.NewClient(ctx)
+	var client *gcp.GCPClient
-	if err != nil {
+	var err error
-		return nil, err
+	if bucket.Spec.SecretRef != nil {
+		secretName := types.NamespacedName{
+			Namespace: bucket.GetNamespace(),
+			Name:      bucket.Spec.SecretRef.Name,
+		}
+
+		var secret corev1.Secret
+		if err := r.Get(ctx, secretName, &secret); err != nil {
+			return nil, fmt.Errorf("credentials secret error: %w", err)
+		}
+		if err := gcp.ValidateSecret(secret.Data, secret.Name); err != nil {
+			return nil, err
+		}
+		serviceAccount := gcp.InitCredentialsWithSecret(secret.Data)
+		client, err = gcp.NewClientWithSAKey(ctx, serviceAccount)
+		if err != nil {
+			return nil, err
+		}
+	} else {
+		client, err = gcp.NewClient(ctx)
+		if err != nil {
+			return nil, err
+		}
 	}
 	return client, nil
+
 }
 
 // authMinio creates a new Minio client to interact with S3

go.mod

@@ -23,6 +23,7 @@ require (
 	github.com/go-git/go-git/v5 v5.4.2
 	github.com/go-logr/logr v0.4.0
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
+	github.com/golang/mock v1.6.0 // indirect
 	github.com/googleapis/gax-go/v2 v2.1.0 // indirect
 	github.com/libgit2/git2go/v31 v31.4.14
 	github.com/minio/minio-go/v7 v7.0.10

go.sum

@@ -415,6 +415,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
 github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
 github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
 github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
+github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
 github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

pkg/gcp/gcp.go

@@ -18,56 +18,171 @@ package gcp
 
 import (
 	"context"
+	"encoding/json"
 	"errors"
+	"fmt"
 	"io"
 	"os"
 	"path/filepath"
 
 	gcpStorage "cloud.google.com/go/storage"
 	interator "google.golang.org/api/iterator"
+	"google.golang.org/api/option"
+)
+
+const (
+	ServiceAccount          = "service_account"
+	AuthUri                 = "https://accounts.google.com/o/oauth2/auth"
+	TokenUri                = "https://oauth2.googleapis.com/token"
+	AuthProviderX509CertUrl = "https://www.googleapis.com/oauth2/v1/certs"
 )
 
 var (
 	// IteratorDone is returned when the looping of objects/content
 	// has reached the end of the iteration.
 	IteratorDone = interator.Done
-	// DirectoryExists is an error returned when the filename provided
+	// ErrorDirectoryExists is an error returned when the filename provided
 	// is a directory.
-	DirectoryExists = errors.New("filename is a directory")
+	ErrorDirectoryExists = errors.New("filename is a directory")
-	// ObjectDoesNotExist is an error returned when the object whose name
+	// ErrorObjectDoesNotExist is an error returned when the object whose name
 	// is provided does not exist.
-	ObjectDoesNotExist = errors.New("object does not exist")
+	ErrorObjectDoesNotExist = errors.New("object does not exist")
 )
 
+type Client interface {
+	Bucket(string) *gcpStorage.BucketHandle
+	Close() error
+}
+
+type BucketHandle interface {
+	Create(context.Context, string, *gcpStorage.BucketAttrs) error
+	Delete(context.Context) error
+	Attrs(context.Context) (*gcpStorage.BucketAttrs, error)
+	Object(string) *gcpStorage.ObjectHandle
+	Objects(context.Context, *gcpStorage.Query) *gcpStorage.ObjectIterator
+}
+
+type ObjectHandle interface {
+	Attrs(context.Context) (*gcpStorage.ObjectAttrs, error)
+	NewRangeReader(context.Context, int64, int64) (*gcpStorage.Reader, error)
+}
 type GCPClient struct {
 	// client for interacting with the Google Cloud
 	// Storage APIs.
-	Client *gcpStorage.Client
+	Client Client
 	// startRange is the starting read value for
 	// reading the object from bucket.
-	startRange int64
+	StartRange int64
 	// endRange is the ending read value for
 	// reading the object from bucket.
-	endRange int64
+	EndRange int64
+}
+
+// CredentialsFile struct representing the GCP Service Account
+// JSON file.
+type CredentialsFile struct {
+	Type                    string `json:"type"`
+	ProjectID               string `json:"project_id"`
+	PrivateKeyID            string `json:"private_key_id"`
+	PrivateKey              string `json:"private_key"`
+	ClientEmail             string `json:"client_email"`
+	ClientID                string `json:"client_id"`
+	AuthUri                 string `json:"auth_uri"`
+	TokenUri                string `json:"token_uri"`
+	AuthProviderX509CertUrl string `json:"auth_provider_x509_cert_url"`
+	ClientX509CertUrl       string `json:"client_x509_cert_url"`
 }
 
 // NewClient creates a new GCP storage client
 // The Google Storage Client will automatically
 // look for the Google Application Credential environment variable
-// or look for the Google Application Credential file
+// or look for the Google Application Credential file.
 func NewClient(ctx context.Context) (*GCPClient, error) {
 	client, err := gcpStorage.NewClient(ctx)
 	if err != nil {
 		return nil, err
 	}
-	return &GCPClient{Client: client, startRange: 0, endRange: -1}, nil
+
+	return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil
+}
+
+// NewClientWithSAKey creates a new GCP storage client
+// It uses the provided JSON file with service account details
+// To authenticate.
+func NewClientWithSAKey(ctx context.Context, credentials *CredentialsFile) (*GCPClient, error) {
+	saAccount, err := credentials.credentailsToJSON()
+	if err != nil {
+		return nil, err
+	}
+
+	client, err := gcpStorage.NewClient(ctx, option.WithCredentialsJSON(saAccount))
+	if err != nil {
+		return nil, err
+	}
+
+	return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil
+}
+
+// credentailsToJSON converts GCP service account credentials struct to JSON.
+func (credentials *CredentialsFile) credentailsToJSON() ([]byte, error) {
+	credentialsJSON, err := json.Marshal(credentials)
+	if err != nil {
+		return nil, err
+	}
+
+	return credentialsJSON, nil
+}
+
+// InitCredentialsWithSecret creates a new credential
+// by initializing a new CredentialsFile struct
+func InitCredentialsWithSecret(secret map[string][]byte) *CredentialsFile {
+	return &CredentialsFile{
+		Type:                    ServiceAccount,
+		ProjectID:               string(secret["projectid"]),
+		PrivateKeyID:            string(secret["privatekeyid"]),
+		PrivateKey:              string(secret["privatekey"]),
+		ClientEmail:             string(secret["clientemail"]),
+		ClientID:                string(secret["clientid"]),
+		AuthUri:                 AuthUri,
+		TokenUri:                TokenUri,
+		AuthProviderX509CertUrl: AuthProviderX509CertUrl,
+		ClientX509CertUrl:       string(secret["certurl"]),
+	}
+}
+
+// ValidateSecret validates the credential secrets
+// It ensures that needed secret fields are not missing.
+func ValidateSecret(secret map[string][]byte, name string) error {
+	if _, exists := secret["projectid"]; !exists {
+		return fmt.Errorf("invalid '%s' secret data: required fields 'projectid'", name)
+	}
+	if _, exists := secret["privatekeyid"]; !exists {
+		return fmt.Errorf("invalid '%s' secret data: required fields 'privatekeyid'", name)
+	}
+	if _, exists := secret["privatekey"]; !exists {
+		return fmt.Errorf("invalid '%s' secret data: required fields 'privatekey'", name)
+	}
+	if _, exists := secret["clientemail"]; !exists {
+		return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name)
+	}
+	if _, exists := secret["clientemail"]; !exists {
+		return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name)
+	}
+	if _, exists := secret["clientid"]; !exists {
+		return fmt.Errorf("invalid '%s' secret data: required fields 'clientid'", name)
+	}
+	if _, exists := secret["certurl"]; !exists {
+		return fmt.Errorf("invalid '%s' secret data: required fields 'certurl'", name)
+	}
+
+	return nil
 }
 
 // SetRange sets the startRange and endRange used to read the Object from
 // the bucket. It is a helper method for resumable downloads.
 func (c *GCPClient) SetRange(start, end int64) {
-	c.startRange = start
+	c.StartRange = start
-	c.endRange = end
+	c.EndRange = end
 }
 
 // BucketExists checks if the bucket with the provided name exists.
@@ -82,15 +197,18 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool,
 	return true, nil
 }
 
-// ObjectExists checks if the object with the provided name exists.
+// ObjectAttributes checks if the object with the provided name exists.
 // If it exists the Object attributes are returned.
-func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) {
+func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) {
 	attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx)
 	// ErrObjectNotExist is returned if the object does not exist
+	if err == gcpStorage.ErrObjectNotExist {
+		return false, nil, err
+	}
 	if err != nil {
 		return false, nil, err
 	}
-	return true, attrs, err
+	return true, attrs, nil
 }
 
 // FGetObject gets the object from the bucket and downloads the object locally
@@ -101,7 +219,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca
 	if err == nil {
 		// If the destination exists and is a directory.
 		if dirStatus.IsDir() {
-			return DirectoryExists
+			return ErrorDirectoryExists
 		}
 	}
 
@@ -124,17 +242,16 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca
 	// ObjectExists verifies if object exists and you have permission to access.
 	// Check if the object exists and if you have permission to access it
 	// The Object attributes are returned if the Object exists.
-	exists, attrs, err := c.ObjectExists(ctx, bucketName, objectName)
+	exists, attrs, err := c.ObjectAttributes(ctx, bucketName, objectName)
 	if err != nil {
 		return err
 	}
 	if !exists {
-		return ObjectDoesNotExist
+		return ErrorObjectDoesNotExist
 	}
 
 	// Write to a temporary file "filename.part.gcp" before saving.
-	filePartPath := localPath + attrs.Etag + ".part.gcp"
+	filePartPath := localPath + ".part.gcp"
-
 	// If exists, open in append mode. If not create it as a part file.
 	filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
 	if err != nil {
@@ -165,25 +282,25 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca
 	}
 
 	// Get Object from GCP Bucket
-	objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.startRange, c.endRange)
+	objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.StartRange, c.EndRange)
 	if err != nil {
 		return err
 	}
 	defer objectReader.Close()
 
 	// Write to the part file.
-	if _, err = io.CopyN(filePart, objectReader, attrs.Size); err != nil {
+	if _, err := io.CopyN(filePart, objectReader, attrs.Size); err != nil {
 		return err
 	}
 
 	// Close the file before rename, this is specifically needed for Windows users.
 	closeAndRemove = false
-	if err = filePart.Close(); err != nil {
+	if err := filePart.Close(); err != nil {
 		return err
 	}
 
 	// Safely completed. Now commit by renaming to actual filename.
-	if err = os.Rename(filePartPath, localPath); err != nil {
+	if err := os.Rename(filePartPath, localPath); err != nil {
 		return err
 	}
 

pkg/gcp/gcp_test.go

@@ -14,128 +14,119 @@ See the License for the specific language governing permissions and
 limitations under the License.
 */
 
-package gcp
+package gcp_test
 
 import (
 	"context"
 	"os"
 	"path/filepath"
 	"testing"
+	"time"
 
-	"gotest.tools/assert"
+	gcpStorage "cloud.google.com/go/storage"
+	"github.com/fluxcd/source-controller/pkg/gcp"
+	"github.com/fluxcd/source-controller/pkg/gcp/mocks"
+	"github.com/golang/mock/gomock"
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
-func TestNewClient(t *testing.T) {
+var (
-	// TODO: Setup GCP mock here
+	MockCtrl         *gomock.Controller
-	t.Skip()
+	MockClient       *mocks.MockClient
-	client, err := NewClient(context.Background())
+	MockBucketHandle *mocks.MockBucketHandle
-	assert.NilError(t, err)
+	MockObjectHandle *mocks.MockObjectHandle
-	assert.Assert(t, client.Client != nil)
+	bucketName       string = "test-bucket"
+	objectName       string = "test.yaml"
+	localPath        string
+)
+
+// mockgen -destination=mocks/mock_gcp_storage.go -package=mocks -source=gcp.go GCPStorageService
+func TestGCPProvider(t *testing.T) {
+	MockCtrl = gomock.NewController(GinkgoT())
+	RegisterFailHandler(Fail)
+	RunSpecs(t, "Test GCP Storage Provider Suite")
 }
 
-func TestSetRange(t *testing.T) {
+var _ = BeforeSuite(func() {
-	// TODO: Setup GCP mock here
+	MockClient = mocks.NewMockClient(MockCtrl)
-	t.Skip()
+	MockBucketHandle = mocks.NewMockBucketHandle(MockCtrl)
-	client, err := NewClient(context.Background())
+	MockObjectHandle = mocks.NewMockObjectHandle(MockCtrl)
-	assert.NilError(t, err)
-	testCases := []struct {
-		title string
-		start int64
-		end   int64
-	}{
-		{
-			title: "Test Case 1",
-			start: 1,
-			end:   5,
-		},
-		{
-			title: "Test Case 2",
-			start: 3,
-			end:   6,
-		},
-		{
-			title: "Test Case 3",
-			start: 4,
-			end:   5,
-		},
-		{
-			title: "Test Case 4",
-			start: 2,
-			end:   7,
-		},
-	}
-	for _, tt := range testCases {
-		t.Run(tt.title, func(t *testing.T) {
-			client.SetRange(tt.start, tt.end)
-			assert.Equal(t, tt.start, client.startRange)
-			assert.Equal(t, tt.end, client.endRange)
-		})
-	}
-}
-
-func TestBucketExists(t *testing.T) {
-	// TODO: Setup GCP mock here
-	t.Skip()
-	ctx := context.Background()
-	bucketName := ""
-	client, err := NewClient(ctx)
-	assert.NilError(t, err)
-	exists, err := client.BucketExists(ctx, bucketName)
-	assert.NilError(t, err)
-	assert.Assert(t, exists)
-}
-
-func TestObjectExists(t *testing.T) {
-	// TODO: Setup GCP mock here
-	t.Skip()
-	ctx := context.Background()
-	// bucketName is the name of the bucket which contains the object
-	bucketName := ""
-	// objectName is the path to the object within the bucket
-	objectName := ""
-	client, err := NewClient(ctx)
-	assert.NilError(t, err)
-	exists, attrs, err := client.ObjectExists(ctx, bucketName, objectName)
-	assert.NilError(t, err)
-	assert.Assert(t, exists)
-	assert.Assert(t, attrs != nil)
-}
-
-func TestListObjects(t *testing.T) {
-	// TODO: Setup GCP mock here
-	t.Skip()
-	ctx := context.Background()
-	// bucketName is the name of the bucket which contains the object
-	bucketName := ""
-	client, err := NewClient(ctx)
-	assert.NilError(t, err)
-	objects := client.ListObjects(ctx, bucketName, nil)
-	assert.NilError(t, err)
-	assert.Assert(t, objects != nil)
-	for {
-		object, err := objects.Next()
-		if err == IteratorDone {
-			break
-		}
-		assert.Assert(t, object != nil)
-	}
-}
-
-func TestFGetObject(t *testing.T) {
-	// TODO: Setup GCP mock here
-	t.Skip()
-	ctx := context.Background()
-	// bucketName is the name of the bucket which contains the object
-	bucketName := ""
-	// objectName is the path to the object within the bucket
-	objectName := ""
 	tempDir, err := os.MkdirTemp("", bucketName)
 	if err != nil {
-		assert.NilError(t, err)
+		Expect(err).ToNot(HaveOccurred())
 	}
-	localPath := filepath.Join(tempDir, objectName)
+	localPath = filepath.Join(tempDir, objectName)
-	client, err := NewClient(ctx)
+	MockClient.EXPECT().Bucket(bucketName).Return(MockBucketHandle).AnyTimes()
-	assert.NilError(t, err)
+	MockBucketHandle.EXPECT().Object(objectName).Return(&gcpStorage.ObjectHandle{}).AnyTimes()
-	objErr := client.FGetObject(ctx, bucketName, objectName, localPath)
+	MockBucketHandle.EXPECT().Attrs(context.Background()).Return(&gcpStorage.BucketAttrs{
-	assert.NilError(t, objErr)
+		Name:    bucketName,
-}
+		Created: time.Now(),
+		Etag:    "test-etag",
+	}, nil).AnyTimes()
+	MockBucketHandle.EXPECT().Objects(gomock.Any(), nil).Return(&gcpStorage.ObjectIterator{}).AnyTimes()
+	MockObjectHandle.EXPECT().Attrs(gomock.Any()).Return(&gcpStorage.ObjectAttrs{
+		Bucket:      bucketName,
+		Name:        objectName,
+		ContentType: "text/x-yaml",
+		Etag:        "test-etag",
+		Size:        125,
+		Created:     time.Now(),
+	}, nil).AnyTimes()
+	MockObjectHandle.EXPECT().NewRangeReader(gomock.Any(), 10, 125).Return(&gcpStorage.Reader{}, nil).AnyTimes()
+})
+
+var _ = Describe("GCP Storage Provider", func() {
+	Describe("Get GCP Storage Provider client from gcp", func() {
+
+		Context("Gcp storage Bucket - BucketExists", func() {
+			It("should not return an error when fetching gcp storage bucket", func() {
+				gcpClient := &gcp.GCPClient{
+					Client:     MockClient,
+					StartRange: 0,
+					EndRange:   -1,
+				}
+				exists, err := gcpClient.BucketExists(context.Background(), bucketName)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(exists).To(BeTrue())
+			})
+		})
+		Context("Gcp storage Bucket - FGetObject", func() {
+			It("should get the object from the bucket and download the object locally", func() {
+				gcpClient := &gcp.GCPClient{
+					Client:     MockClient,
+					StartRange: 0,
+					EndRange:   -1,
+				}
+				err := gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath)
+				Expect(err).ToNot(HaveOccurred())
+			})
+		})
+		Context("Gcp storage Bucket - ObjectAttributes", func() {
+			It("should get the object attributes", func() {
+				gcpClient := &gcp.GCPClient{
+					Client:     MockClient,
+					StartRange: 0,
+					EndRange:   -1,
+				}
+				exists, attrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName)
+				Expect(err).ToNot(HaveOccurred())
+				Expect(exists).To(BeTrue())
+				Expect(attrs).ToNot(BeNil())
+			})
+
+			Context("Gcp storage Bucket - SetRange", func() {
+				It("should set the range of the io reader seeker for the file download", func() {
+					gcpClient := &gcp.GCPClient{
+						Client:     MockClient,
+						StartRange: 0,
+						EndRange:   -1,
+					}
+					gcpClient.SetRange(2, 5)
+					Expect(gcpClient.StartRange).To(Equal(int64(2)))
+					Expect(gcpClient.EndRange).To(Equal(int64(5)))
+				})
+			})
+		})
+	})
+})

pkg/gcp/mocks/mock_gcp_storage.go

@@ -0,0 +1,211 @@
+// Code generated by MockGen. DO NOT EDIT.
+// Source: gcp.go
+
+// Package mocks is a generated GoMock package.
+package mocks
+
+import (
+	context "context"
+	reflect "reflect"
+
+	storage "cloud.google.com/go/storage"
+	gomock "github.com/golang/mock/gomock"
+)
+
+// MockClient is a mock of Client interface.
+type MockClient struct {
+	ctrl     *gomock.Controller
+	recorder *MockClientMockRecorder
+}
+
+// MockClientMockRecorder is the mock recorder for MockClient.
+type MockClientMockRecorder struct {
+	mock *MockClient
+}
+
+// NewMockClient creates a new mock instance.
+func NewMockClient(ctrl *gomock.Controller) *MockClient {
+	mock := &MockClient{ctrl: ctrl}
+	mock.recorder = &MockClientMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockClient) EXPECT() *MockClientMockRecorder {
+	return m.recorder
+}
+
+// Bucket mocks base method.
+func (m *MockClient) Bucket(arg0 string) *storage.BucketHandle {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Bucket", arg0)
+	ret0, _ := ret[0].(*storage.BucketHandle)
+	return ret0
+}
+
+// Bucket indicates an expected call of Bucket.
+func (mr *MockClientMockRecorder) Bucket(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockClient)(nil).Bucket), arg0)
+}
+
+// Close mocks base method.
+func (m *MockClient) Close() error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Close")
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Close indicates an expected call of Close.
+func (mr *MockClientMockRecorder) Close() *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClient)(nil).Close))
+}
+
+// MockBucketHandle is a mock of BucketHandle interface.
+type MockBucketHandle struct {
+	ctrl     *gomock.Controller
+	recorder *MockBucketHandleMockRecorder
+}
+
+// MockBucketHandleMockRecorder is the mock recorder for MockBucketHandle.
+type MockBucketHandleMockRecorder struct {
+	mock *MockBucketHandle
+}
+
+// NewMockBucketHandle creates a new mock instance.
+func NewMockBucketHandle(ctrl *gomock.Controller) *MockBucketHandle {
+	mock := &MockBucketHandle{ctrl: ctrl}
+	mock.recorder = &MockBucketHandleMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockBucketHandle) EXPECT() *MockBucketHandleMockRecorder {
+	return m.recorder
+}
+
+// Attrs mocks base method.
+func (m *MockBucketHandle) Attrs(arg0 context.Context) (*storage.BucketAttrs, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Attrs", arg0)
+	ret0, _ := ret[0].(*storage.BucketAttrs)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Attrs indicates an expected call of Attrs.
+func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0)
+}
+
+// Create mocks base method.
+func (m *MockBucketHandle) Create(arg0 context.Context, arg1 string, arg2 *storage.BucketAttrs) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Create indicates an expected call of Create.
+func (mr *MockBucketHandleMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockBucketHandle)(nil).Create), arg0, arg1, arg2)
+}
+
+// Delete mocks base method.
+func (m *MockBucketHandle) Delete(arg0 context.Context) error {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Delete", arg0)
+	ret0, _ := ret[0].(error)
+	return ret0
+}
+
+// Delete indicates an expected call of Delete.
+func (mr *MockBucketHandleMockRecorder) Delete(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBucketHandle)(nil).Delete), arg0)
+}
+
+// Object mocks base method.
+func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Object", arg0)
+	ret0, _ := ret[0].(*storage.ObjectHandle)
+	return ret0
+}
+
+// Object indicates an expected call of Object.
+func (mr *MockBucketHandleMockRecorder) Object(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Object", reflect.TypeOf((*MockBucketHandle)(nil).Object), arg0)
+}
+
+// Objects mocks base method.
+func (m *MockBucketHandle) Objects(arg0 context.Context, arg1 *storage.Query) *storage.ObjectIterator {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Objects", arg0, arg1)
+	ret0, _ := ret[0].(*storage.ObjectIterator)
+	return ret0
+}
+
+// Objects indicates an expected call of Objects.
+func (mr *MockBucketHandleMockRecorder) Objects(arg0, arg1 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Objects", reflect.TypeOf((*MockBucketHandle)(nil).Objects), arg0, arg1)
+}
+
+// MockObjectHandle is a mock of ObjectHandle interface.
+type MockObjectHandle struct {
+	ctrl     *gomock.Controller
+	recorder *MockObjectHandleMockRecorder
+}
+
+// MockObjectHandleMockRecorder is the mock recorder for MockObjectHandle.
+type MockObjectHandleMockRecorder struct {
+	mock *MockObjectHandle
+}
+
+// NewMockObjectHandle creates a new mock instance.
+func NewMockObjectHandle(ctrl *gomock.Controller) *MockObjectHandle {
+	mock := &MockObjectHandle{ctrl: ctrl}
+	mock.recorder = &MockObjectHandleMockRecorder{mock}
+	return mock
+}
+
+// EXPECT returns an object that allows the caller to indicate expected use.
+func (m *MockObjectHandle) EXPECT() *MockObjectHandleMockRecorder {
+	return m.recorder
+}
+
+// Attrs mocks base method.
+func (m *MockObjectHandle) Attrs(arg0 context.Context) (*storage.ObjectAttrs, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "Attrs", arg0)
+	ret0, _ := ret[0].(*storage.ObjectAttrs)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// Attrs indicates an expected call of Attrs.
+func (mr *MockObjectHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockObjectHandle)(nil).Attrs), arg0)
+}
+
+// NewRangeReader mocks base method.
+func (m *MockObjectHandle) NewRangeReader(arg0 context.Context, arg1, arg2 int64) (*storage.Reader, error) {
+	m.ctrl.T.Helper()
+	ret := m.ctrl.Call(m, "NewRangeReader", arg0, arg1, arg2)
+	ret0, _ := ret[0].(*storage.Reader)
+	ret1, _ := ret[1].(error)
+	return ret0, ret1
+}
+
+// NewRangeReader indicates an expected call of NewRangeReader.
+func (mr *MockObjectHandleMockRecorder) NewRangeReader(arg0, arg1, arg2 interface{}) *gomock.Call {
+	mr.mock.ctrl.T.Helper()
+	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewRangeReader", reflect.TypeOf((*MockObjectHandle)(nil).NewRangeReader), arg0, arg1, arg2)
+}