fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Service Account Key Authentication to GCP Provider 

Signed-off-by: pa250194 <pa250194@ncr.com>
This commit is contained in:
pa250194 2021-09-10 16:01:16 -05:00
parent 90395f426a
commit 0444c6e16d
8 changed files with 512 additions and 176 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
api/v1beta1/bucket_types.go
View File

@ -30,7 +30,7 @@ const (
// BucketSpec defines the desired state of an S3 compatible bucket
type BucketSpec struct {
// The S3 compatible storage provider name, default ('generic').
// +kubebuilder:validation:Enum=generic;aws
// +kubebuilder:validation:Enum=generic;aws;gcp
// +kubebuilder:default:=generic
// +optional
Provider string `json:"provider,omitempty"`

1
config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml
View File

@ -66,6 +66,7 @@ spec:
enum:
- generic
- aws
- gcp
type: string
region:
description: The bucket region.

96
controllers/bucket_controller.go
View File

@ -177,16 +177,21 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr
}
func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, error) {
var tempDir string
var err error
var sourceBucket sourcev1.Bucket
tempDir, err := os.MkdirTemp("", bucket.Name)
if err != nil {
err = fmt.Errorf("tmp dir error: %w", err)
return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err
}
defer os.RemoveAll(tempDir)
if bucket.Spec.Provider == sourcev1.GoogleBucketProvider {
sourceBucket, tempDir, err = r.reconcileWithGCP(ctx, bucket)
sourceBucket, err = r.reconcileWithGCP(ctx, bucket, tempDir)
if err != nil {
return sourceBucket, err
}
} else {
sourceBucket, tempDir, err = r.reconcileWithMinio(ctx, bucket)
sourceBucket, err = r.reconcileWithMinio(ctx, bucket, tempDir)
if err != nil {
return sourceBucket, err
}
@ -261,41 +266,36 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1.
// reconcileWithGCP handles getting objects from a Google Cloud Platform bucket
// using a gcp client
func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) {
func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) {
gcpClient, err := r.authGCP(ctx, bucket)
if err != nil {
err = fmt.Errorf("auth error: %w", err)
return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err
}
defer gcpClient.Client.Close()
// create tmp dir
tempDir, err := os.MkdirTemp("", bucket.Name)
if err != nil {
err = fmt.Errorf("tmp dir error: %w", err)
return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err
}
defer os.RemoveAll(tempDir)
ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration)
defer cancel()
exists, err := gcpClient.BucketExists(ctxTimeout, bucket.Spec.BucketName)
if err != nil {
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
if !exists {
err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName)
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
// Look for file with ignore rules first.
path := filepath.Join(tempDir, sourceignore.IgnoreFile)
if err := gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path); err != nil {
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
if err == gcp.ErrorObjectDoesNotExist && sourceignore.IgnoreFile != ".sourceignore" {
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
}
ps, err := sourceignore.ReadIgnoreFile(path, nil)
if err != nil {
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
// In-spec patterns take precedence
if bucket.Spec.Ignore != nil {
@ -311,7 +311,7 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1
}
if err != nil {
err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, err)
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
if strings.HasSuffix(object.Name, "/") || object.Name == sourceignore.IgnoreFile {
@ -323,42 +323,33 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1
}
localPath := filepath.Join(tempDir, object.Name)
// FGetObject - get and download bucket object
if err = gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Name, localPath); err != nil {
err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err)
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
}
return sourcev1.Bucket{}, tempDir, nil
return sourcev1.Bucket{}, nil
}
// reconcileWithMinio handles getting objects from an S3 compatible bucket
// using a minio client
func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) {
func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) {
s3Client, err := r.authMinio(ctx, bucket)
if err != nil {
err = fmt.Errorf("auth error: %w", err)
return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err
}
// create tmp dir
tempDir, err := os.MkdirTemp("", bucket.Name)
if err != nil {
err = fmt.Errorf("tmp dir error: %w", err)
return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err
}
defer os.RemoveAll(tempDir)
ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration)
defer cancel()
exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName)
if err != nil {
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
if !exists {
err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName)
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
// Look for file with ignore rules first
@ -367,12 +358,12 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source
path := filepath.Join(tempDir, sourceignore.IgnoreFile)
if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil {
if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" {
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
}
ps, err := sourceignore.ReadIgnoreFile(path, nil)
if err != nil {
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
// In-spec patterns take precedence
if bucket.Spec.Ignore != nil {
@ -387,7 +378,7 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source
}) {
if object.Err != nil {
err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err)
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile {
@ -402,20 +393,43 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source
err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{})
if err != nil {
err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err)
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err
return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
}
}
return sourcev1.Bucket{}, tempDir, nil
return sourcev1.Bucket{}, nil
}
// authGCP creates a new Google Cloud Platform storage client
// to interact with the Storage service.
// to interact with the storage service.
func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) (*gcp.GCPClient, error) {
client, err := gcp.NewClient(ctx)
if err != nil {
return nil, err
var client *gcp.GCPClient
var err error
if bucket.Spec.SecretRef != nil {
secretName := types.NamespacedName{
Namespace: bucket.GetNamespace(),
Name: bucket.Spec.SecretRef.Name,
}
var secret corev1.Secret
if err := r.Get(ctx, secretName, &secret); err != nil {
return nil, fmt.Errorf("credentials secret error: %w", err)
}
if err := gcp.ValidateSecret(secret.Data, secret.Name); err != nil {
return nil, err
}
serviceAccount := gcp.InitCredentialsWithSecret(secret.Data)
client, err = gcp.NewClientWithSAKey(ctx, serviceAccount)
if err != nil {
return nil, err
}
} else {
client, err = gcp.NewClient(ctx)
if err != nil {
return nil, err
}
}
return client, nil
}
// authMinio creates a new Minio client to interact with S3

1
go.mod
View File

@ -23,6 +23,7 @@ require (
github.com/go-git/go-git/v5 v5.4.2
github.com/go-logr/logr v0.4.0
github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
github.com/golang/mock v1.6.0 // indirect
github.com/googleapis/gax-go/v2 v2.1.0 // indirect
github.com/libgit2/git2go/v31 v31.4.14
github.com/minio/minio-go/v7 v7.0.10

1
go.sum
View File

@ -415,6 +415,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt
github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw=
github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4=
github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8=
github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc=
github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs=
github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=

163
pkg/gcp/gcp.go
View File

@ -18,56 +18,171 @@ package gcp
import (
"context"
"encoding/json"
"errors"
"fmt"
"io"
"os"
"path/filepath"
gcpStorage "cloud.google.com/go/storage"
interator "google.golang.org/api/iterator"
"google.golang.org/api/option"
)
const (
ServiceAccount = "service_account"
AuthUri = "https://accounts.google.com/o/oauth2/auth"
TokenUri = "https://oauth2.googleapis.com/token"
AuthProviderX509CertUrl = "https://www.googleapis.com/oauth2/v1/certs"
)
var (
// IteratorDone is returned when the looping of objects/content
// has reached the end of the iteration.
IteratorDone = interator.Done
// DirectoryExists is an error returned when the filename provided
// ErrorDirectoryExists is an error returned when the filename provided
// is a directory.
DirectoryExists = errors.New("filename is a directory")
// ObjectDoesNotExist is an error returned when the object whose name
ErrorDirectoryExists = errors.New("filename is a directory")
// ErrorObjectDoesNotExist is an error returned when the object whose name
// is provided does not exist.
ObjectDoesNotExist = errors.New("object does not exist")
ErrorObjectDoesNotExist = errors.New("object does not exist")
)
type Client interface {
Bucket(string) *gcpStorage.BucketHandle
Close() error
}
type BucketHandle interface {
Create(context.Context, string, *gcpStorage.BucketAttrs) error
Delete(context.Context) error
Attrs(context.Context) (*gcpStorage.BucketAttrs, error)
Object(string) *gcpStorage.ObjectHandle
Objects(context.Context, *gcpStorage.Query) *gcpStorage.ObjectIterator
}
type ObjectHandle interface {
Attrs(context.Context) (*gcpStorage.ObjectAttrs, error)
NewRangeReader(context.Context, int64, int64) (*gcpStorage.Reader, error)
}
type GCPClient struct {
// client for interacting with the Google Cloud
// Storage APIs.
Client *gcpStorage.Client
Client Client
// startRange is the starting read value for
// reading the object from bucket.
startRange int64
StartRange int64
// endRange is the ending read value for
// reading the object from bucket.
endRange int64
EndRange int64
}
// CredentialsFile struct representing the GCP Service Account
// JSON file.
type CredentialsFile struct {
Type string `json:"type"`
ProjectID string `json:"project_id"`
PrivateKeyID string `json:"private_key_id"`
PrivateKey string `json:"private_key"`
ClientEmail string `json:"client_email"`
ClientID string `json:"client_id"`
AuthUri string `json:"auth_uri"`
TokenUri string `json:"token_uri"`
AuthProviderX509CertUrl string `json:"auth_provider_x509_cert_url"`
ClientX509CertUrl string `json:"client_x509_cert_url"`
}
// NewClient creates a new GCP storage client
// The Google Storage Client will automatically
// look for the Google Application Credential environment variable
// or look for the Google Application Credential file
// or look for the Google Application Credential file.
func NewClient(ctx context.Context) (*GCPClient, error) {
client, err := gcpStorage.NewClient(ctx)
if err != nil {
return nil, err
}
return &GCPClient{Client: client, startRange: 0, endRange: -1}, nil
return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil
}
// NewClientWithSAKey creates a new GCP storage client
// It uses the provided JSON file with service account details
// To authenticate.
func NewClientWithSAKey(ctx context.Context, credentials *CredentialsFile) (*GCPClient, error) {
saAccount, err := credentials.credentailsToJSON()
if err != nil {
return nil, err
}
client, err := gcpStorage.NewClient(ctx, option.WithCredentialsJSON(saAccount))
if err != nil {
return nil, err
}
return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil
}
// credentailsToJSON converts GCP service account credentials struct to JSON.
func (credentials *CredentialsFile) credentailsToJSON() ([]byte, error) {
credentialsJSON, err := json.Marshal(credentials)
if err != nil {
return nil, err
}
return credentialsJSON, nil
}
// InitCredentialsWithSecret creates a new credential
// by initializing a new CredentialsFile struct
func InitCredentialsWithSecret(secret map[string][]byte) *CredentialsFile {
return &CredentialsFile{
Type: ServiceAccount,
ProjectID: string(secret["projectid"]),
PrivateKeyID: string(secret["privatekeyid"]),
PrivateKey: string(secret["privatekey"]),
ClientEmail: string(secret["clientemail"]),
ClientID: string(secret["clientid"]),
AuthUri: AuthUri,
TokenUri: TokenUri,
AuthProviderX509CertUrl: AuthProviderX509CertUrl,
ClientX509CertUrl: string(secret["certurl"]),
}
}
// ValidateSecret validates the credential secrets
// It ensures that needed secret fields are not missing.
func ValidateSecret(secret map[string][]byte, name string) error {
if _, exists := secret["projectid"]; !exists {
return fmt.Errorf("invalid '%s' secret data: required fields 'projectid'", name)
}
if _, exists := secret["privatekeyid"]; !exists {
return fmt.Errorf("invalid '%s' secret data: required fields 'privatekeyid'", name)
}
if _, exists := secret["privatekey"]; !exists {
return fmt.Errorf("invalid '%s' secret data: required fields 'privatekey'", name)
}
if _, exists := secret["clientemail"]; !exists {
return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name)
}
if _, exists := secret["clientemail"]; !exists {
return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name)
}
if _, exists := secret["clientid"]; !exists {
return fmt.Errorf("invalid '%s' secret data: required fields 'clientid'", name)
}
if _, exists := secret["certurl"]; !exists {
return fmt.Errorf("invalid '%s' secret data: required fields 'certurl'", name)
}
return nil
}
// SetRange sets the startRange and endRange used to read the Object from
// the bucket. It is a helper method for resumable downloads.
func (c *GCPClient) SetRange(start, end int64) {
c.startRange = start
c.endRange = end
c.StartRange = start
c.EndRange = end
}
// BucketExists checks if the bucket with the provided name exists.
@ -82,15 +197,18 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool,
return true, nil
}
// ObjectExists checks if the object with the provided name exists.
// ObjectAttributes checks if the object with the provided name exists.
// If it exists the Object attributes are returned.
func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) {
func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) {
attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx)
// ErrObjectNotExist is returned if the object does not exist
if err == gcpStorage.ErrObjectNotExist {
return false, nil, err
}
if err != nil {
return false, nil, err
}
return true, attrs, err
return true, attrs, nil
}
// FGetObject gets the object from the bucket and downloads the object locally
@ -101,7 +219,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca
if err == nil {
// If the destination exists and is a directory.
if dirStatus.IsDir() {
return DirectoryExists
return ErrorDirectoryExists
}
}
@ -124,17 +242,16 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca
// ObjectExists verifies if object exists and you have permission to access.
// Check if the object exists and if you have permission to access it
// The Object attributes are returned if the Object exists.
exists, attrs, err := c.ObjectExists(ctx, bucketName, objectName)
exists, attrs, err := c.ObjectAttributes(ctx, bucketName, objectName)
if err != nil {
return err
}
if !exists {
return ObjectDoesNotExist
return ErrorObjectDoesNotExist
}
// Write to a temporary file "filename.part.gcp" before saving.
filePartPath := localPath + attrs.Etag + ".part.gcp"
filePartPath := localPath + ".part.gcp"
// If exists, open in append mode. If not create it as a part file.
filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600)
if err != nil {
@ -165,25 +282,25 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca
}
// Get Object from GCP Bucket
objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.startRange, c.endRange)
objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.StartRange, c.EndRange)
if err != nil {
return err
}
defer objectReader.Close()
// Write to the part file.
if _, err = io.CopyN(filePart, objectReader, attrs.Size); err != nil {
if _, err := io.CopyN(filePart, objectReader, attrs.Size); err != nil {
return err
}
// Close the file before rename, this is specifically needed for Windows users.
closeAndRemove = false
if err = filePart.Close(); err != nil {
if err := filePart.Close(); err != nil {
return err
}
// Safely completed. Now commit by renaming to actual filename.
if err = os.Rename(filePartPath, localPath); err != nil {
if err := os.Rename(filePartPath, localPath); err != nil {
return err
}

213
pkg/gcp/gcp_test.go
View File

@ -14,128 +14,119 @@ See the License for the specific language governing permissions and
limitations under the License.
*/
package gcp
package gcp_test
import (
"context"
"os"
"path/filepath"
"testing"
"time"
"gotest.tools/assert"
gcpStorage "cloud.google.com/go/storage"
"github.com/fluxcd/source-controller/pkg/gcp"
"github.com/fluxcd/source-controller/pkg/gcp/mocks"
"github.com/golang/mock/gomock"
. "github.com/onsi/ginkgo"
. "github.com/onsi/gomega"
)
func TestNewClient(t *testing.T) {
// TODO: Setup GCP mock here
t.Skip()
client, err := NewClient(context.Background())
assert.NilError(t, err)
assert.Assert(t, client.Client != nil)
var (
MockCtrl *gomock.Controller
MockClient *mocks.MockClient
MockBucketHandle *mocks.MockBucketHandle
MockObjectHandle *mocks.MockObjectHandle
bucketName string = "test-bucket"
objectName string = "test.yaml"
localPath string
)
// mockgen -destination=mocks/mock_gcp_storage.go -package=mocks -source=gcp.go GCPStorageService
func TestGCPProvider(t *testing.T) {
MockCtrl = gomock.NewController(GinkgoT())
RegisterFailHandler(Fail)
RunSpecs(t, "Test GCP Storage Provider Suite")
}
func TestSetRange(t *testing.T) {
// TODO: Setup GCP mock here
t.Skip()
client, err := NewClient(context.Background())
assert.NilError(t, err)
testCases := []struct {
title string
start int64
end int64
}{
{
title: "Test Case 1",
start: 1,
end: 5,
},
{
title: "Test Case 2",
start: 3,
end: 6,
},
{
title: "Test Case 3",
start: 4,
end: 5,
},
{
title: "Test Case 4",
start: 2,
end: 7,
},
}
for _, tt := range testCases {
t.Run(tt.title, func(t *testing.T) {
client.SetRange(tt.start, tt.end)
assert.Equal(t, tt.start, client.startRange)
assert.Equal(t, tt.end, client.endRange)
})
}
}
func TestBucketExists(t *testing.T) {
// TODO: Setup GCP mock here
t.Skip()
ctx := context.Background()
bucketName := ""
client, err := NewClient(ctx)
assert.NilError(t, err)
exists, err := client.BucketExists(ctx, bucketName)
assert.NilError(t, err)
assert.Assert(t, exists)
}
func TestObjectExists(t *testing.T) {
// TODO: Setup GCP mock here
t.Skip()
ctx := context.Background()
// bucketName is the name of the bucket which contains the object
bucketName := ""
// objectName is the path to the object within the bucket
objectName := ""
client, err := NewClient(ctx)
assert.NilError(t, err)
exists, attrs, err := client.ObjectExists(ctx, bucketName, objectName)
assert.NilError(t, err)
assert.Assert(t, exists)
assert.Assert(t, attrs != nil)
}
func TestListObjects(t *testing.T) {
// TODO: Setup GCP mock here
t.Skip()
ctx := context.Background()
// bucketName is the name of the bucket which contains the object
bucketName := ""
client, err := NewClient(ctx)
assert.NilError(t, err)
objects := client.ListObjects(ctx, bucketName, nil)
assert.NilError(t, err)
assert.Assert(t, objects != nil)
for {
object, err := objects.Next()
if err == IteratorDone {
break
}
assert.Assert(t, object != nil)
}
}
func TestFGetObject(t *testing.T) {
// TODO: Setup GCP mock here
t.Skip()
ctx := context.Background()
// bucketName is the name of the bucket which contains the object
bucketName := ""
// objectName is the path to the object within the bucket
objectName := ""
var _ = BeforeSuite(func() {
MockClient = mocks.NewMockClient(MockCtrl)
MockBucketHandle = mocks.NewMockBucketHandle(MockCtrl)
MockObjectHandle = mocks.NewMockObjectHandle(MockCtrl)
tempDir, err := os.MkdirTemp("", bucketName)
if err != nil {
assert.NilError(t, err)
Expect(err).ToNot(HaveOccurred())
}
localPath := filepath.Join(tempDir, objectName)
client, err := NewClient(ctx)
assert.NilError(t, err)
objErr := client.FGetObject(ctx, bucketName, objectName, localPath)
assert.NilError(t, objErr)
}
localPath = filepath.Join(tempDir, objectName)
MockClient.EXPECT().Bucket(bucketName).Return(MockBucketHandle).AnyTimes()
MockBucketHandle.EXPECT().Object(objectName).Return(&gcpStorage.ObjectHandle{}).AnyTimes()
MockBucketHandle.EXPECT().Attrs(context.Background()).Return(&gcpStorage.BucketAttrs{
Name: bucketName,
Created: time.Now(),
Etag: "test-etag",
}, nil).AnyTimes()
MockBucketHandle.EXPECT().Objects(gomock.Any(), nil).Return(&gcpStorage.ObjectIterator{}).AnyTimes()
MockObjectHandle.EXPECT().Attrs(gomock.Any()).Return(&gcpStorage.ObjectAttrs{
Bucket: bucketName,
Name: objectName,
ContentType: "text/x-yaml",
Etag: "test-etag",
Size: 125,
Created: time.Now(),
}, nil).AnyTimes()
MockObjectHandle.EXPECT().NewRangeReader(gomock.Any(), 10, 125).Return(&gcpStorage.Reader{}, nil).AnyTimes()
})
var _ = Describe("GCP Storage Provider", func() {
Describe("Get GCP Storage Provider client from gcp", func() {
Context("Gcp storage Bucket - BucketExists", func() {
It("should not return an error when fetching gcp storage bucket", func() {
gcpClient := &gcp.GCPClient{
Client: MockClient,
StartRange: 0,
EndRange: -1,
}
exists, err := gcpClient.BucketExists(context.Background(), bucketName)
Expect(err).ToNot(HaveOccurred())
Expect(exists).To(BeTrue())
})
})
Context("Gcp storage Bucket - FGetObject", func() {
It("should get the object from the bucket and download the object locally", func() {
gcpClient := &gcp.GCPClient{
Client: MockClient,
StartRange: 0,
EndRange: -1,
}
err := gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath)
Expect(err).ToNot(HaveOccurred())
})
})
Context("Gcp storage Bucket - ObjectAttributes", func() {
It("should get the object attributes", func() {
gcpClient := &gcp.GCPClient{
Client: MockClient,
StartRange: 0,
EndRange: -1,
}
exists, attrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName)
Expect(err).ToNot(HaveOccurred())
Expect(exists).To(BeTrue())
Expect(attrs).ToNot(BeNil())
})
Context("Gcp storage Bucket - SetRange", func() {
It("should set the range of the io reader seeker for the file download", func() {
gcpClient := &gcp.GCPClient{
Client: MockClient,
StartRange: 0,
EndRange: -1,
}
gcpClient.SetRange(2, 5)
Expect(gcpClient.StartRange).To(Equal(int64(2)))
Expect(gcpClient.EndRange).To(Equal(int64(5)))
})
})
})
})
})

211
pkg/gcp/mocks/mock_gcp_storage.go Normal file
View File

@ -0,0 +1,211 @@
// Code generated by MockGen. DO NOT EDIT.
// Source: gcp.go
// Package mocks is a generated GoMock package.
package mocks
import (
context "context"
reflect "reflect"
storage "cloud.google.com/go/storage"
gomock "github.com/golang/mock/gomock"
)
// MockClient is a mock of Client interface.
type MockClient struct {
ctrl *gomock.Controller
recorder *MockClientMockRecorder
}
// MockClientMockRecorder is the mock recorder for MockClient.
type MockClientMockRecorder struct {
mock *MockClient
}
// NewMockClient creates a new mock instance.
func NewMockClient(ctrl *gomock.Controller) *MockClient {
mock := &MockClient{ctrl: ctrl}
mock.recorder = &MockClientMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockClient) EXPECT() *MockClientMockRecorder {
return m.recorder
}
// Bucket mocks base method.
func (m *MockClient) Bucket(arg0 string) *storage.BucketHandle {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Bucket", arg0)
ret0, _ := ret[0].(*storage.BucketHandle)
return ret0
}
// Bucket indicates an expected call of Bucket.
func (mr *MockClientMockRecorder) Bucket(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockClient)(nil).Bucket), arg0)
}
// Close mocks base method.
func (m *MockClient) Close() error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Close")
ret0, _ := ret[0].(error)
return ret0
}
// Close indicates an expected call of Close.
func (mr *MockClientMockRecorder) Close() *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClient)(nil).Close))
}
// MockBucketHandle is a mock of BucketHandle interface.
type MockBucketHandle struct {
ctrl *gomock.Controller
recorder *MockBucketHandleMockRecorder
}
// MockBucketHandleMockRecorder is the mock recorder for MockBucketHandle.
type MockBucketHandleMockRecorder struct {
mock *MockBucketHandle
}
// NewMockBucketHandle creates a new mock instance.
func NewMockBucketHandle(ctrl *gomock.Controller) *MockBucketHandle {
mock := &MockBucketHandle{ctrl: ctrl}
mock.recorder = &MockBucketHandleMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockBucketHandle) EXPECT() *MockBucketHandleMockRecorder {
return m.recorder
}
// Attrs mocks base method.
func (m *MockBucketHandle) Attrs(arg0 context.Context) (*storage.BucketAttrs, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Attrs", arg0)
ret0, _ := ret[0].(*storage.BucketAttrs)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Attrs indicates an expected call of Attrs.
func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0)
}
// Create mocks base method.
func (m *MockBucketHandle) Create(arg0 context.Context, arg1 string, arg2 *storage.BucketAttrs) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2)
ret0, _ := ret[0].(error)
return ret0
}
// Create indicates an expected call of Create.
func (mr *MockBucketHandleMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockBucketHandle)(nil).Create), arg0, arg1, arg2)
}
// Delete mocks base method.
func (m *MockBucketHandle) Delete(arg0 context.Context) error {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Delete", arg0)
ret0, _ := ret[0].(error)
return ret0
}
// Delete indicates an expected call of Delete.
func (mr *MockBucketHandleMockRecorder) Delete(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBucketHandle)(nil).Delete), arg0)
}
// Object mocks base method.
func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Object", arg0)
ret0, _ := ret[0].(*storage.ObjectHandle)
return ret0
}
// Object indicates an expected call of Object.
func (mr *MockBucketHandleMockRecorder) Object(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Object", reflect.TypeOf((*MockBucketHandle)(nil).Object), arg0)
}
// Objects mocks base method.
func (m *MockBucketHandle) Objects(arg0 context.Context, arg1 *storage.Query) *storage.ObjectIterator {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Objects", arg0, arg1)
ret0, _ := ret[0].(*storage.ObjectIterator)
return ret0
}
// Objects indicates an expected call of Objects.
func (mr *MockBucketHandleMockRecorder) Objects(arg0, arg1 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Objects", reflect.TypeOf((*MockBucketHandle)(nil).Objects), arg0, arg1)
}
// MockObjectHandle is a mock of ObjectHandle interface.
type MockObjectHandle struct {
ctrl *gomock.Controller
recorder *MockObjectHandleMockRecorder
}
// MockObjectHandleMockRecorder is the mock recorder for MockObjectHandle.
type MockObjectHandleMockRecorder struct {
mock *MockObjectHandle
}
// NewMockObjectHandle creates a new mock instance.
func NewMockObjectHandle(ctrl *gomock.Controller) *MockObjectHandle {
mock := &MockObjectHandle{ctrl: ctrl}
mock.recorder = &MockObjectHandleMockRecorder{mock}
return mock
}
// EXPECT returns an object that allows the caller to indicate expected use.
func (m *MockObjectHandle) EXPECT() *MockObjectHandleMockRecorder {
return m.recorder
}
// Attrs mocks base method.
func (m *MockObjectHandle) Attrs(arg0 context.Context) (*storage.ObjectAttrs, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "Attrs", arg0)
ret0, _ := ret[0].(*storage.ObjectAttrs)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// Attrs indicates an expected call of Attrs.
func (mr *MockObjectHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockObjectHandle)(nil).Attrs), arg0)
}
// NewRangeReader mocks base method.
func (m *MockObjectHandle) NewRangeReader(arg0 context.Context, arg1, arg2 int64) (*storage.Reader, error) {
m.ctrl.T.Helper()
ret := m.ctrl.Call(m, "NewRangeReader", arg0, arg1, arg2)
ret0, _ := ret[0].(*storage.Reader)
ret1, _ := ret[1].(error)
return ret0, ret1
}
// NewRangeReader indicates an expected call of NewRangeReader.
func (mr *MockObjectHandleMockRecorder) NewRangeReader(arg0, arg1, arg2 interface{}) *gomock.Call {
mr.mock.ctrl.T.Helper()
return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewRangeReader", reflect.TypeOf((*MockObjectHandle)(nil).NewRangeReader), arg0, arg1, arg2)
}