Added Tests to GCP provider

Signed-off-by: pa250194 <pa250194@ncr.com>

pkg/gcp/gcp.go

@@ -49,26 +49,10 @@ var (
 	ErrorObjectDoesNotExist = errors.New("object does not exist")
 )
 
-type Client interface {
-	Bucket(string) *gcpStorage.BucketHandle
-	Close() error
-}
-
-type BucketHandle interface {
-	Attrs(context.Context) (*gcpStorage.BucketAttrs, error)
-	Object(string) *gcpStorage.ObjectHandle
-	Objects(context.Context, *gcpStorage.Query) *gcpStorage.ObjectIterator
-}
-
-type ObjectHandle interface {
-	Attrs(context.Context) (*gcpStorage.ObjectAttrs, error)
-	NewRangeReader(context.Context, int64, int64) (*gcpStorage.Reader, error)
-}
-
 type GCPClient struct {
 	// client for interacting with the Google Cloud
 	// Storage APIs.
-	Client
+	*gcpStorage.Client
 	// startRange is the starting read value for
 	// reading the object from bucket.
 	StartRange int64

pkg/gcp/gcp_test.go

@@ -18,6 +18,15 @@ package gcp_test
 
 import (
 	"context"
+	"crypto/tls"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
+	"log"
+	"net"
+	"net/http"
+	"net/http/httptest"
 	"os"
 	"path/filepath"
 	"testing"
@@ -25,108 +34,253 @@ import (
 
 	gcpStorage "cloud.google.com/go/storage"
 	"github.com/fluxcd/source-controller/pkg/gcp"
-	"github.com/fluxcd/source-controller/pkg/gcp/mocks"
+	"google.golang.org/api/googleapi"
-	"github.com/golang/mock/gomock"
+	raw "google.golang.org/api/storage/v1"
-	. "github.com/onsi/ginkgo"
+	"gotest.tools/assert"
-	. "github.com/onsi/gomega"
+
+	"google.golang.org/api/option"
+)
+
+const (
+	bucketName string = "test-bucket"
+	objectName string = "test.yaml"
 )
 
 var (
-	MockCtrl         *gomock.Controller
+	Client *gcpStorage.Client
-	MockClient       *mocks.MockClient
+	err    error
-	MockBucketHandle *mocks.MockBucketHandle
-	MockObjectHandle *mocks.MockObjectHandle
-	bucketName       string = "test-bucket"
-	objectName       string = "test.yaml"
-	localPath        string
 )
 
-// mockgen -destination=mocks/mock_gcp_storage.go -package=mocks -source=gcp.go GCPStorageService
+func TestMain(m *testing.M) {
-func TestGCPProvider(t *testing.T) {
+	hc, close := newTestServer(func(w http.ResponseWriter, r *http.Request) {
-	MockCtrl = gomock.NewController(GinkgoT())
+		io.Copy(ioutil.Discard, r.Body)
-	RegisterFailHandler(Fail)
+		w.WriteHeader(200)
-	RunSpecs(t, "Test GCP Storage Provider Suite")
+		if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s?alt=json&prettyPrint=false&projection=full", bucketName) {
-}
+			response := getBucket()
-
+			jsonedResp, err := json.Marshal(response)
-var _ = BeforeSuite(func() {
-	MockClient = mocks.NewMockClient(MockCtrl)
-	MockBucketHandle = mocks.NewMockBucketHandle(MockCtrl)
-	MockObjectHandle = mocks.NewMockObjectHandle(MockCtrl)
-	tempDir, err := os.MkdirTemp("", bucketName)
 			if err != nil {
-		Expect(err).ToNot(HaveOccurred())
+				log.Fatalf("error marshalling resp %v\n", err)
+			}
+			_, err = w.Write(jsonedResp)
+			if err != nil {
+				log.Fatalf("error writing jsonedResp %v\n", err)
+			}
+		} else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) {
+			response := getObject()
+			jsonedResp, err := json.Marshal(response)
+			if err != nil {
+				log.Fatalf("error marshalling resp %v\n", err)
+			}
+			_, err = w.Write(jsonedResp)
+			if err != nil {
+				log.Fatalf("error writing jsonedResp %v\n", err)
+			}
+		} else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o?alt=json&delimiter=&endOffset=&pageToken=&prefix=&prettyPrint=false&projection=full&startOffset=&versions=false", bucketName) {
+			response := getObject()
+			jsonedResp, err := json.Marshal(response)
+			if err != nil {
+				log.Fatalf("error marshalling resp %v\n", err)
+			}
+			_, err = w.Write(jsonedResp)
+			if err != nil {
+				log.Fatalf("error writing jsonedResp %v\n", err)
+			}
+		} else if r.RequestURI == fmt.Sprintf("/%s/test.yaml", bucketName) || r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) {
+			response := getObjectFile()
+			_, err = w.Write([]byte(response))
+			if err != nil {
+				log.Fatalf("error writing jsonedResp %v\n", err)
+			}
 		}
-	localPath = filepath.Join(tempDir, objectName)
-	MockClient.EXPECT().Bucket(bucketName).Return(&gcpStorage.BucketHandle{}).AnyTimes()
-	MockBucketHandle.EXPECT().Object(objectName).Return(&gcpStorage.ObjectHandle{}).AnyTimes()
-	MockBucketHandle.EXPECT().Attrs(context.Background()).Return(&gcpStorage.BucketAttrs{
-		Name:    bucketName,
-		Created: time.Now(),
-		Etag:    "test-etag",
-	}, nil).AnyTimes()
-	MockBucketHandle.EXPECT().Objects(gomock.Any(), nil).Return(&gcpStorage.ObjectIterator{}).AnyTimes()
-	MockObjectHandle.EXPECT().Attrs(gomock.Any()).Return(&gcpStorage.ObjectAttrs{
-		Bucket:      bucketName,
-		Name:        objectName,
-		ContentType: "text/x-yaml",
-		Etag:        "test-etag",
-		Size:        125,
-		Created:     time.Now(),
-	}, nil).AnyTimes()
-	MockObjectHandle.EXPECT().NewRangeReader(gomock.Any(), 10, 125).Return(&gcpStorage.Reader{}, nil).AnyTimes()
 	})
+	ctx := context.Background()
+	Client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc))
+	if err != nil {
+		log.Fatal(err)
+	}
+	run := m.Run()
+	close()
+	os.Exit(run)
+}
 
-var _ = Describe("GCP Storage Provider", func() {
+func TestBucketExists(t *testing.T) {
-	Describe("Get GCP Storage Provider client from gcp", func() {
-
-		Context("Gcp storage Bucket - BucketExists", func() {
-			It("should not return an error when fetching gcp storage bucket", func() {
 	gcpClient := &gcp.GCPClient{
-					Client:     MockClient,
+		Client:     Client,
 		StartRange: 0,
 		EndRange:   -1,
 	}
 	exists, err := gcpClient.BucketExists(context.Background(), bucketName)
-				Expect(err).ToNot(HaveOccurred())
+	assert.NilError(t, err)
-				Expect(exists).To(BeTrue())
+	assert.Assert(t, exists)
-			})
-		})
-		Context("Gcp storage Bucket - FGetObject", func() {
-			It("should get the object from the bucket and download the object locally", func() {
-				gcpClient := &gcp.GCPClient{
-					Client:     MockClient,
-					StartRange: 0,
-					EndRange:   -1,
 }
-				err := gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath)
-				Expect(err).ToNot(HaveOccurred())
-			})
-		})
-		Context("Gcp storage Bucket - ObjectAttributes", func() {
-			It("should get the object attributes", func() {
-				gcpClient := &gcp.GCPClient{
-					Client:     MockClient,
-					StartRange: 0,
-					EndRange:   -1,
-				}
-				exists, attrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName)
-				Expect(err).ToNot(HaveOccurred())
-				Expect(exists).To(BeTrue())
-				Expect(attrs).ToNot(BeNil())
-			})
 
-			Context("Gcp storage Bucket - SetRange", func() {
+func TestObjectAttributes(t *testing.T) {
-				It("should set the range of the io reader seeker for the file download", func() {
 	gcpClient := &gcp.GCPClient{
-						Client:     MockClient,
+		Client:     Client,
+		StartRange: 0,
+		EndRange:   -1,
+	}
+	exists, objectAttrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName)
+	if err == gcpStorage.ErrObjectNotExist {
+		assert.NilError(t, err)
+	}
+	assert.NilError(t, err)
+	assert.Assert(t, exists)
+	assert.Assert(t, objectAttrs != nil)
+}
+
+func TestListObjects(t *testing.T) {
+	gcpClient := &gcp.GCPClient{
+		Client:     Client,
+		StartRange: 0,
+		EndRange:   -1,
+	}
+	objectInterator := gcpClient.ListObjects(context.Background(), bucketName, nil)
+	for {
+		_, err := objectInterator.Next()
+		if err == gcp.IteratorDone {
+			break
+		}
+		assert.NilError(t, err)
+	}
+	assert.Assert(t, objectInterator != nil)
+}
+
+func TestFGetObject(t *testing.T) {
+	tempDir, err := os.MkdirTemp("", bucketName)
+	assert.NilError(t, err)
+	defer os.RemoveAll(tempDir)
+	gcpClient := &gcp.GCPClient{
+		Client:     Client,
+		StartRange: 0,
+		EndRange:   -1,
+	}
+	localPath := filepath.Join(tempDir, objectName)
+	err = gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath)
+	if err != io.EOF {
+		assert.NilError(t, err)
+	}
+}
+
+func TestSetRange(t *testing.T) {
+	gcpClient := &gcp.GCPClient{
+		Client:     Client,
 		StartRange: 0,
 		EndRange:   -1,
 	}
 	gcpClient.SetRange(2, 5)
-					Expect(gcpClient.StartRange).To(Equal(int64(2)))
+	assert.Equal(t, gcpClient.StartRange, int64(2))
-					Expect(gcpClient.EndRange).To(Equal(int64(5)))
+	assert.Equal(t, gcpClient.EndRange, int64(5))
-				})
+}
-			})
+
-		})
+func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*http.Client, func()) {
-	})
+	ts := httptest.NewTLSServer(http.HandlerFunc(handler))
-})
+	tlsConf := &tls.Config{InsecureSkipVerify: true}
+	tr := &http.Transport{
+		TLSClientConfig: tlsConf,
+		DialTLS: func(netw, addr string) (net.Conn, error) {
+			return tls.Dial("tcp", ts.Listener.Addr().String(), tlsConf)
+		},
+	}
+	return &http.Client{Transport: tr}, func() {
+		tr.CloseIdleConnections()
+		ts.Close()
+	}
+}
+
+func getObject() *raw.Object {
+	customTime := time.Now()
+	retTime := customTime.Add(3 * time.Hour)
+	return &raw.Object{
+		Bucket:                  bucketName,
+		Name:                    objectName,
+		EventBasedHold:          false,
+		TemporaryHold:           false,
+		RetentionExpirationTime: retTime.Format(time.RFC3339),
+		ContentType:             "text/x-yaml",
+		ContentLanguage:         "en-us",
+		Size:                    1 << 20,
+		CustomTime:              customTime.Format(time.RFC3339),
+		Md5Hash:                 "bFbHCDvedeecefdgmfmhfuRxBdcedGe96S82XJOAXxjJpk=",
+	}
+}
+
+func getBucket() *raw.Bucket {
+	labels := map[string]string{"a": "b"}
+	matchClasses := []string{"STANDARD"}
+	aTime := time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC)
+	rb := &raw.Bucket{
+		Name:                  bucketName,
+		Location:              "loc",
+		DefaultEventBasedHold: true,
+		Metageneration:        3,
+		StorageClass:          "sc",
+		TimeCreated:           "2021-5-23T04:05:06Z",
+		Versioning:            &raw.BucketVersioning{Enabled: true},
+		Labels:                labels,
+		Billing:               &raw.BucketBilling{RequesterPays: true},
+		Etag:                  "BNaB2y5Xr3&5MHDca4SoTNL79lyhahr7MV87ubwjgdtg6ghs",
+		Lifecycle: &raw.BucketLifecycle{
+			Rule: []*raw.BucketLifecycleRule{{
+				Action: &raw.BucketLifecycleRuleAction{
+					Type:         "SetStorageClass",
+					StorageClass: "NEARLINE",
+				},
+				Condition: &raw.BucketLifecycleRuleCondition{
+					Age:                 10,
+					IsLive:              googleapi.Bool(true),
+					CreatedBefore:       "2021-01-02",
+					MatchesStorageClass: matchClasses,
+					NumNewerVersions:    3,
+				},
+			}},
+		},
+		RetentionPolicy: &raw.BucketRetentionPolicy{
+			RetentionPeriod: 3,
+			EffectiveTime:   aTime.Format(time.RFC3339),
+		},
+		IamConfiguration: &raw.BucketIamConfiguration{
+			BucketPolicyOnly: &raw.BucketIamConfigurationBucketPolicyOnly{
+				Enabled:    true,
+				LockedTime: aTime.Format(time.RFC3339),
+			},
+			UniformBucketLevelAccess: &raw.BucketIamConfigurationUniformBucketLevelAccess{
+				Enabled:    true,
+				LockedTime: aTime.Format(time.RFC3339),
+			},
+		},
+		Cors: []*raw.BucketCors{
+			{
+				MaxAgeSeconds:  3600,
+				Method:         []string{"GET", "POST"},
+				Origin:         []string{"*"},
+				ResponseHeader: []string{"FOO"},
+			},
+		},
+		Acl: []*raw.BucketAccessControl{
+			{Bucket: bucketName, Role: "READER", Email: "test@example.com", Entity: "allUsers"},
+		},
+		LocationType: "dual-region",
+		Encryption:   &raw.BucketEncryption{DefaultKmsKeyName: "key"},
+		Logging:      &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"},
+		Website:      &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"},
+	}
+	return rb
+}
+
+func getObjectFile() string {
+	return `
+	apiVersion: source.toolkit.fluxcd.io/v1beta1
+	kind: Bucket
+	metadata:
+	  name: podinfo
+	  namespace: default
+	spec:
+	  interval: 5m
+	  provider: aws
+	  bucketName: podinfo
+	  endpoint: s3.amazonaws.com
+	  region: us-east-1
+	  timeout: 30s
+	`
+}

pkg/gcp/mocks/mock_gcp_storage.go

@@ -1,183 +0,0 @@
-// Code generated by MockGen. DO NOT EDIT.
-// Source: gcp.go
-
-// Package mocks is a generated GoMock package.
-package mocks
-
-import (
-	context "context"
-	reflect "reflect"
-
-	storage "cloud.google.com/go/storage"
-	gomock "github.com/golang/mock/gomock"
-)
-
-// MockClient is a mock of Client interface.
-type MockClient struct {
-	ctrl     *gomock.Controller
-	recorder *MockClientMockRecorder
-}
-
-// MockClientMockRecorder is the mock recorder for MockClient.
-type MockClientMockRecorder struct {
-	mock *MockClient
-}
-
-// NewMockClient creates a new mock instance.
-func NewMockClient(ctrl *gomock.Controller) *MockClient {
-	mock := &MockClient{ctrl: ctrl}
-	mock.recorder = &MockClientMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockClient) EXPECT() *MockClientMockRecorder {
-	return m.recorder
-}
-
-// Bucket mocks base method.
-func (m *MockClient) Bucket(arg0 string) *storage.BucketHandle {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Bucket", arg0)
-	ret0, _ := ret[0].(*storage.BucketHandle)
-	return ret0
-}
-
-// Bucket indicates an expected call of Bucket.
-func (mr *MockClientMockRecorder) Bucket(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockClient)(nil).Bucket), arg0)
-}
-
-// Close mocks base method.
-func (m *MockClient) Close() error {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Close")
-	ret0, _ := ret[0].(error)
-	return ret0
-}
-
-// Close indicates an expected call of Close.
-func (mr *MockClientMockRecorder) Close() *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClient)(nil).Close))
-}
-
-// MockBucketHandle is a mock of BucketHandle interface.
-type MockBucketHandle struct {
-	ctrl     *gomock.Controller
-	recorder *MockBucketHandleMockRecorder
-}
-
-// MockBucketHandleMockRecorder is the mock recorder for MockBucketHandle.
-type MockBucketHandleMockRecorder struct {
-	mock *MockBucketHandle
-}
-
-// NewMockBucketHandle creates a new mock instance.
-func NewMockBucketHandle(ctrl *gomock.Controller) *MockBucketHandle {
-	mock := &MockBucketHandle{ctrl: ctrl}
-	mock.recorder = &MockBucketHandleMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockBucketHandle) EXPECT() *MockBucketHandleMockRecorder {
-	return m.recorder
-}
-
-// Attrs mocks base method.
-func (m *MockBucketHandle) Attrs(arg0 context.Context) (*storage.BucketAttrs, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Attrs", arg0)
-	ret0, _ := ret[0].(*storage.BucketAttrs)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// Attrs indicates an expected call of Attrs.
-func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0)
-}
-
-// Object mocks base method.
-func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Object", arg0)
-	ret0, _ := ret[0].(*storage.ObjectHandle)
-	return ret0
-}
-
-// Object indicates an expected call of Object.
-func (mr *MockBucketHandleMockRecorder) Object(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Object", reflect.TypeOf((*MockBucketHandle)(nil).Object), arg0)
-}
-
-// Objects mocks base method.
-func (m *MockBucketHandle) Objects(arg0 context.Context, arg1 *storage.Query) *storage.ObjectIterator {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Objects", arg0, arg1)
-	ret0, _ := ret[0].(*storage.ObjectIterator)
-	return ret0
-}
-
-// Objects indicates an expected call of Objects.
-func (mr *MockBucketHandleMockRecorder) Objects(arg0, arg1 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Objects", reflect.TypeOf((*MockBucketHandle)(nil).Objects), arg0, arg1)
-}
-
-// MockObjectHandle is a mock of ObjectHandle interface.
-type MockObjectHandle struct {
-	ctrl     *gomock.Controller
-	recorder *MockObjectHandleMockRecorder
-}
-
-// MockObjectHandleMockRecorder is the mock recorder for MockObjectHandle.
-type MockObjectHandleMockRecorder struct {
-	mock *MockObjectHandle
-}
-
-// NewMockObjectHandle creates a new mock instance.
-func NewMockObjectHandle(ctrl *gomock.Controller) *MockObjectHandle {
-	mock := &MockObjectHandle{ctrl: ctrl}
-	mock.recorder = &MockObjectHandleMockRecorder{mock}
-	return mock
-}
-
-// EXPECT returns an object that allows the caller to indicate expected use.
-func (m *MockObjectHandle) EXPECT() *MockObjectHandleMockRecorder {
-	return m.recorder
-}
-
-// Attrs mocks base method.
-func (m *MockObjectHandle) Attrs(arg0 context.Context) (*storage.ObjectAttrs, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "Attrs", arg0)
-	ret0, _ := ret[0].(*storage.ObjectAttrs)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// Attrs indicates an expected call of Attrs.
-func (mr *MockObjectHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockObjectHandle)(nil).Attrs), arg0)
-}
-
-// NewRangeReader mocks base method.
-func (m *MockObjectHandle) NewRangeReader(arg0 context.Context, arg1, arg2 int64) (*storage.Reader, error) {
-	m.ctrl.T.Helper()
-	ret := m.ctrl.Call(m, "NewRangeReader", arg0, arg1, arg2)
-	ret0, _ := ret[0].(*storage.Reader)
-	ret1, _ := ret[1].(error)
-	return ret0, ret1
-}
-
-// NewRangeReader indicates an expected call of NewRangeReader.
-func (mr *MockObjectHandleMockRecorder) NewRangeReader(arg0, arg1, arg2 interface{}) *gomock.Call {
-	mr.mock.ctrl.T.Helper()
-	return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewRangeReader", reflect.TypeOf((*MockObjectHandle)(nil).NewRangeReader), arg0, arg1, arg2)
-}