diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 9e4eee73..53451954 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -29,6 +29,7 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/s3utils" + "google.golang.org/api/option" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -417,13 +418,12 @@ func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) if err := gcp.ValidateSecret(secret.Data, secret.Name); err != nil { return nil, err } - serviceAccount := gcp.InitCredentialsWithSecret(secret.Data) - client, err = gcp.NewClientWithSAKey(ctx, serviceAccount) + client, err = gcp.NewClient(ctx, option.WithCredentialsJSON(secret.Data["serviceaccount"])) if err != nil { return nil, err } } else { - client, err = gcp.NewClient(ctx) + client, err = gcp.NewClient(ctx, nil) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 1d60520c..3ec4d3e8 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/libgit2/git2go/v31 v31.4.14 github.com/minio/minio-go/v7 v7.0.10 @@ -36,7 +35,7 @@ require ( golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect golang.org/x/text v0.3.7 // indirect - google.golang.org/api v0.54.0 // indirect + google.golang.org/api v0.54.0 google.golang.org/genproto v0.0.0-20210830153122-0bac4d21c8ea // indirect gotest.tools v2.2.0+incompatible helm.sh/helm/v3 v3.6.3 diff --git a/go.sum b/go.sum index be1c5759..a1a0d5dc 100644 --- a/go.sum +++ b/go.sum @@ -42,7 +42,6 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.16.0 h1:1UwAux2OZP4310YXg5ohqBEpV16Y93uZG4+qOX7K2Kg= cloud.google.com/go/storage v1.16.0/go.mod h1:ieKBmUyzcftN5tbxwnXClMKH00CfcQ+xL6NN0r5QfmE= @@ -415,7 +414,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -485,7 +483,6 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index ec56384a..b2274e34 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -18,7 +18,6 @@ package gcp import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -30,13 +29,6 @@ import ( "google.golang.org/api/option" ) -const ( - ServiceAccount = "service_account" - AuthUri = "https://accounts.google.com/o/oauth2/auth" - TokenUri = "https://oauth2.googleapis.com/token" - AuthProviderX509CertUrl = "https://www.googleapis.com/oauth2/v1/certs" -) - var ( // IteratorDone is returned when the looping of objects/content // has reached the end of the iteration. @@ -61,27 +53,12 @@ type GCPClient struct { EndRange int64 } -// CredentialsFile struct representing the GCP Service Account -// JSON file. -type CredentialsFile struct { - Type string `json:"type"` - ProjectID string `json:"project_id"` - PrivateKeyID string `json:"private_key_id"` - PrivateKey string `json:"private_key"` - ClientEmail string `json:"client_email"` - ClientID string `json:"client_id"` - AuthUri string `json:"auth_uri"` - TokenUri string `json:"token_uri"` - AuthProviderX509CertUrl string `json:"auth_provider_x509_cert_url"` - ClientX509CertUrl string `json:"client_x509_cert_url"` -} - // NewClient creates a new GCP storage client // The Google Storage Client will automatically // look for the Google Application Credential environment variable // or look for the Google Application Credential file. -func NewClient(ctx context.Context) (*GCPClient, error) { - client, err := gcpStorage.NewClient(ctx) +func NewClient(ctx context.Context, opts ...option.ClientOption) (*GCPClient, error) { + client, err := gcpStorage.NewClient(ctx, opts...) if err != nil { return nil, err } @@ -89,70 +66,11 @@ func NewClient(ctx context.Context) (*GCPClient, error) { return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil } -// NewClientWithSAKey creates a new GCP storage client -// It uses the provided JSON file with service account details -// To authenticate. -func NewClientWithSAKey(ctx context.Context, credentials *CredentialsFile) (*GCPClient, error) { - saAccount, err := credentials.credentailsToJSON() - if err != nil { - return nil, err - } - - client, err := gcpStorage.NewClient(ctx, option.WithCredentialsJSON(saAccount)) - if err != nil { - return nil, err - } - - return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil -} - -// credentailsToJSON converts GCP service account credentials struct to JSON. -func (credentials *CredentialsFile) credentailsToJSON() ([]byte, error) { - credentialsJSON, err := json.Marshal(credentials) - if err != nil { - return nil, err - } - - return credentialsJSON, nil -} - -// InitCredentialsWithSecret creates a new credential -// by initializing a new CredentialsFile struct -func InitCredentialsWithSecret(secret map[string][]byte) *CredentialsFile { - return &CredentialsFile{ - Type: ServiceAccount, - ProjectID: string(secret["projectid"]), - PrivateKeyID: string(secret["privatekeyid"]), - PrivateKey: string(secret["privatekey"]), - ClientEmail: string(secret["clientemail"]), - ClientID: string(secret["clientid"]), - AuthUri: AuthUri, - TokenUri: TokenUri, - AuthProviderX509CertUrl: AuthProviderX509CertUrl, - ClientX509CertUrl: string(secret["certurl"]), - } -} - // ValidateSecret validates the credential secrets // It ensures that needed secret fields are not missing. func ValidateSecret(secret map[string][]byte, name string) error { - if _, exists := secret["projectid"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'projectid'", name) - } - if _, exists := secret["privatekeyid"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'privatekeyid'", name) - } - if _, exists := secret["privatekey"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'privatekey'", name) - } - if _, exists := secret["clientemail"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) - } - if _, exists := secret["clientid"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'clientid'", name) - } - if _, exists := secret["certurl"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'certurl'", name) + if _, exists := secret["serviceaccount"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'serviceaccount'", name) } return nil diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 64db105f..5c0292e9 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -47,54 +47,61 @@ const ( ) var ( - Client *gcpStorage.Client + hc *http.Client + client *gcpStorage.Client + close func() err error ) func TestMain(m *testing.M) { - hc, close := newTestServer(func(w http.ResponseWriter, r *http.Request) { + hc, close = newTestServer(func(w http.ResponseWriter, r *http.Request) { io.Copy(ioutil.Discard, r.Body) - w.WriteHeader(200) if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s?alt=json&prettyPrint=false&projection=full", bucketName) { + w.WriteHeader(200) response := getBucket() - jsonedResp, err := json.Marshal(response) + jsonResponse, err := json.Marshal(response) if err != nil { - log.Fatalf("error marshalling resp %v\n", err) + log.Fatalf("error marshalling response %v\n", err) } - _, err = w.Write(jsonedResp) + _, err = w.Write(jsonResponse) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing jsonResponse %v\n", err) } } else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) { + w.WriteHeader(200) response := getObject() - jsonedResp, err := json.Marshal(response) + jsonResponse, err := json.Marshal(response) if err != nil { - log.Fatalf("error marshalling resp %v\n", err) + log.Fatalf("error marshalling response %v\n", err) } - _, err = w.Write(jsonedResp) + _, err = w.Write(jsonResponse) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing jsonResponse %v\n", err) } } else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o?alt=json&delimiter=&endOffset=&pageToken=&prefix=&prettyPrint=false&projection=full&startOffset=&versions=false", bucketName) { + w.WriteHeader(200) response := getObject() - jsonedResp, err := json.Marshal(response) + jsonResponse, err := json.Marshal(response) if err != nil { - log.Fatalf("error marshalling resp %v\n", err) + log.Fatalf("error marshalling response %v\n", err) } - _, err = w.Write(jsonedResp) + _, err = w.Write(jsonResponse) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing jsonResponse %v\n", err) } } else if r.RequestURI == fmt.Sprintf("/%s/test.yaml", bucketName) || r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) { + w.WriteHeader(200) response := getObjectFile() _, err = w.Write([]byte(response)) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing response %v\n", err) } + } else { + w.WriteHeader(404) } }) ctx := context.Background() - Client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) + client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { log.Fatal(err) } @@ -103,9 +110,15 @@ func TestMain(m *testing.M) { os.Exit(run) } +func TestNewClient(t *testing.T) { + gcpClient, err := gcp.NewClient(context.Background(), option.WithHTTPClient(hc)) + assert.NilError(t, err) + assert.Assert(t, gcpClient != nil) +} + func TestBucketExists(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -114,9 +127,21 @@ func TestBucketExists(t *testing.T) { assert.Assert(t, exists) } +func TestBucketNotExists(t *testing.T) { + bucket := "notexistsbucket" + gcpClient := &gcp.GCPClient{ + Client: client, + StartRange: 0, + EndRange: -1, + } + exists, err := gcpClient.BucketExists(context.Background(), bucket) + assert.NilError(t, err) + assert.Assert(t, !exists) +} + func TestObjectAttributes(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -131,7 +156,7 @@ func TestObjectAttributes(t *testing.T) { func TestListObjects(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -151,7 +176,7 @@ func TestFGetObject(t *testing.T) { assert.NilError(t, err) defer os.RemoveAll(tempDir) gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -162,9 +187,41 @@ func TestFGetObject(t *testing.T) { } } +func TestFGetObjectNotExists(t *testing.T) { + object := "notexists.txt" + tempDir, err := os.MkdirTemp("", bucketName) + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + gcpClient := &gcp.GCPClient{ + Client: client, + StartRange: 0, + EndRange: -1, + } + localPath := filepath.Join(tempDir, object) + err = gcpClient.FGetObject(context.Background(), bucketName, object, localPath) + if err != io.EOF { + assert.Error(t, err, "storage: object doesn't exist") + } +} + +func TestFGetObjectDirectoryIsFileName(t *testing.T) { + tempDir, err := os.MkdirTemp("", bucketName) + defer os.RemoveAll(tempDir) + assert.NilError(t, err) + gcpClient := &gcp.GCPClient{ + Client: client, + StartRange: 0, + EndRange: -1, + } + err = gcpClient.FGetObject(context.Background(), bucketName, objectName, tempDir) + if err != io.EOF { + assert.Error(t, err, "filename is a directory") + } +} + func TestSetRange(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -173,6 +230,45 @@ func TestSetRange(t *testing.T) { assert.Equal(t, gcpClient.EndRange, int64(5)) } +func TestValidateSecret(t *testing.T) { + t.Parallel() + testCases := []struct { + title string + secret map[string][]byte + name string + error bool + }{ + { + "Test Case 1", + map[string][]byte{ + "serviceaccount": []byte("serviceaccount"), + }, + "Service Account", + false, + }, + { + "Test Case 2", + map[string][]byte{ + "data": []byte("data"), + }, + "Service Account", + true, + }, + } + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.title, func(t *testing.T) { + t.Parallel() + err := gcp.ValidateSecret(testCase.secret, testCase.name) + if testCase.error { + assert.Error(t, err, fmt.Sprintf("invalid '%v' secret data: required fields 'serviceaccount'", testCase.name)) + } else { + assert.NilError(t, err) + } + }) + } +} + func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*http.Client, func()) { ts := httptest.NewTLSServer(http.HandlerFunc(handler)) tlsConf := &tls.Config{InsecureSkipVerify: true}