From 7da9619b58e74dd891898022a6ce104381c8b4c3 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Wed, 1 Sep 2021 14:10:08 -0500 Subject: [PATCH 01/53] Feature: Add Support for Google Cloud Storage along with Workload Identity Added Support for Google Cloud Storage with Workload Identity as Source Provider. This enables the use of GCP without enabling S3 compatible access. Signed-off-by: pa250194 --- api/v1beta1/bucket_types.go | 1 + controllers/bucket_controller.go | 233 ++++++++++++++++++++--------- docs/spec/v1alpha1/buckets.md | 1 + go.mod | 12 +- go.sum | 249 +++++++++++++++++++++++++++++-- pkg/.DS_Store | Bin 0 -> 6148 bytes pkg/gcp/gcp.go | 199 ++++++++++++++++++++++++ pkg/gcp/gcp_test.go | 62 ++++++++ 8 files changed, 674 insertions(+), 83 deletions(-) create mode 100644 pkg/.DS_Store create mode 100644 pkg/gcp/gcp.go create mode 100644 pkg/gcp/gcp_test.go diff --git a/api/v1beta1/bucket_types.go b/api/v1beta1/bucket_types.go index 492002b8..1dc68851 100644 --- a/api/v1beta1/bucket_types.go +++ b/api/v1beta1/bucket_types.go @@ -79,6 +79,7 @@ type BucketSpec struct { const ( GenericBucketProvider string = "generic" AmazonBucketProvider string = "aws" + GoogleBucketProvider string = "gcp" ) // BucketStatus defines the observed state of a bucket diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index e1ca4641..3ec8d5e2 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -46,6 +46,7 @@ import ( "github.com/fluxcd/pkg/runtime/events" "github.com/fluxcd/pkg/runtime/metrics" "github.com/fluxcd/pkg/runtime/predicates" + "github.com/fluxcd/source-controller/pkg/gcp" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/source-controller/pkg/sourceignore" @@ -176,77 +177,20 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, error) { - s3Client, err := r.auth(ctx, bucket) - if err != nil { - err = fmt.Errorf("auth error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err - } - - // create tmp dir - tempDir, err := os.MkdirTemp("", bucket.Name) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err - } - defer os.RemoveAll(tempDir) - - ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) - defer cancel() - - exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName) - if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err - } - if !exists { - err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err - } - - // Look for file with ignore rules first - // NB: S3 has flat filepath keys making it impossible to look - // for files in "subdirectories" without building up a tree first. - path := filepath.Join(tempDir, sourceignore.IgnoreFile) - if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil { - if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err - } - } - ps, err := sourceignore.ReadIgnoreFile(path, nil) - if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err - } - // In-spec patterns take precedence - if bucket.Spec.Ignore != nil { - ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*bucket.Spec.Ignore), nil)...) - } - matcher := sourceignore.NewMatcher(ps) - - // download bucket content - for object := range s3Client.ListObjects(ctxTimeout, bucket.Spec.BucketName, minio.ListObjectsOptions{ - Recursive: true, - UseV1: s3utils.IsGoogleEndpoint(*s3Client.EndpointURL()), - }) { - if object.Err != nil { - err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err - } - - if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile { - continue - } - - if matcher.Match(strings.Split(object.Key, "/"), false) { - continue - } - - localPath := filepath.Join(tempDir, object.Key) - err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{}) + var tempDir string + var err error + var sourceBucket sourcev1.Bucket + if bucket.Spec.Provider == sourcev1.GoogleBucketProvider { + sourceBucket, tempDir, err = r.reconcileWithGCP(ctx, bucket) if err != nil { - err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err + return sourceBucket, err + } + } else { + sourceBucket, tempDir, err = r.reconcileWithAWS(ctx, bucket) + if err != nil { + return sourceBucket, err } } - revision, err := r.checksum(tempDir) if err != nil { return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err @@ -315,6 +259,159 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1. return ctrl.Result{}, nil } +func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { + gcpClient, err := r.authGCP(ctx, bucket) + if err != nil { + err = fmt.Errorf("auth error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err + } + defer gcpClient.Client.Close() + // create tmp dir + tempDir, err := os.MkdirTemp("", bucket.Name) + if err != nil { + err = fmt.Errorf("tmp dir error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err + } + defer os.RemoveAll(tempDir) + + ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) + defer cancel() + + exists, err := gcpClient.BucketExists(ctxTimeout, bucket.Spec.BucketName) + if err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + if !exists { + err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + + // Look for file with ignore rules first. + path := filepath.Join(tempDir, sourceignore.IgnoreFile) + if err := gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path); err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + ps, err := sourceignore.ReadIgnoreFile(path, nil) + if err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + // In-spec patterns take precedence + if bucket.Spec.Ignore != nil { + ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*bucket.Spec.Ignore), nil)...) + } + matcher := sourceignore.NewMatcher(ps) + objects := gcpClient.ListObjects(ctxTimeout, bucket.Spec.BucketName, nil) + // download bucket content + for { + object, err := objects.Next() + if err == gcp.IteratorDone { + break + } + if err != nil { + err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, err) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + + if strings.HasSuffix(object.Name, "/") || object.Name == sourceignore.IgnoreFile { + continue + } + + if matcher.Match(strings.Split(object.Name, "/"), false) { + continue + } + + localPath := filepath.Join(tempDir, object.Name) + // FGetObject - get and download bucket object + if err = gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Name, localPath); err != nil { + err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + } + return sourcev1.Bucket{}, tempDir, nil +} + +func (r *BucketReconciler) reconcileWithAWS(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { + s3Client, err := r.auth(ctx, bucket) + if err != nil { + err = fmt.Errorf("auth error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err + } + + // create tmp dir + tempDir, err := os.MkdirTemp("", bucket.Name) + if err != nil { + err = fmt.Errorf("tmp dir error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err + } + defer os.RemoveAll(tempDir) + + ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) + defer cancel() + + exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName) + if err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + if !exists { + err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + + // Look for file with ignore rules first + // NB: S3 has flat filepath keys making it impossible to look + // for files in "subdirectories" without building up a tree first. + path := filepath.Join(tempDir, sourceignore.IgnoreFile) + if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil { + if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + } + ps, err := sourceignore.ReadIgnoreFile(path, nil) + if err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + // In-spec patterns take precedence + if bucket.Spec.Ignore != nil { + ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*bucket.Spec.Ignore), nil)...) + } + matcher := sourceignore.NewMatcher(ps) + + // download bucket content + for object := range s3Client.ListObjects(ctxTimeout, bucket.Spec.BucketName, minio.ListObjectsOptions{ + Recursive: true, + UseV1: s3utils.IsGoogleEndpoint(*s3Client.EndpointURL()), + }) { + if object.Err != nil { + err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + + if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile { + continue + } + + if matcher.Match(strings.Split(object.Key, "/"), false) { + continue + } + + localPath := filepath.Join(tempDir, object.Key) + err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{}) + if err != nil { + err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + } + return sourcev1.Bucket{}, tempDir, nil +} + +func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) (*gcp.GCPClient, error) { + client, err := gcp.NewClient(ctx) + if err != nil { + return nil, err + } + return client, nil +} + func (r *BucketReconciler) auth(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) { opt := minio.Options{ Region: bucket.Spec.Region, diff --git a/docs/spec/v1alpha1/buckets.md b/docs/spec/v1alpha1/buckets.md index 7addeccd..0ad60f41 100644 --- a/docs/spec/v1alpha1/buckets.md +++ b/docs/spec/v1alpha1/buckets.md @@ -57,6 +57,7 @@ Supported providers: const ( GenericBucketProvider string = "generic" AmazonBucketProvider string = "aws" + GoogleBucketProvider string = "gcp" ) ``` diff --git a/go.mod b/go.mod index 2b033733..49ebac77 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.16 replace github.com/fluxcd/source-controller/api => ./api require ( + cloud.google.com/go v0.93.3 // indirect + cloud.google.com/go/storage v1.16.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/cyphar/filepath-securejoin v0.2.2 github.com/fluxcd/pkg/apis/meta v0.10.0 @@ -20,13 +22,21 @@ require ( github.com/go-git/go-billy/v5 v5.3.1 github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/libgit2/git2go/v31 v31.4.14 github.com/minio/minio-go/v7 v7.0.10 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.14.0 github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b - golang.org/x/sync v0.0.0-20201207232520-09787c993a3a + golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect + golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/api v0.54.0 // indirect + google.golang.org/genproto v0.0.0-20210830153122-0bac4d21c8ea // indirect gotest.tools v2.2.0+incompatible helm.sh/helm/v3 v3.6.3 k8s.io/api v0.21.3 diff --git a/go.sum b/go.sum index 477e3ddb..a1ea7ce9 100644 --- a/go.sum +++ b/go.sum @@ -9,20 +9,43 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3 h1:wPBktZFzYBcCZVARvwVKqH1uEj+aLXofJEtrb4oOsio= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.16.0 h1:1UwAux2OZP4310YXg5ohqBEpV16Y93uZG4+qOX7K2Kg= +cloud.google.com/go/storage v1.16.0/go.mod h1:ieKBmUyzcftN5tbxwnXClMKH00CfcQ+xL6NN0r5QfmE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= @@ -86,6 +109,7 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -137,6 +161,10 @@ github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMn github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59 h1:qWj4qVYZ95vLWwqyNJCQg7rDsG5wPdze0UaPolH7DUk= @@ -216,7 +244,13 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -370,18 +404,24 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -391,9 +431,11 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= @@ -402,20 +444,38 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= @@ -424,7 +484,10 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= @@ -449,6 +512,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -484,6 +548,7 @@ github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= @@ -740,6 +805,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -839,8 +905,11 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= @@ -862,8 +931,12 @@ go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -929,8 +1002,10 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -940,6 +1015,9 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -961,6 +1039,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -972,31 +1051,57 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a h1:bRuuGXV8wwSdGTB+CtJf+FjgO1APK1CoO39T4BN/XBw= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210615190721-d04028783cf1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1040,25 +1145,43 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -1069,8 +1192,10 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1121,13 +1246,33 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1146,13 +1291,32 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.49.0/go.mod h1:BECiH72wsfwUvOVn3+btPD5WHi0LzavZReBndi42L18= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0 h1:ECJUVngj71QI6XEm7b1sAf8BljU5inEhMbKPR8Lxhhk= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= @@ -1174,11 +1338,46 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210624174822-c5cf32407d0a/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210830153122-0bac4d21c8ea h1:5eMUso2GVOxypVH1fR4oKgDobrvi4DHctJ4fVk66s/4= +google.golang.org/genproto v0.0.0-20210830153122-0bac4d21c8ea/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1189,10 +1388,29 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1204,8 +1422,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1235,6 +1454,7 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1261,6 +1481,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= k8s.io/api v0.21.3 h1:cblWILbLO8ar+Fj6xdDGr603HRsf8Wu9E9rngJeprZQ= diff --git a/pkg/.DS_Store b/pkg/.DS_Store new file mode 100644 index 0000000000000000000000000000000000000000..63391478b0cd7ae519be6c18bc8a84af5726952c GIT binary patch literal 6148 zcmeHK!EO^V5FIBeH~~ZsNc4iV7j6+sXoboFX$kG2H>5>y0Mza_t8}Gh9qn$2AO!7i zv_H}B;0yR1cw?K2v~Z<>W~A{Ok7sQ8d1dbqks8k@BceVLX((f>i{=-?*BR(owR5O3dUy@m#RrPTARkhlk zjceDn?&;pG-rMNdETTHD=hZkaUvuwyTpuUieql_0ZpX>XEL|M*?>sX_9cM*a+T?7K zBIM1hteBX^*vyMbW#@_9fYx2zJ?L*Pms>kKgJ85hS`C8b2p9sRWHF z;-^%gUsAfBr|wdsI0hBTH*f_LT2O3)f z06TD7fo*^OfdM;!p~uD{JP_qlfi6|~5<~ggLAi8{hd#b>=+a5ql`)TBS@{Y@`O-rL zHk?%GP+DO?7+7bZ?Yez_|Mz}h|F0)WPZ$se{woGlXE)j%V@v*Ry|g)gYXj&Zl!fCO mhvz9U>{E8Tby{b#nIr literal 0 HcmV?d00001 diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go new file mode 100644 index 00000000..18ea53fe --- /dev/null +++ b/pkg/gcp/gcp.go @@ -0,0 +1,199 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcp + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + + gcpStorage "cloud.google.com/go/storage" + interator "google.golang.org/api/iterator" +) + +var ( + // IteratorDone is returned when the looping of objects/content + // has reached the end of the iteration. + IteratorDone = interator.Done + // DirectoryExists is an error returned when the filename provided + // is a directory. + DirectoryExists = errors.New("filename is a directory") + // ObjectDoesNotExist is an error returned when the object whose name + // is provided does not exist. + ObjectDoesNotExist = errors.New("object does not exist") +) + +type GCPClient struct { + // client for interacting with the Google Cloud + // Storage APIs. + Client *gcpStorage.Client + // startRange is the starting read value for + // reading the object from bucket. + startRange int64 + // endRange is the ending read value for + // reading the object from bucket. + endRange int64 +} + +// NewClient creates a new GCP storage client +// The Google Storage Client will automatically +// look for the Google Application Credential environment variable +// or look for the Google Application Credential file +func NewClient(ctx context.Context) (*GCPClient, error) { + client, err := gcpStorage.NewClient(ctx) + if err != nil { + return nil, err + } + return &GCPClient{Client: client, startRange: 0, endRange: -1}, nil +} + +// SetRange sets the startRange and endRange used to read the Object from +// the bucket. It is a helper method for resumable downloads. +func (c *GCPClient) SetRange(start, end int64) { + c.startRange = start + c.endRange = end +} + +// BucketExists checks if the bucket with the provided name exists. +func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, error) { + _, err := c.Client.Bucket(bucketName).Attrs(ctx) + if err == gcpStorage.ErrBucketNotExist { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +// ObjectExists checks if the object with the provided name exists. +// If it exists the Object attributes are returned. +func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) { + attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) + // ErrObjectNotExist is returned if the object does not exist + if err != nil { + return false, nil, err + } + return true, attrs, err +} + +// FGetObject gets the object from the bucket and downloads the object locally +// A part file is created so the download can be resumable. +func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, localPath string) error { + // Verify if destination already exists. + dirStatus, err := os.Stat(localPath) + if err == nil { + // If the destination exists and is a directory. + if dirStatus.IsDir() { + return DirectoryExists + } + } + + // Proceed if file does not exist. return for all other errors. + if err != nil { + if !os.IsNotExist(err) { + return err + } + } + + // Extract top level directory. + objectDir, _ := filepath.Split(localPath) + if objectDir != "" { + // Create any missing top level directories. + if err := os.MkdirAll(objectDir, 0700); err != nil { + return err + } + } + + // ObjectExists verifies if object exists and you have permission to access. + // Check if the object exists and if you have permission to access it + // The Object attributes are returned if the Object exists. + exists, attrs, err := c.ObjectExists(ctx, bucketName, objectName) + if err != nil { + return err + } + if !exists { + return ObjectDoesNotExist + } + + // Write to a temporary file "filename.part.gcp" before saving. + filePartPath := localPath + attrs.Etag + ".part.gcp" + + // If exists, open in append mode. If not create it as a part file. + filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return err + } + + // If we return early with an error, be sure to close and delete + // filePart. If we have an error along the way there is a chance + // that filePart is somehow damaged, and we should discard it. + closeAndRemove := true + defer func() { + if closeAndRemove { + _ = filePart.Close() + _ = os.Remove(filePartPath) + } + }() + + // Issue Stat to get the current offset. + partFileStat, err := filePart.Stat() + if err != nil { + return err + } + + // Set the File size request range + // If the part file exists + if partFileStat.Size() > 0 { + c.SetRange(partFileStat.Size(), 0) + } + + // Get Object from GCP Bucket + objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.startRange, c.endRange) + if err != nil { + return err + } + defer objectReader.Close() + + // Write to the part file. + if _, err = io.CopyN(filePart, objectReader, attrs.Size); err != nil { + return err + } + + // Close the file before rename, this is specifically needed for Windows users. + closeAndRemove = false + if err = filePart.Close(); err != nil { + return err + } + + // Safely completed. Now commit by renaming to actual filename. + if err = os.Rename(filePartPath, localPath); err != nil { + return err + } + + return nil +} + +// ListObjects lists the objects/contents of the bucket whose bucket name is provided. +// the objects are returned as an Objectiterator and .Next() has to be called on them +// to loop through the Objects. +func (c *GCPClient) ListObjects(ctx context.Context, bucketName string, query *gcpStorage.Query) *gcpStorage.ObjectIterator { + items := c.Client.Bucket(bucketName).Objects(ctx, query) + return items +} diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go new file mode 100644 index 00000000..30412f49 --- /dev/null +++ b/pkg/gcp/gcp_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcp + +import ( + "context" + "testing" + + "gotest.tools/assert" +) + +func TestSetRange(t *testing.T) { + client, err := NewClient(context.Background()) + assert.NilError(t, err) + testCases := []struct { + title string + start int64 + end int64 + }{ + { + title: "Test Case 1", + start: 1, + end: 5, + }, + { + title: "Test Case 2", + start: 3, + end: 6, + }, + { + title: "Test Case 3", + start: 4, + end: 5, + }, + { + title: "Test Case 4", + start: 2, + end: 7, + }, + } + for _, tt := range testCases { + t.Run(tt.title, func(t *testing.T) { + client.SetRange(tt.start, tt.end) + assert.Equal(t, tt.start, client.startRange) + assert.Equal(t, tt.end, client.endRange) + }) + } +} From a5588fb191a99b22b85b032edd72af8ab5b3866b Mon Sep 17 00:00:00 2001 From: pa250194 Date: Wed, 1 Sep 2021 14:41:40 -0500 Subject: [PATCH 02/53] Added Comments for reconcileWithGCP and reconcileWithMinio Signed-off-by: pa250194 --- controllers/bucket_controller.go | 16 ++++++++++++---- docs/api/source.md | 4 ++-- 2 files changed, 14 insertions(+), 6 deletions(-) diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 3ec8d5e2..3ff17f22 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -186,7 +186,7 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket return sourceBucket, err } } else { - sourceBucket, tempDir, err = r.reconcileWithAWS(ctx, bucket) + sourceBucket, tempDir, err = r.reconcileWithMinio(ctx, bucket) if err != nil { return sourceBucket, err } @@ -259,6 +259,8 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1. return ctrl.Result{}, nil } +// reconcileWithGCP handles getting objects from a Google Cloud Platform bucket +// using a gcp client func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { gcpClient, err := r.authGCP(ctx, bucket) if err != nil { @@ -330,8 +332,10 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1 return sourcev1.Bucket{}, tempDir, nil } -func (r *BucketReconciler) reconcileWithAWS(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { - s3Client, err := r.auth(ctx, bucket) +// reconcileWithMinio handles getting objects from an S3 compatible bucket +// using a minio client +func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { + s3Client, err := r.authMinio(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err @@ -404,6 +408,8 @@ func (r *BucketReconciler) reconcileWithAWS(ctx context.Context, bucket sourcev1 return sourcev1.Bucket{}, tempDir, nil } +// authGCP creates a new Google Cloud Platform storage client +// to interact with the Storage service. func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) (*gcp.GCPClient, error) { client, err := gcp.NewClient(ctx) if err != nil { @@ -412,7 +418,9 @@ func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) return client, nil } -func (r *BucketReconciler) auth(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) { +// authMinio creates a new Minio client to interact with S3 +// compatible storage services. +func (r *BucketReconciler) authMinio(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) { opt := minio.Options{ Region: bucket.Spec.Region, Secure: !bucket.Spec.Insecure, diff --git a/docs/api/source.md b/docs/api/source.md index 7b1fede4..f2ac54a0 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -1470,8 +1470,8 @@ Artifact includedArtifacts
- -[]*./api/v1beta1.Artifact + +[]*github.com/fluxcd/source-controller/api/v1beta1.Artifact From 78379ddcd5d8da75b07b6b2a0763d5264d649735 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 2 Sep 2021 08:51:02 -0500 Subject: [PATCH 03/53] Added initial testing for new GCP provider Signed-off-by: pa250194 --- pkg/gcp/gcp_test.go | 79 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 30412f49..459e691a 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -18,12 +18,24 @@ package gcp import ( "context" + "os" + "path/filepath" "testing" "gotest.tools/assert" ) +func TestNewClient(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() + client, err := NewClient(context.Background()) + assert.NilError(t, err) + assert.Assert(t, client.Client != nil) +} + func TestSetRange(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() client, err := NewClient(context.Background()) assert.NilError(t, err) testCases := []struct { @@ -60,3 +72,70 @@ func TestSetRange(t *testing.T) { }) } } + +func TestBucketExists(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() + ctx := context.Background() + bucketName := "" + client, err := NewClient(ctx) + assert.NilError(t, err) + exists, err := client.BucketExists(ctx, bucketName) + assert.NilError(t, err) + assert.Assert(t, exists) +} + +func TestObjectExists(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() + ctx := context.Background() + // bucketName is the name of the bucket which contains the object + bucketName := "" + // objectName is the path to the object within the bucket + objectName := "" + client, err := NewClient(ctx) + assert.NilError(t, err) + exists, attrs, err := client.ObjectExists(ctx, bucketName, objectName) + assert.NilError(t, err) + assert.Assert(t, exists) + assert.Assert(t, attrs != nil) +} + +func TestListObjects(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() + ctx := context.Background() + // bucketName is the name of the bucket which contains the object + bucketName := "" + client, err := NewClient(ctx) + assert.NilError(t, err) + objects := client.ListObjects(ctx, bucketName, nil) + assert.NilError(t, err) + assert.Assert(t, objects != nil) + for { + object, err := objects.Next() + if err == IteratorDone { + break + } + assert.Assert(t, object != nil) + } +} + +func TestFGetObject(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() + ctx := context.Background() + // bucketName is the name of the bucket which contains the object + bucketName := "" + // objectName is the path to the object within the bucket + objectName := "" + tempDir, err := os.MkdirTemp("", bucketName) + if err != nil { + assert.NilError(t, err) + } + localPath := filepath.Join(tempDir, objectName) + client, err := NewClient(ctx) + assert.NilError(t, err) + objErr := client.FGetObject(ctx, bucketName, objectName, localPath) + assert.NilError(t, objErr) +} From 90395f426a460a253c20a2b28899ede7f4e95792 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 2 Sep 2021 08:51:55 -0500 Subject: [PATCH 04/53] Remove .DS_STORE file Signed-off-by: pa250194 --- pkg/.DS_Store | Bin 6148 -> 0 bytes 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 pkg/.DS_Store diff --git a/pkg/.DS_Store b/pkg/.DS_Store deleted file mode 100644 index 63391478b0cd7ae519be6c18bc8a84af5726952c..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 6148 zcmeHK!EO^V5FIBeH~~ZsNc4iV7j6+sXoboFX$kG2H>5>y0Mza_t8}Gh9qn$2AO!7i zv_H}B;0yR1cw?K2v~Z<>W~A{Ok7sQ8d1dbqks8k@BceVLX((f>i{=-?*BR(owR5O3dUy@m#RrPTARkhlk zjceDn?&;pG-rMNdETTHD=hZkaUvuwyTpuUieql_0ZpX>XEL|M*?>sX_9cM*a+T?7K zBIM1hteBX^*vyMbW#@_9fYx2zJ?L*Pms>kKgJ85hS`C8b2p9sRWHF z;-^%gUsAfBr|wdsI0hBTH*f_LT2O3)f z06TD7fo*^OfdM;!p~uD{JP_qlfi6|~5<~ggLAi8{hd#b>=+a5ql`)TBS@{Y@`O-rL zHk?%GP+DO?7+7bZ?Yez_|Mz}h|F0)WPZ$se{woGlXE)j%V@v*Ry|g)gYXj&Zl!fCO mhvz9U>{E8Tby{b#nIr From 0444c6e16d6823c878fcef45c6318fe614fa7837 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Fri, 10 Sep 2021 16:01:16 -0500 Subject: [PATCH 05/53] Service Account Key Authentication to GCP Provider Signed-off-by: pa250194 --- api/v1beta1/bucket_types.go | 2 +- .../source.toolkit.fluxcd.io_buckets.yaml | 1 + controllers/bucket_controller.go | 96 ++++---- go.mod | 1 + go.sum | 1 + pkg/gcp/gcp.go | 163 ++++++++++++-- pkg/gcp/gcp_test.go | 213 +++++++++--------- pkg/gcp/mocks/mock_gcp_storage.go | 211 +++++++++++++++++ 8 files changed, 512 insertions(+), 176 deletions(-) create mode 100644 pkg/gcp/mocks/mock_gcp_storage.go diff --git a/api/v1beta1/bucket_types.go b/api/v1beta1/bucket_types.go index 1dc68851..e046eaa8 100644 --- a/api/v1beta1/bucket_types.go +++ b/api/v1beta1/bucket_types.go @@ -30,7 +30,7 @@ const ( // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws + // +kubebuilder:validation:Enum=generic;aws;gcp // +kubebuilder:default:=generic // +optional Provider string `json:"provider,omitempty"` diff --git a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml index 5905c1d7..a64e98b4 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml @@ -66,6 +66,7 @@ spec: enum: - generic - aws + - gcp type: string region: description: The bucket region. diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 3ff17f22..9e4eee73 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -177,16 +177,21 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, error) { - var tempDir string var err error var sourceBucket sourcev1.Bucket + tempDir, err := os.MkdirTemp("", bucket.Name) + if err != nil { + err = fmt.Errorf("tmp dir error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err + } + defer os.RemoveAll(tempDir) if bucket.Spec.Provider == sourcev1.GoogleBucketProvider { - sourceBucket, tempDir, err = r.reconcileWithGCP(ctx, bucket) + sourceBucket, err = r.reconcileWithGCP(ctx, bucket, tempDir) if err != nil { return sourceBucket, err } } else { - sourceBucket, tempDir, err = r.reconcileWithMinio(ctx, bucket) + sourceBucket, err = r.reconcileWithMinio(ctx, bucket, tempDir) if err != nil { return sourceBucket, err } @@ -261,41 +266,36 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1. // reconcileWithGCP handles getting objects from a Google Cloud Platform bucket // using a gcp client -func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { +func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) { gcpClient, err := r.authGCP(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err } defer gcpClient.Client.Close() - // create tmp dir - tempDir, err := os.MkdirTemp("", bucket.Name) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err - } - defer os.RemoveAll(tempDir) ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) defer cancel() exists, err := gcpClient.BucketExists(ctxTimeout, bucket.Spec.BucketName) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if !exists { err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // Look for file with ignore rules first. path := filepath.Join(tempDir, sourceignore.IgnoreFile) if err := gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path); err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + if err == gcp.ErrorObjectDoesNotExist && sourceignore.IgnoreFile != ".sourceignore" { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err + } } ps, err := sourceignore.ReadIgnoreFile(path, nil) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // In-spec patterns take precedence if bucket.Spec.Ignore != nil { @@ -311,7 +311,7 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1 } if err != nil { err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if strings.HasSuffix(object.Name, "/") || object.Name == sourceignore.IgnoreFile { @@ -323,42 +323,33 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1 } localPath := filepath.Join(tempDir, object.Name) - // FGetObject - get and download bucket object if err = gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Name, localPath); err != nil { err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } } - return sourcev1.Bucket{}, tempDir, nil + return sourcev1.Bucket{}, nil } // reconcileWithMinio handles getting objects from an S3 compatible bucket // using a minio client -func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { +func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) { s3Client, err := r.authMinio(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err } - // create tmp dir - tempDir, err := os.MkdirTemp("", bucket.Name) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err - } - defer os.RemoveAll(tempDir) - ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) defer cancel() exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if !exists { err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // Look for file with ignore rules first @@ -367,12 +358,12 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source path := filepath.Join(tempDir, sourceignore.IgnoreFile) if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil { if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } } ps, err := sourceignore.ReadIgnoreFile(path, nil) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // In-spec patterns take precedence if bucket.Spec.Ignore != nil { @@ -387,7 +378,7 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source }) { if object.Err != nil { err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile { @@ -402,20 +393,43 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{}) if err != nil { err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } } - return sourcev1.Bucket{}, tempDir, nil + return sourcev1.Bucket{}, nil } // authGCP creates a new Google Cloud Platform storage client -// to interact with the Storage service. +// to interact with the storage service. func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) (*gcp.GCPClient, error) { - client, err := gcp.NewClient(ctx) - if err != nil { - return nil, err + var client *gcp.GCPClient + var err error + if bucket.Spec.SecretRef != nil { + secretName := types.NamespacedName{ + Namespace: bucket.GetNamespace(), + Name: bucket.Spec.SecretRef.Name, + } + + var secret corev1.Secret + if err := r.Get(ctx, secretName, &secret); err != nil { + return nil, fmt.Errorf("credentials secret error: %w", err) + } + if err := gcp.ValidateSecret(secret.Data, secret.Name); err != nil { + return nil, err + } + serviceAccount := gcp.InitCredentialsWithSecret(secret.Data) + client, err = gcp.NewClientWithSAKey(ctx, serviceAccount) + if err != nil { + return nil, err + } + } else { + client, err = gcp.NewClient(ctx) + if err != nil { + return nil, err + } } return client, nil + } // authMinio creates a new Minio client to interact with S3 diff --git a/go.mod b/go.mod index 49ebac77..1d60520c 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/mock v1.6.0 // indirect github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/libgit2/git2go/v31 v31.4.14 github.com/minio/minio-go/v7 v7.0.10 diff --git a/go.sum b/go.sum index a1ea7ce9..be1c5759 100644 --- a/go.sum +++ b/go.sum @@ -415,6 +415,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 18ea53fe..8f2f8811 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -18,56 +18,171 @@ package gcp import ( "context" + "encoding/json" "errors" + "fmt" "io" "os" "path/filepath" gcpStorage "cloud.google.com/go/storage" interator "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +const ( + ServiceAccount = "service_account" + AuthUri = "https://accounts.google.com/o/oauth2/auth" + TokenUri = "https://oauth2.googleapis.com/token" + AuthProviderX509CertUrl = "https://www.googleapis.com/oauth2/v1/certs" ) var ( // IteratorDone is returned when the looping of objects/content // has reached the end of the iteration. IteratorDone = interator.Done - // DirectoryExists is an error returned when the filename provided + // ErrorDirectoryExists is an error returned when the filename provided // is a directory. - DirectoryExists = errors.New("filename is a directory") - // ObjectDoesNotExist is an error returned when the object whose name + ErrorDirectoryExists = errors.New("filename is a directory") + // ErrorObjectDoesNotExist is an error returned when the object whose name // is provided does not exist. - ObjectDoesNotExist = errors.New("object does not exist") + ErrorObjectDoesNotExist = errors.New("object does not exist") ) +type Client interface { + Bucket(string) *gcpStorage.BucketHandle + Close() error +} + +type BucketHandle interface { + Create(context.Context, string, *gcpStorage.BucketAttrs) error + Delete(context.Context) error + Attrs(context.Context) (*gcpStorage.BucketAttrs, error) + Object(string) *gcpStorage.ObjectHandle + Objects(context.Context, *gcpStorage.Query) *gcpStorage.ObjectIterator +} + +type ObjectHandle interface { + Attrs(context.Context) (*gcpStorage.ObjectAttrs, error) + NewRangeReader(context.Context, int64, int64) (*gcpStorage.Reader, error) +} type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. - Client *gcpStorage.Client + Client Client // startRange is the starting read value for // reading the object from bucket. - startRange int64 + StartRange int64 // endRange is the ending read value for // reading the object from bucket. - endRange int64 + EndRange int64 +} + +// CredentialsFile struct representing the GCP Service Account +// JSON file. +type CredentialsFile struct { + Type string `json:"type"` + ProjectID string `json:"project_id"` + PrivateKeyID string `json:"private_key_id"` + PrivateKey string `json:"private_key"` + ClientEmail string `json:"client_email"` + ClientID string `json:"client_id"` + AuthUri string `json:"auth_uri"` + TokenUri string `json:"token_uri"` + AuthProviderX509CertUrl string `json:"auth_provider_x509_cert_url"` + ClientX509CertUrl string `json:"client_x509_cert_url"` } // NewClient creates a new GCP storage client // The Google Storage Client will automatically // look for the Google Application Credential environment variable -// or look for the Google Application Credential file +// or look for the Google Application Credential file. func NewClient(ctx context.Context) (*GCPClient, error) { client, err := gcpStorage.NewClient(ctx) if err != nil { return nil, err } - return &GCPClient{Client: client, startRange: 0, endRange: -1}, nil + + return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil +} + +// NewClientWithSAKey creates a new GCP storage client +// It uses the provided JSON file with service account details +// To authenticate. +func NewClientWithSAKey(ctx context.Context, credentials *CredentialsFile) (*GCPClient, error) { + saAccount, err := credentials.credentailsToJSON() + if err != nil { + return nil, err + } + + client, err := gcpStorage.NewClient(ctx, option.WithCredentialsJSON(saAccount)) + if err != nil { + return nil, err + } + + return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil +} + +// credentailsToJSON converts GCP service account credentials struct to JSON. +func (credentials *CredentialsFile) credentailsToJSON() ([]byte, error) { + credentialsJSON, err := json.Marshal(credentials) + if err != nil { + return nil, err + } + + return credentialsJSON, nil +} + +// InitCredentialsWithSecret creates a new credential +// by initializing a new CredentialsFile struct +func InitCredentialsWithSecret(secret map[string][]byte) *CredentialsFile { + return &CredentialsFile{ + Type: ServiceAccount, + ProjectID: string(secret["projectid"]), + PrivateKeyID: string(secret["privatekeyid"]), + PrivateKey: string(secret["privatekey"]), + ClientEmail: string(secret["clientemail"]), + ClientID: string(secret["clientid"]), + AuthUri: AuthUri, + TokenUri: TokenUri, + AuthProviderX509CertUrl: AuthProviderX509CertUrl, + ClientX509CertUrl: string(secret["certurl"]), + } +} + +// ValidateSecret validates the credential secrets +// It ensures that needed secret fields are not missing. +func ValidateSecret(secret map[string][]byte, name string) error { + if _, exists := secret["projectid"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'projectid'", name) + } + if _, exists := secret["privatekeyid"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'privatekeyid'", name) + } + if _, exists := secret["privatekey"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'privatekey'", name) + } + if _, exists := secret["clientemail"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) + } + if _, exists := secret["clientemail"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) + } + if _, exists := secret["clientid"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'clientid'", name) + } + if _, exists := secret["certurl"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'certurl'", name) + } + + return nil } // SetRange sets the startRange and endRange used to read the Object from // the bucket. It is a helper method for resumable downloads. func (c *GCPClient) SetRange(start, end int64) { - c.startRange = start - c.endRange = end + c.StartRange = start + c.EndRange = end } // BucketExists checks if the bucket with the provided name exists. @@ -82,15 +197,18 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, return true, nil } -// ObjectExists checks if the object with the provided name exists. +// ObjectAttributes checks if the object with the provided name exists. // If it exists the Object attributes are returned. -func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) { +func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) { attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) // ErrObjectNotExist is returned if the object does not exist + if err == gcpStorage.ErrObjectNotExist { + return false, nil, err + } if err != nil { return false, nil, err } - return true, attrs, err + return true, attrs, nil } // FGetObject gets the object from the bucket and downloads the object locally @@ -101,7 +219,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca if err == nil { // If the destination exists and is a directory. if dirStatus.IsDir() { - return DirectoryExists + return ErrorDirectoryExists } } @@ -124,17 +242,16 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca // ObjectExists verifies if object exists and you have permission to access. // Check if the object exists and if you have permission to access it // The Object attributes are returned if the Object exists. - exists, attrs, err := c.ObjectExists(ctx, bucketName, objectName) + exists, attrs, err := c.ObjectAttributes(ctx, bucketName, objectName) if err != nil { return err } if !exists { - return ObjectDoesNotExist + return ErrorObjectDoesNotExist } // Write to a temporary file "filename.part.gcp" before saving. - filePartPath := localPath + attrs.Etag + ".part.gcp" - + filePartPath := localPath + ".part.gcp" // If exists, open in append mode. If not create it as a part file. filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) if err != nil { @@ -165,25 +282,25 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca } // Get Object from GCP Bucket - objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.startRange, c.endRange) + objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.StartRange, c.EndRange) if err != nil { return err } defer objectReader.Close() // Write to the part file. - if _, err = io.CopyN(filePart, objectReader, attrs.Size); err != nil { + if _, err := io.CopyN(filePart, objectReader, attrs.Size); err != nil { return err } // Close the file before rename, this is specifically needed for Windows users. closeAndRemove = false - if err = filePart.Close(); err != nil { + if err := filePart.Close(); err != nil { return err } // Safely completed. Now commit by renaming to actual filename. - if err = os.Rename(filePartPath, localPath); err != nil { + if err := os.Rename(filePartPath, localPath); err != nil { return err } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 459e691a..f30774ac 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -14,128 +14,119 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gcp +package gcp_test import ( "context" "os" "path/filepath" "testing" + "time" - "gotest.tools/assert" + gcpStorage "cloud.google.com/go/storage" + "github.com/fluxcd/source-controller/pkg/gcp" + "github.com/fluxcd/source-controller/pkg/gcp/mocks" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) -func TestNewClient(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - client, err := NewClient(context.Background()) - assert.NilError(t, err) - assert.Assert(t, client.Client != nil) +var ( + MockCtrl *gomock.Controller + MockClient *mocks.MockClient + MockBucketHandle *mocks.MockBucketHandle + MockObjectHandle *mocks.MockObjectHandle + bucketName string = "test-bucket" + objectName string = "test.yaml" + localPath string +) + +// mockgen -destination=mocks/mock_gcp_storage.go -package=mocks -source=gcp.go GCPStorageService +func TestGCPProvider(t *testing.T) { + MockCtrl = gomock.NewController(GinkgoT()) + RegisterFailHandler(Fail) + RunSpecs(t, "Test GCP Storage Provider Suite") } -func TestSetRange(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - client, err := NewClient(context.Background()) - assert.NilError(t, err) - testCases := []struct { - title string - start int64 - end int64 - }{ - { - title: "Test Case 1", - start: 1, - end: 5, - }, - { - title: "Test Case 2", - start: 3, - end: 6, - }, - { - title: "Test Case 3", - start: 4, - end: 5, - }, - { - title: "Test Case 4", - start: 2, - end: 7, - }, - } - for _, tt := range testCases { - t.Run(tt.title, func(t *testing.T) { - client.SetRange(tt.start, tt.end) - assert.Equal(t, tt.start, client.startRange) - assert.Equal(t, tt.end, client.endRange) - }) - } -} - -func TestBucketExists(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - bucketName := "" - client, err := NewClient(ctx) - assert.NilError(t, err) - exists, err := client.BucketExists(ctx, bucketName) - assert.NilError(t, err) - assert.Assert(t, exists) -} - -func TestObjectExists(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - // bucketName is the name of the bucket which contains the object - bucketName := "" - // objectName is the path to the object within the bucket - objectName := "" - client, err := NewClient(ctx) - assert.NilError(t, err) - exists, attrs, err := client.ObjectExists(ctx, bucketName, objectName) - assert.NilError(t, err) - assert.Assert(t, exists) - assert.Assert(t, attrs != nil) -} - -func TestListObjects(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - // bucketName is the name of the bucket which contains the object - bucketName := "" - client, err := NewClient(ctx) - assert.NilError(t, err) - objects := client.ListObjects(ctx, bucketName, nil) - assert.NilError(t, err) - assert.Assert(t, objects != nil) - for { - object, err := objects.Next() - if err == IteratorDone { - break - } - assert.Assert(t, object != nil) - } -} - -func TestFGetObject(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - // bucketName is the name of the bucket which contains the object - bucketName := "" - // objectName is the path to the object within the bucket - objectName := "" +var _ = BeforeSuite(func() { + MockClient = mocks.NewMockClient(MockCtrl) + MockBucketHandle = mocks.NewMockBucketHandle(MockCtrl) + MockObjectHandle = mocks.NewMockObjectHandle(MockCtrl) tempDir, err := os.MkdirTemp("", bucketName) if err != nil { - assert.NilError(t, err) + Expect(err).ToNot(HaveOccurred()) } - localPath := filepath.Join(tempDir, objectName) - client, err := NewClient(ctx) - assert.NilError(t, err) - objErr := client.FGetObject(ctx, bucketName, objectName, localPath) - assert.NilError(t, objErr) -} + localPath = filepath.Join(tempDir, objectName) + MockClient.EXPECT().Bucket(bucketName).Return(MockBucketHandle).AnyTimes() + MockBucketHandle.EXPECT().Object(objectName).Return(&gcpStorage.ObjectHandle{}).AnyTimes() + MockBucketHandle.EXPECT().Attrs(context.Background()).Return(&gcpStorage.BucketAttrs{ + Name: bucketName, + Created: time.Now(), + Etag: "test-etag", + }, nil).AnyTimes() + MockBucketHandle.EXPECT().Objects(gomock.Any(), nil).Return(&gcpStorage.ObjectIterator{}).AnyTimes() + MockObjectHandle.EXPECT().Attrs(gomock.Any()).Return(&gcpStorage.ObjectAttrs{ + Bucket: bucketName, + Name: objectName, + ContentType: "text/x-yaml", + Etag: "test-etag", + Size: 125, + Created: time.Now(), + }, nil).AnyTimes() + MockObjectHandle.EXPECT().NewRangeReader(gomock.Any(), 10, 125).Return(&gcpStorage.Reader{}, nil).AnyTimes() +}) + +var _ = Describe("GCP Storage Provider", func() { + Describe("Get GCP Storage Provider client from gcp", func() { + + Context("Gcp storage Bucket - BucketExists", func() { + It("should not return an error when fetching gcp storage bucket", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + exists, err := gcpClient.BucketExists(context.Background(), bucketName) + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + }) + Context("Gcp storage Bucket - FGetObject", func() { + It("should get the object from the bucket and download the object locally", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + err := gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath) + Expect(err).ToNot(HaveOccurred()) + }) + }) + Context("Gcp storage Bucket - ObjectAttributes", func() { + It("should get the object attributes", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + exists, attrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + Expect(attrs).ToNot(BeNil()) + }) + + Context("Gcp storage Bucket - SetRange", func() { + It("should set the range of the io reader seeker for the file download", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + gcpClient.SetRange(2, 5) + Expect(gcpClient.StartRange).To(Equal(int64(2))) + Expect(gcpClient.EndRange).To(Equal(int64(5))) + }) + }) + }) + }) +}) diff --git a/pkg/gcp/mocks/mock_gcp_storage.go b/pkg/gcp/mocks/mock_gcp_storage.go new file mode 100644 index 00000000..54b78be1 --- /dev/null +++ b/pkg/gcp/mocks/mock_gcp_storage.go @@ -0,0 +1,211 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: gcp.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + storage "cloud.google.com/go/storage" + gomock "github.com/golang/mock/gomock" +) + +// MockClient is a mock of Client interface. +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder +} + +// MockClientMockRecorder is the mock recorder for MockClient. +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance. +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// Bucket mocks base method. +func (m *MockClient) Bucket(arg0 string) *storage.BucketHandle { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Bucket", arg0) + ret0, _ := ret[0].(*storage.BucketHandle) + return ret0 +} + +// Bucket indicates an expected call of Bucket. +func (mr *MockClientMockRecorder) Bucket(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockClient)(nil).Bucket), arg0) +} + +// Close mocks base method. +func (m *MockClient) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockClientMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClient)(nil).Close)) +} + +// MockBucketHandle is a mock of BucketHandle interface. +type MockBucketHandle struct { + ctrl *gomock.Controller + recorder *MockBucketHandleMockRecorder +} + +// MockBucketHandleMockRecorder is the mock recorder for MockBucketHandle. +type MockBucketHandleMockRecorder struct { + mock *MockBucketHandle +} + +// NewMockBucketHandle creates a new mock instance. +func NewMockBucketHandle(ctrl *gomock.Controller) *MockBucketHandle { + mock := &MockBucketHandle{ctrl: ctrl} + mock.recorder = &MockBucketHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBucketHandle) EXPECT() *MockBucketHandleMockRecorder { + return m.recorder +} + +// Attrs mocks base method. +func (m *MockBucketHandle) Attrs(arg0 context.Context) (*storage.BucketAttrs, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Attrs", arg0) + ret0, _ := ret[0].(*storage.BucketAttrs) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Attrs indicates an expected call of Attrs. +func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0) +} + +// Create mocks base method. +func (m *MockBucketHandle) Create(arg0 context.Context, arg1 string, arg2 *storage.BucketAttrs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockBucketHandleMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockBucketHandle)(nil).Create), arg0, arg1, arg2) +} + +// Delete mocks base method. +func (m *MockBucketHandle) Delete(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockBucketHandleMockRecorder) Delete(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBucketHandle)(nil).Delete), arg0) +} + +// Object mocks base method. +func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Object", arg0) + ret0, _ := ret[0].(*storage.ObjectHandle) + return ret0 +} + +// Object indicates an expected call of Object. +func (mr *MockBucketHandleMockRecorder) Object(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Object", reflect.TypeOf((*MockBucketHandle)(nil).Object), arg0) +} + +// Objects mocks base method. +func (m *MockBucketHandle) Objects(arg0 context.Context, arg1 *storage.Query) *storage.ObjectIterator { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Objects", arg0, arg1) + ret0, _ := ret[0].(*storage.ObjectIterator) + return ret0 +} + +// Objects indicates an expected call of Objects. +func (mr *MockBucketHandleMockRecorder) Objects(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Objects", reflect.TypeOf((*MockBucketHandle)(nil).Objects), arg0, arg1) +} + +// MockObjectHandle is a mock of ObjectHandle interface. +type MockObjectHandle struct { + ctrl *gomock.Controller + recorder *MockObjectHandleMockRecorder +} + +// MockObjectHandleMockRecorder is the mock recorder for MockObjectHandle. +type MockObjectHandleMockRecorder struct { + mock *MockObjectHandle +} + +// NewMockObjectHandle creates a new mock instance. +func NewMockObjectHandle(ctrl *gomock.Controller) *MockObjectHandle { + mock := &MockObjectHandle{ctrl: ctrl} + mock.recorder = &MockObjectHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockObjectHandle) EXPECT() *MockObjectHandleMockRecorder { + return m.recorder +} + +// Attrs mocks base method. +func (m *MockObjectHandle) Attrs(arg0 context.Context) (*storage.ObjectAttrs, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Attrs", arg0) + ret0, _ := ret[0].(*storage.ObjectAttrs) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Attrs indicates an expected call of Attrs. +func (mr *MockObjectHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockObjectHandle)(nil).Attrs), arg0) +} + +// NewRangeReader mocks base method. +func (m *MockObjectHandle) NewRangeReader(arg0 context.Context, arg1, arg2 int64) (*storage.Reader, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewRangeReader", arg0, arg1, arg2) + ret0, _ := ret[0].(*storage.Reader) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NewRangeReader indicates an expected call of NewRangeReader. +func (mr *MockObjectHandleMockRecorder) NewRangeReader(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewRangeReader", reflect.TypeOf((*MockObjectHandle)(nil).NewRangeReader), arg0, arg1, arg2) +} From eeb38bdf2ce6e1a0dffe1c4979faa3ecc4c9cf84 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Tue, 14 Sep 2021 09:34:02 -0500 Subject: [PATCH 06/53] Tests for GCP Bucket Provider Signed-off-by: pa250194 --- docs/spec/v1alpha1/buckets.md | 2 +- pkg/gcp/gcp.go | 8 ++------ pkg/gcp/gcp_test.go | 2 +- pkg/gcp/mocks/mock_gcp_storage.go | 28 ---------------------------- 4 files changed, 4 insertions(+), 36 deletions(-) diff --git a/docs/spec/v1alpha1/buckets.md b/docs/spec/v1alpha1/buckets.md index 0ad60f41..53838e25 100644 --- a/docs/spec/v1alpha1/buckets.md +++ b/docs/spec/v1alpha1/buckets.md @@ -11,7 +11,7 @@ Bucket: // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws + // +kubebuilder:validation:Enum=generic;aws;gcp // +optional Provider string `json:"provider,omitempty"` diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 8f2f8811..2c372e9e 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -55,8 +55,6 @@ type Client interface { } type BucketHandle interface { - Create(context.Context, string, *gcpStorage.BucketAttrs) error - Delete(context.Context) error Attrs(context.Context) (*gcpStorage.BucketAttrs, error) Object(string) *gcpStorage.ObjectHandle Objects(context.Context, *gcpStorage.Query) *gcpStorage.ObjectIterator @@ -66,10 +64,11 @@ type ObjectHandle interface { Attrs(context.Context) (*gcpStorage.ObjectAttrs, error) NewRangeReader(context.Context, int64, int64) (*gcpStorage.Reader, error) } + type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. - Client Client + Client // startRange is the starting read value for // reading the object from bucket. StartRange int64 @@ -165,9 +164,6 @@ func ValidateSecret(secret map[string][]byte, name string) error { if _, exists := secret["clientemail"]; !exists { return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) } - if _, exists := secret["clientemail"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) - } if _, exists := secret["clientid"]; !exists { return fmt.Errorf("invalid '%s' secret data: required fields 'clientid'", name) } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index f30774ac..ed72f9d2 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -57,7 +57,7 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) } localPath = filepath.Join(tempDir, objectName) - MockClient.EXPECT().Bucket(bucketName).Return(MockBucketHandle).AnyTimes() + MockClient.EXPECT().Bucket(bucketName).Return(&gcpStorage.BucketHandle{}).AnyTimes() MockBucketHandle.EXPECT().Object(objectName).Return(&gcpStorage.ObjectHandle{}).AnyTimes() MockBucketHandle.EXPECT().Attrs(context.Background()).Return(&gcpStorage.BucketAttrs{ Name: bucketName, diff --git a/pkg/gcp/mocks/mock_gcp_storage.go b/pkg/gcp/mocks/mock_gcp_storage.go index 54b78be1..25b5e9c1 100644 --- a/pkg/gcp/mocks/mock_gcp_storage.go +++ b/pkg/gcp/mocks/mock_gcp_storage.go @@ -101,34 +101,6 @@ func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0) } -// Create mocks base method. -func (m *MockBucketHandle) Create(arg0 context.Context, arg1 string, arg2 *storage.BucketAttrs) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// Create indicates an expected call of Create. -func (mr *MockBucketHandleMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockBucketHandle)(nil).Create), arg0, arg1, arg2) -} - -// Delete mocks base method. -func (m *MockBucketHandle) Delete(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Delete indicates an expected call of Delete. -func (mr *MockBucketHandleMockRecorder) Delete(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBucketHandle)(nil).Delete), arg0) -} - // Object mocks base method. func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle { m.ctrl.T.Helper() From c204f6a8eed125b53fd48c0c8ab006ce3fb7b6cd Mon Sep 17 00:00:00 2001 From: pa250194 Date: Wed, 15 Sep 2021 14:42:53 -0500 Subject: [PATCH 07/53] Added Tests to GCP provider Signed-off-by: pa250194 --- pkg/gcp/gcp.go | 18 +- pkg/gcp/gcp_test.go | 342 ++++++++++++++++++++++-------- pkg/gcp/mocks/mock_gcp_storage.go | 183 ---------------- 3 files changed, 249 insertions(+), 294 deletions(-) delete mode 100644 pkg/gcp/mocks/mock_gcp_storage.go diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 2c372e9e..ec56384a 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -49,26 +49,10 @@ var ( ErrorObjectDoesNotExist = errors.New("object does not exist") ) -type Client interface { - Bucket(string) *gcpStorage.BucketHandle - Close() error -} - -type BucketHandle interface { - Attrs(context.Context) (*gcpStorage.BucketAttrs, error) - Object(string) *gcpStorage.ObjectHandle - Objects(context.Context, *gcpStorage.Query) *gcpStorage.ObjectIterator -} - -type ObjectHandle interface { - Attrs(context.Context) (*gcpStorage.ObjectAttrs, error) - NewRangeReader(context.Context, int64, int64) (*gcpStorage.Reader, error) -} - type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. - Client + *gcpStorage.Client // startRange is the starting read value for // reading the object from bucket. StartRange int64 diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index ed72f9d2..64db105f 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -18,6 +18,15 @@ package gcp_test import ( "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "net/http/httptest" "os" "path/filepath" "testing" @@ -25,108 +34,253 @@ import ( gcpStorage "cloud.google.com/go/storage" "github.com/fluxcd/source-controller/pkg/gcp" - "github.com/fluxcd/source-controller/pkg/gcp/mocks" - "github.com/golang/mock/gomock" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "google.golang.org/api/googleapi" + raw "google.golang.org/api/storage/v1" + "gotest.tools/assert" + + "google.golang.org/api/option" +) + +const ( + bucketName string = "test-bucket" + objectName string = "test.yaml" ) var ( - MockCtrl *gomock.Controller - MockClient *mocks.MockClient - MockBucketHandle *mocks.MockBucketHandle - MockObjectHandle *mocks.MockObjectHandle - bucketName string = "test-bucket" - objectName string = "test.yaml" - localPath string + Client *gcpStorage.Client + err error ) -// mockgen -destination=mocks/mock_gcp_storage.go -package=mocks -source=gcp.go GCPStorageService -func TestGCPProvider(t *testing.T) { - MockCtrl = gomock.NewController(GinkgoT()) - RegisterFailHandler(Fail) - RunSpecs(t, "Test GCP Storage Provider Suite") +func TestMain(m *testing.M) { + hc, close := newTestServer(func(w http.ResponseWriter, r *http.Request) { + io.Copy(ioutil.Discard, r.Body) + w.WriteHeader(200) + if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s?alt=json&prettyPrint=false&projection=full", bucketName) { + response := getBucket() + jsonedResp, err := json.Marshal(response) + if err != nil { + log.Fatalf("error marshalling resp %v\n", err) + } + _, err = w.Write(jsonedResp) + if err != nil { + log.Fatalf("error writing jsonedResp %v\n", err) + } + } else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) { + response := getObject() + jsonedResp, err := json.Marshal(response) + if err != nil { + log.Fatalf("error marshalling resp %v\n", err) + } + _, err = w.Write(jsonedResp) + if err != nil { + log.Fatalf("error writing jsonedResp %v\n", err) + } + } else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o?alt=json&delimiter=&endOffset=&pageToken=&prefix=&prettyPrint=false&projection=full&startOffset=&versions=false", bucketName) { + response := getObject() + jsonedResp, err := json.Marshal(response) + if err != nil { + log.Fatalf("error marshalling resp %v\n", err) + } + _, err = w.Write(jsonedResp) + if err != nil { + log.Fatalf("error writing jsonedResp %v\n", err) + } + } else if r.RequestURI == fmt.Sprintf("/%s/test.yaml", bucketName) || r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) { + response := getObjectFile() + _, err = w.Write([]byte(response)) + if err != nil { + log.Fatalf("error writing jsonedResp %v\n", err) + } + } + }) + ctx := context.Background() + Client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) + if err != nil { + log.Fatal(err) + } + run := m.Run() + close() + os.Exit(run) } -var _ = BeforeSuite(func() { - MockClient = mocks.NewMockClient(MockCtrl) - MockBucketHandle = mocks.NewMockBucketHandle(MockCtrl) - MockObjectHandle = mocks.NewMockObjectHandle(MockCtrl) - tempDir, err := os.MkdirTemp("", bucketName) - if err != nil { - Expect(err).ToNot(HaveOccurred()) +func TestBucketExists(t *testing.T) { + gcpClient := &gcp.GCPClient{ + Client: Client, + StartRange: 0, + EndRange: -1, } - localPath = filepath.Join(tempDir, objectName) - MockClient.EXPECT().Bucket(bucketName).Return(&gcpStorage.BucketHandle{}).AnyTimes() - MockBucketHandle.EXPECT().Object(objectName).Return(&gcpStorage.ObjectHandle{}).AnyTimes() - MockBucketHandle.EXPECT().Attrs(context.Background()).Return(&gcpStorage.BucketAttrs{ - Name: bucketName, - Created: time.Now(), - Etag: "test-etag", - }, nil).AnyTimes() - MockBucketHandle.EXPECT().Objects(gomock.Any(), nil).Return(&gcpStorage.ObjectIterator{}).AnyTimes() - MockObjectHandle.EXPECT().Attrs(gomock.Any()).Return(&gcpStorage.ObjectAttrs{ - Bucket: bucketName, - Name: objectName, - ContentType: "text/x-yaml", - Etag: "test-etag", - Size: 125, - Created: time.Now(), - }, nil).AnyTimes() - MockObjectHandle.EXPECT().NewRangeReader(gomock.Any(), 10, 125).Return(&gcpStorage.Reader{}, nil).AnyTimes() -}) + exists, err := gcpClient.BucketExists(context.Background(), bucketName) + assert.NilError(t, err) + assert.Assert(t, exists) +} -var _ = Describe("GCP Storage Provider", func() { - Describe("Get GCP Storage Provider client from gcp", func() { +func TestObjectAttributes(t *testing.T) { + gcpClient := &gcp.GCPClient{ + Client: Client, + StartRange: 0, + EndRange: -1, + } + exists, objectAttrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) + if err == gcpStorage.ErrObjectNotExist { + assert.NilError(t, err) + } + assert.NilError(t, err) + assert.Assert(t, exists) + assert.Assert(t, objectAttrs != nil) +} - Context("Gcp storage Bucket - BucketExists", func() { - It("should not return an error when fetching gcp storage bucket", func() { - gcpClient := &gcp.GCPClient{ - Client: MockClient, - StartRange: 0, - EndRange: -1, - } - exists, err := gcpClient.BucketExists(context.Background(), bucketName) - Expect(err).ToNot(HaveOccurred()) - Expect(exists).To(BeTrue()) - }) - }) - Context("Gcp storage Bucket - FGetObject", func() { - It("should get the object from the bucket and download the object locally", func() { - gcpClient := &gcp.GCPClient{ - Client: MockClient, - StartRange: 0, - EndRange: -1, - } - err := gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath) - Expect(err).ToNot(HaveOccurred()) - }) - }) - Context("Gcp storage Bucket - ObjectAttributes", func() { - It("should get the object attributes", func() { - gcpClient := &gcp.GCPClient{ - Client: MockClient, - StartRange: 0, - EndRange: -1, - } - exists, attrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) - Expect(err).ToNot(HaveOccurred()) - Expect(exists).To(BeTrue()) - Expect(attrs).ToNot(BeNil()) - }) +func TestListObjects(t *testing.T) { + gcpClient := &gcp.GCPClient{ + Client: Client, + StartRange: 0, + EndRange: -1, + } + objectInterator := gcpClient.ListObjects(context.Background(), bucketName, nil) + for { + _, err := objectInterator.Next() + if err == gcp.IteratorDone { + break + } + assert.NilError(t, err) + } + assert.Assert(t, objectInterator != nil) +} - Context("Gcp storage Bucket - SetRange", func() { - It("should set the range of the io reader seeker for the file download", func() { - gcpClient := &gcp.GCPClient{ - Client: MockClient, - StartRange: 0, - EndRange: -1, - } - gcpClient.SetRange(2, 5) - Expect(gcpClient.StartRange).To(Equal(int64(2))) - Expect(gcpClient.EndRange).To(Equal(int64(5))) - }) - }) - }) - }) -}) +func TestFGetObject(t *testing.T) { + tempDir, err := os.MkdirTemp("", bucketName) + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + gcpClient := &gcp.GCPClient{ + Client: Client, + StartRange: 0, + EndRange: -1, + } + localPath := filepath.Join(tempDir, objectName) + err = gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath) + if err != io.EOF { + assert.NilError(t, err) + } +} + +func TestSetRange(t *testing.T) { + gcpClient := &gcp.GCPClient{ + Client: Client, + StartRange: 0, + EndRange: -1, + } + gcpClient.SetRange(2, 5) + assert.Equal(t, gcpClient.StartRange, int64(2)) + assert.Equal(t, gcpClient.EndRange, int64(5)) +} + +func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*http.Client, func()) { + ts := httptest.NewTLSServer(http.HandlerFunc(handler)) + tlsConf := &tls.Config{InsecureSkipVerify: true} + tr := &http.Transport{ + TLSClientConfig: tlsConf, + DialTLS: func(netw, addr string) (net.Conn, error) { + return tls.Dial("tcp", ts.Listener.Addr().String(), tlsConf) + }, + } + return &http.Client{Transport: tr}, func() { + tr.CloseIdleConnections() + ts.Close() + } +} + +func getObject() *raw.Object { + customTime := time.Now() + retTime := customTime.Add(3 * time.Hour) + return &raw.Object{ + Bucket: bucketName, + Name: objectName, + EventBasedHold: false, + TemporaryHold: false, + RetentionExpirationTime: retTime.Format(time.RFC3339), + ContentType: "text/x-yaml", + ContentLanguage: "en-us", + Size: 1 << 20, + CustomTime: customTime.Format(time.RFC3339), + Md5Hash: "bFbHCDvedeecefdgmfmhfuRxBdcedGe96S82XJOAXxjJpk=", + } +} + +func getBucket() *raw.Bucket { + labels := map[string]string{"a": "b"} + matchClasses := []string{"STANDARD"} + aTime := time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC) + rb := &raw.Bucket{ + Name: bucketName, + Location: "loc", + DefaultEventBasedHold: true, + Metageneration: 3, + StorageClass: "sc", + TimeCreated: "2021-5-23T04:05:06Z", + Versioning: &raw.BucketVersioning{Enabled: true}, + Labels: labels, + Billing: &raw.BucketBilling{RequesterPays: true}, + Etag: "BNaB2y5Xr3&5MHDca4SoTNL79lyhahr7MV87ubwjgdtg6ghs", + Lifecycle: &raw.BucketLifecycle{ + Rule: []*raw.BucketLifecycleRule{{ + Action: &raw.BucketLifecycleRuleAction{ + Type: "SetStorageClass", + StorageClass: "NEARLINE", + }, + Condition: &raw.BucketLifecycleRuleCondition{ + Age: 10, + IsLive: googleapi.Bool(true), + CreatedBefore: "2021-01-02", + MatchesStorageClass: matchClasses, + NumNewerVersions: 3, + }, + }}, + }, + RetentionPolicy: &raw.BucketRetentionPolicy{ + RetentionPeriod: 3, + EffectiveTime: aTime.Format(time.RFC3339), + }, + IamConfiguration: &raw.BucketIamConfiguration{ + BucketPolicyOnly: &raw.BucketIamConfigurationBucketPolicyOnly{ + Enabled: true, + LockedTime: aTime.Format(time.RFC3339), + }, + UniformBucketLevelAccess: &raw.BucketIamConfigurationUniformBucketLevelAccess{ + Enabled: true, + LockedTime: aTime.Format(time.RFC3339), + }, + }, + Cors: []*raw.BucketCors{ + { + MaxAgeSeconds: 3600, + Method: []string{"GET", "POST"}, + Origin: []string{"*"}, + ResponseHeader: []string{"FOO"}, + }, + }, + Acl: []*raw.BucketAccessControl{ + {Bucket: bucketName, Role: "READER", Email: "test@example.com", Entity: "allUsers"}, + }, + LocationType: "dual-region", + Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"}, + Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"}, + Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"}, + } + return rb +} + +func getObjectFile() string { + return ` + apiVersion: source.toolkit.fluxcd.io/v1beta1 + kind: Bucket + metadata: + name: podinfo + namespace: default + spec: + interval: 5m + provider: aws + bucketName: podinfo + endpoint: s3.amazonaws.com + region: us-east-1 + timeout: 30s + ` +} diff --git a/pkg/gcp/mocks/mock_gcp_storage.go b/pkg/gcp/mocks/mock_gcp_storage.go deleted file mode 100644 index 25b5e9c1..00000000 --- a/pkg/gcp/mocks/mock_gcp_storage.go +++ /dev/null @@ -1,183 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: gcp.go - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - storage "cloud.google.com/go/storage" - gomock "github.com/golang/mock/gomock" -) - -// MockClient is a mock of Client interface. -type MockClient struct { - ctrl *gomock.Controller - recorder *MockClientMockRecorder -} - -// MockClientMockRecorder is the mock recorder for MockClient. -type MockClientMockRecorder struct { - mock *MockClient -} - -// NewMockClient creates a new mock instance. -func NewMockClient(ctrl *gomock.Controller) *MockClient { - mock := &MockClient{ctrl: ctrl} - mock.recorder = &MockClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockClient) EXPECT() *MockClientMockRecorder { - return m.recorder -} - -// Bucket mocks base method. -func (m *MockClient) Bucket(arg0 string) *storage.BucketHandle { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Bucket", arg0) - ret0, _ := ret[0].(*storage.BucketHandle) - return ret0 -} - -// Bucket indicates an expected call of Bucket. -func (mr *MockClientMockRecorder) Bucket(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockClient)(nil).Bucket), arg0) -} - -// Close mocks base method. -func (m *MockClient) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockClientMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClient)(nil).Close)) -} - -// MockBucketHandle is a mock of BucketHandle interface. -type MockBucketHandle struct { - ctrl *gomock.Controller - recorder *MockBucketHandleMockRecorder -} - -// MockBucketHandleMockRecorder is the mock recorder for MockBucketHandle. -type MockBucketHandleMockRecorder struct { - mock *MockBucketHandle -} - -// NewMockBucketHandle creates a new mock instance. -func NewMockBucketHandle(ctrl *gomock.Controller) *MockBucketHandle { - mock := &MockBucketHandle{ctrl: ctrl} - mock.recorder = &MockBucketHandleMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockBucketHandle) EXPECT() *MockBucketHandleMockRecorder { - return m.recorder -} - -// Attrs mocks base method. -func (m *MockBucketHandle) Attrs(arg0 context.Context) (*storage.BucketAttrs, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Attrs", arg0) - ret0, _ := ret[0].(*storage.BucketAttrs) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Attrs indicates an expected call of Attrs. -func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0) -} - -// Object mocks base method. -func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Object", arg0) - ret0, _ := ret[0].(*storage.ObjectHandle) - return ret0 -} - -// Object indicates an expected call of Object. -func (mr *MockBucketHandleMockRecorder) Object(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Object", reflect.TypeOf((*MockBucketHandle)(nil).Object), arg0) -} - -// Objects mocks base method. -func (m *MockBucketHandle) Objects(arg0 context.Context, arg1 *storage.Query) *storage.ObjectIterator { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Objects", arg0, arg1) - ret0, _ := ret[0].(*storage.ObjectIterator) - return ret0 -} - -// Objects indicates an expected call of Objects. -func (mr *MockBucketHandleMockRecorder) Objects(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Objects", reflect.TypeOf((*MockBucketHandle)(nil).Objects), arg0, arg1) -} - -// MockObjectHandle is a mock of ObjectHandle interface. -type MockObjectHandle struct { - ctrl *gomock.Controller - recorder *MockObjectHandleMockRecorder -} - -// MockObjectHandleMockRecorder is the mock recorder for MockObjectHandle. -type MockObjectHandleMockRecorder struct { - mock *MockObjectHandle -} - -// NewMockObjectHandle creates a new mock instance. -func NewMockObjectHandle(ctrl *gomock.Controller) *MockObjectHandle { - mock := &MockObjectHandle{ctrl: ctrl} - mock.recorder = &MockObjectHandleMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockObjectHandle) EXPECT() *MockObjectHandleMockRecorder { - return m.recorder -} - -// Attrs mocks base method. -func (m *MockObjectHandle) Attrs(arg0 context.Context) (*storage.ObjectAttrs, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Attrs", arg0) - ret0, _ := ret[0].(*storage.ObjectAttrs) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Attrs indicates an expected call of Attrs. -func (mr *MockObjectHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockObjectHandle)(nil).Attrs), arg0) -} - -// NewRangeReader mocks base method. -func (m *MockObjectHandle) NewRangeReader(arg0 context.Context, arg1, arg2 int64) (*storage.Reader, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NewRangeReader", arg0, arg1, arg2) - ret0, _ := ret[0].(*storage.Reader) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// NewRangeReader indicates an expected call of NewRangeReader. -func (mr *MockObjectHandleMockRecorder) NewRangeReader(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewRangeReader", reflect.TypeOf((*MockObjectHandle)(nil).NewRangeReader), arg0, arg1, arg2) -} From 6ff5970fe1a7e473ea282a443387391aa8038f36 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 16 Sep 2021 09:48:33 -0500 Subject: [PATCH 08/53] Added more tests and cleaned up GCP provider logic Signed-off-by: pa250194 --- controllers/bucket_controller.go | 6 +- go.mod | 3 +- go.sum | 3 - pkg/gcp/gcp.go | 90 +------------------- pkg/gcp/gcp_test.go | 140 ++++++++++++++++++++++++++----- 5 files changed, 126 insertions(+), 116 deletions(-) diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 9e4eee73..53451954 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -29,6 +29,7 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/s3utils" + "google.golang.org/api/option" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -417,13 +418,12 @@ func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) if err := gcp.ValidateSecret(secret.Data, secret.Name); err != nil { return nil, err } - serviceAccount := gcp.InitCredentialsWithSecret(secret.Data) - client, err = gcp.NewClientWithSAKey(ctx, serviceAccount) + client, err = gcp.NewClient(ctx, option.WithCredentialsJSON(secret.Data["serviceaccount"])) if err != nil { return nil, err } } else { - client, err = gcp.NewClient(ctx) + client, err = gcp.NewClient(ctx, nil) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 1d60520c..3ec4d3e8 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/libgit2/git2go/v31 v31.4.14 github.com/minio/minio-go/v7 v7.0.10 @@ -36,7 +35,7 @@ require ( golang.org/x/sync v0.0.0-20210220032951-036812b2e83c golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect golang.org/x/text v0.3.7 // indirect - google.golang.org/api v0.54.0 // indirect + google.golang.org/api v0.54.0 google.golang.org/genproto v0.0.0-20210830153122-0bac4d21c8ea // indirect gotest.tools v2.2.0+incompatible helm.sh/helm/v3 v3.6.3 diff --git a/go.sum b/go.sum index be1c5759..a1a0d5dc 100644 --- a/go.sum +++ b/go.sum @@ -42,7 +42,6 @@ cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiy cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= -cloud.google.com/go/storage v1.10.0 h1:STgFzyU5/8miMl0//zKh2aQeTyeaUH3WN9bSUiJ09bA= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.16.0 h1:1UwAux2OZP4310YXg5ohqBEpV16Y93uZG4+qOX7K2Kg= cloud.google.com/go/storage v1.16.0/go.mod h1:ieKBmUyzcftN5tbxwnXClMKH00CfcQ+xL6NN0r5QfmE= @@ -415,7 +414,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= @@ -485,7 +483,6 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= -github.com/googleapis/gax-go/v2 v2.0.5 h1:sjZBwGj9Jlw33ImPtvFviGYvseOtDM7hkSKB7+Tv3SM= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso= github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index ec56384a..b2274e34 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -18,7 +18,6 @@ package gcp import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -30,13 +29,6 @@ import ( "google.golang.org/api/option" ) -const ( - ServiceAccount = "service_account" - AuthUri = "https://accounts.google.com/o/oauth2/auth" - TokenUri = "https://oauth2.googleapis.com/token" - AuthProviderX509CertUrl = "https://www.googleapis.com/oauth2/v1/certs" -) - var ( // IteratorDone is returned when the looping of objects/content // has reached the end of the iteration. @@ -61,27 +53,12 @@ type GCPClient struct { EndRange int64 } -// CredentialsFile struct representing the GCP Service Account -// JSON file. -type CredentialsFile struct { - Type string `json:"type"` - ProjectID string `json:"project_id"` - PrivateKeyID string `json:"private_key_id"` - PrivateKey string `json:"private_key"` - ClientEmail string `json:"client_email"` - ClientID string `json:"client_id"` - AuthUri string `json:"auth_uri"` - TokenUri string `json:"token_uri"` - AuthProviderX509CertUrl string `json:"auth_provider_x509_cert_url"` - ClientX509CertUrl string `json:"client_x509_cert_url"` -} - // NewClient creates a new GCP storage client // The Google Storage Client will automatically // look for the Google Application Credential environment variable // or look for the Google Application Credential file. -func NewClient(ctx context.Context) (*GCPClient, error) { - client, err := gcpStorage.NewClient(ctx) +func NewClient(ctx context.Context, opts ...option.ClientOption) (*GCPClient, error) { + client, err := gcpStorage.NewClient(ctx, opts...) if err != nil { return nil, err } @@ -89,70 +66,11 @@ func NewClient(ctx context.Context) (*GCPClient, error) { return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil } -// NewClientWithSAKey creates a new GCP storage client -// It uses the provided JSON file with service account details -// To authenticate. -func NewClientWithSAKey(ctx context.Context, credentials *CredentialsFile) (*GCPClient, error) { - saAccount, err := credentials.credentailsToJSON() - if err != nil { - return nil, err - } - - client, err := gcpStorage.NewClient(ctx, option.WithCredentialsJSON(saAccount)) - if err != nil { - return nil, err - } - - return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil -} - -// credentailsToJSON converts GCP service account credentials struct to JSON. -func (credentials *CredentialsFile) credentailsToJSON() ([]byte, error) { - credentialsJSON, err := json.Marshal(credentials) - if err != nil { - return nil, err - } - - return credentialsJSON, nil -} - -// InitCredentialsWithSecret creates a new credential -// by initializing a new CredentialsFile struct -func InitCredentialsWithSecret(secret map[string][]byte) *CredentialsFile { - return &CredentialsFile{ - Type: ServiceAccount, - ProjectID: string(secret["projectid"]), - PrivateKeyID: string(secret["privatekeyid"]), - PrivateKey: string(secret["privatekey"]), - ClientEmail: string(secret["clientemail"]), - ClientID: string(secret["clientid"]), - AuthUri: AuthUri, - TokenUri: TokenUri, - AuthProviderX509CertUrl: AuthProviderX509CertUrl, - ClientX509CertUrl: string(secret["certurl"]), - } -} - // ValidateSecret validates the credential secrets // It ensures that needed secret fields are not missing. func ValidateSecret(secret map[string][]byte, name string) error { - if _, exists := secret["projectid"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'projectid'", name) - } - if _, exists := secret["privatekeyid"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'privatekeyid'", name) - } - if _, exists := secret["privatekey"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'privatekey'", name) - } - if _, exists := secret["clientemail"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) - } - if _, exists := secret["clientid"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'clientid'", name) - } - if _, exists := secret["certurl"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'certurl'", name) + if _, exists := secret["serviceaccount"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'serviceaccount'", name) } return nil diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 64db105f..5c0292e9 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -47,54 +47,61 @@ const ( ) var ( - Client *gcpStorage.Client + hc *http.Client + client *gcpStorage.Client + close func() err error ) func TestMain(m *testing.M) { - hc, close := newTestServer(func(w http.ResponseWriter, r *http.Request) { + hc, close = newTestServer(func(w http.ResponseWriter, r *http.Request) { io.Copy(ioutil.Discard, r.Body) - w.WriteHeader(200) if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s?alt=json&prettyPrint=false&projection=full", bucketName) { + w.WriteHeader(200) response := getBucket() - jsonedResp, err := json.Marshal(response) + jsonResponse, err := json.Marshal(response) if err != nil { - log.Fatalf("error marshalling resp %v\n", err) + log.Fatalf("error marshalling response %v\n", err) } - _, err = w.Write(jsonedResp) + _, err = w.Write(jsonResponse) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing jsonResponse %v\n", err) } } else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) { + w.WriteHeader(200) response := getObject() - jsonedResp, err := json.Marshal(response) + jsonResponse, err := json.Marshal(response) if err != nil { - log.Fatalf("error marshalling resp %v\n", err) + log.Fatalf("error marshalling response %v\n", err) } - _, err = w.Write(jsonedResp) + _, err = w.Write(jsonResponse) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing jsonResponse %v\n", err) } } else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o?alt=json&delimiter=&endOffset=&pageToken=&prefix=&prettyPrint=false&projection=full&startOffset=&versions=false", bucketName) { + w.WriteHeader(200) response := getObject() - jsonedResp, err := json.Marshal(response) + jsonResponse, err := json.Marshal(response) if err != nil { - log.Fatalf("error marshalling resp %v\n", err) + log.Fatalf("error marshalling response %v\n", err) } - _, err = w.Write(jsonedResp) + _, err = w.Write(jsonResponse) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing jsonResponse %v\n", err) } } else if r.RequestURI == fmt.Sprintf("/%s/test.yaml", bucketName) || r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) { + w.WriteHeader(200) response := getObjectFile() _, err = w.Write([]byte(response)) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing response %v\n", err) } + } else { + w.WriteHeader(404) } }) ctx := context.Background() - Client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) + client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { log.Fatal(err) } @@ -103,9 +110,15 @@ func TestMain(m *testing.M) { os.Exit(run) } +func TestNewClient(t *testing.T) { + gcpClient, err := gcp.NewClient(context.Background(), option.WithHTTPClient(hc)) + assert.NilError(t, err) + assert.Assert(t, gcpClient != nil) +} + func TestBucketExists(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -114,9 +127,21 @@ func TestBucketExists(t *testing.T) { assert.Assert(t, exists) } +func TestBucketNotExists(t *testing.T) { + bucket := "notexistsbucket" + gcpClient := &gcp.GCPClient{ + Client: client, + StartRange: 0, + EndRange: -1, + } + exists, err := gcpClient.BucketExists(context.Background(), bucket) + assert.NilError(t, err) + assert.Assert(t, !exists) +} + func TestObjectAttributes(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -131,7 +156,7 @@ func TestObjectAttributes(t *testing.T) { func TestListObjects(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -151,7 +176,7 @@ func TestFGetObject(t *testing.T) { assert.NilError(t, err) defer os.RemoveAll(tempDir) gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -162,9 +187,41 @@ func TestFGetObject(t *testing.T) { } } +func TestFGetObjectNotExists(t *testing.T) { + object := "notexists.txt" + tempDir, err := os.MkdirTemp("", bucketName) + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + gcpClient := &gcp.GCPClient{ + Client: client, + StartRange: 0, + EndRange: -1, + } + localPath := filepath.Join(tempDir, object) + err = gcpClient.FGetObject(context.Background(), bucketName, object, localPath) + if err != io.EOF { + assert.Error(t, err, "storage: object doesn't exist") + } +} + +func TestFGetObjectDirectoryIsFileName(t *testing.T) { + tempDir, err := os.MkdirTemp("", bucketName) + defer os.RemoveAll(tempDir) + assert.NilError(t, err) + gcpClient := &gcp.GCPClient{ + Client: client, + StartRange: 0, + EndRange: -1, + } + err = gcpClient.FGetObject(context.Background(), bucketName, objectName, tempDir) + if err != io.EOF { + assert.Error(t, err, "filename is a directory") + } +} + func TestSetRange(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -173,6 +230,45 @@ func TestSetRange(t *testing.T) { assert.Equal(t, gcpClient.EndRange, int64(5)) } +func TestValidateSecret(t *testing.T) { + t.Parallel() + testCases := []struct { + title string + secret map[string][]byte + name string + error bool + }{ + { + "Test Case 1", + map[string][]byte{ + "serviceaccount": []byte("serviceaccount"), + }, + "Service Account", + false, + }, + { + "Test Case 2", + map[string][]byte{ + "data": []byte("data"), + }, + "Service Account", + true, + }, + } + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.title, func(t *testing.T) { + t.Parallel() + err := gcp.ValidateSecret(testCase.secret, testCase.name) + if testCase.error { + assert.Error(t, err, fmt.Sprintf("invalid '%v' secret data: required fields 'serviceaccount'", testCase.name)) + } else { + assert.NilError(t, err) + } + }) + } +} + func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*http.Client, func()) { ts := httptest.NewTLSServer(http.HandlerFunc(handler)) tlsConf := &tls.Config{InsecureSkipVerify: true} From fa8c4ca0965f5ea2ec7a0bd691672eb6c775f737 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 16 Sep 2021 09:56:28 -0500 Subject: [PATCH 09/53] Fix nil pointer dereference Signed-off-by: pa250194 --- controllers/bucket_controller.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 53451954..aa11c261 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -423,7 +423,7 @@ func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) return nil, err } } else { - client, err = gcp.NewClient(ctx, nil) + client, err = gcp.NewClient(ctx) if err != nil { return nil, err } From a6be9c8a2139b441a41ddb71b82c1e52f9cd9ab8 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 16 Sep 2021 12:15:26 -0500 Subject: [PATCH 10/53] Updated docs to include GCP provider instructions Signed-off-by: pa250194 --- docs/spec/v1alpha1/buckets.md | 5 +-- docs/spec/v1beta1/buckets.md | 82 ++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/docs/spec/v1alpha1/buckets.md b/docs/spec/v1alpha1/buckets.md index 53838e25..bb2c07a9 100644 --- a/docs/spec/v1alpha1/buckets.md +++ b/docs/spec/v1alpha1/buckets.md @@ -11,7 +11,7 @@ Bucket: // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws;gcp + // +kubebuilder:validation:Enum=generic;aws // +optional Provider string `json:"provider,omitempty"` @@ -57,7 +57,6 @@ Supported providers: const ( GenericBucketProvider string = "generic" AmazonBucketProvider string = "aws" - GoogleBucketProvider string = "gcp" ) ``` @@ -232,4 +231,4 @@ Wait for ready condition: ```bash kubectl -n gitios-system wait bucket/podinfo --for=condition=ready --timeout=1m -``` +``` \ No newline at end of file diff --git a/docs/spec/v1beta1/buckets.md b/docs/spec/v1beta1/buckets.md index 1bcae604..90a23340 100644 --- a/docs/spec/v1beta1/buckets.md +++ b/docs/spec/v1beta1/buckets.md @@ -11,7 +11,7 @@ Bucket: // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws + // +kubebuilder:validation:Enum=generic;aws;gcp // +optional Provider string `json:"provider,omitempty"` @@ -62,6 +62,7 @@ Supported providers: const ( GenericBucketProvider string = "generic" AmazonBucketProvider string = "aws" + GoogleBucketProvider string = "gcp" ) ``` @@ -182,7 +183,8 @@ data: secretkey: ``` -> **Note:** that for Google Cloud Storage you have to enable +> **Note:** that when using the generic provider +> for Google Cloud Storage you have to enable > S3 compatible access in your GCP project. ### AWS IAM authentication @@ -230,6 +232,82 @@ spec: } ``` +### GCP Provider + +When the provider is `gcp` and the `secretRef` is not specified, +the GCP client authenticates using workload identity. +The GCP client automatically handles authentication in two ways. +The first way being that the GCP client library will automatically +check for the presence of the GOOGLE_APPLICATION_CREDENTIAL +environment variable. If this is not found, the GCP client library +will search for the Google Application Credential file in the config directory: + +```yaml +apiVersion: source.toolkit.fluccd.io/v1beta1 +kind: Bucket +metadata: + name: podinfo + namespace: gitops-system +spec: + interval: 5m + provider: gcp + bucketName: podinfo + endpoint: storage.googleapis.com + region: us-east-1 + timeout: 30s +``` + +When the provider is `gcp` and the `secretRef` is specified, +the GCP client authenticates using a Kubernetes secret named serviceaccount +which is a base 64 encoded string of the GCP service account JSON file: + +```yaml +apiVersion: source.toolkit.fluccd.io/v1beta1 +kind: Bucket +metadata: + name: podinfo + namespace: gitops-system +spec: + interval: 5m + provider: gcp + bucketName: podinfo + endpoint: storage.googleapis.com + region: us-east-1 + timeout: 30s + secretRef: + name: gcp-service-account +--- +apiVersion: v1 +kind: Secret +metadata: + name: gcp-service-account + namespace: gitops-system +type: Opaque +data: + serviceaccount: "ewogICAgInR5cGUiOiAic2VydmljZV9hY2NvdW50IiwKICAgICJwcm9qZWN0X2lkIjogInBvZGluZm8iLAogICAgInByaXZhdGVfa2V5X2lkIjogIjI4cXdnaDNnZGY1aGozZ2I1ZmozZ3N1NXlmZ2gzNGY0NTMyNDU2OGh5MiIsCiAgICAicHJpdmF0ZV9rZXkiOiAiLS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tXG5Id2V0aGd5MTIzaHVnZ2hoaGJkY3U2MzU2ZGd5amhzdmd2R0ZESFlnY2RqYnZjZGhic3g2M2Ncbjc2dGd5Y2ZlaHVoVkdURllmdzZ0N3lkZ3lWZ3lkaGV5aHVnZ3ljdWhland5NnQzNWZ0aHl1aGVndmNldGZcblRGVUhHVHlnZ2h1Ymh4ZTY1eWd0NnRneWVkZ3kzMjZodWN5dnN1aGJoY3Zjc2poY3NqaGNzdmdkdEhGQ0dpXG5IY3llNnR5eWczZ2Z5dWhjaGNzYmh5Z2NpamRiaHl5VEY2NnR1aGNldnVoZGNiaHVoaHZmdGN1aGJoM3VoN3Q2eVxuZ2d2ZnRVSGJoNnQ1cmZ0aGh1R1ZSdGZqaGJmY3JkNXI2N3l1aHV2Z0ZUWWpndnRmeWdoYmZjZHJoeWpoYmZjdGZkZnlodmZnXG50Z3ZnZ3RmeWdodmZ0NnR1Z3ZURjVyNjZ0dWpoZ3ZmcnR5aGhnZmN0Nnk3eXRmcjVjdHZnaGJoaHZ0Z2hoanZjdHRmeWNmXG5mZnhmZ2hqYnZnY2d5dDY3dWpiZ3ZjdGZ5aFZDN3VodmdjeWp2aGhqdnl1amNcbmNnZ2hndmdjZmhnZzc2NTQ1NHRjZnRoaGdmdHloaHZ2eXZ2ZmZnZnJ5eXU3N3JlcmVkc3dmdGhoZ2ZjZnR5Y2ZkcnR0ZmhmL1xuLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLVxuIiwKICAgICJjbGllbnRfZW1haWwiOiAidGVzdEBwb2RpbmZvLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAgICJjbGllbnRfaWQiOiAiMzI2NTc2MzQ2Nzg3NjI1MzY3NDYiLAogICAgImF1dGhfdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi9hdXRoIiwKICAgICJ0b2tlbl91cmkiOiAiaHR0cHM6Ly9vYXV0aDIuZ29vZ2xlYXBpcy5jb20vdG9rZW4iLAogICAgImF1dGhfcHJvdmlkZXJfeDUwOV9jZXJ0X3VybCI6ICJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9vYXV0aDIvdjEvY2VydHMiLAogICAgImNsaWVudF94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3JvYm90L3YxL21ldGFkYXRhL3g1MDkvdGVzdCU0MHBvZGluZm8uaWFtLmdzZXJ2aWNlYWNjb3VudC5jb20iCn0=" +``` + +> **Note:** the serviceaccount secret is a base 64 encoded form of +> the GCP service account json file like so + +```json + { + "type": "service_account", + "project_id": "podinfo", + "private_key_id": "28qwgh3gdf5hj3gb5fj3gsu5yfgh34f45324568hy2", + "private_key": "-----BEGIN PRIVATE KEY-----\nHwethgy123hugghhhbdcu6356dgyjhsvgvGFDHYgcdjbvcdhbsx63c\n76tgycfehuhVGTFYfw6t7ydgyVgydheyhuggycuhejwy6t35fthyuhegvcetf\nTFUHGTygghubhxe65ygt6tgyedgy326hucyvsuhbhcvcsjhcsjhcsvgdtHFCGi\nHcye6tyyg3gfyuhchcsbhygcijdbhyyTF66tuhcevuhdcbhuhhvftcuhbh3uh7t6y\nggvftUHbh6t5rfthhuGVRtfjhbfcrd5r67yuhuvgFTYjgvtfyghbfcdrhyjhbfctfdfyhvfg\ntgvggtfyghvft6tugvTF5r66tujhgvfrtyhhgfct6y7ytfr5ctvghbhhvtghhjvcttfycf\nffxfghjbvgcgyt67ujbgvctfyhVC7uhvgcyjvhhjvyujc\ncgghgvgcfhgg765454tcfthhgftyhhvvyvvffgfryyu77reredswfthhgfcftycfdrttfhf/\n-----END PRIVATE KEY-----\n", + "client_email": "test@podinfo.iam.gserviceaccount.com", + "client_id": "32657634678762536746", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%40podinfo.iam.gserviceaccount.com" + } +``` +> **Note:** that when using the gcp provider for +> Google Cloud Storage you do not have to enable +> S3 compatible access in your GCP project. + ## Status examples Successful download: From 0b971511ad07757538daceca7824bc7f8d3d5158 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 16 Sep 2021 12:28:23 -0500 Subject: [PATCH 11/53] Revert change to doc/api/source.md Signed-off-by: pa250194 --- docs/api/source.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/api/source.md b/docs/api/source.md index f2ac54a0..acce3cb5 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -1470,8 +1470,8 @@ Artifact includedArtifacts
- -[]*github.com/fluxcd/source-controller/api/v1beta1.Artifact + +[]*./api/v1beta1.Artifact @@ -2002,4 +2002,4 @@ string

Source interface must be supported by all API types.

This page was automatically generated with gen-crd-api-reference-docs

-
+ \ No newline at end of file From 057c65e939663cc8c0c0b9a40285b9356d9f3e6a Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 23 Sep 2021 12:38:38 -0500 Subject: [PATCH 12/53] Removed resumable downloads Signed-off-by: pa250194 --- pkg/gcp/gcp.go | 73 +++++++++------------------------------------ pkg/gcp/gcp_test.go | 44 ++++++--------------------- 2 files changed, 23 insertions(+), 94 deletions(-) diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index b2274e34..f332801f 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -45,12 +45,6 @@ type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. *gcpStorage.Client - // startRange is the starting read value for - // reading the object from bucket. - StartRange int64 - // endRange is the ending read value for - // reading the object from bucket. - EndRange int64 } // NewClient creates a new GCP storage client @@ -63,7 +57,7 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*GCPClient, er return nil, err } - return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil + return &GCPClient{Client: client}, nil } // ValidateSecret validates the credential secrets @@ -76,18 +70,11 @@ func ValidateSecret(secret map[string][]byte, name string) error { return nil } -// SetRange sets the startRange and endRange used to read the Object from -// the bucket. It is a helper method for resumable downloads. -func (c *GCPClient) SetRange(start, end int64) { - c.StartRange = start - c.EndRange = end -} - // BucketExists checks if the bucket with the provided name exists. func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, error) { _, err := c.Client.Bucket(bucketName).Attrs(ctx) if err == gcpStorage.ErrBucketNotExist { - return false, nil + return false, err } if err != nil { return false, err @@ -97,16 +84,16 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, // ObjectAttributes checks if the object with the provided name exists. // If it exists the Object attributes are returned. -func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) { - attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) +func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, error) { + _, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) // ErrObjectNotExist is returned if the object does not exist if err == gcpStorage.ErrObjectNotExist { - return false, nil, err + return false, err } if err != nil { - return false, nil, err + return false, err } - return true, attrs, nil + return true, nil } // FGetObject gets the object from the bucket and downloads the object locally @@ -140,7 +127,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca // ObjectExists verifies if object exists and you have permission to access. // Check if the object exists and if you have permission to access it // The Object attributes are returned if the Object exists. - exists, attrs, err := c.ObjectAttributes(ctx, bucketName, objectName) + exists, err := c.ObjectAttributes(ctx, bucketName, objectName) if err != nil { return err } @@ -148,57 +135,25 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca return ErrorObjectDoesNotExist } - // Write to a temporary file "filename.part.gcp" before saving. - filePartPath := localPath + ".part.gcp" - // If exists, open in append mode. If not create it as a part file. - filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + objectFile, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY, 0600) if err != nil { return err } - // If we return early with an error, be sure to close and delete - // filePart. If we have an error along the way there is a chance - // that filePart is somehow damaged, and we should discard it. - closeAndRemove := true - defer func() { - if closeAndRemove { - _ = filePart.Close() - _ = os.Remove(filePartPath) - } - }() - - // Issue Stat to get the current offset. - partFileStat, err := filePart.Stat() - if err != nil { - return err - } - - // Set the File size request range - // If the part file exists - if partFileStat.Size() > 0 { - c.SetRange(partFileStat.Size(), 0) - } - // Get Object from GCP Bucket - objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.StartRange, c.EndRange) + objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewReader(ctx) if err != nil { return err } defer objectReader.Close() - // Write to the part file. - if _, err := io.CopyN(filePart, objectReader, attrs.Size); err != nil { + // Write Object to file. + if _, err := io.Copy(objectFile, objectReader); err != nil { return err } - // Close the file before rename, this is specifically needed for Windows users. - closeAndRemove = false - if err := filePart.Close(); err != nil { - return err - } - - // Safely completed. Now commit by renaming to actual filename. - if err := os.Rename(filePartPath, localPath); err != nil { + // Close the file. + if err := objectFile.Close(); err != nil { return err } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 5c0292e9..76c455e5 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -118,9 +118,7 @@ func TestNewClient(t *testing.T) { func TestBucketExists(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } exists, err := gcpClient.BucketExists(context.Background(), bucketName) assert.NilError(t, err) @@ -130,35 +128,28 @@ func TestBucketExists(t *testing.T) { func TestBucketNotExists(t *testing.T) { bucket := "notexistsbucket" gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } exists, err := gcpClient.BucketExists(context.Background(), bucket) - assert.NilError(t, err) + assert.Error(t, err, "storage: bucket doesn't exist") assert.Assert(t, !exists) } func TestObjectAttributes(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } - exists, objectAttrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) + exists, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) if err == gcpStorage.ErrObjectNotExist { assert.NilError(t, err) } assert.NilError(t, err) assert.Assert(t, exists) - assert.Assert(t, objectAttrs != nil) } func TestListObjects(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } objectInterator := gcpClient.ListObjects(context.Background(), bucketName, nil) for { @@ -176,9 +167,7 @@ func TestFGetObject(t *testing.T) { assert.NilError(t, err) defer os.RemoveAll(tempDir) gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } localPath := filepath.Join(tempDir, objectName) err = gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath) @@ -193,9 +182,7 @@ func TestFGetObjectNotExists(t *testing.T) { assert.NilError(t, err) defer os.RemoveAll(tempDir) gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } localPath := filepath.Join(tempDir, object) err = gcpClient.FGetObject(context.Background(), bucketName, object, localPath) @@ -209,9 +196,7 @@ func TestFGetObjectDirectoryIsFileName(t *testing.T) { defer os.RemoveAll(tempDir) assert.NilError(t, err) gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } err = gcpClient.FGetObject(context.Background(), bucketName, objectName, tempDir) if err != io.EOF { @@ -219,17 +204,6 @@ func TestFGetObjectDirectoryIsFileName(t *testing.T) { } } -func TestSetRange(t *testing.T) { - gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, - } - gcpClient.SetRange(2, 5) - assert.Equal(t, gcpClient.StartRange, int64(2)) - assert.Equal(t, gcpClient.EndRange, int64(5)) -} - func TestValidateSecret(t *testing.T) { t.Parallel() testCases := []struct { From 38be5ed1111483d6c55afc660cc3b58385a71efe Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 23 Sep 2021 12:40:32 -0500 Subject: [PATCH 13/53] Cleanup obsolete comments Signed-off-by: pa250194 --- pkg/gcp/gcp.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index f332801f..c869419b 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -97,7 +97,6 @@ func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName } // FGetObject gets the object from the bucket and downloads the object locally -// A part file is created so the download can be resumable. func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, localPath string) error { // Verify if destination already exists. dirStatus, err := os.Stat(localPath) From 7c0d4c070ea407efc73979bb05343cf69395b9f5 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 23 Sep 2021 13:42:21 -0500 Subject: [PATCH 14/53] Refactor comments and method names Signed-off-by: pa250194 --- pkg/gcp/gcp.go | 10 ++++------ pkg/gcp/gcp_test.go | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index c869419b..470fe227 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -82,9 +82,8 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, return true, nil } -// ObjectAttributes checks if the object with the provided name exists. -// If it exists the Object attributes are returned. -func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, error) { +// ObjectExists checks if the object with the provided name exists. +func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, error) { _, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) // ErrObjectNotExist is returned if the object does not exist if err == gcpStorage.ErrObjectNotExist { @@ -124,9 +123,8 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca } // ObjectExists verifies if object exists and you have permission to access. - // Check if the object exists and if you have permission to access it - // The Object attributes are returned if the Object exists. - exists, err := c.ObjectAttributes(ctx, bucketName, objectName) + // Check if the object exists and if you have permission to access it. + exists, err := c.ObjectExists(ctx, bucketName, objectName) if err != nil { return err } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 76c455e5..8faa5e2c 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -131,15 +131,15 @@ func TestBucketNotExists(t *testing.T) { Client: client, } exists, err := gcpClient.BucketExists(context.Background(), bucket) - assert.Error(t, err, "storage: bucket doesn't exist") + assert.Error(t, err, gcpStorage.ErrBucketNotExist.Error()) assert.Assert(t, !exists) } -func TestObjectAttributes(t *testing.T) { +func TestObjectExists(t *testing.T) { gcpClient := &gcp.GCPClient{ Client: client, } - exists, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) + exists, err := gcpClient.ObjectExists(context.Background(), bucketName, objectName) if err == gcpStorage.ErrObjectNotExist { assert.NilError(t, err) } @@ -147,6 +147,16 @@ func TestObjectAttributes(t *testing.T) { assert.Assert(t, exists) } +func TestObjectNotExists(t *testing.T) { + object := "doesnotexists.yaml" + gcpClient := &gcp.GCPClient{ + Client: client, + } + exists, err := gcpClient.ObjectExists(context.Background(), bucketName, object) + assert.Error(t, err, gcpStorage.ErrObjectNotExist.Error()) + assert.Assert(t, !exists) +} + func TestListObjects(t *testing.T) { gcpClient := &gcp.GCPClient{ Client: client, From 911ecc64b8f8b9dd7f49b0070660d7eb798400d2 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Mon, 11 Oct 2021 08:29:49 -0500 Subject: [PATCH 15/53] Update go.sum Signed-off-by: pa250194 --- go.sum | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go.sum b/go.sum index 723df358..34c48c50 100644 --- a/go.sum +++ b/go.sum @@ -1073,7 +1073,9 @@ golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= @@ -1166,6 +1168,7 @@ golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 69fffa0d27cec60c59a06d4e7496a9a4f428cbaa Mon Sep 17 00:00:00 2001 From: pa250194 Date: Tue, 12 Oct 2021 11:46:48 -0500 Subject: [PATCH 16/53] Fixed spelling and capitalization Signed-off-by: pa250194 --- pkg/gcp/gcp.go | 22 ++++++++++------------ pkg/gcp/gcp_test.go | 18 +++++++++--------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 470fe227..38a0b99f 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -24,15 +24,15 @@ import ( "os" "path/filepath" - gcpStorage "cloud.google.com/go/storage" - interator "google.golang.org/api/iterator" + gcpstorage "cloud.google.com/go/storage" + "google.golang.org/api/iterator" "google.golang.org/api/option" ) var ( // IteratorDone is returned when the looping of objects/content // has reached the end of the iteration. - IteratorDone = interator.Done + IteratorDone = iterator.Done // ErrorDirectoryExists is an error returned when the filename provided // is a directory. ErrorDirectoryExists = errors.New("filename is a directory") @@ -44,15 +44,13 @@ var ( type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. - *gcpStorage.Client + *gcpstorage.Client } -// NewClient creates a new GCP storage client -// The Google Storage Client will automatically -// look for the Google Application Credential environment variable -// or look for the Google Application Credential file. +// NewClient creates a new GCP storage client. The Client will automatically look for the Google Application +// Credential environment variable or look for the Google Application Credential file. func NewClient(ctx context.Context, opts ...option.ClientOption) (*GCPClient, error) { - client, err := gcpStorage.NewClient(ctx, opts...) + client, err := gcpstorage.NewClient(ctx, opts...) if err != nil { return nil, err } @@ -73,7 +71,7 @@ func ValidateSecret(secret map[string][]byte, name string) error { // BucketExists checks if the bucket with the provided name exists. func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, error) { _, err := c.Client.Bucket(bucketName).Attrs(ctx) - if err == gcpStorage.ErrBucketNotExist { + if err == gcpstorage.ErrBucketNotExist { return false, err } if err != nil { @@ -86,7 +84,7 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, error) { _, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) // ErrObjectNotExist is returned if the object does not exist - if err == gcpStorage.ErrObjectNotExist { + if err == gcpstorage.ErrObjectNotExist { return false, err } if err != nil { @@ -160,7 +158,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca // ListObjects lists the objects/contents of the bucket whose bucket name is provided. // the objects are returned as an Objectiterator and .Next() has to be called on them // to loop through the Objects. -func (c *GCPClient) ListObjects(ctx context.Context, bucketName string, query *gcpStorage.Query) *gcpStorage.ObjectIterator { +func (c *GCPClient) ListObjects(ctx context.Context, bucketName string, query *gcpstorage.Query) *gcpstorage.ObjectIterator { items := c.Client.Bucket(bucketName).Objects(ctx, query) return items } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 8faa5e2c..99d72309 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -32,7 +32,7 @@ import ( "testing" "time" - gcpStorage "cloud.google.com/go/storage" + gcpstorage "cloud.google.com/go/storage" "github.com/fluxcd/source-controller/pkg/gcp" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" @@ -48,7 +48,7 @@ const ( var ( hc *http.Client - client *gcpStorage.Client + client *gcpstorage.Client close func() err error ) @@ -101,7 +101,7 @@ func TestMain(m *testing.M) { } }) ctx := context.Background() - client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) + client, err = gcpstorage.NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { log.Fatal(err) } @@ -131,7 +131,7 @@ func TestBucketNotExists(t *testing.T) { Client: client, } exists, err := gcpClient.BucketExists(context.Background(), bucket) - assert.Error(t, err, gcpStorage.ErrBucketNotExist.Error()) + assert.Error(t, err, gcpstorage.ErrBucketNotExist.Error()) assert.Assert(t, !exists) } @@ -140,7 +140,7 @@ func TestObjectExists(t *testing.T) { Client: client, } exists, err := gcpClient.ObjectExists(context.Background(), bucketName, objectName) - if err == gcpStorage.ErrObjectNotExist { + if err == gcpstorage.ErrObjectNotExist { assert.NilError(t, err) } assert.NilError(t, err) @@ -153,7 +153,7 @@ func TestObjectNotExists(t *testing.T) { Client: client, } exists, err := gcpClient.ObjectExists(context.Background(), bucketName, object) - assert.Error(t, err, gcpStorage.ErrObjectNotExist.Error()) + assert.Error(t, err, gcpstorage.ErrObjectNotExist.Error()) assert.Assert(t, !exists) } @@ -161,15 +161,15 @@ func TestListObjects(t *testing.T) { gcpClient := &gcp.GCPClient{ Client: client, } - objectInterator := gcpClient.ListObjects(context.Background(), bucketName, nil) + objectIterator := gcpClient.ListObjects(context.Background(), bucketName, nil) for { - _, err := objectInterator.Next() + _, err := objectIterator.Next() if err == gcp.IteratorDone { break } assert.NilError(t, err) } - assert.Assert(t, objectInterator != nil) + assert.Assert(t, objectIterator != nil) } func TestFGetObject(t *testing.T) { From 572eed74e54d61379ff97f3bb801d58a0f209162 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Wed, 1 Sep 2021 14:10:08 -0500 Subject: [PATCH 17/53] Add Support for GCP storage with workload identity Added Support for Google Cloud Storage with Workload Identity as Source Provider. This enables the use of GCP without enabling S3 compatible access. Signed-off-by: pa250194 --- api/v1beta1/bucket_types.go | 1 + controllers/bucket_controller.go | 233 ++++++++++++++++++++--------- docs/spec/v1alpha1/buckets.md | 1 + go.mod | 12 +- go.sum | 247 +++++++++++++++++++++++++++++-- pkg/gcp/gcp.go | 199 +++++++++++++++++++++++++ pkg/gcp/gcp_test.go | 62 ++++++++ 7 files changed, 672 insertions(+), 83 deletions(-) create mode 100644 pkg/gcp/gcp.go create mode 100644 pkg/gcp/gcp_test.go diff --git a/api/v1beta1/bucket_types.go b/api/v1beta1/bucket_types.go index 492002b8..1dc68851 100644 --- a/api/v1beta1/bucket_types.go +++ b/api/v1beta1/bucket_types.go @@ -79,6 +79,7 @@ type BucketSpec struct { const ( GenericBucketProvider string = "generic" AmazonBucketProvider string = "aws" + GoogleBucketProvider string = "gcp" ) // BucketStatus defines the observed state of a bucket diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index e1ca4641..3ec8d5e2 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -46,6 +46,7 @@ import ( "github.com/fluxcd/pkg/runtime/events" "github.com/fluxcd/pkg/runtime/metrics" "github.com/fluxcd/pkg/runtime/predicates" + "github.com/fluxcd/source-controller/pkg/gcp" sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/source-controller/pkg/sourceignore" @@ -176,77 +177,20 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, error) { - s3Client, err := r.auth(ctx, bucket) - if err != nil { - err = fmt.Errorf("auth error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err - } - - // create tmp dir - tempDir, err := os.MkdirTemp("", bucket.Name) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err - } - defer os.RemoveAll(tempDir) - - ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) - defer cancel() - - exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName) - if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err - } - if !exists { - err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err - } - - // Look for file with ignore rules first - // NB: S3 has flat filepath keys making it impossible to look - // for files in "subdirectories" without building up a tree first. - path := filepath.Join(tempDir, sourceignore.IgnoreFile) - if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil { - if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err - } - } - ps, err := sourceignore.ReadIgnoreFile(path, nil) - if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err - } - // In-spec patterns take precedence - if bucket.Spec.Ignore != nil { - ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*bucket.Spec.Ignore), nil)...) - } - matcher := sourceignore.NewMatcher(ps) - - // download bucket content - for object := range s3Client.ListObjects(ctxTimeout, bucket.Spec.BucketName, minio.ListObjectsOptions{ - Recursive: true, - UseV1: s3utils.IsGoogleEndpoint(*s3Client.EndpointURL()), - }) { - if object.Err != nil { - err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err - } - - if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile { - continue - } - - if matcher.Match(strings.Split(object.Key, "/"), false) { - continue - } - - localPath := filepath.Join(tempDir, object.Key) - err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{}) + var tempDir string + var err error + var sourceBucket sourcev1.Bucket + if bucket.Spec.Provider == sourcev1.GoogleBucketProvider { + sourceBucket, tempDir, err = r.reconcileWithGCP(ctx, bucket) if err != nil { - err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err + return sourceBucket, err + } + } else { + sourceBucket, tempDir, err = r.reconcileWithAWS(ctx, bucket) + if err != nil { + return sourceBucket, err } } - revision, err := r.checksum(tempDir) if err != nil { return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err @@ -315,6 +259,159 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1. return ctrl.Result{}, nil } +func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { + gcpClient, err := r.authGCP(ctx, bucket) + if err != nil { + err = fmt.Errorf("auth error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err + } + defer gcpClient.Client.Close() + // create tmp dir + tempDir, err := os.MkdirTemp("", bucket.Name) + if err != nil { + err = fmt.Errorf("tmp dir error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err + } + defer os.RemoveAll(tempDir) + + ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) + defer cancel() + + exists, err := gcpClient.BucketExists(ctxTimeout, bucket.Spec.BucketName) + if err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + if !exists { + err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + + // Look for file with ignore rules first. + path := filepath.Join(tempDir, sourceignore.IgnoreFile) + if err := gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path); err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + ps, err := sourceignore.ReadIgnoreFile(path, nil) + if err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + // In-spec patterns take precedence + if bucket.Spec.Ignore != nil { + ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*bucket.Spec.Ignore), nil)...) + } + matcher := sourceignore.NewMatcher(ps) + objects := gcpClient.ListObjects(ctxTimeout, bucket.Spec.BucketName, nil) + // download bucket content + for { + object, err := objects.Next() + if err == gcp.IteratorDone { + break + } + if err != nil { + err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, err) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + + if strings.HasSuffix(object.Name, "/") || object.Name == sourceignore.IgnoreFile { + continue + } + + if matcher.Match(strings.Split(object.Name, "/"), false) { + continue + } + + localPath := filepath.Join(tempDir, object.Name) + // FGetObject - get and download bucket object + if err = gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Name, localPath); err != nil { + err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + } + return sourcev1.Bucket{}, tempDir, nil +} + +func (r *BucketReconciler) reconcileWithAWS(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { + s3Client, err := r.auth(ctx, bucket) + if err != nil { + err = fmt.Errorf("auth error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err + } + + // create tmp dir + tempDir, err := os.MkdirTemp("", bucket.Name) + if err != nil { + err = fmt.Errorf("tmp dir error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err + } + defer os.RemoveAll(tempDir) + + ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) + defer cancel() + + exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName) + if err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + if !exists { + err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + + // Look for file with ignore rules first + // NB: S3 has flat filepath keys making it impossible to look + // for files in "subdirectories" without building up a tree first. + path := filepath.Join(tempDir, sourceignore.IgnoreFile) + if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil { + if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + } + ps, err := sourceignore.ReadIgnoreFile(path, nil) + if err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + // In-spec patterns take precedence + if bucket.Spec.Ignore != nil { + ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*bucket.Spec.Ignore), nil)...) + } + matcher := sourceignore.NewMatcher(ps) + + // download bucket content + for object := range s3Client.ListObjects(ctxTimeout, bucket.Spec.BucketName, minio.ListObjectsOptions{ + Recursive: true, + UseV1: s3utils.IsGoogleEndpoint(*s3Client.EndpointURL()), + }) { + if object.Err != nil { + err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + + if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile { + continue + } + + if matcher.Match(strings.Split(object.Key, "/"), false) { + continue + } + + localPath := filepath.Join(tempDir, object.Key) + err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{}) + if err != nil { + err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + } + } + return sourcev1.Bucket{}, tempDir, nil +} + +func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) (*gcp.GCPClient, error) { + client, err := gcp.NewClient(ctx) + if err != nil { + return nil, err + } + return client, nil +} + func (r *BucketReconciler) auth(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) { opt := minio.Options{ Region: bucket.Spec.Region, diff --git a/docs/spec/v1alpha1/buckets.md b/docs/spec/v1alpha1/buckets.md index 7addeccd..0ad60f41 100644 --- a/docs/spec/v1alpha1/buckets.md +++ b/docs/spec/v1alpha1/buckets.md @@ -57,6 +57,7 @@ Supported providers: const ( GenericBucketProvider string = "generic" AmazonBucketProvider string = "aws" + GoogleBucketProvider string = "gcp" ) ``` diff --git a/go.mod b/go.mod index 8dfca6d0..8fabe102 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,8 @@ go 1.16 replace github.com/fluxcd/source-controller/api => ./api require ( + cloud.google.com/go v0.93.3 // indirect + cloud.google.com/go/storage v1.16.0 github.com/Masterminds/semver/v3 v3.1.1 github.com/cyphar/filepath-securejoin v0.2.2 github.com/fluxcd/pkg/apis/meta v0.10.0 @@ -20,13 +22,21 @@ require ( github.com/go-git/go-billy/v5 v5.3.1 github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/libgit2/git2go/v31 v31.6.1 github.com/minio/minio-go/v7 v7.0.10 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.14.0 github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b - golang.org/x/sync v0.0.0-20201207232520-09787c993a3a + golang.org/x/net v0.0.0-20210825183410-e898025ed96a // indirect + golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f // indirect + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c + golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf // indirect + golang.org/x/text v0.3.7 // indirect + google.golang.org/api v0.54.0 + google.golang.org/genproto v0.0.0-20210830153122-0bac4d21c8ea // indirect gotest.tools v2.2.0+incompatible helm.sh/helm/v3 v3.6.3 k8s.io/api v0.21.3 diff --git a/go.sum b/go.sum index 22b70bde..34c48c50 100644 --- a/go.sum +++ b/go.sum @@ -9,20 +9,42 @@ cloud.google.com/go v0.46.3/go.mod h1:a6bKKbmY7er1mI7TEI4lsAkts/mkhTSZK8w33B4RAg cloud.google.com/go v0.50.0/go.mod h1:r9sluTvynVuxRIOHXQEHMFffphuXHOMZMycpNR5e6To= cloud.google.com/go v0.52.0/go.mod h1:pXajvRH/6o3+F9jDHZWQ5PbGhn+o8w9qiu/CffaVdO4= cloud.google.com/go v0.53.0/go.mod h1:fp/UouUEsRkN6ryDKNW/Upv/JBKnv6WDthjR6+vze6M= -cloud.google.com/go v0.54.0 h1:3ithwDMr7/3vpAMXiH+ZQnYbuIsh+OPhUPMFC9enmn0= cloud.google.com/go v0.54.0/go.mod h1:1rq2OEkV3YMf6n/9ZvGWI3GWw0VoqH/1x2nd8Is/bPc= +cloud.google.com/go v0.56.0/go.mod h1:jr7tqZxxKOVYizybht9+26Z/gUq7tiRzu+ACVAMbKVk= +cloud.google.com/go v0.57.0/go.mod h1:oXiQ6Rzq3RAkkY7N6t3TcE6jE+CIBBbA36lwQ1JyzZs= +cloud.google.com/go v0.62.0/go.mod h1:jmCYTdRCQuc1PHIIJ/maLInMho30T/Y0M4hTdTShOYc= +cloud.google.com/go v0.65.0/go.mod h1:O5N8zS7uWy9vkA9vayVHs65eM1ubvY4h553ofrNHObY= +cloud.google.com/go v0.72.0/go.mod h1:M+5Vjvlc2wnp6tjzE102Dw08nGShTscUx2nZMufOKPI= +cloud.google.com/go v0.74.0/go.mod h1:VV1xSbzvo+9QJOxLDaJfTjx5e+MePCpCWwvftOeQmWk= +cloud.google.com/go v0.78.0/go.mod h1:QjdrLG0uq+YwhjoVOLsS1t7TW8fs36kLs4XO5R5ECHg= +cloud.google.com/go v0.79.0/go.mod h1:3bzgcEeQlzbuEAYu4mrWhKqWjmpprinYgKJLgKHnbb8= +cloud.google.com/go v0.81.0/go.mod h1:mk/AM35KwGk/Nm2YSeZbxXdrNK3KZOYHmLkOqC2V6E0= +cloud.google.com/go v0.83.0/go.mod h1:Z7MJUsANfY0pYPdw0lbnivPx4/vhy/e2FEkSkF7vAVY= +cloud.google.com/go v0.84.0/go.mod h1:RazrYuxIK6Kb7YrzzhPoLmCVzl7Sup4NrbKPg8KHSUM= +cloud.google.com/go v0.87.0/go.mod h1:TpDYlFy7vuLzZMMZ+B6iRiELaY7z/gJPaqbMx6mlWcY= +cloud.google.com/go v0.90.0/go.mod h1:kRX0mNRHe0e2rC6oNakvwQqzyDmg57xJ+SZU1eT2aDQ= +cloud.google.com/go v0.93.3 h1:wPBktZFzYBcCZVARvwVKqH1uEj+aLXofJEtrb4oOsio= +cloud.google.com/go v0.93.3/go.mod h1:8utlLll2EF5XMAV15woO4lSbWQlk8rer9aLOfLh7+YI= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= cloud.google.com/go/bigquery v1.3.0/go.mod h1:PjpwJnslEMmckchkHFfq+HTD2DmtT67aNFKH1/VBDHE= cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvftPBK2Dvzc= +cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= +cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= +cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/firestore v1.1.0/go.mod h1:ulACoGHTpvq5r8rxGJ4ddJZBZqakUQqClKRT5SZwBmk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= cloud.google.com/go/pubsub v1.1.0/go.mod h1:EwwdRX2sKPjnvnqCa270oGRyludottCI76h+R3AArQw= cloud.google.com/go/pubsub v1.2.0/go.mod h1:jhfEVHT8odbXTkndysNHCcx0awwzvfOlguIAii9o8iA= +cloud.google.com/go/pubsub v1.3.1/go.mod h1:i+ucay31+CNRpDW4Lu78I4xXG+O1r/MAHgjpRVR+TSU= cloud.google.com/go/storage v1.0.0/go.mod h1:IhtSnM/ZTZV8YYJWCY8RULGVqBDmpoyjwiyrjsg+URw= cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0ZeosJ0Rtdos= cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= +cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= +cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +cloud.google.com/go/storage v1.16.0 h1:1UwAux2OZP4310YXg5ohqBEpV16Y93uZG4+qOX7K2Kg= +cloud.google.com/go/storage v1.16.0/go.mod h1:ieKBmUyzcftN5tbxwnXClMKH00CfcQ+xL6NN0r5QfmE= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= github.com/Azure/azure-sdk-for-go v16.2.1+incompatible/go.mod h1:9XXNKU+eRnpl9moKnB4QOLf1HestfXbmab5FXxiDBjc= github.com/Azure/go-ansiterm v0.0.0-20170929234023-d6e3b3328b78 h1:w+iIsaOQNcT7OZ575w+acHgRric5iCyQh+xv+KJ4HB8= @@ -86,6 +108,7 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= +github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= @@ -139,6 +162,10 @@ github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmE github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= +github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= +github.com/cncf/udpa/go v0.0.0-20200629203442-efcf912fb354/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= +github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59 h1:qWj4qVYZ95vLWwqyNJCQg7rDsG5wPdze0UaPolH7DUk= @@ -219,7 +246,13 @@ github.com/emicklei/go-restful v2.9.5+incompatible/go.mod h1:otzb+WCGbkyDHkqmQmT github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= +github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= +github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= +github.com/envoyproxy/go-control-plane v0.9.7/go.mod h1:cwu0lG7PUMfa9snN8LXBig5ynNVH9qI8YYLbd1fK2po= +github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210217033140-668b12f5399d/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk= +github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/evanphx/json-patch v0.5.2/go.mod h1:ZWS5hhDbVDyob71nXKNL0+PWn6ToqBHMikGIFbs31qQ= github.com/evanphx/json-patch v4.5.0+incompatible/go.mod h1:50XU6AFN0ol/bzJsmQLiYLvXMP4fmwYFNcr97nuDLSk= @@ -375,18 +408,24 @@ github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4er github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= -github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= github.com/golang/mock v1.4.0/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= +github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= +github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= github.com/golang/protobuf v1.3.4/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8= github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA= github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs= @@ -396,9 +435,11 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangplus/testing v0.0.0-20180327235837-af21d9c3145e/go.mod h1:0AA//k/eakGydO4jKRoRL2j92ZKSzTgj9tclaCrvXHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= @@ -407,20 +448,38 @@ github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5a github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.0 h1:Hsa8mG0dQ46ij8Sl2AYJDUv1oA9/d6Vk+3LG99Oe02g= github.com/google/gofuzz v1.1.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= +github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= +github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= +github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= +github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20200708004538-1a94d8640e99/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= @@ -430,6 +489,8 @@ github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.1.0 h1:6DWmvNpomjL1+3liNSZbVns3zsYzzCjm6pRBO1tLeso= +github.com/googleapis/gax-go/v2 v2.1.0/go.mod h1:Q3nei7sK6ybPYH7twZdmQpAd1MKb7pfu6SK+H1/DsU0= github.com/googleapis/gnostic v0.4.1/go.mod h1:LRhVm6pbyptWbWbuZ38d1eyptfvIytN3ir6b65WBswg= github.com/googleapis/gnostic v0.5.5 h1:9fHAtK0uDfpveeqqo1hkEZJcFvYXAiCN3UutL8F9xHw= github.com/googleapis/gnostic v0.5.5/go.mod h1:7+EbHbldMins07ALC74bsA81Ovc97DwqyJO1AENw9kA= @@ -454,6 +515,7 @@ github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.0/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/hashicorp/consul/api v1.1.0/go.mod h1:VmuI/Lkw1nC05EYQWNKwWGbkg+FbDBtguAZLlVdkD9Q= github.com/hashicorp/consul/api v1.3.0/go.mod h1:MmDNSzIMUjNpY/mQ398R4bk2FnqQLoPndWW5VkKPlCE= github.com/hashicorp/consul/sdk v0.1.1/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8= @@ -489,6 +551,7 @@ github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.11/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= @@ -748,6 +811,7 @@ github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= +github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= @@ -852,8 +916,11 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca h1:1CFlNzQhALwjS9mBAUkycX616GzgsuYUOCHA5+HSlXI= github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= +github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43 h1:+lm10QQTNSBd8DVTNGHx7o/IKu9HYDvLMffDhbyLccI= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50 h1:hlE8//ciYMztlGpl/VA+Zm1AcTPHYkHJPbHqE6WJUXE= @@ -875,8 +942,12 @@ go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= go.opencensus.io v0.22.0/go.mod h1:+kGneAE2xo2IficOXnaByMWTGM9T73dGwxeWcUqIpI8= go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= -go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= +go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.23.0 h1:gqCw0LfLxScz8irSi8exQc7fyQ0fKQU/qnC/X8+V/1M= +go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 h1:+FNtrFTmVw0YZGpBGX56XDee331t6JAXeK2bcyhLOOc= go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5/go.mod h1:nmDLcffg48OtT/PSW0Hg7FvpRQsQh5OSqIylirxKC7o= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= @@ -942,8 +1013,10 @@ golang.org/x/lint v0.0.0-20190909230951-414d861bb4ac/go.mod h1:6SW0HCj/g11FgYtHl golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= golang.org/x/lint v0.0.0-20191125180803-fdd1cda4f05f/go.mod h1:5qLYkcX4OjUUV8bRuDixDT3tpyyb+LUpUlRWLxfhWrs= golang.org/x/lint v0.0.0-20200130185559-910be7a94367/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= -golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20201208152925-83fdc39ff7b5/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616 h1:VLliZ0d+/avPrXXH+OakdXhpJuEoBZuwh1m2j7U6Iug= +golang.org/x/lint v0.0.0-20210508222113-6edffad5e616/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mobile v0.0.0-20190312151609-d3739f865fa6/go.mod h1:z+o9i4GpDbdi3rU15maQ/Ox0txvL9dWGYEHz965HBQE= golang.org/x/mobile v0.0.0-20190719004257-d2bd2a29d028/go.mod h1:E/iHnbuqvinMTCcRqshq8CkpyQDoeVncDDYHnLhea+o= golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= @@ -953,6 +1026,9 @@ golang.org/x/mod v0.1.1-0.20191107180719-034126e5016b/go.mod h1:QqPTAvyqsEbceGzB golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.3.1-0.20200828183125-ce943fd02449/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -974,6 +1050,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190619014844-b5b0513f8c1b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -985,32 +1062,58 @@ golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200501053045-e0ff5e5a1de5/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200506145744-7e3656a0809f/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200513185701-a91f0712d120/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= +golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20210316092652-d523dce5a7f4/go.mod h1:RBQZq4jEuRlivfhVLdyRGr576XBO4/greRjx4P4O3yc= golang.org/x/net v0.0.0-20210326060303-6b1517762897/go.mod h1:uSPa2vr4CLtc/ILN5odXGNXS6mhrKVzTaCXzk9m6W3k= -golang.org/x/net v0.0.0-20210428140749-89ef3d95e781 h1:DzZ89McO9/gWPsQXS/FVKAlG02ZjaQ6AlZRBimEYOd0= +golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20210428140749-89ef3d95e781/go.mod h1:OJAsFXCWl8Ukc7SiCT/9KSuxbyM7479/AVlXFRxuMCk= +golang.org/x/net v0.0.0-20210503060351-7fd8e65b6420/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a h1:bRuuGXV8wwSdGTB+CtJf+FjgO1APK1CoO39T4BN/XBw= +golang.org/x/net v0.0.0-20210825183410-e898025ed96a/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d h1:TzXSXBo42m9gQenoE3b9BGiEpg5IG2JkU5FkPIawgtw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210220000619-9bb904979d93/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210313182246-cd4f82c27b84/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210615190721-d04028783cf1/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210628180205-a41e5a781914/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210805134026-6f1e6394065a/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f h1:Qmd2pbz05z7z6lm0DrgQVVPuBm92jqujBKMHMOlOQEw= +golang.org/x/oauth2 v0.0.0-20210819190943-2bc19b11175f/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200317015054-43a5402ce75a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.0.0-20201207232520-09787c993a3a h1:DcqTD9SDLc+1P/r1EmRBwnVsrOwW+kk2vWf9n+1sGhs= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1056,26 +1159,44 @@ golang.org/x/sys v0.0.0-20200212091648-12a6c2dcc1e4/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200523222454-059865788121/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201204225414-ed752295db88/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210112080510-489259a85091/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210119212857-b64e53b001e4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210220050731-9a76102bfb43/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210225134936-a50acf3fe073/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210305230114-8fe3ee5dd75b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210315160823-c6e025ad8005/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210320140829-1e4c9ba3b0c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210324051608-47abb6519492/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210426230700-d19ff857e887/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210502180810-71e4cd670f79/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210514084401-e8d321eab015/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210603081109-ebe580a85c40/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c h1:F1jZWGFhYfh0Ci55sIpILtKKK8p3i2/krTr0H1rg74I= +golang.org/x/sys v0.0.0-20210603125802-9665404d3644/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210630005230-0f9fa26af87c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210806184541-e5e7981a1069/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf h1:2ucpDCmfkl8Bd/FsLtiD653Wf96cW37s+iGx93zsu4k= +golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d h1:SZxvLBoTP5yHO3Frd4z4vrF+DBX9vMVanchswa69toE= @@ -1086,8 +1207,10 @@ golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3 golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= -golang.org/x/text v0.3.6 h1:aRYxNxv6iGQlyVaZmk6ZgYEDa+Jg18DxebPSrd6bg1M= +golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7 h1:olpwvP2KacW1ZWvsR7uQhoyTYvKAupfQrRGBFM352Gk= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -1138,13 +1261,33 @@ golang.org/x/tools v0.0.0-20200204074204-1cc6d1ef6c74/go.mod h1:TB2adYChydJhpapK golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200212150539-ea181f53ac56/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200224181240-023911ca70b2/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200227222343-706bc42d1f0d/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200304193943-95d2e580d8eb/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200312045724-11d5b4c81c7d/go.mod h1:o4KQGtdN14AW+yjsvvwRTJJuXz8XRtIHtEnmAXLyFUw= +golang.org/x/tools v0.0.0-20200331025713-a30bf2db82d4/go.mod h1:Sl4aGygMT6LrqrWclx+PTx3U+LnKx/seiNR+3G19Ar8= +golang.org/x/tools v0.0.0-20200501065659-ab2804fb9c9d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200505023115-26f46d2f7ef8/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200512131952-2bc93b1c0c88/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200515010526-7d3b6ebf133d/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200618134242-20370b0cb4b2/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200619180055-7c47624df98f/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200804011535-6c149bb5ef0d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200825202427-b303f430e36d/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200904185747-39188db58858/go.mod h1:Cj7w3i3Rnn0Xh82ur9kSqwfTHTeVxaDqrfMjpcNT6bE= +golang.org/x/tools v0.0.0-20201110124207-079ba7bd75cd/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201201161351-ac6f37ff4c2a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20201208233053-a543418bbed2/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= -golang.org/x/tools v0.1.0 h1:po9/4sTYwZU9lPhi1tOrb4hCv3qrhiQ77LZfGa2OjwY= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= +golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.5 h1:ouewzE6p+/VEB31YYnTbEJdi8pFqKp4P4n85vwo3DHA= +golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -1163,13 +1306,32 @@ google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsb google.golang.org/api v0.15.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/api v0.17.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.18.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.19.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= google.golang.org/api v0.20.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.22.0/go.mod h1:BwFmGc8tA3vsd7r/7kR8DY7iEEGSU04BFxCo5jP/sfE= +google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE= +google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM= +google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc= +google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= +google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= +google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.41.0/go.mod h1:RkxM5lITDfTzmyKFPt+wGrCJbVfniCr2ool8kTBzRTU= +google.golang.org/api v0.43.0/go.mod h1:nQsDGjRXMo4lvh5hP0TKqF244gqhGcr/YSIykhUk/94= +google.golang.org/api v0.47.0/go.mod h1:Wbvgpq1HddcWVtzsVLyfLp8lDg6AA241LmgIL59tHXo= +google.golang.org/api v0.48.0/go.mod h1:71Pr1vy+TAZRPkPs/xlCf5SsU8WjuAWv1Pfjbtukyy4= +google.golang.org/api v0.49.0/go.mod h1:BECiH72wsfwUvOVn3+btPD5WHi0LzavZReBndi42L18= +google.golang.org/api v0.50.0/go.mod h1:4bNT5pAuq5ji4SRZm+5QIkjny9JAyVD/3gaSihNefaw= +google.golang.org/api v0.51.0/go.mod h1:t4HdrdoNgyN5cbEfm7Lum0lcLDLiise1F8qDKX00sOU= +google.golang.org/api v0.54.0 h1:ECJUVngj71QI6XEm7b1sAf8BljU5inEhMbKPR8Lxhhk= +google.golang.org/api v0.54.0/go.mod h1:7C4bFFOvVDGXjfDTAsgGwDgAxRDeQ4X8NvUedIt6z3k= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/cloud v0.0.0-20151119220103-975617b05ea8/go.mod h1:0H1ncTHf11KCFhTc/+EFRbzSCOZx+VUbRMk55Yv5MYk= @@ -1191,11 +1353,46 @@ google.golang.org/genproto v0.0.0-20200122232147-0452cf42e150/go.mod h1:n3cpQtvx google.golang.org/genproto v0.0.0-20200204135345-fa8e72b47b90/go.mod h1:GmwEX6Z4W5gMy59cAlVYjN9JhxgbQH6Gn+gFDQe2lzA= google.golang.org/genproto v0.0.0-20200212174721-66ed5ce911ce/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200224152610-e50cd9704f63/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200228133532-8c2c7df3a383/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200305110556-506484158171/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200312145019-da6875a35672/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200331122359-1ee6d9798940/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200430143042-b979b6f78d84/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200511104702-f5ebc3bea380/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= +google.golang.org/genproto v0.0.0-20200515170657-fc4c6c6a6587/go.mod h1:YsZOwe1myG/8QRHRsmBRE1LrgQY60beZKjly0O1fX9U= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= +google.golang.org/genproto v0.0.0-20200618031413-b414f8b61790/go.mod h1:jDfRM7FcilCzHH/e9qn6dsT145K34l5v+OpcnNgKAAA= +google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20200904004341-0bd0a958aa1d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= -google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a h1:pOwg4OoaRYScjmR4LlLgdtnyoHYTSAVhhqe5uPdpII8= +google.golang.org/genproto v0.0.0-20201109203340-2640f1f9cdfb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20201110150050-8816d57aaa9a/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201201144952-b05cb90ed32e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210222152913-aa3ee6e6a81c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= +google.golang.org/genproto v0.0.0-20210513213006-bf773b8c8384/go.mod h1:P3QM42oQyzQSnHPnZ/vqoCdDmzH28fzWByN9asMeM8A= +google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210604141403-392c879c8b08/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210608205507-b6d2f5bf0d7d/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20210617175327-b9e0b3197ced/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210624174822-c5cf32407d0a/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210624195500-8bfb893ecb84/go.mod h1:SzzZ/N+nwJDaO1kznhnlzqS8ocJICar6hYhVyhi++24= +google.golang.org/genproto v0.0.0-20210713002101-d411969a0d9a/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210716133855-ce7ef5c701ea/go.mod h1:AxrInvYm1dci+enl5hChSFPOmmUF1+uAa/UsgNRWd7k= +google.golang.org/genproto v0.0.0-20210728212813-7823e685a01f/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210805201207-89edb61ffb67/go.mod h1:ob2IJxKrgPT52GcgX759i1sleT07tiKowYBGbczaW48= +google.golang.org/genproto v0.0.0-20210813162853-db860fec028c/go.mod h1:cFeNkxwySK631ADgubI+/XFU/xp8FD5KIVV4rj8UC5w= +google.golang.org/genproto v0.0.0-20210821163610-241b8fcbd6c8/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= +google.golang.org/genproto v0.0.0-20210830153122-0bac4d21c8ea h1:5eMUso2GVOxypVH1fR4oKgDobrvi4DHctJ4fVk66s/4= +google.golang.org/genproto v0.0.0-20210830153122-0bac4d21c8ea/go.mod h1:eFjDcFEctNawg4eG61bRv87N7iHBWyVhJu7u1kqDUXY= google.golang.org/grpc v0.0.0-20160317175043-d3ddb4469d5a/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -1206,10 +1403,29 @@ google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ij google.golang.org/grpc v1.22.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= -google.golang.org/grpc v1.27.1 h1:zvIju4sqAGvwKspUQOhwnpcqSbzi7/H6QomNNjTL4sk= google.golang.org/grpc v1.27.1/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= +google.golang.org/grpc v1.28.0/go.mod h1:rpkK4SK4GF4Ach/+MFLZUBavHOvF2JJB5uozKKal+60= +google.golang.org/grpc v1.29.1/go.mod h1:itym6AZVZYACWQqET3MqgPpjcuV5QH3BxFS3IjizoKk= +google.golang.org/grpc v1.30.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.0/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM8pak= +google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= +google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= +google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= +google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.37.1/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= +google.golang.org/grpc v1.40.0 h1:AGJ0Ih4mHjSeibYkFGh1dD9KJ/eOtZ93I6hoHhukQ5Q= +google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= +google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -1221,8 +1437,9 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ= +google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20141024133853-64131543e789/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -1252,6 +1469,7 @@ gopkg.in/warnings.v0 v0.1.2/go.mod h1:jksf8JmL6Qr/oQM2OXTHunEvvTAsrWBLb6OOjuVWRN gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= @@ -1278,6 +1496,7 @@ honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWh honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= k8s.io/api v0.21.0/go.mod h1:+YbrhBBGgsxbF6o6Kj4KJPJnBmAKuXDeS3E18bgHNVU= k8s.io/api v0.21.1/go.mod h1:FstGROTmsSHBarKc8bylzXih8BLNYTiS3TZcsoEDg2s= k8s.io/api v0.21.3 h1:cblWILbLO8ar+Fj6xdDGr603HRsf8Wu9E9rngJeprZQ= diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go new file mode 100644 index 00000000..18ea53fe --- /dev/null +++ b/pkg/gcp/gcp.go @@ -0,0 +1,199 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcp + +import ( + "context" + "errors" + "io" + "os" + "path/filepath" + + gcpStorage "cloud.google.com/go/storage" + interator "google.golang.org/api/iterator" +) + +var ( + // IteratorDone is returned when the looping of objects/content + // has reached the end of the iteration. + IteratorDone = interator.Done + // DirectoryExists is an error returned when the filename provided + // is a directory. + DirectoryExists = errors.New("filename is a directory") + // ObjectDoesNotExist is an error returned when the object whose name + // is provided does not exist. + ObjectDoesNotExist = errors.New("object does not exist") +) + +type GCPClient struct { + // client for interacting with the Google Cloud + // Storage APIs. + Client *gcpStorage.Client + // startRange is the starting read value for + // reading the object from bucket. + startRange int64 + // endRange is the ending read value for + // reading the object from bucket. + endRange int64 +} + +// NewClient creates a new GCP storage client +// The Google Storage Client will automatically +// look for the Google Application Credential environment variable +// or look for the Google Application Credential file +func NewClient(ctx context.Context) (*GCPClient, error) { + client, err := gcpStorage.NewClient(ctx) + if err != nil { + return nil, err + } + return &GCPClient{Client: client, startRange: 0, endRange: -1}, nil +} + +// SetRange sets the startRange and endRange used to read the Object from +// the bucket. It is a helper method for resumable downloads. +func (c *GCPClient) SetRange(start, end int64) { + c.startRange = start + c.endRange = end +} + +// BucketExists checks if the bucket with the provided name exists. +func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, error) { + _, err := c.Client.Bucket(bucketName).Attrs(ctx) + if err == gcpStorage.ErrBucketNotExist { + return false, nil + } + if err != nil { + return false, err + } + return true, nil +} + +// ObjectExists checks if the object with the provided name exists. +// If it exists the Object attributes are returned. +func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) { + attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) + // ErrObjectNotExist is returned if the object does not exist + if err != nil { + return false, nil, err + } + return true, attrs, err +} + +// FGetObject gets the object from the bucket and downloads the object locally +// A part file is created so the download can be resumable. +func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, localPath string) error { + // Verify if destination already exists. + dirStatus, err := os.Stat(localPath) + if err == nil { + // If the destination exists and is a directory. + if dirStatus.IsDir() { + return DirectoryExists + } + } + + // Proceed if file does not exist. return for all other errors. + if err != nil { + if !os.IsNotExist(err) { + return err + } + } + + // Extract top level directory. + objectDir, _ := filepath.Split(localPath) + if objectDir != "" { + // Create any missing top level directories. + if err := os.MkdirAll(objectDir, 0700); err != nil { + return err + } + } + + // ObjectExists verifies if object exists and you have permission to access. + // Check if the object exists and if you have permission to access it + // The Object attributes are returned if the Object exists. + exists, attrs, err := c.ObjectExists(ctx, bucketName, objectName) + if err != nil { + return err + } + if !exists { + return ObjectDoesNotExist + } + + // Write to a temporary file "filename.part.gcp" before saving. + filePartPath := localPath + attrs.Etag + ".part.gcp" + + // If exists, open in append mode. If not create it as a part file. + filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + if err != nil { + return err + } + + // If we return early with an error, be sure to close and delete + // filePart. If we have an error along the way there is a chance + // that filePart is somehow damaged, and we should discard it. + closeAndRemove := true + defer func() { + if closeAndRemove { + _ = filePart.Close() + _ = os.Remove(filePartPath) + } + }() + + // Issue Stat to get the current offset. + partFileStat, err := filePart.Stat() + if err != nil { + return err + } + + // Set the File size request range + // If the part file exists + if partFileStat.Size() > 0 { + c.SetRange(partFileStat.Size(), 0) + } + + // Get Object from GCP Bucket + objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.startRange, c.endRange) + if err != nil { + return err + } + defer objectReader.Close() + + // Write to the part file. + if _, err = io.CopyN(filePart, objectReader, attrs.Size); err != nil { + return err + } + + // Close the file before rename, this is specifically needed for Windows users. + closeAndRemove = false + if err = filePart.Close(); err != nil { + return err + } + + // Safely completed. Now commit by renaming to actual filename. + if err = os.Rename(filePartPath, localPath); err != nil { + return err + } + + return nil +} + +// ListObjects lists the objects/contents of the bucket whose bucket name is provided. +// the objects are returned as an Objectiterator and .Next() has to be called on them +// to loop through the Objects. +func (c *GCPClient) ListObjects(ctx context.Context, bucketName string, query *gcpStorage.Query) *gcpStorage.ObjectIterator { + items := c.Client.Bucket(bucketName).Objects(ctx, query) + return items +} diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go new file mode 100644 index 00000000..30412f49 --- /dev/null +++ b/pkg/gcp/gcp_test.go @@ -0,0 +1,62 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package gcp + +import ( + "context" + "testing" + + "gotest.tools/assert" +) + +func TestSetRange(t *testing.T) { + client, err := NewClient(context.Background()) + assert.NilError(t, err) + testCases := []struct { + title string + start int64 + end int64 + }{ + { + title: "Test Case 1", + start: 1, + end: 5, + }, + { + title: "Test Case 2", + start: 3, + end: 6, + }, + { + title: "Test Case 3", + start: 4, + end: 5, + }, + { + title: "Test Case 4", + start: 2, + end: 7, + }, + } + for _, tt := range testCases { + t.Run(tt.title, func(t *testing.T) { + client.SetRange(tt.start, tt.end) + assert.Equal(t, tt.start, client.startRange) + assert.Equal(t, tt.end, client.endRange) + }) + } +} From a600528729258dd768c454e22e6cc89b68b16281 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Wed, 1 Sep 2021 14:41:40 -0500 Subject: [PATCH 18/53] Added Comments for reconcileWithGCP and reconcileWithMinio Signed-off-by: pa250194 --- controllers/bucket_controller.go | 16 ++++++++++++---- docs/api/source.md | 2 +- 2 files changed, 13 insertions(+), 5 deletions(-) diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 3ec8d5e2..3ff17f22 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -186,7 +186,7 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket return sourceBucket, err } } else { - sourceBucket, tempDir, err = r.reconcileWithAWS(ctx, bucket) + sourceBucket, tempDir, err = r.reconcileWithMinio(ctx, bucket) if err != nil { return sourceBucket, err } @@ -259,6 +259,8 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1. return ctrl.Result{}, nil } +// reconcileWithGCP handles getting objects from a Google Cloud Platform bucket +// using a gcp client func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { gcpClient, err := r.authGCP(ctx, bucket) if err != nil { @@ -330,8 +332,10 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1 return sourcev1.Bucket{}, tempDir, nil } -func (r *BucketReconciler) reconcileWithAWS(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { - s3Client, err := r.auth(ctx, bucket) +// reconcileWithMinio handles getting objects from an S3 compatible bucket +// using a minio client +func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { + s3Client, err := r.authMinio(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err @@ -404,6 +408,8 @@ func (r *BucketReconciler) reconcileWithAWS(ctx context.Context, bucket sourcev1 return sourcev1.Bucket{}, tempDir, nil } +// authGCP creates a new Google Cloud Platform storage client +// to interact with the Storage service. func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) (*gcp.GCPClient, error) { client, err := gcp.NewClient(ctx) if err != nil { @@ -412,7 +418,9 @@ func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) return client, nil } -func (r *BucketReconciler) auth(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) { +// authMinio creates a new Minio client to interact with S3 +// compatible storage services. +func (r *BucketReconciler) authMinio(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) { opt := minio.Options{ Region: bucket.Spec.Region, Secure: !bucket.Spec.Insecure, diff --git a/docs/api/source.md b/docs/api/source.md index ca22d432..78aee678 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -2032,4 +2032,4 @@ string

Source interface must be supported by all API types.

This page was automatically generated with gen-crd-api-reference-docs

-
+ \ No newline at end of file From 2cc48fefb17cc27c6b81a10a86a667ffe72c8020 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 2 Sep 2021 08:51:02 -0500 Subject: [PATCH 19/53] Added initial testing for new GCP provider Signed-off-by: pa250194 --- pkg/gcp/gcp_test.go | 79 +++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 79 insertions(+) diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 30412f49..459e691a 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -18,12 +18,24 @@ package gcp import ( "context" + "os" + "path/filepath" "testing" "gotest.tools/assert" ) +func TestNewClient(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() + client, err := NewClient(context.Background()) + assert.NilError(t, err) + assert.Assert(t, client.Client != nil) +} + func TestSetRange(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() client, err := NewClient(context.Background()) assert.NilError(t, err) testCases := []struct { @@ -60,3 +72,70 @@ func TestSetRange(t *testing.T) { }) } } + +func TestBucketExists(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() + ctx := context.Background() + bucketName := "" + client, err := NewClient(ctx) + assert.NilError(t, err) + exists, err := client.BucketExists(ctx, bucketName) + assert.NilError(t, err) + assert.Assert(t, exists) +} + +func TestObjectExists(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() + ctx := context.Background() + // bucketName is the name of the bucket which contains the object + bucketName := "" + // objectName is the path to the object within the bucket + objectName := "" + client, err := NewClient(ctx) + assert.NilError(t, err) + exists, attrs, err := client.ObjectExists(ctx, bucketName, objectName) + assert.NilError(t, err) + assert.Assert(t, exists) + assert.Assert(t, attrs != nil) +} + +func TestListObjects(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() + ctx := context.Background() + // bucketName is the name of the bucket which contains the object + bucketName := "" + client, err := NewClient(ctx) + assert.NilError(t, err) + objects := client.ListObjects(ctx, bucketName, nil) + assert.NilError(t, err) + assert.Assert(t, objects != nil) + for { + object, err := objects.Next() + if err == IteratorDone { + break + } + assert.Assert(t, object != nil) + } +} + +func TestFGetObject(t *testing.T) { + // TODO: Setup GCP mock here + t.Skip() + ctx := context.Background() + // bucketName is the name of the bucket which contains the object + bucketName := "" + // objectName is the path to the object within the bucket + objectName := "" + tempDir, err := os.MkdirTemp("", bucketName) + if err != nil { + assert.NilError(t, err) + } + localPath := filepath.Join(tempDir, objectName) + client, err := NewClient(ctx) + assert.NilError(t, err) + objErr := client.FGetObject(ctx, bucketName, objectName, localPath) + assert.NilError(t, objErr) +} From 57b54c859245e3aea6c9c5d4e892d232100d1ffa Mon Sep 17 00:00:00 2001 From: pa250194 Date: Fri, 10 Sep 2021 16:01:16 -0500 Subject: [PATCH 20/53] Service Account Key Authentication to GCP Provider Signed-off-by: pa250194 --- api/v1beta1/bucket_types.go | 2 +- .../source.toolkit.fluxcd.io_buckets.yaml | 1 + controllers/bucket_controller.go | 96 ++++---- go.mod | 1 + go.sum | 1 + pkg/gcp/gcp.go | 163 ++++++++++++-- pkg/gcp/gcp_test.go | 213 +++++++++--------- pkg/gcp/mocks/mock_gcp_storage.go | 211 +++++++++++++++++ 8 files changed, 512 insertions(+), 176 deletions(-) create mode 100644 pkg/gcp/mocks/mock_gcp_storage.go diff --git a/api/v1beta1/bucket_types.go b/api/v1beta1/bucket_types.go index 1dc68851..e046eaa8 100644 --- a/api/v1beta1/bucket_types.go +++ b/api/v1beta1/bucket_types.go @@ -30,7 +30,7 @@ const ( // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws + // +kubebuilder:validation:Enum=generic;aws;gcp // +kubebuilder:default:=generic // +optional Provider string `json:"provider,omitempty"` diff --git a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml index 5905c1d7..a64e98b4 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml @@ -66,6 +66,7 @@ spec: enum: - generic - aws + - gcp type: string region: description: The bucket region. diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 3ff17f22..9e4eee73 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -177,16 +177,21 @@ func (r *BucketReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctr } func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, error) { - var tempDir string var err error var sourceBucket sourcev1.Bucket + tempDir, err := os.MkdirTemp("", bucket.Name) + if err != nil { + err = fmt.Errorf("tmp dir error: %w", err) + return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err + } + defer os.RemoveAll(tempDir) if bucket.Spec.Provider == sourcev1.GoogleBucketProvider { - sourceBucket, tempDir, err = r.reconcileWithGCP(ctx, bucket) + sourceBucket, err = r.reconcileWithGCP(ctx, bucket, tempDir) if err != nil { return sourceBucket, err } } else { - sourceBucket, tempDir, err = r.reconcileWithMinio(ctx, bucket) + sourceBucket, err = r.reconcileWithMinio(ctx, bucket, tempDir) if err != nil { return sourceBucket, err } @@ -261,41 +266,36 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1. // reconcileWithGCP handles getting objects from a Google Cloud Platform bucket // using a gcp client -func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { +func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) { gcpClient, err := r.authGCP(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err } defer gcpClient.Client.Close() - // create tmp dir - tempDir, err := os.MkdirTemp("", bucket.Name) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err - } - defer os.RemoveAll(tempDir) ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) defer cancel() exists, err := gcpClient.BucketExists(ctxTimeout, bucket.Spec.BucketName) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if !exists { err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // Look for file with ignore rules first. path := filepath.Join(tempDir, sourceignore.IgnoreFile) if err := gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path); err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + if err == gcp.ErrorObjectDoesNotExist && sourceignore.IgnoreFile != ".sourceignore" { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err + } } ps, err := sourceignore.ReadIgnoreFile(path, nil) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // In-spec patterns take precedence if bucket.Spec.Ignore != nil { @@ -311,7 +311,7 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1 } if err != nil { err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if strings.HasSuffix(object.Name, "/") || object.Name == sourceignore.IgnoreFile { @@ -323,42 +323,33 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1 } localPath := filepath.Join(tempDir, object.Name) - // FGetObject - get and download bucket object if err = gcpClient.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Name, localPath); err != nil { err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } } - return sourcev1.Bucket{}, tempDir, nil + return sourcev1.Bucket{}, nil } // reconcileWithMinio handles getting objects from an S3 compatible bucket // using a minio client -func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket) (sourcev1.Bucket, string, error) { +func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) { s3Client, err := r.authMinio(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err } - // create tmp dir - tempDir, err := os.MkdirTemp("", bucket.Name) - if err != nil { - err = fmt.Errorf("tmp dir error: %w", err) - return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), "", err - } - defer os.RemoveAll(tempDir) - ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) defer cancel() exists, err := s3Client.BucketExists(ctxTimeout, bucket.Spec.BucketName) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if !exists { err = fmt.Errorf("bucket '%s' not found", bucket.Spec.BucketName) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // Look for file with ignore rules first @@ -367,12 +358,12 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source path := filepath.Join(tempDir, sourceignore.IgnoreFile) if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil { if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } } ps, err := sourceignore.ReadIgnoreFile(path, nil) if err != nil { - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } // In-spec patterns take precedence if bucket.Spec.Ignore != nil { @@ -387,7 +378,7 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source }) { if object.Err != nil { err = fmt.Errorf("listing objects from bucket '%s' failed: %w", bucket.Spec.BucketName, object.Err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile { @@ -402,20 +393,43 @@ func (r *BucketReconciler) reconcileWithMinio(ctx context.Context, bucket source err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{}) if err != nil { err = fmt.Errorf("downloading object from bucket '%s' failed: %w", bucket.Spec.BucketName, err) - return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), "", err + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } } - return sourcev1.Bucket{}, tempDir, nil + return sourcev1.Bucket{}, nil } // authGCP creates a new Google Cloud Platform storage client -// to interact with the Storage service. +// to interact with the storage service. func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) (*gcp.GCPClient, error) { - client, err := gcp.NewClient(ctx) - if err != nil { - return nil, err + var client *gcp.GCPClient + var err error + if bucket.Spec.SecretRef != nil { + secretName := types.NamespacedName{ + Namespace: bucket.GetNamespace(), + Name: bucket.Spec.SecretRef.Name, + } + + var secret corev1.Secret + if err := r.Get(ctx, secretName, &secret); err != nil { + return nil, fmt.Errorf("credentials secret error: %w", err) + } + if err := gcp.ValidateSecret(secret.Data, secret.Name); err != nil { + return nil, err + } + serviceAccount := gcp.InitCredentialsWithSecret(secret.Data) + client, err = gcp.NewClientWithSAKey(ctx, serviceAccount) + if err != nil { + return nil, err + } + } else { + client, err = gcp.NewClient(ctx) + if err != nil { + return nil, err + } } return client, nil + } // authMinio creates a new Minio client to interact with S3 diff --git a/go.mod b/go.mod index 8fabe102..07b8d82e 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/mock v1.6.0 // indirect github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/libgit2/git2go/v31 v31.6.1 github.com/minio/minio-go/v7 v7.0.10 diff --git a/go.sum b/go.sum index 34c48c50..75df5bf5 100644 --- a/go.sum +++ b/go.sum @@ -419,6 +419,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 18ea53fe..8f2f8811 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -18,56 +18,171 @@ package gcp import ( "context" + "encoding/json" "errors" + "fmt" "io" "os" "path/filepath" gcpStorage "cloud.google.com/go/storage" interator "google.golang.org/api/iterator" + "google.golang.org/api/option" +) + +const ( + ServiceAccount = "service_account" + AuthUri = "https://accounts.google.com/o/oauth2/auth" + TokenUri = "https://oauth2.googleapis.com/token" + AuthProviderX509CertUrl = "https://www.googleapis.com/oauth2/v1/certs" ) var ( // IteratorDone is returned when the looping of objects/content // has reached the end of the iteration. IteratorDone = interator.Done - // DirectoryExists is an error returned when the filename provided + // ErrorDirectoryExists is an error returned when the filename provided // is a directory. - DirectoryExists = errors.New("filename is a directory") - // ObjectDoesNotExist is an error returned when the object whose name + ErrorDirectoryExists = errors.New("filename is a directory") + // ErrorObjectDoesNotExist is an error returned when the object whose name // is provided does not exist. - ObjectDoesNotExist = errors.New("object does not exist") + ErrorObjectDoesNotExist = errors.New("object does not exist") ) +type Client interface { + Bucket(string) *gcpStorage.BucketHandle + Close() error +} + +type BucketHandle interface { + Create(context.Context, string, *gcpStorage.BucketAttrs) error + Delete(context.Context) error + Attrs(context.Context) (*gcpStorage.BucketAttrs, error) + Object(string) *gcpStorage.ObjectHandle + Objects(context.Context, *gcpStorage.Query) *gcpStorage.ObjectIterator +} + +type ObjectHandle interface { + Attrs(context.Context) (*gcpStorage.ObjectAttrs, error) + NewRangeReader(context.Context, int64, int64) (*gcpStorage.Reader, error) +} type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. - Client *gcpStorage.Client + Client Client // startRange is the starting read value for // reading the object from bucket. - startRange int64 + StartRange int64 // endRange is the ending read value for // reading the object from bucket. - endRange int64 + EndRange int64 +} + +// CredentialsFile struct representing the GCP Service Account +// JSON file. +type CredentialsFile struct { + Type string `json:"type"` + ProjectID string `json:"project_id"` + PrivateKeyID string `json:"private_key_id"` + PrivateKey string `json:"private_key"` + ClientEmail string `json:"client_email"` + ClientID string `json:"client_id"` + AuthUri string `json:"auth_uri"` + TokenUri string `json:"token_uri"` + AuthProviderX509CertUrl string `json:"auth_provider_x509_cert_url"` + ClientX509CertUrl string `json:"client_x509_cert_url"` } // NewClient creates a new GCP storage client // The Google Storage Client will automatically // look for the Google Application Credential environment variable -// or look for the Google Application Credential file +// or look for the Google Application Credential file. func NewClient(ctx context.Context) (*GCPClient, error) { client, err := gcpStorage.NewClient(ctx) if err != nil { return nil, err } - return &GCPClient{Client: client, startRange: 0, endRange: -1}, nil + + return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil +} + +// NewClientWithSAKey creates a new GCP storage client +// It uses the provided JSON file with service account details +// To authenticate. +func NewClientWithSAKey(ctx context.Context, credentials *CredentialsFile) (*GCPClient, error) { + saAccount, err := credentials.credentailsToJSON() + if err != nil { + return nil, err + } + + client, err := gcpStorage.NewClient(ctx, option.WithCredentialsJSON(saAccount)) + if err != nil { + return nil, err + } + + return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil +} + +// credentailsToJSON converts GCP service account credentials struct to JSON. +func (credentials *CredentialsFile) credentailsToJSON() ([]byte, error) { + credentialsJSON, err := json.Marshal(credentials) + if err != nil { + return nil, err + } + + return credentialsJSON, nil +} + +// InitCredentialsWithSecret creates a new credential +// by initializing a new CredentialsFile struct +func InitCredentialsWithSecret(secret map[string][]byte) *CredentialsFile { + return &CredentialsFile{ + Type: ServiceAccount, + ProjectID: string(secret["projectid"]), + PrivateKeyID: string(secret["privatekeyid"]), + PrivateKey: string(secret["privatekey"]), + ClientEmail: string(secret["clientemail"]), + ClientID: string(secret["clientid"]), + AuthUri: AuthUri, + TokenUri: TokenUri, + AuthProviderX509CertUrl: AuthProviderX509CertUrl, + ClientX509CertUrl: string(secret["certurl"]), + } +} + +// ValidateSecret validates the credential secrets +// It ensures that needed secret fields are not missing. +func ValidateSecret(secret map[string][]byte, name string) error { + if _, exists := secret["projectid"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'projectid'", name) + } + if _, exists := secret["privatekeyid"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'privatekeyid'", name) + } + if _, exists := secret["privatekey"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'privatekey'", name) + } + if _, exists := secret["clientemail"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) + } + if _, exists := secret["clientemail"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) + } + if _, exists := secret["clientid"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'clientid'", name) + } + if _, exists := secret["certurl"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'certurl'", name) + } + + return nil } // SetRange sets the startRange and endRange used to read the Object from // the bucket. It is a helper method for resumable downloads. func (c *GCPClient) SetRange(start, end int64) { - c.startRange = start - c.endRange = end + c.StartRange = start + c.EndRange = end } // BucketExists checks if the bucket with the provided name exists. @@ -82,15 +197,18 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, return true, nil } -// ObjectExists checks if the object with the provided name exists. +// ObjectAttributes checks if the object with the provided name exists. // If it exists the Object attributes are returned. -func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) { +func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) { attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) // ErrObjectNotExist is returned if the object does not exist + if err == gcpStorage.ErrObjectNotExist { + return false, nil, err + } if err != nil { return false, nil, err } - return true, attrs, err + return true, attrs, nil } // FGetObject gets the object from the bucket and downloads the object locally @@ -101,7 +219,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca if err == nil { // If the destination exists and is a directory. if dirStatus.IsDir() { - return DirectoryExists + return ErrorDirectoryExists } } @@ -124,17 +242,16 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca // ObjectExists verifies if object exists and you have permission to access. // Check if the object exists and if you have permission to access it // The Object attributes are returned if the Object exists. - exists, attrs, err := c.ObjectExists(ctx, bucketName, objectName) + exists, attrs, err := c.ObjectAttributes(ctx, bucketName, objectName) if err != nil { return err } if !exists { - return ObjectDoesNotExist + return ErrorObjectDoesNotExist } // Write to a temporary file "filename.part.gcp" before saving. - filePartPath := localPath + attrs.Etag + ".part.gcp" - + filePartPath := localPath + ".part.gcp" // If exists, open in append mode. If not create it as a part file. filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) if err != nil { @@ -165,25 +282,25 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca } // Get Object from GCP Bucket - objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.startRange, c.endRange) + objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.StartRange, c.EndRange) if err != nil { return err } defer objectReader.Close() // Write to the part file. - if _, err = io.CopyN(filePart, objectReader, attrs.Size); err != nil { + if _, err := io.CopyN(filePart, objectReader, attrs.Size); err != nil { return err } // Close the file before rename, this is specifically needed for Windows users. closeAndRemove = false - if err = filePart.Close(); err != nil { + if err := filePart.Close(); err != nil { return err } // Safely completed. Now commit by renaming to actual filename. - if err = os.Rename(filePartPath, localPath); err != nil { + if err := os.Rename(filePartPath, localPath); err != nil { return err } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 459e691a..f30774ac 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -14,128 +14,119 @@ See the License for the specific language governing permissions and limitations under the License. */ -package gcp +package gcp_test import ( "context" "os" "path/filepath" "testing" + "time" - "gotest.tools/assert" + gcpStorage "cloud.google.com/go/storage" + "github.com/fluxcd/source-controller/pkg/gcp" + "github.com/fluxcd/source-controller/pkg/gcp/mocks" + "github.com/golang/mock/gomock" + . "github.com/onsi/ginkgo" + . "github.com/onsi/gomega" ) -func TestNewClient(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - client, err := NewClient(context.Background()) - assert.NilError(t, err) - assert.Assert(t, client.Client != nil) +var ( + MockCtrl *gomock.Controller + MockClient *mocks.MockClient + MockBucketHandle *mocks.MockBucketHandle + MockObjectHandle *mocks.MockObjectHandle + bucketName string = "test-bucket" + objectName string = "test.yaml" + localPath string +) + +// mockgen -destination=mocks/mock_gcp_storage.go -package=mocks -source=gcp.go GCPStorageService +func TestGCPProvider(t *testing.T) { + MockCtrl = gomock.NewController(GinkgoT()) + RegisterFailHandler(Fail) + RunSpecs(t, "Test GCP Storage Provider Suite") } -func TestSetRange(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - client, err := NewClient(context.Background()) - assert.NilError(t, err) - testCases := []struct { - title string - start int64 - end int64 - }{ - { - title: "Test Case 1", - start: 1, - end: 5, - }, - { - title: "Test Case 2", - start: 3, - end: 6, - }, - { - title: "Test Case 3", - start: 4, - end: 5, - }, - { - title: "Test Case 4", - start: 2, - end: 7, - }, - } - for _, tt := range testCases { - t.Run(tt.title, func(t *testing.T) { - client.SetRange(tt.start, tt.end) - assert.Equal(t, tt.start, client.startRange) - assert.Equal(t, tt.end, client.endRange) - }) - } -} - -func TestBucketExists(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - bucketName := "" - client, err := NewClient(ctx) - assert.NilError(t, err) - exists, err := client.BucketExists(ctx, bucketName) - assert.NilError(t, err) - assert.Assert(t, exists) -} - -func TestObjectExists(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - // bucketName is the name of the bucket which contains the object - bucketName := "" - // objectName is the path to the object within the bucket - objectName := "" - client, err := NewClient(ctx) - assert.NilError(t, err) - exists, attrs, err := client.ObjectExists(ctx, bucketName, objectName) - assert.NilError(t, err) - assert.Assert(t, exists) - assert.Assert(t, attrs != nil) -} - -func TestListObjects(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - // bucketName is the name of the bucket which contains the object - bucketName := "" - client, err := NewClient(ctx) - assert.NilError(t, err) - objects := client.ListObjects(ctx, bucketName, nil) - assert.NilError(t, err) - assert.Assert(t, objects != nil) - for { - object, err := objects.Next() - if err == IteratorDone { - break - } - assert.Assert(t, object != nil) - } -} - -func TestFGetObject(t *testing.T) { - // TODO: Setup GCP mock here - t.Skip() - ctx := context.Background() - // bucketName is the name of the bucket which contains the object - bucketName := "" - // objectName is the path to the object within the bucket - objectName := "" +var _ = BeforeSuite(func() { + MockClient = mocks.NewMockClient(MockCtrl) + MockBucketHandle = mocks.NewMockBucketHandle(MockCtrl) + MockObjectHandle = mocks.NewMockObjectHandle(MockCtrl) tempDir, err := os.MkdirTemp("", bucketName) if err != nil { - assert.NilError(t, err) + Expect(err).ToNot(HaveOccurred()) } - localPath := filepath.Join(tempDir, objectName) - client, err := NewClient(ctx) - assert.NilError(t, err) - objErr := client.FGetObject(ctx, bucketName, objectName, localPath) - assert.NilError(t, objErr) -} + localPath = filepath.Join(tempDir, objectName) + MockClient.EXPECT().Bucket(bucketName).Return(MockBucketHandle).AnyTimes() + MockBucketHandle.EXPECT().Object(objectName).Return(&gcpStorage.ObjectHandle{}).AnyTimes() + MockBucketHandle.EXPECT().Attrs(context.Background()).Return(&gcpStorage.BucketAttrs{ + Name: bucketName, + Created: time.Now(), + Etag: "test-etag", + }, nil).AnyTimes() + MockBucketHandle.EXPECT().Objects(gomock.Any(), nil).Return(&gcpStorage.ObjectIterator{}).AnyTimes() + MockObjectHandle.EXPECT().Attrs(gomock.Any()).Return(&gcpStorage.ObjectAttrs{ + Bucket: bucketName, + Name: objectName, + ContentType: "text/x-yaml", + Etag: "test-etag", + Size: 125, + Created: time.Now(), + }, nil).AnyTimes() + MockObjectHandle.EXPECT().NewRangeReader(gomock.Any(), 10, 125).Return(&gcpStorage.Reader{}, nil).AnyTimes() +}) + +var _ = Describe("GCP Storage Provider", func() { + Describe("Get GCP Storage Provider client from gcp", func() { + + Context("Gcp storage Bucket - BucketExists", func() { + It("should not return an error when fetching gcp storage bucket", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + exists, err := gcpClient.BucketExists(context.Background(), bucketName) + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + }) + }) + Context("Gcp storage Bucket - FGetObject", func() { + It("should get the object from the bucket and download the object locally", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + err := gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath) + Expect(err).ToNot(HaveOccurred()) + }) + }) + Context("Gcp storage Bucket - ObjectAttributes", func() { + It("should get the object attributes", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + exists, attrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) + Expect(err).ToNot(HaveOccurred()) + Expect(exists).To(BeTrue()) + Expect(attrs).ToNot(BeNil()) + }) + + Context("Gcp storage Bucket - SetRange", func() { + It("should set the range of the io reader seeker for the file download", func() { + gcpClient := &gcp.GCPClient{ + Client: MockClient, + StartRange: 0, + EndRange: -1, + } + gcpClient.SetRange(2, 5) + Expect(gcpClient.StartRange).To(Equal(int64(2))) + Expect(gcpClient.EndRange).To(Equal(int64(5))) + }) + }) + }) + }) +}) diff --git a/pkg/gcp/mocks/mock_gcp_storage.go b/pkg/gcp/mocks/mock_gcp_storage.go new file mode 100644 index 00000000..54b78be1 --- /dev/null +++ b/pkg/gcp/mocks/mock_gcp_storage.go @@ -0,0 +1,211 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: gcp.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + storage "cloud.google.com/go/storage" + gomock "github.com/golang/mock/gomock" +) + +// MockClient is a mock of Client interface. +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder +} + +// MockClientMockRecorder is the mock recorder for MockClient. +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance. +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// Bucket mocks base method. +func (m *MockClient) Bucket(arg0 string) *storage.BucketHandle { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Bucket", arg0) + ret0, _ := ret[0].(*storage.BucketHandle) + return ret0 +} + +// Bucket indicates an expected call of Bucket. +func (mr *MockClientMockRecorder) Bucket(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockClient)(nil).Bucket), arg0) +} + +// Close mocks base method. +func (m *MockClient) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockClientMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClient)(nil).Close)) +} + +// MockBucketHandle is a mock of BucketHandle interface. +type MockBucketHandle struct { + ctrl *gomock.Controller + recorder *MockBucketHandleMockRecorder +} + +// MockBucketHandleMockRecorder is the mock recorder for MockBucketHandle. +type MockBucketHandleMockRecorder struct { + mock *MockBucketHandle +} + +// NewMockBucketHandle creates a new mock instance. +func NewMockBucketHandle(ctrl *gomock.Controller) *MockBucketHandle { + mock := &MockBucketHandle{ctrl: ctrl} + mock.recorder = &MockBucketHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBucketHandle) EXPECT() *MockBucketHandleMockRecorder { + return m.recorder +} + +// Attrs mocks base method. +func (m *MockBucketHandle) Attrs(arg0 context.Context) (*storage.BucketAttrs, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Attrs", arg0) + ret0, _ := ret[0].(*storage.BucketAttrs) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Attrs indicates an expected call of Attrs. +func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0) +} + +// Create mocks base method. +func (m *MockBucketHandle) Create(arg0 context.Context, arg1 string, arg2 *storage.BucketAttrs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockBucketHandleMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockBucketHandle)(nil).Create), arg0, arg1, arg2) +} + +// Delete mocks base method. +func (m *MockBucketHandle) Delete(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockBucketHandleMockRecorder) Delete(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBucketHandle)(nil).Delete), arg0) +} + +// Object mocks base method. +func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Object", arg0) + ret0, _ := ret[0].(*storage.ObjectHandle) + return ret0 +} + +// Object indicates an expected call of Object. +func (mr *MockBucketHandleMockRecorder) Object(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Object", reflect.TypeOf((*MockBucketHandle)(nil).Object), arg0) +} + +// Objects mocks base method. +func (m *MockBucketHandle) Objects(arg0 context.Context, arg1 *storage.Query) *storage.ObjectIterator { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Objects", arg0, arg1) + ret0, _ := ret[0].(*storage.ObjectIterator) + return ret0 +} + +// Objects indicates an expected call of Objects. +func (mr *MockBucketHandleMockRecorder) Objects(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Objects", reflect.TypeOf((*MockBucketHandle)(nil).Objects), arg0, arg1) +} + +// MockObjectHandle is a mock of ObjectHandle interface. +type MockObjectHandle struct { + ctrl *gomock.Controller + recorder *MockObjectHandleMockRecorder +} + +// MockObjectHandleMockRecorder is the mock recorder for MockObjectHandle. +type MockObjectHandleMockRecorder struct { + mock *MockObjectHandle +} + +// NewMockObjectHandle creates a new mock instance. +func NewMockObjectHandle(ctrl *gomock.Controller) *MockObjectHandle { + mock := &MockObjectHandle{ctrl: ctrl} + mock.recorder = &MockObjectHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockObjectHandle) EXPECT() *MockObjectHandleMockRecorder { + return m.recorder +} + +// Attrs mocks base method. +func (m *MockObjectHandle) Attrs(arg0 context.Context) (*storage.ObjectAttrs, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Attrs", arg0) + ret0, _ := ret[0].(*storage.ObjectAttrs) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Attrs indicates an expected call of Attrs. +func (mr *MockObjectHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockObjectHandle)(nil).Attrs), arg0) +} + +// NewRangeReader mocks base method. +func (m *MockObjectHandle) NewRangeReader(arg0 context.Context, arg1, arg2 int64) (*storage.Reader, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewRangeReader", arg0, arg1, arg2) + ret0, _ := ret[0].(*storage.Reader) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NewRangeReader indicates an expected call of NewRangeReader. +func (mr *MockObjectHandleMockRecorder) NewRangeReader(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewRangeReader", reflect.TypeOf((*MockObjectHandle)(nil).NewRangeReader), arg0, arg1, arg2) +} From 1fae4f61c189338bf4b8f456134a6d4959119f56 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Tue, 14 Sep 2021 09:34:02 -0500 Subject: [PATCH 21/53] Tests for GCP Bucket Provider Signed-off-by: pa250194 --- docs/spec/v1alpha1/buckets.md | 2 +- pkg/gcp/gcp.go | 8 ++------ pkg/gcp/gcp_test.go | 2 +- pkg/gcp/mocks/mock_gcp_storage.go | 28 ---------------------------- 4 files changed, 4 insertions(+), 36 deletions(-) diff --git a/docs/spec/v1alpha1/buckets.md b/docs/spec/v1alpha1/buckets.md index 0ad60f41..53838e25 100644 --- a/docs/spec/v1alpha1/buckets.md +++ b/docs/spec/v1alpha1/buckets.md @@ -11,7 +11,7 @@ Bucket: // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws + // +kubebuilder:validation:Enum=generic;aws;gcp // +optional Provider string `json:"provider,omitempty"` diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 8f2f8811..2c372e9e 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -55,8 +55,6 @@ type Client interface { } type BucketHandle interface { - Create(context.Context, string, *gcpStorage.BucketAttrs) error - Delete(context.Context) error Attrs(context.Context) (*gcpStorage.BucketAttrs, error) Object(string) *gcpStorage.ObjectHandle Objects(context.Context, *gcpStorage.Query) *gcpStorage.ObjectIterator @@ -66,10 +64,11 @@ type ObjectHandle interface { Attrs(context.Context) (*gcpStorage.ObjectAttrs, error) NewRangeReader(context.Context, int64, int64) (*gcpStorage.Reader, error) } + type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. - Client Client + Client // startRange is the starting read value for // reading the object from bucket. StartRange int64 @@ -165,9 +164,6 @@ func ValidateSecret(secret map[string][]byte, name string) error { if _, exists := secret["clientemail"]; !exists { return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) } - if _, exists := secret["clientemail"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) - } if _, exists := secret["clientid"]; !exists { return fmt.Errorf("invalid '%s' secret data: required fields 'clientid'", name) } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index f30774ac..ed72f9d2 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -57,7 +57,7 @@ var _ = BeforeSuite(func() { Expect(err).ToNot(HaveOccurred()) } localPath = filepath.Join(tempDir, objectName) - MockClient.EXPECT().Bucket(bucketName).Return(MockBucketHandle).AnyTimes() + MockClient.EXPECT().Bucket(bucketName).Return(&gcpStorage.BucketHandle{}).AnyTimes() MockBucketHandle.EXPECT().Object(objectName).Return(&gcpStorage.ObjectHandle{}).AnyTimes() MockBucketHandle.EXPECT().Attrs(context.Background()).Return(&gcpStorage.BucketAttrs{ Name: bucketName, diff --git a/pkg/gcp/mocks/mock_gcp_storage.go b/pkg/gcp/mocks/mock_gcp_storage.go index 54b78be1..25b5e9c1 100644 --- a/pkg/gcp/mocks/mock_gcp_storage.go +++ b/pkg/gcp/mocks/mock_gcp_storage.go @@ -101,34 +101,6 @@ func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0) } -// Create mocks base method. -func (m *MockBucketHandle) Create(arg0 context.Context, arg1 string, arg2 *storage.BucketAttrs) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// Create indicates an expected call of Create. -func (mr *MockBucketHandleMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockBucketHandle)(nil).Create), arg0, arg1, arg2) -} - -// Delete mocks base method. -func (m *MockBucketHandle) Delete(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Delete indicates an expected call of Delete. -func (mr *MockBucketHandleMockRecorder) Delete(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBucketHandle)(nil).Delete), arg0) -} - // Object mocks base method. func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle { m.ctrl.T.Helper() From a46b0f54b8e5c4a0bda48a38ce6d5c8b2c7bfe01 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Wed, 15 Sep 2021 14:42:53 -0500 Subject: [PATCH 22/53] Added Tests to GCP provider Signed-off-by: pa250194 --- pkg/gcp/gcp.go | 18 +- pkg/gcp/gcp_test.go | 342 ++++++++++++++++++++++-------- pkg/gcp/mocks/mock_gcp_storage.go | 183 ---------------- 3 files changed, 249 insertions(+), 294 deletions(-) delete mode 100644 pkg/gcp/mocks/mock_gcp_storage.go diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 2c372e9e..ec56384a 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -49,26 +49,10 @@ var ( ErrorObjectDoesNotExist = errors.New("object does not exist") ) -type Client interface { - Bucket(string) *gcpStorage.BucketHandle - Close() error -} - -type BucketHandle interface { - Attrs(context.Context) (*gcpStorage.BucketAttrs, error) - Object(string) *gcpStorage.ObjectHandle - Objects(context.Context, *gcpStorage.Query) *gcpStorage.ObjectIterator -} - -type ObjectHandle interface { - Attrs(context.Context) (*gcpStorage.ObjectAttrs, error) - NewRangeReader(context.Context, int64, int64) (*gcpStorage.Reader, error) -} - type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. - Client + *gcpStorage.Client // startRange is the starting read value for // reading the object from bucket. StartRange int64 diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index ed72f9d2..64db105f 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -18,6 +18,15 @@ package gcp_test import ( "context" + "crypto/tls" + "encoding/json" + "fmt" + "io" + "io/ioutil" + "log" + "net" + "net/http" + "net/http/httptest" "os" "path/filepath" "testing" @@ -25,108 +34,253 @@ import ( gcpStorage "cloud.google.com/go/storage" "github.com/fluxcd/source-controller/pkg/gcp" - "github.com/fluxcd/source-controller/pkg/gcp/mocks" - "github.com/golang/mock/gomock" - . "github.com/onsi/ginkgo" - . "github.com/onsi/gomega" + "google.golang.org/api/googleapi" + raw "google.golang.org/api/storage/v1" + "gotest.tools/assert" + + "google.golang.org/api/option" +) + +const ( + bucketName string = "test-bucket" + objectName string = "test.yaml" ) var ( - MockCtrl *gomock.Controller - MockClient *mocks.MockClient - MockBucketHandle *mocks.MockBucketHandle - MockObjectHandle *mocks.MockObjectHandle - bucketName string = "test-bucket" - objectName string = "test.yaml" - localPath string + Client *gcpStorage.Client + err error ) -// mockgen -destination=mocks/mock_gcp_storage.go -package=mocks -source=gcp.go GCPStorageService -func TestGCPProvider(t *testing.T) { - MockCtrl = gomock.NewController(GinkgoT()) - RegisterFailHandler(Fail) - RunSpecs(t, "Test GCP Storage Provider Suite") +func TestMain(m *testing.M) { + hc, close := newTestServer(func(w http.ResponseWriter, r *http.Request) { + io.Copy(ioutil.Discard, r.Body) + w.WriteHeader(200) + if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s?alt=json&prettyPrint=false&projection=full", bucketName) { + response := getBucket() + jsonedResp, err := json.Marshal(response) + if err != nil { + log.Fatalf("error marshalling resp %v\n", err) + } + _, err = w.Write(jsonedResp) + if err != nil { + log.Fatalf("error writing jsonedResp %v\n", err) + } + } else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) { + response := getObject() + jsonedResp, err := json.Marshal(response) + if err != nil { + log.Fatalf("error marshalling resp %v\n", err) + } + _, err = w.Write(jsonedResp) + if err != nil { + log.Fatalf("error writing jsonedResp %v\n", err) + } + } else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o?alt=json&delimiter=&endOffset=&pageToken=&prefix=&prettyPrint=false&projection=full&startOffset=&versions=false", bucketName) { + response := getObject() + jsonedResp, err := json.Marshal(response) + if err != nil { + log.Fatalf("error marshalling resp %v\n", err) + } + _, err = w.Write(jsonedResp) + if err != nil { + log.Fatalf("error writing jsonedResp %v\n", err) + } + } else if r.RequestURI == fmt.Sprintf("/%s/test.yaml", bucketName) || r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) { + response := getObjectFile() + _, err = w.Write([]byte(response)) + if err != nil { + log.Fatalf("error writing jsonedResp %v\n", err) + } + } + }) + ctx := context.Background() + Client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) + if err != nil { + log.Fatal(err) + } + run := m.Run() + close() + os.Exit(run) } -var _ = BeforeSuite(func() { - MockClient = mocks.NewMockClient(MockCtrl) - MockBucketHandle = mocks.NewMockBucketHandle(MockCtrl) - MockObjectHandle = mocks.NewMockObjectHandle(MockCtrl) - tempDir, err := os.MkdirTemp("", bucketName) - if err != nil { - Expect(err).ToNot(HaveOccurred()) +func TestBucketExists(t *testing.T) { + gcpClient := &gcp.GCPClient{ + Client: Client, + StartRange: 0, + EndRange: -1, } - localPath = filepath.Join(tempDir, objectName) - MockClient.EXPECT().Bucket(bucketName).Return(&gcpStorage.BucketHandle{}).AnyTimes() - MockBucketHandle.EXPECT().Object(objectName).Return(&gcpStorage.ObjectHandle{}).AnyTimes() - MockBucketHandle.EXPECT().Attrs(context.Background()).Return(&gcpStorage.BucketAttrs{ - Name: bucketName, - Created: time.Now(), - Etag: "test-etag", - }, nil).AnyTimes() - MockBucketHandle.EXPECT().Objects(gomock.Any(), nil).Return(&gcpStorage.ObjectIterator{}).AnyTimes() - MockObjectHandle.EXPECT().Attrs(gomock.Any()).Return(&gcpStorage.ObjectAttrs{ - Bucket: bucketName, - Name: objectName, - ContentType: "text/x-yaml", - Etag: "test-etag", - Size: 125, - Created: time.Now(), - }, nil).AnyTimes() - MockObjectHandle.EXPECT().NewRangeReader(gomock.Any(), 10, 125).Return(&gcpStorage.Reader{}, nil).AnyTimes() -}) + exists, err := gcpClient.BucketExists(context.Background(), bucketName) + assert.NilError(t, err) + assert.Assert(t, exists) +} -var _ = Describe("GCP Storage Provider", func() { - Describe("Get GCP Storage Provider client from gcp", func() { +func TestObjectAttributes(t *testing.T) { + gcpClient := &gcp.GCPClient{ + Client: Client, + StartRange: 0, + EndRange: -1, + } + exists, objectAttrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) + if err == gcpStorage.ErrObjectNotExist { + assert.NilError(t, err) + } + assert.NilError(t, err) + assert.Assert(t, exists) + assert.Assert(t, objectAttrs != nil) +} - Context("Gcp storage Bucket - BucketExists", func() { - It("should not return an error when fetching gcp storage bucket", func() { - gcpClient := &gcp.GCPClient{ - Client: MockClient, - StartRange: 0, - EndRange: -1, - } - exists, err := gcpClient.BucketExists(context.Background(), bucketName) - Expect(err).ToNot(HaveOccurred()) - Expect(exists).To(BeTrue()) - }) - }) - Context("Gcp storage Bucket - FGetObject", func() { - It("should get the object from the bucket and download the object locally", func() { - gcpClient := &gcp.GCPClient{ - Client: MockClient, - StartRange: 0, - EndRange: -1, - } - err := gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath) - Expect(err).ToNot(HaveOccurred()) - }) - }) - Context("Gcp storage Bucket - ObjectAttributes", func() { - It("should get the object attributes", func() { - gcpClient := &gcp.GCPClient{ - Client: MockClient, - StartRange: 0, - EndRange: -1, - } - exists, attrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) - Expect(err).ToNot(HaveOccurred()) - Expect(exists).To(BeTrue()) - Expect(attrs).ToNot(BeNil()) - }) +func TestListObjects(t *testing.T) { + gcpClient := &gcp.GCPClient{ + Client: Client, + StartRange: 0, + EndRange: -1, + } + objectInterator := gcpClient.ListObjects(context.Background(), bucketName, nil) + for { + _, err := objectInterator.Next() + if err == gcp.IteratorDone { + break + } + assert.NilError(t, err) + } + assert.Assert(t, objectInterator != nil) +} - Context("Gcp storage Bucket - SetRange", func() { - It("should set the range of the io reader seeker for the file download", func() { - gcpClient := &gcp.GCPClient{ - Client: MockClient, - StartRange: 0, - EndRange: -1, - } - gcpClient.SetRange(2, 5) - Expect(gcpClient.StartRange).To(Equal(int64(2))) - Expect(gcpClient.EndRange).To(Equal(int64(5))) - }) - }) - }) - }) -}) +func TestFGetObject(t *testing.T) { + tempDir, err := os.MkdirTemp("", bucketName) + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + gcpClient := &gcp.GCPClient{ + Client: Client, + StartRange: 0, + EndRange: -1, + } + localPath := filepath.Join(tempDir, objectName) + err = gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath) + if err != io.EOF { + assert.NilError(t, err) + } +} + +func TestSetRange(t *testing.T) { + gcpClient := &gcp.GCPClient{ + Client: Client, + StartRange: 0, + EndRange: -1, + } + gcpClient.SetRange(2, 5) + assert.Equal(t, gcpClient.StartRange, int64(2)) + assert.Equal(t, gcpClient.EndRange, int64(5)) +} + +func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*http.Client, func()) { + ts := httptest.NewTLSServer(http.HandlerFunc(handler)) + tlsConf := &tls.Config{InsecureSkipVerify: true} + tr := &http.Transport{ + TLSClientConfig: tlsConf, + DialTLS: func(netw, addr string) (net.Conn, error) { + return tls.Dial("tcp", ts.Listener.Addr().String(), tlsConf) + }, + } + return &http.Client{Transport: tr}, func() { + tr.CloseIdleConnections() + ts.Close() + } +} + +func getObject() *raw.Object { + customTime := time.Now() + retTime := customTime.Add(3 * time.Hour) + return &raw.Object{ + Bucket: bucketName, + Name: objectName, + EventBasedHold: false, + TemporaryHold: false, + RetentionExpirationTime: retTime.Format(time.RFC3339), + ContentType: "text/x-yaml", + ContentLanguage: "en-us", + Size: 1 << 20, + CustomTime: customTime.Format(time.RFC3339), + Md5Hash: "bFbHCDvedeecefdgmfmhfuRxBdcedGe96S82XJOAXxjJpk=", + } +} + +func getBucket() *raw.Bucket { + labels := map[string]string{"a": "b"} + matchClasses := []string{"STANDARD"} + aTime := time.Date(2021, 1, 2, 0, 0, 0, 0, time.UTC) + rb := &raw.Bucket{ + Name: bucketName, + Location: "loc", + DefaultEventBasedHold: true, + Metageneration: 3, + StorageClass: "sc", + TimeCreated: "2021-5-23T04:05:06Z", + Versioning: &raw.BucketVersioning{Enabled: true}, + Labels: labels, + Billing: &raw.BucketBilling{RequesterPays: true}, + Etag: "BNaB2y5Xr3&5MHDca4SoTNL79lyhahr7MV87ubwjgdtg6ghs", + Lifecycle: &raw.BucketLifecycle{ + Rule: []*raw.BucketLifecycleRule{{ + Action: &raw.BucketLifecycleRuleAction{ + Type: "SetStorageClass", + StorageClass: "NEARLINE", + }, + Condition: &raw.BucketLifecycleRuleCondition{ + Age: 10, + IsLive: googleapi.Bool(true), + CreatedBefore: "2021-01-02", + MatchesStorageClass: matchClasses, + NumNewerVersions: 3, + }, + }}, + }, + RetentionPolicy: &raw.BucketRetentionPolicy{ + RetentionPeriod: 3, + EffectiveTime: aTime.Format(time.RFC3339), + }, + IamConfiguration: &raw.BucketIamConfiguration{ + BucketPolicyOnly: &raw.BucketIamConfigurationBucketPolicyOnly{ + Enabled: true, + LockedTime: aTime.Format(time.RFC3339), + }, + UniformBucketLevelAccess: &raw.BucketIamConfigurationUniformBucketLevelAccess{ + Enabled: true, + LockedTime: aTime.Format(time.RFC3339), + }, + }, + Cors: []*raw.BucketCors{ + { + MaxAgeSeconds: 3600, + Method: []string{"GET", "POST"}, + Origin: []string{"*"}, + ResponseHeader: []string{"FOO"}, + }, + }, + Acl: []*raw.BucketAccessControl{ + {Bucket: bucketName, Role: "READER", Email: "test@example.com", Entity: "allUsers"}, + }, + LocationType: "dual-region", + Encryption: &raw.BucketEncryption{DefaultKmsKeyName: "key"}, + Logging: &raw.BucketLogging{LogBucket: "lb", LogObjectPrefix: "p"}, + Website: &raw.BucketWebsite{MainPageSuffix: "mps", NotFoundPage: "404"}, + } + return rb +} + +func getObjectFile() string { + return ` + apiVersion: source.toolkit.fluxcd.io/v1beta1 + kind: Bucket + metadata: + name: podinfo + namespace: default + spec: + interval: 5m + provider: aws + bucketName: podinfo + endpoint: s3.amazonaws.com + region: us-east-1 + timeout: 30s + ` +} diff --git a/pkg/gcp/mocks/mock_gcp_storage.go b/pkg/gcp/mocks/mock_gcp_storage.go deleted file mode 100644 index 25b5e9c1..00000000 --- a/pkg/gcp/mocks/mock_gcp_storage.go +++ /dev/null @@ -1,183 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: gcp.go - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - storage "cloud.google.com/go/storage" - gomock "github.com/golang/mock/gomock" -) - -// MockClient is a mock of Client interface. -type MockClient struct { - ctrl *gomock.Controller - recorder *MockClientMockRecorder -} - -// MockClientMockRecorder is the mock recorder for MockClient. -type MockClientMockRecorder struct { - mock *MockClient -} - -// NewMockClient creates a new mock instance. -func NewMockClient(ctrl *gomock.Controller) *MockClient { - mock := &MockClient{ctrl: ctrl} - mock.recorder = &MockClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockClient) EXPECT() *MockClientMockRecorder { - return m.recorder -} - -// Bucket mocks base method. -func (m *MockClient) Bucket(arg0 string) *storage.BucketHandle { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Bucket", arg0) - ret0, _ := ret[0].(*storage.BucketHandle) - return ret0 -} - -// Bucket indicates an expected call of Bucket. -func (mr *MockClientMockRecorder) Bucket(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockClient)(nil).Bucket), arg0) -} - -// Close mocks base method. -func (m *MockClient) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockClientMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClient)(nil).Close)) -} - -// MockBucketHandle is a mock of BucketHandle interface. -type MockBucketHandle struct { - ctrl *gomock.Controller - recorder *MockBucketHandleMockRecorder -} - -// MockBucketHandleMockRecorder is the mock recorder for MockBucketHandle. -type MockBucketHandleMockRecorder struct { - mock *MockBucketHandle -} - -// NewMockBucketHandle creates a new mock instance. -func NewMockBucketHandle(ctrl *gomock.Controller) *MockBucketHandle { - mock := &MockBucketHandle{ctrl: ctrl} - mock.recorder = &MockBucketHandleMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockBucketHandle) EXPECT() *MockBucketHandleMockRecorder { - return m.recorder -} - -// Attrs mocks base method. -func (m *MockBucketHandle) Attrs(arg0 context.Context) (*storage.BucketAttrs, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Attrs", arg0) - ret0, _ := ret[0].(*storage.BucketAttrs) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Attrs indicates an expected call of Attrs. -func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0) -} - -// Object mocks base method. -func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Object", arg0) - ret0, _ := ret[0].(*storage.ObjectHandle) - return ret0 -} - -// Object indicates an expected call of Object. -func (mr *MockBucketHandleMockRecorder) Object(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Object", reflect.TypeOf((*MockBucketHandle)(nil).Object), arg0) -} - -// Objects mocks base method. -func (m *MockBucketHandle) Objects(arg0 context.Context, arg1 *storage.Query) *storage.ObjectIterator { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Objects", arg0, arg1) - ret0, _ := ret[0].(*storage.ObjectIterator) - return ret0 -} - -// Objects indicates an expected call of Objects. -func (mr *MockBucketHandleMockRecorder) Objects(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Objects", reflect.TypeOf((*MockBucketHandle)(nil).Objects), arg0, arg1) -} - -// MockObjectHandle is a mock of ObjectHandle interface. -type MockObjectHandle struct { - ctrl *gomock.Controller - recorder *MockObjectHandleMockRecorder -} - -// MockObjectHandleMockRecorder is the mock recorder for MockObjectHandle. -type MockObjectHandleMockRecorder struct { - mock *MockObjectHandle -} - -// NewMockObjectHandle creates a new mock instance. -func NewMockObjectHandle(ctrl *gomock.Controller) *MockObjectHandle { - mock := &MockObjectHandle{ctrl: ctrl} - mock.recorder = &MockObjectHandleMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockObjectHandle) EXPECT() *MockObjectHandleMockRecorder { - return m.recorder -} - -// Attrs mocks base method. -func (m *MockObjectHandle) Attrs(arg0 context.Context) (*storage.ObjectAttrs, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Attrs", arg0) - ret0, _ := ret[0].(*storage.ObjectAttrs) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Attrs indicates an expected call of Attrs. -func (mr *MockObjectHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockObjectHandle)(nil).Attrs), arg0) -} - -// NewRangeReader mocks base method. -func (m *MockObjectHandle) NewRangeReader(arg0 context.Context, arg1, arg2 int64) (*storage.Reader, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NewRangeReader", arg0, arg1, arg2) - ret0, _ := ret[0].(*storage.Reader) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// NewRangeReader indicates an expected call of NewRangeReader. -func (mr *MockObjectHandleMockRecorder) NewRangeReader(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewRangeReader", reflect.TypeOf((*MockObjectHandle)(nil).NewRangeReader), arg0, arg1, arg2) -} From b02a7625eaac358a32355d31e682fa207977402c Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 16 Sep 2021 09:48:33 -0500 Subject: [PATCH 23/53] Added more tests and cleaned up GCP provider logic Signed-off-by: pa250194 --- controllers/bucket_controller.go | 4 +- go.mod | 1 - go.sum | 1 - pkg/gcp/gcp.go | 90 +------------------- pkg/gcp/gcp_test.go | 140 ++++++++++++++++++++++++++----- 5 files changed, 124 insertions(+), 112 deletions(-) diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 9e4eee73..aa11c261 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -29,6 +29,7 @@ import ( "github.com/minio/minio-go/v7" "github.com/minio/minio-go/v7/pkg/credentials" "github.com/minio/minio-go/v7/pkg/s3utils" + "google.golang.org/api/option" corev1 "k8s.io/api/core/v1" apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -417,8 +418,7 @@ func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) if err := gcp.ValidateSecret(secret.Data, secret.Name); err != nil { return nil, err } - serviceAccount := gcp.InitCredentialsWithSecret(secret.Data) - client, err = gcp.NewClientWithSAKey(ctx, serviceAccount) + client, err = gcp.NewClient(ctx, option.WithCredentialsJSON(secret.Data["serviceaccount"])) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 07b8d82e..8fabe102 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/libgit2/git2go/v31 v31.6.1 github.com/minio/minio-go/v7 v7.0.10 diff --git a/go.sum b/go.sum index 75df5bf5..34c48c50 100644 --- a/go.sum +++ b/go.sum @@ -419,7 +419,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index ec56384a..b2274e34 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -18,7 +18,6 @@ package gcp import ( "context" - "encoding/json" "errors" "fmt" "io" @@ -30,13 +29,6 @@ import ( "google.golang.org/api/option" ) -const ( - ServiceAccount = "service_account" - AuthUri = "https://accounts.google.com/o/oauth2/auth" - TokenUri = "https://oauth2.googleapis.com/token" - AuthProviderX509CertUrl = "https://www.googleapis.com/oauth2/v1/certs" -) - var ( // IteratorDone is returned when the looping of objects/content // has reached the end of the iteration. @@ -61,27 +53,12 @@ type GCPClient struct { EndRange int64 } -// CredentialsFile struct representing the GCP Service Account -// JSON file. -type CredentialsFile struct { - Type string `json:"type"` - ProjectID string `json:"project_id"` - PrivateKeyID string `json:"private_key_id"` - PrivateKey string `json:"private_key"` - ClientEmail string `json:"client_email"` - ClientID string `json:"client_id"` - AuthUri string `json:"auth_uri"` - TokenUri string `json:"token_uri"` - AuthProviderX509CertUrl string `json:"auth_provider_x509_cert_url"` - ClientX509CertUrl string `json:"client_x509_cert_url"` -} - // NewClient creates a new GCP storage client // The Google Storage Client will automatically // look for the Google Application Credential environment variable // or look for the Google Application Credential file. -func NewClient(ctx context.Context) (*GCPClient, error) { - client, err := gcpStorage.NewClient(ctx) +func NewClient(ctx context.Context, opts ...option.ClientOption) (*GCPClient, error) { + client, err := gcpStorage.NewClient(ctx, opts...) if err != nil { return nil, err } @@ -89,70 +66,11 @@ func NewClient(ctx context.Context) (*GCPClient, error) { return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil } -// NewClientWithSAKey creates a new GCP storage client -// It uses the provided JSON file with service account details -// To authenticate. -func NewClientWithSAKey(ctx context.Context, credentials *CredentialsFile) (*GCPClient, error) { - saAccount, err := credentials.credentailsToJSON() - if err != nil { - return nil, err - } - - client, err := gcpStorage.NewClient(ctx, option.WithCredentialsJSON(saAccount)) - if err != nil { - return nil, err - } - - return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil -} - -// credentailsToJSON converts GCP service account credentials struct to JSON. -func (credentials *CredentialsFile) credentailsToJSON() ([]byte, error) { - credentialsJSON, err := json.Marshal(credentials) - if err != nil { - return nil, err - } - - return credentialsJSON, nil -} - -// InitCredentialsWithSecret creates a new credential -// by initializing a new CredentialsFile struct -func InitCredentialsWithSecret(secret map[string][]byte) *CredentialsFile { - return &CredentialsFile{ - Type: ServiceAccount, - ProjectID: string(secret["projectid"]), - PrivateKeyID: string(secret["privatekeyid"]), - PrivateKey: string(secret["privatekey"]), - ClientEmail: string(secret["clientemail"]), - ClientID: string(secret["clientid"]), - AuthUri: AuthUri, - TokenUri: TokenUri, - AuthProviderX509CertUrl: AuthProviderX509CertUrl, - ClientX509CertUrl: string(secret["certurl"]), - } -} - // ValidateSecret validates the credential secrets // It ensures that needed secret fields are not missing. func ValidateSecret(secret map[string][]byte, name string) error { - if _, exists := secret["projectid"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'projectid'", name) - } - if _, exists := secret["privatekeyid"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'privatekeyid'", name) - } - if _, exists := secret["privatekey"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'privatekey'", name) - } - if _, exists := secret["clientemail"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'clientemail'", name) - } - if _, exists := secret["clientid"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'clientid'", name) - } - if _, exists := secret["certurl"]; !exists { - return fmt.Errorf("invalid '%s' secret data: required fields 'certurl'", name) + if _, exists := secret["serviceaccount"]; !exists { + return fmt.Errorf("invalid '%s' secret data: required fields 'serviceaccount'", name) } return nil diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 64db105f..5c0292e9 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -47,54 +47,61 @@ const ( ) var ( - Client *gcpStorage.Client + hc *http.Client + client *gcpStorage.Client + close func() err error ) func TestMain(m *testing.M) { - hc, close := newTestServer(func(w http.ResponseWriter, r *http.Request) { + hc, close = newTestServer(func(w http.ResponseWriter, r *http.Request) { io.Copy(ioutil.Discard, r.Body) - w.WriteHeader(200) if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s?alt=json&prettyPrint=false&projection=full", bucketName) { + w.WriteHeader(200) response := getBucket() - jsonedResp, err := json.Marshal(response) + jsonResponse, err := json.Marshal(response) if err != nil { - log.Fatalf("error marshalling resp %v\n", err) + log.Fatalf("error marshalling response %v\n", err) } - _, err = w.Write(jsonedResp) + _, err = w.Write(jsonResponse) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing jsonResponse %v\n", err) } } else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) { + w.WriteHeader(200) response := getObject() - jsonedResp, err := json.Marshal(response) + jsonResponse, err := json.Marshal(response) if err != nil { - log.Fatalf("error marshalling resp %v\n", err) + log.Fatalf("error marshalling response %v\n", err) } - _, err = w.Write(jsonedResp) + _, err = w.Write(jsonResponse) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing jsonResponse %v\n", err) } } else if r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o?alt=json&delimiter=&endOffset=&pageToken=&prefix=&prettyPrint=false&projection=full&startOffset=&versions=false", bucketName) { + w.WriteHeader(200) response := getObject() - jsonedResp, err := json.Marshal(response) + jsonResponse, err := json.Marshal(response) if err != nil { - log.Fatalf("error marshalling resp %v\n", err) + log.Fatalf("error marshalling response %v\n", err) } - _, err = w.Write(jsonedResp) + _, err = w.Write(jsonResponse) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing jsonResponse %v\n", err) } } else if r.RequestURI == fmt.Sprintf("/%s/test.yaml", bucketName) || r.RequestURI == fmt.Sprintf("/storage/v1/b/%s/o/%s?alt=json&prettyPrint=false&projection=full", bucketName, objectName) { + w.WriteHeader(200) response := getObjectFile() _, err = w.Write([]byte(response)) if err != nil { - log.Fatalf("error writing jsonedResp %v\n", err) + log.Fatalf("error writing response %v\n", err) } + } else { + w.WriteHeader(404) } }) ctx := context.Background() - Client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) + client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { log.Fatal(err) } @@ -103,9 +110,15 @@ func TestMain(m *testing.M) { os.Exit(run) } +func TestNewClient(t *testing.T) { + gcpClient, err := gcp.NewClient(context.Background(), option.WithHTTPClient(hc)) + assert.NilError(t, err) + assert.Assert(t, gcpClient != nil) +} + func TestBucketExists(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -114,9 +127,21 @@ func TestBucketExists(t *testing.T) { assert.Assert(t, exists) } +func TestBucketNotExists(t *testing.T) { + bucket := "notexistsbucket" + gcpClient := &gcp.GCPClient{ + Client: client, + StartRange: 0, + EndRange: -1, + } + exists, err := gcpClient.BucketExists(context.Background(), bucket) + assert.NilError(t, err) + assert.Assert(t, !exists) +} + func TestObjectAttributes(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -131,7 +156,7 @@ func TestObjectAttributes(t *testing.T) { func TestListObjects(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -151,7 +176,7 @@ func TestFGetObject(t *testing.T) { assert.NilError(t, err) defer os.RemoveAll(tempDir) gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -162,9 +187,41 @@ func TestFGetObject(t *testing.T) { } } +func TestFGetObjectNotExists(t *testing.T) { + object := "notexists.txt" + tempDir, err := os.MkdirTemp("", bucketName) + assert.NilError(t, err) + defer os.RemoveAll(tempDir) + gcpClient := &gcp.GCPClient{ + Client: client, + StartRange: 0, + EndRange: -1, + } + localPath := filepath.Join(tempDir, object) + err = gcpClient.FGetObject(context.Background(), bucketName, object, localPath) + if err != io.EOF { + assert.Error(t, err, "storage: object doesn't exist") + } +} + +func TestFGetObjectDirectoryIsFileName(t *testing.T) { + tempDir, err := os.MkdirTemp("", bucketName) + defer os.RemoveAll(tempDir) + assert.NilError(t, err) + gcpClient := &gcp.GCPClient{ + Client: client, + StartRange: 0, + EndRange: -1, + } + err = gcpClient.FGetObject(context.Background(), bucketName, objectName, tempDir) + if err != io.EOF { + assert.Error(t, err, "filename is a directory") + } +} + func TestSetRange(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: Client, + Client: client, StartRange: 0, EndRange: -1, } @@ -173,6 +230,45 @@ func TestSetRange(t *testing.T) { assert.Equal(t, gcpClient.EndRange, int64(5)) } +func TestValidateSecret(t *testing.T) { + t.Parallel() + testCases := []struct { + title string + secret map[string][]byte + name string + error bool + }{ + { + "Test Case 1", + map[string][]byte{ + "serviceaccount": []byte("serviceaccount"), + }, + "Service Account", + false, + }, + { + "Test Case 2", + map[string][]byte{ + "data": []byte("data"), + }, + "Service Account", + true, + }, + } + for _, testCase := range testCases { + testCase := testCase + t.Run(testCase.title, func(t *testing.T) { + t.Parallel() + err := gcp.ValidateSecret(testCase.secret, testCase.name) + if testCase.error { + assert.Error(t, err, fmt.Sprintf("invalid '%v' secret data: required fields 'serviceaccount'", testCase.name)) + } else { + assert.NilError(t, err) + } + }) + } +} + func newTestServer(handler func(w http.ResponseWriter, r *http.Request)) (*http.Client, func()) { ts := httptest.NewTLSServer(http.HandlerFunc(handler)) tlsConf := &tls.Config{InsecureSkipVerify: true} From 57ef719f74adf9668bc2ddd6a58df7c39d287458 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 16 Sep 2021 12:15:26 -0500 Subject: [PATCH 24/53] Updated docs to include GCP provider instructions Signed-off-by: pa250194 --- docs/spec/v1alpha1/buckets.md | 5 +-- docs/spec/v1beta1/buckets.md | 82 ++++++++++++++++++++++++++++++++++- 2 files changed, 82 insertions(+), 5 deletions(-) diff --git a/docs/spec/v1alpha1/buckets.md b/docs/spec/v1alpha1/buckets.md index 53838e25..bb2c07a9 100644 --- a/docs/spec/v1alpha1/buckets.md +++ b/docs/spec/v1alpha1/buckets.md @@ -11,7 +11,7 @@ Bucket: // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws;gcp + // +kubebuilder:validation:Enum=generic;aws // +optional Provider string `json:"provider,omitempty"` @@ -57,7 +57,6 @@ Supported providers: const ( GenericBucketProvider string = "generic" AmazonBucketProvider string = "aws" - GoogleBucketProvider string = "gcp" ) ``` @@ -232,4 +231,4 @@ Wait for ready condition: ```bash kubectl -n gitios-system wait bucket/podinfo --for=condition=ready --timeout=1m -``` +``` \ No newline at end of file diff --git a/docs/spec/v1beta1/buckets.md b/docs/spec/v1beta1/buckets.md index 1bcae604..90a23340 100644 --- a/docs/spec/v1beta1/buckets.md +++ b/docs/spec/v1beta1/buckets.md @@ -11,7 +11,7 @@ Bucket: // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws + // +kubebuilder:validation:Enum=generic;aws;gcp // +optional Provider string `json:"provider,omitempty"` @@ -62,6 +62,7 @@ Supported providers: const ( GenericBucketProvider string = "generic" AmazonBucketProvider string = "aws" + GoogleBucketProvider string = "gcp" ) ``` @@ -182,7 +183,8 @@ data: secretkey: ``` -> **Note:** that for Google Cloud Storage you have to enable +> **Note:** that when using the generic provider +> for Google Cloud Storage you have to enable > S3 compatible access in your GCP project. ### AWS IAM authentication @@ -230,6 +232,82 @@ spec: } ``` +### GCP Provider + +When the provider is `gcp` and the `secretRef` is not specified, +the GCP client authenticates using workload identity. +The GCP client automatically handles authentication in two ways. +The first way being that the GCP client library will automatically +check for the presence of the GOOGLE_APPLICATION_CREDENTIAL +environment variable. If this is not found, the GCP client library +will search for the Google Application Credential file in the config directory: + +```yaml +apiVersion: source.toolkit.fluccd.io/v1beta1 +kind: Bucket +metadata: + name: podinfo + namespace: gitops-system +spec: + interval: 5m + provider: gcp + bucketName: podinfo + endpoint: storage.googleapis.com + region: us-east-1 + timeout: 30s +``` + +When the provider is `gcp` and the `secretRef` is specified, +the GCP client authenticates using a Kubernetes secret named serviceaccount +which is a base 64 encoded string of the GCP service account JSON file: + +```yaml +apiVersion: source.toolkit.fluccd.io/v1beta1 +kind: Bucket +metadata: + name: podinfo + namespace: gitops-system +spec: + interval: 5m + provider: gcp + bucketName: podinfo + endpoint: storage.googleapis.com + region: us-east-1 + timeout: 30s + secretRef: + name: gcp-service-account +--- +apiVersion: v1 +kind: Secret +metadata: + name: gcp-service-account + namespace: gitops-system +type: Opaque +data: + serviceaccount: "ewogICAgInR5cGUiOiAic2VydmljZV9hY2NvdW50IiwKICAgICJwcm9qZWN0X2lkIjogInBvZGluZm8iLAogICAgInByaXZhdGVfa2V5X2lkIjogIjI4cXdnaDNnZGY1aGozZ2I1ZmozZ3N1NXlmZ2gzNGY0NTMyNDU2OGh5MiIsCiAgICAicHJpdmF0ZV9rZXkiOiAiLS0tLS1CRUdJTiBQUklWQVRFIEtFWS0tLS0tXG5Id2V0aGd5MTIzaHVnZ2hoaGJkY3U2MzU2ZGd5amhzdmd2R0ZESFlnY2RqYnZjZGhic3g2M2Ncbjc2dGd5Y2ZlaHVoVkdURllmdzZ0N3lkZ3lWZ3lkaGV5aHVnZ3ljdWhland5NnQzNWZ0aHl1aGVndmNldGZcblRGVUhHVHlnZ2h1Ymh4ZTY1eWd0NnRneWVkZ3kzMjZodWN5dnN1aGJoY3Zjc2poY3NqaGNzdmdkdEhGQ0dpXG5IY3llNnR5eWczZ2Z5dWhjaGNzYmh5Z2NpamRiaHl5VEY2NnR1aGNldnVoZGNiaHVoaHZmdGN1aGJoM3VoN3Q2eVxuZ2d2ZnRVSGJoNnQ1cmZ0aGh1R1ZSdGZqaGJmY3JkNXI2N3l1aHV2Z0ZUWWpndnRmeWdoYmZjZHJoeWpoYmZjdGZkZnlodmZnXG50Z3ZnZ3RmeWdodmZ0NnR1Z3ZURjVyNjZ0dWpoZ3ZmcnR5aGhnZmN0Nnk3eXRmcjVjdHZnaGJoaHZ0Z2hoanZjdHRmeWNmXG5mZnhmZ2hqYnZnY2d5dDY3dWpiZ3ZjdGZ5aFZDN3VodmdjeWp2aGhqdnl1amNcbmNnZ2hndmdjZmhnZzc2NTQ1NHRjZnRoaGdmdHloaHZ2eXZ2ZmZnZnJ5eXU3N3JlcmVkc3dmdGhoZ2ZjZnR5Y2ZkcnR0ZmhmL1xuLS0tLS1FTkQgUFJJVkFURSBLRVktLS0tLVxuIiwKICAgICJjbGllbnRfZW1haWwiOiAidGVzdEBwb2RpbmZvLmlhbS5nc2VydmljZWFjY291bnQuY29tIiwKICAgICJjbGllbnRfaWQiOiAiMzI2NTc2MzQ2Nzg3NjI1MzY3NDYiLAogICAgImF1dGhfdXJpIjogImh0dHBzOi8vYWNjb3VudHMuZ29vZ2xlLmNvbS9vL29hdXRoMi9hdXRoIiwKICAgICJ0b2tlbl91cmkiOiAiaHR0cHM6Ly9vYXV0aDIuZ29vZ2xlYXBpcy5jb20vdG9rZW4iLAogICAgImF1dGhfcHJvdmlkZXJfeDUwOV9jZXJ0X3VybCI6ICJodHRwczovL3d3dy5nb29nbGVhcGlzLmNvbS9vYXV0aDIvdjEvY2VydHMiLAogICAgImNsaWVudF94NTA5X2NlcnRfdXJsIjogImh0dHBzOi8vd3d3Lmdvb2dsZWFwaXMuY29tL3JvYm90L3YxL21ldGFkYXRhL3g1MDkvdGVzdCU0MHBvZGluZm8uaWFtLmdzZXJ2aWNlYWNjb3VudC5jb20iCn0=" +``` + +> **Note:** the serviceaccount secret is a base 64 encoded form of +> the GCP service account json file like so + +```json + { + "type": "service_account", + "project_id": "podinfo", + "private_key_id": "28qwgh3gdf5hj3gb5fj3gsu5yfgh34f45324568hy2", + "private_key": "-----BEGIN PRIVATE KEY-----\nHwethgy123hugghhhbdcu6356dgyjhsvgvGFDHYgcdjbvcdhbsx63c\n76tgycfehuhVGTFYfw6t7ydgyVgydheyhuggycuhejwy6t35fthyuhegvcetf\nTFUHGTygghubhxe65ygt6tgyedgy326hucyvsuhbhcvcsjhcsjhcsvgdtHFCGi\nHcye6tyyg3gfyuhchcsbhygcijdbhyyTF66tuhcevuhdcbhuhhvftcuhbh3uh7t6y\nggvftUHbh6t5rfthhuGVRtfjhbfcrd5r67yuhuvgFTYjgvtfyghbfcdrhyjhbfctfdfyhvfg\ntgvggtfyghvft6tugvTF5r66tujhgvfrtyhhgfct6y7ytfr5ctvghbhhvtghhjvcttfycf\nffxfghjbvgcgyt67ujbgvctfyhVC7uhvgcyjvhhjvyujc\ncgghgvgcfhgg765454tcfthhgftyhhvvyvvffgfryyu77reredswfthhgfcftycfdrttfhf/\n-----END PRIVATE KEY-----\n", + "client_email": "test@podinfo.iam.gserviceaccount.com", + "client_id": "32657634678762536746", + "auth_uri": "https://accounts.google.com/o/oauth2/auth", + "token_uri": "https://oauth2.googleapis.com/token", + "auth_provider_x509_cert_url": "https://www.googleapis.com/oauth2/v1/certs", + "client_x509_cert_url": "https://www.googleapis.com/robot/v1/metadata/x509/test%40podinfo.iam.gserviceaccount.com" + } +``` +> **Note:** that when using the gcp provider for +> Google Cloud Storage you do not have to enable +> S3 compatible access in your GCP project. + ## Status examples Successful download: From 02102de2c78a762effbed73216c1ac0f16b7a3e1 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 23 Sep 2021 12:38:38 -0500 Subject: [PATCH 25/53] Removed resumable downloads Signed-off-by: pa250194 --- pkg/gcp/gcp.go | 74 +++++++++------------------------------------ pkg/gcp/gcp_test.go | 44 ++++++--------------------- 2 files changed, 23 insertions(+), 95 deletions(-) diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index b2274e34..c869419b 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -45,12 +45,6 @@ type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. *gcpStorage.Client - // startRange is the starting read value for - // reading the object from bucket. - StartRange int64 - // endRange is the ending read value for - // reading the object from bucket. - EndRange int64 } // NewClient creates a new GCP storage client @@ -63,7 +57,7 @@ func NewClient(ctx context.Context, opts ...option.ClientOption) (*GCPClient, er return nil, err } - return &GCPClient{Client: client, StartRange: 0, EndRange: -1}, nil + return &GCPClient{Client: client}, nil } // ValidateSecret validates the credential secrets @@ -76,18 +70,11 @@ func ValidateSecret(secret map[string][]byte, name string) error { return nil } -// SetRange sets the startRange and endRange used to read the Object from -// the bucket. It is a helper method for resumable downloads. -func (c *GCPClient) SetRange(start, end int64) { - c.StartRange = start - c.EndRange = end -} - // BucketExists checks if the bucket with the provided name exists. func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, error) { _, err := c.Client.Bucket(bucketName).Attrs(ctx) if err == gcpStorage.ErrBucketNotExist { - return false, nil + return false, err } if err != nil { return false, err @@ -97,20 +84,19 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, // ObjectAttributes checks if the object with the provided name exists. // If it exists the Object attributes are returned. -func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, *gcpStorage.ObjectAttrs, error) { - attrs, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) +func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, error) { + _, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) // ErrObjectNotExist is returned if the object does not exist if err == gcpStorage.ErrObjectNotExist { - return false, nil, err + return false, err } if err != nil { - return false, nil, err + return false, err } - return true, attrs, nil + return true, nil } // FGetObject gets the object from the bucket and downloads the object locally -// A part file is created so the download can be resumable. func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, localPath string) error { // Verify if destination already exists. dirStatus, err := os.Stat(localPath) @@ -140,7 +126,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca // ObjectExists verifies if object exists and you have permission to access. // Check if the object exists and if you have permission to access it // The Object attributes are returned if the Object exists. - exists, attrs, err := c.ObjectAttributes(ctx, bucketName, objectName) + exists, err := c.ObjectAttributes(ctx, bucketName, objectName) if err != nil { return err } @@ -148,57 +134,25 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca return ErrorObjectDoesNotExist } - // Write to a temporary file "filename.part.gcp" before saving. - filePartPath := localPath + ".part.gcp" - // If exists, open in append mode. If not create it as a part file. - filePart, err := os.OpenFile(filePartPath, os.O_CREATE|os.O_APPEND|os.O_WRONLY, 0600) + objectFile, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY, 0600) if err != nil { return err } - // If we return early with an error, be sure to close and delete - // filePart. If we have an error along the way there is a chance - // that filePart is somehow damaged, and we should discard it. - closeAndRemove := true - defer func() { - if closeAndRemove { - _ = filePart.Close() - _ = os.Remove(filePartPath) - } - }() - - // Issue Stat to get the current offset. - partFileStat, err := filePart.Stat() - if err != nil { - return err - } - - // Set the File size request range - // If the part file exists - if partFileStat.Size() > 0 { - c.SetRange(partFileStat.Size(), 0) - } - // Get Object from GCP Bucket - objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewRangeReader(ctx, c.StartRange, c.EndRange) + objectReader, err := c.Client.Bucket(bucketName).Object(objectName).NewReader(ctx) if err != nil { return err } defer objectReader.Close() - // Write to the part file. - if _, err := io.CopyN(filePart, objectReader, attrs.Size); err != nil { + // Write Object to file. + if _, err := io.Copy(objectFile, objectReader); err != nil { return err } - // Close the file before rename, this is specifically needed for Windows users. - closeAndRemove = false - if err := filePart.Close(); err != nil { - return err - } - - // Safely completed. Now commit by renaming to actual filename. - if err := os.Rename(filePartPath, localPath); err != nil { + // Close the file. + if err := objectFile.Close(); err != nil { return err } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 5c0292e9..76c455e5 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -118,9 +118,7 @@ func TestNewClient(t *testing.T) { func TestBucketExists(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } exists, err := gcpClient.BucketExists(context.Background(), bucketName) assert.NilError(t, err) @@ -130,35 +128,28 @@ func TestBucketExists(t *testing.T) { func TestBucketNotExists(t *testing.T) { bucket := "notexistsbucket" gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } exists, err := gcpClient.BucketExists(context.Background(), bucket) - assert.NilError(t, err) + assert.Error(t, err, "storage: bucket doesn't exist") assert.Assert(t, !exists) } func TestObjectAttributes(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } - exists, objectAttrs, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) + exists, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) if err == gcpStorage.ErrObjectNotExist { assert.NilError(t, err) } assert.NilError(t, err) assert.Assert(t, exists) - assert.Assert(t, objectAttrs != nil) } func TestListObjects(t *testing.T) { gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } objectInterator := gcpClient.ListObjects(context.Background(), bucketName, nil) for { @@ -176,9 +167,7 @@ func TestFGetObject(t *testing.T) { assert.NilError(t, err) defer os.RemoveAll(tempDir) gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } localPath := filepath.Join(tempDir, objectName) err = gcpClient.FGetObject(context.Background(), bucketName, objectName, localPath) @@ -193,9 +182,7 @@ func TestFGetObjectNotExists(t *testing.T) { assert.NilError(t, err) defer os.RemoveAll(tempDir) gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } localPath := filepath.Join(tempDir, object) err = gcpClient.FGetObject(context.Background(), bucketName, object, localPath) @@ -209,9 +196,7 @@ func TestFGetObjectDirectoryIsFileName(t *testing.T) { defer os.RemoveAll(tempDir) assert.NilError(t, err) gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, + Client: client, } err = gcpClient.FGetObject(context.Background(), bucketName, objectName, tempDir) if err != io.EOF { @@ -219,17 +204,6 @@ func TestFGetObjectDirectoryIsFileName(t *testing.T) { } } -func TestSetRange(t *testing.T) { - gcpClient := &gcp.GCPClient{ - Client: client, - StartRange: 0, - EndRange: -1, - } - gcpClient.SetRange(2, 5) - assert.Equal(t, gcpClient.StartRange, int64(2)) - assert.Equal(t, gcpClient.EndRange, int64(5)) -} - func TestValidateSecret(t *testing.T) { t.Parallel() testCases := []struct { From 751243ce50a85911b631fce712640a80541dbcaa Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 23 Sep 2021 13:42:21 -0500 Subject: [PATCH 26/53] Refactor comments and method names Signed-off-by: pa250194 --- pkg/gcp/gcp.go | 10 ++++------ pkg/gcp/gcp_test.go | 16 +++++++++++++--- 2 files changed, 17 insertions(+), 9 deletions(-) diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index c869419b..470fe227 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -82,9 +82,8 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, return true, nil } -// ObjectAttributes checks if the object with the provided name exists. -// If it exists the Object attributes are returned. -func (c *GCPClient) ObjectAttributes(ctx context.Context, bucketName, objectName string) (bool, error) { +// ObjectExists checks if the object with the provided name exists. +func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, error) { _, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) // ErrObjectNotExist is returned if the object does not exist if err == gcpStorage.ErrObjectNotExist { @@ -124,9 +123,8 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca } // ObjectExists verifies if object exists and you have permission to access. - // Check if the object exists and if you have permission to access it - // The Object attributes are returned if the Object exists. - exists, err := c.ObjectAttributes(ctx, bucketName, objectName) + // Check if the object exists and if you have permission to access it. + exists, err := c.ObjectExists(ctx, bucketName, objectName) if err != nil { return err } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 76c455e5..8faa5e2c 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -131,15 +131,15 @@ func TestBucketNotExists(t *testing.T) { Client: client, } exists, err := gcpClient.BucketExists(context.Background(), bucket) - assert.Error(t, err, "storage: bucket doesn't exist") + assert.Error(t, err, gcpStorage.ErrBucketNotExist.Error()) assert.Assert(t, !exists) } -func TestObjectAttributes(t *testing.T) { +func TestObjectExists(t *testing.T) { gcpClient := &gcp.GCPClient{ Client: client, } - exists, err := gcpClient.ObjectAttributes(context.Background(), bucketName, objectName) + exists, err := gcpClient.ObjectExists(context.Background(), bucketName, objectName) if err == gcpStorage.ErrObjectNotExist { assert.NilError(t, err) } @@ -147,6 +147,16 @@ func TestObjectAttributes(t *testing.T) { assert.Assert(t, exists) } +func TestObjectNotExists(t *testing.T) { + object := "doesnotexists.yaml" + gcpClient := &gcp.GCPClient{ + Client: client, + } + exists, err := gcpClient.ObjectExists(context.Background(), bucketName, object) + assert.Error(t, err, gcpStorage.ErrObjectNotExist.Error()) + assert.Assert(t, !exists) +} + func TestListObjects(t *testing.T) { gcpClient := &gcp.GCPClient{ Client: client, From 116906cca408acc7e2032c8dd8aa7dc3e8afc0d2 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Tue, 12 Oct 2021 11:46:48 -0500 Subject: [PATCH 27/53] Fixed spelling and capitalization Signed-off-by: pa250194 --- pkg/gcp/gcp.go | 22 ++++++++++------------ pkg/gcp/gcp_test.go | 18 +++++++++--------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 470fe227..38a0b99f 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -24,15 +24,15 @@ import ( "os" "path/filepath" - gcpStorage "cloud.google.com/go/storage" - interator "google.golang.org/api/iterator" + gcpstorage "cloud.google.com/go/storage" + "google.golang.org/api/iterator" "google.golang.org/api/option" ) var ( // IteratorDone is returned when the looping of objects/content // has reached the end of the iteration. - IteratorDone = interator.Done + IteratorDone = iterator.Done // ErrorDirectoryExists is an error returned when the filename provided // is a directory. ErrorDirectoryExists = errors.New("filename is a directory") @@ -44,15 +44,13 @@ var ( type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. - *gcpStorage.Client + *gcpstorage.Client } -// NewClient creates a new GCP storage client -// The Google Storage Client will automatically -// look for the Google Application Credential environment variable -// or look for the Google Application Credential file. +// NewClient creates a new GCP storage client. The Client will automatically look for the Google Application +// Credential environment variable or look for the Google Application Credential file. func NewClient(ctx context.Context, opts ...option.ClientOption) (*GCPClient, error) { - client, err := gcpStorage.NewClient(ctx, opts...) + client, err := gcpstorage.NewClient(ctx, opts...) if err != nil { return nil, err } @@ -73,7 +71,7 @@ func ValidateSecret(secret map[string][]byte, name string) error { // BucketExists checks if the bucket with the provided name exists. func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, error) { _, err := c.Client.Bucket(bucketName).Attrs(ctx) - if err == gcpStorage.ErrBucketNotExist { + if err == gcpstorage.ErrBucketNotExist { return false, err } if err != nil { @@ -86,7 +84,7 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, error) { _, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) // ErrObjectNotExist is returned if the object does not exist - if err == gcpStorage.ErrObjectNotExist { + if err == gcpstorage.ErrObjectNotExist { return false, err } if err != nil { @@ -160,7 +158,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca // ListObjects lists the objects/contents of the bucket whose bucket name is provided. // the objects are returned as an Objectiterator and .Next() has to be called on them // to loop through the Objects. -func (c *GCPClient) ListObjects(ctx context.Context, bucketName string, query *gcpStorage.Query) *gcpStorage.ObjectIterator { +func (c *GCPClient) ListObjects(ctx context.Context, bucketName string, query *gcpstorage.Query) *gcpstorage.ObjectIterator { items := c.Client.Bucket(bucketName).Objects(ctx, query) return items } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 8faa5e2c..99d72309 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -32,7 +32,7 @@ import ( "testing" "time" - gcpStorage "cloud.google.com/go/storage" + gcpstorage "cloud.google.com/go/storage" "github.com/fluxcd/source-controller/pkg/gcp" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" @@ -48,7 +48,7 @@ const ( var ( hc *http.Client - client *gcpStorage.Client + client *gcpstorage.Client close func() err error ) @@ -101,7 +101,7 @@ func TestMain(m *testing.M) { } }) ctx := context.Background() - client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) + client, err = gcpstorage.NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { log.Fatal(err) } @@ -131,7 +131,7 @@ func TestBucketNotExists(t *testing.T) { Client: client, } exists, err := gcpClient.BucketExists(context.Background(), bucket) - assert.Error(t, err, gcpStorage.ErrBucketNotExist.Error()) + assert.Error(t, err, gcpstorage.ErrBucketNotExist.Error()) assert.Assert(t, !exists) } @@ -140,7 +140,7 @@ func TestObjectExists(t *testing.T) { Client: client, } exists, err := gcpClient.ObjectExists(context.Background(), bucketName, objectName) - if err == gcpStorage.ErrObjectNotExist { + if err == gcpstorage.ErrObjectNotExist { assert.NilError(t, err) } assert.NilError(t, err) @@ -153,7 +153,7 @@ func TestObjectNotExists(t *testing.T) { Client: client, } exists, err := gcpClient.ObjectExists(context.Background(), bucketName, object) - assert.Error(t, err, gcpStorage.ErrObjectNotExist.Error()) + assert.Error(t, err, gcpstorage.ErrObjectNotExist.Error()) assert.Assert(t, !exists) } @@ -161,15 +161,15 @@ func TestListObjects(t *testing.T) { gcpClient := &gcp.GCPClient{ Client: client, } - objectInterator := gcpClient.ListObjects(context.Background(), bucketName, nil) + objectIterator := gcpClient.ListObjects(context.Background(), bucketName, nil) for { - _, err := objectInterator.Next() + _, err := objectIterator.Next() if err == gcp.IteratorDone { break } assert.NilError(t, err) } - assert.Assert(t, objectInterator != nil) + assert.Assert(t, objectIterator != nil) } func TestFGetObject(t *testing.T) { From f62571bcecf97a678c53625719fac58cd4efa759 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 14 Oct 2021 09:39:53 -0500 Subject: [PATCH 28/53] Added log for GCP provider auth error Signed-off-by: pa250194 --- controllers/bucket_controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index aa11c261..ddb1ea94 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -268,9 +268,11 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1. // reconcileWithGCP handles getting objects from a Google Cloud Platform bucket // using a gcp client func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) { + log := logr.FromContext(ctx) gcpClient, err := r.authGCP(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) + log.Error(err, "GCP Provider") return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err } defer gcpClient.Client.Close() From f797fbfdf080e9c1b06a8e9e62bc487f85ae218c Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 14 Oct 2021 09:51:49 -0500 Subject: [PATCH 29/53] Added Logger to closing GCP client Signed-off-by: pa250194 --- controllers/bucket_controller.go | 3 +-- pkg/gcp/gcp.go | 8 ++++++++ 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index ddb1ea94..002c95c6 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -272,10 +272,9 @@ func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1 gcpClient, err := r.authGCP(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) - log.Error(err, "GCP Provider") return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err } - defer gcpClient.Client.Close() + defer gcpClient.Close(log) ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) defer cancel() diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 38a0b99f..9127fcde 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -25,6 +25,7 @@ import ( "path/filepath" gcpstorage "cloud.google.com/go/storage" + "github.com/go-logr/logr" "google.golang.org/api/iterator" "google.golang.org/api/option" ) @@ -162,3 +163,10 @@ func (c *GCPClient) ListObjects(ctx context.Context, bucketName string, query *g items := c.Client.Bucket(bucketName).Objects(ctx, query) return items } + +// Close closes the GCP Client and logs any useful errors +func (c *GCPClient) Close(log logr.Logger) { + if err := c.Client.Close(); err != nil { + log.Error(err, "GCP Provider") + } +} From 869c7960e3dda63adf1d4aefce241b25748132d7 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 10 Sep 2021 12:39:40 +0200 Subject: [PATCH 30/53] Update github.com/libgit2/git2go to v31.6.1 This commit updates `github.com/libgit2/git2go` to `v31.6.1` (with `libgit2` `1.1.1`), and changes the container image build process so that it makes use of `ghcr.io/hiddeco/golang-with-libgit2`. This image provides a pre-build dynamic `libgit2` dependency linked against OpenSSL and LibSSH2 (without gcrypt), and a set of cross-compile build tools (see [rationale](https://github.com/hiddeco/golang-with-libgit2#rationale) and [usage](https://github.co/hiddeco/golang-with-libgit2#usage) for more detailed information). The linked set of dependency should solve most known issues around unsupport private key types, but does not resolve the issues with ECDSA* and ED25519 hostkeys yet. Solving this requires a newer version of `libgit2` (`>=1.2.0`), which currently does not seem to work properly with `git2go/v32`. Some small changes have been made to the `libgit2` package to address (future) deprecations. Signed-off-by: Hidde Beydals --- .dockerignore | 1 + .github/actions/run-tests/Dockerfile | 15 +-- .gitignore | 3 + Dockerfile | 78 ++++++-------- Makefile | 153 ++++++++++++++++----------- go.mod | 2 +- go.sum | 4 +- internal/fs/rename.go | 1 + internal/fs/rename_windows.go | 1 + pkg/git/libgit2/checkout.go | 2 +- pkg/git/libgit2/checkout_test.go | 7 +- pkg/git/libgit2/transport.go | 22 ++-- 12 files changed, 151 insertions(+), 138 deletions(-) create mode 100644 .dockerignore diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 00000000..1e2f1e16 --- /dev/null +++ b/.dockerignore @@ -0,0 +1 @@ +hack/libgit2/ diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index 1e891956..93c9f549 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,17 +1,4 @@ -FROM golang:1.16-buster as builder - -# Up-to-date libgit2 dependencies are only available in -# unstable, as libssh2 in testing/bullseye has been linked -# against gcrypt which causes issues with PKCS* formats. -# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271 -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list -RUN set -eux; \ - apt-get update \ - && apt-get install -y libgit2-dev/unstable \ - && apt-get clean \ - && apt-get autoremove --purge -y \ - && rm -rf /var/lib/apt/lists/* +FROM ghcr.io/hiddeco/golang-with-libgit2:dev as builder # Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ diff --git a/.gitignore b/.gitignore index 8f19ec80..b9375325 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,6 @@ # vendor/ bin/ config/release/ + +# Exclude all libgit2 related files +hack/libgit2/ diff --git a/Dockerfile b/Dockerfile index 059a25bb..083fb1e6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,69 +1,57 @@ -FROM golang:1.16-buster as builder - -# Up-to-date libgit2 dependencies are only available in -# unstable, as libssh2 in testing/bullseye has been linked -# against gcrypt which causes issues with PKCS* formats. -# Explicitly listing all build dependencies is required because -# they can only be automagically found for AMD64 builds. -# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271 -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list -RUN set -eux; \ - apt-get update \ - && apt-get install -y \ - libgit2-dev/unstable \ - zlib1g-dev/unstable \ - libssh2-1-dev/unstable \ - libpcre3-dev/unstable \ - && apt-get clean \ - && apt-get autoremove --purge -y \ - && rm -rf /var/lib/apt/lists/* +ARG BASE_IMG=ghcr.io/hiddeco/golang-with-libgit2 +ARG BASE_TAG=dev +FROM ${BASE_IMG}:${BASE_TAG} AS build +# Configure workspace WORKDIR /workspace -# copy api submodule +# Copy api submodule COPY api/ api/ -# copy modules manifests +# Copy modules manifests COPY go.mod go.mod COPY go.sum go.sum -# cache modules +# Cache modules RUN go mod download -# copy source code +# Copy source code COPY main.go main.go COPY controllers/ controllers/ COPY pkg/ pkg/ COPY internal/ internal/ -# build without specifing the arch -RUN CGO_ENABLED=1 go build -o source-controller main.go +# Build the binary +ENV CGO_ENABLED=1 +ARG TARGETPLATFORM +RUN xx-go build -o source-controller -trimpath \ + main.go -FROM debian:buster-slim as controller +FROM debian:bullseye-slim as controller -# link repo to the GitHub Container Registry image +# Link repo to the GitHub Container Registry image LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller" -# Up-to-date libgit2 dependencies are only available in -# unstable, as libssh2 in testing/bullseye has been linked -# against gcrypt which causes issues with PKCS* formats. -# Ref: https://bugs.debian.org/cgi-bin/bugreport.cgi?bug=668271 -RUN echo "deb http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list \ - && echo "deb-src http://deb.debian.org/debian unstable main" >> /etc/apt/sources.list -RUN set -eux; \ - apt-get update \ - && apt-get install -y \ - ca-certificates \ - libgit2-1.1 \ - && apt-get clean \ - && apt-get autoremove --purge -y \ - && rm -rf /var/lib/apt/lists/* - -COPY --from=builder /workspace/source-controller /usr/local/bin/ - +# Configure user RUN groupadd controller && \ useradd --gid controller --shell /bin/sh --create-home controller +# Copy libgit2 +COPY --from=build /libgit2/lib/* /usr/local/lib/ +RUN ldconfig + +# Upgrade packages and install runtime dependencies +RUN echo "deb http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \ + && echo "deb-src http://deb.debian.org/debian sid main" >> /etc/apt/sources.list \ + && apt update \ + && apt install --no-install-recommends -y zlib1g/sid libssl1.1/sid libssh2-1/sid \ + && apt install --no-install-recommends -y ca-certificates \ + && apt clean \ + && apt autoremove --purge -y \ + && rm -rf /var/lib/apt/lists/* + +# Copy over binary from build +COPY --from=build /workspace/source-controller /usr/local/bin/ + USER controller ENTRYPOINT [ "source-controller" ] diff --git a/Makefile b/Makefile index 8f1ecc80..167c4804 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,32 @@ # Image URL to use all building/pushing image targets IMG ?= fluxcd/source-controller:latest + +# Base image used to build the Go binary +BASE_IMG ?= ghcr.io/hiddeco/golang-with-libgit2 +BASE_TAG ?= dev + # Produce CRDs that work back to Kubernetes 1.16 CRD_OPTIONS ?= crd:crdVersions=v1 -ENVTEST_BIN_VERSION?=1.19.2 -KUBEBUILDER_ASSETS?=$(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path) +# Repository root based on Git metadata +REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) + +# Dependency versions +LIBGIT2_VERSION ?= 1.1.1 +ENVTEST_BIN_VERSION ?= 1.19.2 +KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path) + +# libgit2 related magical paths +# These are used to determine if the target libgit2 version is already available on +# the system, or where they should be installed to +SYSTEM_LIBGIT2_VERSION := $(shell pkg-config --modversion libgit2 2>/dev/null) +LIBGIT2_PATH := $(REPOSITORY_ROOT)/hack/libgit2 +LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib +LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.so.$(LIBGIT2_VERSION) + +# API (doc) generation utilities +CONTROLLER_GEN_VERSION ?= v0.5.0 +GEN_API_REF_DOCS_VERSION ?= 0.3.0 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) @@ -13,121 +35,130 @@ else GOBIN=$(shell go env GOBIN) endif -all: manager +all: build -# Run tests -test: generate fmt vet manifests api-docs setup-envtest - KUBEBUILDER_ASSETS=$(KUBEBUILDER_ASSETS) go test ./... -coverprofile cover.out - cd api; go test ./... -coverprofile cover.out - -# Build manager binary -manager: generate fmt vet +build: $(LIBGIT2) ## Build manager binary + PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig/ \ go build -o bin/manager main.go -# Run against the configured Kubernetes cluster in ~/.kube/config -run: generate fmt vet manifests +test: $(LIBGIT2) test-api ## Run tests + LD_LIBRARY_PATH=$(LIBGIT2_LIB_PATH) \ + PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig/ \ + go test ./... -coverprofile cover.out + +test-api: ## Run api tests + cd api; go test ./... -coverprofile cover.out + +run: $(LIBGIT2) generate fmt vet manifests ## Run against the configured Kubernetes cluster in ~/.kube/config + LD_LIBRARY_PATH=$(LIBGIT2_LIB_PATH) \ go run ./main.go -# Install CRDs into a cluster -install: manifests +install: manifests ## Install CRDs into a cluster kustomize build config/crd | kubectl apply -f - -# Uninstall CRDs from a cluster -uninstall: manifests +uninstall: manifests ## Uninstall CRDs from a cluster kustomize build config/crd | kubectl delete -f - -# Deploy controller in the configured Kubernetes cluster in ~/.kube/config -deploy: manifests +deploy: manifests ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config cd config/manager && kustomize edit set image fluxcd/source-controller=${IMG} kustomize build config/default | kubectl apply -f - -# Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config -dev-deploy: +dev-deploy: ## Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config mkdir -p config/dev && cp config/default/* config/dev cd config/dev && kustomize edit set image fluxcd/source-controller=${IMG} kustomize build config/dev | kubectl apply -f - rm -rf config/dev -# Generate manifests e.g. CRD, RBAC etc. -manifests: controller-gen +manifests: controller-gen ## Generate manifests, e.g. CRD, RBAC, etc. $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="config/crd/bases" cd api; $(CONTROLLER_GEN) $(CRD_OPTIONS) rbac:roleName=manager-role paths="./..." output:crd:artifacts:config="../config/crd/bases" -# Generate API reference documentation -api-docs: gen-crd-api-reference-docs +api-docs: gen-crd-api-reference-docs ## Generate API reference documentation $(API_REF_GEN) -api-dir=./api/v1beta1 -config=./hack/api-docs/config.json -template-dir=./hack/api-docs/template -out-file=./docs/api/source.md -# Run go mod tidy -tidy: +tidy: ## Run go mod tidy go mod tidy cd api; go mod tidy -# Run go fmt against code -fmt: +fmt: ## Run go fmt against code go fmt ./... cd api; go fmt ./... -# Run go vet against code -vet: +vet: ## Run go vet against code + PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig \ go vet ./... cd api; go vet ./... -# Generate code -generate: controller-gen +generate: controller-gen ## Generate API code cd api; $(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..." -# Build the docker image -docker-build: - docker build . -t ${IMG} +docker-build: ## Build the docker image + docker build \ + --build-arg BASE_IMG=$(BASE_IMG) \ + --build-arg BASE_TAG=$(BASE_TAG) \ + -t ${IMG} . -# Push the docker image -docker-push: +docker-push: ## Push docker image docker push ${IMG} -# Find or download controller-gen -controller-gen: +controller-gen: ## Find or download controller-gen ifeq (, $(shell which controller-gen)) @{ \ - set -e ;\ - CONTROLLER_GEN_TMP_DIR=$$(mktemp -d) ;\ - cd $$CONTROLLER_GEN_TMP_DIR ;\ - go mod init tmp ;\ - go get sigs.k8s.io/controller-tools/cmd/controller-gen@v0.5.0 ;\ - rm -rf $$CONTROLLER_GEN_TMP_DIR ;\ + set -e; \ + CONTROLLER_GEN_TMP_DIR=$$(mktemp -d); \ + cd $$CONTROLLER_GEN_TMP_DIR; \ + go mod init tmp; \ + go get sigs.k8s.io/controller-tools/cmd/controller-gen@$(CONTROLLER_GEN_VERSION); \ + rm -rf $$CONTROLLER_GEN_TMP_DIR; \ } CONTROLLER_GEN=$(GOBIN)/controller-gen else CONTROLLER_GEN=$(shell which controller-gen) endif -# Find or download gen-crd-api-reference-docs -gen-crd-api-reference-docs: +gen-crd-api-reference-docs: ## Find or download gen-crd-api-reference-docs ifeq (, $(shell which gen-crd-api-reference-docs)) @{ \ - set -e ;\ - API_REF_GEN_TMP_DIR=$$(mktemp -d) ;\ - cd $$API_REF_GEN_TMP_DIR ;\ - go mod init tmp ;\ - go get github.com/ahmetb/gen-crd-api-reference-docs@v0.3.0 ;\ - rm -rf $$API_REF_GEN_TMP_DIR ;\ + set -e; \ + API_REF_GEN_TMP_DIR=$$(mktemp -d); \ + cd $$API_REF_GEN_TMP_DIR; \ + go mod init tmp; \ + go get github.com/ahmetb/gen-crd-api-reference-docs@$(GEN_API_REF_DOCS_VERSION); \ + rm -rf $$API_REF_GEN_TMP_DIR; \ } API_REF_GEN=$(GOBIN)/gen-crd-api-reference-docs else API_REF_GEN=$(shell which gen-crd-api-reference-docs) endif -# Find or download setup-envtest -setup-envtest: +setup-envtest: ## Find or download setup-envtest ifeq (, $(shell which setup-envtest)) @{ \ - set -e ;\ - SETUP_ENVTEST_TMP_DIR=$$(mktemp -d) ;\ - cd $$SETUP_ENVTEST_TMP_DIR ;\ - go mod init tmp ;\ - go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest ;\ - rm -rf $$SETUP_ENVTEST_TMP_DIR ;\ + set -e; \ + SETUP_ENVTEST_TMP_DIR=$$(mktemp -d); \ + cd $$SETUP_ENVTEST_TMP_DIR; \ + go mod init tmp; \ + go get sigs.k8s.io/controller-runtime/tools/setup-envtest@latest; \ + rm -rf $$SETUP_ENVTEST_TMP_DIR; \ } SETUP_ENVTEST=$(GOBIN)/setup-envtest else SETUP_ENVTEST=$(shell which setup-envtest) endif + +libgit2: $(LIBGIT2) ## Detect or download libgit2 library + +$(LIBGIT2): +ifeq ($(LIBGIT2_VERSION),$(SYSTEM_LIBGIT2_VERSION)) +else + @{ \ + set -e; \ + mkdir -p $(LIBGIT2_PATH); \ + docker cp $(shell docker create --rm $(BASE_IMG):$(BASE_TAG)):/libgit2/Makefile $(LIBGIT2_PATH); \ + INSTALL_PREFIX=$(LIBGIT2_PATH) make -C $(LIBGIT2_PATH); \ + } +endif + +.PHONY: help +help: ## Display this help menu + @awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-20s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST) diff --git a/go.mod b/go.mod index 3ec4d3e8..26038e28 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/go-logr/logr v0.4.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/googleapis/gax-go/v2 v2.1.0 // indirect - github.com/libgit2/git2go/v31 v31.4.14 + github.com/libgit2/git2go/v31 v31.6.1 github.com/minio/minio-go/v7 v7.0.10 github.com/onsi/ginkgo v1.16.4 github.com/onsi/gomega v1.14.0 diff --git a/go.sum b/go.sum index a1a0d5dc..ce31fea3 100644 --- a/go.sum +++ b/go.sum @@ -608,8 +608,8 @@ github.com/lann/ps v0.0.0-20150810152359-62de8c46ede0/go.mod h1:vmVJ0l/dxyfGW6Fm github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.0 h1:Zx5DJFEYQXio93kgXnQ09fXNiUKsqv4OUEu2UtGcB1E= github.com/lib/pq v1.10.0/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= -github.com/libgit2/git2go/v31 v31.4.14 h1:6GOd3965D9e/+gjxCwZF4eQ+vB9kKB4yKFqdQr6XZ2E= -github.com/libgit2/git2go/v31 v31.4.14/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec= +github.com/libgit2/git2go/v31 v31.6.1 h1:FnKHHDDBgltSsu9RpKuL4rSR8dQ1JTf9dfvFhZ1y7Aw= +github.com/libgit2/git2go/v31 v31.6.1/go.mod h1:c/rkJcBcUFx6wHaT++UwNpKvIsmPNqCeQ/vzO4DrEec= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de h1:9TO3cAIGXtEhnIaL+V+BEER86oLrvS+kWobKpbJuye0= github.com/liggitt/tabwriter v0.0.0-20181228230101-89fcab3d43de/go.mod h1:zAbeS9B/r2mtpb6U+EI2rYA5OAXxsYw6wTamcNW+zcE= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= diff --git a/internal/fs/rename.go b/internal/fs/rename.go index a1b4a411..bad1f477 100644 --- a/internal/fs/rename.go +++ b/internal/fs/rename.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build !windows // +build !windows package fs diff --git a/internal/fs/rename_windows.go b/internal/fs/rename_windows.go index 3b565057..fa9a0b4d 100644 --- a/internal/fs/rename_windows.go +++ b/internal/fs/rename_windows.go @@ -2,6 +2,7 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. +//go:build windows // +build windows package fs diff --git a/pkg/git/libgit2/checkout.go b/pkg/git/libgit2/checkout.go index 01363f8f..74c976fa 100644 --- a/pkg/git/libgit2/checkout.go +++ b/pkg/git/libgit2/checkout.go @@ -156,7 +156,7 @@ func (c *CheckoutCommit) Checkout(ctx context.Context, path, url string, auth *g if err != nil { return nil, "", fmt.Errorf("git worktree error: %w", err) } - err = repo.CheckoutTree(tree, &git2go.CheckoutOpts{ + err = repo.CheckoutTree(tree, &git2go.CheckoutOptions{ Strategy: git2go.CheckoutForce, }) if err != nil { diff --git a/pkg/git/libgit2/checkout_test.go b/pkg/git/libgit2/checkout_test.go index 6de5484d..4b06f584 100644 --- a/pkg/git/libgit2/checkout_test.go +++ b/pkg/git/libgit2/checkout_test.go @@ -32,7 +32,7 @@ import ( func TestCheckoutTagSemVer_Checkout(t *testing.T) { certCallback := func(cert *git2go.Certificate, valid bool, hostname string) git2go.ErrorCode { - return 0 + return git2go.ErrorCodeOK } auth := &git.Auth{CertCallback: certCallback} @@ -57,9 +57,10 @@ func TestCheckoutTagSemVer_Checkout(t *testing.T) { if _, err := io.Copy(h, f); err != nil { t.Error(err) } + const expectedHash = "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302" fileHash := hex.EncodeToString(h.Sum(nil)) - if fileHash != "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302" { - t.Errorf("expected files not checked out. Expected hash %s, got %s", "2bd1707542a11f987ee24698dcc095a9f57639f401133ef6a29da97bf8f3f302", fileHash) + if fileHash != expectedHash { + t.Errorf("expected files not checked out. Expected hash %s, got %s", expectedHash, fileHash) } semVer := CheckoutSemVer{ diff --git a/pkg/git/libgit2/transport.go b/pkg/git/libgit2/transport.go index da3d04e9..6329a490 100644 --- a/pkg/git/libgit2/transport.go +++ b/pkg/git/libgit2/transport.go @@ -66,8 +66,8 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) { password = string(d) } if username != "" && password != "" { - credCallback = func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) { - cred, err := git2go.NewCredUserpassPlaintext(username, password) + credCallback = func(url string, usernameFromURL string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) { + cred, err := git2go.NewCredentialUserpassPlaintext(username, password) if err != nil { return nil, err } @@ -81,7 +81,7 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) { roots := x509.NewCertPool() ok := roots.AppendCertsFromPEM(caFile) if !ok { - return git2go.ErrCertificate + return git2go.ErrorCodeCertificate } opts := x509.VerifyOptions{ @@ -90,9 +90,9 @@ func (s *BasicAuth) Method(secret corev1.Secret) (*git.Auth, error) { } _, err := cert.X509.Verify(opts) if err != nil { - return git2go.ErrCertificate + return git2go.ErrorCodeCertificate } - return git2go.ErrOk + return git2go.ErrorCodeOK } } @@ -137,8 +137,8 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) { user = git.DefaultPublicKeyAuthUser } - credCallback := func(url string, usernameFromURL string, allowedTypes git2go.CredType) (*git2go.Cred, error) { - cred, err := git2go.NewCredSshKeyFromMemory(user, "", string(identity), string(password)) + credCallback := func(url string, usernameFromURL string, allowedTypes git2go.CredentialType) (*git2go.Credential, error) { + cred, err := git2go.NewCredentialSSHKeyFromMemory(user, "", string(identity), string(password)) if err != nil { return nil, err } @@ -157,20 +157,20 @@ func (s *PublicKeyAuth) Method(secret corev1.Secret) (*git.Auth, error) { // Check if the configured host matches the hostname given to // the callback. if host != hostname { - return git2go.ErrUser + return git2go.ErrorCodeUser } // We are now certain that the configured host and the hostname // given to the callback match. Use the configured host (that - // includes the port), and normalize it so we can check if there + // includes the port), and normalize it, so we can check if there // is an entry for the hostname _and_ port. host = knownhosts.Normalize(s.host) for _, k := range kk { if k.matches(host, cert.Hostkey) { - return git2go.ErrOk + return git2go.ErrorCodeOK } } - return git2go.ErrCertificate + return git2go.ErrorCodeCertificate } return &git.Auth{CredCallback: credCallback, CertCallback: certCallback}, nil From c9e3f97470fb9c544324ffc67d442a0043074eed Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Tue, 28 Sep 2021 00:12:39 +0200 Subject: [PATCH 31/53] Add `docker-buildx` target to `Makefile` To allow building a multi-platform container image using `buildx`. Various configuration flags allow for fine(r)-grain control over the build process: - `BASE_IMG`: FQDN of the base image that should be used, without a tag. - `BASE_TAG: tag of the base image that should be used. Allows checksum sum to be included. - `BUILDX_PLATFORMS`: platforms to target for the final container image. - `BUILDX_ARGS`: additional `docker buildx build` arguments, e.g. `--push` to push the result to a (local) image registry. Signed-off-by: Hidde Beydals --- .github/actions/run-tests/Dockerfile | 2 +- .github/workflows/e2e.yaml | 4 ++-- Dockerfile | 2 +- Makefile | 31 +++++++++++++++++++++------- 4 files changed, 27 insertions(+), 12 deletions(-) diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index 93c9f549..f07783ac 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/hiddeco/golang-with-libgit2:dev as builder +FROM ghcr.io/hiddeco/golang-with-libgit2:dev as build # Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index 085724d5..b75e7059 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -44,11 +44,11 @@ jobs: exit 1 fi - name: Build container image - run: make docker-build IMG=test/source-controller:latest + run: make docker-build IMG=test/source-controller TAG=latest - name: Load test image run: kind load docker-image test/source-controller:latest - name: Deploy controller - run: make dev-deploy IMG=test/source-controller:latest + run: make dev-deploy IMG=test/source-controller TAG=latest - name: Run smoke tests run: | kubectl -n source-system apply -f ./config/samples diff --git a/Dockerfile b/Dockerfile index 083fb1e6..08661246 100644 --- a/Dockerfile +++ b/Dockerfile @@ -37,7 +37,7 @@ RUN groupadd controller && \ useradd --gid controller --shell /bin/sh --create-home controller # Copy libgit2 -COPY --from=build /libgit2/lib/* /usr/local/lib/ +COPY --from=build /libgit2/lib/ /usr/local/lib/ RUN ldconfig # Upgrade packages and install runtime dependencies diff --git a/Makefile b/Makefile index 167c4804..a1a6ee9c 100644 --- a/Makefile +++ b/Makefile @@ -1,10 +1,17 @@ # Image URL to use all building/pushing image targets -IMG ?= fluxcd/source-controller:latest +IMG ?= fluxcd/source-controller +TAG ?= latest # Base image used to build the Go binary BASE_IMG ?= ghcr.io/hiddeco/golang-with-libgit2 BASE_TAG ?= dev +# Allows for defining additional Docker buildx arguments, +# e.g. '--push'. +BUILDX_ARGS ?= +# Architectures to build images for +BUILDX_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7 + # Produce CRDs that work back to Kubernetes 1.16 CRD_OPTIONS ?= crd:crdVersions=v1 @@ -60,12 +67,12 @@ uninstall: manifests ## Uninstall CRDs from a cluster kustomize build config/crd | kubectl delete -f - deploy: manifests ## Deploy controller in the configured Kubernetes cluster in ~/.kube/config - cd config/manager && kustomize edit set image fluxcd/source-controller=${IMG} + cd config/manager && kustomize edit set image fluxcd/source-controller=$(IMG):$(TAG) kustomize build config/default | kubectl apply -f - dev-deploy: ## Deploy controller dev image in the configured Kubernetes cluster in ~/.kube/config mkdir -p config/dev && cp config/default/* config/dev - cd config/dev && kustomize edit set image fluxcd/source-controller=${IMG} + cd config/dev && kustomize edit set image fluxcd/source-controller=$(IMG):$(TAG) kustomize build config/dev | kubectl apply -f - rm -rf config/dev @@ -84,7 +91,7 @@ fmt: ## Run go fmt against code go fmt ./... cd api; go fmt ./... -vet: ## Run go vet against code +vet: $(LIBGIT2) ## Run go vet against code PKG_CONFIG_PATH=$(LIBGIT2_LIB_PATH)/pkgconfig \ go vet ./... cd api; go vet ./... @@ -92,14 +99,22 @@ vet: ## Run go vet against code generate: controller-gen ## Generate API code cd api; $(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..." -docker-build: ## Build the docker image +docker-build: ## Build the Docker image docker build \ --build-arg BASE_IMG=$(BASE_IMG) \ --build-arg BASE_TAG=$(BASE_TAG) \ - -t ${IMG} . + -t $(IMG):$(TAG) . + +docker-buildx: ## Build the cross-platform Docker image + docker buildx build \ + --build-arg BASE_IMG=$(BASE_IMG) \ + --build-arg BASE_TAG=$(BASE_TAG) \ + --platform=$(BUILDX_PLATFORMS) \ + -t $(IMG):$(TAG) \ + $(BUILDX_ARGS) . -docker-push: ## Push docker image - docker push ${IMG} +docker-push: ## Push Docker image + docker push $(IMG):$(TAG) controller-gen: ## Find or download controller-gen ifeq (, $(shell which controller-gen)) From b283e3e9c5f120479c517890fb22abc0cbe4fad7 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 30 Sep 2021 16:24:32 +0200 Subject: [PATCH 32/53] Change image to image under Flux organization Signed-off-by: Hidde Beydals --- .github/actions/run-tests/Dockerfile | 2 +- Dockerfile | 4 ++-- Makefile | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index f07783ac..25e4f232 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/hiddeco/golang-with-libgit2:dev as build +FROM ghcr.io/fluxcd/golang-with-libgit2:1.16.8-bullseye-libgit2-1.1.1 as build # Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ diff --git a/Dockerfile b/Dockerfile index 08661246..a97bac5d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ -ARG BASE_IMG=ghcr.io/hiddeco/golang-with-libgit2 -ARG BASE_TAG=dev +ARG BASE_IMG=ghcr.io/fluxcd/golang-with-libgit2 +ARG BASE_TAG=1.16.8-bullseye-libgit2-1.1.1 FROM ${BASE_IMG}:${BASE_TAG} AS build # Configure workspace diff --git a/Makefile b/Makefile index a1a6ee9c..73fcd508 100644 --- a/Makefile +++ b/Makefile @@ -3,8 +3,8 @@ IMG ?= fluxcd/source-controller TAG ?= latest # Base image used to build the Go binary -BASE_IMG ?= ghcr.io/hiddeco/golang-with-libgit2 -BASE_TAG ?= dev +BASE_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 +BASE_TAG ?= 1.16.8-bullseye-libgit2-1.1.1 # Allows for defining additional Docker buildx arguments, # e.g. '--push'. From 500d0aeda01c4cdf6a89ef4ecc236a454faa3036 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 1 Oct 2021 19:08:59 +0200 Subject: [PATCH 33/53] Update base image to version with Darwin detection To provide a better (contributing) experience to those with Apple machines, as determining the correct paths there is a bit harder. Signed-off-by: Hidde Beydals --- .github/actions/run-tests/Dockerfile | 2 +- Dockerfile | 2 +- Makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index 25e4f232..afc7d4b5 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,4 +1,4 @@ -FROM ghcr.io/fluxcd/golang-with-libgit2:1.16.8-bullseye-libgit2-1.1.1 as build +FROM ghcr.io/fluxcd/golang-with-libgit2:1.16.8-bullseye-libgit2-1.1.1-1 as build # Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ diff --git a/Dockerfile b/Dockerfile index a97bac5d..68d65124 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,5 +1,5 @@ ARG BASE_IMG=ghcr.io/fluxcd/golang-with-libgit2 -ARG BASE_TAG=1.16.8-bullseye-libgit2-1.1.1 +ARG BASE_TAG=1.16.8-bullseye-libgit2-1.1.1-1 FROM ${BASE_IMG}:${BASE_TAG} AS build # Configure workspace diff --git a/Makefile b/Makefile index 73fcd508..c8769a6f 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ TAG ?= latest # Base image used to build the Go binary BASE_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 -BASE_TAG ?= 1.16.8-bullseye-libgit2-1.1.1 +BASE_TAG ?= 1.16.8-bullseye-libgit2-1.1.1-1 # Allows for defining additional Docker buildx arguments, # e.g. '--push'. From 1b11e11a90651d41c4e1c69b24df3c2d18c8ca23 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 4 Oct 2021 13:51:04 +0200 Subject: [PATCH 34/53] Allow libgit2 build to be enforced This can be useful on machines where libgit2 is installed due to other applications depending on it, but where the composition of this installation does not properly work with the controller. Reason the system version is still preferred, is because this lowers the barrier for drive-by contributors, as a working set of (Git) dependencies should only really be required if you are going to perform work in that domain. Signed-off-by: Hidde Beydals --- Makefile | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/Makefile b/Makefile index c8769a6f..bdfa840f 100644 --- a/Makefile +++ b/Makefile @@ -18,8 +18,10 @@ CRD_OPTIONS ?= crd:crdVersions=v1 # Repository root based on Git metadata REPOSITORY_ROOT := $(shell git rev-parse --show-toplevel) -# Dependency versions +# Libgit2 version LIBGIT2_VERSION ?= 1.1.1 + +# Other dependency versions ENVTEST_BIN_VERSION ?= 1.19.2 KUBEBUILDER_ASSETS ?= $(shell $(SETUP_ENVTEST) use -i $(ENVTEST_BIN_VERSION) -p path) @@ -31,6 +33,10 @@ LIBGIT2_PATH := $(REPOSITORY_ROOT)/hack/libgit2 LIBGIT2_LIB_PATH := $(LIBGIT2_PATH)/lib LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.so.$(LIBGIT2_VERSION) +ifneq ($(LIBGIT2_VERSION),$(SYSTEM_LIBGIT2_VERSION)) + LIBGIT2_FORCE ?= 1 +endif + # API (doc) generation utilities CONTROLLER_GEN_VERSION ?= v0.5.0 GEN_API_REF_DOCS_VERSION ?= 0.3.0 @@ -164,8 +170,7 @@ endif libgit2: $(LIBGIT2) ## Detect or download libgit2 library $(LIBGIT2): -ifeq ($(LIBGIT2_VERSION),$(SYSTEM_LIBGIT2_VERSION)) -else +ifeq (1, $(LIBGIT2_FORCE)) @{ \ set -e; \ mkdir -p $(LIBGIT2_PATH); \ From cc01df231e0c89949579cf6f8bb18d2712d02d4b Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 4 Oct 2021 13:56:59 +0200 Subject: [PATCH 35/53] Detect macOS produced libgit2.dylib on Darwin Signed-off-by: Hidde Beydals --- Makefile | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/Makefile b/Makefile index bdfa840f..63e65106 100644 --- a/Makefile +++ b/Makefile @@ -37,6 +37,10 @@ ifneq ($(LIBGIT2_VERSION),$(SYSTEM_LIBGIT2_VERSION)) LIBGIT2_FORCE ?= 1 endif +ifeq ($(shell uname -s),Darwin) + LIBGIT2 := $(LIBGIT2_LIB_PATH)/libgit2.$(LIBGIT2_VERSION).dylib +endif + # API (doc) generation utilities CONTROLLER_GEN_VERSION ?= v0.5.0 GEN_API_REF_DOCS_VERSION ?= 0.3.0 From 153b122970cbe0c34a427cdc621f9696811fcf64 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 4 Oct 2021 14:19:34 +0200 Subject: [PATCH 36/53] Document libgit2 build behavior in CONTRIBUTING.md Signed-off-by: Hidde Beydals --- CONTRIBUTING.md | 52 ++++++++++++++++++++++++------------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 29ce578b..76c62687 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -30,37 +30,37 @@ meeting](https://docs.google.com/document/d/1l_M0om0qUEN_NNiGgpqJ2tvsF2iioHkaARD ### Installing required dependencies -The dependency [libgit2](https://libgit2.org/) needs to be installed to be able to run -Source Controller or its test-suite locally (not in a container). +The dependency [libgit2](https://libgit2.org/) needs to be installed to be able +to run source-controller or its test-suite locally (not in a container). -**macOS** -``` -brew install libgit2 +In case this dependency is not present on your system (at the expected +version), the first invocation of a `make` target that requires the +dependency will attempt to compile it locally to `hack/libgit2`. For this build +to succeed; CMake, Docker, OpenSSL 1.1 and LibSSH2 must be present on the system. + +Triggering a manual build of the dependency is possible as well by running +`make libgit2`. To enforce the build, for example if your system dependencies +match but are not linked in a compatible way, append `LIBGIT2_FORCE=1` to the +`make` command. + +#### macOS + +```console +$ # Ensure libgit2 dependencies are available +$ brew install cmake openssl@1.1 libssh2 +$ LIBGIT2_FORCE=1 make libgit2 ``` -**Arch Linux** -``` -pacman -S libgit2 +#### Linux + +```console +$ # Ensure libgit2 dependencies are available +$ pacman -S cmake openssl libssh2 +$ LIBGIT2_FORCE=1 make libgit2 ``` -**Building from source** - -1. Ensure [`cmake`](https://cmake.org) is available on your system. -1. Download and unarchive [the right `libgit2` version](https://github.com/libgit2/git2go#which-go-version-to-use) - for our current `git2go` dependency: - - ```console - $ LIBGIT2_VER=1.1.0 - $ curl -L https://github.com/libgit2/libgit2/releases/download/v$LIBGIT2_VER/libgit2-$LIBGIT2_VER.tar.gz -o /tmp/libgit2.tar.gz - $ tar -xvf /tmp/libgit2.tar.gz -C /tmp/libgit2-$LIBGIT2_VER - ``` -1. Build and install the library on your system: - - ```console - $ mkdir /tmp/libgit2-$LIBGIT2_VER/build && cd /tmp/libgit2-$LIBGIT2_VER/build - $ cmake .. -DCMAKE_INSTALL_PREFIX=/usr - $ sudo cmake --build . --target install - ``` +**Note:** Example shown is for Arch Linux, but likewise procedure can be +followed using any other package manager, e.g. `apt`. ### How to run the test suite From d04c532461c894b556433a47a5c9f6de8ccf934f Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 7 Oct 2021 17:46:54 +0200 Subject: [PATCH 37/53] Switch to scratch based libgit2 container image This moves the `libgit2` compilation to the image, to ensure it can be build on builders that aren't backed by AMD64. The image is structured in such a way that e.g. running nightly builds targeting a different Go version, or targeting a different OS vendor would be possible in the future via build arguments. Signed-off-by: Hidde Beydals --- .github/workflows/e2e.yaml | 2 +- Dockerfile | 48 ++++++++++++++++++++++++++++++++++---- Makefile | 26 ++++++++------------- 3 files changed, 55 insertions(+), 21 deletions(-) diff --git a/.github/workflows/e2e.yaml b/.github/workflows/e2e.yaml index b75e7059..12472092 100644 --- a/.github/workflows/e2e.yaml +++ b/.github/workflows/e2e.yaml @@ -44,7 +44,7 @@ jobs: exit 1 fi - name: Build container image - run: make docker-build IMG=test/source-controller TAG=latest + run: make docker-build IMG=test/source-controller TAG=latest BUILD_PLATFORMS=linux/amd64 BUILD_ARGS=--load - name: Load test image run: kind load docker-image test/source-controller:latest - name: Deploy controller diff --git a/Dockerfile b/Dockerfile index 68d65124..2f03fd84 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,37 @@ -ARG BASE_IMG=ghcr.io/fluxcd/golang-with-libgit2 -ARG BASE_TAG=1.16.8-bullseye-libgit2-1.1.1-1 -FROM ${BASE_IMG}:${BASE_TAG} AS build +ARG BASE_VARIANT=bullseye +ARG GO_VERSION=1.16.8 +ARG XX_VERSION=1.0.0-rc.2 + +ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2 +ARG LIBGIT2_TAG=libgit2-1.1.1 + +FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx +FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2 + +FROM --platform=$BUILDPLATFORM golang:${GO_VERSION}-${BASE_VARIANT} as gostable +FROM --platform=$BUILDPLATFORM golang:1.17rc1-${BASE_VARIANT} AS golatest + +FROM gostable AS go-linux + +FROM go-${TARGETOS} AS build-base-bullseye + +# Copy the build utiltiies +COPY --from=xx / / +COPY --from=libgit2 /Makefile /libgit2/ + +# Install the libgit2 build dependencies +RUN make -C /libgit2 cmake + +ARG TARGETPLATFORM +RUN make -C /libgit2 dependencies + +FROM build-base-${BASE_VARIANT} as libgit2-bullseye + +# Compile and install libgit2 +ARG TARGETPLATFORM +RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2 + +FROM libgit2-${BASE_VARIANT} as build-bullseye # Configure workspace WORKDIR /workspace @@ -27,7 +58,16 @@ ARG TARGETPLATFORM RUN xx-go build -o source-controller -trimpath \ main.go -FROM debian:bullseye-slim as controller +FROM build-${BASE_VARIANT} as prepare-bullseye + +# Move libgit2 lib to generic and predictable location +ARG TARGETPLATFORM +RUN mkdir -p /libgit2/lib/ \ + && cp -d /usr/lib/$(xx-info triple)/libgit2.so* /libgit2/lib/ + +FROM prepare-${BASE_VARIANT} as build + +FROM debian:${BASE_VARIANT}-slim as controller # Link repo to the GitHub Container Registry image LABEL org.opencontainers.image.source="https://github.com/fluxcd/source-controller" diff --git a/Makefile b/Makefile index 63e65106..47243bd2 100644 --- a/Makefile +++ b/Makefile @@ -3,14 +3,14 @@ IMG ?= fluxcd/source-controller TAG ?= latest # Base image used to build the Go binary -BASE_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 -BASE_TAG ?= 1.16.8-bullseye-libgit2-1.1.1-1 +LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 +LIBGIT2_TAG ?= libgit2-1.1.1 # Allows for defining additional Docker buildx arguments, # e.g. '--push'. -BUILDX_ARGS ?= +BUILD_ARGS ?= # Architectures to build images for -BUILDX_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7 +BUILD_PLATFORMS ?= linux/amd64,linux/arm64,linux/arm/v7 # Produce CRDs that work back to Kubernetes 1.16 CRD_OPTIONS ?= crd:crdVersions=v1 @@ -110,18 +110,12 @@ generate: controller-gen ## Generate API code cd api; $(CONTROLLER_GEN) object:headerFile="../hack/boilerplate.go.txt" paths="./..." docker-build: ## Build the Docker image - docker build \ - --build-arg BASE_IMG=$(BASE_IMG) \ - --build-arg BASE_TAG=$(BASE_TAG) \ - -t $(IMG):$(TAG) . - -docker-buildx: ## Build the cross-platform Docker image docker buildx build \ - --build-arg BASE_IMG=$(BASE_IMG) \ - --build-arg BASE_TAG=$(BASE_TAG) \ - --platform=$(BUILDX_PLATFORMS) \ + --build-arg LIBGIT2_IMG=$(LIBGIT2_IMG) \ + --build-arg LIBGIT2_TAG=$(LIBGIT2_TAG) \ + --platform=$(BUILD_PLATFORMS) \ -t $(IMG):$(TAG) \ - $(BUILDX_ARGS) . + $(BUILD_ARGS) . docker-push: ## Push Docker image docker push $(IMG):$(TAG) @@ -178,8 +172,8 @@ ifeq (1, $(LIBGIT2_FORCE)) @{ \ set -e; \ mkdir -p $(LIBGIT2_PATH); \ - docker cp $(shell docker create --rm $(BASE_IMG):$(BASE_TAG)):/libgit2/Makefile $(LIBGIT2_PATH); \ - INSTALL_PREFIX=$(LIBGIT2_PATH) make -C $(LIBGIT2_PATH); \ + curl -sL https://raw.githubusercontent.com/fluxcd/golang-with-libgit2/$(LIBGIT2_TAG)/hack/Makefile -o $(LIBGIT2_PATH)/Makefile; \ + INSTALL_PREFIX=$(LIBGIT2_PATH) make -C $(LIBGIT2_PATH) libgit2; \ } endif From 610131956e259f07131d7264506b04b8e834d4d6 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Thu, 7 Oct 2021 17:53:03 +0200 Subject: [PATCH 38/53] Update Dockerfile used in tests as well This ensures the Dockerfile used for testing is making use of the same scratch image to compile `libgit2` as the actual application image. In a future iteration we should restructure our GitHub Action workflows to re-use the application image, saving us an additional Dockerfile and a duplicate build. Inspiration for this (which makes use of a local registry for the duration of the build) can be found at: https://github.com/fluxcd/golang-with-libgit2/blob/main/.github/workflows/build.yaml Signed-off-by: Hidde Beydals --- .github/actions/run-tests/Dockerfile | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index afc7d4b5..6024ca21 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -1,4 +1,26 @@ -FROM ghcr.io/fluxcd/golang-with-libgit2:1.16.8-bullseye-libgit2-1.1.1-1 as build +ARG BASE_VARIANT=bullseye +ARG GO_VERSION=1.16.8 +ARG XX_VERSION=1.0.0-rc.2 + +ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2 +ARG LIBGIT2_TAG=libgit2-1.1.1 + +FROM tonistiigi/xx:${XX_VERSION} AS xx +FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2 + +FROM golang:${GO_VERSION}-${BASE_VARIANT} as gostable + +# Copy the build utiltiies +COPY --from=xx / / +COPY --from=libgit2 /Makefile /libgit2/ + +# Install the libgit2 build dependencies +RUN make -C /libgit2 cmake + +RUN make -C /libgit2 dependencies + +# Compile and install libgit2 +RUN FLAGS=$(xx-clang --print-cmake-defines) make -C /libgit2 libgit2 # Use the GitHub Actions uid:gid combination for proper fs permissions RUN groupadd -g 116 test && \ From 66fffe103efa52dc3e37db2401dc2a90d0ff5ad8 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 8 Oct 2021 09:40:02 +0200 Subject: [PATCH 39/53] CONTRIBUTING: include pkg-config as macOS dep As this isn't available on Darwin by default, unlike on most Linux distributions. Signed-off-by: Hidde Beydals --- CONTRIBUTING.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 76c62687..c933bbab 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -47,7 +47,7 @@ match but are not linked in a compatible way, append `LIBGIT2_FORCE=1` to the ```console $ # Ensure libgit2 dependencies are available -$ brew install cmake openssl@1.1 libssh2 +$ brew install cmake openssl@1.1 libssh2 pkg-config $ LIBGIT2_FORCE=1 make libgit2 ``` From 6fe6f07d5e97b13a5fb6bc7b74e47219777dd508 Mon Sep 17 00:00:00 2001 From: Stefan Prodan Date: Fri, 8 Oct 2021 11:22:44 +0300 Subject: [PATCH 40/53] Update containerd and runc to fix CVEs Signed-off-by: Stefan Prodan --- go.mod | 6 ++++++ go.sum | 27 +++++++++++++++++++++------ 2 files changed, 27 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 26038e28..e20ee6bf 100644 --- a/go.mod +++ b/go.mod @@ -48,3 +48,9 @@ require ( // required by https://github.com/helm/helm/blob/v3.6.0/go.mod replace github.com/docker/distribution => github.com/docker/distribution v0.0.0-20191216044856-a8371794149d + +// fix CVE-2021-41103 +replace github.com/containerd/containerd => github.com/containerd/containerd v1.4.11 + +// fix CVE-2021-30465 +replace github.com/opencontainers/runc => github.com/opencontainers/runc v1.0.0-rc95 diff --git a/go.sum b/go.sum index ce31fea3..10175849 100644 --- a/go.sum +++ b/go.sum @@ -154,10 +154,12 @@ github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghf github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chai2010/gettext-go v0.0.0-20160711120539-c6fed771bfd5/go.mod h1:/iP1qXHoty45bqomnu2LM+VVyAEdWN+vtSHGlQgyxbw= +github.com/checkpoint-restore/go-criu/v5 v5.0.0/go.mod h1:cfwC0EG7HMUenopBsUf9d89JlCLQIfgVcNsNN0t6T2M= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/cilium/ebpf v0.0.0-20200110133405-4032b1d8aae3/go.mod h1:MA5e5Lr8slmEg9bt0VpxxWqJlO4iwu3FBdHUzV7wQVg= +github.com/cilium/ebpf v0.5.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/clbanning/x2j v0.0.0-20191024224557-825249438eec/go.mod h1:jMjuTZXRI4dUb/I5gc9Hdhagfvm9+RyrPryS/auMzxE= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= @@ -169,9 +171,9 @@ github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59 h1:qWj4qVYZ95vLWwqyNJCQg7rDsG5wPdze0UaPolH7DUk= github.com/containerd/cgroups v0.0.0-20200531161412-0dbf7f05ba59/go.mod h1:pA0z1pT8KYB3TCXK/ocprsh7MAkoW8bZVzPdih9snmM= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= -github.com/containerd/containerd v1.3.2/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= -github.com/containerd/containerd v1.4.4 h1:rtRG4N6Ct7GNssATwgpvMGfnjnwfjnu/Zs9W3Ikzq+M= -github.com/containerd/containerd v1.4.4/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= +github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= +github.com/containerd/containerd v1.4.11 h1:QCGOUN+i70jEEL/A6JVIbhy4f4fanzAzSR4kNG7SlcE= +github.com/containerd/containerd v1.4.11/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y= github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7 h1:6ejg6Lkk8dskcM7wQ28gONkukbQkM4qpj4RnYbpFzrI= github.com/containerd/continuity v0.0.0-20201208142359-180525291bb7/go.mod h1:kR3BEg7bDFaEddKm54WSmrol1fKWDU1nKYkgrcgZT7Y= @@ -189,6 +191,7 @@ github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.0.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= +github.com/coreos/go-systemd/v22 v22.3.1/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= @@ -285,6 +288,7 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9 h1:hsms1Qyu0jgnwNXIxa+/V/PDsU6CfLf6CNO8H7IWoS4= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= @@ -388,6 +392,7 @@ github.com/gobuffalo/packr/v2 v2.7.1/go.mod h1:qYEvAazPaVxy7Y7KR0W8qYEE+RymX74kE github.com/gobwas/glob v0.2.3 h1:A4xDbljILXROh+kObIiy5kIaPYD8e96x1tgBhUI5J+Y= github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godror/godror v0.13.3/go.mod h1:2ouUT4kdhUBk7TAkHWD4SN0CdI0pgEQbo8FVHhbSKWg= github.com/gofrs/flock v0.8.0/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gogo/googleapis v1.1.0/go.mod h1:gf4bu3Q80BeJ6H1S1vYPm8/ELATdvryBaNFGgqEef3s= @@ -672,6 +677,7 @@ github.com/mitchellh/reflectwalk v1.0.1 h1:FVzMWA5RllMAKIdUSC8mdWo3XtwoecrH79BY7 github.com/mitchellh/reflectwalk v1.0.1/go.mod h1:mSTlrgnPZtwu0c4WaC2kGObEpuNDbx0jmZXqmk4esnw= github.com/moby/spdystream v0.2.0 h1:cjW1zVyyoiM0T7b6UoySUFqzXMoqRckQtXwGPiBhOM8= github.com/moby/spdystream v0.2.0/go.mod h1:f7i0iNDQJ059oMTcWxx8MA/zKFIuD/lY+0GqbN2Wy8c= +github.com/moby/sys/mountinfo v0.4.1/go.mod h1:rEr8tzG/lsIZHBtN/JjGG+LMYx9eXgW2JI+6q0qou+A= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635 h1:rzf0wL0CHVc8CEsgyygG0Mn9CNCCPZqOPaz8RiiHYQk= github.com/moby/term v0.0.0-20201216013528-df9cb8a40635/go.mod h1:FBS0z0QWA44HXygs7VXDUOGoN/1TV3RuWkLO04am3wc= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -684,6 +690,7 @@ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00 h1:n6/ github.com/monochromegane/go-gitignore v0.0.0-20200626010858-205db1a8cc00/go.mod h1:Pm3mSP3c5uWn86xMLZ5Sa7JB9GsEZySvHYXCTK4E9q4= github.com/morikuni/aec v1.0.0 h1:nP9CBfwrvYnBRgY6qfDQkygYDmYwOilePFkwzv4dU8A= github.com/morikuni/aec v1.0.0/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= +github.com/mrunalp/fileutils v0.5.0/go.mod h1:M1WthSahJixYnrXQl/DFQuteStB1weuxD2QJNHXfbSQ= github.com/munnerz/goautoneg v0.0.0-20120707110453-a547fc61f48d/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822/go.mod h1:+n7T8mK8HuQTcFwEeznm/DIxMOiR9yIdICNftLE1DvQ= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= @@ -732,10 +739,11 @@ github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3I github.com/opencontainers/image-spec v1.0.0/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= github.com/opencontainers/image-spec v1.0.1 h1:JMemWkRwHx4Zj+fVxWoMCFm/8sYGGrUVojFA6h/TRcI= github.com/opencontainers/image-spec v1.0.1/go.mod h1:BtxoFyWECRxE4U/7sNtV5W15zMzWCbyJoFRP3s7yZA0= -github.com/opencontainers/runc v0.0.0-20190115041553-12f6a991201f/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= -github.com/opencontainers/runc v0.1.1 h1:GlxAyO6x8rfZYN9Tt0Kti5a/cP41iuiO2yYT0IJGY8Y= -github.com/opencontainers/runc v0.1.1/go.mod h1:qT5XzbpPznkRYVz/mWwUaVBUv2rmF59PVA73FjuZG0U= +github.com/opencontainers/runc v1.0.0-rc95 h1:RMuWVfY3E1ILlVsC3RhIq38n4sJtlOFwU9gfFZSqrd0= +github.com/opencontainers/runc v1.0.0-rc95/go.mod h1:z+bZxa/+Tz/FmYVWkhUajJdzFeOqjc5vrqskhVyHGUM= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/selinux v1.8.0/go.mod h1:RScLhm78qiWa2gbVCcGkC7tCGdgk3ogry1nUQF8Evvo= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -821,6 +829,7 @@ github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0 github.com/satori/go.uuid v1.2.0 h1:0uYX9dsZ2yD7q2RtLRtPSdGDWzjeM3TbMJP9utgA0ww= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc= +github.com/seccomp/libseccomp-golang v0.9.1/go.mod h1:GbW5+tmTXfcxTToHLXlScSlAvWlF4P2Ca7zGrPiEpWo= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sergi/go-diff v1.1.0 h1:we8PVUC3FE2uYfodKH/nBHMSetSfHDR6scGdBi+erh0= github.com/sergi/go-diff v1.1.0/go.mod h1:STckp+ISIX8hZLjrqAeVduY0gWCT9IjLuqbuNXdaHfM= @@ -882,6 +891,7 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/subosito/gotenv v1.2.0/go.mod h1:N0PQaV/YGNqwC0u51sEeR/aUtSLEXKX9iv69rRypqCw= +github.com/syndtr/gocapability v0.0.0-20200815063812-42c35b437635/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= @@ -891,6 +901,9 @@ github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijb github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYppBueQtXaqoE= +github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= +github.com/willf/bitset v1.1.11/go.mod h1:83CECat5yLh5zVOf4P1ErAgKA5UDvKtgyUABdr3+MjI= github.com/xanzy/ssh-agent v0.3.0 h1:wUMzuKtKilRgBAD1sUb8gOwwRr2FGoBVumcjoOACClI= github.com/xanzy/ssh-agent v0.3.0/go.mod h1:3s9xbODqPuuhK9JV1R321M/FlMZSBvE5aY6eAcqrDh0= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f h1:J9EGpcZtP0E/raorCMxlFGSTBrsSlaDGf3jU/qvAE2c= @@ -1119,6 +1132,7 @@ golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190515120540-06a5c4944438/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190602015325-4c4f7f33c9ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606203320-7fc4e5ec1444/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190616124812-15dcb6c0061f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1130,6 +1144,7 @@ golang.org/x/sys v0.0.0-20191002063906-3421d5a6bb1c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191022100944-742c48ecaeb7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191115151921-52ab43148777/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191204072324-ce4227a45e2e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191220142924-d4481acd189f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 5e6abae9e82448962eae2b0f3ee0905f09ad1741 Mon Sep 17 00:00:00 2001 From: Dylan Arbour Date: Fri, 12 Mar 2021 23:46:32 -0500 Subject: [PATCH 41/53] Add ReconcileStrategy to HelmChart This commit adds a `ReconcileStrategy` field to the `HelmChart` resource, which allows defining when a new chart should be packaged and/or published if it originates from a `Bucket` or `GitRepository` resource. The two available strategies are: - `ChartVersion`: creates a new artifact when the version of the Helm chart as defined in the `Chart.yaml` from the Source is different from the current version. - `Revision`: creates a new artifact when the revision of the Source is different from the current revision. For the `Revision` strategy, the (checksum part of the) revision of the artifact the chart originatesfrom is added as SemVer metadata. A chart from a `GitRepository` with Artifact revision `main/f0faacd5164a875ebdbd9e3fab778f49c5aadbbc` and a chart with e.g. SemVer `0.1.0` will be published as `0.1.0+f0faacd5164a875ebdbd9e3fab778f49c5aadbbc`. A chart from a `Bucket` with Artifact revision `f0faacd5164a875ebdbd9e3fab778f49c5aadbbc` and a chart with e.g. SemVer `0.1.0` will be published as `0.1.0+f0faacd5164a875ebdbd9e3fab778f49c5aadbbc`. Signed-off-by: Dylan Arbour --- api/v1beta1/helmchart_types.go | 17 +++++++++ .../source.toolkit.fluxcd.io_helmcharts.yaml | 7 ++++ controllers/helmchart_controller.go | 25 +++++++++++- controllers/helmchart_controller_test.go | 17 ++++++++- docs/api/source.md | 30 +++++++++++++++ docs/spec/v1beta1/helmcharts.md | 38 +++++++++++++++++++ 6 files changed, 131 insertions(+), 3 deletions(-) diff --git a/api/v1beta1/helmchart_types.go b/api/v1beta1/helmchart_types.go index 96f02780..01fde150 100644 --- a/api/v1beta1/helmchart_types.go +++ b/api/v1beta1/helmchart_types.go @@ -45,6 +45,15 @@ type HelmChartSpec struct { // +required Interval metav1.Duration `json:"interval"` + // Determines what enables the creation of a new artifact. Valid values are + // ('ChartVersion', 'Revision'). + // See the documentation of the values for an explanation on their behavior. + // Defaults to ChartVersion when omitted. + // +kubebuilder:validation:Enum=ChartVersion;Revision + // +kubebuilder:default:=ChartVersion + // +optional + ReconcileStrategy string `json:"reconcileStrategy,omitempty"` + // Alternative list of values files to use as the chart values (values.yaml // is not included by default), expected to be a relative path in the SourceRef. // Values files are merged in the order of this list with the last file overriding @@ -65,6 +74,14 @@ type HelmChartSpec struct { Suspend bool `json:"suspend,omitempty"` } +const ( + // ReconcileStrategyChartVersion reconciles when the version of the Helm chart is different. + ReconcileStrategyChartVersion string = "ChartVersion" + + // ReconcileStrategyRevision reconciles when the Revision of the source is different. + ReconcileStrategyRevision string = "Revision" +) + // LocalHelmChartSourceReference contains enough information to let you locate // the typed referenced object at namespace level. type LocalHelmChartSourceReference struct { diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml index 21a499a8..fe40562b 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml @@ -62,6 +62,13 @@ spec: interval: description: The interval at which to check the Source for updates. type: string + reconcileStrategy: + default: ChartVersion + description: Determines what enables the creation of a new artifact. Valid values are ('ChartVersion', 'Revision'). See the documentation of the values for an explanation on their behavior. Defaults to ChartVersion when omitted. + enum: + - ChartVersion + - Revision + type: string sourceRef: description: The reference to the Source the chart is available at. properties: diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 50583bca..ee6b93e9 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -27,6 +27,7 @@ import ( "strings" "time" + "github.com/Masterminds/semver/v3" securejoin "github.com/cyphar/filepath-securejoin" "github.com/go-logr/logr" helmchart "helm.sh/helm/v3/pkg/chart" @@ -526,9 +527,29 @@ func (r *HelmChartReconciler) reconcileFromTarballArtifact(ctx context.Context, return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err } + v, err := semver.NewVersion(helmChart.Metadata.Version) + if err != nil { + err = fmt.Errorf("semver error: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + } + + version := v.String() + if chart.Spec.ReconcileStrategy == sourcev1.ReconcileStrategyRevision { + // Isolate the commit SHA from GitRepository type artifacts by removing the branch/ prefix. + splitRev := strings.Split(artifact.Revision, "/") + v, err := v.SetMetadata(splitRev[len(splitRev)-1]) + if err != nil { + err = fmt.Errorf("semver error: %w", err) + return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err + } + + version = v.String() + helmChart.Metadata.Version = v.String() + } + // Return early if the revision is still the same as the current chart artifact - newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(), helmChart.Metadata.Version, - fmt.Sprintf("%s-%s.tgz", helmChart.Metadata.Name, helmChart.Metadata.Version)) + newArtifact := r.Storage.NewArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(), version, + fmt.Sprintf("%s-%s.tgz", helmChart.Metadata.Name, version)) if !force && apimeta.IsStatusConditionTrue(chart.Status.Conditions, meta.ReadyCondition) && chart.GetArtifact().HasRevision(newArtifact.Revision) { if newArtifact.URL != artifact.URL { r.Storage.SetArtifactURL(chart.GetArtifact()) diff --git a/controllers/helmchart_controller_test.go b/controllers/helmchart_controller_test.go index ba107daa..de3f7ad3 100644 --- a/controllers/helmchart_controller_test.go +++ b/controllers/helmchart_controller_test.go @@ -709,7 +709,7 @@ var _ = Describe("HelmChartReconciler", func() { err = f.Close() Expect(err).NotTo(HaveOccurred()) - _, err = wt.Commit("Chart version bump", &git.CommitOptions{ + commit, err := wt.Commit("Chart version bump", &git.CommitOptions{ Author: &object.Signature{ Name: "John Doe", Email: "john@example.com", @@ -735,6 +735,21 @@ var _ = Describe("HelmChartReconciler", func() { Expect(helmChart.Values["testDefault"]).To(BeTrue()) Expect(helmChart.Values["testOverride"]).To(BeFalse()) + When("Setting reconcileStrategy to Revision", func() { + updated := &sourcev1.HelmChart{} + Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed()) + updated.Spec.ReconcileStrategy = sourcev1.ReconcileStrategyRevision + Expect(k8sClient.Update(context.Background(), updated)).To(Succeed()) + got := &sourcev1.HelmChart{} + Eventually(func() bool { + _ = k8sClient.Get(context.Background(), key, got) + return got.Status.Artifact.Revision != updated.Status.Artifact.Revision && + storage.ArtifactExist(*got.Status.Artifact) + }, timeout, interval).Should(BeTrue()) + Expect(got.Status.Artifact.Revision).To(ContainSubstring(updated.Status.Artifact.Revision)) + Expect(got.Status.Artifact.Revision).To(ContainSubstring(commit.String())) + }) + When("Setting valid valuesFiles attribute", func() { updated := &sourcev1.HelmChart{} Expect(k8sClient.Get(context.Background(), key, updated)).To(Succeed()) diff --git a/docs/api/source.md b/docs/api/source.md index acce3cb5..eeb579a2 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -555,6 +555,21 @@ Kubernetes meta/v1.Duration +reconcileStrategy
+ +string + + + +(Optional) +

Determines what enables reconciliation. Valid values are (‘ChartVersion’, +‘Revision’). See the documentation of the values for an explanation on their +behavior. +Defaults to ChartVersion when omitted.

+ + + + valuesFiles
[]string @@ -1613,6 +1628,21 @@ Kubernetes meta/v1.Duration +reconcileStrategy
+ +string + + + +(Optional) +

Determines what enables reconciliation. Valid values are (‘ChartVersion’, +‘Revision’). See the documentation of the values for an explanation on their +behavior. +Defaults to ChartVersion when omitted.

+ + + + valuesFiles
[]string diff --git a/docs/spec/v1beta1/helmcharts.md b/docs/spec/v1beta1/helmcharts.md index 067004f9..6c4461c2 100644 --- a/docs/spec/v1beta1/helmcharts.md +++ b/docs/spec/v1beta1/helmcharts.md @@ -28,6 +28,15 @@ type HelmChartSpec struct { // +required Interval metav1.Duration `json:"interval"` + // Determines what enables the creation of a new artifact. Valid values are + // ('ChartVersion', 'Revision'). + // See the documentation of the values for an explanation on their behavior. + // Defaults to ChartVersion when omitted. + // +kubebuilder:validation:Enum=ChartVersion;Revision + // +kubebuilder:default:=ChartVersion + // +optional + ReconcileStrategy string `json:"reconcileStrategy,omitempty"` + // Alternative list of values files to use as the chart values (values.yaml // is not included by default), expected to be a relative path in the SourceRef. // Values files are merged in the order of this list with the last file overriding @@ -49,6 +58,18 @@ type HelmChartSpec struct { } ``` +### Reconciliation strategies + +```go +const ( + // ReconcileStrategyChartVersion creates a new chart artifact when the version of the Helm chart is different. + ReconcileStrategyChartVersion string = "ChartVersion" + + // ReconcileStrategyRevision creates a new chart artifact when the Revision of the SourceRef is different. + ReconcileStrategyRevision string = "Revision" +) +``` + ### Reference types ```go @@ -230,6 +251,23 @@ spec: - ./charts/podinfo/values-production.yaml ``` +Reconcile with every change to the source revision: + +```yaml +apiVersion: source.toolkit.fluxcd.io/v1beta1 +kind: HelmChart +metadata: + name: podinfo + namespace: default +spec: + chart: ./charts/podinfo + sourceRef: + name: podinfo + kind: GitRepository + interval: 10m + reconcileStrategy: Revision +``` + ## Status examples Successful chart pull: From 96ab646cd4a5a210054b1c42a281aecb4c8294c0 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 8 Oct 2021 11:58:54 +0200 Subject: [PATCH 42/53] Release v0.16.0 Signed-off-by: Hidde Beydals --- CHANGELOG.md | 24 ++++++++++++++++++++++++ config/manager/kustomization.yaml | 2 +- go.mod | 2 +- 3 files changed, 26 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dfd55ad2..483f1a5f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,30 @@ All notable changes to this project are documented in this file. +## 0.16.0 + +**Release date:** 2021-10-08 + +This prerelease improves the configuration of the `libgit2` C library, solving +most issues around private key formats (e.g. PKCS#8 and ED25519) by ensuring +it is linked against OpenSSL and LibSSH2. + +In addition, the `HelmChart` resource does now allow setting a `ReconcileStrategy` +to define when a new artifact for a chart should be created for charts from +`Bucket` and `GitRepository` sources. By setting this to `Revision`, you no +longer have to bump the version in the `Chart.yaml` file, but a new chart will +automatically be made available when the revision of the Source changes. + +Fixes: +* Update containerd and runc to fix CVEs + [#446](https://github.com/fluxcd/source-controller/pull/446) + +Improvements: +* Add reconcile strategy for HelmCharts + [#308](https://github.com/fluxcd/source-controller/pull/308) +* Update github.com/libgit2/git2go to v31.6.1 + [#437](https://github.com/fluxcd/source-controller/pull/437) + ## 0.15.4 **Release date:** 2021-08-05 diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index 88ce2820..0cba2457 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -6,4 +6,4 @@ resources: images: - name: fluxcd/source-controller newName: fluxcd/source-controller - newTag: v0.15.4 + newTag: v0.16.0 diff --git a/go.mod b/go.mod index e20ee6bf..8fabe102 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,7 @@ require ( github.com/fluxcd/pkg/ssh v0.1.0 github.com/fluxcd/pkg/untar v0.1.0 github.com/fluxcd/pkg/version v0.1.0 - github.com/fluxcd/source-controller/api v0.15.4 + github.com/fluxcd/source-controller/api v0.16.0 github.com/go-git/go-billy/v5 v5.3.1 github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 From c2495ae4080bce620e96c7949daf074b0b06430f Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 8 Oct 2021 13:14:40 +0200 Subject: [PATCH 43/53] Fix generation of API documentation The version was accidentally set to an invalid version, causing the API documentation generation to fail. Signed-off-by: Hidde Beydals --- Makefile | 2 +- docs/api/source.md | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/Makefile b/Makefile index 47243bd2..e940205d 100644 --- a/Makefile +++ b/Makefile @@ -43,7 +43,7 @@ endif # API (doc) generation utilities CONTROLLER_GEN_VERSION ?= v0.5.0 -GEN_API_REF_DOCS_VERSION ?= 0.3.0 +GEN_API_REF_DOCS_VERSION ?= v0.3.0 # Get the currently used golang install path (in GOPATH/bin, unless GOBIN is set) ifeq (,$(shell go env GOBIN)) diff --git a/docs/api/source.md b/docs/api/source.md index eeb579a2..78aee678 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -562,9 +562,9 @@ string (Optional) -

Determines what enables reconciliation. Valid values are (‘ChartVersion’, -‘Revision’). See the documentation of the values for an explanation on their -behavior. +

Determines what enables the creation of a new artifact. Valid values are +(‘ChartVersion’, ‘Revision’). +See the documentation of the values for an explanation on their behavior. Defaults to ChartVersion when omitted.

@@ -1635,9 +1635,9 @@ string (Optional) -

Determines what enables reconciliation. Valid values are (‘ChartVersion’, -‘Revision’). See the documentation of the values for an explanation on their -behavior. +

Determines what enables the creation of a new artifact. Valid values are +(‘ChartVersion’, ‘Revision’). +See the documentation of the values for an explanation on their behavior. Defaults to ChartVersion when omitted.

From e2548cbe57c76618e67f31cf8e325aa84c5e747a Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Fri, 8 Oct 2021 15:48:59 +0200 Subject: [PATCH 44/53] Update fluxcd/golang-with-libgit2 to 1.1.1-1 This includes a tiny fix for Darwin to ensure the generated `.pc` file includes the right paths. Signed-off-by: Hidde Beydals --- .github/actions/run-tests/Dockerfile | 2 +- Dockerfile | 2 +- Makefile | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/actions/run-tests/Dockerfile b/.github/actions/run-tests/Dockerfile index 6024ca21..ee9bd04e 100644 --- a/.github/actions/run-tests/Dockerfile +++ b/.github/actions/run-tests/Dockerfile @@ -3,7 +3,7 @@ ARG GO_VERSION=1.16.8 ARG XX_VERSION=1.0.0-rc.2 ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2 -ARG LIBGIT2_TAG=libgit2-1.1.1 +ARG LIBGIT2_TAG=libgit2-1.1.1-1 FROM tonistiigi/xx:${XX_VERSION} AS xx FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2 diff --git a/Dockerfile b/Dockerfile index 2f03fd84..8b818e9c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,7 +3,7 @@ ARG GO_VERSION=1.16.8 ARG XX_VERSION=1.0.0-rc.2 ARG LIBGIT2_IMG=ghcr.io/fluxcd/golang-with-libgit2 -ARG LIBGIT2_TAG=libgit2-1.1.1 +ARG LIBGIT2_TAG=libgit2-1.1.1-1 FROM --platform=$BUILDPLATFORM tonistiigi/xx:${XX_VERSION} AS xx FROM ${LIBGIT2_IMG}:${LIBGIT2_TAG} as libgit2 diff --git a/Makefile b/Makefile index e940205d..fb4d9998 100644 --- a/Makefile +++ b/Makefile @@ -4,7 +4,7 @@ TAG ?= latest # Base image used to build the Go binary LIBGIT2_IMG ?= ghcr.io/fluxcd/golang-with-libgit2 -LIBGIT2_TAG ?= libgit2-1.1.1 +LIBGIT2_TAG ?= libgit2-1.1.1-1 # Allows for defining additional Docker buildx arguments, # e.g. '--push'. From 38bf4d9859aa295f1c0a695d40378a110c0dab7e Mon Sep 17 00:00:00 2001 From: pa250194 Date: Tue, 12 Oct 2021 11:46:48 -0500 Subject: [PATCH 45/53] Fixed spelling and capitalization Signed-off-by: pa250194 --- pkg/gcp/gcp.go | 22 ++++++++++------------ pkg/gcp/gcp_test.go | 18 +++++++++--------- 2 files changed, 19 insertions(+), 21 deletions(-) diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 470fe227..38a0b99f 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -24,15 +24,15 @@ import ( "os" "path/filepath" - gcpStorage "cloud.google.com/go/storage" - interator "google.golang.org/api/iterator" + gcpstorage "cloud.google.com/go/storage" + "google.golang.org/api/iterator" "google.golang.org/api/option" ) var ( // IteratorDone is returned when the looping of objects/content // has reached the end of the iteration. - IteratorDone = interator.Done + IteratorDone = iterator.Done // ErrorDirectoryExists is an error returned when the filename provided // is a directory. ErrorDirectoryExists = errors.New("filename is a directory") @@ -44,15 +44,13 @@ var ( type GCPClient struct { // client for interacting with the Google Cloud // Storage APIs. - *gcpStorage.Client + *gcpstorage.Client } -// NewClient creates a new GCP storage client -// The Google Storage Client will automatically -// look for the Google Application Credential environment variable -// or look for the Google Application Credential file. +// NewClient creates a new GCP storage client. The Client will automatically look for the Google Application +// Credential environment variable or look for the Google Application Credential file. func NewClient(ctx context.Context, opts ...option.ClientOption) (*GCPClient, error) { - client, err := gcpStorage.NewClient(ctx, opts...) + client, err := gcpstorage.NewClient(ctx, opts...) if err != nil { return nil, err } @@ -73,7 +71,7 @@ func ValidateSecret(secret map[string][]byte, name string) error { // BucketExists checks if the bucket with the provided name exists. func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, error) { _, err := c.Client.Bucket(bucketName).Attrs(ctx) - if err == gcpStorage.ErrBucketNotExist { + if err == gcpstorage.ErrBucketNotExist { return false, err } if err != nil { @@ -86,7 +84,7 @@ func (c *GCPClient) BucketExists(ctx context.Context, bucketName string) (bool, func (c *GCPClient) ObjectExists(ctx context.Context, bucketName, objectName string) (bool, error) { _, err := c.Client.Bucket(bucketName).Object(objectName).Attrs(ctx) // ErrObjectNotExist is returned if the object does not exist - if err == gcpStorage.ErrObjectNotExist { + if err == gcpstorage.ErrObjectNotExist { return false, err } if err != nil { @@ -160,7 +158,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca // ListObjects lists the objects/contents of the bucket whose bucket name is provided. // the objects are returned as an Objectiterator and .Next() has to be called on them // to loop through the Objects. -func (c *GCPClient) ListObjects(ctx context.Context, bucketName string, query *gcpStorage.Query) *gcpStorage.ObjectIterator { +func (c *GCPClient) ListObjects(ctx context.Context, bucketName string, query *gcpstorage.Query) *gcpstorage.ObjectIterator { items := c.Client.Bucket(bucketName).Objects(ctx, query) return items } diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 8faa5e2c..99d72309 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -32,7 +32,7 @@ import ( "testing" "time" - gcpStorage "cloud.google.com/go/storage" + gcpstorage "cloud.google.com/go/storage" "github.com/fluxcd/source-controller/pkg/gcp" "google.golang.org/api/googleapi" raw "google.golang.org/api/storage/v1" @@ -48,7 +48,7 @@ const ( var ( hc *http.Client - client *gcpStorage.Client + client *gcpstorage.Client close func() err error ) @@ -101,7 +101,7 @@ func TestMain(m *testing.M) { } }) ctx := context.Background() - client, err = gcpStorage.NewClient(ctx, option.WithHTTPClient(hc)) + client, err = gcpstorage.NewClient(ctx, option.WithHTTPClient(hc)) if err != nil { log.Fatal(err) } @@ -131,7 +131,7 @@ func TestBucketNotExists(t *testing.T) { Client: client, } exists, err := gcpClient.BucketExists(context.Background(), bucket) - assert.Error(t, err, gcpStorage.ErrBucketNotExist.Error()) + assert.Error(t, err, gcpstorage.ErrBucketNotExist.Error()) assert.Assert(t, !exists) } @@ -140,7 +140,7 @@ func TestObjectExists(t *testing.T) { Client: client, } exists, err := gcpClient.ObjectExists(context.Background(), bucketName, objectName) - if err == gcpStorage.ErrObjectNotExist { + if err == gcpstorage.ErrObjectNotExist { assert.NilError(t, err) } assert.NilError(t, err) @@ -153,7 +153,7 @@ func TestObjectNotExists(t *testing.T) { Client: client, } exists, err := gcpClient.ObjectExists(context.Background(), bucketName, object) - assert.Error(t, err, gcpStorage.ErrObjectNotExist.Error()) + assert.Error(t, err, gcpstorage.ErrObjectNotExist.Error()) assert.Assert(t, !exists) } @@ -161,15 +161,15 @@ func TestListObjects(t *testing.T) { gcpClient := &gcp.GCPClient{ Client: client, } - objectInterator := gcpClient.ListObjects(context.Background(), bucketName, nil) + objectIterator := gcpClient.ListObjects(context.Background(), bucketName, nil) for { - _, err := objectInterator.Next() + _, err := objectIterator.Next() if err == gcp.IteratorDone { break } assert.NilError(t, err) } - assert.Assert(t, objectInterator != nil) + assert.Assert(t, objectIterator != nil) } func TestFGetObject(t *testing.T) { From 39811ed46af4c156e8feebaba75aed2bb218236f Mon Sep 17 00:00:00 2001 From: pa250194 Date: Wed, 1 Sep 2021 14:10:08 -0500 Subject: [PATCH 46/53] Add Support for GCP storage with workload identity Added Support for Google Cloud Storage with Workload Identity as Source Provider. This enables the use of GCP without enabling S3 compatible access. Signed-off-by: pa250194 --- docs/spec/v1alpha1/buckets.md | 1 + go.sum | 2 ++ 2 files changed, 3 insertions(+) diff --git a/docs/spec/v1alpha1/buckets.md b/docs/spec/v1alpha1/buckets.md index bb2c07a9..49dcfbb8 100644 --- a/docs/spec/v1alpha1/buckets.md +++ b/docs/spec/v1alpha1/buckets.md @@ -57,6 +57,7 @@ Supported providers: const ( GenericBucketProvider string = "generic" AmazonBucketProvider string = "aws" + GoogleBucketProvider string = "gcp" ) ``` diff --git a/go.sum b/go.sum index 10175849..34c48c50 100644 --- a/go.sum +++ b/go.sum @@ -1074,6 +1074,7 @@ golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwY golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= +golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210224082022-3d97a244fca7/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= @@ -1168,6 +1169,7 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200831180312-196b9ba8737a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200909081042-eff7692f9009/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= From 2baa8a289c7925d2d06f5de0819a737a9c8e39cd Mon Sep 17 00:00:00 2001 From: pa250194 Date: Wed, 1 Sep 2021 14:41:40 -0500 Subject: [PATCH 47/53] Added Comments for reconcileWithGCP and reconcileWithMinio Signed-off-by: pa250194 --- controllers/bucket_controller.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index aa11c261..86efc75c 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -432,9 +432,7 @@ func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) } -// authMinio creates a new Minio client to interact with S3 -// compatible storage services. -func (r *BucketReconciler) authMinio(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) { +func (r *BucketReconciler) auth(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) { opt := minio.Options{ Region: bucket.Spec.Region, Secure: !bucket.Spec.Insecure, From be1ed50ac47a5db425dd71180dd714cbad389302 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Fri, 10 Sep 2021 16:01:16 -0500 Subject: [PATCH 48/53] Service Account Key Authentication to GCP Provider Signed-off-by: pa250194 --- go.mod | 1 + go.sum | 1 + pkg/gcp/gcp.go | 2 +- pkg/gcp/gcp_test.go | 1 - pkg/gcp/mocks/mock_gcp_storage.go | 211 ++++++++++++++++++++++++++++++ 5 files changed, 214 insertions(+), 2 deletions(-) create mode 100644 pkg/gcp/mocks/mock_gcp_storage.go diff --git a/go.mod b/go.mod index 8fabe102..07b8d82e 100644 --- a/go.mod +++ b/go.mod @@ -23,6 +23,7 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/mock v1.6.0 // indirect github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/libgit2/git2go/v31 v31.6.1 github.com/minio/minio-go/v7 v7.0.10 diff --git a/go.sum b/go.sum index 34c48c50..75df5bf5 100644 --- a/go.sum +++ b/go.sum @@ -419,6 +419,7 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= +github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index 38a0b99f..cd4bedac 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -127,7 +127,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca return err } if !exists { - return ErrorObjectDoesNotExist + return ObjectDoesNotExist } objectFile, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY, 0600) diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index 99d72309..f6618498 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -30,7 +30,6 @@ import ( "os" "path/filepath" "testing" - "time" gcpstorage "cloud.google.com/go/storage" "github.com/fluxcd/source-controller/pkg/gcp" diff --git a/pkg/gcp/mocks/mock_gcp_storage.go b/pkg/gcp/mocks/mock_gcp_storage.go new file mode 100644 index 00000000..54b78be1 --- /dev/null +++ b/pkg/gcp/mocks/mock_gcp_storage.go @@ -0,0 +1,211 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: gcp.go + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + storage "cloud.google.com/go/storage" + gomock "github.com/golang/mock/gomock" +) + +// MockClient is a mock of Client interface. +type MockClient struct { + ctrl *gomock.Controller + recorder *MockClientMockRecorder +} + +// MockClientMockRecorder is the mock recorder for MockClient. +type MockClientMockRecorder struct { + mock *MockClient +} + +// NewMockClient creates a new mock instance. +func NewMockClient(ctrl *gomock.Controller) *MockClient { + mock := &MockClient{ctrl: ctrl} + mock.recorder = &MockClientMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockClient) EXPECT() *MockClientMockRecorder { + return m.recorder +} + +// Bucket mocks base method. +func (m *MockClient) Bucket(arg0 string) *storage.BucketHandle { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Bucket", arg0) + ret0, _ := ret[0].(*storage.BucketHandle) + return ret0 +} + +// Bucket indicates an expected call of Bucket. +func (mr *MockClientMockRecorder) Bucket(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockClient)(nil).Bucket), arg0) +} + +// Close mocks base method. +func (m *MockClient) Close() error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Close") + ret0, _ := ret[0].(error) + return ret0 +} + +// Close indicates an expected call of Close. +func (mr *MockClientMockRecorder) Close() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClient)(nil).Close)) +} + +// MockBucketHandle is a mock of BucketHandle interface. +type MockBucketHandle struct { + ctrl *gomock.Controller + recorder *MockBucketHandleMockRecorder +} + +// MockBucketHandleMockRecorder is the mock recorder for MockBucketHandle. +type MockBucketHandleMockRecorder struct { + mock *MockBucketHandle +} + +// NewMockBucketHandle creates a new mock instance. +func NewMockBucketHandle(ctrl *gomock.Controller) *MockBucketHandle { + mock := &MockBucketHandle{ctrl: ctrl} + mock.recorder = &MockBucketHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockBucketHandle) EXPECT() *MockBucketHandleMockRecorder { + return m.recorder +} + +// Attrs mocks base method. +func (m *MockBucketHandle) Attrs(arg0 context.Context) (*storage.BucketAttrs, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Attrs", arg0) + ret0, _ := ret[0].(*storage.BucketAttrs) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Attrs indicates an expected call of Attrs. +func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0) +} + +// Create mocks base method. +func (m *MockBucketHandle) Create(arg0 context.Context, arg1 string, arg2 *storage.BucketAttrs) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Create indicates an expected call of Create. +func (mr *MockBucketHandleMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockBucketHandle)(nil).Create), arg0, arg1, arg2) +} + +// Delete mocks base method. +func (m *MockBucketHandle) Delete(arg0 context.Context) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Delete", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// Delete indicates an expected call of Delete. +func (mr *MockBucketHandleMockRecorder) Delete(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBucketHandle)(nil).Delete), arg0) +} + +// Object mocks base method. +func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Object", arg0) + ret0, _ := ret[0].(*storage.ObjectHandle) + return ret0 +} + +// Object indicates an expected call of Object. +func (mr *MockBucketHandleMockRecorder) Object(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Object", reflect.TypeOf((*MockBucketHandle)(nil).Object), arg0) +} + +// Objects mocks base method. +func (m *MockBucketHandle) Objects(arg0 context.Context, arg1 *storage.Query) *storage.ObjectIterator { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Objects", arg0, arg1) + ret0, _ := ret[0].(*storage.ObjectIterator) + return ret0 +} + +// Objects indicates an expected call of Objects. +func (mr *MockBucketHandleMockRecorder) Objects(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Objects", reflect.TypeOf((*MockBucketHandle)(nil).Objects), arg0, arg1) +} + +// MockObjectHandle is a mock of ObjectHandle interface. +type MockObjectHandle struct { + ctrl *gomock.Controller + recorder *MockObjectHandleMockRecorder +} + +// MockObjectHandleMockRecorder is the mock recorder for MockObjectHandle. +type MockObjectHandleMockRecorder struct { + mock *MockObjectHandle +} + +// NewMockObjectHandle creates a new mock instance. +func NewMockObjectHandle(ctrl *gomock.Controller) *MockObjectHandle { + mock := &MockObjectHandle{ctrl: ctrl} + mock.recorder = &MockObjectHandleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockObjectHandle) EXPECT() *MockObjectHandleMockRecorder { + return m.recorder +} + +// Attrs mocks base method. +func (m *MockObjectHandle) Attrs(arg0 context.Context) (*storage.ObjectAttrs, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Attrs", arg0) + ret0, _ := ret[0].(*storage.ObjectAttrs) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Attrs indicates an expected call of Attrs. +func (mr *MockObjectHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockObjectHandle)(nil).Attrs), arg0) +} + +// NewRangeReader mocks base method. +func (m *MockObjectHandle) NewRangeReader(arg0 context.Context, arg1, arg2 int64) (*storage.Reader, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NewRangeReader", arg0, arg1, arg2) + ret0, _ := ret[0].(*storage.Reader) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NewRangeReader indicates an expected call of NewRangeReader. +func (mr *MockObjectHandleMockRecorder) NewRangeReader(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewRangeReader", reflect.TypeOf((*MockObjectHandle)(nil).NewRangeReader), arg0, arg1, arg2) +} From 99c79bffaa47e2177d552d0ea78f9abe48022cc6 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Tue, 14 Sep 2021 09:34:02 -0500 Subject: [PATCH 49/53] Tests for GCP Bucket Provider Signed-off-by: pa250194 --- docs/spec/v1alpha1/buckets.md | 2 +- pkg/gcp/mocks/mock_gcp_storage.go | 28 ---------------------------- 2 files changed, 1 insertion(+), 29 deletions(-) diff --git a/docs/spec/v1alpha1/buckets.md b/docs/spec/v1alpha1/buckets.md index 49dcfbb8..843524df 100644 --- a/docs/spec/v1alpha1/buckets.md +++ b/docs/spec/v1alpha1/buckets.md @@ -11,7 +11,7 @@ Bucket: // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws + // +kubebuilder:validation:Enum=generic;aws;gcp // +optional Provider string `json:"provider,omitempty"` diff --git a/pkg/gcp/mocks/mock_gcp_storage.go b/pkg/gcp/mocks/mock_gcp_storage.go index 54b78be1..25b5e9c1 100644 --- a/pkg/gcp/mocks/mock_gcp_storage.go +++ b/pkg/gcp/mocks/mock_gcp_storage.go @@ -101,34 +101,6 @@ func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0) } -// Create mocks base method. -func (m *MockBucketHandle) Create(arg0 context.Context, arg1 string, arg2 *storage.BucketAttrs) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Create", arg0, arg1, arg2) - ret0, _ := ret[0].(error) - return ret0 -} - -// Create indicates an expected call of Create. -func (mr *MockBucketHandleMockRecorder) Create(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Create", reflect.TypeOf((*MockBucketHandle)(nil).Create), arg0, arg1, arg2) -} - -// Delete mocks base method. -func (m *MockBucketHandle) Delete(arg0 context.Context) error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Delete", arg0) - ret0, _ := ret[0].(error) - return ret0 -} - -// Delete indicates an expected call of Delete. -func (mr *MockBucketHandleMockRecorder) Delete(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delete", reflect.TypeOf((*MockBucketHandle)(nil).Delete), arg0) -} - // Object mocks base method. func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle { m.ctrl.T.Helper() From c98130548e58c6c601fc1cd7485e0e872a604e25 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Wed, 15 Sep 2021 14:42:53 -0500 Subject: [PATCH 50/53] Added Tests to GCP provider Signed-off-by: pa250194 --- pkg/gcp/mocks/mock_gcp_storage.go | 183 ------------------------------ 1 file changed, 183 deletions(-) delete mode 100644 pkg/gcp/mocks/mock_gcp_storage.go diff --git a/pkg/gcp/mocks/mock_gcp_storage.go b/pkg/gcp/mocks/mock_gcp_storage.go deleted file mode 100644 index 25b5e9c1..00000000 --- a/pkg/gcp/mocks/mock_gcp_storage.go +++ /dev/null @@ -1,183 +0,0 @@ -// Code generated by MockGen. DO NOT EDIT. -// Source: gcp.go - -// Package mocks is a generated GoMock package. -package mocks - -import ( - context "context" - reflect "reflect" - - storage "cloud.google.com/go/storage" - gomock "github.com/golang/mock/gomock" -) - -// MockClient is a mock of Client interface. -type MockClient struct { - ctrl *gomock.Controller - recorder *MockClientMockRecorder -} - -// MockClientMockRecorder is the mock recorder for MockClient. -type MockClientMockRecorder struct { - mock *MockClient -} - -// NewMockClient creates a new mock instance. -func NewMockClient(ctrl *gomock.Controller) *MockClient { - mock := &MockClient{ctrl: ctrl} - mock.recorder = &MockClientMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockClient) EXPECT() *MockClientMockRecorder { - return m.recorder -} - -// Bucket mocks base method. -func (m *MockClient) Bucket(arg0 string) *storage.BucketHandle { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Bucket", arg0) - ret0, _ := ret[0].(*storage.BucketHandle) - return ret0 -} - -// Bucket indicates an expected call of Bucket. -func (mr *MockClientMockRecorder) Bucket(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Bucket", reflect.TypeOf((*MockClient)(nil).Bucket), arg0) -} - -// Close mocks base method. -func (m *MockClient) Close() error { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Close") - ret0, _ := ret[0].(error) - return ret0 -} - -// Close indicates an expected call of Close. -func (mr *MockClientMockRecorder) Close() *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Close", reflect.TypeOf((*MockClient)(nil).Close)) -} - -// MockBucketHandle is a mock of BucketHandle interface. -type MockBucketHandle struct { - ctrl *gomock.Controller - recorder *MockBucketHandleMockRecorder -} - -// MockBucketHandleMockRecorder is the mock recorder for MockBucketHandle. -type MockBucketHandleMockRecorder struct { - mock *MockBucketHandle -} - -// NewMockBucketHandle creates a new mock instance. -func NewMockBucketHandle(ctrl *gomock.Controller) *MockBucketHandle { - mock := &MockBucketHandle{ctrl: ctrl} - mock.recorder = &MockBucketHandleMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockBucketHandle) EXPECT() *MockBucketHandleMockRecorder { - return m.recorder -} - -// Attrs mocks base method. -func (m *MockBucketHandle) Attrs(arg0 context.Context) (*storage.BucketAttrs, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Attrs", arg0) - ret0, _ := ret[0].(*storage.BucketAttrs) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Attrs indicates an expected call of Attrs. -func (mr *MockBucketHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockBucketHandle)(nil).Attrs), arg0) -} - -// Object mocks base method. -func (m *MockBucketHandle) Object(arg0 string) *storage.ObjectHandle { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Object", arg0) - ret0, _ := ret[0].(*storage.ObjectHandle) - return ret0 -} - -// Object indicates an expected call of Object. -func (mr *MockBucketHandleMockRecorder) Object(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Object", reflect.TypeOf((*MockBucketHandle)(nil).Object), arg0) -} - -// Objects mocks base method. -func (m *MockBucketHandle) Objects(arg0 context.Context, arg1 *storage.Query) *storage.ObjectIterator { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Objects", arg0, arg1) - ret0, _ := ret[0].(*storage.ObjectIterator) - return ret0 -} - -// Objects indicates an expected call of Objects. -func (mr *MockBucketHandleMockRecorder) Objects(arg0, arg1 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Objects", reflect.TypeOf((*MockBucketHandle)(nil).Objects), arg0, arg1) -} - -// MockObjectHandle is a mock of ObjectHandle interface. -type MockObjectHandle struct { - ctrl *gomock.Controller - recorder *MockObjectHandleMockRecorder -} - -// MockObjectHandleMockRecorder is the mock recorder for MockObjectHandle. -type MockObjectHandleMockRecorder struct { - mock *MockObjectHandle -} - -// NewMockObjectHandle creates a new mock instance. -func NewMockObjectHandle(ctrl *gomock.Controller) *MockObjectHandle { - mock := &MockObjectHandle{ctrl: ctrl} - mock.recorder = &MockObjectHandleMockRecorder{mock} - return mock -} - -// EXPECT returns an object that allows the caller to indicate expected use. -func (m *MockObjectHandle) EXPECT() *MockObjectHandleMockRecorder { - return m.recorder -} - -// Attrs mocks base method. -func (m *MockObjectHandle) Attrs(arg0 context.Context) (*storage.ObjectAttrs, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Attrs", arg0) - ret0, _ := ret[0].(*storage.ObjectAttrs) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// Attrs indicates an expected call of Attrs. -func (mr *MockObjectHandleMockRecorder) Attrs(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Attrs", reflect.TypeOf((*MockObjectHandle)(nil).Attrs), arg0) -} - -// NewRangeReader mocks base method. -func (m *MockObjectHandle) NewRangeReader(arg0 context.Context, arg1, arg2 int64) (*storage.Reader, error) { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "NewRangeReader", arg0, arg1, arg2) - ret0, _ := ret[0].(*storage.Reader) - ret1, _ := ret[1].(error) - return ret0, ret1 -} - -// NewRangeReader indicates an expected call of NewRangeReader. -func (mr *MockObjectHandleMockRecorder) NewRangeReader(arg0, arg1, arg2 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NewRangeReader", reflect.TypeOf((*MockObjectHandle)(nil).NewRangeReader), arg0, arg1, arg2) -} From 5077c1f9f63baeff18a717cc458ab5dd8bd16517 Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 16 Sep 2021 09:48:33 -0500 Subject: [PATCH 51/53] Added more tests and cleaned up GCP provider logic Signed-off-by: pa250194 --- go.mod | 1 - go.sum | 1 - 2 files changed, 2 deletions(-) diff --git a/go.mod b/go.mod index 07b8d82e..8fabe102 100644 --- a/go.mod +++ b/go.mod @@ -23,7 +23,6 @@ require ( github.com/go-git/go-git/v5 v5.4.2 github.com/go-logr/logr v0.4.0 github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/mock v1.6.0 // indirect github.com/googleapis/gax-go/v2 v2.1.0 // indirect github.com/libgit2/git2go/v31 v31.6.1 github.com/minio/minio-go/v7 v7.0.10 diff --git a/go.sum b/go.sum index 75df5bf5..34c48c50 100644 --- a/go.sum +++ b/go.sum @@ -419,7 +419,6 @@ github.com/golang/mock v1.4.1/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt github.com/golang/mock v1.4.3/go.mod h1:UOMv5ysSaYNkG+OFQykRIcU/QvvxJf3p21QfJ2Bt3cw= github.com/golang/mock v1.4.4/go.mod h1:l3mdAwkq5BuhzHwde/uurv3sEJeZMXNpwsxVWU71h+4= github.com/golang/mock v1.5.0/go.mod h1:CWnOUgYIOo4TcNZ0wHX3YZCqsaM1I1Jvs6v3mP3KVu8= -github.com/golang/mock v1.6.0 h1:ErTB+efbowRARo13NNdxyJji2egdxLGQhRaY+DUumQc= github.com/golang/mock v1.6.0/go.mod h1:p6yTPP+5HYm5mzsMV8JkE6ZKdX+/wYM6Hr+LicevLPs= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= From 7921caf05649cbc1254b774b1d113cd46e6c7afe Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 16 Sep 2021 12:15:26 -0500 Subject: [PATCH 52/53] Updated docs to include GCP provider instructions Signed-off-by: pa250194 --- docs/spec/v1alpha1/buckets.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/docs/spec/v1alpha1/buckets.md b/docs/spec/v1alpha1/buckets.md index 843524df..bb2c07a9 100644 --- a/docs/spec/v1alpha1/buckets.md +++ b/docs/spec/v1alpha1/buckets.md @@ -11,7 +11,7 @@ Bucket: // BucketSpec defines the desired state of an S3 compatible bucket type BucketSpec struct { // The S3 compatible storage provider name, default ('generic'). - // +kubebuilder:validation:Enum=generic;aws;gcp + // +kubebuilder:validation:Enum=generic;aws // +optional Provider string `json:"provider,omitempty"` @@ -57,7 +57,6 @@ Supported providers: const ( GenericBucketProvider string = "generic" AmazonBucketProvider string = "aws" - GoogleBucketProvider string = "gcp" ) ``` From c4e4b3928cbc36efcf0e64bfabc1f4d76d48e03e Mon Sep 17 00:00:00 2001 From: pa250194 Date: Thu, 14 Oct 2021 09:39:53 -0500 Subject: [PATCH 53/53] Added Logger to closing GCP client Signed-off-by: pa250194 Added log for GCP provider auth error Signed-off-by: pa250194 --- controllers/bucket_controller.go | 7 +++++-- pkg/gcp/gcp.go | 10 +++++++++- pkg/gcp/gcp_test.go | 1 + 3 files changed, 15 insertions(+), 3 deletions(-) diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 86efc75c..002c95c6 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -268,12 +268,13 @@ func (r *BucketReconciler) reconcileDelete(ctx context.Context, bucket sourcev1. // reconcileWithGCP handles getting objects from a Google Cloud Platform bucket // using a gcp client func (r *BucketReconciler) reconcileWithGCP(ctx context.Context, bucket sourcev1.Bucket, tempDir string) (sourcev1.Bucket, error) { + log := logr.FromContext(ctx) gcpClient, err := r.authGCP(ctx, bucket) if err != nil { err = fmt.Errorf("auth error: %w", err) return sourcev1.BucketNotReady(bucket, sourcev1.AuthenticationFailedReason, err.Error()), err } - defer gcpClient.Client.Close() + defer gcpClient.Close(log) ctxTimeout, cancel := context.WithTimeout(ctx, bucket.Spec.Timeout.Duration) defer cancel() @@ -432,7 +433,9 @@ func (r *BucketReconciler) authGCP(ctx context.Context, bucket sourcev1.Bucket) } -func (r *BucketReconciler) auth(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) { +// authMinio creates a new Minio client to interact with S3 +// compatible storage services. +func (r *BucketReconciler) authMinio(ctx context.Context, bucket sourcev1.Bucket) (*minio.Client, error) { opt := minio.Options{ Region: bucket.Spec.Region, Secure: !bucket.Spec.Insecure, diff --git a/pkg/gcp/gcp.go b/pkg/gcp/gcp.go index cd4bedac..9127fcde 100644 --- a/pkg/gcp/gcp.go +++ b/pkg/gcp/gcp.go @@ -25,6 +25,7 @@ import ( "path/filepath" gcpstorage "cloud.google.com/go/storage" + "github.com/go-logr/logr" "google.golang.org/api/iterator" "google.golang.org/api/option" ) @@ -127,7 +128,7 @@ func (c *GCPClient) FGetObject(ctx context.Context, bucketName, objectName, loca return err } if !exists { - return ObjectDoesNotExist + return ErrorObjectDoesNotExist } objectFile, err := os.OpenFile(localPath, os.O_CREATE|os.O_WRONLY, 0600) @@ -162,3 +163,10 @@ func (c *GCPClient) ListObjects(ctx context.Context, bucketName string, query *g items := c.Client.Bucket(bucketName).Objects(ctx, query) return items } + +// Close closes the GCP Client and logs any useful errors +func (c *GCPClient) Close(log logr.Logger) { + if err := c.Client.Close(); err != nil { + log.Error(err, "GCP Provider") + } +} diff --git a/pkg/gcp/gcp_test.go b/pkg/gcp/gcp_test.go index f6618498..99d72309 100644 --- a/pkg/gcp/gcp_test.go +++ b/pkg/gcp/gcp_test.go @@ -30,6 +30,7 @@ import ( "os" "path/filepath" "testing" + "time" gcpstorage "cloud.google.com/go/storage" "github.com/fluxcd/source-controller/pkg/gcp"