This commit is contained in:
Josh van Leeuwen 2025-05-22 18:02:30 +01:00 committed by GitHub
commit 26926a956b
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 17 additions and 28 deletions

View File

@ -30,7 +30,6 @@ import (
"cloud.google.com/go/storage"
"github.com/google/uuid"
"go.uber.org/multierr"
"google.golang.org/api/googleapi"
"google.golang.org/api/iterator"
"google.golang.org/api/option"
@ -430,32 +429,33 @@ func (g *GCPStorage) bulkGet(ctx context.Context, req *bindings.InvokeRequest) (
var allObjs []*storage.ObjectAttrs
it := g.client.Bucket(g.metadata.Bucket).Objects(ctx, nil)
for {
attrs, err2 := it.Next()
if err2 == iterator.Done {
var attrs *storage.ObjectAttrs
attrs, err = it.Next()
if err == iterator.Done {
break
}
allObjs = append(allObjs, attrs)
}
var wg sync.WaitGroup
objectsCh := make(chan objectData, len(allObjs))
errCh := make(chan error, len(allObjs))
wg.Add(len(allObjs))
objects := make([]objectData, len(allObjs))
errs := make([]error, len(allObjs))
for i, obj := range allObjs {
wg.Add(1)
go func(idx int, object *storage.ObjectAttrs) {
defer wg.Done()
rc, err3 := g.client.Bucket(g.metadata.Bucket).Object(object.Name).NewReader(ctx)
if err3 != nil {
errCh <- err3
rc, gerr := g.client.Bucket(g.metadata.Bucket).Object(object.Name).NewReader(ctx)
if gerr != nil {
errs[idx] = err
return
}
defer rc.Close()
data, readErr := io.ReadAll(rc)
if readErr != nil {
errCh <- readErr
data, gerr := io.ReadAll(rc)
if gerr != nil {
errs[idx] = err
return
}
@ -464,7 +464,7 @@ func (g *GCPStorage) bulkGet(ctx context.Context, req *bindings.InvokeRequest) (
data = []byte(encoded)
}
objectsCh <- objectData{
objects[idx] = objectData{
Name: object.Name,
Data: data,
Attrs: *object,
@ -473,28 +473,17 @@ func (g *GCPStorage) bulkGet(ctx context.Context, req *bindings.InvokeRequest) (
}
wg.Wait()
close(errCh)
var multiErr error
for err := range errCh {
multierr.AppendInto(&multiErr, err)
if err = errors.Join(errs...); err != nil {
return nil, fmt.Errorf("gcp bucket binding error while reading objects: %w", err)
}
if multiErr != nil {
return nil, multiErr
}
response := make([]objectData, 0, len(allObjs))
for obj := range objectsCh {
response = append(response, obj)
}
jsonResponse, err := json.Marshal(response)
response, err := json.Marshal(allObjs)
if err != nil {
return nil, fmt.Errorf("gcp bucket binding error while marshalling bulk get response: %w", err)
}
return &bindings.InvokeResponse{
Data: jsonResponse,
Data: response,
}, nil
}