fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #1228 from fluxcd/bucket-prefix 

bucket: Add prefix filtering capability
This commit is contained in:
Stefan Prodan 2023-10-17 15:24:11 +03:00 committed by GitHub
parent 3073c2c261 674c287dbd
commit f2a1814aea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
11 changed files with 60 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

5
api/v1beta2/bucket_types.go
View File

@ -23,6 +23,7 @@ import (
"github.com/fluxcd/pkg/apis/acl"
"github.com/fluxcd/pkg/apis/meta"
apiv1 "github.com/fluxcd/source-controller/api/v1"
)
@ -73,6 +74,10 @@ type BucketSpec struct {
// +optional
Region string `json:"region,omitempty"`
// Prefix to use for server-side filtering of files in the Bucket.
// +optional
Prefix string `json:"prefix,omitempty"`
// SecretRef specifies the Secret containing authentication credentials
// for the Bucket.
// +optional

4
config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml
View File

@ -331,6 +331,10 @@ spec:
to ensure efficient use of resources.
pattern: ^([0-9]+(\.[0-9]+)?(ms|s|m|h))+$
type: string
prefix:
description: Prefix to use for server-side filtering of files in the
Bucket.
type: string
provider:
default: generic
description: Provider of the object storage bucket. Defaults to 'generic',

24
docs/api/v1beta2/source.md
View File

@ -138,6 +138,18 @@ string
</tr>
<tr>
<td>
<code>prefix</code><br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>Prefix to use for server-side filtering of files in the Bucket.</p>
</td>
</tr>
<tr>
<td>
<code>secretRef</code><br>
<em>
<a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">
@ -1422,6 +1434,18 @@ string
</tr>
<tr>
<td>
<code>prefix</code><br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>Prefix to use for server-side filtering of files in the Bucket.</p>
</td>
</tr>
<tr>
<td>
<code>secretRef</code><br>
<em>
<a href="https://pkg.go.dev/github.com/fluxcd/pkg/apis/meta#LocalObjectReference">

9
docs/spec/v1beta2/buckets.md
View File

@ -785,6 +785,15 @@ credentials for the object storage. For some `.spec.provider` implementations
the presence of the field is required, see [Provider](#provider) for more
details and examples.
### Prefix
`.spec.prefix` is an optional field to enable server-side filtering
of files in the Bucket.
**Note:** The server-side filtering works only with the `generic`, `aws`
and `gcp` [provider](#provider) and is preferred over [`.spec.ignore`](#ignore)
as a more efficient way of excluding files.
### Ignore
`.spec.ignore` is an optional field to specify rules in [the `.gitignore`

4
internal/controller/bucket_controller.go
View File

@ -145,7 +145,7 @@ type BucketProvider interface {
// bucket, calling visit for every item.
// If the underlying client or the visit callback returns an error,
// it returns early.
VisitObjects(ctx context.Context, bucketName string, visit func(key, etag string) error) error
VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(key, etag string) error) error
// ObjectIsNotFound returns true if the given error indicates an object
// could not be found.
ObjectIsNotFound(error) bool
@ -742,7 +742,7 @@ func fetchEtagIndex(ctx context.Context, provider BucketProvider, obj *bucketv1.
matcher := sourceignore.NewMatcher(ps)
// Build up index
err = provider.VisitObjects(ctxTimeout, obj.Spec.BucketName, func(key, etag string) error {
err = provider.VisitObjects(ctxTimeout, obj.Spec.BucketName, obj.Spec.Prefix, func(key, etag string) error {
if strings.HasSuffix(key, "/") || key == sourceignore.IgnoreFile {
return nil
}

2
internal/controller/bucket_controller_fetch_test.go
View File

@ -69,7 +69,7 @@ func (m mockBucketClient) ObjectIsNotFound(e error) bool {
return e == errMockNotFound
}
func (m mockBucketClient) VisitObjects(_ context.Context, _ string, f func(key, etag string) error) error {
func (m mockBucketClient) VisitObjects(_ context.Context, _ string, _ string, f func(key, etag string) error) error {
for key, obj := range m.objects {
if err := f(key, obj.etag); err != nil {
return err

2
pkg/azure/blob.go
View File

@ -265,7 +265,7 @@ func (c *BlobClient) FGetObject(ctx context.Context, bucketName, objectName, loc
// bucket, calling visit for every item.
// If the underlying client or the visit callback returns an error,
// it returns early.
func (c *BlobClient) VisitObjects(ctx context.Context, bucketName string, visit func(path, etag string) error) error {
func (c *BlobClient) VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(path, etag string) error) error {
items := c.NewListBlobsFlatPager(bucketName, nil)
for items.More() {
resp, err := items.NextPage(ctx)

6
pkg/gcp/gcp.go
View File

@ -165,8 +165,10 @@ func (c *GCSClient) FGetObject(ctx context.Context, bucketName, objectName, loca
// bucket, calling visit for every item.
// If the underlying client or the visit callback returns an error,
// it returns early.
func (c *GCSClient) VisitObjects(ctx context.Context, bucketName string, visit func(path, etag string) error) error {
items := c.Client.Bucket(bucketName).Objects(ctx, nil)
func (c *GCSClient) VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(path, etag string) error) error {
items := c.Client.Bucket(bucketName).Objects(ctx, &gcpstorage.Query{
Prefix: prefix,
})
for {
object, err := items.Next()
if err == IteratorDone {

6
pkg/gcp/gcp_test.go
View File

@ -170,7 +170,7 @@ func TestVisitObjects(t *testing.T) {
}
keys := []string{}
etags := []string{}
err := gcpClient.VisitObjects(context.Background(), bucketName, func(key, etag string) error {
err := gcpClient.VisitObjects(context.Background(), bucketName, "", func(key, etag string) error {
keys = append(keys, key)
etags = append(etags, etag)
return nil
@ -185,7 +185,7 @@ func TestVisitObjectsErr(t *testing.T) {
Client: client,
}
badBucketName := "bad-bucket"
err := gcpClient.VisitObjects(context.Background(), badBucketName, func(key, etag string) error {
err := gcpClient.VisitObjects(context.Background(), badBucketName, "", func(key, etag string) error {
return nil
})
assert.Error(t, err, fmt.Sprintf("listing objects from bucket '%s' failed: storage: bucket doesn't exist", badBucketName))
@ -196,7 +196,7 @@ func TestVisitObjectsCallbackErr(t *testing.T) {
Client: client,
}
mockErr := fmt.Errorf("mock")
err := gcpClient.VisitObjects(context.Background(), bucketName, func(key, etag string) error {
err := gcpClient.VisitObjects(context.Background(), bucketName, "", func(key, etag string) error {
return mockErr
})
assert.Error(t, err, mockErr.Error())

3
pkg/minio/minio.go
View File

@ -105,9 +105,10 @@ func (c *MinioClient) FGetObject(ctx context.Context, bucketName, objectName, lo
// bucket, calling visit for every item.
// If the underlying client or the visit callback returns an error,
// it returns early.
func (c *MinioClient) VisitObjects(ctx context.Context, bucketName string, visit func(key, etag string) error) error {
func (c *MinioClient) VisitObjects(ctx context.Context, bucketName string, prefix string, visit func(key, etag string) error) error {
for object := range c.Client.ListObjects(ctx, bucketName, minio.ListObjectsOptions{
Recursive: true,
Prefix: prefix,
UseV1: s3utils.IsGoogleEndpoint(*c.Client.EndpointURL()),
}) {
if object.Err != nil {

8
pkg/minio/minio_test.go
View File

@ -36,6 +36,7 @@ import (
"github.com/fluxcd/pkg/apis/meta"
"github.com/fluxcd/pkg/sourceignore"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
)
@ -62,6 +63,7 @@ var (
var (
bucketName = "test-bucket-minio" + uuid.New().String()
prefix = ""
secret = corev1.Secret{
ObjectMeta: v1.ObjectMeta{
Name: "minio-secret",
@ -228,7 +230,7 @@ func TestFGetObjectNotExists(t *testing.T) {
func TestVisitObjects(t *testing.T) {
keys := []string{}
etags := []string{}
err := testMinioClient.VisitObjects(context.TODO(), bucketName, func(key, etag string) error {
err := testMinioClient.VisitObjects(context.TODO(), bucketName, prefix, func(key, etag string) error {
keys = append(keys, key)
etags = append(etags, etag)
return nil
@ -241,7 +243,7 @@ func TestVisitObjects(t *testing.T) {
func TestVisitObjectsErr(t *testing.T) {
ctx := context.Background()
badBucketName := "bad-bucket"
err := testMinioClient.VisitObjects(ctx, badBucketName, func(string, string) error {
err := testMinioClient.VisitObjects(ctx, badBucketName, prefix, func(string, string) error {
return nil
})
assert.Error(t, err, fmt.Sprintf("listing objects from bucket '%s' failed: The specified bucket does not exist", badBucketName))
@ -249,7 +251,7 @@ func TestVisitObjectsErr(t *testing.T) {
func TestVisitObjectsCallbackErr(t *testing.T) {
mockErr := fmt.Errorf("mock")
err := testMinioClient.VisitObjects(context.TODO(), bucketName, func(key, etag string) error {
err := testMinioClient.VisitObjects(context.TODO(), bucketName, prefix, func(key, etag string) error {
return mockErr
})
assert.Error(t, err, mockErr.Error())