Check ignore matches before Bucket item downloads

Signed-off-by: Hidde Beydals <hello@hidde.co>

controllers/bucket_controller.go

@@ -17,6 +17,7 @@ limitations under the License.
 package controllers
 
 import (
+	"bytes"
 	"context"
 	"crypto/sha1"
 	"fmt"
@@ -49,6 +50,7 @@ import (
 	"github.com/fluxcd/pkg/runtime/predicates"
 
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
+	"github.com/fluxcd/source-controller/pkg/sourceignore"
 )
 
 // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=buckets,verbs=get;list;watch;create;update;patch;delete
@@ -202,6 +204,9 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket
 		return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err
 	}
 
+	ps := sourceignore.GetPatterns(bytes.NewBufferString(*bucket.Spec.Ignore), nil)
+	matcher := sourceignore.NewMatcher(ps)
+
 	// download bucket content
 	for object := range s3Client.ListObjects(ctxTimeout, bucket.Spec.BucketName, minio.ListObjectsOptions{
 		Recursive: true,
@@ -216,6 +221,10 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket
 			continue
 		}
 
+		if matcher.Match([]string{object.Key}, false) {
+			continue
+		}
+
 		localPath := filepath.Join(tempDir, object.Key)
 		err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, object.Key, localPath, minio.GetObjectOptions{})
 		if err != nil {

controllers/storage.go

@@ -18,8 +18,6 @@ package controllers
 
 import (
 	"archive/tar"
-	"bufio"
-	"bytes"
 	"compress/gzip"
 	"crypto/sha1"
 	"fmt"
@@ -39,6 +37,7 @@ import (
 
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
 	"github.com/fluxcd/source-controller/internal/fs"
+	"github.com/fluxcd/source-controller/pkg/sourceignore"
 )
 
 const (
@@ -159,11 +158,11 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, ignore *strin
 		return fmt.Errorf("invalid dir path: %s", dir)
 	}
 
-	ps, err := loadExcludePatterns(dir, ignore)
+	ps, err := sourceignore.LoadExcludePatterns(dir, ignore)
 	if err != nil {
 		return err
 	}
-	matcher := gitignore.NewMatcher(ps)
+	matcher := sourceignore.NewMatcher(ps)
 
 	localPath := s.LocalPath(*artifact)
 	tf, err := ioutil.TempFile(filepath.Split(localPath))
@@ -400,51 +399,6 @@ func (s *Storage) LocalPath(artifact sourcev1.Artifact) string {
 	return filepath.Join(s.BasePath, artifact.Path)
 }
 
-// getPatterns collects ignore patterns from the given reader and returns them
-// as a gitignore.Pattern slice.
-func getPatterns(reader io.Reader, path []string) []gitignore.Pattern {
-	var ps []gitignore.Pattern
-	scanner := bufio.NewScanner(reader)
-
-	for scanner.Scan() {
-		s := scanner.Text()
-		if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
-			ps = append(ps, gitignore.ParsePattern(s, path))
-		}
-	}
-
-	return ps
-}
-
-// loadExcludePatterns loads the excluded patterns from sourceignore or other
-// sources.
-func loadExcludePatterns(dir string, ignore *string) ([]gitignore.Pattern, error) {
-	path := strings.Split(dir, "/")
-
-	var ps []gitignore.Pattern
-	for _, p := range strings.Split(excludeVCS, ",") {
-		ps = append(ps, gitignore.ParsePattern(p, path))
-	}
-
-	if ignore == nil {
-		all := strings.Join([]string{excludeExt, excludeCI, excludeExtra}, ",")
-		for _, p := range strings.Split(all, ",") {
-			ps = append(ps, gitignore.ParsePattern(p, path))
-		}
-
-		if f, err := os.Open(filepath.Join(dir, excludeFile)); err == nil {
-			defer f.Close()
-			ps = append(ps, getPatterns(f, path)...)
-		} else if !os.IsNotExist(err) {
-			return nil, err
-		}
-	} else {
-		ps = append(ps, getPatterns(bytes.NewBufferString(*ignore), path)...)
-	}
-
-	return ps, nil
-}
-
 // newHash returns a new SHA1 hash.
 func newHash() hash.Hash {
 	return sha1.New()

pkg/sourceignore/sourceignore.go

@@ -0,0 +1,87 @@
+/*
+Copyright 2021 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sourceignore
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
+)
+
+const (
+	ExcludeFile  = ".sourceignore"
+	ExcludeVCS   = ".git/,.gitignore,.gitmodules,.gitattributes"
+	ExcludeExt   = "*.jpg,*.jpeg,*.gif,*.png,*.wmv,*.flv,*.tar.gz,*.zip"
+	ExcludeCI    = ".github/,.circleci/,.travis.yml,.gitlab-ci.yml,appveyor.yml,.drone.yml,cloudbuild.yaml,codeship-services.yml,codeship-steps.yml"
+	ExcludeExtra = "**/.goreleaser.yml,**/.sops.yaml,**/.flux.yaml"
+)
+
+// NewMatcher returns a gitignore.Matcher for the given gitignore.Pattern
+// slice. It mainly exists to compliment the API.
+func NewMatcher(ps []gitignore.Pattern) gitignore.Matcher {
+	return gitignore.NewMatcher(ps)
+}
+
+// GetPatterns collects ignore patterns from the given reader and
+// returns them as a gitignore.Pattern slice.
+func GetPatterns(reader io.Reader, path []string) []gitignore.Pattern {
+	var ps []gitignore.Pattern
+	scanner := bufio.NewScanner(reader)
+
+	for scanner.Scan() {
+		s := scanner.Text()
+		if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
+			ps = append(ps, gitignore.ParsePattern(s, path))
+		}
+	}
+
+	return ps
+}
+
+// LoadExcludePatterns loads the excluded patterns from .sourceignore or other
+// sources and returns the gitignore.Pattern slice.
+func LoadExcludePatterns(dir string, ignore *string) ([]gitignore.Pattern, error) {
+	path := strings.Split(dir, "/")
+
+	var ps []gitignore.Pattern
+	for _, p := range strings.Split(ExcludeVCS, ",") {
+		ps = append(ps, gitignore.ParsePattern(p, path))
+	}
+
+	if ignore == nil {
+		all := strings.Join([]string{ExcludeExt, ExcludeCI, ExcludeExtra}, ",")
+		for _, p := range strings.Split(all, ",") {
+			ps = append(ps, gitignore.ParsePattern(p, path))
+		}
+
+		if f, err := os.Open(filepath.Join(dir, ExcludeFile)); err == nil {
+			defer f.Close()
+			ps = append(ps, GetPatterns(f, path)...)
+		} else if !os.IsNotExist(err) {
+			return nil, err
+		}
+	} else {
+		ps = append(ps, GetPatterns(bytes.NewBufferString(*ignore), path)...)
+	}
+
+	return ps, nil
+}