diff --git a/controllers/bucket_controller.go b/controllers/bucket_controller.go index 4f04f603..3b5fb4a9 100644 --- a/controllers/bucket_controller.go +++ b/controllers/bucket_controller.go @@ -17,7 +17,6 @@ limitations under the License. package controllers import ( - "bytes" "context" "crypto/sha1" "fmt" @@ -204,7 +203,23 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } - ps := sourceignore.GetPatterns(bytes.NewBufferString(*bucket.Spec.Ignore), nil) + // Look for file with ignore rules first + // NB: S3 has flat filepath keys making it impossible to look + // for files in "subdirectories" without building up a tree first. + path := filepath.Join(tempDir, sourceignore.IgnoreFile) + if err := s3Client.FGetObject(ctxTimeout, bucket.Spec.BucketName, sourceignore.IgnoreFile, path, minio.GetObjectOptions{}); err != nil { + if resp, ok := err.(minio.ErrorResponse); ok && resp.Code != "NoSuchKey" { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err + } + } + ps, err := sourceignore.ReadIgnoreFile(path, nil) + if err != nil { + return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err + } + // In-spec patterns take precedence + if bucket.Spec.Ignore != nil { + ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*bucket.Spec.Ignore), nil)...) + } matcher := sourceignore.NewMatcher(ps) // download bucket content @@ -217,7 +232,7 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket return sourcev1.BucketNotReady(bucket, sourcev1.BucketOperationFailedReason, err.Error()), err } - if strings.HasSuffix(object.Key, "/") { + if strings.HasSuffix(object.Key, "/") || object.Key == sourceignore.IgnoreFile { continue } @@ -264,7 +279,7 @@ func (r *BucketReconciler) reconcile(ctx context.Context, bucket sourcev1.Bucket defer unlock() // archive artifact and check integrity - if err := r.Storage.Archive(&artifact, tempDir, bucket.Spec.Ignore); err != nil { + if err := r.Storage.Archive(&artifact, tempDir, nil); err != nil { err = fmt.Errorf("storage archive error: %w", err) return sourcev1.BucketNotReady(bucket, sourcev1.StorageOperationFailedReason, err.Error()), err } diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index 00986aee..db3bd54e 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -21,6 +21,7 @@ import ( "fmt" "io/ioutil" "os" + "strings" "time" "github.com/go-logr/logr" @@ -45,6 +46,7 @@ import ( sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" "github.com/fluxcd/source-controller/pkg/git" "github.com/fluxcd/source-controller/pkg/git/strategy" + "github.com/fluxcd/source-controller/pkg/sourceignore" ) // +kubebuilder:rbac:groups=source.toolkit.fluxcd.io,resources=gitrepositories,verbs=get;list;watch;create;update;patch;delete @@ -270,7 +272,15 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour defer unlock() // archive artifact and check integrity - if err := r.Storage.Archive(&artifact, tmpGit, repository.Spec.Ignore); err != nil { + ps, err := sourceignore.LoadIgnorePatterns(tmpGit, nil) + if err != nil { + err = fmt.Errorf(".sourceignore error: %w", err) + return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err + } + if repository.Spec.Ignore != nil { + ps = append(ps, sourceignore.ReadPatterns(strings.NewReader(*repository.Spec.Ignore), nil)...) + } + if err := r.Storage.Archive(&artifact, tmpGit, SourceIgnoreFilter(ps, nil)); err != nil { err = fmt.Errorf("storage archive error: %w", err) return sourcev1.GitRepositoryNotReady(repository, sourcev1.StorageOperationFailedReason, err.Error()), err } diff --git a/controllers/storage.go b/controllers/storage.go index a5f6f576..206f755f 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -40,14 +40,6 @@ import ( "github.com/fluxcd/source-controller/pkg/sourceignore" ) -const ( - excludeFile = ".sourceignore" - excludeVCS = ".git/,.gitignore,.gitmodules,.gitattributes" - excludeExt = "*.jpg,*.jpeg,*.gif,*.png,*.wmv,*.flv,*.tar.gz,*.zip" - excludeCI = ".github/,.circleci/,.travis.yml,.gitlab-ci.yml,appveyor.yml,.drone.yml,cloudbuild.yaml,codeship-services.yml,codeship-steps.yml" - excludeExtra = "**/.goreleaser.yml,**/.sops.yaml,**/.flux.yaml" -) - // Storage manages artifacts type Storage struct { // BasePath is the local directory path where the source artifacts are stored. @@ -150,20 +142,36 @@ func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool { return fi.Mode().IsRegular() } -// Archive atomically archives the given directory as a tarball to the given v1beta1.Artifact -// path, excluding any VCS specific files and directories, or any of the excludes defined in -// the excludeFiles. If successful, it sets the checksum and last update time on the artifact. -func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, ignore *string) (err error) { +// ArchiveFileFilter must return true if a file should not be included +// in the archive after inspecting the given path and/or os.FileInfo. +type ArchiveFileFilter func(p string, fi os.FileInfo) bool + +// SourceIgnoreFilter returns an ArchiveFileFilter that filters out +// files matching sourceignore.VCSPatterns and any of the provided +// patterns. If an empty gitignore.Pattern slice is given, the matcher +// is set to sourceignore.NewDefaultMatcher. +func SourceIgnoreFilter(ps []gitignore.Pattern, domain []string) ArchiveFileFilter { + matcher := sourceignore.NewDefaultMatcher(ps, domain) + if len(ps) > 0 { + ps = append(sourceignore.VCSPatterns(domain), ps...) + matcher = sourceignore.NewMatcher(ps) + } + return func(p string, fi os.FileInfo) bool { + // The directory is always false as the archiver does already skip + // directories. + return matcher.Match(strings.Split(p, string(filepath.Separator)), false) + } +} + +// Archive atomically archives the given directory as a tarball to the +// given v1beta1.Artifact path, excluding directories and any +// ArchiveFileFilter matches. If successful, it sets the checksum and +// last update time on the artifact. +func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter ArchiveFileFilter) (err error) { if f, err := os.Stat(dir); os.IsNotExist(err) || !f.IsDir() { return fmt.Errorf("invalid dir path: %s", dir) } - ps, err := sourceignore.LoadExcludePatterns(dir, ignore) - if err != nil { - return err - } - matcher := sourceignore.NewMatcher(ps) - localPath := s.LocalPath(*artifact) tf, err := ioutil.TempFile(filepath.Split(localPath)) if err != nil { @@ -181,7 +189,52 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, ignore *strin gw := gzip.NewWriter(mw) tw := tar.NewWriter(gw) - if err := writeToArchiveExcludeMatches(dir, matcher, tw); err != nil { + if err := filepath.Walk(dir, func(p string, fi os.FileInfo, err error) error { + if err != nil { + return err + } + + // Ignore anything that is not a file (directories, symlinks) + if !fi.Mode().IsRegular() { + return nil + } + + // Skip filtered files + if filter != nil && filter(p, fi) { + return nil + } + + header, err := tar.FileInfoHeader(fi, p) + if err != nil { + return err + } + // The name needs to be modified to maintain directory structure + // as tar.FileInfoHeader only has access to the base name of the file. + // Ref: https://golang.org/src/archive/tar/common.go?#L626 + relFilePath := p + if filepath.IsAbs(dir) { + relFilePath, err = filepath.Rel(dir, p) + if err != nil { + return err + } + } + header.Name = relFilePath + + if err := tw.WriteHeader(header); err != nil { + return err + } + + f, err := os.Open(p) + if err != nil { + f.Close() + return err + } + if _, err := io.Copy(tw, f); err != nil { + f.Close() + return err + } + return f.Close() + }); err != nil { tw.Close() gw.Close() tf.Close() @@ -214,58 +267,6 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, ignore *strin return nil } -// writeToArchiveExcludeMatches walks over the given dir and writes any regular file that does -// not match the given gitignore.Matcher. -func writeToArchiveExcludeMatches(dir string, matcher gitignore.Matcher, writer *tar.Writer) error { - fn := func(p string, fi os.FileInfo, err error) error { - if err != nil { - return err - } - - // Ignore anything that is not a file (directories, symlinks) - if !fi.Mode().IsRegular() { - return nil - } - - // Ignore excluded extensions and files - if matcher.Match(strings.Split(p, "/"), false) { - return nil - } - - header, err := tar.FileInfoHeader(fi, p) - if err != nil { - return err - } - // The name needs to be modified to maintain directory structure - // as tar.FileInfoHeader only has access to the base name of the file. - // Ref: https://golang.org/src/archive/tar/common.go?#L626 - relFilePath := p - if filepath.IsAbs(dir) { - relFilePath, err = filepath.Rel(dir, p) - if err != nil { - return err - } - } - header.Name = relFilePath - - if err := writer.WriteHeader(header); err != nil { - return err - } - - f, err := os.Open(p) - if err != nil { - f.Close() - return err - } - if _, err := io.Copy(writer, f); err != nil { - f.Close() - return err - } - return f.Close() - } - return filepath.Walk(dir, fn) -} - // AtomicWriteFile atomically writes the io.Reader contents to the v1beta1.Artifact path. // If successful, it sets the checksum and last update time on the artifact. func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader, mode os.FileMode) (err error) { diff --git a/controllers/storage_test.go b/controllers/storage_test.go index 3271c579..a79df6a1 100644 --- a/controllers/storage_test.go +++ b/controllers/storage_test.go @@ -1,3 +1,19 @@ +/* +Copyright 2020, 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package controllers import ( @@ -7,28 +23,16 @@ import ( "io" "io/ioutil" "os" - "os/exec" "path" "path/filepath" "testing" "time" + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + sourcev1 "github.com/fluxcd/source-controller/api/v1beta1" ) -type ignoreMap map[string]bool - -var remoteRepository = "https://github.com/fluxcd/source-controller" - -func init() { - // if this remote repo ever gets in your way, this is an escape; just set - // this to the url you want to clone. Be the source you want to be. - s := os.Getenv("REMOTE_REPOSITORY") - if s != "" { - remoteRepository = s - } -} - func createStoragePath() (string, error) { return ioutil.TempDir("", "") } @@ -67,16 +71,16 @@ func TestStorageConstructor(t *testing.T) { // walks a tar.gz and looks for paths with the basename. It does not match // symlinks properly at this time because that's painful. -func walkTar(tarFile string, match string) (bool, error) { +func walkTar(tarFile string, match string) (int64, bool, error) { f, err := os.Open(tarFile) if err != nil { - return false, fmt.Errorf("could not open file: %w", err) + return 0, false, fmt.Errorf("could not open file: %w", err) } defer f.Close() gzr, err := gzip.NewReader(f) if err != nil { - return false, fmt.Errorf("could not unzip file: %w", err) + return 0, false, fmt.Errorf("could not unzip file: %w", err) } defer gzr.Close() @@ -86,177 +90,156 @@ func walkTar(tarFile string, match string) (bool, error) { if err == io.EOF { break } else if err != nil { - return false, fmt.Errorf("Corrupt tarball reading header: %w", err) + return 0, false, fmt.Errorf("corrupt tarball reading header: %w", err) } switch header.Typeflag { case tar.TypeDir, tar.TypeReg: - if filepath.Base(header.Name) == match { - return true, nil + if header.Name == match { + return header.Size, true, nil } default: // skip } } - return false, nil + return 0, false, nil } -func testPatterns(t *testing.T, storage *Storage, artifact sourcev1.Artifact, table ignoreMap) { - for name, expected := range table { - res, err := walkTar(storage.LocalPath(artifact), name) - if err != nil { - t.Fatalf("while reading tarball: %v", err) - } +func TestStorage_Archive(t *testing.T) { + dir, err := createStoragePath() + if err != nil { + t.Fatal(err) + } + t.Cleanup(cleanupStoragePath(dir)) - if res != expected { - if expected { - t.Fatalf("Could not find repository file matching %q in tarball for repo %q", name, remoteRepository) - } else { - t.Fatalf("Repository contained ignored file %q in tarball for repo %q", name, remoteRepository) + storage, err := NewStorage(dir, "hostname", time.Minute) + if err != nil { + t.Fatalf("error while bootstrapping storage: %v", err) + } + + createFiles := func(files map[string][]byte) (dir string, err error) { + defer func() { + if err != nil && dir != "" { + os.RemoveAll(dir) + } + }() + dir, err = ioutil.TempDir("", "archive-test-files-") + if err != nil { + return + } + for name, b := range files { + absPath := filepath.Join(dir, name) + if err = os.MkdirAll(filepath.Dir(absPath), 0755); err != nil { + return + } + f, err := os.Create(absPath) + if err != nil { + return "", fmt.Errorf("could not create file %q: %w", absPath, err) + } + if n, err := f.Write(b); err != nil { + f.Close() + return "", fmt.Errorf("could not write %d bytes to file %q: %w", n, f.Name(), err) + } + f.Close() + } + return + } + + matchFiles := func(t *testing.T, storage *Storage, artifact sourcev1.Artifact, files map[string][]byte) { + for name, b := range files { + mustExist := !(name[0:1] == "!") + if !mustExist { + name = name[1:] + } + s, exist, err := walkTar(storage.LocalPath(artifact), name) + if err != nil { + t.Fatalf("failed reading tarball: %v", err) + } + if bs := int64(len(b)); s != bs { + t.Fatalf("%q size %v != %v", name, s, bs) + } + if exist != mustExist { + if mustExist { + t.Errorf("could not find file %q in tarball", name) + } else { + t.Errorf("tarball contained excluded file %q", name) + } } } } -} -func createArchive(t *testing.T, storage *Storage, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact { - gitDir, err := ioutil.TempDir("", "") - if err != nil { - t.Fatalf("could not create temporary directory: %v", err) + tests := []struct { + name string + files map[string][]byte + filter ArchiveFileFilter + want map[string][]byte + wantErr bool + }{ + { + name: "no filter", + files: map[string][]byte{ + ".git/config": nil, + "file.jpg": []byte(`contents`), + "manifest.yaml": nil, + }, + filter: nil, + want: map[string][]byte{ + ".git/config": nil, + "file.jpg": []byte(`contents`), + "manifest.yaml": nil, + }, + }, + { + name: "exclude VCS", + files: map[string][]byte{ + ".git/config": nil, + "manifest.yaml": nil, + }, + filter: SourceIgnoreFilter(nil, nil), + want: map[string][]byte{ + "!.git/config": nil, + "manifest.yaml": nil, + }, + }, + { + name: "custom", + files: map[string][]byte{ + ".git/config": nil, + "custom": nil, + "horse.jpg": nil, + }, + filter: SourceIgnoreFilter([]gitignore.Pattern{ + gitignore.ParsePattern("custom", nil), + }, nil), + want: map[string][]byte{ + "!git/config": nil, + "!custom": nil, + "horse.jpg": nil, + }, + wantErr: false, + }, } - t.Cleanup(func() { os.RemoveAll(gitDir) }) - - if err := exec.Command("git", "clone", remoteRepository, gitDir).Run(); err != nil { - t.Fatalf("Could not clone remote repository: %v", err) + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + dir, err := createFiles(tt.files) + if err != nil { + t.Error(err) + return + } + defer os.RemoveAll(dir) + artifact := sourcev1.Artifact{ + Path: filepath.Join(randStringRunes(10), randStringRunes(10), randStringRunes(10)+".tar.gz"), + } + if err := storage.MkdirAll(artifact); err != nil { + t.Fatalf("artifact directory creation failed: %v", err) + } + if err := storage.Archive(&artifact, dir, tt.filter); (err != nil) != tt.wantErr { + t.Errorf("Archive() error = %v, wantErr %v", err, tt.wantErr) + } + matchFiles(t, storage, artifact, tt.want) + }) } - - // inject files.. just empty files - for _, name := range filenames { - f, err := os.Create(filepath.Join(gitDir, name)) - if err != nil { - t.Fatalf("Could not inject filename %q: %v", name, err) - } - f.Close() - } - - // inject sourceignore if not empty - if sourceIgnore != "" { - si, err := os.Create(filepath.Join(gitDir, ".sourceignore")) - if err != nil { - t.Fatalf("Could not create .sourceignore: %v", err) - } - - if _, err := io.WriteString(si, sourceIgnore); err != nil { - t.Fatalf("Could not write to .sourceignore: %v", err) - } - - si.Close() - } - artifact := sourcev1.Artifact{ - Path: filepath.Join(randStringRunes(10), randStringRunes(10), randStringRunes(10)+".tar.gz"), - } - if err := storage.MkdirAll(artifact); err != nil { - t.Fatalf("artifact directory creation failed: %v", err) - } - - if err := storage.Archive(&artifact, gitDir, spec.Ignore); err != nil { - t.Fatalf("archiving failed: %v", err) - } - - if !storage.ArtifactExist(artifact) { - t.Fatalf("artifact was created but does not exist: %+v", artifact) - } - - return artifact -} - -func stringPtr(s string) *string { - return &s -} - -func TestArchiveBasic(t *testing.T) { - table := ignoreMap{ - "README.md": true, - ".gitignore": false, - } - - dir, err := createStoragePath() - if err != nil { - t.Fatal(err) - } - t.Cleanup(cleanupStoragePath(dir)) - - storage, err := NewStorage(dir, "hostname", time.Minute) - if err != nil { - t.Fatalf("Error while bootstrapping storage: %v", err) - } - - testPatterns(t, storage, createArchive(t, storage, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table) -} - -func TestArchiveIgnore(t *testing.T) { - // this is a list of files that will be created in the repository for each - // subtest. it is manipulated later on. - filenames := []string{ - "foo.tar.gz", - "bar.jpg", - "bar.gif", - "foo.jpeg", - "video.flv", - "video.wmv", - "bar.png", - "foo.zip", - ".drone.yml", - ".flux.yaml", - } - - // this is the table of ignored files and their values. true means that it's - // present in the resulting tarball. - table := ignoreMap{} - for _, item := range filenames { - table[item] = false - } - - dir, err := createStoragePath() - if err != nil { - t.Fatal(err) - } - t.Cleanup(cleanupStoragePath(dir)) - - storage, err := NewStorage(dir, "hostname", time.Minute) - if err != nil { - t.Fatalf("Error while bootstrapping storage: %v", err) - } - - t.Run("automatically ignored files", func(t *testing.T) { - testPatterns(t, storage, createArchive(t, storage, filenames, "", sourcev1.GitRepositorySpec{}), table) - }) - - table = ignoreMap{} - for _, item := range filenames { - table[item] = true - } - - t.Run("only vcs ignored files", func(t *testing.T) { - testPatterns(t, storage, createArchive(t, storage, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table) - }) - - filenames = append(filenames, "test.txt") - table["test.txt"] = false - sourceIgnoreFile := "*.txt" - - t.Run("sourceignore injected via CRD", func(t *testing.T) { - testPatterns(t, storage, createArchive(t, storage, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table) - }) - - table = ignoreMap{} - for _, item := range filenames { - table[item] = false - } - - t.Run("sourceignore injected via filename", func(t *testing.T) { - testPatterns(t, storage, createArchive(t, storage, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table) - }) } func TestStorageRemoveAllButCurrent(t *testing.T) { diff --git a/go.mod b/go.mod index eac89da8..52d18006 100644 --- a/go.mod +++ b/go.mod @@ -28,6 +28,7 @@ require ( github.com/spf13/pflag v1.0.5 golang.org/x/crypto v0.0.0-20210322153248-0c34fe9e7dc2 golang.org/x/sync v0.0.0-20201207232520-09787c993a3a + gotest.tools v2.2.0+incompatible helm.sh/helm/v3 v3.5.3 k8s.io/api v0.20.2 k8s.io/apimachinery v0.20.2 diff --git a/pkg/sourceignore/sourceignore.go b/pkg/sourceignore/sourceignore.go index dc65468f..b4e0bf50 100644 --- a/pkg/sourceignore/sourceignore.go +++ b/pkg/sourceignore/sourceignore.go @@ -18,8 +18,8 @@ package sourceignore import ( "bufio" - "bytes" "io" + "io/ioutil" "os" "path/filepath" "strings" @@ -28,7 +28,7 @@ import ( ) const ( - ExcludeFile = ".sourceignore" + IgnoreFile = ".sourceignore" ExcludeVCS = ".git/,.gitignore,.gitmodules,.gitattributes" ExcludeExt = "*.jpg,*.jpeg,*.gif,*.png,*.wmv,*.flv,*.tar.gz,*.zip" ExcludeCI = ".github/,.circleci/,.travis.yml,.gitlab-ci.yml,appveyor.yml,.drone.yml,cloudbuild.yaml,codeship-services.yml,codeship-steps.yml" @@ -41,47 +41,85 @@ func NewMatcher(ps []gitignore.Pattern) gitignore.Matcher { return gitignore.NewMatcher(ps) } -// GetPatterns collects ignore patterns from the given reader and -// returns them as a gitignore.Pattern slice. -func GetPatterns(reader io.Reader, path []string) []gitignore.Pattern { +// NewDefaultMatcher returns a gitignore.Matcher with the DefaultPatterns +// as lowest priority patterns. +func NewDefaultMatcher(ps []gitignore.Pattern, domain []string) gitignore.Matcher { + var defaultPs []gitignore.Pattern + defaultPs = append(defaultPs, VCSPatterns(domain)...) + defaultPs = append(defaultPs, DefaultPatterns(domain)...) + ps = append(defaultPs, ps...) + return gitignore.NewMatcher(ps) +} + +// VCSPatterns returns a gitignore.Pattern slice with ExcludeVCS +// patterns. +func VCSPatterns(domain []string) []gitignore.Pattern { var ps []gitignore.Pattern - scanner := bufio.NewScanner(reader) - - for scanner.Scan() { - s := scanner.Text() - if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 { - ps = append(ps, gitignore.ParsePattern(s, path)) - } + for _, p := range strings.Split(ExcludeVCS, ",") { + ps = append(ps, gitignore.ParsePattern(p, domain)) } - return ps } -// LoadExcludePatterns loads the excluded patterns from .sourceignore or other -// sources and returns the gitignore.Pattern slice. -func LoadExcludePatterns(dir string, ignore *string) ([]gitignore.Pattern, error) { - path := strings.Split(dir, "/") - +// DefaultPatterns returns a gitignore.Pattern slice with the default +// ExcludeExt, ExcludeCI, ExcludeExtra patterns. +func DefaultPatterns(domain []string) []gitignore.Pattern { + all := strings.Join([]string{ExcludeExt, ExcludeCI, ExcludeExtra}, ",") var ps []gitignore.Pattern - for _, p := range strings.Split(ExcludeVCS, ",") { - ps = append(ps, gitignore.ParsePattern(p, path)) + for _, p := range strings.Split(all, ",") { + ps = append(ps, gitignore.ParsePattern(p, domain)) } + return ps +} - if ignore == nil { - all := strings.Join([]string{ExcludeExt, ExcludeCI, ExcludeExtra}, ",") - for _, p := range strings.Split(all, ",") { - ps = append(ps, gitignore.ParsePattern(p, path)) +// ReadPatterns collects ignore patterns from the given reader and +// returns them as a gitignore.Pattern slice. +// If a domain is supplied, this is used as the scope of the read +// patterns. +func ReadPatterns(reader io.Reader, domain []string) []gitignore.Pattern { + var ps []gitignore.Pattern + scanner := bufio.NewScanner(reader) + for scanner.Scan() { + s := scanner.Text() + if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 { + ps = append(ps, gitignore.ParsePattern(s, domain)) } - - if f, err := os.Open(filepath.Join(dir, ExcludeFile)); err == nil { - defer f.Close() - ps = append(ps, GetPatterns(f, path)...) - } else if !os.IsNotExist(err) { - return nil, err - } - } else { - ps = append(ps, GetPatterns(bytes.NewBufferString(*ignore), path)...) } + return ps +} +// ReadIgnoreFile attempts to read the file at the given path and +// returns the read patterns. +func ReadIgnoreFile(path string, domain []string) ([]gitignore.Pattern, error) { + var ps []gitignore.Pattern + if f, err := os.Open(path); err == nil { + defer f.Close() + ps = append(ps, ReadPatterns(f, domain)...) + } else if !os.IsNotExist(err) { + return nil, err + } + return ps, nil +} + +// LoadIgnorePatterns recursively loads the the IgnoreFile patterns found +// in the directory. +func LoadIgnorePatterns(dir string, domain []string) ([]gitignore.Pattern, error) { + ps, err := ReadIgnoreFile(filepath.Join(dir, IgnoreFile), domain) + if err != nil { + return nil, err + } + fis, err := ioutil.ReadDir(dir) + if err != nil { + return nil, err + } + for _, fi := range fis { + if fi.IsDir() && fi.Name() != ".git" { + var subps []gitignore.Pattern + subps, err = LoadIgnorePatterns(filepath.Join(dir, fi.Name()), append(domain, fi.Name())) + if len(subps) > 0 { + ps = append(ps, subps...) + } + } + } return ps, nil } diff --git a/pkg/sourceignore/sourceignore_test.go b/pkg/sourceignore/sourceignore_test.go new file mode 100644 index 00000000..98a88d7e --- /dev/null +++ b/pkg/sourceignore/sourceignore_test.go @@ -0,0 +1,261 @@ +/* +Copyright 2021 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package sourceignore + +import ( + "io/ioutil" + "os" + "path/filepath" + "reflect" + "strings" + "testing" + + "github.com/go-git/go-git/v5/plumbing/format/gitignore" + "gotest.tools/assert" +) + +func TestReadPatterns(t *testing.T) { + tests := []struct { + name string + ignore string + domain []string + matches []string + mismatches []string + }{ + { + name: "simple", + ignore: `ignore-dir/* +!ignore-dir/include +`, + matches: []string{"ignore-dir/file.yaml"}, + mismatches: []string{"file.yaml", "ignore-dir/include"}, + }, + { + name: "with comments", + ignore: `ignore-dir/* +# !ignore-dir/include`, + matches: []string{"ignore-dir/file.yaml", "ignore-dir/include"}, + }, + { + name: "domain scoped", + domain: []string{"domain", "scoped"}, + ignore: "ignore-dir/*", + matches: []string{"domain/scoped/ignore-dir/file.yaml"}, + mismatches: []string{"ignore-dir/file.yaml"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + reader := strings.NewReader(tt.ignore) + ps := ReadPatterns(reader, tt.domain) + matcher := NewMatcher(ps) + for _, m := range tt.matches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m) + } + for _, m := range tt.mismatches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m) + } + }) + } +} + +func TestReadIgnoreFile(t *testing.T) { + f, err := ioutil.TempFile("", IgnoreFile) + if err != nil { + t.Fatal(err) + } + defer os.Remove(f.Name()) + if _, err = f.Write([]byte(`# .sourceignore +ignore-this.txt`)); err != nil { + t.Fatal(err) + } + f.Close() + + tests := []struct { + name string + path string + domain []string + want []gitignore.Pattern + }{ + { + name: IgnoreFile, + path: f.Name(), + want: []gitignore.Pattern{ + gitignore.ParsePattern("ignore-this.txt", nil), + }, + }, + { + name: "with domain", + path: f.Name(), + domain: strings.Split(filepath.Dir(f.Name()), string(filepath.Separator)), + want: []gitignore.Pattern{ + gitignore.ParsePattern("ignore-this.txt", strings.Split(filepath.Dir(f.Name()), string(filepath.Separator))), + }, + }, + { + name: "non existing", + path: "", + want: nil, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := ReadIgnoreFile(tt.path, tt.domain) + if err != nil { + t.Error(err) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("ReadIgnoreFile() got = %d, want %#v", got, tt.want) + } + }) + } +} + +func TestVCSPatterns(t *testing.T) { + tests := []struct { + name string + domain []string + patterns []gitignore.Pattern + matches []string + mismatches []string + }{ + { + name: "simple matches", + matches: []string{".git/config", ".gitignore"}, + mismatches: []string{"workload.yaml", "workload.yml", "simple.txt"}, + }, + { + name: "domain scoped matches", + domain: []string{"directory"}, + matches: []string{"directory/.git/config", "directory/.gitignore"}, + mismatches: []string{"other/.git/config"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := NewDefaultMatcher(tt.patterns, tt.domain) + for _, m := range tt.matches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m) + } + for _, m := range tt.mismatches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m) + } + }) + } +} + +func TestDefaultPatterns(t *testing.T) { + tests := []struct { + name string + domain []string + patterns []gitignore.Pattern + matches []string + mismatches []string + }{ + { + name: "simple matches", + matches: []string{"image.jpg", "archive.tar.gz", ".github/workflows/workflow.yaml", "subdir/.flux.yaml", "subdir2/.sops.yaml"}, + mismatches: []string{"workload.yaml", "workload.yml", "simple.txt"}, + }, + { + name: "domain scoped matches", + domain: []string{"directory"}, + matches: []string{"directory/image.jpg", "directory/archive.tar.gz"}, + mismatches: []string{"other/image.jpg", "other/archive.tar.gz"}, + }, + { + name: "patterns", + patterns: []gitignore.Pattern{gitignore.ParsePattern("!*.jpg", nil)}, + mismatches: []string{"image.jpg"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + matcher := NewDefaultMatcher(tt.patterns, tt.domain) + for _, m := range tt.matches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m) + } + for _, m := range tt.mismatches { + assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m) + } + }) + } +} + +func TestLoadExcludePatterns(t *testing.T) { + tmpDir, err := ioutil.TempDir("", "sourceignore-load-") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpDir) + files := map[string]string{ + ".sourceignore": "root.txt", + "d/.gitignore": "ignored", + "z/.sourceignore": "last.txt", + "a/b/.sourceignore": "subdir.txt", + } + for n, c := range files { + if err = os.MkdirAll(filepath.Join(tmpDir, filepath.Dir(n)), 0755); err != nil { + t.Fatal(err) + } + if err = os.WriteFile(filepath.Join(tmpDir, n), []byte(c), 0644); err != nil { + t.Fatal(err) + } + } + tests := []struct { + name string + dir string + domain []string + want []gitignore.Pattern + }{ + { + name: "traverse loads", + dir: tmpDir, + want: []gitignore.Pattern{ + gitignore.ParsePattern("root.txt", nil), + gitignore.ParsePattern("subdir.txt", []string{"a", "b"}), + gitignore.ParsePattern("last.txt", []string{"z"}), + }, + }, + { + name: "domain", + dir: tmpDir, + domain: strings.Split(tmpDir, string(filepath.Separator)), + want: []gitignore.Pattern{ + gitignore.ParsePattern("root.txt", strings.Split(tmpDir, string(filepath.Separator))), + gitignore.ParsePattern("subdir.txt", append(strings.Split(tmpDir, string(filepath.Separator)), "a", "b")), + gitignore.ParsePattern("last.txt", append(strings.Split(tmpDir, string(filepath.Separator)), "z")), + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := LoadIgnorePatterns(tt.dir, tt.domain) + if err != nil { + t.Error(err) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("LoadIgnorePatterns() got = %#v, want %#v", got, tt.want) + for _, v := range got { + t.Error(v) + } + } + }) + } +}