Use sourceignore from fluxcd/pkg

Signed-off-by: Stefan Prodan <stefan.prodan@gmail.com>
2022-08-16 14:50:42 +03:00 · 2022-08-16 14:50:42 +03:00 · f873d71ec5
parent fc5dc4dc7c
commit f873d71ec5
8 changed files with 10 additions and 389 deletions
--- a/controllers/bucket_controller.go
+++ b/controllers/bucket_controller.go
@ -50,13 +50,13 @@ import (
 	"github.com/fluxcd/pkg/runtime/patch"
 	"github.com/fluxcd/pkg/runtime/predicates"

+	"github.com/fluxcd/pkg/sourceignore"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 	serror "github.com/fluxcd/source-controller/internal/error"
 	sreconcile "github.com/fluxcd/source-controller/internal/reconcile"
 	"github.com/fluxcd/source-controller/internal/reconcile/summarize"
 	"github.com/fluxcd/source-controller/pkg/gcp"
 	"github.com/fluxcd/source-controller/pkg/minio"
-	"github.com/fluxcd/source-controller/pkg/sourceignore"
 )

 // maxConcurrentBucketFetches is the upper bound on the goroutines used to
--- a/controllers/gitrepository_controller.go
+++ b/controllers/gitrepository_controller.go
@ -49,6 +49,7 @@ import (
 	"github.com/fluxcd/pkg/runtime/patch"
 	"github.com/fluxcd/pkg/runtime/predicates"

+	"github.com/fluxcd/pkg/sourceignore"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 	serror "github.com/fluxcd/source-controller/internal/error"
 	"github.com/fluxcd/source-controller/internal/features"
@ -57,7 +58,6 @@ import (
 	"github.com/fluxcd/source-controller/internal/util"
 	"github.com/fluxcd/source-controller/pkg/git"
 	"github.com/fluxcd/source-controller/pkg/git/strategy"
-	"github.com/fluxcd/source-controller/pkg/sourceignore"
 )

 // gitRepositoryReadyCondition contains the information required to summarize a
--- a/controllers/storage.go
+++ b/controllers/storage.go
@ -39,9 +39,9 @@ import (
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"

+	"github.com/fluxcd/pkg/sourceignore"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 	sourcefs "github.com/fluxcd/source-controller/internal/fs"
-	"github.com/fluxcd/source-controller/pkg/sourceignore"
 )

 const GarbageCountLimit = 1000
--- a/go.mod
+++ b/go.mod
@ -38,8 +38,9 @@ require (
 	github.com/fluxcd/pkg/helmtestserver v0.7.4
 	github.com/fluxcd/pkg/lockedfile v0.1.0
 	github.com/fluxcd/pkg/masktoken v0.0.1
-	github.com/fluxcd/pkg/oci v0.3.0
+	github.com/fluxcd/pkg/oci v0.5.0
 	github.com/fluxcd/pkg/runtime v0.16.2
+	github.com/fluxcd/pkg/sourceignore v0.1.0
 	github.com/fluxcd/pkg/ssh v0.5.0
 	github.com/fluxcd/pkg/testserver v0.2.0
 	github.com/fluxcd/pkg/untar v0.1.0
--- a/go.sum
+++ b/go.sum
@ -401,10 +401,12 @@ github.com/fluxcd/pkg/lockedfile v0.1.0 h1:YsYFAkd6wawMCcD74ikadAKXA4s2sukdxrn7w
 github.com/fluxcd/pkg/lockedfile v0.1.0/go.mod h1:EJLan8t9MiOcgTs8+puDjbE6I/KAfHbdvIy9VUgIjm8=
 github.com/fluxcd/pkg/masktoken v0.0.1 h1:egWR/ibTzf4L3PxE8TauKO1srD1Ye/aalgQRQuKKRdU=
 github.com/fluxcd/pkg/masktoken v0.0.1/go.mod h1:sQmMtX4s5RwdGlByJazzNasWFFgBdmtNcgeZcGBI72Y=
-github.com/fluxcd/pkg/oci v0.3.0 h1:GFn6JZeg5fV2K4vsQ0s5lJFid6qrpA4RybLXL+7qUbQ=
-github.com/fluxcd/pkg/oci v0.3.0/go.mod h1:c1pj9E/G5927gSa6ooACAyZe+HwjgmPk9johL7oXDHw=
+github.com/fluxcd/pkg/oci v0.5.0 h1:ghVKxvDmHoMXohRRsyDIISZPAXbA6hxz7IGX/EyvUZw=
+github.com/fluxcd/pkg/oci v0.5.0/go.mod h1:qEzBvOJvWKrIKIhCKw1Nlbgod9ClD8C6A7F2cylXpsA=
 github.com/fluxcd/pkg/runtime v0.16.2 h1:CexfMmJK+r12sHTvKWyAax0pcPomjd6VnaHXcxjUrRY=
 github.com/fluxcd/pkg/runtime v0.16.2/go.mod h1:OHSKsrO+T+Ym8WZRS2oidrnauWRARuE2nfm8ewevm7M=
+github.com/fluxcd/pkg/sourceignore v0.1.0 h1:v36Rqp6FDB7Ntjy7NakdwscOfxFPk14peKa+VMBNugo=
+github.com/fluxcd/pkg/sourceignore v0.1.0/go.mod h1:m9/q+YLMNSWjXns1n/5q3ucwzSSddti+D6ExbNaCo6s=
 github.com/fluxcd/pkg/ssh v0.5.0 h1:jE9F2XvUXC2mgseeXMATvO014fLqdB30/VzlPLKsk20=
 github.com/fluxcd/pkg/ssh v0.5.0/go.mod h1:KGgOUOy1uI6RC6+qxIBLvP1AeOOs/nLB25Ca6TZMIXE=
 github.com/fluxcd/pkg/testserver v0.2.0 h1:Mj0TapmKaywI6Fi5wvt1LAZpakUHmtzWQpJNKQ0Krt4=
--- a/pkg/minio/minio_test.go
+++ b/pkg/minio/minio_test.go
@ -26,8 +26,8 @@ import (
 	"testing"

 	"github.com/fluxcd/pkg/apis/meta"
+	"github.com/fluxcd/pkg/sourceignore"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
-	"github.com/fluxcd/source-controller/pkg/sourceignore"

 	"github.com/google/uuid"
 	miniov7 "github.com/minio/minio-go/v7"
--- a/pkg/sourceignore/sourceignore.go
+++ b/pkg/sourceignore/sourceignore.go
@ -1,126 +0,0 @@
-/*
-Copyright 2021 The Flux authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package sourceignore
-
-import (
-	"bufio"
-	"io"
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
-)
-
-const (
-	IgnoreFile   = ".sourceignore"
-	ExcludeVCS   = ".git/,.gitignore,.gitmodules,.gitattributes"
-	ExcludeExt   = "*.jpg,*.jpeg,*.gif,*.png,*.wmv,*.flv,*.tar.gz,*.zip"
-	ExcludeCI    = ".github/,.circleci/,.travis.yml,.gitlab-ci.yml,appveyor.yml,.drone.yml,cloudbuild.yaml,codeship-services.yml,codeship-steps.yml"
-	ExcludeExtra = "**/.goreleaser.yml,**/.sops.yaml,**/.flux.yaml"
-)
-
-// NewMatcher returns a gitignore.Matcher for the given gitignore.Pattern
-// slice. It mainly exists to compliment the API.
-func NewMatcher(ps []gitignore.Pattern) gitignore.Matcher {
-	return gitignore.NewMatcher(ps)
-}
-
-// NewDefaultMatcher returns a gitignore.Matcher with the DefaultPatterns
-// as lowest priority patterns.
-func NewDefaultMatcher(ps []gitignore.Pattern, domain []string) gitignore.Matcher {
-	var defaultPs []gitignore.Pattern
-	defaultPs = append(defaultPs, VCSPatterns(domain)...)
-	defaultPs = append(defaultPs, DefaultPatterns(domain)...)
-	ps = append(defaultPs, ps...)
-	return gitignore.NewMatcher(ps)
-}
-
-// VCSPatterns returns a gitignore.Pattern slice with ExcludeVCS
-// patterns.
-func VCSPatterns(domain []string) []gitignore.Pattern {
-	var ps []gitignore.Pattern
-	for _, p := range strings.Split(ExcludeVCS, ",") {
-		ps = append(ps, gitignore.ParsePattern(p, domain))
-	}
-	return ps
-}
-
-// DefaultPatterns returns a gitignore.Pattern slice with the default
-// ExcludeExt, ExcludeCI, ExcludeExtra patterns.
-func DefaultPatterns(domain []string) []gitignore.Pattern {
-	all := strings.Join([]string{ExcludeExt, ExcludeCI, ExcludeExtra}, ",")
-	var ps []gitignore.Pattern
-	for _, p := range strings.Split(all, ",") {
-		ps = append(ps, gitignore.ParsePattern(p, domain))
-	}
-	return ps
-}
-
-// ReadPatterns collects ignore patterns from the given reader and
-// returns them as a gitignore.Pattern slice.
-// If a domain is supplied, this is used as the scope of the read
-// patterns.
-func ReadPatterns(reader io.Reader, domain []string) []gitignore.Pattern {
-	var ps []gitignore.Pattern
-	scanner := bufio.NewScanner(reader)
-	for scanner.Scan() {
-		s := scanner.Text()
-		if !strings.HasPrefix(s, "#") && len(strings.TrimSpace(s)) > 0 {
-			ps = append(ps, gitignore.ParsePattern(s, domain))
-		}
-	}
-	return ps
-}
-
-// ReadIgnoreFile attempts to read the file at the given path and
-// returns the read patterns.
-func ReadIgnoreFile(path string, domain []string) ([]gitignore.Pattern, error) {
-	var ps []gitignore.Pattern
-	if f, err := os.Open(path); err == nil {
-		defer f.Close()
-		ps = append(ps, ReadPatterns(f, domain)...)
-	} else if !os.IsNotExist(err) {
-		return nil, err
-	}
-	return ps, nil
-}
-
-// LoadIgnorePatterns recursively loads the IgnoreFile patterns found
-// in the directory.
-func LoadIgnorePatterns(dir string, domain []string) ([]gitignore.Pattern, error) {
-	ps, err := ReadIgnoreFile(filepath.Join(dir, IgnoreFile), domain)
-	if err != nil {
-		return nil, err
-	}
-	fis, err := os.ReadDir(dir)
-	if err != nil {
-		return nil, err
-	}
-	for _, fi := range fis {
-		if fi.IsDir() && fi.Name() != ".git" {
-			var subps []gitignore.Pattern
-			if subps, err = LoadIgnorePatterns(filepath.Join(dir, fi.Name()), append(domain, fi.Name())); err != nil {
-				return nil, err
-			}
-			if len(subps) > 0 {
-				ps = append(ps, subps...)
-			}
-		}
-	}
-	return ps, nil
-}
--- a/pkg/sourceignore/sourceignore_test.go
+++ b/pkg/sourceignore/sourceignore_test.go
@ -1,256 +0,0 @@
-/*
-Copyright 2021 The Flux authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package sourceignore
-
-import (
-	"os"
-	"path/filepath"
-	"reflect"
-	"strings"
-	"testing"
-
-	"github.com/go-git/go-git/v5/plumbing/format/gitignore"
-	"gotest.tools/assert"
-)
-
-func TestReadPatterns(t *testing.T) {
-	tests := []struct {
-		name       string
-		ignore     string
-		domain     []string
-		matches    []string
-		mismatches []string
-	}{
-		{
-			name: "simple",
-			ignore: `ignore-dir/*
-!ignore-dir/include
-`,
-			matches:    []string{"ignore-dir/file.yaml"},
-			mismatches: []string{"file.yaml", "ignore-dir/include"},
-		},
-		{
-			name: "with comments",
-			ignore: `ignore-dir/*
-# !ignore-dir/include`,
-			matches: []string{"ignore-dir/file.yaml", "ignore-dir/include"},
-		},
-		{
-			name:       "domain scoped",
-			domain:     []string{"domain", "scoped"},
-			ignore:     "ignore-dir/*",
-			matches:    []string{"domain/scoped/ignore-dir/file.yaml"},
-			mismatches: []string{"ignore-dir/file.yaml"},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			reader := strings.NewReader(tt.ignore)
-			ps := ReadPatterns(reader, tt.domain)
-			matcher := NewMatcher(ps)
-			for _, m := range tt.matches {
-				assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m)
-			}
-			for _, m := range tt.mismatches {
-				assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m)
-			}
-		})
-	}
-}
-
-func TestReadIgnoreFile(t *testing.T) {
-	f, err := os.CreateTemp("", IgnoreFile)
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.Remove(f.Name())
-	if _, err = f.Write([]byte(`# .sourceignore
-ignore-this.txt`)); err != nil {
-		t.Fatal(err)
-	}
-	f.Close()
-
-	tests := []struct {
-		name   string
-		path   string
-		domain []string
-		want   []gitignore.Pattern
-	}{
-		{
-			name: IgnoreFile,
-			path: f.Name(),
-			want: []gitignore.Pattern{
-				gitignore.ParsePattern("ignore-this.txt", nil),
-			},
-		},
-		{
-			name:   "with domain",
-			path:   f.Name(),
-			domain: strings.Split(filepath.Dir(f.Name()), string(filepath.Separator)),
-			want: []gitignore.Pattern{
-				gitignore.ParsePattern("ignore-this.txt", strings.Split(filepath.Dir(f.Name()), string(filepath.Separator))),
-			},
-		},
-		{
-			name: "non existing",
-			path: "",
-			want: nil,
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := ReadIgnoreFile(tt.path, tt.domain)
-			if err != nil {
-				t.Error(err)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("ReadIgnoreFile() got = %d, want %#v", got, tt.want)
-			}
-		})
-	}
-}
-
-func TestVCSPatterns(t *testing.T) {
-	tests := []struct {
-		name       string
-		domain     []string
-		patterns   []gitignore.Pattern
-		matches    []string
-		mismatches []string
-	}{
-		{
-			name:       "simple matches",
-			matches:    []string{".git/config", ".gitignore"},
-			mismatches: []string{"workload.yaml", "workload.yml", "simple.txt"},
-		},
-		{
-			name:       "domain scoped matches",
-			domain:     []string{"directory"},
-			matches:    []string{"directory/.git/config", "directory/.gitignore"},
-			mismatches: []string{"other/.git/config"},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			matcher := NewDefaultMatcher(tt.patterns, tt.domain)
-			for _, m := range tt.matches {
-				assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m)
-			}
-			for _, m := range tt.mismatches {
-				assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m)
-			}
-		})
-	}
-}
-
-func TestDefaultPatterns(t *testing.T) {
-	tests := []struct {
-		name       string
-		domain     []string
-		patterns   []gitignore.Pattern
-		matches    []string
-		mismatches []string
-	}{
-		{
-			name:       "simple matches",
-			matches:    []string{"image.jpg", "archive.tar.gz", ".github/workflows/workflow.yaml", "subdir/.flux.yaml", "subdir2/.sops.yaml"},
-			mismatches: []string{"workload.yaml", "workload.yml", "simple.txt"},
-		},
-		{
-			name:       "domain scoped matches",
-			domain:     []string{"directory"},
-			matches:    []string{"directory/image.jpg", "directory/archive.tar.gz"},
-			mismatches: []string{"other/image.jpg", "other/archive.tar.gz"},
-		},
-		{
-			name:       "patterns",
-			patterns:   []gitignore.Pattern{gitignore.ParsePattern("!*.jpg", nil)},
-			mismatches: []string{"image.jpg"},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			matcher := NewDefaultMatcher(tt.patterns, tt.domain)
-			for _, m := range tt.matches {
-				assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), true, "expected %s to match", m)
-			}
-			for _, m := range tt.mismatches {
-				assert.Equal(t, matcher.Match(strings.Split(m, "/"), false), false, "expected %s to not match", m)
-			}
-		})
-	}
-}
-
-func TestLoadExcludePatterns(t *testing.T) {
-	tmpDir := t.TempDir()
-	files := map[string]string{
-		".sourceignore":     "root.txt",
-		"d/.gitignore":      "ignored",
-		"z/.sourceignore":   "last.txt",
-		"a/b/.sourceignore": "subdir.txt",
-	}
-	for n, c := range files {
-		if err := os.MkdirAll(filepath.Join(tmpDir, filepath.Dir(n)), 0o750); err != nil {
-			t.Fatal(err)
-		}
-		if err := os.WriteFile(filepath.Join(tmpDir, n), []byte(c), 0o640); err != nil {
-			t.Fatal(err)
-		}
-	}
-	tests := []struct {
-		name   string
-		dir    string
-		domain []string
-		want   []gitignore.Pattern
-	}{
-		{
-			name: "traverse loads",
-			dir:  tmpDir,
-			want: []gitignore.Pattern{
-				gitignore.ParsePattern("root.txt", nil),
-				gitignore.ParsePattern("subdir.txt", []string{"a", "b"}),
-				gitignore.ParsePattern("last.txt", []string{"z"}),
-			},
-		},
-		{
-			name:   "domain",
-			dir:    tmpDir,
-			domain: strings.Split(tmpDir, string(filepath.Separator)),
-			want: []gitignore.Pattern{
-				gitignore.ParsePattern("root.txt", strings.Split(tmpDir, string(filepath.Separator))),
-				gitignore.ParsePattern("subdir.txt", append(strings.Split(tmpDir, string(filepath.Separator)), "a", "b")),
-				gitignore.ParsePattern("last.txt", append(strings.Split(tmpDir, string(filepath.Separator)), "z")),
-			},
-		},
-	}
-	for _, tt := range tests {
-		t.Run(tt.name, func(t *testing.T) {
-			got, err := LoadIgnorePatterns(tt.dir, tt.domain)
-			if err != nil {
-				t.Error(err)
-				return
-			}
-			if !reflect.DeepEqual(got, tt.want) {
-				t.Errorf("LoadIgnorePatterns() got = %#v, want %#v", got, tt.want)
-				for _, v := range got {
-					t.Error(v)
-				}
-			}
-		})
-	}
-}