helm: copy internal ignore and sympath modules

We require these to be able to mimic Helm's own directory loader, and
surprisingly (for `ignore` at least), these are not public.

Signed-off-by: Hidde Beydals <hello@hidde.co>

internal/helm/chart/loader/ignore/doc.go

@@ -0,0 +1,67 @@
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*Package ignore provides tools for writing ignore files (a la .gitignore).
+
+This provides both an ignore parser and a file-aware processor.
+
+The format of ignore files closely follows, but does not exactly match, the
+format for .gitignore files (https://git-scm.com/docs/gitignore).
+
+The formatting rules are as follows:
+
+	- Parsing is line-by-line
+	- Empty lines are ignored
+	- Lines the begin with # (comments) will be ignored
+	- Leading and trailing spaces are always ignored
+	- Inline comments are NOT supported ('foo* # Any foo' does not contain a comment)
+	- There is no support for multi-line patterns
+	- Shell glob patterns are supported. See Go's "path/filepath".Match
+	- If a pattern begins with a leading !, the match will be negated.
+	- If a pattern begins with a leading /, only paths relatively rooted will match.
+	- If the pattern ends with a trailing /, only directories will match
+	- If a pattern contains no slashes, file basenames are tested (not paths)
+	- The pattern sequence "**", while legal in a glob, will cause an error here
+	  (to indicate incompatibility with .gitignore).
+
+Example:
+
+	# Match any file named foo.txt
+	foo.txt
+
+	# Match any text file
+	*.txt
+
+	# Match only directories named mydir
+	mydir/
+
+	# Match only text files in the top-level directory
+	/*.txt
+
+	# Match only the file foo.txt in the top-level directory
+	/foo.txt
+
+	# Match any file named ab.txt, ac.txt, or ad.txt
+	a[b-d].txt
+
+Notable differences from .gitignore:
+	- The '**' syntax is not supported.
+	- The globbing library is Go's 'filepath.Match', not fnmatch(3)
+	- Trailing spaces are always ignored (there is no supported escape sequence)
+	- The evaluation of escape sequences has not been tested for compatibility
+	- There is no support for '\!' as a special leading sequence.
+*/
+package ignore

internal/helm/chart/loader/ignore/rules.go

@@ -0,0 +1,228 @@
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ignore
+
+import (
+	"bufio"
+	"bytes"
+	"io"
+	"log"
+	"os"
+	"path/filepath"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+// HelmIgnore default name of an ignorefile.
+const HelmIgnore = ".helmignore"
+
+// Rules is a collection of path matching rules.
+//
+// Parse() and ParseFile() will construct and populate new Rules.
+// Empty() will create an immutable empty ruleset.
+type Rules struct {
+	patterns []*pattern
+}
+
+// Empty builds an empty ruleset.
+func Empty() *Rules {
+	return &Rules{patterns: []*pattern{}}
+}
+
+// AddDefaults adds default ignore patterns.
+//
+// Ignore all dotfiles in "templates/"
+func (r *Rules) AddDefaults() {
+	r.parseRule(`templates/.?*`)
+}
+
+// ParseFile parses a helmignore file and returns the *Rules.
+func ParseFile(file string) (*Rules, error) {
+	f, err := os.Open(file)
+	if err != nil {
+		return nil, err
+	}
+	defer f.Close()
+	return Parse(f)
+}
+
+// Parse parses a rules file
+func Parse(file io.Reader) (*Rules, error) {
+	r := &Rules{patterns: []*pattern{}}
+
+	s := bufio.NewScanner(file)
+	currentLine := 0
+	utf8bom := []byte{0xEF, 0xBB, 0xBF}
+	for s.Scan() {
+		scannedBytes := s.Bytes()
+		// We trim UTF8 BOM
+		if currentLine == 0 {
+			scannedBytes = bytes.TrimPrefix(scannedBytes, utf8bom)
+		}
+		line := string(scannedBytes)
+		currentLine++
+
+		if err := r.parseRule(line); err != nil {
+			return r, err
+		}
+	}
+	return r, s.Err()
+}
+
+// Ignore evaluates the file at the given path, and returns true if it should be ignored.
+//
+// Ignore evaluates path against the rules in order. Evaluation stops when a match
+// is found. Matching a negative rule will stop evaluation.
+func (r *Rules) Ignore(path string, fi os.FileInfo) bool {
+	// Don't match on empty dirs.
+	if path == "" {
+		return false
+	}
+
+	// Disallow ignoring the current working directory.
+	// See issue:
+	// 1776 (New York City) Hamilton: "Pardon me, are you Aaron Burr, sir?"
+	if path == "." || path == "./" {
+		return false
+	}
+	for _, p := range r.patterns {
+		if p.match == nil {
+			log.Printf("ignore: no matcher supplied for %q", p.raw)
+			return false
+		}
+
+		// For negative rules, we need to capture and return non-matches,
+		// and continue for matches.
+		if p.negate {
+			if p.mustDir && !fi.IsDir() {
+				return true
+			}
+			if !p.match(path, fi) {
+				return true
+			}
+			continue
+		}
+
+		// If the rule is looking for directories, and this is not a directory,
+		// skip it.
+		if p.mustDir && !fi.IsDir() {
+			continue
+		}
+		if p.match(path, fi) {
+			return true
+		}
+	}
+	return false
+}
+
+// parseRule parses a rule string and creates a pattern, which is then stored in the Rules object.
+func (r *Rules) parseRule(rule string) error {
+	rule = strings.TrimSpace(rule)
+
+	// Ignore blank lines
+	if rule == "" {
+		return nil
+	}
+	// Comment
+	if strings.HasPrefix(rule, "#") {
+		return nil
+	}
+
+	// Fail any rules that contain **
+	if strings.Contains(rule, "**") {
+		return errors.New("double-star (**) syntax is not supported")
+	}
+
+	// Fail any patterns that can't compile. A non-empty string must be
+	// given to Match() to avoid optimization that skips rule evaluation.
+	if _, err := filepath.Match(rule, "abc"); err != nil {
+		return err
+	}
+
+	p := &pattern{raw: rule}
+
+	// Negation is handled at a higher level, so strip the leading ! from the
+	// string.
+	if strings.HasPrefix(rule, "!") {
+		p.negate = true
+		rule = rule[1:]
+	}
+
+	// Directory verification is handled by a higher level, so the trailing /
+	// is removed from the rule. That way, a directory named "foo" matches,
+	// even if the supplied string does not contain a literal slash character.
+	if strings.HasSuffix(rule, "/") {
+		p.mustDir = true
+		rule = strings.TrimSuffix(rule, "/")
+	}
+
+	if strings.HasPrefix(rule, "/") {
+		// Require path matches the root path.
+		p.match = func(n string, fi os.FileInfo) bool {
+			rule = strings.TrimPrefix(rule, "/")
+			ok, err := filepath.Match(rule, n)
+			if err != nil {
+				log.Printf("Failed to compile %q: %s", rule, err)
+				return false
+			}
+			return ok
+		}
+	} else if strings.Contains(rule, "/") {
+		// require structural match.
+		p.match = func(n string, fi os.FileInfo) bool {
+			ok, err := filepath.Match(rule, n)
+			if err != nil {
+				log.Printf("Failed to compile %q: %s", rule, err)
+				return false
+			}
+			return ok
+		}
+	} else {
+		p.match = func(n string, fi os.FileInfo) bool {
+			// When there is no slash in the pattern, we evaluate ONLY the
+			// filename.
+			n = filepath.Base(n)
+			ok, err := filepath.Match(rule, n)
+			if err != nil {
+				log.Printf("Failed to compile %q: %s", rule, err)
+				return false
+			}
+			return ok
+		}
+	}
+
+	r.patterns = append(r.patterns, p)
+	return nil
+}
+
+// matcher is a function capable of computing a match.
+//
+// It returns true if the rule matches.
+type matcher func(name string, fi os.FileInfo) bool
+
+// pattern describes a pattern to be matched in a rule set.
+type pattern struct {
+	// raw is the unparsed string, with nothing stripped.
+	raw string
+	// match is the matcher function.
+	match matcher
+	// negate indicates that the rule's outcome should be negated.
+	negate bool
+	// mustDir indicates that the matched file must be a directory.
+	mustDir bool
+}

internal/helm/chart/loader/ignore/rules_test.go

@@ -0,0 +1,155 @@
+/*
+Copyright The Helm Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package ignore
+
+import (
+	"bytes"
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+var testdata = "./testdata"
+
+func TestParse(t *testing.T) {
+	rules := `#ignore
+
+	#ignore
+foo
+bar/*
+baz/bar/foo.txt
+
+one/more
+`
+	r, err := parseString(rules)
+	if err != nil {
+		t.Fatalf("Error parsing rules: %s", err)
+	}
+
+	if len(r.patterns) != 4 {
+		t.Errorf("Expected 4 rules, got %d", len(r.patterns))
+	}
+
+	expects := []string{"foo", "bar/*", "baz/bar/foo.txt", "one/more"}
+	for i, p := range r.patterns {
+		if p.raw != expects[i] {
+			t.Errorf("Expected %q, got %q", expects[i], p.raw)
+		}
+		if p.match == nil {
+			t.Errorf("Expected %s to have a matcher function.", p.raw)
+		}
+	}
+}
+
+func TestParseFail(t *testing.T) {
+	shouldFail := []string{"foo/**/bar", "[z-"}
+	for _, fail := range shouldFail {
+		_, err := parseString(fail)
+		if err == nil {
+			t.Errorf("Rule %q should have failed", fail)
+		}
+	}
+}
+
+func TestParseFile(t *testing.T) {
+	f := filepath.Join(testdata, HelmIgnore)
+	if _, err := os.Stat(f); err != nil {
+		t.Fatalf("Fixture %s missing: %s", f, err)
+	}
+
+	r, err := ParseFile(f)
+	if err != nil {
+		t.Fatalf("Failed to parse rules file: %s", err)
+	}
+
+	if len(r.patterns) != 3 {
+		t.Errorf("Expected 3 patterns, got %d", len(r.patterns))
+	}
+}
+
+func TestIgnore(t *testing.T) {
+	// Test table: Given pattern and name, Ignore should return expect.
+	tests := []struct {
+		pattern string
+		name    string
+		expect  bool
+	}{
+		// Glob tests
+		{`helm.txt`, "helm.txt", true},
+		{`helm.*`, "helm.txt", true},
+		{`helm.*`, "rudder.txt", false},
+		{`*.txt`, "tiller.txt", true},
+		{`*.txt`, "cargo/a.txt", true},
+		{`cargo/*.txt`, "cargo/a.txt", true},
+		{`cargo/*.*`, "cargo/a.txt", true},
+		{`cargo/*.txt`, "mast/a.txt", false},
+		{`ru[c-e]?er.txt`, "rudder.txt", true},
+		{`templates/.?*`, "templates/.dotfile", true},
+		// "." should never get ignored. https://github.com/helm/helm/issues/1776
+		{`.*`, ".", false},
+		{`.*`, "./", false},
+		{`.*`, ".joonix", true},
+		{`.*`, "helm.txt", false},
+		{`.*`, "", false},
+
+		// Directory tests
+		{`cargo/`, "cargo", true},
+		{`cargo/`, "cargo/", true},
+		{`cargo/`, "mast/", false},
+		{`helm.txt/`, "helm.txt", false},
+
+		// Negation tests
+		{`!helm.txt`, "helm.txt", false},
+		{`!helm.txt`, "tiller.txt", true},
+		{`!*.txt`, "cargo", true},
+		{`!cargo/`, "mast/", true},
+
+		// Absolute path tests
+		{`/a.txt`, "a.txt", true},
+		{`/a.txt`, "cargo/a.txt", false},
+		{`/cargo/a.txt`, "cargo/a.txt", true},
+	}
+
+	for _, test := range tests {
+		r, err := parseString(test.pattern)
+		if err != nil {
+			t.Fatalf("Failed to parse: %s", err)
+		}
+		fi, err := os.Stat(filepath.Join(testdata, test.name))
+		if err != nil {
+			t.Fatalf("Fixture missing: %s", err)
+		}
+
+		if r.Ignore(test.name, fi) != test.expect {
+			t.Errorf("Expected %q to be %v for pattern %q", test.name, test.expect, test.pattern)
+		}
+	}
+}
+
+func TestAddDefaults(t *testing.T) {
+	r := Rules{}
+	r.AddDefaults()
+
+	if len(r.patterns) != 1 {
+		t.Errorf("Expected 1 default patterns, got %d", len(r.patterns))
+	}
+}
+
+func parseString(str string) (*Rules, error) {
+	b := bytes.NewBuffer([]byte(str))
+	return Parse(b)
+}

internal/helm/chart/loader/ignore/testdata/.helmignore

@@ -0,0 +1,3 @@
+mast/a.txt
+.DS_Store
+.git

internal/helm/chart/loader/sympath/walk.go

@@ -0,0 +1,119 @@
+/*
+Copyright (c) for portions of walk.go are held by The Go Authors, 2009 and are
+provided under the BSD license.
+
+https://github.com/golang/go/blob/master/LICENSE
+
+Copyright The Helm Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sympath
+
+import (
+	"log"
+	"os"
+	"path/filepath"
+	"sort"
+
+	"github.com/pkg/errors"
+)
+
+// Walk walks the file tree rooted at root, calling walkFn for each file or directory
+// in the tree, including root. All errors that arise visiting files and directories
+// are filtered by walkFn. The files are walked in lexical order, which makes the
+// output deterministic but means that for very large directories Walk can be
+// inefficient. Walk follows symbolic links.
+func Walk(root string, walkFn filepath.WalkFunc) error {
+	info, err := os.Lstat(root)
+	if err != nil {
+		err = walkFn(root, nil, err)
+	} else {
+		err = symwalk(root, info, walkFn)
+	}
+	if err == filepath.SkipDir {
+		return nil
+	}
+	return err
+}
+
+// readDirNames reads the directory named by dirname and returns
+// a sorted list of directory entries.
+func readDirNames(dirname string) ([]string, error) {
+	f, err := os.Open(dirname)
+	if err != nil {
+		return nil, err
+	}
+	names, err := f.Readdirnames(-1)
+	f.Close()
+	if err != nil {
+		return nil, err
+	}
+	sort.Strings(names)
+	return names, nil
+}
+
+// symwalk recursively descends path, calling walkFn.
+func symwalk(path string, info os.FileInfo, walkFn filepath.WalkFunc) error {
+	// Recursively walk symlinked directories.
+	if IsSymlink(info) {
+		resolved, err := filepath.EvalSymlinks(path)
+		if err != nil {
+			return errors.Wrapf(err, "error evaluating symlink %s", path)
+		}
+		log.Printf("found symbolic link in path: %s resolves to %s", path, resolved)
+		if info, err = os.Lstat(resolved); err != nil {
+			return err
+		}
+		if err := symwalk(path, info, walkFn); err != nil && err != filepath.SkipDir {
+			return err
+		}
+		return nil
+	}
+
+	if err := walkFn(path, info, nil); err != nil {
+		return err
+	}
+
+	if !info.IsDir() {
+		return nil
+	}
+
+	names, err := readDirNames(path)
+	if err != nil {
+		return walkFn(path, info, err)
+	}
+
+	for _, name := range names {
+		filename := filepath.Join(path, name)
+		fileInfo, err := os.Lstat(filename)
+		if err != nil {
+			if err := walkFn(filename, fileInfo, err); err != nil && err != filepath.SkipDir {
+				return err
+			}
+		} else {
+			err = symwalk(filename, fileInfo, walkFn)
+			if err != nil {
+				if (!fileInfo.IsDir() && !IsSymlink(fileInfo)) || err != filepath.SkipDir {
+					return err
+				}
+			}
+		}
+	}
+	return nil
+}
+
+// IsSymlink is used to determine if the fileinfo is a symbolic link.
+func IsSymlink(fi os.FileInfo) bool {
+	return fi.Mode()&os.ModeSymlink != 0
+}

internal/helm/chart/loader/sympath/walk_test.go

@@ -0,0 +1,151 @@
+/*
+Copyright (c) for portions of walk_test.go are held by The Go Authors, 2009 and are
+provided under the BSD license.
+
+https://github.com/golang/go/blob/master/LICENSE
+
+Copyright The Helm Authors.
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package sympath
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+)
+
+type Node struct {
+	name          string
+	entries       []*Node // nil if the entry is a file
+	marks         int
+	expectedMarks int
+	symLinkedTo   string
+}
+
+var tree = &Node{
+	"testdata",
+	[]*Node{
+		{"a", nil, 0, 1, ""},
+		{"b", []*Node{}, 0, 1, ""},
+		{"c", nil, 0, 2, ""},
+		{"d", nil, 0, 0, "c"},
+		{
+			"e",
+			[]*Node{
+				{"x", nil, 0, 1, ""},
+				{"y", []*Node{}, 0, 1, ""},
+				{
+					"z",
+					[]*Node{
+						{"u", nil, 0, 1, ""},
+						{"v", nil, 0, 1, ""},
+						{"w", nil, 0, 1, ""},
+					},
+					0,
+					1,
+					"",
+				},
+			},
+			0,
+			1,
+			"",
+		},
+	},
+	0,
+	1,
+	"",
+}
+
+func walkTree(n *Node, path string, f func(path string, n *Node)) {
+	f(path, n)
+	for _, e := range n.entries {
+		walkTree(e, filepath.Join(path, e.name), f)
+	}
+}
+
+func makeTree(t *testing.T) {
+	walkTree(tree, tree.name, func(path string, n *Node) {
+		if n.entries == nil {
+			if n.symLinkedTo != "" {
+				if err := os.Symlink(n.symLinkedTo, path); err != nil {
+					t.Fatalf("makeTree: %v", err)
+				}
+			} else {
+				fd, err := os.Create(path)
+				if err != nil {
+					t.Fatalf("makeTree: %v", err)
+					return
+				}
+				fd.Close()
+			}
+		} else {
+			if err := os.Mkdir(path, 0770); err != nil {
+				t.Fatalf("makeTree: %v", err)
+			}
+		}
+	})
+}
+
+func checkMarks(t *testing.T, report bool) {
+	walkTree(tree, tree.name, func(path string, n *Node) {
+		if n.marks != n.expectedMarks && report {
+			t.Errorf("node %s mark = %d; expected %d", path, n.marks, n.expectedMarks)
+		}
+		n.marks = 0
+	})
+}
+
+// Assumes that each node name is unique. Good enough for a test.
+// If clear is true, any incoming error is cleared before return. The errors
+// are always accumulated, though.
+func mark(info os.FileInfo, err error, errors *[]error, clear bool) error {
+	if err != nil {
+		*errors = append(*errors, err)
+		if clear {
+			return nil
+		}
+		return err
+	}
+	name := info.Name()
+	walkTree(tree, tree.name, func(path string, n *Node) {
+		if n.name == name {
+			n.marks++
+		}
+	})
+	return nil
+}
+
+func TestWalk(t *testing.T) {
+	makeTree(t)
+	errors := make([]error, 0, 10)
+	clear := true
+	markFn := func(path string, info os.FileInfo, err error) error {
+		return mark(info, err, &errors, clear)
+	}
+	// Expect no errors.
+	err := Walk(tree.name, markFn)
+	if err != nil {
+		t.Fatalf("no error expected, found: %s", err)
+	}
+	if len(errors) != 0 {
+		t.Fatalf("unexpected errors: %s", errors)
+	}
+	checkMarks(t, true)
+
+	// cleanup
+	if err := os.RemoveAll(tree.name); err != nil {
+		t.Errorf("removeTree: %v", err)
+	}
+}