source-controller/controllers/storage_test.go

package controllers

import (
	"archive/tar"
	"compress/gzip"
	"fmt"
	"io"
	"io/ioutil"
	"os"
	"os/exec"
	"path"
	"path/filepath"
	"testing"
	"time"

	sourcev1 "github.com/fluxcd/source-controller/api/v1beta1"
)

type ignoreMap map[string]bool

var remoteRepository = "https://github.com/fluxcd/source-controller"

func init() {
	// if this remote repo ever gets in your way, this is an escape; just set
	// this to the url you want to clone. Be the source you want to be.
	s := os.Getenv("REMOTE_REPOSITORY")
	if s != "" {
		remoteRepository = s
	}
}

func createStoragePath() (string, error) {
	return ioutil.TempDir("", "")
}

func cleanupStoragePath(dir string) func() {
	return func() { os.RemoveAll(dir) }
}

func TestStorageConstructor(t *testing.T) {
	dir, err := createStoragePath()
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(cleanupStoragePath(dir))

	if _, err := NewStorage("/nonexistent", "hostname", time.Minute); err == nil {
		t.Fatal("nonexistent path was allowable in storage constructor")
	}

	f, err := ioutil.TempFile(dir, "")
	if err != nil {
		t.Fatalf("while creating temporary file: %v", err)
	}
	f.Close()

	if _, err := NewStorage(f.Name(), "hostname", time.Minute); err == nil {
		os.Remove(f.Name())
		t.Fatal("file path was accepted as basedir")
	}
	os.Remove(f.Name())

	if _, err := NewStorage(dir, "hostname", time.Minute); err != nil {
		t.Fatalf("Valid path did not successfully return: %v", err)
	}
}

// walks a tar.gz and looks for paths with the basename. It does not match
// symlinks properly at this time because that's painful.
func walkTar(tarFile string, match string) (bool, error) {
	f, err := os.Open(tarFile)
	if err != nil {
		return false, fmt.Errorf("could not open file: %w", err)
	}
	defer f.Close()

	gzr, err := gzip.NewReader(f)
	if err != nil {
		return false, fmt.Errorf("could not unzip file: %w", err)
	}
	defer gzr.Close()

	tr := tar.NewReader(gzr)
	for {
		header, err := tr.Next()
		if err == io.EOF {
			break
		} else if err != nil {
			return false, fmt.Errorf("Corrupt tarball reading header: %w", err)
		}

		switch header.Typeflag {
		case tar.TypeDir, tar.TypeReg:
			if filepath.Base(header.Name) == match {
				return true, nil
			}
		default:
			// skip
		}
	}

	return false, nil
}

func testPatterns(t *testing.T, storage *Storage, artifact sourcev1.Artifact, table ignoreMap) {
	for name, expected := range table {
		res, err := walkTar(storage.LocalPath(artifact), name)
		if err != nil {
			t.Fatalf("while reading tarball: %v", err)
		}

		if res != expected {
			if expected {
				t.Fatalf("Could not find repository file matching %q in tarball for repo %q", name, remoteRepository)
			} else {
				t.Fatalf("Repository contained ignored file %q in tarball for repo %q", name, remoteRepository)
			}
		}
	}
}

func createArchive(t *testing.T, storage *Storage, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact {
	gitDir, err := ioutil.TempDir("", "")
	if err != nil {
		t.Fatalf("could not create temporary directory: %v", err)
	}
	t.Cleanup(func() { os.RemoveAll(gitDir) })

	if err := exec.Command("git", "clone", remoteRepository, gitDir).Run(); err != nil {
		t.Fatalf("Could not clone remote repository: %v", err)
	}

	// inject files.. just empty files
	for _, name := range filenames {
		f, err := os.Create(filepath.Join(gitDir, name))
		if err != nil {
			t.Fatalf("Could not inject filename %q: %v", name, err)
		}
		f.Close()
	}

	// inject sourceignore if not empty
	if sourceIgnore != "" {
		si, err := os.Create(filepath.Join(gitDir, ".sourceignore"))
		if err != nil {
			t.Fatalf("Could not create .sourceignore: %v", err)
		}

		if _, err := io.WriteString(si, sourceIgnore); err != nil {
			t.Fatalf("Could not write to .sourceignore: %v", err)
		}

		si.Close()
	}
	artifact := sourcev1.Artifact{
		Path: filepath.Join(randStringRunes(10), randStringRunes(10), randStringRunes(10)+".tar.gz"),
	}
	if err := storage.MkdirAll(artifact); err != nil {
		t.Fatalf("artifact directory creation failed: %v", err)
	}

	if err := storage.Archive(&artifact, gitDir, spec.Ignore); err != nil {
		t.Fatalf("archiving failed: %v", err)
	}

	if !storage.ArtifactExist(artifact) {
		t.Fatalf("artifact was created but does not exist: %+v", artifact)
	}

	return artifact
}

func stringPtr(s string) *string {
	return &s
}

func TestArchiveBasic(t *testing.T) {
	table := ignoreMap{
		"README.md":  true,
		".gitignore": false,
	}

	dir, err := createStoragePath()
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(cleanupStoragePath(dir))

	storage, err := NewStorage(dir, "hostname", time.Minute)
	if err != nil {
		t.Fatalf("Error while bootstrapping storage: %v", err)
	}

	testPatterns(t, storage, createArchive(t, storage, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table)
}

func TestArchiveIgnore(t *testing.T) {
	// this is a list of files that will be created in the repository for each
	// subtest. it is manipulated later on.
	filenames := []string{
		"foo.tar.gz",
		"bar.jpg",
		"bar.gif",
		"foo.jpeg",
		"video.flv",
		"video.wmv",
		"bar.png",
		"foo.zip",
	}

	// this is the table of ignored files and their values. true means that it's
	// present in the resulting tarball.
	table := ignoreMap{}
	for _, item := range filenames {
		table[item] = false
	}

	dir, err := createStoragePath()
	if err != nil {
		t.Fatal(err)
	}
	t.Cleanup(cleanupStoragePath(dir))

	storage, err := NewStorage(dir, "hostname", time.Minute)
	if err != nil {
		t.Fatalf("Error while bootstrapping storage: %v", err)
	}

	t.Run("automatically ignored files", func(t *testing.T) {
		testPatterns(t, storage, createArchive(t, storage, filenames, "", sourcev1.GitRepositorySpec{}), table)
	})

	table = ignoreMap{}
	for _, item := range filenames {
		table[item] = true
	}

	t.Run("only vcs ignored files", func(t *testing.T) {
		testPatterns(t, storage, createArchive(t, storage, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table)
	})

	filenames = append(filenames, "test.txt")
	table["test.txt"] = false
	sourceIgnoreFile := "*.txt"

	t.Run("sourceignore injected via CRD", func(t *testing.T) {
		testPatterns(t, storage, createArchive(t, storage, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table)
	})

	table = ignoreMap{}
	for _, item := range filenames {
		table[item] = false
	}

	t.Run("sourceignore injected via filename", func(t *testing.T) {
		testPatterns(t, storage, createArchive(t, storage, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table)
	})
}

func TestStorageRemoveAllButCurrent(t *testing.T) {
	t.Run("bad directory in archive", func(t *testing.T) {
		dir, err := ioutil.TempDir("", "")
		if err != nil {
			t.Fatal(err)
		}
		t.Cleanup(func() { os.RemoveAll(dir) })

		s, err := NewStorage(dir, "hostname", time.Minute)
		if err != nil {
			t.Fatalf("Valid path did not successfully return: %v", err)
		}

		if err := s.RemoveAllButCurrent(sourcev1.Artifact{Path: path.Join(dir, "really", "nonexistent")}); err == nil {
			t.Fatal("Did not error while pruning non-existent path")
		}
	})
}