fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

storage: only store relative path in artifact 

As the storage base directory is determined during runtime, and
artifacts may live longer than that if they are e.g. stored in a
persistent volume but the mount path configuration changes.
This commit is contained in:
Hidde Beydals 2020-09-08 17:21:56 +02:00
parent f8c4bd31ca
commit 0b752178b1
4 changed files with 55 additions and 34 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/v1alpha1/source.go
View File

@ -1,6 +1,8 @@
package v1alpha1
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// Source interface must be supported by all API types.
// +k8s:deepcopy-gen=false

7
controllers/helmchart_controller.go
View File

@ -188,7 +188,8 @@ func (r *HelmChartReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts
func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
repository sourcev1.HelmRepository, chart sourcev1.HelmChart) (sourcev1.HelmChart, error) {
cv, err := helm.GetDownloadableChartVersionFromIndex(repository.Status.Artifact.Path, chart.Spec.Chart, chart.Spec.Version)
cv, err := helm.GetDownloadableChartVersionFromIndex(r.Storage.LocalPath(*repository.GetArtifact()),
chart.Spec.Chart, chart.Spec.Version)
if err != nil {
return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
}
@ -333,7 +334,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
defer os.RemoveAll(tmpDir)
// open file
f, err := os.Open(repository.GetArtifact().Path)
f, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact()))
if err != nil {
err = fmt.Errorf("artifact open error: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
@ -384,7 +385,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
// package chart
pkg := action.NewPackage()
pkg.Destination = filepath.Dir(artifact.Path)
pkg.Destination = filepath.Dir(r.Storage.LocalPath(artifact))
_, err = pkg.Run(chartPath, nil)
if err != nil {
err = fmt.Errorf("chart package error: %w", err)

44
controllers/storage.go
View File

@ -69,7 +69,7 @@ func NewStorage(basePath string, hostname string, timeout time.Duration) (*Stora
}, nil
}
// ArtifactFor returns an artifact for the given Kubernetes object
// ArtifactFor returns an artifact for the v1alpha1.Source.
func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision string) sourcev1.Artifact {
path := sourcev1.ArtifactPath(kind, metadata.GetNamespace(), metadata.GetName(), fileName)
localPath := filepath.Join(s.BasePath, path)
@ -83,21 +83,22 @@ func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, rev
}
}
// MkdirAll calls os.MkdirAll for the given artifact base dir
// MkdirAll calls os.MkdirAll for the given v1alpha1.Artifact base dir.
func (s *Storage) MkdirAll(artifact sourcev1.Artifact) error {
dir := filepath.Dir(artifact.Path)
dir := filepath.Dir(s.LocalPath(artifact))
return os.MkdirAll(dir, 0777)
}
// RemoveAll calls os.RemoveAll for the given artifact base dir
// RemoveAll calls os.RemoveAll for the given v1alpha1.Artifact base dir.
func (s *Storage) RemoveAll(artifact sourcev1.Artifact) error {
dir := filepath.Dir(artifact.Path)
dir := filepath.Dir(s.LocalPath(artifact))
return os.RemoveAll(dir)
}
// RemoveAllButCurrent removes all files for the given artifact base dir excluding the current one
func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
dir := filepath.Dir(artifact.Path)
localPath := s.LocalPath(artifact)
dir := filepath.Dir(localPath)
var errors []string
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
@ -105,7 +106,7 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
return nil
}
if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
if path != localPath && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
if err := os.Remove(path); err != nil {
errors = append(errors, info.Name())
}
@ -122,7 +123,7 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
// ArtifactExist returns a boolean indicating whether the artifact exists in storage and is a
// regular file.
func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool {
fi, err := os.Lstat(artifact.Path)
fi, err := os.Lstat(s.LocalPath(artifact))
if err != nil {
return false
}
@ -143,7 +144,7 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.
matcher := gitignore.NewMatcher(ps)
gzFile, err := os.Create(artifact.Path)
gzFile, err := os.Create(s.LocalPath(artifact))
if err != nil {
return err
}
@ -204,20 +205,22 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.
// WriteFile writes the given bytes to the artifact path if the checksum differs
func (s *Storage) WriteFile(artifact sourcev1.Artifact, data []byte) error {
localPath := s.LocalPath(artifact)
sum := s.Checksum(data)
if file, err := os.Stat(artifact.Path); !os.IsNotExist(err) && !file.IsDir() {
if fb, err := ioutil.ReadFile(artifact.Path); err == nil && sum == s.Checksum(fb) {
if file, err := os.Stat(localPath); !os.IsNotExist(err) && !file.IsDir() {
if fb, err := ioutil.ReadFile(localPath); err == nil && sum == s.Checksum(fb) {
return nil
}
}
return ioutil.WriteFile(artifact.Path, data, 0644)
return ioutil.WriteFile(localPath, data, 0644)
}
// Symlink creates or updates a symbolic link for the given artifact
// and returns the URL for the symlink
// and returns the URL for the symlink.
func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string, error) {
dir := filepath.Dir(artifact.Path)
localPath := s.LocalPath(artifact)
dir := filepath.Dir(localPath)
link := filepath.Join(dir, linkName)
tmpLink := link + ".tmp"
@ -225,7 +228,7 @@ func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string,
return "", err
}
if err := os.Symlink(artifact.Path, tmpLink); err != nil {
if err := os.Symlink(localPath, tmpLink); err != nil {
return "", err
}
@ -245,11 +248,20 @@ func (s *Storage) Checksum(b []byte) string {
// Lock creates a file lock for the given artifact
func (s *Storage) Lock(artifact sourcev1.Artifact) (unlock func(), err error) {
lockFile := artifact.Path + ".lock"
lockFile := s.LocalPath(artifact) + ".lock"
mutex := lockedfile.MutexAt(lockFile)
return mutex.Lock()
}
// LocalPath returns the local path of the given artifact (that is: relative to
// the Storage.BasePath).
func (s *Storage) LocalPath(artifact sourcev1.Artifact) string {
if artifact.Path == "" {
return ""
}
return filepath.Join(s.BasePath, artifact.Path)
}
func getPatterns(reader io.Reader, path []string) []gitignore.Pattern {
var ps []gitignore.Pattern
scanner := bufio.NewScanner(reader)

34
controllers/storage_test.go
View File

@ -113,9 +113,9 @@ func walkTar(tarFile string, match string) (bool, error) {
return false, nil
}
func testPatterns(t *testing.T, artifact sourcev1.Artifact, table ignoreMap) {
func testPatterns(t *testing.T, dir string, artifact sourcev1.Artifact, table ignoreMap) {
for name, expected := range table {
res, err := walkTar(artifact.Path, name)
res, err := walkTar(filepath.Join(dir, artifact.Path), name)
if err != nil {
t.Fatalf("while reading tarball: %v", err)
}
@ -130,13 +130,7 @@ func testPatterns(t *testing.T, artifact sourcev1.Artifact, table ignoreMap) {
}
}
func createArchive(t *testing.T, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact {
dir, err := createStoragePath()
if err != nil {
t.Fatal(err)
}
t.Cleanup(cleanupStoragePath(dir))
func createArchive(t *testing.T, dir string, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact {
storage, err := NewStorage(dir, "hostname", time.Minute)
if err != nil {
t.Fatalf("Error while bootstrapping storage: %v", err)
@ -197,7 +191,13 @@ func TestArchiveBasic(t *testing.T) {
".gitignore": false,
}
testPatterns(t, createArchive(t, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table)
dir, err := createStoragePath()
if err != nil {
t.Fatal(err)
}
t.Cleanup(cleanupStoragePath(dir))
testPatterns(t, dir, createArchive(t, dir, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table)
}
func TestArchiveIgnore(t *testing.T) {
@ -221,8 +221,14 @@ func TestArchiveIgnore(t *testing.T) {
table[item] = false
}
dir, err := createStoragePath()
if err != nil {
t.Fatal(err)
}
t.Cleanup(cleanupStoragePath(dir))
t.Run("automatically ignored files", func(t *testing.T) {
testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{}), table)
testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{}), table)
})
table = ignoreMap{}
@ -231,7 +237,7 @@ func TestArchiveIgnore(t *testing.T) {
}
t.Run("only vcs ignored files", func(t *testing.T) {
testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table)
testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table)
})
filenames = append(filenames, "test.txt")
@ -239,7 +245,7 @@ func TestArchiveIgnore(t *testing.T) {
sourceIgnoreFile := "*.txt"
t.Run("sourceignore injected via CRD", func(t *testing.T) {
testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table)
testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table)
})
table = ignoreMap{}
@ -248,7 +254,7 @@ func TestArchiveIgnore(t *testing.T) {
}
t.Run("sourceignore injected via filename", func(t *testing.T) {
testPatterns(t, createArchive(t, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table)
testPatterns(t, dir, createArchive(t, dir, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table)
})
}