diff --git a/api/v1alpha1/artifact.go b/api/v1alpha1/artifact.go
index 131ea863..b9bac76b 100644
--- a/api/v1alpha1/artifact.go
+++ b/api/v1alpha1/artifact.go
@@ -17,14 +17,15 @@ limitations under the License.
package v1alpha1
import (
- "fmt"
+ "path"
+ "strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
-// Artifact represents the output of a source synchronisation
+// Artifact represents the output of a source synchronisation.
type Artifact struct {
- // Path is the local file path of this artifact.
+ // Path is the relative file path of this artifact.
// +required
Path string `json:"path"`
@@ -32,20 +33,31 @@ type Artifact struct {
// +required
URL string `json:"url"`
- // Revision is a human readable identifier traceable in the origin source system.
- // It can be a commit sha, git tag, a helm index timestamp,
- // a helm chart version, a checksum, etc.
+ // Revision is a human readable identifier traceable in the origin
+ // source system. It can be a Git commit sha, Git tag, a Helm index
+ // timestamp, a Helm chart version, etc.
// +optional
Revision string `json:"revision"`
+ // Checksum is the SHA1 checksum of the artifact.
+ // +optional
+ Checksum string `json:"checksum"`
+
// LastUpdateTime is the timestamp corresponding to the last
// update of this artifact.
// +required
LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
}
-// ArtifactPath returns the artifact path in the form of
-// ///
-func ArtifactPath(kind, namespace, name, filename string) string {
- return fmt.Sprintf("%s/%s/%s/%s", kind, namespace, name, filename)
+// ArtifactDir returns the artifact dir path in the form of
+// //.
+func ArtifactDir(kind, namespace, name string) string {
+ kind = strings.ToLower(kind)
+ return path.Join(kind, namespace, name)
+}
+
+// ArtifactPath returns the artifact path in the form of
+// ///.
+func ArtifactPath(kind, namespace, name, filename string) string {
+ return path.Join(ArtifactDir(kind, namespace, name), filename)
}
diff --git a/api/v1alpha1/source.go b/api/v1alpha1/source.go
index a4c3dfe8..4c918416 100644
--- a/api/v1alpha1/source.go
+++ b/api/v1alpha1/source.go
@@ -1,6 +1,8 @@
package v1alpha1
-import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+import (
+ metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+)
// Source interface must be supported by all API types.
// +k8s:deepcopy-gen=false
diff --git a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml
index 2a339ee9..e60d5852 100644
--- a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml
+++ b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml
@@ -129,18 +129,21 @@ spec:
description: Artifact represents the output of the last successful
repository sync.
properties:
+ checksum:
+ description: Checksum is the SHA1 checksum of the artifact.
+ type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of this artifact.
format: date-time
type: string
path:
- description: Path is the local file path of this artifact.
+ description: Path is the relative file path of this artifact.
type: string
revision:
description: Revision is a human readable identifier traceable
- in the origin source system. It can be a commit sha, git tag,
- a helm index timestamp, a helm chart version, a checksum, etc.
+ in the origin source system. It can be a Git commit sha, Git
+ tag, a Helm index timestamp, a Helm chart version, etc.
type: string
url:
description: URL is the HTTP address of this artifact.
diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml
index e66ef34d..d1125575 100644
--- a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml
+++ b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml
@@ -101,18 +101,21 @@ spec:
description: Artifact represents the output of the last successful
chart sync.
properties:
+ checksum:
+ description: Checksum is the SHA1 checksum of the artifact.
+ type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of this artifact.
format: date-time
type: string
path:
- description: Path is the local file path of this artifact.
+ description: Path is the relative file path of this artifact.
type: string
revision:
description: Revision is a human readable identifier traceable
- in the origin source system. It can be a commit sha, git tag,
- a helm index timestamp, a helm chart version, a checksum, etc.
+ in the origin source system. It can be a Git commit sha, Git
+ tag, a Helm index timestamp, a Helm chart version, etc.
type: string
url:
description: URL is the HTTP address of this artifact.
diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml
index e165ee19..8f7f675e 100644
--- a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml
+++ b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml
@@ -81,18 +81,21 @@ spec:
description: Artifact represents the output of the last successful
repository sync.
properties:
+ checksum:
+ description: Checksum is the SHA1 checksum of the artifact.
+ type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of this artifact.
format: date-time
type: string
path:
- description: Path is the local file path of this artifact.
+ description: Path is the relative file path of this artifact.
type: string
revision:
description: Revision is a human readable identifier traceable
- in the origin source system. It can be a commit sha, git tag,
- a helm index timestamp, a helm chart version, a checksum, etc.
+ in the origin source system. It can be a Git commit sha, Git
+ tag, a Helm index timestamp, a Helm chart version, etc.
type: string
url:
description: URL is the HTTP address of this artifact.
diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go
index b1aa96fc..4ce52033 100644
--- a/controllers/gitrepository_controller.go
+++ b/controllers/gitrepository_controller.go
@@ -212,8 +212,10 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
}
}
+ // TODO(hidde): implement checksum when https://github.com/fluxcd/source-controller/pull/133
+ // has been merged.
artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(),
- fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision)
+ fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision, "")
// create artifact dir
err = r.Storage.MkdirAll(artifact)
@@ -300,10 +302,10 @@ func (r *GitRepositoryReconciler) verify(ctx context.Context, publicKeySecret ty
// gc performs a garbage collection on all but current artifacts of
// the given repository.
func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository, all bool) error {
+ if all {
+ return r.Storage.RemoveAll(r.Storage.ArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "", ""))
+ }
if repository.Status.Artifact != nil {
- if all {
- return r.Storage.RemoveAll(*repository.Status.Artifact)
- }
return r.Storage.RemoveAllButCurrent(*repository.Status.Artifact)
}
return nil
diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go
index 0eaca70b..e1eb1291 100644
--- a/controllers/helmchart_controller.go
+++ b/controllers/helmchart_controller.go
@@ -188,7 +188,8 @@ func (r *HelmChartReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts
func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
repository sourcev1.HelmRepository, chart sourcev1.HelmChart) (sourcev1.HelmChart, error) {
- cv, err := helm.GetDownloadableChartVersionFromIndex(repository.Status.Artifact.Path, chart.Spec.Chart, chart.Spec.Version)
+ cv, err := helm.GetDownloadableChartVersionFromIndex(r.Storage.LocalPath(*repository.GetArtifact()),
+ chart.Spec.Chart, chart.Spec.Version)
if err != nil {
return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
}
@@ -260,7 +261,7 @@ func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
sum := r.Storage.Checksum(chartBytes)
artifact := r.Storage.ArtifactFor(chart.Kind, chart.GetObjectMeta(),
- fmt.Sprintf("%s-%s-%s.tgz", cv.Name, cv.Version, sum), cv.Version)
+ fmt.Sprintf("%s-%s-%s.tgz", cv.Name, cv.Version, sum), cv.Version, sum)
// create artifact dir
err = r.Storage.MkdirAll(artifact)
@@ -333,7 +334,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
defer os.RemoveAll(tmpDir)
// open file
- f, err := os.Open(repository.GetArtifact().Path)
+ f, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact()))
if err != nil {
err = fmt.Errorf("artifact open error: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
@@ -364,8 +365,10 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
return chart, nil
}
+ // TODO(hidde): implement checksum when https://github.com/fluxcd/source-controller/pull/133
+ // has been merged.
artifact := r.Storage.ArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(),
- fmt.Sprintf("%s-%s.tgz", chartMetadata.Name, chartMetadata.Version), chartMetadata.Version)
+ fmt.Sprintf("%s-%s.tgz", chartMetadata.Name, chartMetadata.Version), chartMetadata.Version, "")
// create artifact dir
err = r.Storage.MkdirAll(artifact)
@@ -384,7 +387,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
// package chart
pkg := action.NewPackage()
- pkg.Destination = filepath.Dir(artifact.Path)
+ pkg.Destination = filepath.Dir(r.Storage.LocalPath(artifact))
_, err = pkg.Run(chartPath, nil)
if err != nil {
err = fmt.Errorf("chart package error: %w", err)
@@ -432,10 +435,10 @@ func (r *HelmChartReconciler) getGitRepositoryWithArtifact(ctx context.Context,
// gc performs a garbage collection on all but current artifacts of
// the given chart.
func (r *HelmChartReconciler) gc(chart sourcev1.HelmChart, all bool) error {
+ if all {
+ return r.Storage.RemoveAll(r.Storage.ArtifactFor(chart.Kind, chart.GetObjectMeta(), "", "", ""))
+ }
if chart.Status.Artifact != nil {
- if all {
- return r.Storage.RemoveAll(*chart.Status.Artifact)
- }
return r.Storage.RemoveAllButCurrent(*chart.Status.Artifact)
}
return nil
diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go
index 1bd37268..0ea38533 100644
--- a/controllers/helmrepository_controller.go
+++ b/controllers/helmrepository_controller.go
@@ -229,7 +229,7 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
sum := r.Storage.Checksum(index)
artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(),
- fmt.Sprintf("index-%s.yaml", sum), sum)
+ fmt.Sprintf("index-%s.yaml", sum), i.Generated.Format(time.RFC3339Nano), sum)
// create artifact dir
err = r.Storage.MkdirAll(artifact)
@@ -294,10 +294,10 @@ func (r *HelmRepositoryReconciler) shouldResetStatus(repository sourcev1.HelmRep
// gc performs a garbage collection on all but current artifacts of
// the given repository.
func (r *HelmRepositoryReconciler) gc(repository sourcev1.HelmRepository, all bool) error {
+ if all {
+ return r.Storage.RemoveAll(r.Storage.ArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "", ""))
+ }
if repository.Status.Artifact != nil {
- if all {
- return r.Storage.RemoveAll(*repository.Status.Artifact)
- }
return r.Storage.RemoveAllButCurrent(*repository.Status.Artifact)
}
return nil
diff --git a/controllers/storage.go b/controllers/storage.go
index 56e4ea48..6df68106 100644
--- a/controllers/storage.go
+++ b/controllers/storage.go
@@ -69,9 +69,8 @@ func NewStorage(basePath string, hostname string, timeout time.Duration) (*Stora
}, nil
}
-// ArtifactFor returns an artifact for the given Kubernetes object
-func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision string) sourcev1.Artifact {
- kind = strings.ToLower(kind)
+// ArtifactFor returns an artifact for the v1alpha1.Source.
+func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision, checksum string) sourcev1.Artifact {
path := sourcev1.ArtifactPath(kind, metadata.GetNamespace(), metadata.GetName(), fileName)
localPath := filepath.Join(s.BasePath, path)
url := fmt.Sprintf("http://%s/%s", s.Hostname, path)
@@ -80,25 +79,27 @@ func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, rev
Path: localPath,
URL: url,
Revision: revision,
+ Checksum: checksum,
LastUpdateTime: metav1.Now(),
}
}
-// MkdirAll calls os.MkdirAll for the given artifact base dir
+// MkdirAll calls os.MkdirAll for the given v1alpha1.Artifact base dir.
func (s *Storage) MkdirAll(artifact sourcev1.Artifact) error {
- dir := filepath.Dir(artifact.Path)
+ dir := filepath.Dir(s.LocalPath(artifact))
return os.MkdirAll(dir, 0777)
}
-// RemoveAll calls os.RemoveAll for the given artifact base dir
+// RemoveAll calls os.RemoveAll for the given v1alpha1.Artifact base dir.
func (s *Storage) RemoveAll(artifact sourcev1.Artifact) error {
- dir := filepath.Dir(artifact.Path)
+ dir := filepath.Dir(s.LocalPath(artifact))
return os.RemoveAll(dir)
}
// RemoveAllButCurrent removes all files for the given artifact base dir excluding the current one
func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
- dir := filepath.Dir(artifact.Path)
+ localPath := s.LocalPath(artifact)
+ dir := filepath.Dir(localPath)
var errors []string
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
@@ -106,7 +107,7 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
return nil
}
- if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
+ if path != localPath && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
if err := os.Remove(path); err != nil {
errors = append(errors, info.Name())
}
@@ -123,7 +124,7 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
// ArtifactExist returns a boolean indicating whether the artifact exists in storage and is a
// regular file.
func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool {
- fi, err := os.Lstat(artifact.Path)
+ fi, err := os.Lstat(s.LocalPath(artifact))
if err != nil {
return false
}
@@ -144,7 +145,7 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.
matcher := gitignore.NewMatcher(ps)
- gzFile, err := os.Create(artifact.Path)
+ gzFile, err := os.Create(s.LocalPath(artifact))
if err != nil {
return err
}
@@ -205,20 +206,22 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.
// WriteFile writes the given bytes to the artifact path if the checksum differs
func (s *Storage) WriteFile(artifact sourcev1.Artifact, data []byte) error {
+ localPath := s.LocalPath(artifact)
sum := s.Checksum(data)
- if file, err := os.Stat(artifact.Path); !os.IsNotExist(err) && !file.IsDir() {
- if fb, err := ioutil.ReadFile(artifact.Path); err == nil && sum == s.Checksum(fb) {
+ if file, err := os.Stat(localPath); !os.IsNotExist(err) && !file.IsDir() {
+ if fb, err := ioutil.ReadFile(localPath); err == nil && sum == s.Checksum(fb) {
return nil
}
}
- return ioutil.WriteFile(artifact.Path, data, 0644)
+ return ioutil.WriteFile(localPath, data, 0644)
}
// Symlink creates or updates a symbolic link for the given artifact
-// and returns the URL for the symlink
+// and returns the URL for the symlink.
func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string, error) {
- dir := filepath.Dir(artifact.Path)
+ localPath := s.LocalPath(artifact)
+ dir := filepath.Dir(localPath)
link := filepath.Join(dir, linkName)
tmpLink := link + ".tmp"
@@ -226,7 +229,7 @@ func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string,
return "", err
}
- if err := os.Symlink(artifact.Path, tmpLink); err != nil {
+ if err := os.Symlink(localPath, tmpLink); err != nil {
return "", err
}
@@ -246,11 +249,20 @@ func (s *Storage) Checksum(b []byte) string {
// Lock creates a file lock for the given artifact
func (s *Storage) Lock(artifact sourcev1.Artifact) (unlock func(), err error) {
- lockFile := artifact.Path + ".lock"
+ lockFile := s.LocalPath(artifact) + ".lock"
mutex := lockedfile.MutexAt(lockFile)
return mutex.Lock()
}
+// LocalPath returns the local path of the given artifact (that is: relative to
+// the Storage.BasePath).
+func (s *Storage) LocalPath(artifact sourcev1.Artifact) string {
+ if artifact.Path == "" {
+ return ""
+ }
+ return filepath.Join(s.BasePath, artifact.Path)
+}
+
func getPatterns(reader io.Reader, path []string) []gitignore.Pattern {
var ps []gitignore.Pattern
scanner := bufio.NewScanner(reader)
diff --git a/controllers/storage_test.go b/controllers/storage_test.go
index 268c4488..f27e76a1 100644
--- a/controllers/storage_test.go
+++ b/controllers/storage_test.go
@@ -113,9 +113,9 @@ func walkTar(tarFile string, match string) (bool, error) {
return false, nil
}
-func testPatterns(t *testing.T, artifact sourcev1.Artifact, table ignoreMap) {
+func testPatterns(t *testing.T, dir string, artifact sourcev1.Artifact, table ignoreMap) {
for name, expected := range table {
- res, err := walkTar(artifact.Path, name)
+ res, err := walkTar(filepath.Join(dir, artifact.Path), name)
if err != nil {
t.Fatalf("while reading tarball: %v", err)
}
@@ -130,13 +130,7 @@ func testPatterns(t *testing.T, artifact sourcev1.Artifact, table ignoreMap) {
}
}
-func createArchive(t *testing.T, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact {
- dir, err := createStoragePath()
- if err != nil {
- t.Fatal(err)
- }
- t.Cleanup(cleanupStoragePath(dir))
-
+func createArchive(t *testing.T, dir string, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact {
storage, err := NewStorage(dir, "hostname", time.Minute)
if err != nil {
t.Fatalf("Error while bootstrapping storage: %v", err)
@@ -197,7 +191,13 @@ func TestArchiveBasic(t *testing.T) {
".gitignore": false,
}
- testPatterns(t, createArchive(t, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table)
+ dir, err := createStoragePath()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(cleanupStoragePath(dir))
+
+ testPatterns(t, dir, createArchive(t, dir, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table)
}
func TestArchiveIgnore(t *testing.T) {
@@ -221,8 +221,14 @@ func TestArchiveIgnore(t *testing.T) {
table[item] = false
}
+ dir, err := createStoragePath()
+ if err != nil {
+ t.Fatal(err)
+ }
+ t.Cleanup(cleanupStoragePath(dir))
+
t.Run("automatically ignored files", func(t *testing.T) {
- testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{}), table)
+ testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{}), table)
})
table = ignoreMap{}
@@ -231,7 +237,7 @@ func TestArchiveIgnore(t *testing.T) {
}
t.Run("only vcs ignored files", func(t *testing.T) {
- testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table)
+ testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table)
})
filenames = append(filenames, "test.txt")
@@ -239,7 +245,7 @@ func TestArchiveIgnore(t *testing.T) {
sourceIgnoreFile := "*.txt"
t.Run("sourceignore injected via CRD", func(t *testing.T) {
- testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table)
+ testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table)
})
table = ignoreMap{}
@@ -248,7 +254,7 @@ func TestArchiveIgnore(t *testing.T) {
}
t.Run("sourceignore injected via filename", func(t *testing.T) {
- testPatterns(t, createArchive(t, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table)
+ testPatterns(t, dir, createArchive(t, dir, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table)
})
}
diff --git a/docs/api/source.md b/docs/api/source.md
index b32eea5f..b7294757 100644
--- a/docs/api/source.md
+++ b/docs/api/source.md
@@ -458,7 +458,7 @@ HelmRepositoryStatus
HelmChartStatus,
HelmRepositoryStatus)
-Artifact represents the output of a source synchronisation
+Artifact represents the output of a source synchronisation.