diff --git a/api/v1alpha1/artifact.go b/api/v1alpha1/artifact.go index 131ea863..b9bac76b 100644 --- a/api/v1alpha1/artifact.go +++ b/api/v1alpha1/artifact.go @@ -17,14 +17,15 @@ limitations under the License. package v1alpha1 import ( - "fmt" + "path" + "strings" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) -// Artifact represents the output of a source synchronisation +// Artifact represents the output of a source synchronisation. type Artifact struct { - // Path is the local file path of this artifact. + // Path is the relative file path of this artifact. // +required Path string `json:"path"` @@ -32,20 +33,31 @@ type Artifact struct { // +required URL string `json:"url"` - // Revision is a human readable identifier traceable in the origin source system. - // It can be a commit sha, git tag, a helm index timestamp, - // a helm chart version, a checksum, etc. + // Revision is a human readable identifier traceable in the origin + // source system. It can be a Git commit sha, Git tag, a Helm index + // timestamp, a Helm chart version, etc. // +optional Revision string `json:"revision"` + // Checksum is the SHA1 checksum of the artifact. + // +optional + Checksum string `json:"checksum"` + // LastUpdateTime is the timestamp corresponding to the last // update of this artifact. // +required LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"` } -// ArtifactPath returns the artifact path in the form of -// /// -func ArtifactPath(kind, namespace, name, filename string) string { - return fmt.Sprintf("%s/%s/%s/%s", kind, namespace, name, filename) +// ArtifactDir returns the artifact dir path in the form of +// //. +func ArtifactDir(kind, namespace, name string) string { + kind = strings.ToLower(kind) + return path.Join(kind, namespace, name) +} + +// ArtifactPath returns the artifact path in the form of +// ///. +func ArtifactPath(kind, namespace, name, filename string) string { + return path.Join(ArtifactDir(kind, namespace, name), filename) } diff --git a/api/v1alpha1/source.go b/api/v1alpha1/source.go index a4c3dfe8..4c918416 100644 --- a/api/v1alpha1/source.go +++ b/api/v1alpha1/source.go @@ -1,6 +1,8 @@ package v1alpha1 -import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +import ( + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) // Source interface must be supported by all API types. // +k8s:deepcopy-gen=false diff --git a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml index 2a339ee9..e60d5852 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml @@ -129,18 +129,21 @@ spec: description: Artifact represents the output of the last successful repository sync. properties: + checksum: + description: Checksum is the SHA1 checksum of the artifact. + type: string lastUpdateTime: description: LastUpdateTime is the timestamp corresponding to the last update of this artifact. format: date-time type: string path: - description: Path is the local file path of this artifact. + description: Path is the relative file path of this artifact. type: string revision: description: Revision is a human readable identifier traceable - in the origin source system. It can be a commit sha, git tag, - a helm index timestamp, a helm chart version, a checksum, etc. + in the origin source system. It can be a Git commit sha, Git + tag, a Helm index timestamp, a Helm chart version, etc. type: string url: description: URL is the HTTP address of this artifact. diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml index e66ef34d..d1125575 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml @@ -101,18 +101,21 @@ spec: description: Artifact represents the output of the last successful chart sync. properties: + checksum: + description: Checksum is the SHA1 checksum of the artifact. + type: string lastUpdateTime: description: LastUpdateTime is the timestamp corresponding to the last update of this artifact. format: date-time type: string path: - description: Path is the local file path of this artifact. + description: Path is the relative file path of this artifact. type: string revision: description: Revision is a human readable identifier traceable - in the origin source system. It can be a commit sha, git tag, - a helm index timestamp, a helm chart version, a checksum, etc. + in the origin source system. It can be a Git commit sha, Git + tag, a Helm index timestamp, a Helm chart version, etc. type: string url: description: URL is the HTTP address of this artifact. diff --git a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml index e165ee19..8f7f675e 100644 --- a/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml +++ b/config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml @@ -81,18 +81,21 @@ spec: description: Artifact represents the output of the last successful repository sync. properties: + checksum: + description: Checksum is the SHA1 checksum of the artifact. + type: string lastUpdateTime: description: LastUpdateTime is the timestamp corresponding to the last update of this artifact. format: date-time type: string path: - description: Path is the local file path of this artifact. + description: Path is the relative file path of this artifact. type: string revision: description: Revision is a human readable identifier traceable - in the origin source system. It can be a commit sha, git tag, - a helm index timestamp, a helm chart version, a checksum, etc. + in the origin source system. It can be a Git commit sha, Git + tag, a Helm index timestamp, a Helm chart version, etc. type: string url: description: URL is the HTTP address of this artifact. diff --git a/controllers/gitrepository_controller.go b/controllers/gitrepository_controller.go index b1aa96fc..4ce52033 100644 --- a/controllers/gitrepository_controller.go +++ b/controllers/gitrepository_controller.go @@ -212,8 +212,10 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour } } + // TODO(hidde): implement checksum when https://github.com/fluxcd/source-controller/pull/133 + // has been merged. artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(), - fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision) + fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision, "") // create artifact dir err = r.Storage.MkdirAll(artifact) @@ -300,10 +302,10 @@ func (r *GitRepositoryReconciler) verify(ctx context.Context, publicKeySecret ty // gc performs a garbage collection on all but current artifacts of // the given repository. func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository, all bool) error { + if all { + return r.Storage.RemoveAll(r.Storage.ArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "", "")) + } if repository.Status.Artifact != nil { - if all { - return r.Storage.RemoveAll(*repository.Status.Artifact) - } return r.Storage.RemoveAllButCurrent(*repository.Status.Artifact) } return nil diff --git a/controllers/helmchart_controller.go b/controllers/helmchart_controller.go index 0eaca70b..e1eb1291 100644 --- a/controllers/helmchart_controller.go +++ b/controllers/helmchart_controller.go @@ -188,7 +188,8 @@ func (r *HelmChartReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context, repository sourcev1.HelmRepository, chart sourcev1.HelmChart) (sourcev1.HelmChart, error) { - cv, err := helm.GetDownloadableChartVersionFromIndex(repository.Status.Artifact.Path, chart.Spec.Chart, chart.Spec.Version) + cv, err := helm.GetDownloadableChartVersionFromIndex(r.Storage.LocalPath(*repository.GetArtifact()), + chart.Spec.Chart, chart.Spec.Version) if err != nil { return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err } @@ -260,7 +261,7 @@ func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context, sum := r.Storage.Checksum(chartBytes) artifact := r.Storage.ArtifactFor(chart.Kind, chart.GetObjectMeta(), - fmt.Sprintf("%s-%s-%s.tgz", cv.Name, cv.Version, sum), cv.Version) + fmt.Sprintf("%s-%s-%s.tgz", cv.Name, cv.Version, sum), cv.Version, sum) // create artifact dir err = r.Storage.MkdirAll(artifact) @@ -333,7 +334,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context, defer os.RemoveAll(tmpDir) // open file - f, err := os.Open(repository.GetArtifact().Path) + f, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact())) if err != nil { err = fmt.Errorf("artifact open error: %w", err) return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err @@ -364,8 +365,10 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context, return chart, nil } + // TODO(hidde): implement checksum when https://github.com/fluxcd/source-controller/pull/133 + // has been merged. artifact := r.Storage.ArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(), - fmt.Sprintf("%s-%s.tgz", chartMetadata.Name, chartMetadata.Version), chartMetadata.Version) + fmt.Sprintf("%s-%s.tgz", chartMetadata.Name, chartMetadata.Version), chartMetadata.Version, "") // create artifact dir err = r.Storage.MkdirAll(artifact) @@ -384,7 +387,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context, // package chart pkg := action.NewPackage() - pkg.Destination = filepath.Dir(artifact.Path) + pkg.Destination = filepath.Dir(r.Storage.LocalPath(artifact)) _, err = pkg.Run(chartPath, nil) if err != nil { err = fmt.Errorf("chart package error: %w", err) @@ -432,10 +435,10 @@ func (r *HelmChartReconciler) getGitRepositoryWithArtifact(ctx context.Context, // gc performs a garbage collection on all but current artifacts of // the given chart. func (r *HelmChartReconciler) gc(chart sourcev1.HelmChart, all bool) error { + if all { + return r.Storage.RemoveAll(r.Storage.ArtifactFor(chart.Kind, chart.GetObjectMeta(), "", "", "")) + } if chart.Status.Artifact != nil { - if all { - return r.Storage.RemoveAll(*chart.Status.Artifact) - } return r.Storage.RemoveAllButCurrent(*chart.Status.Artifact) } return nil diff --git a/controllers/helmrepository_controller.go b/controllers/helmrepository_controller.go index 1bd37268..0ea38533 100644 --- a/controllers/helmrepository_controller.go +++ b/controllers/helmrepository_controller.go @@ -229,7 +229,7 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou sum := r.Storage.Checksum(index) artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(), - fmt.Sprintf("index-%s.yaml", sum), sum) + fmt.Sprintf("index-%s.yaml", sum), i.Generated.Format(time.RFC3339Nano), sum) // create artifact dir err = r.Storage.MkdirAll(artifact) @@ -294,10 +294,10 @@ func (r *HelmRepositoryReconciler) shouldResetStatus(repository sourcev1.HelmRep // gc performs a garbage collection on all but current artifacts of // the given repository. func (r *HelmRepositoryReconciler) gc(repository sourcev1.HelmRepository, all bool) error { + if all { + return r.Storage.RemoveAll(r.Storage.ArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "", "")) + } if repository.Status.Artifact != nil { - if all { - return r.Storage.RemoveAll(*repository.Status.Artifact) - } return r.Storage.RemoveAllButCurrent(*repository.Status.Artifact) } return nil diff --git a/controllers/storage.go b/controllers/storage.go index 56e4ea48..6df68106 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -69,9 +69,8 @@ func NewStorage(basePath string, hostname string, timeout time.Duration) (*Stora }, nil } -// ArtifactFor returns an artifact for the given Kubernetes object -func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision string) sourcev1.Artifact { - kind = strings.ToLower(kind) +// ArtifactFor returns an artifact for the v1alpha1.Source. +func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision, checksum string) sourcev1.Artifact { path := sourcev1.ArtifactPath(kind, metadata.GetNamespace(), metadata.GetName(), fileName) localPath := filepath.Join(s.BasePath, path) url := fmt.Sprintf("http://%s/%s", s.Hostname, path) @@ -80,25 +79,27 @@ func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, rev Path: localPath, URL: url, Revision: revision, + Checksum: checksum, LastUpdateTime: metav1.Now(), } } -// MkdirAll calls os.MkdirAll for the given artifact base dir +// MkdirAll calls os.MkdirAll for the given v1alpha1.Artifact base dir. func (s *Storage) MkdirAll(artifact sourcev1.Artifact) error { - dir := filepath.Dir(artifact.Path) + dir := filepath.Dir(s.LocalPath(artifact)) return os.MkdirAll(dir, 0777) } -// RemoveAll calls os.RemoveAll for the given artifact base dir +// RemoveAll calls os.RemoveAll for the given v1alpha1.Artifact base dir. func (s *Storage) RemoveAll(artifact sourcev1.Artifact) error { - dir := filepath.Dir(artifact.Path) + dir := filepath.Dir(s.LocalPath(artifact)) return os.RemoveAll(dir) } // RemoveAllButCurrent removes all files for the given artifact base dir excluding the current one func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error { - dir := filepath.Dir(artifact.Path) + localPath := s.LocalPath(artifact) + dir := filepath.Dir(localPath) var errors []string _ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error { if err != nil { @@ -106,7 +107,7 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error { return nil } - if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink { + if path != localPath && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink { if err := os.Remove(path); err != nil { errors = append(errors, info.Name()) } @@ -123,7 +124,7 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error { // ArtifactExist returns a boolean indicating whether the artifact exists in storage and is a // regular file. func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool { - fi, err := os.Lstat(artifact.Path) + fi, err := os.Lstat(s.LocalPath(artifact)) if err != nil { return false } @@ -144,7 +145,7 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1. matcher := gitignore.NewMatcher(ps) - gzFile, err := os.Create(artifact.Path) + gzFile, err := os.Create(s.LocalPath(artifact)) if err != nil { return err } @@ -205,20 +206,22 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1. // WriteFile writes the given bytes to the artifact path if the checksum differs func (s *Storage) WriteFile(artifact sourcev1.Artifact, data []byte) error { + localPath := s.LocalPath(artifact) sum := s.Checksum(data) - if file, err := os.Stat(artifact.Path); !os.IsNotExist(err) && !file.IsDir() { - if fb, err := ioutil.ReadFile(artifact.Path); err == nil && sum == s.Checksum(fb) { + if file, err := os.Stat(localPath); !os.IsNotExist(err) && !file.IsDir() { + if fb, err := ioutil.ReadFile(localPath); err == nil && sum == s.Checksum(fb) { return nil } } - return ioutil.WriteFile(artifact.Path, data, 0644) + return ioutil.WriteFile(localPath, data, 0644) } // Symlink creates or updates a symbolic link for the given artifact -// and returns the URL for the symlink +// and returns the URL for the symlink. func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string, error) { - dir := filepath.Dir(artifact.Path) + localPath := s.LocalPath(artifact) + dir := filepath.Dir(localPath) link := filepath.Join(dir, linkName) tmpLink := link + ".tmp" @@ -226,7 +229,7 @@ func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string, return "", err } - if err := os.Symlink(artifact.Path, tmpLink); err != nil { + if err := os.Symlink(localPath, tmpLink); err != nil { return "", err } @@ -246,11 +249,20 @@ func (s *Storage) Checksum(b []byte) string { // Lock creates a file lock for the given artifact func (s *Storage) Lock(artifact sourcev1.Artifact) (unlock func(), err error) { - lockFile := artifact.Path + ".lock" + lockFile := s.LocalPath(artifact) + ".lock" mutex := lockedfile.MutexAt(lockFile) return mutex.Lock() } +// LocalPath returns the local path of the given artifact (that is: relative to +// the Storage.BasePath). +func (s *Storage) LocalPath(artifact sourcev1.Artifact) string { + if artifact.Path == "" { + return "" + } + return filepath.Join(s.BasePath, artifact.Path) +} + func getPatterns(reader io.Reader, path []string) []gitignore.Pattern { var ps []gitignore.Pattern scanner := bufio.NewScanner(reader) diff --git a/controllers/storage_test.go b/controllers/storage_test.go index 268c4488..f27e76a1 100644 --- a/controllers/storage_test.go +++ b/controllers/storage_test.go @@ -113,9 +113,9 @@ func walkTar(tarFile string, match string) (bool, error) { return false, nil } -func testPatterns(t *testing.T, artifact sourcev1.Artifact, table ignoreMap) { +func testPatterns(t *testing.T, dir string, artifact sourcev1.Artifact, table ignoreMap) { for name, expected := range table { - res, err := walkTar(artifact.Path, name) + res, err := walkTar(filepath.Join(dir, artifact.Path), name) if err != nil { t.Fatalf("while reading tarball: %v", err) } @@ -130,13 +130,7 @@ func testPatterns(t *testing.T, artifact sourcev1.Artifact, table ignoreMap) { } } -func createArchive(t *testing.T, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact { - dir, err := createStoragePath() - if err != nil { - t.Fatal(err) - } - t.Cleanup(cleanupStoragePath(dir)) - +func createArchive(t *testing.T, dir string, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact { storage, err := NewStorage(dir, "hostname", time.Minute) if err != nil { t.Fatalf("Error while bootstrapping storage: %v", err) @@ -197,7 +191,13 @@ func TestArchiveBasic(t *testing.T) { ".gitignore": false, } - testPatterns(t, createArchive(t, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table) + dir, err := createStoragePath() + if err != nil { + t.Fatal(err) + } + t.Cleanup(cleanupStoragePath(dir)) + + testPatterns(t, dir, createArchive(t, dir, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table) } func TestArchiveIgnore(t *testing.T) { @@ -221,8 +221,14 @@ func TestArchiveIgnore(t *testing.T) { table[item] = false } + dir, err := createStoragePath() + if err != nil { + t.Fatal(err) + } + t.Cleanup(cleanupStoragePath(dir)) + t.Run("automatically ignored files", func(t *testing.T) { - testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{}), table) + testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{}), table) }) table = ignoreMap{} @@ -231,7 +237,7 @@ func TestArchiveIgnore(t *testing.T) { } t.Run("only vcs ignored files", func(t *testing.T) { - testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table) + testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table) }) filenames = append(filenames, "test.txt") @@ -239,7 +245,7 @@ func TestArchiveIgnore(t *testing.T) { sourceIgnoreFile := "*.txt" t.Run("sourceignore injected via CRD", func(t *testing.T) { - testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table) + testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table) }) table = ignoreMap{} @@ -248,7 +254,7 @@ func TestArchiveIgnore(t *testing.T) { } t.Run("sourceignore injected via filename", func(t *testing.T) { - testPatterns(t, createArchive(t, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table) + testPatterns(t, dir, createArchive(t, dir, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table) }) } diff --git a/docs/api/source.md b/docs/api/source.md index b32eea5f..b7294757 100644 --- a/docs/api/source.md +++ b/docs/api/source.md @@ -458,7 +458,7 @@ HelmRepositoryStatus HelmChartStatus, HelmRepositoryStatus)

-

Artifact represents the output of a source synchronisation

+

Artifact represents the output of a source synchronisation.

@@ -477,7 +477,7 @@ string @@ -500,9 +500,21 @@ string + + + + diff --git a/docs/spec/v1alpha1/common.md b/docs/spec/v1alpha1/common.md index ae8910d9..1007ef79 100644 --- a/docs/spec/v1alpha1/common.md +++ b/docs/spec/v1alpha1/common.md @@ -54,9 +54,9 @@ kubectl annotate --overwrite gitrepository/podinfo fluxcd.io/reconcileAt="$(date Source objects should contain a status sub-resource that embeds an artifact object: ```go -// Artifact represents the output of a source synchronisation +// Artifact represents the output of a source synchronisation. type Artifact struct { - // Path is the local file path of this artifact. + // Path is the relative file path of this artifact. // +required Path string `json:"path"` @@ -64,12 +64,16 @@ type Artifact struct { // +required URL string `json:"url"` - // Revision is a human readable identifier traceable in the origin source system. - // It can be a commit sha, git tag, a helm index timestamp, - // a helm chart version, a checksum, etc. + // Revision is a human readable identifier traceable in the origin + // source system. It can be a Git commit sha, Git tag, a Helm index + // timestamp, a Helm chart version, etc. // +optional Revision string `json:"revision"` + // Checksum is the SHA1 checksum of the artifact. + // +optional + Checksum string `json:"checksum"` + // LastUpdateTime is the timestamp corresponding to the last // update of this artifact. // +required
-

Path is the local file path of this artifact.

+

Path is the relative file path of this artifact.

(Optional) -

Revision is a human readable identifier traceable in the origin source system. -It can be a commit sha, git tag, a helm index timestamp, -a helm chart version, a checksum, etc.

+

Revision is a human readable identifier traceable in the origin +source system. It can be a Git commit sha, Git tag, a Helm index +timestamp, a Helm chart version, etc.

+
+checksum
+ +string + +
+(Optional) +

Checksum is the SHA1 checksum of the artifact.