fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Merge pull request #134 from fluxcd/artifact-api-improvements

This commit is contained in:
Hidde Beydals 2020-09-09 14:10:33 +02:00 committed by GitHub
parent ecaa98012f 68947cfca6
commit ec51edae43
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 140 additions and 78 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
api/v1alpha1/artifact.go
View File

@ -17,14 +17,15 @@ limitations under the License.
package v1alpha1
import (
"fmt"
"path"
"strings"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// Artifact represents the output of a source synchronisation
// Artifact represents the output of a source synchronisation.
type Artifact struct {
// Path is the local file path of this artifact.
// Path is the relative file path of this artifact.
// +required
Path string `json:"path"`
@ -32,20 +33,31 @@ type Artifact struct {
// +required
URL string `json:"url"`
// Revision is a human readable identifier traceable in the origin source system.
// It can be a commit sha, git tag, a helm index timestamp,
// a helm chart version, a checksum, etc.
// Revision is a human readable identifier traceable in the origin
// source system. It can be a Git commit sha, Git tag, a Helm index
// timestamp, a Helm chart version, etc.
// +optional
Revision string `json:"revision"`
// Checksum is the SHA1 checksum of the artifact.
// +optional
Checksum string `json:"checksum"`
// LastUpdateTime is the timestamp corresponding to the last
// update of this artifact.
// +required
LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
}
// ArtifactPath returns the artifact path in the form of
// <source-kind>/<source-namespace>/<source-name>/<artifact-filename>
func ArtifactPath(kind, namespace, name, filename string) string {
return fmt.Sprintf("%s/%s/%s/%s", kind, namespace, name, filename)
// ArtifactDir returns the artifact dir path in the form of
// <source-kind>/<source-namespace>/<source-name>.
func ArtifactDir(kind, namespace, name string) string {
kind = strings.ToLower(kind)
return path.Join(kind, namespace, name)
}
// ArtifactPath returns the artifact path in the form of
// <source-kind>/<source-namespace>/<source-name>/<artifact-filename>.
func ArtifactPath(kind, namespace, name, filename string) string {
return path.Join(ArtifactDir(kind, namespace, name), filename)
}

4
api/v1alpha1/source.go
View File

@ -1,6 +1,8 @@
package v1alpha1
import metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
import (
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
)
// Source interface must be supported by all API types.
// +k8s:deepcopy-gen=false

9
config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml
View File

@ -129,18 +129,21 @@ spec:
description: Artifact represents the output of the last successful
repository sync.
properties:
checksum:
description: Checksum is the SHA1 checksum of the artifact.
type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of this artifact.
format: date-time
type: string
path:
description: Path is the local file path of this artifact.
description: Path is the relative file path of this artifact.
type: string
revision:
description: Revision is a human readable identifier traceable
in the origin source system. It can be a commit sha, git tag,
a helm index timestamp, a helm chart version, a checksum, etc.
in the origin source system. It can be a Git commit sha, Git
tag, a Helm index timestamp, a Helm chart version, etc.
type: string
url:
description: URL is the HTTP address of this artifact.

9
config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml
View File

@ -101,18 +101,21 @@ spec:
description: Artifact represents the output of the last successful
chart sync.
properties:
checksum:
description: Checksum is the SHA1 checksum of the artifact.
type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of this artifact.
format: date-time
type: string
path:
description: Path is the local file path of this artifact.
description: Path is the relative file path of this artifact.
type: string
revision:
description: Revision is a human readable identifier traceable
in the origin source system. It can be a commit sha, git tag,
a helm index timestamp, a helm chart version, a checksum, etc.
in the origin source system. It can be a Git commit sha, Git
tag, a Helm index timestamp, a Helm chart version, etc.
type: string
url:
description: URL is the HTTP address of this artifact.

9
config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml
View File

@ -81,18 +81,21 @@ spec:
description: Artifact represents the output of the last successful
repository sync.
properties:
checksum:
description: Checksum is the SHA1 checksum of the artifact.
type: string
lastUpdateTime:
description: LastUpdateTime is the timestamp corresponding to
the last update of this artifact.
format: date-time
type: string
path:
description: Path is the local file path of this artifact.
description: Path is the relative file path of this artifact.
type: string
revision:
description: Revision is a human readable identifier traceable
in the origin source system. It can be a commit sha, git tag,
a helm index timestamp, a helm chart version, a checksum, etc.
in the origin source system. It can be a Git commit sha, Git
tag, a Helm index timestamp, a Helm chart version, etc.
type: string
url:
description: URL is the HTTP address of this artifact.

10
controllers/gitrepository_controller.go
View File

@ -212,8 +212,10 @@ func (r *GitRepositoryReconciler) reconcile(ctx context.Context, repository sour
}
}
// TODO(hidde): implement checksum when https://github.com/fluxcd/source-controller/pull/133
// has been merged.
artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(),
fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision)
fmt.Sprintf("%s.tar.gz", commit.Hash.String()), revision, "")
// create artifact dir
err = r.Storage.MkdirAll(artifact)
@ -300,10 +302,10 @@ func (r *GitRepositoryReconciler) verify(ctx context.Context, publicKeySecret ty
// gc performs a garbage collection on all but current artifacts of
// the given repository.
func (r *GitRepositoryReconciler) gc(repository sourcev1.GitRepository, all bool) error {
if all {
return r.Storage.RemoveAll(r.Storage.ArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "", ""))
}
if repository.Status.Artifact != nil {
if all {
return r.Storage.RemoveAll(*repository.Status.Artifact)
}
return r.Storage.RemoveAllButCurrent(*repository.Status.Artifact)
}
return nil

19
controllers/helmchart_controller.go
View File

@ -188,7 +188,8 @@ func (r *HelmChartReconciler) SetupWithManagerAndOptions(mgr ctrl.Manager, opts
func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
repository sourcev1.HelmRepository, chart sourcev1.HelmChart) (sourcev1.HelmChart, error) {
cv, err := helm.GetDownloadableChartVersionFromIndex(repository.Status.Artifact.Path, chart.Spec.Chart, chart.Spec.Version)
cv, err := helm.GetDownloadableChartVersionFromIndex(r.Storage.LocalPath(*repository.GetArtifact()),
chart.Spec.Chart, chart.Spec.Version)
if err != nil {
return sourcev1.HelmChartNotReady(chart, sourcev1.ChartPullFailedReason, err.Error()), err
}
@ -260,7 +261,7 @@ func (r *HelmChartReconciler) reconcileFromHelmRepository(ctx context.Context,
sum := r.Storage.Checksum(chartBytes)
artifact := r.Storage.ArtifactFor(chart.Kind, chart.GetObjectMeta(),
fmt.Sprintf("%s-%s-%s.tgz", cv.Name, cv.Version, sum), cv.Version)
fmt.Sprintf("%s-%s-%s.tgz", cv.Name, cv.Version, sum), cv.Version, sum)
// create artifact dir
err = r.Storage.MkdirAll(artifact)
@ -333,7 +334,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
defer os.RemoveAll(tmpDir)
// open file
f, err := os.Open(repository.GetArtifact().Path)
f, err := os.Open(r.Storage.LocalPath(*repository.GetArtifact()))
if err != nil {
err = fmt.Errorf("artifact open error: %w", err)
return sourcev1.HelmChartNotReady(chart, sourcev1.StorageOperationFailedReason, err.Error()), err
@ -364,8 +365,10 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
return chart, nil
}
// TODO(hidde): implement checksum when https://github.com/fluxcd/source-controller/pull/133
// has been merged.
artifact := r.Storage.ArtifactFor(chart.Kind, chart.ObjectMeta.GetObjectMeta(),
fmt.Sprintf("%s-%s.tgz", chartMetadata.Name, chartMetadata.Version), chartMetadata.Version)
fmt.Sprintf("%s-%s.tgz", chartMetadata.Name, chartMetadata.Version), chartMetadata.Version, "")
// create artifact dir
err = r.Storage.MkdirAll(artifact)
@ -384,7 +387,7 @@ func (r *HelmChartReconciler) reconcileFromGitRepository(ctx context.Context,
// package chart
pkg := action.NewPackage()
pkg.Destination = filepath.Dir(artifact.Path)
pkg.Destination = filepath.Dir(r.Storage.LocalPath(artifact))
_, err = pkg.Run(chartPath, nil)
if err != nil {
err = fmt.Errorf("chart package error: %w", err)
@ -432,10 +435,10 @@ func (r *HelmChartReconciler) getGitRepositoryWithArtifact(ctx context.Context,
// gc performs a garbage collection on all but current artifacts of
// the given chart.
func (r *HelmChartReconciler) gc(chart sourcev1.HelmChart, all bool) error {
if all {
return r.Storage.RemoveAll(r.Storage.ArtifactFor(chart.Kind, chart.GetObjectMeta(), "", "", ""))
}
if chart.Status.Artifact != nil {
if all {
return r.Storage.RemoveAll(*chart.Status.Artifact)
}
return r.Storage.RemoveAllButCurrent(*chart.Status.Artifact)
}
return nil

8
controllers/helmrepository_controller.go
View File

@ -229,7 +229,7 @@ func (r *HelmRepositoryReconciler) reconcile(ctx context.Context, repository sou
sum := r.Storage.Checksum(index)
artifact := r.Storage.ArtifactFor(repository.Kind, repository.ObjectMeta.GetObjectMeta(),
fmt.Sprintf("index-%s.yaml", sum), sum)
fmt.Sprintf("index-%s.yaml", sum), i.Generated.Format(time.RFC3339Nano), sum)
// create artifact dir
err = r.Storage.MkdirAll(artifact)
@ -294,10 +294,10 @@ func (r *HelmRepositoryReconciler) shouldResetStatus(repository sourcev1.HelmRep
// gc performs a garbage collection on all but current artifacts of
// the given repository.
func (r *HelmRepositoryReconciler) gc(repository sourcev1.HelmRepository, all bool) error {
if all {
return r.Storage.RemoveAll(r.Storage.ArtifactFor(repository.Kind, repository.GetObjectMeta(), "", "", ""))
}
if repository.Status.Artifact != nil {
if all {
return r.Storage.RemoveAll(*repository.Status.Artifact)
}
return r.Storage.RemoveAllButCurrent(*repository.Status.Artifact)
}
return nil

48
controllers/storage.go
View File

@ -69,9 +69,8 @@ func NewStorage(basePath string, hostname string, timeout time.Duration) (*Stora
}, nil
}
// ArtifactFor returns an artifact for the given Kubernetes object
func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision string) sourcev1.Artifact {
kind = strings.ToLower(kind)
// ArtifactFor returns an artifact for the v1alpha1.Source.
func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, revision, checksum string) sourcev1.Artifact {
path := sourcev1.ArtifactPath(kind, metadata.GetNamespace(), metadata.GetName(), fileName)
localPath := filepath.Join(s.BasePath, path)
url := fmt.Sprintf("http://%s/%s", s.Hostname, path)
@ -80,25 +79,27 @@ func (s *Storage) ArtifactFor(kind string, metadata metav1.Object, fileName, rev
Path: localPath,
URL: url,
Revision: revision,
Checksum: checksum,
LastUpdateTime: metav1.Now(),
}
}
// MkdirAll calls os.MkdirAll for the given artifact base dir
// MkdirAll calls os.MkdirAll for the given v1alpha1.Artifact base dir.
func (s *Storage) MkdirAll(artifact sourcev1.Artifact) error {
dir := filepath.Dir(artifact.Path)
dir := filepath.Dir(s.LocalPath(artifact))
return os.MkdirAll(dir, 0777)
}
// RemoveAll calls os.RemoveAll for the given artifact base dir
// RemoveAll calls os.RemoveAll for the given v1alpha1.Artifact base dir.
func (s *Storage) RemoveAll(artifact sourcev1.Artifact) error {
dir := filepath.Dir(artifact.Path)
dir := filepath.Dir(s.LocalPath(artifact))
return os.RemoveAll(dir)
}
// RemoveAllButCurrent removes all files for the given artifact base dir excluding the current one
func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
dir := filepath.Dir(artifact.Path)
localPath := s.LocalPath(artifact)
dir := filepath.Dir(localPath)
var errors []string
_ = filepath.Walk(dir, func(path string, info os.FileInfo, err error) error {
if err != nil {
@ -106,7 +107,7 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
return nil
}
if path != artifact.Path && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
if path != localPath && !info.IsDir() && info.Mode()&os.ModeSymlink != os.ModeSymlink {
if err := os.Remove(path); err != nil {
errors = append(errors, info.Name())
}
@ -123,7 +124,7 @@ func (s *Storage) RemoveAllButCurrent(artifact sourcev1.Artifact) error {
// ArtifactExist returns a boolean indicating whether the artifact exists in storage and is a
// regular file.
func (s *Storage) ArtifactExist(artifact sourcev1.Artifact) bool {
fi, err := os.Lstat(artifact.Path)
fi, err := os.Lstat(s.LocalPath(artifact))
if err != nil {
return false
}
@ -144,7 +145,7 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.
matcher := gitignore.NewMatcher(ps)
gzFile, err := os.Create(artifact.Path)
gzFile, err := os.Create(s.LocalPath(artifact))
if err != nil {
return err
}
@ -205,20 +206,22 @@ func (s *Storage) Archive(artifact sourcev1.Artifact, dir string, spec sourcev1.
// WriteFile writes the given bytes to the artifact path if the checksum differs
func (s *Storage) WriteFile(artifact sourcev1.Artifact, data []byte) error {
localPath := s.LocalPath(artifact)
sum := s.Checksum(data)
if file, err := os.Stat(artifact.Path); !os.IsNotExist(err) && !file.IsDir() {
if fb, err := ioutil.ReadFile(artifact.Path); err == nil && sum == s.Checksum(fb) {
if file, err := os.Stat(localPath); !os.IsNotExist(err) && !file.IsDir() {
if fb, err := ioutil.ReadFile(localPath); err == nil && sum == s.Checksum(fb) {
return nil
}
}
return ioutil.WriteFile(artifact.Path, data, 0644)
return ioutil.WriteFile(localPath, data, 0644)
}
// Symlink creates or updates a symbolic link for the given artifact
// and returns the URL for the symlink
// and returns the URL for the symlink.
func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string, error) {
dir := filepath.Dir(artifact.Path)
localPath := s.LocalPath(artifact)
dir := filepath.Dir(localPath)
link := filepath.Join(dir, linkName)
tmpLink := link + ".tmp"
@ -226,7 +229,7 @@ func (s *Storage) Symlink(artifact sourcev1.Artifact, linkName string) (string,
return "", err
}
if err := os.Symlink(artifact.Path, tmpLink); err != nil {
if err := os.Symlink(localPath, tmpLink); err != nil {
return "", err
}
@ -246,11 +249,20 @@ func (s *Storage) Checksum(b []byte) string {
// Lock creates a file lock for the given artifact
func (s *Storage) Lock(artifact sourcev1.Artifact) (unlock func(), err error) {
lockFile := artifact.Path + ".lock"
lockFile := s.LocalPath(artifact) + ".lock"
mutex := lockedfile.MutexAt(lockFile)
return mutex.Lock()
}
// LocalPath returns the local path of the given artifact (that is: relative to
// the Storage.BasePath).
func (s *Storage) LocalPath(artifact sourcev1.Artifact) string {
if artifact.Path == "" {
return ""
}
return filepath.Join(s.BasePath, artifact.Path)
}
func getPatterns(reader io.Reader, path []string) []gitignore.Pattern {
var ps []gitignore.Pattern
scanner := bufio.NewScanner(reader)

34
controllers/storage_test.go
View File

@ -113,9 +113,9 @@ func walkTar(tarFile string, match string) (bool, error) {
return false, nil
}
func testPatterns(t *testing.T, artifact sourcev1.Artifact, table ignoreMap) {
func testPatterns(t *testing.T, dir string, artifact sourcev1.Artifact, table ignoreMap) {
for name, expected := range table {
res, err := walkTar(artifact.Path, name)
res, err := walkTar(filepath.Join(dir, artifact.Path), name)
if err != nil {
t.Fatalf("while reading tarball: %v", err)
}
@ -130,13 +130,7 @@ func testPatterns(t *testing.T, artifact sourcev1.Artifact, table ignoreMap) {
}
}
func createArchive(t *testing.T, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact {
dir, err := createStoragePath()
if err != nil {
t.Fatal(err)
}
t.Cleanup(cleanupStoragePath(dir))
func createArchive(t *testing.T, dir string, filenames []string, sourceIgnore string, spec sourcev1.GitRepositorySpec) sourcev1.Artifact {
storage, err := NewStorage(dir, "hostname", time.Minute)
if err != nil {
t.Fatalf("Error while bootstrapping storage: %v", err)
@ -197,7 +191,13 @@ func TestArchiveBasic(t *testing.T) {
".gitignore": false,
}
testPatterns(t, createArchive(t, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table)
dir, err := createStoragePath()
if err != nil {
t.Fatal(err)
}
t.Cleanup(cleanupStoragePath(dir))
testPatterns(t, dir, createArchive(t, dir, []string{"README.md", ".gitignore"}, "", sourcev1.GitRepositorySpec{}), table)
}
func TestArchiveIgnore(t *testing.T) {
@ -221,8 +221,14 @@ func TestArchiveIgnore(t *testing.T) {
table[item] = false
}
dir, err := createStoragePath()
if err != nil {
t.Fatal(err)
}
t.Cleanup(cleanupStoragePath(dir))
t.Run("automatically ignored files", func(t *testing.T) {
testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{}), table)
testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{}), table)
})
table = ignoreMap{}
@ -231,7 +237,7 @@ func TestArchiveIgnore(t *testing.T) {
}
t.Run("only vcs ignored files", func(t *testing.T) {
testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table)
testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr("")}), table)
})
filenames = append(filenames, "test.txt")
@ -239,7 +245,7 @@ func TestArchiveIgnore(t *testing.T) {
sourceIgnoreFile := "*.txt"
t.Run("sourceignore injected via CRD", func(t *testing.T) {
testPatterns(t, createArchive(t, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table)
testPatterns(t, dir, createArchive(t, dir, filenames, "", sourcev1.GitRepositorySpec{Ignore: stringPtr(sourceIgnoreFile)}), table)
})
table = ignoreMap{}
@ -248,7 +254,7 @@ func TestArchiveIgnore(t *testing.T) {
}
t.Run("sourceignore injected via filename", func(t *testing.T) {
testPatterns(t, createArchive(t, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table)
testPatterns(t, dir, createArchive(t, dir, filenames, sourceIgnoreFile, sourcev1.GitRepositorySpec{}), table)
})
}

22
docs/api/source.md
View File

@ -458,7 +458,7 @@ HelmRepositoryStatus
<a href="#source.toolkit.fluxcd.io/v1alpha1.HelmChartStatus">HelmChartStatus</a>,
<a href="#source.toolkit.fluxcd.io/v1alpha1.HelmRepositoryStatus">HelmRepositoryStatus</a>)
</p>
<p>Artifact represents the output of a source synchronisation</p>
<p>Artifact represents the output of a source synchronisation.</p>
<div class="md-typeset__scrollwrap">
<div class="md-typeset__table">
<table>
@ -477,7 +477,7 @@ string
</em>
</td>
<td>
<p>Path is the local file path of this artifact.</p>
<p>Path is the relative file path of this artifact.</p>
</td>
</tr>
<tr>
@ -500,9 +500,21 @@ string
</td>
<td>
<em>(Optional)</em>
<p>Revision is a human readable identifier traceable in the origin source system.
It can be a commit sha, git tag, a helm index timestamp,
a helm chart version, a checksum, etc.</p>
<p>Revision is a human readable identifier traceable in the origin
source system. It can be a Git commit sha, Git tag, a Helm index
timestamp, a Helm chart version, etc.</p>
</td>
</tr>
<tr>
<td>
<code>checksum</code><br>
<em>
string
</em>
</td>
<td>
<em>(Optional)</em>
<p>Checksum is the SHA1 checksum of the artifact.</p>
</td>
</tr>
<tr>

14
docs/spec/v1alpha1/common.md
View File

@ -54,9 +54,9 @@ kubectl annotate --overwrite gitrepository/podinfo fluxcd.io/reconcileAt="$(date
Source objects should contain a status sub-resource that embeds an artifact object:
```go
// Artifact represents the output of a source synchronisation
// Artifact represents the output of a source synchronisation.
type Artifact struct {
// Path is the local file path of this artifact.
// Path is the relative file path of this artifact.
// +required
Path string `json:"path"`
@ -64,12 +64,16 @@ type Artifact struct {
// +required
URL string `json:"url"`
// Revision is a human readable identifier traceable in the origin source system.
// It can be a commit sha, git tag, a helm index timestamp,
// a helm chart version, a checksum, etc.
// Revision is a human readable identifier traceable in the origin
// source system. It can be a Git commit sha, Git tag, a Helm index
// timestamp, a Helm chart version, etc.
// +optional
Revision string `json:"revision"`
// Checksum is the SHA1 checksum of the artifact.
// +optional
Checksum string `json:"checksum"`
// LastUpdateTime is the timestamp corresponding to the last
// update of this artifact.
// +required