fluxcd
/
source-controller
mirror of https://github.com/fluxcd/source-controller.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Implement Size field on archived artifacts 

This adds a Size field to Artifacts, which reflects the number of bytes
written to the artifact when it's being archived.

Signed-off-by: Kevin McDermott <bigkevmcd@gmail.com>
This commit is contained in:
Kevin McDermott 2022-02-22 10:30:58 +00:00 committed by Hidde Beydals
parent 25e6e16a75
commit f7105ea736
14 changed files with 95 additions and 5 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
api/v1beta2/artifact_types.go
View File

@ -51,6 +51,10 @@ type Artifact struct {
// artifact.
// +required
LastUpdateTime metav1.Time `json:"lastUpdateTime,omitempty"`
// Size is the number of bytes in the file.
// +optional
Size *int64 `json:"size,omitempty"`
}
// HasRevision returns true if the given revision matches the current Revision

5
api/v1beta2/zz_generated.deepcopy.go
View File

@ -32,6 +32,11 @@ import (
func (in *Artifact) DeepCopyInto(out *Artifact) {
*out = *in
in.LastUpdateTime.DeepCopyInto(&out.LastUpdateTime)
if in.Size != nil {
in, out := &in.Size, &out.Size
*out = new(int64)
**out = **in
}
}
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new Artifact.

4
config/crd/bases/source.toolkit.fluxcd.io_buckets.yaml
View File

@ -391,6 +391,10 @@ spec:
in the origin source system. It can be a Git commit SHA, Git
tag, a Helm index timestamp, a Helm chart version, etc.
type: string
size:
description: Size is the number of bytes in the file.
format: int64
type: integer
url:
description: URL is the HTTP address of this artifact. It is used
by the consumers of the artifacts to fetch and use the artifacts.

8
config/crd/bases/source.toolkit.fluxcd.io_gitrepositories.yaml
View File

@ -559,6 +559,10 @@ spec:
in the origin source system. It can be a Git commit SHA, Git
tag, a Helm index timestamp, a Helm chart version, etc.
type: string
size:
description: Size is the number of bytes in the file.
format: int64
type: integer
url:
description: URL is the HTTP address of this artifact. It is used
by the consumers of the artifacts to fetch and use the artifacts.
@ -663,6 +667,10 @@ spec:
in the origin source system. It can be a Git commit SHA, Git
tag, a Helm index timestamp, a Helm chart version, etc.
type: string
size:
description: Size is the number of bytes in the file.
format: int64
type: integer
url:
description: URL is the HTTP address of this artifact. It is
used by the consumers of the artifacts to fetch and use the

4
config/crd/bases/source.toolkit.fluxcd.io_helmcharts.yaml
View File

@ -438,6 +438,10 @@ spec:
in the origin source system. It can be a Git commit SHA, Git
tag, a Helm index timestamp, a Helm chart version, etc.
type: string
size:
description: Size is the number of bytes in the file.
format: int64
type: integer
url:
description: URL is the HTTP address of this artifact. It is used
by the consumers of the artifacts to fetch and use the artifacts.

4
config/crd/bases/source.toolkit.fluxcd.io_helmrepositories.yaml
View File

@ -364,6 +364,10 @@ spec:
in the origin source system. It can be a Git commit SHA, Git
tag, a Helm index timestamp, a Helm chart version, etc.
type: string
size:
description: Size is the number of bytes in the file.
format: int64
type: integer
url:
description: URL is the HTTP address of this artifact. It is used
by the consumers of the artifacts to fetch and use the artifacts.

3
controllers/artifact_matchers_test.go
View File

@ -54,6 +54,9 @@ func (m matchArtifact) Match(actual interface{}) (success bool, err error) {
if ok, err = Equal(m.expected.Checksum).Match(actualArtifact.Checksum); !ok {
return ok, err
}
if ok, err = Equal(m.expected.Size).Match(actualArtifact.Size); !ok {
return ok, err
}
return ok, err
}

2
controllers/bucket_controller_test.go
View File

@ -200,6 +200,7 @@ func TestBucketReconciler_reconcileStorage(t *testing.T) {
Revision: "c",
Checksum: "2e7d2c03a9507ae265ecf5b5356885a53393a2029d241394997265a1a25aefc6",
URL: testStorage.Hostname + "/reconcile-storage/c.txt",
Size: int64p(int64(len("c"))),
},
assertPaths: []string{
"/reconcile-storage/c.txt",
@ -251,6 +252,7 @@ func TestBucketReconciler_reconcileStorage(t *testing.T) {
Revision: "f",
Checksum: "3b9c358f36f0a31b6ad3e14f309c7cf198ac9246e8316f9ce543d5b19ac02b80",
URL: testStorage.Hostname + "/reconcile-storage/hostname.txt",
Size: int64p(int64(len("file"))),
},
},
}

15
controllers/gitrepository_controller_test.go
View File

@ -19,6 +19,7 @@ package controllers
import (
"context"
"fmt"
"net/http"
"net/url"
"os"
"path/filepath"
@ -818,6 +819,16 @@ func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) {
wantErr: true,
},
}
artifactSize := func(g *WithT, artifactURL string) *int64 {
if artifactURL == "" {
return nil
}
res, err := http.Get(artifactURL)
g.Expect(err).NotTo(HaveOccurred())
g.Expect(res.StatusCode).To(Equal(http.StatusOK))
defer res.Body.Close()
return &res.ContentLength
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
@ -850,6 +861,10 @@ func TestGitRepositoryReconciler_reconcileArtifact(t *testing.T) {
g.Expect(err != nil).To(Equal(tt.wantErr))
g.Expect(got).To(Equal(tt.want))
if obj.Status.Artifact != nil {
g.Expect(obj.Status.Artifact.Size).To(Equal(artifactSize(g, obj.Status.Artifact.URL)))
}
if tt.afterFunc != nil {
tt.afterFunc(g, obj)
}

2
controllers/helmchart_controller_test.go
View File

@ -198,6 +198,7 @@ func TestHelmChartReconciler_reconcileStorage(t *testing.T) {
Revision: "c",
Checksum: "2e7d2c03a9507ae265ecf5b5356885a53393a2029d241394997265a1a25aefc6",
URL: testStorage.Hostname + "/reconcile-storage/c.txt",
Size: int64p(int64(len("c"))),
},
assertPaths: []string{
"/reconcile-storage/c.txt",
@ -250,6 +251,7 @@ func TestHelmChartReconciler_reconcileStorage(t *testing.T) {
Revision: "f",
Checksum: "3b9c358f36f0a31b6ad3e14f309c7cf198ac9246e8316f9ce543d5b19ac02b80",
URL: testStorage.Hostname + "/reconcile-storage/hostname.txt",
Size: int64p(int64(len("file"))),
},
},
}

2
controllers/helmrepository_controller_test.go
View File

@ -166,6 +166,7 @@ func TestHelmRepositoryReconciler_reconcileStorage(t *testing.T) {
Revision: "c",
Checksum: "2e7d2c03a9507ae265ecf5b5356885a53393a2029d241394997265a1a25aefc6",
URL: testStorage.Hostname + "/reconcile-storage/c.txt",
Size: int64p(int64(len("c"))),
},
assertPaths: []string{
"/reconcile-storage/c.txt",
@ -218,6 +219,7 @@ func TestHelmRepositoryReconciler_reconcileStorage(t *testing.T) {
Revision: "f",
Checksum: "3b9c358f36f0a31b6ad3e14f309c7cf198ac9246e8316f9ce543d5b19ac02b80",
URL: testStorage.Hostname + "/reconcile-storage/hostname.txt",
Size: int64p(int64(len("file"))),
},
},
}

27
controllers/storage.go
View File

@ -194,7 +194,8 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
}()
h := newHash()
mw := io.MultiWriter(h, tf)
sz := &writeCounter{}
mw := io.MultiWriter(h, tf, sz)
gw := gzip.NewWriter(mw)
tw := tar.NewWriter(gw)
@ -286,6 +287,8 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
artifact.LastUpdateTime = metav1.Now()
artifact.Size = &sz.written
return nil
}
@ -305,7 +308,8 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
}()
h := newHash()
mw := io.MultiWriter(h, tf)
sz := &writeCounter{}
mw := io.MultiWriter(h, tf, sz)
if _, err := io.Copy(mw, reader); err != nil {
tf.Close()
@ -325,6 +329,8 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
artifact.LastUpdateTime = metav1.Now()
artifact.Size = &sz.written
return nil
}
@ -344,7 +350,8 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error
}()
h := newHash()
mw := io.MultiWriter(h, tf)
sz := &writeCounter{}
mw := io.MultiWriter(h, tf, sz)
if _, err := io.Copy(mw, reader); err != nil {
tf.Close()
@ -360,6 +367,8 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error
artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
artifact.LastUpdateTime = metav1.Now()
artifact.Size = &sz.written
return nil
}
@ -471,3 +480,15 @@ func (s *Storage) LocalPath(artifact sourcev1.Artifact) string {
func newHash() hash.Hash {
return sha256.New()
}
// writecounter is an implementation of io.Writer that only records the number
// of bytes written.
type writeCounter struct {
written int64
}
func (wc *writeCounter) Write(p []byte) (int, error) {
n := len(p)
wc.written += int64(n)
return n, nil
}

4
controllers/suite_test.go
View File

@ -193,3 +193,7 @@ func randStringRunes(n int) string {
}
return string(b)
}
func int64p(i int64) *int64 {
return &i
}

16
docs/api/source.md
View File

@ -931,6 +931,18 @@ Kubernetes meta/v1.Time
artifact.</p>
</td>
</tr>
<tr>
<td>
<code>size</code><br>
<em>
int64
</em>
</td>
<td>
<em>(Optional)</em>
<p>Size is the number of bytes in the file.</p>
</td>
</tr>
</tbody>
</table>
</div>
@ -1568,8 +1580,8 @@ Artifact
<td>
<code>includedArtifacts</code><br>
<em>
<a href="#source.toolkit.fluxcd.io/v1beta2.*github.com/fluxcd/source-controller/api/v1beta2.Artifact">
[]*github.com/fluxcd/source-controller/api/v1beta2.Artifact
<a href="#source.toolkit.fluxcd.io/v1beta2.*./api/v1beta2.Artifact">
[]*./api/v1beta2.Artifact
</a>
</em>
</td>