storage: calculate `Digest` for `Artifact`

Signed-off-by: Hidde Beydals <hello@hidde.co>
This commit is contained in:
Hidde Beydals 2022-11-07 12:54:28 +00:00
parent 964b2d3f00
commit 6e0a6f11d4
7 changed files with 349 additions and 15 deletions

View File

@ -33,15 +33,17 @@ import (
"time"
securejoin "github.com/cyphar/filepath-securejoin"
"github.com/fluxcd/go-git/v5/plumbing/format/gitignore"
"github.com/fluxcd/pkg/lockedfile"
"github.com/fluxcd/pkg/untar"
digestlib "github.com/opencontainers/go-digest"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
kerrors "k8s.io/apimachinery/pkg/util/errors"
"github.com/fluxcd/pkg/lockedfile"
"github.com/fluxcd/pkg/sourceignore"
"github.com/fluxcd/pkg/untar"
sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
"github.com/fluxcd/source-controller/internal/digest"
sourcefs "github.com/fluxcd/source-controller/internal/fs"
)
@ -358,9 +360,12 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
}
}()
h := newHash()
md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256)
if err != nil {
return fmt.Errorf("failed to create digester: %w", err)
}
sz := &writeCounter{}
mw := io.MultiWriter(h, tf, sz)
mw := io.MultiWriter(md, tf, sz)
gw := gzip.NewWriter(mw)
tw := tar.NewWriter(gw)
@ -450,7 +455,8 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
return err
}
artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
artifact.Digest = md.Digest(digest.Canonical).String()
artifact.Checksum = md.Digest(digestlib.SHA256).Encoded()
artifact.LastUpdateTime = metav1.Now()
artifact.Size = &sz.written
@ -472,9 +478,12 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
}
}()
h := newHash()
md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256)
if err != nil {
return fmt.Errorf("failed to create digester: %w", err)
}
sz := &writeCounter{}
mw := io.MultiWriter(h, tf, sz)
mw := io.MultiWriter(md, tf, sz)
if _, err := io.Copy(mw, reader); err != nil {
tf.Close()
@ -492,7 +501,8 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
return err
}
artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
artifact.Digest = md.Digest(digest.Canonical).String()
artifact.Checksum = md.Digest(digestlib.SHA256).Encoded()
artifact.LastUpdateTime = metav1.Now()
artifact.Size = &sz.written
@ -514,9 +524,12 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error
}
}()
h := newHash()
md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256)
if err != nil {
return fmt.Errorf("failed to create digester: %w", err)
}
sz := &writeCounter{}
mw := io.MultiWriter(h, tf, sz)
mw := io.MultiWriter(md, tf, sz)
if _, err := io.Copy(mw, reader); err != nil {
tf.Close()
@ -530,7 +543,8 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error
return err
}
artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
artifact.Digest = md.Digest(digest.Canonical).String()
artifact.Checksum = md.Digest(digestlib.SHA256).Encoded()
artifact.LastUpdateTime = metav1.Now()
artifact.Size = &sz.written

8
go.mod
View File

@ -10,6 +10,10 @@ replace github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.16.0
// The util.Walk func was never release as a tag.
replace github.com/go-git/go-billy/v5 => github.com/go-git/go-billy/v5 v5.0.0-20210804024030-7ab80d7c013d
// Replace digest lib to master to gather access to BLAKE3.
// xref: https://github.com/opencontainers/go-digest/pull/66
replace github.com/opencontainers/go-digest => github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be
require (
cloud.google.com/go/storage v1.29.0
github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1
@ -45,6 +49,8 @@ require (
github.com/google/uuid v1.3.0
github.com/minio/minio-go/v7 v7.0.47
github.com/onsi/gomega v1.26.0
github.com/opencontainers/go-digest v1.0.0
github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be
github.com/ory/dockertest/v3 v3.9.1
github.com/otiai10/copy v1.9.0
github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
@ -277,7 +283,6 @@ require (
github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
github.com/oklog/ulid v1.3.1 // indirect
github.com/olekukonko/tablewriter v0.0.5 // indirect
github.com/opencontainers/go-digest v1.0.0 // indirect
github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
github.com/opencontainers/runc v1.1.2 // indirect
github.com/opentracing/opentracing-go v1.2.0 // indirect
@ -334,6 +339,7 @@ require (
github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect
github.com/yvasiyarov/gorelic v0.0.7 // indirect
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect
github.com/zeebo/blake3 v0.1.1 // indirect
github.com/zeebo/errs v1.2.2 // indirect
go.etcd.io/bbolt v1.3.6 // indirect
go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 // indirect

13
go.sum
View File

@ -1253,8 +1253,10 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9
github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q=
github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be h1:f2PlhC9pm5sqpBZFvnAoKj+KzXRzbjFMA+TqXfJdgho=
github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be h1:yJISmqboKE7zWqC2Nlg3pBkelqCblzZBoMHv2nbrUjQ=
github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be/go.mod h1:amaK2C3q0MwQTE9OgeDacYr8Qac7uKwICGry1fn3UrI=
github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
github.com/opencontainers/runc v1.1.2 h1:2VSZwLx5k/BfsBxMMipG/LYUnmqOD/BPkIVgQUcTlLw=
@ -1600,8 +1602,14 @@ github.com/yvasiyarov/gorelic v0.0.7/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96Tg
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 h1:AsFN8kXcCVkUFHyuzp1FtYbzp1nCO/H6+1uPSGEyPzM=
github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
github.com/zalando/go-keyring v0.1.0/go.mod h1:RaxNwUITJaHVdQ0VC7pELPZ3tOWn13nr0gZMZEhpVU0=
github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
github.com/zeebo/blake3 v0.1.1 h1:Nbsts7DdKThRHHd+YNlqiGlRqGEF2bE2eXN+xQ1hsEs=
github.com/zeebo/blake3 v0.1.1/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4=
github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g=
github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
github.com/zeebo/pcg v1.0.0 h1:dt+dx+HvX8g7Un32rY9XWoYnd0NmKmrIzpHF7qiTDj0=
github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@ -1994,6 +2002,7 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

39
internal/digest/digest.go Normal file
View File

@ -0,0 +1,39 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package digest
import (
_ "crypto/sha256"
_ "crypto/sha512"
"fmt"
"github.com/opencontainers/go-digest"
_ "github.com/opencontainers/go-digest/blake3"
)
// Canonical is the primary digest algorithm used to calculate checksums.
const Canonical = digest.SHA256
// AlgorithmForName returns the digest algorithm for the given name, or an
// error of type digest.ErrDigestUnsupported if the algorithm is unavailable.
func AlgorithmForName(name string) (digest.Algorithm, error) {
a := digest.Algorithm(name)
if !a.Available() {
return "", fmt.Errorf("%w: %s", digest.ErrDigestUnsupported, name)
}
return a, nil
}

View File

@ -0,0 +1,71 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package digest
import (
"errors"
"testing"
. "github.com/onsi/gomega"
"github.com/opencontainers/go-digest"
)
func TestAlgorithmForName(t *testing.T) {
tests := []struct {
name string
want digest.Algorithm
wantErr error
}{
{
name: "sha256",
want: digest.SHA256,
},
{
name: "sha384",
want: digest.SHA384,
},
{
name: "sha512",
want: digest.SHA512,
},
{
name: "blake3",
want: digest.BLAKE3,
},
{
name: "sha1",
want: SHA1,
},
{
name: "not-available",
wantErr: digest.ErrDigestUnsupported,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
g := NewWithT(t)
got, err := AlgorithmForName(tt.name)
if tt.wantErr != nil {
g.Expect(err).To(HaveOccurred())
g.Expect(errors.Is(err, tt.wantErr)).To(BeTrue())
return
}
g.Expect(err).ToNot(HaveOccurred())
g.Expect(got).To(Equal(tt.want))
})
}
}

71
internal/digest/writer.go Normal file
View File

@ -0,0 +1,71 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package digest
import (
"fmt"
"io"
"github.com/opencontainers/go-digest"
)
// MultiDigester is a digester that writes to multiple digesters to calculate
// the checksum of different algorithms.
type MultiDigester struct {
d map[digest.Algorithm]digest.Digester
}
// NewMultiDigester returns a new MultiDigester that writes to newly
// initialized digesters for the given algorithms. If a provided algorithm is
// not available, it returns a digest.ErrDigestUnsupported error.
func NewMultiDigester(algos ...digest.Algorithm) (*MultiDigester, error) {
d := make(map[digest.Algorithm]digest.Digester, len(algos))
for _, a := range algos {
if _, ok := d[a]; ok {
continue
}
if !a.Available() {
return nil, fmt.Errorf("%w: %s", digest.ErrDigestUnsupported, a)
}
d[a] = a.Digester()
}
return &MultiDigester{d: d}, nil
}
// Write writes p to all underlying digesters.
func (w *MultiDigester) Write(p []byte) (n int, err error) {
for _, d := range w.d {
n, err = d.Hash().Write(p)
if err != nil {
return
}
if n != len(p) {
err = io.ErrShortWrite
return
}
}
return len(p), nil
}
// Digest returns the digest of the data written to the digester of the given
// algorithm, or an empty digest if the algorithm is not available.
func (w *MultiDigester) Digest(algo digest.Algorithm) digest.Digest {
if d, ok := w.d[algo]; ok {
return d.Digest()
}
return ""
}

View File

@ -0,0 +1,124 @@
/*
Copyright 2022 The Flux authors
Licensed under the Apache License, Version 2.0 (the "License");
you may not use this file except in compliance with the License.
You may obtain a copy of the License at
http://www.apache.org/licenses/LICENSE-2.0
Unless required by applicable law or agreed to in writing, software
distributed under the License is distributed on an "AS IS" BASIS,
WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
See the License for the specific language governing permissions and
limitations under the License.
*/
package digest
import (
"crypto/rand"
"testing"
. "github.com/onsi/gomega"
"github.com/opencontainers/go-digest"
)
func TestNewMultiDigester(t *testing.T) {
t.Run("constructs a MultiDigester", func(t *testing.T) {
g := NewWithT(t)
d, err := NewMultiDigester(Canonical, digest.SHA512)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(d.d).To(HaveLen(2))
})
t.Run("returns an error if an algorithm is not available", func(t *testing.T) {
g := NewWithT(t)
_, err := NewMultiDigester(digest.Algorithm("not-available"))
g.Expect(err).To(HaveOccurred())
})
}
func TestMultiDigester_Write(t *testing.T) {
t.Run("writes to all digesters", func(t *testing.T) {
g := NewWithT(t)
d, err := NewMultiDigester(Canonical, digest.SHA512)
g.Expect(err).ToNot(HaveOccurred())
n, err := d.Write([]byte("hello"))
g.Expect(err).ToNot(HaveOccurred())
g.Expect(n).To(Equal(5))
n, err = d.Write([]byte(" world"))
g.Expect(err).ToNot(HaveOccurred())
g.Expect(n).To(Equal(6))
g.Expect(d.Digest(Canonical)).To(BeEquivalentTo("sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"))
g.Expect(d.Digest(digest.SHA512)).To(BeEquivalentTo("sha512:309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f"))
})
}
func TestMultiDigester_Digest(t *testing.T) {
t.Run("returns the digest for the given algorithm", func(t *testing.T) {
g := NewWithT(t)
d, err := NewMultiDigester(Canonical, digest.SHA512)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(d.Digest(Canonical)).To(BeEquivalentTo("sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"))
g.Expect(d.Digest(digest.SHA512)).To(BeEquivalentTo("sha512:cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"))
})
t.Run("returns an empty digest if the algorithm is not supported", func(t *testing.T) {
g := NewWithT(t)
d, err := NewMultiDigester(Canonical, digest.SHA512)
g.Expect(err).ToNot(HaveOccurred())
g.Expect(d.Digest(digest.Algorithm("not-available"))).To(BeEmpty())
})
}
func benchmarkMultiDigesterWrite(b *testing.B, algos []digest.Algorithm, pSize int64) {
md, err := NewMultiDigester(algos...)
if err != nil {
b.Fatal(err)
}
p := make([]byte, pSize)
if _, err = rand.Read(p); err != nil {
b.Fatal(err)
}
b.ResetTimer()
for i := 0; i < b.N; i++ {
md.Write(p)
}
}
func BenchmarkMultiDigester_Write(b *testing.B) {
const pSize = 1024 * 2
b.Run("sha256", func(b *testing.B) {
benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256}, pSize)
})
b.Run("blake3", func(b *testing.B) {
benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.BLAKE3}, pSize)
})
b.Run("sha256+sha384", func(b *testing.B) {
benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256, digest.SHA384}, pSize)
})
b.Run("sha256+sha512", func(b *testing.B) {
benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256, digest.SHA512}, pSize)
})
b.Run("sha256+blake3", func(b *testing.B) {
benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256, digest.BLAKE3}, pSize)
})
}