From 6e0a6f11d44d544a351bb12c4bd5eb031dbccd04 Mon Sep 17 00:00:00 2001 From: Hidde Beydals Date: Mon, 7 Nov 2022 12:54:28 +0000 Subject: [PATCH] storage: calculate `Digest` for `Artifact` Signed-off-by: Hidde Beydals --- controllers/storage.go | 38 ++++++---- go.mod | 8 ++- go.sum | 13 +++- internal/digest/digest.go | 39 +++++++++++ internal/digest/digest_test.go | 71 +++++++++++++++++++ internal/digest/writer.go | 71 +++++++++++++++++++ internal/digest/writer_test.go | 124 +++++++++++++++++++++++++++++++++ 7 files changed, 349 insertions(+), 15 deletions(-) create mode 100644 internal/digest/digest.go create mode 100644 internal/digest/digest_test.go create mode 100644 internal/digest/writer.go create mode 100644 internal/digest/writer_test.go diff --git a/controllers/storage.go b/controllers/storage.go index 57993a0a..52c51134 100644 --- a/controllers/storage.go +++ b/controllers/storage.go @@ -33,15 +33,17 @@ import ( "time" securejoin "github.com/cyphar/filepath-securejoin" - "github.com/fluxcd/go-git/v5/plumbing/format/gitignore" - "github.com/fluxcd/pkg/lockedfile" - "github.com/fluxcd/pkg/untar" + digestlib "github.com/opencontainers/go-digest" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" kerrors "k8s.io/apimachinery/pkg/util/errors" + "github.com/fluxcd/pkg/lockedfile" "github.com/fluxcd/pkg/sourceignore" + "github.com/fluxcd/pkg/untar" + sourcev1 "github.com/fluxcd/source-controller/api/v1beta2" + "github.com/fluxcd/source-controller/internal/digest" sourcefs "github.com/fluxcd/source-controller/internal/fs" ) @@ -358,9 +360,12 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv } }() - h := newHash() + md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256) + if err != nil { + return fmt.Errorf("failed to create digester: %w", err) + } sz := &writeCounter{} - mw := io.MultiWriter(h, tf, sz) + mw := io.MultiWriter(md, tf, sz) gw := gzip.NewWriter(mw) tw := tar.NewWriter(gw) @@ -450,7 +455,8 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv return err } - artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil)) + artifact.Digest = md.Digest(digest.Canonical).String() + artifact.Checksum = md.Digest(digestlib.SHA256).Encoded() artifact.LastUpdateTime = metav1.Now() artifact.Size = &sz.written @@ -472,9 +478,12 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader, } }() - h := newHash() + md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256) + if err != nil { + return fmt.Errorf("failed to create digester: %w", err) + } sz := &writeCounter{} - mw := io.MultiWriter(h, tf, sz) + mw := io.MultiWriter(md, tf, sz) if _, err := io.Copy(mw, reader); err != nil { tf.Close() @@ -492,7 +501,8 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader, return err } - artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil)) + artifact.Digest = md.Digest(digest.Canonical).String() + artifact.Checksum = md.Digest(digestlib.SHA256).Encoded() artifact.LastUpdateTime = metav1.Now() artifact.Size = &sz.written @@ -514,9 +524,12 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error } }() - h := newHash() + md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256) + if err != nil { + return fmt.Errorf("failed to create digester: %w", err) + } sz := &writeCounter{} - mw := io.MultiWriter(h, tf, sz) + mw := io.MultiWriter(md, tf, sz) if _, err := io.Copy(mw, reader); err != nil { tf.Close() @@ -530,7 +543,8 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error return err } - artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil)) + artifact.Digest = md.Digest(digest.Canonical).String() + artifact.Checksum = md.Digest(digestlib.SHA256).Encoded() artifact.LastUpdateTime = metav1.Now() artifact.Size = &sz.written diff --git a/go.mod b/go.mod index 107d8140..102925a8 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,10 @@ replace github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.16.0 // The util.Walk func was never release as a tag. replace github.com/go-git/go-billy/v5 => github.com/go-git/go-billy/v5 v5.0.0-20210804024030-7ab80d7c013d +// Replace digest lib to master to gather access to BLAKE3. +// xref: https://github.com/opencontainers/go-digest/pull/66 +replace github.com/opencontainers/go-digest => github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be + require ( cloud.google.com/go/storage v1.29.0 github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1 @@ -45,6 +49,8 @@ require ( github.com/google/uuid v1.3.0 github.com/minio/minio-go/v7 v7.0.47 github.com/onsi/gomega v1.26.0 + github.com/opencontainers/go-digest v1.0.0 + github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be github.com/ory/dockertest/v3 v3.9.1 github.com/otiai10/copy v1.9.0 github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5 @@ -277,7 +283,6 @@ require ( github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect github.com/oklog/ulid v1.3.1 // indirect github.com/olekukonko/tablewriter v0.0.5 // indirect - github.com/opencontainers/go-digest v1.0.0 // indirect github.com/opencontainers/image-spec v1.1.0-rc2 // indirect github.com/opencontainers/runc v1.1.2 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect @@ -334,6 +339,7 @@ require ( github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect github.com/yvasiyarov/gorelic v0.0.7 // indirect github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect + github.com/zeebo/blake3 v0.1.1 // indirect github.com/zeebo/errs v1.2.2 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 // indirect diff --git a/go.sum b/go.sum index aabb676f..a3af4555 100644 --- a/go.sum +++ b/go.sum @@ -1253,8 +1253,10 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9 github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q= github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= -github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= -github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be h1:f2PlhC9pm5sqpBZFvnAoKj+KzXRzbjFMA+TqXfJdgho= +github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= +github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be h1:yJISmqboKE7zWqC2Nlg3pBkelqCblzZBoMHv2nbrUjQ= +github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be/go.mod h1:amaK2C3q0MwQTE9OgeDacYr8Qac7uKwICGry1fn3UrI= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opencontainers/runc v1.1.2 h1:2VSZwLx5k/BfsBxMMipG/LYUnmqOD/BPkIVgQUcTlLw= @@ -1600,8 +1602,14 @@ github.com/yvasiyarov/gorelic v0.0.7/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96Tg github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 h1:AsFN8kXcCVkUFHyuzp1FtYbzp1nCO/H6+1uPSGEyPzM= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= github.com/zalando/go-keyring v0.1.0/go.mod h1:RaxNwUITJaHVdQ0VC7pELPZ3tOWn13nr0gZMZEhpVU0= +github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY= +github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0= +github.com/zeebo/blake3 v0.1.1 h1:Nbsts7DdKThRHHd+YNlqiGlRqGEF2bE2eXN+xQ1hsEs= +github.com/zeebo/blake3 v0.1.1/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4= github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g= github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4= +github.com/zeebo/pcg v1.0.0 h1:dt+dx+HvX8g7Un32rY9XWoYnd0NmKmrIzpHF7qiTDj0= +github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -1994,6 +2002,7 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= diff --git a/internal/digest/digest.go b/internal/digest/digest.go new file mode 100644 index 00000000..9fcca642 --- /dev/null +++ b/internal/digest/digest.go @@ -0,0 +1,39 @@ +/* +Copyright 2022 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package digest + +import ( + _ "crypto/sha256" + _ "crypto/sha512" + "fmt" + + "github.com/opencontainers/go-digest" + _ "github.com/opencontainers/go-digest/blake3" +) + +// Canonical is the primary digest algorithm used to calculate checksums. +const Canonical = digest.SHA256 + +// AlgorithmForName returns the digest algorithm for the given name, or an +// error of type digest.ErrDigestUnsupported if the algorithm is unavailable. +func AlgorithmForName(name string) (digest.Algorithm, error) { + a := digest.Algorithm(name) + if !a.Available() { + return "", fmt.Errorf("%w: %s", digest.ErrDigestUnsupported, name) + } + return a, nil +} diff --git a/internal/digest/digest_test.go b/internal/digest/digest_test.go new file mode 100644 index 00000000..3030c2d1 --- /dev/null +++ b/internal/digest/digest_test.go @@ -0,0 +1,71 @@ +/* +Copyright 2022 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package digest + +import ( + "errors" + "testing" + + . "github.com/onsi/gomega" + "github.com/opencontainers/go-digest" +) + +func TestAlgorithmForName(t *testing.T) { + tests := []struct { + name string + want digest.Algorithm + wantErr error + }{ + { + name: "sha256", + want: digest.SHA256, + }, + { + name: "sha384", + want: digest.SHA384, + }, + { + name: "sha512", + want: digest.SHA512, + }, + { + name: "blake3", + want: digest.BLAKE3, + }, + { + name: "sha1", + want: SHA1, + }, + { + name: "not-available", + wantErr: digest.ErrDigestUnsupported, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + g := NewWithT(t) + got, err := AlgorithmForName(tt.name) + if tt.wantErr != nil { + g.Expect(err).To(HaveOccurred()) + g.Expect(errors.Is(err, tt.wantErr)).To(BeTrue()) + return + } + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(got).To(Equal(tt.want)) + }) + } +} diff --git a/internal/digest/writer.go b/internal/digest/writer.go new file mode 100644 index 00000000..4783f8b8 --- /dev/null +++ b/internal/digest/writer.go @@ -0,0 +1,71 @@ +/* +Copyright 2022 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package digest + +import ( + "fmt" + "io" + + "github.com/opencontainers/go-digest" +) + +// MultiDigester is a digester that writes to multiple digesters to calculate +// the checksum of different algorithms. +type MultiDigester struct { + d map[digest.Algorithm]digest.Digester +} + +// NewMultiDigester returns a new MultiDigester that writes to newly +// initialized digesters for the given algorithms. If a provided algorithm is +// not available, it returns a digest.ErrDigestUnsupported error. +func NewMultiDigester(algos ...digest.Algorithm) (*MultiDigester, error) { + d := make(map[digest.Algorithm]digest.Digester, len(algos)) + for _, a := range algos { + if _, ok := d[a]; ok { + continue + } + if !a.Available() { + return nil, fmt.Errorf("%w: %s", digest.ErrDigestUnsupported, a) + } + d[a] = a.Digester() + } + return &MultiDigester{d: d}, nil +} + +// Write writes p to all underlying digesters. +func (w *MultiDigester) Write(p []byte) (n int, err error) { + for _, d := range w.d { + n, err = d.Hash().Write(p) + if err != nil { + return + } + if n != len(p) { + err = io.ErrShortWrite + return + } + } + return len(p), nil +} + +// Digest returns the digest of the data written to the digester of the given +// algorithm, or an empty digest if the algorithm is not available. +func (w *MultiDigester) Digest(algo digest.Algorithm) digest.Digest { + if d, ok := w.d[algo]; ok { + return d.Digest() + } + return "" +} diff --git a/internal/digest/writer_test.go b/internal/digest/writer_test.go new file mode 100644 index 00000000..d58518ef --- /dev/null +++ b/internal/digest/writer_test.go @@ -0,0 +1,124 @@ +/* +Copyright 2022 The Flux authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package digest + +import ( + "crypto/rand" + "testing" + + . "github.com/onsi/gomega" + "github.com/opencontainers/go-digest" +) + +func TestNewMultiDigester(t *testing.T) { + t.Run("constructs a MultiDigester", func(t *testing.T) { + g := NewWithT(t) + + d, err := NewMultiDigester(Canonical, digest.SHA512) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(d.d).To(HaveLen(2)) + }) + + t.Run("returns an error if an algorithm is not available", func(t *testing.T) { + g := NewWithT(t) + + _, err := NewMultiDigester(digest.Algorithm("not-available")) + g.Expect(err).To(HaveOccurred()) + }) +} + +func TestMultiDigester_Write(t *testing.T) { + t.Run("writes to all digesters", func(t *testing.T) { + g := NewWithT(t) + + d, err := NewMultiDigester(Canonical, digest.SHA512) + g.Expect(err).ToNot(HaveOccurred()) + + n, err := d.Write([]byte("hello")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(n).To(Equal(5)) + + n, err = d.Write([]byte(" world")) + g.Expect(err).ToNot(HaveOccurred()) + g.Expect(n).To(Equal(6)) + + g.Expect(d.Digest(Canonical)).To(BeEquivalentTo("sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9")) + g.Expect(d.Digest(digest.SHA512)).To(BeEquivalentTo("sha512:309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f")) + }) +} + +func TestMultiDigester_Digest(t *testing.T) { + t.Run("returns the digest for the given algorithm", func(t *testing.T) { + g := NewWithT(t) + + d, err := NewMultiDigester(Canonical, digest.SHA512) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(d.Digest(Canonical)).To(BeEquivalentTo("sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855")) + g.Expect(d.Digest(digest.SHA512)).To(BeEquivalentTo("sha512:cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e")) + }) + + t.Run("returns an empty digest if the algorithm is not supported", func(t *testing.T) { + g := NewWithT(t) + + d, err := NewMultiDigester(Canonical, digest.SHA512) + g.Expect(err).ToNot(HaveOccurred()) + + g.Expect(d.Digest(digest.Algorithm("not-available"))).To(BeEmpty()) + }) +} + +func benchmarkMultiDigesterWrite(b *testing.B, algos []digest.Algorithm, pSize int64) { + md, err := NewMultiDigester(algos...) + if err != nil { + b.Fatal(err) + } + + p := make([]byte, pSize) + if _, err = rand.Read(p); err != nil { + b.Fatal(err) + } + + b.ResetTimer() + for i := 0; i < b.N; i++ { + md.Write(p) + } +} + +func BenchmarkMultiDigester_Write(b *testing.B) { + const pSize = 1024 * 2 + + b.Run("sha256", func(b *testing.B) { + benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256}, pSize) + }) + + b.Run("blake3", func(b *testing.B) { + benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.BLAKE3}, pSize) + }) + + b.Run("sha256+sha384", func(b *testing.B) { + benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256, digest.SHA384}, pSize) + }) + + b.Run("sha256+sha512", func(b *testing.B) { + benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256, digest.SHA512}, pSize) + }) + + b.Run("sha256+blake3", func(b *testing.B) { + benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256, digest.BLAKE3}, pSize) + }) +}