storage: calculate `Digest` for `Artifact`

Signed-off-by: Hidde Beydals <hello@hidde.co>

controllers/storage.go

@@ -33,15 +33,17 @@ import (
 	"time"
 
 	securejoin "github.com/cyphar/filepath-securejoin"
-
 	"github.com/fluxcd/go-git/v5/plumbing/format/gitignore"
-	"github.com/fluxcd/pkg/lockedfile"
+	digestlib "github.com/opencontainers/go-digest"
-	"github.com/fluxcd/pkg/untar"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	kerrors "k8s.io/apimachinery/pkg/util/errors"
 
+	"github.com/fluxcd/pkg/lockedfile"
 	"github.com/fluxcd/pkg/sourceignore"
+	"github.com/fluxcd/pkg/untar"
+
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
+	"github.com/fluxcd/source-controller/internal/digest"
 	sourcefs "github.com/fluxcd/source-controller/internal/fs"
 )
 
@@ -358,9 +360,12 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
 		}
 	}()
 
-	h := newHash()
+	md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256)
+	if err != nil {
+		return fmt.Errorf("failed to create digester: %w", err)
+	}
 	sz := &writeCounter{}
-	mw := io.MultiWriter(h, tf, sz)
+	mw := io.MultiWriter(md, tf, sz)
 
 	gw := gzip.NewWriter(mw)
 	tw := tar.NewWriter(gw)
@@ -450,7 +455,8 @@ func (s *Storage) Archive(artifact *sourcev1.Artifact, dir string, filter Archiv
 		return err
 	}
 
-	artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
+	artifact.Digest = md.Digest(digest.Canonical).String()
+	artifact.Checksum = md.Digest(digestlib.SHA256).Encoded()
 	artifact.LastUpdateTime = metav1.Now()
 	artifact.Size = &sz.written
 
@@ -472,9 +478,12 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
 		}
 	}()
 
-	h := newHash()
+	md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256)
+	if err != nil {
+		return fmt.Errorf("failed to create digester: %w", err)
+	}
 	sz := &writeCounter{}
-	mw := io.MultiWriter(h, tf, sz)
+	mw := io.MultiWriter(md, tf, sz)
 
 	if _, err := io.Copy(mw, reader); err != nil {
 		tf.Close()
@@ -492,7 +501,8 @@ func (s *Storage) AtomicWriteFile(artifact *sourcev1.Artifact, reader io.Reader,
 		return err
 	}
 
-	artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
+	artifact.Digest = md.Digest(digest.Canonical).String()
+	artifact.Checksum = md.Digest(digestlib.SHA256).Encoded()
 	artifact.LastUpdateTime = metav1.Now()
 	artifact.Size = &sz.written
 
@@ -514,9 +524,12 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error
 		}
 	}()
 
-	h := newHash()
+	md, err := digest.NewMultiDigester(digest.Canonical, digestlib.SHA256)
+	if err != nil {
+		return fmt.Errorf("failed to create digester: %w", err)
+	}
 	sz := &writeCounter{}
-	mw := io.MultiWriter(h, tf, sz)
+	mw := io.MultiWriter(md, tf, sz)
 
 	if _, err := io.Copy(mw, reader); err != nil {
 		tf.Close()
@@ -530,7 +543,8 @@ func (s *Storage) Copy(artifact *sourcev1.Artifact, reader io.Reader) (err error
 		return err
 	}
 
-	artifact.Checksum = fmt.Sprintf("%x", h.Sum(nil))
+	artifact.Digest = md.Digest(digest.Canonical).String()
+	artifact.Checksum = md.Digest(digestlib.SHA256).Encoded()
 	artifact.LastUpdateTime = metav1.Now()
 	artifact.Size = &sz.written
 

go.mod

@@ -10,6 +10,10 @@ replace github.com/emicklei/go-restful => github.com/emicklei/go-restful v2.16.0
 // The util.Walk func was never release as a tag.
 replace github.com/go-git/go-billy/v5 => github.com/go-git/go-billy/v5 v5.0.0-20210804024030-7ab80d7c013d
 
+// Replace digest lib to master to gather access to BLAKE3.
+// xref: https://github.com/opencontainers/go-digest/pull/66
+replace github.com/opencontainers/go-digest => github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be
+
 require (
 	cloud.google.com/go/storage v1.29.0
 	github.com/AdaLogics/go-fuzz-headers v0.0.0-20230106234847-43070de90fa1
@@ -45,6 +49,8 @@ require (
 	github.com/google/uuid v1.3.0
 	github.com/minio/minio-go/v7 v7.0.47
 	github.com/onsi/gomega v1.26.0
+	github.com/opencontainers/go-digest v1.0.0
+	github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be
 	github.com/ory/dockertest/v3 v3.9.1
 	github.com/otiai10/copy v1.9.0
 	github.com/phayes/freeport v0.0.0-20220201140144-74d24b5ae9f5
@@ -277,7 +283,6 @@ require (
 	github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/olekukonko/tablewriter v0.0.5 // indirect
-	github.com/opencontainers/go-digest v1.0.0 // indirect
 	github.com/opencontainers/image-spec v1.1.0-rc2 // indirect
 	github.com/opencontainers/runc v1.1.2 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
@@ -334,6 +339,7 @@ require (
 	github.com/yvasiyarov/go-metrics v0.0.0-20150112132944-c25f46c4b940 // indirect
 	github.com/yvasiyarov/gorelic v0.0.7 // indirect
 	github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 // indirect
+	github.com/zeebo/blake3 v0.1.1 // indirect
 	github.com/zeebo/errs v1.2.2 // indirect
 	go.etcd.io/bbolt v1.3.6 // indirect
 	go.etcd.io/etcd/api/v3 v3.6.0-alpha.0 // indirect

go.sum

@@ -1253,8 +1253,10 @@ github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9
 github.com/onsi/gomega v1.26.0 h1:03cDLK28U6hWvCAns6NeydX3zIm4SF3ci69ulidS32Q=
 github.com/onsi/gomega v1.26.0/go.mod h1:r+zV744Re+DiYCIPRlYOTxn0YkOLcAnW8k1xXdMPGhM=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
-github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
+github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be h1:f2PlhC9pm5sqpBZFvnAoKj+KzXRzbjFMA+TqXfJdgho=
-github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/go-digest v1.0.1-0.20220411205349-bde1400a84be/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
+github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be h1:yJISmqboKE7zWqC2Nlg3pBkelqCblzZBoMHv2nbrUjQ=
+github.com/opencontainers/go-digest/blake3 v0.0.0-20220411205349-bde1400a84be/go.mod h1:amaK2C3q0MwQTE9OgeDacYr8Qac7uKwICGry1fn3UrI=
 github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034=
 github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ=
 github.com/opencontainers/runc v1.1.2 h1:2VSZwLx5k/BfsBxMMipG/LYUnmqOD/BPkIVgQUcTlLw=
@@ -1600,8 +1602,14 @@ github.com/yvasiyarov/gorelic v0.0.7/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96Tg
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9 h1:AsFN8kXcCVkUFHyuzp1FtYbzp1nCO/H6+1uPSGEyPzM=
 github.com/yvasiyarov/newrelic_platform_go v0.0.0-20160601141957-9c099fbc30e9/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg=
 github.com/zalando/go-keyring v0.1.0/go.mod h1:RaxNwUITJaHVdQ0VC7pELPZ3tOWn13nr0gZMZEhpVU0=
+github.com/zeebo/assert v1.1.0 h1:hU1L1vLTHsnO8x8c9KAR5GmM5QscxHg5RNU5z5qbUWY=
+github.com/zeebo/assert v1.1.0/go.mod h1:Pq9JiuJQpG8JLJdtkwrJESF0Foym2/D9XMU5ciN/wJ0=
+github.com/zeebo/blake3 v0.1.1 h1:Nbsts7DdKThRHHd+YNlqiGlRqGEF2bE2eXN+xQ1hsEs=
+github.com/zeebo/blake3 v0.1.1/go.mod h1:G9pM4qQwjRzF1/v7+vabMj/c5mWpGZ2Wzo3Eb4z0pb4=
 github.com/zeebo/errs v1.2.2 h1:5NFypMTuSdoySVTqlNs1dEoU21QVamMQJxW/Fii5O7g=
 github.com/zeebo/errs v1.2.2/go.mod h1:sgbWHsvVuTPHcqJJGQ1WhI5KbWlHYz+2+2C/LSEtCw4=
+github.com/zeebo/pcg v1.0.0 h1:dt+dx+HvX8g7Un32rY9XWoYnd0NmKmrIzpHF7qiTDj0=
+github.com/zeebo/pcg v1.0.0/go.mod h1:09F0S9iiKrwn9rlI5yjLkmrug154/YRW6KnnXVDM/l4=
 github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0=
 go.etcd.io/bbolt v1.3.2/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
 go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU=
@@ -1994,6 +2002,7 @@ golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201009025420-dfb3f7c4e634/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20201014080544-cc95f250f6bc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210104204734-6f8348627aad/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=

internal/digest/digest.go

@@ -0,0 +1,39 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package digest
+
+import (
+	_ "crypto/sha256"
+	_ "crypto/sha512"
+	"fmt"
+
+	"github.com/opencontainers/go-digest"
+	_ "github.com/opencontainers/go-digest/blake3"
+)
+
+// Canonical is the primary digest algorithm used to calculate checksums.
+const Canonical = digest.SHA256
+
+// AlgorithmForName returns the digest algorithm for the given name, or an
+// error of type digest.ErrDigestUnsupported if the algorithm is unavailable.
+func AlgorithmForName(name string) (digest.Algorithm, error) {
+	a := digest.Algorithm(name)
+	if !a.Available() {
+		return "", fmt.Errorf("%w: %s", digest.ErrDigestUnsupported, name)
+	}
+	return a, nil
+}

internal/digest/digest_test.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package digest
+
+import (
+	"errors"
+	"testing"
+
+	. "github.com/onsi/gomega"
+	"github.com/opencontainers/go-digest"
+)
+
+func TestAlgorithmForName(t *testing.T) {
+	tests := []struct {
+		name    string
+		want    digest.Algorithm
+		wantErr error
+	}{
+		{
+			name: "sha256",
+			want: digest.SHA256,
+		},
+		{
+			name: "sha384",
+			want: digest.SHA384,
+		},
+		{
+			name: "sha512",
+			want: digest.SHA512,
+		},
+		{
+			name: "blake3",
+			want: digest.BLAKE3,
+		},
+		{
+			name: "sha1",
+			want: SHA1,
+		},
+		{
+			name:    "not-available",
+			wantErr: digest.ErrDigestUnsupported,
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			g := NewWithT(t)
+			got, err := AlgorithmForName(tt.name)
+			if tt.wantErr != nil {
+				g.Expect(err).To(HaveOccurred())
+				g.Expect(errors.Is(err, tt.wantErr)).To(BeTrue())
+				return
+			}
+			g.Expect(err).ToNot(HaveOccurred())
+			g.Expect(got).To(Equal(tt.want))
+		})
+	}
+}

internal/digest/writer.go

@@ -0,0 +1,71 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package digest
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/opencontainers/go-digest"
+)
+
+// MultiDigester is a digester that writes to multiple digesters to calculate
+// the checksum of different algorithms.
+type MultiDigester struct {
+	d map[digest.Algorithm]digest.Digester
+}
+
+// NewMultiDigester returns a new MultiDigester that writes to newly
+// initialized digesters for the given algorithms. If a provided algorithm is
+// not available, it returns a digest.ErrDigestUnsupported error.
+func NewMultiDigester(algos ...digest.Algorithm) (*MultiDigester, error) {
+	d := make(map[digest.Algorithm]digest.Digester, len(algos))
+	for _, a := range algos {
+		if _, ok := d[a]; ok {
+			continue
+		}
+		if !a.Available() {
+			return nil, fmt.Errorf("%w: %s", digest.ErrDigestUnsupported, a)
+		}
+		d[a] = a.Digester()
+	}
+	return &MultiDigester{d: d}, nil
+}
+
+// Write writes p to all underlying digesters.
+func (w *MultiDigester) Write(p []byte) (n int, err error) {
+	for _, d := range w.d {
+		n, err = d.Hash().Write(p)
+		if err != nil {
+			return
+		}
+		if n != len(p) {
+			err = io.ErrShortWrite
+			return
+		}
+	}
+	return len(p), nil
+}
+
+// Digest returns the digest of the data written to the digester of the given
+// algorithm, or an empty digest if the algorithm is not available.
+func (w *MultiDigester) Digest(algo digest.Algorithm) digest.Digest {
+	if d, ok := w.d[algo]; ok {
+		return d.Digest()
+	}
+	return ""
+}

internal/digest/writer_test.go

@@ -0,0 +1,124 @@
+/*
+Copyright 2022 The Flux authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package digest
+
+import (
+	"crypto/rand"
+	"testing"
+
+	. "github.com/onsi/gomega"
+	"github.com/opencontainers/go-digest"
+)
+
+func TestNewMultiDigester(t *testing.T) {
+	t.Run("constructs a MultiDigester", func(t *testing.T) {
+		g := NewWithT(t)
+
+		d, err := NewMultiDigester(Canonical, digest.SHA512)
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(d.d).To(HaveLen(2))
+	})
+
+	t.Run("returns an error if an algorithm is not available", func(t *testing.T) {
+		g := NewWithT(t)
+
+		_, err := NewMultiDigester(digest.Algorithm("not-available"))
+		g.Expect(err).To(HaveOccurred())
+	})
+}
+
+func TestMultiDigester_Write(t *testing.T) {
+	t.Run("writes to all digesters", func(t *testing.T) {
+		g := NewWithT(t)
+
+		d, err := NewMultiDigester(Canonical, digest.SHA512)
+		g.Expect(err).ToNot(HaveOccurred())
+
+		n, err := d.Write([]byte("hello"))
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(n).To(Equal(5))
+
+		n, err = d.Write([]byte(" world"))
+		g.Expect(err).ToNot(HaveOccurred())
+		g.Expect(n).To(Equal(6))
+
+		g.Expect(d.Digest(Canonical)).To(BeEquivalentTo("sha256:b94d27b9934d3e08a52e52d7da7dabfac484efe37a5380ee9088f7ace2efcde9"))
+		g.Expect(d.Digest(digest.SHA512)).To(BeEquivalentTo("sha512:309ecc489c12d6eb4cc40f50c902f2b4d0ed77ee511a7c7a9bcd3ca86d4cd86f989dd35bc5ff499670da34255b45b0cfd830e81f605dcf7dc5542e93ae9cd76f"))
+	})
+}
+
+func TestMultiDigester_Digest(t *testing.T) {
+	t.Run("returns the digest for the given algorithm", func(t *testing.T) {
+		g := NewWithT(t)
+
+		d, err := NewMultiDigester(Canonical, digest.SHA512)
+		g.Expect(err).ToNot(HaveOccurred())
+
+		g.Expect(d.Digest(Canonical)).To(BeEquivalentTo("sha256:e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855"))
+		g.Expect(d.Digest(digest.SHA512)).To(BeEquivalentTo("sha512:cf83e1357eefb8bdf1542850d66d8007d620e4050b5715dc83f4a921d36ce9ce47d0d13c5d85f2b0ff8318d2877eec2f63b931bd47417a81a538327af927da3e"))
+	})
+
+	t.Run("returns an empty digest if the algorithm is not supported", func(t *testing.T) {
+		g := NewWithT(t)
+
+		d, err := NewMultiDigester(Canonical, digest.SHA512)
+		g.Expect(err).ToNot(HaveOccurred())
+
+		g.Expect(d.Digest(digest.Algorithm("not-available"))).To(BeEmpty())
+	})
+}
+
+func benchmarkMultiDigesterWrite(b *testing.B, algos []digest.Algorithm, pSize int64) {
+	md, err := NewMultiDigester(algos...)
+	if err != nil {
+		b.Fatal(err)
+	}
+
+	p := make([]byte, pSize)
+	if _, err = rand.Read(p); err != nil {
+		b.Fatal(err)
+	}
+
+	b.ResetTimer()
+	for i := 0; i < b.N; i++ {
+		md.Write(p)
+	}
+}
+
+func BenchmarkMultiDigester_Write(b *testing.B) {
+	const pSize = 1024 * 2
+
+	b.Run("sha256", func(b *testing.B) {
+		benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256}, pSize)
+	})
+
+	b.Run("blake3", func(b *testing.B) {
+		benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.BLAKE3}, pSize)
+	})
+
+	b.Run("sha256+sha384", func(b *testing.B) {
+		benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256, digest.SHA384}, pSize)
+	})
+
+	b.Run("sha256+sha512", func(b *testing.B) {
+		benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256, digest.SHA512}, pSize)
+	})
+
+	b.Run("sha256+blake3", func(b *testing.B) {
+		benchmarkMultiDigesterWrite(b, []digest.Algorithm{digest.SHA256, digest.BLAKE3}, pSize)
+	})
+}