Merge pull request #743 from fluxcd/refactor-artifact-fetcher

Refactor: Acquire artifacts with `fluxcd/pkg/http/fetch`

controllers/kustomization_controller.go

@@ -50,12 +50,14 @@ import (
 
 	apiacl "github.com/fluxcd/pkg/apis/acl"
 	"github.com/fluxcd/pkg/apis/meta"
+	"github.com/fluxcd/pkg/http/fetch"
 	"github.com/fluxcd/pkg/runtime/acl"
 	runtimeClient "github.com/fluxcd/pkg/runtime/client"
 	"github.com/fluxcd/pkg/runtime/events"
 	"github.com/fluxcd/pkg/runtime/metrics"
 	"github.com/fluxcd/pkg/runtime/predicates"
 	"github.com/fluxcd/pkg/ssa"
+	"github.com/fluxcd/pkg/tar"
 	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
 
 	kustomizev1 "github.com/fluxcd/kustomize-controller/api/v1beta2"
@@ -74,7 +76,7 @@ import (
 // KustomizationReconciler reconciles a Kustomization object
 type KustomizationReconciler struct {
 	client.Client
-	artifactFetcher       *ArtifactFetcher
+	artifactFetcher       *fetch.ArchiveFetcher
 	requeueDependency     time.Duration
 	Scheme                *runtime.Scheme
 	EventRecorder         kuberecorder.EventRecorder
@@ -124,7 +126,7 @@ func (r *KustomizationReconciler) SetupWithManager(mgr ctrl.Manager, opts Kustom
 
 	r.requeueDependency = opts.DependencyRequeueInterval
 	r.statusManager = fmt.Sprintf("gotk-%s", r.ControllerName)
-	r.artifactFetcher = NewArtifactFetcher(opts.HTTPRetry)
+	r.artifactFetcher = fetch.NewArchiveFetcher(opts.HTTPRetry, tar.UnlimitedUntarSize, os.Getenv("SOURCE_CONTROLLER_LOCALHOST"))
 
 	return ctrl.NewControllerManagedBy(mgr).
 		For(&kustomizev1.Kustomization{}, builder.WithPredicates(
@@ -270,7 +272,7 @@ func (r *KustomizationReconciler) Reconcile(ctx context.Context, req ctrl.Reques
 	reconciledKustomization, reconcileErr := r.reconcile(ctx, *kustomization.DeepCopy(), source)
 
 	// requeue if the artifact is not found
-	if reconcileErr == ArtifactNotFoundError {
+	if reconcileErr == fetch.FileNotFoundError {
 		msg := fmt.Sprintf("Source is not ready, artifact not found, retrying in %s", r.requeueDependency.String())
 		log.Info(msg)
 		if err := r.patchStatus(ctx, req, kustomizev1.KustomizationProgressing(kustomization, msg).Status); err != nil {
@@ -332,7 +334,7 @@ func (r *KustomizationReconciler) reconcile(
 	defer os.RemoveAll(tmpDir)
 
 	// download artifact and extract files
-	err = r.artifactFetcher.Fetch(source.GetArtifact(), tmpDir)
+	err = r.artifactFetcher.Fetch(source.GetArtifact().URL, source.GetArtifact().Checksum, tmpDir)
 	if err != nil {
 		return kustomizev1.KustomizationNotReady(
 			kustomization,

controllers/kustomization_decryptor_test.go

@@ -66,11 +66,11 @@ func TestKustomizationReconciler_Decryptor(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret")
 
 	artifactName := "sops-" + randStringRunes(5)
-	artifactChecksum, err := createArtifact(testServer, "testdata/sops", artifactName)
+	artifactChecksum, err := testServer.ArtifactFromDir("testdata/sops", artifactName)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	overlayArtifactName := "sops-" + randStringRunes(5)
-	overlayChecksum, err := createArtifact(testServer, "testdata/test-dotenv", overlayArtifactName)
+	overlayChecksum, err := testServer.ArtifactFromDir("testdata/test-dotenv", overlayArtifactName)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	repositoryName := types.NamespacedName{

controllers/kustomization_fetcher.go

@@ -1,127 +0,0 @@
-/*
-Copyright 2022 The Flux authors
-
-Licensed under the Apache License, Version 2.0 (the "License");
-you may not use this file except in compliance with the License.
-You may obtain a copy of the License at
-
-    http://www.apache.org/licenses/LICENSE-2.0
-
-Unless required by applicable law or agreed to in writing, software
-distributed under the License is distributed on an "AS IS" BASIS,
-WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-See the License for the specific language governing permissions and
-limitations under the License.
-*/
-
-package controllers
-
-import (
-	"bytes"
-	"crypto/sha1"
-	"crypto/sha256"
-	"errors"
-	"fmt"
-	"io"
-	"net/http"
-	"net/url"
-	"os"
-	"time"
-
-	"github.com/fluxcd/pkg/untar"
-	sourcev1 "github.com/fluxcd/source-controller/api/v1beta2"
-	"github.com/hashicorp/go-retryablehttp"
-)
-
-// ArtifactFetcher holds the HTTP client that reties with back off when
-// the artifact server is offline.
-type ArtifactFetcher struct {
-	httpClient *retryablehttp.Client
-}
-
-// ArtifactNotFoundError is an error type used to signal 404 HTTP status code responses.
-var ArtifactNotFoundError = errors.New("artifact not found")
-
-// NewArtifactFetcher configures the retryable http client used for fetching artifacts.
-// By default, it retries 10 times within a 3.5 minutes window.
-func NewArtifactFetcher(retries int) *ArtifactFetcher {
-	httpClient := retryablehttp.NewClient()
-	httpClient.RetryWaitMin = 5 * time.Second
-	httpClient.RetryWaitMax = 30 * time.Second
-	httpClient.RetryMax = retries
-	httpClient.Logger = nil
-
-	return &ArtifactFetcher{httpClient: httpClient}
-}
-
-// Fetch downloads, verifies and extracts the artifact content to the specified directory.
-// If the artifact server responds with 5xx errors, the download operation is retried.
-// If the artifact server responds with 404, the returned error is of type ArtifactNotFoundError.
-// If the artifact server is unavailable for more than 3 minutes, the returned error contains the original status code.
-func (r *ArtifactFetcher) Fetch(artifact *sourcev1.Artifact, dir string) error {
-	artifactURL := artifact.URL
-	if hostname := os.Getenv("SOURCE_CONTROLLER_LOCALHOST"); hostname != "" {
-		u, err := url.Parse(artifactURL)
-		if err != nil {
-			return err
-		}
-		u.Host = hostname
-		artifactURL = u.String()
-	}
-
-	req, err := retryablehttp.NewRequest(http.MethodGet, artifactURL, nil)
-	if err != nil {
-		return fmt.Errorf("failed to create a new request: %w", err)
-	}
-
-	resp, err := r.httpClient.Do(req)
-	if err != nil {
-		return fmt.Errorf("failed to download artifact, error: %w", err)
-	}
-	defer resp.Body.Close()
-
-	if code := resp.StatusCode; code != http.StatusOK {
-		if code == http.StatusNotFound {
-			return ArtifactNotFoundError
-		}
-		return fmt.Errorf("failed to download artifact from %s, status: %s", artifactURL, resp.Status)
-	}
-
-	var buf bytes.Buffer
-
-	// verify checksum matches origin
-	if err := r.Verify(artifact, &buf, resp.Body); err != nil {
-		return err
-	}
-
-	// extract
-	if _, err = untar.Untar(&buf, dir); err != nil {
-		return fmt.Errorf("failed to untar artifact, error: %w", err)
-	}
-
-	return nil
-}
-
-// Verify computes the checksum of the tarball and returns an error if the computed value
-// does not match the artifact advertised checksum.
-func (r *ArtifactFetcher) Verify(artifact *sourcev1.Artifact, buf *bytes.Buffer, reader io.Reader) error {
-	hasher := sha256.New()
-
-	// for backwards compatibility with source-controller v0.17.2 and older
-	if len(artifact.Checksum) == 40 {
-		hasher = sha1.New()
-	}
-
-	// compute checksum
-	mw := io.MultiWriter(hasher, buf)
-	if _, err := io.Copy(mw, reader); err != nil {
-		return err
-	}
-
-	if checksum := fmt.Sprintf("%x", hasher.Sum(nil)); checksum != artifact.Checksum {
-		return fmt.Errorf("failed to verify artifact: computed checksum '%s' doesn't match advertised '%s'",
-			checksum, artifact.Checksum)
-	}
-
-	return nil
-}

controllers/kustomization_transformer_test.go

@@ -50,7 +50,7 @@ func TestKustomizationReconciler_KustomizeTransformer(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 
 	artifactFile := "patch-" + randStringRunes(5)
-	artifactChecksum, err := createArtifact(testServer, "testdata/transformers", artifactFile)
+	artifactChecksum, err := testServer.ArtifactFromDir("testdata/transformers", artifactFile)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	repositoryName := types.NamespacedName{
@@ -173,7 +173,7 @@ func TestKustomizationReconciler_KustomizeTransformerFiles(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred())
 
 	artifactFile := "patch-" + randStringRunes(5)
-	artifactChecksum, err := createArtifact(testServer, "testdata/file-transformer", artifactFile)
+	artifactChecksum, err := testServer.ArtifactFromDir("testdata/file-transformer", artifactFile)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	repositoryName := types.NamespacedName{
@@ -292,7 +292,7 @@ func TestKustomizationReconciler_FluxTransformers(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret")
 
 	artifactFile := "patch-" + randStringRunes(5)
-	artifactChecksum, err := createArtifact(testServer, "testdata/patch", artifactFile)
+	artifactChecksum, err := testServer.ArtifactFromDir("testdata/patch", artifactFile)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	repositoryName := types.NamespacedName{

controllers/kustomization_validation_test.go

@@ -43,11 +43,11 @@ func TestKustomizationReconciler_Validation(t *testing.T) {
 	g.Expect(err).NotTo(HaveOccurred(), "failed to create kubeconfig secret")
 
 	artifactName := "val-" + randStringRunes(5)
-	artifactChecksum, err := createArtifact(testServer, "testdata/invalid/plain", artifactName)
+	artifactChecksum, err := testServer.ArtifactFromDir("testdata/invalid/plain", artifactName)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	overlayArtifactName := "val-" + randStringRunes(5)
-	overlayChecksum, err := createArtifact(testServer, "testdata/invalid/overlay", overlayArtifactName)
+	overlayChecksum, err := testServer.ArtifactFromDir("testdata/invalid/overlay", overlayArtifactName)
 	g.Expect(err).ToNot(HaveOccurred())
 
 	repositoryName := types.NamespacedName{

controllers/suite_test.go

@@ -17,17 +17,12 @@ limitations under the License.
 package controllers
 
 import (
-	"archive/tar"
-	"compress/gzip"
 	"context"
-	"crypto/sha1"
 	"crypto/sha256"
 	"fmt"
-	"io"
 	"math/rand"
 	"os"
 	"path/filepath"
-	"strings"
 	"testing"
 	"time"
 
@@ -288,95 +283,6 @@ func applyGitRepository(objKey client.ObjectKey, artifactName string, revision s
 	return nil
 }
 
-func createArtifact(artifactServer *testserver.ArtifactServer, fixture, path string) (string, error) {
-	if f, err := os.Stat(fixture); os.IsNotExist(err) || !f.IsDir() {
-		return "", fmt.Errorf("invalid fixture path: %s", fixture)
-	}
-	f, err := os.Create(filepath.Join(artifactServer.Root(), path))
-	if err != nil {
-		return "", err
-	}
-	defer func() {
-		if err != nil {
-			os.Remove(f.Name())
-		}
-	}()
-
-	h := sha1.New()
-
-	mw := io.MultiWriter(h, f)
-	gw := gzip.NewWriter(mw)
-	tw := tar.NewWriter(gw)
-
-	if err = filepath.Walk(fixture, func(p string, fi os.FileInfo, err error) error {
-		if err != nil {
-			return err
-		}
-
-		// Ignore anything that is not a file (directories, symlinks)
-		if !fi.Mode().IsRegular() {
-			return nil
-		}
-
-		// Ignore dotfiles
-		if strings.HasPrefix(fi.Name(), ".") {
-			return nil
-		}
-
-		header, err := tar.FileInfoHeader(fi, p)
-		if err != nil {
-			return err
-		}
-		// The name needs to be modified to maintain directory structure
-		// as tar.FileInfoHeader only has access to the base name of the file.
-		// Ref: https://golang.org/src/archive/tar/common.go?#L626
-		relFilePath := p
-		if filepath.IsAbs(fixture) {
-			relFilePath, err = filepath.Rel(fixture, p)
-			if err != nil {
-				return err
-			}
-		}
-		header.Name = relFilePath
-
-		if err := tw.WriteHeader(header); err != nil {
-			return err
-		}
-
-		f, err := os.Open(p)
-		if err != nil {
-			f.Close()
-			return err
-		}
-		if _, err := io.Copy(tw, f); err != nil {
-			f.Close()
-			return err
-		}
-		return f.Close()
-	}); err != nil {
-		return "", err
-	}
-
-	if err := tw.Close(); err != nil {
-		gw.Close()
-		f.Close()
-		return "", err
-	}
-	if err := gw.Close(); err != nil {
-		f.Close()
-		return "", err
-	}
-	if err := f.Close(); err != nil {
-		return "", err
-	}
-
-	if err := os.Chmod(f.Name(), 0644); err != nil {
-		return "", err
-	}
-
-	return fmt.Sprintf("%x", h.Sum(nil)), nil
-}
-
 func createVaultTestInstance() (*dockertest.Pool, *dockertest.Resource, error) {
 	// uses a sensible default on windows (tcp/http) and linux/osx (socket)
 	pool, err := dockertest.NewPool("")

go.mod

@@ -24,15 +24,15 @@ require (
 	github.com/fluxcd/pkg/apis/acl v0.1.0
 	github.com/fluxcd/pkg/apis/kustomize v0.6.0
 	github.com/fluxcd/pkg/apis/meta v0.17.0
+	github.com/fluxcd/pkg/http/fetch v0.1.0
 	github.com/fluxcd/pkg/kustomize v0.8.0
 	github.com/fluxcd/pkg/runtime v0.20.0
 	github.com/fluxcd/pkg/ssa v0.21.0
-	github.com/fluxcd/pkg/testserver v0.3.0
-	github.com/fluxcd/pkg/untar v0.2.0
+	github.com/fluxcd/pkg/tar v0.1.0
+	github.com/fluxcd/pkg/testserver v0.4.0
 	github.com/fluxcd/source-controller/api v0.30.0
-	github.com/hashicorp/go-retryablehttp v0.7.1
 	github.com/hashicorp/vault/api v1.8.0
-	github.com/onsi/gomega v1.20.2
+	github.com/onsi/gomega v1.21.1
 	github.com/ory/dockertest v3.3.5+incompatible
 	github.com/otiai10/copy v1.7.0
 	github.com/spf13/pflag v1.0.5
@@ -150,6 +150,7 @@ require (
 	github.com/hashicorp/go-immutable-radix v1.3.1 // indirect
 	github.com/hashicorp/go-multierror v1.1.1 // indirect
 	github.com/hashicorp/go-plugin v1.4.3 // indirect
+	github.com/hashicorp/go-retryablehttp v0.7.1 // indirect
 	github.com/hashicorp/go-rootcerts v1.0.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/mlock v0.1.2 // indirect
 	github.com/hashicorp/go-secure-stdlib/parseutil v0.1.6 // indirect

go.sum

@@ -290,16 +290,18 @@ github.com/fluxcd/pkg/apis/kustomize v0.6.0 h1:Afxv3Uv+xiuettzqm3sP0ceWikDZTfHdH
 github.com/fluxcd/pkg/apis/kustomize v0.6.0/go.mod h1:iY0zSpK6eUiPfNt/yR6g0q/wQP+wH+Ax/L7KBOx5x2M=
 github.com/fluxcd/pkg/apis/meta v0.17.0 h1:Y2dfo1syHZDb9Mexjr2SWdcj1FnxnRXm015hEnhl6wU=
 github.com/fluxcd/pkg/apis/meta v0.17.0/go.mod h1:GrOVzWXiu22XjLNgLLe2EBYhQPqZetes5SIADb4bmHE=
+github.com/fluxcd/pkg/http/fetch v0.1.0 h1:Ig/kZuM0+jHBJnwHn5UUseTKIYD5w8X4bInJyuyOZKI=
+github.com/fluxcd/pkg/http/fetch v0.1.0/go.mod h1:1CjOSfn7aOeHf2ZRA2+GTKHg442zN6X/fSys3a0KLC0=
 github.com/fluxcd/pkg/kustomize v0.8.0 h1:8AdEvp6y38ISZzoi0H82Si5zkmLXClbeX10W7HevB00=
 github.com/fluxcd/pkg/kustomize v0.8.0/go.mod h1:zGtCZF6V3hMWcf46SqrQc10fS9yUlKzi2UcFUeabDAE=
 github.com/fluxcd/pkg/runtime v0.20.0 h1:F9q9wap0BhjQszboUroJrYOB1C831zkQwTAk2tlMIQc=
 github.com/fluxcd/pkg/runtime v0.20.0/go.mod h1:KVHNQMhccuLTjMDFVCr/SF+4Z554bcMH1LncC4sQf8o=
 github.com/fluxcd/pkg/ssa v0.21.0 h1:aeoTohPNf5x7jQjHidyLJAOHw3EyHOQoQN3mN2i+4cc=
 github.com/fluxcd/pkg/ssa v0.21.0/go.mod h1:jumyhUbEMDnduN7anSlKfxl2fEoyeyv+Ta5hWCbxI5Q=
-github.com/fluxcd/pkg/testserver v0.3.0 h1:oyZW6YWHVZR7FRVNu7lN9F5H808TD2jCzBm8CenFoi0=
-github.com/fluxcd/pkg/testserver v0.3.0/go.mod h1:gjOKX41okmrGYOa4oOF2fiLedDAfPo1XaG/EzrUUGBI=
-github.com/fluxcd/pkg/untar v0.2.0 h1:sJXU+FbJcNUb2ffLJNjeR3hwt3X2loVpOMlCUjyFw6E=
-github.com/fluxcd/pkg/untar v0.2.0/go.mod h1:33AyoWaPpjX/xXpczcfhQh2AkB63TFwiR2YwROtv23E=
+github.com/fluxcd/pkg/tar v0.1.0 h1:ObyUml8NJtGQtz/cRgexd7HU2mQsTmgjz2dtX4xdnng=
+github.com/fluxcd/pkg/tar v0.1.0/go.mod h1:w0/TOC7kwBJhnSJn7TCABkc/I7ib1f2Yz6vOsbLBnhw=
+github.com/fluxcd/pkg/testserver v0.4.0 h1:pDZ3gistqYhwlf3sAjn1Q8NzN4Qe6I1BEmHMHi46lMg=
+github.com/fluxcd/pkg/testserver v0.4.0/go.mod h1:gjOKX41okmrGYOa4oOF2fiLedDAfPo1XaG/EzrUUGBI=
 github.com/fluxcd/source-controller/api v0.30.0 h1:rPVPpwXcYG2n0DTRcRagfGDiccvCib5S09K5iMjlpRU=
 github.com/fluxcd/source-controller/api v0.30.0/go.mod h1:UkjAqQ6QAXNNesNQDTArTeiTp+UuhOUIA+JyFhGP/+Q=
 github.com/form3tech-oss/jwt-go v3.2.2+incompatible/go.mod h1:pbq4aXjuKjdthFRnoDwaVPLA+WlJuPGy+QneDUgJi2k=
@@ -676,8 +678,8 @@ github.com/onsi/ginkgo/v2 v2.1.6 h1:Fx2POJZfKRQcM1pH49qSZiYeu319wji004qX+GDovrU=
 github.com/onsi/gomega v0.0.0-20170829124025-dcabb60a477c/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
 github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
-github.com/onsi/gomega v1.20.2 h1:8uQq0zMgLEfa0vRrrBgaJF2gyW9Da9BmfGV+OyUzfkY=
-github.com/onsi/gomega v1.20.2/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
+github.com/onsi/gomega v1.21.1 h1:OB/euWYIExnPBohllTicTHmGTrMaqJ67nIu80j0/uEM=
+github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc=
 github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U=
 github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM=
 github.com/opencontainers/image-spec v1.0.2 h1:9yCKha/T5XdGtO0q9Q9a6T5NUCsTn/DrBg0D7ufOcFM=