controllers: test HelmRepository HTTP basic auth

controllers/helmrepository_controller_test.go

@@ -2,25 +2,20 @@ package controllers
 
 import (
 	"context"
-	"fmt"
+	"net/http"
-	"io/ioutil"
 	"os"
 	"path"
-	"path/filepath"
+	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"helm.sh/helm/v3/pkg/action"
-	"helm.sh/helm/v3/pkg/getter"
-	"helm.sh/helm/v3/pkg/repo/repotest"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
-	"k8s.io/client-go/kubernetes/scheme"
-	ctrl "sigs.k8s.io/controller-runtime"
 
 	sourcev1 "github.com/fluxcd/source-controller/api/v1alpha1"
+	"github.com/fluxcd/source-controller/internal/testserver"
 )
 
 var _ = Describe("HelmRepositoryReconciler", func() {
@@ -32,8 +27,7 @@ var _ = Describe("HelmRepositoryReconciler", func() {
 
 	var (
 		namespace  *corev1.Namespace
-		storage     *Storage
+		helmServer *testserver.Helm
-		helmRepoSrv *repotest.Server
 		err        error
 	)
 
@@ -43,93 +37,60 @@ var _ = Describe("HelmRepositoryReconciler", func() {
 		}
 		err = k8sClient.Create(context.Background(), namespace)
 		Expect(err).NotTo(HaveOccurred(), "failed to create test namespace")
-
-		tmpStoragePath, err := ioutil.TempDir("", "helmrepository")
-		Expect(err).NotTo(HaveOccurred(), "failed to create tmp storage dir")
-
-		storage, err = NewStorage(tmpStoragePath, "localhost", timeout)
-		Expect(err).NotTo(HaveOccurred(), "failed to create tmp storage")
-
-		helmRepoSrv, err = makeHelmRepoSrv()
-		Expect(err).NotTo(HaveOccurred(), "failed to setup tmp helm repository server")
-		helmRepoSrv.Start()
-
-		err = (&HelmRepositoryReconciler{
-			Client:  k8sClient,
-			Log:     ctrl.Log.WithName("controllers").WithName("HelmRepository"),
-			Scheme:  scheme.Scheme,
-			Storage: storage,
-			Getters: getter.Providers{getter.Provider{
-				Schemes: []string{"http", "https"},
-				New:     getter.NewHTTPGetter,
-			}},
-		}).SetupWithManager(k8sManager)
-		Expect(err).ToNot(HaveOccurred(), "failed to setup reconciler")
-
-		go func() {
-			err = k8sManager.Start(ctrl.SetupSignalHandler())
-			Expect(err).ToNot(HaveOccurred())
-		}()
 	})
 
 	AfterEach(func() {
-		if storage != nil {
+		err = k8sClient.Delete(context.Background(), namespace)
-			os.RemoveAll(storage.BasePath)
-		}
-		if helmRepoSrv != nil {
-			helmRepoSrv.Stop()
-			os.RemoveAll(filepath.Dir(helmRepoSrv.Root()))
-		}
-
-		err := k8sClient.Delete(context.Background(), namespace)
 		Expect(err).NotTo(HaveOccurred(), "failed to delete test namespace")
 	})
 
 	Context("HelmRepository", func() {
-		It("Should create successfully", func() {
+		It("Creates artifacts for", func() {
+			helmServer, err = testserver.NewTempHelmServer()
+			Expect(err).To(Succeed())
+			defer os.RemoveAll(helmServer.Root())
+			defer helmServer.Stop()
+			helmServer.Start()
+
+			Expect(helmServer.PackageChart(path.Join("testdata/helmchart"))).Should(Succeed())
+			Expect(helmServer.GenerateIndex()).Should(Succeed())
+
 			key := types.NamespacedName{
 				Name:      "helmrepository-sample-" + randStringRunes(5),
 				Namespace: namespace.Name,
 			}
-
 			created := &sourcev1.HelmRepository{
 				ObjectMeta: metav1.ObjectMeta{
 					Name:      key.Name,
 					Namespace: key.Namespace,
 				},
 				Spec: sourcev1.HelmRepositorySpec{
-					URL:      helmRepoSrv.URL(),
+					URL:      helmServer.URL(),
 					Interval: metav1.Duration{Duration: interval},
 				},
 			}
-
 			Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
 
-			got := &sourcev1.HelmRepository{}
 			By("Expecting artifact")
+			got := &sourcev1.HelmRepository{}
 			Eventually(func() bool {
 				_ = k8sClient.Get(context.Background(), key, got)
-				return got.Status.Artifact != nil
+				return got.Status.Artifact != nil && storage.ArtifactExist(*got.Status.Artifact)
 			}, timeout, interval).Should(BeTrue())
-			Eventually(func() bool {
-				return storage.ArtifactExist(*got.Status.Artifact)
-			}).Should(BeTrue())
 
 			By("Updating the chart index")
 			// Regenerating the index is sufficient to make the revision change
-			Expect(helmRepoSrv.CreateIndex()).Should(Succeed())
+			Expect(helmServer.GenerateIndex()).Should(Succeed())
 			Eventually(func() bool {
-				r := &sourcev1.HelmRepository{}
+				now := &sourcev1.HelmRepository{}
-				_ = k8sClient.Get(context.Background(), key, r)
+				_ = k8sClient.Get(context.Background(), key, now)
-				if r.Status.Artifact == nil {
+				// Test revision change and garbage collection
-					return false
+				return now.Status.Artifact.Revision != got.Status.Artifact.Revision &&
-				}
+					!storage.ArtifactExist(*got.Status.Artifact)
-				return r.Status.Artifact.Revision != got.Status.Artifact.Revision
 			}, timeout, interval).Should(BeTrue())
 
 			updated := &sourcev1.HelmRepository{}
 			Expect(k8sClient.Get(context.Background(), key, updated)).Should(Succeed())
-
 			updated.Spec.Interval = metav1.Duration{Duration: 60 * time.Second}
 			Expect(k8sClient.Update(context.Background(), updated)).Should(Succeed())
 
@@ -145,31 +106,98 @@ var _ = Describe("HelmRepositoryReconciler", func() {
 				r := &sourcev1.HelmRepository{}
 				return k8sClient.Get(context.Background(), key, r)
 			}).ShouldNot(Succeed())
+			Eventually(storage.ArtifactExist(*got.Status.Artifact), timeout, interval).ShouldNot(BeTrue())
+		})
+
+		It("Authenticates when basic auth credentials are provided", func() {
+			helmServer, err = testserver.NewTempHelmServer()
+			Expect(err).NotTo(HaveOccurred())
+
+			var username, password = "john", "doe"
+			helmServer.WithMiddleware(func(handler http.Handler) http.Handler {
+				return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+					u, p, ok := r.BasicAuth()
+					if !ok || username != u || password != p {
+						w.WriteHeader(401)
+						return
+					}
+					handler.ServeHTTP(w, r)
+				})
+			})
+			defer os.RemoveAll(helmServer.Root())
+			defer helmServer.Stop()
+			helmServer.Start()
+
+			Expect(helmServer.PackageChart(path.Join("testdata/helmchart"))).Should(Succeed())
+			Expect(helmServer.GenerateIndex()).Should(Succeed())
+
+			secretKey := types.NamespacedName{
+				Name:      "helmrepository-auth-" + randStringRunes(5),
+				Namespace: namespace.Name,
+			}
+			secret := &corev1.Secret{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      secretKey.Name,
+					Namespace: secretKey.Namespace,
+				},
+				Data: map[string][]byte{},
+			}
+			Expect(k8sClient.Create(context.Background(), secret)).Should(Succeed())
+
+			key := types.NamespacedName{
+				Name:      "helmrepository-sample-" + randStringRunes(5),
+				Namespace: namespace.Name,
+			}
+			created := &sourcev1.HelmRepository{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      key.Name,
+					Namespace: key.Namespace,
+				},
+				Spec: sourcev1.HelmRepositorySpec{
+					URL: helmServer.URL(),
+					SecretRef: &corev1.LocalObjectReference{
+						Name: secretKey.Name,
+					},
+					Interval: metav1.Duration{Duration: interval},
+				},
+			}
+			Expect(k8sClient.Create(context.Background(), created)).Should(Succeed())
+
+			By("Expecting 401")
 			Eventually(func() bool {
-				return storage.ArtifactExist(*got.Status.Artifact)
+				got := &sourcev1.HelmRepository{}
-			}).ShouldNot(BeTrue())
+				_ = k8sClient.Get(context.Background(), key, got)
+				for _, c := range got.Status.Conditions {
+					if c.Reason == sourcev1.IndexationFailedReason &&
+						strings.Contains(c.Message, "401 Unauthorized") {
+						return true
+					}
+				}
+				return false
+			}, timeout, interval).Should(BeTrue())
+
+			By("Expecting missing field error")
+			secret.Data["username"] = []byte(username)
+			Expect(k8sClient.Update(context.Background(), secret)).Should(Succeed())
+			Eventually(func() bool {
+				got := &sourcev1.HelmRepository{}
+				_ = k8sClient.Get(context.Background(), key, got)
+				for _, c := range got.Status.Conditions {
+					if c.Reason == sourcev1.AuthenticationFailedReason {
+						return true
+					}
+				}
+				return false
+			}, timeout, interval).Should(BeTrue())
+
+			By("Expecting artifact")
+			secret.Data["password"] = []byte(password)
+			Expect(k8sClient.Update(context.Background(), secret)).Should(Succeed())
+			Eventually(func() bool {
+				got := &sourcev1.HelmRepository{}
+				_ = k8sClient.Get(context.Background(), key, got)
+				return got.Status.Artifact != nil
+			}, timeout, interval).Should(BeTrue())
 		})
 	})
 })
-
-func makeHelmRepoSrv() (*repotest.Server, error) {
-	tmpDir, err := ioutil.TempDir("", "helm-repo-srv")
-	if err != nil {
-		return nil, fmt.Errorf("failed to create tmp helm repository dir: %w", err)
-	}
-
-	pkg := action.NewPackage()
-	pkg.Destination = tmpDir
-	_, err = pkg.Run("testdata/helmchart", nil)
-	if err != nil {
-		os.RemoveAll(tmpDir)
-		return nil, fmt.Errorf("failed to package helm chart: %w", err)
-	}
-
-	srv := repotest.NewServer(path.Join(tmpDir, "*.tgz"))
-	if err = srv.CreateIndex(); err != nil {
-		os.RemoveAll(tmpDir)
-		return nil, fmt.Errorf("failed to create index for tmp helm repository: %w", err)
-	}
-	return srv, nil
-}

controllers/suite_test.go

@@ -17,6 +17,7 @@ limitations under the License.
 package controllers
 
 import (
+	"io/ioutil"
 	"math/rand"
 	"os"
 	"path/filepath"
@@ -25,7 +26,7 @@ import (
 
 	. "github.com/onsi/ginkgo"
 	. "github.com/onsi/gomega"
-	"github.com/onsi/gomega/gexec"
+	"helm.sh/helm/v3/pkg/getter"
 	"k8s.io/client-go/kubernetes/scheme"
 	"k8s.io/client-go/rest"
 	ctrl "sigs.k8s.io/controller-runtime"
@@ -46,6 +47,7 @@ var cfg *rest.Config
 var k8sClient client.Client
 var k8sManager ctrl.Manager
 var testEnv *envtest.Environment
+var storage *Storage
 
 func TestAPIs(t *testing.T) {
 	RegisterFailHandler(Fail)
@@ -86,11 +88,34 @@ var _ = BeforeSuite(func(done Done) {
 
 	// +kubebuilder:scaffold:scheme
 
+	tmpStoragePath, err := ioutil.TempDir("", "helmrepository")
+	Expect(err).NotTo(HaveOccurred(), "failed to create tmp storage dir")
+
+	storage, err = NewStorage(tmpStoragePath, "localhost", time.Second*30)
+	Expect(err).NotTo(HaveOccurred(), "failed to create tmp storage")
+
 	k8sManager, err = ctrl.NewManager(cfg, ctrl.Options{
 		Scheme: scheme.Scheme,
 	})
 	Expect(err).ToNot(HaveOccurred())
 
+	err = (&HelmRepositoryReconciler{
+		Client:  k8sManager.GetClient(),
+		Log:     ctrl.Log.WithName("controllers").WithName("HelmRepository"),
+		Scheme:  scheme.Scheme,
+		Storage: storage,
+		Getters: getter.Providers{getter.Provider{
+			Schemes: []string{"http", "https"},
+			New:     getter.NewHTTPGetter,
+		}},
+	}).SetupWithManager(k8sManager)
+	Expect(err).ToNot(HaveOccurred(), "failed to setup HelmRepositoryReconciler")
+
+	go func() {
+		err = k8sManager.Start(ctrl.SetupSignalHandler())
+		Expect(err).ToNot(HaveOccurred())
+	}()
+
 	k8sClient = k8sManager.GetClient()
 	Expect(k8sClient).ToNot(BeNil())
 
@@ -99,7 +124,10 @@ var _ = BeforeSuite(func(done Done) {
 
 var _ = AfterSuite(func() {
 	By("tearing down the test environment")
-	gexec.KillAndWait(5 * time.Second)
+	if storage != nil {
+		err := os.RemoveAll(storage.BasePath)
+		Expect(err).NotTo(HaveOccurred())
+	}
 	err := testEnv.Stop()
 	Expect(err).ToNot(HaveOccurred())
 })

internal/testserver/helm.go

@@ -0,0 +1,43 @@
+package testserver
+
+import (
+	"io/ioutil"
+	"path/filepath"
+
+	"helm.sh/helm/v3/pkg/action"
+	"helm.sh/helm/v3/pkg/repo"
+	"sigs.k8s.io/yaml"
+)
+
+func NewTempHelmServer() (*Helm, error) {
+	server, err := NewTempHTTPServer()
+	if err != nil {
+		return nil, err
+	}
+	helm := &Helm{server}
+	return helm, nil
+}
+
+type Helm struct {
+	*HTTP
+}
+
+func (s *Helm) GenerateIndex() error {
+	index, err := repo.IndexDirectory(s.HTTP.docroot, s.HTTP.URL())
+	if err != nil {
+		return err
+	}
+	d, err := yaml.Marshal(index)
+	if err != nil {
+		return err
+	}
+	f := filepath.Join(s.HTTP.docroot, "index.yaml")
+	return ioutil.WriteFile(f, d, 0644)
+}
+
+func (s *Helm) PackageChart(path string) error {
+	pkg := action.NewPackage()
+	pkg.Destination = s.HTTP.docroot
+	_, err := pkg.Run(path, nil)
+	return err
+}

internal/testserver/http.go

@@ -0,0 +1,65 @@
+package testserver
+
+import (
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+)
+
+func NewTempHTTPServer() (*HTTP, error) {
+	tmpDir, err := ioutil.TempDir("", "http-test-")
+	if err != nil {
+		return nil, err
+	}
+	srv := NewHTTPServer(tmpDir)
+	return srv, nil
+}
+
+func NewHTTPServer(docroot string) *HTTP {
+	root, err := filepath.Abs(docroot)
+	if err != nil {
+		panic(err)
+	}
+	return &HTTP{
+		docroot: root,
+	}
+}
+
+type HTTP struct {
+	docroot    string
+	middleware func(http.Handler) http.Handler
+	server     *httptest.Server
+}
+
+func (s *HTTP) WithMiddleware(m func(handler http.Handler) http.Handler) *HTTP {
+	s.middleware = m
+	return s
+}
+
+func (s *HTTP) Start() {
+	s.server = httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		handler := http.FileServer(http.Dir(s.docroot))
+		if s.middleware != nil {
+			s.middleware(handler).ServeHTTP(w, r)
+			return
+		}
+		handler.ServeHTTP(w, r)
+	}))
+}
+
+func (s *HTTP) Stop() {
+	if s.server != nil {
+		s.server.Close()
+	}
+}
+
+func (s *HTTP) Root() string {
+	return s.docroot
+}
+func (s *HTTP) URL() string {
+	if s.server != nil {
+		return s.server.URL
+	}
+	return ""
+}