Chart tracker now supports logos in data urls (#355)

Closes #328

Signed-off-by: Sergio Castaño Arteaga <tegioz@icloud.com>

cmd/chart-tracker/dispatcher.go

@@ -32,7 +32,7 @@ type Job struct {
 	Kind         JobKind
 	Repo         *hub.ChartRepository
 	ChartVersion *repo.ChartVersion
-	DownloadLogo bool
+	GetLogo      bool
 }
 
 // Dispatcher is in charge of generating jobs to register or unregister charts
@@ -105,9 +105,9 @@ func (d *Dispatcher) generateSyncJobs(wg *sync.WaitGroup, r *hub.ChartRepository
 	chartsAvailable := make(map[string]struct{})
 	for _, chartVersions := range indexFile.Entries {
 		for i, chartVersion := range chartVersions {
-			var downloadLogo bool
+			var getLogo bool
 			if i == 0 {
-				downloadLogo = true
+				getLogo = true
 			}
 			key := fmt.Sprintf("%s@%s", chartVersion.Metadata.Name, chartVersion.Metadata.Version)
 			chartsAvailable[key] = struct{}{}
@@ -116,7 +116,7 @@ func (d *Dispatcher) generateSyncJobs(wg *sync.WaitGroup, r *hub.ChartRepository
 					Kind:         Register,
 					Repo:         r,
 					ChartVersion: chartVersion,
-					DownloadLogo: downloadLogo,
+					GetLogo:      getLogo,
 				}
 			}
 			select {

cmd/chart-tracker/dispatcher_test.go

@@ -114,7 +114,7 @@ func TestDispatcher(t *testing.T) {
 						Kind:         Register,
 						Repo:         repo1,
 						ChartVersion: pkg1V1,
-						DownloadLogo: true,
+						GetLogo:      true,
 					},
 				},
 			},
@@ -139,13 +139,13 @@ func TestDispatcher(t *testing.T) {
 						Kind:         Register,
 						Repo:         repo1,
 						ChartVersion: pkg1V1,
-						DownloadLogo: true,
+						GetLogo:      true,
 					},
 					{
 						Kind:         Register,
 						Repo:         repo1,
 						ChartVersion: pkg1V2,
-						DownloadLogo: false,
+						GetLogo:      false,
 					},
 				},
 			},
@@ -219,7 +219,7 @@ func TestDispatcher(t *testing.T) {
 						Kind:         Register,
 						Repo:         repo1,
 						ChartVersion: pkg1V2,
-						DownloadLogo: false,
+						GetLogo:      false,
 					},
 				},
 			},
@@ -252,7 +252,7 @@ func TestDispatcher(t *testing.T) {
 						Kind:         Register,
 						Repo:         repo1,
 						ChartVersion: pkg2V1,
-						DownloadLogo: true,
+						GetLogo:      true,
 					},
 				},
 			},
@@ -288,25 +288,25 @@ func TestDispatcher(t *testing.T) {
 						Kind:         Register,
 						Repo:         repo1,
 						ChartVersion: pkg1V1,
-						DownloadLogo: true,
+						GetLogo:      true,
 					},
 					{
 						Kind:         Register,
 						Repo:         repo1,
 						ChartVersion: pkg1V2,
-						DownloadLogo: false,
+						GetLogo:      false,
 					},
 					{
 						Kind:         Register,
 						Repo:         repo1,
 						ChartVersion: pkg2V1,
-						DownloadLogo: true,
+						GetLogo:      true,
 					},
 					{
 						Kind:         Register,
 						Repo:         repo2,
 						ChartVersion: pkg3V1,
-						DownloadLogo: true,
+						GetLogo:      true,
 					},
 				},
 			},

cmd/chart-tracker/worker.go

@@ -10,12 +10,14 @@ import (
 	"net/url"
 	"path"
 	"runtime/debug"
+	"strings"
 	"sync"
 
 	"github.com/artifacthub/hub/internal/hub"
 	"github.com/artifacthub/hub/internal/img"
 	"github.com/rs/zerolog"
 	"github.com/rs/zerolog/log"
+	"github.com/vincent-petithory/dataurl"
 	"helm.sh/helm/v3/pkg/chart"
 	"helm.sh/helm/v3/pkg/chart/loader"
 )
@@ -96,8 +98,19 @@ func (w *Worker) Run(wg *sync.WaitGroup, queue chan *Job) {
 	}
 }
 
-// downloadImage downloads the image located at the url provided.
-func (w *Worker) downloadImage(u string) ([]byte, error) {
+// getImage gets the image located at the url provided. If it's a data url the
+// image is extracted from it. Otherwise it's downloaded using the url.
+func (w *Worker) getImage(u string) ([]byte, error) {
+	// Image in data url
+	if strings.HasPrefix(u, "data:") {
+		dataURL, err := dataurl.DecodeString(u)
+		if err != nil {
+			return nil, err
+		}
+		return dataURL.Data, nil
+	}
+
+	// Download image using url provided
 	resp, err := w.hg.Get(u)
 	if err != nil {
 		return nil, err
@@ -154,13 +167,13 @@ func (w *Worker) handleRegisterJob(j *Job) error {
 
 	// Store chart logo when available if requested
 	var logoURL, logoImageID string
-	if j.DownloadLogo {
+	if j.GetLogo {
 		if md.Icon != "" {
 			logoURL = md.Icon
-			data, err := w.downloadImage(md.Icon)
+			data, err := w.getImage(md.Icon)
 			if err != nil {
-				w.ec.Append(j.Repo.ChartRepositoryID, fmt.Errorf("error dowloading logo %s: %w", md.Icon, err))
-				w.logger.Debug().Err(err).Str("url", md.Icon).Msg("image download failed")
+				w.ec.Append(j.Repo.ChartRepositoryID, fmt.Errorf("error getting logo image %s: %w", md.Icon, err))
+				w.logger.Debug().Err(err).Str("url", md.Icon).Msg("get image failed")
 			} else {
 				logoImageID, err = w.is.SaveImage(w.ctx, data)
 				if err != nil && !errors.Is(err, image.ErrFormat) {

cmd/chart-tracker/worker_test.go

@@ -33,11 +33,20 @@ func TestWorker(t *testing.T) {
 				"http://tests/pkg1-1.0.0.tgz",
 			},
 		}
+		pkg2V1 := &repo.ChartVersion{
+			Metadata: &chart.Metadata{
+				Name:    "pkg2",
+				Version: "1.0.0",
+			},
+			URLs: []string{
+				"http://tests/pkg2-1.0.0.tgz",
+			},
+		}
 		job := &Job{
 			Kind:         Register,
 			Repo:         repo1,
 			ChartVersion: pkg1V1,
-			DownloadLogo: true,
+			GetLogo:      true,
 		}
 
 		t.Run("error downloading chart", func(t *testing.T) {
@@ -176,6 +185,31 @@ func TestWorker(t *testing.T) {
 			ww.w.Run(ww.wg, ww.queue)
 			ww.assertExpectations(t)
 		})
+
+		t.Run("package with logo in data url registered successfully", func(t *testing.T) {
+			// Setup worker and expectations
+			ww := newWorkerWrapper(context.Background())
+			job := &Job{
+				Kind:         Register,
+				Repo:         repo1,
+				ChartVersion: pkg2V1,
+				GetLogo:      true,
+			}
+			ww.queue <- job
+			close(ww.queue)
+			f, _ := os.Open("testdata/" + path.Base(job.ChartVersion.URLs[0]))
+			ww.hg.On("Get", job.ChartVersion.URLs[0]).Return(&http.Response{
+				Body:       f,
+				StatusCode: http.StatusOK,
+			}, nil)
+			expectedLogoData, _ := ioutil.ReadFile("testdata/red-dot.png")
+			ww.is.On("SaveImage", mock.Anything, expectedLogoData).Return("imageID", nil)
+			ww.pm.On("Register", mock.Anything, mock.Anything).Return(nil)
+
+			// Run worker and check expectations
+			ww.w.Run(ww.wg, ww.queue)
+			ww.assertExpectations(t)
+		})
 	})
 
 	t.Run("handle unregister job", func(t *testing.T) {

go.mod

@@ -34,6 +34,7 @@ require (
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
 	github.com/spf13/viper v1.6.3
 	github.com/stretchr/testify v1.5.1
+	github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50
 	golang.org/x/crypto v0.0.0-20200422194213-44a606286825
 	golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect
 	golang.org/x/net v0.0.0-20200421231249-e086a090c8fd // indirect

go.sum

@@ -625,6 +625,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT
 github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw=
+github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 h1:uxE3GYdXIOfhMv3unJKETJEhw78gvzuQqRX/rVirc2A=
+github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
 github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
 github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=