diff --git a/cmd/chart-tracker/dispatcher.go b/cmd/chart-tracker/dispatcher.go index fbf98bec..87c1c34c 100644 --- a/cmd/chart-tracker/dispatcher.go +++ b/cmd/chart-tracker/dispatcher.go @@ -32,7 +32,7 @@ type Job struct { Kind JobKind Repo *hub.ChartRepository ChartVersion *repo.ChartVersion - DownloadLogo bool + GetLogo bool } // Dispatcher is in charge of generating jobs to register or unregister charts @@ -105,9 +105,9 @@ func (d *Dispatcher) generateSyncJobs(wg *sync.WaitGroup, r *hub.ChartRepository chartsAvailable := make(map[string]struct{}) for _, chartVersions := range indexFile.Entries { for i, chartVersion := range chartVersions { - var downloadLogo bool + var getLogo bool if i == 0 { - downloadLogo = true + getLogo = true } key := fmt.Sprintf("%s@%s", chartVersion.Metadata.Name, chartVersion.Metadata.Version) chartsAvailable[key] = struct{}{} @@ -116,7 +116,7 @@ func (d *Dispatcher) generateSyncJobs(wg *sync.WaitGroup, r *hub.ChartRepository Kind: Register, Repo: r, ChartVersion: chartVersion, - DownloadLogo: downloadLogo, + GetLogo: getLogo, } } select { diff --git a/cmd/chart-tracker/dispatcher_test.go b/cmd/chart-tracker/dispatcher_test.go index 0790f929..097a7096 100644 --- a/cmd/chart-tracker/dispatcher_test.go +++ b/cmd/chart-tracker/dispatcher_test.go @@ -114,7 +114,7 @@ func TestDispatcher(t *testing.T) { Kind: Register, Repo: repo1, ChartVersion: pkg1V1, - DownloadLogo: true, + GetLogo: true, }, }, }, @@ -139,13 +139,13 @@ func TestDispatcher(t *testing.T) { Kind: Register, Repo: repo1, ChartVersion: pkg1V1, - DownloadLogo: true, + GetLogo: true, }, { Kind: Register, Repo: repo1, ChartVersion: pkg1V2, - DownloadLogo: false, + GetLogo: false, }, }, }, @@ -219,7 +219,7 @@ func TestDispatcher(t *testing.T) { Kind: Register, Repo: repo1, ChartVersion: pkg1V2, - DownloadLogo: false, + GetLogo: false, }, }, }, @@ -252,7 +252,7 @@ func TestDispatcher(t *testing.T) { Kind: Register, Repo: repo1, ChartVersion: pkg2V1, - DownloadLogo: true, + GetLogo: true, }, }, }, @@ -288,25 +288,25 @@ func TestDispatcher(t *testing.T) { Kind: Register, Repo: repo1, ChartVersion: pkg1V1, - DownloadLogo: true, + GetLogo: true, }, { Kind: Register, Repo: repo1, ChartVersion: pkg1V2, - DownloadLogo: false, + GetLogo: false, }, { Kind: Register, Repo: repo1, ChartVersion: pkg2V1, - DownloadLogo: true, + GetLogo: true, }, { Kind: Register, Repo: repo2, ChartVersion: pkg3V1, - DownloadLogo: true, + GetLogo: true, }, }, }, diff --git a/cmd/chart-tracker/testdata/pkg2-1.0.0.tgz b/cmd/chart-tracker/testdata/pkg2-1.0.0.tgz new file mode 100644 index 00000000..a91b439f Binary files /dev/null and b/cmd/chart-tracker/testdata/pkg2-1.0.0.tgz differ diff --git a/cmd/chart-tracker/testdata/red-dot.png b/cmd/chart-tracker/testdata/red-dot.png new file mode 100644 index 00000000..7a3361fb Binary files /dev/null and b/cmd/chart-tracker/testdata/red-dot.png differ diff --git a/cmd/chart-tracker/worker.go b/cmd/chart-tracker/worker.go index d95051c5..d031486a 100644 --- a/cmd/chart-tracker/worker.go +++ b/cmd/chart-tracker/worker.go @@ -10,12 +10,14 @@ import ( "net/url" "path" "runtime/debug" + "strings" "sync" "github.com/artifacthub/hub/internal/hub" "github.com/artifacthub/hub/internal/img" "github.com/rs/zerolog" "github.com/rs/zerolog/log" + "github.com/vincent-petithory/dataurl" "helm.sh/helm/v3/pkg/chart" "helm.sh/helm/v3/pkg/chart/loader" ) @@ -96,8 +98,19 @@ func (w *Worker) Run(wg *sync.WaitGroup, queue chan *Job) { } } -// downloadImage downloads the image located at the url provided. -func (w *Worker) downloadImage(u string) ([]byte, error) { +// getImage gets the image located at the url provided. If it's a data url the +// image is extracted from it. Otherwise it's downloaded using the url. +func (w *Worker) getImage(u string) ([]byte, error) { + // Image in data url + if strings.HasPrefix(u, "data:") { + dataURL, err := dataurl.DecodeString(u) + if err != nil { + return nil, err + } + return dataURL.Data, nil + } + + // Download image using url provided resp, err := w.hg.Get(u) if err != nil { return nil, err @@ -154,13 +167,13 @@ func (w *Worker) handleRegisterJob(j *Job) error { // Store chart logo when available if requested var logoURL, logoImageID string - if j.DownloadLogo { + if j.GetLogo { if md.Icon != "" { logoURL = md.Icon - data, err := w.downloadImage(md.Icon) + data, err := w.getImage(md.Icon) if err != nil { - w.ec.Append(j.Repo.ChartRepositoryID, fmt.Errorf("error dowloading logo %s: %w", md.Icon, err)) - w.logger.Debug().Err(err).Str("url", md.Icon).Msg("image download failed") + w.ec.Append(j.Repo.ChartRepositoryID, fmt.Errorf("error getting logo image %s: %w", md.Icon, err)) + w.logger.Debug().Err(err).Str("url", md.Icon).Msg("get image failed") } else { logoImageID, err = w.is.SaveImage(w.ctx, data) if err != nil && !errors.Is(err, image.ErrFormat) { diff --git a/cmd/chart-tracker/worker_test.go b/cmd/chart-tracker/worker_test.go index 44e40dd7..d7fe009b 100644 --- a/cmd/chart-tracker/worker_test.go +++ b/cmd/chart-tracker/worker_test.go @@ -33,11 +33,20 @@ func TestWorker(t *testing.T) { "http://tests/pkg1-1.0.0.tgz", }, } + pkg2V1 := &repo.ChartVersion{ + Metadata: &chart.Metadata{ + Name: "pkg2", + Version: "1.0.0", + }, + URLs: []string{ + "http://tests/pkg2-1.0.0.tgz", + }, + } job := &Job{ Kind: Register, Repo: repo1, ChartVersion: pkg1V1, - DownloadLogo: true, + GetLogo: true, } t.Run("error downloading chart", func(t *testing.T) { @@ -176,6 +185,31 @@ func TestWorker(t *testing.T) { ww.w.Run(ww.wg, ww.queue) ww.assertExpectations(t) }) + + t.Run("package with logo in data url registered successfully", func(t *testing.T) { + // Setup worker and expectations + ww := newWorkerWrapper(context.Background()) + job := &Job{ + Kind: Register, + Repo: repo1, + ChartVersion: pkg2V1, + GetLogo: true, + } + ww.queue <- job + close(ww.queue) + f, _ := os.Open("testdata/" + path.Base(job.ChartVersion.URLs[0])) + ww.hg.On("Get", job.ChartVersion.URLs[0]).Return(&http.Response{ + Body: f, + StatusCode: http.StatusOK, + }, nil) + expectedLogoData, _ := ioutil.ReadFile("testdata/red-dot.png") + ww.is.On("SaveImage", mock.Anything, expectedLogoData).Return("imageID", nil) + ww.pm.On("Register", mock.Anything, mock.Anything).Return(nil) + + // Run worker and check expectations + ww.w.Run(ww.wg, ww.queue) + ww.assertExpectations(t) + }) }) t.Run("handle unregister job", func(t *testing.T) { diff --git a/go.mod b/go.mod index e26261aa..5d8d6dfb 100644 --- a/go.mod +++ b/go.mod @@ -34,6 +34,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.6.3 github.com/stretchr/testify v1.5.1 + github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 golang.org/x/crypto v0.0.0-20200422194213-44a606286825 golang.org/x/image v0.0.0-20200119044424-58c23975cae1 // indirect golang.org/x/net v0.0.0-20200421231249-e086a090c8fd // indirect diff --git a/go.sum b/go.sum index 6d216548..f806a988 100644 --- a/go.sum +++ b/go.sum @@ -625,6 +625,8 @@ github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljT github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= +github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50 h1:uxE3GYdXIOfhMv3unJKETJEhw78gvzuQqRX/rVirc2A= +github.com/vincent-petithory/dataurl v0.0.0-20191104211930-d1553a71de50/go.mod h1:FHafX5vmDzyP+1CQATJn7WFKc9CvnvxyvZy6I1MrG/U= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ= github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs=