Add endpoint for Harbor replication adapter (#1001)

Closes #997

Signed-off-by: Sergio Castaño Arteaga <tegioz@icloud.com>

cmd/hub/handlers/handlers.go

@@ -272,6 +272,15 @@ func (h *Handlers) setupRouter() {
 
 		// Images
 		r.With(h.Users.RequireLogin).Post("/images", h.Static.SaveImage)
+
+		// Harbor replication
+		//
+		// This endpoint is used by the Harbor replication Artifact Hub adapter.
+		// It returns some information about all packages versions of Helm kind
+		// available so that they can be synchronized in Harbor deployments. It
+		// will probably start being used in Harbor 2.2.0, so we need to be
+		// careful to not introduce breaking changes.
+		r.Get("/harborReplication", h.Packages.GetHarborReplicationDump)
 	})
 
 	// Monocular compatible search API

cmd/hub/handlers/pkg/handlers.go

@@ -67,6 +67,19 @@ func (h *Handlers) GetChangeLog(w http.ResponseWriter, r *http.Request) {
 	helpers.RenderJSON(w, dataJSON, helpers.DefaultAPICacheMaxAge, http.StatusOK)
 }
 
+// GetHarborReplicationDump is an http handler used to get a summary of all
+// available packages versions of kind Helm in the hub database so that they
+// can be synchronized in Harbor.
+func (h *Handlers) GetHarborReplicationDump(w http.ResponseWriter, r *http.Request) {
+	dataJSON, err := h.pkgManager.GetHarborReplicationDumpJSON(r.Context())
+	if err != nil {
+		h.logger.Error().Err(err).Str("method", "GetHarborReplicationDump").Send()
+		helpers.RenderErrorJSON(w, err)
+		return
+	}
+	helpers.RenderJSON(w, dataJSON, 1*time.Hour, http.StatusOK)
+}
+
 // GetRandom is an http handler used to get some random packages from the hub
 // database.
 func (h *Handlers) GetRandom(w http.ResponseWriter, r *http.Request) {

cmd/hub/handlers/pkg/handlers_test.go

@@ -134,6 +134,43 @@ func TestGetChangeLog(t *testing.T) {
 	})
 }
 
+func TestGetHarborReplicationDump(t *testing.T) {
+	t.Run("get harbor replication dump succeeded", func(t *testing.T) {
+		t.Parallel()
+		w := httptest.NewRecorder()
+		r, _ := http.NewRequest("GET", "/", nil)
+
+		hw := newHandlersWrapper()
+		hw.pm.On("GetHarborReplicationDumpJSON", r.Context()).Return([]byte("dataJSON"), nil)
+		hw.h.GetHarborReplicationDump(w, r)
+		resp := w.Result()
+		defer resp.Body.Close()
+		h := resp.Header
+		data, _ := ioutil.ReadAll(resp.Body)
+
+		assert.Equal(t, http.StatusOK, resp.StatusCode)
+		assert.Equal(t, "application/json", h.Get("Content-Type"))
+		assert.Equal(t, helpers.BuildCacheControlHeader(1*time.Hour), h.Get("Cache-Control"))
+		assert.Equal(t, []byte("dataJSON"), data)
+		hw.pm.AssertExpectations(t)
+	})
+
+	t.Run("error getting harbor replication dump", func(t *testing.T) {
+		t.Parallel()
+		w := httptest.NewRecorder()
+		r, _ := http.NewRequest("GET", "/", nil)
+
+		hw := newHandlersWrapper()
+		hw.pm.On("GetHarborReplicationDumpJSON", r.Context()).Return(nil, tests.ErrFakeDB)
+		hw.h.GetHarborReplicationDump(w, r)
+		resp := w.Result()
+		defer resp.Body.Close()
+
+		assert.Equal(t, http.StatusInternalServerError, resp.StatusCode)
+		hw.pm.AssertExpectations(t)
+	})
+}
+
 func TestGetRandom(t *testing.T) {
 	t.Run("get random packages succeeded", func(t *testing.T) {
 		t.Parallel()

database/migrations/functions/001_load_functions.sql

@@ -30,6 +30,7 @@
 {{ template "organizations/user_belongs_to_organization.sql" }}
 
 {{ template "packages/generate_package_tsdoc.sql" }}
+{{ template "packages/get_harbor_replication_dump.sql" }}
 {{ template "packages/get_package.sql" }}
 {{ template "packages/get_package_changelog.sql" }}
 {{ template "packages/get_package_summary.sql" }}

database/migrations/functions/packages/get_harbor_replication_dump.sql

@@ -0,0 +1,17 @@
+-- get_harbor_replication_dump returns a json list with all packages versions
+-- of kind Helm available so that they can be synchronized in Harbor.
+create or replace function get_harbor_replication_dump()
+returns setof json as $$
+    select coalesce(json_agg(json_build_object(
+        'repository', r.name,
+        'package', p.normalized_name,
+        'version', s.version,
+        'url', s.content_url
+    )), '[]')
+    from package p
+    join repository r using (repository_id)
+    join snapshot s using (package_id)
+    where r.repository_kind_id = 0
+    and (s.deprecated is null or s.deprecated = false)
+    and s.content_url is not null;
+$$ language sql;

database/tests/functions/packages/get_harbor_replication_dump.sql

@@ -0,0 +1,132 @@
+-- Start transaction and plan tests
+begin;
+select plan(2);
+
+-- Declare some variables
+\set org1ID '00000000-0000-0000-0000-000000000001'
+\set repo1ID '00000000-0000-0000-0000-000000000001'
+\set repo2ID '00000000-0000-0000-0000-000000000002'
+\set repo3ID '00000000-0000-0000-0000-000000000003'
+\set package1ID '00000000-0000-0000-0000-000000000001'
+\set package2ID '00000000-0000-0000-0000-000000000002'
+\set package3ID '00000000-0000-0000-0000-000000000003'
+\set package4ID '00000000-0000-0000-0000-000000000004'
+
+-- No packages at this point
+select is(
+    get_harbor_replication_dump()::jsonb,
+    '[]'::jsonb,
+    'No packages in db yet, empty dump expected'
+);
+
+-- Seed some data
+insert into organization (organization_id, name, display_name, description, home_url)
+values (:'org1ID', 'org1', 'Organization 1', 'Description 1', 'https://org1.com');
+insert into repository (repository_id, name, display_name, url, repository_kind_id, organization_id)
+values (:'repo1ID', 'repo1', 'Repo 1', 'https://repo1.com', 0, :'org1ID');
+insert into repository (repository_id, name, display_name, url, repository_kind_id, organization_id)
+values (:'repo2ID', 'repo2', 'Repo 2', 'https://repo2.com', 0, :'org1ID');
+insert into repository (repository_id, name, display_name, url, repository_kind_id, organization_id)
+values (:'repo3ID', 'repo3', 'Repo 3', 'https://repo3.com', 1, :'org1ID');
+insert into package (
+    package_id,
+    name,
+    latest_version,
+    repository_id
+) values (
+    :'package1ID',
+    'package1',
+    '1.0.0',
+    :'repo1ID'
+);
+insert into snapshot (
+    package_id,
+    version,
+    content_url
+) values (
+    :'package1ID',
+    '1.0.0',
+    'package1_1.0.0_url'
+);
+insert into package (
+    package_id,
+    name,
+    latest_version,
+    repository_id
+) values (
+    :'package2ID',
+    'package2',
+    '1.0.0',
+    :'repo2ID'
+);
+insert into snapshot (
+    package_id,
+    version,
+    content_url
+) values (
+    :'package2ID',
+    '1.0.0',
+    'package2_1.0.0_url'
+);
+insert into package (
+    package_id,
+    name,
+    latest_version,
+    repository_id
+) values (
+    :'package3ID',
+    'package3',
+    '1.0.0',
+    :'repo2ID'
+);
+insert into snapshot (
+    package_id,
+    version
+) values (
+    :'package3ID',
+    '1.0.0'
+);
+insert into package (
+    package_id,
+    name,
+    latest_version,
+    repository_id
+) values (
+    :'package4ID',
+    'package4',
+    '1.0.0',
+    :'repo3ID'
+);
+insert into snapshot (
+    package_id,
+    version,
+    content_url
+) values (
+    :'package4ID',
+    '1.0.0',
+    'package4_1.0.0_url'
+);
+
+-- Run some tests
+select is(
+    get_harbor_replication_dump()::jsonb,
+    '[
+        {
+            "repository": "repo1",
+            "package": "package1",
+            "version": "1.0.0",
+            "url": "package1_1.0.0_url"
+        },
+        {
+            "repository": "repo2",
+            "package": "package2",
+            "version": "1.0.0",
+            "url": "package2_1.0.0_url"
+        }
+    ]'::jsonb,
+    'Two packages expected in dump'
+);
+
+-- Finish tests and rollback transaction
+select * from finish();
+rollback;

database/tests/schema/schema.sql

@@ -1,6 +1,6 @@
 -- Start transaction and plan tests
 begin;
-select plan(131);
+select plan(132);
 
 -- Check default_text_search_config is correct
 select results_eq(
@@ -372,6 +372,7 @@ select has_function('update_organization');
 select has_function('user_belongs_to_organization');
 -- Packages
 select has_function('generate_package_tsdoc');
+select has_function('get_harbor_replication_dump');
 select has_function('get_package');
 select has_function('get_package_changelog');
 select has_function('get_package_summary');

docs/api/openapi.yaml

@@ -25,6 +25,8 @@ tags:
     description: ""
   - name: Availability checks
     description: ""
+  - name: Integrations
+    description: ""
 paths:
   /users:
     post:
@@ -1841,6 +1843,84 @@ paths:
           $ref: "#/components/responses/TooManyRequests"
         "500":
           $ref: "#/components/responses/InternalServerError"
+  /harborReplication:
+    get:
+      tags:
+        - Integrations
+      summary: Get Harbor replication dump
+      responses:
+        "200":
+          description: ""
+          content:
+            application/json:
+              schema:
+                type: array
+                items:
+                  type: object
+                  required:
+                    - repository
+                    - package
+                    - version
+                    - url
+                properties:
+                  repository:
+                    type: string
+                    nullable: false
+                  package:
+                    type: string
+                    nullable: false
+                  version:
+                    type: string
+                    nullable: false
+                  url:
+                    type: string
+                    format: uri
+                    nullable: false
+                example:
+                  - repository: bitnami
+                    package: nginx-ingress-controller
+                    version: 5.6.10
+                    url: https://charts.bitnami.com/bitnami/nginx-ingress-controller-5.6.10.tgz
+                  - repository: bitnami
+                    package: mediawiki
+                    version: 6.3.6
+                    url: https://charts.bitnami.com/bitnami/mediawiki-6.3.6.tgz
+                  - repository: crossplane
+                    package: crossplane
+                    version: 0.0.0-793.c9c71f8
+                    url: https://charts.crossplane.io/master/crossplane-0.0.0-793.c9c71f8.tgz
+                  - repository: choerodon
+                    package: devops-service
+                    version: 0.22.0
+                    url: https://openchart.choerodon.com.cn/choerodon/c7n/charts/devops-service-0.22.0.tgz
+                  - repository: t3n
+                    package: snipeit
+                    version: 2.4.0
+                    url: https://storage.googleapis.com/t3n-helm-charts/snipeit-2.4.0.tgz
+                  - repository: bitnami
+                    package: osclass
+                    version: 7.0.10
+                    url: https://charts.bitnami.com/bitnami/osclass-7.0.10.tgz
+                  - repository: choerodon
+                    package: asgard-service
+                    version: 0.13.0
+                    url: https://openchart.choerodon.com.cn/choerodon/c7n/charts/asgard-service-0.13.0.tgz
+                  - repository: bitnami
+                    package: osclass
+                    version: 3.2.0
+                    url: https://charts.bitnami.com/bitnami/osclass-3.2.0.tgz
+                  - repository: cronce
+                    package: torrentor
+                    version: 0.3.0
+                    url: https://charts.cronce.io/charts/torrentor-0.3.0.tgz
+                  - repository: banzaicloud-stable
+                    package: kafka-operator
+                    version: 0.0.12
+                    url: https://kubernetes-charts.banzaicloud.com/charts/kafka-operator-0.0.12.tgz
+        "429":
+          $ref: "#/components/responses/TooManyRequests"
+        "500":
+          $ref: "#/components/responses/InternalServerError"
 components:
   securitySchemes:
     ApiKeyAuth:

internal/hub/pkg.go

@@ -94,6 +94,7 @@ type Package struct {
 type PackageManager interface {
 	Get(ctx context.Context, input *GetPackageInput) (*Package, error)
 	GetChangeLogJSON(ctx context.Context, pkgID string) ([]byte, error)
+	GetHarborReplicationDumpJSON(ctx context.Context) ([]byte, error)
 	GetJSON(ctx context.Context, input *GetPackageInput) ([]byte, error)
 	GetRandomJSON(ctx context.Context) ([]byte, error)
 	GetSnapshotSecurityReportJSON(ctx context.Context, pkgID, version string) ([]byte, error)

internal/pkg/manager.go

@@ -15,6 +15,7 @@ import (
 
 const (
 	// Database queries
+	getHarborReplicationDumpDBQ     = `select get_harbor_replication_dump()`
 	getPkgDBQ                       = `select get_package($1::jsonb)`
 	getPkgChangeLogDBQ              = `select get_package_changelog($1::uuid)`
 	getPkgStarsDBQ                  = `select get_package_stars($1::uuid, $2::uuid)`
@@ -73,6 +74,12 @@ func (m *Manager) GetChangeLogJSON(ctx context.Context, pkgID string) ([]byte, e
 	return util.DBQueryJSON(ctx, m.db, getPkgChangeLogDBQ, pkgID)
 }
 
+// GetHarborReplicationDumpJSON returns a json list with all packages versions
+// of kind Helm available so that they can be synchronized in Harbor.
+func (m *Manager) GetHarborReplicationDumpJSON(ctx context.Context) ([]byte, error) {
+	return util.DBQueryJSON(ctx, m.db, getHarborReplicationDumpDBQ)
+}
+
 // GetJSON returns the package identified by the input provided as a json
 // object. The json object is built by the database.
 func (m *Manager) GetJSON(ctx context.Context, input *hub.GetPackageInput) ([]byte, error) {

internal/pkg/manager_test.go

@@ -260,6 +260,34 @@ func TestGetChangeLogJSON(t *testing.T) {
 	})
 }
 
+func TestGetHarborReplicationDumpJSON(t *testing.T) {
+	ctx := context.Background()
+
+	t.Run("database query succeeded", func(t *testing.T) {
+		t.Parallel()
+		db := &tests.DBMock{}
+		db.On("QueryRow", ctx, getHarborReplicationDumpDBQ).Return([]byte("dataJSON"), nil)
+		m := NewManager(db)
+
+		dataJSON, err := m.GetHarborReplicationDumpJSON(ctx)
+		assert.NoError(t, err)
+		assert.Equal(t, []byte("dataJSON"), dataJSON)
+		db.AssertExpectations(t)
+	})
+
+	t.Run("database error", func(t *testing.T) {
+		t.Parallel()
+		db := &tests.DBMock{}
+		db.On("QueryRow", ctx, getHarborReplicationDumpDBQ).Return(nil, tests.ErrFakeDB)
+		m := NewManager(db)
+
+		dataJSON, err := m.GetHarborReplicationDumpJSON(ctx)
+		assert.Equal(t, tests.ErrFakeDB, err)
+		assert.Nil(t, dataJSON)
+		db.AssertExpectations(t)
+	})
+}
+
 func TestGetJSON(t *testing.T) {
 	ctx := context.Background()
 

internal/pkg/mock.go

@@ -26,6 +26,13 @@ func (m *ManagerMock) GetChangeLogJSON(ctx context.Context, pkgID string) ([]byt
 	return data, args.Error(1)
 }
 
+// GetHarborReplicationDumpJSON implements the PackageManager interface.
+func (m *ManagerMock) GetHarborReplicationDumpJSON(ctx context.Context) ([]byte, error) {
+	args := m.Called(ctx)
+	data, _ := args.Get(0).([]byte)
+	return data, args.Error(1)
+}
+
 // GetJSON implements the PackageManager interface.
 func (m *ManagerMock) GetJSON(ctx context.Context, input *hub.GetPackageInput) ([]byte, error) {
 	args := m.Called(ctx, input)