From 617105b00c49fcc2cf11de5485db42c98d27b87c Mon Sep 17 00:00:00 2001 From: "Sergio C. Arteaga" Date: Tue, 5 Jan 2021 16:23:46 +0100 Subject: [PATCH] Add endpoint for Harbor replication adapter (#1001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Closes #997 Signed-off-by: Sergio Castaño Arteaga --- cmd/hub/handlers/handlers.go | 9 ++ cmd/hub/handlers/pkg/handlers.go | 13 ++ cmd/hub/handlers/pkg/handlers_test.go | 37 +++++ .../functions/001_load_functions.sql | 1 + .../packages/get_harbor_replication_dump.sql | 17 +++ .../packages/get_harbor_replication_dump.sql | 132 ++++++++++++++++++ database/tests/schema/schema.sql | 3 +- docs/api/openapi.yaml | 80 +++++++++++ internal/hub/pkg.go | 1 + internal/pkg/manager.go | 7 + internal/pkg/manager_test.go | 28 ++++ internal/pkg/mock.go | 7 + 12 files changed, 334 insertions(+), 1 deletion(-) create mode 100644 database/migrations/functions/packages/get_harbor_replication_dump.sql create mode 100644 database/tests/functions/packages/get_harbor_replication_dump.sql diff --git a/cmd/hub/handlers/handlers.go b/cmd/hub/handlers/handlers.go index 266e85dc..d6e494ae 100644 --- a/cmd/hub/handlers/handlers.go +++ b/cmd/hub/handlers/handlers.go @@ -272,6 +272,15 @@ func (h *Handlers) setupRouter() { // Images r.With(h.Users.RequireLogin).Post("/images", h.Static.SaveImage) + + // Harbor replication + // + // This endpoint is used by the Harbor replication Artifact Hub adapter. + // It returns some information about all packages versions of Helm kind + // available so that they can be synchronized in Harbor deployments. It + // will probably start being used in Harbor 2.2.0, so we need to be + // careful to not introduce breaking changes. + r.Get("/harborReplication", h.Packages.GetHarborReplicationDump) }) // Monocular compatible search API diff --git a/cmd/hub/handlers/pkg/handlers.go b/cmd/hub/handlers/pkg/handlers.go index 32654d4a..4b126679 100644 --- a/cmd/hub/handlers/pkg/handlers.go +++ b/cmd/hub/handlers/pkg/handlers.go @@ -67,6 +67,19 @@ func (h *Handlers) GetChangeLog(w http.ResponseWriter, r *http.Request) { helpers.RenderJSON(w, dataJSON, helpers.DefaultAPICacheMaxAge, http.StatusOK) } +// GetHarborReplicationDump is an http handler used to get a summary of all +// available packages versions of kind Helm in the hub database so that they +// can be synchronized in Harbor. +func (h *Handlers) GetHarborReplicationDump(w http.ResponseWriter, r *http.Request) { + dataJSON, err := h.pkgManager.GetHarborReplicationDumpJSON(r.Context()) + if err != nil { + h.logger.Error().Err(err).Str("method", "GetHarborReplicationDump").Send() + helpers.RenderErrorJSON(w, err) + return + } + helpers.RenderJSON(w, dataJSON, 1*time.Hour, http.StatusOK) +} + // GetRandom is an http handler used to get some random packages from the hub // database. func (h *Handlers) GetRandom(w http.ResponseWriter, r *http.Request) { diff --git a/cmd/hub/handlers/pkg/handlers_test.go b/cmd/hub/handlers/pkg/handlers_test.go index 8f70b964..2feef141 100644 --- a/cmd/hub/handlers/pkg/handlers_test.go +++ b/cmd/hub/handlers/pkg/handlers_test.go @@ -134,6 +134,43 @@ func TestGetChangeLog(t *testing.T) { }) } +func TestGetHarborReplicationDump(t *testing.T) { + t.Run("get harbor replication dump succeeded", func(t *testing.T) { + t.Parallel() + w := httptest.NewRecorder() + r, _ := http.NewRequest("GET", "/", nil) + + hw := newHandlersWrapper() + hw.pm.On("GetHarborReplicationDumpJSON", r.Context()).Return([]byte("dataJSON"), nil) + hw.h.GetHarborReplicationDump(w, r) + resp := w.Result() + defer resp.Body.Close() + h := resp.Header + data, _ := ioutil.ReadAll(resp.Body) + + assert.Equal(t, http.StatusOK, resp.StatusCode) + assert.Equal(t, "application/json", h.Get("Content-Type")) + assert.Equal(t, helpers.BuildCacheControlHeader(1*time.Hour), h.Get("Cache-Control")) + assert.Equal(t, []byte("dataJSON"), data) + hw.pm.AssertExpectations(t) + }) + + t.Run("error getting harbor replication dump", func(t *testing.T) { + t.Parallel() + w := httptest.NewRecorder() + r, _ := http.NewRequest("GET", "/", nil) + + hw := newHandlersWrapper() + hw.pm.On("GetHarborReplicationDumpJSON", r.Context()).Return(nil, tests.ErrFakeDB) + hw.h.GetHarborReplicationDump(w, r) + resp := w.Result() + defer resp.Body.Close() + + assert.Equal(t, http.StatusInternalServerError, resp.StatusCode) + hw.pm.AssertExpectations(t) + }) +} + func TestGetRandom(t *testing.T) { t.Run("get random packages succeeded", func(t *testing.T) { t.Parallel() diff --git a/database/migrations/functions/001_load_functions.sql b/database/migrations/functions/001_load_functions.sql index 48b3d79d..88284fee 100644 --- a/database/migrations/functions/001_load_functions.sql +++ b/database/migrations/functions/001_load_functions.sql @@ -30,6 +30,7 @@ {{ template "organizations/user_belongs_to_organization.sql" }} {{ template "packages/generate_package_tsdoc.sql" }} +{{ template "packages/get_harbor_replication_dump.sql" }} {{ template "packages/get_package.sql" }} {{ template "packages/get_package_changelog.sql" }} {{ template "packages/get_package_summary.sql" }} diff --git a/database/migrations/functions/packages/get_harbor_replication_dump.sql b/database/migrations/functions/packages/get_harbor_replication_dump.sql new file mode 100644 index 00000000..0a384176 --- /dev/null +++ b/database/migrations/functions/packages/get_harbor_replication_dump.sql @@ -0,0 +1,17 @@ +-- get_harbor_replication_dump returns a json list with all packages versions +-- of kind Helm available so that they can be synchronized in Harbor. +create or replace function get_harbor_replication_dump() +returns setof json as $$ + select coalesce(json_agg(json_build_object( + 'repository', r.name, + 'package', p.normalized_name, + 'version', s.version, + 'url', s.content_url + )), '[]') + from package p + join repository r using (repository_id) + join snapshot s using (package_id) + where r.repository_kind_id = 0 + and (s.deprecated is null or s.deprecated = false) + and s.content_url is not null; +$$ language sql; diff --git a/database/tests/functions/packages/get_harbor_replication_dump.sql b/database/tests/functions/packages/get_harbor_replication_dump.sql new file mode 100644 index 00000000..ac61f3d7 --- /dev/null +++ b/database/tests/functions/packages/get_harbor_replication_dump.sql @@ -0,0 +1,132 @@ +-- Start transaction and plan tests +begin; +select plan(2); + +-- Declare some variables +\set org1ID '00000000-0000-0000-0000-000000000001' +\set repo1ID '00000000-0000-0000-0000-000000000001' +\set repo2ID '00000000-0000-0000-0000-000000000002' +\set repo3ID '00000000-0000-0000-0000-000000000003' +\set package1ID '00000000-0000-0000-0000-000000000001' +\set package2ID '00000000-0000-0000-0000-000000000002' +\set package3ID '00000000-0000-0000-0000-000000000003' +\set package4ID '00000000-0000-0000-0000-000000000004' + +-- No packages at this point +select is( + get_harbor_replication_dump()::jsonb, + '[]'::jsonb, + 'No packages in db yet, empty dump expected' +); + +-- Seed some data +insert into organization (organization_id, name, display_name, description, home_url) +values (:'org1ID', 'org1', 'Organization 1', 'Description 1', 'https://org1.com'); +insert into repository (repository_id, name, display_name, url, repository_kind_id, organization_id) +values (:'repo1ID', 'repo1', 'Repo 1', 'https://repo1.com', 0, :'org1ID'); +insert into repository (repository_id, name, display_name, url, repository_kind_id, organization_id) +values (:'repo2ID', 'repo2', 'Repo 2', 'https://repo2.com', 0, :'org1ID'); +insert into repository (repository_id, name, display_name, url, repository_kind_id, organization_id) +values (:'repo3ID', 'repo3', 'Repo 3', 'https://repo3.com', 1, :'org1ID'); +insert into package ( + package_id, + name, + latest_version, + repository_id +) values ( + :'package1ID', + 'package1', + '1.0.0', + :'repo1ID' +); +insert into snapshot ( + package_id, + version, + content_url +) values ( + :'package1ID', + '1.0.0', + 'package1_1.0.0_url' +); +insert into package ( + package_id, + name, + latest_version, + repository_id +) values ( + :'package2ID', + 'package2', + '1.0.0', + :'repo2ID' +); +insert into snapshot ( + package_id, + version, + content_url +) values ( + :'package2ID', + '1.0.0', + 'package2_1.0.0_url' +); +insert into package ( + package_id, + name, + latest_version, + repository_id +) values ( + :'package3ID', + 'package3', + '1.0.0', + :'repo2ID' +); +insert into snapshot ( + package_id, + version +) values ( + :'package3ID', + '1.0.0' +); +insert into package ( + package_id, + name, + latest_version, + repository_id +) values ( + :'package4ID', + 'package4', + '1.0.0', + :'repo3ID' +); +insert into snapshot ( + package_id, + version, + content_url +) values ( + :'package4ID', + '1.0.0', + 'package4_1.0.0_url' +); + +-- Run some tests +select is( + get_harbor_replication_dump()::jsonb, + '[ + { + "repository": "repo1", + "package": "package1", + "version": "1.0.0", + "url": "package1_1.0.0_url" + }, + { + "repository": "repo2", + "package": "package2", + "version": "1.0.0", + "url": "package2_1.0.0_url" + } + ]'::jsonb, + 'Two packages expected in dump' +); + +-- Finish tests and rollback transaction +select * from finish(); +rollback; diff --git a/database/tests/schema/schema.sql b/database/tests/schema/schema.sql index 91442b4d..efbd6147 100644 --- a/database/tests/schema/schema.sql +++ b/database/tests/schema/schema.sql @@ -1,6 +1,6 @@ -- Start transaction and plan tests begin; -select plan(131); +select plan(132); -- Check default_text_search_config is correct select results_eq( @@ -372,6 +372,7 @@ select has_function('update_organization'); select has_function('user_belongs_to_organization'); -- Packages select has_function('generate_package_tsdoc'); +select has_function('get_harbor_replication_dump'); select has_function('get_package'); select has_function('get_package_changelog'); select has_function('get_package_summary'); diff --git a/docs/api/openapi.yaml b/docs/api/openapi.yaml index fdaecfa3..730f5fb5 100644 --- a/docs/api/openapi.yaml +++ b/docs/api/openapi.yaml @@ -25,6 +25,8 @@ tags: description: "" - name: Availability checks description: "" + - name: Integrations + description: "" paths: /users: post: @@ -1841,6 +1843,84 @@ paths: $ref: "#/components/responses/TooManyRequests" "500": $ref: "#/components/responses/InternalServerError" + /harborReplication: + get: + tags: + - Integrations + summary: Get Harbor replication dump + responses: + "200": + description: "" + content: + application/json: + schema: + type: array + items: + type: object + required: + - repository + - package + - version + - url + properties: + repository: + type: string + nullable: false + package: + type: string + nullable: false + version: + type: string + nullable: false + url: + type: string + format: uri + nullable: false + example: + - repository: bitnami + package: nginx-ingress-controller + version: 5.6.10 + url: https://charts.bitnami.com/bitnami/nginx-ingress-controller-5.6.10.tgz + - repository: bitnami + package: mediawiki + version: 6.3.6 + url: https://charts.bitnami.com/bitnami/mediawiki-6.3.6.tgz + - repository: crossplane + package: crossplane + version: 0.0.0-793.c9c71f8 + url: https://charts.crossplane.io/master/crossplane-0.0.0-793.c9c71f8.tgz + - repository: choerodon + package: devops-service + version: 0.22.0 + url: https://openchart.choerodon.com.cn/choerodon/c7n/charts/devops-service-0.22.0.tgz + - repository: t3n + package: snipeit + version: 2.4.0 + url: https://storage.googleapis.com/t3n-helm-charts/snipeit-2.4.0.tgz + - repository: bitnami + package: osclass + version: 7.0.10 + url: https://charts.bitnami.com/bitnami/osclass-7.0.10.tgz + - repository: choerodon + package: asgard-service + version: 0.13.0 + url: https://openchart.choerodon.com.cn/choerodon/c7n/charts/asgard-service-0.13.0.tgz + - repository: bitnami + package: osclass + version: 3.2.0 + url: https://charts.bitnami.com/bitnami/osclass-3.2.0.tgz + - repository: cronce + package: torrentor + version: 0.3.0 + url: https://charts.cronce.io/charts/torrentor-0.3.0.tgz + - repository: banzaicloud-stable + package: kafka-operator + version: 0.0.12 + url: https://kubernetes-charts.banzaicloud.com/charts/kafka-operator-0.0.12.tgz + "429": + $ref: "#/components/responses/TooManyRequests" + "500": + $ref: "#/components/responses/InternalServerError" components: securitySchemes: ApiKeyAuth: diff --git a/internal/hub/pkg.go b/internal/hub/pkg.go index 9ff4b5db..82f6e33b 100644 --- a/internal/hub/pkg.go +++ b/internal/hub/pkg.go @@ -94,6 +94,7 @@ type Package struct { type PackageManager interface { Get(ctx context.Context, input *GetPackageInput) (*Package, error) GetChangeLogJSON(ctx context.Context, pkgID string) ([]byte, error) + GetHarborReplicationDumpJSON(ctx context.Context) ([]byte, error) GetJSON(ctx context.Context, input *GetPackageInput) ([]byte, error) GetRandomJSON(ctx context.Context) ([]byte, error) GetSnapshotSecurityReportJSON(ctx context.Context, pkgID, version string) ([]byte, error) diff --git a/internal/pkg/manager.go b/internal/pkg/manager.go index f0201b7e..2215e674 100644 --- a/internal/pkg/manager.go +++ b/internal/pkg/manager.go @@ -15,6 +15,7 @@ import ( const ( // Database queries + getHarborReplicationDumpDBQ = `select get_harbor_replication_dump()` getPkgDBQ = `select get_package($1::jsonb)` getPkgChangeLogDBQ = `select get_package_changelog($1::uuid)` getPkgStarsDBQ = `select get_package_stars($1::uuid, $2::uuid)` @@ -73,6 +74,12 @@ func (m *Manager) GetChangeLogJSON(ctx context.Context, pkgID string) ([]byte, e return util.DBQueryJSON(ctx, m.db, getPkgChangeLogDBQ, pkgID) } +// GetHarborReplicationDumpJSON returns a json list with all packages versions +// of kind Helm available so that they can be synchronized in Harbor. +func (m *Manager) GetHarborReplicationDumpJSON(ctx context.Context) ([]byte, error) { + return util.DBQueryJSON(ctx, m.db, getHarborReplicationDumpDBQ) +} + // GetJSON returns the package identified by the input provided as a json // object. The json object is built by the database. func (m *Manager) GetJSON(ctx context.Context, input *hub.GetPackageInput) ([]byte, error) { diff --git a/internal/pkg/manager_test.go b/internal/pkg/manager_test.go index 30a851ec..c93817d5 100644 --- a/internal/pkg/manager_test.go +++ b/internal/pkg/manager_test.go @@ -260,6 +260,34 @@ func TestGetChangeLogJSON(t *testing.T) { }) } +func TestGetHarborReplicationDumpJSON(t *testing.T) { + ctx := context.Background() + + t.Run("database query succeeded", func(t *testing.T) { + t.Parallel() + db := &tests.DBMock{} + db.On("QueryRow", ctx, getHarborReplicationDumpDBQ).Return([]byte("dataJSON"), nil) + m := NewManager(db) + + dataJSON, err := m.GetHarborReplicationDumpJSON(ctx) + assert.NoError(t, err) + assert.Equal(t, []byte("dataJSON"), dataJSON) + db.AssertExpectations(t) + }) + + t.Run("database error", func(t *testing.T) { + t.Parallel() + db := &tests.DBMock{} + db.On("QueryRow", ctx, getHarborReplicationDumpDBQ).Return(nil, tests.ErrFakeDB) + m := NewManager(db) + + dataJSON, err := m.GetHarborReplicationDumpJSON(ctx) + assert.Equal(t, tests.ErrFakeDB, err) + assert.Nil(t, dataJSON) + db.AssertExpectations(t) + }) +} + func TestGetJSON(t *testing.T) { ctx := context.Background() diff --git a/internal/pkg/mock.go b/internal/pkg/mock.go index 4dceab4a..22d6a4b5 100644 --- a/internal/pkg/mock.go +++ b/internal/pkg/mock.go @@ -26,6 +26,13 @@ func (m *ManagerMock) GetChangeLogJSON(ctx context.Context, pkgID string) ([]byt return data, args.Error(1) } +// GetHarborReplicationDumpJSON implements the PackageManager interface. +func (m *ManagerMock) GetHarborReplicationDumpJSON(ctx context.Context) ([]byte, error) { + args := m.Called(ctx) + data, _ := args.Get(0).([]byte) + return data, args.Error(1) +} + // GetJSON implements the PackageManager interface. func (m *ManagerMock) GetJSON(ctx context.Context, input *hub.GetPackageInput) ([]byte, error) { args := m.Called(ctx, input)