From b13bde9b4967f5c8b02b788a40a89dd4cec5b78a Mon Sep 17 00:00:00 2001 From: Lance Ball Date: Tue, 15 Feb 2022 13:06:42 -0600 Subject: [PATCH] feat: precompile cloudevent schema (#471) * feat: precompile cloudevent schema This commit modifies the build pipleline so that the cloudevent schema is precompiled for runtime validation. This eliminates the need to compile the schema at runtime, improving both performance and security. Fixes: https://github.com/cloudevents/sdk-javascript/issues/423 Signed-off-by: Lance Ball --- .eslintrc | 1 + .gitignore | 1 + package-lock.json | 137 +++++++++++++++++++++++---- package.json | 10 +- src/event/schemas.ts | 86 ----------------- src/event/spec.ts | 17 +--- src/schema/cloudevent.json | 128 +++++++++++++++++++++++++ src/schema/formats.js | 10 ++ test/integration/cloud_event_test.ts | 12 +-- test/integration/sdk_test.ts | 4 +- test/integration/spec_1_tests.ts | 2 +- 11 files changed, 275 insertions(+), 133 deletions(-) delete mode 100644 src/event/schemas.ts create mode 100644 src/schema/cloudevent.json create mode 100644 src/schema/formats.js diff --git a/.eslintrc b/.eslintrc index d515f85..5b0543c 100644 --- a/.eslintrc +++ b/.eslintrc @@ -15,6 +15,7 @@ "plugins": [ "header" ], + "ignorePatterns": ["**/schema/*"], "rules": { "no-var": "error", "standard/no-callback-literal": "off", diff --git a/.gitignore b/.gitignore index 1a0ce74..90afece 100644 --- a/.gitignore +++ b/.gitignore @@ -13,6 +13,7 @@ index.js /bundles /dist /docs +src/schema/v1.js # Runtime data pids diff --git a/package-lock.json b/package-lock.json index 4f63bfa..60955ed 100644 --- a/package-lock.json +++ b/package-lock.json @@ -509,11 +509,29 @@ "strip-json-comments": "^3.1.1" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true } } }, @@ -651,15 +669,6 @@ "defer-to-connect": "^2.0.0" } }, - "@types/ajv": { - "version": "1.0.0", - "resolved": "https://registry.npmjs.org/@types/ajv/-/ajv-1.0.0.tgz", - "integrity": "sha1-T7JEB0Ly9sMOf7B5e4OfxvaWaCo=", - "dev": true, - "requires": { - "ajv": "*" - } - }, "@types/cacheable-request": { "version": "6.0.2", "resolved": "https://registry.npmjs.org/@types/cacheable-request/-/cacheable-request-6.0.2.tgz", @@ -1162,16 +1171,40 @@ } }, "ajv": { - "version": "6.12.6", - "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", - "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "version": "8.10.0", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-8.10.0.tgz", + "integrity": "sha512-bzqAEZOjkrUMl2afH8dknrq5KEk2SrwdBROR+vH1EKVQTqaUbJVPdc/gEdggTMM0Se+s+Ja4ju4TlNcStKl2Hw==", "requires": { "fast-deep-equal": "^3.1.1", - "fast-json-stable-stringify": "^2.0.0", - "json-schema-traverse": "^0.4.1", + "json-schema-traverse": "^1.0.0", + "require-from-string": "^2.0.2", "uri-js": "^4.2.2" } }, + "ajv-cli": { + "version": "5.0.0", + "resolved": "https://registry.npmjs.org/ajv-cli/-/ajv-cli-5.0.0.tgz", + "integrity": "sha512-LY4m6dUv44HTyhV+u2z5uX4EhPYTM38Iv1jdgDJJJCyOOuqB8KtZEGjPZ2T+sh5ZIJrXUfgErYx/j3gLd3+PlQ==", + "dev": true, + "requires": { + "ajv": "^8.0.0", + "fast-json-patch": "^2.0.0", + "glob": "^7.1.0", + "js-yaml": "^3.14.0", + "json-schema-migrate": "^2.0.0", + "json5": "^2.1.3", + "minimist": "^1.2.0" + } + }, + "ajv-formats": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ajv-formats/-/ajv-formats-2.1.1.tgz", + "integrity": "sha512-Wx0Kx52hxE7C18hkMEggYlEifqWZtYaRgouJor+WMdPnQyEK13vgEWyVNup7SoeeoLMsr4kf5h6dOW11I15MUA==", + "dev": true, + "requires": { + "ajv": "^8.0.0" + } + }, "ajv-keywords": { "version": "3.5.2", "resolved": "https://registry.npmjs.org/ajv-keywords/-/ajv-keywords-3.5.2.tgz", @@ -2040,11 +2073,29 @@ "v8-compile-cache": "^2.0.3" }, "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, "ignore": { "version": "4.0.6", "resolved": "https://registry.npmjs.org/ignore/-/ignore-4.0.6.tgz", "integrity": "sha512-cyFDKrqc/YdcWFniJhzI42+AzS+gNwmUzOSFcRCQYwySuBBBy/KjuxWLZ/FHEH6Moq1NizMOBWyTcv8O4OZIMg==", "dev": true + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true } } }, @@ -2370,10 +2421,28 @@ "micromatch": "^4.0.4" } }, + "fast-json-patch": { + "version": "2.2.1", + "resolved": "https://registry.npmjs.org/fast-json-patch/-/fast-json-patch-2.2.1.tgz", + "integrity": "sha512-4j5uBaTnsYAV5ebkidvxiLUYOwjQ+JSFljeqfTxCrH9bDmlCQaOJFS84oDJ2rAXZq2yskmk3ORfoP9DCwqFNig==", + "dev": true, + "requires": { + "fast-deep-equal": "^2.0.1" + }, + "dependencies": { + "fast-deep-equal": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-2.0.1.tgz", + "integrity": "sha1-ewUhjd+WZ79/Nwv3/bLLFf3Qqkk=", + "dev": true + } + } + }, "fast-json-stable-stringify": { "version": "2.1.0", "resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.1.0.tgz", - "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==" + "integrity": "sha512-lhd/wF+Lk98HZoTCtlVraHtfh5XYijIjalXck7saUtuanSDyLMxnHhSXEDJqHxD7msR8D0uCmqlkwjCV8xvwHw==", + "dev": true }, "fast-levenshtein": { "version": "2.0.6", @@ -3381,10 +3450,19 @@ "integrity": "sha512-xyFwyhro/JEof6Ghe2iz2NcXoj2sloNsWr/XsERDK/oiPCfaNhl5ONfp+jQdAZRQQ0IJWNzH9zIZF7li91kh2w==", "dev": true }, + "json-schema-migrate": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/json-schema-migrate/-/json-schema-migrate-2.0.0.tgz", + "integrity": "sha512-r38SVTtojDRp4eD6WsCqiE0eNDt4v1WalBXb9cyZYw9ai5cGtBwzRNWjHzJl38w6TxFkXAIA7h+fyX3tnrAFhQ==", + "dev": true, + "requires": { + "ajv": "^8.0.0" + } + }, "json-schema-traverse": { - "version": "0.4.1", - "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", - "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==" + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-1.0.0.tgz", + "integrity": "sha512-NM8/P9n3XjXhIZn1lLhkFaACTOURQXjWhV4BA/RnOv8xvgqtqpAX9IO4mRQxSx1Rlo4tqzeqb0sOlruaOy3dug==" }, "json-stable-stringify-without-jsonify": { "version": "1.0.1", @@ -5090,8 +5168,7 @@ "require-from-string": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/require-from-string/-/require-from-string-2.0.2.tgz", - "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==", - "dev": true + "integrity": "sha512-Xf0nWe6RseziFMu+Ap9biiUbmplq6S9/p+7w7YXP/JBHhrUDDUhwa+vANyubuqfZWTveU//DYVGsDG7RKL/vEw==" }, "require-main-filename": { "version": "2.0.0", @@ -5211,6 +5288,26 @@ "@types/json-schema": "^7.0.8", "ajv": "^6.12.5", "ajv-keywords": "^3.5.2" + }, + "dependencies": { + "ajv": { + "version": "6.12.6", + "resolved": "https://registry.npmjs.org/ajv/-/ajv-6.12.6.tgz", + "integrity": "sha512-j3fVLgvTo527anyYyJOGTYJbG+vnnQYvE0m5mmkc1TK+nxAppkCLMIL0aZ4dblVCNoGShhm+kzE4ZUykBoMg4g==", + "dev": true, + "requires": { + "fast-deep-equal": "^3.1.1", + "fast-json-stable-stringify": "^2.0.0", + "json-schema-traverse": "^0.4.1", + "uri-js": "^4.2.2" + } + }, + "json-schema-traverse": { + "version": "0.4.1", + "resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.4.1.tgz", + "integrity": "sha512-xbbCH5dCYU5T8LcEhhuh7HJ88HXuW3qsI3Y0zOZFKfZEHcpWiHU/Jxzk629Brsab/mMiHQti9wMP+845RPe3Vg==", + "dev": true + } } }, "seed-random": { diff --git a/package.json b/package.json index abb8f63..d8787b8 100644 --- a/package.json +++ b/package.json @@ -5,7 +5,10 @@ "main": "dist/index.js", "scripts": { "watch": "tsc --project tsconfig.json --watch", - "build": "tsc --project tsconfig.json && tsc --project tsconfig.browser.json && webpack", + "build:src": "tsc --project tsconfig.json", + "build:browser": "tsc --project tsconfig.browser.json && webpack", + "build:schema": "ajv compile -c ./src/schema/formats.js -s src/schema/cloudevent.json --strict-types false -o src/schema/v1.js", + "build": "npm run build:schema && npm run build:src && npm run build:browser", "lint": "npm run lint:md && npm run lint:js", "lint:js": "eslint 'src/**/*.{js,ts}' 'test/**/*.{js,ts}' cucumber.js", "lint:md": "remark .", @@ -106,13 +109,12 @@ }, "homepage": "https://github.com/cloudevents/sdk-javascript#readme", "dependencies": { - "ajv": "~6.12.3", + "ajv": "^8.6.3", "util": "^0.12.4", "uuid": "~8.3.0" }, "devDependencies": { "@cucumber/cucumber": "^8.0.0-rc.1", - "@types/ajv": "^1.0.0", "@types/chai": "^4.2.11", "@types/cucumber": "^6.0.1", "@types/got": "^9.6.11", @@ -122,6 +124,8 @@ "@types/uuid": "^8.0.0", "@typescript-eslint/eslint-plugin": "^4.29.0", "@typescript-eslint/parser": "^4.29.0", + "ajv-cli": "^5.0.0", + "ajv-formats": "^2.1.1", "axios": "^0.21.3", "chai": "~4.2.0", "eslint": "^7.32.0", diff --git a/src/event/schemas.ts b/src/event/schemas.ts deleted file mode 100644 index 9444a98..0000000 --- a/src/event/schemas.ts +++ /dev/null @@ -1,86 +0,0 @@ -/* - Copyright 2021 The CloudEvents Authors - SPDX-License-Identifier: Apache-2.0 -*/ - -export const schemaV1 = { - $ref: "#/definitions/event", - definitions: { - specversion: { - type: "string", - minLength: 1, - const: "1.0", - }, - datacontenttype: { - type: "string", - }, - data: { - type: ["object", "string", "array", "number", "boolean", "null"], - }, - data_base64: { - type: "string", - }, - event: { - properties: { - specversion: { - $ref: "#/definitions/specversion", - }, - datacontenttype: { - $ref: "#/definitions/datacontenttype", - }, - data: { - $ref: "#/definitions/data", - }, - data_base64: { - $ref: "#/definitions/data_base64", - }, - id: { - $ref: "#/definitions/id", - }, - time: { - $ref: "#/definitions/time", - }, - dataschema: { - $ref: "#/definitions/dataschema", - }, - subject: { - $ref: "#/definitions/subject", - }, - type: { - $ref: "#/definitions/type", - }, - source: { - $ref: "#/definitions/source", - }, - }, - required: ["specversion", "id", "type", "source"], - type: "object", - }, - id: { - type: "string", - minLength: 1, - }, - time: { - format: "js-date-time", - type: "string", - }, - dataschema: { - type: "string", - format: "uri", - }, - subject: { - type: "string", - minLength: 1, - }, - type: { - type: "string", - minLength: 1, - }, - source: { - format: "uri-reference", - type: "string", - minLength: 1, - }, - }, - type: "object", -}; diff --git a/src/event/spec.ts b/src/event/spec.ts index e1f514f..25def7d 100644 --- a/src/event/spec.ts +++ b/src/event/spec.ts @@ -3,28 +3,17 @@ SPDX-License-Identifier: Apache-2.0 */ -import Ajv, { Options } from "ajv"; import { ValidationError } from "./validation"; import { CloudEventV1 } from "./interfaces"; -import { schemaV1 } from "./schemas"; import { Version } from "./cloudevent"; +import validate from "../schema/v1"; -const ajv = new Ajv({ extendRefs: true } as Options); - -// handle date-time format specially because a user could pass -// Date().toString(), which is not spec compliant date-time format -ajv.addFormat("js-date-time", function (dateTimeString) { - const date = new Date(Date.parse(dateTimeString)); - return date.toString() !== "Invalid Date"; -}); - -const isValidAgainstSchemaV1 = ajv.compile(schemaV1); export function validateCloudEvent(event: CloudEventV1): boolean { if (event.specversion === Version.V1) { - if (!isValidAgainstSchemaV1(event)) { - throw new ValidationError("invalid payload", isValidAgainstSchemaV1.errors); + if (!validate(event)) { + throw new ValidationError("invalid payload", (validate as any).errors); } } else { return false; diff --git a/src/schema/cloudevent.json b/src/schema/cloudevent.json new file mode 100644 index 0000000..b9d5550 --- /dev/null +++ b/src/schema/cloudevent.json @@ -0,0 +1,128 @@ +{ + "$schema": "http://json-schema.org/draft-07/schema#", + "description": "CloudEvents Specification JSON Schema", + "type": "object", + "properties": { + "id": { + "description": "Identifies the event.", + "$ref": "#/definitions/iddef", + "examples": [ + "A234-1234-1234" + ] + }, + "source": { + "description": "Identifies the context in which an event happened.", + "$ref": "#/definitions/sourcedef", + "examples" : [ + "https://github.com/cloudevents", + "mailto:cncf-wg-serverless@lists.cncf.io", + "urn:uuid:6e8bc430-9c3a-11d9-9669-0800200c9a66", + "cloudevents/spec/pull/123", + "/sensors/tn-1234567/alerts", + "1-555-123-4567" + ] + }, + "specversion": { + "description": "The version of the CloudEvents specification which the event uses.", + "$ref": "#/definitions/specversiondef", + "examples": [ + "1.0" + ] + }, + "type": { + "description": "Describes the type of event related to the originating occurrence.", + "$ref": "#/definitions/typedef", + "examples" : [ + "com.github.pull_request.opened", + "com.example.object.deleted.v2" + ] + }, + "datacontenttype": { + "description": "Content type of the data value. Must adhere to RFC 2046 format.", + "$ref": "#/definitions/datacontenttypedef", + "examples": [ + "text/xml", + "application/json", + "image/png", + "multipart/form-data" + ] + }, + "dataschema": { + "description": "Identifies the schema that data adheres to.", + "$ref": "#/definitions/dataschemadef" + }, + "subject": { + "description": "Describes the subject of the event in the context of the event producer (identified by source).", + "$ref": "#/definitions/subjectdef", + "examples": [ + "mynewfile.jpg" + ] + }, + "time": { + "description": "Timestamp of when the occurrence happened. Must adhere to RFC 3339.", + "$ref": "#/definitions/timedef", + "examples": [ + "2018-04-05T17:31:00Z" + ] + }, + "data": { + "description": "The event payload.", + "$ref": "#/definitions/datadef", + "examples": [ + "" + ] + }, + "data_base64": { + "description": "Base64 encoded event payload. Must adhere to RFC4648.", + "$ref": "#/definitions/data_base64def", + "examples": [ + "Zm9vYg==" + ] + } + }, + "required": ["id", "source", "specversion", "type"], + "definitions": { + "iddef": { + "type": "string", + "minLength": 1 + }, + "sourcedef": { + "type": "string", + "format": "uri-reference", + "minLength": 1 + }, + "specversiondef": { + "type": "string", + "minLength": 1 + }, + "typedef": { + "type": "string", + "minLength": 1 + }, + "datacontenttypedef": { + "type": ["string", "null"], + "minLength": 1 + }, + "dataschemadef": { + "type": ["string", "null"], + "format": "uri", + "minLength": 1 + }, + "subjectdef": { + "type": ["string", "null"], + "minLength": 1 + }, + "timedef": { + "type": ["string", "null"], + "format": "date-time", + "minLength": 1 + }, + "datadef": { + "type": ["object", "string", "number", "array", "boolean", "null"] + }, + "data_base64def": { + "type": ["string", "null"], + "contentEncoding": "base64" + } + } +} diff --git a/src/schema/formats.js b/src/schema/formats.js new file mode 100644 index 0000000..f19ee1f --- /dev/null +++ b/src/schema/formats.js @@ -0,0 +1,10 @@ +/* + Copyright 2021 The CloudEvents Authors + SPDX-License-Identifier: Apache-2.0 +*/ + +function formats(ajv) { + require("ajv-formats")(ajv); +} + +module.exports = formats; diff --git a/test/integration/cloud_event_test.ts b/test/integration/cloud_event_test.ts index ba40ce5..f7c95c0 100644 --- a/test/integration/cloud_event_test.ts +++ b/test/integration/cloud_event_test.ts @@ -9,7 +9,6 @@ import fs from "fs"; import { expect } from "chai"; import { CloudEvent, ValidationError, Version } from "../../src"; import { asBase64 } from "../../src/event/validation"; -import { ErrorObject } from "schema-utils/declarations/validate"; const type = "org.cncf.cloudevents.example"; const source = "http://unit.test"; @@ -225,13 +224,12 @@ describe("A 1.0 CloudEvent", () => { type: "my.event.type", source: "", }); - } catch (err) { + } catch (err: any) { expect(err).to.be.instanceOf(ValidationError); - const e = err as unknown as ValidationError; - const errors = e.errors as ErrorObject[]; - expect(e.message).to.include("invalid payload"); - expect(errors[0].dataPath).to.equal(".source"); - expect(errors[0].keyword).to.equal("minLength"); + const error = err.errors[0] as any; + expect(err.message).to.include("invalid payload"); + expect(error.instancePath).to.equal("/source"); + expect(error.keyword).to.equal("minLength"); } }); }); diff --git a/test/integration/sdk_test.ts b/test/integration/sdk_test.ts index 37dc4fd..04483c0 100644 --- a/test/integration/sdk_test.ts +++ b/test/integration/sdk_test.ts @@ -21,12 +21,12 @@ describe("The SDK Requirements", () => { }); describe("v0.3", () => { - it("should create an event using the right spec version", () => { + it("should create an (invalid) event using the right spec version", () => { expect( new CloudEvent({ ...fixture, specversion: Version.V03, - }).specversion, + }, false).specversion, ).to.equal(Version.V03); }); }); diff --git a/test/integration/spec_1_tests.ts b/test/integration/spec_1_tests.ts index e83decd..e2bbd62 100644 --- a/test/integration/spec_1_tests.ts +++ b/test/integration/spec_1_tests.ts @@ -150,7 +150,7 @@ describe("CloudEvents Spec v1.0", () => { describe("'time'", () => { it("must adhere to the format specified in RFC 3339", () => { const d = new Date(); - cloudevent = cloudevent.cloneWith({ time: d.toString() }); + cloudevent = cloudevent.cloneWith({ time: d.toString() }, false); // ensure that we always get back the same thing we passed in expect(cloudevent.time).to.equal(d.toString()); // ensure that when stringified, the timestamp is in RFC3339 format