From 1a9e7cd7c76e2f9b40bb8d51763db32647a74ab3 Mon Sep 17 00:00:00 2001 From: murgatroid99 Date: Tue, 18 Dec 2018 14:17:06 -0800 Subject: [PATCH] Add message type information to package definition output. --- gulpfile.ts | 2 +- packages/proto-loader/gulpfile.ts | 13 +- packages/proto-loader/src/index.ts | 141 +++++++++++++++--- .../proto-loader/test/descriptor_type_test.ts | 53 +++++++ packages/proto-loader/test_protos/enums.proto | 27 ++++ .../proto-loader/test_protos/messages.proto | 28 ++++ packages/proto-loader/tsconfig.json | 3 +- 7 files changed, 245 insertions(+), 22 deletions(-) create mode 100644 packages/proto-loader/test/descriptor_type_test.ts create mode 100644 packages/proto-loader/test_protos/enums.proto create mode 100644 packages/proto-loader/test_protos/messages.proto diff --git a/gulpfile.ts b/gulpfile.ts index 89d9e290..50b10c03 100644 --- a/gulpfile.ts +++ b/gulpfile.ts @@ -99,7 +99,7 @@ gulp.task('native.test', 'Run tests of native code', (callback) => { }); gulp.task('test.only', 'Run tests without rebuilding anything', - ['js.core.test', 'native.test.only']); + ['js.core.test', 'native.test.only', 'protobuf.test']); gulp.task('test', 'Run all tests', (callback) => { runSequence('build', 'test.only', 'internal.test.test', callback); diff --git a/packages/proto-loader/gulpfile.ts b/packages/proto-loader/gulpfile.ts index 550ce364..e655826e 100644 --- a/packages/proto-loader/gulpfile.ts +++ b/packages/proto-loader/gulpfile.ts @@ -54,7 +54,16 @@ gulp.task('clean', 'Deletes transpiled code.', ['install'], gulp.task('clean.all', 'Deletes all files added by targets', ['clean']); /** - * Transpiles TypeScript files in src/ to JavaScript according to the settings + * Transpiles TypeScript files in src/ and test/ to JavaScript according to the settings * found in tsconfig.json. */ -gulp.task('compile', 'Transpiles src/.', () => execNpmCommand('compile')); +gulp.task('compile', 'Transpiles src/ and test/.', () => execNpmCommand('compile')); + +/** + * Transpiles src/ and test/, and then runs all tests. + */ +gulp.task('test', 'Runs all tests.', () => { + return gulp.src(`${outDir}/test/**/*.js`) + .pipe(mocha({reporter: 'mocha-jenkins-reporter', + require: ['ts-node/register']})); +}); diff --git a/packages/proto-loader/src/index.ts b/packages/proto-loader/src/index.ts index 1c09fbaa..9b8704c2 100644 --- a/packages/proto-loader/src/index.ts +++ b/packages/proto-loader/src/index.ts @@ -16,10 +16,25 @@ * */ import * as Protobuf from 'protobufjs'; +import * as descriptor from 'protobufjs/ext/descriptor'; import * as fs from 'fs'; import * as path from 'path'; import camelCase = require('lodash.camelcase'); +declare module 'protobufjs' { + interface Type { + toDescriptor(protoVersion: string): Protobuf.Message & descriptor.IDescriptorProto; + } + + interface Root { + toDescriptor(protoVersion: string): Protobuf.Message & descriptor.IFileDescriptorSet; + } + + interface Enum { + toDescriptor(protoVersion: string): Protobuf.Message & descriptor.IEnumDescriptorProto; + } +} + export interface Serialize { (value: T): Buffer; } @@ -28,6 +43,20 @@ export interface Deserialize { (bytes: Buffer): T; } +export interface ProtobufTypeDefinition { + format: string; + type: object; + fileDescriptorProtos: Buffer[]; +} + +export interface MessageTypeDefinition extends ProtobufTypeDefinition { + format: 'Protocol Buffer 3 DescriptorProto'; +} + +export interface EnumTypeDefinition extends ProtobufTypeDefinition { + format: 'Protocol Buffer 3 EnumDescriptorProto'; +} + export interface MethodDefinition { path: string; requestStream: boolean; @@ -37,20 +66,33 @@ export interface MethodDefinition { requestDeserialize: Deserialize; responseDeserialize: Deserialize; originalName?: string; + requestType: MessageTypeDefinition; + responseType: MessageTypeDefinition; } export interface ServiceDefinition { [index: string]: MethodDefinition; } +export type AnyDefinition = ServiceDefinition | MessageTypeDefinition | EnumTypeDefinition; + export interface PackageDefinition { - [index: string]: ServiceDefinition; + [index: string]: AnyDefinition; } export type Options = Protobuf.IParseOptions & Protobuf.IConversionOptions & { includeDirs?: string[]; }; +const descriptorOptions: Protobuf.IConversionOptions = { + longs: String, + enums: String, + bytes: String, + defaults: true, + oneofs: true, + json: true +}; + function joinName(baseName: string, name: string): string { if (baseName === '') { return name; @@ -59,19 +101,28 @@ function joinName(baseName: string, name: string): string { } } -function getAllServices(obj: Protobuf.NamespaceBase, parentName: string): Array<[string, Protobuf.Service]> { +type HandledReflectionObject = Protobuf.Service | Protobuf.Type | Protobuf.Enum; + +function isHandledReflectionObject(obj: Protobuf.ReflectionObject): obj is HandledReflectionObject { + return obj instanceof Protobuf.Service || obj instanceof Protobuf.Type || obj instanceof Protobuf.Enum; +} + +function isNamespaceBase(obj: Protobuf.ReflectionObject): obj is Protobuf.NamespaceBase { + return obj instanceof Protobuf.Namespace || obj instanceof Protobuf.Root; +} + +function getAllHandledReflectionObjects(obj: Protobuf.ReflectionObject, parentName: string): Array<[string, HandledReflectionObject]> { const objName = joinName(parentName, obj.name); - if (obj.hasOwnProperty('methods')) { - return [[objName, obj as Protobuf.Service]]; + if (isHandledReflectionObject(obj)) { + return [[objName, obj]]; } else { - return obj.nestedArray.map((child) => { - if (child.hasOwnProperty('nested')) { - return getAllServices(child as Protobuf.NamespaceBase, objName); - } else { - return []; - } - }).reduce((accumulator, currentValue) => accumulator.concat(currentValue), []); + if (isNamespaceBase(obj) && typeof obj.nested !== undefined) { + return Object.keys(obj.nested!).map((name) => { + return getAllHandledReflectionObjects(obj.nested![name], objName); + }).reduce((accumulator, currentValue) => accumulator.concat(currentValue), []); + } } + return []; } function createDeserializer(cls: Protobuf.Type, options: Options): Deserialize { @@ -88,16 +139,22 @@ function createSerializer(cls: Protobuf.Type): Serialize { } function createMethodDefinition(method: Protobuf.Method, serviceName: string, options: Options): MethodDefinition { + /* This is only ever called after the corresponding root.resolveAll(), so we + * can assume that the resolved request and response types are non-null */ + const requestType: Protobuf.Type = method.resolvedRequestType!; + const responseType: Protobuf.Type = method.resolvedResponseType!; return { path: '/' + serviceName + '/' + method.name, requestStream: !!method.requestStream, responseStream: !!method.responseStream, - requestSerialize: createSerializer(method.resolvedRequestType as Protobuf.Type), - requestDeserialize: createDeserializer(method.resolvedRequestType as Protobuf.Type, options), - responseSerialize: createSerializer(method.resolvedResponseType as Protobuf.Type), - responseDeserialize: createDeserializer(method.resolvedResponseType as Protobuf.Type, options), + requestSerialize: createSerializer(requestType), + requestDeserialize: createDeserializer(requestType, options), + responseSerialize: createSerializer(responseType), + responseDeserialize: createDeserializer(responseType, options), // TODO(murgatroid99): Find a better way to handle this - originalName: camelCase(method.name) + originalName: camelCase(method.name), + requestType: createMessageDefinition(requestType), + responseType: createMessageDefinition(responseType) }; } @@ -109,10 +166,58 @@ function createServiceDefinition(service: Protobuf.Service, name: string, option return def; } +const fileDescriptorCache: Map = new Map(); +function getFileDescriptors(root: Protobuf.Root): Buffer[] { + if (fileDescriptorCache.has(root)) { + return fileDescriptorCache.get(root)!; + } else { + const descriptorList: descriptor.IFileDescriptorProto[] = root.toDescriptor('proto3').file; + const bufferList: Buffer[] = descriptorList.map(value => Buffer.from(descriptor.FileDescriptorProto.encode(value).finish())); + fileDescriptorCache.set(root, bufferList); + return bufferList; + } +} + +function createMessageDefinition(message: Protobuf.Type): MessageTypeDefinition { + const messageDescriptor: protobuf.Message = message.toDescriptor('proto3'); + return { + format: 'Protocol Buffer 3 DescriptorProto', + type: messageDescriptor.$type.toObject(messageDescriptor, descriptorOptions), + fileDescriptorProtos: getFileDescriptors(message.root) + }; +} + +function createEnumDefinition(enumType: Protobuf.Enum): EnumTypeDefinition { + const enumDescriptor: protobuf.Message = enumType.toDescriptor('proto3'); + return { + format: 'Protocol Buffer 3 EnumDescriptorProto', + type: enumDescriptor.$type.toObject(enumDescriptor, descriptorOptions), + fileDescriptorProtos: getFileDescriptors(enumType.root) + }; +} + +/** + * function createDefinition(obj: Protobuf.Service, name: string, options: Options): ServiceDefinition; + * function createDefinition(obj: Protobuf.Type, name: string, options: Options): MessageTypeDefinition; + * function createDefinition(obj: Protobuf.Enum, name: string, options: Options): EnumTypeDefinition; + */ +function createDefinition(obj: HandledReflectionObject, name: string, options: Options): AnyDefinition { + if (obj instanceof Protobuf.Service) { + return createServiceDefinition(obj, name, options); + } else if (obj instanceof Protobuf.Type) { + return createMessageDefinition(obj); + } else if (obj instanceof Protobuf.Enum) { + return createEnumDefinition(obj); + } else { + throw new Error('Type mismatch in reflection object handling'); + } +} + function createPackageDefinition(root: Protobuf.Root, options: Options): PackageDefinition { const def: PackageDefinition = {}; - for (const [name, service] of getAllServices(root, '')) { - def[name] = createServiceDefinition(service, name, options); + root.resolveAll(); + for (const [name, obj] of getAllHandledReflectionObjects(root, '')) { + def[name] = createDefinition(obj, name, options); } return def; } diff --git a/packages/proto-loader/test/descriptor_type_test.ts b/packages/proto-loader/test/descriptor_type_test.ts new file mode 100644 index 00000000..06327e02 --- /dev/null +++ b/packages/proto-loader/test/descriptor_type_test.ts @@ -0,0 +1,53 @@ +import * as assert from 'assert'; + +import * as proto_loader from '../src/index'; + +// Relative path from build output directory to test_protos directory +const TEST_PROTO_DIR = `${__dirname}/../../test_protos/`; + +type TypeDefinition = proto_loader.EnumTypeDefinition | proto_loader.MessageTypeDefinition; + +function isTypeObject(obj: proto_loader.AnyDefinition): obj is TypeDefinition { + return 'format' in obj; +} + +describe('Descriptor types', () => { + it('Should be output for each enum', (done) => { + proto_loader.load(`${TEST_PROTO_DIR}/enums.proto`).then((packageDefinition) => { + assert('Enum1' in packageDefinition); + assert(isTypeObject(packageDefinition.Enum1)); + // Need additional check because compiler doesn't understand asserts + if(isTypeObject(packageDefinition.Enum1)) { + const enum1Def: TypeDefinition = packageDefinition.Enum1; + assert.strictEqual(enum1Def.format, 'Protocol Buffer 3 EnumDescriptorProto'); + } + + assert('Enum2' in packageDefinition); + assert(isTypeObject(packageDefinition.Enum2)); + // Need additional check because compiler doesn't understand asserts + if(isTypeObject(packageDefinition.Enum2)) { + const enum2Def: TypeDefinition = packageDefinition.Enum2; + assert.strictEqual(enum2Def.format, 'Protocol Buffer 3 EnumDescriptorProto'); + } + done(); + }, (error) => {done(error);}); + }); + it('Should be output for each message', (done) => { + proto_loader.load(`${TEST_PROTO_DIR}/messages.proto`).then((packageDefinition) => { + assert('LongValues' in packageDefinition); + assert(isTypeObject(packageDefinition.LongValues)); + if(isTypeObject(packageDefinition.LongValues)) { + const longValuesDef: TypeDefinition = packageDefinition.LongValues; + assert.strictEqual(longValuesDef.format, 'Protocol Buffer 3 DescriptorProto'); + } + + assert('SequenceValues' in packageDefinition); + assert(isTypeObject(packageDefinition.SequenceValues)); + if(isTypeObject(packageDefinition.SequenceValues)) { + const sequenceValuesDef: TypeDefinition = packageDefinition.SequenceValues; + assert.strictEqual(sequenceValuesDef.format, 'Protocol Buffer 3 DescriptorProto'); + } + done(); + }, (error) => {done(error);}); + }); +}); \ No newline at end of file diff --git a/packages/proto-loader/test_protos/enums.proto b/packages/proto-loader/test_protos/enums.proto new file mode 100644 index 00000000..58e32532 --- /dev/null +++ b/packages/proto-loader/test_protos/enums.proto @@ -0,0 +1,27 @@ +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +enum Enum1 { + DEFAULT = 0; + VALUE1 = 1; + VALUE2 = 2; +} + +enum Enum2 { + DEFAULT = 0; + ABC = 5; + DEF = 10; +} \ No newline at end of file diff --git a/packages/proto-loader/test_protos/messages.proto b/packages/proto-loader/test_protos/messages.proto new file mode 100644 index 00000000..70627451 --- /dev/null +++ b/packages/proto-loader/test_protos/messages.proto @@ -0,0 +1,28 @@ +// Copyright 2019 gRPC authors. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +syntax = "proto3"; + +message LongValues { + int64 int_64 = 1; + uint64 uint_64 = 2; + sint64 sint_64 = 3; + fixed64 fixed_64 = 4; + sfixed64 sfixed_64 = 5; +} + +message SequenceValues { + bytes bytes_field = 1; + repeated int32 repeated_field = 2; +} \ No newline at end of file diff --git a/packages/proto-loader/tsconfig.json b/packages/proto-loader/tsconfig.json index f893d7af..26f33cd1 100644 --- a/packages/proto-loader/tsconfig.json +++ b/packages/proto-loader/tsconfig.json @@ -5,6 +5,7 @@ "outDir": "build" }, "include": [ - "src/*.ts" + "src/*.ts", + "test/*.ts" ] }