feat(otlp-grpc-exporter-base): use statically generated protobuf code (#3705)

Co-authored-by: Daniel Dyla <dyladan@users.noreply.github.com>

experimental/CHANGELOG.md

@@ -11,6 +11,7 @@ All notable changes to experimental packages in this project will be documented
 * feat(otlp-transformer): support log records. [#3712](https://github.com/open-telemetry/opentelemetry-js/pull/3712/) @llc1123
 * feat(otlp-grpc-exporter-base): support log records. [#3712](https://github.com/open-telemetry/opentelemetry-js/pull/3712/) @llc1123
 * feat(exporter-logs-otlp-grpc): otlp-grpc exporter for logs. [#3712](https://github.com/open-telemetry/opentelemetry-js/pull/3712/) @llc1123
+* feat(otlp-grpc-exporter-base): use statically generated protobuf code [#3705](https://github.com/open-telemetry/opentelemetry-js/pull/3705) @pichlermarc
 * refactor(otlp-transformer): refine metric transformers. [#3770](https://github.com/open-telemetry/opentelemetry-js/pull/3770/) @llc1123
 
 ### :bug: (Bug Fix)

experimental/packages/otlp-grpc-exporter-base/.gitignore

@@ -0,0 +1,2 @@
+src/generated/*
+!src/generated/.gitkeep

experimental/packages/otlp-grpc-exporter-base/package.json

@@ -8,17 +8,17 @@
   "scripts": {
     "prepublishOnly": "npm run compile",
     "codecov": "nyc report --reporter=json && codecov -f coverage/*.json -p ../../../",
-    "compile": "tsc --build",
+    "compile": "npm run protos && tsc --build",
     "clean": "tsc --build --clean",
     "lint": "eslint . --ext .ts",
     "lint:fix": "eslint . --ext .ts --fix",
-    "postcompile": "npm run submodule && npm run protos:copy",
-    "protos:copy": "cpx protos/opentelemetry/**/*.* build/protos/opentelemetry",
+    "protos": "npm run submodule && npm run protos:generate",
+    "protos:generate": "node ../../../scripts/generate-protos.js",
     "submodule": "git submodule sync --recursive && git submodule update --init --recursive",
     "tdd": "npm run test -- --watch-extensions ts --watch",
     "test": "nyc ts-mocha -p tsconfig.json 'test/**/*.test.ts'",
     "version": "node ../../../scripts/version-update.js",
-    "watch": "npm run protos:copy && tsc -w",
+    "watch": "npm run protos && tsc -w",
     "precompile": "lerna run version --scope $(npm pkg get name) --include-dependencies",
     "prewatch": "npm run precompile"
   },
@@ -40,7 +40,6 @@
     "build/src/**/*.js",
     "build/src/**/*.js.map",
     "build/src/**/*.d.ts",
-    "build/protos/**/*.proto",
     "doc",
     "LICENSE",
     "README.md"
@@ -64,16 +63,17 @@
     "sinon": "15.0.0",
     "ts-loader": "8.4.0",
     "ts-mocha": "10.0.0",
-    "typescript": "4.4.4"
+    "typescript": "4.4.4",
+    "protobufjs-cli": "1.0.2"
   },
   "peerDependencies": {
     "@opentelemetry/api": "^1.0.0"
   },
   "dependencies": {
     "@grpc/grpc-js": "^1.7.1",
-    "@grpc/proto-loader": "^0.7.3",
     "@opentelemetry/core": "1.12.0",
-    "@opentelemetry/otlp-exporter-base": "0.38.0"
+    "@opentelemetry/otlp-exporter-base": "0.38.0",
+    "protobufjs": "^7.2.2"
   },
   "homepage": "https://github.com/open-telemetry/opentelemetry-js/tree/main/experimental/packages/otlp-grpc-exporter-base",
   "sideEffects": false

experimental/packages/otlp-grpc-exporter-base/src/LogsExportServiceClient.ts

@@ -0,0 +1,51 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as root from './generated/root';
+import * as grpc from '@grpc/grpc-js';
+import { IExportLogsServiceRequest } from '@opentelemetry/otlp-transformer';
+import { ExportType } from './internal-types';
+import IExportLogsServiceResponse = root.opentelemetry.proto.collector.logs.v1.IExportLogsServiceResponse;
+
+const responseType = root.opentelemetry.proto.collector.logs.v1
+  .ExportLogsServiceResponse as ExportType<IExportLogsServiceResponse>;
+
+const requestType = root.opentelemetry.proto.collector.logs.v1
+  .ExportLogsServiceRequest as ExportType<IExportLogsServiceRequest>;
+
+const logsServiceDefinition = {
+  export: {
+    path: '/opentelemetry.proto.collector.logs.v1.LogsService/Export',
+    requestStream: false,
+    responseStream: false,
+    requestSerialize: (arg: IExportLogsServiceRequest) => {
+      return Buffer.from(requestType.encode(arg).finish());
+    },
+    requestDeserialize: (arg: Buffer) => {
+      return requestType.decode(arg);
+    },
+    responseSerialize: (arg: IExportLogsServiceResponse) => {
+      return Buffer.from(responseType.encode(arg).finish());
+    },
+    responseDeserialize: (arg: Buffer) => {
+      return responseType.decode(arg);
+    },
+  },
+};
+
+// Creates a new instance of a gRPC service client for OTLP logs
+export const LogsExportServiceClient: grpc.ServiceClientConstructor =
+  grpc.makeGenericClientConstructor(logsServiceDefinition, 'LogsExportService');

experimental/packages/otlp-grpc-exporter-base/src/MetricsExportServiceClient.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as root from './generated/root';
+import * as grpc from '@grpc/grpc-js';
+import { IExportMetricsServiceRequest } from '@opentelemetry/otlp-transformer';
+import { ExportType } from './internal-types';
+import IExportMetricsServiceResponse = root.opentelemetry.proto.collector.metrics.v1.IExportMetricsServiceResponse;
+
+const responseType = root.opentelemetry.proto.collector.metrics.v1
+  .ExportMetricsServiceResponse as ExportType<IExportMetricsServiceResponse>;
+
+const requestType = root.opentelemetry.proto.collector.metrics.v1
+  .ExportMetricsServiceRequest as ExportType<IExportMetricsServiceRequest>;
+
+const metricsServiceDefinition = {
+  export: {
+    path: '/opentelemetry.proto.collector.metrics.v1.MetricsService/Export',
+    requestStream: false,
+    responseStream: false,
+    requestSerialize: (arg: IExportMetricsServiceRequest) => {
+      return Buffer.from(requestType.encode(arg).finish());
+    },
+    requestDeserialize: (arg: Buffer) => {
+      return requestType.decode(arg);
+    },
+    responseSerialize: (arg: IExportMetricsServiceResponse) => {
+      return Buffer.from(responseType.encode(arg).finish());
+    },
+    responseDeserialize: (arg: Buffer) => {
+      return responseType.decode(arg);
+    },
+  },
+};
+
+// Creates a new instance of a gRPC service client for OTLP metrics
+export const MetricExportServiceClient: grpc.ServiceClientConstructor =
+  grpc.makeGenericClientConstructor(
+    metricsServiceDefinition,
+    'MetricsExportService'
+  );

experimental/packages/otlp-grpc-exporter-base/src/TraceExportServiceClient.ts

@@ -0,0 +1,54 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import * as root from './generated/root';
+import * as grpc from '@grpc/grpc-js';
+import { IExportTraceServiceRequest } from '@opentelemetry/otlp-transformer';
+import { ExportType } from './internal-types';
+import IExportTraceServiceResponse = root.opentelemetry.proto.collector.trace.v1.IExportTraceServiceResponse;
+
+const responseType = root.opentelemetry.proto.collector.trace.v1
+  .ExportTraceServiceResponse as ExportType<IExportTraceServiceResponse>;
+
+const requestType = root.opentelemetry.proto.collector.trace.v1
+  .ExportTraceServiceRequest as ExportType<IExportTraceServiceRequest>;
+
+const traceServiceDefinition = {
+  export: {
+    path: '/opentelemetry.proto.collector.trace.v1.TraceService/Export',
+    requestStream: false,
+    responseStream: false,
+    requestSerialize: (arg: IExportTraceServiceRequest) => {
+      return Buffer.from(requestType.encode(arg).finish());
+    },
+    requestDeserialize: (arg: Buffer) => {
+      return requestType.decode(arg);
+    },
+    responseSerialize: (arg: IExportTraceServiceResponse) => {
+      return Buffer.from(responseType.encode(arg).finish());
+    },
+    responseDeserialize: (arg: Buffer) => {
+      return responseType.decode(arg);
+    },
+  },
+};
+
+// Creates a new instance of a gRPC service client for exporting OTLP traces
+export const TraceExportServiceClient: grpc.ServiceClientConstructor =
+  grpc.makeGenericClientConstructor(
+    traceServiceDefinition,
+    'TraceExportService'
+  );

experimental/packages/otlp-grpc-exporter-base/src/internal-types.ts

@@ -0,0 +1,22 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import type * as protobuf from 'protobufjs';
+
+export interface ExportType<T, R = T & { toJSON: () => unknown }> {
+  encode(message: T, writer?: protobuf.Writer): protobuf.Writer;
+  decode(reader: protobuf.Reader | Uint8Array, length?: number): R;
+}

experimental/packages/otlp-grpc-exporter-base/src/types.ts

@@ -34,13 +34,15 @@ export interface GRPCQueueItem<ExportedItem> {
 /**
  * Service Client for sending spans/metrics/logs
  */
-export interface ServiceClient extends grpc.Client {
+export interface ServiceClient {
   export: (
     request: any,
     metadata: grpc.Metadata,
     options: grpc.CallOptions,
     callback: Function
   ) => {};
+
+  close(): void;
 }
 
 /**

experimental/packages/otlp-grpc-exporter-base/src/util.ts

@@ -15,7 +15,6 @@
  */
 
 import * as grpc from '@grpc/grpc-js';
-import * as protoLoader from '@grpc/proto-loader';
 import { diag } from '@opentelemetry/api';
 import { getEnv, globalErrorHandler } from '@opentelemetry/core';
 import * as path from 'path';
@@ -28,11 +27,15 @@ import {
   ServiceClientType,
 } from './types';
 import {
-  CompressionAlgorithm,
   ExportServiceError,
   OTLPExporterError,
+  CompressionAlgorithm,
 } from '@opentelemetry/otlp-exporter-base';
 
+import { MetricExportServiceClient } from './MetricsExportServiceClient';
+import { TraceExportServiceClient } from './TraceExportServiceClient';
+import { LogsExportServiceClient } from './LogsExportServiceClient';
+
 export const DEFAULT_COLLECTOR_URL = 'http://localhost:4317';
 
 export function onInit<ExportItem, ServiceRequest>(
@@ -46,61 +49,41 @@ export function onInit<ExportItem, ServiceRequest>(
     collector.getUrlFromConfig(config)
   );
 
-  const includeDirs = [path.resolve(__dirname, '..', 'protos')];
+  try {
+    if (collector.getServiceClientType() === ServiceClientType.SPANS) {
+      const client = new TraceExportServiceClient(collector.url, credentials, {
+        'grpc.default_compression_algorithm': collector.compression.valueOf(),
+      });
 
-  protoLoader
-    .load(collector.getServiceProtoPath(), {
-      keepCase: false,
-      longs: String,
-      enums: String,
-      defaults: true,
-      oneofs: true,
-      includeDirs,
-    })
-    .then(packageDefinition => {
-      const packageObject: any = grpc.loadPackageDefinition(packageDefinition);
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      collector.serviceClient = client;
+    } else if (collector.getServiceClientType() === ServiceClientType.METRICS) {
+      const client = new MetricExportServiceClient(collector.url, credentials, {
+        'grpc.default_compression_algorithm': collector.compression.valueOf(),
+      });
 
-      const options = {
-        'grpc.default_compression_algorithm': collector.compression,
-      };
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      collector.serviceClient = client;
+    } else if (collector.getServiceClientType() === ServiceClientType.LOGS) {
+      const client = new LogsExportServiceClient(collector.url, credentials, {
+        'grpc.default_compression_algorithm': collector.compression.valueOf(),
+      });
 
-      switch (collector.getServiceClientType()) {
-        case ServiceClientType.SPANS:
-          collector.serviceClient =
-            new packageObject.opentelemetry.proto.collector.trace.v1.TraceService(
-              collector.url,
-              credentials,
-              options
-            );
-          break;
-        case ServiceClientType.METRICS:
-          collector.serviceClient =
-            new packageObject.opentelemetry.proto.collector.metrics.v1.MetricsService(
-              collector.url,
-              credentials,
-              options
-            );
-          break;
-        case ServiceClientType.LOGS:
-          collector.serviceClient =
-            new packageObject.opentelemetry.proto.collector.logs.v1.LogsService(
-              collector.url,
-              credentials,
-              options
-            );
-          break;
-      }
-
-      if (collector.grpcQueue.length > 0) {
-        const queue = collector.grpcQueue.splice(0);
-        queue.forEach((item: GRPCQueueItem<ExportItem>) => {
-          collector.send(item.objects, item.onSuccess, item.onError);
-        });
-      }
-    })
-    .catch(err => {
-      globalErrorHandler(err);
+      // eslint-disable-next-line @typescript-eslint/ban-ts-comment
+      // @ts-ignore
+      collector.serviceClient = client;
+    }
+  } catch (err) {
+    globalErrorHandler(err);
+  }
+  if (collector.grpcQueue.length > 0) {
+    const queue = collector.grpcQueue.splice(0);
+    queue.forEach((item: GRPCQueueItem<ExportItem>) => {
+      collector.send(item.objects, item.onSuccess, item.onError);
     });
+  }
 }
 
 export function send<ExportItem, ServiceRequest>(

experimental/packages/otlp-grpc-exporter-base/tsconfig.json

@@ -1,12 +1,14 @@
 {
   "extends": "../../../tsconfig.base.json",
   "compilerOptions": {
+    "allowJs": true,
     "outDir": "build",
     "rootDir": "."
   },
   "include": [
     "src/**/*.ts",
-    "src/generated/*.js",
+    "src/generated/**/*.js",
+    "src/generated/**/*.ts",
     "test/**/*.ts"
   ],
   "references": [

experimental/packages/otlp-proto-exporter-base/package.json

@@ -19,7 +19,8 @@
     "clean": "tsc --build --clean tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
     "lint": "eslint . --ext .ts",
     "lint:fix": "eslint . --ext .ts --fix",
-    "protos": "npm run submodule && node scripts/protos.js",
+    "protos": "npm run submodule && npm run protos:generate",
+    "protos:generate": "node ../../../scripts/generate-protos.js",
     "submodule": "git submodule sync --recursive && git submodule update --init --recursive",
     "version": "node ../../../scripts/version-update.js",
     "watch": "npm run protos && tsc -w tsconfig.json tsconfig.esm.json tsconfig.esnext.json",

package.json

@@ -3,15 +3,13 @@
   "version": "0.1.0",
   "description": "OpenTelemetry is a distributed tracing and stats collection framework.",
   "scripts": {
-    "precompile": "lerna run version",
-    "compile": "lerna run protos && tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
+    "precompile": "lerna run version && npm run submodule",
+    "compile": "lerna run protos:generate && tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
     "prewatch": "npm run precompile",
     "watch": "tsc --build --watch tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
     "clean": "tsc --build --clean tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
     "postinstall": "npm run update-ts-configs && npm run bootstrap",
-    "postcompile": "npm run submodule && npm run protos:copy",
     "submodule": "git submodule sync --recursive && git submodule update --init --recursive",
-    "protos:copy": "lerna run protos:copy",
     "version:update": "lerna run version:update",
     "test": "lerna run test",
     "test:browser": "lerna run test:browser",

scripts/generate-protos.js

@@ -3,8 +3,10 @@
 const cp = require('child_process');
 const path = require('path');
 
-const generatedPath = path.resolve(__dirname, '../src/generated');
-const protosPath = path.resolve(__dirname, '../protos');
+const appRoot = process.cwd();
+
+const generatedPath = path.resolve(appRoot, './src/generated');
+const protosPath = path.resolve(appRoot, './protos');
 const protos = [
   'opentelemetry/proto/common/v1/common.proto',
   'opentelemetry/proto/resource/v1/resource.proto',
@@ -12,6 +14,8 @@ const protos = [
   'opentelemetry/proto/collector/trace/v1/trace_service.proto',
   'opentelemetry/proto/metrics/v1/metrics.proto',
   'opentelemetry/proto/collector/metrics/v1/metrics_service.proto',
+  'opentelemetry/proto/logs/v1/logs.proto',
+  'opentelemetry/proto/collector/logs/v1/logs_service.proto',
 ].map(it => {
   return path.join(protosPath, it);
 });
@@ -33,7 +37,7 @@ function exec(command, argv) {
 }
 
 function pbts(pbjsOutFile) {
-  const pbtsPath = path.resolve(__dirname, '../node_modules/.bin/pbts');
+  const pbtsPath = path.resolve(appRoot, './node_modules/.bin/pbts');
   const pbtsOptions = [
     '-o', path.join(generatedPath, 'root.d.ts'),
   ];
@@ -41,7 +45,7 @@ function pbts(pbjsOutFile) {
 }
 
 async function pbjs(files) {
-  const pbjsPath = path.resolve(__dirname, '../node_modules/.bin/pbjs');
+  const pbjsPath = path.resolve(appRoot, './node_modules/.bin/pbjs');
   const outFile = path.join(generatedPath, 'root.js');
   const pbjsOptions = [
     '-t', 'static-module',