feat(otlp-transformer): replace protobufjs metrics serialization with custom implementation (#6629)

Author: pichlermarc

bundler-tests/browser/nextjs-15-edge/package.json

@@ -4,7 +4,7 @@
   "private": true,
   "scripts": {
     "build": "rm -rf .next && next build",
-    "test:bundle": "npm i && node test-bundle.mjs"
+    "test:bundle": "npm i && npm run build"
   },
   "dependencies": {
     "@opentelemetry/api": "file:../../../api",

bundler-tests/browser/nextjs-15-edge/test-bundle.mjs

@@ -10,6 +10,7 @@ For notes on migrating to 2.x / 0.200.x see [the upgrade guide](doc/upgrade-to-2
 
 ### :rocket: Features
 
+* feat(otlp-transformer): replace protobufjs metrics serialization with custom implementation [#6625](https://github.com/open-telemetry/opentelemetry-js/pull/6629) @pichlermarc
 * feat(configuration): show all config validation errors, if there are multiple [#6683](https://github.com/open-telemetry/opentelemetry-js/pull/6683) @trentm
 * feat(sdk-node): allow startNodeSDK() without an arg [#6688](https://github.com/open-telemetry/opentelemetry-js/pull/6688) @trentm
 

experimental/CHANGELOG.md

@@ -17,11 +17,11 @@
     "compile": "tsc --build tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
     "clean": "tsc --build --clean tsconfig.json tsconfig.esm.json tsconfig.esnext.json",
     "protos": "npm run submodule && npm run protos:generate",
-    "protos:generate:js": "pbjs -t static-module -p ./protos -w commonjs --null-defaults -o ./src/generated/root.js ./protos/opentelemetry/proto/common/v1/common.proto ./protos/opentelemetry/proto/resource/v1/resource.proto ./protos/opentelemetry/proto/metrics/v1/metrics.proto ./protos/opentelemetry/proto/collector/metrics/v1/metrics_service.proto && npm run protos:generate:js:tests",
+    "protos:generate:js": "npm run protos:generate:js:tests",
     "protos:generate:js:tests": "npm run protos:generate:js:testbed && npm run protos:generate:js:test-signals",
     "protos:generate:js:testbed": "pbjs -t static-module -p ./test/fixtures -w commonjs --null-defaults -o ./test/generated/testbed.js ./test/fixtures/testbed.proto",
-    "protos:generate:js:test-signals": "pbjs -r signals -t static-module -p ./protos -w commonjs --null-defaults -o ./test/generated/signals.js ./protos/opentelemetry/proto/common/v1/common.proto ./protos/opentelemetry/proto/resource/v1/resource.proto ./protos/opentelemetry/proto/logs/v1/logs.proto ./protos/opentelemetry/proto/collector/logs/v1/logs_service.proto ./protos/opentelemetry/proto/trace/v1/trace.proto ./protos/opentelemetry/proto/collector/trace/v1/trace_service.proto",
-    "protos:generate:ts": "pbts -o ./src/generated/root.d.ts ./src/generated/root.js && pbts -o ./test/generated/testbed.d.ts ./test/generated/testbed.js && pbts -o ./test/generated/signals.d.ts ./test/generated/signals.js",
+    "protos:generate:js:test-signals": "pbjs -r signals -t static-module -p ./protos -w commonjs --null-defaults -o ./test/generated/signals.js ./protos/opentelemetry/proto/common/v1/common.proto ./protos/opentelemetry/proto/resource/v1/resource.proto ./protos/opentelemetry/proto/logs/v1/logs.proto ./protos/opentelemetry/proto/collector/logs/v1/logs_service.proto ./protos/opentelemetry/proto/trace/v1/trace.proto ./protos/opentelemetry/proto/collector/trace/v1/trace_service.proto ./protos/opentelemetry/proto/metrics/v1/metrics.proto ./protos/opentelemetry/proto/collector/metrics/v1/metrics_service.proto",
+    "protos:generate:ts": "pbts -o ./test/generated/testbed.d.ts ./test/generated/testbed.js && pbts -o ./test/generated/signals.d.ts ./test/generated/signals.js",
     "protos:generate": "npm run protos:generate:js && npm run protos:generate:ts",
     "lint": "eslint . --ext .ts",
     "lint:fix": "eslint . --ext .ts --fix",
@@ -79,6 +79,7 @@
     "karma-webpack": "5.0.1",
     "mocha": "11.7.5",
     "nyc": "17.1.0",
+    "protobufjs": "8.0.1",
     "protobufjs-cli": "2.0.1",
     "ts-loader": "9.5.4",
     "typescript": "5.0.4",
@@ -90,8 +91,7 @@
     "@opentelemetry/resources": "2.7.1",
     "@opentelemetry/sdk-logs": "0.217.0",
     "@opentelemetry/sdk-metrics": "2.7.1",
-    "@opentelemetry/sdk-trace-base": "2.7.1",
-    "protobufjs": "8.0.1"
+    "@opentelemetry/sdk-trace-base": "2.7.1"
   },
   "homepage": "https://github.com/open-telemetry/opentelemetry-js/tree/main/experimental/packages/otlp-transformer",
   "sideEffects": false

experimental/packages/otlp-transformer/package.json

@@ -4,6 +4,8 @@
  */
 import type { Attributes, HrTime } from '@opentelemetry/api';
 import type { AnyValue, LogAttributes } from '@opentelemetry/api-logs';
+import type { InstrumentationScope } from '@opentelemetry/core';
+import type { Resource } from '@opentelemetry/resources';
 import type { IProtobufWriter } from './i-protobuf-writer';
 
 /**
@@ -187,3 +189,60 @@ export function writeAnyValue(writer: IProtobufWriter, value: AnyValue): void {
   }
   // Else: unsupported type, write nothing
 }
+
+/**
+ * Write an InstrumentationScope message.
+ *
+ * Proto fields (InstrumentationScope):
+ *   1  name     string  (wire type 2)
+ *   2  version  string  (wire type 2)
+ */
+export function writeInstrumentationScope(
+  writer: IProtobufWriter,
+  scope: InstrumentationScope,
+  fieldNumber: number
+): void {
+  writer.writeTag(fieldNumber, 2);
+  const start = writer.startLengthDelimited();
+  const startPos = writer.pos;
+
+  // name (field 1, string)
+  writer.writeTag(1, 2);
+  writer.writeString(scope.name);
+
+  // version (field 2, string) - skip if empty
+  if (scope.version) {
+    writer.writeTag(2, 2);
+    writer.writeString(scope.version);
+  }
+
+  writer.finishLengthDelimited(start, writer.pos - startPos);
+}
+
+/**
+ * Write a Resource message and its enclosing tag.
+ *
+ * Proto fields (Resource):
+ *   1  attributes                repeated KeyValue  (wire type 2)
+ *   2  dropped_attributes_count  uint32             (wire type 0)
+ */
+export function writeResource(
+  writer: IProtobufWriter,
+  resource: Resource,
+  fieldNumber: number
+): void {
+  writer.writeTag(fieldNumber, 2);
+  const resourceStart = writer.startLengthDelimited();
+  const resourceStartPos = writer.pos;
+
+  // attributes (field 1, repeated KeyValue)
+  if (resource.attributes) {
+    writeAttributes(writer, resource.attributes, 1);
+  }
+
+  // dropped_attributes_count (field 2, uint32) - set to 0 as we don't track this
+  writer.writeTag(2, 0);
+  writer.writeVarint(0);
+
+  writer.finishLengthDelimited(resourceStart, writer.pos - resourceStartPos);
+}

experimental/packages/otlp-transformer/src/common/protobuf/common-serializer.ts

@@ -7,6 +7,8 @@ export interface IProtobufWriter {
   pos: number;
   writeTag(fieldNumber: number, wireType: number): void;
   writeVarint(value: number): void;
+  writeSint32(value: number): void;
+  writeSfixed64(value: number): void;
   writeFixed32(value: number): void;
   writeFixed64(low: number, high: number): void;
   writeBytes(bytes: Uint8Array): void;

experimental/packages/otlp-transformer/src/common/protobuf/i-protobuf-writer.ts

@@ -37,22 +37,37 @@ export class ProtobufReader {
   /**
    * Read a base-128 varint.
    * Returns a JS `number`; precision above 2^53 is silently lost.
+   * Throws if the buffer is truncated mid-varint.
    */
   readVarint(): number {
     let result = 0;
     let shift = 0;
+    let terminated = false;
     while (this.pos < this._buf.length) {
       const b = this._buf[this.pos++];
       result += (b & 0x7f) * Math.pow(2, shift);
       shift += 7;
-      if ((b & 0x80) === 0) break;
+      if ((b & 0x80) === 0) {
+        terminated = true;
+        break;
+      }
+    }
+    if (!terminated) {
+      throw new Error(
+        'Truncated buffer: unexpected end of data while reading varint'
+      );
     }
     return result;
   }
 
   /** Read a length-delimited byte sequence (bytes field or embedded message). */
   readBytes(): Uint8Array {
     const len = this.readVarint();
+    if (this.pos + len > this._buf.length) {
+      throw new Error(
+        `Truncated buffer: expected ${len} bytes at position ${this.pos}, but only ${this._buf.length - this.pos} available`
+      );
+    }
     const slice = this._buf.subarray(this.pos, this.pos + len);
     this.pos += len;
     return slice;

experimental/packages/otlp-transformer/src/common/protobuf/protobuf-reader.ts

@@ -51,6 +51,15 @@ export class ProtobufSizeEstimator implements IProtobufWriter {
     this.pos += estimateVarintSize(value);
   }
 
+  writeSint32(value: number): void {
+    // Zigzag encode: (n << 1) ^ (n >> 31)
+    this.pos += estimateVarintSize(((value << 1) ^ (value >> 31)) >>> 0);
+  }
+
+  writeSfixed64(_value: number): void {
+    this.pos += 8;
+  }
+
   writeFixed32(_value: number): void {
     this.pos += 4;
   }

experimental/packages/otlp-transformer/src/common/protobuf/protobuf-size-estimator.ts

@@ -126,6 +126,42 @@ export class ProtobufWriter implements IProtobufWriter {
     }
   }
 
+  /**
+   * Write a sint32 value using zigzag encoding
+   */
+  writeSint32(value: number): void {
+    // Zigzag encode: (n << 1) ^ (n >> 31)
+    this.writeVarint(((value << 1) ^ (value >> 31)) >>> 0);
+  }
+
+  /**
+   * Write a signed 64-bit fixed integer (sfixed64) from a JS number.
+   * Handles negative values via two's complement.
+   */
+  writeSfixed64(value: number): void {
+    let low: number;
+    let high: number;
+
+    if (value >= 0) {
+      low = value >>> 0;
+      high = (value / 0x100000000) >>> 0;
+    } else {
+      // Two's complement for negative values
+      const abs = Math.abs(value);
+      low = abs >>> 0;
+      high = (abs / 0x100000000) >>> 0;
+      // Invert bits and add 1
+      low = ~low >>> 0;
+      high = ~high >>> 0;
+      low = (low + 1) >>> 0;
+      if (low === 0) {
+        high = (high + 1) >>> 0;
+      }
+    }
+
+    this.writeFixed64(low, high);
+  }
+
   /**
    * Write a varint (variable-length integer)
    */

experimental/packages/otlp-transformer/src/common/protobuf/protobuf-writer.ts

@@ -8,5 +8,14 @@
  */
 export interface ISerializer<Request, Response> {
   serializeRequest(request: Request): Uint8Array | undefined;
+
+  /**
+   * Deserialize the response from the backend. The response is expected to be in the form of a
+   * {@link Uint8Array} and will be deserialized into the expected response type.
+   *
+   * @param data
+   * @throws {unknown} if the deserialization fails
+   * @returns the deserialized response
+   */
   deserializeResponse(data: Uint8Array): Response;
 }