Random failures in decode(encode(data))

[hybrist]

protobuf.js version: 6.4.4

It looks like for the schema & input below encode is generating random output. Some of which can be decoded but most of them cannot. Shouldn't encode always generate the same output for the same input?

I tried to find a minimal example. Notes:

• Changing "value" to anything but 0 in the input makes the problem go away, so it looks to be connected to omitting default values.
• The field at the end of Model seems to be required to trigger the issue but the type of field doesn't seem to matter.

'use strict';
var _ = require('lodash');
var protobuf = require('protobufjs');

var $root = new protobuf.Root();
protobuf.parse(`syntax = "proto3";

message DoubleQuantity {
    double value = 1;
}

message Model {
    repeated DoubleQuantity samples = 1;
    string end = 2;
}
`, $root);

var Model = $root.lookup('Model');
var input = {"samples":[{"value":0}],"end":"x"};

var model = Model.from(input);
var hexes = _.range(50000).map(() =>
  Model.encode(Model.from(_.cloneDeep(model))).finish().toString('hex'));
var uniqHexes = _.uniq(hexes);
if (uniqHexes.length > 1) {
  console.log('%d conflicting ways of serializing the same data', uniqHexes.length);
  console.log(uniqHexes.slice(0, 5));
}

let errorCount = 0;
uniqHexes.forEach((hex, idx) => {
  const buffer = new Buffer(hex, 'hex');
  try {
    Model.decode(buffer);
  } catch (e) {
    ++errorCount;
    if (errorCount < 10) {
      console.error('Failed to decode hex#%d:\n  ', idx, e.message);
    }
  }
});

if (uniqHexes.length === 1 && errorCount === 0) {
  console.log('Everything is ok!');
} else {
  console.log('%d/%d failed to decode', errorCount, uniqHexes.length);
}

> node test-case.js
826 conflicting ways of serializing the same data
[ '0a00000000',
  '0a00bf5fff',
  '0a00020401',
  '0a0074696f',
  '0a007b0d0a' ]
Failed to decode hex#0:
   index out of range: 5 + 1 > 5
Failed to decode hex#1:
   invalid wire type 7 at offset 4
Failed to decode hex#2:
   index out of range: 4 + 4 > 5
Failed to decode hex#3:
   invalid wire type 4 at offset 3
Failed to decode hex#4:
   index out of range: 4 + 4 > 5
Failed to decode hex#5:
   index out of range: 5 + 10 > 5
Failed to decode hex#6:
   index out of range: 5 + 8 > 5
Failed to decode hex#7:
   index out of range: 5 + 10 > 5
Failed to decode hex#8:
   index out of range: 3 + 8 > 5
776/826 failed to decode

[dcodeIO]

Thank you for reporting this. Your example has been quite helpful!

The underlying issue here was a missing check for an empty forked state. This led to a corruption of a writer's operations list, which then allocated a properly sized buffer but missed some data in the final write due to non-execution of missing operations. The random data you see there is from Buffer.allocUnsafe, which does not initialize reused memory to all zeroes - hence, this is also relevant for security.

[hybrist]

Wow - that was super fast! Thanks a lot! :)

EDIT: Verified that everything's working now with a larger sample. 👍