Process huge strings in chunks, see #555; Run bench in tests for a while; Raw alloc benchmark

Author: dcodeIO

.travis.yml

@@ -6,6 +6,10 @@ node_js:
   - 6
   - 7
 
+script:
+  - npm test
+  - npm run bench
+
 branches:
   only:
     - master

README.md

@@ -81,7 +81,7 @@ Production:
 
 Or [download](https://github.com/dcodeIO/protobuf.js/tree/master/dist) the library.
 
-The `protobuf` namespace will always be available globally but also supports AMD.
+The `protobuf` namespace will always be available globally / also supports AMD loaders.
 
 Examples
 --------
@@ -354,11 +354,11 @@ usage: pbts [options] file1.js file2.js ...
 
 ### Descriptors vs. static modules
 
-While .proto and JSON files require the full library (about 18kb gzipped), pretty much all code but the relatively short descriptors is shared.
+While .proto and JSON files require the full library (about 18kb gzipped), pretty much all code but the relatively short descriptors is shared. Hence, it usually doesn't become much larger than that.
 
-Static code, on the other hand, requires just the minimal runtime (about 5.5kb gzipped), but generates relatively large code bases without any reflection features.
+Static code, on the other hand, requires just the minimal runtime (about 5.5kb gzipped, no reflection features), but generates relatively large codebases that you can edit or strip down by hand.
 
-When `new Function` is supported (and it usually is), there is no difference performance-wise as the code generated statically is the same generated at runtime.
+When `new Function` is supported (and it usually is), there is no difference performance-wise as the code generated statically is the same as generated at runtime.
 
 Building
 --------

bench/alloc.js

@@ -0,0 +1,45 @@
+var protobuf = require(".."),
+    newSuite = require("./suite");
+
+var pool = require("../src/util/pool"),
+    poolAlloc = pool(function(size) {
+        return new Uint8Array(size);
+    }, Uint8Array.prototype.subarray);
+
+[
+    64,
+    256,
+    1024
+]
+.forEach(function(size) {
+
+    newSuite("buffer[" + size + "]")
+    .add("new Uint8Array", function() {
+        var buf = new Uint8Array(size);
+    })
+    .add("Buffer.alloc", function() {
+        var buf = Buffer.alloc(size);
+    })
+    .add("poolAlloc<Uint8Array>", function() {
+        var buf = poolAlloc(size);
+    })
+    .add("Buffer.allocUnsafe", function() {
+        var buf = Buffer.allocUnsafe(size);
+    })
+    .add("new Buffer", function() {
+        var buf = new Buffer(size);
+    })
+    .run();
+
+    var ab = new ArrayBuffer(size);
+
+    newSuite("wrap[" + size + "]")
+    .add("new Uint8Array(ArrayBuffer)", function() {
+        var buf = new Uint8Array(ab);
+    })
+    .add("Buffer.from(ArrayBuffer)", function() {
+        var buf = Buffer.from(ab);
+    })
+    .run();
+
+});

bench/index.js

@@ -1,4 +1,4 @@
-var protobuf  = require("../src/index"),
+var protobuf  = require(".."),
     newSuite  = require("./suite"),
     data      = require("./bench.json");
 
@@ -35,6 +35,11 @@ for (var i = 0; i < 500000; ++i)
     Test.verify(data);
 console.log("");
 
+if (!Buffer.from)
+    Buffer.from = function from(str, enc) {
+        return new Buffer(str, enc);
+    };
+
 // give the optimizer some time to do its job
 setTimeout(function() {
     var str    = JSON.stringify(data),
@@ -48,7 +53,7 @@ setTimeout(function() {
         JSON.stringify(data);
     })
     .add("JSON.stringify to buffer", function() {
-        new Buffer(JSON.stringify(data), "utf8");
+        Buffer.from(JSON.stringify(data), "utf8");
     })
     .run();
 
@@ -72,7 +77,7 @@ setTimeout(function() {
         JSON.parse(JSON.stringify(data));
     })
     .add("JSON to/from buffer", function() {
-        JSON.parse(new Buffer(JSON.stringify(data), "utf8").toString("utf8"));
+        JSON.parse(Buffer.from(JSON.stringify(data), "utf8").toString("utf8"));
     })
     .run();
 

bench/read.js

@@ -1,4 +1,4 @@
-var protobuf = require("../src/index"),
+var protobuf = require(".."),
     newSuite = require("./suite"),
     ieee754  = require("../lib/ieee754");
 

bench/write.js

@@ -1,4 +1,4 @@
-var protobuf = require("../src/index"),
+var protobuf = require(".."),
     newSuite = require("./suite"),
     ieee754  = require("../lib/ieee754");
 

cli/pbjs.js

@@ -54,6 +54,13 @@ exports.main = function(args) {
             "",
             "  -r, --root      Specifies an alternative protobuf.roots name for *-module targets.",
             "",
+            "  Static code generation only:",
+            "",
+            "  --no-encode     Does not generate encode functions.",
+            "  --no-decode     Does not generate decode functions.",
+            "  --no-verify     Does not generate verify functions.",
+            "  --no-delimited  Does not generate delimited encode/decode functions.",
+            "",
             "usage: " + chalk.bold.green(path.basename(process.argv[1])) + " [options] file1.proto file2.json ..."
         ].join("\n"));
         return 1;

cli/targets/static.js

@@ -13,10 +13,12 @@ var Type      = protobuf.Type,
 
 var out = [];
 var indent = 0;
+var config = {};
 
 static_target.description = "Static code without reflection";
 
 function static_target(root, options, callback) {
+    config = options;
     try {
         push("// Lazily resolved type references");
         push("var $lazyTypes = [];");
@@ -51,6 +53,7 @@ function static_target(root, options, callback) {
     } finally {
         out = [];
         indent = 0;
+        config = {};
     }
 }
 
@@ -266,72 +269,91 @@ function buildType(ref, type) {
         push("});");
     });
 
-    // #encode
-    push("");
-    pushComment([
-        "Encodes the specified " + type.name + ".",
-        "@function",
-        "@param {" + fullName + "|Object} message " + type.name + " or plain object to encode",
-        "@param {Writer} [writer] Writer to encode to",
-        "@returns {Writer} Writer"
-    ]);
-    buildFunction(type, "encode", protobuf.encode.generate(type), {
-        Writer : "$protobuf.Writer",
-        util   : "$protobuf.util"
-    });
+    if (config.encode !== false) {
 
-    // #encodeDelimited
-    push("");
-    pushComment([
-        "Encodes the specified " + type.name + ", length delimited.",
-        "@param {" + fullName + "|Object} message " + type.name + " or plain object to encode",
-        "@param {Writer} [writer] Writer to encode to",
-        "@returns {Writer} Writer"
-    ]);
-    push(name(type.name) + ".encodeDelimited = function encodeDelimited(message, writer) {");
-    ++indent;
-    push("return this.encode(message, writer).ldelim();");
-    --indent;
-    push("};");
+        // #encode
+        push("");
+        pushComment([
+            "Encodes the specified " + type.name + ".",
+            "@function",
+            "@param {" + fullName + "|Object} message " + type.name + " or plain object to encode",
+            "@param {Writer} [writer] Writer to encode to",
+            "@returns {Writer} Writer"
+        ]);
+        buildFunction(type, "encode", protobuf.encode.generate(type), {
+            Writer : "$protobuf.Writer",
+            util   : "$protobuf.util"
+        });
 
-    // #decode
-    push("");
-    pushComment([
-        "Decodes a " + type.name + " from the specified reader or buffer.",
-        "@function",
-        "@param {Reader|Uint8Array} readerOrBuffer Reader or buffer to decode from",
-        "@param {number} [length] Message length if known beforehand",
-        "@returns {" + fullName + "} " + type.name
-    ]);
-    buildFunction(type, "decode", protobuf.decode.generate(type), {
-        Reader : "$protobuf.Reader",
-        util   : "$protobuf.util"
-    });
+        if (config.delimited !== false) {
 
-    // #decodeDelimited
-    push("");
-    pushComment([
-        "Decodes a " + type.name + " from the specified reader or buffer, length delimited.",
-        "@param {Reader|Uint8Array} readerOrBuffer Reader or buffer to decode from",
-        "@returns {" + fullName + "} " + type.name
-    ]);
-    push(name(type.name) + ".decodeDelimited = function decodeDelimited(readerOrBuffer) {");
-    ++indent;
-    push("readerOrBuffer = readerOrBuffer instanceof $protobuf.Reader ? readerOrBuffer : $protobuf.Reader(readerOrBuffer);");
-    push("return this.decode(readerOrBuffer, readerOrBuffer.uint32());");
-    --indent;
-    push("};");
+            // #encodeDelimited
+            push("");
+            pushComment([
+                "Encodes the specified " + type.name + ", length delimited.",
+                "@param {" + fullName + "|Object} message " + type.name + " or plain object to encode",
+                "@param {Writer} [writer] Writer to encode to",
+                "@returns {Writer} Writer"
+            ]);
+            push(name(type.name) + ".encodeDelimited = function encodeDelimited(message, writer) {");
+            ++indent;
+            push("return this.encode(message, writer).ldelim();");
+            --indent;
+            push("};");
 
-    // #verify
-    push("");
-    pushComment([
-        "Verifies a " + type.name + ".",
-        "@param {" + fullName + "|Object} message " + type.name + " or plain object to verify",
-        "@returns {?string} `null` if valid, otherwise the reason why it is not"
-    ]);
-    buildFunction(type, "verify", protobuf.verify.generate(type), {
-        util : "$protobuf.util"
-    });
+        }
+
+    }
+
+    if (config.decode !== false) {
+
+        // #decode
+        push("");
+        pushComment([
+            "Decodes a " + type.name + " from the specified reader or buffer.",
+            "@function",
+            "@param {Reader|Uint8Array} readerOrBuffer Reader or buffer to decode from",
+            "@param {number} [length] Message length if known beforehand",
+            "@returns {" + fullName + "} " + type.name
+        ]);
+        buildFunction(type, "decode", protobuf.decode.generate(type), {
+            Reader : "$protobuf.Reader",
+            util   : "$protobuf.util"
+        });
+
+        if (config.delimited !== false) {
+
+            // #decodeDelimited
+            push("");
+            pushComment([
+                "Decodes a " + type.name + " from the specified reader or buffer, length delimited.",
+                "@param {Reader|Uint8Array} readerOrBuffer Reader or buffer to decode from",
+                "@returns {" + fullName + "} " + type.name
+            ]);
+            push(name(type.name) + ".decodeDelimited = function decodeDelimited(readerOrBuffer) {");
+            ++indent;
+            push("readerOrBuffer = readerOrBuffer instanceof $protobuf.Reader ? readerOrBuffer : $protobuf.Reader(readerOrBuffer);");
+            push("return this.decode(readerOrBuffer, readerOrBuffer.uint32());");
+            --indent;
+            push("};");
+
+        }
+    }
+
+    if (config.verify !== false) {
+
+        // #verify
+        push("");
+        pushComment([
+            "Verifies a " + type.name + ".",
+            "@param {" + fullName + "|Object} message " + type.name + " or plain object to verify",
+            "@returns {?string} `null` if valid, otherwise the reason why it is not"
+        ]);
+        buildFunction(type, "verify", protobuf.verify.generate(type), {
+            util : "$protobuf.util"
+        });
+
+    }
 }
 
 function buildService(ref, service) {