Add unknown fields support to Protobuf.js

components/protobuf/dist/ProtoBuf.js

@@ -15,14 +15,14 @@
  */
 
 /**
- * @license ProtoBuf.js (c) 2013 Daniel Wirtz <dcode@dcode.io>
+ * @license protobuf.js (c) 2013 Daniel Wirtz <dcode@dcode.io>
  * Released under the Apache License, Version 2.0
- * see: https://github.com/dcodeIO/ProtoBuf.js for details
+ * see: https://github.com/dcodeIO/protobuf.js for details
  */
 (function(global, factory) {
 
     /* AMD */ if (typeof define === 'function' && define["amd"])
-        define(["ByteBuffer"], factory);
+        define(["bytebuffer"], factory);
     /* CommonJS */ else if (typeof require === "function" && typeof module === "object" && module && module["exports"])
         module["exports"] = factory(require("bytebuffer"), true);
     /* Global */ else
@@ -57,7 +57,7 @@
      * @const
      * @expose
      */
-    ProtoBuf.VERSION = "4.1.2";
+    ProtoBuf.VERSION = "5.0.1";
 
     /**
      * Wire types.
@@ -725,7 +725,8 @@
                 // "syntax": undefined
             };
             var token,
-                head = true;
+                head = true,
+                weak;
             try {
                 while (token = this.tn.next()) {
                     switch (token) {
@@ -742,11 +743,12 @@
                             if (!head)
                                 throw Error("unexpected 'import'");
                             token = this.tn.peek();
-                            if (token === "public") // ignored
+                            if (token === "public" || (weak = token === "weak")) // token ignored
                                 this.tn.next();
                             token = this._readString();
                             this.tn.skip(";");
-                            topLevel["imports"].push(token);
+                            if (!weak) // import ignored
+                                topLevel["imports"].push(token);
                             break;
                         case 'syntax':
                             if (!head)
@@ -1071,6 +1073,7 @@
                 "enums": [],
                 "messages": [],
                 "options": {},
+                "services": [],
                 "oneofs": {}
                 // "extensions": undefined
             };
@@ -1097,8 +1100,12 @@
                     this._parseMessage(msg);
                 else if (token === "option")
                     this._parseOption(msg);
+                else if (token === "service")
+                    this._parseService(msg);
                 else if (token === "extensions")
-                    this._parseExtensions(msg);
+                    msg["extensions"] = this._parseExtensionRanges();
+                else if (token === "reserved")
+                    this._parseIgnored(); // TODO
                 else if (token === "extend")
                     this._parseExtend(msg);
                 else if (Lang.TYPEREF.test(token)) {
@@ -1113,6 +1120,16 @@
             return msg;
         };
 
+        /**
+         * Parses an ignored statement.
+         * @private
+         */
+        ParserPrototype._parseIgnored = function() {
+            while (this.tn.peek() !== ';')
+                this.tn.next();
+            this.tn.skip(";");
+        };
+
         /**
          * Parses a message field.
          * @param {!Object} msg Message definition
@@ -1275,29 +1292,43 @@
         };
 
         /**
-         * Parses an extensions statement.
+         * Parses extension / reserved ranges.
-         * @param {!Object} msg Message object
+         * @returns {!Array.<!Array.<number>>}
          * @private
          */
-        ParserPrototype._parseExtensions = function(msg) {
+        ParserPrototype._parseExtensionRanges = function() {
-            var token = this.tn.next(),
+            var ranges = [];
+            var token,
+                range,
+                value;
+            do {
                 range = [];
-            if (token === "min")
+                while (true) {
-                range.push(ProtoBuf.ID_MIN);
+                    token = this.tn.next();
-            else if (token === "max")
+                    switch (token) {
-                range.push(ProtoBuf.ID_MAX);
+                        case "min":
-            else
+                            value = ProtoBuf.ID_MIN;
-                range.push(mkNumber(token));
+                            break;
-            this.tn.skip("to");
+                        case "max":
-            token = this.tn.next();
+                            value = ProtoBuf.ID_MAX;
-            if (token === "min")
+                            break;
-                range.push(ProtoBuf.ID_MIN);
+                        default:
-            else if (token === "max")
+                            value = mkNumber(token);
-                range.push(ProtoBuf.ID_MAX);
+                            break;
-            else
+                    }
-                range.push(mkNumber(token));
+                    range.push(value);
+                    if (range.length === 2)
+                        break;
+                    if (this.tn.peek() !== "to") {
+                        range.push(value);
+                        break;
+                    }
+                    this.tn.next();
+                }
+                ranges.push(range);
+            } while (this.tn.omit(","));
             this.tn.skip(";");
-            msg["extensions"] = range;
+            return ranges;
         };
 
         /**
@@ -1765,9 +1796,10 @@
          * @expose
          */
         ElementPrototype.verifyValue = function(value) {
-            var fail = function(val, msg) {
+            var self = this;
-                throw Error("Illegal value for "+this.toString(true)+" of type "+this.type.name+": "+val+" ("+msg+")");
+            function fail(val, msg) {
-            }.bind(this);
+                throw Error("Illegal value for "+self.toString(true)+" of type "+self.type.name+": "+val+" ("+msg+")");
+            }
             switch (this.type) {
                 // Signed 32bit
                 case ProtoBuf.TYPES["int32"]:
@@ -2257,10 +2289,10 @@
 
             /**
              * Extensions range.
-             * @type {!Array.<number>}
+             * @type {!Array.<number>|undefined}
              * @expose
              */
-            this.extensions = [ProtoBuf.ID_MIN, ProtoBuf.ID_MAX];
+            this.extensions = undefined;
 
             /**
              * Runtime message class.
@@ -2374,6 +2406,14 @@
                  */
                 var MessagePrototype = Message.prototype = Object.create(ProtoBuf.Builder.Message.prototype);
 
+
+                Object.defineProperty(MessagePrototype, '__unknownFields', {
+                    configurable: true,
+                    enumerable: false,
+                    value: null,
+                    writable: true,
+                });
+
                 /**
                  * Adds a value to a repeated field.
                  * @name ProtoBuf.Builder.Message#add
@@ -2659,18 +2699,19 @@
                  * @name ProtoBuf.Builder.Message#encodeDelimited
                  * @function
                  * @param {(!ByteBuffer|boolean)=} buffer ByteBuffer to encode to. Will create a new one and flip it if omitted.
+                 * @param {boolean=} noVerify Whether to not verify field values, defaults to `false`
                  * @return {!ByteBuffer} Encoded message as a ByteBuffer
                  * @throws {Error} If the message cannot be encoded or if required fields are missing. The later still
                  *  returns the encoded ByteBuffer in the `encoded` property on the error.
                  * @expose
                  */
-                MessagePrototype.encodeDelimited = function(buffer) {
+                MessagePrototype.encodeDelimited = function(buffer, noVerify) {
                     var isNew = false;
                     if (!buffer)
                         buffer = new ByteBuffer(),
                         isNew = true;
                     var enc = new ByteBuffer().LE();
-                    T.encode(this, enc).flip();
+                    T.encode(this, enc, noVerify).flip();
                     buffer.writeVarint32(enc.remaining());
                     buffer.append(enc);
                     return isNew ? buffer.flip() : buffer;
@@ -2817,7 +2858,7 @@
                         return binaryAsBase64 ? obj.toBase64() : obj.toBuffer();
                     // Convert Longs to proper objects or strings
                     if (ProtoBuf.Long.isLong(obj))
-                        return longsAsStrings ? obj.toString() : new ProtoBuf.Long(obj);
+                        return longsAsStrings ? obj.toString() : ProtoBuf.Long.fromValue(obj);
                     var clone;
                     // Clone arrays
                     if (Array.isArray(obj)) {
@@ -2879,6 +2920,7 @@
                  * @name ProtoBuf.Builder.Message.decode
                  * @function
                  * @param {!ByteBuffer|!ArrayBuffer|!Buffer|string} buffer Buffer to decode from
+                 * @param {(number|string)=} length Message length. Defaults to decode all the remainig data.
                  * @param {string=} enc Encoding if buffer is a string: hex, utf8 (not recommended), defaults to base64
                  * @return {!ProtoBuf.Builder.Message} Decoded message
                  * @throws {Error} If the message cannot be decoded or if required fields are missing. The later still
@@ -2887,7 +2929,10 @@
                  * @see ProtoBuf.Builder.Message.decode64
                  * @see ProtoBuf.Builder.Message.decodeHex
                  */
-                Message.decode = function(buffer, enc) {
+                Message.decode = function(buffer, length, enc) {
+                    if (typeof length === 'string')
+                        enc = length,
+                        length = -1;
                     if (typeof buffer === 'string')
                         buffer = ByteBuffer.wrap(buffer, enc ? enc : "base64");
                     buffer = ByteBuffer.isByteBuffer(buffer) ? buffer : ByteBuffer.wrap(buffer); // May throw
@@ -3082,6 +3127,9 @@
                 err["encoded"] = buffer; // Still expose what we got
                 throw(err);
             }
+            if (message.__unknownFields) {
+                buffer.append(message.__unknownFields);
+            }
             return buffer;
         };
 
@@ -3148,7 +3196,7 @@
         /**
          * Decodes an encoded message and returns the decoded message.
          * @param {ByteBuffer} buffer ByteBuffer to decode from
-         * @param {number=} length Message length. Defaults to decode all the available data.
+         * @param {number=} length Message length. Defaults to decode all remaining data.
          * @param {number=} expectedGroupEndId Expected GROUPEND id if this is a legacy group
          * @return {ProtoBuf.Builder.Message} Decoded message
          * @throws {Error} If the message cannot be decoded
@@ -3168,8 +3216,16 @@
                         throw Error("Illegal group end indicator for "+this.toString(true)+": "+id+" ("+(expectedGroupEndId ? expectedGroupEndId+" expected" : "not a group")+")");
                     break;
                 }
+                // "messages created by your new code can be parsed by your old code: old binaries simply append the buffer to unknownFields when parsing.
                 if (!(field = this._fieldsById[id])) {
-                    // "messages created by your new code can be parsed by your old code: old binaries simply ignore the new field when parsing."
+                    // Finds the starting offset to slice
+                    let start = buffer.offset;
+                    do {
+                        --start;
+                        buffer.offset = start;
+                    } while (buffer.readVarint32() !== tag);
+
+                    // Skip the piece in the buffer
                     switch (wireType) {
                         case ProtoBuf.WIRE_TYPES.VARINT:
                             buffer.readVarint32();
@@ -3190,6 +3246,14 @@
                         default:
                             throw Error("Illegal wire type for unknown field "+id+" in "+this.toString(true)+"#decode: "+wireType);
                     }
+
+                    // Slice the part of the buffer we can't parse and add it to unknownFields
+                    const unknownFields = msg.__unknownFields ? msg.__unknownFields : new ByteBuffer(0);
+                    const slicedBuffer = buffer.slice(start, buffer.offset);
+                    msg.__unknownFields = ByteBuffer.concat([
+                        unknownFields,
+                        slicedBuffer
+                    ]);
                     continue;
                 }
                 if (field.repeated && !field.options["packed"]) {
@@ -3398,9 +3462,10 @@
          */
         FieldPrototype.verifyValue = function(value, skipRepeated) {
             skipRepeated = skipRepeated || false;
-            var fail = function(val, msg) {
+            var self = this;
-                throw Error("Illegal value for "+this.toString(true)+" of type "+this.type.name+": "+val+" ("+msg+")");
+            function fail(val, msg) {
-            }.bind(this);
+                throw Error("Illegal value for "+self.toString(true)+" of type "+self.type.name+": "+val+" ("+msg+")");
+            }
             if (value === null) { // NULL values for optional fields
                 if (this.required)
                     fail(typeof value, "required");
@@ -4014,6 +4079,9 @@
                                         callback(err);
                                         return;
                                     }
+                                    // Coalesce to empty string when service response has empty content
+                                    if (res === null)
+                                        res = ''
                                     try { res = method.resolvedResponseType.clazz.decode(res); } catch (notABuffer) {}
                                     if (!res || !(res instanceof method.resolvedResponseType.clazz)) {
                                         callback(Error("Illegal response type received in service method "+ T.name+"#"+method.name));
@@ -4457,13 +4525,12 @@
                                 subObj.push(svc);
                             });
 
-                        // Set extension range
+                        // Set extension ranges
                         if (def["extensions"]) {
-                            obj.extensions = def["extensions"];
+                            if (typeof def["extensions"][0] === 'number') // pre 5.0.1
-                            if (obj.extensions[0] < ProtoBuf.ID_MIN)
+                                obj.extensions = [ def["extensions"] ];
-                                obj.extensions[0] = ProtoBuf.ID_MIN;
+                            else
-                            if (obj.extensions[1] > ProtoBuf.ID_MAX)
+                                obj.extensions = def["extensions"];
-                                obj.extensions[1] = ProtoBuf.ID_MAX;
                         }
 
                         // Create on top of current namespace
@@ -4502,8 +4569,16 @@
                             def["fields"].forEach(function(fld) {
                                 if (obj.getChild(fld['id']|0) !== null)
                                     throw Error("duplicate extended field id in "+obj.name+": "+fld['id']);
-                                if (fld['id'] < obj.extensions[0] || fld['id'] > obj.extensions[1])
+                                // Check if field id is allowed to be extended
-                                    throw Error("illegal extended field id in "+obj.name+": "+fld['id']+" ("+obj.extensions.join(' to ')+" expected)");
+                                if (obj.extensions) {
+                                    var valid = false;
+                                    obj.extensions.forEach(function(range) {
+                                        if (fld["id"] >= range[0] && fld["id"] <= range[1])
+                                            valid = true;
+                                    });
+                                    if (!valid)
+                                        throw Error("illegal extended field id in "+obj.name+": "+fld['id']+" (not within valid ranges)");
+                                }
                                 // Convert extension field names to camel case notation if the override is set
                                 var name = fld["name"];
                                 if (this.options['convertFieldsToCamelCase'])

test/index.html

@@ -385,6 +385,7 @@
   <script type="text/javascript" src="crypto_test.js"></script>
   <script type="text/javascript" src="database_test.js"></script>
   <script type="text/javascript" src="i18n_test.js"></script>
+  <script type="text/javascript" src="protobuf_test.js"></script>
 
   <!-- Comment out to turn off code coverage. Useful for getting real callstacks. -->
   <!--   NOTE: blanket doesn't support modern syntax and will choke until we find a replacement. :0( -->

test/protobuf_test.js

@@ -0,0 +1,123 @@
+describe('ProtoBuf.js', () => {
+  const { ProtoBuf } = window.dcodeIO;
+
+  const sampleProto = `message Simple_v1 {
+    optional string knownName  = 1;
+    optional string knownValue = 3;
+  }
+  message Simple_v2 {
+    optional string knownName  = 1;
+    optional int32  unknownFlags = 2;
+    optional string knownValue = 3;
+    optional string unknownString = 4;
+  }`;
+
+  it('retains unknown fields', () => {
+    const builder = ProtoBuf.loadProto(sampleProto);
+    const protos = builder.build();
+
+    const v2 = new protos.Simple_v2();
+    v2.knownName = 'version2';
+    v2.unknownFlags = 42;
+    v2.knownValue = 'known value';
+    v2.unknownString = 'f';
+
+    const v1 = protos.Simple_v1.decode(v2.encode());
+
+    const result = protos.Simple_v2.decode(v1.encode());
+
+    assert.equal(result.knownName, v2.knownName, 'known fields');
+    assert.equal(42, result.unknownFlags, 'unknown flag');
+    assert.equal('f', result.unknownString, 'unknown string');
+    assert.equal('known value', result.knownValue, 'known value');
+  });
+
+  it('supports nested unknown fields', () => {
+    const nestedProto = `
+    ${sampleProto}
+    message Container_v1 {
+      optional Simple_v1 elem = 1;
+    }
+    message Container_v2 {
+      optional Simple_v2 elem = 1;
+    }`;
+
+    const builder = ProtoBuf.loadProto(nestedProto);
+    const protos = builder.build();
+
+    const v2 = new protos.Container_v2();
+    v2.elem = {
+      knownName: 'nested v2',
+      unknownFlags: 10,
+      knownValue: 'hello world',
+    };
+
+    const v1 = protos.Container_v1.decode(v2.encode());
+
+    const result = protos.Container_v2.decode(v1.encode());
+
+    assert.equal(
+      v2.elem.knownName,
+      result.elem.knownName,
+      'nested: known fields'
+    );
+    assert.equal(10, result.elem.unknownFlags, 'nested: unknown flags');
+    assert.equal('hello world', result.elem.knownValue, 'known value');
+  });
+
+  it('allows multi-byte id', () => {
+    const proto = `message Simple_v1 {
+      optional string knownName  = 1;
+      optional string knownValue = 3;
+    }
+    message Simple_v2 {
+      optional string knownName  = 1;
+      optional int32  unknownFlags = 296;
+      optional string knownValue = 3;
+    }`;
+
+    const builder = ProtoBuf.loadProto(proto);
+    const protos = builder.build();
+
+    const v2 = new protos.Simple_v2();
+    v2.knownName = 'v2 multibyte';
+    v2.unknownFlags = 16;
+    v2.knownValue = 'foo bar';
+
+    const v1 = protos.Simple_v1.decode(v2.encode());
+
+    const result = protos.Simple_v2.decode(v1.encode());
+
+    assert.equal(result.knownName, v2.knownName, 'multibyte: known fields');
+    assert.equal(16, result.unknownFlags, 'multibyte: unknown fields');
+    assert.equal('foo bar', result.knownValue, 'multibyte: known value');
+  });
+
+  it('retains fields with 64bit type', () => {
+    const proto = `message Simple_v1 {
+      optional string knownName  = 1;
+      optional string knownValue = 3;
+    }
+    message Simple_v2 {
+      optional string knownName  = 1;
+      optional double unknownFlags = 2;
+      optional string knownValue = 3;
+    }`;
+
+    const builder = ProtoBuf.loadProto(proto);
+    const protos = builder.build();
+
+    const v2 = new protos.Simple_v2();
+    v2.knownName = 'v2 double';
+    v2.unknownFlags = 0;
+    v2.knownValue = 'double double';
+
+    const v1 = protos.Simple_v1.decode(v2.encode());
+
+    const result = protos.Simple_v2.decode(v1.encode());
+
+    assert.equal(result.knownName, v2.knownName, 'double: known fields');
+    assert.equal(0, result.unknownFlags, 'double: unknown fields');
+    assert.equal('double double', result.knownValue, 'double: known value');
+  });
+});