Project import generated by Copybara

PiperOrigin-RevId: 301311140
pull/7307/head
Rafi Kamal 5 years ago committed by Copybara-Service
parent 422053f3bc
commit da1c46401b
  1. 130
      js/experimental/runtime/kernel/binary_storage.js
  2. 30
      js/experimental/runtime/kernel/binary_storage_test.js
  3. 8
      js/experimental/runtime/kernel/indexer.js
  4. 6
      js/experimental/runtime/kernel/indexer_test.js
  5. 5
      js/experimental/runtime/kernel/kernel.js
  6. 112
      js/experimental/runtime/kernel/storage.js

@ -0,0 +1,130 @@
goog.module('protobuf.runtime.BinaryStorage');
const Storage = goog.require('protobuf.runtime.Storage');
const {checkDefAndNotNull} = goog.require('protobuf.internal.checks');
/**
* Class storing all the fields of a binary protobuf message.
*
* @package
* @template FieldType
* @implements {Storage}
*/
class BinaryStorage {
/**
* @param {number=} pivot
*/
constructor(pivot = Storage.DEFAULT_PIVOT) {
/**
* Fields having a field number no greater than the pivot value are stored
* into an array for fast access. A field with field number X is stored into
* the array position X - 1.
*
* @private @const {!Array<!FieldType|undefined>}
*/
this.array_ = new Array(pivot);
/**
* Fields having a field number higher than the pivot value are stored into
* the map. We create the map only when it's needed, since even an empty map
* takes up a significant amount of memory.
*
* @private {?Map<number, !FieldType>}
*/
this.map_ = null;
}
/**
* Fields having a field number no greater than the pivot value are stored
* into an array for fast access. A field with field number X is stored into
* the array position X - 1.
* @return {number}
* @override
*/
getPivot() {
return this.array_.length;
}
/**
* Sets a field in the specified field number.
*
* @param {number} fieldNumber
* @param {!FieldType} field
* @override
*/
set(fieldNumber, field) {
if (fieldNumber <= this.getPivot()) {
this.array_[fieldNumber - 1] = field;
} else {
if (this.map_) {
this.map_.set(fieldNumber, field);
} else {
this.map_ = new Map([[fieldNumber, field]]);
}
}
}
/**
* Returns a field at the specified field number.
*
* @param {number} fieldNumber
* @return {!FieldType|undefined}
* @override
*/
get(fieldNumber) {
if (fieldNumber <= this.getPivot()) {
return this.array_[fieldNumber - 1];
} else {
return this.map_ ? this.map_.get(fieldNumber) : undefined;
}
}
/**
* Deletes a field from the specified field number.
*
* @param {number} fieldNumber
* @override
*/
delete(fieldNumber) {
if (fieldNumber <= this.getPivot()) {
delete this.array_[fieldNumber - 1];
} else {
if (this.map_) {
this.map_.delete(fieldNumber);
}
}
}
/**
* Executes the provided function once for each field.
*
* @param {function(!FieldType, number): void} callback
* @override
*/
forEach(callback) {
this.array_.forEach((field, fieldNumber) => {
if (field) {
callback(checkDefAndNotNull(field), fieldNumber + 1);
}
});
if (this.map_) {
this.map_.forEach(callback);
}
}
/**
* Creates a shallow copy of the storage.
*
* @return {!BinaryStorage}
* @override
*/
shallowCopy() {
const copy = new BinaryStorage(this.getPivot());
this.forEach(
(field, fieldNumber) =>
void copy.set(fieldNumber, field.shallowCopy()));
return copy;
}
}
exports = BinaryStorage;

@ -1,11 +1,11 @@
/** /**
* @fileoverview Tests for storage.js. * @fileoverview Tests for storage.js.
*/ */
goog.module('protobuf.binary.StorageTest'); goog.module('protobuf.runtime.BinaryStorageTest');
goog.setTestOnly(); goog.setTestOnly();
const Storage = goog.require('protobuf.binary.Storage'); const BinaryStorage = goog.require('protobuf.runtime.BinaryStorage');
const {Field} = goog.require('protobuf.binary.field'); const {Field} = goog.require('protobuf.binary.field');
/** /**
@ -25,7 +25,7 @@ const /** !Field */ field4 =
/** /**
* Returns the number of fields stored. * Returns the number of fields stored.
* *
* @param {!Storage} storage * @param {!BinaryStorage} storage
* @return {number} * @return {number}
*/ */
function getStorageSize(storage) { function getStorageSize(storage) {
@ -34,9 +34,9 @@ function getStorageSize(storage) {
return size; return size;
} }
describe('Storage', () => { describe('BinaryStorage', () => {
it('sets and gets a field not greater than the pivot', () => { it('sets and gets a field not greater than the pivot', () => {
const storage = new Storage(DEFAULT_PIVOT); const storage = new BinaryStorage(DEFAULT_PIVOT);
storage.set(1, field1); storage.set(1, field1);
storage.set(DEFAULT_PIVOT, field2); storage.set(DEFAULT_PIVOT, field2);
@ -47,7 +47,7 @@ describe('Storage', () => {
}); });
it('sets and gets a field greater than the pivot', () => { it('sets and gets a field greater than the pivot', () => {
const storage = new Storage(DEFAULT_PIVOT); const storage = new BinaryStorage(DEFAULT_PIVOT);
storage.set(DEFAULT_PIVOT + 1, field1); storage.set(DEFAULT_PIVOT + 1, field1);
storage.set(100000, field2); storage.set(100000, field2);
@ -57,7 +57,7 @@ describe('Storage', () => {
}); });
it('sets and gets a field when pivot is zero', () => { it('sets and gets a field when pivot is zero', () => {
const storage = new Storage(0); const storage = new BinaryStorage(0);
storage.set(0, field1); storage.set(0, field1);
storage.set(100000, field2); storage.set(100000, field2);
@ -68,7 +68,7 @@ describe('Storage', () => {
}); });
it('sets and gets a field when pivot is undefined', () => { it('sets and gets a field when pivot is undefined', () => {
const storage = new Storage(); const storage = new BinaryStorage();
storage.set(0, field1); storage.set(0, field1);
storage.set(DEFAULT_PIVOT, field2); storage.set(DEFAULT_PIVOT, field2);
@ -81,7 +81,7 @@ describe('Storage', () => {
}); });
it('returns undefined for nonexistent fields', () => { it('returns undefined for nonexistent fields', () => {
const storage = new Storage(DEFAULT_PIVOT); const storage = new BinaryStorage(DEFAULT_PIVOT);
expect(storage.get(1)).toBeUndefined(); expect(storage.get(1)).toBeUndefined();
expect(storage.get(DEFAULT_PIVOT)).toBeUndefined(); expect(storage.get(DEFAULT_PIVOT)).toBeUndefined();
@ -91,7 +91,7 @@ describe('Storage', () => {
it('returns undefined for nonexistent fields after map initialization', it('returns undefined for nonexistent fields after map initialization',
() => { () => {
const storage = new Storage(DEFAULT_PIVOT); const storage = new BinaryStorage(DEFAULT_PIVOT);
storage.set(100001, field1); storage.set(100001, field1);
expect(storage.get(1)).toBeUndefined(); expect(storage.get(1)).toBeUndefined();
@ -101,7 +101,7 @@ describe('Storage', () => {
}); });
it('deletes a field in delete() when values are only in array', () => { it('deletes a field in delete() when values are only in array', () => {
const storage = new Storage(DEFAULT_PIVOT); const storage = new BinaryStorage(DEFAULT_PIVOT);
storage.set(1, field1); storage.set(1, field1);
storage.delete(1); storage.delete(1);
@ -111,7 +111,7 @@ describe('Storage', () => {
it('deletes a field in delete() when values are both in array and map', it('deletes a field in delete() when values are both in array and map',
() => { () => {
const storage = new Storage(DEFAULT_PIVOT); const storage = new BinaryStorage(DEFAULT_PIVOT);
storage.set(DEFAULT_PIVOT, field2); storage.set(DEFAULT_PIVOT, field2);
storage.set(DEFAULT_PIVOT + 1, field3); storage.set(DEFAULT_PIVOT + 1, field3);
@ -123,7 +123,7 @@ describe('Storage', () => {
}); });
it('deletes a field in delete() when values are only in map', () => { it('deletes a field in delete() when values are only in map', () => {
const storage = new Storage(DEFAULT_PIVOT); const storage = new BinaryStorage(DEFAULT_PIVOT);
storage.set(100000, field4); storage.set(100000, field4);
storage.delete(100000); storage.delete(100000);
@ -132,7 +132,7 @@ describe('Storage', () => {
}); });
it('loops over all the elements in forEach()', () => { it('loops over all the elements in forEach()', () => {
const storage = new Storage(DEFAULT_PIVOT); const storage = new BinaryStorage(DEFAULT_PIVOT);
storage.set(1, field1); storage.set(1, field1);
storage.set(DEFAULT_PIVOT, field2); storage.set(DEFAULT_PIVOT, field2);
storage.set(DEFAULT_PIVOT + 1, field3); storage.set(DEFAULT_PIVOT + 1, field3);
@ -150,7 +150,7 @@ describe('Storage', () => {
}); });
it('creates a shallow copy of the storage in shallowCopy()', () => { it('creates a shallow copy of the storage in shallowCopy()', () => {
const storage = new Storage(DEFAULT_PIVOT); const storage = new BinaryStorage(DEFAULT_PIVOT);
storage.set(1, field1); storage.set(1, field1);
storage.set(100000, field2); storage.set(100000, field2);

@ -4,15 +4,15 @@
*/ */
goog.module('protobuf.binary.indexer'); goog.module('protobuf.binary.indexer');
const BinaryStorage = goog.require('protobuf.runtime.BinaryStorage');
const BufferDecoder = goog.require('protobuf.binary.BufferDecoder'); const BufferDecoder = goog.require('protobuf.binary.BufferDecoder');
const Storage = goog.require('protobuf.binary.Storage');
const WireType = goog.require('protobuf.binary.WireType'); const WireType = goog.require('protobuf.binary.WireType');
const {Field} = goog.require('protobuf.binary.field'); const {Field} = goog.require('protobuf.binary.field');
const {checkCriticalElementIndex, checkCriticalState} = goog.require('protobuf.internal.checks'); const {checkCriticalElementIndex, checkCriticalState} = goog.require('protobuf.internal.checks');
/** /**
* Appends a new entry in the index array for the given field number. * Appends a new entry in the index array for the given field number.
* @param {!Storage<!Field>} storage * @param {!BinaryStorage<!Field>} storage
* @param {number} fieldNumber * @param {number} fieldNumber
* @param {!WireType} wireType * @param {!WireType} wireType
* @param {number} startIndex * @param {number} startIndex
@ -50,13 +50,13 @@ function tagToFieldNumber(tag) {
* Creates an index of field locations in a given binary protobuf. * Creates an index of field locations in a given binary protobuf.
* @param {!BufferDecoder} bufferDecoder * @param {!BufferDecoder} bufferDecoder
* @param {number|undefined} pivot * @param {number|undefined} pivot
* @return {!Storage<!Field>} * @return {!BinaryStorage<!Field>}
* @package * @package
*/ */
function buildIndex(bufferDecoder, pivot) { function buildIndex(bufferDecoder, pivot) {
bufferDecoder.setCursor(bufferDecoder.startIndex()); bufferDecoder.setCursor(bufferDecoder.startIndex());
const storage = new Storage(pivot); const storage = new BinaryStorage(pivot);
while (bufferDecoder.hasNext()) { while (bufferDecoder.hasNext()) {
const tag = bufferDecoder.getUnsignedVarint32(); const tag = bufferDecoder.getUnsignedVarint32();
const wireType = tagToWireType(tag); const wireType = tagToWireType(tag);

@ -10,8 +10,8 @@ goog.setTestOnly();
// in this file have to know which checking level is enabled to make correct // in this file have to know which checking level is enabled to make correct
// assertions. // assertions.
// Test are run in all checking levels. // Test are run in all checking levels.
const BinaryStorage = goog.require('protobuf.runtime.BinaryStorage');
const BufferDecoder = goog.require('protobuf.binary.BufferDecoder'); const BufferDecoder = goog.require('protobuf.binary.BufferDecoder');
const Storage = goog.require('protobuf.binary.Storage');
const WireType = goog.require('protobuf.binary.WireType'); const WireType = goog.require('protobuf.binary.WireType');
const {CHECK_CRITICAL_STATE} = goog.require('protobuf.internal.checks'); const {CHECK_CRITICAL_STATE} = goog.require('protobuf.internal.checks');
const {Field, IndexEntry} = goog.require('protobuf.binary.field'); const {Field, IndexEntry} = goog.require('protobuf.binary.field');
@ -21,7 +21,7 @@ const {createBufferDecoder} = goog.require('protobuf.binary.bufferDecoderHelper'
/** /**
* Returns the number of fields stored. * Returns the number of fields stored.
* *
* @param {!Storage} storage * @param {!BinaryStorage} storage
* @return {number} * @return {number}
*/ */
function getStorageSize(storage) { function getStorageSize(storage) {
@ -37,7 +37,7 @@ const PIVOT = 1;
/** /**
* Asserts a single IndexEntry at a given field number. * Asserts a single IndexEntry at a given field number.
* @param {!Storage} storage * @param {!BinaryStorage} storage
* @param {number} fieldNumber * @param {number} fieldNumber
* @param {...!IndexEntry} expectedEntries * @param {...!IndexEntry} expectedEntries
*/ */

@ -14,11 +14,12 @@
*/ */
goog.module('protobuf.runtime.Kernel'); goog.module('protobuf.runtime.Kernel');
const BinaryStorage = goog.require('protobuf.runtime.BinaryStorage');
const BufferDecoder = goog.require('protobuf.binary.BufferDecoder'); const BufferDecoder = goog.require('protobuf.binary.BufferDecoder');
const ByteString = goog.require('protobuf.ByteString'); const ByteString = goog.require('protobuf.ByteString');
const Int64 = goog.require('protobuf.Int64'); const Int64 = goog.require('protobuf.Int64');
const InternalMessage = goog.require('protobuf.binary.InternalMessage'); const InternalMessage = goog.require('protobuf.binary.InternalMessage');
const Storage = goog.require('protobuf.binary.Storage'); const Storage = goog.require('protobuf.runtime.Storage');
const WireType = goog.require('protobuf.binary.WireType'); const WireType = goog.require('protobuf.binary.WireType');
const Writer = goog.require('protobuf.binary.Writer'); const Writer = goog.require('protobuf.binary.Writer');
const reader = goog.require('protobuf.binary.reader'); const reader = goog.require('protobuf.binary.reader');
@ -278,7 +279,7 @@ class Kernel {
* @return {!Kernel} * @return {!Kernel}
*/ */
static createEmpty(pivot = undefined) { static createEmpty(pivot = undefined) {
return new Kernel(/* bufferDecoder= */ null, new Storage(pivot)); return new Kernel(/* bufferDecoder= */ null, new BinaryStorage(pivot));
} }
/** /**

@ -1,57 +1,19 @@
goog.module('protobuf.binary.Storage'); goog.module('protobuf.runtime.Storage');
const {checkDefAndNotNull} = goog.require('protobuf.internal.checks');
/**
* 85% of the proto fields have a field number <= 24:
* https://plx.corp.google.com/scripts2/script_5d._f02af6_0000_23b1_a15f_001a1139dd02
*
* @type {number}
*/
// LINT.IfChange
const DEFAULT_PIVOT = 24;
// LINT.ThenChange(//depot/google3/third_party/protobuf/javascript/runtime/kernel/storage_test.js,
// //depot/google3/net/proto2/contrib/js_proto/internal/kernel_message_generator.cc)
/** /**
* Class storing all the fields of a protobuf message. * Interface for getting and storing fields of a protobuf message.
* *
* @interface
* @package * @package
* @template FieldType * @template FieldType
*/ */
class Storage { class Storage {
/** /**
* @param {number=} pivot * Returns the pivot value.
*/ *
constructor(pivot = DEFAULT_PIVOT) {
/**
* Fields having a field number no greater than the pivot value are stored
* into an array for fast access. A field with field number X is stored into
* the array position X - 1.
*
* @private @const {!Array<!FieldType|undefined>}
*/
this.array_ = new Array(pivot);
/**
* Fields having a field number higher than the pivot value are stored into
* the map. We create the map only when it's needed, since even an empty map
* takes up a significant amount of memory.
*
* @private {?Map<number, !FieldType>}
*/
this.map_ = null;
}
/**
* Fields having a field number no greater than the pivot value are stored
* into an array for fast access. A field with field number X is stored into
* the array position X - 1.
* @return {number} * @return {number}
*/ */
getPivot() { getPivot() {}
return this.array_.length;
}
/** /**
* Sets a field in the specified field number. * Sets a field in the specified field number.
@ -59,17 +21,7 @@ class Storage {
* @param {number} fieldNumber * @param {number} fieldNumber
* @param {!FieldType} field * @param {!FieldType} field
*/ */
set(fieldNumber, field) { set(fieldNumber, field) {}
if (fieldNumber <= this.getPivot()) {
this.array_[fieldNumber - 1] = field;
} else {
if (this.map_) {
this.map_.set(fieldNumber, field);
} else {
this.map_ = new Map([[fieldNumber, field]]);
}
}
}
/** /**
* Returns a field at the specified field number. * Returns a field at the specified field number.
@ -77,57 +29,39 @@ class Storage {
* @param {number} fieldNumber * @param {number} fieldNumber
* @return {!FieldType|undefined} * @return {!FieldType|undefined}
*/ */
get(fieldNumber) { get(fieldNumber) {}
if (fieldNumber <= this.getPivot()) {
return this.array_[fieldNumber - 1];
} else {
return this.map_ ? this.map_.get(fieldNumber) : undefined;
}
}
/** /**
* Deletes a field from the specified field number. * Deletes a field from the specified field number.
* *
* @param {number} fieldNumber * @param {number} fieldNumber
*/ */
delete(fieldNumber) { delete(fieldNumber) {}
if (fieldNumber <= this.getPivot()) {
delete this.array_[fieldNumber - 1];
} else {
if (this.map_) {
this.map_.delete(fieldNumber);
}
}
}
/** /**
* Executes the provided function once for each array element. * Executes the provided function once for each field.
* *
* @param {function(!FieldType, number): void} callback * @param {function(!FieldType, number): void} callback
*/ */
forEach(callback) { forEach(callback) {}
this.array_.forEach((field, fieldNumber) => {
if (field) {
callback(checkDefAndNotNull(field), fieldNumber + 1);
}
});
if (this.map_) {
this.map_.forEach(callback);
}
}
/** /**
* Creates a shallow copy of the storage. * Creates a shallow copy of the storage.
* *
* @return {!Storage} * @return {!Storage}
*/ */
shallowCopy() { shallowCopy() {}
const copy = new Storage(this.getPivot());
this.forEach(
(field, fieldNumber) =>
void copy.set(fieldNumber, field.shallowCopy()));
return copy;
}
} }
/**
* 85% of the proto fields have a field number <= 24:
* https://plx.corp.google.com/scripts2/script_5d._f02af6_0000_23b1_a15f_001a1139dd02
*
* @type {number}
*/
// LINT.IfChange
Storage.DEFAULT_PIVOT = 24;
// LINT.ThenChange(//depot/google3/third_party/protobuf/javascript/runtime/kernel/binary_storage_test.js,
// //depot/google3/net/proto2/contrib/js_proto/internal/kernel_message_generator.cc)
exports = Storage; exports = Storage;

Loading…
Cancel
Save