index.js 15 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586
  1. /* Copyright 2015-present Facebook, Inc.
  2. * Licensed under the Apache License, Version 2.0 */
  3. var EE = require('events').EventEmitter;
  4. var util = require('util');
  5. var os = require('os');
  6. var assert = require('assert');
  7. var Int64 = require('node-int64');
  8. // BSER uses the local endianness to reduce byte swapping overheads
  9. // (the protocol is expressly local IPC only). We need to tell node
  10. // to use the native endianness when reading various native values.
  11. var isBigEndian = os.endianness() == 'BE';
  12. // Find the next power-of-2 >= size
  13. function nextPow2(size) {
  14. return Math.pow(2, Math.ceil(Math.log(size) / Math.LN2));
  15. }
  16. // Expandable buffer that we can provide a size hint for
  17. function Accumulator(initsize) {
  18. this.buf = new Buffer(nextPow2(initsize || 8192));
  19. this.readOffset = 0;
  20. this.writeOffset = 0;
  21. }
  22. // For testing
  23. exports.Accumulator = Accumulator
  24. // How much we can write into this buffer without allocating
  25. Accumulator.prototype.writeAvail = function() {
  26. return this.buf.length - this.writeOffset;
  27. }
  28. // How much we can read
  29. Accumulator.prototype.readAvail = function() {
  30. return this.writeOffset - this.readOffset;
  31. }
  32. // Ensure that we have enough space for size bytes
  33. Accumulator.prototype.reserve = function(size) {
  34. if (size < this.writeAvail()) {
  35. return;
  36. }
  37. // If we can make room by shunting down, do so
  38. if (this.readOffset > 0) {
  39. this.buf.copy(this.buf, 0, this.readOffset, this.writeOffset);
  40. this.writeOffset -= this.readOffset;
  41. this.readOffset = 0;
  42. }
  43. // If we made enough room, no need to allocate more
  44. if (size < this.writeAvail()) {
  45. return;
  46. }
  47. // Allocate a replacement and copy it in
  48. var buf = new Buffer(nextPow2(this.buf.length + size - this.writeAvail()));
  49. this.buf.copy(buf);
  50. this.buf = buf;
  51. }
  52. // Append buffer or string. Will resize as needed
  53. Accumulator.prototype.append = function(buf) {
  54. if (Buffer.isBuffer(buf)) {
  55. this.reserve(buf.length);
  56. buf.copy(this.buf, this.writeOffset, 0, buf.length);
  57. this.writeOffset += buf.length;
  58. } else {
  59. var size = Buffer.byteLength(buf);
  60. this.reserve(size);
  61. this.buf.write(buf, this.writeOffset);
  62. this.writeOffset += size;
  63. }
  64. }
  65. Accumulator.prototype.assertReadableSize = function(size) {
  66. if (this.readAvail() < size) {
  67. throw new Error("wanted to read " + size +
  68. " bytes but only have " + this.readAvail());
  69. }
  70. }
  71. Accumulator.prototype.peekString = function(size) {
  72. this.assertReadableSize(size);
  73. return this.buf.toString('utf-8', this.readOffset, this.readOffset + size);
  74. }
  75. Accumulator.prototype.readString = function(size) {
  76. var str = this.peekString(size);
  77. this.readOffset += size;
  78. return str;
  79. }
  80. Accumulator.prototype.peekInt = function(size) {
  81. this.assertReadableSize(size);
  82. switch (size) {
  83. case 1:
  84. return this.buf.readInt8(this.readOffset, size);
  85. case 2:
  86. return isBigEndian ?
  87. this.buf.readInt16BE(this.readOffset, size) :
  88. this.buf.readInt16LE(this.readOffset, size);
  89. case 4:
  90. return isBigEndian ?
  91. this.buf.readInt32BE(this.readOffset, size) :
  92. this.buf.readInt32LE(this.readOffset, size);
  93. case 8:
  94. var big = this.buf.slice(this.readOffset, this.readOffset + 8);
  95. if (isBigEndian) {
  96. // On a big endian system we can simply pass the buffer directly
  97. return new Int64(big);
  98. }
  99. // Otherwise we need to byteswap
  100. return new Int64(byteswap64(big));
  101. default:
  102. throw new Error("invalid integer size " + size);
  103. }
  104. }
  105. Accumulator.prototype.readInt = function(bytes) {
  106. var ival = this.peekInt(bytes);
  107. if (ival instanceof Int64 && isFinite(ival.valueOf())) {
  108. ival = ival.valueOf();
  109. }
  110. this.readOffset += bytes;
  111. return ival;
  112. }
  113. Accumulator.prototype.peekDouble = function() {
  114. this.assertReadableSize(8);
  115. return isBigEndian ?
  116. this.buf.readDoubleBE(this.readOffset) :
  117. this.buf.readDoubleLE(this.readOffset);
  118. }
  119. Accumulator.prototype.readDouble = function() {
  120. var dval = this.peekDouble();
  121. this.readOffset += 8;
  122. return dval;
  123. }
  124. Accumulator.prototype.readAdvance = function(size) {
  125. if (size > 0) {
  126. this.assertReadableSize(size);
  127. } else if (size < 0 && this.readOffset + size < 0) {
  128. throw new Error("advance with negative offset " + size +
  129. " would seek off the start of the buffer");
  130. }
  131. this.readOffset += size;
  132. }
  133. Accumulator.prototype.writeByte = function(value) {
  134. this.reserve(1);
  135. this.buf.writeInt8(value, this.writeOffset);
  136. ++this.writeOffset;
  137. }
  138. Accumulator.prototype.writeInt = function(value, size) {
  139. this.reserve(size);
  140. switch (size) {
  141. case 1:
  142. this.buf.writeInt8(value, this.writeOffset);
  143. break;
  144. case 2:
  145. if (isBigEndian) {
  146. this.buf.writeInt16BE(value, this.writeOffset);
  147. } else {
  148. this.buf.writeInt16LE(value, this.writeOffset);
  149. }
  150. break;
  151. case 4:
  152. if (isBigEndian) {
  153. this.buf.writeInt32BE(value, this.writeOffset);
  154. } else {
  155. this.buf.writeInt32LE(value, this.writeOffset);
  156. }
  157. break;
  158. default:
  159. throw new Error("unsupported integer size " + size);
  160. }
  161. this.writeOffset += size;
  162. }
  163. Accumulator.prototype.writeDouble = function(value) {
  164. this.reserve(8);
  165. if (isBigEndian) {
  166. this.buf.writeDoubleBE(value, this.writeOffset);
  167. } else {
  168. this.buf.writeDoubleLE(value, this.writeOffset);
  169. }
  170. this.writeOffset += 8;
  171. }
  172. var BSER_ARRAY = 0x00;
  173. var BSER_OBJECT = 0x01;
  174. var BSER_STRING = 0x02;
  175. var BSER_INT8 = 0x03;
  176. var BSER_INT16 = 0x04;
  177. var BSER_INT32 = 0x05;
  178. var BSER_INT64 = 0x06;
  179. var BSER_REAL = 0x07;
  180. var BSER_TRUE = 0x08;
  181. var BSER_FALSE = 0x09;
  182. var BSER_NULL = 0x0a;
  183. var BSER_TEMPLATE = 0x0b;
  184. var BSER_SKIP = 0x0c;
  185. var ST_NEED_PDU = 0; // Need to read and decode PDU length
  186. var ST_FILL_PDU = 1; // Know the length, need to read whole content
  187. var MAX_INT8 = 127;
  188. var MAX_INT16 = 32767;
  189. var MAX_INT32 = 2147483647;
  190. function BunserBuf() {
  191. EE.call(this);
  192. this.buf = new Accumulator();
  193. this.state = ST_NEED_PDU;
  194. }
  195. util.inherits(BunserBuf, EE);
  196. exports.BunserBuf = BunserBuf;
  197. BunserBuf.prototype.append = function(buf, synchronous) {
  198. if (synchronous) {
  199. this.buf.append(buf);
  200. return this.process(synchronous);
  201. }
  202. try {
  203. this.buf.append(buf);
  204. } catch (err) {
  205. this.emit('error', err);
  206. return;
  207. }
  208. // Arrange to decode later. This allows the consuming
  209. // application to make progress with other work in the
  210. // case that we have a lot of subscription updates coming
  211. // in from a large tree.
  212. this.processLater();
  213. }
  214. BunserBuf.prototype.processLater = function() {
  215. var self = this;
  216. process.nextTick(function() {
  217. try {
  218. self.process(false);
  219. } catch (err) {
  220. self.emit('error', err);
  221. }
  222. });
  223. }
  224. // Do something with the buffer to advance our state.
  225. // If we're running synchronously we'll return either
  226. // the value we've decoded or undefined if we don't
  227. // yet have enought data.
  228. // If we're running asynchronously, we'll emit the value
  229. // when it becomes ready and schedule another invocation
  230. // of process on the next tick if we still have data we
  231. // can process.
  232. BunserBuf.prototype.process = function(synchronous) {
  233. if (this.state == ST_NEED_PDU) {
  234. if (this.buf.readAvail() < 2) {
  235. return;
  236. }
  237. // Validate BSER header
  238. this.expectCode(0);
  239. this.expectCode(1);
  240. this.pduLen = this.decodeInt(true /* relaxed */);
  241. if (this.pduLen === false) {
  242. // Need more data, walk backwards
  243. this.buf.readAdvance(-2);
  244. return;
  245. }
  246. // Ensure that we have a big enough buffer to read the rest of the PDU
  247. this.buf.reserve(this.pduLen);
  248. this.state = ST_FILL_PDU;
  249. }
  250. if (this.state == ST_FILL_PDU) {
  251. if (this.buf.readAvail() < this.pduLen) {
  252. // Need more data
  253. return;
  254. }
  255. // We have enough to decode it
  256. var val = this.decodeAny();
  257. if (synchronous) {
  258. return val;
  259. }
  260. this.emit('value', val);
  261. this.state = ST_NEED_PDU;
  262. }
  263. if (!synchronous && this.buf.readAvail() > 0) {
  264. this.processLater();
  265. }
  266. }
  267. BunserBuf.prototype.raise = function(reason) {
  268. throw new Error(reason + ", in Buffer of length " +
  269. this.buf.buf.length + " (" + this.buf.readAvail() +
  270. " readable) at offset " + this.buf.readOffset + " buffer: " +
  271. JSON.stringify(this.buf.buf.slice(
  272. this.buf.readOffset, this.buf.readOffset + 32).toJSON()));
  273. }
  274. BunserBuf.prototype.expectCode = function(expected) {
  275. var code = this.buf.readInt(1);
  276. if (code != expected) {
  277. this.raise("expected bser opcode " + expected + " but got " + code);
  278. }
  279. }
  280. BunserBuf.prototype.decodeAny = function() {
  281. var code = this.buf.peekInt(1);
  282. switch (code) {
  283. case BSER_INT8:
  284. case BSER_INT16:
  285. case BSER_INT32:
  286. case BSER_INT64:
  287. return this.decodeInt();
  288. case BSER_REAL:
  289. this.buf.readAdvance(1);
  290. return this.buf.readDouble();
  291. case BSER_TRUE:
  292. this.buf.readAdvance(1);
  293. return true;
  294. case BSER_FALSE:
  295. this.buf.readAdvance(1);
  296. return false;
  297. case BSER_NULL:
  298. this.buf.readAdvance(1);
  299. return null;
  300. case BSER_STRING:
  301. return this.decodeString();
  302. case BSER_ARRAY:
  303. return this.decodeArray();
  304. case BSER_OBJECT:
  305. return this.decodeObject();
  306. case BSER_TEMPLATE:
  307. return this.decodeTemplate();
  308. default:
  309. this.raise("unhandled bser opcode " + code);
  310. }
  311. }
  312. BunserBuf.prototype.decodeArray = function() {
  313. this.expectCode(BSER_ARRAY);
  314. var nitems = this.decodeInt();
  315. var arr = [];
  316. for (var i = 0; i < nitems; ++i) {
  317. arr.push(this.decodeAny());
  318. }
  319. return arr;
  320. }
  321. BunserBuf.prototype.decodeObject = function() {
  322. this.expectCode(BSER_OBJECT);
  323. var nitems = this.decodeInt();
  324. var res = {};
  325. for (var i = 0; i < nitems; ++i) {
  326. var key = this.decodeString();
  327. var val = this.decodeAny();
  328. res[key] = val;
  329. }
  330. return res;
  331. }
  332. BunserBuf.prototype.decodeTemplate = function() {
  333. this.expectCode(BSER_TEMPLATE);
  334. var keys = this.decodeArray();
  335. var nitems = this.decodeInt();
  336. var arr = [];
  337. for (var i = 0; i < nitems; ++i) {
  338. var obj = {};
  339. for (var keyidx = 0; keyidx < keys.length; ++keyidx) {
  340. if (this.buf.peekInt(1) == BSER_SKIP) {
  341. this.buf.readAdvance(1);
  342. continue;
  343. }
  344. var val = this.decodeAny();
  345. obj[keys[keyidx]] = val;
  346. }
  347. arr.push(obj);
  348. }
  349. return arr;
  350. }
  351. BunserBuf.prototype.decodeString = function() {
  352. this.expectCode(BSER_STRING);
  353. var len = this.decodeInt();
  354. return this.buf.readString(len);
  355. }
  356. // This is unusual compared to the other decode functions in that
  357. // we may not have enough data available to satisfy the read, and
  358. // we don't want to throw. This is only true when we're reading
  359. // the PDU length from the PDU header; we'll set relaxSizeAsserts
  360. // in that case.
  361. BunserBuf.prototype.decodeInt = function(relaxSizeAsserts) {
  362. if (relaxSizeAsserts && !this.buf.readAvail(1)) {
  363. return false;
  364. } else {
  365. this.buf.assertReadableSize(1);
  366. }
  367. var code = this.buf.peekInt(1);
  368. var size = 0;
  369. switch (code) {
  370. case BSER_INT8:
  371. size = 1;
  372. break;
  373. case BSER_INT16:
  374. size = 2;
  375. break;
  376. case BSER_INT32:
  377. size = 4;
  378. break;
  379. case BSER_INT64:
  380. size = 8;
  381. break;
  382. default:
  383. this.raise("invalid bser int encoding " + code);
  384. }
  385. if (relaxSizeAsserts && !this.buf.readAvail(1 + size)) {
  386. return false;
  387. }
  388. this.buf.readAdvance(1);
  389. return this.buf.readInt(size);
  390. }
  391. // synchronously BSER decode a string and return the value
  392. function loadFromBuffer(input) {
  393. var buf = new BunserBuf();
  394. var result = buf.append(input, true);
  395. if (buf.buf.readAvail()) {
  396. throw Error(
  397. 'excess data found after input buffer, use BunserBuf instead');
  398. }
  399. if (typeof result === 'undefined') {
  400. throw Error(
  401. 'no bser found in string and no error raised!?');
  402. }
  403. return result;
  404. }
  405. exports.loadFromBuffer = loadFromBuffer
  406. // Byteswap an arbitrary buffer, flipping from one endian
  407. // to the other, returning a new buffer with the resultant data
  408. function byteswap64(buf) {
  409. var swap = new Buffer(buf.length);
  410. for (var i = 0; i < buf.length; i++) {
  411. swap[i] = buf[buf.length -1 - i];
  412. }
  413. return swap;
  414. }
  415. function dump_int64(buf, val) {
  416. // Get the raw bytes. The Int64 buffer is big endian
  417. var be = val.toBuffer();
  418. if (isBigEndian) {
  419. // We're a big endian system, so the buffer is exactly how we
  420. // want it to be
  421. buf.writeByte(BSER_INT64);
  422. buf.append(be);
  423. return;
  424. }
  425. // We need to byte swap to get the correct representation
  426. var le = byteswap64(be);
  427. buf.writeByte(BSER_INT64);
  428. buf.append(le);
  429. }
  430. function dump_int(buf, val) {
  431. var abs = Math.abs(val);
  432. if (abs <= MAX_INT8) {
  433. buf.writeByte(BSER_INT8);
  434. buf.writeInt(val, 1);
  435. } else if (abs <= MAX_INT16) {
  436. buf.writeByte(BSER_INT16);
  437. buf.writeInt(val, 2);
  438. } else if (abs <= MAX_INT32) {
  439. buf.writeByte(BSER_INT32);
  440. buf.writeInt(val, 4);
  441. } else {
  442. dump_int64(buf, new Int64(val));
  443. }
  444. }
  445. function dump_any(buf, val) {
  446. switch (typeof(val)) {
  447. case 'number':
  448. // check if it is an integer or a float
  449. if (isFinite(val) && Math.floor(val) === val) {
  450. dump_int(buf, val);
  451. } else {
  452. buf.writeByte(BSER_REAL);
  453. buf.writeDouble(val);
  454. }
  455. return;
  456. case 'string':
  457. buf.writeByte(BSER_STRING);
  458. dump_int(buf, Buffer.byteLength(val));
  459. buf.append(val);
  460. return;
  461. case 'boolean':
  462. buf.writeByte(val ? BSER_TRUE : BSER_FALSE);
  463. return;
  464. case 'object':
  465. if (val === null) {
  466. buf.writeByte(BSER_NULL);
  467. return;
  468. }
  469. if (val instanceof Int64) {
  470. dump_int64(buf, val);
  471. return;
  472. }
  473. if (Array.isArray(val)) {
  474. buf.writeByte(BSER_ARRAY);
  475. dump_int(buf, val.length);
  476. for (var i = 0; i < val.length; ++i) {
  477. dump_any(buf, val[i]);
  478. }
  479. return;
  480. }
  481. buf.writeByte(BSER_OBJECT);
  482. var keys = Object.keys(val);
  483. // First pass to compute number of defined keys
  484. var num_keys = keys.length;
  485. for (var i = 0; i < keys.length; ++i) {
  486. var key = keys[i];
  487. var v = val[key];
  488. if (typeof(v) == 'undefined') {
  489. num_keys--;
  490. }
  491. }
  492. dump_int(buf, num_keys);
  493. for (var i = 0; i < keys.length; ++i) {
  494. var key = keys[i];
  495. var v = val[key];
  496. if (typeof(v) == 'undefined') {
  497. // Don't include it
  498. continue;
  499. }
  500. dump_any(buf, key);
  501. try {
  502. dump_any(buf, v);
  503. } catch (e) {
  504. throw new Error(
  505. e.message + ' (while serializing object property with name `' +
  506. key + "')");
  507. }
  508. }
  509. return;
  510. default:
  511. throw new Error('cannot serialize type ' + typeof(val) + ' to BSER');
  512. }
  513. }
  514. // BSER encode value and return a buffer of the contents
  515. function dumpToBuffer(val) {
  516. var buf = new Accumulator();
  517. // Build out the header
  518. buf.writeByte(0);
  519. buf.writeByte(1);
  520. // Reserve room for an int32 to hold our PDU length
  521. buf.writeByte(BSER_INT32);
  522. buf.writeInt(0, 4); // We'll come back and fill this in at the end
  523. dump_any(buf, val);
  524. // Compute PDU length
  525. var off = buf.writeOffset;
  526. var len = off - 7 /* the header length */;
  527. buf.writeOffset = 3; // The length value to fill in
  528. buf.writeInt(len, 4); // write the length in the space we reserved
  529. buf.writeOffset = off;
  530. return buf.buf.slice(0, off);
  531. }
  532. exports.dumpToBuffer = dumpToBuffer