utrie.es5.js 47 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243244245246247248249250251252253254255256257258259260261262263264265266267268269270271272273274275276277278279280281282283284285286287288289290291292293294295296297298299300301302303304305306307308309310311312313314315316317318319320321322323324325326327328329330331332333334335336337338339340341342343344345346347348349350351352353354355356357358359360361362363364365366367368369370371372373374375376377378379380381382383384385386387388389390391392393394395396397398399400401402403404405406407408409410411412413414415416417418419420421422423424425426427428429430431432433434435436437438439440441442443444445446447448449450451452453454455456457458459460461462463464465466467468469470471472473474475476477478479480481482483484485486487488489490491492493494495496497498499500501502503504505506507508509510511512513514515516517518519520521522523524525526527528529530531532533534535536537538539540541542543544545546547548549550551552553554555556557558559560561562563564565566567568569570571572573574575576577578579580581582583584585586587588589590591592593594595596597598599600601602603604605606607608609610611612613614615616617618619620621622623624625626627628629630631632633634635636637638639640641642643644645646647648649650651652653654655656657658659660661662663664665666667668669670671672673674675676677678679680681682683684685686687688689690691692693694695696697698699700701702703704705706707708709710711712713714715716717718719720721722723724725726727728729730731732733734735736737738739740741742743744745746747748749750751752753754755756757758759760761762763764765766767768769770771772773774775776777778779780781782783784785786787788789790791792793794795796797798799800801802803804805806807808809810811812813814815816817818819820821822823824825826827828829830831832833834835836837838839840841842843844845846847848849850851852853854855856857858859860861862863864865866867868869870871872873874875876877878879880881882883884885886887888889890891892893894895896897898899900901902903904905906907908909910911912913914915916917918919920921922923924925926927928929930931932933934935936937938939940941942943944945946947948949950951952953954955956957958959960961962963964965966967968969970971972973974975976977978979980981982983984985986987988989990991992993994995996997998999100010011002100310041005100610071008100910101011101210131014101510161017101810191020102110221023102410251026102710281029103010311032103310341035103610371038103910401041104210431044104510461047104810491050105110521053105410551056105710581059106010611062106310641065106610671068106910701071107210731074107510761077107810791080108110821083108410851086108710881089109010911092109310941095109610971098109911001101
  1. /*
  2. * utrie 1.0.2 <https://github.com/niklasvh/utrie>
  3. * Copyright (c) 2022 Niklas von Hertzen <https://hertzen.com>
  4. * Released under MIT License
  5. */
  6. var chars$1 = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
  7. // Use a lookup table to find the index.
  8. var lookup$1 = typeof Uint8Array === 'undefined' ? [] : new Uint8Array(256);
  9. for (var i$1 = 0; i$1 < chars$1.length; i$1++) {
  10. lookup$1[chars$1.charCodeAt(i$1)] = i$1;
  11. }
  12. var decode = function (base64) {
  13. var bufferLength = base64.length * 0.75, len = base64.length, i, p = 0, encoded1, encoded2, encoded3, encoded4;
  14. if (base64[base64.length - 1] === '=') {
  15. bufferLength--;
  16. if (base64[base64.length - 2] === '=') {
  17. bufferLength--;
  18. }
  19. }
  20. var buffer = typeof ArrayBuffer !== 'undefined' &&
  21. typeof Uint8Array !== 'undefined' &&
  22. typeof Uint8Array.prototype.slice !== 'undefined'
  23. ? new ArrayBuffer(bufferLength)
  24. : new Array(bufferLength);
  25. var bytes = Array.isArray(buffer) ? buffer : new Uint8Array(buffer);
  26. for (i = 0; i < len; i += 4) {
  27. encoded1 = lookup$1[base64.charCodeAt(i)];
  28. encoded2 = lookup$1[base64.charCodeAt(i + 1)];
  29. encoded3 = lookup$1[base64.charCodeAt(i + 2)];
  30. encoded4 = lookup$1[base64.charCodeAt(i + 3)];
  31. bytes[p++] = (encoded1 << 2) | (encoded2 >> 4);
  32. bytes[p++] = ((encoded2 & 15) << 4) | (encoded3 >> 2);
  33. bytes[p++] = ((encoded3 & 3) << 6) | (encoded4 & 63);
  34. }
  35. return buffer;
  36. };
  37. var polyUint16Array = function (buffer) {
  38. var length = buffer.length;
  39. var bytes = [];
  40. for (var i = 0; i < length; i += 2) {
  41. bytes.push((buffer[i + 1] << 8) | buffer[i]);
  42. }
  43. return bytes;
  44. };
  45. var polyUint32Array = function (buffer) {
  46. var length = buffer.length;
  47. var bytes = [];
  48. for (var i = 0; i < length; i += 4) {
  49. bytes.push((buffer[i + 3] << 24) | (buffer[i + 2] << 16) | (buffer[i + 1] << 8) | buffer[i]);
  50. }
  51. return bytes;
  52. };
  53. /** Shift size for getting the index-2 table offset. */
  54. var UTRIE2_SHIFT_2 = 5;
  55. /** Shift size for getting the index-1 table offset. */
  56. var UTRIE2_SHIFT_1 = 6 + 5;
  57. /**
  58. * Shift size for shifting left the index array values.
  59. * Increases possible data size with 16-bit index values at the cost
  60. * of compactability.
  61. * This requires data blocks to be aligned by UTRIE2_DATA_GRANULARITY.
  62. */
  63. var UTRIE2_INDEX_SHIFT = 2;
  64. /**
  65. * Difference between the two shift sizes,
  66. * for getting an index-1 offset from an index-2 offset. 6=11-5
  67. */
  68. var UTRIE2_SHIFT_1_2 = UTRIE2_SHIFT_1 - UTRIE2_SHIFT_2;
  69. /**
  70. * The part of the index-2 table for U+D800..U+DBFF stores values for
  71. * lead surrogate code _units_ not code _points_.
  72. * Values for lead surrogate code _points_ are indexed with this portion of the table.
  73. * Length=32=0x20=0x400>>UTRIE2_SHIFT_2. (There are 1024=0x400 lead surrogates.)
  74. */
  75. var UTRIE2_LSCP_INDEX_2_OFFSET = 0x10000 >> UTRIE2_SHIFT_2;
  76. /** Number of entries in a data block. 32=0x20 */
  77. var UTRIE2_DATA_BLOCK_LENGTH = 1 << UTRIE2_SHIFT_2;
  78. /** Mask for getting the lower bits for the in-data-block offset. */
  79. var UTRIE2_DATA_MASK = UTRIE2_DATA_BLOCK_LENGTH - 1;
  80. var UTRIE2_LSCP_INDEX_2_LENGTH = 0x400 >> UTRIE2_SHIFT_2;
  81. /** Count the lengths of both BMP pieces. 2080=0x820 */
  82. var UTRIE2_INDEX_2_BMP_LENGTH = UTRIE2_LSCP_INDEX_2_OFFSET + UTRIE2_LSCP_INDEX_2_LENGTH;
  83. /**
  84. * The 2-byte UTF-8 version of the index-2 table follows at offset 2080=0x820.
  85. * Length 32=0x20 for lead bytes C0..DF, regardless of UTRIE2_SHIFT_2.
  86. */
  87. var UTRIE2_UTF8_2B_INDEX_2_OFFSET = UTRIE2_INDEX_2_BMP_LENGTH;
  88. var UTRIE2_UTF8_2B_INDEX_2_LENGTH = 0x800 >> 6; /* U+0800 is the first code point after 2-byte UTF-8 */
  89. /**
  90. * The index-1 table, only used for supplementary code points, at offset 2112=0x840.
  91. * Variable length, for code points up to highStart, where the last single-value range starts.
  92. * Maximum length 512=0x200=0x100000>>UTRIE2_SHIFT_1.
  93. * (For 0x100000 supplementary code points U+10000..U+10ffff.)
  94. *
  95. * The part of the index-2 table for supplementary code points starts
  96. * after this index-1 table.
  97. *
  98. * Both the index-1 table and the following part of the index-2 table
  99. * are omitted completely if there is only BMP data.
  100. */
  101. var UTRIE2_INDEX_1_OFFSET = UTRIE2_UTF8_2B_INDEX_2_OFFSET + UTRIE2_UTF8_2B_INDEX_2_LENGTH;
  102. /**
  103. * Number of index-1 entries for the BMP. 32=0x20
  104. * This part of the index-1 table is omitted from the serialized form.
  105. */
  106. var UTRIE2_OMITTED_BMP_INDEX_1_LENGTH = 0x10000 >> UTRIE2_SHIFT_1;
  107. /** Number of entries in an index-2 block. 64=0x40 */
  108. var UTRIE2_INDEX_2_BLOCK_LENGTH = 1 << UTRIE2_SHIFT_1_2;
  109. /** Mask for getting the lower bits for the in-index-2-block offset. */
  110. var UTRIE2_INDEX_2_MASK = UTRIE2_INDEX_2_BLOCK_LENGTH - 1;
  111. var slice16 = function (view, start, end) {
  112. if (view.slice) {
  113. return view.slice(start, end);
  114. }
  115. return new Uint16Array(Array.prototype.slice.call(view, start, end));
  116. };
  117. var slice32 = function (view, start, end) {
  118. if (view.slice) {
  119. return view.slice(start, end);
  120. }
  121. return new Uint32Array(Array.prototype.slice.call(view, start, end));
  122. };
  123. var createTrieFromBase64 = function (base64, _byteLength) {
  124. var buffer = decode(base64);
  125. var view32 = Array.isArray(buffer) ? polyUint32Array(buffer) : new Uint32Array(buffer);
  126. var view16 = Array.isArray(buffer) ? polyUint16Array(buffer) : new Uint16Array(buffer);
  127. var headerLength = 24;
  128. var index = slice16(view16, headerLength / 2, view32[4] / 2);
  129. var data = view32[5] === 2
  130. ? slice16(view16, (headerLength + view32[4]) / 2)
  131. : slice32(view32, Math.ceil((headerLength + view32[4]) / 4));
  132. return new Trie(view32[0], view32[1], view32[2], view32[3], index, data);
  133. };
  134. var Trie = /** @class */ (function () {
  135. function Trie(initialValue, errorValue, highStart, highValueIndex, index, data) {
  136. this.initialValue = initialValue;
  137. this.errorValue = errorValue;
  138. this.highStart = highStart;
  139. this.highValueIndex = highValueIndex;
  140. this.index = index;
  141. this.data = data;
  142. }
  143. /**
  144. * Get the value for a code point as stored in the Trie.
  145. *
  146. * @param codePoint the code point
  147. * @return the value
  148. */
  149. Trie.prototype.get = function (codePoint) {
  150. var ix;
  151. if (codePoint >= 0) {
  152. if (codePoint < 0x0d800 || (codePoint > 0x0dbff && codePoint <= 0x0ffff)) {
  153. // Ordinary BMP code point, excluding leading surrogates.
  154. // BMP uses a single level lookup. BMP index starts at offset 0 in the Trie2 index.
  155. // 16 bit data is stored in the index array itself.
  156. ix = this.index[codePoint >> UTRIE2_SHIFT_2];
  157. ix = (ix << UTRIE2_INDEX_SHIFT) + (codePoint & UTRIE2_DATA_MASK);
  158. return this.data[ix];
  159. }
  160. if (codePoint <= 0xffff) {
  161. // Lead Surrogate Code Point. A Separate index section is stored for
  162. // lead surrogate code units and code points.
  163. // The main index has the code unit data.
  164. // For this function, we need the code point data.
  165. // Note: this expression could be refactored for slightly improved efficiency, but
  166. // surrogate code points will be so rare in practice that it's not worth it.
  167. ix = this.index[UTRIE2_LSCP_INDEX_2_OFFSET + ((codePoint - 0xd800) >> UTRIE2_SHIFT_2)];
  168. ix = (ix << UTRIE2_INDEX_SHIFT) + (codePoint & UTRIE2_DATA_MASK);
  169. return this.data[ix];
  170. }
  171. if (codePoint < this.highStart) {
  172. // Supplemental code point, use two-level lookup.
  173. ix = UTRIE2_INDEX_1_OFFSET - UTRIE2_OMITTED_BMP_INDEX_1_LENGTH + (codePoint >> UTRIE2_SHIFT_1);
  174. ix = this.index[ix];
  175. ix += (codePoint >> UTRIE2_SHIFT_2) & UTRIE2_INDEX_2_MASK;
  176. ix = this.index[ix];
  177. ix = (ix << UTRIE2_INDEX_SHIFT) + (codePoint & UTRIE2_DATA_MASK);
  178. return this.data[ix];
  179. }
  180. if (codePoint <= 0x10ffff) {
  181. return this.data[this.highValueIndex];
  182. }
  183. }
  184. // Fall through. The code point is outside of the legal range of 0..0x10ffff.
  185. return this.errorValue;
  186. };
  187. return Trie;
  188. }());
  189. /*
  190. * base64-arraybuffer 1.0.2 <https://github.com/niklasvh/base64-arraybuffer>
  191. * Copyright (c) 2022 Niklas von Hertzen <https://hertzen.com>
  192. * Released under MIT License
  193. */
  194. var chars = 'ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/';
  195. // Use a lookup table to find the index.
  196. var lookup = typeof Uint8Array === 'undefined' ? [] : new Uint8Array(256);
  197. for (var i = 0; i < chars.length; i++) {
  198. lookup[chars.charCodeAt(i)] = i;
  199. }
  200. var encode = function (arraybuffer) {
  201. var bytes = new Uint8Array(arraybuffer), i, len = bytes.length, base64 = '';
  202. for (i = 0; i < len; i += 3) {
  203. base64 += chars[bytes[i] >> 2];
  204. base64 += chars[((bytes[i] & 3) << 4) | (bytes[i + 1] >> 4)];
  205. base64 += chars[((bytes[i + 1] & 15) << 2) | (bytes[i + 2] >> 6)];
  206. base64 += chars[bytes[i + 2] & 63];
  207. }
  208. if (len % 3 === 2) {
  209. base64 = base64.substring(0, base64.length - 1) + '=';
  210. }
  211. else if (len % 3 === 1) {
  212. base64 = base64.substring(0, base64.length - 2) + '==';
  213. }
  214. return base64;
  215. };
  216. /**
  217. * Trie2 constants, defining shift widths, index array lengths, etc.
  218. *
  219. * These are needed for the runtime macros but users can treat these as
  220. * implementation details and skip to the actual public API further below.
  221. */
  222. // const UTRIE2_OPTIONS_VALUE_BITS_MASK = 0x000f;
  223. /** Number of code points per index-1 table entry. 2048=0x800 */
  224. var UTRIE2_CP_PER_INDEX_1_ENTRY = 1 << UTRIE2_SHIFT_1;
  225. /** The alignment size of a data block. Also the granularity for compaction. */
  226. var UTRIE2_DATA_GRANULARITY = 1 << UTRIE2_INDEX_SHIFT;
  227. /* Fixed layout of the first part of the index array. ------------------- */
  228. /**
  229. * The BMP part of the index-2 table is fixed and linear and starts at offset 0.
  230. * Length=2048=0x800=0x10000>>UTRIE2_SHIFT_2.
  231. */
  232. var UTRIE2_INDEX_2_OFFSET = 0;
  233. var UTRIE2_MAX_INDEX_1_LENGTH = 0x100000 >> UTRIE2_SHIFT_1;
  234. /*
  235. * Fixed layout of the first part of the data array. -----------------------
  236. * Starts with 4 blocks (128=0x80 entries) for ASCII.
  237. */
  238. /**
  239. * The illegal-UTF-8 data block follows the ASCII block, at offset 128=0x80.
  240. * Used with linear access for single bytes 0..0xbf for simple error handling.
  241. * Length 64=0x40, not UTRIE2_DATA_BLOCK_LENGTH.
  242. */
  243. var UTRIE2_BAD_UTF8_DATA_OFFSET = 0x80;
  244. /** The start of non-linear-ASCII data blocks, at offset 192=0xc0. */
  245. var UTRIE2_DATA_START_OFFSET = 0xc0;
  246. /* Building a Trie2 ---------------------------------------------------------- */
  247. /*
  248. * These definitions are mostly needed by utrie2_builder.c, but also by
  249. * utrie2_get32() and utrie2_enum().
  250. */
  251. /*
  252. * At build time, leave a gap in the index-2 table,
  253. * at least as long as the maximum lengths of the 2-byte UTF-8 index-2 table
  254. * and the supplementary index-1 table.
  255. * Round up to UTRIE2_INDEX_2_BLOCK_LENGTH for proper compacting.
  256. */
  257. var UNEWTRIE2_INDEX_GAP_OFFSET = UTRIE2_INDEX_2_BMP_LENGTH;
  258. var UNEWTRIE2_INDEX_GAP_LENGTH = (UTRIE2_UTF8_2B_INDEX_2_LENGTH + UTRIE2_MAX_INDEX_1_LENGTH + UTRIE2_INDEX_2_MASK) & ~UTRIE2_INDEX_2_MASK;
  259. /**
  260. * Maximum length of the build-time index-2 array.
  261. * Maximum number of Unicode code points (0x110000) shifted right by UTRIE2_SHIFT_2,
  262. * plus the part of the index-2 table for lead surrogate code points,
  263. * plus the build-time index gap,
  264. * plus the null index-2 block.
  265. */
  266. var UNEWTRIE2_MAX_INDEX_2_LENGTH = (0x110000 >> UTRIE2_SHIFT_2) +
  267. UTRIE2_LSCP_INDEX_2_LENGTH +
  268. UNEWTRIE2_INDEX_GAP_LENGTH +
  269. UTRIE2_INDEX_2_BLOCK_LENGTH;
  270. var UNEWTRIE2_INDEX_1_LENGTH = 0x110000 >> UTRIE2_SHIFT_1;
  271. /**
  272. * Maximum length of the build-time data array.
  273. * One entry per 0x110000 code points, plus the illegal-UTF-8 block and the null block,
  274. * plus values for the 0x400 surrogate code units.
  275. */
  276. var UNEWTRIE2_MAX_DATA_LENGTH = 0x110000 + 0x40 + 0x40 + 0x400;
  277. /* Start with allocation of 16k data entries. */
  278. var UNEWTRIE2_INITIAL_DATA_LENGTH = 1 << 14;
  279. /* Grow about 8x each time. */
  280. var UNEWTRIE2_MEDIUM_DATA_LENGTH = 1 << 17;
  281. /** The null index-2 block, following the gap in the index-2 table. */
  282. var UNEWTRIE2_INDEX_2_NULL_OFFSET = UNEWTRIE2_INDEX_GAP_OFFSET + UNEWTRIE2_INDEX_GAP_LENGTH;
  283. /** The start of allocated index-2 blocks. */
  284. var UNEWTRIE2_INDEX_2_START_OFFSET = UNEWTRIE2_INDEX_2_NULL_OFFSET + UTRIE2_INDEX_2_BLOCK_LENGTH;
  285. /**
  286. * The null data block.
  287. * Length 64=0x40 even if UTRIE2_DATA_BLOCK_LENGTH is smaller,
  288. * to work with 6-bit trail bytes from 2-byte UTF-8.
  289. */
  290. var UNEWTRIE2_DATA_NULL_OFFSET = UTRIE2_DATA_START_OFFSET;
  291. /** The start of allocated data blocks. */
  292. var UNEWTRIE2_DATA_START_OFFSET = UNEWTRIE2_DATA_NULL_OFFSET + 0x40;
  293. /**
  294. * The start of data blocks for U+0800 and above.
  295. * Below, compaction uses a block length of 64 for 2-byte UTF-8.
  296. * From here on, compaction uses UTRIE2_DATA_BLOCK_LENGTH.
  297. * Data values for 0x780 code points beyond ASCII.
  298. */
  299. var UNEWTRIE2_DATA_0800_OFFSET = UNEWTRIE2_DATA_START_OFFSET + 0x780;
  300. /**
  301. * Maximum length of the runtime index array.
  302. * Limited by its own 16-bit index values, and by uint16_t UTrie2Header.indexLength.
  303. * (The actual maximum length is lower,
  304. * (0x110000>>UTRIE2_SHIFT_2)+UTRIE2_UTF8_2B_INDEX_2_LENGTH+UTRIE2_MAX_INDEX_1_LENGTH.)
  305. */
  306. var UTRIE2_MAX_INDEX_LENGTH = 0xffff;
  307. /**
  308. * Maximum length of the runtime data array.
  309. * Limited by 16-bit index values that are left-shifted by UTRIE2_INDEX_SHIFT,
  310. * and by uint16_t UTrie2Header.shiftedDataLength.
  311. */
  312. var UTRIE2_MAX_DATA_LENGTH = 0xffff << UTRIE2_INDEX_SHIFT;
  313. var BITS_16 = 16;
  314. var BITS_32 = 32;
  315. var isHighSurrogate = function (c) { return c >= 0xd800 && c <= 0xdbff; };
  316. var equalInt = function (a, s, t, length) {
  317. for (var i = 0; i < length; i++) {
  318. if (a[s + i] !== a[t + i]) {
  319. return false;
  320. }
  321. }
  322. return true;
  323. };
  324. var TrieBuilder = /** @class */ (function () {
  325. function TrieBuilder(initialValue, errorValue) {
  326. if (initialValue === void 0) { initialValue = 0; }
  327. if (errorValue === void 0) { errorValue = 0; }
  328. this.initialValue = initialValue;
  329. this.errorValue = errorValue;
  330. this.highStart = 0x110000;
  331. this.data = new Uint32Array(UNEWTRIE2_INITIAL_DATA_LENGTH);
  332. this.dataCapacity = UNEWTRIE2_INITIAL_DATA_LENGTH;
  333. this.highStart = 0x110000;
  334. this.firstFreeBlock = 0; /* no free block in the list */
  335. this.isCompacted = false;
  336. this.index1 = new Uint32Array(UNEWTRIE2_INDEX_1_LENGTH);
  337. this.index2 = new Uint32Array(UNEWTRIE2_MAX_INDEX_2_LENGTH);
  338. /*
  339. * Multi-purpose per-data-block table.
  340. *
  341. * Before compacting:
  342. *
  343. * Per-data-block reference counters/free-block list.
  344. * 0: unused
  345. * >0: reference counter (number of index-2 entries pointing here)
  346. * <0: next free data block in free-block list
  347. *
  348. * While compacting:
  349. *
  350. * Map of adjusted indexes, used in compactData() and compactIndex2().
  351. * Maps from original indexes to new ones.
  352. */
  353. this.map = new Uint32Array(UNEWTRIE2_MAX_DATA_LENGTH >> UTRIE2_SHIFT_2);
  354. /*
  355. * preallocate and reset
  356. * - ASCII
  357. * - the bad-UTF-8-data block
  358. * - the null data block
  359. */
  360. var i, j;
  361. for (i = 0; i < 0x80; ++i) {
  362. this.data[i] = initialValue;
  363. }
  364. for (; i < 0xc0; ++i) {
  365. this.data[i] = errorValue;
  366. }
  367. for (i = UNEWTRIE2_DATA_NULL_OFFSET; i < UNEWTRIE2_DATA_START_OFFSET; ++i) {
  368. this.data[i] = initialValue;
  369. }
  370. this.dataNullOffset = UNEWTRIE2_DATA_NULL_OFFSET;
  371. this.dataLength = UNEWTRIE2_DATA_START_OFFSET;
  372. /* set the index-2 indexes for the 2=0x80>>UTRIE2_SHIFT_2 ASCII data blocks */
  373. for (i = 0, j = 0; j < 0x80; ++i, j += UTRIE2_DATA_BLOCK_LENGTH) {
  374. this.index2[i] = j;
  375. this.map[i] = 1;
  376. }
  377. /* reference counts for the bad-UTF-8-data block */
  378. for (; j < 0xc0; ++i, j += UTRIE2_DATA_BLOCK_LENGTH) {
  379. this.map[i] = 0;
  380. }
  381. /*
  382. * Reference counts for the null data block: all blocks except for the ASCII blocks.
  383. * Plus 1 so that we don't drop this block during compaction.
  384. * Plus as many as needed for lead surrogate code points.
  385. */
  386. /* i==newTrie->dataNullOffset */
  387. this.map[i++] = (0x110000 >> UTRIE2_SHIFT_2) - (0x80 >> UTRIE2_SHIFT_2) + 1 + UTRIE2_LSCP_INDEX_2_LENGTH;
  388. j += UTRIE2_DATA_BLOCK_LENGTH;
  389. for (; j < UNEWTRIE2_DATA_START_OFFSET; ++i, j += UTRIE2_DATA_BLOCK_LENGTH) {
  390. this.map[i] = 0;
  391. }
  392. /*
  393. * set the remaining indexes in the BMP index-2 block
  394. * to the null data block
  395. */
  396. for (i = 0x80 >> UTRIE2_SHIFT_2; i < UTRIE2_INDEX_2_BMP_LENGTH; ++i) {
  397. this.index2[i] = UNEWTRIE2_DATA_NULL_OFFSET;
  398. }
  399. /*
  400. * Fill the index gap with impossible values so that compaction
  401. * does not overlap other index-2 blocks with the gap.
  402. */
  403. for (i = 0; i < UNEWTRIE2_INDEX_GAP_LENGTH; ++i) {
  404. this.index2[UNEWTRIE2_INDEX_GAP_OFFSET + i] = -1;
  405. }
  406. /* set the indexes in the null index-2 block */
  407. for (i = 0; i < UTRIE2_INDEX_2_BLOCK_LENGTH; ++i) {
  408. this.index2[UNEWTRIE2_INDEX_2_NULL_OFFSET + i] = UNEWTRIE2_DATA_NULL_OFFSET;
  409. }
  410. this.index2NullOffset = UNEWTRIE2_INDEX_2_NULL_OFFSET;
  411. this.index2Length = UNEWTRIE2_INDEX_2_START_OFFSET;
  412. /* set the index-1 indexes for the linear index-2 block */
  413. for (i = 0, j = 0; i < UTRIE2_OMITTED_BMP_INDEX_1_LENGTH; ++i, j += UTRIE2_INDEX_2_BLOCK_LENGTH) {
  414. this.index1[i] = j;
  415. }
  416. /* set the remaining index-1 indexes to the null index-2 block */
  417. for (; i < UNEWTRIE2_INDEX_1_LENGTH; ++i) {
  418. this.index1[i] = UNEWTRIE2_INDEX_2_NULL_OFFSET;
  419. }
  420. /*
  421. * Preallocate and reset data for U+0080..U+07ff,
  422. * for 2-byte UTF-8 which will be compacted in 64-blocks
  423. * even if UTRIE2_DATA_BLOCK_LENGTH is smaller.
  424. */
  425. for (i = 0x80; i < 0x800; i += UTRIE2_DATA_BLOCK_LENGTH) {
  426. this.set(i, initialValue);
  427. }
  428. }
  429. /**
  430. * Set a value for a code point.
  431. *
  432. * @param c the code point
  433. * @param value the value
  434. */
  435. TrieBuilder.prototype.set = function (c, value) {
  436. if (c < 0 || c > 0x10ffff) {
  437. throw new Error('Invalid code point.');
  438. }
  439. this._set(c, true, value);
  440. return this;
  441. };
  442. /**
  443. * Set a value in a range of code points [start..end].
  444. * All code points c with start<=c<=end will get the value if
  445. * overwrite is TRUE or if the old value is the initial value.
  446. *
  447. * @param start the first code point to get the value
  448. * @param end the last code point to get the value (inclusive)
  449. * @param value the value
  450. * @param overwrite flag for whether old non-initial values are to be overwritten
  451. */
  452. TrieBuilder.prototype.setRange = function (start, end, value, overwrite) {
  453. if (overwrite === void 0) { overwrite = false; }
  454. /*
  455. * repeat value in [start..end]
  456. * mark index values for repeat-data blocks by setting bit 31 of the index values
  457. * fill around existing values if any, if(overwrite)
  458. */
  459. var block, rest, repeatBlock;
  460. if (start > 0x10ffff || start < 0 || end > 0x10ffff || end < 0 || start > end) {
  461. throw new Error('Invalid code point range.');
  462. }
  463. if (!overwrite && value === this.initialValue) {
  464. return this; /* nothing to do */
  465. }
  466. if (this.isCompacted) {
  467. throw new Error('Trie was already compacted');
  468. }
  469. var limit = end + 1;
  470. if ((start & UTRIE2_DATA_MASK) !== 0) {
  471. /* set partial block at [start..following block boundary[ */
  472. block = this.getDataBlock(start, true);
  473. var nextStart = (start + UTRIE2_DATA_BLOCK_LENGTH) & ~UTRIE2_DATA_MASK;
  474. if (nextStart <= limit) {
  475. this.fillBlock(block, start & UTRIE2_DATA_MASK, UTRIE2_DATA_BLOCK_LENGTH, value, this.initialValue, overwrite);
  476. start = nextStart;
  477. }
  478. else {
  479. this.fillBlock(block, start & UTRIE2_DATA_MASK, limit & UTRIE2_DATA_MASK, value, this.initialValue, overwrite);
  480. return this;
  481. }
  482. }
  483. /* number of positions in the last, partial block */
  484. rest = limit & UTRIE2_DATA_MASK;
  485. /* round down limit to a block boundary */
  486. limit &= ~UTRIE2_DATA_MASK;
  487. /* iterate over all-value blocks */
  488. repeatBlock = value === this.initialValue ? this.dataNullOffset : -1;
  489. while (start < limit) {
  490. var i2 = void 0;
  491. var setRepeatBlock = false;
  492. if (value === this.initialValue && this.isInNullBlock(start, true)) {
  493. start += UTRIE2_DATA_BLOCK_LENGTH; /* nothing to do */
  494. continue;
  495. }
  496. /* get index value */
  497. i2 = this.getIndex2Block(start, true);
  498. i2 += (start >> UTRIE2_SHIFT_2) & UTRIE2_INDEX_2_MASK;
  499. block = this.index2[i2];
  500. if (this.isWritableBlock(block)) {
  501. /* already allocated */
  502. if (overwrite && block >= UNEWTRIE2_DATA_0800_OFFSET) {
  503. /*
  504. * We overwrite all values, and it's not a
  505. * protected (ASCII-linear or 2-byte UTF-8) block:
  506. * replace with the repeatBlock.
  507. */
  508. setRepeatBlock = true;
  509. }
  510. else {
  511. /* !overwrite, or protected block: just write the values into this block */
  512. this.fillBlock(block, 0, UTRIE2_DATA_BLOCK_LENGTH, value, this.initialValue, overwrite);
  513. }
  514. }
  515. else if (this.data[block] !== value && (overwrite || block === this.dataNullOffset)) {
  516. /*
  517. * Set the repeatBlock instead of the null block or previous repeat block:
  518. *
  519. * If !isWritableBlock() then all entries in the block have the same value
  520. * because it's the null block or a range block (the repeatBlock from a previous
  521. * call to utrie2_setRange32()).
  522. * No other blocks are used multiple times before compacting.
  523. *
  524. * The null block is the only non-writable block with the initialValue because
  525. * of the repeatBlock initialization above. (If value==initialValue, then
  526. * the repeatBlock will be the null data block.)
  527. *
  528. * We set our repeatBlock if the desired value differs from the block's value,
  529. * and if we overwrite any data or if the data is all initial values
  530. * (which is the same as the block being the null block, see above).
  531. */
  532. setRepeatBlock = true;
  533. }
  534. if (setRepeatBlock) {
  535. if (repeatBlock >= 0) {
  536. this.setIndex2Entry(i2, repeatBlock);
  537. }
  538. else {
  539. /* create and set and fill the repeatBlock */
  540. repeatBlock = this.getDataBlock(start, true);
  541. this.writeBlock(repeatBlock, value);
  542. }
  543. }
  544. start += UTRIE2_DATA_BLOCK_LENGTH;
  545. }
  546. if (rest > 0) {
  547. /* set partial block at [last block boundary..limit[ */
  548. block = this.getDataBlock(start, true);
  549. this.fillBlock(block, 0, rest, value, this.initialValue, overwrite);
  550. }
  551. return this;
  552. };
  553. /**
  554. * Get the value for a code point as stored in the Trie2.
  555. *
  556. * @param codePoint the code point
  557. * @return the value
  558. */
  559. TrieBuilder.prototype.get = function (codePoint) {
  560. if (codePoint < 0 || codePoint > 0x10ffff) {
  561. return this.errorValue;
  562. }
  563. else {
  564. return this._get(codePoint, true);
  565. }
  566. };
  567. TrieBuilder.prototype._get = function (c, fromLSCP) {
  568. var i2;
  569. if (c >= this.highStart && (!(c >= 0xd800 && c < 0xdc00) || fromLSCP)) {
  570. return this.data[this.dataLength - UTRIE2_DATA_GRANULARITY];
  571. }
  572. if (c >= 0xd800 && c < 0xdc00 && fromLSCP) {
  573. i2 = UTRIE2_LSCP_INDEX_2_OFFSET - (0xd800 >> UTRIE2_SHIFT_2) + (c >> UTRIE2_SHIFT_2);
  574. }
  575. else {
  576. i2 = this.index1[c >> UTRIE2_SHIFT_1] + ((c >> UTRIE2_SHIFT_2) & UTRIE2_INDEX_2_MASK);
  577. }
  578. var block = this.index2[i2];
  579. return this.data[block + (c & UTRIE2_DATA_MASK)];
  580. };
  581. TrieBuilder.prototype.freeze = function (valueBits) {
  582. if (valueBits === void 0) { valueBits = BITS_32; }
  583. var i;
  584. var allIndexesLength;
  585. var dataMove; /* >0 if the data is moved to the end of the index array */
  586. /* compact if necessary */
  587. if (!this.isCompacted) {
  588. this.compactTrie();
  589. }
  590. allIndexesLength = this.highStart <= 0x10000 ? UTRIE2_INDEX_1_OFFSET : this.index2Length;
  591. if (valueBits === BITS_16) {
  592. // dataMove = allIndexesLength;
  593. dataMove = 0;
  594. }
  595. else {
  596. dataMove = 0;
  597. }
  598. /* are indexLength and dataLength within limits? */
  599. if (
  600. /* for unshifted indexLength */
  601. allIndexesLength > UTRIE2_MAX_INDEX_LENGTH ||
  602. /* for unshifted dataNullOffset */
  603. dataMove + this.dataNullOffset > 0xffff ||
  604. /* for unshifted 2-byte UTF-8 index-2 values */
  605. dataMove + UNEWTRIE2_DATA_0800_OFFSET > 0xffff ||
  606. /* for shiftedDataLength */
  607. dataMove + this.dataLength > UTRIE2_MAX_DATA_LENGTH) {
  608. throw new Error('Trie data is too large.');
  609. }
  610. var index = new Uint16Array(allIndexesLength);
  611. /* write the index-2 array values shifted right by UTRIE2_INDEX_SHIFT, after adding dataMove */
  612. var destIdx = 0;
  613. for (i = 0; i < UTRIE2_INDEX_2_BMP_LENGTH; i++) {
  614. index[destIdx++] = (this.index2[i] + dataMove) >> UTRIE2_INDEX_SHIFT;
  615. }
  616. /* write UTF-8 2-byte index-2 values, not right-shifted */
  617. for (i = 0; i < 0xc2 - 0xc0; ++i) {
  618. /* C0..C1 */
  619. index[destIdx++] = dataMove + UTRIE2_BAD_UTF8_DATA_OFFSET;
  620. }
  621. for (; i < 0xe0 - 0xc0; ++i) {
  622. /* C2..DF */
  623. index[destIdx++] = dataMove + this.index2[i << (6 - UTRIE2_SHIFT_2)];
  624. }
  625. if (this.highStart > 0x10000) {
  626. var index1Length = (this.highStart - 0x10000) >> UTRIE2_SHIFT_1;
  627. var index2Offset = UTRIE2_INDEX_2_BMP_LENGTH + UTRIE2_UTF8_2B_INDEX_2_LENGTH + index1Length;
  628. /* write 16-bit index-1 values for supplementary code points */
  629. for (i = 0; i < index1Length; i++) {
  630. index[destIdx++] = UTRIE2_INDEX_2_OFFSET + this.index1[i + UTRIE2_OMITTED_BMP_INDEX_1_LENGTH];
  631. }
  632. /*
  633. * write the index-2 array values for supplementary code points,
  634. * shifted right by UTRIE2_INDEX_SHIFT, after adding dataMove
  635. */
  636. for (i = 0; i < this.index2Length - index2Offset; i++) {
  637. index[destIdx++] = (dataMove + this.index2[index2Offset + i]) >> UTRIE2_INDEX_SHIFT;
  638. }
  639. }
  640. /* write the 16/32-bit data array */
  641. switch (valueBits) {
  642. case BITS_16:
  643. /* write 16-bit data values */
  644. var data16 = new Uint16Array(this.dataLength);
  645. for (i = 0; i < this.dataLength; i++) {
  646. data16[i] = this.data[i];
  647. }
  648. return new Trie(this.initialValue, this.errorValue, this.highStart, dataMove + this.dataLength - UTRIE2_DATA_GRANULARITY, index, data16);
  649. case BITS_32:
  650. /* write 32-bit data values */
  651. var data32 = new Uint32Array(this.dataLength);
  652. for (i = 0; i < this.dataLength; i++) {
  653. data32[i] = this.data[i];
  654. }
  655. return new Trie(this.initialValue, this.errorValue, this.highStart, dataMove + this.dataLength - UTRIE2_DATA_GRANULARITY, index, data32);
  656. default:
  657. throw new Error('Bits should be either 16 or 32');
  658. }
  659. };
  660. /*
  661. * Find the start of the last range in the trie by enumerating backward.
  662. * Indexes for supplementary code points higher than this will be omitted.
  663. */
  664. TrieBuilder.prototype.findHighStart = function (highValue) {
  665. var value;
  666. var i2, j, i2Block, prevI2Block, block, prevBlock;
  667. /* set variables for previous range */
  668. if (highValue === this.initialValue) {
  669. prevI2Block = this.index2NullOffset;
  670. prevBlock = this.dataNullOffset;
  671. }
  672. else {
  673. prevI2Block = -1;
  674. prevBlock = -1;
  675. }
  676. var prev = 0x110000;
  677. /* enumerate index-2 blocks */
  678. var i1 = UNEWTRIE2_INDEX_1_LENGTH;
  679. var c = prev;
  680. while (c > 0) {
  681. i2Block = this.index1[--i1];
  682. if (i2Block === prevI2Block) {
  683. /* the index-2 block is the same as the previous one, and filled with highValue */
  684. c -= UTRIE2_CP_PER_INDEX_1_ENTRY;
  685. continue;
  686. }
  687. prevI2Block = i2Block;
  688. if (i2Block === this.index2NullOffset) {
  689. /* this is the null index-2 block */
  690. if (highValue !== this.initialValue) {
  691. return c;
  692. }
  693. c -= UTRIE2_CP_PER_INDEX_1_ENTRY;
  694. }
  695. else {
  696. /* enumerate data blocks for one index-2 block */
  697. for (i2 = UTRIE2_INDEX_2_BLOCK_LENGTH; i2 > 0;) {
  698. block = this.index2[i2Block + --i2];
  699. if (block === prevBlock) {
  700. /* the block is the same as the previous one, and filled with highValue */
  701. c -= UTRIE2_DATA_BLOCK_LENGTH;
  702. continue;
  703. }
  704. prevBlock = block;
  705. if (block === this.dataNullOffset) {
  706. /* this is the null data block */
  707. if (highValue !== this.initialValue) {
  708. return c;
  709. }
  710. c -= UTRIE2_DATA_BLOCK_LENGTH;
  711. }
  712. else {
  713. for (j = UTRIE2_DATA_BLOCK_LENGTH; j > 0;) {
  714. value = this.data[block + --j];
  715. if (value !== highValue) {
  716. return c;
  717. }
  718. --c;
  719. }
  720. }
  721. }
  722. }
  723. }
  724. /* deliver last range */
  725. return 0;
  726. };
  727. /*
  728. * Compact a build-time trie.
  729. *
  730. * The compaction
  731. * - removes blocks that are identical with earlier ones
  732. * - overlaps adjacent blocks as much as possible (if overlap==TRUE)
  733. * - moves blocks in steps of the data granularity
  734. * - moves and overlaps blocks that overlap with multiple values in the overlap region
  735. *
  736. * It does not
  737. * - try to move and overlap blocks that are not already adjacent
  738. */
  739. TrieBuilder.prototype.compactData = function () {
  740. var start, movedStart;
  741. var blockLength, overlap;
  742. var i, mapIndex, blockCount;
  743. /* do not compact linear-ASCII data */
  744. var newStart = UTRIE2_DATA_START_OFFSET;
  745. for (start = 0, i = 0; start < newStart; start += UTRIE2_DATA_BLOCK_LENGTH, ++i) {
  746. this.map[i] = start;
  747. }
  748. /*
  749. * Start with a block length of 64 for 2-byte UTF-8,
  750. * then switch to UTRIE2_DATA_BLOCK_LENGTH.
  751. */
  752. blockLength = 64;
  753. blockCount = blockLength >> UTRIE2_SHIFT_2;
  754. for (start = newStart; start < this.dataLength;) {
  755. /*
  756. * start: index of first entry of current block
  757. * newStart: index where the current block is to be moved
  758. * (right after current end of already-compacted data)
  759. */
  760. if (start === UNEWTRIE2_DATA_0800_OFFSET) {
  761. blockLength = UTRIE2_DATA_BLOCK_LENGTH;
  762. blockCount = 1;
  763. }
  764. /* skip blocks that are not used */
  765. if (this.map[start >> UTRIE2_SHIFT_2] <= 0) {
  766. /* advance start to the next block */
  767. start += blockLength;
  768. /* leave newStart with the previous block! */
  769. continue;
  770. }
  771. /* search for an identical block */
  772. movedStart = this.findSameDataBlock(newStart, start, blockLength);
  773. if (movedStart >= 0) {
  774. /* found an identical block, set the other block's index value for the current block */
  775. for (i = blockCount, mapIndex = start >> UTRIE2_SHIFT_2; i > 0; --i) {
  776. this.map[mapIndex++] = movedStart;
  777. movedStart += UTRIE2_DATA_BLOCK_LENGTH;
  778. }
  779. /* advance start to the next block */
  780. start += blockLength;
  781. /* leave newStart with the previous block! */
  782. continue;
  783. }
  784. /* see if the beginning of this block can be overlapped with the end of the previous block */
  785. /* look for maximum overlap (modulo granularity) with the previous, adjacent block */
  786. for (overlap = blockLength - UTRIE2_DATA_GRANULARITY; overlap > 0 && !equalInt(this.data, newStart - overlap, start, overlap); overlap -= UTRIE2_DATA_GRANULARITY) { }
  787. if (overlap > 0 || newStart < start) {
  788. /* some overlap, or just move the whole block */
  789. movedStart = newStart - overlap;
  790. for (i = blockCount, mapIndex = start >> UTRIE2_SHIFT_2; i > 0; --i) {
  791. this.map[mapIndex++] = movedStart;
  792. movedStart += UTRIE2_DATA_BLOCK_LENGTH;
  793. }
  794. /* move the non-overlapping indexes to their new positions */
  795. start += overlap;
  796. for (i = blockLength - overlap; i > 0; --i) {
  797. this.data[newStart++] = this.data[start++];
  798. }
  799. }
  800. else {
  801. /* no overlap && newStart==start */
  802. for (i = blockCount, mapIndex = start >> UTRIE2_SHIFT_2; i > 0; --i) {
  803. this.map[mapIndex++] = start;
  804. start += UTRIE2_DATA_BLOCK_LENGTH;
  805. }
  806. newStart = start;
  807. }
  808. }
  809. /* now adjust the index-2 table */
  810. for (i = 0; i < this.index2Length; ++i) {
  811. if (i === UNEWTRIE2_INDEX_GAP_OFFSET) {
  812. /* Gap indexes are invalid (-1). Skip over the gap. */
  813. i += UNEWTRIE2_INDEX_GAP_LENGTH;
  814. }
  815. this.index2[i] = this.map[this.index2[i] >> UTRIE2_SHIFT_2];
  816. }
  817. this.dataNullOffset = this.map[this.dataNullOffset >> UTRIE2_SHIFT_2];
  818. /* ensure dataLength alignment */
  819. while ((newStart & (UTRIE2_DATA_GRANULARITY - 1)) !== 0) {
  820. this.data[newStart++] = this.initialValue;
  821. }
  822. this.dataLength = newStart;
  823. };
  824. TrieBuilder.prototype.findSameDataBlock = function (dataLength, otherBlock, blockLength) {
  825. var block = 0;
  826. /* ensure that we do not even partially get past dataLength */
  827. dataLength -= blockLength;
  828. for (; block <= dataLength; block += UTRIE2_DATA_GRANULARITY) {
  829. if (equalInt(this.data, block, otherBlock, blockLength)) {
  830. return block;
  831. }
  832. }
  833. return -1;
  834. };
  835. TrieBuilder.prototype.compactTrie = function () {
  836. var highValue = this.get(0x10ffff);
  837. /* find highStart and round it up */
  838. var localHighStart = this.findHighStart(highValue);
  839. localHighStart = (localHighStart + (UTRIE2_CP_PER_INDEX_1_ENTRY - 1)) & ~(UTRIE2_CP_PER_INDEX_1_ENTRY - 1);
  840. if (localHighStart === 0x110000) {
  841. highValue = this.errorValue;
  842. }
  843. /*
  844. * Set trie->highStart only after utrie2_get32(trie, highStart).
  845. * Otherwise utrie2_get32(trie, highStart) would try to read the highValue.
  846. */
  847. this.highStart = localHighStart;
  848. if (this.highStart < 0x110000) {
  849. /* Blank out [highStart..10ffff] to release associated data blocks. */
  850. var suppHighStart = this.highStart <= 0x10000 ? 0x10000 : this.highStart;
  851. this.setRange(suppHighStart, 0x10ffff, this.initialValue, true);
  852. }
  853. this.compactData();
  854. if (this.highStart > 0x10000) {
  855. this.compactIndex2();
  856. }
  857. /*
  858. * Store the highValue in the data array and round up the dataLength.
  859. * Must be done after compactData() because that assumes that dataLength
  860. * is a multiple of UTRIE2_DATA_BLOCK_LENGTH.
  861. */
  862. this.data[this.dataLength++] = highValue;
  863. while ((this.dataLength & (UTRIE2_DATA_GRANULARITY - 1)) !== 0) {
  864. this.data[this.dataLength++] = this.initialValue;
  865. }
  866. this.isCompacted = true;
  867. };
  868. TrieBuilder.prototype.compactIndex2 = function () {
  869. var i, start, movedStart, overlap;
  870. /* do not compact linear-BMP index-2 blocks */
  871. var newStart = UTRIE2_INDEX_2_BMP_LENGTH;
  872. for (start = 0, i = 0; start < newStart; start += UTRIE2_INDEX_2_BLOCK_LENGTH, ++i) {
  873. this.map[i] = start;
  874. }
  875. /* Reduce the index table gap to what will be needed at runtime. */
  876. newStart += UTRIE2_UTF8_2B_INDEX_2_LENGTH + ((this.highStart - 0x10000) >> UTRIE2_SHIFT_1);
  877. for (start = UNEWTRIE2_INDEX_2_NULL_OFFSET; start < this.index2Length;) {
  878. /*
  879. * start: index of first entry of current block
  880. * newStart: index where the current block is to be moved
  881. * (right after current end of already-compacted data)
  882. */
  883. /* search for an identical block */
  884. if ((movedStart = this.findSameIndex2Block(newStart, start)) >= 0) {
  885. /* found an identical block, set the other block's index value for the current block */
  886. this.map[start >> UTRIE2_SHIFT_1_2] = movedStart;
  887. /* advance start to the next block */
  888. start += UTRIE2_INDEX_2_BLOCK_LENGTH;
  889. /* leave newStart with the previous block! */
  890. continue;
  891. }
  892. /* see if the beginning of this block can be overlapped with the end of the previous block */
  893. /* look for maximum overlap with the previous, adjacent block */
  894. for (overlap = UTRIE2_INDEX_2_BLOCK_LENGTH - 1; overlap > 0 && !equalInt(this.index2, newStart - overlap, start, overlap); --overlap) { }
  895. if (overlap > 0 || newStart < start) {
  896. /* some overlap, or just move the whole block */
  897. this.map[start >> UTRIE2_SHIFT_1_2] = newStart - overlap;
  898. /* move the non-overlapping indexes to their new positions */
  899. start += overlap;
  900. for (i = UTRIE2_INDEX_2_BLOCK_LENGTH - overlap; i > 0; --i) {
  901. this.index2[newStart++] = this.index2[start++];
  902. }
  903. }
  904. else {
  905. /* no overlap && newStart==start */ this.map[start >> UTRIE2_SHIFT_1_2] = start;
  906. start += UTRIE2_INDEX_2_BLOCK_LENGTH;
  907. newStart = start;
  908. }
  909. }
  910. /* now adjust the index-1 table */
  911. for (i = 0; i < UNEWTRIE2_INDEX_1_LENGTH; ++i) {
  912. this.index1[i] = this.map[this.index1[i] >> UTRIE2_SHIFT_1_2];
  913. }
  914. this.index2NullOffset = this.map[this.index2NullOffset >> UTRIE2_SHIFT_1_2];
  915. /*
  916. * Ensure data table alignment:
  917. * Needs to be granularity-aligned for 16-bit trie
  918. * (so that dataMove will be down-shiftable),
  919. * and 2-aligned for uint32_t data.
  920. */
  921. while ((newStart & ((UTRIE2_DATA_GRANULARITY - 1) | 1)) !== 0) {
  922. /* Arbitrary value: 0x3fffc not possible for real data. */
  923. this.index2[newStart++] = 0x0000ffff << UTRIE2_INDEX_SHIFT;
  924. }
  925. this.index2Length = newStart;
  926. };
  927. TrieBuilder.prototype.findSameIndex2Block = function (index2Length, otherBlock) {
  928. /* ensure that we do not even partially get past index2Length */
  929. index2Length -= UTRIE2_INDEX_2_BLOCK_LENGTH;
  930. for (var block = 0; block <= index2Length; ++block) {
  931. if (equalInt(this.index2, block, otherBlock, UTRIE2_INDEX_2_BLOCK_LENGTH)) {
  932. return block;
  933. }
  934. }
  935. return -1;
  936. };
  937. TrieBuilder.prototype._set = function (c, forLSCP, value) {
  938. if (this.isCompacted) {
  939. throw new Error('Trie was already compacted');
  940. }
  941. var block = this.getDataBlock(c, forLSCP);
  942. this.data[block + (c & UTRIE2_DATA_MASK)] = value;
  943. return this;
  944. };
  945. TrieBuilder.prototype.writeBlock = function (block, value) {
  946. var limit = block + UTRIE2_DATA_BLOCK_LENGTH;
  947. while (block < limit) {
  948. this.data[block++] = value;
  949. }
  950. };
  951. TrieBuilder.prototype.isInNullBlock = function (c, forLSCP) {
  952. var i2 = isHighSurrogate(c) && forLSCP
  953. ? UTRIE2_LSCP_INDEX_2_OFFSET - (0xd800 >> UTRIE2_SHIFT_2) + (c >> UTRIE2_SHIFT_2)
  954. : this.index1[c >> UTRIE2_SHIFT_1] + ((c >> UTRIE2_SHIFT_2) & UTRIE2_INDEX_2_MASK);
  955. var block = this.index2[i2];
  956. return block === this.dataNullOffset;
  957. };
  958. TrieBuilder.prototype.fillBlock = function (block, start, limit, value, initialValue, overwrite) {
  959. var pLimit = block + limit;
  960. if (overwrite) {
  961. for (var i = block + start; i < pLimit; i++) {
  962. this.data[i] = value;
  963. }
  964. }
  965. else {
  966. for (var i = block + start; i < pLimit; i++) {
  967. if (this.data[i] === initialValue) {
  968. this.data[i] = value;
  969. }
  970. }
  971. }
  972. };
  973. TrieBuilder.prototype.setIndex2Entry = function (i2, block) {
  974. ++this.map[block >> UTRIE2_SHIFT_2]; /* increment first, in case block==oldBlock! */
  975. var oldBlock = this.index2[i2];
  976. if (0 === --this.map[oldBlock >> UTRIE2_SHIFT_2]) {
  977. this.releaseDataBlock(oldBlock);
  978. }
  979. this.index2[i2] = block;
  980. };
  981. TrieBuilder.prototype.releaseDataBlock = function (block) {
  982. /* put this block at the front of the free-block chain */
  983. this.map[block >> UTRIE2_SHIFT_2] = -this.firstFreeBlock;
  984. this.firstFreeBlock = block;
  985. };
  986. TrieBuilder.prototype.getDataBlock = function (c, forLSCP) {
  987. var i2 = this.getIndex2Block(c, forLSCP);
  988. i2 += (c >> UTRIE2_SHIFT_2) & UTRIE2_INDEX_2_MASK;
  989. var oldBlock = this.index2[i2];
  990. if (this.isWritableBlock(oldBlock)) {
  991. return oldBlock;
  992. }
  993. /* allocate a new data block */
  994. var newBlock = this.allocDataBlock(oldBlock);
  995. this.setIndex2Entry(i2, newBlock);
  996. return newBlock;
  997. };
  998. TrieBuilder.prototype.isWritableBlock = function (block) {
  999. return block !== this.dataNullOffset && 1 === this.map[block >> UTRIE2_SHIFT_2];
  1000. };
  1001. TrieBuilder.prototype.getIndex2Block = function (c, forLSCP) {
  1002. if (c >= 0xd800 && c < 0xdc00 && forLSCP) {
  1003. return UTRIE2_LSCP_INDEX_2_OFFSET;
  1004. }
  1005. var i1 = c >> UTRIE2_SHIFT_1;
  1006. var i2 = this.index1[i1];
  1007. if (i2 === this.index2NullOffset) {
  1008. i2 = this.allocIndex2Block();
  1009. this.index1[i1] = i2;
  1010. }
  1011. return i2;
  1012. };
  1013. TrieBuilder.prototype.allocDataBlock = function (copyBlock) {
  1014. var newBlock;
  1015. if (this.firstFreeBlock !== 0) {
  1016. /* get the first free block */
  1017. newBlock = this.firstFreeBlock;
  1018. this.firstFreeBlock = -this.map[newBlock >> UTRIE2_SHIFT_2];
  1019. }
  1020. else {
  1021. /* get a new block from the high end */
  1022. newBlock = this.dataLength;
  1023. var newTop = newBlock + UTRIE2_DATA_BLOCK_LENGTH;
  1024. if (newTop > this.dataCapacity) {
  1025. var capacity = void 0;
  1026. /* out of memory in the data array */
  1027. if (this.dataCapacity < UNEWTRIE2_MEDIUM_DATA_LENGTH) {
  1028. capacity = UNEWTRIE2_MEDIUM_DATA_LENGTH;
  1029. }
  1030. else if (this.dataCapacity < UNEWTRIE2_MAX_DATA_LENGTH) {
  1031. capacity = UNEWTRIE2_MAX_DATA_LENGTH;
  1032. }
  1033. else {
  1034. /*
  1035. * Should never occur.
  1036. * Either UNEWTRIE2_MAX_DATA_LENGTH is incorrect,
  1037. * or the code writes more values than should be possible.
  1038. */
  1039. throw new Error('Internal error in Trie creation.');
  1040. }
  1041. var newData = new Uint32Array(capacity);
  1042. newData.set(this.data.subarray(0, this.dataLength));
  1043. this.data = newData;
  1044. this.dataCapacity = capacity;
  1045. }
  1046. this.dataLength = newTop;
  1047. }
  1048. this.data.set(this.data.subarray(copyBlock, copyBlock + UTRIE2_DATA_BLOCK_LENGTH), newBlock);
  1049. this.map[newBlock >> UTRIE2_SHIFT_2] = 0;
  1050. return newBlock;
  1051. };
  1052. TrieBuilder.prototype.allocIndex2Block = function () {
  1053. var newBlock = this.index2Length;
  1054. var newTop = newBlock + UTRIE2_INDEX_2_BLOCK_LENGTH;
  1055. if (newTop > this.index2.length) {
  1056. throw new Error('Internal error in Trie creation.');
  1057. /*
  1058. * Should never occur.
  1059. * Either UTRIE2_MAX_BUILD_TIME_INDEX_LENGTH is incorrect,
  1060. * or the code writes more values than should be possible.
  1061. */
  1062. }
  1063. this.index2Length = newTop;
  1064. this.index2.set(this.index2.subarray(this.index2NullOffset, this.index2NullOffset + UTRIE2_INDEX_2_BLOCK_LENGTH), newBlock);
  1065. return newBlock;
  1066. };
  1067. return TrieBuilder;
  1068. }());
  1069. var serializeBase64 = function (trie) {
  1070. var index = trie.index;
  1071. var data = trie.data;
  1072. if (!(index instanceof Uint16Array) || !(data instanceof Uint16Array || data instanceof Uint32Array)) {
  1073. throw new Error('TrieBuilder serializer only support TypedArrays');
  1074. }
  1075. var headerLength = Uint32Array.BYTES_PER_ELEMENT * 6;
  1076. var bufferLength = headerLength + index.byteLength + data.byteLength;
  1077. var buffer = new ArrayBuffer(Math.ceil(bufferLength / 4) * 4);
  1078. var view32 = new Uint32Array(buffer);
  1079. var view16 = new Uint16Array(buffer);
  1080. view32[0] = trie.initialValue;
  1081. view32[1] = trie.errorValue;
  1082. view32[2] = trie.highStart;
  1083. view32[3] = trie.highValueIndex;
  1084. view32[4] = index.byteLength;
  1085. // $FlowFixMe
  1086. view32[5] = data.BYTES_PER_ELEMENT;
  1087. view16.set(index, headerLength / Uint16Array.BYTES_PER_ELEMENT);
  1088. if (data.BYTES_PER_ELEMENT === Uint16Array.BYTES_PER_ELEMENT) {
  1089. view16.set(data, (headerLength + index.byteLength) / Uint16Array.BYTES_PER_ELEMENT);
  1090. }
  1091. else {
  1092. view32.set(data, Math.ceil((headerLength + index.byteLength) / Uint32Array.BYTES_PER_ELEMENT));
  1093. }
  1094. return [encode(new Uint8Array(buffer)), buffer.byteLength];
  1095. };
  1096. export { Trie, TrieBuilder, createTrieFromBase64, serializeBase64 };
  1097. //# sourceMappingURL=utrie.es5.js.map