utils.js 7.2 KB

123456789101112131415161718192021222324252627282930313233343536373839404142434445464748495051525354555657585960616263646566676869707172737475767778798081828384858687888990919293949596979899100101102103104105106107108109110111112113114115116117118119120121122123124125126127128129130131132133134135136137138139140141142143144145146147148149150151152153154155156157158159160161162163164165166167168169170171172173174175176177178179180181182183184185186187188189190191192193194195196197198199200201202203204205206207208209210211212213214215216217218219220221222223224225226227228229230231232233234235236237238239240241242243
  1. var charCodeDef = require('./char-code-definitions');
  2. var isDigit = charCodeDef.isDigit;
  3. var isHexDigit = charCodeDef.isHexDigit;
  4. var isUppercaseLetter = charCodeDef.isUppercaseLetter;
  5. var isName = charCodeDef.isName;
  6. var isWhiteSpace = charCodeDef.isWhiteSpace;
  7. var isValidEscape = charCodeDef.isValidEscape;
  8. function getCharCode(source, offset) {
  9. return offset < source.length ? source.charCodeAt(offset) : 0;
  10. }
  11. function getNewlineLength(source, offset, code) {
  12. if (code === 13 /* \r */ && getCharCode(source, offset + 1) === 10 /* \n */) {
  13. return 2;
  14. }
  15. return 1;
  16. }
  17. function cmpChar(testStr, offset, referenceCode) {
  18. var code = testStr.charCodeAt(offset);
  19. // code.toLowerCase() for A..Z
  20. if (isUppercaseLetter(code)) {
  21. code = code | 32;
  22. }
  23. return code === referenceCode;
  24. }
  25. function cmpStr(testStr, start, end, referenceStr) {
  26. if (end - start !== referenceStr.length) {
  27. return false;
  28. }
  29. if (start < 0 || end > testStr.length) {
  30. return false;
  31. }
  32. for (var i = start; i < end; i++) {
  33. var testCode = testStr.charCodeAt(i);
  34. var referenceCode = referenceStr.charCodeAt(i - start);
  35. // testCode.toLowerCase() for A..Z
  36. if (isUppercaseLetter(testCode)) {
  37. testCode = testCode | 32;
  38. }
  39. if (testCode !== referenceCode) {
  40. return false;
  41. }
  42. }
  43. return true;
  44. }
  45. function findWhiteSpaceStart(source, offset) {
  46. for (; offset >= 0; offset--) {
  47. if (!isWhiteSpace(source.charCodeAt(offset))) {
  48. break;
  49. }
  50. }
  51. return offset + 1;
  52. }
  53. function findWhiteSpaceEnd(source, offset) {
  54. for (; offset < source.length; offset++) {
  55. if (!isWhiteSpace(source.charCodeAt(offset))) {
  56. break;
  57. }
  58. }
  59. return offset;
  60. }
  61. function findDecimalNumberEnd(source, offset) {
  62. for (; offset < source.length; offset++) {
  63. if (!isDigit(source.charCodeAt(offset))) {
  64. break;
  65. }
  66. }
  67. return offset;
  68. }
  69. // § 4.3.7. Consume an escaped code point
  70. function consumeEscaped(source, offset) {
  71. // It assumes that the U+005C REVERSE SOLIDUS (\) has already been consumed and
  72. // that the next input code point has already been verified to be part of a valid escape.
  73. offset += 2;
  74. // hex digit
  75. if (isHexDigit(getCharCode(source, offset - 1))) {
  76. // Consume as many hex digits as possible, but no more than 5.
  77. // Note that this means 1-6 hex digits have been consumed in total.
  78. for (var maxOffset = Math.min(source.length, offset + 5); offset < maxOffset; offset++) {
  79. if (!isHexDigit(getCharCode(source, offset))) {
  80. break;
  81. }
  82. }
  83. // If the next input code point is whitespace, consume it as well.
  84. var code = getCharCode(source, offset);
  85. if (isWhiteSpace(code)) {
  86. offset += getNewlineLength(source, offset, code);
  87. }
  88. }
  89. return offset;
  90. }
  91. // §4.3.11. Consume a name
  92. // Note: This algorithm does not do the verification of the first few code points that are necessary
  93. // to ensure the returned code points would constitute an <ident-token>. If that is the intended use,
  94. // ensure that the stream starts with an identifier before calling this algorithm.
  95. function consumeName(source, offset) {
  96. // Let result initially be an empty string.
  97. // Repeatedly consume the next input code point from the stream:
  98. for (; offset < source.length; offset++) {
  99. var code = source.charCodeAt(offset);
  100. // name code point
  101. if (isName(code)) {
  102. // Append the code point to result.
  103. continue;
  104. }
  105. // the stream starts with a valid escape
  106. if (isValidEscape(code, getCharCode(source, offset + 1))) {
  107. // Consume an escaped code point. Append the returned code point to result.
  108. offset = consumeEscaped(source, offset) - 1;
  109. continue;
  110. }
  111. // anything else
  112. // Reconsume the current input code point. Return result.
  113. break;
  114. }
  115. return offset;
  116. }
  117. // §4.3.12. Consume a number
  118. function consumeNumber(source, offset) {
  119. var code = source.charCodeAt(offset);
  120. // 2. If the next input code point is U+002B PLUS SIGN (+) or U+002D HYPHEN-MINUS (-),
  121. // consume it and append it to repr.
  122. if (code === 0x002B || code === 0x002D) {
  123. code = source.charCodeAt(offset += 1);
  124. }
  125. // 3. While the next input code point is a digit, consume it and append it to repr.
  126. if (isDigit(code)) {
  127. offset = findDecimalNumberEnd(source, offset + 1);
  128. code = source.charCodeAt(offset);
  129. }
  130. // 4. If the next 2 input code points are U+002E FULL STOP (.) followed by a digit, then:
  131. if (code === 0x002E && isDigit(source.charCodeAt(offset + 1))) {
  132. // 4.1 Consume them.
  133. // 4.2 Append them to repr.
  134. code = source.charCodeAt(offset += 2);
  135. // 4.3 Set type to "number".
  136. // TODO
  137. // 4.4 While the next input code point is a digit, consume it and append it to repr.
  138. offset = findDecimalNumberEnd(source, offset);
  139. }
  140. // 5. If the next 2 or 3 input code points are U+0045 LATIN CAPITAL LETTER E (E)
  141. // or U+0065 LATIN SMALL LETTER E (e), ... , followed by a digit, then:
  142. if (cmpChar(source, offset, 101 /* e */)) {
  143. var sign = 0;
  144. code = source.charCodeAt(offset + 1);
  145. // ... optionally followed by U+002D HYPHEN-MINUS (-) or U+002B PLUS SIGN (+) ...
  146. if (code === 0x002D || code === 0x002B) {
  147. sign = 1;
  148. code = source.charCodeAt(offset + 2);
  149. }
  150. // ... followed by a digit
  151. if (isDigit(code)) {
  152. // 5.1 Consume them.
  153. // 5.2 Append them to repr.
  154. // 5.3 Set type to "number".
  155. // TODO
  156. // 5.4 While the next input code point is a digit, consume it and append it to repr.
  157. offset = findDecimalNumberEnd(source, offset + 1 + sign + 1);
  158. }
  159. }
  160. return offset;
  161. }
  162. // § 4.3.14. Consume the remnants of a bad url
  163. // ... its sole use is to consume enough of the input stream to reach a recovery point
  164. // where normal tokenizing can resume.
  165. function consumeBadUrlRemnants(source, offset) {
  166. // Repeatedly consume the next input code point from the stream:
  167. for (; offset < source.length; offset++) {
  168. var code = source.charCodeAt(offset);
  169. // U+0029 RIGHT PARENTHESIS ())
  170. // EOF
  171. if (code === 0x0029) {
  172. // Return.
  173. offset++;
  174. break;
  175. }
  176. if (isValidEscape(code, getCharCode(source, offset + 1))) {
  177. // Consume an escaped code point.
  178. // Note: This allows an escaped right parenthesis ("\)") to be encountered
  179. // without ending the <bad-url-token>. This is otherwise identical to
  180. // the "anything else" clause.
  181. offset = consumeEscaped(source, offset);
  182. }
  183. }
  184. return offset;
  185. }
  186. module.exports = {
  187. consumeEscaped: consumeEscaped,
  188. consumeName: consumeName,
  189. consumeNumber: consumeNumber,
  190. consumeBadUrlRemnants: consumeBadUrlRemnants,
  191. cmpChar: cmpChar,
  192. cmpStr: cmpStr,
  193. getNewlineLength: getNewlineLength,
  194. findWhiteSpaceStart: findWhiteSpaceStart,
  195. findWhiteSpaceEnd: findWhiteSpaceEnd
  196. };