tokenizer.js 1.3 KB

12345678910111213141516171819202122232425262728293031323334353637383940414243444546474849505152535455
  1. var SyntaxError = require('./SyntaxError');
  2. var TAB = 9;
  3. var N = 10;
  4. var F = 12;
  5. var R = 13;
  6. var SPACE = 32;
  7. var Tokenizer = function(str) {
  8. this.str = str;
  9. this.pos = 0;
  10. };
  11. Tokenizer.prototype = {
  12. charCodeAt: function(pos) {
  13. return pos < this.str.length ? this.str.charCodeAt(pos) : 0;
  14. },
  15. charCode: function() {
  16. return this.charCodeAt(this.pos);
  17. },
  18. nextCharCode: function() {
  19. return this.charCodeAt(this.pos + 1);
  20. },
  21. nextNonWsCode: function(pos) {
  22. return this.charCodeAt(this.findWsEnd(pos));
  23. },
  24. findWsEnd: function(pos) {
  25. for (; pos < this.str.length; pos++) {
  26. var code = this.str.charCodeAt(pos);
  27. if (code !== R && code !== N && code !== F && code !== SPACE && code !== TAB) {
  28. break;
  29. }
  30. }
  31. return pos;
  32. },
  33. substringToPos: function(end) {
  34. return this.str.substring(this.pos, this.pos = end);
  35. },
  36. eat: function(code) {
  37. if (this.charCode() !== code) {
  38. this.error('Expect `' + String.fromCharCode(code) + '`');
  39. }
  40. this.pos++;
  41. },
  42. peek: function() {
  43. return this.pos < this.str.length ? this.str.charAt(this.pos++) : '';
  44. },
  45. error: function(message) {
  46. throw new SyntaxError(message, this.str, this.pos);
  47. }
  48. };
  49. module.exports = Tokenizer;