From 7f325344906bafe741e7c11449614caffedcb3cd Mon Sep 17 00:00:00 2001 From: zodiac Date: Tue, 31 Mar 2009 17:19:21 +0000 Subject: [PATCH] Added a tokenizer that can recognize BASIC tokens, instead of just splitting on white space. git-svn-id: https://shellinabox.googlecode.com/svn/trunk@104 0da03de8-d603-11dd-86c2-0f8696b7b6f9 --- demo/demo.js | 112 ++++++++++++++++++++++++++++++++------------- demo/demo.jspp | 122 ++++++++++++++++++++++++++++++++++--------------- 2 files changed, 163 insertions(+), 71 deletions(-) diff --git a/demo/demo.js b/demo/demo.js index 25527e3..3022b40 100644 --- a/demo/demo.js +++ b/demo/demo.js @@ -233,16 +233,16 @@ Demo.prototype.doReadLine = function() { Demo.prototype.doCommand = function() { this.gotoState(2 /* STATE_PROMPT */); - var tokens = new Tokens(this.line); + var tokens = new this.Tokens(this.line); this.line = ''; var cmd = tokens.nextToken(); - if (cmd != undefined) { + if (cmd) { cmd = cmd.toUpperCase(); if (cmd.match(/^[0-9]+$/)) { tokens.removeLineNumber(); var lineNumber = parseInt(cmd); var index = this.findLine(lineNumber); - if (tokens.nextToken() == undefined) { + if (tokens.nextToken() == null) { if (index > 0) { // Delete line from program this.program.splice(index, 1); @@ -253,7 +253,8 @@ Demo.prototype.doCommand = function() { this.program[index].setTokens(tokens); } else { // Add new line to program - this.program.splice(-index - 1, 0, new Line(lineNumber, tokens)); + this.program.splice(-index - 1, 0, + new this.Line(lineNumber, tokens)); } } } else { @@ -273,14 +274,14 @@ Demo.prototype.doEval = function() { this.vt100('Supported commands:\r\n' + ' HELP LIST NEW RUN\r\n'); } else if (cmd == "LIST") { - if (this.tokens.nextToken() != undefined) { + if (this.tokens.nextToken() != null) { this.error(); } else { for (var i = 0; i < this.program.length; i++) { var line = this.program[i]; this.vt100('' + line.lineNumber()); line.tokens().reset(); - for (var token; (token = line.tokens().nextToken()) != undefined; ) { + for (var token; (token = line.tokens().nextToken()) != null; ) { this.vt100(' ' + token); } line.tokens().reset(); @@ -297,7 +298,7 @@ Demo.prototype.doEval = function() { this.gotoState(7 /* STATE_NEW_Y_N */); } } else if (cmd == "RUN") { - if (this.tokens.nextToken() != undefined) { + if (this.tokens.nextToken() != null) { this.error(); } else if (this.program.length > 0) { this.currentLineIndex = 0; @@ -365,53 +366,98 @@ Demo.prototype.findLine = function(lineNumber) { return -l - 1; }; -function Tokens(line) { - this.tokens = line.split(' '); - this.index = 0; +Demo.prototype.Tokens = function(line) { + this.line = line; + this.tokens = line; }; -Tokens.prototype.nextToken = function() { - while (this.index < this.tokens.length) { - var token = this.tokens[this.index++]; - token = token.replace(/^[ \t]*/, ''); - token = token.replace(/[ \t]*$/, ''); - if (token.length) { - return token; +Demo.prototype.Tokens.prototype.nextToken = function() { + var tokens = this.tokens.replace(/^[ \t]*/, ''); + if (!tokens.length) { + return null; + } + var token = tokens.charAt(0); + switch (token) { + case '<': + if (tokens.length > 1) { + if (tokens.charAt(1) == '>') { + token = '<>'; + } else if (tokens.charAt(1) == '=') { + token = '<='; + } + } + break; + case '>': + if (tokens.charAt(1) == '=') { + token = '>='; + } + break; + case '=': + case '+': + case '-': + case '*': + case '/': + case '(': + case ')': + case '?': + case ',': + case ';': + case '"': + case ':': + case '$': + case '%': + case '#': + break; + default: + if (token >= '0' && token <= '9' || token == '.') { + token = tokens.match(/^[0-9]*(?:[.][0-9]*)?(?:[eE][-+]?[0-9]+)?/); + if (token) { + token = token[0]; + } + } else if (token >= 'A' && token <= 'Z' || + token >= 'a' && token <= 'z') { + token = tokens.match(/^[A-Za-z][A-Za-z0-9_]*/); + if (token) { + token = token[0]; + } + } else { + token = ''; } } - return undefined; -}; - -Tokens.prototype.removeLineNumber = function() { - if (this.tokens.length > 0) { - this.tokens.splice(0, 1); - if (this.index > 0) { - this.index--; - } + if (token) { + this.tokens = tokens.substr(token.length); + } else { + this.tokens = tokens.substr(1); + token = undefined; } + return token; }; -Tokens.prototype.reset = function() { - this.index = 0; +Demo.prototype.Tokens.prototype.removeLineNumber = function() { + this.line = this.line.replace(/^[0-9]*[ \t]*/, ''); }; -function Line(lineNumber, tokens) { +Demo.prototype.Tokens.prototype.reset = function() { + this.tokens = this.line; +}; + +Demo.prototype.Line = function(lineNumber, tokens) { this.lineNumber_ = lineNumber; this.tokens_ = tokens; }; -Line.prototype.lineNumber = function() { +Demo.prototype.Line.prototype.lineNumber = function() { return this.lineNumber_; }; -Line.prototype.tokens = function() { +Demo.prototype.Line.prototype.tokens = function() { return this.tokens_; }; -Line.prototype.setTokens = function(tokens) { +Demo.prototype.Line.prototype.setTokens = function(tokens) { this.tokens_ = tokens; }; -Line.prototype.sort = function(a, b) { +Demo.prototype.Line.prototype.sort = function(a, b) { return a.lineNumber_ - b.lineNumber_; }; diff --git a/demo/demo.jspp b/demo/demo.jspp index 56ddca0..5fa78ca 100644 --- a/demo/demo.jspp +++ b/demo/demo.jspp @@ -233,16 +233,16 @@ Demo.prototype.doReadLine = function() { Demo.prototype.doCommand = function() { this.gotoState(STATE_PROMPT); - var tokens = new Tokens(this.line); + var tokens = new this.Tokens(this.line); this.line = ''; var cmd = tokens.nextToken(); - if (cmd != undefined) { + if (cmd) { cmd = cmd.toUpperCase(); if (cmd.match(/^[0-9]+$/)) { tokens.removeLineNumber(); var lineNumber = parseInt(cmd); var index = this.findLine(lineNumber); - if (tokens.nextToken() == undefined) { + if (tokens.nextToken() == null) { if (index > 0) { // Delete line from program this.program.splice(index, 1); @@ -253,7 +253,8 @@ Demo.prototype.doCommand = function() { this.program[index].setTokens(tokens); } else { // Add new line to program - this.program.splice(-index - 1, 0, new Line(lineNumber, tokens)); + this.program.splice(-index - 1, 0, + new this.Line(lineNumber, tokens)); } } } else { @@ -268,26 +269,26 @@ Demo.prototype.doCommand = function() { Demo.prototype.doEval = function() { this.gotoState(STATE_PROMPT); - var cmd = this.tokens.nextToken().toUpperCase(); - if (cmd == "HELP") { + var token = this.tokens.nextToken().toUpperCase(); + if (token == "HELP") { this.vt100('Supported commands:\r\n' + ' HELP LIST NEW RUN\r\n'); - } else if (cmd == "LIST") { - if (this.tokens.nextToken() != undefined) { + } else if (token == "LIST") { + if (this.tokens.nextToken() != null) { this.error(); } else { for (var i = 0; i < this.program.length; i++) { var line = this.program[i]; this.vt100('' + line.lineNumber()); line.tokens().reset(); - for (var token; (token = line.tokens().nextToken()) != undefined; ) { + for (var token; (token = line.tokens().nextToken()) != null; ) { this.vt100(' ' + token); } line.tokens().reset(); this.vt100('\r\n'); } } - } else if (cmd == "NEW") { + } else if (token == "NEW") { if (this.currentLineIndex >= 0) { this.error('Cannot call NEW from a program'); } else if (this.program.length == 0) { @@ -296,8 +297,8 @@ Demo.prototype.doEval = function() { this.vt100('Do you really want to delete the program (y/N) '); this.gotoState(STATE_NEW_Y_N); } - } else if (cmd == "RUN") { - if (this.tokens.nextToken() != undefined) { + } else if (token == "RUN") { + if (this.tokens.nextToken() != null) { this.error(); } else if (this.program.length > 0) { this.currentLineIndex = 0; @@ -365,53 +366,98 @@ Demo.prototype.findLine = function(lineNumber) { return -l - 1; }; -function Tokens(line) { - this.tokens = line.split(' '); - this.index = 0; +Demo.prototype.Tokens = function(line) { + this.line = line; + this.tokens = line; }; -Tokens.prototype.nextToken = function() { - while (this.index < this.tokens.length) { - var token = this.tokens[this.index++]; - token = token.replace(/^[ \t]*/, ''); - token = token.replace(/[ \t]*$/, ''); - if (token.length) { - return token; +Demo.prototype.Tokens.prototype.nextToken = function() { + var tokens = this.tokens.replace(/^[ \t]*/, ''); + if (!tokens.length) { + return null; + } + var token = tokens.charAt(0); + switch (token) { + case '<': + if (tokens.length > 1) { + if (tokens.charAt(1) == '>') { + token = '<>'; + } else if (tokens.charAt(1) == '=') { + token = '<='; + } + } + break; + case '>': + if (tokens.charAt(1) == '=') { + token = '>='; + } + break; + case '=': + case '+': + case '-': + case '*': + case '/': + case '(': + case ')': + case '?': + case ',': + case ';': + case '"': + case ':': + case '$': + case '%': + case '#': + break; + default: + if (token >= '0' && token <= '9' || token == '.') { + token = tokens.match(/^[0-9]*(?:[.][0-9]*)?(?:[eE][-+]?[0-9]+)?/); + if (token) { + token = token[0]; + } + } else if (token >= 'A' && token <= 'Z' || + token >= 'a' && token <= 'z') { + token = tokens.match(/^[A-Za-z][A-Za-z0-9_]*/); + if (token) { + token = token[0]; + } + } else { + token = ''; } } - return undefined; -}; - -Tokens.prototype.removeLineNumber = function() { - if (this.tokens.length > 0) { - this.tokens.splice(0, 1); - if (this.index > 0) { - this.index--; - } + if (token) { + this.tokens = tokens.substr(token.length); + } else { + this.tokens = tokens.substr(1); + token = undefined; } + return token; }; -Tokens.prototype.reset = function() { - this.index = 0; +Demo.prototype.Tokens.prototype.removeLineNumber = function() { + this.line = this.line.replace(/^[0-9]*[ \t]*/, ''); }; -function Line(lineNumber, tokens) { +Demo.prototype.Tokens.prototype.reset = function() { + this.tokens = this.line; +}; + +Demo.prototype.Line = function(lineNumber, tokens) { this.lineNumber_ = lineNumber; this.tokens_ = tokens; }; -Line.prototype.lineNumber = function() { +Demo.prototype.Line.prototype.lineNumber = function() { return this.lineNumber_; }; -Line.prototype.tokens = function() { +Demo.prototype.Line.prototype.tokens = function() { return this.tokens_; }; -Line.prototype.setTokens = function(tokens) { +Demo.prototype.Line.prototype.setTokens = function(tokens) { this.tokens_ = tokens; }; -Line.prototype.sort = function(a, b) { +Demo.prototype.Line.prototype.sort = function(a, b) { return a.lineNumber_ - b.lineNumber_; };