simono41/shellinabox
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

Added a tokenizer that can recognize BASIC tokens, instead of just

Browse source 
splitting on white space.


git-svn-id: https://shellinabox.googlecode.com/svn/trunk@104 0da03de8-d603-11dd-86c2-0f8696b7b6f9
This commit is contained in:
zodiac 2009-03-31 17:19:21 +00:00
parent 6021dd8aaa
commit 7f32534490
2 changed files with 163 additions and 71 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

112
demo/demo.js
View file

@ -233,16 +233,16 @@ Demo.prototype.doReadLine = function() {
Demo.prototype.doCommand = function() { Demo.prototype.doCommand = function() {
this.gotoState(2 /* STATE_PROMPT */); this.gotoState(2 /* STATE_PROMPT */);
var tokens = new Tokens(this.line); var tokens = new this.Tokens(this.line);
this.line = ''; this.line = '';
var cmd = tokens.nextToken(); var cmd = tokens.nextToken();
if (cmd != undefined) { if (cmd) {
cmd = cmd.toUpperCase(); cmd = cmd.toUpperCase();
if (cmd.match(/^[0-9]+$/)) { if (cmd.match(/^[0-9]+$/)) {
tokens.removeLineNumber(); tokens.removeLineNumber();
var lineNumber = parseInt(cmd); var lineNumber = parseInt(cmd);
var index = this.findLine(lineNumber); var index = this.findLine(lineNumber);
if (tokens.nextToken() == undefined) { if (tokens.nextToken() == null) {
if (index > 0) { if (index > 0) {
// Delete line from program // Delete line from program
this.program.splice(index, 1); this.program.splice(index, 1);
@ -253,7 +253,8 @@ Demo.prototype.doCommand = function() {
this.program[index].setTokens(tokens); this.program[index].setTokens(tokens);
} else { } else {
// Add new line to program // Add new line to program
this.program.splice(-index - 1, 0, new Line(lineNumber, tokens)); this.program.splice(-index - 1, 0,
new this.Line(lineNumber, tokens));
} }
} }
} else { } else {
@ -273,14 +274,14 @@ Demo.prototype.doEval = function() {
this.vt100('Supported commands:\r\n' + this.vt100('Supported commands:\r\n' +
' HELP LIST NEW RUN\r\n'); ' HELP LIST NEW RUN\r\n');
} else if (cmd == "LIST") { } else if (cmd == "LIST") {
if (this.tokens.nextToken() != undefined) { if (this.tokens.nextToken() != null) {
this.error(); this.error();
} else { } else {
for (var i = 0; i < this.program.length; i++) { for (var i = 0; i < this.program.length; i++) {
var line = this.program[i]; var line = this.program[i];
this.vt100('' + line.lineNumber()); this.vt100('' + line.lineNumber());
line.tokens().reset(); line.tokens().reset();
for (var token; (token = line.tokens().nextToken()) != undefined; ) { for (var token; (token = line.tokens().nextToken()) != null; ) {
this.vt100(' ' + token); this.vt100(' ' + token);
} }
line.tokens().reset(); line.tokens().reset();
@ -297,7 +298,7 @@ Demo.prototype.doEval = function() {
this.gotoState(7 /* STATE_NEW_Y_N */); this.gotoState(7 /* STATE_NEW_Y_N */);
} }
} else if (cmd == "RUN") { } else if (cmd == "RUN") {
if (this.tokens.nextToken() != undefined) { if (this.tokens.nextToken() != null) {
this.error(); this.error();
} else if (this.program.length > 0) { } else if (this.program.length > 0) {
this.currentLineIndex = 0; this.currentLineIndex = 0;
@ -365,53 +366,98 @@ Demo.prototype.findLine = function(lineNumber) {
return -l - 1; return -l - 1;
}; };
function Tokens(line) { Demo.prototype.Tokens = function(line) {
this.tokens = line.split(' '); this.line = line;
this.index = 0; this.tokens = line;
}; };
Tokens.prototype.nextToken = function() { Demo.prototype.Tokens.prototype.nextToken = function() {
while (this.index < this.tokens.length) { var tokens = this.tokens.replace(/^[ \t]*/, '');
var token = this.tokens[this.index++]; if (!tokens.length) {
token = token.replace(/^[ \t]*/, ''); return null;
token = token.replace(/[ \t]*$/, ''); }
if (token.length) { var token = tokens.charAt(0);
return token; switch (token) {
case '<':
if (tokens.length > 1) {
if (tokens.charAt(1) == '>') {
token = '<>';
} else if (tokens.charAt(1) == '=') {
token = '<=';
}
}
break;
case '>':
if (tokens.charAt(1) == '=') {
token = '>=';
}
break;
case '=':
case '+':
case '-':
case '*':
case '/':
case '(':
case ')':
case '?':
case ',':
case ';':
case '"':
case ':':
case '$':
case '%':
case '#':
break;
default:
if (token >= '0' && token <= '9' || token == '.') {
token = tokens.match(/^[0-9]*(?:[.][0-9]*)?(?:[eE][-+]?[0-9]+)?/);
if (token) {
token = token[0];
}
} else if (token >= 'A' && token <= 'Z' ||
token >= 'a' && token <= 'z') {
token = tokens.match(/^[A-Za-z][A-Za-z0-9_]*/);
if (token) {
token = token[0];
}
} else {
token = '';
} }
} }
return undefined; if (token) {
}; this.tokens = tokens.substr(token.length);
} else {
Tokens.prototype.removeLineNumber = function() { this.tokens = tokens.substr(1);
if (this.tokens.length > 0) { token = undefined;
this.tokens.splice(0, 1);
if (this.index > 0) {
this.index--;
}
} }
return token;
}; };
Tokens.prototype.reset = function() { Demo.prototype.Tokens.prototype.removeLineNumber = function() {
this.index = 0; this.line = this.line.replace(/^[0-9]*[ \t]*/, '');
}; };
function Line(lineNumber, tokens) { Demo.prototype.Tokens.prototype.reset = function() {
this.tokens = this.line;
};
Demo.prototype.Line = function(lineNumber, tokens) {
this.lineNumber_ = lineNumber; this.lineNumber_ = lineNumber;
this.tokens_ = tokens; this.tokens_ = tokens;
}; };
Line.prototype.lineNumber = function() { Demo.prototype.Line.prototype.lineNumber = function() {
return this.lineNumber_; return this.lineNumber_;
}; };
Line.prototype.tokens = function() { Demo.prototype.Line.prototype.tokens = function() {
return this.tokens_; return this.tokens_;
}; };
Line.prototype.setTokens = function(tokens) { Demo.prototype.Line.prototype.setTokens = function(tokens) {
this.tokens_ = tokens; this.tokens_ = tokens;
}; };
Line.prototype.sort = function(a, b) { Demo.prototype.Line.prototype.sort = function(a, b) {
return a.lineNumber_ - b.lineNumber_; return a.lineNumber_ - b.lineNumber_;
}; };

122
demo/demo.jspp
View file

@ -233,16 +233,16 @@ Demo.prototype.doReadLine = function() {
Demo.prototype.doCommand = function() { Demo.prototype.doCommand = function() {
this.gotoState(STATE_PROMPT); this.gotoState(STATE_PROMPT);
var tokens = new Tokens(this.line); var tokens = new this.Tokens(this.line);
this.line = ''; this.line = '';
var cmd = tokens.nextToken(); var cmd = tokens.nextToken();
if (cmd != undefined) { if (cmd) {
cmd = cmd.toUpperCase(); cmd = cmd.toUpperCase();
if (cmd.match(/^[0-9]+$/)) { if (cmd.match(/^[0-9]+$/)) {
tokens.removeLineNumber(); tokens.removeLineNumber();
var lineNumber = parseInt(cmd); var lineNumber = parseInt(cmd);
var index = this.findLine(lineNumber); var index = this.findLine(lineNumber);
if (tokens.nextToken() == undefined) { if (tokens.nextToken() == null) {
if (index > 0) { if (index > 0) {
// Delete line from program // Delete line from program
this.program.splice(index, 1); this.program.splice(index, 1);
@ -253,7 +253,8 @@ Demo.prototype.doCommand = function() {
this.program[index].setTokens(tokens); this.program[index].setTokens(tokens);
} else { } else {
// Add new line to program // Add new line to program
this.program.splice(-index - 1, 0, new Line(lineNumber, tokens)); this.program.splice(-index - 1, 0,
new this.Line(lineNumber, tokens));
} }
} }
} else { } else {
@ -268,26 +269,26 @@ Demo.prototype.doCommand = function() {
Demo.prototype.doEval = function() { Demo.prototype.doEval = function() {
this.gotoState(STATE_PROMPT); this.gotoState(STATE_PROMPT);
var cmd = this.tokens.nextToken().toUpperCase(); var token = this.tokens.nextToken().toUpperCase();
if (cmd == "HELP") { if (token == "HELP") {
this.vt100('Supported commands:\r\n' + this.vt100('Supported commands:\r\n' +
' HELP LIST NEW RUN\r\n'); ' HELP LIST NEW RUN\r\n');
} else if (cmd == "LIST") { } else if (token == "LIST") {
if (this.tokens.nextToken() != undefined) { if (this.tokens.nextToken() != null) {
this.error(); this.error();
} else { } else {
for (var i = 0; i < this.program.length; i++) { for (var i = 0; i < this.program.length; i++) {
var line = this.program[i]; var line = this.program[i];
this.vt100('' + line.lineNumber()); this.vt100('' + line.lineNumber());
line.tokens().reset(); line.tokens().reset();
for (var token; (token = line.tokens().nextToken()) != undefined; ) { for (var token; (token = line.tokens().nextToken()) != null; ) {
this.vt100(' ' + token); this.vt100(' ' + token);
} }
line.tokens().reset(); line.tokens().reset();
this.vt100('\r\n'); this.vt100('\r\n');
} }
} }
} else if (cmd == "NEW") { } else if (token == "NEW") {
if (this.currentLineIndex >= 0) { if (this.currentLineIndex >= 0) {
this.error('Cannot call NEW from a program'); this.error('Cannot call NEW from a program');
} else if (this.program.length == 0) { } else if (this.program.length == 0) {
@ -296,8 +297,8 @@ Demo.prototype.doEval = function() {
this.vt100('Do you really want to delete the program (y/N) '); this.vt100('Do you really want to delete the program (y/N) ');
this.gotoState(STATE_NEW_Y_N); this.gotoState(STATE_NEW_Y_N);
} }
} else if (cmd == "RUN") { } else if (token == "RUN") {
if (this.tokens.nextToken() != undefined) { if (this.tokens.nextToken() != null) {
this.error(); this.error();
} else if (this.program.length > 0) { } else if (this.program.length > 0) {
this.currentLineIndex = 0; this.currentLineIndex = 0;
@ -365,53 +366,98 @@ Demo.prototype.findLine = function(lineNumber) {
return -l - 1; return -l - 1;
}; };
function Tokens(line) { Demo.prototype.Tokens = function(line) {
this.tokens = line.split(' '); this.line = line;
this.index = 0; this.tokens = line;
}; };
Tokens.prototype.nextToken = function() { Demo.prototype.Tokens.prototype.nextToken = function() {
while (this.index < this.tokens.length) { var tokens = this.tokens.replace(/^[ \t]*/, '');
var token = this.tokens[this.index++]; if (!tokens.length) {
token = token.replace(/^[ \t]*/, ''); return null;
token = token.replace(/[ \t]*$/, ''); }
if (token.length) { var token = tokens.charAt(0);
return token; switch (token) {
case '<':
if (tokens.length > 1) {
if (tokens.charAt(1) == '>') {
token = '<>';
} else if (tokens.charAt(1) == '=') {
token = '<=';
}
}
break;
case '>':
if (tokens.charAt(1) == '=') {
token = '>=';
}
break;
case '=':
case '+':
case '-':
case '*':
case '/':
case '(':
case ')':
case '?':
case ',':
case ';':
case '"':
case ':':
case '$':
case '%':
case '#':
break;
default:
if (token >= '0' && token <= '9' || token == '.') {
token = tokens.match(/^[0-9]*(?:[.][0-9]*)?(?:[eE][-+]?[0-9]+)?/);
if (token) {
token = token[0];
}
} else if (token >= 'A' && token <= 'Z' ||
token >= 'a' && token <= 'z') {
token = tokens.match(/^[A-Za-z][A-Za-z0-9_]*/);
if (token) {
token = token[0];
}
} else {
token = '';
} }
} }
return undefined; if (token) {
}; this.tokens = tokens.substr(token.length);
} else {
Tokens.prototype.removeLineNumber = function() { this.tokens = tokens.substr(1);
if (this.tokens.length > 0) { token = undefined;
this.tokens.splice(0, 1);
if (this.index > 0) {
this.index--;
}
} }
return token;
}; };
Tokens.prototype.reset = function() { Demo.prototype.Tokens.prototype.removeLineNumber = function() {
this.index = 0; this.line = this.line.replace(/^[0-9]*[ \t]*/, '');
}; };
function Line(lineNumber, tokens) { Demo.prototype.Tokens.prototype.reset = function() {
this.tokens = this.line;
};
Demo.prototype.Line = function(lineNumber, tokens) {
this.lineNumber_ = lineNumber; this.lineNumber_ = lineNumber;
this.tokens_ = tokens; this.tokens_ = tokens;
}; };
Line.prototype.lineNumber = function() { Demo.prototype.Line.prototype.lineNumber = function() {
return this.lineNumber_; return this.lineNumber_;
}; };
Line.prototype.tokens = function() { Demo.prototype.Line.prototype.tokens = function() {
return this.tokens_; return this.tokens_;
}; };
Line.prototype.setTokens = function(tokens) { Demo.prototype.Line.prototype.setTokens = function(tokens) {
this.tokens_ = tokens; this.tokens_ = tokens;
}; };
Line.prototype.sort = function(a, b) { Demo.prototype.Line.prototype.sort = function(a, b) {
return a.lineNumber_ - b.lineNumber_; return a.lineNumber_ - b.lineNumber_;
}; };