Tokenizer.java
|
// FILE. . . . . /home/hak/ilt/src/ilog/rcl/Tokenizer.java // EDIT BY . . . Hassan Ait-Kaci // ON MACHINE. . Latitude407.Ilog.Biz // STARTED ON. . Wed May 3 14:37:03 2006 package ilog.rcl;
This is a skeleton tokenizer for building a java parser from the Jacc
grammar specified in RCL.grm. It is
mininal in that it relies on just enough information to make a toy
parser. It is rather self-explanatory.
|
import java.io.Reader; import java.io.FileReader; import java.io.BufferedReader; import java.io.IOException; import ilog.language.io.StreamTokenizer; import ilog.language.syntax.*; public class Tokenizer implements FileTokenizer { int lineno; String file; Reader reader; StreamTokenizer st; boolean prompt; public Tokenizer () { } public Tokenizer (String file) throws IOException { setFile(file); } public void setFile (String file) throws IOException { setFileName(file); setReader(new BufferedReader(new FileReader(file))); } public final int lineNumber() { return lineno; } public final void setReader (Reader rd) { reader = rd; st = new StreamTokenizer(reader); st.ordinaryChars("+-.(){}[]<>;,:=!&|*/%"); st.setType('.',st.NUMERIC); st.quoteChar('\''); st.quoteChar('"'); st.wordChars("_@?$"); } public final Reader getReader () { return reader; } public final String fileName () { return file; } public final void setFileName (String file) { this.file = file; } final ParseNode locate (ParseNode node) { return ((ParseNode)node.setStart(st.tokenStart()).setEnd(st.tokenEnd())).setFile(file); } final ParseNode integer () { return locate(Parser.numberToken("Data",(int)st.nval)); } final ParseNode floatingPoint () { return locate(Parser.numberToken("Data",st.nval)); } final ParseNode string () { return locate(Parser.symbolToken("Data","\""+st.sval+"\"")); } final ParseNode character () { return locate(Parser.symbolToken("Data","'"+st.sval+"'")); } final ParseNode object (String symbol) { return locate(Parser.symbolToken("Ind",symbol)); } final ParseNode variable (String symbol) { return locate(Parser.symbolToken("Var",symbol.substring(1))); } final ParseNode function (String symbol) { return locate(Parser.symbolToken("Fun",symbol)); } final ParseNode relation (String symbol) { return locate(Parser.symbolToken("Rel",symbol)); } final ParseNode literal (String s) { return locate(Parser.literalToken(s)); } final ParseNode literal (int c) { return locate(Parser.literalToken(String.valueOf((char)c))); } public final ParseNode nextToken () throws IOException { ParseNode t = null; int tk = st.nextToken(); int nextChar = st.peek(); lineno = st.lineno(); switch (tk) { case StreamTokenizer.TT_SPECIAL: return nextToken(); case StreamTokenizer.TT_EOF: t = locate(Parser.eoi()); break; case StreamTokenizer.TT_NUMBER: if (st.isInteger) t = integer(); else t = floatingPoint(); break; case '"': t = string(); break; case '\'': // NB: this does NOT check what's between single quotes t = character(); break; case StreamTokenizer.TT_WORD: if (Lexicon.isVariable(st.sval)) { t = variable(st.sval); break; } if (Lexicon.isReserved(st.sval)) { t = literal(st.sval); break; } if (Lexicon.isFunction(st.sval)) { t = function(st.sval); break; } if (Lexicon.isRelation(st.sval)) { t = relation(st.sval); break; } t = object(st.sval); break; case '=': case '!': case '<': case '>': case '&': case '|': case '+': case '-': case '*': case '/': case '%': if (nextChar == '=' || (tk == nextChar && (tk == '+' || tk == '-' || tk == '&' || tk == '|'))) { t = literal(String.valueOf((char)tk)+((char)nextChar)); st.skipChar(true); break; } default: t = literal(tk); } return t; } }
This file was generated on Wed May 03 20:59:55 PDT 2006 from file Tokenizer.java
by the ilog.language.tools.Hilite Java tool written by Hassan Aït-Kaci