Tokenizer.java
|
// FILE. . . . . /home/hak/ilt/src/ilog/rif/Tokenizer.java // EDIT BY . . . Hassan Ait-Kaci // ON MACHINE. . Latitude407.Ilog.Biz // STARTED ON. . Wed May 3 14:37:03 2006 package ilog.rif;
This is a skeleton tokenizer for building a java parser from the Jacc
grammar specified in HRL.grm. It is
mininal in that it relies on just enough information to make a toy
parser. It is rather self-explanatory.
|
import java.io.Reader; import java.io.FileReader; import java.io.BufferedReader; import java.io.IOException; import ilog.language.io.StreamTokenizer; import ilog.language.syntax.*; public class Tokenizer implements FileTokenizer { int lineno; String file; Reader reader; StreamTokenizer st; boolean prompt; public Tokenizer () { } public Tokenizer (String file) throws IOException { setFile(file); } public void setFile (String file) throws IOException { setFileName(file); setReader(new BufferedReader(new FileReader(file))); } public final int lineNumber() { return lineno; } public final void setReader (Reader rd) { reader = rd; st = new StreamTokenizer(reader); st.ordinaryChars("+-.(){}[]<>;,:=!&|*/%"); st.setType('.',st.NUMERIC); st.quoteChar('\''); st.quoteChar('"'); st.wordChars("_@?$"); } public final Reader getReader () { return reader; } public final String fileName () { return file; } public final void setFileName (String file) { this.file = file; } final ParseNode locate (ParseNode node) { return ((ParseNode)node.setStart(st.tokenStart()).setEnd(st.tokenEnd())).setFile(file); } final ParseNode integer () { return locate(Parser.numberToken("Value",(int)st.nval)); } final ParseNode floatingPoint () { return locate(Parser.numberToken("Value",st.nval)); } final ParseNode string () { return locate(Parser.symbolToken("Value","\""+st.sval+"\"")); } final ParseNode character () { return locate(Parser.symbolToken("Value","'"+st.sval+"'")); } final ParseNode name (String symbol) { return locate(Parser.symbolToken("Name",symbol)); } final ParseNode variable (String symbol) { return locate(Parser.symbolToken("Var",symbol.substring(1))); } final ParseNode function (String symbol) { return locate(Parser.symbolToken("Fun",symbol)); } final ParseNode relation (String symbol) { return locate(Parser.symbolToken("Rel",symbol)); } final ParseNode literal (String s) { return locate(Parser.literalToken(s)); } final ParseNode literal (int c) { return locate(Parser.literalToken(String.valueOf((char)c))); } public final ParseNode nextToken () throws IOException { ParseNode t = null; int tk = st.nextToken(); int nextChar = st.peek(); lineno = st.lineno(); switch (tk) { case StreamTokenizer.TT_SPECIAL: return nextToken(); case StreamTokenizer.TT_EOF: t = locate(Parser.eoi()); break; case StreamTokenizer.TT_NUMBER: if (st.isInteger) t = integer(); else t = floatingPoint(); break; case '"': t = string(); break; case '\'': // NB: this does NOT check what's between single quotes t = character(); break; case StreamTokenizer.TT_WORD: if (Lexicon.isVariable(st.sval)) { t = variable(st.sval); break; } if (Lexicon.isReserved(st.sval)) { t = literal(st.sval); break; } if (Lexicon.isFunction(st.sval)) { t = function(st.sval); break; } if (Lexicon.isRelation(st.sval)) { t = relation(st.sval); break; } t = name(st.sval); break; case ':': if (nextChar == '-') { t = literal(String.valueOf((char)tk)+((char)nextChar)); st.skipChar(true); break; } t = literal(tk); break; case '=': case '!': case '<': case '>': case '&': case '|': case '+': case '-': case '*': case '/': case '%': if (nextChar == '=' || (tk == nextChar && (tk == '+' || tk == '-' || tk == '&' || tk == '|'))) { t = literal(String.valueOf((char)tk)+((char)nextChar)); st.skipChar(true); break; } default: t = literal(tk); } // System.out.println("Reading token: "+t); return t; } }
This file was generated on Mon Mar 31 17:18:18 PDT 2008 from file Tokenizer.java
by the ilog.language.tools.Hilite Java tool written by Hassan Aït-Kaci