using System; using System.IO; using System.Text; using Ilog.Language.IO; using Ilog.Language.Util; using Ilog.Language.Parsing; public class TermTokenizer : Tokenizer { TextReader reader; StreamTokenizer input; string file = "stdin"; public TermTokenizer () { SetReader(Console.In); interactive = true; Banner(); Prompt(); } public TermTokenizer (string stream, bool isFileName) { if (isFileName) { SetReader(new StreamReader(file = stream)); interactive = false; Banner(); Console.WriteLine("*** Parsing file: "+stream); } else { SetReader(new StringReader(stream)); interactive = false; } } public int LineNumber { get { return input.GetLineNumber(); } } public TextReader GetReader () { return reader; } public void SetReader (TextReader rd) { input = new StreamTokenizer(reader = rd); input.OrdinaryChars("+-./"); input.QuoteChar('\'',0); // no escape allowed } private bool interactive; public bool IsInteractive { get { return interactive; } } private static string promptString = ">"; public static void Prompt () { Console.Write(promptString+" "); } static public void SetPrompt (string p) { promptString = p; } private static string banner = "*** Welcome to the dynamic operator term parser! [C# version]\n" + "*** Generated on " + DateTime.Now.ToLongDateString() + ", at " + DateTime.Now.ToLongTimeString(); public static void Banner () { Console.WriteLine(banner); } private ParseNode Locate (ParseNode node) { return ((ParseNode)node.SetStart(input.TokenStart()).SetEnd(input.TokenEnd())) .SetFile(file); } private ParseNode Locate (ParseNode node, Location start) { return ((ParseNode)node.SetStart(start).SetEnd(input.TokenEnd())) .SetFile(file); } private bool IsOtherChar (int c) { return StreamTokenizer.IsOrdinaryChar(c) && !(StreamTokenizer.IsWhiteSpaceChar(c) || c == '(' || c == ')' || c == ','); } public ParseNode NextToken () { ParseNode t = null; switch (input.NextToken()) { case StreamTokenizer.TT_EOF: reader.Close(); Locate(t = GenericParser.E_O_I); break; case '\'': case '"': case StreamTokenizer.TT_WORD: if (input.SValue == "exit") t = DynamicParser.LiteralToken("exit"); else t = GenericParser.SymbolToken("FUNCTOR",input.SValue); Locate(t); break; case StreamTokenizer.TT_NUMBER: if (input.IsInteger) t = GenericParser.NumberToken("NUMBER",(int)input.NValue); else t = GenericParser.NumberToken("NUMBER",input.NValue); Locate(t); break; case '(': case ')': case ',': case '.': Locate(t = DynamicParser.LiteralToken(((char)input.TType).ToString())); break; default: // read the longest possible token and return it as a symbol Location start = input.TokenStart(); StringBuilder functor = new StringBuilder(((char)input.TType).ToString()); input.SpaceIsSignificant(true); while (IsOtherChar(input.Peek())) functor.Append((char)input.NextToken()); input.SpaceIsSignificant(false); t = GenericParser.SymbolToken("FUNCTOR",functor.ToString()); Locate(t,start); break; } return t; } }