// FILE. . . . . /home/hak/hlt/src/hlt/osfv2/apps/v2/sources/OsfV2Tokenizer.java
// EDIT BY . . . Hassan Ait-Kaci
// ON MACHINE. . Hak-Laptop
// STARTED ON. . Tue Jan 21 08:19:21 2014

package hlt.osf.v2;

import java.io.IOException;
import java.io.Reader;
import java.io.FileReader;
import java.io.StringReader;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.FileNotFoundException;

import hlt.language.io.CircularInclusionException;
import hlt.language.io.IncludeReader;
import hlt.language.io.StreamTokenizer;
import hlt.language.io.IO;

import hlt.language.tools.Misc;
// import hlt.language.tools.Debug;

import hlt.language.syntax.*;

public class OsfV2Tokenizer extends AbstractTokenizer
{
  int lineno;
  IncludeReader reader;
  StreamTokenizer st;
  boolean isInteractive;

  public final void setInteractive ()
    {
      setReader(new InputStreamReader(System.in));
      isInteractive = true;
      prompt();
    }

  public final int lineNumber()
    {
      return lineno;
    }

  public final void setReader (Reader rd)
    {
      reader = rd instanceof IncludeReader ? (IncludeReader)rd : new IncludeReader(rd);
      st = new StreamTokenizer(reader);
      st.ordinaryChars("@+.(){}[]<>;,?:#%^&|!=-~`*/\\");
      st.setType('.',st.NUMERIC);
      // st.setType('.',st.ORDINARY);
      st.parseDigitsOnly();
      st.quoteChar('\'');
      st.quoteChar('"');
      st.wordChars("-_");

      // Debug.setDebugFlag(true);
    }

  public final Reader getReader ()
    {
      return reader;
    }

  final void setFile (String file)
    {
      reader.setFile(file);
    }

  public final void include (String file) throws FileNotFoundException, CircularInclusionException
    {
      reader.include(file);
    }

  public final void setIsInteractive (boolean flag)
    {
      isInteractive = flag;
    }

  private final boolean promptIsNeeded ()
    {
      boolean isWaiting = isInteractive;

      try
        {
	  isWaiting &= (!reader.ready() && reader.depth() == 0);
	            // || (reader.ready() && reader.depth() == 1 && reader.reachedEndOfInclusion());

	  // if (Debug.flagIsOn())
	  //   {
	  //     System.out.println("Reading from "+(reader.getFile() == null ? "console" : reader.getFile()));
	  //     System.out.println("Reader ready? = "+reader.ready());
	  //     System.out.println("Reader depth = "+reader.depth());
	  //     System.out.println("Latest char = "+Misc.pform(reader.latestChar()));
	  //   }
        }
      catch (IOException e)
        {
          e.printStackTrace();
        }

      return isWaiting || reader.reachedEndOfInclusion();
    }

  public final void prompt ()
    {
      if (promptIsNeeded())
	{
	  // if (OsfV2Parser.isTiming)
	  //   {
	  //     OsfV2Parser.time = System.currentTimeMillis() - OsfV2Parser.time;
	  //     OsfV2Parser.displayLine("*** Processing time = "+OsfV2Parser.time+" ms");
	  //   }
	  System.out.print(promptString());
	}
    }
  
  public final String promptString ()
    {
      return Misc.numberString(lineno+1,3,'*') + " " + promptLabel + "> ";
    }

  String promptLabel = "OSF V2.0";  

  public final void setPrompt (String p)
    {
      promptLabel = p;
    }

  final ParseNode locate (ParseNode node)
    {
      return ((ParseNode)node
	      .setStart(st.tokenStart())
	      .setEnd(st.tokenEnd())).setFile(reader.getFile());
    }

  /* ************************************************************************ */

  final ParseNode integer ()
    {
      return locate(GenericParser.numberToken("INTEGER",(int)st.nval));
      // return locate(GenericParser.symbolToken("IDENTIFIER",Integer.toString((int)st.nval)));
    }

  final ParseNode character ()
    {
      return locate(GenericParser.symbolToken("CHAR",st.sval));
    }

  final ParseNode floatingPoint ()
    {
      return locate(GenericParser.numberToken("FLOAT",st.nval));
    }

  final ParseNode string ()
    {
      return locate(GenericParser.symbolToken("STRING",st.sval));
    }

  final ParseNode identifier ()
    {
      return locate(GenericParser.symbolToken("IDENTIFIER",st.sval));
    }

  final ParseNode truthvalue ()
    {
      return locate(GenericParser.symbolToken("BOOLEAN",st.sval));
    }

  final ParseNode pragma ()
    {
      return locate(GenericParser.symbolToken("PRAGMA",st.sval.intern()));
    }

  final ParseNode nameTag ()
    {
      return locate(GenericParser.symbolToken("TAG",("#"+st.sval).intern()));
    }

  final ParseNode numberTag ()
    {
      return locate(GenericParser.symbolToken("TAG",("#"+(int)st.nval).intern()));
    }

  final ParseNode top ()
    {
      return locate(GenericParser.symbolToken("TOP","@"));
    }

  final ParseNode bottom ()
    {
      return locate(GenericParser.symbolToken("BOTTOM","{}"));
    }

  final ParseNode isa ()
    {
      return locate(GenericParser.symbolToken("ISA","<|"));
    }

  final ParseNode arrow ()
    {
      return locate(GenericParser.symbolToken("ARROW","=>"));
    }

  final ParseNode and ()
    {
      return locate(GenericParser.symbolToken("AND","&"));
    }

  final ParseNode or ()
    {
      return locate(GenericParser.symbolToken("OR","|"));
    }

  final ParseNode not ()
    {
      return locate(GenericParser.symbolToken("NOT","!"));
    }

  final ParseNode butnot ()
    {
      return locate(GenericParser.symbolToken("BUTNOT","\\"));
    }

  final ParseNode eos ()
    {
      return locate(GenericParser.symbolToken("EOS","."));
    }

  final ParseNode literal (String s)
    {
      return locate(GenericParser.literalToken(s));
    }

  final ParseNode literal (int c)
    {
      return locate(GenericParser.literalToken(String.valueOf((char)c)));
    }

  /* ************************************************************************ */

  public final ParseNode nextToken () throws IOException
    {
      ParseNode t = null;
      int tk = st.nextToken();
      int nt = 0;
      lineno = st.lineno();

      // if (Debug.flagIsOn())
      // 	{
      // 	  System.out.println("Stream tokenizer's current state: "+st);
      // 	}

      switch (tk)
        {
        case StreamTokenizer.TT_SPECIAL:
          return nextToken();
        case IO.EOI:
	  System.out.println("*** END OF INPUT");
	  return nextToken();
        case StreamTokenizer.TT_EOF:
	  // if (Debug.flagIsOn())
	  //   System.out.println("READ END OF FILE");
          t = locate(GenericParser.eoi());
          break;
        case StreamTokenizer.TT_WORD:
	  if (st.sval == "true" || st.sval == "false")
	    t = truthvalue();
	  else
	    t = identifier();
	  break;
        case '%':
	  if ((nt = st.nextToken()) == StreamTokenizer.TT_WORD)
	    t = pragma();
	  else
	    {
	      st.pushBack();
	      t = literal("%");
	    }
	  break;
        case '#':
	  if ((nt = st.nextToken()) == StreamTokenizer.TT_WORD)
	    t = nameTag();
	  else
	    if (nt == StreamTokenizer.TT_NUMBER && st.isInteger)
	      t = numberTag();
	      else
		{
		  st.pushBack();
		  t = literal("#");
		}
	  break;
        case '"':
          t = string();
          break;
        case '\'':    // NB: this does NOT check the contents between single quotes
          t = character();
          break;
	case '@':
	  t = top();
	  break;
	case '{':
          if (st.peek() == '}')
            {
	      t = bottom();
              st.skipChar(true);
	    }
	  else
	    t = literal("{");
          break;	  
        case '<':
          if (st.peek() == '|')
            {
	      t = isa();
              st.skipChar(true);
	    }
	  else
	    t = literal("<");
          break;
        case '=':
          if (st.peek() == '>')
            {
	      t = arrow();
              st.skipChar(true);
	    }
	  else
	    t = literal("=");
          break;
        case '&':
	  t = and();
          break;
        case '|':
	  t = or();
          break;
        case '!':
	  t = not();
          break;	
        case '\\':
	  t = butnot();
          break;	
        case StreamTokenizer.TT_NUMBER:
          if (st.isInteger)
            t = integer();
          else
            t = floatingPoint();
          break;
        case '.':
	  t = eos();
          break;	
	default:
	  t = literal(tk);
        }
      // System.out.println("Read Token: "+t.nodeInfo());
      return t;
    }
}

This file was generated on Mon Jun 02 09:43:20 CEST 2014 from file OsfV2Tokenizer.java
by the hlt.language.tools.Hilite Java tool written by Hassan Aït-Kaci