HootTokenizer.html

HootTokenizer.java

//  FILE. . . . . d:/hak/hlt/src/hlt/osf/hoot/sources/HootTokenizer.java
//  EDIT BY . . . Hassan Ait-Kaci
//  ON MACHINE. . Hak-Laptop
//  STARTED ON. . Mon May  6 07:12:31 2019

Copyright: © by the author

Author: Hassan Aït-Kaci

Version: Last modified on Sat Sep 14 14:24:37 2019 by hak



package hlt.osf.hoot;

import java.io.IOException;
import java.io.Reader;
import java.io.FileReader;
import java.io.StringReader;
import java.io.InputStreamReader;
import java.io.IOException;
import java.io.FileNotFoundException;

import hlt.language.io.CircularInclusionException;
import hlt.language.io.IncludeReader;
import hlt.language.io.StreamTokenizer;
import hlt.language.io.IO;

import hlt.language.tools.Misc;
import hlt.language.tools.Debug;

import hlt.language.syntax.*;

This is the tokenizer class for the HOOT language grammar. It defines the lexical units used in the grammar rules HootRules.grm and declared in HootDeclarations.grm.


public class HootTokenizer extends AbstractTokenizer
{
  int lineno;
  IncludeReader reader;
  StreamTokenizer st;
  boolean isInteractive;

  public final void setInteractive ()
    {
      setReader(new InputStreamReader(System.in));
      isInteractive = true;
      prompt();
    }

  public final int lineNumber()
    {
      return lineno;
    }

  public final void setReader (Reader rd)
    {
      reader = rd instanceof IncludeReader ? (IncludeReader)rd : new IncludeReader(rd);
      st = new StreamTokenizer(reader);
      st.ordinaryChars("@+.(){}[]<>;,?:#%^&|!=-~`*/\\");
      st.setType('.',st.NUMERIC);
      // st.setType('.',st.ORDINARY);
      st.parseDigitsOnly();
      st.quoteChar('\'');
      st.quoteChar('"');
      st.wordChars("-_");

      Debug.setDebugFlag(true);
    }

  public final Reader getReader ()
    {
      return reader;
    }

  final void setFile (String file)
    {
      reader.setFile(file);
    }

  public final void include (String file) throws FileNotFoundException, CircularInclusionException
    {
      reader.include(file);
    }

  public final void setIsInteractive (boolean flag)
    {
      isInteractive = flag;
    }

  private final boolean promptIsNeeded ()
    {
      boolean isWaiting = isInteractive;

      try
        {
	  isWaiting &= (!reader.ready() && reader.depth() == 0)
	            || (reader.ready() && reader.depth() == 1 && reader.reachedEndOfInclusion());

	  if (Debug.flagIsOn())
	    {
	      System.out.println("Reading from "+(reader.getFile() == null ? "console" : reader.getFile()));
	      System.out.println("Reader ready? = "+reader.ready());
	      System.out.println("Reader depth = "+reader.depth());
	      System.out.println("Latest char = "+Misc.pform(reader.latestChar()));
	    }
        }
      catch (IOException e)
        {
          e.printStackTrace();
        }

      return isWaiting || reader.reachedEndOfInclusion();
    }

  public final void prompt ()
    {
      if (promptIsNeeded())
	{
	  // if (HootParser.isTiming)
	  //   {
	  //     HootParser.time = System.currentTimeMillis() - HootParser.time;
	  //     HootParser.displayLine("*** Processing time = "+HootParser.time+" ms");
	  //   }
	  System.out.print(promptString());
	}
    }

An interactive HOOT-session prompt line consists of an input-statement number (counted since the start of the session), followed by the HOOT prompt label and "> ".


  public final String promptString ()
    {
      return Misc.numberString(lineno+1,3,'*') + " " + promptLabel + "> ";
    }

The HOOT prompt label consists of "HOOT" followed by its version number.


  String promptLabel = "HOOT v"+HootMain.version;  

  public final void setPrompt (String p)
    {
      promptLabel = p;
    }

  final ParseNode locate (ParseNode node)
    {
      return ((ParseNode)node
	      .setStart(st.tokenStart())
	      .setEnd(st.tokenEnd())).setFile(reader.getFile());
    }

  /* ************************************************************************ */

  final ParseNode integer ()
    {
      return locate(GenericParser.numberToken("INTEGER",(int)st.nval));
      // return locate(GenericParser.symbolToken("IDENTIFIER",Integer.toString((int)st.nval)));
    }

  final ParseNode character ()
    {
      return locate(GenericParser.symbolToken("CHAR",st.sval));
    }

  final ParseNode floatingPoint ()
    {
      return locate(GenericParser.numberToken("FLOAT",st.nval));
    }

  final ParseNode string ()
    {
      return locate(GenericParser.symbolToken("STRING",st.sval));
    }

  final ParseNode identifier ()
    {
      return locate(GenericParser.symbolToken("IDENTIFIER",st.sval));
    }

  final ParseNode truthvalue ()
    {
      return locate(GenericParser.symbolToken("BOOLEAN",st.sval));
    }

  final ParseNode pragma ()
    {
      String pragma = st.sval.intern(); 
      switch (pragma)
	{
	case "exit":
	  return locate(GenericParser.symbolToken("EXIT_PRAGMA",pragma));
	case "include":
	  return locate(GenericParser.symbolToken("INCLUDE_PRAGMA",pragma));
	case "encode":
	  return locate(GenericParser.symbolToken("ENCODE_PRAGMA",pragma));
	case "eval":
	  return locate(GenericParser.symbolToken("EVAL_PRAGMA",pragma));
	case "sorts":
	  return locate(GenericParser.symbolToken("SORTS_PRAGMA",pragma));
	case "size":
	  return locate(GenericParser.symbolToken("SIZE_PRAGMA",pragma));
	case "height":
	  return locate(GenericParser.symbolToken("HEIGHT_PRAGMA",pragma));
	case "width":
	  return locate(GenericParser.symbolToken("WIDTH_PRAGMA",pragma));	
	case "children":
	  return locate(GenericParser.symbolToken("CHILDREN_PRAGMA",pragma));
	case "descendants":
	  return locate(GenericParser.symbolToken("DESCENDANTS_PRAGMA",pragma));
	case "parents":
	  return locate(GenericParser.symbolToken("PARENTS_PRAGMA",pragma));
	case "ancestors":
	  return locate(GenericParser.symbolToken("ANCESTORS_PRAGMA",pragma));
	case "enumsize":
	  return locate(GenericParser.symbolToken("ENUMSIZE_PRAGMA",pragma));
	case "isa":
	  return locate(GenericParser.symbolToken("ISA_PRAGMA",pragma));
	case "refresh":
	  return locate(GenericParser.symbolToken("REFRESH_PRAGMA",pragma));
	case "tree":
	  return locate(GenericParser.symbolToken("TREE_PRAGMA",pragma));
	case "timing":
	  return locate(GenericParser.symbolToken("TIMING_PRAGMA",pragma));
	case "gc":
	  return locate(GenericParser.symbolToken("GC_PRAGMA",pragma));
	case "syntax":
	  return locate(GenericParser.symbolToken("SYNTAX_PRAGMA",pragma));
	case "trace":
	  return locate(GenericParser.symbolToken("TRACE_PRAGMA",pragma));
	case "defined":
	  return locate(GenericParser.symbolToken("DEFINED_PRAGMA",pragma));
	case "symbols":
	  return locate(GenericParser.symbolToken("SYMBOLS_PRAGMA",pragma));
	case "help":
	  return locate(GenericParser.symbolToken("HELP_PRAGMA",pragma));
	default:
	  break;	  
	}

      return locate(GenericParser.symbolToken("UNKOWN_PRAGMA",pragma));	
    }

  final ParseNode nameTag ()
    {
      return locate(GenericParser.symbolToken("TAG",("#"+st.sval).intern()));
    }

  final ParseNode numberTag ()
    {
      return locate(GenericParser.symbolToken("TAG",("#"+(int)st.nval).intern()));
    }

  final ParseNode top ()
    {
      return locate(GenericParser.symbolToken("TOP","@"));
    }

  final ParseNode bottom ()
    {
      return locate(GenericParser.symbolToken("BOTTOM","{}"));
    }

  final ParseNode isa ()
    {
      return locate(GenericParser.symbolToken("ISA","<|"));
    }

  final ParseNode arrow ()
    {
      return locate(GenericParser.symbolToken("ARROW","=>"));
    }

  final ParseNode and ()
    {
      return locate(GenericParser.symbolToken("AND","&"));
    }

  final ParseNode or ()
    {
      return locate(GenericParser.symbolToken("OR","|"));
    }

  final ParseNode not ()
    {
      return locate(GenericParser.symbolToken("NOT","!"));
    }

  final ParseNode butnot ()
    {
      return locate(GenericParser.symbolToken("BUTNOT","\\"));
    }

  final ParseNode eos ()
    {
      return locate(GenericParser.symbolToken("EOS","."));
    }

  final ParseNode literal (String s)
    {
      return locate(GenericParser.literalToken(s));
    }

  final ParseNode literal (int c)
    {
      return locate(GenericParser.literalToken(String.valueOf((char)c)));
    }

  /* ************************************************************************ */

  public final ParseNode nextToken () throws IOException
    {
      ParseNode t = null;
      int tk = st.nextToken();
      int nt = 0;
      lineno = st.lineno();

      if (Debug.flagIsOn())
      	{
      	  System.out.println("Stream tokenizer's current state: "+st);
      	}

      switch (tk)
        {
        case StreamTokenizer.TT_SPECIAL:
          return nextToken();
        case IO.EOI:
	  System.out.println("*** END OF INPUT");
	  return nextToken();
        case StreamTokenizer.TT_EOF:
	  if (Debug.flagIsOn())
	    System.out.println("READ END OF FILE");
          t = locate(GenericParser.eoi());
          break;
        case StreamTokenizer.TT_WORD:
	  if (st.sval == "true" || st.sval == "false")
	    t = truthvalue();
	  else
	    t = identifier();
	  break;
        case '%':
	  if ((nt = st.nextToken()) == StreamTokenizer.TT_WORD)
	    t = pragma();
	  else
	    {
	      st.pushBack();
	      t = literal("%");
	    }
	  break;
        case '#':
	  if ((nt = st.nextToken()) == StreamTokenizer.TT_WORD)
	    t = nameTag();
	  else
	    if (nt == StreamTokenizer.TT_NUMBER && st.isInteger)
	      t = numberTag();
	      else
		{
		  st.pushBack();
		  t = literal("#");
		}
	  break;
        case '"':
          t = string();
          break;
        case '\'':    // NB: this does NOT check the contents between single quotes
          t = character();
          break;
	case '@':
	  t = top();
	  break;
	case '{':
	  while (st.isWhitespaceChar(st.peek())) // skip possible white space
	    st.skipChar();
          if (st.peek() == '}')
            {
	      t = bottom();
              st.skipChar(true);
	    }
	  else
	    t = literal("{");
          break;	  
        case '<':
          if (st.peek() == '|')
            {
	      t = isa();
              st.skipChar(true);
	    }
	  else
	    t = literal("<");
          break;
        case '=':
          if (st.peek() == '>')
            {
	      t = arrow();
              st.skipChar(true);
	    }
	  else
	    t = literal("=");
          break;
        case '&':
	  t = and();
          break;
        case '|':
	  t = or();
          break;
        case '!':
	  t = not();
          break;	
        case '\\':
	  t = butnot();
          break;	
        case StreamTokenizer.TT_NUMBER:
          if (st.isInteger)
            t = integer();
          else
            t = floatingPoint();
          break;
        case '.':
	  t = eos();
          break;	
	default:
	  t = literal(tk);
        }
      //      System.out.println("Read Token: "+t.nodeInfo());
      return t;
    }
}

This file was generated on Wed Oct 09 17:17:20 PDT 2019 from file HootTokenizer.java
by the hlt.language.tools.Hilite Java tool written by Hassan Aït-Kaci