# -*- coding: UTF-8 -*-
#    This file is part of arena.
#   
#    arena is free software: you can redistribute it and/or modify it
#    under the terms of the GNU General Public License as published by the
#    Free Software Foundation; either version 3 of the License or, at your
#    option, any later version.
#
#    arena is distributed in the hope that it will be useful, but
#    WITHOUT ANY WARRANTY; without even the implied warranty of MERCHANTABLITY
#    or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public
#    License for more details.
#
#    You should have received a copy of the GNU General Public License along
#    with arena.  If not, see <http://www.gnu.org/licenses/>. 

## @file 
#  This file defines several classes that are used to retrieve tokens from a
#  stream.

from Library.Base.Classwork import AnObject
from Library.IO.Streams import IsStreamable
from Library.String.Utilities import IsPrintable, String
from Library.Parse.Lex.Languages import CharacterType, TokenType

## A generic token.
#
#  This class serves as the base for all other token types that can be parsed
#  from a stream.  It provides methods for reading the token from a given
#  input stream and for writing it to another stream.  It also provides a class
#  method that can be called to construct and return the appropriate token type
#  depending on the current character retrieved from a given input stream.
class AToken(AnObject, IsPrintable, IsStreamable):
    ## Constructor
    #
    #  `ThisLanguage` is a reference to an instance of AComputerLanguage,
    #  which will be used to match specific characters and identifiers to
    #  keywords and operators.
    def __init__(Self, ThisLanguage):
        AnObject.__init__(Self)
        
        Self._Language = ThisLanguage

    ## Initializer    
    def init(Self):
        result = AnObject.init(Self)
        
        if result:
            # Establish private attributes
            # The token type or opcode
            Self._code = 0
            # The token string literal
            Self._literal = ""
            # The language used to parse the token
            Self._Language = None
            
            # Establish public attributes
            # Whether or not the token is to be appended to parser output
            Self.silent = False
            
        return result
        
    ## Construct a new token based on the next character read from the
    #  specified stream.
    #
    #  `ThisSource` should be a reference to the input stream from which
    #  tokens will be parsed.
    #
    #  `ThisLanguage` should be a reference to an instance of AComputerLanguage,
    #  which will be used to determine the type of the character read from the
    #  input stream.
    @staticmethod
    def fromSource(ThisSource, ThisLanguage):
        Result = None
        
        if (ThisSource is not None) and (ThisLanguage is not None):
            if ThisSource.hasEnded() == False:
                # Read the next character from the source and then put it back
                inchar = ThisSource.read(ch=" ")
                ch = inchar["ch"]
                ThisSource.rewindBy(len(ch))
                
                # Check the character type and construct the matching token type
                charType = ThisLanguage.typeOf(ch)
                
                if charType == CharacterType.letter:
                    Result = AWordToken(ThisLanguage)
                    
                elif charType == CharacterType.symbol:
                    Result = ASpecialToken(ThisLanguage)
                    
                elif charType == CharacterType.digit:
                    Result = ANumberToken(ThisLanguage)
                    
                elif charType == CharacterType.quote:
                    Result = AStringToken(ThisLanguage)
                    
                elif charType == CharacterType.space:
                    Result = ASpaceToken(ThisLanguage)
                    
                elif charType == CharacterType.endOfLine:
                    Result = ALineEndingToken(ThisLanguage)
                    
                elif charType == CharacterType.endOfStream:
                    Result = AStreamEndingToken(ThisLanguage)
                    
                else:
                    Result = AnErrorToken(ThisLanguage)
                
            else:
                Result = AStreamEndingToken(ThisLanguage)
                          
        if Result is not None:
            # Read the token
            Result.readFrom(ThisSource)
            
        return Result   
        
    ## Return the token to the input stream.
    #
    #  This methods allows a form of token "peeking", in which a parser can
    #  retrieve a token from the stream to see if matches a predefined list of
    #  tokens, and then return it to the stream so that it can be read again.
    #
    #  The base implementation of this method simply calls AStream.rewindBy() on
    #  `ThisSource` to rewind the stream by the length of the token literal
    #  string.
    def rewind(Self, ThisSource):
        if len(Self._literal) > 0:
            ThisSource.rewindBy(len(Self._literal))
        
    # IsPrintable implementation -----------------------------------------------
    ## Return a string representation of the token.  This representation
    #  will be a quoted form of the token text, unless the token text is 
    #  already quoted.
    def toString(Self):
        if Self._literal[0] not in \
          Self._Language.characters[CharacterType.quote]:
            return String.quote(Self._literal)
            
        else:
            return Self._literal
            
    # IsStreamable implementation ----------------------------------------------
    ## Read the token from the specified stream.
    #    
    #  This routine should be implemented by descendant classes.
    def readFrom(Self, Source):
        return 0
        
    ## Write the token to the specified stream.
    #    
    #  In the base implementation, this routine simply writes the token code
    #  to the stream.  Descendant classes may override this routine in order
    #  to output additional information. 
    def writeTo(Self, Dest):
        result = 0
        
        if (Dest is not None) and (Self.silent == False):
            result = Dest.write(Self._code)
        
        return result
        
    # Properties ---------------------------------------------------------------
    ## Retrieve the token code
    def code(Self):
        return Self._code
        
    ## Retrieve the token literal
    def literal(Self):
        return Self._literal
        
    ## Retrieve the language used to parse the token.
    def Language(Self):
        return Self._Language
        
## A token that represents a single word or identifier.
#
#  This token collects characters from its input stream until it finds one that
#  is not allowed by the language to be part of a keyword or identifier.  It
#  then compares the accumulated string against a list of registered keywords
#  to determine whether a keyword or identifier has been encountered.
class AWordToken(AToken):
    ## Constructor
    def __init__(Self, ThisLanguage):
        AToken.__init__(Self, ThisLanguage)
            
    ## Initializer
    def init(Self):
        result = AToken.init(Self)
        
        if result:
            # Establish private attributes
            # The scope of the token (if it is an identifier)
            Self._scope = 0
            # The index of the token within its symbol table
            Self._index = 0
            
        return result
        
    # IsStreamable implementation ----------------------------------------------
    ## Write the token to the specified stream.
    #
    #  The information written includes the token code, the token scope,
    #  and the token index within its symbol table.
    def writeTo(Self, Dest):
        result = 0
        
        if (Dest is not None) and (Self.silent == False):
            result = AToken.writeTo(Self)
            
            # Certain tokens are identifiers
            if Self._code in (TokenType.identifier, TokenType.number,
              TokenType.string):
                result += Dest.write(Self._scope)
                result += Dest.write(Self._index)
                
        return result
        
    ## Read the token from the specified stream.
    #
    #  This routine is used to tokenize a text-based input stream.
    def readFrom(Self, Source):
        result = 0
        
        if Source is not None:
            ch = " "
            text = ""
            
            # Loop to read the token
            inchar = Source.read(ch=" ")
            ch = inchar["ch"]
            result += inchar["result"]
            while True:
                text += ch
                if Source.hasEnded():
                    break
                    
                inchar = Source.read(ch=" ")
                ch = inchar["ch"]
                result += inchar["result"]

                if Self._Language.typeOf(ch) not in (CharacterType.letter,
                  CharacterType.alpha, CharacterType.digit):
                    break
            
            Self._literal = text
            
            # Check to see if the token is a keyword or operator
            Self._code = Self._Language.meaningOf(Self._literal)
            
            if Self._code == TokenType.dummy:
                Self._code = TokenType.identifier
                
            if Source.hasEnded() == False:
                # Put back the non-word character 
                Source.rewindBy(len(ch))
                result -= len(ch)
                
        return result
        
    # Properties ---------------------------------------------------------------
    ## Retrieve the token scope (the symbol table in which it is located)
    def scope(Self):
        return Self._scope
        
    ## Retrieve the token index within its symbol table
    def index(Self):
        return Self._index
        
## A token that represents a numeric value.
#
#  As with AWordToken, this class simply accumulates characters from the input
#  stream until it encounters one that is not allowed by the language to be
#  part of a number.  It then attempts to convert that string into a numeric
#  value; if it fails to do so, it signals that it has an erroneous number by
#  changing its \p code() to TokenType.error.
class ANumberToken(AWordToken):
    ## Constructor
    def __init__(Self, ThisLanguage):
        AWordToken.__init__(Self, ThisLanguage)
            
    ## Initializer
    def init(Self):
        result = AWordToken.init(Self)
        
        if result:
            # Establish private attributes
            Self._code = TokenType.number
            # The numeric value of the token
            Self._value = 0
            
        return result
                  
    # IsStreamable implementation ----------------------------------------------
    ## Read the token from the specified stream.
    #
    #  This routine is used to tokenize a text-based input stream.
    def readFrom(Self, Source):
        result = 0
        
        if Source is not None:
            ch = " "
            text = ""
            
            inchar = Source.read(ch=" ")
            ch = inchar["ch"]
            result += inchar["result"]
                        
            # Loop to fetch the number
            while True:
                text += ch
                if Source.hasEnded():
                    break
                    
                inchar = Source.read(ch=" ")
                ch = inchar["ch"]
                result += inchar["result"]
                
                if Self._Language.typeOf(ch) not in (CharacterType.digit,
                  CharacterType.numeric):
                    break
                    
            Self._literal = text
            # Attempt to convert the text to a number (float or long)
            try:
                Self._value = float(Self._literal)
                if Self._value.is_integer():
                    Self._value = long(Self._value)
                
            except ValueError:
                # Failed to convert the number, so it is invalid
                Self._code = TokenType.error
                
            # Put back the non-numeric character
            if Source.hasEnded() == False:
                Source.rewindBy(len(ch))
                result -= len(ch)
        
        return result
        
    # Properties ---------------------------------------------------------------
    ## Determine whether or not the number represented is an integer value.
    def isInteger(Self):
        return (isinstance(Self._value, int)) or \
          (isinstance(Self._value, long))

    ## Retrieve the numeric value of the token.
    def value(Self):
        return Self._value

## A token that represents a string literal.
#
#  This class accumulates characters until the end of the string is found.  The
#  end of the string is demarcated by the same character which began the string.
#  After accumulating the string, the class sets its value() to the value of
#  the string, minus quote characters.
class AStringToken(AWordToken):
    ## Constructor
    def __init__(Self, ThisLanguage):
        AWordToken.__init__(Self, ThisLanguage)
            
    ## Initializer
    def init(Self):
        result = AWordToken.init(Self)
        
        if result:
            # Establish private attributes
            # The value of the string literal (without quotes)
            Self._value = ""
            Self._code = TokenType.string
            
        return result
        
    # IsStreamable implementation ----------------------------------------------
    ## Read the token from the specified stream.
    #
    #  This routine is used to tokenize text from the specified stream.
    def readFrom(Self, Source):
        result = 0
        
        if Source is not None:
            ch = " "
            text = ""
            
            # Loop to read the token
            inchar = Source.read(ch=" ")
            ch = inchar["ch"]
            result += inchar["result"]
            while True:
                text += ch
                if Source.hasEnded():
                    break
                    
                inchar = Source.read(ch=" ")
                ch = inchar["ch"]
                result += inchar["result"]
                
                if (ch == text[0]) or \
                  (Self._Language.typeOf(ch) == CharacterType.endOfLine):
                    break

            Self._literal = text
            
            # If the first and last characters of the string match...
            if ch == Self._literal[0]:
                Self._literal += ch
                # Store the string literal without quotes
                Self._value = Self._literal[1:-1] 
                
            # Otherwise, the string is invalid
            else:
                Self._code = TokenType.error
                
                if Source.hasEnded() == False:
                    # Put back the non-string character 
                    Source.rewindBy(len(ch))
                    result -= len(ch)
                    
        return result   
        
    # Properties ---------------------------------------------------------------
    ## Retrieve the value of the string (the string literal, minus its quote
    #  characters).
    def value(Self):
        return Self._value

## A token that represents a special character which is not necessarily a 
#  keyword or operator (such a list delimiter, brackets, etc.).
#
#  As with AWordToken, this class accumulates characters until it encounters one
#  that is not allowed by the language to be a symbol.  It then checks the
#  accumulated string against a list of registered keywords and operators.  If
#  no match is found, the class returns the rightmost character from its string
#  to the stream and checks again.  The process continues until a match is found 
#  or the token literal is only a single character in length.
class ASpecialToken(AToken):
    ## Constructor
    def __init__(Self, ThisLanguage):
        AToken.__init__(Self, ThisLanguage)
            
    # IsStreamable implementation ----------------------------------------------
    ## Read the token from the specified stream.
    #
    #  This routine is used to tokenize text in the specified stream.
    def readFrom(Self, Source):
        result = 0
        
        if Source is not None:
            ch = " "
            text = ""
            
            inchar = Source.read(ch=" ")
            ch = inchar["ch"]
            result += inchar["result"]
            
            # Loop to retrieve the token
            while True:
                text += ch
                if Source.hasEnded():
                    break
                    
                inchar = Source.read(ch=" ")
                ch = inchar["ch"]
                result += inchar["result"]
                
                if Self._Language.typeOf(ch) != CharacterType.symbol:
                    break
                    
            # Return the non-special character to the stream
            Source.rewindBy(len(ch))
            result -= len(ch)
            
            # Check to see if the token is a keyword or operator
            Self._code = Self._Language.meaningOf(text)
            while (Self._code == TokenType.dummy) and (len(text) > 1):
                # Return the character to the source
                Source.rewindBy(len(ch))
                result -= len(ch)
                # Shave a character from the token string
                text = text[0:-1]
                # Check again
                Self._code = Self._Language.meaningOf(text)
                
            # If no matching keyword, operator, or delimiter is found, then...
            if Self._code == TokenType.dummy:
                Self._code = TokenType.error
                
            Self._literal = text
        
        return result
    
## A token that represents whitespace (not a token from space).
#
#  This class accumulates whitespace characters until it encounters one that is
#  not allowed by the language to be a space.
class ASpaceToken(AToken):
    ## Constructor
    def __init__(Self, ThisLanguage):
        AToken.__init__(Self, ThisLanguage)
            
    ## Initializer
    def init(Self):
        result = AToken.init(Self)
        
        if result:
            # Establish private attributes
            # The token code
            Self._code = TokenType.space
            
        return result
        
    # IsStreamable implementation ----------------------------------------------
    ## Read the token from the specified stream.
    #
    #  This routine is used to tokenize text in the specified stream.
    def readFrom(Self, Source):
        result = 0
        
        if Source is not None:
            ch = " "
            text = ""
            
            inchar = Source.read(ch=" ")
            ch = inchar["ch"]
            result += inchar["result"]
            
            # Collect the whitespace
            while True:
                text += ch
                if Source.hasEnded():
                    break
                    
                inchar = Source.read(ch=" ")
                ch = inchar["ch"]
                result += inchar["result"]
            
                if Self._Language.typeOf(ch) != CharacterType.space:
                    break
                    
            Self._literal = text
            
            # Put back the non-whitespace character
            if Source.hasEnded() == False:
                Source.rewindBy(len(ch))
                result -= len(ch)
        
        return result

## A token which represents an end of line character (or sequence of
#  characters, for Windows-derived streams).
#
#  This token collects either one or two newline characters from the input
#  stream, depending upon whether the stream is a Unix-style stream or a 
#  Windows-style stream.
class ALineEndingToken(AToken):
    ## Constructor
    def __init__(Self, ThisLanguage):
        AToken.__init__(Self, ThisLanguage)
            
    ## Initializer
    def init(Self):
        result = AToken.init(Self)
        
        if result:
            # Establish private attributes
            # The token code
            Self._code = TokenType.endOfLine
            
        return result
        
    # IsStreamable implementation ----------------------------------------------
    ## Read the token from the specified stream.
    #
    #  This routine is used to tokenize text in the specified stream.
    def readFrom(Self, Source):
        result = 0
        
        if Source is not None:
            ch = " "
            text = ""
            
            inchar = Source.read(ch=" ")
            ch = inchar["ch"]
            result += inchar["result"]
            text += ch    
            
            # For Windows encodings, we check for one additional character
            if ch == "\x0D":              
                if Source.hasEnded() == False:                
                    inchar = Source.read(ch=" ")
                    ch = inchar["ch"]
                    result += inchar["result"]
                
                    if Self._Language.typeOf(ch) == CharacterType.endOfLine:
                        text += ch
            
            Self._literal = text
            # Put back the non-EOL character
            if Self._Language.typeOf(ch) != CharacterType.endOfLine:
                if Source.hasEnded() == False:
                    Source.rewindBy(len(ch))
                    result -= len(ch)
        
        return result

## A token that represents the end of the stream. 
# 
#  This token does not actually read from the stream; it is created 
#  automatically by AToken.readFrom() when the end of the stream is 
#  encountered.
class AStreamEndingToken(AToken):
    ## Constructor
    def __init__(Self, ThisLanguage):
        AToken.__init__(Self, ThisLanguage)
            
    ## Initializer
    def init(Self):
        result = AToken.init(Self)
        
        if result:
            # Establish private attributes
            # The token type
            Self._code = TokenType.endOfStream
            ## The token text
            Self._literal = "end of stream"
            
        return result
        
    # IsStreamable implementation ----------------------------------------------
    ## Read the token from the specified stream.
    #
    #  This routine does nothing.
    def readFrom(Self, Source):
        # Do nothing
        return 0
  
## An erroneous token read from the source.
#
#  This token accumulates error characters from the stream and returns them
#  within its literal value.
class AnErrorToken(AToken):
    ## Constructor
    def __init__(Self, ThisLanguage):
        AToken.__init__(Self, ThisLanguage)
            
    ## Initializer
    def init(Self):
        result = AToken.init(Self)
        
        if result:
            # Establish private attributes
            # The token code
            Self._code = TokenType.error
            
        return result
        
    # IsStreamable implementation ----------------------------------------------
    ## Read the token from the specified stream.
    #
    #  This routine is used to tokenize text from the specified stream.
    def readFrom(Self, Source):
        result = 0
        
        if Source is not None:
            ch = " "
            text = ""
            
            inchar = Source.read(ch=" ")
            ch = inchar["ch"]
            result += inchar["result"]
            
            # Collect the token
            while True:
                text += ch
                
                if Source.hasEnded():
                    break
                    
                inchar = Source.read(ch=" ")
                ch = inchar["ch"]
                result += inchar["result"]
                
                if Self._Language.typeOf(ch) != CharacterType.dummy:
                    break
                    
            Self._literal = text
            
            # Put back the non-erroneous character
            if Source.hasEnded() == False:
                Source.rewindBy(len(ch))
                result -= len(ch)
        
        return result
