nx_post_support/server/libs/tclint/lexer.py

import ply.lex as lex
from typing import Tuple

TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE"
TOK_BACKSLASH_SUB = "BACKSLASH_SUB"
TOK_NEWLINE = "NEWLINE"
TOK_SEMI = "SEMI"
TOK_WS = "WS"
TOK_QUOTE = "QUOTE"
TOK_ARG_EXPANSION = "ARG_EXPANSION"
TOK_LBRACE = "LBRACE"
TOK_RBRACE = "RBRACE"
TOK_STAR = "STAR"
TOK_LBRACKET = "LBRACKET"
TOK_RBRACKET = "RBRACKET"
TOK_DOLLAR = "DOLLAR"
TOK_LPAREN = "LPAREN"
TOK_RPAREN = "RPAREN"
TOK_HASH = "HASH"
TOK_ALPHA_CHARS = "ALPHA_CHARS"
TOK_NUM_CHARS = "NUM_CHARS"
TOK_NAMESPACE_SEP = "NAMESPACE_SEP"
TOK_CHAR = "CHAR"
TOK_CONTENTS = "CONTENTS"
TOK_EOF = None

STATE_BRACEDWORD = "bracedword"


class TclSyntaxError(Exception):
    def __init__(self, message, start: Tuple[int, int], end: Tuple[int, int]):
        super().__init__(message)
        self.start = start
        self.end = end


class _LexTable:
    tokens = (
        TOK_BACKSLASH_NEWLINE,
        TOK_BACKSLASH_SUB,
        TOK_NEWLINE,
        TOK_SEMI,
        TOK_WS,
        TOK_QUOTE,
        TOK_ARG_EXPANSION,
        TOK_LBRACE,
        TOK_RBRACE,
        TOK_STAR,
        TOK_LBRACKET,
        TOK_RBRACKET,
        TOK_DOLLAR,
        TOK_LPAREN,
        TOK_RPAREN,
        TOK_HASH,
        TOK_ALPHA_CHARS,
        TOK_NUM_CHARS,
        TOK_NAMESPACE_SEP,
        TOK_CHAR,
        TOK_CONTENTS,
    )

    # This defines a conditional lexing state for parsing braced words. This is a
    # performance optimization; since there are few special characters in this context,
    # we can use a smaller set of tokens to parse them faster. This has a large impact
    # since most Tcl programs have a large number of braced words. Any token with
    # `bracedword` in its name is included in this state. Tokens that are included in
    # this state and the default state also include `INITIAL` in their name.
    states = ((STATE_BRACEDWORD, "exclusive"),)

    def _tok(self, t):
        pos = (t.lexer.lineno, t.lexer.colno)
        t.lexer.lineno += t.value.count("\n")
        index = t.value.rfind("\n")
        if index == -1:
            t.lexer.colno += len(t.value)
        else:
            remaining = t.value[index + 1 :]
            t.lexer.colno = len(remaining) + 1

        t.value = (t.value, pos)
        return t

    # Priority important
    def t_bracedword_INITIAL_BACKSLASH_NEWLINE(self, t):
        r"\\\n"
        return self._tok(t)

    # Priority important
    def t_bracedword_INITIAL_BACKSLASH_SUB(self, t):
        r"\\."
        return self._tok(t)

    def t_NEWLINE(self, t):
        r"\n"
        return self._tok(t)

    def t_SEMI(self, t):
        r";"
        return self._tok(t)

    # TODO: should use \s?
    def t_WS(self, t):
        r"[\t\v\f\r ]+"
        return self._tok(t)

    def t_QUOTE(self, t):
        r'"'
        return self._tok(t)

    # Must be higher priority than LBRACE
    def t_ARG_EXPANSION(self, t):
        r"\{\*\}"
        return self._tok(t)

    def t_bracedword_INITIAL_LBRACE(self, t):
        r"\{"
        return self._tok(t)

    def t_bracedword_INITIAL_RBRACE(self, t):
        r"\}"
        return self._tok(t)

    def t_STAR(self, t):
        r"\*"
        return self._tok(t)

    def t_LBRACKET(self, t):
        r"\["
        return self._tok(t)

    def t_RBRACKET(self, t):
        r"\]"
        return self._tok(t)

    def t_DOLLAR(self, t):
        r"\$"
        return self._tok(t)

    def t_LPAREN(self, t):
        r"\("
        return self._tok(t)

    def t_RPAREN(self, t):
        r"\)"
        return self._tok(t)

    def t_HASH(self, t):
        r"\#"
        return self._tok(t)

    # Valid non-numeric chars in variable names
    def t_ALPHA_CHARS(self, t):
        r"[A-Za-z_]+"
        return self._tok(t)

    # Valid numeric chars in variable names
    # This is split up from the above to facilitate expression parsing, since
    # e.g. 1eq1 can't be a single token.
    def t_NUM_CHARS(self, t):
        r"[0-9]+"
        return self._tok(t)

    def t_NAMESPACE_SEP(self, t):
        r"::+"
        return self._tok(t)

    def t_bracedword_CONTENTS(self, t):
        r"[^{}\\]+"
        return self._tok(t)

    # Catch-all. TODO: inefficient, should probably munch multiple chars
    def t_CHAR(self, t):
        r"."
        return self._tok(t)

    # Error handling rule
    # TODO: do we need this? since we have a catch-all...
    # there is a warning
    def t_bracedword_INITIAL_error(self, t):
        print("Illegal character '%s'" % t.value[0])
        t.lexer.skip(1)

    def __init__(self):
        self.lexer = lex.lex(object=self)
        self.lexer.lineno = 1
        self.lexer.colno = 1

    def new_lexer(self, pos=None):
        lexer = self.lexer.clone()
        lexer.lineno = 1
        lexer.colno = 1

        if pos is not None:
            line, col = pos
            lexer.lineno = line
            lexer.colno = col

        return lexer


# Calling `lex.lex()` performs an expensive reflection process to generate the lexer.
# This singleton class holds a preinitialized lexer that can then be cloned to create
# individual instances.
LexTable = _LexTable()


class Lexer:
    def __init__(self, pos=None):
        self.lexer = LexTable.new_lexer(pos)
        self.current = None

    def input(self, text):
        self.lexer.input(text)
        self.current = self.lexer.token()

    def type(self):
        if self.current is None:
            return TOK_EOF
        return self.current.type

    def value(self):
        if self.current is None:
            return None
        return self.current.value[0]

    def pos(self):
        if self.current is None:
            return (self.lexer.lineno, self.lexer.colno)
        return self.current.value[1]

    def next(self):
        self.current = self.lexer.token()

    def expect(self, *tokens, message, pos):
        if self.type() not in tokens:
            self.next()  # munch another token to update position
            raise TclSyntaxError(message, pos, self.pos())

        self.next()

    def assert_(self, *tokens):
        assert self.current.type in tokens
        self.next()