server/src/tools/lexer.py

@@ -0,0 +1,234 @@
+from enum import Enum
+import ply.lex as lex
+from typing import Tuple
+
+
+class Tok(str, Enum):
+    TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE"
+    TOK_BACKSLASH_SUB = "BACKSLASH_SUB"
+    TOK_NEWLINE = "NEWLINE"
+    TOK_SEMI = "SEMI"
+    TOK_WS = "WS"
+    TOK_QUOTE = "QUOTE"
+    TOK_ARG_EXPANSION = "ARG_EXPANSION"
+    TOK_LBRACE = "LBRACE"
+    TOK_RBRACE = "RBRACE"
+    TOK_STAR = "STAR"
+    TOK_LBRACKET = "LBRACKET"
+    TOK_RBRACKET = "RBRACKET"
+    TOK_DOLLAR = "DOLLAR"
+    TOK_LPAREN = "LPAREN"
+    TOK_RPAREN = "RPAREN"
+    TOK_HASH = "HASH"
+    TOK_ALPHA_CHARS = "ALPHA_CHARS"
+    TOK_NUM_CHARS = "NUM_CHARS"
+    TOK_NAMESPACE_SEP = "NAMESPACE_SEP"
+    TOK_CHAR = "CHAR"
+    TOK_CONTENTS = "CONTENTS"
+
+
+STATE_BRACEDWORD = "bracedword"
+TOK_EOF = None
+
+
+class TclSyntaxError(Exception):
+    def __init__(self, message, start: Tuple[int, int], end: Tuple[int, int]):
+        super().__init__(message)
+        self.start = start
+        self.end = end
+
+
+class _LexTable:
+    tokens = tuple(t.value for t in Tok)
+
+    states = ((STATE_BRACEDWORD, "exclusive"),)
+
+    def __init__(self):
+        self.lexer = lex.lex(object=self)
+        self.lexer.lineno = 1
+        self.lexer.colno = 1
+
+    def new_lexer(self, pos=None):
+        lexer = self.lexer.clone()
+        lexer.lineno = 1
+        lexer.colno = 1
+
+        if pos is not None:
+            line, col = pos
+            lexer.lineno = line
+            lexer.colno = col
+
+        return lexer
+
+    def _tok(self, t):
+        pos = (t.lexer.lineno, t.lexer.colno)
+        t.lexer.lineno += t.value.count("\n")
+        index = t.value.rfind("\n")
+        if index == -1:
+            t.lexer.colno += len(t.value)
+        else:
+            remaining = t.value[index + 1 :]
+            t.lexer.colno = len(remaining) + 1
+
+        t.value = (t.value, pos)
+        return t
+
+    # Priority important
+    def t_bracedword_INITIAL_BACKSLASH_NEWLINE(self, t):
+        r"\\\n"
+        return self._tok(t)
+
+    # Priority important
+    def t_bracedword_INITIAL_BACKSLASH_SUB(self, t):
+        r"\\."
+        return self._tok(t)
+
+    def t_NEWLINE(self, t):
+        r"\n"
+        return self._tok(t)
+
+    def t_SEMI(self, t):
+        r";"
+        return self._tok(t)
+
+    # TODO: should use \s?
+    def t_WS(self, t):
+        r"[\t\v\f\r ]+"
+        return self._tok(t)
+
+    def t_QUOTE(self, t):
+        r'"'
+        return self._tok(t)
+
+    # Must be higher priority than LBRACE
+    def t_ARG_EXPANSION(self, t):
+        r"\{\*\}"
+        return self._tok(t)
+
+    def t_bracedword_INITIAL_LBRACE(self, t):
+        r"\{"
+        return self._tok(t)
+
+    def t_bracedword_INITIAL_RBRACE(self, t):
+        r"\}"
+        return self._tok(t)
+
+    def t_STAR(self, t):
+        r"\*"
+        return self._tok(t)
+
+    def t_LBRACKET(self, t):
+        r"\["
+        return self._tok(t)
+
+    def t_RBRACKET(self, t):
+        r"\]"
+        return self._tok(t)
+
+    def t_DOLLAR(self, t):
+        r"\$"
+        return self._tok(t)
+
+    def t_LPAREN(self, t):
+        r"\("
+        return self._tok(t)
+
+    def t_RPAREN(self, t):
+        r"\)"
+        return self._tok(t)
+
+    def t_HASH(self, t):
+        r"\#"
+        return self._tok(t)
+
+    # Valid non-numeric chars in variable names
+    def t_ALPHA_CHARS(self, t):
+        r"[A-Za-z_]+"
+        return self._tok(t)
+
+    def t_NUM_CHARS(self, t):
+        r"[0-9]+"
+        return self._tok(t)
+
+    def t_NAMESPACE_SEP(self, t):
+        r"::+"
+        return self._tok(t)
+
+    def t_bracedword_CONTENTS(self, t):
+        r"[^{}\\]+"
+        return self._tok(t)
+
+    # Catch-all. TODO: inefficient, should probably munch multiple chars
+    def t_CHAR(self, t):
+        r"."
+        return self._tok(t)
+
+    # Error handling rule
+    # TODO: do we need this? since we have a catch-all...
+    # there is a warning
+    def t_bracedword_INITIAL_error(self, t):
+        print("Illegal character '%s'" % t.value[0])
+        t.lexer.skip(1)
+
+
+# Calling `lex.lex()` performs an expensive reflection process to generate the lexer.
+# This singleton class holds a preinitialized lexer that can then be cloned to create
+# individual instances.
+LexTable = _LexTable()
+
+
+class Lexer:
+    def __init__(self, pos=None):
+        self.lexer = LexTable.new_lexer(pos)
+        self.current = None
+
+    def input(self, text):
+        self.lexer.input(text)
+        self.current = self.lexer.token()
+
+    def type(self):
+        if self.current is None:
+            return TOK_EOF
+        return self.current.type
+
+    def value(self):
+        if self.current is None:
+            return None
+        return self.current.value[0]
+
+    def pos(self):
+        if self.current is None:
+            return (self.lexer.lineno, self.lexer.colno)
+        return self.current.value[1]
+
+    def next(self):
+        self.current = self.lexer.token()
+
+    def expect(self, *tokens, message, pos):
+        if self.type() not in tokens:
+            self.next()  # munch another token to update position
+            raise TclSyntaxError(message, pos, self.pos())
+
+        self.next()
+
+    def assert_(self, *tokens):
+        assert self.current.type in tokens
+        self.next()
+
+
+def dump_tokens(code):
+    lx = Lexer()
+    lx.input(code)
+    out = []
+    while lx.type() is not TOK_EOF:
+        out.append((lx.type(), lx.value(), lx.pos()))
+        lx.next()
+    return out
+
+
+if __name__ == "__main__":
+    code = (
+        "set a 1\nputs $a\nnamespace eval test {}\n proc myProc {arg1 {optArg 10}} {}"
+    )
+    for ttype, val, (ln, col) in dump_tokens(code):
+        print(f"{ttype:<18} {val!r:<10} @ ({ln},{col})")

server/src/tools/parser.py

@@ -0,0 +1,9 @@
+class _Word:
+    def __init__(self):
+        self.segements = []
+        self.current_segment = ""
+        self.current_start = None
+
+    def add_tok(self, tok):
+        if self.current_start is None:
+            self.current_start = tok.value[1]

server/src/tools/syntax_tree.py

@@ -0,0 +1,59 @@
+"""Classes for representing and interacting with Tcl syntax trees."""
+
+
+class Visitor:
+    """Abstract base class for Visitors that operate on syntax tree."""
+
+    def visit_script(self, script):
+        pass
+
+    def visit_comment(self, comment):
+        pass
+
+    def visit_command(self, command):
+        pass
+
+    def visit_command_sub(self, command_sub):
+        pass
+
+    def visit_bare_word(self, word):
+        pass
+
+    def visit_braced_word(self, word):
+        pass
+
+    def visit_quoted_word(self, word):
+        pass
+
+    def visit_compound_bare_word(self, word):
+        pass
+
+    def visit_var_sub(self, var_sub):
+        pass
+
+    def visit_arg_expansion(self, arg_expansion):
+        pass
+
+    def visit_list(self, list):
+        pass
+
+    def visit_expression(self, expression):
+        pass
+
+    def visit_braced_expression(self, expression):
+        pass
+
+    def visit_paren_expression(self, expression):
+        pass
+
+    def visit_unary_op(self, unary_op):
+        pass
+
+    def visit_binary_op(self, binary_op):
+        pass
+
+    def visit_ternary_op(self, ternary_op):
+        pass
+
+    def visit_function(self, function):
+        pass