From 64fa802cd6e648a2826977238f8e6176768186cf Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Tue, 22 Jul 2025 18:41:20 +0200 Subject: [PATCH] update parser --- server/src/tools/lexer.py | 234 ++++++++++++++++++++++++++++++++ server/src/tools/parser.py | 9 ++ server/src/tools/syntax_tree.py | 59 ++++++++ 3 files changed, 302 insertions(+) create mode 100644 server/src/tools/lexer.py create mode 100644 server/src/tools/parser.py create mode 100644 server/src/tools/syntax_tree.py diff --git a/server/src/tools/lexer.py b/server/src/tools/lexer.py new file mode 100644 index 0000000..1375c4b --- /dev/null +++ b/server/src/tools/lexer.py @@ -0,0 +1,234 @@ +from enum import Enum +import ply.lex as lex +from typing import Tuple + + +class Tok(str, Enum): + TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE" + TOK_BACKSLASH_SUB = "BACKSLASH_SUB" + TOK_NEWLINE = "NEWLINE" + TOK_SEMI = "SEMI" + TOK_WS = "WS" + TOK_QUOTE = "QUOTE" + TOK_ARG_EXPANSION = "ARG_EXPANSION" + TOK_LBRACE = "LBRACE" + TOK_RBRACE = "RBRACE" + TOK_STAR = "STAR" + TOK_LBRACKET = "LBRACKET" + TOK_RBRACKET = "RBRACKET" + TOK_DOLLAR = "DOLLAR" + TOK_LPAREN = "LPAREN" + TOK_RPAREN = "RPAREN" + TOK_HASH = "HASH" + TOK_ALPHA_CHARS = "ALPHA_CHARS" + TOK_NUM_CHARS = "NUM_CHARS" + TOK_NAMESPACE_SEP = "NAMESPACE_SEP" + TOK_CHAR = "CHAR" + TOK_CONTENTS = "CONTENTS" + + +STATE_BRACEDWORD = "bracedword" +TOK_EOF = None + + +class TclSyntaxError(Exception): + def __init__(self, message, start: Tuple[int, int], end: Tuple[int, int]): + super().__init__(message) + self.start = start + self.end = end + + +class _LexTable: + tokens = tuple(t.value for t in Tok) + + states = ((STATE_BRACEDWORD, "exclusive"),) + + def __init__(self): + self.lexer = lex.lex(object=self) + self.lexer.lineno = 1 + self.lexer.colno = 1 + + def new_lexer(self, pos=None): + lexer = self.lexer.clone() + lexer.lineno = 1 + lexer.colno = 1 + + if pos is not None: + line, col = pos + lexer.lineno = line + lexer.colno = col + + return lexer + + def _tok(self, t): + pos = (t.lexer.lineno, t.lexer.colno) + t.lexer.lineno += t.value.count("\n") + index = t.value.rfind("\n") + if index == -1: + t.lexer.colno += len(t.value) + else: + remaining = t.value[index + 1 :] + t.lexer.colno = len(remaining) + 1 + + t.value = (t.value, pos) + return t + + # Priority important + def t_bracedword_INITIAL_BACKSLASH_NEWLINE(self, t): + r"\\\n" + return self._tok(t) + + # Priority important + def t_bracedword_INITIAL_BACKSLASH_SUB(self, t): + r"\\." + return self._tok(t) + + def t_NEWLINE(self, t): + r"\n" + return self._tok(t) + + def t_SEMI(self, t): + r";" + return self._tok(t) + + # TODO: should use \s? + def t_WS(self, t): + r"[\t\v\f\r ]+" + return self._tok(t) + + def t_QUOTE(self, t): + r'"' + return self._tok(t) + + # Must be higher priority than LBRACE + def t_ARG_EXPANSION(self, t): + r"\{\*\}" + return self._tok(t) + + def t_bracedword_INITIAL_LBRACE(self, t): + r"\{" + return self._tok(t) + + def t_bracedword_INITIAL_RBRACE(self, t): + r"\}" + return self._tok(t) + + def t_STAR(self, t): + r"\*" + return self._tok(t) + + def t_LBRACKET(self, t): + r"\[" + return self._tok(t) + + def t_RBRACKET(self, t): + r"\]" + return self._tok(t) + + def t_DOLLAR(self, t): + r"\$" + return self._tok(t) + + def t_LPAREN(self, t): + r"\(" + return self._tok(t) + + def t_RPAREN(self, t): + r"\)" + return self._tok(t) + + def t_HASH(self, t): + r"\#" + return self._tok(t) + + # Valid non-numeric chars in variable names + def t_ALPHA_CHARS(self, t): + r"[A-Za-z_]+" + return self._tok(t) + + def t_NUM_CHARS(self, t): + r"[0-9]+" + return self._tok(t) + + def t_NAMESPACE_SEP(self, t): + r"::+" + return self._tok(t) + + def t_bracedword_CONTENTS(self, t): + r"[^{}\\]+" + return self._tok(t) + + # Catch-all. TODO: inefficient, should probably munch multiple chars + def t_CHAR(self, t): + r"." + return self._tok(t) + + # Error handling rule + # TODO: do we need this? since we have a catch-all... + # there is a warning + def t_bracedword_INITIAL_error(self, t): + print("Illegal character '%s'" % t.value[0]) + t.lexer.skip(1) + + +# Calling `lex.lex()` performs an expensive reflection process to generate the lexer. +# This singleton class holds a preinitialized lexer that can then be cloned to create +# individual instances. +LexTable = _LexTable() + + +class Lexer: + def __init__(self, pos=None): + self.lexer = LexTable.new_lexer(pos) + self.current = None + + def input(self, text): + self.lexer.input(text) + self.current = self.lexer.token() + + def type(self): + if self.current is None: + return TOK_EOF + return self.current.type + + def value(self): + if self.current is None: + return None + return self.current.value[0] + + def pos(self): + if self.current is None: + return (self.lexer.lineno, self.lexer.colno) + return self.current.value[1] + + def next(self): + self.current = self.lexer.token() + + def expect(self, *tokens, message, pos): + if self.type() not in tokens: + self.next() # munch another token to update position + raise TclSyntaxError(message, pos, self.pos()) + + self.next() + + def assert_(self, *tokens): + assert self.current.type in tokens + self.next() + + +def dump_tokens(code): + lx = Lexer() + lx.input(code) + out = [] + while lx.type() is not TOK_EOF: + out.append((lx.type(), lx.value(), lx.pos())) + lx.next() + return out + + +if __name__ == "__main__": + code = ( + "set a 1\nputs $a\nnamespace eval test {}\n proc myProc {arg1 {optArg 10}} {}" + ) + for ttype, val, (ln, col) in dump_tokens(code): + print(f"{ttype:<18} {val!r:<10} @ ({ln},{col})") diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py new file mode 100644 index 0000000..3a5d5b9 --- /dev/null +++ b/server/src/tools/parser.py @@ -0,0 +1,9 @@ +class _Word: + def __init__(self): + self.segements = [] + self.current_segment = "" + self.current_start = None + + def add_tok(self, tok): + if self.current_start is None: + self.current_start = tok.value[1] diff --git a/server/src/tools/syntax_tree.py b/server/src/tools/syntax_tree.py new file mode 100644 index 0000000..d77de03 --- /dev/null +++ b/server/src/tools/syntax_tree.py @@ -0,0 +1,59 @@ +"""Classes for representing and interacting with Tcl syntax trees.""" + + +class Visitor: + """Abstract base class for Visitors that operate on syntax tree.""" + + def visit_script(self, script): + pass + + def visit_comment(self, comment): + pass + + def visit_command(self, command): + pass + + def visit_command_sub(self, command_sub): + pass + + def visit_bare_word(self, word): + pass + + def visit_braced_word(self, word): + pass + + def visit_quoted_word(self, word): + pass + + def visit_compound_bare_word(self, word): + pass + + def visit_var_sub(self, var_sub): + pass + + def visit_arg_expansion(self, arg_expansion): + pass + + def visit_list(self, list): + pass + + def visit_expression(self, expression): + pass + + def visit_braced_expression(self, expression): + pass + + def visit_paren_expression(self, expression): + pass + + def visit_unary_op(self, unary_op): + pass + + def visit_binary_op(self, binary_op): + pass + + def visit_ternary_op(self, ternary_op): + pass + + def visit_function(self, function): + pass