Christoph/nx_post_support
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases 25 Wiki Activity

update parser

Browse Source
This commit is contained in:
Christoph Brandau
2025-07-25 16:02:39 +02:00
parent 94181497bd
commit c34dac847a
16 changed files with 168 additions and 1667 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/tools/parser.py
+96 -104
View File
@@ -1,64 +1,40 @@
import string
import re
from src.tools.lexer import (
Lexer,
TclSyntaxError,
STATE_BRACEDWORD,
TOK_BACKSLASH_NEWLINE,
TOK_NEWLINE,
TOK_SEMI,
TOK_WS,
TOK_QUOTE,
TOK_ARG_EXPANSION,
TOK_LBRACE,
TOK_RBRACE,
TOK_LBRACKET,
TOK_RBRACKET,
TOK_DOLLAR,
TOK_LPAREN,
TOK_RPAREN,
TOK_HASH,
TOK_ALPHA_CHARS,
TOK_NUM_CHARS,
TOK_NAMESPACE_SEP,
TOK_EOF,
)
from src.tools.syntax_tree import (
Script,
Comment,
from tools.commands.checks import CommandArgError, check_command
from tools.lexer import Lexer, TclSyntaxError, Tok, TOK_EOF, STATE_BRACEDWORD
from tools.syntax_tree import (
ArgExpansion,
BareWord,
BinaryOp,
BracedExpression,
BracedWord,
Command,
CommandSub,
ArgExpansion,
VarSub,
BareWord,
BracedWord,
QuotedWord,
Comment,
CompoundBareWord,
List,
Expression,
BracedExpression,
ParenExpression,
UnaryOp,
BinaryOp,
TernaryOp,
Function,
List,
ParenExpression,
QuotedWord,
Script,
TernaryOp,
UnaryOp,
VarSub,
)
from src.tools.commands import CommandArgError, get_commands
from src.tools.commands.checks import check_command
from src.tools.violations import Rule, Violation
def _strip_ws(parse_func):
"""Decorator used by expression parser for stripping whitespace around a node."""
def func(parser, ts):
while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}:
while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE, Tok.TOK_NEWLINE}:
ts.next()
node = parse_func(parser, ts)
while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}:
while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE, Tok.TOK_NEWLINE}:
ts.next()
return node
@@ -98,15 +74,12 @@ class _Word:
class Parser:
def __init__(self, debug=False, command_plugins=None):
def __init__(self, debug=False):
self._debug = debug
self._debug_indent = 0
# TODO: better way to handle this?
self.violations = []
if command_plugins is None:
command_plugins = []
self._commands = get_commands(command_plugins)
self._commands = []
def debug(self, *msg):
if self._debug:
@@ -197,12 +170,12 @@ class Parser:
script = Script(pos=pos)
while ts.type() is not TOK_EOF:
if ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE}:
if ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE}:
# strip whitespace at start of command
ts.next()
continue
if ts.type() == TOK_HASH:
if ts.type() == Tok.TOK_HASH:
script.add(self.parse_comment(ts))
else:
cmd = self.parse_command(ts, in_command_sub=in_command_sub)
@@ -210,13 +183,13 @@ class Parser:
script.add(cmd)
# when in command sub mode, a script is terminated by ]
if in_command_sub and ts.type() == TOK_RBRACKET:
if in_command_sub and ts.type() == Tok.TOK_RBRACKET:
return script
ts.expect(
TOK_EOF,
TOK_NEWLINE,
TOK_SEMI,
Tok.TOK_NEWLINE,
Tok.TOK_SEMI,
message=f"expected newline or semicolon, got {ts.value()}",
pos=ts.pos(),
)
@@ -236,10 +209,10 @@ class Parser:
self.debug(f"parse_comment({ts.current})")
pos = ts.pos()
ts.assert_(TOK_HASH)
ts.assert_(Tok.TOK_HASH)
value = ""
while ts.type() not in {TOK_NEWLINE, TOK_EOF}:
while ts.type() not in {Tok.TOK_NEWLINE, TOK_EOF}:
value += ts.value()
ts.next()
@@ -261,10 +234,10 @@ class Parser:
args = []
while True:
if ts.type() not in {TOK_WS, TOK_BACKSLASH_NEWLINE}:
if ts.type() not in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE}:
break
while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE}:
while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE}:
ts.next()
word = self.parse_word(ts, in_command_sub)
@@ -276,9 +249,10 @@ class Parser:
self._debug_indent -= 1
try:
pass
parsed_args = self._parse_command_args(routine.contents, args)
except CommandArgError as e:
self.violations.append(Violation(Rule.COMMAND_ARGS, str(e), pos, ts.pos()))
# self.violations.append(Violation(Rule.COMMAND_ARGS, str(e), pos, ts.pos()))
parsed_args = args
children = [routine, *parsed_args]
@@ -289,11 +263,11 @@ class Parser:
def parse_word(self, ts, in_command_sub):
self.debug(f"parse_word({ts.current})")
if ts.type() == TOK_ARG_EXPANSION:
if ts.type() == Tok.TOK_ARG_EXPANSION:
return self.parse_arg_expansion(ts, in_command_sub)
elif ts.type() == TOK_LBRACE:
elif ts.type() == Tok.TOK_LBRACE:
return self.parse_braced_word(ts)
elif ts.type() == TOK_QUOTE:
elif ts.type() == Tok.TOK_QUOTE:
return self.parse_quoted_word(ts)
else:
return self.parse_bare_word(ts, in_command_sub)
@@ -302,11 +276,17 @@ class Parser:
self.debug(f"parse_arg_expansion({ts.current})")
pos = ts.pos()
ts.assert_(TOK_ARG_EXPANSION)
ts.assert_(Tok.TOK_ARG_EXPANSION)
delimiters = [TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE, TOK_SEMI, TOK_EOF]
delimiters = [
Tok.TOK_WS,
Tok.TOK_BACKSLASH_NEWLINE,
Tok.TOK_NEWLINE,
Tok.TOK_SEMI,
TOK_EOF,
]
if in_command_sub:
delimiters.append(TOK_RBRACKET)
delimiters.append(Tok.TOK_RBRACKET)
# Arg expansion is just a regular braced word if followed by whitespace,
# or other word boundaries such as semicolon or right bracket
@@ -323,18 +303,18 @@ class Parser:
self._debug_indent += 1
pos = ts.pos()
ts.assert_(TOK_QUOTE)
ts.assert_(Tok.TOK_QUOTE)
word = _Word()
while ts.type() not in {TOK_QUOTE, TOK_EOF}:
if ts.type() == TOK_DOLLAR:
while ts.type() not in {Tok.TOK_QUOTE, TOK_EOF}:
if ts.type() == Tok.TOK_DOLLAR:
dollar_tok = ts.current
var_sub = self.parse_var_sub(ts)
if var_sub:
word.add_node(var_sub)
else:
word.add_tok(dollar_tok)
elif ts.type() == TOK_LBRACKET:
elif ts.type() == Tok.TOK_LBRACKET:
command_sub = self.parse_command_sub(ts)
word.add_node(command_sub)
else:
@@ -344,7 +324,9 @@ class Parser:
res = word.resolve(ts.pos())
ts.expect(
TOK_QUOTE, message="reached EOF without finding match for quote", pos=pos
Tok.TOK_QUOTE,
message="reached EOF without finding match for quote",
pos=pos,
)
self._debug_indent -= 1
@@ -360,7 +342,7 @@ class Parser:
ts.lexer.push_state(STATE_BRACEDWORD)
ts.assert_(TOK_LBRACE)
ts.assert_(Tok.TOK_LBRACE)
word = ""
# store position for each brace we want to match, facilitating good
@@ -375,9 +357,9 @@ class Parser:
ts.pos(),
)
if toktype == TOK_LBRACE:
if toktype == Tok.TOK_LBRACE:
expected_braces.append(ts.pos())
elif toktype == TOK_RBRACE:
elif toktype == Tok.TOK_RBRACE:
try:
expected_braces.pop()
except IndexError:
@@ -404,21 +386,27 @@ class Parser:
pos = ts.pos()
word = _Word()
delimiters = [TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE, TOK_SEMI, TOK_EOF]
delimiters = [
Tok.TOK_WS,
Tok.TOK_BACKSLASH_NEWLINE,
Tok.TOK_NEWLINE,
Tok.TOK_SEMI,
TOK_EOF,
]
# In command sub mode, words are ended by ]
if in_command_sub:
delimiters.append(TOK_RBRACKET)
delimiters.append(Tok.TOK_RBRACKET)
while ts.type() not in delimiters:
if ts.type() == TOK_DOLLAR:
if ts.type() == Tok.TOK_DOLLAR:
dollar_tok = ts.current
var_sub = self.parse_var_sub(ts)
if var_sub:
word.add_node(var_sub)
else:
word.add_tok(dollar_tok)
elif ts.type() == TOK_LBRACKET:
elif ts.type() == Tok.TOK_LBRACKET:
command_sub = self.parse_command_sub(ts)
word.add_node(command_sub)
else:
@@ -439,13 +427,13 @@ class Parser:
self.debug(f"parse_var_sub({ts.current})")
pos = ts.pos()
ts.assert_(TOK_DOLLAR)
ts.assert_(Tok.TOK_DOLLAR)
var = ""
if ts.type() == TOK_LBRACE:
if ts.type() == Tok.TOK_LBRACE:
brace_pos = ts.pos()
ts.next()
while ts.type() != TOK_RBRACE:
while ts.type() != Tok.TOK_RBRACE:
if ts.type() is TOK_EOF:
raise TclSyntaxError(
"reached EOF without finding match for brace",
@@ -458,7 +446,11 @@ class Parser:
return VarSub(var, pos=pos, end_pos=ts.pos(), braced=True)
while ts.type() in {TOK_ALPHA_CHARS, TOK_NUM_CHARS, TOK_NAMESPACE_SEP}:
while ts.type() in {
Tok.TOK_ALPHA_CHARS,
Tok.TOK_NUM_CHARS,
Tok.TOK_NAMESPACE_SEP,
}:
var += ts.value()
ts.next()
@@ -466,25 +458,25 @@ class Parser:
return None
index_nodes = []
if ts.type() == TOK_LPAREN:
if ts.type() == Tok.TOK_LPAREN:
paren_pos = ts.pos()
index = _Word()
ts.next()
while ts.type() != TOK_RPAREN:
while ts.type() != Tok.TOK_RPAREN:
if ts.type() == TOK_EOF:
raise TclSyntaxError(
"reached EOF without finding match for paren",
paren_pos,
ts.pos(),
)
if ts.type() == TOK_DOLLAR:
if ts.type() == Tok.TOK_DOLLAR:
dollar_tok = ts.current
var_sub = self.parse_var_sub(ts)
if var_sub:
index.add_node(var_sub)
else:
index.add_tok(dollar_tok)
elif ts.type() == TOK_LBRACKET:
elif ts.type() == Tok.TOK_LBRACKET:
command_sub = self.parse_command_sub(ts)
index.add_node(command_sub)
else:
@@ -506,11 +498,11 @@ class Parser:
self._debug_indent += 1
pos = ts.pos()
ts.assert_(TOK_LBRACKET)
ts.assert_(Tok.TOK_LBRACKET)
script = self._parse_script(ts, in_command_sub=True)
ts.assert_(TOK_RBRACKET)
ts.assert_(Tok.TOK_RBRACKET)
end_pos = ts.pos()
script.line = pos[0]
@@ -537,7 +529,7 @@ class Parser:
ts = Lexer(pos=node.contents_pos)
ts.input(node.contents)
DELIMITERS = {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}
DELIMITERS = {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE, Tok.TOK_NEWLINE}
list_node = List(pos=node.pos, end_pos=node.end_pos)
while ts.type() is not TOK_EOF:
@@ -547,24 +539,24 @@ class Parser:
if ts.type() is TOK_EOF:
break
if ts.type() == TOK_LBRACE:
if ts.type() == Tok.TOK_LBRACE:
# we can reuse parse_braced_word, since it doesn't use
# substitutions in any case
list_node.add(self.parse_braced_word(ts))
elif ts.type() == TOK_QUOTE:
elif ts.type() == Tok.TOK_QUOTE:
quote_word_pos = ts.pos()
ts.assert_(TOK_QUOTE)
ts.assert_(Tok.TOK_QUOTE)
bare_word_pos = ts.pos()
contents = ""
while ts.type() not in {TOK_QUOTE, TOK_EOF}:
while ts.type() not in {Tok.TOK_QUOTE, TOK_EOF}:
contents += ts.value()
ts.next()
word = BareWord(contents, pos=bare_word_pos, end_pos=ts.pos())
ts.expect(
TOK_QUOTE,
Tok.TOK_QUOTE,
message="reached EOF without finding match for quote",
pos=quote_word_pos,
)
@@ -607,7 +599,7 @@ class Parser:
expr = op1
# last condition is hack to break out of expression in case we're in ternary op
if ts.type() not in {TOK_EOF, TOK_RPAREN} and ts.value() not in {":", ","}:
if ts.type() not in {TOK_EOF, Tok.TOK_RPAREN} and ts.value() not in {":", ","}:
if ts.value() == "?":
# weird hack to record operator
start = ts.pos()
@@ -637,27 +629,27 @@ class Parser:
op2 = self._parse_expression(ts)
expr = BinaryOp(op1, operator, op2, pos=op1.pos, end_pos=op2.end_pos)
if ts.type() != TOK_RPAREN and ts.value() not in {":", ","}:
if ts.type() != Tok.TOK_RPAREN and ts.value() not in {":", ","}:
ts.expect(TOK_EOF, message="expected end of expression", pos=ts.pos())
return expr
@_strip_ws
def _parse_operand(self, ts):
if ts.type() == TOK_DOLLAR:
if ts.type() == Tok.TOK_DOLLAR:
return self.parse_var_sub(ts)
if ts.type() == TOK_QUOTE:
if ts.type() == Tok.TOK_QUOTE:
return self.parse_quoted_word(ts)
if ts.type() == TOK_LBRACE:
if ts.type() == Tok.TOK_LBRACE:
return self.parse_braced_word(ts)
if ts.type() == TOK_LBRACKET:
if ts.type() == Tok.TOK_LBRACKET:
return self.parse_command_sub(ts)
if ts.type() == TOK_LPAREN:
if ts.type() == Tok.TOK_LPAREN:
start = ts.pos()
ts.next()
expr = self._parse_expression(ts)
ts.expect(
TOK_RPAREN,
Tok.TOK_RPAREN,
message="reached EOF without finding match for paren",
pos=expr.pos,
)
@@ -693,7 +685,7 @@ class Parser:
# move on. If not, we keep consuming tokens that may correspond to a
# valid bareword (pretty much just alphanumeric chars).
if not (_is_int_literal(operand) or _is_float_literal(operand)):
while ts.type() in {TOK_ALPHA_CHARS, TOK_NUM_CHARS}:
while ts.type() in {Tok.TOK_ALPHA_CHARS, Tok.TOK_NUM_CHARS}:
operand += ts.value()
ts.next()
@@ -763,16 +755,16 @@ class Parser:
return BareWord(operator, pos=pos, end_pos=ts.pos())
def _parse_function(self, ts, name):
while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE}:
while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE}:
ts.next()
ts.expect(
TOK_LPAREN,
Tok.TOK_LPAREN,
message="expected open paren after function name",
pos=name.pos,
)
delims = {TOK_RPAREN, TOK_EOF}
delims = {Tok.TOK_RPAREN, TOK_EOF}
arguments = []
if ts.type() not in delims:
@@ -791,7 +783,7 @@ class Parser:
arguments.append(self._parse_expression(ts))
ts.expect(
TOK_RPAREN,
Tok.TOK_RPAREN,
message="expected close paren after function arguments",
pos=name.pos,
)