add more semantic tokens
This commit is contained in:
+216
-5
@@ -1,14 +1,225 @@
|
||||
from tclint.parser import Parser, _strip_ws
|
||||
from tclint.parser import (
|
||||
Parser,
|
||||
_is_bool_literal,
|
||||
_is_float_literal,
|
||||
_is_float_prefix,
|
||||
_is_function,
|
||||
_is_int_literal,
|
||||
_is_int_prefix,
|
||||
)
|
||||
from tclint.lexer import (
|
||||
TOK_WS,
|
||||
STATE_BRACEDWORD,
|
||||
TOK_BACKSLASH_NEWLINE,
|
||||
TOK_EOF,
|
||||
TOK_RPAREN,
|
||||
TOK_LBRACE,
|
||||
TOK_RBRACE,
|
||||
TOK_WS,
|
||||
TOK_NEWLINE,
|
||||
TOK_ALPHA_CHARS,
|
||||
TOK_RPAREN,
|
||||
TOK_LPAREN,
|
||||
TOK_DOLLAR,
|
||||
TOK_QUOTE,
|
||||
TOK_LBRACKET,
|
||||
TOK_NUM_CHARS,
|
||||
TclSyntaxError,
|
||||
)
|
||||
from tclint.syntax_tree import BareWord, TernaryOp, BinaryOp
|
||||
from tclint.syntax_tree import (
|
||||
BracedWord,
|
||||
BareWord,
|
||||
BinaryOp,
|
||||
TernaryOp,
|
||||
ParenExpression,
|
||||
UnaryOp,
|
||||
)
|
||||
|
||||
|
||||
class CustomParser(Parser):
|
||||
pass
|
||||
def _strip_ws(parse_func):
|
||||
"""Decorator used by expression parser for stripping whitespace around a node."""
|
||||
|
||||
def func(parser, ts):
|
||||
while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}:
|
||||
ts.next()
|
||||
|
||||
node = parse_func(parser, ts)
|
||||
|
||||
while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}:
|
||||
ts.next()
|
||||
|
||||
return node
|
||||
|
||||
return func
|
||||
|
||||
@_strip_ws
|
||||
def _parse_expression(self, ts):
|
||||
op1 = self._parse_operand(ts)
|
||||
expr = op1
|
||||
|
||||
# last condition is hack to break out of expression in case we're in ternary op
|
||||
if ts.type() not in {TOK_EOF, TOK_RPAREN} and ts.value() not in {":", ","}:
|
||||
if ts.value() == "?":
|
||||
# weird hack to record operator
|
||||
start = ts.pos()
|
||||
ts.next()
|
||||
q = BareWord("?", pos=start, end_pos=ts.pos())
|
||||
|
||||
op2 = self._parse_expression(ts)
|
||||
if ts.value() != ":":
|
||||
start = ts.pos()
|
||||
ts.next()
|
||||
end = ts.pos()
|
||||
raise TclSyntaxError(
|
||||
"expected ':' to continue ternary expression", start, end
|
||||
)
|
||||
|
||||
# weird hack again
|
||||
start = ts.pos()
|
||||
ts.next()
|
||||
colon = BareWord(":", pos=start, end_pos=ts.pos())
|
||||
|
||||
op3 = self._parse_expression(ts)
|
||||
expr = TernaryOp(
|
||||
op1, q, op2, colon, op3, pos=op1.pos, end_pos=op3.end_pos
|
||||
)
|
||||
else:
|
||||
operator = self._parse_operator(ts)
|
||||
op2 = self._parse_expression(ts)
|
||||
expr = BinaryOp(op1, operator, op2, pos=op1.pos, end_pos=op2.end_pos)
|
||||
|
||||
if ts.type() != TOK_RPAREN and ts.value() not in {":", ","}:
|
||||
ts.expect(TOK_EOF, message="expected end of expression", pos=ts.pos())
|
||||
|
||||
return expr
|
||||
|
||||
@_strip_ws
|
||||
def _parse_operand(self, ts):
|
||||
if ts.type() == TOK_DOLLAR:
|
||||
return self.parse_var_sub(ts)
|
||||
if ts.type() == TOK_QUOTE:
|
||||
return self.parse_quoted_word(ts)
|
||||
if ts.type() == TOK_LBRACE:
|
||||
return self.parse_braced_word(ts)
|
||||
if ts.type() == TOK_LBRACKET:
|
||||
return self.parse_command_sub(ts)
|
||||
if ts.type() == TOK_LPAREN:
|
||||
start = ts.pos()
|
||||
ts.next()
|
||||
expr = self._parse_expression(ts)
|
||||
ts.expect(
|
||||
TOK_RPAREN,
|
||||
message="reached EOF without finding match for paren",
|
||||
pos=expr.pos,
|
||||
)
|
||||
end = ts.pos()
|
||||
return ParenExpression(expr, start, end)
|
||||
if ts.value() in {"-", "+", "~", "!"}:
|
||||
operator_val = ts.value()
|
||||
operator_pos = ts.pos()
|
||||
ts.next()
|
||||
operator = BareWord(operator_val, pos=operator_pos, end_pos=ts.pos())
|
||||
operand = self._parse_operand(ts)
|
||||
# Since _parse_operand() munches whitespace after the operand, we
|
||||
# set the end of the UnaryOp to the end of the operand rather than
|
||||
# ts.pos(). Otherwise, the bounds of the UnaryOp would include all
|
||||
# that whitespace.
|
||||
return UnaryOp(operator, operand, pos=operator_pos, end_pos=operand.end_pos)
|
||||
|
||||
# If none of these, collect tokens that may comprise an operand
|
||||
operand = ""
|
||||
operand_pos = ts.pos()
|
||||
|
||||
# First, we want to check for numeric operands (either ints or numeric
|
||||
# floats) by consuming tokens as long as they comprise the prefix of a
|
||||
# numeric operand
|
||||
while ts.type() != TOK_EOF and (
|
||||
_is_int_prefix(operand + ts.value())
|
||||
or _is_float_prefix(operand + ts.value())
|
||||
):
|
||||
operand += ts.value()
|
||||
ts.next()
|
||||
|
||||
# Next, we check if we've consumed an entire numeric literal. If so, we
|
||||
# move on. If not, we keep consuming tokens that may correspond to a
|
||||
# valid bareword (pretty much just alphanumeric chars).
|
||||
if not (_is_int_literal(operand) or _is_float_literal(operand)):
|
||||
while ts.type() in {TOK_ALPHA_CHARS, TOK_NUM_CHARS}:
|
||||
operand += ts.value()
|
||||
ts.next()
|
||||
|
||||
# The above method is a little hacky. Note that it doesn't parse things
|
||||
# exactly the same as Tcl. E.g. if a script includes `expr {1foo}`,
|
||||
# tclint will report an invalid operator "foo", whereas tclsh will
|
||||
# report an invalid bareword "1foo". Despite reporting them differently
|
||||
# both tools should still catch the same syntax errors, since there are
|
||||
# no legal barewords that begin with a numeric literal prefix, and tclsh
|
||||
# will stop parsing numeric operands if they're actually followed by a
|
||||
# legal operator (e.g. `expr {1eq1}` will be handled properly).
|
||||
|
||||
is_func = _is_function(operand)
|
||||
|
||||
if not (
|
||||
_is_int_literal(operand)
|
||||
or _is_float_literal(operand)
|
||||
or _is_bool_literal(operand)
|
||||
or is_func
|
||||
):
|
||||
raise TclSyntaxError(
|
||||
f"invalid bareword in expression: {operand}", operand_pos, ts.pos()
|
||||
)
|
||||
|
||||
node = BareWord(operand, pos=operand_pos, end_pos=ts.pos())
|
||||
|
||||
if is_func:
|
||||
node = self._parse_function(ts, node)
|
||||
|
||||
return node
|
||||
|
||||
def parse_braced_word(self, ts):
|
||||
self.debug(f"parse_braced_word({ts.current})")
|
||||
pos = ts.pos()
|
||||
|
||||
ts.lexer.push_state(STATE_BRACEDWORD)
|
||||
|
||||
ts.assert_(TOK_LBRACE)
|
||||
|
||||
word = ""
|
||||
expected_braces = [pos] # Stack für geschachtelte Klammern
|
||||
|
||||
while True:
|
||||
toktype = ts.type()
|
||||
if toktype == TOK_EOF:
|
||||
raise TclSyntaxError(
|
||||
"reached EOF without finding match for brace",
|
||||
expected_braces[-1],
|
||||
ts.pos(),
|
||||
)
|
||||
|
||||
if toktype == TOK_BACKSLASH_NEWLINE:
|
||||
# TCL-spezifisch: Zeilenumbruch mit Backslash ignorieren
|
||||
ts.next()
|
||||
continue
|
||||
|
||||
if toktype == TOK_LBRACE:
|
||||
expected_braces.append(ts.pos())
|
||||
elif toktype == TOK_RBRACE:
|
||||
try:
|
||||
expected_braces.pop()
|
||||
except IndexError:
|
||||
start = ts.pos()
|
||||
ts.next()
|
||||
end = ts.pos()
|
||||
raise TclSyntaxError(
|
||||
"found closing brace without matching open brace", start, end
|
||||
)
|
||||
|
||||
if len(expected_braces) == 0:
|
||||
ts.lexer.pop_state()
|
||||
ts.next()
|
||||
break
|
||||
|
||||
word += ts.value()
|
||||
ts.next()
|
||||
|
||||
end_pos = ts.pos()
|
||||
return BracedWord(word, pos=pos, end_pos=end_pos)
|
||||
|
||||
Reference in New Issue
Block a user