From c9a07ae1b926393792ac4745176a14af37e3bfd2 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Wed, 23 Jul 2025 17:12:24 +0200 Subject: [PATCH] use tclint --- server/src/tools/checks.py | 240 ++++++++ server/src/tools/commands/__init__.py | 37 ++ server/src/tools/parser.py | 847 +++++++++++++++++++++++++- server/src/tools/plugins.py | 81 +++ server/src/tools/schema.py | 35 ++ server/src/tools/syntax_tree.py | 379 ++++++++++++ 6 files changed, 1618 insertions(+), 1 deletion(-) create mode 100644 server/src/tools/checks.py create mode 100644 server/src/tools/commands/__init__.py create mode 100644 server/src/tools/plugins.py create mode 100644 server/src/tools/schema.py diff --git a/server/src/tools/checks.py b/server/src/tools/checks.py new file mode 100644 index 0000000..08be911 --- /dev/null +++ b/server/src/tools/checks.py @@ -0,0 +1,240 @@ +"""Helpers for checking command arguments.""" + +from collections.abc import Callable +from typing import List, Optional, Union + +from syntax_tree import ArgExpansion, QuotedWord, BracedWord, BareWord, Node + + +class CommandArgError(Exception): + pass + + +def arg_count(args, parser): + # TODO: graceful handling of argsub going into things with recursive parsing. + # if the argsub happens to be "concrete", we can technically do the right + # thing (although this should probably be flagged as a readability issue...) + # otherwise, we should flag that the non-concrete argsub is not okay for + # these cases. however, I think its not okay-ness doesn't need to be absolute, e.g. + # I think we could allow: + # + # catch {puts "my script"} {*}$catchopts + # + + arg_count = 0 + has_arg_expansion = False + for arg in args: + if isinstance(arg, ArgExpansion): + if arg.contents is None: + has_arg_expansion = True + continue + arg_count += len(parser.parse_list(arg.contents)) + else: + arg_count += 1 + + return arg_count, has_arg_expansion + + +def check_count(command, min=None, max=None, args_name="args"): + def check(args, parser): + if min is None and max is None: + return None + + count, has_arg_expansion = arg_count(args, parser) + + if not has_arg_expansion and min == max and count != min: + raise CommandArgError( + f"wrong # of {args_name} for {command}: got {count}, expected {min}" + ) + + if not has_arg_expansion and min is not None and count < min: + raise CommandArgError( + f"not enough {args_name} for {command}: got {count}, expected at least" + f" {min}" + ) + + if max is not None and count > max: + raise CommandArgError( + f"too many {args_name} for {command}: got {count}, expected no more" + f" than {max}" + ) + + return None + + return check + + +def eval(args, parser, command): + if len(args) > 1 and any(isinstance(arg, (QuotedWord, BracedWord)) for arg in args): + # Slightly odd restriction, but our syntax tree doesn't have a great way + # to handle this case. We require each command argument to correspond to + # one child node, but multiple quoted or braced word arguments can be + # combined into a single subcommand when interpreted eval-style. This + # requirement exists to facilitate style checking, if we had a separate + # CST for style checks and AST for logical checks we may be able to + # handle it. + + raise CommandArgError( + f"unable to parse multiple {command} arguments when one includes a braced" + " or quoted word" + ) + + # Construct the body of the eval taking whitespace into account to ensure we get + # style checking. + + eval_script = "" + prev_arg_end_pos = None + for arg in args: + contents = arg.contents + if contents is None: + # TODO: flag sort of eval-specific violation? Common patterns will + # often trigger this, and it seems useful to be able to turn it off + raise CommandArgError( + f"{command} received an argument with a substitution, unable to parse" + " its arguments" + ) + + if prev_arg_end_pos is not None: + if prev_arg_end_pos[0] != arg.line: + # If we have multiple args on the same line, we know there must be a + # backslash newline. Add it so the parsing works. + eval_script += "\\\n" * (arg.line - prev_arg_end_pos[0]) + eval_script += " " * (arg.col - 1) + else: + eval_script += " " * (arg.col - prev_arg_end_pos[1]) + eval_script += contents + + prev_arg_end_pos = arg.end_pos + + script = parser.parse(eval_script, pos=(args[0].pos)) + script.end_pos = args[-1].end_pos + + return [script] + + +def check_command( + command: str, args: List[Node], parser, command_spec: Union[Callable, dict, None] +) -> Optional[List[Node]]: + if command_spec is None: + return None + + if isinstance(command_spec, dict): + return check_arg_spec(command, args, parser, command_spec) + + return command_spec(args, parser) + + +def check_arg_spec( + command: str, args: List[Node], parser, arg_spec: dict +) -> Optional[List[Node]]: + if "subcommands" in arg_spec: + subcommands = arg_spec["subcommands"] + try: + subcommand = args[0].contents + except IndexError: + subcommand = None + + if subcommand in subcommands: + new_args = check_command( + f"{command} {subcommand}", args[1:], parser, subcommands[subcommand] + ) + if new_args is None: + return new_args + return args[0:1] + new_args + + if "" in subcommands: + return check_command(command, args, parser, subcommands[""]) + + if subcommand is not None: + msg = f"invalid subcommand for {command}: got {subcommand}" + else: + msg = f"no subcommand provided for {command}" + + raise CommandArgError(f"{msg}, expected one of {', '.join(subcommands.keys())}") + + switches = arg_spec["switches"] + args_allowed = set(switches) + args_required = {switch for switch in switches if switches[switch]["required"]} + positional_args = [] + + args = list(args) + while len(args) > 0: + arg = args.pop(0) + + # To facilitate better error messages, we expect that switches are always + # specified as BareWords that start with "-" or ">". This lets us throw an + # error when a switch-like thing doesn't match any supported arguments, + # rather than counting it towards the positional arguments (which usually + # ends up in a vague "too many arguments" error). To make tclint interpret a + # switch-like word as a positional argument, users should wrap it in "", and + # any switches should be BareWords. + contents = arg.contents + if not (isinstance(arg, BareWord) and contents and contents[0] in {"-", ">"}): + positional_args.append(arg) + continue + + # TODO check required arguments + if contents in args_allowed: + if switches[contents]["value"]: + try: + args.pop(0) + except IndexError: + raise CommandArgError( + f"invalid arguments for {command}: expected value after" + f" {contents}" + ) + if not switches[contents]["repeated"]: + args_allowed.remove(contents) + if contents in args_required: + args_required.remove(contents) + elif contents in arg_spec: + raise CommandArgError(f"duplicate argument for {command}: {contents}") + else: + prefix_matches = [] + for switch in switches: + if switch.startswith(contents): + prefix_matches.append(switch) + + if len(prefix_matches) == 1: + raise CommandArgError( + f"shortened argument for {command}: expand {contents} to" + f" {prefix_matches[0]}" + ) + + if len(prefix_matches) > 1: + raise CommandArgError( + f"ambiguous argument for {command}: {contents} could be any of" + f" {', '.join(prefix_matches)}" + ) + + raise CommandArgError(f"unrecognized argument for {command}: {contents}") + + if len(args_required) > 1: + raise CommandArgError( + f"missing required arguments for {command}: {', '.join(args_required)}" + ) + elif len(args_required) == 1: + raise CommandArgError( + f"missing required argument for {command}: {args_required.pop()}" + ) + + min_positionals = 0 + max_positionals: Optional[int] = 0 + for positional in arg_spec["positionals"]: + if positional["value"]["type"] == "variadic": + max_positionals = None + + if positional["required"]: + min_positionals += 1 + if max_positionals is not None: + max_positionals += 1 + + check = check_count( + command, + min=min_positionals, + max=max_positionals, + args_name="positional args", + ) + check(positional_args, None) + + return None diff --git a/server/src/tools/commands/__init__.py b/server/src/tools/commands/__init__.py new file mode 100644 index 0000000..1202114 --- /dev/null +++ b/server/src/tools/commands/__init__.py @@ -0,0 +1,37 @@ +import pathlib +from typing import List, Dict, Union + +from tools.commands import builtin as _builtin +from tools.commands.plugins import PluginManager + +# import to expose in package +from tools.commands.checks import CommandArgError + +__all__ = ["CommandArgError", "validate_command_plugins", "get_commands"] + + +def validate_command_plugins(plugins: List[str]) -> List[str]: + valid_plugins = [] + for plugin in set(plugins): + if PluginManager.load(plugin) is not None: + valid_plugins.append(plugin) + + return valid_plugins + + +def get_commands(plugins: List[Union[str, pathlib.Path]]) -> Dict: + commands = {} + commands.update(_builtin.commands) + + for plugin in plugins: + if isinstance(plugin, str): + plugin_commands = PluginManager.load(plugin) + elif isinstance(plugin, pathlib.Path): + plugin_commands = PluginManager.load_from_spec(plugin) + else: + raise TypeError(f"Plugins must be strings or paths, got {type(plugin)}") + + if plugin_commands is not None: + commands.update(plugin_commands) + + return commands diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py index 3a5d5b9..abf6553 100644 --- a/server/src/tools/parser.py +++ b/server/src/tools/parser.py @@ -1,9 +1,854 @@ +from tools.lexer import Lexer, TclSyntaxError, Tok, TOK_EOF +from tools import syntax_tree as st +from tools.commands import CommandArgError, get_commands +from tools.checks import check_command + + +def _strip_ws(parse_func): + """Decorator used by expression parser for stripping whitespace around a node.""" + + def func(parser, ts): + while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE, Tok.TOK_NEWLINE}: + ts.next() + + node = parse_func(parser, ts) + + while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE, Tok.TOK_NEWLINE}: + ts.next() + + return node + + return func + + class _Word: def __init__(self): - self.segements = [] + self.segments = [] self.current_segment = "" self.current_start = None def add_tok(self, tok): if self.current_start is None: self.current_start = tok.value[1] + + def add_node(self, node): + if self.current_segment != "": + self.segments.append( + st.BareWord( + self.current_segment, pos=self.current_start, end_pos=node.pos + ) + ) + self.current_segment = "" + self.current_start = None + self.segments.append(node) + + def resolve(self, end_pos): + if self.current_segment: + self.segments.append( + st.BareWord( + self.current_segment, pos=self.current_start, end_pos=end_pos + ) + ) + + return self.segments + + +class Parser: + def __init__(self, debug=False, command_plugins=None): + self._debug = debug + self._debug_indent = 0 + self.violations = [] + if command_plugins is None: + command_plugins = [] + self._commands = get_commands(command_plugins) + + def debug(self, *msg): + if self._debug: + print(" " * self._debug_indent, end="") + print(*msg) + + def parse(self, script, pos=None): + lexer = Lexer(pos=pos) + lexer.input(script) + tree = self._parse_script(lexer, in_command_sub=False) + assert lexer.type() == TOK_EOF, ( + "Didn't reach EOF parsing script, please file a bug report." + ) + + return tree + + def _parse_command_args(self, routine, args): + """Since many built-in Tcl commands take in Tcl scripts or expressions + as arguments, building a complete parse tree requires checking command + names and possibly parsing their arguments. + + The node of any argument that gets parsed by this method is replaced with + the parse tree of that argument. Since this process requires checking + the arguments provided to these commands, this method may report lint + violations. + + This parsing process is analogous to how the Tcl interpreter interprets + scripts, and better handles weird edge cases compared to a traditional + parsing technique. For example, this may look like valid Tcl: + + proc foo {a} { + # output } + puts "}" + } + + But really, it is invalid since the } in the comment terminates the body + of the proc - Tcl blindly constructs the body of the proc until it + reaches the first }. tclint handles this correctly. + """ + if routine not in self._commands: + return args + + spec = self._commands[routine] + + try: + new_args = check_command(routine, args, self, spec) + except TclSyntaxError as e: + raise e + except CommandArgError as e: + raise e + except Exception: + if self._debug: + raise + raise CommandArgError( + f"error parsing command arguments, possibly malformed {routine} command" + ) + + if new_args is None: + return args + + return new_args + + def parse_script(self, node): + if node.contents is None: + raise CommandArgError( + "expected braced word or word without substitutions in argument" + " interpreted as script" + ) + + script = self.parse(node.contents, pos=node.contents_pos) + if isinstance(node, BracedWord): + script.braced = True + + script.line = node.line + script.col = node.col + script.end_pos = node.end_pos + + return script + + def _parse_script(self, ts, in_command_sub): + self.debug(f"parse_script({ts.current})") + self._debug_indent += 1 + pos = ts.pos() + + if in_command_sub: + script = CommandSub(pos=pos) + else: + script = Script(pos=pos) + + while ts.type() is not TOK_EOF: + if ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE}: + # strip whitespace at start of command + ts.next() + continue + + if ts.type() == TOK_HASH: + script.add(self.parse_comment(ts)) + else: + cmd = self.parse_command(ts, in_command_sub=in_command_sub) + if cmd is not None: + script.add(cmd) + + # when in command sub mode, a script is terminated by ] + if in_command_sub and ts.type() == TOK_RBRACKET: + return script + + ts.expect( + TOK_EOF, + TOK_NEWLINE, + TOK_SEMI, + message=f"expected newline or semicolon, got {ts.value()}", + pos=ts.pos(), + ) + + if in_command_sub and ts.type() is TOK_EOF: + raise TclSyntaxError( + "reached EOF without finding end of command substitution", pos, ts.pos() + ) + + self._debug_indent -= 1 + + script.end_pos = ts.pos() + + return script + + def parse_comment(self, ts): + self.debug(f"parse_comment({ts.current})") + pos = ts.pos() + + ts.assert_(TOK_HASH) + + value = "" + while ts.type() not in {TOK_NEWLINE, TOK_EOF}: + value += ts.value() + ts.next() + + # Stripping trailing whitespace from comments here allows tclfmt to clean it up + # without affecting the AST. + value = value.rstrip() + + return Comment(value, pos=pos, end_pos=ts.pos()) + + def parse_command(self, ts, in_command_sub): + self.debug(f"parse_command({ts.current})") + self._debug_indent += 1 + pos = ts.pos() + + routine = self.parse_word(ts, in_command_sub) + if routine is None: + self._debug_indent -= 1 + return None + + args = [] + while True: + if ts.type() not in {TOK_WS, TOK_BACKSLASH_NEWLINE}: + break + + while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE}: + ts.next() + + word = self.parse_word(ts, in_command_sub) + if word is None: + break + + args.append(word) + + self._debug_indent -= 1 + + try: + parsed_args = self._parse_command_args(routine.contents, args) + except CommandArgError as e: + self.violations.append(Violation(Rule.COMMAND_ARGS, str(e), pos, ts.pos())) + parsed_args = args + + children = [routine, *parsed_args] + # We need to inherit end pos of last child to prevent us from counting + # extra whitespace at end of command, which is important for + # spaces-in-braces check. + return Command(*children, pos=pos, end_pos=children[-1].end_pos) + + def parse_word(self, ts, in_command_sub): + self.debug(f"parse_word({ts.current})") + if ts.type() == TOK_ARG_EXPANSION: + return self.parse_arg_expansion(ts, in_command_sub) + elif ts.type() == TOK_LBRACE: + return self.parse_braced_word(ts) + elif ts.type() == TOK_QUOTE: + return self.parse_quoted_word(ts) + else: + return self.parse_bare_word(ts, in_command_sub) + + def parse_arg_expansion(self, ts, in_command_sub): + self.debug(f"parse_arg_expansion({ts.current})") + pos = ts.pos() + + ts.assert_(TOK_ARG_EXPANSION) + + delimiters = [TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE, TOK_SEMI, TOK_EOF] + if in_command_sub: + delimiters.append(TOK_RBRACKET) + + # Arg expansion is just a regular braced word if followed by whitespace, + # or other word boundaries such as semicolon or right bracket + # (in command substitution) + if ts.type() in delimiters: + return BracedWord("*", pos=pos, end_pos=ts.pos()) + + return ArgExpansion( + self.parse_word(ts, in_command_sub), pos=pos, end_pos=ts.pos() + ) + + def parse_quoted_word(self, ts): + self.debug(f"parse_quoted_word({ts.current})") + self._debug_indent += 1 + pos = ts.pos() + + ts.assert_(TOK_QUOTE) + + word = _Word() + while ts.type() not in {TOK_QUOTE, TOK_EOF}: + if ts.type() == TOK_DOLLAR: + dollar_tok = ts.current + var_sub = self.parse_var_sub(ts) + if var_sub: + word.add_node(var_sub) + else: + word.add_tok(dollar_tok) + elif ts.type() == TOK_LBRACKET: + command_sub = self.parse_command_sub(ts) + word.add_node(command_sub) + else: + word.add_tok(ts.current) + ts.next() + + res = word.resolve(ts.pos()) + + ts.expect( + TOK_QUOTE, message="reached EOF without finding match for quote", pos=pos + ) + + self._debug_indent -= 1 + + if not res: + res = [] + + return QuotedWord(*res, pos=pos, end_pos=ts.pos()) + + def parse_braced_word(self, ts): + self.debug(f"parse_braced_word({ts.current})") + pos = ts.pos() + + ts.lexer.push_state(STATE_BRACEDWORD) + + ts.assert_(TOK_LBRACE) + + word = "" + # store position for each brace we want to match, facilitating good + # error messages + expected_braces = [pos] + while True: + toktype = ts.type() + if toktype == TOK_EOF: + raise TclSyntaxError( + "reached EOF without finding match for brace", + expected_braces[-1], + ts.pos(), + ) + + if toktype == TOK_LBRACE: + expected_braces.append(ts.pos()) + elif toktype == TOK_RBRACE: + try: + expected_braces.pop() + except IndexError: + start = ts.pos() + ts.next() + end = ts.pos() + raise TclSyntaxError( + "found closing brace without matching open brace", start, end + ) + + if len(expected_braces) == 0: + ts.lexer.pop_state() + ts.next() + break + word += ts.value() + ts.next() + + end_pos = ts.pos() + return BracedWord(word, pos=pos, end_pos=end_pos) + + def parse_bare_word(self, ts, in_command_sub): + self.debug(f"parse_bare_word({ts.current})") + self._debug_indent += 1 + pos = ts.pos() + + word = _Word() + delimiters = [TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE, TOK_SEMI, TOK_EOF] + + # In command sub mode, words are ended by ] + if in_command_sub: + delimiters.append(TOK_RBRACKET) + + while ts.type() not in delimiters: + if ts.type() == TOK_DOLLAR: + dollar_tok = ts.current + var_sub = self.parse_var_sub(ts) + if var_sub: + word.add_node(var_sub) + else: + word.add_tok(dollar_tok) + elif ts.type() == TOK_LBRACKET: + command_sub = self.parse_command_sub(ts) + word.add_node(command_sub) + else: + word.add_tok(ts.current) + ts.next() + + res = word.resolve(ts.pos()) + + self._debug_indent -= 1 + + if not res: + return None + if len(res) == 1: + return res[0] + return CompoundBareWord(*res, pos=pos, end_pos=ts.pos()) + + def parse_var_sub(self, ts): + self.debug(f"parse_var_sub({ts.current})") + pos = ts.pos() + + ts.assert_(TOK_DOLLAR) + + var = "" + if ts.type() == TOK_LBRACE: + brace_pos = ts.pos() + ts.next() + while ts.type() != TOK_RBRACE: + if ts.type() is TOK_EOF: + raise TclSyntaxError( + "reached EOF without finding match for brace", + brace_pos, + ts.pos(), + ) + var += ts.value() + ts.next() + ts.next() + + return VarSub(var, pos=pos, end_pos=ts.pos(), braced=True) + + while ts.type() in {TOK_ALPHA_CHARS, TOK_NUM_CHARS, TOK_NAMESPACE_SEP}: + var += ts.value() + ts.next() + + if not var: + return None + + index_nodes = [] + if ts.type() == TOK_LPAREN: + paren_pos = ts.pos() + index = _Word() + ts.next() + while ts.type() != TOK_RPAREN: + if ts.type() == TOK_EOF: + raise TclSyntaxError( + "reached EOF without finding match for paren", + paren_pos, + ts.pos(), + ) + if ts.type() == TOK_DOLLAR: + dollar_tok = ts.current + var_sub = self.parse_var_sub(ts) + if var_sub: + index.add_node(var_sub) + else: + index.add_tok(dollar_tok) + elif ts.type() == TOK_LBRACKET: + command_sub = self.parse_command_sub(ts) + index.add_node(command_sub) + else: + index.add_tok(ts.current) + ts.next() + + index_nodes = index.resolve(ts.pos()) + ts.next() + + var_sub = VarSub(var, pos=pos, end_pos=ts.pos()) + + for index_segment in index_nodes: + var_sub.add(index_segment) + + return var_sub + + def parse_command_sub(self, ts): + self.debug(f"parse_command_sub({ts.current})") + self._debug_indent += 1 + + pos = ts.pos() + ts.assert_(TOK_LBRACKET) + + script = self._parse_script(ts, in_command_sub=True) + + ts.assert_(TOK_RBRACKET) + end_pos = ts.pos() + + script.line = pos[0] + script.col = pos[1] + script.end_pos = end_pos + + self._debug_indent -= 1 + return script + + def parse_list(self, node): + """Parse contents of node as Tcl list. This is a distinct entry point + that doesn't get used when generating the main syntax tree, but is used + in command-specific argument parsing. + """ + if isinstance(node, List): + return node + + if node.contents is None: + raise CommandArgError( + "expected braced word or word without substitutions in argument" + " interpreted as list" + ) + + ts = Lexer(pos=node.contents_pos) + ts.input(node.contents) + + DELIMITERS = {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE} + + list_node = List(pos=node.pos, end_pos=node.end_pos) + while ts.type() is not TOK_EOF: + while ts.type() in DELIMITERS: + ts.next() + + if ts.type() is TOK_EOF: + break + + if ts.type() == TOK_LBRACE: + # we can reuse parse_braced_word, since it doesn't use + # substitutions in any case + list_node.add(self.parse_braced_word(ts)) + elif ts.type() == TOK_QUOTE: + quote_word_pos = ts.pos() + + ts.assert_(TOK_QUOTE) + + bare_word_pos = ts.pos() + contents = "" + while ts.type() not in {TOK_QUOTE, TOK_EOF}: + contents += ts.value() + ts.next() + word = BareWord(contents, pos=bare_word_pos, end_pos=ts.pos()) + + ts.expect( + TOK_QUOTE, + message="reached EOF without finding match for quote", + pos=quote_word_pos, + ) + + list_node.add(QuotedWord(word, pos=quote_word_pos, end_pos=ts.pos())) + else: + pos = ts.pos() + contents = "" + while ts.type() not in {*DELIMITERS, TOK_EOF}: + contents += ts.value() + ts.next() + list_node.add(BareWord(contents, pos=pos, end_pos=ts.pos())) + + return list_node + + def parse_expression(self, node): + if node.contents is None: + raise CommandArgError( + "expected braced word or word without substitutions in argument" + " interpreted as expr" + ) + + ts = Lexer(pos=node.contents_pos) + ts.input(node.contents) + + contents = self._parse_expression(ts) + ts.expect( + TOK_EOF, + message=f"expected end of expression, got {ts.value()}", + pos=ts.pos(), + ) + if isinstance(node, BracedWord): + return BracedExpression(contents, pos=node.pos, end_pos=node.end_pos) + + return Expression(contents, pos=node.pos, end_pos=node.end_pos) + + @_strip_ws + def _parse_expression(self, ts): + op1 = self._parse_operand(ts) + expr = op1 + + # last condition is hack to break out of expression in case we're in ternary op + if ts.type() not in {TOK_EOF, TOK_RPAREN} and ts.value() not in {":", ","}: + if ts.value() == "?": + # weird hack to record operator + start = ts.pos() + ts.next() + q = BareWord("?", pos=start, end_pos=ts.pos()) + + op2 = self._parse_expression(ts) + if ts.value() != ":": + start = ts.pos() + ts.next() + end = ts.pos() + raise TclSyntaxError( + "expected ':' to continue ternary expression", start, end + ) + + # weird hack again + start = ts.pos() + ts.next() + colon = BareWord(":", pos=start, end_pos=ts.pos()) + + op3 = self._parse_expression(ts) + expr = TernaryOp( + op1, q, op2, colon, op3, pos=op1.pos, end_pos=op3.end_pos + ) + else: + operator = self._parse_operator(ts) + op2 = self._parse_expression(ts) + expr = BinaryOp(op1, operator, op2, pos=op1.pos, end_pos=op2.end_pos) + + if ts.type() != TOK_RPAREN and ts.value() not in {":", ","}: + ts.expect(TOK_EOF, message="expected end of expression", pos=ts.pos()) + + return expr + + @_strip_ws + def _parse_operand(self, ts): + if ts.type() == TOK_DOLLAR: + return self.parse_var_sub(ts) + if ts.type() == TOK_QUOTE: + return self.parse_quoted_word(ts) + if ts.type() == TOK_LBRACE: + return self.parse_braced_word(ts) + if ts.type() == TOK_LBRACKET: + return self.parse_command_sub(ts) + if ts.type() == TOK_LPAREN: + start = ts.pos() + ts.next() + expr = self._parse_expression(ts) + ts.expect( + TOK_RPAREN, + message="reached EOF without finding match for paren", + pos=expr.pos, + ) + end = ts.pos() + return ParenExpression(expr, start, end) + if ts.value() in {"-", "+", "~", "!"}: + operator_val = ts.value() + operator_pos = ts.pos() + ts.next() + operator = BareWord(operator_val, pos=operator_pos, end_pos=ts.pos()) + operand = self._parse_operand(ts) + # Since _parse_operand() munches whitespace after the operand, we + # set the end of the UnaryOp to the end of the operand rather than + # ts.pos(). Otherwise, the bounds of the UnaryOp would include all + # that whitespace. + return UnaryOp(operator, operand, pos=operator_pos, end_pos=operand.end_pos) + + # If none of these, collect tokens that may comprise an operand + operand = "" + operand_pos = ts.pos() + + # First, we want to check for numeric operands (either ints or numeric + # floats) by consuming tokens as long as they comprise the prefix of a + # numeric operand + while ts.type() != TOK_EOF and ( + _is_int_prefix(operand + ts.value()) + or _is_float_prefix(operand + ts.value()) + ): + operand += ts.value() + ts.next() + + # Next, we check if we've consumed an entire numeric literal. If so, we + # move on. If not, we keep consuming tokens that may correspond to a + # valid bareword (pretty much just alphanumeric chars). + if not (_is_int_literal(operand) or _is_float_literal(operand)): + while ts.type() in {TOK_ALPHA_CHARS, TOK_NUM_CHARS}: + operand += ts.value() + ts.next() + + # The above method is a little hacky. Note that it doesn't parse things + # exactly the same as Tcl. E.g. if a script includes `expr {1foo}`, + # tclint will report an invalid operator "foo", whereas tclsh will + # report an invalid bareword "1foo". Despite reporting them differently + # both tools should still catch the same syntax errors, since there are + # no legal barewords that begin with a numeric literal prefix, and tclsh + # will stop parsing numeric operands if they're actually followed by a + # legal operator (e.g. `expr {1eq1}` will be handled properly). + + is_func = _is_function(operand) + + if not ( + _is_int_literal(operand) + or _is_float_literal(operand) + or _is_bool_literal(operand) + or is_func + ): + raise TclSyntaxError( + f"invalid bareword in expression: {operand}", operand_pos, ts.pos() + ) + + node = BareWord(operand, pos=operand_pos, end_pos=ts.pos()) + + if is_func: + node = self._parse_function(ts, node) + + return node + + def _parse_operator(self, ts): + pos = ts.pos() + + # hacky logic to handle parsing legal operators + + if ts.value() in {"*", "&", "|"}: + # one or two of these characters are legal operators + operator = ts.value() + ts.next() + if ts.value() == operator: + operator += ts.value() + ts.next() + elif ts.value() in {"<", ">"}: + operator = ts.value() + ts.next() + if ts.value() in {operator, "="}: + operator += ts.value() + ts.next() + elif ts.value() in {"=", "!"}: + operator = ts.value() + ts.next() + if ts.value() != "=": + raise TclSyntaxError( + f"invalid operator in expression: {operator}", pos, ts.pos() + ) + operator += ts.value() + ts.next() + elif ts.value() in {"*", "/", "%", "+", "-", "^", "eq", "ne", "in", "ni"}: + operator = ts.value() + ts.next() + else: + raise TclSyntaxError( + f"invalid operator in expression: {ts.value()}", pos, ts.pos() + ) + + return BareWord(operator, pos=pos, end_pos=ts.pos()) + + def _parse_function(self, ts, name): + while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE}: + ts.next() + + ts.expect( + TOK_LPAREN, + message="expected open paren after function name", + pos=name.pos, + ) + + delims = {Tok.TOK_RPAREN, TOK_EOF} + + arguments = [] + if ts.type() not in delims: + arguments.append(self._parse_expression(ts)) + + while ts.type() not in delims: + if ts.value() != ",": + start = ts.pos() + ts.next() + end = ts.pos() + raise TclSyntaxError( + "expected comma between function arguments", start, end + ) + ts.next() + + arguments.append(self._parse_expression(ts)) + + ts.expect( + Tok.TOK_RPAREN, + message="expected close paren after function arguments", + pos=name.pos, + ) + return st.Function(name, *arguments, pos=name.pos, end_pos=ts.pos()) + + +def _all(_list, non_empty=False): + """Like all(), but if non_empty is True, list must also have at least 1 element.""" + if non_empty and len(_list) == 0: + return False + return all(_list) + + +def _is_int(operand, full=False): + """Returns whether operand is a valid Tcl integer literal. + + If full is False, will also return True if operand is the prefix of an + integer literal. An empty string is not a valid full literal, but is a + valid prefix. + """ + # prefixes + if operand.startswith("0b"): + return _all([digit in "01" for digit in operand[2:]], non_empty=full) + if operand.startswith("0o"): + return _all( + [digit in string.octdigits for digit in operand[2:]], non_empty=full + ) + if operand.startswith("0x"): + return _all( + [digit in string.hexdigits for digit in operand[2:]], non_empty=full + ) + if operand.startswith("0"): + # fun fact: apparently a lone 0 prefix is interpreted as octal + return _all( + [digit in string.octdigits for digit in operand[1:]], non_empty=full + ) + + return _all([digit in string.digits for digit in operand], non_empty=full) + + +def _is_int_literal(operand): + return _is_int(operand, full=True) + + +def _is_int_prefix(operand): + return _is_int(operand, full=False) + + +def _is_float_literal(operand): + if operand.lower() in {"nan", "inf"}: + return True + + return ( + operand != "" and re.fullmatch(r"\d*\.?\d*([Ee][+-]?\d+)?", operand) is not None + ) + + +def _is_float_prefix(operand): + """Returns whether operand is the prefix of a valid numeric float literal.""" + return re.fullmatch(r"\d*\.?\d*([Ee][+-]?)?\d*", operand) is not None + + +def _is_bool_literal(operand): + return operand in {"false", "no", "off", "true", "yes", "on"} + + +# map of function names to # arguments accepted +# None indicates 1 or more arguments +# TODO: use these values in an actual separate check. they might want to live elsewhere +_FUNCTIONS = { + "abs": 1, + "acos": 1, + "asin": 1, + "atan": 1, + "atan2": 2, + "bool": 1, + "ceil": 1, + "cos": 1, + "cosh": 1, + "double": 1, + "entier": 1, + "exp": 1, + "floor": 1, + "fmod": 2, + "hypot": 2, + "int": 1, + "isqrt": 1, + "log": 1, + "log10": 1, + "max": None, + "min": None, + "pow": 2, + "rand": 0, + "round": 1, + "sin": 1, + "sinh": 1, + "sqrt": 1, + "srand": 1, + "tan": 1, + "tanh": 1, + "wide": 1, +} + + +def _is_function(operand): + return operand in _FUNCTIONS.keys() diff --git a/server/src/tools/plugins.py b/server/src/tools/plugins.py new file mode 100644 index 0000000..927e874 --- /dev/null +++ b/server/src/tools/plugins.py @@ -0,0 +1,81 @@ +from importlib.metadata import entry_points +import json +import pathlib +from typing import Dict, Optional + + +class _PluginManager: + def __init__(self): + self._loaded = {} + self._installed = {} + self._loaded_specs = {} + for plugin in entry_points(group="tclint.plugins"): + if plugin.name in self._installed: + print(f"Warning: found duplicate definitions for plugin {plugin.name}") + self._installed[plugin.name] = plugin + + def load(self, name: str) -> Optional[Dict]: + if name in self._loaded: + return self._loaded[name] + + mod = self._load(name) + self._loaded[name] = mod + return mod + + def load_from_spec(self, path: pathlib.Path) -> Optional[Dict]: + if path in self._loaded_specs: + return self._loaded_specs[path] + + spec = self._load_from_spec(path) + self._loaded_specs[path] = spec + return spec + + def _load_from_spec(self, path: pathlib.Path) -> Optional[Dict]: + try: + with open(path.expanduser(), "r") as f: + spec = json.load(f) + except (FileNotFoundError, RuntimeError): + print(f"Warning: command spec {path} not found, skipping...") + return None + + try: + # Apply defaults and validate the spec. + spec = command_schema(spec) + except voluptuous.Invalid as e: + print(f"Warning: invalid command spec {path}: {e}") + return None + + return spec["commands"] + + def get_mod(self, name: str) -> Optional[ModuleType]: + if name not in self._installed: + print(f"Warning: plugin {name} is not installed") + return None + + plugin = self._installed[name] + + try: + module = plugin.load() + except Exception as e: + print(f"Warning: error loading plugin {name}: {e}") + return None + + return module + + def _load(self, name: str): + module = self.get_mod(name) + if module is None: + print(f"Skipping requested plugin {name}") + return None + + if not hasattr(module, "commands"): + print(f"Warning: skipping plugin {name} since it does not define commands") + return None + + return getattr(module, "commands") + + +# TODO: we'll probably want to construct this in the tclint entry point and pass +# it around rather than using a singleton instance, but this made for an easier +# refactor. +PluginManager = _PluginManager() diff --git a/server/src/tools/schema.py b/server/src/tools/schema.py new file mode 100644 index 0000000..7662a10 --- /dev/null +++ b/server/src/tools/schema.py @@ -0,0 +1,35 @@ +from collections.abc import Callable +from voluptuous import Schema, Optional, Or, Self + +# Need to define this as a Schema with required=True to ensure that this requirement +# persists through the Or in the main schema definition. +_command_args = Schema( + { + Optional("positionals", default=[]): [ + { + "name": str, + "required": bool, + "value": Or({"type": "any"}, {"type": "variadic"}), + } + ], + Optional("switches", default={}): { + Optional(str): { + "required": bool, + "repeated": bool, + "value": Or({"type": "any"}, None), + Optional("metavar"): str, + } + }, + }, + required=True, +) + +commands_schema = Schema( + {Optional(str): Or(_command_args, None, {"subcommands": Self}, Callable)}, + required=True, +) + +schema = Schema( + {"name": str, "commands": commands_schema}, + required=True, +) diff --git a/server/src/tools/syntax_tree.py b/server/src/tools/syntax_tree.py index d77de03..97f646c 100644 --- a/server/src/tools/syntax_tree.py +++ b/server/src/tools/syntax_tree.py @@ -57,3 +57,382 @@ class Visitor: def visit_function(self, function): pass + + +class Node: + """ + Invariants: + - self.value is some sort of base Python type + - self.children is a list of Node types + """ + + def __init__(self, *init, pos=None, end_pos=None): + """pos: line, column of first character of parsed region (1-indexed) + end_pos: line, column of first character after parsed region (1-indexed) + """ + self.line = None + self.col = None + if pos is not None: + self.line, self.col = pos + self.end_pos = end_pos + + self.value = None + if len(init) > 0 and not isinstance(init[0], Node): + self.value = init[0] + init = init[1:] + + if not all(isinstance(v, Node) for v in init): + raise TypeError("Children must be Node instances") + + self.children = list(init) + + def add(self, node): + self.children.append(node) + + @property + def contents(self): + """This is overloaded by word Nodes that may have concrete contents. + + TODO: I prefer the name value, but that's currently taken... + """ + return None + + @property + def contents_pos(self): + """This is overloaded by word Nodes that may have concrete contents. + + Returns the position at which the contents start. + + TODO: consider combining with with `contents`? + """ + return None + + def _pos_str(self): + start_pos_str = "?" + if self.pos is not None: + start_pos_str = f"{self.pos[0]}:{self.pos[1]}" + end_pos_str = "?" + if self.end_pos is not None: + end_pos_str = f"{self.end_pos[0]}:{self.end_pos[1]}" + + return f" # {start_pos_str}-{end_pos_str}" + + def _make_str(self, indent=None, positions=False): + if indent is not None: + s = " " * indent + else: + s = "" + + s += self.__class__.__name__ + s += "(" + + if self.value: + s += repr(self.value) + if self.children: + s += ", " + + if positions and self.children: + s += self._pos_str() + + for i, child in enumerate(self.children): + if indent is not None: + s += "\n" + s += child._make_str( + indent=None if indent is None else indent + 1, positions=positions + ) + if i < len(self.children) - 1: + s += ", " + s += ")" + + if positions and not self.children: + s += self._pos_str() + + return s + + def pretty(self, positions=False): + return self._make_str(indent=0, positions=positions) + + def __str__(self): + return self._make_str() + + def __eq__(self, other): + if type(self) is not type(other): + return False + + if self.value != other.value: + return False + + if len(self.children) != len(other.children): + return False + for my_child, other_child in zip(self.children, other.children): + if my_child != other_child: + return False + + return True + + def diff(self, other, indent_depth=0): + lines = [] + indent = " " * indent_depth + + my_cls = self.__class__.__name__ + other_cls = other.__class__.__name__ + + if my_cls != other_cls: + lines += [f"{indent}-{my_cls}("] + lines += [f"{indent}+{other_cls}("] + return lines + + if self.value != other.value: + lines += [f'{indent}-{my_cls}("{self.value}"'] + lines += [f'{indent}+{other_cls}("{other.value}"'] + return lines + + if len(self.children) != len(other.children): + my_children = ",".join( + [child.__class__.__name__ for child in self.children] + ) + other_children = ",".join( + [child.__class__.__name__ for child in other.children] + ) + + lines += [f"{indent}-{my_cls}({my_children})"] + lines += [f"{indent}+{other_cls}({other_children})"] + return lines + + if self.value is not None: + lines += [f"{indent}{my_cls}({self.value}"] + else: + lines += [f"{indent}{my_cls}("] + + for my_child, other_child in zip(self.children, other.children): + lines += my_child.diff(other_child, indent_depth=indent_depth + 1) + + lines += [f"{indent})"] + + return lines + + @property + def pos(self): + if self.line is None or self.col is None: + return None + + return (self.line, self.col) + + def _recurse(self, visitor): + for child in self.children: + child.accept(visitor, recurse=True) + + +class Script(Node): + def __init__(self, *args, **kwargs): + super().__init__(*args, **kwargs) + # hack for spaces-in-braces check + self.braced = False + + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_script(self) + + +class Comment(Node): + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_comment(self) + + +class Command(Node): + def __init__(self, routine, *args, pos=None, end_pos=None): + self.routine = routine + self.args = args + super().__init__(routine, *args, pos=pos, end_pos=end_pos) + + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_command(self) + + +class CommandSub(Node): + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_command_sub(self) + + +class BareWord(Node): + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_bare_word(self) + + @property + def contents(self): + return self.value + + @property + def contents_pos(self): + return self.pos + + +class BracedWord(Node): + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_braced_word(self) + + @property + def contents(self): + return self.value + + @property + def contents_pos(self): + return (self.line, self.col + 1) + + +class QuotedWord(Node): + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_quoted_word(self) + + @property + def contents(self): + """A QuotedWord with concrete contents is put in the tree as + QuotedWord(BareWord()), so return contents of its child.""" + if len(self.children) > 1: + return None + if len(self.children) == 0: + # weird special case to handle blank quoted string ("") - it doesn't + # make sense to put in a child, but setting/returning the value is + # also inconsistent + return "" + + return self.children[0].contents + + @property + def contents_pos(self): + if self.contents is None: + return None + return (self.line, self.col + 1) + + +class CompoundBareWord(Node): + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_compound_bare_word(self) + + +class VarSub(Node): + def __init__(self, *args, braced=False, **kwargs): + self.braced = braced + return super().__init__(*args, **kwargs) + + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_var_sub(self) + + +class ArgExpansion(Node): + def __init__(self, list, pos=None, end_pos=None): + self.list = list + super().__init__(list, pos=pos, end_pos=end_pos) + + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_arg_expansion(self) + + +class List(Node): + """This Node currently exists exclusively for implementing the switch + command in a way that facilitates style checks. Might be nice to find + another way to handle this that doesn't require a special Node.""" + + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_list(self) + + +class Expression(Node): + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_expression(self) + + +class BracedExpression(Node): + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_braced_expression(self) + + +class ParenExpression(Node): + def __init__(self, body: Expression, pos=None, end_pos=None): + self.body = body + super().__init__(body, pos=pos, end_pos=end_pos) + + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_paren_expression(self) + + +class UnaryOp(Node): + def __init__(self, operator, operand, pos=None, end_pos=None): + self.operator = operator + self.operand = operand + super().__init__(operator, operand, pos=pos, end_pos=end_pos) + + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_unary_op(self) + + +class BinaryOp(Node): + def __init__(self, operator1, operand, operator2, pos=None, end_pos=None): + self.operator1 = operator1 + self.operand = operand + self.operator2 = operator2 + super().__init__(operator1, operand, operator2, pos=pos, end_pos=end_pos) + + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_binary_op(self) + + +class TernaryOp(Node): + def __init__(self, condition, question, true, colon, false, pos=None, end_pos=None): + self.condition = condition + self.question = question + self.true = true + self.colon = colon + self.false = false + + super().__init__( + condition, question, true, colon, false, pos=pos, end_pos=end_pos + ) + + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_ternary_op(self) + + +class Function(Node): + def __init__(self, name, *args, pos=None, end_pos=None): + self.name = name + self.args = args + super().__init__(name, *args, pos=pos, end_pos=end_pos) + + def accept(self, visitor, recurse=False): + if recurse: + self._recurse(visitor) + visitor.visit_function(self)