From c34dac847a9a452be5514c5def1bfa26d1376472 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Fri, 25 Jul 2025 16:02:39 +0200 Subject: [PATCH] update parser --- server/src/test_tcl.py | 10 + server/src/tools/__init__.py | 0 server/src/tools/checks.py | 227 ------------ server/src/tools/commands/__init__.py | 28 +- server/src/tools/commands/builtin.py | 6 +- server/src/tools/commands/checks.py | 2 +- server/src/tools/commands/plugins.py | 86 ----- server/src/tools/comments.py | 91 ----- server/src/tools/config.py | 429 ----------------------- server/src/tools/format.py | 480 -------------------------- server/src/tools/lexer.py | 92 ++--- server/src/tools/parser.py | 200 ++++++----- server/src/tools/plugins.py | 81 ----- server/src/tools/schema.py | 35 -- server/src/tools/syntax_tree.py | 18 +- server/src/tools/violations.py | 50 --- 16 files changed, 168 insertions(+), 1667 deletions(-) create mode 100644 server/src/test_tcl.py delete mode 100644 server/src/tools/__init__.py delete mode 100644 server/src/tools/checks.py delete mode 100644 server/src/tools/commands/plugins.py delete mode 100644 server/src/tools/comments.py delete mode 100644 server/src/tools/config.py delete mode 100644 server/src/tools/format.py delete mode 100644 server/src/tools/plugins.py delete mode 100644 server/src/tools/schema.py delete mode 100644 server/src/tools/violations.py diff --git a/server/src/test_tcl.py b/server/src/test_tcl.py new file mode 100644 index 0000000..89cb451 --- /dev/null +++ b/server/src/test_tcl.py @@ -0,0 +1,10 @@ +from tools.parser import Parser + + +def main(): + parser = Parser(True) + parser.parse("puts hello") + + +if __name__ == "__main__": + main() diff --git a/server/src/tools/__init__.py b/server/src/tools/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/server/src/tools/checks.py b/server/src/tools/checks.py deleted file mode 100644 index cc36ae4..0000000 --- a/server/src/tools/checks.py +++ /dev/null @@ -1,227 +0,0 @@ -import re - -from src.tools.commands import get_commands -from src.tools.violations import Rule, Violation - -from src.tools.syntax_tree import ( - Visitor, - BracedExpression, - Expression, - BracedWord, - QuotedWord, - CommandSub, -) - - -class LineLengthChecker: - """Ensures lines aren't too long. - - Reports 'line-length' violations. - """ - - # ref: https://github.com/eslint/eslint/blob/b29a16b22f234f6134475efb6c7be5ac946556ee/lib/rules/max-len.js#L101 # noqa: E501 - # ^ ironic lint waiver... - URL_RE = re.compile(r"[^:/?#]:\/\/[^?#]") - - def check(self, input, _, config): - violations = [] - for i, line in enumerate(input.split("\n")): - if self.URL_RE.search(line) is not None: - # ignore URLs - continue - - lineno = i + 1 - if len(line) > config.style_line_length: - start = (lineno, 1) - end = (lineno, len(line) + 1) - violations.append( - Violation( - Rule.LINE_LENGTH, - f"line length is {len(line)}, maximum allowed is" - f" {config.style_line_length}", - start, - end, - ) - ) - - return violations - - -class TrailingWhitespaceChecker: - """Ensures lines don't include trailing whitespace. - - Reports 'trailing-whitespace' violations. - """ - - def check(self, input, _, config): - violations = [] - for i, line in enumerate(input.split("\n")): - lineno = i + 1 - - WHITESPACE = (" ", "\t") - if line.endswith(WHITESPACE): - start_col = len(line.rstrip("".join(WHITESPACE))) - start = (lineno, start_col + 1) - end = (lineno, len(line) + 1) - violations.append( - Violation( - Rule.TRAILING_WHITESPACE, - "line has trailing whitespace", - start, - end, - ) - ) - - return violations - - -class RedefinedBuiltinChecker(Visitor): - """Ensures names of built-in commands aren't reused by proc definitions. - - Reports 'redefined-builtin' violations. - """ - - def check(self, _, tree, config): - self._violations = [] - - plugins = [config.commands] if config.commands is not None else [] - commands = get_commands(plugins) - self._commands = commands.keys() - - tree.accept(self, recurse=True) - - return self._violations - - def visit_command(self, command): - if command.routine.contents != "proc": - return - - if len(command.args) == 0: - # This is a syntax error, but should already be caught as a command-args - # error by the parser's `proc` command handling. - return - - name = command.args[0].contents - - if name in self._commands: - self._violations.append( - Violation( - Rule.REDEFINED_BUILTIN, - f"redefinition of built-in command '{name}'", - command.pos, - command.args[1].end_pos, - ) - ) - - -class UnbracedExprChecker(Visitor): - def check(self, _, tree, __): - self._violations = [] - tree.accept(self, recurse=True) - return self._violations - - def visit_command(self, command): - if command.routine.contents != "expr": - return - - if len(command.args) == 0: - # This is a syntax error, but should already be caught as a command-args - # error by the parser's `expr` command handling. - return - - if len(command.args) == 1 and isinstance( - command.args[0], (BracedExpression, Expression) - ): - return - - # If we got here, tclint had trouble parsing the expression due to one of the - # two following cases. - - for child in command.args: - if child.contents is None: - self._violations.append( - Violation( - Rule.UNBRACED_EXPR, - "expression with substitutions should be enclosed by braces", - command.args[0].pos, - command.args[-1].end_pos, - ) - ) - return - - for child in command.args: - if isinstance(child, (BracedWord, QuotedWord)): - self._violations.append( - Violation( - Rule.UNBRACED_EXPR, - "expression containing braced or quoted words should be" - " enclosed by braces", - command.args[0].pos, - command.args[-1].end_pos, - ) - ) - return - - # If we reach here, there's probably a bug in expr parsing logic. - assert False, ( - "Children of expr node were different than expected, please file a bug" - " report" - ) - - -class RedundantExprChecker(Visitor): - def check(self, _, tree, __): - self._violations = [] - tree.accept(self, recurse=True) - return self._violations - - def _check_operand(self, operand): - if not isinstance(operand, CommandSub) or len(operand.children) != 1: - return - - command = operand.children[0] - if command.routine.contents == "expr": - self._violations.append( - Violation( - Rule.REDUNDANT_EXPR, - "unnecessary command substitution within expression", - operand.pos, - operand.end_pos, - ) - ) - - def visit_braced_expression(self, expression): - if len(expression.children) == 1: - self._check_operand(expression.children[0]) - - def visit_expression(self, expression): - if len(expression.children) == 1: - self._check_operand(expression.children[0]) - - def visit_unary_op(self, expr): - self._check_operand(expr.children[1]) - - def visit_binary_op(self, expr): - self._check_operand(expr.children[0]) - self._check_operand(expr.children[2]) - - def visit_ternary_op(self, expr): - self._check_operand(expr.children[0]) - self._check_operand(expr.children[2]) - self._check_operand(expr.children[4]) - - def visit_function(self, function): - for arg in function.children[1:]: - self._check_operand(arg) - - -def get_checkers(): - checkers = ( - RedefinedBuiltinChecker(), - UnbracedExprChecker(), - RedundantExprChecker(), - LineLengthChecker(), - TrailingWhitespaceChecker(), - ) - - return checkers diff --git a/server/src/tools/commands/__init__.py b/server/src/tools/commands/__init__.py index f69a211..f498d2c 100644 --- a/server/src/tools/commands/__init__.py +++ b/server/src/tools/commands/__init__.py @@ -1,37 +1,17 @@ import pathlib from typing import List, Dict, Union -from src.tools.commands import builtin as _builtin -from src.tools.commands.plugins import PluginManager +from tools.commands import builtin as _builtin + # import to expose in package -from src.tools.commands.checks import CommandArgError +from tools.commands.checks import CommandArgError -__all__ = ["CommandArgError", "validate_command_plugins", "get_commands"] - - -def validate_command_plugins(plugins: List[str]) -> List[str]: - valid_plugins = [] - for plugin in set(plugins): - if PluginManager.load(plugin) is not None: - valid_plugins.append(plugin) - - return valid_plugins +__all__ = ["CommandArgError", "get_commands"] def get_commands(plugins: List[Union[str, pathlib.Path]]) -> Dict: commands = {} commands.update(_builtin.commands) - for plugin in plugins: - if isinstance(plugin, str): - plugin_commands = PluginManager.load(plugin) - elif isinstance(plugin, pathlib.Path): - plugin_commands = PluginManager.load_from_spec(plugin) - else: - raise TypeError(f"Plugins must be strings or paths, got {type(plugin)}") - - if plugin_commands is not None: - commands.update(plugin_commands) - return commands diff --git a/server/src/tools/commands/builtin.py b/server/src/tools/commands/builtin.py index b0aec14..ce12405 100644 --- a/server/src/tools/commands/builtin.py +++ b/server/src/tools/commands/builtin.py @@ -32,13 +32,13 @@ these would be helpful for your use case, please file an issue. - https://www.tcl.tk/man/tcl/TclCmd/mathop.html """ -from src.tools.commands.checks import ( +from tools.commands.checks import ( CommandArgError, check_count, eval, ) -from src.tools.commands.schema import commands_schema -from src.tools.syntax_tree import BareWord +from tools.commands.schema import commands_schema +from tools.syntax_tree import BareWord def _check_code(arg): diff --git a/server/src/tools/commands/checks.py b/server/src/tools/commands/checks.py index a325f59..558bb3d 100644 --- a/server/src/tools/commands/checks.py +++ b/server/src/tools/commands/checks.py @@ -3,7 +3,7 @@ from collections.abc import Callable from typing import List, Optional, Union -from src.tools.syntax_tree import ArgExpansion, QuotedWord, BracedWord, BareWord, Node +from tools.syntax_tree import ArgExpansion, QuotedWord, BracedWord, BareWord, Node class CommandArgError(Exception): diff --git a/server/src/tools/commands/plugins.py b/server/src/tools/commands/plugins.py deleted file mode 100644 index 9c41c5b..0000000 --- a/server/src/tools/commands/plugins.py +++ /dev/null @@ -1,86 +0,0 @@ -from importlib.metadata import entry_points -import json -import pathlib -from typing import Dict, Optional -from types import ModuleType - -import voluptuous - -from src.tools.commands.schema import schema as command_schema - - -class _PluginManager: - def __init__(self): - self._loaded = {} - self._installed = {} - self._loaded_specs = {} - for plugin in entry_points(group="tclint.plugins"): - if plugin.name in self._installed: - print(f"Warning: found duplicate definitions for plugin {plugin.name}") - self._installed[plugin.name] = plugin - - def load(self, name: str) -> Optional[Dict]: - if name in self._loaded: - return self._loaded[name] - - mod = self._load(name) - self._loaded[name] = mod - return mod - - def load_from_spec(self, path: pathlib.Path) -> Optional[Dict]: - if path in self._loaded_specs: - return self._loaded_specs[path] - - spec = self._load_from_spec(path) - self._loaded_specs[path] = spec - return spec - - def _load_from_spec(self, path: pathlib.Path) -> Optional[Dict]: - try: - with open(path.expanduser(), "r") as f: - spec = json.load(f) - except (FileNotFoundError, RuntimeError): - print(f"Warning: command spec {path} not found, skipping...") - return None - - try: - # Apply defaults and validate the spec. - spec = command_schema(spec) - except voluptuous.Invalid as e: - print(f"Warning: invalid command spec {path}: {e}") - return None - - return spec["commands"] - - def get_mod(self, name: str) -> Optional[ModuleType]: - if name not in self._installed: - print(f"Warning: plugin {name} is not installed") - return None - - plugin = self._installed[name] - - try: - module = plugin.load() - except Exception as e: - print(f"Warning: error loading plugin {name}: {e}") - return None - - return module - - def _load(self, name: str): - module = self.get_mod(name) - if module is None: - print(f"Skipping requested plugin {name}") - return None - - if not hasattr(module, "commands"): - print(f"Warning: skipping plugin {name} since it does not define commands") - return None - - return getattr(module, "commands") - - -# TODO: we'll probably want to construct this in the tclint entry point and pass -# it around rather than using a singleton instance, but this made for an easier -# refactor. -PluginManager = _PluginManager() diff --git a/server/src/tools/comments.py b/server/src/tools/comments.py deleted file mode 100644 index 293a997..0000000 --- a/server/src/tools/comments.py +++ /dev/null @@ -1,91 +0,0 @@ -from collections import defaultdict - -from src.tools.syntax_tree import Visitor -from src.tools.violations import ALL_RULES, Rule - - -class CommentVisitor(Visitor): - """Scans the tree for lint waiver comments.""" - - def __init__(self): - # line -> [rule] - self.ignore_lines = defaultdict(set) - - self._disable_regions = { - # rule -> line - } - - def run(self, tree, path): - self._path = path - tree.accept(self, recurse=True) - - # resolve remaining disabled regions - last_line = tree.end_pos[0] - for rule, start_line in self._disable_regions.items(): - for line in range(start_line, last_line + 1): - self.ignore_lines[line].add(rule) - - return self.ignore_lines - - def visit_comment(self, comment): - contents = comment.value.strip() - - if not contents.startswith("tclint-"): - return - - split = contents.split(" ", 1) - - command = split[0] - - rule_strs = [] - if len(split) > 1: - rest = split[-1] - rule_strs = rest.split("--", 1)[0] - rule_strs = rule_strs.replace(" ", "") - rule_strs = rule_strs.split(",") - - rules = [] - if not rule_strs: - # default if no rules specified is all violation types - rules = ALL_RULES - else: - for rule in rule_strs: - try: - rules.append(Rule(rule)) - except ValueError: - self._warning( - f"unknown rule '{rule}' provided to '{command}'", comment.pos - ) - - if command == "tclint-disable": - for rule in rules: - # if in dictionary, already disabled - this has no effect - if rule not in self._disable_regions: - self._disable_regions[rule] = comment.line - elif command == "tclint-disable-line": - line = comment.line - self.ignore_lines[line].update(rules) - elif command == "tclint-disable-next-line": - line = comment.line + 1 - self.ignore_lines[line].update(rules) - elif command == "tclint-enable": - for rule in rules: - if rule in self._disable_regions: - disable_start_line = self._disable_regions[rule] - disable_end_line = comment.line - - for line in range(disable_start_line, disable_end_line + 1): - self.ignore_lines[line].add(rule) - - del self._disable_regions[rule] - else: - self._warning( - f"comment starts with '{command}', which looks like a tclint keyword." - " Is this a typo?", - comment.pos, - ) - - def _warning(self, message, pos): - # TODO: formal warning mechanism - prefix = self._path if self._path is not None else "(stdin)" - print(f"Warning: {prefix}:{pos[0]}:{pos[1]}: {message}") diff --git a/server/src/tools/config.py b/server/src/tools/config.py deleted file mode 100644 index 87cf234..0000000 --- a/server/src/tools/config.py +++ /dev/null @@ -1,429 +0,0 @@ -import argparse -import pathlib -from typing import Union, List -from typing import Optional as OptionalType -import dataclasses -import sys - -if sys.version_info >= (3, 11): - import tomllib -else: - import tomli as tomllib - -from voluptuous import Schema, Optional, And, Coerce, Invalid, Range - -from src.tools.violations import Rule - - -@dataclasses.dataclass -class Config: - """This dataclass defines the supported Config fields and their default - values. It provides an external interface for accessing config values. - - The type annotations defined here are fairly loose - more specific type - validation (and normalization) is defined by `validators` below. - """ - - exclude: List[str] = dataclasses.field(default_factory=list) - ignore: List[Rule] = dataclasses.field(default_factory=list) - commands: OptionalType[pathlib.Path] = dataclasses.field(default=None) - extensions: List[str] = dataclasses.field( - default_factory=lambda: ["tcl", "sdc", "xdc", "upf"] - ) - style_indent: OptionalType[Union[str, int]] = dataclasses.field(default=None) - style_line_length: int = dataclasses.field(default=100) - style_max_blank_lines: int = dataclasses.field(default=2) - style_indent_namespace_eval: bool = dataclasses.field(default=True) - style_spaces_in_braces: bool = dataclasses.field(default=False) - - def apply_cli_args(self, args): - args_dict = vars(args) - for field in dataclasses.fields(self): - if field.name in args_dict and args_dict[field.name] is not None: - setattr(self, field.name, args_dict[field.name]) - - # Special arguments that aren't handled automatically - if "extend_exclude" in args_dict and args_dict["extend_exclude"] is not None: - self.exclude.extend(args_dict["extend_exclude"]) - - if "extend_ignore" in args_dict and args_dict["extend_ignore"] is not None: - self.ignore.extend(args_dict["extend_ignore"]) - - def get_indent(self) -> str: - """Get indent setting as string. - - This helper does two things. One, it's a helpful utility to factor out the logic - required for calculating the indent. Two, it lets us ergonomically store if the - indentation is not set in style_indent, which the LSP relies on. - """ - if self.style_indent is None: - # Default indent - return " " * 4 - elif self.style_indent == "tab": - return "\t" - elif isinstance(self.style_indent, int): - return " " * self.style_indent - - # Should be unreachable, validated on ingestion of config - raise ValueError( - f"unexpected value for config.style_indent: {self.style_indent}" - ) - - -# Validators using `voluptuous` library that check and normalize config inputs. -# Used for checking both config files as well as config-related CLI args. - -# Using these for CLI args adds a constraint that all non-boolean validators -# need to be able to normalize a value from a string. This means one could put -# e.g. a string representation of a list into a .toml config file, but we shouldn't -# document this, since it won't be considered stable behavior. - - -def _str2list(s): - """Handles string-to-list normalization.""" - if isinstance(s, str): - if s == "": - return [] - return [v.strip() for v in s.split(",")] - return s - - -_VALIDATORS = { - # note: it's ok if paths don't exist - allows for generic - # configurations with directories like .git/ excluded - "exclude": _str2list, - "ignore": And( - _str2list, - [ - Coerce(Rule, msg="invalid rule ID"), - ], - ), - "commands": Coerce(pathlib.Path), - "extensions": _str2list, - "style_indent": Coerce( - lambda v: v if v == "tab" else int(v), msg="expected integer or 'tab'" - ), - "style_line_length": Coerce(int), - "style_max_blank_lines": And( - Coerce(int), - # we could technically support i >= 0, but I think 0 would be a weird - # setting and this lets us ignore pluralizing the violation message :) - Range(min=1), - ), - "style_indent_namespace_eval": bool, - "style_spaces_in_braces": bool, -} - - -def _validate_config(config): - """Validates dictionary read from TOML config file. Individual value validators - are implemented in the global dict, this defines the actual structure of the - schema.""" - - base_config = { - Optional("ignore"): _VALIDATORS["ignore"], - Optional("commands"): _VALIDATORS["commands"], - Optional("style"): { - Optional("indent"): _VALIDATORS["style_indent"], - Optional("line-length"): _VALIDATORS["style_line_length"], - Optional("max-blank-lines"): _VALIDATORS["style_max_blank_lines"], - Optional("indent-namespace-eval"): _VALIDATORS[ - "style_indent_namespace_eval" - ], - Optional("spaces-in-braces"): _VALIDATORS["style_spaces_in_braces"], - }, - } - - schema = Schema( - { - # exclude and extensions can only be used in global context - Optional("exclude"): _VALIDATORS["exclude"], - Optional("extensions"): _VALIDATORS["extensions"], - **base_config, - Optional("fileset"): Schema( - [{"paths": [Coerce(pathlib.Path)], **base_config}], required=True - ), - } - ) - - try: - return schema(config) - except Invalid as e: - if not e.path: - raise ConfigError(e.error_message) - - # Stringify error path to my own taste. - path = [] - for item in e.path: - if isinstance(item, int): - # Brackets around indices - if len(path) > 0: - path[-1] += f"[{item}]" - else: - path.append(f"[{item}]") - else: - path.append(str(item)) - - raise ConfigError(f"{e.error_message} ({'.'.join(path)})") - - -def _validator(key): - def func(s): - try: - return Schema(_VALIDATORS[key])(s) - except Invalid as e: - raise argparse.ArgumentTypeError(str(e)) - - return func - - -def _add_bool(group, parser, dest, yes_flag, no_flag): - mutex_group = group.add_mutually_exclusive_group(required=False) - mutex_group.add_argument(yes_flag, dest=dest, action="store_true") - mutex_group.add_argument(no_flag, dest=dest, action="store_false") - parser.set_defaults(**{dest: None}) - - -def setup_common_config_cli_args(config_group): - config_group.add_argument( - "--exclude", type=_validator("exclude"), metavar='"path1, path2, ..."' - ) - config_group.add_argument( - "--extend-exclude", type=_validator("exclude"), metavar='"path1, path2, ..."' - ) - config_group.add_argument( - "--extensions", type=_validator("extensions"), metavar='"tcl, xdc, ..."' - ) - config_group.add_argument( - "--commands", type=_validator("commands"), metavar="" - ) - - -def setup_config_cli_args(parser): - """This method defines config-related CLI arguments. - - The destvars of these switches should match the fields of Config. - """ - config_group = parser.add_argument_group("configuration arguments") - - config_group.add_argument( - "--ignore", type=_validator("ignore"), metavar='"rule1, rule2, ..."' - ) - config_group.add_argument( - "--extend-ignore", type=_validator("ignore"), metavar='"rule1, rule2, ..."' - ) - setup_common_config_cli_args(config_group) - config_group.add_argument( - "--style-line-length", - type=_validator("style_line_length"), - metavar="", - ) - - -def setup_tclfmt_config_cli_args(parser): - """This method defines the subset of config-related CLI arguments used by tclfmt. - - The destvars of these switches should match the fields of Config. - """ - config_group = parser.add_argument_group("configuration arguments") - - setup_common_config_cli_args(config_group) - - config_group.add_argument( - "--indent", - type=_validator("style_indent"), - metavar="", - dest="style_indent", - ) - config_group.add_argument( - "--max-blank-lines", - type=_validator("style_max_blank_lines"), - metavar="", - dest="style_max_blank_lines", - ) - _add_bool( - config_group, - parser, - "style_indent_namespace_eval", - "--indent-namespace-eval", - "--no-indent-namespace-eval", - ) - _add_bool( - config_group, - parser, - "style_spaces_in_braces", - "--spaces-in-braces", - "--no-spaces-in-braces", - ) - - -def _flatten(d, prefix=None): - """Flattens TOML config dictionary structure to match the flat set of fields - expected by Config dataclass.""" - if prefix is None: - prefix = [] - - flat = {} - for k, v in d.items(): - if isinstance(v, dict): - flat.update(_flatten(v, prefix=prefix + [k])) - else: - flat["_".join(prefix + [k]).replace("-", "_")] = v - - return flat - - -class RunConfig: - """Class that holds information about both global and fileset configs. User - code can get a Config object that applies to a particular file by calling - get_from_path() and supplying that file's path.""" - - def __init__(self, global_config=None, fileset_configs=None): - if global_config is not None: - self._global_config = global_config - else: - self._global_config = Config() - - self._fileset_configs = [ - # ([pathlib.Path...], Config]) - ] - if fileset_configs is not None: - self._fileset_configs = fileset_configs - - @property - def exclude(self): - return self._global_config.exclude - - @property - def extensions(self): - return self._global_config.extensions - - @classmethod - def from_dict(cls, config_dict: dict, root: pathlib.Path): - config_dict = _validate_config(config_dict) - try: - fileset_config_dicts = config_dict.pop("fileset") - except KeyError: - fileset_config_dicts = [] - - config_dict = _flatten(config_dict) - global_config = Config(**config_dict) - - fileset_configs = [] - for fileset_config in fileset_config_dicts: - paths = [] - for path in fileset_config.pop("paths"): - if not path.is_absolute(): - path = root / path - paths.append(path.resolve()) - - fileset_config = _flatten(fileset_config) - - # pull in default values from global config - full_fileset_config = config_dict.copy() - full_fileset_config.update(fileset_config) - - fileset_configs.append((paths, Config(**full_fileset_config))) - - return cls(global_config, fileset_configs) - - @classmethod - def from_path(cls, path: Union[str, pathlib.Path], root: pathlib.Path): - path = pathlib.Path(path) - - if not path.exists(): - raise FileNotFoundError - - with open(path, "rb") as f: - try: - data = tomllib.load(f) - except tomllib.TOMLDecodeError as e: - raise ConfigError(f"{path}: {e}") - - try: - return cls.from_dict(data, root) - except ConfigError as e: - raise ConfigError(f"{path}: {e}") - - @classmethod - def from_pyproject(cls, directory=None): - if directory is None: - directory = pathlib.Path(".") - else: - directory = pathlib.Path(directory) - - path = directory / "pyproject.toml" - - if not path.exists(): - raise FileNotFoundError - - with open(path, "rb") as f: - data = tomllib.load(f) - - tclint_config = data.get("tool", {})["tclint"] - - try: - return cls.from_dict(tclint_config, directory) - except ConfigError as e: - raise ConfigError(f"pyproject.toml: {e}") - - def get_for_path(self, path) -> Config: - if path is None: - return self._global_config - - path = path.resolve() - for fileset_paths, config in self._fileset_configs: - for fileset_path in fileset_paths: - if path.is_relative_to(fileset_path): - return config - - return self._global_config - - def apply_cli_args(self, args): - self._global_config.apply_cli_args(args) - for _, fileset_config in self._fileset_configs: - fileset_config.apply_cli_args(args) - - -class ConfigError(Exception): - pass - - -DEFAULT_CONFIGS = ("tclint.toml", ".tclint") - - -def get_config( - config_path: OptionalType[pathlib.Path], root: pathlib.Path -) -> OptionalType[RunConfig]: - """Loads a config file. - - If `config_path` is supplied, attempts to read config file from this path. If the - path can't be found, raises a ConfigError. - - Otherwise, attempts to read config from `root`/{tclint.toml, .tclint, - pyproject.toml} (in that order). If none of these files can be found, returns None. - - `root` is also used to resolve some relative paths in the config file. - """ - # user-supplied - if config_path is not None: - try: - return RunConfig.from_path(config_path, root) - except FileNotFoundError: - raise ConfigError(f"path {config_path} doesn't exist") - - for path in DEFAULT_CONFIGS: - try: - return RunConfig.from_path(root / path, root) - except FileNotFoundError: - pass - - try: - return RunConfig.from_pyproject(directory=root) - except ConfigError as e: - raise e - except (FileNotFoundError, tomllib.TOMLDecodeError, KeyError): - # just skip if file doesn't exist, contains TOML errors, or tclint key not found - pass - - return None diff --git a/server/src/tools/format.py b/server/src/tools/format.py deleted file mode 100644 index 66441e5..0000000 --- a/server/src/tools/format.py +++ /dev/null @@ -1,480 +0,0 @@ -import dataclasses -import itertools -import textwrap -from typing import List, Tuple, Union -import sys - -from src.tools.syntax_tree import ( - Node, - Script, - Command, - Comment, - CommandSub, - BareWord, - QuotedWord, - BracedWord, - CompoundBareWord, - VarSub, - ArgExpansion, - Expression, - BracedExpression, - ParenExpression, - UnaryOp, - BinaryOp, - TernaryOp, - Function, -) -from src.tools.parser import Parser -from src.tools.syntax_tree import List as ListNode - - -@dataclasses.dataclass -class LiteralBlock: - block: List[str] - pos: Tuple[int, int] - end_pos: Tuple[int, int] - - -@dataclasses.dataclass -class FormatterOpts: - indent: str - spaces_in_braces: bool - max_blank_lines: int - indent_namespace_eval: bool - - -class Formatter: - def __init__(self, opts: FormatterOpts): - self.opts = opts - - def _indent(self, lines: List[str], indent: str) -> List[str]: - indented = [] - for line in lines: - if line == "": - indented.append("") - else: - indented.append(indent + line) - - return indented - - def _brace(self, lines: List[str]) -> List[str]: - spaces_in_braces = " " if self.opts.spaces_in_braces else "" - if lines == [""]: - return ["{" + spaces_in_braces + "}"] - - braced_lines = lines[:] - braced_lines[0] = "{" + spaces_in_braces + lines[0] - braced_lines[-1] += spaces_in_braces + "}" - return braced_lines - - def format(self, *nodes: Union[Node, LiteralBlock]) -> List[str]: - formatted = [] - for node in nodes: - if isinstance(node, Script): - formatted += self.format_script(node) - elif isinstance(node, Command): - formatted += self.format_command(node) - elif isinstance(node, Comment): - formatted += self.format_comment(node) - elif isinstance(node, CommandSub): - formatted += self.format_command_sub(node) - elif isinstance(node, BareWord): - formatted += self.format_bare_word(node) - elif isinstance(node, QuotedWord): - formatted += self.format_quoted_word(node) - elif isinstance(node, BracedWord): - formatted += self.format_braced_word(node) - elif isinstance(node, CompoundBareWord): - formatted += self.format_compound_bare_word(node) - elif isinstance(node, VarSub): - formatted += self.format_var_sub(node) - elif isinstance(node, ArgExpansion): - formatted += self.format_arg_expansion(node) - elif isinstance(node, ListNode): - formatted += self.format_list(node) - elif isinstance(node, Expression): - formatted += self.format_expression(node) - elif isinstance(node, BracedExpression): - formatted += self.format_braced_expression(node) - elif isinstance(node, ParenExpression): - formatted += self.format_paren_expression(node) - elif isinstance(node, UnaryOp): - formatted += self.format_unary_op(node) - elif isinstance(node, BinaryOp): - formatted += self.format_binary_op(node) - elif isinstance(node, TernaryOp): - formatted += self.format_ternary_op(node) - elif isinstance(node, Function): - formatted += self.format_function(node) - elif isinstance(node, LiteralBlock): - formatted += node.block - else: - assert False, f"unrecognized node: {type(node)}" - - return formatted - - def format_top(self, script: str, parser: Parser) -> str: - tree = parser.parse(script) - self.script = script.split("\n") - return "\n".join(self.format_script_contents(tree)) + "\n" - - def format_partial(self, script: str, parser: Parser) -> str: - """Formats a partial Tcl script. - - This function formats a partial script according to the gofmt partial formatting - rules, "[preserving] leading indentation as well as leading and trailing spaces" - (ref: https://pkg.go.dev/cmd/gofmt#pkg-overview). Unlike Go, we have no way of - detecting if a given script is a program fragment, hence the distinct method - from `format_top` . - """ - leading = "".join(itertools.takewhile(str.isspace, script)) - try: - leading, indent = leading.rsplit("\n", 1) - leading += "\n" - except ValueError: - leading, indent = "", leading - trailing = "".join(itertools.takewhile(str.isspace, reversed(script)))[::-1] - - script = script.strip() - tree = parser.parse(script) - self.script = script.split("\n") - - formatted = "\n".join(self.format_script_contents(tree)) - - return leading + textwrap.indent(formatted, indent) + trailing - - def format_script_contents(self, script: Union[Script, CommandSub]) -> List[str]: - to_format = [] - skip_formatting_start = None - for child in script.children: - if skip_formatting_start is None: - to_format.append(child) - - if isinstance(child, Comment): - if child.value.strip() == "tclfmt-disable": - if skip_formatting_start is not None: - print( - "Warning: encountered 'tclint-disable' while formatting is" - " already disabled, ignoring...", - file=sys.stderr, - ) - else: - skip_formatting_start = child.pos[0] - elif child.value.strip() == "tclfmt-enable": - if skip_formatting_start is None: - print( - "Warning: encountered 'tclint-enable' while formatting is" - " already disabled, ignoring...", - file=sys.stderr, - ) - else: - skip_formatting_end = child.pos[0] - block = self.script[skip_formatting_start:skip_formatting_end] - to_format.append( - LiteralBlock( - block, - pos=(skip_formatting_start + 1, 1), - end_pos=(skip_formatting_end, 1), - ) - ) - skip_formatting_start = None - - if skip_formatting_start is not None: - print("Warning: missing 'tclint-enable'", file=sys.stderr) - to_format.append( - LiteralBlock( - self.script[skip_formatting_start:], - pos=(skip_formatting_start + 1, 1), - end_pos=script.end_pos, - ) - ) - - formatted = [""] - last_line = None - for child in to_format: - if last_line is not None: - if last_line == child.pos[0]: - if isinstance(child, Comment): - formatted[-1] += " ;" - else: - formatted[-1] += "; " - else: - newlines = child.pos[0] - last_line - newlines = min(newlines, self.opts.max_blank_lines + 1) - formatted.extend([""] * newlines) - last_line = child.end_pos[0] - - lines = self.format(child) - formatted[-1] += lines[0] - formatted.extend(lines[1:]) - - return formatted - - def format_script(self, script: Script, should_indent=True) -> List[str]: - lines = self.format_script_contents(script) - if script.pos[0] == script.end_pos[0]: - return self._brace(lines) - - # Usually, we enforce that multi-line scripts start on a new line after the open - # brace. However, if a comment was originally on the same line as the open brace - # we preserve it, since it's probably meant to be associated with this line - # (e.g. a tclint-disable-line). - open_brace = "{" - if ( - len(script.children) > 0 - and isinstance(script.children[0], Comment) - and script.pos[0] == script.children[0].pos[0] - ): - open_brace += " " + lines[0] - lines = lines[1:] - - if should_indent: - return [open_brace] + self._indent(lines, self.opts.indent) + ["}"] - else: - return [open_brace] + lines + ["}"] - - def format_command(self, command: Command) -> List[str]: - is_namespace_eval = ( - command.routine.contents == "namespace" - and len(command.args) > 0 - and command.args[0].contents == "eval" - ) - should_indent = not is_namespace_eval or self.opts.indent_namespace_eval - - hanging_indent = False - formatted = self.format(command.routine) - last_line = command.routine.end_pos[0] - for child in command.args: - if isinstance(child, Script): - child_lines = self.format_script(child, should_indent=should_indent) - else: - child_lines = self.format(child) - - if last_line == child.pos[0]: - formatted[-1] += " " - formatted[-1] += child_lines[0] - else: - formatted[-1] += " \\" - formatted.append(self.opts.indent + child_lines[0]) - hanging_indent = True - - if hanging_indent: - formatted.extend(self._indent(child_lines[1:], self.opts.indent)) - else: - formatted.extend(child_lines[1:]) - - last_line = child.end_pos[0] - - return formatted - - def format_comment(self, comment: Comment) -> List[str]: - return [f"#{comment.value}"] - - def format_command_sub(self, command_sub): - if len(command_sub.children) == 0: - return ["[]"] - - formatted = [] - contents = self.format_script_contents(command_sub) - if len(command_sub.children) > 1 and len(contents) > 1: - formatted.append("[") - formatted.extend(self._indent(contents, self.opts.indent)) - formatted.append("]") - else: - formatted.append("[" + contents[0]) - formatted.extend(contents[1:]) - formatted[-1] += "]" - - return formatted - - def format_bare_word(self, word) -> List[str]: - # Property enforced by parser - assert word.contents is not None - return [word.contents] - - def format_quoted_word(self, word) -> List[str]: - if word.contents is not None: - return [f'"{word.contents}"'] - - formatted = "" - for child in word.children: - formatted += "\n".join(self.format(child)) - - return [f'"{formatted}"'] - - def format_braced_word(self, word) -> List[str]: - assert word.contents is not None - return [f"{{{word.contents}}}"] - - def format_compound_bare_word(self, word) -> List[str]: - formatted = [""] - for child in word.children: - child_lines = self.format(child) - formatted[-1] += child_lines[0] - formatted.extend(child_lines[1:]) - - return formatted - - def format_var_sub(self, varsub) -> List[str]: - # We might be able to make the formatter infer whether braces are required, and - # remove them from the syntax tree. For now it's easier to just mimic the - # original format. - if varsub.braced: - formatted = [f"${{{varsub.value}}}"] - else: - formatted = [f"${varsub.value}"] - - if varsub.children: - # We just concatenate everything as is, since changes in whitespace are - # semantically meaningful in this context. Any newlines are captured by - # BareWords. - formatted[-1] += "(" - for child in varsub.children: - child_lines = self.format(child) - formatted[-1] += child_lines[0] - formatted.extend(child_lines[1:]) - formatted[-1] += ")" - - return formatted - - def format_arg_expansion(self, arg_expansion) -> List[str]: - lines = self.format(arg_expansion.list) - lines[0] = "{*}" + lines[0] - - return lines - - def format_list(self, list_node) -> List[str]: - # Similar to Script, but the contents are a bit more straightforward. - contents = [""] - last_line = None - for child in list_node.children: - if last_line is not None: - if last_line == child.pos[0]: - contents[-1] += " " - else: - newlines = child.pos[0] - last_line - newlines = min(newlines, 3) - contents.extend([""] * newlines) - - lines = self.format(child) - contents[-1] += lines[0] - contents.extend(lines[1:]) - - last_line = child.end_pos[0] - - if list_node.pos[0] == list_node.end_pos[0]: - return self._brace(contents) - - return ["{"] + self._indent(contents, self.opts.indent) + ["}"] - - def format_expression(self, expr) -> List[str]: - formatted = [""] - for child in expr.children: - lines = self.format(child) - formatted[-1] += lines[0] - for line in lines[1:]: - formatted[-1] += " \\" - formatted += self._indent([line], self.opts.indent) - - # Trick: we know there are quotes around the expression if the start of the - # expression is a different column than its first child. - quoted = expr.pos[1] != expr.children[0].pos[1] - if quoted: - formatted[0] = '"' + formatted[0] - formatted[-1] += '"' - - return formatted - - def format_braced_expression(self, expr) -> List[str]: - formatted = [""] - for child in expr.children: - lines = self.format(child) - formatted[-1] += lines[0] - formatted.extend(lines[1:]) - - if expr.pos[0] == expr.end_pos[0]: - return self._brace(formatted) - - return ["{"] + self._indent(formatted, self.opts.indent) + ["}"] - - def format_paren_expression(self, expr) -> List[str]: - body = expr.body - - formatted = ["("] - lines = self.format(body) - if expr.pos[0] != body.pos[0]: - formatted.extend(lines) - else: - formatted[-1] += lines[0] - formatted.extend(lines[1:]) - - formatted = formatted[0:1] + self._indent(formatted[1:], self.opts.indent) - - if expr.end_pos[0] != body.end_pos[0]: - formatted.append(")") - else: - formatted[-1] += ")" - - return formatted - - def format_unary_op(self, expr): - op = self.format(expr.operator) - assert len(op) == 1 - - lines = self.format(expr.operand) - lines[0] = op[0] + lines[0] - return lines - - def _format_op(self, expr) -> List[str]: - nodes = expr.children - formatted = self.format(nodes[0]) - - last = nodes[0] - for next in nodes[1:]: - lines = self.format(next) - if last.end_pos[0] != next.pos[0]: - formatted.extend(lines) - else: - formatted[-1] += " " - formatted[-1] += lines[0] - formatted.extend(lines[1:]) - last = next - - return formatted - - def format_binary_op(self, expr) -> List[str]: - return self._format_op(expr) - - def format_ternary_op(self, expr) -> List[str]: - return self._format_op(expr) - - def format_function(self, function): - name = self.format(function.name) - assert len(name) == 1 - name = name[0] - - formatted = [f"{name}("] - - last = function.name - for i, child in enumerate(function.args): - if i > 0: - formatted[-1] += "," - lines = self.format(child) - if last.end_pos[0] != child.pos[0]: - formatted.extend(lines) - else: - if i > 0: - formatted[-1] += " " - formatted[-1] += lines[0] - formatted.extend(lines[1:]) - last = child - - # indent any continuation lines, but we leave the closing paren dedented - formatted = formatted[0:1] + self._indent(formatted[1:], self.opts.indent) - - if last.end_pos[0] != function.end_pos[0]: - formatted.append(")") - else: - formatted[-1] += ")" - - return formatted diff --git a/server/src/tools/lexer.py b/server/src/tools/lexer.py index dc18312..dcea50e 100644 --- a/server/src/tools/lexer.py +++ b/server/src/tools/lexer.py @@ -1,27 +1,32 @@ +from enum import Enum import ply.lex as lex from typing import Tuple -TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE" -TOK_BACKSLASH_SUB = "BACKSLASH_SUB" -TOK_NEWLINE = "NEWLINE" -TOK_SEMI = "SEMI" -TOK_WS = "WS" -TOK_QUOTE = "QUOTE" -TOK_ARG_EXPANSION = "ARG_EXPANSION" -TOK_LBRACE = "LBRACE" -TOK_RBRACE = "RBRACE" -TOK_STAR = "STAR" -TOK_LBRACKET = "LBRACKET" -TOK_RBRACKET = "RBRACKET" -TOK_DOLLAR = "DOLLAR" -TOK_LPAREN = "LPAREN" -TOK_RPAREN = "RPAREN" -TOK_HASH = "HASH" -TOK_ALPHA_CHARS = "ALPHA_CHARS" -TOK_NUM_CHARS = "NUM_CHARS" -TOK_NAMESPACE_SEP = "NAMESPACE_SEP" -TOK_CHAR = "CHAR" -TOK_CONTENTS = "CONTENTS" + +class Tok(str, Enum): + TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE" + TOK_BACKSLASH_SUB = "BACKSLASH_SUB" + TOK_NEWLINE = "NEWLINE" + TOK_SEMI = "SEMI" + TOK_WS = "WS" + TOK_QUOTE = "QUOTE" + TOK_ARG_EXPANSION = "ARG_EXPANSION" + TOK_LBRACE = "LBRACE" + TOK_RBRACE = "RBRACE" + TOK_STAR = "STAR" + TOK_LBRACKET = "LBRACKET" + TOK_RBRACKET = "RBRACKET" + TOK_DOLLAR = "DOLLAR" + TOK_LPAREN = "LPAREN" + TOK_RPAREN = "RPAREN" + TOK_HASH = "HASH" + TOK_ALPHA_CHARS = "ALPHA_CHARS" + TOK_NUM_CHARS = "NUM_CHARS" + TOK_NAMESPACE_SEP = "NAMESPACE_SEP" + TOK_CHAR = "CHAR" + TOK_CONTENTS = "CONTENTS" + + TOK_EOF = None STATE_BRACEDWORD = "bracedword" @@ -35,30 +40,7 @@ class TclSyntaxError(Exception): class _LexTable: - tokens = ( - TOK_BACKSLASH_NEWLINE, - TOK_BACKSLASH_SUB, - TOK_NEWLINE, - TOK_SEMI, - TOK_WS, - TOK_QUOTE, - TOK_ARG_EXPANSION, - TOK_LBRACE, - TOK_RBRACE, - TOK_STAR, - TOK_LBRACKET, - TOK_RBRACKET, - TOK_DOLLAR, - TOK_LPAREN, - TOK_RPAREN, - TOK_HASH, - TOK_ALPHA_CHARS, - TOK_NUM_CHARS, - TOK_NAMESPACE_SEP, - TOK_CHAR, - TOK_CONTENTS, - ) - + tokens = tuple(t.value for t in Tok) # This defines a conditional lexing state for parsing braced words. This is a # performance optimization; since there are few special characters in this context, # we can use a smaller set of tokens to parse them faster. This has a large impact @@ -241,3 +223,23 @@ class Lexer: def assert_(self, *tokens): assert self.current.type in tokens self.next() + + +def main(): + code = 'puts "Hello, World!"\nset x 42\n' + lexer = Lexer() + lexer.input(code) + + print("Lexing input:\n", code) + print("\nTokens:\n" + "-" * 30) + + while lexer.type() is not None: + tok_type = lexer.type() + tok_value = lexer.value() + tok_pos = lexer.pos() + print(f"Type: {tok_type:20} | Value: {repr(tok_value):20} | Pos: {tok_pos}") + lexer.next() + + +if __name__ == "__main__": + main() diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py index 5819a75..27145d8 100644 --- a/server/src/tools/parser.py +++ b/server/src/tools/parser.py @@ -1,64 +1,40 @@ import string import re -from src.tools.lexer import ( - Lexer, - TclSyntaxError, - STATE_BRACEDWORD, - TOK_BACKSLASH_NEWLINE, - TOK_NEWLINE, - TOK_SEMI, - TOK_WS, - TOK_QUOTE, - TOK_ARG_EXPANSION, - TOK_LBRACE, - TOK_RBRACE, - TOK_LBRACKET, - TOK_RBRACKET, - TOK_DOLLAR, - TOK_LPAREN, - TOK_RPAREN, - TOK_HASH, - TOK_ALPHA_CHARS, - TOK_NUM_CHARS, - TOK_NAMESPACE_SEP, - TOK_EOF, -) -from src.tools.syntax_tree import ( - Script, - Comment, +from tools.commands.checks import CommandArgError, check_command +from tools.lexer import Lexer, TclSyntaxError, Tok, TOK_EOF, STATE_BRACEDWORD +from tools.syntax_tree import ( + ArgExpansion, + BareWord, + BinaryOp, + BracedExpression, + BracedWord, Command, CommandSub, - ArgExpansion, - VarSub, - BareWord, - BracedWord, - QuotedWord, + Comment, CompoundBareWord, - List, Expression, - BracedExpression, - ParenExpression, - UnaryOp, - BinaryOp, - TernaryOp, Function, + List, + ParenExpression, + QuotedWord, + Script, + TernaryOp, + UnaryOp, + VarSub, ) -from src.tools.commands import CommandArgError, get_commands -from src.tools.commands.checks import check_command -from src.tools.violations import Rule, Violation def _strip_ws(parse_func): """Decorator used by expression parser for stripping whitespace around a node.""" def func(parser, ts): - while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: + while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE, Tok.TOK_NEWLINE}: ts.next() node = parse_func(parser, ts) - while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: + while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE, Tok.TOK_NEWLINE}: ts.next() return node @@ -98,15 +74,12 @@ class _Word: class Parser: - def __init__(self, debug=False, command_plugins=None): + def __init__(self, debug=False): self._debug = debug self._debug_indent = 0 - # TODO: better way to handle this? self.violations = [] - if command_plugins is None: - command_plugins = [] - self._commands = get_commands(command_plugins) + self._commands = [] def debug(self, *msg): if self._debug: @@ -197,12 +170,12 @@ class Parser: script = Script(pos=pos) while ts.type() is not TOK_EOF: - if ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE}: + if ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE}: # strip whitespace at start of command ts.next() continue - if ts.type() == TOK_HASH: + if ts.type() == Tok.TOK_HASH: script.add(self.parse_comment(ts)) else: cmd = self.parse_command(ts, in_command_sub=in_command_sub) @@ -210,13 +183,13 @@ class Parser: script.add(cmd) # when in command sub mode, a script is terminated by ] - if in_command_sub and ts.type() == TOK_RBRACKET: + if in_command_sub and ts.type() == Tok.TOK_RBRACKET: return script ts.expect( TOK_EOF, - TOK_NEWLINE, - TOK_SEMI, + Tok.TOK_NEWLINE, + Tok.TOK_SEMI, message=f"expected newline or semicolon, got {ts.value()}", pos=ts.pos(), ) @@ -236,10 +209,10 @@ class Parser: self.debug(f"parse_comment({ts.current})") pos = ts.pos() - ts.assert_(TOK_HASH) + ts.assert_(Tok.TOK_HASH) value = "" - while ts.type() not in {TOK_NEWLINE, TOK_EOF}: + while ts.type() not in {Tok.TOK_NEWLINE, TOK_EOF}: value += ts.value() ts.next() @@ -261,10 +234,10 @@ class Parser: args = [] while True: - if ts.type() not in {TOK_WS, TOK_BACKSLASH_NEWLINE}: + if ts.type() not in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE}: break - while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE}: + while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE}: ts.next() word = self.parse_word(ts, in_command_sub) @@ -276,9 +249,10 @@ class Parser: self._debug_indent -= 1 try: + pass parsed_args = self._parse_command_args(routine.contents, args) except CommandArgError as e: - self.violations.append(Violation(Rule.COMMAND_ARGS, str(e), pos, ts.pos())) + # self.violations.append(Violation(Rule.COMMAND_ARGS, str(e), pos, ts.pos())) parsed_args = args children = [routine, *parsed_args] @@ -289,11 +263,11 @@ class Parser: def parse_word(self, ts, in_command_sub): self.debug(f"parse_word({ts.current})") - if ts.type() == TOK_ARG_EXPANSION: + if ts.type() == Tok.TOK_ARG_EXPANSION: return self.parse_arg_expansion(ts, in_command_sub) - elif ts.type() == TOK_LBRACE: + elif ts.type() == Tok.TOK_LBRACE: return self.parse_braced_word(ts) - elif ts.type() == TOK_QUOTE: + elif ts.type() == Tok.TOK_QUOTE: return self.parse_quoted_word(ts) else: return self.parse_bare_word(ts, in_command_sub) @@ -302,11 +276,17 @@ class Parser: self.debug(f"parse_arg_expansion({ts.current})") pos = ts.pos() - ts.assert_(TOK_ARG_EXPANSION) + ts.assert_(Tok.TOK_ARG_EXPANSION) - delimiters = [TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE, TOK_SEMI, TOK_EOF] + delimiters = [ + Tok.TOK_WS, + Tok.TOK_BACKSLASH_NEWLINE, + Tok.TOK_NEWLINE, + Tok.TOK_SEMI, + TOK_EOF, + ] if in_command_sub: - delimiters.append(TOK_RBRACKET) + delimiters.append(Tok.TOK_RBRACKET) # Arg expansion is just a regular braced word if followed by whitespace, # or other word boundaries such as semicolon or right bracket @@ -323,18 +303,18 @@ class Parser: self._debug_indent += 1 pos = ts.pos() - ts.assert_(TOK_QUOTE) + ts.assert_(Tok.TOK_QUOTE) word = _Word() - while ts.type() not in {TOK_QUOTE, TOK_EOF}: - if ts.type() == TOK_DOLLAR: + while ts.type() not in {Tok.TOK_QUOTE, TOK_EOF}: + if ts.type() == Tok.TOK_DOLLAR: dollar_tok = ts.current var_sub = self.parse_var_sub(ts) if var_sub: word.add_node(var_sub) else: word.add_tok(dollar_tok) - elif ts.type() == TOK_LBRACKET: + elif ts.type() == Tok.TOK_LBRACKET: command_sub = self.parse_command_sub(ts) word.add_node(command_sub) else: @@ -344,7 +324,9 @@ class Parser: res = word.resolve(ts.pos()) ts.expect( - TOK_QUOTE, message="reached EOF without finding match for quote", pos=pos + Tok.TOK_QUOTE, + message="reached EOF without finding match for quote", + pos=pos, ) self._debug_indent -= 1 @@ -360,7 +342,7 @@ class Parser: ts.lexer.push_state(STATE_BRACEDWORD) - ts.assert_(TOK_LBRACE) + ts.assert_(Tok.TOK_LBRACE) word = "" # store position for each brace we want to match, facilitating good @@ -375,9 +357,9 @@ class Parser: ts.pos(), ) - if toktype == TOK_LBRACE: + if toktype == Tok.TOK_LBRACE: expected_braces.append(ts.pos()) - elif toktype == TOK_RBRACE: + elif toktype == Tok.TOK_RBRACE: try: expected_braces.pop() except IndexError: @@ -404,21 +386,27 @@ class Parser: pos = ts.pos() word = _Word() - delimiters = [TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE, TOK_SEMI, TOK_EOF] + delimiters = [ + Tok.TOK_WS, + Tok.TOK_BACKSLASH_NEWLINE, + Tok.TOK_NEWLINE, + Tok.TOK_SEMI, + TOK_EOF, + ] # In command sub mode, words are ended by ] if in_command_sub: - delimiters.append(TOK_RBRACKET) + delimiters.append(Tok.TOK_RBRACKET) while ts.type() not in delimiters: - if ts.type() == TOK_DOLLAR: + if ts.type() == Tok.TOK_DOLLAR: dollar_tok = ts.current var_sub = self.parse_var_sub(ts) if var_sub: word.add_node(var_sub) else: word.add_tok(dollar_tok) - elif ts.type() == TOK_LBRACKET: + elif ts.type() == Tok.TOK_LBRACKET: command_sub = self.parse_command_sub(ts) word.add_node(command_sub) else: @@ -439,13 +427,13 @@ class Parser: self.debug(f"parse_var_sub({ts.current})") pos = ts.pos() - ts.assert_(TOK_DOLLAR) + ts.assert_(Tok.TOK_DOLLAR) var = "" - if ts.type() == TOK_LBRACE: + if ts.type() == Tok.TOK_LBRACE: brace_pos = ts.pos() ts.next() - while ts.type() != TOK_RBRACE: + while ts.type() != Tok.TOK_RBRACE: if ts.type() is TOK_EOF: raise TclSyntaxError( "reached EOF without finding match for brace", @@ -458,7 +446,11 @@ class Parser: return VarSub(var, pos=pos, end_pos=ts.pos(), braced=True) - while ts.type() in {TOK_ALPHA_CHARS, TOK_NUM_CHARS, TOK_NAMESPACE_SEP}: + while ts.type() in { + Tok.TOK_ALPHA_CHARS, + Tok.TOK_NUM_CHARS, + Tok.TOK_NAMESPACE_SEP, + }: var += ts.value() ts.next() @@ -466,25 +458,25 @@ class Parser: return None index_nodes = [] - if ts.type() == TOK_LPAREN: + if ts.type() == Tok.TOK_LPAREN: paren_pos = ts.pos() index = _Word() ts.next() - while ts.type() != TOK_RPAREN: + while ts.type() != Tok.TOK_RPAREN: if ts.type() == TOK_EOF: raise TclSyntaxError( "reached EOF without finding match for paren", paren_pos, ts.pos(), ) - if ts.type() == TOK_DOLLAR: + if ts.type() == Tok.TOK_DOLLAR: dollar_tok = ts.current var_sub = self.parse_var_sub(ts) if var_sub: index.add_node(var_sub) else: index.add_tok(dollar_tok) - elif ts.type() == TOK_LBRACKET: + elif ts.type() == Tok.TOK_LBRACKET: command_sub = self.parse_command_sub(ts) index.add_node(command_sub) else: @@ -506,11 +498,11 @@ class Parser: self._debug_indent += 1 pos = ts.pos() - ts.assert_(TOK_LBRACKET) + ts.assert_(Tok.TOK_LBRACKET) script = self._parse_script(ts, in_command_sub=True) - ts.assert_(TOK_RBRACKET) + ts.assert_(Tok.TOK_RBRACKET) end_pos = ts.pos() script.line = pos[0] @@ -537,7 +529,7 @@ class Parser: ts = Lexer(pos=node.contents_pos) ts.input(node.contents) - DELIMITERS = {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE} + DELIMITERS = {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE, Tok.TOK_NEWLINE} list_node = List(pos=node.pos, end_pos=node.end_pos) while ts.type() is not TOK_EOF: @@ -547,24 +539,24 @@ class Parser: if ts.type() is TOK_EOF: break - if ts.type() == TOK_LBRACE: + if ts.type() == Tok.TOK_LBRACE: # we can reuse parse_braced_word, since it doesn't use # substitutions in any case list_node.add(self.parse_braced_word(ts)) - elif ts.type() == TOK_QUOTE: + elif ts.type() == Tok.TOK_QUOTE: quote_word_pos = ts.pos() - ts.assert_(TOK_QUOTE) + ts.assert_(Tok.TOK_QUOTE) bare_word_pos = ts.pos() contents = "" - while ts.type() not in {TOK_QUOTE, TOK_EOF}: + while ts.type() not in {Tok.TOK_QUOTE, TOK_EOF}: contents += ts.value() ts.next() word = BareWord(contents, pos=bare_word_pos, end_pos=ts.pos()) ts.expect( - TOK_QUOTE, + Tok.TOK_QUOTE, message="reached EOF without finding match for quote", pos=quote_word_pos, ) @@ -607,7 +599,7 @@ class Parser: expr = op1 # last condition is hack to break out of expression in case we're in ternary op - if ts.type() not in {TOK_EOF, TOK_RPAREN} and ts.value() not in {":", ","}: + if ts.type() not in {TOK_EOF, Tok.TOK_RPAREN} and ts.value() not in {":", ","}: if ts.value() == "?": # weird hack to record operator start = ts.pos() @@ -637,27 +629,27 @@ class Parser: op2 = self._parse_expression(ts) expr = BinaryOp(op1, operator, op2, pos=op1.pos, end_pos=op2.end_pos) - if ts.type() != TOK_RPAREN and ts.value() not in {":", ","}: + if ts.type() != Tok.TOK_RPAREN and ts.value() not in {":", ","}: ts.expect(TOK_EOF, message="expected end of expression", pos=ts.pos()) return expr @_strip_ws def _parse_operand(self, ts): - if ts.type() == TOK_DOLLAR: + if ts.type() == Tok.TOK_DOLLAR: return self.parse_var_sub(ts) - if ts.type() == TOK_QUOTE: + if ts.type() == Tok.TOK_QUOTE: return self.parse_quoted_word(ts) - if ts.type() == TOK_LBRACE: + if ts.type() == Tok.TOK_LBRACE: return self.parse_braced_word(ts) - if ts.type() == TOK_LBRACKET: + if ts.type() == Tok.TOK_LBRACKET: return self.parse_command_sub(ts) - if ts.type() == TOK_LPAREN: + if ts.type() == Tok.TOK_LPAREN: start = ts.pos() ts.next() expr = self._parse_expression(ts) ts.expect( - TOK_RPAREN, + Tok.TOK_RPAREN, message="reached EOF without finding match for paren", pos=expr.pos, ) @@ -693,7 +685,7 @@ class Parser: # move on. If not, we keep consuming tokens that may correspond to a # valid bareword (pretty much just alphanumeric chars). if not (_is_int_literal(operand) or _is_float_literal(operand)): - while ts.type() in {TOK_ALPHA_CHARS, TOK_NUM_CHARS}: + while ts.type() in {Tok.TOK_ALPHA_CHARS, Tok.TOK_NUM_CHARS}: operand += ts.value() ts.next() @@ -763,16 +755,16 @@ class Parser: return BareWord(operator, pos=pos, end_pos=ts.pos()) def _parse_function(self, ts, name): - while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE}: + while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE}: ts.next() ts.expect( - TOK_LPAREN, + Tok.TOK_LPAREN, message="expected open paren after function name", pos=name.pos, ) - delims = {TOK_RPAREN, TOK_EOF} + delims = {Tok.TOK_RPAREN, TOK_EOF} arguments = [] if ts.type() not in delims: @@ -791,7 +783,7 @@ class Parser: arguments.append(self._parse_expression(ts)) ts.expect( - TOK_RPAREN, + Tok.TOK_RPAREN, message="expected close paren after function arguments", pos=name.pos, ) diff --git a/server/src/tools/plugins.py b/server/src/tools/plugins.py deleted file mode 100644 index 927e874..0000000 --- a/server/src/tools/plugins.py +++ /dev/null @@ -1,81 +0,0 @@ -from importlib.metadata import entry_points -import json -import pathlib -from typing import Dict, Optional - - -class _PluginManager: - def __init__(self): - self._loaded = {} - self._installed = {} - self._loaded_specs = {} - for plugin in entry_points(group="tclint.plugins"): - if plugin.name in self._installed: - print(f"Warning: found duplicate definitions for plugin {plugin.name}") - self._installed[plugin.name] = plugin - - def load(self, name: str) -> Optional[Dict]: - if name in self._loaded: - return self._loaded[name] - - mod = self._load(name) - self._loaded[name] = mod - return mod - - def load_from_spec(self, path: pathlib.Path) -> Optional[Dict]: - if path in self._loaded_specs: - return self._loaded_specs[path] - - spec = self._load_from_spec(path) - self._loaded_specs[path] = spec - return spec - - def _load_from_spec(self, path: pathlib.Path) -> Optional[Dict]: - try: - with open(path.expanduser(), "r") as f: - spec = json.load(f) - except (FileNotFoundError, RuntimeError): - print(f"Warning: command spec {path} not found, skipping...") - return None - - try: - # Apply defaults and validate the spec. - spec = command_schema(spec) - except voluptuous.Invalid as e: - print(f"Warning: invalid command spec {path}: {e}") - return None - - return spec["commands"] - - def get_mod(self, name: str) -> Optional[ModuleType]: - if name not in self._installed: - print(f"Warning: plugin {name} is not installed") - return None - - plugin = self._installed[name] - - try: - module = plugin.load() - except Exception as e: - print(f"Warning: error loading plugin {name}: {e}") - return None - - return module - - def _load(self, name: str): - module = self.get_mod(name) - if module is None: - print(f"Skipping requested plugin {name}") - return None - - if not hasattr(module, "commands"): - print(f"Warning: skipping plugin {name} since it does not define commands") - return None - - return getattr(module, "commands") - - -# TODO: we'll probably want to construct this in the tclint entry point and pass -# it around rather than using a singleton instance, but this made for an easier -# refactor. -PluginManager = _PluginManager() diff --git a/server/src/tools/schema.py b/server/src/tools/schema.py deleted file mode 100644 index 7662a10..0000000 --- a/server/src/tools/schema.py +++ /dev/null @@ -1,35 +0,0 @@ -from collections.abc import Callable -from voluptuous import Schema, Optional, Or, Self - -# Need to define this as a Schema with required=True to ensure that this requirement -# persists through the Or in the main schema definition. -_command_args = Schema( - { - Optional("positionals", default=[]): [ - { - "name": str, - "required": bool, - "value": Or({"type": "any"}, {"type": "variadic"}), - } - ], - Optional("switches", default={}): { - Optional(str): { - "required": bool, - "repeated": bool, - "value": Or({"type": "any"}, None), - Optional("metavar"): str, - } - }, - }, - required=True, -) - -commands_schema = Schema( - {Optional(str): Or(_command_args, None, {"subcommands": Self}, Callable)}, - required=True, -) - -schema = Schema( - {"name": str, "commands": commands_schema}, - required=True, -) diff --git a/server/src/tools/syntax_tree.py b/server/src/tools/syntax_tree.py index d11f6df..fdfac56 100644 --- a/server/src/tools/syntax_tree.py +++ b/server/src/tools/syntax_tree.py @@ -1,4 +1,4 @@ -"""Classes for representing and interacting with Tcl syntax trees. """ +"""Classes for representing and interacting with Tcl syntax trees.""" class Visitor: @@ -67,9 +67,6 @@ class Node: """ def __init__(self, *init, pos=None, end_pos=None): - """pos: line, column of first character of parsed region (1-indexed) - end_pos: line, column of first character after parsed region (1-indexed) - """ self.line = None self.col = None if pos is not None: @@ -80,7 +77,6 @@ class Node: if len(init) > 0 and not isinstance(init[0], Node): self.value = init[0] init = init[1:] - if not all(isinstance(v, Node) for v in init): raise TypeError("Children must be Node instances") @@ -188,12 +184,12 @@ class Node: return lines if len(self.children) != len(other.children): - my_children = ",".join([ - child.__class__.__name__ for child in self.children - ]) - other_children = ",".join([ - child.__class__.__name__ for child in other.children - ]) + my_children = ",".join( + [child.__class__.__name__ for child in self.children] + ) + other_children = ",".join( + [child.__class__.__name__ for child in other.children] + ) lines += [f"{indent}-{my_cls}({my_children})"] lines += [f"{indent}+{other_cls}({other_children})"] diff --git a/server/src/tools/violations.py b/server/src/tools/violations.py deleted file mode 100644 index 8b08329..0000000 --- a/server/src/tools/violations.py +++ /dev/null @@ -1,50 +0,0 @@ -from enum import Enum -from typing import Tuple - - -class Rule(Enum): - """This enum serves a few purposes: - - 1) define symbols for rule IDs to be used in code - 2) map these symbols to names in the UI - 3) collect all rule IDs/provide validation for IDs - """ - - LINE_LENGTH = "line-length" - TRAILING_WHITESPACE = "trailing-whitespace" - COMMAND_ARGS = "command-args" - REDEFINED_BUILTIN = "redefined-builtin" - UNBRACED_EXPR = "unbraced-expr" - REDUNDANT_EXPR = "redundant-expr" - - def __str__(self): - return self.value - - -ALL_RULES = [rule for rule in Rule] - - -class Violation: - def __init__( - self, id: Rule, message: str, start: Tuple[int, int], end: Tuple[int, int] - ): - self.id = id - self.message = message - self.start = start - self.end = end - - def __lt__(self, other): - return self.start < other.start - - def __str__(self): - line, col = self.start - rule = str(self.id) - - return f"{line}:{col}: {self.message} [{rule}]" - - @classmethod - def create(cls, id): - def func(message: str, start: Tuple[int, int], end: Tuple[int, int]): - return cls(id, message, start, end) - - return func