From 0ea2cbec87de00f0a65cd260214f72948a2b6578 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Mon, 28 Jul 2025 22:21:49 +0200 Subject: [PATCH 1/7] add noxfile.py --- .vscodeignore | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/.vscodeignore b/.vscodeignore index a054988..a94eca4 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -14,4 +14,5 @@ esbuild.js **/__pycache__/** **/requirements.txt **/requirements.in -**/server/src/_debug_server.py \ No newline at end of file +**/server/src/_debug_server.py +noxfile.py \ No newline at end of file -- 2.54.0 From ed045aa459440376c40c2cf958fb804feeb37fa4 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Wed, 30 Jul 2025 17:28:18 +0200 Subject: [PATCH 2/7] update completion items --- server/src/common/load_data.py | 9 ++ server/src/lsp_server.py | 64 ++++++---- server/src/test_tcl.py | 45 ++----- server/src/tools/completion_items.py | 37 ++++++ server/src/tools/parser.py | 49 ++------ server/src/tools/semantic_tokens.py | 170 +++++++++++++++++---------- 6 files changed, 208 insertions(+), 166 deletions(-) create mode 100644 server/src/tools/completion_items.py diff --git a/server/src/common/load_data.py b/server/src/common/load_data.py index 67837ff..dc22c99 100644 --- a/server/src/common/load_data.py +++ b/server/src/common/load_data.py @@ -21,6 +21,7 @@ class StandardCompletionItems: self.__tcl_keyword_list = self.__load_tcl_keyword() self.__nx_procs = self.__load_nx_procs() self.__nx_variables = self.__load_nx_variables() + self.__custom_functions = list[lsp.CompletionItem] @property def json_data(self): @@ -38,6 +39,14 @@ class StandardCompletionItems: def nx_variables(self): return self.__nx_variables + @property + def custom_functions(self) -> list[lsp.CompletionItem]: + return self.__custom_functions + + @custom_functions.setter + def custom_functions(self, value: lsp.CompletionItem): + self.__custom_functions.append(value) + def __load_json(self) -> dict: with open( pathlib.Path(__file__).parent.joinpath("completion_list.json"), "r" diff --git a/server/src/lsp_server.py b/server/src/lsp_server.py index 707f401..4e2b91b 100644 --- a/server/src/lsp_server.py +++ b/server/src/lsp_server.py @@ -14,6 +14,8 @@ import sysconfig import traceback from typing import Any, List, Optional, Sequence, Tuple import re +import operator +from functools import reduce # ********************************************************** @@ -44,18 +46,15 @@ import lsprotocol.types as lsp from pygls import server, uris, workspace from pygls.workspace.text_document import TextDocument from common.load_data import standard_items -from common.formatter import format_tcl from tclint.parser import Parser from tclint.lexer import TclSyntaxError from tclint.format import Formatter, FormatterOpts from tclint.violations import Violation -from tools.semantic_tokens import ( - SemanticTokenCollector, - collect_semantic_tokens, - encode_tokens, -) + from plugins.poco_plugin import commands from tools import checks +from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier +from tools.completion_items import _Completion DIAGNOSTIC_SOURCE = "nx-post-support" @@ -78,7 +77,6 @@ class TclLanguageServer(server.LanguageServer): # parser._commands.update(commands) indent = "\t" if not options.insert_spaces else " " * options.tab_size - formatter = Formatter( FormatterOpts( indent=indent, @@ -102,7 +100,7 @@ class TclLanguageServer(server.LanguageServer): self.parser.violations = [] tree = self.parser.parse(document.source) violations += self.parser.violations - + # log_to_output(tree.pretty(2)) for checker in checks.get_checkers(): violations += checker.check(document.source, tree) return violations @@ -170,16 +168,6 @@ WORKSPACE_SETTINGS = {} GLOBAL_SETTINGS = {} RUNNER = pathlib.Path(__file__).parent / "lsp_runner.py" -TOKEN_TYPES = [ - "command", - "variable", - "function", - "string", - "number", - "keyword", - "comment", -] -TOKEN_MODIFIERS = [] MAX_WORKERS = 5 LSP_SERVER = TclLanguageServer( @@ -254,22 +242,48 @@ def document_diagnostic(params: lsp.DocumentDiagnosticParams): @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_COMPLETION) def on_completion(params: lsp.CompletionParams) -> list[lsp.CompletionItem]: + document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) + ci = _Completion() + tree = LSP_SERVER.parser.parse(document.source) + tree.accept(ci, recurse=True) + items = ( standard_items.tcl_keyword_list + standard_items.nx_procs + standard_items.nx_variables + + ci.custom_functions ) return lsp.CompletionList(is_incomplete=False, items=items) -@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL) -def on_semantic_tokens(params: lsp.SemanticTokensParams): - doc = LSP_SERVER.workspace.get_document(params.text_document.uri) - code = doc.source +@LSP_SERVER.feature( + lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL, + lsp.SemanticTokensLegend( + token_types=TOKEN_TYPES, + token_modifiers=[m.name for m in TokenModifier], + ), +) +def semantic_tokens(params: lsp.SemanticTokensParams): + document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) - tokens = collect_semantic_tokens(code) - data = encode_tokens(tokens) + data = [] + plugins = [] + hl = _Highlighter(plugins, log_to_output=log_to_output) + tree = LSP_SERVER.parser.parse(document.source) + tree.accept(hl, recurse=True) + + tokens = hl.tokens() + for token in tokens: + data.extend( + [ + token.line, + token.offset, + token.lenght, + TOKEN_TYPES.index(token.tok_type), + reduce(operator.or_, token.tok_modifiers, 0), + ] + ) return lsp.SemanticTokens(data=data) @@ -401,7 +415,7 @@ def initialize(params: lsp.InitializeParams) -> lsp.InitializeResult: ) semantic_tokens_legend = lsp.SemanticTokensLegend( token_types=TOKEN_TYPES, - token_modifiers=TOKEN_MODIFIERS, + token_modifiers=TokenModifier, ) return lsp.InitializeResult( capabilities=lsp.ServerCapabilities( diff --git a/server/src/test_tcl.py b/server/src/test_tcl.py index 783c406..4ffbfeb 100644 --- a/server/src/test_tcl.py +++ b/server/src/test_tcl.py @@ -1,7 +1,5 @@ from tclint.parser import Parser as tcLintParser from tclint.lexer import Lexer as tclingLexer -from parser.lexer import Lexer -from parser.parser import Parser def main(): @@ -28,22 +26,9 @@ LIB_SPF_prepend MOM_strt Start_Lib { def lexer_test(): lexer = tclingLexer() - tree = lexer.input("""puts hello - proc myProc {arg {arg7 0}} {} - set myVar 123 - puts puts - LIB_SPF_prepend MOM_strt Start_Lib { - set somthing 1 - set more 2 -} myTag - -LIB_SPF_prepend MOM_strt Start_Lib { - proc test {} { - puts "Hello" - } - set somthing 1 - set someting 3 -} myTag""") + tree = lexer.input(""" +if {$oem(custom_clamp_4th) == 1 && $oem(custom_clamp_5th) == 1 \\ + && $oem(status_clamp_4th) == "off" && $oem(status_clamp_5th) == "off"}""") # print("Lexing input:\n", code) # print("\nTokens:\n" + "-" * 30) @@ -57,26 +42,18 @@ LIB_SPF_prepend MOM_strt Start_Lib { def test_1(): - code = """ - proc myProc {arg } { - set myVar 1 - } + from tclint.lexer import Lexer, TOK_BACKSLASH_NEWLINE - namespace eval myNS { - proc innerProc {} { - MOM_abort_program "Test" - } - } - set result [myNS::innerProc] - """ + code = "expr {1 == 2 \\\n&& 3 == 4}" lexer = Lexer() lexer.input(code) - parser = Parser(lexer) - ast = parser.parse() - visitor = NodeVisitor() - ast.accept(visitor) + while lexer.type() is not None: + print( + f"Type: {lexer.type():<20} | Value: {lexer.value()!r} | Pos: {lexer.pos()}" + ) + lexer.next() class NodeVisitor: @@ -98,4 +75,4 @@ class NodeVisitor: if __name__ == "__main__": - main() + test_1() diff --git a/server/src/tools/completion_items.py b/server/src/tools/completion_items.py new file mode 100644 index 0000000..39fc1c9 --- /dev/null +++ b/server/src/tools/completion_items.py @@ -0,0 +1,37 @@ +from tclint.syntax_tree import Visitor +import lsprotocol.types as lsp + + +class CompletionItems: + def __init__(self): + self._custom_functions: list[lsp.CompletionItem] = [] + + @property + def custom_functions(self) -> list[lsp.CompletionItem]: + return self._custom_functions + + @custom_functions.setter + def custom_functions(self, value: lsp.CompletionItem): + self._custom_functions.append(value) + + +class _Completion(Visitor): + def __init__(self): + super().__init__() + self._custom_functions: list[lsp.CompletionItem] = [] + + @property + def custom_functions(self) -> list[lsp.CompletionItem]: + return self._custom_functions + + def visit_command(self, command): + routine = command.routine + + if routine.contents == "proc" and command.args: + first_arg = command.args[0] + if hasattr(first_arg, "value"): + self._custom_functions.append( + lsp.CompletionItem( + label=first_arg.value, kind=lsp.CompletionItemKind.Function + ) + ) diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py index 07c10a7..7e81eb2 100644 --- a/server/src/tools/parser.py +++ b/server/src/tools/parser.py @@ -1,49 +1,14 @@ -import textwrap -from tclint.parser import Parser +from tclint.parser import Parser, _strip_ws from tclint.lexer import ( - STATE_BRACEDWORD, - TOK_LBRACE, + TOK_WS, + TOK_BACKSLASH_NEWLINE, TOK_EOF, - TOK_RBRACE, + TOK_RPAREN, + TOK_NEWLINE, TclSyntaxError, ) -from tclint.syntax_tree import BracedWord +from tclint.syntax_tree import BareWord, TernaryOp, BinaryOp class CustomParser(Parser): - def parse_braced_word(self, ts): - """ - Ersetzt BracedWord durch echtes Script, wenn mehrzeilig. - """ - pos = ts.pos() - ts.lexer.push_state(STATE_BRACEDWORD) - ts.assert_(TOK_LBRACE) - - content = "" - expected = [pos] - while True: - t = ts.type() - if t == TOK_EOF: - raise TclSyntaxError( - "reached EOF without finding match for brace", - expected[-1], - ts.pos(), - ) - if t == TOK_LBRACE: - expected.append(ts.pos()) - elif t == TOK_RBRACE: - expected.pop() - if not expected: - ts.lexer.pop_state() - ts.next() - break - content += ts.value() - ts.next() - - end_pos = ts.pos() - # Mehrzeilig? Dann als Script parsen: - if "\n" in content.strip(): - self.parse_script(content) - - # Einzeilig: unverändert als Literal - return BracedWord(content, pos=pos, end_pos=end_pos) + pass diff --git a/server/src/tools/semantic_tokens.py b/server/src/tools/semantic_tokens.py index ffc80ac..0a5fff1 100644 --- a/server/src/tools/semantic_tokens.py +++ b/server/src/tools/semantic_tokens.py @@ -1,76 +1,116 @@ -from tclint.parser import Parser -from tclint.syntax_tree import Visitor, BareWord, VarSub, Comment, Command, Function - -TOKEN_TYPES = { - "command": 0, - "variable": 1, - "function": 2, - "string": 3, - "number": 4, - "keyword": 5, - "comment": 6, -} +import enum +from typing import List +from tclint.syntax_tree import Visitor +from tclint.commands import get_commands +import attrs +from common.load_data import standard_items +import lsprotocol.types as lsp -class SemanticTokenCollector(Visitor): - def __init__(self): - self.tokens = [] - - def _add_token(self, node, token_type): - if not node.pos or not node.end_pos: - return - - line, col = node.pos - end_line, end_col = node.end_pos - length = (end_col - col) if line == end_line else 1 - - self.tokens.append((line - 1, col - 1, length, token_type, 0)) - - def visit_command(self, command: Command): - self._add_token(command.routine, "command") - for arg in command.args: - arg.accept(self, recurse=True) - - def visit_comment(self, comment: Comment): - self._add_token(comment, "comment") - - def visit_bare_word(self, word: BareWord): - if word.value.isdigit(): - self._add_token(word, "number") - else: - self._add_token(word, "string") - - def visit_var_sub(self, var_sub: VarSub): - self._add_token(var_sub, "variable") - - def visit_function(self, function: Function): - self._add_token(function.name, "function") - for arg in function.args: - arg.accept(self, recurse=True) +class TokenModifier(enum.IntFlag): + deprecated = enum.auto() + readonly = enum.auto() + defaultLibrary = enum.auto() + definition = enum.auto() -def collect_semantic_tokens(code: str): - parser = Parser() - tree = parser.parse(code) - visitor = SemanticTokenCollector() - tree.accept(visitor, recurse=True) - return visitor.tokens +@attrs.define +class Token: + line: int + offset: int + lenght: int + + tok_type: str = "" + tok_modifiers: List[TokenModifier] = attrs.field(factory=list) -def encode_tokens(tokens): - tokens.sort() - encoded = [] +TOKEN_TYPES = [ + "keyword", + "variable", + "function", + "operator", + "parameter", + "type", + "class", +] - last_line = 0 - last_char = 0 - for line, char, length, token_type, modifiers in tokens: - delta_line = line - last_line - delta_start = char - last_char if delta_line == 0 else char +class _Highlighter(Visitor): + def __init__(self, plugins, log_to_output): + self._commands = get_commands(plugins) + self._tokens = [] + self.log_to_output = log_to_output - encoded.extend([delta_line, delta_start, length, token_type, modifiers]) + def visit_command(self, command): + routine = command.routine + self.log_to_output(str(command.routine)) + if routine.contents in self._commands: + line, col = routine.pos + self._tokens.append(((line - 1, col - 1), len(routine.contents), "keyword")) - last_line = line - last_char = char if delta_line == 0 else 0 + if routine.contents == "set" and command.args: + first_arg = command.args[0] + if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): + line, col = first_arg.pos + self._tokens.append( + (((line - 1, col - 1), len(first_arg.value), "variable")) + ) + if routine.contents == "proc" and command.args: + first_arg = command.args[0] + if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): + line, col = first_arg.pos + self._tokens.append( + (((line - 1, col - 1), len(first_arg.value), "function")) + ) - return encoded + if routine.contents == "namespace" and command.args: + first_arg = command.args[1] + if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): + line, col = first_arg.pos + self._tokens.append( + (((line - 1, col - 1), len(first_arg.value), "class")) + ) + + def visit_var_sub(self, var_sub): + self.log_to_output(str(var_sub)) + pass + + def tokens(self) -> list[Token]: + """Encode tokens as described in + https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens. + """ + tokens = [] + last_line = 0 + last_col = 0 + for (line, col), length, tok_type in sorted(self._tokens, key=lambda x: x[0]): + line_delta = line - last_line + col_delta = col + if line == last_line: + col_delta -= last_col + + tokens.append(Token(line_delta, col_delta, length, tok_type)) + last_line, last_col = line, col + + return tokens + + +# @server.feature( +# lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL, +# lsp.SemanticTokensLegend(token_types=["keyword"], token_modifiers=[]), +# ) +# def semantic_tokens(ls: TclspServer, params: lsp.SemanticTokensParams): +# logging.debug("Received %s: %s", lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL, params) +# document = ls.workspace.get_text_document(params.text_document.uri) + +# path = Path(document.path) +# root = ls.get_root(path) +# config = ls.get_config(path, root) + +# plugins = [config.commands] if config.commands is not None else [] +# parser = Parser(command_plugins=plugins) +# hl = _Highlighter(plugins) + +# tree = parser.parse(document.source) +# tree.accept(hl, recurse=True) + +# return lsp.SemanticTokens(data=hl.tokens()) -- 2.54.0 From fed8b84b4689ba37904f3ba237e0a6e0f0539385 Mon Sep 17 00:00:00 2001 From: christoph_xd Date: Wed, 30 Jul 2025 22:10:27 +0200 Subject: [PATCH 3/7] add more semantic tokens --- server/src/lsp_server.py | 6 +- server/src/tools/parser.py | 221 +++++++++++++++++++++++++++- server/src/tools/semantic_tokens.py | 30 +++- test/test.tcl | 17 +-- 4 files changed, 253 insertions(+), 21 deletions(-) diff --git a/server/src/lsp_server.py b/server/src/lsp_server.py index 4e2b91b..7a157fd 100644 --- a/server/src/lsp_server.py +++ b/server/src/lsp_server.py @@ -52,7 +52,7 @@ from tclint.format import Formatter, FormatterOpts from tclint.violations import Violation from plugins.poco_plugin import commands -from tools import checks +from tools import checks, parser from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier from tools.completion_items import _Completion @@ -62,7 +62,7 @@ DIAGNOSTIC_SOURCE = "nx-post-support" class TclLanguageServer(server.LanguageServer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.parser = Parser() + self.parser = parser.CustomParser() # Parser() for command in commands: self.parser._commands.update(command) self.diagnostics = {} @@ -201,6 +201,8 @@ def did_open(params: lsp.DidOpenTextDocumentParams) -> None: def did_save(params: lsp.DidSaveTextDocumentParams) -> None: """LSP handler for textDocument/didSave request.""" document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) + tree = LSP_SERVER.parser.parse(document.source) + log_to_output(tree.pretty(2)) @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_CLOSE) diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py index 7e81eb2..b57a43d 100644 --- a/server/src/tools/parser.py +++ b/server/src/tools/parser.py @@ -1,14 +1,225 @@ -from tclint.parser import Parser, _strip_ws +from tclint.parser import ( + Parser, + _is_bool_literal, + _is_float_literal, + _is_float_prefix, + _is_function, + _is_int_literal, + _is_int_prefix, +) from tclint.lexer import ( - TOK_WS, + STATE_BRACEDWORD, TOK_BACKSLASH_NEWLINE, TOK_EOF, - TOK_RPAREN, + TOK_LBRACE, + TOK_RBRACE, + TOK_WS, TOK_NEWLINE, + TOK_ALPHA_CHARS, + TOK_RPAREN, + TOK_LPAREN, + TOK_DOLLAR, + TOK_QUOTE, + TOK_LBRACKET, + TOK_NUM_CHARS, TclSyntaxError, ) -from tclint.syntax_tree import BareWord, TernaryOp, BinaryOp +from tclint.syntax_tree import ( + BracedWord, + BareWord, + BinaryOp, + TernaryOp, + ParenExpression, + UnaryOp, +) class CustomParser(Parser): - pass + def _strip_ws(parse_func): + """Decorator used by expression parser for stripping whitespace around a node.""" + + def func(parser, ts): + while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: + ts.next() + + node = parse_func(parser, ts) + + while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: + ts.next() + + return node + + return func + + @_strip_ws + def _parse_expression(self, ts): + op1 = self._parse_operand(ts) + expr = op1 + + # last condition is hack to break out of expression in case we're in ternary op + if ts.type() not in {TOK_EOF, TOK_RPAREN} and ts.value() not in {":", ","}: + if ts.value() == "?": + # weird hack to record operator + start = ts.pos() + ts.next() + q = BareWord("?", pos=start, end_pos=ts.pos()) + + op2 = self._parse_expression(ts) + if ts.value() != ":": + start = ts.pos() + ts.next() + end = ts.pos() + raise TclSyntaxError( + "expected ':' to continue ternary expression", start, end + ) + + # weird hack again + start = ts.pos() + ts.next() + colon = BareWord(":", pos=start, end_pos=ts.pos()) + + op3 = self._parse_expression(ts) + expr = TernaryOp( + op1, q, op2, colon, op3, pos=op1.pos, end_pos=op3.end_pos + ) + else: + operator = self._parse_operator(ts) + op2 = self._parse_expression(ts) + expr = BinaryOp(op1, operator, op2, pos=op1.pos, end_pos=op2.end_pos) + + if ts.type() != TOK_RPAREN and ts.value() not in {":", ","}: + ts.expect(TOK_EOF, message="expected end of expression", pos=ts.pos()) + + return expr + + @_strip_ws + def _parse_operand(self, ts): + if ts.type() == TOK_DOLLAR: + return self.parse_var_sub(ts) + if ts.type() == TOK_QUOTE: + return self.parse_quoted_word(ts) + if ts.type() == TOK_LBRACE: + return self.parse_braced_word(ts) + if ts.type() == TOK_LBRACKET: + return self.parse_command_sub(ts) + if ts.type() == TOK_LPAREN: + start = ts.pos() + ts.next() + expr = self._parse_expression(ts) + ts.expect( + TOK_RPAREN, + message="reached EOF without finding match for paren", + pos=expr.pos, + ) + end = ts.pos() + return ParenExpression(expr, start, end) + if ts.value() in {"-", "+", "~", "!"}: + operator_val = ts.value() + operator_pos = ts.pos() + ts.next() + operator = BareWord(operator_val, pos=operator_pos, end_pos=ts.pos()) + operand = self._parse_operand(ts) + # Since _parse_operand() munches whitespace after the operand, we + # set the end of the UnaryOp to the end of the operand rather than + # ts.pos(). Otherwise, the bounds of the UnaryOp would include all + # that whitespace. + return UnaryOp(operator, operand, pos=operator_pos, end_pos=operand.end_pos) + + # If none of these, collect tokens that may comprise an operand + operand = "" + operand_pos = ts.pos() + + # First, we want to check for numeric operands (either ints or numeric + # floats) by consuming tokens as long as they comprise the prefix of a + # numeric operand + while ts.type() != TOK_EOF and ( + _is_int_prefix(operand + ts.value()) + or _is_float_prefix(operand + ts.value()) + ): + operand += ts.value() + ts.next() + + # Next, we check if we've consumed an entire numeric literal. If so, we + # move on. If not, we keep consuming tokens that may correspond to a + # valid bareword (pretty much just alphanumeric chars). + if not (_is_int_literal(operand) or _is_float_literal(operand)): + while ts.type() in {TOK_ALPHA_CHARS, TOK_NUM_CHARS}: + operand += ts.value() + ts.next() + + # The above method is a little hacky. Note that it doesn't parse things + # exactly the same as Tcl. E.g. if a script includes `expr {1foo}`, + # tclint will report an invalid operator "foo", whereas tclsh will + # report an invalid bareword "1foo". Despite reporting them differently + # both tools should still catch the same syntax errors, since there are + # no legal barewords that begin with a numeric literal prefix, and tclsh + # will stop parsing numeric operands if they're actually followed by a + # legal operator (e.g. `expr {1eq1}` will be handled properly). + + is_func = _is_function(operand) + + if not ( + _is_int_literal(operand) + or _is_float_literal(operand) + or _is_bool_literal(operand) + or is_func + ): + raise TclSyntaxError( + f"invalid bareword in expression: {operand}", operand_pos, ts.pos() + ) + + node = BareWord(operand, pos=operand_pos, end_pos=ts.pos()) + + if is_func: + node = self._parse_function(ts, node) + + return node + + def parse_braced_word(self, ts): + self.debug(f"parse_braced_word({ts.current})") + pos = ts.pos() + + ts.lexer.push_state(STATE_BRACEDWORD) + + ts.assert_(TOK_LBRACE) + + word = "" + expected_braces = [pos] # Stack für geschachtelte Klammern + + while True: + toktype = ts.type() + if toktype == TOK_EOF: + raise TclSyntaxError( + "reached EOF without finding match for brace", + expected_braces[-1], + ts.pos(), + ) + + if toktype == TOK_BACKSLASH_NEWLINE: + # TCL-spezifisch: Zeilenumbruch mit Backslash ignorieren + ts.next() + continue + + if toktype == TOK_LBRACE: + expected_braces.append(ts.pos()) + elif toktype == TOK_RBRACE: + try: + expected_braces.pop() + except IndexError: + start = ts.pos() + ts.next() + end = ts.pos() + raise TclSyntaxError( + "found closing brace without matching open brace", start, end + ) + + if len(expected_braces) == 0: + ts.lexer.pop_state() + ts.next() + break + + word += ts.value() + ts.next() + + end_pos = ts.pos() + return BracedWord(word, pos=pos, end_pos=end_pos) diff --git a/server/src/tools/semantic_tokens.py b/server/src/tools/semantic_tokens.py index 0a5fff1..543dee1 100644 --- a/server/src/tools/semantic_tokens.py +++ b/server/src/tools/semantic_tokens.py @@ -3,8 +3,6 @@ from typing import List from tclint.syntax_tree import Visitor from tclint.commands import get_commands import attrs -from common.load_data import standard_items -import lsprotocol.types as lsp class TokenModifier(enum.IntFlag): @@ -63,6 +61,34 @@ class _Highlighter(Visitor): (((line - 1, col - 1), len(first_arg.value), "function")) ) + if len(command.args) >= 2: + param_list = command.args[1] + + # BracedWord oder Liste erwartet + if hasattr(param_list, "children"): + for child in param_list.children: + # Parameter kann einfaches Wort sein + if hasattr(child, "value") and child.value is not None: + line, col = child.pos + self._tokens.append( + ((line - 1, col - 1), len(child.value), "variable") + ) + + # Parameter mit Default-Wert ist meist eine List (z. B. {arg default}) + elif hasattr(child, "children") and len(child.children) >= 1: + name_node = child.children[0] + if hasattr(name_node, "value") and hasattr( + name_node, "pos" + ): + line, col = name_node.pos + self._tokens.append( + ( + (line - 1, col - 1), + len(name_node.value), + "variable", + ) + ) + if routine.contents == "namespace" and command.args: first_arg = command.args[1] if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): diff --git a/test/test.tcl b/test/test.tcl index 8779fbf..bbbc3b2 100644 --- a/test/test.tcl +++ b/test/test.tcl @@ -1,13 +1,6 @@ -proc myProc {arg {opt 1}} { - -} -set myVar 1 - -namespace eval myNameSpace { - proc namespaceProc {} {} -} - -set result [myNameSpace::namespaceProc] - -MOM_abort_program "Test" \ No newline at end of file +set main 1 +if {$main == 1 \ +&& 1 == 1} { + puts "main" +} {} -- 2.54.0 From d90f9b5425bb4a184375ee8948383d44b1d7431c Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Thu, 31 Jul 2025 09:38:02 +0200 Subject: [PATCH 4/7] adjsut completion Items --- server/src/lsp_server.py | 9 +++++---- server/src/tools/completion_items.py | 4 +++- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/server/src/lsp_server.py b/server/src/lsp_server.py index 4e2b91b..1d5893d 100644 --- a/server/src/lsp_server.py +++ b/server/src/lsp_server.py @@ -213,6 +213,10 @@ def did_change(params: lsp.DidChangeTextDocumentParams) -> None: """LSP handler for textDocument/didChange request""" document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) LSP_SERVER.compute_diagnostics(document) + ci = _Completion() + tree = LSP_SERVER.parser.parse(document.source) + tree.accept(ci, recurse=True) + LSP_SERVER._custom_functions = ci.custom_functions @LSP_SERVER.feature( @@ -243,15 +247,12 @@ def document_diagnostic(params: lsp.DocumentDiagnosticParams): @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_COMPLETION) def on_completion(params: lsp.CompletionParams) -> list[lsp.CompletionItem]: document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) - ci = _Completion() - tree = LSP_SERVER.parser.parse(document.source) - tree.accept(ci, recurse=True) items = ( standard_items.tcl_keyword_list + standard_items.nx_procs + standard_items.nx_variables - + ci.custom_functions + + LSP_SERVER._custom_functions ) return lsp.CompletionList(is_incomplete=False, items=items) diff --git a/server/src/tools/completion_items.py b/server/src/tools/completion_items.py index 39fc1c9..0e5aba3 100644 --- a/server/src/tools/completion_items.py +++ b/server/src/tools/completion_items.py @@ -29,7 +29,9 @@ class _Completion(Visitor): if routine.contents == "proc" and command.args: first_arg = command.args[0] - if hasattr(first_arg, "value"): + if hasattr(first_arg, "value") and not any( + item.label == first_arg.value for item in self._custom_functions + ): self._custom_functions.append( lsp.CompletionItem( label=first_arg.value, kind=lsp.CompletionItemKind.Function -- 2.54.0 From 98e2d104a8835c909c356d71a2e885476372673e Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Thu, 31 Jul 2025 16:19:09 +0200 Subject: [PATCH 5/7] update lsp_server --- package.json | 2 +- server/src/common/completion_list.json | 25 +++ server/src/lsp_server.py | 12 +- server/src/tools/checks.py | 55 ++++++- server/src/tools/completion_items.py | 6 + server/src/tools/parser.py | 215 +++++++------------------ server/src/tools/semantic_tokens.py | 87 ++++++---- test/test.tcl | 97 ++++++++++- 8 files changed, 299 insertions(+), 200 deletions(-) diff --git a/package.json b/package.json index 19e7fee..79ca231 100644 --- a/package.json +++ b/package.json @@ -123,4 +123,4 @@ "prettier": "^3.4.2", "typescript": "^5.7.2" } -} +} \ No newline at end of file diff --git a/server/src/common/completion_list.json b/server/src/common/completion_list.json index aec2eb7..ce9021b 100644 --- a/server/src/common/completion_list.json +++ b/server/src/common/completion_list.json @@ -514,6 +514,31 @@ "set syslog [MOM_ask_syslog_name]" ] }, + { + "label": "MOM_ask_ude_info", + "kind": "function", + "description": "This command is used to retrieve the information about a user-defined event (UDE) of a specified object.", + "format": "MOM_ask_ude_info object_name object_type ", + "parameters": [ + { + "name": "Object_name", + "desc": "The name of the object to which the UDE is attached. It can be a group, an operation, a tool, a geometry, or a method." + }, + { + "name": "object_type", + "desc": "The type of the object to which the UDE is attached. The following types are available.(group || operation/oper || tool || geometry/geom || method/meth)" + }, + { + "name": "Start/End/\"\"", + "desc": "This parameter is optional. Indicates whether you want to retrieve the UDE attached to the start event, end event, or both. If it is empty, returns the UDEs attached to the specified operation." + } + ], + "returns": [ + "0 - The UDE information retrieval failed.", + "1 - The UDE information successfully retrieved." + ], + "example": [] + }, { "label": "MOM_cancel_suppress_force_once_per_event", "kind": "function", diff --git a/server/src/lsp_server.py b/server/src/lsp_server.py index 222703a..4e3d7d1 100644 --- a/server/src/lsp_server.py +++ b/server/src/lsp_server.py @@ -54,7 +54,7 @@ from tclint.violations import Violation from plugins.poco_plugin import commands from tools import checks, parser from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier -from tools.completion_items import _Completion +from tools.completion_items import completion DIAGNOSTIC_SOURCE = "nx-post-support" @@ -62,7 +62,7 @@ DIAGNOSTIC_SOURCE = "nx-post-support" class TclLanguageServer(server.LanguageServer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.parser = parser.CustomParser() # Parser() + self.parser = Parser() for command in commands: self.parser._commands.update(command) self.diagnostics = {} @@ -215,10 +215,10 @@ def did_change(params: lsp.DidChangeTextDocumentParams) -> None: """LSP handler for textDocument/didChange request""" document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) LSP_SERVER.compute_diagnostics(document) - ci = _Completion() + completion.reset() tree = LSP_SERVER.parser.parse(document.source) - tree.accept(ci, recurse=True) - LSP_SERVER._custom_functions = ci.custom_functions + tree.accept(completion, recurse=True) + log_to_output(tree.pretty(2)) @LSP_SERVER.feature( @@ -254,7 +254,7 @@ def on_completion(params: lsp.CompletionParams) -> list[lsp.CompletionItem]: standard_items.tcl_keyword_list + standard_items.nx_procs + standard_items.nx_variables - + LSP_SERVER._custom_functions + + completion.custom_functions ) return lsp.CompletionList(is_incomplete=False, items=items) diff --git a/server/src/tools/checks.py b/server/src/tools/checks.py index f714128..70d6886 100644 --- a/server/src/tools/checks.py +++ b/server/src/tools/checks.py @@ -1,4 +1,57 @@ +from enum import Enum +from tclint.syntax_tree import Visitor, BareWord, Command +from tclint.violations import Violation, Rule + + +class Rules(Enum): + VALIDATION = "validation" + OPTIONAL_ARG_POSITION = "optinal_args" + + def __str__(self): + return self.value + + +class CommandArgsCheck(Visitor): + def __init__(self): + self._violations = [] + + def check(self, _, tree): + self._violations.clear() + tree.accept(self, recurse=True) + return self._violations + + def visit_command(self, command: Command): + if ( + not hasattr(command.routine, "contents") + or command.routine.contents != "proc" + ): + return + if len(command.args) < 2: + return + + args_node = command.args[1] + if not hasattr(args_node, "children"): + return + + found_optional = False + for arg in args_node.children: + # Required argument + if isinstance(arg, BareWord): + if found_optional: + self._violations.append( + Violation( + Rules.OPTIONAL_ARG_POSITION, + "Required argument follows optional one", + arg.pos, + arg.end_pos, + ) + ) + # Optional argument + elif hasattr(arg, "children") and len(arg.children) >= 2: + found_optional = True + + def get_checkers(): - checkers = () + checkers = (CommandArgsCheck(),) return checkers diff --git a/server/src/tools/completion_items.py b/server/src/tools/completion_items.py index 0e5aba3..4001ae3 100644 --- a/server/src/tools/completion_items.py +++ b/server/src/tools/completion_items.py @@ -24,6 +24,9 @@ class _Completion(Visitor): def custom_functions(self) -> list[lsp.CompletionItem]: return self._custom_functions + def reset(self): + self._custom_functions = [] + def visit_command(self, command): routine = command.routine @@ -37,3 +40,6 @@ class _Completion(Visitor): label=first_arg.value, kind=lsp.CompletionItemKind.Function ) ) + + +completion = _Completion() diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py index b57a43d..5a10ce3 100644 --- a/server/src/tools/parser.py +++ b/server/src/tools/parser.py @@ -1,12 +1,4 @@ -from tclint.parser import ( - Parser, - _is_bool_literal, - _is_float_literal, - _is_float_prefix, - _is_function, - _is_int_literal, - _is_int_prefix, -) +from tclint.parser import Parser, _strip_ws from tclint.lexer import ( STATE_BRACEDWORD, TOK_BACKSLASH_NEWLINE, @@ -35,29 +27,21 @@ from tclint.syntax_tree import ( class CustomParser(Parser): - def _strip_ws(parse_func): - """Decorator used by expression parser for stripping whitespace around a node.""" - - def func(parser, ts): - while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: - ts.next() - - node = parse_func(parser, ts) - - while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: - ts.next() - - return node - - return func - @_strip_ws def _parse_expression(self, ts): op1 = self._parse_operand(ts) expr = op1 + # Add TOK_BACKSLASH_NEWLINE to the tokens we need to skip + while ts.type() == TOK_BACKSLASH_NEWLINE: + ts.next() + # last condition is hack to break out of expression in case we're in ternary op - if ts.type() not in {TOK_EOF, TOK_RPAREN} and ts.value() not in {":", ","}: + if ts.type() not in { + TOK_EOF, + TOK_RPAREN, + TOK_BACKSLASH_NEWLINE, + } and ts.value() not in {":", ","}: if ts.value() == "?": # weird hack to record operator start = ts.pos() @@ -84,142 +68,67 @@ class CustomParser(Parser): ) else: operator = self._parse_operator(ts) + while ts.type() == TOK_BACKSLASH_NEWLINE: + ts.next() op2 = self._parse_expression(ts) expr = BinaryOp(op1, operator, op2, pos=op1.pos, end_pos=op2.end_pos) - if ts.type() != TOK_RPAREN and ts.value() not in {":", ","}: + if ts.type() not in ( + TOK_RPAREN, + TOK_BACKSLASH_NEWLINE, + ) and ts.value() not in {":", ","}: ts.expect(TOK_EOF, message="expected end of expression", pos=ts.pos()) return expr - @_strip_ws - def _parse_operand(self, ts): - if ts.type() == TOK_DOLLAR: - return self.parse_var_sub(ts) - if ts.type() == TOK_QUOTE: - return self.parse_quoted_word(ts) - if ts.type() == TOK_LBRACE: - return self.parse_braced_word(ts) - if ts.type() == TOK_LBRACKET: - return self.parse_command_sub(ts) - if ts.type() == TOK_LPAREN: - start = ts.pos() - ts.next() - expr = self._parse_expression(ts) - ts.expect( - TOK_RPAREN, - message="reached EOF without finding match for paren", - pos=expr.pos, - ) - end = ts.pos() - return ParenExpression(expr, start, end) - if ts.value() in {"-", "+", "~", "!"}: - operator_val = ts.value() - operator_pos = ts.pos() - ts.next() - operator = BareWord(operator_val, pos=operator_pos, end_pos=ts.pos()) - operand = self._parse_operand(ts) - # Since _parse_operand() munches whitespace after the operand, we - # set the end of the UnaryOp to the end of the operand rather than - # ts.pos(). Otherwise, the bounds of the UnaryOp would include all - # that whitespace. - return UnaryOp(operator, operand, pos=operator_pos, end_pos=operand.end_pos) - - # If none of these, collect tokens that may comprise an operand - operand = "" - operand_pos = ts.pos() - - # First, we want to check for numeric operands (either ints or numeric - # floats) by consuming tokens as long as they comprise the prefix of a - # numeric operand - while ts.type() != TOK_EOF and ( - _is_int_prefix(operand + ts.value()) - or _is_float_prefix(operand + ts.value()) - ): - operand += ts.value() - ts.next() - - # Next, we check if we've consumed an entire numeric literal. If so, we - # move on. If not, we keep consuming tokens that may correspond to a - # valid bareword (pretty much just alphanumeric chars). - if not (_is_int_literal(operand) or _is_float_literal(operand)): - while ts.type() in {TOK_ALPHA_CHARS, TOK_NUM_CHARS}: - operand += ts.value() - ts.next() - - # The above method is a little hacky. Note that it doesn't parse things - # exactly the same as Tcl. E.g. if a script includes `expr {1foo}`, - # tclint will report an invalid operator "foo", whereas tclsh will - # report an invalid bareword "1foo". Despite reporting them differently - # both tools should still catch the same syntax errors, since there are - # no legal barewords that begin with a numeric literal prefix, and tclsh - # will stop parsing numeric operands if they're actually followed by a - # legal operator (e.g. `expr {1eq1}` will be handled properly). - - is_func = _is_function(operand) - - if not ( - _is_int_literal(operand) - or _is_float_literal(operand) - or _is_bool_literal(operand) - or is_func - ): - raise TclSyntaxError( - f"invalid bareword in expression: {operand}", operand_pos, ts.pos() - ) - - node = BareWord(operand, pos=operand_pos, end_pos=ts.pos()) - - if is_func: - node = self._parse_function(ts, node) - - return node - - def parse_braced_word(self, ts): - self.debug(f"parse_braced_word({ts.current})") + def _parse_operator(self, ts): pos = ts.pos() - ts.lexer.push_state(STATE_BRACEDWORD) + # hacky logic to handle parsing legal operators - ts.assert_(TOK_LBRACE) - - word = "" - expected_braces = [pos] # Stack für geschachtelte Klammern - - while True: - toktype = ts.type() - if toktype == TOK_EOF: - raise TclSyntaxError( - "reached EOF without finding match for brace", - expected_braces[-1], - ts.pos(), - ) - - if toktype == TOK_BACKSLASH_NEWLINE: - # TCL-spezifisch: Zeilenumbruch mit Backslash ignorieren - ts.next() - continue - - if toktype == TOK_LBRACE: - expected_braces.append(ts.pos()) - elif toktype == TOK_RBRACE: - try: - expected_braces.pop() - except IndexError: - start = ts.pos() - ts.next() - end = ts.pos() - raise TclSyntaxError( - "found closing brace without matching open brace", start, end - ) - - if len(expected_braces) == 0: - ts.lexer.pop_state() - ts.next() - break - - word += ts.value() + # Skip any backslash-newlines before the operator + while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: ts.next() - end_pos = ts.pos() - return BracedWord(word, pos=pos, end_pos=end_pos) + if ts.value() in {"&&", "and"}: # Add explicit handling for logical AND + operator = ts.value() + ts.next() + return BareWord(operator, pos=pos, end_pos=ts.pos()) + elif ts.value() in {"*", "&", "|"}: + # one or two of these characters are legal operators + operator = ts.value() + ts.next() + if ts.value() == operator: + operator += ts.value() + ts.next() + elif ts.value() in {"<", ">"}: + operator = ts.value() + ts.next() + if ts.value() in {operator, "="}: + operator += ts.value() + ts.next() + elif ts.value() in {"=", "!"}: + operator = ts.value() + ts.next() + if ts.value() != "=": + raise TclSyntaxError( + f"invalid operator in expression: {operator}", pos, ts.pos() + ) + operator += ts.value() + ts.next() + elif ts.value() in {"*", "/", "%", "+", "-", "^", "eq", "ne", "in", "ni"}: + operator = ts.value() + ts.next() + else: + while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: + ts.next() + + if ts.value() in {"&&", "and"}: # Try again after whitespace + operator = ts.value() + ts.next() + else: + raise TclSyntaxError( + f"invalid operator in expression: {ts.value()}", pos, ts.pos() + ) + + return BareWord(operator, pos=pos, end_pos=ts.pos()) diff --git a/server/src/tools/semantic_tokens.py b/server/src/tools/semantic_tokens.py index 543dee1..fff965e 100644 --- a/server/src/tools/semantic_tokens.py +++ b/server/src/tools/semantic_tokens.py @@ -1,8 +1,10 @@ import enum from typing import List -from tclint.syntax_tree import Visitor +from tclint.syntax_tree import Visitor, QuotedWord, Command, BareWord from tclint.commands import get_commands import attrs +from common.load_data import standard_items +from tools.completion_items import completion class TokenModifier(enum.IntFlag): @@ -10,6 +12,7 @@ class TokenModifier(enum.IntFlag): readonly = enum.auto() defaultLibrary = enum.auto() definition = enum.auto() + declaration = enum.auto() @attrs.define @@ -30,6 +33,8 @@ TOKEN_TYPES = [ "parameter", "type", "class", + "string", + "parameter", ] @@ -39,26 +44,52 @@ class _Highlighter(Visitor): self._tokens = [] self.log_to_output = log_to_output - def visit_command(self, command): + def visit_quoted_word(self, word: QuotedWord): + if not word.contents: + return + line, col = word.contents_pos + self._tokens.append(((line - 1, col - 1), len(word.contents), "string", [])) + pass + + def visit_bare_word(self, word: BareWord): + if any(item.label == word.value for item in standard_items.nx_procs) or any( + item.label == word.value for item in completion.custom_functions + ): + line, col = word.pos + self._tokens.append( + (((line - 1, col - 1), len(word.value), "function", [])) + ) + + def visit_command(self, command: Command): routine = command.routine - self.log_to_output(str(command.routine)) - if routine.contents in self._commands: - line, col = routine.pos - self._tokens.append(((line - 1, col - 1), len(routine.contents), "keyword")) if routine.contents == "set" and command.args: first_arg = command.args[0] if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): line, col = first_arg.pos self._tokens.append( - (((line - 1, col - 1), len(first_arg.value), "variable")) + ( + ( + (line - 1, col - 1), + len(first_arg.value), + "variable", + [TokenModifier.declaration], + ) + ) ) if routine.contents == "proc" and command.args: first_arg = command.args[0] if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): line, col = first_arg.pos self._tokens.append( - (((line - 1, col - 1), len(first_arg.value), "function")) + ( + ( + (line - 1, col - 1), + len(first_arg.value), + "function", + [TokenModifier.declaration], + ) + ) ) if len(command.args) >= 2: @@ -71,7 +102,12 @@ class _Highlighter(Visitor): if hasattr(child, "value") and child.value is not None: line, col = child.pos self._tokens.append( - ((line - 1, col - 1), len(child.value), "variable") + ( + (line - 1, col - 1), + len(child.value), + "parameter", + [TokenModifier.declaration], + ) ) # Parameter mit Default-Wert ist meist eine List (z. B. {arg default}) @@ -85,7 +121,8 @@ class _Highlighter(Visitor): ( (line - 1, col - 1), len(name_node.value), - "variable", + "parameter", + [TokenModifier.declaration], ) ) @@ -94,7 +131,7 @@ class _Highlighter(Visitor): if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): line, col = first_arg.pos self._tokens.append( - (((line - 1, col - 1), len(first_arg.value), "class")) + (((line - 1, col - 1), len(first_arg.value), "class", [])) ) def visit_var_sub(self, var_sub): @@ -108,35 +145,15 @@ class _Highlighter(Visitor): tokens = [] last_line = 0 last_col = 0 - for (line, col), length, tok_type in sorted(self._tokens, key=lambda x: x[0]): + for (line, col), length, tok_type, tok_modifier in sorted( + self._tokens, key=lambda x: x[0] + ): line_delta = line - last_line col_delta = col if line == last_line: col_delta -= last_col - tokens.append(Token(line_delta, col_delta, length, tok_type)) + tokens.append(Token(line_delta, col_delta, length, tok_type, tok_modifier)) last_line, last_col = line, col return tokens - - -# @server.feature( -# lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL, -# lsp.SemanticTokensLegend(token_types=["keyword"], token_modifiers=[]), -# ) -# def semantic_tokens(ls: TclspServer, params: lsp.SemanticTokensParams): -# logging.debug("Received %s: %s", lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL, params) -# document = ls.workspace.get_text_document(params.text_document.uri) - -# path = Path(document.path) -# root = ls.get_root(path) -# config = ls.get_config(path, root) - -# plugins = [config.commands] if config.commands is not None else [] -# parser = Parser(command_plugins=plugins) -# hl = _Highlighter(plugins) - -# tree = parser.parse(document.source) -# tree.accept(hl, recurse=True) - -# return lsp.SemanticTokens(data=hl.tokens()) diff --git a/test/test.tcl b/test/test.tcl index bbbc3b2..4aef606 100644 --- a/test/test.tcl +++ b/test/test.tcl @@ -1,6 +1,95 @@ - set main 1 -if {$main == 1 \ -&& 1 == 1} { +if {$main == 1 && 1 == 1} { puts "main" -} {} +} + +proc test {} { + puts "main" +} +LIB_GE_command_buffer_edit_insert MOM_tool_change_LIB TOOL_CHANGE_AUTO {CUSTOM_after_tool_change_call} mytag after @TOOL_CHANGE_AUTO + +MOM_abort + + +#_________________________________________________________________________________________________ +# +# Function to output a spacer line or empty line +#_________________________________________________________________________________________________ +proc SERVICE_spacer_output {type {length 20} {line_num 0} {output 1} check} { + LIB_GE_message [string repeat $type $length] "output_$output" $line_num +} + + + +#_________________________________________________________________________________________________ +# +# Function to delete the file +#_________________________________________________________________________________________________ +proc SERVICE_remove_file {file} { + if {![SERVICE_check_file_exists $file]} {return} + MOM_remove_file $file +} + +#_________________________________________________________________________________________________ +# +# Function to check if the file exists +#_________________________________________________________________________________________________ +proc SERVICE_check_file_exists {file} { + if {[file exists $file]} {return 1} + return 0 +} + +#_________________________________________________________________________________________________ +# +# Ask UDE Info for the Tool +#_________________________________________________________________________________________________ +proc SERVICE_ask_ude_tool {pos ude_name tool_name} { + MOM_ask_ude_info $tool_name "tool" $pos + + if {[lsearch $::mom_result $ude_name] != -1} { + return 1 + } + return 0 +} + +#_________________________________________________________________________________________________ +# +# Ask UDE Info for the Operation +#_________________________________________________________________________________________________ +proc SERVICE_ask_ude_operation {pos ude_name path_name} { + MOM_ask_ude_info $path_name "operation" $pos + + if {[lsearch $::mom_result $ude_name] != -1} { + return 1 + } + return 0 +} + +#_________________________________________________________________________________________________ +# +# output suppress or dont suppress +# [SERVICE_output_handling "ingore_output"] ignores the output +# arg options: ingore_output +# restore +#_________________________________________________________________________________________________ +proc SERVICE_output_handling {handler} { + set ::lib_ge(hidden_output) $handler +} + +#_________________________________________________________________________________________________ +# +# write the mom_tool_data to store tool information +# this function is called in start of program +#_________________________________________________________________________________________________ +proc SERVICE_get_tool_data {} { + global mom_tool_data + global mom_operation_info + + set mom_tool_data(toollist) "" + set operations $::mom_operation_name_list + foreach operation $operations { + if {[lsearch -exact $mom_tool_data(toollist) $mom_operation_info($operation,tool_name)] == -1} { + lappend mom_tool_data(toollist) $mom_operation_info($operation,tool_name) + } + } +} -- 2.54.0 From 5e2e85830bf4cdedf3b1401605eb140b5e7bbc28 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Fri, 1 Aug 2025 13:35:02 +0200 Subject: [PATCH 6/7] update --- client/src/extension.ts | 10 ++-- server/src/lsp_server.py | 25 +++++++++ server/src/tools/completion_items.py | 27 ++++++++- server/src/tools/inlay_hint.py | 29 ++++++++++ server/src/tools/symbols.py | 82 ++++++++++++++++++++++++++++ test/test.tcl | 14 ++++- 6 files changed, 178 insertions(+), 9 deletions(-) create mode 100644 server/src/tools/inlay_hint.py create mode 100644 server/src/tools/symbols.py diff --git a/client/src/extension.ts b/client/src/extension.ts index 0474159..a9ee2dd 100644 --- a/client/src/extension.ts +++ b/client/src/extension.ts @@ -179,11 +179,11 @@ export async function activate(context: vscode.ExtensionContext) { context.subscriptions.push(formatDefProvider) - const tclOutlineProvider = vscode.languages.registerDocumentSymbolProvider( - { scheme: "file", language: "tcl" }, - { provideDocumentSymbols: tclDocumentSymbolProvider } - ) - context.subscriptions.push(tclOutlineProvider) + // const tclOutlineProvider = vscode.languages.registerDocumentSymbolProvider( + // { scheme: "file", language: "tcl" }, + // { provideDocumentSymbols: tclDocumentSymbolProvider } + // ) + // context.subscriptions.push(tclOutlineProvider) // Diagnostics collection const diagnosticCollectionCdl = vscode.languages.createDiagnosticCollection("cdl") diff --git a/server/src/lsp_server.py b/server/src/lsp_server.py index 4e3d7d1..4a417d4 100644 --- a/server/src/lsp_server.py +++ b/server/src/lsp_server.py @@ -55,6 +55,8 @@ from plugins.poco_plugin import commands from tools import checks, parser from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier from tools.completion_items import completion +from tools.inlay_hint import InlayHintGenerator +from tools.symbols import OutlineVisitor DIAGNOSTIC_SOURCE = "nx-post-support" @@ -259,6 +261,29 @@ def on_completion(params: lsp.CompletionParams) -> list[lsp.CompletionItem]: return lsp.CompletionList(is_incomplete=False, items=items) +@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DOCUMENT_SYMBOL) +def document_symbols(params: lsp.DocumentSymbolParams): + doc = LSP_SERVER.workspace.get_text_document(params.text_document.uri) + tree = LSP_SERVER.parser.parse(doc.source) + + visitor = OutlineVisitor(LSP_SERVER.parser) + tree.accept(visitor, recurse=True) + + return visitor.stack[0] + + +@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_INLAY_HINT) +def inlay_hints(params: lsp.InlayHintParams): + document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) + tree = LSP_SERVER.parser.parse(document.source) + + # Inlay Hints sammeln + generator = InlayHintGenerator(completion.proc_signatures) + tree.accept(generator, recurse=True) + + return generator.hints + + @LSP_SERVER.feature( lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL, lsp.SemanticTokensLegend( diff --git a/server/src/tools/completion_items.py b/server/src/tools/completion_items.py index 4001ae3..9684812 100644 --- a/server/src/tools/completion_items.py +++ b/server/src/tools/completion_items.py @@ -1,4 +1,4 @@ -from tclint.syntax_tree import Visitor +from tclint.syntax_tree import Visitor, Command, BareWord, List import lsprotocol.types as lsp @@ -19,15 +19,21 @@ class _Completion(Visitor): def __init__(self): super().__init__() self._custom_functions: list[lsp.CompletionItem] = [] + self._proc_signatures = {} @property def custom_functions(self) -> list[lsp.CompletionItem]: return self._custom_functions + @property + def proc_signatures(self): + return self._proc_signatures + def reset(self): self._custom_functions = [] + self._proc_signatures = {} - def visit_command(self, command): + def visit_command(self, command: Command): routine = command.routine if routine.contents == "proc" and command.args: @@ -40,6 +46,23 @@ class _Completion(Visitor): label=first_arg.value, kind=lsp.CompletionItemKind.Function ) ) + if len(command.args) < 2: + return + + param_list_node = command.args[1] + if not hasattr(param_list_node, "children"): + return + + param_names = [] + for arg in param_list_node.children: + if isinstance(arg, BareWord): + param_names.append(arg.value) + elif isinstance(arg, List) and len(arg.children) >= 1: + first = arg.children[0] + if isinstance(first, BareWord): + param_names.append(first.value) + + self._proc_signatures[first_arg.value] = param_names completion = _Completion() diff --git a/server/src/tools/inlay_hint.py b/server/src/tools/inlay_hint.py new file mode 100644 index 0000000..ea4f3e6 --- /dev/null +++ b/server/src/tools/inlay_hint.py @@ -0,0 +1,29 @@ +import lsprotocol.types as lsp +from tclint.syntax_tree import Visitor, Command + + +class InlayHintGenerator(Visitor): + def __init__(self, proc_signatures): + self.proc_signatures = proc_signatures + self.hints = [] + + def visit_command(self, command: Command): + name = getattr(command.routine, "contents", None) + if name not in self.proc_signatures: + return + + param_names = self.proc_signatures[name] + for idx, arg in enumerate(command.args): + if idx >= len(param_names): + break + param_name = param_names[idx] + + if arg.pos: + line, col = arg.pos + self.hints.append( + lsp.InlayHint( + position=lsp.Position(line=line - 1, character=col - 1), + label=f"{param_name}:", + kind=lsp.InlayHintKind.Parameter, + ) + ) diff --git a/server/src/tools/symbols.py b/server/src/tools/symbols.py new file mode 100644 index 0000000..71b3635 --- /dev/null +++ b/server/src/tools/symbols.py @@ -0,0 +1,82 @@ +from tclint.syntax_tree import Visitor, BareWord, BracedWord, Script, Command +from tclint.parser import Parser +import lsprotocol.types as lsp + + +class OutlineVisitor(Visitor): + def __init__(self, parser: Parser): + self.parser = parser + self.stack = [[]] # Root symbol list + + def _range(self, node) -> lsp.Range: + line = node.line - 1 + col = node.col - 1 + if node.end_pos: + end_line = node.end_pos[0] - 1 + end_col = node.end_pos[1] - 1 + else: + end_line = line + end_col = col + 1 + + return lsp.Range( + start=lsp.Position(line=line, character=col), + end=lsp.Position(line=end_line, character=end_col), + ) + + def _add(self, name: str, kind: lsp.SymbolKind, node, children=None): + symbol = lsp.DocumentSymbol( + name=name, + kind=kind, + range=self._range(node), + selection_range=self._range(node), + children=children or [], + ) + self.stack[-1].append(symbol) + return symbol + + def visit_script(self, script): + for child in script.children: + child.accept(self, recurse=False) + + def visit_command(self, command: Command): + if not isinstance(command.routine, BareWord): + return + name = command.routine.contents + + # --- NAMESPACE EVAL --- + if name == "namespace" and len(command.args) >= 3: + subcmd = command.args[0] + if isinstance(subcmd, BareWord) and subcmd.contents == "eval": + ns_arg = command.args[1] + ns_name = ( + ns_arg.contents if isinstance(ns_arg, BareWord) else "" + ) + ns_body = command.args[2] + + ns_symbol = self._add( + ns_name, lsp.SymbolKind.Namespace, command, children=[] + ) + self.stack.append(ns_symbol.children) + + if isinstance(ns_body, BracedWord): + try: + subtree = self.parser.parse_script(ns_body) + subtree.accept(self, recurse=False) + except Exception as e: + print(f"Failed parsing namespace body: {e}") + + self.stack.pop() + + # --- PROC --- + elif name == "proc" and len(command.args) >= 1: + proc_arg = command.args[0] + proc_name = ( + proc_arg.contents if isinstance(proc_arg, BareWord) else "" + ) + self._add(proc_name, lsp.SymbolKind.Function, command) + + # --- SET --- + elif name == "set" and len(command.args) >= 1: + var_arg = command.args[0] + var_name = var_arg.contents if isinstance(var_arg, BareWord) else "" + self._add(var_name, lsp.SymbolKind.Variable, command) diff --git a/test/test.tcl b/test/test.tcl index 4aef606..f53d014 100644 --- a/test/test.tcl +++ b/test/test.tcl @@ -5,21 +5,31 @@ if {$main == 1 && 1 == 1} { proc test {} { puts "main" + proc llll {} {} + set rrrrrrr } LIB_GE_command_buffer_edit_insert MOM_tool_change_LIB TOOL_CHANGE_AUTO {CUSTOM_after_tool_change_call} mytag after @TOOL_CHANGE_AUTO MOM_abort +namespace eval myns { + proc add {a b} { + set sum [expr {$a + $b}] + return $sum + } + set config "debug" +} #_________________________________________________________________________________________________ # # Function to output a spacer line or empty line #_________________________________________________________________________________________________ -proc SERVICE_spacer_output {type {length 20} {line_num 0} {output 1} check} { +proc SERVICE_spacer_output {type {length 20} {line_num 0} {output 1}} { LIB_GE_message [string repeat $type $length] "output_$output" $line_num } - +SERVICE_spacer_output "*" 50 0 1 +SERVICE_remove_file $filename #_________________________________________________________________________________________________ # -- 2.54.0 From 0318532308609b6aa2ffa647b29621ce6528e3fe Mon Sep 17 00:00:00 2001 From: christoph_xd Date: Sun, 3 Aug 2025 21:04:14 +0200 Subject: [PATCH 7/7] add some features --- client/src/extension.ts | 10 +- server/src/common/formatter.py | 63 ---- server/src/lsp_server.py | 43 +-- server/src/tools/formatter.py | 0 server/src/tools/parser.py | 459 ++++++++++++++++++++-------- server/src/tools/semantic_tokens.py | 16 +- server/src/tools/symbols.py | 82 ----- test/test.tcl | 6 +- 8 files changed, 374 insertions(+), 305 deletions(-) delete mode 100644 server/src/common/formatter.py delete mode 100644 server/src/tools/formatter.py diff --git a/client/src/extension.ts b/client/src/extension.ts index a9ee2dd..0474159 100644 --- a/client/src/extension.ts +++ b/client/src/extension.ts @@ -179,11 +179,11 @@ export async function activate(context: vscode.ExtensionContext) { context.subscriptions.push(formatDefProvider) - // const tclOutlineProvider = vscode.languages.registerDocumentSymbolProvider( - // { scheme: "file", language: "tcl" }, - // { provideDocumentSymbols: tclDocumentSymbolProvider } - // ) - // context.subscriptions.push(tclOutlineProvider) + const tclOutlineProvider = vscode.languages.registerDocumentSymbolProvider( + { scheme: "file", language: "tcl" }, + { provideDocumentSymbols: tclDocumentSymbolProvider } + ) + context.subscriptions.push(tclOutlineProvider) // Diagnostics collection const diagnosticCollectionCdl = vscode.languages.createDiagnosticCollection("cdl") diff --git a/server/src/common/formatter.py b/server/src/common/formatter.py deleted file mode 100644 index 29f72eb..0000000 --- a/server/src/common/formatter.py +++ /dev/null @@ -1,63 +0,0 @@ -import re - - -def format_tcl(src: str, indent_str=" ") -> str: - """ - Simple Tcl formatter with special handling for: - 1) Single-line 'if {cond} {action}' blocks remain on one line. - 2) Combined closing-and-opening lines like '} else {' dedent then re-indent. - 3) Lines like '} Tag' stay on the same line: '} Tag'. - 4) Standard multi-line blocks for 'if', 'elseif', 'else', '{', '}'. - """ - level = 0 - out_lines = [] - - for raw_line in src.splitlines(): - stripped = raw_line.strip() - - # Comments are ignored - if stripped.startswith("#"): - out_lines.append(indent_str * level + stripped) - continue - - # Single-line 'if {cond} {action}' → no indent change - if re.match(r"^(if|elseif)\s*\{[^}]+\}\s*\{[^}]+\}$", stripped): - out_lines.append(indent_str * level + stripped) - continue - - # Combined '} else {' → dedent, print, then indent - if re.match(r"^\}\s*(elseif|else)\b.*\{$", stripped): - level = max(level - 1, 0) - out_lines.append(indent_str * level + stripped) - level += 1 - continue - - # SPECIAL: closing brace plus tag on same line: '} Tag' - m = re.match(r"^\}\s+(.+)", stripped) - if m and not stripped.startswith("#"): - # close one block - level = max(level - 1, 0) - # stay on one line: "} Tag" - out_lines.append(f"{indent_str * level}}} {m.group(1)}") - continue - - # Pure '}' → dedent then print - if stripped == "}": - level = max(level - 1, 0) - out_lines.append(f"{indent_str * level}{stripped}") - continue - - # 'elseif' or 'else' alone → align with matching 'if' - if re.match(r"^(elseif|else)\b(?!.*\{)", stripped): - level = max(level - 1, 0) - out_lines.append(f"{indent_str * level}{stripped}") - continue - - # Default: print at current indent - out_lines.append(f"{indent_str * level}{stripped}") - - # Open a new block on lines ending with '{' - if re.match(r"^(if|elseif)\b.*\{$", stripped) or stripped.endswith("{"): - level += 1 - - return "\n".join(out_lines) diff --git a/server/src/lsp_server.py b/server/src/lsp_server.py index 4a417d4..bd1aa55 100644 --- a/server/src/lsp_server.py +++ b/server/src/lsp_server.py @@ -4,16 +4,12 @@ from __future__ import annotations -import copy import json import os import pathlib import re import sys -import sysconfig -import traceback -from typing import Any, List, Optional, Sequence, Tuple -import re +from typing import Any, List, Optional, Tuple import operator from functools import reduce @@ -41,22 +37,18 @@ update_sys_path( # ********************************************************** # pylint: disable=wrong-import-position,import-error import lsp_jsonrpc as jsonrpc -import lsp_utils as utils import lsprotocol.types as lsp from pygls import server, uris, workspace from pygls.workspace.text_document import TextDocument from common.load_data import standard_items -from tclint.parser import Parser from tclint.lexer import TclSyntaxError from tclint.format import Formatter, FormatterOpts from tclint.violations import Violation - from plugins.poco_plugin import commands from tools import checks, parser from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier from tools.completion_items import completion from tools.inlay_hint import InlayHintGenerator -from tools.symbols import OutlineVisitor DIAGNOSTIC_SOURCE = "nx-post-support" @@ -64,7 +56,7 @@ DIAGNOSTIC_SOURCE = "nx-post-support" class TclLanguageServer(server.LanguageServer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.parser = Parser() + self.parser = parser.CustomParser() for command in commands: self.parser._commands.update(command) self.diagnostics = {} @@ -197,14 +189,15 @@ def did_open(params: lsp.DidOpenTextDocumentParams) -> None: """LSP handler for textDocument/didOpen request.""" document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) LSP_SERVER.compute_diagnostics(document) + completion.reset() + tree = LSP_SERVER.parser.parse(document.source) + tree.accept(completion, recurse=True) @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_SAVE) def did_save(params: lsp.DidSaveTextDocumentParams) -> None: """LSP handler for textDocument/didSave request.""" - document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) - tree = LSP_SERVER.parser.parse(document.source) - log_to_output(tree.pretty(2)) + _ = LSP_SERVER.workspace.get_text_document(params.text_document.uri) @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_CLOSE) @@ -220,7 +213,6 @@ def did_change(params: lsp.DidChangeTextDocumentParams) -> None: completion.reset() tree = LSP_SERVER.parser.parse(document.source) tree.accept(completion, recurse=True) - log_to_output(tree.pretty(2)) @LSP_SERVER.feature( @@ -250,7 +242,7 @@ def document_diagnostic(params: lsp.DocumentDiagnosticParams): @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_COMPLETION) def on_completion(params: lsp.CompletionParams) -> list[lsp.CompletionItem]: - document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) + _ = LSP_SERVER.workspace.get_text_document(params.text_document.uri) items = ( standard_items.tcl_keyword_list @@ -261,15 +253,10 @@ def on_completion(params: lsp.CompletionParams) -> list[lsp.CompletionItem]: return lsp.CompletionList(is_incomplete=False, items=items) -@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DOCUMENT_SYMBOL) -def document_symbols(params: lsp.DocumentSymbolParams): - doc = LSP_SERVER.workspace.get_text_document(params.text_document.uri) - tree = LSP_SERVER.parser.parse(doc.source) - - visitor = OutlineVisitor(LSP_SERVER.parser) - tree.accept(visitor, recurse=True) - - return visitor.stack[0] +# @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DOCUMENT_SYMBOL) +# def document_symbols(params: lsp.DocumentSymbolParams): +# doc = LSP_SERVER.workspace.get_text_document(params.text_document.uri) +# return [] @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_INLAY_HINT) @@ -277,7 +264,7 @@ def inlay_hints(params: lsp.InlayHintParams): document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) tree = LSP_SERVER.parser.parse(document.source) - # Inlay Hints sammeln + # collect Inlay Hints generator = InlayHintGenerator(completion.proc_signatures) tree.accept(generator, recurse=True) @@ -405,12 +392,12 @@ def formatting(params: lsp.DocumentFormattingParams) -> list[lsp.TextEdit] | Non start = lsp.Position(line=0, character=0) last_line = source.rsplit("\n", 1)[-1] end = lsp.Position(line=source.count("\n"), character=len(last_line)) - - formatted = LSP_SERVER.format(doc, params.options) + if GLOBAL_SETTINGS.get("formatter", True): + source = LSP_SERVER.format(doc, params.options) return [ lsp.TextEdit( range=lsp.Range(start=start, end=end), - new_text=formatted, + new_text=source, ) ] diff --git a/server/src/tools/formatter.py b/server/src/tools/formatter.py deleted file mode 100644 index e69de29..0000000 diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py index 5a10ce3..408ab28 100644 --- a/server/src/tools/parser.py +++ b/server/src/tools/parser.py @@ -1,134 +1,347 @@ -from tclint.parser import Parser, _strip_ws -from tclint.lexer import ( - STATE_BRACEDWORD, - TOK_BACKSLASH_NEWLINE, - TOK_EOF, - TOK_LBRACE, - TOK_RBRACE, - TOK_WS, - TOK_NEWLINE, - TOK_ALPHA_CHARS, - TOK_RPAREN, - TOK_LPAREN, - TOK_DOLLAR, - TOK_QUOTE, - TOK_LBRACKET, - TOK_NUM_CHARS, - TclSyntaxError, -) +from tclint.parser import Parser +from tclint.commands import CommandArgError from tclint.syntax_tree import ( BracedWord, BareWord, - BinaryOp, - TernaryOp, - ParenExpression, - UnaryOp, + BracedExpression, + List, + QuotedWord, + Expression, ) +import ply.lex as lex +from typing import Tuple + +TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE" +TOK_BACKSLASH_SUB = "BACKSLASH_SUB" +TOK_NEWLINE = "NEWLINE" +TOK_SEMI = "SEMI" +TOK_WS = "WS" +TOK_QUOTE = "QUOTE" +TOK_ARG_EXPANSION = "ARG_EXPANSION" +TOK_LBRACE = "LBRACE" +TOK_RBRACE = "RBRACE" +TOK_STAR = "STAR" +TOK_LBRACKET = "LBRACKET" +TOK_RBRACKET = "RBRACKET" +TOK_DOLLAR = "DOLLAR" +TOK_LPAREN = "LPAREN" +TOK_RPAREN = "RPAREN" +TOK_HASH = "HASH" +TOK_ALPHA_CHARS = "ALPHA_CHARS" +TOK_NUM_CHARS = "NUM_CHARS" +TOK_NAMESPACE_SEP = "NAMESPACE_SEP" +TOK_CHAR = "CHAR" +TOK_CONTENTS = "CONTENTS" +TOK_EOF = None + +STATE_BRACEDWORD = "bracedword" + + +class TclSyntaxError(Exception): + def __init__(self, message, start: Tuple[int, int], end: Tuple[int, int]): + super().__init__(message) + self.start = start + self.end = end + + +class _LexTable: + tokens = ( + TOK_BACKSLASH_NEWLINE, + TOK_BACKSLASH_SUB, + TOK_NEWLINE, + TOK_SEMI, + TOK_WS, + TOK_QUOTE, + TOK_ARG_EXPANSION, + TOK_LBRACE, + TOK_RBRACE, + TOK_STAR, + TOK_LBRACKET, + TOK_RBRACKET, + TOK_DOLLAR, + TOK_LPAREN, + TOK_RPAREN, + TOK_HASH, + TOK_ALPHA_CHARS, + TOK_NUM_CHARS, + TOK_NAMESPACE_SEP, + TOK_CHAR, + TOK_CONTENTS, + ) + + # This defines a conditional lexing state for parsing braced words. This is a + # performance optimization; since there are few special characters in this context, + # we can use a smaller set of tokens to parse them faster. This has a large impact + # since most Tcl programs have a large number of braced words. Any token with + # `bracedword` in its name is included in this state. Tokens that are included in + # this state and the default state also include `INITIAL` in their name. + states = ((STATE_BRACEDWORD, "exclusive"),) + + def _tok(self, t): + pos = (t.lexer.lineno, t.lexer.colno) + t.lexer.lineno += t.value.count("\n") + index = t.value.rfind("\n") + if index == -1: + t.lexer.colno += len(t.value) + else: + remaining = t.value[index + 1 :] + t.lexer.colno = len(remaining) + 1 + + t.value = (t.value, pos) + return t + + # Priority important + def t_bracedword_INITIAL_BACKSLASH_NEWLINE(self, t): + r"\\\r?\n" + return self._tok(t) + + # Priority important + def t_bracedword_INITIAL_BACKSLASH_SUB(self, t): + r"\\." + return self._tok(t) + + def t_NEWLINE(self, t): + r"\n" + return self._tok(t) + + def t_SEMI(self, t): + r";" + return self._tok(t) + + # TODO: should use \s? + def t_WS(self, t): + r"[\t\v\f\r ]+" + return self._tok(t) + + def t_QUOTE(self, t): + r'"' + return self._tok(t) + + # Must be higher priority than LBRACE + def t_ARG_EXPANSION(self, t): + r"\{\*\}" + return self._tok(t) + + def t_bracedword_INITIAL_LBRACE(self, t): + r"\{" + return self._tok(t) + + def t_bracedword_INITIAL_RBRACE(self, t): + r"\}" + return self._tok(t) + + def t_STAR(self, t): + r"\*" + return self._tok(t) + + def t_LBRACKET(self, t): + r"\[" + return self._tok(t) + + def t_RBRACKET(self, t): + r"\]" + return self._tok(t) + + def t_DOLLAR(self, t): + r"\$" + return self._tok(t) + + def t_LPAREN(self, t): + r"\(" + return self._tok(t) + + def t_RPAREN(self, t): + r"\)" + return self._tok(t) + + def t_HASH(self, t): + r"\#" + return self._tok(t) + + # Valid non-numeric chars in variable names + def t_ALPHA_CHARS(self, t): + r"[A-Za-z_]+" + return self._tok(t) + + # Valid numeric chars in variable names + # This is split up from the above to facilitate expression parsing, since + # e.g. 1eq1 can't be a single token. + def t_NUM_CHARS(self, t): + r"[0-9]+" + return self._tok(t) + + def t_NAMESPACE_SEP(self, t): + r"::+" + return self._tok(t) + + def t_bracedword_CONTENTS(self, t): + r"[^{}\\]+" + return self._tok(t) + + # Catch-all. TODO: inefficient, should probably munch multiple chars + def t_CHAR(self, t): + r"." + return self._tok(t) + + # Error handling rule + # TODO: do we need this? since we have a catch-all... + # there is a warning + def t_bracedword_INITIAL_error(self, t): + print("Illegal character '%s'" % t.value[0]) + t.lexer.skip(1) + + def __init__(self): + self.lexer = lex.lex(object=self) + self.lexer.lineno = 1 + self.lexer.colno = 1 + + def new_lexer(self, pos=None): + lexer = self.lexer.clone() + lexer.lineno = 1 + lexer.colno = 1 + + if pos is not None: + line, col = pos + lexer.lineno = line + lexer.colno = col + + return lexer + + +# Calling `lex.lex()` performs an expensive reflection process to generate the lexer. +# This singleton class holds a preinitialized lexer that can then be cloned to create +# individual instances. +LexTable = _LexTable() + + +class Lexer: + def __init__(self, pos=None): + self.lexer = LexTable.new_lexer(pos) + self.current = None + + def input(self, text): + self.lexer.input(text) + self.current = self.lexer.token() + + def type(self): + if self.current is None: + return TOK_EOF + return self.current.type + + def value(self): + if self.current is None: + return None + return self.current.value[0] + + def pos(self): + if self.current is None: + return (self.lexer.lineno, self.lexer.colno) + return self.current.value[1] + + def next(self): + self.current = self.lexer.token() + + def expect(self, *tokens, message, pos): + if self.type() not in tokens: + self.next() # munch another token to update position + raise TclSyntaxError(message, pos, self.pos()) + + self.next() + + def assert_(self, *tokens): + assert self.current.type in tokens + self.next() + class CustomParser(Parser): - @_strip_ws - def _parse_expression(self, ts): - op1 = self._parse_operand(ts) - expr = op1 + def parse(self, script, pos=None): + lexer = Lexer(pos=pos) + lexer.input(script) + tree = self._parse_script(lexer, in_command_sub=False) + assert lexer.type() == TOK_EOF, ( + "Didn't reach EOF parsing script, please file a bug report." + ) - # Add TOK_BACKSLASH_NEWLINE to the tokens we need to skip - while ts.type() == TOK_BACKSLASH_NEWLINE: - ts.next() + return tree - # last condition is hack to break out of expression in case we're in ternary op - if ts.type() not in { + def parse_list(self, node): + """Parse contents of node as Tcl list. This is a distinct entry point + that doesn't get used when generating the main syntax tree, but is used + in command-specific argument parsing. + """ + if isinstance(node, List): + return node + + if node.contents is None: + raise CommandArgError( + "expected braced word or word without substitutions in argument" + " interpreted as list" + ) + + ts = Lexer(pos=node.contents_pos) + ts.input(node.contents) + + DELIMITERS = {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE} + + list_node = List(pos=node.pos, end_pos=node.end_pos) + while ts.type() is not TOK_EOF: + while ts.type() in DELIMITERS: + ts.next() + + if ts.type() is TOK_EOF: + break + + if ts.type() == TOK_LBRACE: + # we can reuse parse_braced_word, since it doesn't use + # substitutions in any case + list_node.add(self.parse_braced_word(ts)) + elif ts.type() == TOK_QUOTE: + quote_word_pos = ts.pos() + + ts.assert_(TOK_QUOTE) + + bare_word_pos = ts.pos() + contents = "" + while ts.type() not in {TOK_QUOTE, TOK_EOF}: + contents += ts.value() + ts.next() + word = BareWord(contents, pos=bare_word_pos, end_pos=ts.pos()) + + ts.expect( + TOK_QUOTE, + message="reached EOF without finding match for quote", + pos=quote_word_pos, + ) + + list_node.add(QuotedWord(word, pos=quote_word_pos, end_pos=ts.pos())) + else: + pos = ts.pos() + contents = "" + while ts.type() not in {*DELIMITERS, TOK_EOF}: + contents += ts.value() + ts.next() + list_node.add(BareWord(contents, pos=pos, end_pos=ts.pos())) + + return list_node + + def parse_expression(self, node): + if node.contents is None: + raise CommandArgError( + "expected braced word or word without substitutions in argument" + " interpreted as expr" + ) + + ts = Lexer(pos=node.contents_pos) + ts.input(node.contents) + + contents = self._parse_expression(ts) + ts.expect( TOK_EOF, - TOK_RPAREN, - TOK_BACKSLASH_NEWLINE, - } and ts.value() not in {":", ","}: - if ts.value() == "?": - # weird hack to record operator - start = ts.pos() - ts.next() - q = BareWord("?", pos=start, end_pos=ts.pos()) + message=f"expected end of expression, got {ts.value()}", + pos=ts.pos(), + ) + if isinstance(node, BracedWord): + return BracedExpression(contents, pos=node.pos, end_pos=node.end_pos) - op2 = self._parse_expression(ts) - if ts.value() != ":": - start = ts.pos() - ts.next() - end = ts.pos() - raise TclSyntaxError( - "expected ':' to continue ternary expression", start, end - ) - - # weird hack again - start = ts.pos() - ts.next() - colon = BareWord(":", pos=start, end_pos=ts.pos()) - - op3 = self._parse_expression(ts) - expr = TernaryOp( - op1, q, op2, colon, op3, pos=op1.pos, end_pos=op3.end_pos - ) - else: - operator = self._parse_operator(ts) - while ts.type() == TOK_BACKSLASH_NEWLINE: - ts.next() - op2 = self._parse_expression(ts) - expr = BinaryOp(op1, operator, op2, pos=op1.pos, end_pos=op2.end_pos) - - if ts.type() not in ( - TOK_RPAREN, - TOK_BACKSLASH_NEWLINE, - ) and ts.value() not in {":", ","}: - ts.expect(TOK_EOF, message="expected end of expression", pos=ts.pos()) - - return expr - - def _parse_operator(self, ts): - pos = ts.pos() - - # hacky logic to handle parsing legal operators - - # Skip any backslash-newlines before the operator - while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: - ts.next() - - if ts.value() in {"&&", "and"}: # Add explicit handling for logical AND - operator = ts.value() - ts.next() - return BareWord(operator, pos=pos, end_pos=ts.pos()) - elif ts.value() in {"*", "&", "|"}: - # one or two of these characters are legal operators - operator = ts.value() - ts.next() - if ts.value() == operator: - operator += ts.value() - ts.next() - elif ts.value() in {"<", ">"}: - operator = ts.value() - ts.next() - if ts.value() in {operator, "="}: - operator += ts.value() - ts.next() - elif ts.value() in {"=", "!"}: - operator = ts.value() - ts.next() - if ts.value() != "=": - raise TclSyntaxError( - f"invalid operator in expression: {operator}", pos, ts.pos() - ) - operator += ts.value() - ts.next() - elif ts.value() in {"*", "/", "%", "+", "-", "^", "eq", "ne", "in", "ni"}: - operator = ts.value() - ts.next() - else: - while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: - ts.next() - - if ts.value() in {"&&", "and"}: # Try again after whitespace - operator = ts.value() - ts.next() - else: - raise TclSyntaxError( - f"invalid operator in expression: {ts.value()}", pos, ts.pos() - ) - - return BareWord(operator, pos=pos, end_pos=ts.pos()) + return Expression(contents, pos=node.pos, end_pos=node.end_pos) diff --git a/server/src/tools/semantic_tokens.py b/server/src/tools/semantic_tokens.py index fff965e..a5a0c68 100644 --- a/server/src/tools/semantic_tokens.py +++ b/server/src/tools/semantic_tokens.py @@ -13,6 +13,7 @@ class TokenModifier(enum.IntFlag): defaultLibrary = enum.auto() definition = enum.auto() declaration = enum.auto() + builtin = enum.auto() @attrs.define @@ -63,6 +64,20 @@ class _Highlighter(Visitor): def visit_command(self, command: Command): routine = command.routine + if routine.contents == "puts": + line, col = routine.contents_pos + self._tokens.append( + ( + ( + (line - 1, col - 1), + len(routine.contents), + "function", + [TokenModifier.builtin], + ) + ) + ) + pass + if routine.contents == "set" and command.args: first_arg = command.args[0] if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): @@ -135,7 +150,6 @@ class _Highlighter(Visitor): ) def visit_var_sub(self, var_sub): - self.log_to_output(str(var_sub)) pass def tokens(self) -> list[Token]: diff --git a/server/src/tools/symbols.py b/server/src/tools/symbols.py index 71b3635..e69de29 100644 --- a/server/src/tools/symbols.py +++ b/server/src/tools/symbols.py @@ -1,82 +0,0 @@ -from tclint.syntax_tree import Visitor, BareWord, BracedWord, Script, Command -from tclint.parser import Parser -import lsprotocol.types as lsp - - -class OutlineVisitor(Visitor): - def __init__(self, parser: Parser): - self.parser = parser - self.stack = [[]] # Root symbol list - - def _range(self, node) -> lsp.Range: - line = node.line - 1 - col = node.col - 1 - if node.end_pos: - end_line = node.end_pos[0] - 1 - end_col = node.end_pos[1] - 1 - else: - end_line = line - end_col = col + 1 - - return lsp.Range( - start=lsp.Position(line=line, character=col), - end=lsp.Position(line=end_line, character=end_col), - ) - - def _add(self, name: str, kind: lsp.SymbolKind, node, children=None): - symbol = lsp.DocumentSymbol( - name=name, - kind=kind, - range=self._range(node), - selection_range=self._range(node), - children=children or [], - ) - self.stack[-1].append(symbol) - return symbol - - def visit_script(self, script): - for child in script.children: - child.accept(self, recurse=False) - - def visit_command(self, command: Command): - if not isinstance(command.routine, BareWord): - return - name = command.routine.contents - - # --- NAMESPACE EVAL --- - if name == "namespace" and len(command.args) >= 3: - subcmd = command.args[0] - if isinstance(subcmd, BareWord) and subcmd.contents == "eval": - ns_arg = command.args[1] - ns_name = ( - ns_arg.contents if isinstance(ns_arg, BareWord) else "" - ) - ns_body = command.args[2] - - ns_symbol = self._add( - ns_name, lsp.SymbolKind.Namespace, command, children=[] - ) - self.stack.append(ns_symbol.children) - - if isinstance(ns_body, BracedWord): - try: - subtree = self.parser.parse_script(ns_body) - subtree.accept(self, recurse=False) - except Exception as e: - print(f"Failed parsing namespace body: {e}") - - self.stack.pop() - - # --- PROC --- - elif name == "proc" and len(command.args) >= 1: - proc_arg = command.args[0] - proc_name = ( - proc_arg.contents if isinstance(proc_arg, BareWord) else "" - ) - self._add(proc_name, lsp.SymbolKind.Function, command) - - # --- SET --- - elif name == "set" and len(command.args) >= 1: - var_arg = command.args[0] - var_name = var_arg.contents if isinstance(var_arg, BareWord) else "" - self._add(var_name, lsp.SymbolKind.Variable, command) diff --git a/test/test.tcl b/test/test.tcl index f53d014..9a65346 100644 --- a/test/test.tcl +++ b/test/test.tcl @@ -28,8 +28,8 @@ proc SERVICE_spacer_output {type {length 20} {line_num 0} {output 1}} { LIB_GE_message [string repeat $type $length] "output_$output" $line_num } -SERVICE_spacer_output "*" 50 0 1 -SERVICE_remove_file $filename + +SERVICE_spacer_output "*" 2 0 0 #_________________________________________________________________________________________________ # @@ -93,7 +93,7 @@ proc SERVICE_output_handling {handler} { #_________________________________________________________________________________________________ proc SERVICE_get_tool_data {} { global mom_tool_data - global mom_operation_info +global mom_operation_info set mom_tool_data(toollist) "" set operations $::mom_operation_name_list -- 2.54.0