From ed045aa459440376c40c2cf958fb804feeb37fa4 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Wed, 30 Jul 2025 17:28:18 +0200 Subject: [PATCH] update completion items --- server/src/common/load_data.py | 9 ++ server/src/lsp_server.py | 64 ++++++---- server/src/test_tcl.py | 45 ++----- server/src/tools/completion_items.py | 37 ++++++ server/src/tools/parser.py | 49 ++------ server/src/tools/semantic_tokens.py | 170 +++++++++++++++++---------- 6 files changed, 208 insertions(+), 166 deletions(-) create mode 100644 server/src/tools/completion_items.py diff --git a/server/src/common/load_data.py b/server/src/common/load_data.py index 67837ff..dc22c99 100644 --- a/server/src/common/load_data.py +++ b/server/src/common/load_data.py @@ -21,6 +21,7 @@ class StandardCompletionItems: self.__tcl_keyword_list = self.__load_tcl_keyword() self.__nx_procs = self.__load_nx_procs() self.__nx_variables = self.__load_nx_variables() + self.__custom_functions = list[lsp.CompletionItem] @property def json_data(self): @@ -38,6 +39,14 @@ class StandardCompletionItems: def nx_variables(self): return self.__nx_variables + @property + def custom_functions(self) -> list[lsp.CompletionItem]: + return self.__custom_functions + + @custom_functions.setter + def custom_functions(self, value: lsp.CompletionItem): + self.__custom_functions.append(value) + def __load_json(self) -> dict: with open( pathlib.Path(__file__).parent.joinpath("completion_list.json"), "r" diff --git a/server/src/lsp_server.py b/server/src/lsp_server.py index 707f401..4e2b91b 100644 --- a/server/src/lsp_server.py +++ b/server/src/lsp_server.py @@ -14,6 +14,8 @@ import sysconfig import traceback from typing import Any, List, Optional, Sequence, Tuple import re +import operator +from functools import reduce # ********************************************************** @@ -44,18 +46,15 @@ import lsprotocol.types as lsp from pygls import server, uris, workspace from pygls.workspace.text_document import TextDocument from common.load_data import standard_items -from common.formatter import format_tcl from tclint.parser import Parser from tclint.lexer import TclSyntaxError from tclint.format import Formatter, FormatterOpts from tclint.violations import Violation -from tools.semantic_tokens import ( - SemanticTokenCollector, - collect_semantic_tokens, - encode_tokens, -) + from plugins.poco_plugin import commands from tools import checks +from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier +from tools.completion_items import _Completion DIAGNOSTIC_SOURCE = "nx-post-support" @@ -78,7 +77,6 @@ class TclLanguageServer(server.LanguageServer): # parser._commands.update(commands) indent = "\t" if not options.insert_spaces else " " * options.tab_size - formatter = Formatter( FormatterOpts( indent=indent, @@ -102,7 +100,7 @@ class TclLanguageServer(server.LanguageServer): self.parser.violations = [] tree = self.parser.parse(document.source) violations += self.parser.violations - + # log_to_output(tree.pretty(2)) for checker in checks.get_checkers(): violations += checker.check(document.source, tree) return violations @@ -170,16 +168,6 @@ WORKSPACE_SETTINGS = {} GLOBAL_SETTINGS = {} RUNNER = pathlib.Path(__file__).parent / "lsp_runner.py" -TOKEN_TYPES = [ - "command", - "variable", - "function", - "string", - "number", - "keyword", - "comment", -] -TOKEN_MODIFIERS = [] MAX_WORKERS = 5 LSP_SERVER = TclLanguageServer( @@ -254,22 +242,48 @@ def document_diagnostic(params: lsp.DocumentDiagnosticParams): @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_COMPLETION) def on_completion(params: lsp.CompletionParams) -> list[lsp.CompletionItem]: + document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) + ci = _Completion() + tree = LSP_SERVER.parser.parse(document.source) + tree.accept(ci, recurse=True) + items = ( standard_items.tcl_keyword_list + standard_items.nx_procs + standard_items.nx_variables + + ci.custom_functions ) return lsp.CompletionList(is_incomplete=False, items=items) -@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL) -def on_semantic_tokens(params: lsp.SemanticTokensParams): - doc = LSP_SERVER.workspace.get_document(params.text_document.uri) - code = doc.source +@LSP_SERVER.feature( + lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL, + lsp.SemanticTokensLegend( + token_types=TOKEN_TYPES, + token_modifiers=[m.name for m in TokenModifier], + ), +) +def semantic_tokens(params: lsp.SemanticTokensParams): + document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) - tokens = collect_semantic_tokens(code) - data = encode_tokens(tokens) + data = [] + plugins = [] + hl = _Highlighter(plugins, log_to_output=log_to_output) + tree = LSP_SERVER.parser.parse(document.source) + tree.accept(hl, recurse=True) + + tokens = hl.tokens() + for token in tokens: + data.extend( + [ + token.line, + token.offset, + token.lenght, + TOKEN_TYPES.index(token.tok_type), + reduce(operator.or_, token.tok_modifiers, 0), + ] + ) return lsp.SemanticTokens(data=data) @@ -401,7 +415,7 @@ def initialize(params: lsp.InitializeParams) -> lsp.InitializeResult: ) semantic_tokens_legend = lsp.SemanticTokensLegend( token_types=TOKEN_TYPES, - token_modifiers=TOKEN_MODIFIERS, + token_modifiers=TokenModifier, ) return lsp.InitializeResult( capabilities=lsp.ServerCapabilities( diff --git a/server/src/test_tcl.py b/server/src/test_tcl.py index 783c406..4ffbfeb 100644 --- a/server/src/test_tcl.py +++ b/server/src/test_tcl.py @@ -1,7 +1,5 @@ from tclint.parser import Parser as tcLintParser from tclint.lexer import Lexer as tclingLexer -from parser.lexer import Lexer -from parser.parser import Parser def main(): @@ -28,22 +26,9 @@ LIB_SPF_prepend MOM_strt Start_Lib { def lexer_test(): lexer = tclingLexer() - tree = lexer.input("""puts hello - proc myProc {arg {arg7 0}} {} - set myVar 123 - puts puts - LIB_SPF_prepend MOM_strt Start_Lib { - set somthing 1 - set more 2 -} myTag - -LIB_SPF_prepend MOM_strt Start_Lib { - proc test {} { - puts "Hello" - } - set somthing 1 - set someting 3 -} myTag""") + tree = lexer.input(""" +if {$oem(custom_clamp_4th) == 1 && $oem(custom_clamp_5th) == 1 \\ + && $oem(status_clamp_4th) == "off" && $oem(status_clamp_5th) == "off"}""") # print("Lexing input:\n", code) # print("\nTokens:\n" + "-" * 30) @@ -57,26 +42,18 @@ LIB_SPF_prepend MOM_strt Start_Lib { def test_1(): - code = """ - proc myProc {arg } { - set myVar 1 - } + from tclint.lexer import Lexer, TOK_BACKSLASH_NEWLINE - namespace eval myNS { - proc innerProc {} { - MOM_abort_program "Test" - } - } - set result [myNS::innerProc] - """ + code = "expr {1 == 2 \\\n&& 3 == 4}" lexer = Lexer() lexer.input(code) - parser = Parser(lexer) - ast = parser.parse() - visitor = NodeVisitor() - ast.accept(visitor) + while lexer.type() is not None: + print( + f"Type: {lexer.type():<20} | Value: {lexer.value()!r} | Pos: {lexer.pos()}" + ) + lexer.next() class NodeVisitor: @@ -98,4 +75,4 @@ class NodeVisitor: if __name__ == "__main__": - main() + test_1() diff --git a/server/src/tools/completion_items.py b/server/src/tools/completion_items.py new file mode 100644 index 0000000..39fc1c9 --- /dev/null +++ b/server/src/tools/completion_items.py @@ -0,0 +1,37 @@ +from tclint.syntax_tree import Visitor +import lsprotocol.types as lsp + + +class CompletionItems: + def __init__(self): + self._custom_functions: list[lsp.CompletionItem] = [] + + @property + def custom_functions(self) -> list[lsp.CompletionItem]: + return self._custom_functions + + @custom_functions.setter + def custom_functions(self, value: lsp.CompletionItem): + self._custom_functions.append(value) + + +class _Completion(Visitor): + def __init__(self): + super().__init__() + self._custom_functions: list[lsp.CompletionItem] = [] + + @property + def custom_functions(self) -> list[lsp.CompletionItem]: + return self._custom_functions + + def visit_command(self, command): + routine = command.routine + + if routine.contents == "proc" and command.args: + first_arg = command.args[0] + if hasattr(first_arg, "value"): + self._custom_functions.append( + lsp.CompletionItem( + label=first_arg.value, kind=lsp.CompletionItemKind.Function + ) + ) diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py index 07c10a7..7e81eb2 100644 --- a/server/src/tools/parser.py +++ b/server/src/tools/parser.py @@ -1,49 +1,14 @@ -import textwrap -from tclint.parser import Parser +from tclint.parser import Parser, _strip_ws from tclint.lexer import ( - STATE_BRACEDWORD, - TOK_LBRACE, + TOK_WS, + TOK_BACKSLASH_NEWLINE, TOK_EOF, - TOK_RBRACE, + TOK_RPAREN, + TOK_NEWLINE, TclSyntaxError, ) -from tclint.syntax_tree import BracedWord +from tclint.syntax_tree import BareWord, TernaryOp, BinaryOp class CustomParser(Parser): - def parse_braced_word(self, ts): - """ - Ersetzt BracedWord durch echtes Script, wenn mehrzeilig. - """ - pos = ts.pos() - ts.lexer.push_state(STATE_BRACEDWORD) - ts.assert_(TOK_LBRACE) - - content = "" - expected = [pos] - while True: - t = ts.type() - if t == TOK_EOF: - raise TclSyntaxError( - "reached EOF without finding match for brace", - expected[-1], - ts.pos(), - ) - if t == TOK_LBRACE: - expected.append(ts.pos()) - elif t == TOK_RBRACE: - expected.pop() - if not expected: - ts.lexer.pop_state() - ts.next() - break - content += ts.value() - ts.next() - - end_pos = ts.pos() - # Mehrzeilig? Dann als Script parsen: - if "\n" in content.strip(): - self.parse_script(content) - - # Einzeilig: unverändert als Literal - return BracedWord(content, pos=pos, end_pos=end_pos) + pass diff --git a/server/src/tools/semantic_tokens.py b/server/src/tools/semantic_tokens.py index ffc80ac..0a5fff1 100644 --- a/server/src/tools/semantic_tokens.py +++ b/server/src/tools/semantic_tokens.py @@ -1,76 +1,116 @@ -from tclint.parser import Parser -from tclint.syntax_tree import Visitor, BareWord, VarSub, Comment, Command, Function - -TOKEN_TYPES = { - "command": 0, - "variable": 1, - "function": 2, - "string": 3, - "number": 4, - "keyword": 5, - "comment": 6, -} +import enum +from typing import List +from tclint.syntax_tree import Visitor +from tclint.commands import get_commands +import attrs +from common.load_data import standard_items +import lsprotocol.types as lsp -class SemanticTokenCollector(Visitor): - def __init__(self): - self.tokens = [] - - def _add_token(self, node, token_type): - if not node.pos or not node.end_pos: - return - - line, col = node.pos - end_line, end_col = node.end_pos - length = (end_col - col) if line == end_line else 1 - - self.tokens.append((line - 1, col - 1, length, token_type, 0)) - - def visit_command(self, command: Command): - self._add_token(command.routine, "command") - for arg in command.args: - arg.accept(self, recurse=True) - - def visit_comment(self, comment: Comment): - self._add_token(comment, "comment") - - def visit_bare_word(self, word: BareWord): - if word.value.isdigit(): - self._add_token(word, "number") - else: - self._add_token(word, "string") - - def visit_var_sub(self, var_sub: VarSub): - self._add_token(var_sub, "variable") - - def visit_function(self, function: Function): - self._add_token(function.name, "function") - for arg in function.args: - arg.accept(self, recurse=True) +class TokenModifier(enum.IntFlag): + deprecated = enum.auto() + readonly = enum.auto() + defaultLibrary = enum.auto() + definition = enum.auto() -def collect_semantic_tokens(code: str): - parser = Parser() - tree = parser.parse(code) - visitor = SemanticTokenCollector() - tree.accept(visitor, recurse=True) - return visitor.tokens +@attrs.define +class Token: + line: int + offset: int + lenght: int + + tok_type: str = "" + tok_modifiers: List[TokenModifier] = attrs.field(factory=list) -def encode_tokens(tokens): - tokens.sort() - encoded = [] +TOKEN_TYPES = [ + "keyword", + "variable", + "function", + "operator", + "parameter", + "type", + "class", +] - last_line = 0 - last_char = 0 - for line, char, length, token_type, modifiers in tokens: - delta_line = line - last_line - delta_start = char - last_char if delta_line == 0 else char +class _Highlighter(Visitor): + def __init__(self, plugins, log_to_output): + self._commands = get_commands(plugins) + self._tokens = [] + self.log_to_output = log_to_output - encoded.extend([delta_line, delta_start, length, token_type, modifiers]) + def visit_command(self, command): + routine = command.routine + self.log_to_output(str(command.routine)) + if routine.contents in self._commands: + line, col = routine.pos + self._tokens.append(((line - 1, col - 1), len(routine.contents), "keyword")) - last_line = line - last_char = char if delta_line == 0 else 0 + if routine.contents == "set" and command.args: + first_arg = command.args[0] + if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): + line, col = first_arg.pos + self._tokens.append( + (((line - 1, col - 1), len(first_arg.value), "variable")) + ) + if routine.contents == "proc" and command.args: + first_arg = command.args[0] + if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): + line, col = first_arg.pos + self._tokens.append( + (((line - 1, col - 1), len(first_arg.value), "function")) + ) - return encoded + if routine.contents == "namespace" and command.args: + first_arg = command.args[1] + if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): + line, col = first_arg.pos + self._tokens.append( + (((line - 1, col - 1), len(first_arg.value), "class")) + ) + + def visit_var_sub(self, var_sub): + self.log_to_output(str(var_sub)) + pass + + def tokens(self) -> list[Token]: + """Encode tokens as described in + https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens. + """ + tokens = [] + last_line = 0 + last_col = 0 + for (line, col), length, tok_type in sorted(self._tokens, key=lambda x: x[0]): + line_delta = line - last_line + col_delta = col + if line == last_line: + col_delta -= last_col + + tokens.append(Token(line_delta, col_delta, length, tok_type)) + last_line, last_col = line, col + + return tokens + + +# @server.feature( +# lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL, +# lsp.SemanticTokensLegend(token_types=["keyword"], token_modifiers=[]), +# ) +# def semantic_tokens(ls: TclspServer, params: lsp.SemanticTokensParams): +# logging.debug("Received %s: %s", lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL, params) +# document = ls.workspace.get_text_document(params.text_document.uri) + +# path = Path(document.path) +# root = ls.get_root(path) +# config = ls.get_config(path, root) + +# plugins = [config.commands] if config.commands is not None else [] +# parser = Parser(command_plugins=plugins) +# hl = _Highlighter(plugins) + +# tree = parser.parse(document.source) +# tree.accept(hl, recurse=True) + +# return lsp.SemanticTokens(data=hl.tokens())