update completion items

server/src/common/load_data.py

@@ -21,6 +21,7 @@ class StandardCompletionItems:
         self.__tcl_keyword_list = self.__load_tcl_keyword()
         self.__nx_procs = self.__load_nx_procs()
         self.__nx_variables = self.__load_nx_variables()
+        self.__custom_functions = list[lsp.CompletionItem]
 
     @property
     def json_data(self):
@@ -38,6 +39,14 @@ class StandardCompletionItems:
     def nx_variables(self):
         return self.__nx_variables
 
+    @property
+    def custom_functions(self) -> list[lsp.CompletionItem]:
+        return self.__custom_functions
+
+    @custom_functions.setter
+    def custom_functions(self, value: lsp.CompletionItem):
+        self.__custom_functions.append(value)
+
     def __load_json(self) -> dict:
         with open(
             pathlib.Path(__file__).parent.joinpath("completion_list.json"), "r"

server/src/lsp_server.py

@@ -14,6 +14,8 @@ import sysconfig
 import traceback
 from typing import Any, List, Optional, Sequence, Tuple
 import re
+import operator
+from functools import reduce
 
 
 # **********************************************************
@@ -44,18 +46,15 @@ import lsprotocol.types as lsp
 from pygls import server, uris, workspace
 from pygls.workspace.text_document import TextDocument
 from common.load_data import standard_items
-from common.formatter import format_tcl
 from tclint.parser import Parser
 from tclint.lexer import TclSyntaxError
 from tclint.format import Formatter, FormatterOpts
 from tclint.violations import Violation
-from tools.semantic_tokens import (
+
-    SemanticTokenCollector,
-    collect_semantic_tokens,
-    encode_tokens,
-)
 from plugins.poco_plugin import commands
 from tools import checks
+from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier
+from tools.completion_items import _Completion
 
 DIAGNOSTIC_SOURCE = "nx-post-support"
 
@@ -78,7 +77,6 @@ class TclLanguageServer(server.LanguageServer):
         # parser._commands.update(commands)
 
         indent = "\t" if not options.insert_spaces else " " * options.tab_size
-
         formatter = Formatter(
             FormatterOpts(
                 indent=indent,
@@ -102,7 +100,7 @@ class TclLanguageServer(server.LanguageServer):
         self.parser.violations = []
         tree = self.parser.parse(document.source)
         violations += self.parser.violations
-
+        # log_to_output(tree.pretty(2))
         for checker in checks.get_checkers():
             violations += checker.check(document.source, tree)
         return violations
@@ -170,16 +168,6 @@ WORKSPACE_SETTINGS = {}
 GLOBAL_SETTINGS = {}
 RUNNER = pathlib.Path(__file__).parent / "lsp_runner.py"
 
-TOKEN_TYPES = [
-    "command",
-    "variable",
-    "function",
-    "string",
-    "number",
-    "keyword",
-    "comment",
-]
-TOKEN_MODIFIERS = []
 
 MAX_WORKERS = 5
 LSP_SERVER = TclLanguageServer(
@@ -254,22 +242,48 @@ def document_diagnostic(params: lsp.DocumentDiagnosticParams):
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_COMPLETION)
 def on_completion(params: lsp.CompletionParams) -> list[lsp.CompletionItem]:
+    document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+    ci = _Completion()
+    tree = LSP_SERVER.parser.parse(document.source)
+    tree.accept(ci, recurse=True)
+
     items = (
         standard_items.tcl_keyword_list
         + standard_items.nx_procs
         + standard_items.nx_variables
+        + ci.custom_functions
     )
     return lsp.CompletionList(is_incomplete=False, items=items)
 
 
-@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL)
+@LSP_SERVER.feature(
-def on_semantic_tokens(params: lsp.SemanticTokensParams):
+    lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
-    doc = LSP_SERVER.workspace.get_document(params.text_document.uri)
+    lsp.SemanticTokensLegend(
-    code = doc.source
+        token_types=TOKEN_TYPES,
+        token_modifiers=[m.name for m in TokenModifier],
+    ),
+)
+def semantic_tokens(params: lsp.SemanticTokensParams):
+    document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
 
-    tokens = collect_semantic_tokens(code)
+    data = []
-    data = encode_tokens(tokens)
+    plugins = []
+    hl = _Highlighter(plugins, log_to_output=log_to_output)
 
+    tree = LSP_SERVER.parser.parse(document.source)
+    tree.accept(hl, recurse=True)
+
+    tokens = hl.tokens()
+    for token in tokens:
+        data.extend(
+            [
+                token.line,
+                token.offset,
+                token.lenght,
+                TOKEN_TYPES.index(token.tok_type),
+                reduce(operator.or_, token.tok_modifiers, 0),
+            ]
+        )
     return lsp.SemanticTokens(data=data)
 
 
@@ -401,7 +415,7 @@ def initialize(params: lsp.InitializeParams) -> lsp.InitializeResult:
     )
     semantic_tokens_legend = lsp.SemanticTokensLegend(
         token_types=TOKEN_TYPES,
-        token_modifiers=TOKEN_MODIFIERS,
+        token_modifiers=TokenModifier,
     )
     return lsp.InitializeResult(
         capabilities=lsp.ServerCapabilities(

server/src/test_tcl.py

@@ -1,7 +1,5 @@
 from tclint.parser import Parser as tcLintParser
 from tclint.lexer import Lexer as tclingLexer
-from parser.lexer import Lexer
-from parser.parser import Parser
 
 
 def main():
@@ -28,22 +26,9 @@ LIB_SPF_prepend MOM_strt Start_Lib {
 
 def lexer_test():
     lexer = tclingLexer()
-    tree = lexer.input("""puts hello
+    tree = lexer.input(""" 
-                 proc myProc {arg {arg7 0}} {}
+if {$oem(custom_clamp_4th) == 1 && $oem(custom_clamp_5th) == 1 \\
-                 set myVar 123
+        && $oem(status_clamp_4th) == "off" && $oem(status_clamp_5th) == "off"}""")
-                 puts puts
-                 LIB_SPF_prepend MOM_strt Start_Lib {
-        set somthing 1
-        set more 2
-} myTag
-
-LIB_SPF_prepend MOM_strt Start_Lib {            
-    proc test {} {
-        puts "Hello"
-    }       
-    set somthing 1
-    set someting 3
-} myTag""")
 
     # print("Lexing input:\n", code)
     # print("\nTokens:\n" + "-" * 30)
@@ -57,26 +42,18 @@ LIB_SPF_prepend MOM_strt Start_Lib {
 
 
 def test_1():
-    code = """
+    from tclint.lexer import Lexer, TOK_BACKSLASH_NEWLINE
-        proc myProc {arg } {
-            set myVar 1
-        }
 
-        namespace eval myNS {
+    code = "expr {1 == 2 \\\n&& 3 == 4}"
-            proc innerProc {} {
-                MOM_abort_program "Test"
-            }
-        }
-        set result [myNS::innerProc]
-        """
 
     lexer = Lexer()
     lexer.input(code)
-    parser = Parser(lexer)
-    ast = parser.parse()
 
-    visitor = NodeVisitor()
+    while lexer.type() is not None:
-    ast.accept(visitor)
+        print(
+            f"Type: {lexer.type():<20} | Value: {lexer.value()!r} | Pos: {lexer.pos()}"
+        )
+        lexer.next()
 
 
 class NodeVisitor:
@@ -98,4 +75,4 @@ class NodeVisitor:
 
 
 if __name__ == "__main__":
-    main()
+    test_1()

server/src/tools/completion_items.py

@@ -0,0 +1,37 @@
+from tclint.syntax_tree import Visitor
+import lsprotocol.types as lsp
+
+
+class CompletionItems:
+    def __init__(self):
+        self._custom_functions: list[lsp.CompletionItem] = []
+
+    @property
+    def custom_functions(self) -> list[lsp.CompletionItem]:
+        return self._custom_functions
+
+    @custom_functions.setter
+    def custom_functions(self, value: lsp.CompletionItem):
+        self._custom_functions.append(value)
+
+
+class _Completion(Visitor):
+    def __init__(self):
+        super().__init__()
+        self._custom_functions: list[lsp.CompletionItem] = []
+
+    @property
+    def custom_functions(self) -> list[lsp.CompletionItem]:
+        return self._custom_functions
+
+    def visit_command(self, command):
+        routine = command.routine
+
+        if routine.contents == "proc" and command.args:
+            first_arg = command.args[0]
+            if hasattr(first_arg, "value"):
+                self._custom_functions.append(
+                    lsp.CompletionItem(
+                        label=first_arg.value, kind=lsp.CompletionItemKind.Function
+                    )
+                )

server/src/tools/parser.py

@@ -1,49 +1,14 @@
-import textwrap
+from tclint.parser import Parser, _strip_ws
-from tclint.parser import Parser
 from tclint.lexer import (
-    STATE_BRACEDWORD,
+    TOK_WS,
-    TOK_LBRACE,
+    TOK_BACKSLASH_NEWLINE,
     TOK_EOF,
-    TOK_RBRACE,
+    TOK_RPAREN,
+    TOK_NEWLINE,
     TclSyntaxError,
 )
-from tclint.syntax_tree import BracedWord
+from tclint.syntax_tree import BareWord, TernaryOp, BinaryOp
 
 
 class CustomParser(Parser):
-    def parse_braced_word(self, ts):
+    pass
-        """
-        Ersetzt BracedWord durch echtes Script, wenn mehrzeilig.
-        """
-        pos = ts.pos()
-        ts.lexer.push_state(STATE_BRACEDWORD)
-        ts.assert_(TOK_LBRACE)
-
-        content = ""
-        expected = [pos]
-        while True:
-            t = ts.type()
-            if t == TOK_EOF:
-                raise TclSyntaxError(
-                    "reached EOF without finding match for brace",
-                    expected[-1],
-                    ts.pos(),
-                )
-            if t == TOK_LBRACE:
-                expected.append(ts.pos())
-            elif t == TOK_RBRACE:
-                expected.pop()
-                if not expected:
-                    ts.lexer.pop_state()
-                    ts.next()
-                    break
-            content += ts.value()
-            ts.next()
-
-        end_pos = ts.pos()
-        # Mehrzeilig? Dann als Script parsen:
-        if "\n" in content.strip():
-            self.parse_script(content)
-
-        # Einzeilig: unverändert als Literal
-        return BracedWord(content, pos=pos, end_pos=end_pos)

server/src/tools/semantic_tokens.py

@@ -1,76 +1,116 @@
-from tclint.parser import Parser
+import enum
-from tclint.syntax_tree import Visitor, BareWord, VarSub, Comment, Command, Function
+from typing import List
-
+from tclint.syntax_tree import Visitor
-TOKEN_TYPES = {
+from tclint.commands import get_commands
-    "command": 0,
+import attrs
-    "variable": 1,
+from common.load_data import standard_items
-    "function": 2,
+import lsprotocol.types as lsp
-    "string": 3,
-    "number": 4,
-    "keyword": 5,
-    "comment": 6,
-}
 
 
-class SemanticTokenCollector(Visitor):
+class TokenModifier(enum.IntFlag):
-    def __init__(self):
+    deprecated = enum.auto()
-        self.tokens = []
+    readonly = enum.auto()
-
+    defaultLibrary = enum.auto()
-    def _add_token(self, node, token_type):
+    definition = enum.auto()
-        if not node.pos or not node.end_pos:
-            return
-
-        line, col = node.pos
-        end_line, end_col = node.end_pos
-        length = (end_col - col) if line == end_line else 1
-
-        self.tokens.append((line - 1, col - 1, length, token_type, 0))
-
-    def visit_command(self, command: Command):
-        self._add_token(command.routine, "command")
-        for arg in command.args:
-            arg.accept(self, recurse=True)
-
-    def visit_comment(self, comment: Comment):
-        self._add_token(comment, "comment")
-
-    def visit_bare_word(self, word: BareWord):
-        if word.value.isdigit():
-            self._add_token(word, "number")
-        else:
-            self._add_token(word, "string")
-
-    def visit_var_sub(self, var_sub: VarSub):
-        self._add_token(var_sub, "variable")
-
-    def visit_function(self, function: Function):
-        self._add_token(function.name, "function")
-        for arg in function.args:
-            arg.accept(self, recurse=True)
 
 
-def collect_semantic_tokens(code: str):
+@attrs.define
-    parser = Parser()
+class Token:
-    tree = parser.parse(code)
+    line: int
-    visitor = SemanticTokenCollector()
+    offset: int
-    tree.accept(visitor, recurse=True)
+    lenght: int
-    return visitor.tokens
+
+    tok_type: str = ""
+    tok_modifiers: List[TokenModifier] = attrs.field(factory=list)
 
 
-def encode_tokens(tokens):
+TOKEN_TYPES = [
-    tokens.sort()
+    "keyword",
-    encoded = []
+    "variable",
+    "function",
+    "operator",
+    "parameter",
+    "type",
+    "class",
+]
 
-    last_line = 0
-    last_char = 0
 
-    for line, char, length, token_type, modifiers in tokens:
+class _Highlighter(Visitor):
-        delta_line = line - last_line
+    def __init__(self, plugins, log_to_output):
-        delta_start = char - last_char if delta_line == 0 else char
+        self._commands = get_commands(plugins)
+        self._tokens = []
+        self.log_to_output = log_to_output
 
-        encoded.extend([delta_line, delta_start, length, token_type, modifiers])
+    def visit_command(self, command):
+        routine = command.routine
+        self.log_to_output(str(command.routine))
+        if routine.contents in self._commands:
+            line, col = routine.pos
+            self._tokens.append(((line - 1, col - 1), len(routine.contents), "keyword"))
 
-        last_line = line
+        if routine.contents == "set" and command.args:
-        last_char = char if delta_line == 0 else 0
+            first_arg = command.args[0]
+            if hasattr(first_arg, "pos") and hasattr(first_arg, "value"):
+                line, col = first_arg.pos
+                self._tokens.append(
+                    (((line - 1, col - 1), len(first_arg.value), "variable"))
+                )
+        if routine.contents == "proc" and command.args:
+            first_arg = command.args[0]
+            if hasattr(first_arg, "pos") and hasattr(first_arg, "value"):
+                line, col = first_arg.pos
+                self._tokens.append(
+                    (((line - 1, col - 1), len(first_arg.value), "function"))
+                )
 
-    return encoded
+        if routine.contents == "namespace" and command.args:
+            first_arg = command.args[1]
+            if hasattr(first_arg, "pos") and hasattr(first_arg, "value"):
+                line, col = first_arg.pos
+                self._tokens.append(
+                    (((line - 1, col - 1), len(first_arg.value), "class"))
+                )
+
+    def visit_var_sub(self, var_sub):
+        self.log_to_output(str(var_sub))
+        pass
+
+    def tokens(self) -> list[Token]:
+        """Encode tokens as described in
+        https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens.
+        """
+        tokens = []
+        last_line = 0
+        last_col = 0
+        for (line, col), length, tok_type in sorted(self._tokens, key=lambda x: x[0]):
+            line_delta = line - last_line
+            col_delta = col
+            if line == last_line:
+                col_delta -= last_col
+
+            tokens.append(Token(line_delta, col_delta, length, tok_type))
+            last_line, last_col = line, col
+
+        return tokens
+
+
+# @server.feature(
+#     lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
+#     lsp.SemanticTokensLegend(token_types=["keyword"], token_modifiers=[]),
+# )
+# def semantic_tokens(ls: TclspServer, params: lsp.SemanticTokensParams):
+#     logging.debug("Received %s: %s", lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL, params)
+#     document = ls.workspace.get_text_document(params.text_document.uri)
+
+#     path = Path(document.path)
+#     root = ls.get_root(path)
+#     config = ls.get_config(path, root)
+
+#     plugins = [config.commands] if config.commands is not None else []
+#     parser = Parser(command_plugins=plugins)
+#     hl = _Highlighter(plugins)
+
+#     tree = parser.parse(document.source)
+#     tree.accept(hl, recurse=True)
+
+#     return lsp.SemanticTokens(data=hl.tokens())