Merge pull request 'tcl_language_support' (#19) from tcl_language_support into main

Reviewed-on: #19

.vscodeignore

@@ -15,3 +15,4 @@ esbuild.js
 **/requirements.txt
 **/requirements.in
 **/server/src/_debug_server.py
+noxfile.py

server/src/common/completion_list.json

@@ -514,6 +514,31 @@
                 "set syslog [MOM_ask_syslog_name]"
             ]
         },
+        {
+            "label": "MOM_ask_ude_info",
+            "kind": "function",
+            "description": "This command is used to retrieve the information about a user-defined event (UDE) of a specified object.",
+            "format": "MOM_ask_ude_info object_name object_type <Start/End/\"\">",
+            "parameters": [
+                {
+                    "name": "Object_name",
+                    "desc": "The name of the object to which the UDE is attached. It can be a group, an operation, a tool, a geometry, or a method."
+                },
+                {
+                    "name": "object_type",
+                    "desc": "The type of the object to which the UDE is attached. The following types are available.(group || operation/oper || tool || geometry/geom || method/meth)"
+                },
+                {
+                    "name": "Start/End/\"\"",
+                    "desc": "This parameter is optional. Indicates whether you want to retrieve the UDE attached to the start event, end event, or both. If it is empty, returns the UDEs attached to the specified operation."
+                }
+            ],
+            "returns": [
+                "0 - The UDE information retrieval failed.",
+                "1 - The UDE information successfully retrieved."
+            ],
+            "example": []
+        },
         {
             "label": "MOM_cancel_suppress_force_once_per_event",
             "kind": "function",

server/src/common/formatter.py

@@ -1,63 +0,0 @@
-import re
-
-
-def format_tcl(src: str, indent_str="    ") -> str:
-    """
-    Simple Tcl formatter with special handling for:
-    1) Single-line 'if {cond} {action}' blocks remain on one line.
-    2) Combined closing-and-opening lines like '} else {' dedent then re-indent.
-    3) Lines like '} Tag' stay on the same line: '} Tag'.
-    4) Standard multi-line blocks for 'if', 'elseif', 'else', '{', '}'.
-    """
-    level = 0
-    out_lines = []
-
-    for raw_line in src.splitlines():
-        stripped = raw_line.strip()
-
-        # Comments are ignored
-        if stripped.startswith("#"):
-            out_lines.append(indent_str * level + stripped)
-            continue
-
-        # Single-line 'if {cond} {action}' → no indent change
-        if re.match(r"^(if|elseif)\s*\{[^}]+\}\s*\{[^}]+\}$", stripped):
-            out_lines.append(indent_str * level + stripped)
-            continue
-
-        # Combined '} else {' → dedent, print, then indent
-        if re.match(r"^\}\s*(elseif|else)\b.*\{$", stripped):
-            level = max(level - 1, 0)
-            out_lines.append(indent_str * level + stripped)
-            level += 1
-            continue
-
-        # SPECIAL: closing brace plus tag on same line: '} Tag'
-        m = re.match(r"^\}\s+(.+)", stripped)
-        if m and not stripped.startswith("#"):
-            # close one block
-            level = max(level - 1, 0)
-            # stay on one line: "} Tag"
-            out_lines.append(f"{indent_str * level}}} {m.group(1)}")
-            continue
-
-        # Pure '}' → dedent then print
-        if stripped == "}":
-            level = max(level - 1, 0)
-            out_lines.append(f"{indent_str * level}{stripped}")
-            continue
-
-        # 'elseif' or 'else' alone → align with matching 'if'
-        if re.match(r"^(elseif|else)\b(?!.*\{)", stripped):
-            level = max(level - 1, 0)
-            out_lines.append(f"{indent_str * level}{stripped}")
-            continue
-
-        # Default: print at current indent
-        out_lines.append(f"{indent_str * level}{stripped}")
-
-        # Open a new block on lines ending with '{'
-        if re.match(r"^(if|elseif)\b.*\{$", stripped) or stripped.endswith("{"):
-            level += 1
-
-    return "\n".join(out_lines)

server/src/common/load_data.py

@@ -21,6 +21,7 @@ class StandardCompletionItems:
         self.__tcl_keyword_list = self.__load_tcl_keyword()
         self.__nx_procs = self.__load_nx_procs()
         self.__nx_variables = self.__load_nx_variables()
+        self.__custom_functions = list[lsp.CompletionItem]
 
     @property
     def json_data(self):
@@ -38,6 +39,14 @@ class StandardCompletionItems:
     def nx_variables(self):
         return self.__nx_variables
 
+    @property
+    def custom_functions(self) -> list[lsp.CompletionItem]:
+        return self.__custom_functions
+
+    @custom_functions.setter
+    def custom_functions(self, value: lsp.CompletionItem):
+        self.__custom_functions.append(value)
+
     def __load_json(self) -> dict:
         with open(
             pathlib.Path(__file__).parent.joinpath("completion_list.json"), "r"

server/src/lsp_server.py

@@ -4,16 +4,14 @@
 
 from __future__ import annotations
 
-import copy
 import json
 import os
 import pathlib
 import re
 import sys
-import sysconfig
+from typing import Any, List, Optional, Tuple
-import traceback
+import operator
-from typing import Any, List, Optional, Sequence, Tuple
+from functools import reduce
-import re
 
 
 # **********************************************************
@@ -39,23 +37,18 @@ update_sys_path(
 # **********************************************************
 # pylint: disable=wrong-import-position,import-error
 import lsp_jsonrpc as jsonrpc
-import lsp_utils as utils
 import lsprotocol.types as lsp
 from pygls import server, uris, workspace
 from pygls.workspace.text_document import TextDocument
 from common.load_data import standard_items
-from common.formatter import format_tcl
-from tclint.parser import Parser
 from tclint.lexer import TclSyntaxError
 from tclint.format import Formatter, FormatterOpts
 from tclint.violations import Violation
-from tools.semantic_tokens import (
-    SemanticTokenCollector,
-    collect_semantic_tokens,
-    encode_tokens,
-)
 from plugins.poco_plugin import commands
-from tools import checks
+from tools import checks, parser
+from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier
+from tools.completion_items import completion
+from tools.inlay_hint import InlayHintGenerator
 
 DIAGNOSTIC_SOURCE = "nx-post-support"
 
@@ -63,7 +56,7 @@ DIAGNOSTIC_SOURCE = "nx-post-support"
 class TclLanguageServer(server.LanguageServer):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.parser = Parser()
+        self.parser = parser.CustomParser()
         for command in commands:
             self.parser._commands.update(command)
         self.diagnostics = {}
@@ -78,7 +71,6 @@ class TclLanguageServer(server.LanguageServer):
         # parser._commands.update(commands)
 
         indent = "\t" if not options.insert_spaces else " " * options.tab_size
-
         formatter = Formatter(
             FormatterOpts(
                 indent=indent,
@@ -102,7 +94,7 @@ class TclLanguageServer(server.LanguageServer):
         self.parser.violations = []
         tree = self.parser.parse(document.source)
         violations += self.parser.violations
-
+        # log_to_output(tree.pretty(2))
         for checker in checks.get_checkers():
             violations += checker.check(document.source, tree)
         return violations
@@ -170,16 +162,6 @@ WORKSPACE_SETTINGS = {}
 GLOBAL_SETTINGS = {}
 RUNNER = pathlib.Path(__file__).parent / "lsp_runner.py"
 
-TOKEN_TYPES = [
-    "command",
-    "variable",
-    "function",
-    "string",
-    "number",
-    "keyword",
-    "comment",
-]
-TOKEN_MODIFIERS = []
 
 MAX_WORKERS = 5
 LSP_SERVER = TclLanguageServer(
@@ -207,12 +189,15 @@ def did_open(params: lsp.DidOpenTextDocumentParams) -> None:
     """LSP handler for textDocument/didOpen request."""
     document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
     LSP_SERVER.compute_diagnostics(document)
+    completion.reset()
+    tree = LSP_SERVER.parser.parse(document.source)
+    tree.accept(completion, recurse=True)
 
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_SAVE)
 def did_save(params: lsp.DidSaveTextDocumentParams) -> None:
     """LSP handler for textDocument/didSave request."""
-    document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+    _ = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
 
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_CLOSE)
@@ -225,6 +210,9 @@ def did_change(params: lsp.DidChangeTextDocumentParams) -> None:
     """LSP handler for textDocument/didChange request"""
     document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
     LSP_SERVER.compute_diagnostics(document)
+    completion.reset()
+    tree = LSP_SERVER.parser.parse(document.source)
+    tree.accept(completion, recurse=True)
 
 
 @LSP_SERVER.feature(
@@ -254,22 +242,63 @@ def document_diagnostic(params: lsp.DocumentDiagnosticParams):
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_COMPLETION)
 def on_completion(params: lsp.CompletionParams) -> list[lsp.CompletionItem]:
+    _ = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+
     items = (
         standard_items.tcl_keyword_list
         + standard_items.nx_procs
         + standard_items.nx_variables
+        + completion.custom_functions
     )
     return lsp.CompletionList(is_incomplete=False, items=items)
 
 
-@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL)
+# @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DOCUMENT_SYMBOL)
-def on_semantic_tokens(params: lsp.SemanticTokensParams):
+# def document_symbols(params: lsp.DocumentSymbolParams):
-    doc = LSP_SERVER.workspace.get_document(params.text_document.uri)
+#     doc = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
-    code = doc.source
+#     return []
 
-    tokens = collect_semantic_tokens(code)
-    data = encode_tokens(tokens)
 
+@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_INLAY_HINT)
+def inlay_hints(params: lsp.InlayHintParams):
+    document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+    tree = LSP_SERVER.parser.parse(document.source)
+
+    # collect Inlay Hints
+    generator = InlayHintGenerator(completion.proc_signatures)
+    tree.accept(generator, recurse=True)
+
+    return generator.hints
+
+
+@LSP_SERVER.feature(
+    lsp.TEXT_DOCUMENT_SEMANTIC_TOKENS_FULL,
+    lsp.SemanticTokensLegend(
+        token_types=TOKEN_TYPES,
+        token_modifiers=[m.name for m in TokenModifier],
+    ),
+)
+def semantic_tokens(params: lsp.SemanticTokensParams):
+    document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+
+    data = []
+    plugins = []
+    hl = _Highlighter(plugins, log_to_output=log_to_output)
+
+    tree = LSP_SERVER.parser.parse(document.source)
+    tree.accept(hl, recurse=True)
+
+    tokens = hl.tokens()
+    for token in tokens:
+        data.extend(
+            [
+                token.line,
+                token.offset,
+                token.lenght,
+                TOKEN_TYPES.index(token.tok_type),
+                reduce(operator.or_, token.tok_modifiers, 0),
+            ]
+        )
     return lsp.SemanticTokens(data=data)
 
 
@@ -363,12 +392,12 @@ def formatting(params: lsp.DocumentFormattingParams) -> list[lsp.TextEdit] | Non
     start = lsp.Position(line=0, character=0)
     last_line = source.rsplit("\n", 1)[-1]
     end = lsp.Position(line=source.count("\n"), character=len(last_line))
-
+    if GLOBAL_SETTINGS.get("formatter", True):
-    formatted = LSP_SERVER.format(doc, params.options)
+        source = LSP_SERVER.format(doc, params.options)
     return [
         lsp.TextEdit(
             range=lsp.Range(start=start, end=end),
-            new_text=formatted,
+            new_text=source,
         )
     ]
 
@@ -401,7 +430,7 @@ def initialize(params: lsp.InitializeParams) -> lsp.InitializeResult:
     )
     semantic_tokens_legend = lsp.SemanticTokensLegend(
         token_types=TOKEN_TYPES,
-        token_modifiers=TOKEN_MODIFIERS,
+        token_modifiers=TokenModifier,
     )
     return lsp.InitializeResult(
         capabilities=lsp.ServerCapabilities(

server/src/test_tcl.py

@@ -1,7 +1,5 @@
 from tclint.parser import Parser as tcLintParser
 from tclint.lexer import Lexer as tclingLexer
-from parser.lexer import Lexer
-from parser.parser import Parser
 
 
 def main():
@@ -28,22 +26,9 @@ LIB_SPF_prepend MOM_strt Start_Lib {
 
 def lexer_test():
     lexer = tclingLexer()
-    tree = lexer.input("""puts hello
+    tree = lexer.input(""" 
-                 proc myProc {arg {arg7 0}} {}
+if {$oem(custom_clamp_4th) == 1 && $oem(custom_clamp_5th) == 1 \\
-                 set myVar 123
+        && $oem(status_clamp_4th) == "off" && $oem(status_clamp_5th) == "off"}""")
-                 puts puts
-                 LIB_SPF_prepend MOM_strt Start_Lib {
-        set somthing 1
-        set more 2
-} myTag
-
-LIB_SPF_prepend MOM_strt Start_Lib {            
-    proc test {} {
-        puts "Hello"
-    }       
-    set somthing 1
-    set someting 3
-} myTag""")
 
     # print("Lexing input:\n", code)
     # print("\nTokens:\n" + "-" * 30)
@@ -57,26 +42,18 @@ LIB_SPF_prepend MOM_strt Start_Lib {
 
 
 def test_1():
-    code = """
+    from tclint.lexer import Lexer, TOK_BACKSLASH_NEWLINE
-        proc myProc {arg } {
-            set myVar 1
-        }
 
-        namespace eval myNS {
+    code = "expr {1 == 2 \\\n&& 3 == 4}"
-            proc innerProc {} {
-                MOM_abort_program "Test"
-            }
-        }
-        set result [myNS::innerProc]
-        """
 
     lexer = Lexer()
     lexer.input(code)
-    parser = Parser(lexer)
-    ast = parser.parse()
 
-    visitor = NodeVisitor()
+    while lexer.type() is not None:
-    ast.accept(visitor)
+        print(
+            f"Type: {lexer.type():<20} | Value: {lexer.value()!r} | Pos: {lexer.pos()}"
+        )
+        lexer.next()
 
 
 class NodeVisitor:
@@ -98,4 +75,4 @@ class NodeVisitor:
 
 
 if __name__ == "__main__":
-    main()
+    test_1()

server/src/tools/checks.py

@@ -1,4 +1,57 @@
+from enum import Enum
+from tclint.syntax_tree import Visitor, BareWord, Command
+from tclint.violations import Violation, Rule
+
+
+class Rules(Enum):
+    VALIDATION = "validation"
+    OPTIONAL_ARG_POSITION = "optinal_args"
+
+    def __str__(self):
+        return self.value
+
+
+class CommandArgsCheck(Visitor):
+    def __init__(self):
+        self._violations = []
+
+    def check(self, _, tree):
+        self._violations.clear()
+        tree.accept(self, recurse=True)
+        return self._violations
+
+    def visit_command(self, command: Command):
+        if (
+            not hasattr(command.routine, "contents")
+            or command.routine.contents != "proc"
+        ):
+            return
+        if len(command.args) < 2:
+            return
+
+        args_node = command.args[1]
+        if not hasattr(args_node, "children"):
+            return
+
+        found_optional = False
+        for arg in args_node.children:
+            # Required argument
+            if isinstance(arg, BareWord):
+                if found_optional:
+                    self._violations.append(
+                        Violation(
+                            Rules.OPTIONAL_ARG_POSITION,
+                            "Required argument follows optional one",
+                            arg.pos,
+                            arg.end_pos,
+                        )
+                    )
+            # Optional argument
+            elif hasattr(arg, "children") and len(arg.children) >= 2:
+                found_optional = True
+
+
 def get_checkers():
-    checkers = ()
+    checkers = (CommandArgsCheck(),)
 
     return checkers

server/src/tools/completion_items.py

@@ -0,0 +1,68 @@
+from tclint.syntax_tree import Visitor, Command, BareWord, List
+import lsprotocol.types as lsp
+
+
+class CompletionItems:
+    def __init__(self):
+        self._custom_functions: list[lsp.CompletionItem] = []
+
+    @property
+    def custom_functions(self) -> list[lsp.CompletionItem]:
+        return self._custom_functions
+
+    @custom_functions.setter
+    def custom_functions(self, value: lsp.CompletionItem):
+        self._custom_functions.append(value)
+
+
+class _Completion(Visitor):
+    def __init__(self):
+        super().__init__()
+        self._custom_functions: list[lsp.CompletionItem] = []
+        self._proc_signatures = {}
+
+    @property
+    def custom_functions(self) -> list[lsp.CompletionItem]:
+        return self._custom_functions
+
+    @property
+    def proc_signatures(self):
+        return self._proc_signatures
+
+    def reset(self):
+        self._custom_functions = []
+        self._proc_signatures = {}
+
+    def visit_command(self, command: Command):
+        routine = command.routine
+
+        if routine.contents == "proc" and command.args:
+            first_arg = command.args[0]
+            if hasattr(first_arg, "value") and not any(
+                item.label == first_arg.value for item in self._custom_functions
+            ):
+                self._custom_functions.append(
+                    lsp.CompletionItem(
+                        label=first_arg.value, kind=lsp.CompletionItemKind.Function
+                    )
+                )
+            if len(command.args) < 2:
+                return
+
+            param_list_node = command.args[1]
+            if not hasattr(param_list_node, "children"):
+                return
+
+            param_names = []
+            for arg in param_list_node.children:
+                if isinstance(arg, BareWord):
+                    param_names.append(arg.value)
+                elif isinstance(arg, List) and len(arg.children) >= 1:
+                    first = arg.children[0]
+                    if isinstance(first, BareWord):
+                        param_names.append(first.value)
+
+            self._proc_signatures[first_arg.value] = param_names
+
+
+completion = _Completion()

server/src/tools/inlay_hint.py

@@ -0,0 +1,29 @@
+import lsprotocol.types as lsp
+from tclint.syntax_tree import Visitor, Command
+
+
+class InlayHintGenerator(Visitor):
+    def __init__(self, proc_signatures):
+        self.proc_signatures = proc_signatures
+        self.hints = []
+
+    def visit_command(self, command: Command):
+        name = getattr(command.routine, "contents", None)
+        if name not in self.proc_signatures:
+            return
+
+        param_names = self.proc_signatures[name]
+        for idx, arg in enumerate(command.args):
+            if idx >= len(param_names):
+                break
+            param_name = param_names[idx]
+
+            if arg.pos:
+                line, col = arg.pos
+                self.hints.append(
+                    lsp.InlayHint(
+                        position=lsp.Position(line=line - 1, character=col - 1),
+                        label=f"{param_name}:",
+                        kind=lsp.InlayHintKind.Parameter,
+                    )
+                )

server/src/tools/parser.py

@@ -1,49 +1,347 @@
-import textwrap
 from tclint.parser import Parser
-from tclint.lexer import (
+from tclint.commands import CommandArgError
-    STATE_BRACEDWORD,
+from tclint.syntax_tree import (
-    TOK_LBRACE,
+    BracedWord,
-    TOK_EOF,
+    BareWord,
-    TOK_RBRACE,
+    BracedExpression,
-    TclSyntaxError,
+    List,
+    QuotedWord,
+    Expression,
 )
-from tclint.syntax_tree import BracedWord
+
+import ply.lex as lex
+from typing import Tuple
+
+TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE"
+TOK_BACKSLASH_SUB = "BACKSLASH_SUB"
+TOK_NEWLINE = "NEWLINE"
+TOK_SEMI = "SEMI"
+TOK_WS = "WS"
+TOK_QUOTE = "QUOTE"
+TOK_ARG_EXPANSION = "ARG_EXPANSION"
+TOK_LBRACE = "LBRACE"
+TOK_RBRACE = "RBRACE"
+TOK_STAR = "STAR"
+TOK_LBRACKET = "LBRACKET"
+TOK_RBRACKET = "RBRACKET"
+TOK_DOLLAR = "DOLLAR"
+TOK_LPAREN = "LPAREN"
+TOK_RPAREN = "RPAREN"
+TOK_HASH = "HASH"
+TOK_ALPHA_CHARS = "ALPHA_CHARS"
+TOK_NUM_CHARS = "NUM_CHARS"
+TOK_NAMESPACE_SEP = "NAMESPACE_SEP"
+TOK_CHAR = "CHAR"
+TOK_CONTENTS = "CONTENTS"
+TOK_EOF = None
+
+STATE_BRACEDWORD = "bracedword"
+
+
+class TclSyntaxError(Exception):
+    def __init__(self, message, start: Tuple[int, int], end: Tuple[int, int]):
+        super().__init__(message)
+        self.start = start
+        self.end = end
+
+
+class _LexTable:
+    tokens = (
+        TOK_BACKSLASH_NEWLINE,
+        TOK_BACKSLASH_SUB,
+        TOK_NEWLINE,
+        TOK_SEMI,
+        TOK_WS,
+        TOK_QUOTE,
+        TOK_ARG_EXPANSION,
+        TOK_LBRACE,
+        TOK_RBRACE,
+        TOK_STAR,
+        TOK_LBRACKET,
+        TOK_RBRACKET,
+        TOK_DOLLAR,
+        TOK_LPAREN,
+        TOK_RPAREN,
+        TOK_HASH,
+        TOK_ALPHA_CHARS,
+        TOK_NUM_CHARS,
+        TOK_NAMESPACE_SEP,
+        TOK_CHAR,
+        TOK_CONTENTS,
+    )
+
+    # This defines a conditional lexing state for parsing braced words. This is a
+    # performance optimization; since there are few special characters in this context,
+    # we can use a smaller set of tokens to parse them faster. This has a large impact
+    # since most Tcl programs have a large number of braced words. Any token with
+    # `bracedword` in its name is included in this state. Tokens that are included in
+    # this state and the default state also include `INITIAL` in their name.
+    states = ((STATE_BRACEDWORD, "exclusive"),)
+
+    def _tok(self, t):
+        pos = (t.lexer.lineno, t.lexer.colno)
+        t.lexer.lineno += t.value.count("\n")
+        index = t.value.rfind("\n")
+        if index == -1:
+            t.lexer.colno += len(t.value)
+        else:
+            remaining = t.value[index + 1 :]
+            t.lexer.colno = len(remaining) + 1
+
+        t.value = (t.value, pos)
+        return t
+
+    # Priority important
+    def t_bracedword_INITIAL_BACKSLASH_NEWLINE(self, t):
+        r"\\\r?\n"
+        return self._tok(t)
+
+    # Priority important
+    def t_bracedword_INITIAL_BACKSLASH_SUB(self, t):
+        r"\\."
+        return self._tok(t)
+
+    def t_NEWLINE(self, t):
+        r"\n"
+        return self._tok(t)
+
+    def t_SEMI(self, t):
+        r";"
+        return self._tok(t)
+
+    # TODO: should use \s?
+    def t_WS(self, t):
+        r"[\t\v\f\r ]+"
+        return self._tok(t)
+
+    def t_QUOTE(self, t):
+        r'"'
+        return self._tok(t)
+
+    # Must be higher priority than LBRACE
+    def t_ARG_EXPANSION(self, t):
+        r"\{\*\}"
+        return self._tok(t)
+
+    def t_bracedword_INITIAL_LBRACE(self, t):
+        r"\{"
+        return self._tok(t)
+
+    def t_bracedword_INITIAL_RBRACE(self, t):
+        r"\}"
+        return self._tok(t)
+
+    def t_STAR(self, t):
+        r"\*"
+        return self._tok(t)
+
+    def t_LBRACKET(self, t):
+        r"\["
+        return self._tok(t)
+
+    def t_RBRACKET(self, t):
+        r"\]"
+        return self._tok(t)
+
+    def t_DOLLAR(self, t):
+        r"\$"
+        return self._tok(t)
+
+    def t_LPAREN(self, t):
+        r"\("
+        return self._tok(t)
+
+    def t_RPAREN(self, t):
+        r"\)"
+        return self._tok(t)
+
+    def t_HASH(self, t):
+        r"\#"
+        return self._tok(t)
+
+    # Valid non-numeric chars in variable names
+    def t_ALPHA_CHARS(self, t):
+        r"[A-Za-z_]+"
+        return self._tok(t)
+
+    # Valid numeric chars in variable names
+    # This is split up from the above to facilitate expression parsing, since
+    # e.g. 1eq1 can't be a single token.
+    def t_NUM_CHARS(self, t):
+        r"[0-9]+"
+        return self._tok(t)
+
+    def t_NAMESPACE_SEP(self, t):
+        r"::+"
+        return self._tok(t)
+
+    def t_bracedword_CONTENTS(self, t):
+        r"[^{}\\]+"
+        return self._tok(t)
+
+    # Catch-all. TODO: inefficient, should probably munch multiple chars
+    def t_CHAR(self, t):
+        r"."
+        return self._tok(t)
+
+    # Error handling rule
+    # TODO: do we need this? since we have a catch-all...
+    # there is a warning
+    def t_bracedword_INITIAL_error(self, t):
+        print("Illegal character '%s'" % t.value[0])
+        t.lexer.skip(1)
+
+    def __init__(self):
+        self.lexer = lex.lex(object=self)
+        self.lexer.lineno = 1
+        self.lexer.colno = 1
+
+    def new_lexer(self, pos=None):
+        lexer = self.lexer.clone()
+        lexer.lineno = 1
+        lexer.colno = 1
+
+        if pos is not None:
+            line, col = pos
+            lexer.lineno = line
+            lexer.colno = col
+
+        return lexer
+
+
+# Calling `lex.lex()` performs an expensive reflection process to generate the lexer.
+# This singleton class holds a preinitialized lexer that can then be cloned to create
+# individual instances.
+LexTable = _LexTable()
+
+
+class Lexer:
+    def __init__(self, pos=None):
+        self.lexer = LexTable.new_lexer(pos)
+        self.current = None
+
+    def input(self, text):
+        self.lexer.input(text)
+        self.current = self.lexer.token()
+
+    def type(self):
+        if self.current is None:
+            return TOK_EOF
+        return self.current.type
+
+    def value(self):
+        if self.current is None:
+            return None
+        return self.current.value[0]
+
+    def pos(self):
+        if self.current is None:
+            return (self.lexer.lineno, self.lexer.colno)
+        return self.current.value[1]
+
+    def next(self):
+        self.current = self.lexer.token()
+
+    def expect(self, *tokens, message, pos):
+        if self.type() not in tokens:
+            self.next()  # munch another token to update position
+            raise TclSyntaxError(message, pos, self.pos())
+
+        self.next()
+
+    def assert_(self, *tokens):
+        assert self.current.type in tokens
+        self.next()
 
 
 class CustomParser(Parser):
-    def parse_braced_word(self, ts):
+    def parse(self, script, pos=None):
-        """
+        lexer = Lexer(pos=pos)
-        Ersetzt BracedWord durch echtes Script, wenn mehrzeilig.
+        lexer.input(script)
-        """
+        tree = self._parse_script(lexer, in_command_sub=False)
-        pos = ts.pos()
+        assert lexer.type() == TOK_EOF, (
-        ts.lexer.push_state(STATE_BRACEDWORD)
+            "Didn't reach EOF parsing script, please file a bug report."
-        ts.assert_(TOK_LBRACE)
-
-        content = ""
-        expected = [pos]
-        while True:
-            t = ts.type()
-            if t == TOK_EOF:
-                raise TclSyntaxError(
-                    "reached EOF without finding match for brace",
-                    expected[-1],
-                    ts.pos(),
         )
-            if t == TOK_LBRACE:
+
-                expected.append(ts.pos())
+        return tree
-            elif t == TOK_RBRACE:
+
-                expected.pop()
+    def parse_list(self, node):
-                if not expected:
+        """Parse contents of node as Tcl list. This is a distinct entry point
-                    ts.lexer.pop_state()
+        that doesn't get used when generating the main syntax tree, but is used
+        in command-specific argument parsing.
+        """
+        if isinstance(node, List):
+            return node
+
+        if node.contents is None:
+            raise CommandArgError(
+                "expected braced word or word without substitutions in argument"
+                " interpreted as list"
+            )
+
+        ts = Lexer(pos=node.contents_pos)
+        ts.input(node.contents)
+
+        DELIMITERS = {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}
+
+        list_node = List(pos=node.pos, end_pos=node.end_pos)
+        while ts.type() is not TOK_EOF:
+            while ts.type() in DELIMITERS:
                 ts.next()
+
+            if ts.type() is TOK_EOF:
                 break
-            content += ts.value()
+
+            if ts.type() == TOK_LBRACE:
+                # we can reuse parse_braced_word, since it doesn't use
+                # substitutions in any case
+                list_node.add(self.parse_braced_word(ts))
+            elif ts.type() == TOK_QUOTE:
+                quote_word_pos = ts.pos()
+
+                ts.assert_(TOK_QUOTE)
+
+                bare_word_pos = ts.pos()
+                contents = ""
+                while ts.type() not in {TOK_QUOTE, TOK_EOF}:
+                    contents += ts.value()
                     ts.next()
+                word = BareWord(contents, pos=bare_word_pos, end_pos=ts.pos())
 
-        end_pos = ts.pos()
+                ts.expect(
-        # Mehrzeilig? Dann als Script parsen:
+                    TOK_QUOTE,
-        if "\n" in content.strip():
+                    message="reached EOF without finding match for quote",
-            self.parse_script(content)
+                    pos=quote_word_pos,
+                )
 
-        # Einzeilig: unverändert als Literal
+                list_node.add(QuotedWord(word, pos=quote_word_pos, end_pos=ts.pos()))
-        return BracedWord(content, pos=pos, end_pos=end_pos)
+            else:
+                pos = ts.pos()
+                contents = ""
+                while ts.type() not in {*DELIMITERS, TOK_EOF}:
+                    contents += ts.value()
+                    ts.next()
+                list_node.add(BareWord(contents, pos=pos, end_pos=ts.pos()))
+
+        return list_node
+
+    def parse_expression(self, node):
+        if node.contents is None:
+            raise CommandArgError(
+                "expected braced word or word without substitutions in argument"
+                " interpreted as expr"
+            )
+
+        ts = Lexer(pos=node.contents_pos)
+        ts.input(node.contents)
+
+        contents = self._parse_expression(ts)
+        ts.expect(
+            TOK_EOF,
+            message=f"expected end of expression, got {ts.value()}",
+            pos=ts.pos(),
+        )
+        if isinstance(node, BracedWord):
+            return BracedExpression(contents, pos=node.pos, end_pos=node.end_pos)
+
+        return Expression(contents, pos=node.pos, end_pos=node.end_pos)

server/src/tools/semantic_tokens.py

@@ -1,76 +1,173 @@
-from tclint.parser import Parser
+import enum
-from tclint.syntax_tree import Visitor, BareWord, VarSub, Comment, Command, Function
+from typing import List
-
+from tclint.syntax_tree import Visitor, QuotedWord, Command, BareWord
-TOKEN_TYPES = {
+from tclint.commands import get_commands
-    "command": 0,
+import attrs
-    "variable": 1,
+from common.load_data import standard_items
-    "function": 2,
+from tools.completion_items import completion
-    "string": 3,
-    "number": 4,
-    "keyword": 5,
-    "comment": 6,
-}
 
 
-class SemanticTokenCollector(Visitor):
+class TokenModifier(enum.IntFlag):
-    def __init__(self):
+    deprecated = enum.auto()
-        self.tokens = []
+    readonly = enum.auto()
+    defaultLibrary = enum.auto()
+    definition = enum.auto()
+    declaration = enum.auto()
+    builtin = enum.auto()
 
-    def _add_token(self, node, token_type):
+
-        if not node.pos or not node.end_pos:
+@attrs.define
+class Token:
+    line: int
+    offset: int
+    lenght: int
+
+    tok_type: str = ""
+    tok_modifiers: List[TokenModifier] = attrs.field(factory=list)
+
+
+TOKEN_TYPES = [
+    "keyword",
+    "variable",
+    "function",
+    "operator",
+    "parameter",
+    "type",
+    "class",
+    "string",
+    "parameter",
+]
+
+
+class _Highlighter(Visitor):
+    def __init__(self, plugins, log_to_output):
+        self._commands = get_commands(plugins)
+        self._tokens = []
+        self.log_to_output = log_to_output
+
+    def visit_quoted_word(self, word: QuotedWord):
+        if not word.contents:
             return
-
+        line, col = word.contents_pos
-        line, col = node.pos
+        self._tokens.append(((line - 1, col - 1), len(word.contents), "string", []))
-        end_line, end_col = node.end_pos
+        pass
-        length = (end_col - col) if line == end_line else 1
-
-        self.tokens.append((line - 1, col - 1, length, token_type, 0))
-
-    def visit_command(self, command: Command):
-        self._add_token(command.routine, "command")
-        for arg in command.args:
-            arg.accept(self, recurse=True)
-
-    def visit_comment(self, comment: Comment):
-        self._add_token(comment, "comment")
 
     def visit_bare_word(self, word: BareWord):
-        if word.value.isdigit():
+        if any(item.label == word.value for item in standard_items.nx_procs) or any(
-            self._add_token(word, "number")
+            item.label == word.value for item in completion.custom_functions
-        else:
+        ):
-            self._add_token(word, "string")
+            line, col = word.pos
+            self._tokens.append(
+                (((line - 1, col - 1), len(word.value), "function", []))
+            )
 
-    def visit_var_sub(self, var_sub: VarSub):
+    def visit_command(self, command: Command):
-        self._add_token(var_sub, "variable")
+        routine = command.routine
 
-    def visit_function(self, function: Function):
+        if routine.contents == "puts":
-        self._add_token(function.name, "function")
+            line, col = routine.contents_pos
-        for arg in function.args:
+            self._tokens.append(
-            arg.accept(self, recurse=True)
+                (
+                    (
+                        (line - 1, col - 1),
+                        len(routine.contents),
+                        "function",
+                        [TokenModifier.builtin],
+                    )
+                )
+            )
+            pass
 
+        if routine.contents == "set" and command.args:
+            first_arg = command.args[0]
+            if hasattr(first_arg, "pos") and hasattr(first_arg, "value"):
+                line, col = first_arg.pos
+                self._tokens.append(
+                    (
+                        (
+                            (line - 1, col - 1),
+                            len(first_arg.value),
+                            "variable",
+                            [TokenModifier.declaration],
+                        )
+                    )
+                )
+        if routine.contents == "proc" and command.args:
+            first_arg = command.args[0]
+            if hasattr(first_arg, "pos") and hasattr(first_arg, "value"):
+                line, col = first_arg.pos
+                self._tokens.append(
+                    (
+                        (
+                            (line - 1, col - 1),
+                            len(first_arg.value),
+                            "function",
+                            [TokenModifier.declaration],
+                        )
+                    )
+                )
 
-def collect_semantic_tokens(code: str):
+            if len(command.args) >= 2:
-    parser = Parser()
+                param_list = command.args[1]
-    tree = parser.parse(code)
-    visitor = SemanticTokenCollector()
-    tree.accept(visitor, recurse=True)
-    return visitor.tokens
 
+                # BracedWord oder Liste erwartet
+                if hasattr(param_list, "children"):
+                    for child in param_list.children:
+                        # Parameter kann einfaches Wort sein
+                        if hasattr(child, "value") and child.value is not None:
+                            line, col = child.pos
+                            self._tokens.append(
+                                (
+                                    (line - 1, col - 1),
+                                    len(child.value),
+                                    "parameter",
+                                    [TokenModifier.declaration],
+                                )
+                            )
 
-def encode_tokens(tokens):
+                        # Parameter mit Default-Wert ist meist eine List (z. B. {arg default})
-    tokens.sort()
+                        elif hasattr(child, "children") and len(child.children) >= 1:
-    encoded = []
+                            name_node = child.children[0]
+                            if hasattr(name_node, "value") and hasattr(
+                                name_node, "pos"
+                            ):
+                                line, col = name_node.pos
+                                self._tokens.append(
+                                    (
+                                        (line - 1, col - 1),
+                                        len(name_node.value),
+                                        "parameter",
+                                        [TokenModifier.declaration],
+                                    )
+                                )
 
+        if routine.contents == "namespace" and command.args:
+            first_arg = command.args[1]
+            if hasattr(first_arg, "pos") and hasattr(first_arg, "value"):
+                line, col = first_arg.pos
+                self._tokens.append(
+                    (((line - 1, col - 1), len(first_arg.value), "class", []))
+                )
+
+    def visit_var_sub(self, var_sub):
+        pass
+
+    def tokens(self) -> list[Token]:
+        """Encode tokens as described in
+        https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens.
+        """
+        tokens = []
         last_line = 0
-    last_char = 0
+        last_col = 0
+        for (line, col), length, tok_type, tok_modifier in sorted(
+            self._tokens, key=lambda x: x[0]
+        ):
+            line_delta = line - last_line
+            col_delta = col
+            if line == last_line:
+                col_delta -= last_col
 
-    for line, char, length, token_type, modifiers in tokens:
+            tokens.append(Token(line_delta, col_delta, length, tok_type, tok_modifier))
-        delta_line = line - last_line
+            last_line, last_col = line, col
-        delta_start = char - last_char if delta_line == 0 else char
 
-        encoded.extend([delta_line, delta_start, length, token_type, modifiers])
+        return tokens
-
-        last_line = line
-        last_char = char if delta_line == 0 else 0
-
-    return encoded

test/test.tcl

@@ -1,13 +1,105 @@
-proc myProc {arg {opt 1}} {
+set main 1
-    
+if {$main == 1 && 1 == 1} {
+    puts "main"
 }
 
-set myVar 1
+proc test {} {
+    puts "main"
+    proc llll {} {}
+    set rrrrrrr
+}
+LIB_GE_command_buffer_edit_insert MOM_tool_change_LIB TOOL_CHANGE_AUTO {CUSTOM_after_tool_change_call} mytag after @TOOL_CHANGE_AUTO
 
-namespace eval myNameSpace {
+MOM_abort
-    proc namespaceProc {} {}
+
+namespace eval myns {
+    proc add {a b} {
+        set sum [expr {$a + $b}]
+        return $sum
+    }
+    set config "debug"
 }
 
-set result [myNameSpace::namespaceProc]
+#_________________________________________________________________________________________________
+# <Documentation>
+# Function to output a spacer line or empty line
+#_________________________________________________________________________________________________
+proc SERVICE_spacer_output {type {length 20} {line_num 0} {output 1}} {
+    LIB_GE_message [string repeat $type $length] "output_$output" $line_num
+}
 
-MOM_abort_program "Test"
+
+SERVICE_spacer_output "*" 2 0 0
+
+#_________________________________________________________________________________________________
+# <Documentation>
+# Function to delete the file
+#_________________________________________________________________________________________________
+proc SERVICE_remove_file {file} {
+    if {![SERVICE_check_file_exists $file]} {return}
+    MOM_remove_file $file
+}
+
+#_________________________________________________________________________________________________
+# <Documentation>
+# Function to check if the file exists
+#_________________________________________________________________________________________________
+proc SERVICE_check_file_exists {file} {
+    if {[file exists $file]} {return 1}
+    return 0
+}
+
+#_________________________________________________________________________________________________
+# <Documentation>
+# Ask UDE Info for the Tool
+#_________________________________________________________________________________________________
+proc SERVICE_ask_ude_tool {pos ude_name tool_name} {
+    MOM_ask_ude_info $tool_name "tool" $pos
+
+    if {[lsearch $::mom_result $ude_name] != -1} {
+        return 1
+    }
+    return 0
+}
+
+#_________________________________________________________________________________________________
+# <Documentation>
+# Ask UDE Info for the Operation
+#_________________________________________________________________________________________________
+proc SERVICE_ask_ude_operation {pos ude_name path_name} {
+    MOM_ask_ude_info $path_name "operation" $pos
+
+    if {[lsearch $::mom_result $ude_name] != -1} {
+        return 1
+    }
+    return 0
+}
+
+#_________________________________________________________________________________________________
+# <Documentation>
+# output suppress or dont suppress
+# [SERVICE_output_handling "ingore_output"] ignores the output
+# arg options: ingore_output
+#              restore
+#_________________________________________________________________________________________________
+proc SERVICE_output_handling {handler} {
+    set ::lib_ge(hidden_output) $handler
+}
+
+#_________________________________________________________________________________________________
+# <Documentation>
+# write the mom_tool_data to store tool information
+# this function is called in start of program
+#_________________________________________________________________________________________________
+proc SERVICE_get_tool_data {} {
+    global mom_tool_data
+global mom_operation_info
+
+    set mom_tool_data(toollist) ""
+    set operations $::mom_operation_name_list
+    foreach operation $operations {
+        if {[lsearch -exact $mom_tool_data(toollist) $mom_operation_info($operation,tool_name)] == -1} {
+            lappend mom_tool_data(toollist) $mom_operation_info($operation,tool_name)
+        }
+    }
+}