Update version to 2025.08.1

Reviewed-on: #24

.gitea/workflows/build_and_puplish.yml

@@ -1,7 +1,6 @@
 on:
-    push:
-        tags:
-            - "*"
+    release:
+        types: [published]
 jobs:
     build_and_publish:
         runs-on: ubuntu-latest
@@ -35,14 +34,22 @@ jobs:
                   git add package.json
                   git commit -m "Update version to ${{ github.ref_name }}"
                   git push origin HEAD:main
-            - name: Publish to Visual Studio Marketplace
+            - name: Publish to Visual Studio Marketplace (Pre-Release)
+              if: ${{ github.event.release.prerelease }}
+              run: vsce publish --pre-release
+              env:
+                  VSCE_PAT: ${{ secrets.VSCODE_MARKETPALCE }}
+            - name: Publish to Visual Studio Marketplace (Stable)
+              if: ${{ !github.event.release.prerelease }}
               run: vsce publish
               env:
                   VSCE_PAT: ${{ secrets.VSCODE_MARKETPALCE }}
+            - name: Pack Extension for Release
+              run: vsce pack
             - name: Upload VSIX to Release
               uses: softprops/action-gh-release@v1
               with:
-                  files: "*.vsix"
+                  files: "nx-post-support-*.vsix"
                   # generate_release_notes: true  # Automatische Release Notes
                   # draft: false                  # N
                   # prerelease: false

.vscode/settings.json

@@ -2,6 +2,7 @@
     "ruff.configuration": {
         "lint": {
             "extend-ignore": ["F821", "E402"]
-        }
+        },
+        "line-length": 200
     }
 }

.vscodeignore

@@ -11,8 +11,11 @@ esbuild.js
 .gitea
 .venv/**
 .nox/**
+server/.venv/**
+server/.nox/**
 **/__pycache__/**
 **/requirements.txt
 **/requirements.in
 **/server/src/_debug_server.py
-noxfile.py
+server/noxfile.py
+client/**

README.md

@@ -0,0 +1,49 @@
+# NX Postprocessor Support
+
+A comprehensive VS Code extension providing language support for NX CAD postprocessor development, including CDL (Cutter Definition Language), TCL, and DEF files.
+
+## Features
+
+- **Syntax Highlighting** - Full syntax highlighting for CDL, TCL, and DEF files
+- **Code Formatting** - Automatic code formatting for all supported file types
+- **Multi-language Support** - Supports NX CDL, TCL, and DEF file formats
+- **Intelligent Code Analysis** - Linting and error detection for postprocessor code
+- **Auto-completion** - Context-aware code completion for faster development
+
+## Supported File Types
+
+- `.cdl` - NX Cutter Definition Language files
+- `.tcl` - NX TCL postprocessor files
+- `.def` - NX Definition files
+
+## Installation
+
+1. Install from the VS Code Marketplace
+2. Open any `.cdl`, `.tcl`, or `.def` file
+3. The extension will automatically activate and provide language support
+
+## Configuration
+
+The extension can be configured through VS Code settings:
+
+- `nx-post-support.interpreter` - Specify custom Python interpreter path for the language server
+- `nx-post-support.formatter` - Enable/disable the TCL formatter (default: false)
+- `nx-post-support.inlayHint` - Enable/disable inlay Hints (default: true)
+
+## Usage
+
+Simply open any supported file type and enjoy:
+
+- Syntax highlighting
+- Error detection and linting
+- Code completion
+- Code formatting (Format Document command)
+- Hover information
+
+## Contributing
+
+This extension is actively maintained. For issues or feature requests, please visit our [repository](https://git.cbsk-tech.de/Christoph/nx_post_support.git).
+
+## License
+
+MIT License - see LICENSE file for details.

package.json

@@ -2,8 +2,9 @@
   "name": "nx-post-support",
   "displayName": "NX Postprocessor Support",
   "description": "",
-  "version": "0.4.11",
+  "version": "2025.8.1",
   "publisher": "Christoph",
+  "icon": "images/nx-1.png",
   "serverInfo": {
     "name": "NX Postprocessor Support",
     "module": "nx-post-support"

server/src/_debug_server.py

@@ -32,7 +32,7 @@ if debugger_path:
     # This will ensure that execution is paused as soon as the debugger
     # connects to VS Code. If you don't want to pause here comment this
     # line and set breakpoints as appropriate.
-    debugpy.breakpoint()
+    # debugpy.breakpoint()
 
 SERVER_PATH = os.fspath(pathlib.Path(__file__).parent / "lsp_server.py")
 # NOTE: Set breakpoint in `lsp_server.py` before continuing.

server/src/lsp_server.py

@@ -47,8 +47,9 @@ from tclint.violations import Violation
 from plugins.poco_plugin import commands
 from tools import checks, parser
 from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier
-from tools.completion_items import completion
+from tools.completion_items import completion, remove_existing_items, remove_shared_keys
 from tools.inlay_hint import InlayHintGenerator
+from tools.file_sourcing import get_all_psc_files, read_psc_file
 
 DIAGNOSTIC_SOURCE = "nx-post-support"
 
@@ -60,6 +61,8 @@ class TclLanguageServer(server.LanguageServer):
         for command in commands:
             self.parser._commands.update(command)
         self.diagnostics = {}
+        self.poco_completion: dict = {}
+        self.proc_signatures: dict = {}
 
     def format(
         self,
@@ -121,12 +124,8 @@ class TclLanguageServer(server.LanguageServer):
         for violation in violations:
             message = violation.message
             severity = lsp.DiagnosticSeverity.Warning
-            start = lsp.Position(
-                line=violation.start[0] - 1, character=violation.start[1] - 1
-            )
-            end = lsp.Position(
-                line=violation.end[0] - 1, character=violation.end[1] - 1
-            )
+            start = lsp.Position(line=violation.start[0] - 1, character=violation.start[1] - 1)
+            end = lsp.Position(line=violation.end[0] - 1, character=violation.end[1] - 1)
 
             diagnostics.append(
                 lsp.Diagnostic(
@@ -164,9 +163,7 @@ RUNNER = pathlib.Path(__file__).parent / "lsp_runner.py"
 
 
 MAX_WORKERS = 5
-LSP_SERVER = TclLanguageServer(
-    name="NX Postprocessor Support", version="0.0.1", max_workers=MAX_WORKERS
-)
+LSP_SERVER = TclLanguageServer(name="NX Postprocessor Support", version="0.0.1", max_workers=MAX_WORKERS)
 
 # **********************************************************
 # Tool specific code goes below this.
@@ -189,9 +186,6 @@ def did_open(params: lsp.DidOpenTextDocumentParams) -> None:
     """LSP handler for textDocument/didOpen request."""
     document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
     LSP_SERVER.compute_diagnostics(document)
-    completion.reset()
-    tree = LSP_SERVER.parser.parse(document.source)
-    tree.accept(completion, recurse=True)
 
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_SAVE)
@@ -210,9 +204,6 @@ def did_change(params: lsp.DidChangeTextDocumentParams) -> None:
     """LSP handler for textDocument/didChange request"""
     document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
     LSP_SERVER.compute_diagnostics(document)
-    completion.reset()
-    tree = LSP_SERVER.parser.parse(document.source)
-    tree.accept(completion, recurse=True)
 
 
 @LSP_SERVER.feature(
@@ -243,20 +234,21 @@ def document_diagnostic(params: lsp.DocumentDiagnosticParams):
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_COMPLETION)
 def on_completion(params: lsp.CompletionParams) -> list[lsp.CompletionItem]:
     _ = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+    poco = []
 
-    items = (
-        standard_items.tcl_keyword_list
-        + standard_items.nx_procs
-        + standard_items.nx_variables
-        + completion.custom_functions
-    )
+    for key, value in LSP_SERVER.poco_completion.items():
+        poco.extend(value)
+    items = standard_items.tcl_keyword_list + standard_items.nx_procs + standard_items.nx_variables + poco
     return lsp.CompletionList(is_incomplete=False, items=items)
 
 
 # @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DOCUMENT_SYMBOL)
 # def document_symbols(params: lsp.DocumentSymbolParams):
 #     doc = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
-#     return []
+#     ast = LSP_SERVER.parser.parse(doc.source)
+#     symbols = LSP_SERVER.extract_tcl_symbols(ast)
+
+#     return symbols
 
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_INLAY_HINT)
@@ -265,10 +257,12 @@ def inlay_hints(params: lsp.InlayHintParams):
     tree = LSP_SERVER.parser.parse(document.source)
 
     # collect Inlay Hints
-    generator = InlayHintGenerator(completion.proc_signatures)
+    hints = []
+    for key, value in LSP_SERVER.proc_signatures.items():
+        generator = InlayHintGenerator(LSP_SERVER.proc_signatures[key])
         tree.accept(generator, recurse=True)
-
-    return generator.hints
+        hints += generator.hints
+    return hints
 
 
 @LSP_SERVER.feature(
@@ -283,7 +277,7 @@ def semantic_tokens(params: lsp.SemanticTokensParams):
 
     data = []
     plugins = []
-    hl = _Highlighter(plugins, log_to_output=log_to_output)
+    hl = _Highlighter(plugins, LSP_SERVER.poco_completion)
 
     tree = LSP_SERVER.parser.parse(document.source)
     tree.accept(hl, recurse=True)
@@ -336,9 +330,7 @@ def hover(params: lsp.HoverParams) -> lsp.Hover:
     label = match.get("label", "")
 
     parameters = match.get("parameters", [])
-    param_lines = (
-        "\n".join(f"- `{p['name']}`: {p['desc']}" for p in parameters) or "_None_"
-    )
+    param_lines = "\n".join(f"- `{p['name']}`: {p['desc']}" for p in parameters) or "_None_"
 
     example_data = match.get("example", [])
     example_md = "\n".join(f"{line}" for line in example_data)
@@ -422,12 +414,8 @@ def initialize(params: lsp.InitializeParams) -> lsp.InitializeResult:
 
     settings = params.initialization_options["settings"]
     _update_workspace_settings(settings)
-    log_to_output(
-        f"Settings used to run Server:\r\n{json.dumps(settings, indent=4, ensure_ascii=False)}\r\n"
-    )
-    log_to_output(
-        f"Global settings:\r\n{json.dumps(GLOBAL_SETTINGS, indent=4, ensure_ascii=False)}\r\n"
-    )
+    log_to_output(f"Settings used to run Server:\r\n{json.dumps(settings, indent=4, ensure_ascii=False)}\r\n")
+    log_to_output(f"Global settings:\r\n{json.dumps(GLOBAL_SETTINGS, indent=4, ensure_ascii=False)}\r\n")
     semantic_tokens_legend = lsp.SemanticTokensLegend(
         token_types=TOKEN_TYPES,
         token_modifiers=TokenModifier,
@@ -435,13 +423,37 @@ def initialize(params: lsp.InitializeParams) -> lsp.InitializeResult:
     return lsp.InitializeResult(
         capabilities=lsp.ServerCapabilities(
             document_formatting_provider=GLOBAL_SETTINGS.get("formatter", True),
-            semantic_tokens_provider=lsp.SemanticTokensOptions(
-                legend=semantic_tokens_legend, full=True, range=False
-            ),
+            semantic_tokens_provider=lsp.SemanticTokensOptions(legend=semantic_tokens_legend, full=True, range=False),
         )
     )
 
 
+@LSP_SERVER.feature(lsp.INITIALIZED)
+def initialized(params: lsp.InitializedParams):
+    root = LSP_SERVER.workspace.root_path
+    psc_files = get_all_psc_files(pathlib.Path(root))
+    for psc_file in psc_files:
+        poco_files = read_psc_file(psc_file)
+        for sourced_layer in poco_files:
+            completion.reset()
+            try:
+                file_root = pathlib.Path(root).joinpath(sourced_layer.subfolder if sourced_layer.subfolder else "")
+                for tcl_file in sourced_layer.files:
+                    filepath = pathlib.Path(file_root).joinpath(f"{tcl_file}.tcl")
+                    if not filepath.exists():
+                        continue
+                    completion.reset()
+                    document = LSP_SERVER.workspace.get_text_document(filepath.as_uri())  # filepath.read_text(encoding="utf-8")
+                    tree = LSP_SERVER.parser.parse(document.source)
+                    tree.accept(completion, recurse=True)
+                    remove_existing_items(completion.custom_functions, LSP_SERVER.poco_completion)
+                    LSP_SERVER.poco_completion[str(filepath)] = completion.custom_functions
+                    remove_shared_keys(LSP_SERVER.proc_signatures, completion.proc_signatures)
+                    LSP_SERVER.proc_signatures[str(filepath)] = completion.proc_signatures
+            except Exception as e:
+                log_to_output(f"Fehler beim Parsen von {filepath}: {e}")
+
+
 @LSP_SERVER.feature(lsp.EXIT)
 def on_exit(_params: Optional[Any] = None) -> None:
     """Handle clean up on exit."""
@@ -533,9 +545,7 @@ def _get_settings_by_document(document: workspace.Document | None):
 # *****************************************************
 # Logging and notification.
 # *****************************************************
-def log_to_output(
-    message: str, msg_type: lsp.MessageType = lsp.MessageType.Log
-) -> None:
+def log_to_output(message: str, msg_type: lsp.MessageType = lsp.MessageType.Log) -> None:
     LSP_SERVER.show_message_log(message, msg_type)
 
 

server/src/plugins/poco_plugin.py

@@ -36,6 +36,12 @@ def lib_ge_command_buffer_edit_prepend(args, parser):
     )
 
 
+def lib_ge_command_buffer_edit_replace(args, parser):
+    _lib_ge_command_buffer_edit(
+        args, parser, "LIB_GE_command_buffer_edit_replace", 3, 5
+    )
+
+
 def lib_ge_command_buffer_edit_insert(args, parser):
     _lib_ge_command_buffer_edit(args, parser, "LIB_GE_command_buffer_edit_insert", 2, 6)
 
@@ -44,5 +50,6 @@ commands = [
     {"LIB_GE_command_buffer_edit_append": lib_ge_command_buffer_edit_append},
     {"LIB_GE_command_buffer_edit_prepend": lib_ge_command_buffer_edit_prepend},
     {"LIB_GE_command_buffer_edit_insert": lib_ge_command_buffer_edit_insert},
+    {"LIB_GE_command_buffer_edit_replace": lib_ge_command_buffer_edit_replace},
     {"LIB_GE_command_buffer": _lib_ge_command_buffer},
 ]

server/src/test_tcl.py

@@ -1,78 +0,0 @@
-from tclint.parser import Parser as tcLintParser
-from tclint.lexer import Lexer as tclingLexer
-
-
-def main():
-    parser = tcLintParser(True)
-    tree = parser.parse("""puts hello
-                 proc myProc {arg {arg7 0}} {}
-                 set myVar 123
-                 puts puts
-                 LIB_SPF_prepend MOM_strt Start_Lib {
-        set somthing 1
-        set more 2
-} myTag
-
-LIB_SPF_prepend MOM_strt Start_Lib {            
-    proc test {} {
-        puts "Hello"
-    }       
-    set somthing 1
-    set someting 3
-} myTag""")
-
-    print(tree.pretty(2))
-
-
-def lexer_test():
-    lexer = tclingLexer()
-    tree = lexer.input(""" 
-if {$oem(custom_clamp_4th) == 1 && $oem(custom_clamp_5th) == 1 \\
-        && $oem(status_clamp_4th) == "off" && $oem(status_clamp_5th) == "off"}""")
-
-    # print("Lexing input:\n", code)
-    # print("\nTokens:\n" + "-" * 30)
-
-    while lexer.type() is not None:
-        tok_type = lexer.type()
-        tok_value = lexer.value()
-        tok_pos = lexer.pos()
-        print(f"Type: {tok_type:20} | Value: {repr(tok_value):20} | Pos: {tok_pos}")
-        lexer.next()
-
-
-def test_1():
-    from tclint.lexer import Lexer, TOK_BACKSLASH_NEWLINE
-
-    code = "expr {1 == 2 \\\n&& 3 == 4}"
-
-    lexer = Lexer()
-    lexer.input(code)
-
-    while lexer.type() is not None:
-        print(
-            f"Type: {lexer.type():<20} | Value: {lexer.value()!r} | Pos: {lexer.pos()}"
-        )
-        lexer.next()
-
-
-class NodeVisitor:
-    def visit_script(self, node):
-        for stmt in node.statements:
-            stmt.accept(self)
-
-    def visit_proc(self, node):
-        print(f"Proc: {node.name}, args={node.args}")
-
-    def visit_set(self, node):
-        print(f"Set: {node.varname} = {node.value}")
-
-    def visit_namespace(self, node):
-        print(f"Namespace: {node.name}")
-
-    def visit_command(self, node):
-        print(f"Command: {node.name} {node.args}")
-
-
-if __name__ == "__main__":
-    test_1()

server/src/tools/completion_items.py

@@ -1,5 +1,6 @@
 from tclint.syntax_tree import Visitor, Command, BareWord, List
 import lsprotocol.types as lsp
+from common.load_data import standard_items
 
 
 class CompletionItems:
@@ -38,14 +39,18 @@ class _Completion(Visitor):
 
         if routine.contents == "proc" and command.args:
             first_arg = command.args[0]
-            if hasattr(first_arg, "value") and not any(
-                item.label == first_arg.value for item in self._custom_functions
-            ):
-                self._custom_functions.append(
-                    lsp.CompletionItem(
-                        label=first_arg.value, kind=lsp.CompletionItemKind.Function
-                    )
-                )
+            if not first_arg.value:
+                return
+
+            if any(item.label == first_arg.value for item in standard_items.nx_procs):
+                return
+
+            try:
+                self._custom_functions.remove(first_arg.value)
+            except ValueError:
+                pass
+
+            self._custom_functions.append(lsp.CompletionItem(label=first_arg.value, kind=lsp.CompletionItemKind.Function))
             if len(command.args) < 2:
                 return
 
@@ -65,4 +70,33 @@ class _Completion(Visitor):
             self._proc_signatures[first_arg.value] = param_names
 
 
+def remove_existing_items(items: list[lsp.CompletionItem], store: dict) -> None:
+    """
+    Entfernt alle CompletionItems aus dem store, deren label in der items-Liste vorkommt.
+    Änderungen erfolgen in-place.
+    """
+    labels_to_remove = {item.label for item in items}
+
+    for key in list(store.keys()):
+        filtered = [ci for ci in store[key] if ci.label not in labels_to_remove]
+        if filtered:
+            store[key] = filtered
+        else:
+            del store[key]
+
+
+def remove_shared_keys(nested_dict: dict[str, dict[str, list]], flat_dict: dict[str, list]) -> None:
+    """
+    Entfernt alle Keys aus nested_dict[file][func], wenn func auch in flat_dict vorhanden ist.
+    Änderungen erfolgen in-place.
+    """
+    for file_path, func_dict in list(nested_dict.items()):
+        for func_name in list(func_dict.keys()):
+            if func_name in flat_dict:
+                del nested_dict[file_path][func_name]
+
+        if not nested_dict[file_path]:
+            del nested_dict[file_path]
+
+
 completion = _Completion()

server/src/tools/file_sourcing.py

@@ -0,0 +1,51 @@
+import xml.etree.ElementTree as ET
+from dataclasses import dataclass
+from typing import List, Optional
+from pathlib import Path
+
+
+@dataclass
+class SourcedFile:
+    layer_name: str
+    subfolder: Optional[str]
+    files: List[str]
+
+
+def read_psc_file(psc_file: Path) -> List[SourcedFile]:
+    tree = ET.parse(psc_file)
+    root = tree.getroot()
+
+    layers = root.findall(".//Layer")
+
+    layer_info_list: List[SourcedFile] = []
+    for layer in layers:
+        layer_name = layer.attrib.get("Name")
+        subfolder = layer.attrib.get("SubFolder")
+        # Scripts-Filenames
+        scripts = layer.find("Scripts")
+        script_names = []
+        if scripts is not None:
+            for filename in scripts.findall("Filename"):
+                name = filename.attrib.get("Name")
+                if name:
+                    script_names.append(name)
+
+        layer_info_list.append(
+            SourcedFile(layer_name=layer_name, subfolder=subfolder, files=script_names)
+        )
+    return layer_info_list
+
+
+def get_all_psc_files(root_path: Path) -> list[Path]:
+    return [path for path in root_path.rglob("*.psc")]
+
+
+if __name__ == "__main__":
+    test = get_all_psc_files(
+        Path(
+            r"H:\janus-engineering-customers\KSB_Frankenthal\custom\library\machine\installed_machines\ksb_pe_grob_g550_sone\postprocessor"
+        )
+    )
+    print(test)
+    for pp in test:
+        read_psc_file(pp)

server/src/tools/parser.py

@@ -1,3 +1,5 @@
+import io
+from typing import Optional, Tuple
 from tclint.parser import Parser
 from tclint.commands import CommandArgError
 from tclint.syntax_tree import (
@@ -8,254 +10,17 @@ from tclint.syntax_tree import (
     QuotedWord,
     Expression,
 )
-
-import ply.lex as lex
-from typing import Tuple
-
-TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE"
-TOK_BACKSLASH_SUB = "BACKSLASH_SUB"
-TOK_NEWLINE = "NEWLINE"
-TOK_SEMI = "SEMI"
-TOK_WS = "WS"
-TOK_QUOTE = "QUOTE"
-TOK_ARG_EXPANSION = "ARG_EXPANSION"
-TOK_LBRACE = "LBRACE"
-TOK_RBRACE = "RBRACE"
-TOK_STAR = "STAR"
-TOK_LBRACKET = "LBRACKET"
-TOK_RBRACKET = "RBRACKET"
-TOK_DOLLAR = "DOLLAR"
-TOK_LPAREN = "LPAREN"
-TOK_RPAREN = "RPAREN"
-TOK_HASH = "HASH"
-TOK_ALPHA_CHARS = "ALPHA_CHARS"
-TOK_NUM_CHARS = "NUM_CHARS"
-TOK_NAMESPACE_SEP = "NAMESPACE_SEP"
-TOK_CHAR = "CHAR"
-TOK_CONTENTS = "CONTENTS"
-TOK_EOF = None
-
-STATE_BRACEDWORD = "bracedword"
-
-
-class TclSyntaxError(Exception):
-    def __init__(self, message, start: Tuple[int, int], end: Tuple[int, int]):
-        super().__init__(message)
-        self.start = start
-        self.end = end
-
-
-class _LexTable:
-    tokens = (
-        TOK_BACKSLASH_NEWLINE,
-        TOK_BACKSLASH_SUB,
-        TOK_NEWLINE,
-        TOK_SEMI,
-        TOK_WS,
-        TOK_QUOTE,
-        TOK_ARG_EXPANSION,
-        TOK_LBRACE,
-        TOK_RBRACE,
-        TOK_STAR,
-        TOK_LBRACKET,
-        TOK_RBRACKET,
-        TOK_DOLLAR,
-        TOK_LPAREN,
-        TOK_RPAREN,
-        TOK_HASH,
-        TOK_ALPHA_CHARS,
-        TOK_NUM_CHARS,
-        TOK_NAMESPACE_SEP,
-        TOK_CHAR,
-        TOK_CONTENTS,
-    )
-
-    # This defines a conditional lexing state for parsing braced words. This is a
-    # performance optimization; since there are few special characters in this context,
-    # we can use a smaller set of tokens to parse them faster. This has a large impact
-    # since most Tcl programs have a large number of braced words. Any token with
-    # `bracedword` in its name is included in this state. Tokens that are included in
-    # this state and the default state also include `INITIAL` in their name.
-    states = ((STATE_BRACEDWORD, "exclusive"),)
-
-    def _tok(self, t):
-        pos = (t.lexer.lineno, t.lexer.colno)
-        t.lexer.lineno += t.value.count("\n")
-        index = t.value.rfind("\n")
-        if index == -1:
-            t.lexer.colno += len(t.value)
-        else:
-            remaining = t.value[index + 1 :]
-            t.lexer.colno = len(remaining) + 1
-
-        t.value = (t.value, pos)
-        return t
-
-    # Priority important
-    def t_bracedword_INITIAL_BACKSLASH_NEWLINE(self, t):
-        r"\\\r?\n"
-        return self._tok(t)
-
-    # Priority important
-    def t_bracedword_INITIAL_BACKSLASH_SUB(self, t):
-        r"\\."
-        return self._tok(t)
-
-    def t_NEWLINE(self, t):
-        r"\n"
-        return self._tok(t)
-
-    def t_SEMI(self, t):
-        r";"
-        return self._tok(t)
-
-    # TODO: should use \s?
-    def t_WS(self, t):
-        r"[\t\v\f\r ]+"
-        return self._tok(t)
-
-    def t_QUOTE(self, t):
-        r'"'
-        return self._tok(t)
-
-    # Must be higher priority than LBRACE
-    def t_ARG_EXPANSION(self, t):
-        r"\{\*\}"
-        return self._tok(t)
-
-    def t_bracedword_INITIAL_LBRACE(self, t):
-        r"\{"
-        return self._tok(t)
-
-    def t_bracedword_INITIAL_RBRACE(self, t):
-        r"\}"
-        return self._tok(t)
-
-    def t_STAR(self, t):
-        r"\*"
-        return self._tok(t)
-
-    def t_LBRACKET(self, t):
-        r"\["
-        return self._tok(t)
-
-    def t_RBRACKET(self, t):
-        r"\]"
-        return self._tok(t)
-
-    def t_DOLLAR(self, t):
-        r"\$"
-        return self._tok(t)
-
-    def t_LPAREN(self, t):
-        r"\("
-        return self._tok(t)
-
-    def t_RPAREN(self, t):
-        r"\)"
-        return self._tok(t)
-
-    def t_HASH(self, t):
-        r"\#"
-        return self._tok(t)
-
-    # Valid non-numeric chars in variable names
-    def t_ALPHA_CHARS(self, t):
-        r"[A-Za-z_]+"
-        return self._tok(t)
-
-    # Valid numeric chars in variable names
-    # This is split up from the above to facilitate expression parsing, since
-    # e.g. 1eq1 can't be a single token.
-    def t_NUM_CHARS(self, t):
-        r"[0-9]+"
-        return self._tok(t)
-
-    def t_NAMESPACE_SEP(self, t):
-        r"::+"
-        return self._tok(t)
-
-    def t_bracedword_CONTENTS(self, t):
-        r"[^{}\\]+"
-        return self._tok(t)
-
-    # Catch-all. TODO: inefficient, should probably munch multiple chars
-    def t_CHAR(self, t):
-        r"."
-        return self._tok(t)
-
-    # Error handling rule
-    # TODO: do we need this? since we have a catch-all...
-    # there is a warning
-    def t_bracedword_INITIAL_error(self, t):
-        print("Illegal character '%s'" % t.value[0])
-        t.lexer.skip(1)
-
-    def __init__(self):
-        self.lexer = lex.lex(object=self)
-        self.lexer.lineno = 1
-        self.lexer.colno = 1
-
-    def new_lexer(self, pos=None):
-        lexer = self.lexer.clone()
-        lexer.lineno = 1
-        lexer.colno = 1
-
-        if pos is not None:
-            line, col = pos
-            lexer.lineno = line
-            lexer.colno = col
-
-        return lexer
-
-
-# Calling `lex.lex()` performs an expensive reflection process to generate the lexer.
-# This singleton class holds a preinitialized lexer that can then be cloned to create
-# individual instances.
-LexTable = _LexTable()
-
-
-class Lexer:
-    def __init__(self, pos=None):
-        self.lexer = LexTable.new_lexer(pos)
-        self.current = None
-
-    def input(self, text):
-        self.lexer.input(text)
-        self.current = self.lexer.token()
-
-    def type(self):
-        if self.current is None:
-            return TOK_EOF
-        return self.current.type
-
-    def value(self):
-        if self.current is None:
-            return None
-        return self.current.value[0]
-
-    def pos(self):
-        if self.current is None:
-            return (self.lexer.lineno, self.lexer.colno)
-        return self.current.value[1]
-
-    def next(self):
-        self.current = self.lexer.token()
-
-    def expect(self, *tokens, message, pos):
-        if self.type() not in tokens:
-            self.next()  # munch another token to update position
-            raise TclSyntaxError(message, pos, self.pos())
-
-        self.next()
-
-    def assert_(self, *tokens):
-        assert self.current.type in tokens
-        self.next()
+from tclint.lexer import TclSyntaxError, Lexer, TOK_EOF
 
 
 class CustomParser(Parser):
-    def parse(self, script, pos=None):
+    def __init__(self, debug=False, command_plugins=None):
+        super().__init__(debug, command_plugins)
+        # Used to normalize newlines consistently with open()'s universal newlines mode.
+        self._decoder = io.IncrementalNewlineDecoder(None, True)
+
+    def parse(self, script: str, pos: Optional[Tuple[int, int]] = None):
+        script = self._decoder.decode(script, True)
         lexer = Lexer(pos=pos)
         lexer.input(script)
         tree = self._parse_script(lexer, in_command_sub=False)
@@ -265,83 +30,44 @@ class CustomParser(Parser):
 
         return tree
 
-    def parse_list(self, node):
-        """Parse contents of node as Tcl list. This is a distinct entry point
-        that doesn't get used when generating the main syntax tree, but is used
-        in command-specific argument parsing.
-        """
-        if isinstance(node, List):
-            return node
-
-        if node.contents is None:
-            raise CommandArgError(
-                "expected braced word or word without substitutions in argument"
-                " interpreted as list"
-            )
-
-        ts = Lexer(pos=node.contents_pos)
-        ts.input(node.contents)
-
-        DELIMITERS = {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}
-
-        list_node = List(pos=node.pos, end_pos=node.end_pos)
-        while ts.type() is not TOK_EOF:
-            while ts.type() in DELIMITERS:
-                ts.next()
-
-            if ts.type() is TOK_EOF:
-                break
-
-            if ts.type() == TOK_LBRACE:
-                # we can reuse parse_braced_word, since it doesn't use
-                # substitutions in any case
-                list_node.add(self.parse_braced_word(ts))
-            elif ts.type() == TOK_QUOTE:
-                quote_word_pos = ts.pos()
-
-                ts.assert_(TOK_QUOTE)
-
-                bare_word_pos = ts.pos()
-                contents = ""
-                while ts.type() not in {TOK_QUOTE, TOK_EOF}:
-                    contents += ts.value()
-                    ts.next()
-                word = BareWord(contents, pos=bare_word_pos, end_pos=ts.pos())
-
-                ts.expect(
-                    TOK_QUOTE,
-                    message="reached EOF without finding match for quote",
-                    pos=quote_word_pos,
-                )
-
-                list_node.add(QuotedWord(word, pos=quote_word_pos, end_pos=ts.pos()))
-            else:
+    def _parse_operator(self, ts):
         pos = ts.pos()
-                contents = ""
-                while ts.type() not in {*DELIMITERS, TOK_EOF}:
-                    contents += ts.value()
+
+        # hacky logic to handle parsing legal operators
+
+        if ts.value() in {"*", "&", "|"}:
+            # one or two of these characters are legal operators
+            operator = ts.value()
             ts.next()
-                list_node.add(BareWord(contents, pos=pos, end_pos=ts.pos()))
-
-        return list_node
-
-    def parse_expression(self, node):
-        if node.contents is None:
-            raise CommandArgError(
-                "expected braced word or word without substitutions in argument"
-                " interpreted as expr"
+            if ts.value() == operator:
+                operator += ts.value()
+                ts.next()
+        elif ts.value() in {"<", ">"}:
+            operator = ts.value()
+            ts.next()
+            if ts.value() in {operator, "="}:
+                operator += ts.value()
+                ts.next()
+        elif ts.value() in {"=", "!"}:
+            operator = ts.value()
+            ts.next()
+            if ts.value() != "=":
+                raise TclSyntaxError(
+                    f"invalid operator in expression: {operator}", pos, ts.pos()
                 )
-
-        ts = Lexer(pos=node.contents_pos)
-        ts.input(node.contents)
-
-        contents = self._parse_expression(ts)
-        ts.expect(
-            TOK_EOF,
-            message=f"expected end of expression, got {ts.value()}",
-            pos=ts.pos(),
+            operator += ts.value()
+            ts.next()
+        elif ts.value() in {"*", "/", "%", "+", "-", "^", "eq", "ne", "in", "ni"}:
+            operator = ts.value()
+            ts.next()
+        else:
+            message = "invalid operator in expression: "
+            if ts.value() == "\\ ":
+                message += (
+                    "\\ (check for trailing whitespace if it's the end of the line)"
                 )
-        if isinstance(node, BracedWord):
-            return BracedExpression(contents, pos=node.pos, end_pos=node.end_pos)
+            else:
+                message += ts.value()
+            raise TclSyntaxError(message, pos, ts.pos())
 
-        return Expression(contents, pos=node.pos, end_pos=node.end_pos)
+        return BareWord(operator, pos=pos, end_pos=ts.pos())

server/src/tools/semantic_tokens.py

@@ -5,6 +5,7 @@ from tclint.commands import get_commands
 import attrs
 from common.load_data import standard_items
 from tools.completion_items import completion
+import lsprotocol.types as lsp
 
 
 class TokenModifier(enum.IntFlag):
@@ -40,10 +41,37 @@ TOKEN_TYPES = [
 
 
 class _Highlighter(Visitor):
-    def __init__(self, plugins, log_to_output):
+    def __init__(self, plugins, custom_functions: dict[str : list[lsp.CompletionItem]]):
         self._commands = get_commands(plugins)
         self._tokens = []
-        self.log_to_output = log_to_output
+        self.custom_functions = custom_functions
+
+    def _get_token_info(self, node):
+        """Hilfsmethode um Token-Informationen aus verschiedenen Node-Typen zu extrahieren."""
+        if not hasattr(node, "pos"):
+            return None
+
+        # Einfacher Fall: Node hat direkten value
+        if hasattr(node, "value") and node.value is not None:
+            line, col = node.pos
+            return (line - 1, col - 1), len(node.value)
+
+        # CompoundBareWord: versuche erstes Segment
+        if hasattr(node, "children") and node.children:
+            first_segment = node.children[0]
+            if hasattr(first_segment, "value") and first_segment.value is not None and hasattr(first_segment, "pos"):
+                line, col = first_segment.pos
+                return (line - 1, col - 1), len(first_segment.value)
+
+        # Fallback: Gesamtlänge aus Positionen berechnen
+        if hasattr(node, "end_pos"):
+            start_line, start_col = node.pos
+            end_line, end_col = node.end_pos
+            if start_line == end_line:
+                length = end_col - start_col
+                return (start_line - 1, start_col - 1), length
+
+        return None
 
     def visit_quoted_word(self, word: QuotedWord):
         if not word.contents:
@@ -53,13 +81,15 @@ class _Highlighter(Visitor):
         pass
 
     def visit_bare_word(self, word: BareWord):
-        if any(item.label == word.value for item in standard_items.nx_procs) or any(
-            item.label == word.value for item in completion.custom_functions
-        ):
+        name = word.value
+
+        in_standard = any(item.label == name for item in standard_items.nx_procs)
+
+        in_custom = any(item.label == name for items in self.custom_functions.values() for item in items)
+
+        if in_standard or in_custom:
             line, col = word.pos
-            self._tokens.append(
-                (((line - 1, col - 1), len(word.value), "function", []))
-            )
+            self._tokens.append((((line - 1, col - 1), len(name), "function", [])))
 
     def visit_command(self, command: Command):
         routine = command.routine
@@ -76,17 +106,16 @@ class _Highlighter(Visitor):
                     )
                 )
             )
-            pass
-
         if routine.contents == "set" and command.args:
             first_arg = command.args[0]
-            if hasattr(first_arg, "pos") and hasattr(first_arg, "value"):
-                line, col = first_arg.pos
+            token_info = self._get_token_info(first_arg)
+            if token_info:
+                (line, col), length = token_info
                 self._tokens.append(
                     (
                         (
-                            (line - 1, col - 1),
-                            len(first_arg.value),
+                            (line, col),
+                            length,
                             "variable",
                             [TokenModifier.declaration],
                         )
@@ -128,9 +157,7 @@ class _Highlighter(Visitor):
                         # Parameter mit Default-Wert ist meist eine List (z. B. {arg default})
                         elif hasattr(child, "children") and len(child.children) >= 1:
                             name_node = child.children[0]
-                            if hasattr(name_node, "value") and hasattr(
-                                name_node, "pos"
-                            ):
+                            if hasattr(name_node, "value") and hasattr(name_node, "pos"):
                                 line, col = name_node.pos
                                 self._tokens.append(
                                     (
@@ -140,17 +167,11 @@ class _Highlighter(Visitor):
                                         [TokenModifier.declaration],
                                     )
                                 )
-
         if routine.contents == "namespace" and command.args:
             first_arg = command.args[1]
-            if hasattr(first_arg, "pos") and hasattr(first_arg, "value"):
+            if hasattr(first_arg, "pos") and first_arg.value is not None:
                 line, col = first_arg.pos
-                self._tokens.append(
-                    (((line - 1, col - 1), len(first_arg.value), "class", []))
-                )
-
-    def visit_var_sub(self, var_sub):
-        pass
+                self._tokens.append((((line - 1, col - 1), len(first_arg.value), "class", [])))
 
     def tokens(self) -> list[Token]:
         """Encode tokens as described in
@@ -159,9 +180,7 @@ class _Highlighter(Visitor):
         tokens = []
         last_line = 0
         last_col = 0
-        for (line, col), length, tok_type, tok_modifier in sorted(
-            self._tokens, key=lambda x: x[0]
-        ):
+        for (line, col), length, tok_type, tok_modifier in sorted(self._tokens, key=lambda x: x[0]):
             line_delta = line - last_line
             col_delta = col
             if line == last_line:

server/src/tools/symbols.py

@@ -0,0 +1,46 @@
+import logging
+from collections import defaultdict
+from typing import List, DefaultDict, Union
+
+from tclint.syntax_tree import Visitor, Command, CommandSub, Node, Script
+
+
+class SymbolTable:
+    """Holds a symbol table (links symbols to nodes)."""
+
+    def __init__(self):
+        self.proc_def: DefaultDict[str, list[Node]] = defaultdict(list)
+
+    def add_proc_definition(self, command: Command) -> None:
+        """Add definition of procedure"""
+        # command holds the "proc" keyword, so the proc name is 1st argument
+        proc_name_node = command.args[0]
+        proc_name = proc_name_node.contents
+        if not proc_name:
+            return
+        logging.debug(
+            f"Definition of proc '{proc_name}' at {proc_name_node._pos_str()}"
+        )
+        self.proc_def[proc_name].append(proc_name_node)
+
+    def lookup_proc_definitions(self, symbol_text: str) -> List[Node]:
+        """Lookup definitions of the procedure pointed at by node"""
+        if symbol_text is None or symbol_text not in self.proc_def:
+            return []
+        return self.proc_def[symbol_text]
+
+
+class SymbolTableBuilder(Visitor):
+    """Builds a symbol table."""
+
+    def __init__(self):
+        self.table = SymbolTable()
+
+    def build(self, tree: Union[CommandSub, Script]) -> SymbolTable:
+        """Run the builder visitor through the syntax tree, building a table."""
+        tree.accept(self, recurse=True)
+        return self.table
+
+    def visit_command(self, command: Command) -> None:
+        if command.routine.contents == "proc":
+            self.table.add_proc_definition(command)

test/test.tcl

@@ -1,4 +1,10 @@
-set main 1
+set ::custom_flag(from_move,$::mom_path_name) 1
+# set ::custom_flag(from_move,$::mom_path_name) 1
+
+set te875st 11111
+
+#set ::custom_flag(from_move,$::mom_path_name) 1
+
 if {$main == 1 && 1 == 1} {
     puts "main"
 }
@@ -103,3 +109,8 @@ global mom_operation_info
         }
     }
 }
+
+
+LIB_GE_command_buffer_edit_replace MOM_end_of_program_LIB END_OF_PROGRAM @END_OF_PROG {
+    MOM_do_template "end_of_program_rewind"
+} EndOfProgramRewind