feat(lsp): add incremental indexing and file ops support

Adds a thread-safe incremental index and snapshot API for LSP.
Introduces cache invalidation and file operation hooks for delete
and rename. This keeps indices in sync with disk changes.
Supports reindexing TCL files from disk when needed.

- Adds workspace file change handlers to sync indices on delete/rename.
- Introduces locking and snapshot helpers to safely access shared state.
- Refactors to invalidate caches on edits and reindex TCL files.

server/src/lsp_server.py

@@ -44,11 +44,11 @@ from pygls import uris, workspace
 from common.load_data import standard_items
 from tools.folding_ranges import build_folding_ranges
 from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier
-from tools.completion_items import completion, remove_existing_items, remove_shared_keys
 from tools.inlay_hint import InlayHintGenerator
 from tools.signature_help import build_signature_help
 from tools.file_sourcing import get_all_psc_files, read_psc_file
 from lsp_tclserver import TclLanguageServer
+from pygls.workspace.text_document import TextDocument
 
 
 WORKSPACE_SETTINGS = {}
@@ -78,6 +78,7 @@ LSP_SERVER = TclLanguageServer(
 def did_open(params: lsp.DidOpenTextDocumentParams) -> None:
     """LSP handler for textDocument/didOpen request."""
     document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+    LSP_SERVER.clear_cache_for_uri(document.uri)
     LSP_SERVER.compute_diagnostics(document)
     # Also update custom completion and proc docs for this file
     LSP_SERVER.update_poco_completion_for_file(document)
@@ -90,18 +91,81 @@ def did_save(params: lsp.DidSaveTextDocumentParams) -> None:
 
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_CLOSE)
-def did_close(_: lsp.DidCloseTextDocumentParams) -> None:
+def did_close(params: lsp.DidCloseTextDocumentParams) -> None:
     """LSP handler for textDocument/didClose request."""
+    uri = params.text_document.uri
+    LSP_SERVER.remove_file_state(uri)
+    _index_tcl_file_from_disk(uri)
 
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_CHANGE)
 def did_change(params: lsp.DidChangeTextDocumentParams) -> None:
     """LSP handler for textDocument/didChange request"""
     document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+    LSP_SERVER.clear_cache_for_uri(document.uri)
     LSP_SERVER.compute_diagnostics(document)
     LSP_SERVER.update_poco_completion_for_file(document)
 
 
+FILE_OPERATION_OPTIONS = lsp.FileOperationRegistrationOptions(
+    filters=[
+        lsp.FileOperationFilter(
+            scheme="file",
+            pattern=lsp.FileOperationPattern(glob="**/*"),
+        )
+    ]
+)
+
+
+def _index_tcl_file_from_disk(uri: str) -> None:
+    if not uri.startswith("file:"):
+        return
+
+    path = pathlib.Path(uris.to_fs_path(uri))
+    if path.suffix.lower() != ".tcl" or not path.is_file():
+        return
+
+    try:
+        document = TextDocument(uri=uri, language_id="tcl")
+        LSP_SERVER.update_poco_completion_for_file(
+            document,
+            cache_tree=False,
+            require_file_exists=True,
+        )
+    except (OSError, UnicodeError) as error:
+        log_warning(f"Could not re-index {path}: {error}")
+
+
+@LSP_SERVER.feature(lsp.WORKSPACE_DID_DELETE_FILES, FILE_OPERATION_OPTIONS)
+def did_delete_files(params: lsp.DeleteFilesParams) -> None:
+    for deleted_file in params.files:
+        LSP_SERVER.remove_file_state(deleted_file.uri)
+
+
+@LSP_SERVER.feature(lsp.WORKSPACE_DID_RENAME_FILES, FILE_OPERATION_OPTIONS)
+def did_rename_files(params: lsp.RenameFilesParams) -> None:
+    for renamed_file in params.files:
+        old_path = pathlib.Path(uris.to_fs_path(renamed_file.old_uri))
+        new_path = pathlib.Path(uris.to_fs_path(renamed_file.new_uri))
+        indexed_paths = LSP_SERVER.indexed_paths_under_uri(renamed_file.old_uri)
+        new_index_paths: set[pathlib.Path] = set()
+
+        for indexed_path in indexed_paths:
+            if LSP_SERVER.paths_equal(indexed_path, old_path):
+                new_index_paths.add(new_path)
+            else:
+                relative_path = os.path.relpath(indexed_path, old_path)
+                new_index_paths.add(new_path / relative_path)
+
+        # Also handles renaming a previously unindexed file to a TCL file.
+        if new_path.suffix.lower() == ".tcl":
+            new_index_paths.add(new_path)
+
+        LSP_SERVER.remove_file_state(renamed_file.old_uri)
+        for new_index_path in new_index_paths:
+            _index_tcl_file_from_disk(new_index_path.as_uri())
+
+
 @LSP_SERVER.feature(
     lsp.TEXT_DOCUMENT_DIAGNOSTIC,
     lsp.DiagnosticOptions(
@@ -112,13 +176,18 @@ def did_change(params: lsp.DidChangeTextDocumentParams) -> None:
 )
 def document_diagnostic(params: lsp.DocumentDiagnosticParams):
     """Return diagnostics for the requested document"""
-    was_cached = True
-    if (uri := params.text_document.uri) not in LSP_SERVER.diagnostics:
-        was_cached = False
+    uri = params.text_document.uri
+    diagnostic_state = LSP_SERVER.diagnostic_snapshot(uri)
+    was_cached = diagnostic_state is not None
+    if diagnostic_state is None:
         doc = LSP_SERVER.workspace.get_text_document(uri)
         LSP_SERVER.compute_diagnostics(doc)
+        diagnostic_state = LSP_SERVER.diagnostic_snapshot(uri)
 
-    version, diagnostics = LSP_SERVER.diagnostics[uri]
+    if diagnostic_state is None:
+        return lsp.FullDocumentDiagnosticReport(items=[])
+
+    version, diagnostics = diagnostic_state
     result_id = f"{uri}@{version}"
 
     if was_cached and result_id == params.previous_result_id:
@@ -135,7 +204,8 @@ def on_completion(params: lsp.CompletionParams) -> lsp.CompletionList:
     doc = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
 
     # Base items
-    poco = [item for items in LSP_SERVER.poco_completion.values() for item in items]
+    poco_completion, _, _ = LSP_SERVER.index_snapshot()
+    poco = [item for items in poco_completion.values() for item in items]
     base_items = (
         standard_items.tcl_keyword_list
         + standard_items.nx_procs
@@ -170,14 +240,15 @@ def on_completion(params: lsp.CompletionParams) -> lsp.CompletionList:
                         )
                 break
 
-    # Merge with de-duplication for variables only
+    # Merge with de-duplication. Each file keeps its complete index, so a proc
+    # declared in multiple files must only appear once in the completion list.
     merged: list[lsp.CompletionItem] = []
-    seen_var_labels: set[str] = set()
+    seen_items: set[tuple[str, lsp.CompletionItemKind | None]] = set()
     for it in base_items + dynamic_items:
-        if getattr(it, "kind", None) == lsp.CompletionItemKind.Variable:
-            if it.label in seen_var_labels:
-                continue
-            seen_var_labels.add(it.label)
+        key = (it.label, getattr(it, "kind", None))
+        if key in seen_items:
+            continue
+        seen_items.add(key)
         merged.append(it)
 
     return lsp.CompletionList(is_incomplete=False, items=merged)
@@ -197,18 +268,19 @@ def signature_help(params: lsp.SignatureHelpParams) -> lsp.SignatureHelp | None:
     filepath = str(pathlib.Path(uris.to_fs_path(document.uri)))
     custom_signatures: dict[str, list[str]] = {}
     custom_docs: dict[str, str] = {}
+    _, proc_signatures, proc_docs = LSP_SERVER.index_snapshot()
 
     # Prefer declarations from the current document if duplicate proc names
     # exist in the workspace.
-    for indexed_path, signatures in LSP_SERVER.proc_signatures.items():
+    for indexed_path, signatures in proc_signatures.items():
         if indexed_path != filepath:
             custom_signatures.update(signatures)
-    custom_signatures.update(LSP_SERVER.proc_signatures.get(filepath, {}))
+    custom_signatures.update(proc_signatures.get(filepath, {}))
 
-    for indexed_path, docs in LSP_SERVER.proc_docs.items():
+    for indexed_path, docs in proc_docs.items():
         if indexed_path != filepath:
             custom_docs.update(docs)
-    custom_docs.update(LSP_SERVER.proc_docs.get(filepath, {}))
+    custom_docs.update(proc_docs.get(filepath, {}))
 
     return build_signature_help(
         document.source,
@@ -248,7 +320,8 @@ def inlay_hints(params: lsp.InlayHintParams):
 
     # Merge proc signatures across files and traverse once
     merged_signatures = {}
-    for sigs in LSP_SERVER.proc_signatures.values():
+    _, proc_signatures, _ = LSP_SERVER.index_snapshot()
+    for sigs in proc_signatures.values():
         merged_signatures.update(sigs)
 
     generator = InlayHintGenerator(merged_signatures)
@@ -268,7 +341,8 @@ def semantic_tokens(params: lsp.SemanticTokensParams):
 
     data = []
     plugins = []
-    hl = _Highlighter(plugins, LSP_SERVER.poco_completion)
+    poco_completion, _, _ = LSP_SERVER.index_snapshot()
+    hl = _Highlighter(plugins, poco_completion)
 
     # Reuse cached AST
     tree = LSP_SERVER.get_tree(document)
@@ -360,7 +434,8 @@ def hover(params: lsp.HoverParams) -> lsp.Hover:
     # 2) Otherwise, check if the token is a custom proc and show its preceding doc block
     # Build a merged map of proc -> docs gathered during initialization and updates
     proc_docs: dict[str, str] = {}
-    for file_docs in LSP_SERVER.proc_docs.values():
+    _, _, indexed_proc_docs = LSP_SERVER.index_snapshot()
+    for file_docs in indexed_proc_docs.values():
         proc_docs.update(file_docs)
 
     if token in proc_docs:
@@ -420,7 +495,8 @@ def goto_definition(params: lsp.DefinitionParams):
     # 2) Search in indexed files from proc_signatures
     # Build list of candidate files that declare this token as a proc
     candidate_files: list[str] = []
-    for file_path, procs in LSP_SERVER.proc_signatures.items():
+    _, proc_signatures, _ = LSP_SERVER.index_snapshot()
+    for file_path, procs in proc_signatures.items():
         if token in procs:
             candidate_files.append(file_path)
 
@@ -530,42 +606,24 @@ def initialized(_params: lsp.InitializedParams):
             for psc_file in psc_files:
                 poco_files = read_psc_file(psc_file)
                 for sourced_layer in poco_files:
-                    completion.reset()
-                    try:
-                        file_root = pathlib.Path(root).joinpath(
-                            sourced_layer.subfolder if sourced_layer.subfolder else ""
-                        )
-                        for tcl_file in sourced_layer.files:
-                            filepath = pathlib.Path(file_root).joinpath(
-                                f"{tcl_file}.tcl"
+                    file_root = pathlib.Path(root).joinpath(
+                        sourced_layer.subfolder if sourced_layer.subfolder else ""
+                    )
+                    for tcl_file in sourced_layer.files:
+                        filepath = pathlib.Path(file_root).joinpath(f"{tcl_file}.tcl")
+                        if not filepath.exists():
+                            continue
+                        try:
+                            document = TextDocument(
+                                uri=filepath.as_uri(), language_id="tcl"
                             )
-                            if not filepath.exists():
-                                continue
-                            completion.reset()
-                            document = LSP_SERVER.workspace.get_text_document(
-                                filepath.as_uri()
+                            LSP_SERVER.update_poco_completion_for_file(
+                                document,
+                                cache_tree=False,
+                                require_file_exists=True,
                             )
-                            tree = LSP_SERVER.parser.parse(document.source)
-                            tree.accept(completion, recurse=True)
-                            remove_existing_items(
-                                completion.custom_functions, LSP_SERVER.poco_completion
-                            )
-                            LSP_SERVER.poco_completion[str(filepath)] = (
-                                completion.custom_functions
-                            )
-                            remove_shared_keys(
-                                LSP_SERVER.proc_signatures, completion.proc_signatures
-                            )
-                            LSP_SERVER.proc_signatures[str(filepath)] = (
-                                completion.proc_signatures
-                            )
-                            from tools.proc_docs import build_proc_docs
-
-                            LSP_SERVER.proc_docs[str(filepath)] = build_proc_docs(
-                                tree, document.source
-                            )
-                    except Exception as e:
-                        log_to_output(f"Fehler beim Parsen von {filepath}: {e}")
+                        except Exception as error:
+                            log_to_output(f"Fehler beim Parsen von {filepath}: {error}")
             log_to_output("Background indexing completed.")
         except Exception as e:
             log_to_output(f"Background indexing failed: {e}")