refactor(lsp): cache analysis results and debounce diagnostics

This change adds cached and incremental analysis for the LSP
server to improve responsiveness. The client now debounces
diagnostic updates to avoid excessive recomputation. The
server introduces per-document line caches and various
caches for completions, inlay hints, and metadata to
support faster, incremental updates.

- Debounce diagnostics on text changes to reduce noise.
- Add caches for completions, inlay hints, and metadata.
- Introduce incremental analysis with per-document line caches.

server/src/lsp_server.py

@@ -5,15 +5,15 @@
 from __future__ import annotations
 
 import json
+import operator
 import os
 import pathlib
 import re
 import sys
 import threading
-
-from typing import Any, Optional
-import operator
+from collections import ChainMap
 from functools import reduce
+from typing import Any, Optional
 
 
 # **********************************************************
@@ -40,14 +40,14 @@ update_sys_path(
 # pylint: disable=wrong-import-position,import-error
 import lsp_jsonrpc as jsonrpc
 import lsprotocol.types as lsp
-from pygls import uris, workspace
 from common.load_data import standard_items
+from lsp_tclserver import TclLanguageServer
+from pygls import uris, workspace
+from pygls.workspace.text_document import TextDocument
 from tools.folding_ranges import build_folding_ranges
-from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier
 from tools.inlay_hint import (
     InlayHintGenerator,
     build_builtin_inlay_signatures,
-    build_custom_inlay_signatures,
 )
 from tools.navigation import (
     SymbolIdentity,
@@ -56,10 +56,13 @@ from tools.navigation import (
     symbol_at_position,
     workspace_symbols,
 )
+from tools.semantic_tokens import (
+    TOKEN_TYPE_INDEX,
+    TOKEN_TYPES,
+    TokenModifier,
+    _Highlighter,
+)
 from tools.signature_help import build_signature_help
-from lsp_tclserver import TclLanguageServer
-from pygls.workspace.text_document import TextDocument
-
 
 WORKSPACE_SETTINGS = {}
 GLOBAL_SETTINGS = {}
@@ -75,6 +78,23 @@ BUILTIN_PROC_NAMES = {
     for item in standard_items.tcl_keyword_list + standard_items.nx_procs
 }
 BUILTIN_VARIABLE_NAMES = {item.label for item in standard_items.nx_variables}
+STATIC_COMPLETION_ITEMS = tuple(
+    standard_items.tcl_keyword_list
+    + standard_items.nx_procs
+    + standard_items.nx_variables
+)
+STATIC_COMPLETION_KEYS = frozenset(
+    (item.label, getattr(item, "kind", None)) for item in STATIC_COMPLETION_ITEMS
+)
+BUILTIN_INLAY_SIGNATURES = build_builtin_inlay_signatures(
+    standard_items.json_data.get("MOM_procs", [])
+)
+BUILTIN_HOVER_ITEMS = {}
+for _hover_item in (
+    standard_items.json_data.get("MOM_procs", [])
+    + standard_items.json_data.get("mom_variables", [])
+):
+    BUILTIN_HOVER_ITEMS.setdefault(_hover_item.get("label"), _hover_item)
 
 # **********************************************************
 # Tool specific code goes below this.
@@ -95,15 +115,14 @@ def did_open(params: lsp.DidOpenTextDocumentParams) -> None:
     """LSP handler for textDocument/didOpen request."""
     document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
     LSP_SERVER.clear_cache_for_uri(document.uri)
-    LSP_SERVER.compute_diagnostics(document)
-    # Also update custom completion and proc docs for this file
-    LSP_SERVER.update_poco_completion_for_file(document)
+    LSP_SERVER.analyze_document_now(document)
 
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_SAVE)
 def did_save(params: lsp.DidSaveTextDocumentParams) -> None:
     """LSP handler for textDocument/didSave request."""
-    _ = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+    document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+    LSP_SERVER.analyze_document_now(document)
 
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_CLOSE)
@@ -119,8 +138,7 @@ def did_change(params: lsp.DidChangeTextDocumentParams) -> None:
     """LSP handler for textDocument/didChange request"""
     document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
     LSP_SERVER.clear_cache_for_uri(document.uri)
-    LSP_SERVER.compute_diagnostics(document)
-    LSP_SERVER.update_poco_completion_for_file(document)
+    LSP_SERVER.schedule_document_analysis(document)
 
 
 FILE_OPERATION_OPTIONS = lsp.FileOperationRegistrationOptions(
@@ -203,10 +221,12 @@ def did_change_watched_files(params: lsp.DidChangeWatchedFilesParams) -> None:
 def document_diagnostic(params: lsp.DocumentDiagnosticParams):
     """Return diagnostics for the requested document"""
     uri = params.text_document.uri
+    doc = LSP_SERVER.workspace.get_text_document(uri)
     diagnostic_state = LSP_SERVER.diagnostic_snapshot(uri)
-    was_cached = diagnostic_state is not None
-    if diagnostic_state is None:
-        doc = LSP_SERVER.workspace.get_text_document(uri)
+    was_cached = (
+        diagnostic_state is not None and diagnostic_state[0] == doc.version
+    )
+    if not was_cached:
         LSP_SERVER.compute_diagnostics(doc)
         diagnostic_state = LSP_SERVER.diagnostic_snapshot(uri)
 
@@ -224,24 +244,15 @@ def document_diagnostic(params: lsp.DocumentDiagnosticParams):
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_COMPLETION)
 def on_completion(params: lsp.CompletionParams) -> lsp.CompletionList:
-    from tools.variable_index import build_variable_index
     from tools.completion_items import BUILTIN_VAR_LABELS
 
     doc = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
 
-    # Base items
-    poco_completion, _, _ = LSP_SERVER.index_snapshot()
-    poco = [item for items in poco_completion.values() for item in items]
-    base_items = (
-        standard_items.tcl_keyword_list
-        + standard_items.nx_procs
-        + standard_items.nx_variables
-        + poco
-    )
-
-    # Build variable index from current document
+    workspace_items = LSP_SERVER.completion_items_snapshot()
     tree = LSP_SERVER.get_tree(doc)
-    globals_set, procs_locals, proc_ranges = build_variable_index(doc.source, tree)
+    globals_set, procs_locals, proc_ranges = LSP_SERVER.variable_index_for_document(
+        doc, tree
+    )
 
     # Always include globals (excluding built-ins)
     dynamic_items = []
@@ -268,9 +279,11 @@ def on_completion(params: lsp.CompletionParams) -> lsp.CompletionList:
 
     # Merge with de-duplication. Each file keeps its complete index, so a proc
     # declared in multiple files must only appear once in the completion list.
-    merged: list[lsp.CompletionItem] = []
-    seen_items: set[tuple[str, lsp.CompletionItemKind | None]] = set()
-    for it in base_items + dynamic_items:
+    merged: list[lsp.CompletionItem] = list(STATIC_COMPLETION_ITEMS)
+    seen_items: set[tuple[str, lsp.CompletionItemKind | None]] = set(
+        STATIC_COMPLETION_KEYS
+    )
+    for it in (*workspace_items, *dynamic_items):
         key = (it.label, getattr(it, "kind", None))
         if key in seen_items:
             continue
@@ -291,22 +304,9 @@ def signature_help(params: lsp.SignatureHelpParams) -> lsp.SignatureHelp | None:
     document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
     tree = LSP_SERVER.get_tree(document)
 
-    filepath = str(pathlib.Path(uris.to_fs_path(document.uri)))
-    custom_signatures: dict[str, list[str]] = {}
-    custom_docs: dict[str, str] = {}
-    _, proc_signatures, proc_docs = LSP_SERVER.index_snapshot()
-
-    # Prefer declarations from the current document if duplicate proc names
-    # exist in the workspace.
-    for indexed_path, signatures in proc_signatures.items():
-        if indexed_path != filepath:
-            custom_signatures.update(signatures)
-    custom_signatures.update(proc_signatures.get(filepath, {}))
-
-    for indexed_path, docs in proc_docs.items():
-        if indexed_path != filepath:
-            custom_docs.update(docs)
-    custom_docs.update(proc_docs.get(filepath, {}))
+    custom_signatures, custom_docs = LSP_SERVER.proc_metadata_snapshot(
+        document.path
+    )
 
     return build_signature_help(
         document.source,
@@ -352,30 +352,22 @@ def inlay_hints(params: lsp.InlayHintParams):
 
     # Built-in NX procedures are the fallback. Workspace procedures replace them,
     # and a declaration in the current file wins over duplicate workspace names.
-    signatures = build_builtin_inlay_signatures(
-        standard_items.json_data.get("MOM_procs", [])
-    )
-    _, proc_signatures, proc_docs = LSP_SERVER.index_snapshot()
-    signatures.update(
-        build_custom_inlay_signatures(
-            proc_signatures,
-            proc_docs,
-            LSP_SERVER.navigation_snapshot(),
-            document.path,
-        )
+    custom_signatures = LSP_SERVER.custom_inlay_signatures_snapshot(
+        document.path
     )
+    signatures = ChainMap(custom_signatures, BUILTIN_INLAY_SIGNATURES)
 
     generator = InlayHintGenerator(
         document.source,
         signatures,
+        source_lines=LSP_SERVER.get_lines(document),
         requested_range=params.range,
         parameter_names=parameter_names,
         suppress_when_argument_matches_name=inlay_settings.get(
             "suppressWhenArgumentMatchesName", True
         ),
     )
-    tree.accept(generator, recurse=True)
-    return generator.hints
+    return generator.generate(tree)
 
 
 @LSP_SERVER.feature(
@@ -390,8 +382,7 @@ def semantic_tokens(params: lsp.SemanticTokensParams):
 
     data = []
     plugins = []
-    poco_completion, _, _ = LSP_SERVER.index_snapshot()
-    hl = _Highlighter(plugins, poco_completion)
+    hl = _Highlighter(plugins, LSP_SERVER.custom_function_names_snapshot())
 
     # Reuse cached AST
     tree = LSP_SERVER.get_tree(document)
@@ -404,7 +395,7 @@ def semantic_tokens(params: lsp.SemanticTokensParams):
                 token.line,
                 token.offset,
                 token.length,
-                TOKEN_TYPES.index(token.tok_type),
+                TOKEN_TYPE_INDEX[token.tok_type],
                 reduce(operator.or_, token.tok_modifiers, 0),
             ]
         )
@@ -426,14 +417,14 @@ def hover(params: lsp.HoverParams) -> lsp.Hover:
     col = params.position.character
 
     try:
-        line = document.lines[pos.line]
+        line = LSP_SERVER.get_lines(document)[pos.line]
     except IndexError:
         return None
 
     # Do not show hover for proc name in its declaration
-    from tools.proc_docs import is_proc_declaration_position
+    from tools.proc_docs import is_proc_declaration_line
 
-    if is_proc_declaration_position(document.source, pos.line, pos.character):
+    if is_proc_declaration_line(line, pos.character):
         return None
 
     # Identify the token under the cursor
@@ -445,11 +436,7 @@ def hover(params: lsp.HoverParams) -> lsp.Hover:
         return None
 
     # 1) If token is a known MOM proc/variable, return built-in hover
-    command = token
-    data = standard_items.json_data
-    all_items = data.get("MOM_procs", []) + data.get("mom_variables", [])
-
-    match = next((item for item in all_items if item["label"] == command), None)
+    match = BUILTIN_HOVER_ITEMS.get(token)
     if match and match.get("kind") == "function":
         label = match.get("label", "")
         parameters = match.get("parameters", [])
@@ -482,14 +469,10 @@ def hover(params: lsp.HoverParams) -> lsp.Hover:
 
     # 2) Otherwise, check if the token is a custom proc and show its preceding doc block
     # Build a merged map of proc -> docs gathered during initialization and updates
-    proc_docs: dict[str, str] = {}
-    _, _, indexed_proc_docs = LSP_SERVER.index_snapshot()
-    for file_docs in indexed_proc_docs.values():
-        proc_docs.update(file_docs)
-
-    if token in proc_docs:
+    proc_doc = LSP_SERVER.proc_documentation(token, document.path)
+    if proc_doc is not None:
         return lsp.Hover(
-            lsp.MarkupContent(kind=lsp.MarkupKind.Markdown, value=proc_docs[token])
+            lsp.MarkupContent(kind=lsp.MarkupKind.Markdown, value=proc_doc)
         )
 
     return None
@@ -517,7 +500,7 @@ def _navigation_context(uri: str, position: lsp.Position):
     indexes = LSP_SERVER.navigation_snapshot()
     filepath = str(pathlib.Path(uris.to_fs_path(uri)))
     index = indexes.get(filepath)
-    if index is None:
+    if index is None or LSP_SERVER.index_update_pending(filepath):
         document = LSP_SERVER.workspace.get_text_document(uri)
         LSP_SERVER.update_poco_completion_for_file(document)
         indexes = LSP_SERVER.navigation_snapshot()