feat(indexing): add persistent index cache and incremental reparse

- Pass extension storage path to the server (client/ changes) so the
  server can persist a workspace index.
- Introduce IndexCache (server/tools/index_cache.py) and load/save it on
  initialization and after background indexing. Index entries are stored
  only when the file's stat hasn't changed while being read.
- Add incremental reparse logic (server/tools/incremental_parse.py) and
  use a per-file _last_parse cache in the language server to reparse only
  the top-level Tcl commands touched by an edit, falling back to a full
  parse when necessary.
- Use a new _FileIndex dataclass and _build_file_index helper to unify
  what is stored/loaded for a file; update update_poco_completion_for_file
  to use the persistent cache for disk-read files (from_disk/source_stat).
- Keep background indexing non-blocking and persist the index at the
  end of the run. Add basic unit tests for incremental parse and index cache.

Before: edits and background work always required full parsing of files
and no persistent cross-restart index. After: some edits reuse previous
ASTs and files read from disk can use a persisted index to skip
re-indexing across restarts.

CHANGELOG.md

@@ -1,5 +1,10 @@
 ## Unreleased
 
+- Reparse only the top-level TCL commands touched by an edit instead of the whole file
+- Cache the workspace index in the extension storage so restarts skip reparsing unchanged files; the cache is discarded automatically when the server or bundled tclint changes
+- Speed up TclOO completion, signature help, inlay hints, and Go to Definition by reusing the cached syntax tree and skipping files without classes
+- Speed up references, document highlights, and call hierarchy with cached definition lookups
+- Keep background indexing from blocking requests that need a fresh syntax tree
 - Add Go to Definition for TclOO classes, constructors, and resolved methods, including PSC library definitions
 - Index PSC layer scripts (including external paths and legacy Windows encoding) and share TclOO class metadata across files for completion, signatures, inlay hints, and highlighting
 - Add document-local TclOO method completion for `new`/`create` instances, `my`, and statically inferred return chains

client/src/common/server.ts

@@ -22,7 +22,12 @@ import {
 import { getLSClientTraceLevel, getProjectRoot } from "./utilities"
 import { isVirtualWorkspace } from "./vscodeapi"
 
-export type IInitOptions = { settings: ISettings[]; globalSettings: ISettings }
+export type IInitOptions = {
+    settings: ISettings[]
+    globalSettings: ISettings
+    // Folder for the server's persistent index cache; omitted without a workspace.
+    indexCachePath?: string
+}
 
 let _disposables: Disposable[] = []
 
@@ -114,7 +119,8 @@ export async function restartServer(
     serverId: string,
     serverName: string,
     outputChannel: LogOutputChannel,
-    lsClient?: LanguageClient
+    lsClient?: LanguageClient,
+    indexCachePath?: string
 ): Promise<LanguageClient | undefined> {
     if (lsClient) {
         traceInfo(`Server: Stop requested`)
@@ -132,7 +138,8 @@ export async function restartServer(
             outputChannel,
             {
                 settings: await getExtensionSettings(serverId, true),
-                globalSettings: await getGlobalSettings(serverId, false)
+                globalSettings: await getGlobalSettings(serverId, false),
+                indexCachePath
             }
         )
         traceInfo(`Server: Start requested.`)

client/src/extension.ts

@@ -73,7 +73,13 @@ export async function activate(context: vscode.ExtensionContext) {
                 traceVerbose(
                     `Using interpreter from ${serverInfo.module}.interpreter: ${interpreter.join(" ")}`
                 )
-                client = await restartServer(serverId, serverName, outputChannel, client)
+                client = await restartServer(
+                    serverId,
+                    serverName,
+                    outputChannel,
+                    client,
+                    context.storageUri?.fsPath
+                )
             }
             return
         }
@@ -83,7 +89,13 @@ export async function activate(context: vscode.ExtensionContext) {
             traceVerbose(
                 `Using interpreter from Python extension: ${interpreterDetails.path.join(" ")}`
             )
-            client = await restartServer(serverId, serverName, outputChannel, client)
+            client = await restartServer(
+                serverId,
+                serverName,
+                outputChannel,
+                client,
+                context.storageUri?.fsPath
+            )
             return
         }
 

server/src/lsp_server.py

@@ -11,6 +11,7 @@ import pathlib
 import re
 import sys
 import threading
+import time
 from collections import ChainMap
 from functools import reduce
 from typing import Any, Optional
@@ -52,6 +53,7 @@ from tools.completion_items import (
     ranked_completion_items,
 )
 from tools.folding_ranges import build_folding_ranges
+from tools.index_cache import IndexCache
 from tools.inlay_hint import (
     InlayHintGenerator,
     build_builtin_inlay_signatures,
@@ -89,6 +91,8 @@ from tools.tcl_command_completion import (
 
 WORKSPACE_SETTINGS = {}
 GLOBAL_SETTINGS = {}
+# Extension storage folder of the workspace; without it nothing is persisted.
+INDEX_CACHE_PATH: dict[str, str | None] = {}
 
 
 MAX_WORKERS = 5
@@ -175,6 +179,7 @@ def _index_tcl_file_from_disk(uri: str) -> None:
             document,
             cache_tree=False,
             require_file_exists=True,
+            from_disk=True,
         )
     except (OSError, UnicodeError) as error:
         log_warning(f"Could not re-index {path}: {error}")
@@ -861,6 +866,7 @@ def initialize(params: lsp.InitializeParams) -> lsp.InitializeResult:
     log_to_output(f"sys.path used to run Server:\r\n   {paths}")
 
     GLOBAL_SETTINGS.update(**params.initialization_options.get("globalSettings", {}))
+    INDEX_CACHE_PATH["path"] = params.initialization_options.get("indexCachePath")
 
     settings = params.initialization_options["settings"]
     _update_workspace_settings(settings)
@@ -899,6 +905,8 @@ def initialized(_params: lsp.InitializedParams):
                 log_to_output("Background indexing skipped: no workspace folder is open.")
                 return
             log_to_output("Background indexing started...")
+            started = time.perf_counter()
+            LSP_SERVER.index_cache = IndexCache.load(INDEX_CACHE_PATH.get("path"))
             root_path = pathlib.Path(root)
             skipped_directories = {
                 ".git",
@@ -916,11 +924,13 @@ def initialized(_params: lsp.InitializedParams):
                         document,
                         cache_tree=False,
                         require_file_exists=True,
+                        from_disk=True,
                     )
                 except Exception as error:
                     log_to_output(f"Fehler beim Parsen von {filepath}: {error}")
             _refresh_psc_index()
-            log_to_output("Background indexing completed.")
+            LSP_SERVER.index_cache.save()
+            log_to_output(f"Background indexing completed in {time.perf_counter() - started:.1f}s.")
         except Exception as e:
             log_to_output(f"Background indexing failed: {e}")
 

server/src/lsp_tclserver.py

@@ -3,6 +3,7 @@ import os
 import pathlib
 import threading
 import xml.etree.ElementTree as ET
+from dataclasses import dataclass
 from typing import List, Optional, Tuple
 
 import lsprotocol.types as lsp
@@ -13,12 +14,13 @@ from pygls.workspace.text_document import TextDocument
 from tclint.format import FormatterOpts
 from tclint.lexer import TclSyntaxError
 from tclint.violations import Violation
-from tools import checks, parser
+from tools import checks, incremental_parse, parser
 from tools.completion_items import CompletionCollector
 from tools.tcloo_symbols import class_completion_items
 from tools.tcloo_completion import indexed_classes
 from tools.file_sourcing import get_all_psc_files, psc_script_files
 from tools.formatter import NxFormatter as Formatter
+from tools.index_cache import FileStat, IndexCache, file_stat
 from tools.inlay_hint import InlayHintSignature, build_custom_inlay_signatures
 from tools.navigation import (
     FileSymbolIndex,
@@ -33,6 +35,18 @@ DIAGNOSTIC_SOURCE = "nx-post-support"
 LOGGER = logging.getLogger(__name__)
 
 
+@dataclass(frozen=True)
+class _FileIndex:
+    """Everything indexed for one file; also the persistent cache entry."""
+
+    completion_items: list[lsp.CompletionItem]
+    proc_signatures: dict[str, list[str]]
+    proc_docs: dict[str, str]
+    classes: dict
+    navigation_index: FileSymbolIndex
+    variable_index: tuple[set[str], dict[str, set[str]], list[ProcRange]]
+
+
 class TclLanguageServer(LanguageServer):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
@@ -60,6 +74,10 @@ class TclLanguageServer(LanguageServer):
         self._line_cache: dict[tuple[str, int | None], tuple[str, ...]] = {}
         self._parser_lock = threading.RLock()
         self._thread_parsers = threading.local()
+        # uri -> (normalized source, tree, violations) of the last successful
+        # parse; survives version changes so edits can be reparsed partially.
+        self._last_parse: dict[str, tuple[str, object, list]] = {}
+        self.index_cache = IndexCache()
         self._index_lock = threading.RLock()
         self._index_tokens: dict[str, int] = {}
         self._index_versions: dict[str, int | None] = {}
@@ -87,11 +105,32 @@ class TclLanguageServer(LanguageServer):
         self._analysis_tokens: dict[str, int] = {}
         self._next_analysis_token = 0
 
-    def _parse_source(self, source: str):
+    def _parse_source(self, source: str, pos=None):
         self.parser.violations = []
-        tree = self.parser.parse(source)
+        tree = self.parser.parse(source, pos=pos)
         return tree, list(self.parser.violations)
 
+    def _parse_document(self, document: TextDocument):
+        """Parse a document version, reusing unchanged parts of the last one.
+
+        Callers hold the parser lock.
+        """
+        source = incremental_parse.normalize_newlines(document.source)
+        previous = self._last_parse.get(document.uri)
+        result = None
+        if previous is not None:
+            try:
+                result = incremental_parse.reparse(
+                    *previous, source, self._parse_source
+                )
+            except TclSyntaxError:
+                # E.g. a quote opened in the edit closes further down.
+                result = None
+        if result is None:
+            result = self._parse_source(source)
+        self._last_parse[document.uri] = (source, *result)
+        return result
+
     def parse_source(self, source: str):
         """Parse without retaining an AST, on a parser owned by this thread.
 
@@ -111,7 +150,7 @@ class TclLanguageServer(LanguageServer):
             cached = self._ast_cache.get(key)
             if cached is not None:
                 return cached[0]
-            tree, violations = self._parse_source(document.source)
+            tree, violations = self._parse_document(document)
             self._ast_cache[key] = (tree, violations)
             return tree
 
@@ -121,7 +160,7 @@ class TclLanguageServer(LanguageServer):
             cached = self._ast_cache.get(key)
             if cached is not None:
                 return cached
-            tree, violations = self._parse_source(document.source)
+            tree, violations = self._parse_document(document)
             self._ast_cache[key] = (tree, violations)
             return tree, violations
 
@@ -250,7 +289,9 @@ class TclLanguageServer(LanguageServer):
                     report(f"PSC script not found: {path}")
                     continue
                 try:
+                    source_stat = None
                     if document is None:
+                        source_stat = file_stat(path_string)
                         data = path.read_bytes()
                         try:
                             source = data.decode("utf-8-sig")
@@ -258,7 +299,7 @@ class TclLanguageServer(LanguageServer):
                             # Older Windows NX layers use the ANSI code page.
                             source = data.decode("cp1252")
                         document = TextDocument(uri=uri, source=source, language_id="tcl")
-                    if not self.update_poco_completion_for_file(document, cache_tree=False):
+                    if not self.update_poco_completion_for_file(document, cache_tree=False, source_stat=source_stat):
                         report(f"Could not index PSC script: {path}")
                 except (OSError, UnicodeError) as error:
                     report(f"Could not read PSC script {path}: {error}")
@@ -632,6 +673,33 @@ class TclLanguageServer(LanguageServer):
                 if self._is_same_or_child(cached_path, target):
                     self._ast_cache.pop(key, None)
                     self._line_cache.pop(key, None)
+            for uri in list(self._last_parse):
+                try:
+                    parsed_path = pathlib.Path(uris.to_fs_path(uri))
+                except (TypeError, ValueError):
+                    continue
+                if self._is_same_or_child(parsed_path, target):
+                    del self._last_parse[uri]
+
+    def _build_file_index(
+        self, document: TextDocument, filepath: str, cache_tree: bool
+    ) -> "_FileIndex":
+        tree = (
+            self.get_tree(document)
+            if cache_tree
+            else self.parse_source(document.source)
+        )
+        collector = CompletionCollector()
+        tree.accept(collector, recurse=True)
+        collector.custom_functions.extend(class_completion_items(tree))
+        return _FileIndex(
+            completion_items=list(collector.custom_functions),
+            proc_signatures=dict(collector.proc_signatures),
+            proc_docs=build_proc_docs(tree, document.source),
+            classes=indexed_classes(tree, document.uri, document.source),
+            navigation_index=build_file_symbol_index(filepath, document.uri, tree),
+            variable_index=build_variable_index(document.source, tree),
+        )
 
     def update_poco_completion_for_file(
         self,
@@ -639,32 +707,32 @@ class TclLanguageServer(LanguageServer):
         *,
         cache_tree: bool = True,
         require_file_exists: bool = False,
+        from_disk: bool = False,
+        source_stat: FileStat | None = None,
     ):
-        """Update poco_completion for a specific file when it changes"""
+        """Update poco_completion for a specific file when it changes.
+
+        `from_disk` marks documents that read their source from disk lazily;
+        `source_stat` is the file's stat taken before a caller read it. Such
+        results are served from and stored in the persistent index cache.
+        """
         filepath = str(pathlib.Path(uris.to_fs_path(document.uri)))
         token = self._begin_index_update(filepath, document.version)
         if token is None:
             return False
 
-        collector = CompletionCollector()
+        stat = source_stat or (file_stat(filepath) if from_disk else None)
+        index = self.index_cache.get(filepath, stat) if stat is not None else None
+        if index is None:
             try:
-            tree = (
+                index = self._build_file_index(document, filepath, cache_tree)
-                self.get_tree(document)
-                if cache_tree
-                else self.parse_source(document.source)
-            )
-            tree.accept(collector, recurse=True)
-            collector.custom_functions.extend(class_completion_items(tree))
-            classes = indexed_classes(tree, document.uri, document.source)
-            docs = build_proc_docs(tree, document.source)
-            navigation_index = build_file_symbol_index(
-                filepath, document.uri, tree
-            )
-            variable_index = build_variable_index(document.source, tree)
             except Exception as e:
                 LOGGER.debug("Error parsing %s: %s", filepath, e)
                 self._discard_index_update(filepath, token)
                 return False
+            # Only cache results whose file did not change while being read.
+            if stat is not None and file_stat(filepath) == stat:
+                self.index_cache.put(filepath, stat, index)
 
         if require_file_exists and not pathlib.Path(filepath).is_file():
             self._discard_index_update(filepath, token)
@@ -673,12 +741,12 @@ class TclLanguageServer(LanguageServer):
         with self._index_lock:
             if self._index_tokens.get(filepath) != token:
                 return False
-            self.poco_completion[filepath] = list(collector.custom_functions)
+            self.poco_completion[filepath] = list(index.completion_items)
-            self.proc_signatures[filepath] = dict(collector.proc_signatures)
+            self.proc_signatures[filepath] = dict(index.proc_signatures)
-            self.proc_docs[filepath] = docs
+            self.proc_docs[filepath] = index.proc_docs
-            self.class_indexes[filepath] = classes
+            self.class_indexes[filepath] = index.classes
-            self.navigation_indexes[filepath] = navigation_index
+            self.navigation_indexes[filepath] = index.navigation_index
-            self.variable_indexes[filepath] = (document.version, variable_index)
+            self.variable_indexes[filepath] = (document.version, index.variable_index)
             self._committed_index_versions[filepath] = document.version
             self._invalidate_workspace_caches_locked()
         return True

server/src/tools/incremental_parse.py

@@ -0,0 +1,156 @@
+"""Reparse only the top-level commands touched by an edit.
+
+Tcl top-level commands are independent once the previous command ended on its
+own line: the parser keeps no state between them. An edit is therefore
+reparsed from the first to the last top-level command it touches, commands
+before it are reused as-is and commands after it are reused with their line
+numbers shifted. Whenever that assumption could break, the caller falls back
+to a full parse.
+"""
+
+from __future__ import annotations
+
+import copy
+from collections.abc import Callable
+
+from tclint.syntax_tree import Node, Script
+from tclint.violations import Violation
+
+ParseChunk = Callable[[str, tuple[int, int]], tuple[Script, list[Violation]]]
+
+
+def normalize_newlines(source: str) -> str:
+    """Match the universal newline handling of the tclint parser."""
+    return source.replace("\r\n", "\n").replace("\r", "\n")
+
+
+def _shifted_tree(node: Node, delta: int) -> Node:
+    """Copy a subtree with its line numbers moved by `delta`.
+
+    Cached trees may still be read by other requests, so nodes are never
+    mutated. Attributes such as `Command.routine` alias entries of `children`,
+    so every reference is remapped to the same copy.
+    """
+    copies: dict[int, Node] = {}
+
+    def shifted(original: Node) -> Node:
+        existing = copies.get(id(original))
+        if existing is not None:
+            return existing
+        clone = object.__new__(type(original))
+        copies[id(original)] = clone
+        state = dict(original.__dict__)
+        if state.get("line") is not None:
+            state["line"] += delta
+        end = state.get("end_pos")
+        if end is not None:
+            state["end_pos"] = (end[0] + delta, end[1])
+        for key, value in state.items():
+            if isinstance(value, Node):
+                state[key] = shifted(value)
+            elif isinstance(value, (list, tuple)) and value and isinstance(value[0], Node):
+                state[key] = type(value)(shifted(item) for item in value)
+        clone.__dict__.update(state)
+        return clone
+
+    return shifted(node)
+
+
+def _shifted_violation(violation: Violation, delta: int) -> Violation:
+    clone = copy.copy(violation)
+    clone.start = (violation.start[0] + delta, violation.start[1])
+    clone.end = (violation.end[0] + delta, violation.end[1])
+    return clone
+
+
+def reparse(
+    old_source: str,
+    old_tree: Script,
+    old_violations: list[Violation],
+    new_source: str,
+    parse_chunk: ParseChunk,
+) -> tuple[Script, list[Violation]] | None:
+    """Return the tree of `new_source`, or None when a full parse is needed.
+
+    Both sources must already be newline-normalized. `parse_chunk` parses a
+    top-level fragment starting at the given (line, column) and may raise
+    TclSyntaxError, which the caller handles like any failed parse.
+    """
+    if old_source == new_source:
+        return old_tree, list(old_violations)
+
+    # Changed line range (1-indexed); lines outside it are identical. A pure
+    # insertion leaves last_changed_old == first_changed_line - 1.
+    old_lines = old_source.split("\n")
+    new_lines = new_source.split("\n")
+    limit = min(len(old_lines), len(new_lines))
+    same_before = 0
+    while same_before < limit and old_lines[same_before] == new_lines[same_before]:
+        same_before += 1
+    same_after = 0
+    while (
+        same_after < limit - same_before
+        and old_lines[-1 - same_after] == new_lines[-1 - same_after]
+    ):
+        same_after += 1
+
+    first_changed_line = same_before + 1
+    last_changed_old = len(old_lines) - same_after
+    delta = len(new_lines) - len(old_lines)
+
+    commands = old_tree.children
+    if any(command.line is None or command.end_pos is None for command in commands):
+        return None
+
+    # Commands overlapping the changed lines, widened so that no reused
+    # command shares a line with the reparsed range.
+    first = next(
+        (index for index, command in enumerate(commands) if command.end_pos[0] >= first_changed_line),
+        len(commands),
+    )
+    start_line = first_changed_line
+    if first < len(commands):
+        start_line = min(start_line, commands[first].line)
+    while first > 0 and commands[first - 1].end_pos[0] >= start_line:
+        first -= 1
+        start_line = min(start_line, commands[first].line)
+
+    last = first - 1
+    end_line_old = last_changed_old
+    while last + 1 < len(commands) and commands[last + 1].line <= end_line_old:
+        last += 1
+        end_line_old = max(end_line_old, commands[last].end_pos[0])
+    end_line_new = end_line_old + delta
+
+    if end_line_new < start_line - 1 or end_line_new > len(new_lines):
+        return None
+    # A trailing backslash joins a line with the next one across the boundary.
+    if start_line > 1 and new_lines[start_line - 2].endswith("\\"):
+        return None
+    if end_line_new >= start_line and new_lines[end_line_new - 1].endswith("\\"):
+        return None
+
+    chunk_commands: list[Node] = []
+    chunk_violations: list[Violation] = []
+    if end_line_new >= start_line:
+        chunk = "\n".join(new_lines[start_line - 1 : end_line_new])
+        chunk_tree, chunk_violations = parse_chunk(chunk, (start_line, 1))
+        chunk_commands = chunk_tree.children
+
+    reused_after = [_shifted_tree(command, delta) for command in commands[last + 1 :]]
+    tree = Script(
+        *commands[:first],
+        *chunk_commands,
+        *reused_after,
+        pos=(old_tree.line, old_tree.col),
+    )
+    tree.end_pos = (len(new_lines), len(new_lines[-1]) + 1)
+
+    violations = [violation for violation in old_violations if violation.start[0] < start_line]
+    violations += chunk_violations
+    violations += [
+        _shifted_violation(violation, delta)
+        for violation in old_violations
+        if violation.start[0] > end_line_old
+    ]
+    return tree, violations

server/src/tools/index_cache.py

@@ -0,0 +1,128 @@
+"""Persist per-file index results across server restarts.
+
+Entries are keyed by path and validated by the file's size and mtime. The
+whole cache is tied to a fingerprint of the code that produced it: the
+indexing sources of this server, the bundled tclint sources and the versions
+of all bundled libraries. Any change to them, including a tclint update or a
+local patch, discards the cache instead of loading stale results.
+"""
+
+from __future__ import annotations
+
+import hashlib
+import logging
+import os
+import pathlib
+import pickle
+import sys
+import tempfile
+import threading
+import zlib
+from typing import Any
+
+LOGGER = logging.getLogger(__name__)
+
+# Bump when the cached data layout changes without a source change above.
+CACHE_FORMAT = 1
+CACHE_FILE = "index-cache.pickle.z"
+
+_SRC_DIR = pathlib.Path(__file__).resolve().parent.parent
+_LIBS_DIR = _SRC_DIR.parent / "libs"
+
+FileStat = tuple[int, int]
+
+
+def code_fingerprint() -> str:
+    digest = hashlib.sha256()
+    digest.update(f"{CACHE_FORMAT}|{sys.version}".encode())
+    sources = [
+        _SRC_DIR / "lsp_tclserver.py",
+        *sorted((_SRC_DIR / "tools").glob("*.py")),
+        *sorted((_SRC_DIR / "plugins").glob("*.py")),
+        *sorted((_LIBS_DIR / "tclint").rglob("*.py")),
+    ]
+    for source in sources:
+        digest.update(source.relative_to(_SRC_DIR.parent).as_posix().encode())
+        digest.update(source.read_bytes())
+    for dist_info in sorted(_LIBS_DIR.glob("*.dist-info")):
+        digest.update(dist_info.name.encode())
+    return digest.hexdigest()
+
+
+def file_stat(path: str) -> FileStat | None:
+    try:
+        stat = os.stat(path)
+    except OSError:
+        return None
+    return stat.st_mtime_ns, stat.st_size
+
+
+class IndexCache:
+    def __init__(self, directory: pathlib.Path | None = None, fingerprint: str = ""):
+        self._path = directory / CACHE_FILE if directory is not None else None
+        self._fingerprint = fingerprint
+        self._entries: dict[str, tuple[FileStat, Any]] = {}
+        self._used: set[str] = set()
+        self._dirty = False
+        self._lock = threading.Lock()
+
+    @classmethod
+    def load(cls, directory: pathlib.Path | str | None) -> IndexCache:
+        """Open the cache in `directory`; without one, nothing is persisted."""
+        if not directory:
+            return cls()
+        cache = cls(pathlib.Path(directory), code_fingerprint())
+        try:
+            with open(cache._path, "rb") as file:
+                fingerprint, entries = pickle.loads(zlib.decompress(file.read()))
+        except FileNotFoundError:
+            return cache
+        except Exception as error:  # A damaged cache must never stop indexing.
+            LOGGER.warning("Ignoring unreadable index cache %s: %s", cache._path, error)
+            cache._dirty = True
+            return cache
+        if fingerprint == cache._fingerprint:
+            cache._entries = entries
+        else:
+            cache._dirty = True
+        return cache
+
+    def get(self, path: str, stat: FileStat) -> Any | None:
+        with self._lock:
+            entry = self._entries.get(path)
+            if entry is None or entry[0] != stat:
+                return None
+            self._used.add(path)
+            return entry[1]
+
+    def put(self, path: str, stat: FileStat, data: Any) -> None:
+        if self._path is None:
+            return
+        with self._lock:
+            self._entries[path] = (stat, data)
+            self._used.add(path)
+            self._dirty = True
+
+    def save(self) -> None:
+        """Write entries used in this session atomically; others are dropped."""
+        if self._path is None:
+            return
+        with self._lock:
+            if not self._dirty and self._used == self._entries.keys():
+                return
+            entries = {path: self._entries[path] for path in self._used if path in self._entries}
+            self._entries = entries
+            self._dirty = False
+        try:
+            self._path.parent.mkdir(parents=True, exist_ok=True)
+            with tempfile.NamedTemporaryFile(dir=self._path.parent, delete=False) as file:
+                data = pickle.dumps((self._fingerprint, entries), protocol=pickle.HIGHEST_PROTOCOL)
+                # Pickled indexes are very repetitive; fast compression cuts ~90%.
+                file.write(zlib.compress(data, 1))
+            os.replace(file.name, self._path)
+        except Exception as error:
+            LOGGER.warning("Could not write index cache %s: %s", self._path, error)
+            try:
+                os.unlink(file.name)
+            except (OSError, NameError):
+                pass

server/src/tools/navigation.py

@@ -265,6 +265,12 @@ def build_file_symbol_index(
     filepath: str, uri: str, tree: Node
 ) -> FileSymbolIndex:
     occurrences: list[SymbolOccurrence] = []
+    # Most occurrences repeat a few identities; sharing one object per identity
+    # keeps the index (and its persistent cache) small.
+    identities: dict[SymbolIdentity, SymbolIdentity] = {}
+
+    def shared(identity: SymbolIdentity | None) -> SymbolIdentity | None:
+        return None if identity is None else identities.setdefault(identity, identity)
 
     def add_proc(
         node: Node,
@@ -274,7 +280,7 @@ def build_file_symbol_index(
         is_definition: bool,
         declaration_range: lsp.Range | None = None,
     ) -> None:
-        identity = _proc_identity(raw_name, scope.namespace)
+        identity = shared(_proc_identity(raw_name, scope.namespace))
         caller = None
         if not is_definition:
             caller = (
@@ -288,14 +294,14 @@ def build_file_symbol_index(
                 fallback_identity=(
                     None
                     if is_definition
-                    else _proc_fallback(raw_name, scope.namespace)
+                    else shared(_proc_fallback(raw_name, scope.namespace))
                 ),
                 range=_name_range(node, raw_name),
                 placeholder=_basename(raw_name),
                 is_definition=is_definition,
                 symbol_kind=lsp.SymbolKind.Function,
                 container_name=_container_name(identity),
-                caller=caller,
+                caller=shared(caller),
                 declaration_range=declaration_range,
             )
         )
@@ -309,7 +315,7 @@ def build_file_symbol_index(
         variable_sub: bool = False,
         identity: SymbolIdentity | None = None,
     ) -> None:
-        symbol_identity = identity or _variable_identity(raw_name, scope)
+        symbol_identity = shared(identity or _variable_identity(raw_name, scope))
         occurrences.append(
             SymbolOccurrence(
                 identity=symbol_identity,

server/tests/python_tests/test_incremental_parse.py

@@ -0,0 +1,131 @@
+import sys
+from pathlib import Path
+
+import pytest
+
+THIS_DIR = Path(__file__).parent
+SRC_DIR = THIS_DIR.parent.parent / "src"
+if str(SRC_DIR) not in sys.path:
+    sys.path.insert(0, str(SRC_DIR))
+
+from lsp_tclserver import TclLanguageServer  # noqa: E402
+from pygls.workspace.text_document import TextDocument  # noqa: E402
+from tclint.lexer import TclSyntaxError  # noqa: E402
+from tclint.syntax_tree import Node  # noqa: E402
+from tools.incremental_parse import reparse  # noqa: E402
+from tools.parser import CustomParser  # noqa: E402
+
+SOURCE = """\
+# header comment
+set a 1; set b 2
+proc first {x} {
+    global mom_pos
+    if {$x > 0} {
+        MOM_output_literal "first $x"
+    }
+    return [expr {$x + 1}]
+}
+
+proc second {} {
+    set list [list a b \\
+        c d]
+    return $list
+}
+lappend ::handlers {second}
+"""
+
+
+def _parse(text, pos=None):
+    parser = CustomParser()
+    tree = parser.parse(text, pos=pos)
+    return tree, list(parser.violations)
+
+
+def _differences(a, b, path="root"):
+    if type(a) is not type(b):
+        return f"{path}: {type(a).__name__} != {type(b).__name__}"
+    for key in a.__dict__.keys() | b.__dict__.keys():
+        first, second = a.__dict__.get(key), b.__dict__.get(key)
+        if isinstance(first, Node):
+            difference = _differences(first, second, f"{path}.{key}")
+        elif isinstance(first, (list, tuple)) and first and isinstance(first[0], Node):
+            if len(first) != len(second):
+                return f"{path}.{key}: {len(first)} != {len(second)}"
+            difference = next(
+                (d for i, (x, y) in enumerate(zip(first, second)) if (d := _differences(x, y, f"{path}.{key}[{i}]"))),
+                None,
+            )
+        else:
+            difference = None if first == second else f"{path}.{key}: {first!r} != {second!r}"
+        if difference:
+            return difference
+    return None
+
+
+def _violations(violations):
+    return [(str(v.id), v.message, v.start, v.end) for v in violations]
+
+
+@pytest.mark.parametrize("old, new", [
+    ("MOM_output_literal \"first $x\"", "MOM_output_literal \"first $x\" extra"),
+    ("    return $list\n", "    return $list\n    puts done\n"),
+    ("proc second {} {", "\nproc second {} {"),
+    ("set a 1; set b 2\n", ""),
+    ("# header comment\n", "# header comment\nset inserted 0\n"),
+    ("lappend ::handlers {second}\n", "lappend ::handlers {second}\nproc third {} {}\n"),
+    ("        c d]", "        c d e]"),
+    ("global mom_pos", "global mom_pos mom_out_angle_pos"),
+])
+def test_incremental_tree_matches_full_parse(old, new):
+    edited = SOURCE.replace(old, new, 1)
+    assert edited != SOURCE
+    previous = (SOURCE, *_parse(SOURCE))
+    result = reparse(*previous, edited, _parse)
+    assert result is not None
+    expected = _parse(edited)
+    assert _differences(result[0], expected[0]) is None
+    assert _violations(result[1]) == _violations(expected[1])
+
+
+def test_continuation_across_the_edit_forces_full_parse():
+    edited = SOURCE.replace("set a 1; set b 2", "set a 1; set b 2 \\")
+    assert reparse(SOURCE, *_parse(SOURCE), edited, _parse) is None
+
+
+def test_quote_closing_outside_the_edit_is_left_to_the_full_parse():
+    edited = SOURCE.replace("set a 1; set b 2", 'set a "1; set b 2')
+    try:
+        result = reparse(SOURCE, *_parse(SOURCE), edited, _parse)
+    except TclSyntaxError:
+        result = None
+    assert result is None
+
+
+def test_unchanged_commands_are_reused_and_never_mutated():
+    tree, violations = _parse(SOURCE)
+    edited = SOURCE.replace("return $list", "return [lsort $list]")
+    new_tree, _ = reparse(SOURCE, tree, violations, edited, _parse)
+    assert new_tree.children[0] is tree.children[0]
+    # Commands after the edit are shifted copies; the old tree stays valid.
+    inserted = SOURCE.replace("proc first", "\n\nproc first")
+    shifted_tree, _ = reparse(SOURCE, tree, violations, inserted, _parse)
+    assert shifted_tree.children[-1] is not tree.children[-1]
+    assert shifted_tree.children[-1].line == tree.children[-1].line + 2
+    assert tree.children[-1].line == _parse(SOURCE)[0].children[-1].line
+
+
+def test_server_reparses_edits_incrementally(tmp_path, monkeypatch):
+    server = TclLanguageServer(name="incremental-test", version="1", max_workers=1)
+    uri = (tmp_path / "edit.tcl").as_uri()
+    first = server.get_tree(TextDocument(uri=uri, source=SOURCE, version=1, language_id="tcl"))
+
+    parsed_sources = []
+    parse_source = server._parse_source
+    monkeypatch.setattr(server, "_parse_source", lambda text, pos=None: parsed_sources.append(text) or parse_source(text, pos))
+    server.clear_cache_for_uri(uri)
+    edited = SOURCE.replace("return $list", "return [lsort $list]")
+    second = server.get_tree(TextDocument(uri=uri, source=edited, version=2, language_id="tcl"))
+
+    assert parsed_sources and all(text != edited for text in parsed_sources)
+    assert second.children[0] is first.children[0]
+    assert _differences(second, _parse(edited)[0]) is None

server/tests/python_tests/test_index_cache.py

@@ -0,0 +1,109 @@
+import os
+import sys
+from pathlib import Path
+
+THIS_DIR = Path(__file__).parent
+SRC_DIR = THIS_DIR.parent.parent / "src"
+if str(SRC_DIR) not in sys.path:
+    sys.path.insert(0, str(SRC_DIR))
+
+import tools.index_cache as index_cache  # noqa: E402
+from lsp_tclserver import TclLanguageServer  # noqa: E402
+from pygls.workspace.text_document import TextDocument  # noqa: E402
+from tools.index_cache import IndexCache  # noqa: E402
+
+
+def _index_from_disk(server, path: Path) -> bool:
+    return server.update_poco_completion_for_file(
+        TextDocument(uri=path.as_uri(), language_id="tcl"),
+        cache_tree=False,
+        require_file_exists=True,
+        from_disk=True,
+    )
+
+
+def _warm_server(cache_dir: Path) -> TclLanguageServer:
+    server = TclLanguageServer(name="cache-test", version="1", max_workers=1)
+    server.index_cache = IndexCache.load(cache_dir)
+    return server
+
+
+def _fail_build(*_args, **_kwargs):
+    raise AssertionError("file was parsed although it is cached")
+
+
+def test_second_start_uses_cached_index(tmp_path: Path, monkeypatch):
+    source = tmp_path / "post.tcl"
+    source.write_text("proc cached_proc {a b} { return $a }\n", encoding="utf-8")
+    cache_dir = tmp_path / "storage"
+
+    server = _warm_server(cache_dir)
+    assert _index_from_disk(server, source)
+    server.index_cache.save()
+
+    restarted = _warm_server(cache_dir)
+    monkeypatch.setattr(restarted, "_build_file_index", _fail_build)
+    assert _index_from_disk(restarted, source)
+    assert "cached_proc" in restarted.custom_function_names_snapshot()
+    assert restarted.proc_metadata_snapshot(str(source))[0]["cached_proc"] == ["a", "b"]
+
+
+def test_changed_file_is_parsed_again(tmp_path: Path):
+    source = tmp_path / "post.tcl"
+    source.write_text("proc old_proc {} {}\n", encoding="utf-8")
+    cache_dir = tmp_path / "storage"
+    server = _warm_server(cache_dir)
+    assert _index_from_disk(server, source)
+    server.index_cache.save()
+
+    source.write_text("proc new_proc {} {}\n", encoding="utf-8")
+    stat = source.stat()
+    os.utime(source, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000))
+    restarted = _warm_server(cache_dir)
+    assert _index_from_disk(restarted, source)
+    names = restarted.custom_function_names_snapshot()
+    assert "new_proc" in names and "old_proc" not in names
+
+
+def test_code_change_discards_the_cache(tmp_path: Path, monkeypatch):
+    source = tmp_path / "post.tcl"
+    source.write_text("proc cached_proc {} {}\n", encoding="utf-8")
+    cache_dir = tmp_path / "storage"
+    server = _warm_server(cache_dir)
+    assert _index_from_disk(server, source)
+    server.index_cache.save()
+
+    # E.g. an updated tclint: a different fingerprint must not load old entries.
+    monkeypatch.setattr(index_cache, "code_fingerprint", lambda: "other tclint")
+    restarted = _warm_server(cache_dir)
+    built = []
+    build = restarted._build_file_index
+    monkeypatch.setattr(restarted, "_build_file_index", lambda *args: built.append(args) or build(*args))
+    assert _index_from_disk(restarted, source)
+    assert built
+
+
+def test_damaged_cache_is_ignored(tmp_path: Path, monkeypatch):
+    cache_dir = tmp_path / "storage"
+    cache_dir.mkdir()
+    (cache_dir / index_cache.CACHE_FILE).write_bytes(b"not a cache")
+    source = tmp_path / "post.tcl"
+    source.write_text("proc fresh_proc {} {}\n", encoding="utf-8")
+
+    server = _warm_server(cache_dir)
+    assert _index_from_disk(server, source)
+    server.index_cache.save()
+
+    restarted = _warm_server(cache_dir)
+    monkeypatch.setattr(restarted, "_build_file_index", _fail_build)
+    assert _index_from_disk(restarted, source)
+
+
+def test_open_documents_never_touch_the_cache(tmp_path: Path):
+    source = tmp_path / "post.tcl"
+    source.write_text("proc on_disk {} {}\n", encoding="utf-8")
+    server = _warm_server(tmp_path / "storage")
+    unsaved = TextDocument(uri=source.as_uri(), source="proc unsaved {} {}\n", version=3, language_id="tcl")
+    assert server.update_poco_completion_for_file(unsaved)
+    server.index_cache.save()
+    assert not (tmp_path / "storage" / index_cache.CACHE_FILE).exists()