From 01e8670cc19a600fe9cca0529759ab93c008cd76 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Wed, 23 Sep 2026 13:20:09 +0200 Subject: [PATCH] feat(indexing): add persistent index cache and incremental reparse - Pass extension storage path to the server (client/ changes) so the server can persist a workspace index. - Introduce IndexCache (server/tools/index_cache.py) and load/save it on initialization and after background indexing. Index entries are stored only when the file's stat hasn't changed while being read. - Add incremental reparse logic (server/tools/incremental_parse.py) and use a per-file _last_parse cache in the language server to reparse only the top-level Tcl commands touched by an edit, falling back to a full parse when necessary. - Use a new _FileIndex dataclass and _build_file_index helper to unify what is stored/loaded for a file; update update_poco_completion_for_file to use the persistent cache for disk-read files (from_disk/source_stat). - Keep background indexing non-blocking and persist the index at the end of the run. Add basic unit tests for incremental parse and index cache. Before: edits and background work always required full parsing of files and no persistent cross-restart index. After: some edits reuse previous ASTs and files read from disk can use a persisted index to skip re-indexing across restarts. --- CHANGELOG.md | 5 + client/src/common/server.ts | 13 +- client/src/extension.ts | 16 +- server/src/lsp_server.py | 12 +- server/src/lsp_tclserver.py | 132 +++++++++++---- server/src/tools/incremental_parse.py | 156 ++++++++++++++++++ server/src/tools/index_cache.py | 128 ++++++++++++++ server/src/tools/navigation.py | 14 +- .../python_tests/test_incremental_parse.py | 131 +++++++++++++++ server/tests/python_tests/test_index_cache.py | 109 ++++++++++++ 10 files changed, 674 insertions(+), 42 deletions(-) create mode 100644 server/src/tools/incremental_parse.py create mode 100644 server/src/tools/index_cache.py create mode 100644 server/tests/python_tests/test_incremental_parse.py create mode 100644 server/tests/python_tests/test_index_cache.py diff --git a/CHANGELOG.md b/CHANGELOG.md index bd31eb8..4214d81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## Unreleased +- Reparse only the top-level TCL commands touched by an edit instead of the whole file +- Cache the workspace index in the extension storage so restarts skip reparsing unchanged files; the cache is discarded automatically when the server or bundled tclint changes +- Speed up TclOO completion, signature help, inlay hints, and Go to Definition by reusing the cached syntax tree and skipping files without classes +- Speed up references, document highlights, and call hierarchy with cached definition lookups +- Keep background indexing from blocking requests that need a fresh syntax tree - Add Go to Definition for TclOO classes, constructors, and resolved methods, including PSC library definitions - Index PSC layer scripts (including external paths and legacy Windows encoding) and share TclOO class metadata across files for completion, signatures, inlay hints, and highlighting - Add document-local TclOO method completion for `new`/`create` instances, `my`, and statically inferred return chains diff --git a/client/src/common/server.ts b/client/src/common/server.ts index 8b8300f..3de053d 100644 --- a/client/src/common/server.ts +++ b/client/src/common/server.ts @@ -22,7 +22,12 @@ import { import { getLSClientTraceLevel, getProjectRoot } from "./utilities" import { isVirtualWorkspace } from "./vscodeapi" -export type IInitOptions = { settings: ISettings[]; globalSettings: ISettings } +export type IInitOptions = { + settings: ISettings[] + globalSettings: ISettings + // Folder for the server's persistent index cache; omitted without a workspace. + indexCachePath?: string +} let _disposables: Disposable[] = [] @@ -114,7 +119,8 @@ export async function restartServer( serverId: string, serverName: string, outputChannel: LogOutputChannel, - lsClient?: LanguageClient + lsClient?: LanguageClient, + indexCachePath?: string ): Promise { if (lsClient) { traceInfo(`Server: Stop requested`) @@ -132,7 +138,8 @@ export async function restartServer( outputChannel, { settings: await getExtensionSettings(serverId, true), - globalSettings: await getGlobalSettings(serverId, false) + globalSettings: await getGlobalSettings(serverId, false), + indexCachePath } ) traceInfo(`Server: Start requested.`) diff --git a/client/src/extension.ts b/client/src/extension.ts index bf27936..6876c00 100644 --- a/client/src/extension.ts +++ b/client/src/extension.ts @@ -73,7 +73,13 @@ export async function activate(context: vscode.ExtensionContext) { traceVerbose( `Using interpreter from ${serverInfo.module}.interpreter: ${interpreter.join(" ")}` ) - client = await restartServer(serverId, serverName, outputChannel, client) + client = await restartServer( + serverId, + serverName, + outputChannel, + client, + context.storageUri?.fsPath + ) } return } @@ -83,7 +89,13 @@ export async function activate(context: vscode.ExtensionContext) { traceVerbose( `Using interpreter from Python extension: ${interpreterDetails.path.join(" ")}` ) - client = await restartServer(serverId, serverName, outputChannel, client) + client = await restartServer( + serverId, + serverName, + outputChannel, + client, + context.storageUri?.fsPath + ) return } diff --git a/server/src/lsp_server.py b/server/src/lsp_server.py index 8fe17e9..9b7f440 100644 --- a/server/src/lsp_server.py +++ b/server/src/lsp_server.py @@ -11,6 +11,7 @@ import pathlib import re import sys import threading +import time from collections import ChainMap from functools import reduce from typing import Any, Optional @@ -52,6 +53,7 @@ from tools.completion_items import ( ranked_completion_items, ) from tools.folding_ranges import build_folding_ranges +from tools.index_cache import IndexCache from tools.inlay_hint import ( InlayHintGenerator, build_builtin_inlay_signatures, @@ -89,6 +91,8 @@ from tools.tcl_command_completion import ( WORKSPACE_SETTINGS = {} GLOBAL_SETTINGS = {} +# Extension storage folder of the workspace; without it nothing is persisted. +INDEX_CACHE_PATH: dict[str, str | None] = {} MAX_WORKERS = 5 @@ -175,6 +179,7 @@ def _index_tcl_file_from_disk(uri: str) -> None: document, cache_tree=False, require_file_exists=True, + from_disk=True, ) except (OSError, UnicodeError) as error: log_warning(f"Could not re-index {path}: {error}") @@ -861,6 +866,7 @@ def initialize(params: lsp.InitializeParams) -> lsp.InitializeResult: log_to_output(f"sys.path used to run Server:\r\n {paths}") GLOBAL_SETTINGS.update(**params.initialization_options.get("globalSettings", {})) + INDEX_CACHE_PATH["path"] = params.initialization_options.get("indexCachePath") settings = params.initialization_options["settings"] _update_workspace_settings(settings) @@ -899,6 +905,8 @@ def initialized(_params: lsp.InitializedParams): log_to_output("Background indexing skipped: no workspace folder is open.") return log_to_output("Background indexing started...") + started = time.perf_counter() + LSP_SERVER.index_cache = IndexCache.load(INDEX_CACHE_PATH.get("path")) root_path = pathlib.Path(root) skipped_directories = { ".git", @@ -916,11 +924,13 @@ def initialized(_params: lsp.InitializedParams): document, cache_tree=False, require_file_exists=True, + from_disk=True, ) except Exception as error: log_to_output(f"Fehler beim Parsen von {filepath}: {error}") _refresh_psc_index() - log_to_output("Background indexing completed.") + LSP_SERVER.index_cache.save() + log_to_output(f"Background indexing completed in {time.perf_counter() - started:.1f}s.") except Exception as e: log_to_output(f"Background indexing failed: {e}") diff --git a/server/src/lsp_tclserver.py b/server/src/lsp_tclserver.py index 637f814..1241433 100644 --- a/server/src/lsp_tclserver.py +++ b/server/src/lsp_tclserver.py @@ -3,6 +3,7 @@ import os import pathlib import threading import xml.etree.ElementTree as ET +from dataclasses import dataclass from typing import List, Optional, Tuple import lsprotocol.types as lsp @@ -13,12 +14,13 @@ from pygls.workspace.text_document import TextDocument from tclint.format import FormatterOpts from tclint.lexer import TclSyntaxError from tclint.violations import Violation -from tools import checks, parser +from tools import checks, incremental_parse, parser from tools.completion_items import CompletionCollector from tools.tcloo_symbols import class_completion_items from tools.tcloo_completion import indexed_classes from tools.file_sourcing import get_all_psc_files, psc_script_files from tools.formatter import NxFormatter as Formatter +from tools.index_cache import FileStat, IndexCache, file_stat from tools.inlay_hint import InlayHintSignature, build_custom_inlay_signatures from tools.navigation import ( FileSymbolIndex, @@ -33,6 +35,18 @@ DIAGNOSTIC_SOURCE = "nx-post-support" LOGGER = logging.getLogger(__name__) +@dataclass(frozen=True) +class _FileIndex: + """Everything indexed for one file; also the persistent cache entry.""" + + completion_items: list[lsp.CompletionItem] + proc_signatures: dict[str, list[str]] + proc_docs: dict[str, str] + classes: dict + navigation_index: FileSymbolIndex + variable_index: tuple[set[str], dict[str, set[str]], list[ProcRange]] + + class TclLanguageServer(LanguageServer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -60,6 +74,10 @@ class TclLanguageServer(LanguageServer): self._line_cache: dict[tuple[str, int | None], tuple[str, ...]] = {} self._parser_lock = threading.RLock() self._thread_parsers = threading.local() + # uri -> (normalized source, tree, violations) of the last successful + # parse; survives version changes so edits can be reparsed partially. + self._last_parse: dict[str, tuple[str, object, list]] = {} + self.index_cache = IndexCache() self._index_lock = threading.RLock() self._index_tokens: dict[str, int] = {} self._index_versions: dict[str, int | None] = {} @@ -87,11 +105,32 @@ class TclLanguageServer(LanguageServer): self._analysis_tokens: dict[str, int] = {} self._next_analysis_token = 0 - def _parse_source(self, source: str): + def _parse_source(self, source: str, pos=None): self.parser.violations = [] - tree = self.parser.parse(source) + tree = self.parser.parse(source, pos=pos) return tree, list(self.parser.violations) + def _parse_document(self, document: TextDocument): + """Parse a document version, reusing unchanged parts of the last one. + + Callers hold the parser lock. + """ + source = incremental_parse.normalize_newlines(document.source) + previous = self._last_parse.get(document.uri) + result = None + if previous is not None: + try: + result = incremental_parse.reparse( + *previous, source, self._parse_source + ) + except TclSyntaxError: + # E.g. a quote opened in the edit closes further down. + result = None + if result is None: + result = self._parse_source(source) + self._last_parse[document.uri] = (source, *result) + return result + def parse_source(self, source: str): """Parse without retaining an AST, on a parser owned by this thread. @@ -111,7 +150,7 @@ class TclLanguageServer(LanguageServer): cached = self._ast_cache.get(key) if cached is not None: return cached[0] - tree, violations = self._parse_source(document.source) + tree, violations = self._parse_document(document) self._ast_cache[key] = (tree, violations) return tree @@ -121,7 +160,7 @@ class TclLanguageServer(LanguageServer): cached = self._ast_cache.get(key) if cached is not None: return cached - tree, violations = self._parse_source(document.source) + tree, violations = self._parse_document(document) self._ast_cache[key] = (tree, violations) return tree, violations @@ -250,7 +289,9 @@ class TclLanguageServer(LanguageServer): report(f"PSC script not found: {path}") continue try: + source_stat = None if document is None: + source_stat = file_stat(path_string) data = path.read_bytes() try: source = data.decode("utf-8-sig") @@ -258,7 +299,7 @@ class TclLanguageServer(LanguageServer): # Older Windows NX layers use the ANSI code page. source = data.decode("cp1252") document = TextDocument(uri=uri, source=source, language_id="tcl") - if not self.update_poco_completion_for_file(document, cache_tree=False): + if not self.update_poco_completion_for_file(document, cache_tree=False, source_stat=source_stat): report(f"Could not index PSC script: {path}") except (OSError, UnicodeError) as error: report(f"Could not read PSC script {path}: {error}") @@ -632,6 +673,33 @@ class TclLanguageServer(LanguageServer): if self._is_same_or_child(cached_path, target): self._ast_cache.pop(key, None) self._line_cache.pop(key, None) + for uri in list(self._last_parse): + try: + parsed_path = pathlib.Path(uris.to_fs_path(uri)) + except (TypeError, ValueError): + continue + if self._is_same_or_child(parsed_path, target): + del self._last_parse[uri] + + def _build_file_index( + self, document: TextDocument, filepath: str, cache_tree: bool + ) -> "_FileIndex": + tree = ( + self.get_tree(document) + if cache_tree + else self.parse_source(document.source) + ) + collector = CompletionCollector() + tree.accept(collector, recurse=True) + collector.custom_functions.extend(class_completion_items(tree)) + return _FileIndex( + completion_items=list(collector.custom_functions), + proc_signatures=dict(collector.proc_signatures), + proc_docs=build_proc_docs(tree, document.source), + classes=indexed_classes(tree, document.uri, document.source), + navigation_index=build_file_symbol_index(filepath, document.uri, tree), + variable_index=build_variable_index(document.source, tree), + ) def update_poco_completion_for_file( self, @@ -639,32 +707,32 @@ class TclLanguageServer(LanguageServer): *, cache_tree: bool = True, require_file_exists: bool = False, + from_disk: bool = False, + source_stat: FileStat | None = None, ): - """Update poco_completion for a specific file when it changes""" + """Update poco_completion for a specific file when it changes. + + `from_disk` marks documents that read their source from disk lazily; + `source_stat` is the file's stat taken before a caller read it. Such + results are served from and stored in the persistent index cache. + """ filepath = str(pathlib.Path(uris.to_fs_path(document.uri))) token = self._begin_index_update(filepath, document.version) if token is None: return False - collector = CompletionCollector() - try: - tree = ( - self.get_tree(document) - if cache_tree - else self.parse_source(document.source) - ) - tree.accept(collector, recurse=True) - collector.custom_functions.extend(class_completion_items(tree)) - classes = indexed_classes(tree, document.uri, document.source) - docs = build_proc_docs(tree, document.source) - navigation_index = build_file_symbol_index( - filepath, document.uri, tree - ) - variable_index = build_variable_index(document.source, tree) - except Exception as e: - LOGGER.debug("Error parsing %s: %s", filepath, e) - self._discard_index_update(filepath, token) - return False + stat = source_stat or (file_stat(filepath) if from_disk else None) + index = self.index_cache.get(filepath, stat) if stat is not None else None + if index is None: + try: + index = self._build_file_index(document, filepath, cache_tree) + except Exception as e: + LOGGER.debug("Error parsing %s: %s", filepath, e) + self._discard_index_update(filepath, token) + return False + # Only cache results whose file did not change while being read. + if stat is not None and file_stat(filepath) == stat: + self.index_cache.put(filepath, stat, index) if require_file_exists and not pathlib.Path(filepath).is_file(): self._discard_index_update(filepath, token) @@ -673,12 +741,12 @@ class TclLanguageServer(LanguageServer): with self._index_lock: if self._index_tokens.get(filepath) != token: return False - self.poco_completion[filepath] = list(collector.custom_functions) - self.proc_signatures[filepath] = dict(collector.proc_signatures) - self.proc_docs[filepath] = docs - self.class_indexes[filepath] = classes - self.navigation_indexes[filepath] = navigation_index - self.variable_indexes[filepath] = (document.version, variable_index) + self.poco_completion[filepath] = list(index.completion_items) + self.proc_signatures[filepath] = dict(index.proc_signatures) + self.proc_docs[filepath] = index.proc_docs + self.class_indexes[filepath] = index.classes + self.navigation_indexes[filepath] = index.navigation_index + self.variable_indexes[filepath] = (document.version, index.variable_index) self._committed_index_versions[filepath] = document.version self._invalidate_workspace_caches_locked() return True diff --git a/server/src/tools/incremental_parse.py b/server/src/tools/incremental_parse.py new file mode 100644 index 0000000..a0ab1de --- /dev/null +++ b/server/src/tools/incremental_parse.py @@ -0,0 +1,156 @@ +"""Reparse only the top-level commands touched by an edit. + +Tcl top-level commands are independent once the previous command ended on its +own line: the parser keeps no state between them. An edit is therefore +reparsed from the first to the last top-level command it touches, commands +before it are reused as-is and commands after it are reused with their line +numbers shifted. Whenever that assumption could break, the caller falls back +to a full parse. +""" + +from __future__ import annotations + +import copy +from collections.abc import Callable + +from tclint.syntax_tree import Node, Script +from tclint.violations import Violation + +ParseChunk = Callable[[str, tuple[int, int]], tuple[Script, list[Violation]]] + + +def normalize_newlines(source: str) -> str: + """Match the universal newline handling of the tclint parser.""" + return source.replace("\r\n", "\n").replace("\r", "\n") + + +def _shifted_tree(node: Node, delta: int) -> Node: + """Copy a subtree with its line numbers moved by `delta`. + + Cached trees may still be read by other requests, so nodes are never + mutated. Attributes such as `Command.routine` alias entries of `children`, + so every reference is remapped to the same copy. + """ + copies: dict[int, Node] = {} + + def shifted(original: Node) -> Node: + existing = copies.get(id(original)) + if existing is not None: + return existing + clone = object.__new__(type(original)) + copies[id(original)] = clone + state = dict(original.__dict__) + if state.get("line") is not None: + state["line"] += delta + end = state.get("end_pos") + if end is not None: + state["end_pos"] = (end[0] + delta, end[1]) + for key, value in state.items(): + if isinstance(value, Node): + state[key] = shifted(value) + elif isinstance(value, (list, tuple)) and value and isinstance(value[0], Node): + state[key] = type(value)(shifted(item) for item in value) + clone.__dict__.update(state) + return clone + + return shifted(node) + + +def _shifted_violation(violation: Violation, delta: int) -> Violation: + clone = copy.copy(violation) + clone.start = (violation.start[0] + delta, violation.start[1]) + clone.end = (violation.end[0] + delta, violation.end[1]) + return clone + + +def reparse( + old_source: str, + old_tree: Script, + old_violations: list[Violation], + new_source: str, + parse_chunk: ParseChunk, +) -> tuple[Script, list[Violation]] | None: + """Return the tree of `new_source`, or None when a full parse is needed. + + Both sources must already be newline-normalized. `parse_chunk` parses a + top-level fragment starting at the given (line, column) and may raise + TclSyntaxError, which the caller handles like any failed parse. + """ + if old_source == new_source: + return old_tree, list(old_violations) + + # Changed line range (1-indexed); lines outside it are identical. A pure + # insertion leaves last_changed_old == first_changed_line - 1. + old_lines = old_source.split("\n") + new_lines = new_source.split("\n") + limit = min(len(old_lines), len(new_lines)) + same_before = 0 + while same_before < limit and old_lines[same_before] == new_lines[same_before]: + same_before += 1 + same_after = 0 + while ( + same_after < limit - same_before + and old_lines[-1 - same_after] == new_lines[-1 - same_after] + ): + same_after += 1 + + first_changed_line = same_before + 1 + last_changed_old = len(old_lines) - same_after + delta = len(new_lines) - len(old_lines) + + commands = old_tree.children + if any(command.line is None or command.end_pos is None for command in commands): + return None + + # Commands overlapping the changed lines, widened so that no reused + # command shares a line with the reparsed range. + first = next( + (index for index, command in enumerate(commands) if command.end_pos[0] >= first_changed_line), + len(commands), + ) + start_line = first_changed_line + if first < len(commands): + start_line = min(start_line, commands[first].line) + while first > 0 and commands[first - 1].end_pos[0] >= start_line: + first -= 1 + start_line = min(start_line, commands[first].line) + + last = first - 1 + end_line_old = last_changed_old + while last + 1 < len(commands) and commands[last + 1].line <= end_line_old: + last += 1 + end_line_old = max(end_line_old, commands[last].end_pos[0]) + end_line_new = end_line_old + delta + + if end_line_new < start_line - 1 or end_line_new > len(new_lines): + return None + # A trailing backslash joins a line with the next one across the boundary. + if start_line > 1 and new_lines[start_line - 2].endswith("\\"): + return None + if end_line_new >= start_line and new_lines[end_line_new - 1].endswith("\\"): + return None + + chunk_commands: list[Node] = [] + chunk_violations: list[Violation] = [] + if end_line_new >= start_line: + chunk = "\n".join(new_lines[start_line - 1 : end_line_new]) + chunk_tree, chunk_violations = parse_chunk(chunk, (start_line, 1)) + chunk_commands = chunk_tree.children + + reused_after = [_shifted_tree(command, delta) for command in commands[last + 1 :]] + tree = Script( + *commands[:first], + *chunk_commands, + *reused_after, + pos=(old_tree.line, old_tree.col), + ) + tree.end_pos = (len(new_lines), len(new_lines[-1]) + 1) + + violations = [violation for violation in old_violations if violation.start[0] < start_line] + violations += chunk_violations + violations += [ + _shifted_violation(violation, delta) + for violation in old_violations + if violation.start[0] > end_line_old + ] + return tree, violations diff --git a/server/src/tools/index_cache.py b/server/src/tools/index_cache.py new file mode 100644 index 0000000..33e68cd --- /dev/null +++ b/server/src/tools/index_cache.py @@ -0,0 +1,128 @@ +"""Persist per-file index results across server restarts. + +Entries are keyed by path and validated by the file's size and mtime. The +whole cache is tied to a fingerprint of the code that produced it: the +indexing sources of this server, the bundled tclint sources and the versions +of all bundled libraries. Any change to them, including a tclint update or a +local patch, discards the cache instead of loading stale results. +""" + +from __future__ import annotations + +import hashlib +import logging +import os +import pathlib +import pickle +import sys +import tempfile +import threading +import zlib +from typing import Any + +LOGGER = logging.getLogger(__name__) + +# Bump when the cached data layout changes without a source change above. +CACHE_FORMAT = 1 +CACHE_FILE = "index-cache.pickle.z" + +_SRC_DIR = pathlib.Path(__file__).resolve().parent.parent +_LIBS_DIR = _SRC_DIR.parent / "libs" + +FileStat = tuple[int, int] + + +def code_fingerprint() -> str: + digest = hashlib.sha256() + digest.update(f"{CACHE_FORMAT}|{sys.version}".encode()) + sources = [ + _SRC_DIR / "lsp_tclserver.py", + *sorted((_SRC_DIR / "tools").glob("*.py")), + *sorted((_SRC_DIR / "plugins").glob("*.py")), + *sorted((_LIBS_DIR / "tclint").rglob("*.py")), + ] + for source in sources: + digest.update(source.relative_to(_SRC_DIR.parent).as_posix().encode()) + digest.update(source.read_bytes()) + for dist_info in sorted(_LIBS_DIR.glob("*.dist-info")): + digest.update(dist_info.name.encode()) + return digest.hexdigest() + + +def file_stat(path: str) -> FileStat | None: + try: + stat = os.stat(path) + except OSError: + return None + return stat.st_mtime_ns, stat.st_size + + +class IndexCache: + def __init__(self, directory: pathlib.Path | None = None, fingerprint: str = ""): + self._path = directory / CACHE_FILE if directory is not None else None + self._fingerprint = fingerprint + self._entries: dict[str, tuple[FileStat, Any]] = {} + self._used: set[str] = set() + self._dirty = False + self._lock = threading.Lock() + + @classmethod + def load(cls, directory: pathlib.Path | str | None) -> IndexCache: + """Open the cache in `directory`; without one, nothing is persisted.""" + if not directory: + return cls() + cache = cls(pathlib.Path(directory), code_fingerprint()) + try: + with open(cache._path, "rb") as file: + fingerprint, entries = pickle.loads(zlib.decompress(file.read())) + except FileNotFoundError: + return cache + except Exception as error: # A damaged cache must never stop indexing. + LOGGER.warning("Ignoring unreadable index cache %s: %s", cache._path, error) + cache._dirty = True + return cache + if fingerprint == cache._fingerprint: + cache._entries = entries + else: + cache._dirty = True + return cache + + def get(self, path: str, stat: FileStat) -> Any | None: + with self._lock: + entry = self._entries.get(path) + if entry is None or entry[0] != stat: + return None + self._used.add(path) + return entry[1] + + def put(self, path: str, stat: FileStat, data: Any) -> None: + if self._path is None: + return + with self._lock: + self._entries[path] = (stat, data) + self._used.add(path) + self._dirty = True + + def save(self) -> None: + """Write entries used in this session atomically; others are dropped.""" + if self._path is None: + return + with self._lock: + if not self._dirty and self._used == self._entries.keys(): + return + entries = {path: self._entries[path] for path in self._used if path in self._entries} + self._entries = entries + self._dirty = False + try: + self._path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=self._path.parent, delete=False) as file: + data = pickle.dumps((self._fingerprint, entries), protocol=pickle.HIGHEST_PROTOCOL) + # Pickled indexes are very repetitive; fast compression cuts ~90%. + file.write(zlib.compress(data, 1)) + os.replace(file.name, self._path) + except Exception as error: + LOGGER.warning("Could not write index cache %s: %s", self._path, error) + try: + os.unlink(file.name) + except (OSError, NameError): + pass diff --git a/server/src/tools/navigation.py b/server/src/tools/navigation.py index c90707c..fb5fd52 100644 --- a/server/src/tools/navigation.py +++ b/server/src/tools/navigation.py @@ -265,6 +265,12 @@ def build_file_symbol_index( filepath: str, uri: str, tree: Node ) -> FileSymbolIndex: occurrences: list[SymbolOccurrence] = [] + # Most occurrences repeat a few identities; sharing one object per identity + # keeps the index (and its persistent cache) small. + identities: dict[SymbolIdentity, SymbolIdentity] = {} + + def shared(identity: SymbolIdentity | None) -> SymbolIdentity | None: + return None if identity is None else identities.setdefault(identity, identity) def add_proc( node: Node, @@ -274,7 +280,7 @@ def build_file_symbol_index( is_definition: bool, declaration_range: lsp.Range | None = None, ) -> None: - identity = _proc_identity(raw_name, scope.namespace) + identity = shared(_proc_identity(raw_name, scope.namespace)) caller = None if not is_definition: caller = ( @@ -288,14 +294,14 @@ def build_file_symbol_index( fallback_identity=( None if is_definition - else _proc_fallback(raw_name, scope.namespace) + else shared(_proc_fallback(raw_name, scope.namespace)) ), range=_name_range(node, raw_name), placeholder=_basename(raw_name), is_definition=is_definition, symbol_kind=lsp.SymbolKind.Function, container_name=_container_name(identity), - caller=caller, + caller=shared(caller), declaration_range=declaration_range, ) ) @@ -309,7 +315,7 @@ def build_file_symbol_index( variable_sub: bool = False, identity: SymbolIdentity | None = None, ) -> None: - symbol_identity = identity or _variable_identity(raw_name, scope) + symbol_identity = shared(identity or _variable_identity(raw_name, scope)) occurrences.append( SymbolOccurrence( identity=symbol_identity, diff --git a/server/tests/python_tests/test_incremental_parse.py b/server/tests/python_tests/test_incremental_parse.py new file mode 100644 index 0000000..1bed913 --- /dev/null +++ b/server/tests/python_tests/test_incremental_parse.py @@ -0,0 +1,131 @@ +import sys +from pathlib import Path + +import pytest + +THIS_DIR = Path(__file__).parent +SRC_DIR = THIS_DIR.parent.parent / "src" +if str(SRC_DIR) not in sys.path: + sys.path.insert(0, str(SRC_DIR)) + +from lsp_tclserver import TclLanguageServer # noqa: E402 +from pygls.workspace.text_document import TextDocument # noqa: E402 +from tclint.lexer import TclSyntaxError # noqa: E402 +from tclint.syntax_tree import Node # noqa: E402 +from tools.incremental_parse import reparse # noqa: E402 +from tools.parser import CustomParser # noqa: E402 + +SOURCE = """\ +# header comment +set a 1; set b 2 +proc first {x} { + global mom_pos + if {$x > 0} { + MOM_output_literal "first $x" + } + return [expr {$x + 1}] +} + +proc second {} { + set list [list a b \\ + c d] + return $list +} +lappend ::handlers {second} +""" + + +def _parse(text, pos=None): + parser = CustomParser() + tree = parser.parse(text, pos=pos) + return tree, list(parser.violations) + + +def _differences(a, b, path="root"): + if type(a) is not type(b): + return f"{path}: {type(a).__name__} != {type(b).__name__}" + for key in a.__dict__.keys() | b.__dict__.keys(): + first, second = a.__dict__.get(key), b.__dict__.get(key) + if isinstance(first, Node): + difference = _differences(first, second, f"{path}.{key}") + elif isinstance(first, (list, tuple)) and first and isinstance(first[0], Node): + if len(first) != len(second): + return f"{path}.{key}: {len(first)} != {len(second)}" + difference = next( + (d for i, (x, y) in enumerate(zip(first, second)) if (d := _differences(x, y, f"{path}.{key}[{i}]"))), + None, + ) + else: + difference = None if first == second else f"{path}.{key}: {first!r} != {second!r}" + if difference: + return difference + return None + + +def _violations(violations): + return [(str(v.id), v.message, v.start, v.end) for v in violations] + + +@pytest.mark.parametrize("old, new", [ + ("MOM_output_literal \"first $x\"", "MOM_output_literal \"first $x\" extra"), + (" return $list\n", " return $list\n puts done\n"), + ("proc second {} {", "\nproc second {} {"), + ("set a 1; set b 2\n", ""), + ("# header comment\n", "# header comment\nset inserted 0\n"), + ("lappend ::handlers {second}\n", "lappend ::handlers {second}\nproc third {} {}\n"), + (" c d]", " c d e]"), + ("global mom_pos", "global mom_pos mom_out_angle_pos"), +]) +def test_incremental_tree_matches_full_parse(old, new): + edited = SOURCE.replace(old, new, 1) + assert edited != SOURCE + previous = (SOURCE, *_parse(SOURCE)) + result = reparse(*previous, edited, _parse) + assert result is not None + expected = _parse(edited) + assert _differences(result[0], expected[0]) is None + assert _violations(result[1]) == _violations(expected[1]) + + +def test_continuation_across_the_edit_forces_full_parse(): + edited = SOURCE.replace("set a 1; set b 2", "set a 1; set b 2 \\") + assert reparse(SOURCE, *_parse(SOURCE), edited, _parse) is None + + +def test_quote_closing_outside_the_edit_is_left_to_the_full_parse(): + edited = SOURCE.replace("set a 1; set b 2", 'set a "1; set b 2') + try: + result = reparse(SOURCE, *_parse(SOURCE), edited, _parse) + except TclSyntaxError: + result = None + assert result is None + + +def test_unchanged_commands_are_reused_and_never_mutated(): + tree, violations = _parse(SOURCE) + edited = SOURCE.replace("return $list", "return [lsort $list]") + new_tree, _ = reparse(SOURCE, tree, violations, edited, _parse) + assert new_tree.children[0] is tree.children[0] + # Commands after the edit are shifted copies; the old tree stays valid. + inserted = SOURCE.replace("proc first", "\n\nproc first") + shifted_tree, _ = reparse(SOURCE, tree, violations, inserted, _parse) + assert shifted_tree.children[-1] is not tree.children[-1] + assert shifted_tree.children[-1].line == tree.children[-1].line + 2 + assert tree.children[-1].line == _parse(SOURCE)[0].children[-1].line + + +def test_server_reparses_edits_incrementally(tmp_path, monkeypatch): + server = TclLanguageServer(name="incremental-test", version="1", max_workers=1) + uri = (tmp_path / "edit.tcl").as_uri() + first = server.get_tree(TextDocument(uri=uri, source=SOURCE, version=1, language_id="tcl")) + + parsed_sources = [] + parse_source = server._parse_source + monkeypatch.setattr(server, "_parse_source", lambda text, pos=None: parsed_sources.append(text) or parse_source(text, pos)) + server.clear_cache_for_uri(uri) + edited = SOURCE.replace("return $list", "return [lsort $list]") + second = server.get_tree(TextDocument(uri=uri, source=edited, version=2, language_id="tcl")) + + assert parsed_sources and all(text != edited for text in parsed_sources) + assert second.children[0] is first.children[0] + assert _differences(second, _parse(edited)[0]) is None diff --git a/server/tests/python_tests/test_index_cache.py b/server/tests/python_tests/test_index_cache.py new file mode 100644 index 0000000..78b84f4 --- /dev/null +++ b/server/tests/python_tests/test_index_cache.py @@ -0,0 +1,109 @@ +import os +import sys +from pathlib import Path + +THIS_DIR = Path(__file__).parent +SRC_DIR = THIS_DIR.parent.parent / "src" +if str(SRC_DIR) not in sys.path: + sys.path.insert(0, str(SRC_DIR)) + +import tools.index_cache as index_cache # noqa: E402 +from lsp_tclserver import TclLanguageServer # noqa: E402 +from pygls.workspace.text_document import TextDocument # noqa: E402 +from tools.index_cache import IndexCache # noqa: E402 + + +def _index_from_disk(server, path: Path) -> bool: + return server.update_poco_completion_for_file( + TextDocument(uri=path.as_uri(), language_id="tcl"), + cache_tree=False, + require_file_exists=True, + from_disk=True, + ) + + +def _warm_server(cache_dir: Path) -> TclLanguageServer: + server = TclLanguageServer(name="cache-test", version="1", max_workers=1) + server.index_cache = IndexCache.load(cache_dir) + return server + + +def _fail_build(*_args, **_kwargs): + raise AssertionError("file was parsed although it is cached") + + +def test_second_start_uses_cached_index(tmp_path: Path, monkeypatch): + source = tmp_path / "post.tcl" + source.write_text("proc cached_proc {a b} { return $a }\n", encoding="utf-8") + cache_dir = tmp_path / "storage" + + server = _warm_server(cache_dir) + assert _index_from_disk(server, source) + server.index_cache.save() + + restarted = _warm_server(cache_dir) + monkeypatch.setattr(restarted, "_build_file_index", _fail_build) + assert _index_from_disk(restarted, source) + assert "cached_proc" in restarted.custom_function_names_snapshot() + assert restarted.proc_metadata_snapshot(str(source))[0]["cached_proc"] == ["a", "b"] + + +def test_changed_file_is_parsed_again(tmp_path: Path): + source = tmp_path / "post.tcl" + source.write_text("proc old_proc {} {}\n", encoding="utf-8") + cache_dir = tmp_path / "storage" + server = _warm_server(cache_dir) + assert _index_from_disk(server, source) + server.index_cache.save() + + source.write_text("proc new_proc {} {}\n", encoding="utf-8") + stat = source.stat() + os.utime(source, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + restarted = _warm_server(cache_dir) + assert _index_from_disk(restarted, source) + names = restarted.custom_function_names_snapshot() + assert "new_proc" in names and "old_proc" not in names + + +def test_code_change_discards_the_cache(tmp_path: Path, monkeypatch): + source = tmp_path / "post.tcl" + source.write_text("proc cached_proc {} {}\n", encoding="utf-8") + cache_dir = tmp_path / "storage" + server = _warm_server(cache_dir) + assert _index_from_disk(server, source) + server.index_cache.save() + + # E.g. an updated tclint: a different fingerprint must not load old entries. + monkeypatch.setattr(index_cache, "code_fingerprint", lambda: "other tclint") + restarted = _warm_server(cache_dir) + built = [] + build = restarted._build_file_index + monkeypatch.setattr(restarted, "_build_file_index", lambda *args: built.append(args) or build(*args)) + assert _index_from_disk(restarted, source) + assert built + + +def test_damaged_cache_is_ignored(tmp_path: Path, monkeypatch): + cache_dir = tmp_path / "storage" + cache_dir.mkdir() + (cache_dir / index_cache.CACHE_FILE).write_bytes(b"not a cache") + source = tmp_path / "post.tcl" + source.write_text("proc fresh_proc {} {}\n", encoding="utf-8") + + server = _warm_server(cache_dir) + assert _index_from_disk(server, source) + server.index_cache.save() + + restarted = _warm_server(cache_dir) + monkeypatch.setattr(restarted, "_build_file_index", _fail_build) + assert _index_from_disk(restarted, source) + + +def test_open_documents_never_touch_the_cache(tmp_path: Path): + source = tmp_path / "post.tcl" + source.write_text("proc on_disk {} {}\n", encoding="utf-8") + server = _warm_server(tmp_path / "storage") + unsaved = TextDocument(uri=source.as_uri(), source="proc unsaved {} {}\n", version=3, language_id="tcl") + assert server.update_poco_completion_for_file(unsaved) + server.index_cache.save() + assert not (tmp_path / "storage" / index_cache.CACHE_FILE).exists()