Christoph/nx_post_support
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects 1 Releases 39 Wiki Activity

feat(indexing): add persistent index cache and incremental reparse

Browse Source 
- Pass extension storage path to the server (client/ changes) so the
  server can persist a workspace index.
- Introduce IndexCache (server/tools/index_cache.py) and load/save it on
  initialization and after background indexing. Index entries are stored
  only when the file's stat hasn't changed while being read.
- Add incremental reparse logic (server/tools/incremental_parse.py) and
  use a per-file _last_parse cache in the language server to reparse only
  the top-level Tcl commands touched by an edit, falling back to a full
  parse when necessary.
- Use a new _FileIndex dataclass and _build_file_index helper to unify
  what is stored/loaded for a file; update update_poco_completion_for_file
  to use the persistent cache for disk-read files (from_disk/source_stat).
- Keep background indexing non-blocking and persist the index at the
  end of the run. Add basic unit tests for incremental parse and index cache.

Before: edits and background work always required full parsing of files
and no persistent cross-restart index. After: some edits reuse previous
ASTs and files read from disk can use a persisted index to skip
re-indexing across restarts.
This commit is contained in:
Christoph Brandau
2026-09-23 13:20:09 +02:00
parent b2e6e9d250
commit 01e8670cc1
10 changed files with 674 additions and 42 deletions
Download Patch File Download Diff File Expand all files Collapse all files
server/src/lsp_tclserver.py
+100 -32
View File
@@ -3,6 +3,7 @@ import os
import pathlib
import threading
import xml.etree.ElementTree as ET
from dataclasses import dataclass
from typing import List, Optional, Tuple
import lsprotocol.types as lsp
@@ -13,12 +14,13 @@ from pygls.workspace.text_document import TextDocument
from tclint.format import FormatterOpts
from tclint.lexer import TclSyntaxError
from tclint.violations import Violation
from tools import checks, parser
from tools import checks, incremental_parse, parser
from tools.completion_items import CompletionCollector
from tools.tcloo_symbols import class_completion_items
from tools.tcloo_completion import indexed_classes
from tools.file_sourcing import get_all_psc_files, psc_script_files
from tools.formatter import NxFormatter as Formatter
from tools.index_cache import FileStat, IndexCache, file_stat
from tools.inlay_hint import InlayHintSignature, build_custom_inlay_signatures
from tools.navigation import (
FileSymbolIndex,
@@ -33,6 +35,18 @@ DIAGNOSTIC_SOURCE = "nx-post-support"
LOGGER = logging.getLogger(__name__)
@dataclass(frozen=True)
class _FileIndex:
"""Everything indexed for one file; also the persistent cache entry."""
completion_items: list[lsp.CompletionItem]
proc_signatures: dict[str, list[str]]
proc_docs: dict[str, str]
classes: dict
navigation_index: FileSymbolIndex
variable_index: tuple[set[str], dict[str, set[str]], list[ProcRange]]
class TclLanguageServer(LanguageServer):
def __init__(self, *args, **kwargs):
super().__init__(*args, **kwargs)
@@ -60,6 +74,10 @@ class TclLanguageServer(LanguageServer):
self._line_cache: dict[tuple[str, int | None], tuple[str, ...]] = {}
self._parser_lock = threading.RLock()
self._thread_parsers = threading.local()
# uri -> (normalized source, tree, violations) of the last successful
# parse; survives version changes so edits can be reparsed partially.
self._last_parse: dict[str, tuple[str, object, list]] = {}
self.index_cache = IndexCache()
self._index_lock = threading.RLock()
self._index_tokens: dict[str, int] = {}
self._index_versions: dict[str, int | None] = {}
@@ -87,11 +105,32 @@ class TclLanguageServer(LanguageServer):
self._analysis_tokens: dict[str, int] = {}
self._next_analysis_token = 0
def _parse_source(self, source: str):
def _parse_source(self, source: str, pos=None):
self.parser.violations = []
tree = self.parser.parse(source)
tree = self.parser.parse(source, pos=pos)
return tree, list(self.parser.violations)
def _parse_document(self, document: TextDocument):
"""Parse a document version, reusing unchanged parts of the last one.
Callers hold the parser lock.
"""
source = incremental_parse.normalize_newlines(document.source)
previous = self._last_parse.get(document.uri)
result = None
if previous is not None:
try:
result = incremental_parse.reparse(
*previous, source, self._parse_source
)
except TclSyntaxError:
# E.g. a quote opened in the edit closes further down.
result = None
if result is None:
result = self._parse_source(source)
self._last_parse[document.uri] = (source, *result)
return result
def parse_source(self, source: str):
"""Parse without retaining an AST, on a parser owned by this thread.
@@ -111,7 +150,7 @@ class TclLanguageServer(LanguageServer):
cached = self._ast_cache.get(key)
if cached is not None:
return cached[0]
tree, violations = self._parse_source(document.source)
tree, violations = self._parse_document(document)
self._ast_cache[key] = (tree, violations)
return tree
@@ -121,7 +160,7 @@ class TclLanguageServer(LanguageServer):
cached = self._ast_cache.get(key)
if cached is not None:
return cached
tree, violations = self._parse_source(document.source)
tree, violations = self._parse_document(document)
self._ast_cache[key] = (tree, violations)
return tree, violations
@@ -250,7 +289,9 @@ class TclLanguageServer(LanguageServer):
report(f"PSC script not found: {path}")
continue
try:
source_stat = None
if document is None:
source_stat = file_stat(path_string)
data = path.read_bytes()
try:
source = data.decode("utf-8-sig")
@@ -258,7 +299,7 @@ class TclLanguageServer(LanguageServer):
# Older Windows NX layers use the ANSI code page.
source = data.decode("cp1252")
document = TextDocument(uri=uri, source=source, language_id="tcl")
if not self.update_poco_completion_for_file(document, cache_tree=False):
if not self.update_poco_completion_for_file(document, cache_tree=False, source_stat=source_stat):
report(f"Could not index PSC script: {path}")
except (OSError, UnicodeError) as error:
report(f"Could not read PSC script {path}: {error}")
@@ -632,6 +673,33 @@ class TclLanguageServer(LanguageServer):
if self._is_same_or_child(cached_path, target):
self._ast_cache.pop(key, None)
self._line_cache.pop(key, None)
for uri in list(self._last_parse):
try:
parsed_path = pathlib.Path(uris.to_fs_path(uri))
except (TypeError, ValueError):
continue
if self._is_same_or_child(parsed_path, target):
del self._last_parse[uri]
def _build_file_index(
self, document: TextDocument, filepath: str, cache_tree: bool
) -> "_FileIndex":
tree = (
self.get_tree(document)
if cache_tree
else self.parse_source(document.source)
)
collector = CompletionCollector()
tree.accept(collector, recurse=True)
collector.custom_functions.extend(class_completion_items(tree))
return _FileIndex(
completion_items=list(collector.custom_functions),
proc_signatures=dict(collector.proc_signatures),
proc_docs=build_proc_docs(tree, document.source),
classes=indexed_classes(tree, document.uri, document.source),
navigation_index=build_file_symbol_index(filepath, document.uri, tree),
variable_index=build_variable_index(document.source, tree),
)
def update_poco_completion_for_file(
self,
@@ -639,32 +707,32 @@ class TclLanguageServer(LanguageServer):
*,
cache_tree: bool = True,
require_file_exists: bool = False,
from_disk: bool = False,
source_stat: FileStat | None = None,
):
"""Update poco_completion for a specific file when it changes"""
"""Update poco_completion for a specific file when it changes.
`from_disk` marks documents that read their source from disk lazily;
`source_stat` is the file's stat taken before a caller read it. Such
results are served from and stored in the persistent index cache.
"""
filepath = str(pathlib.Path(uris.to_fs_path(document.uri)))
token = self._begin_index_update(filepath, document.version)
if token is None:
return False
collector = CompletionCollector()
try:
tree = (
self.get_tree(document)
if cache_tree
else self.parse_source(document.source)
)
tree.accept(collector, recurse=True)
collector.custom_functions.extend(class_completion_items(tree))
classes = indexed_classes(tree, document.uri, document.source)
docs = build_proc_docs(tree, document.source)
navigation_index = build_file_symbol_index(
filepath, document.uri, tree
)
variable_index = build_variable_index(document.source, tree)
except Exception as e:
LOGGER.debug("Error parsing %s: %s", filepath, e)
self._discard_index_update(filepath, token)
return False
stat = source_stat or (file_stat(filepath) if from_disk else None)
index = self.index_cache.get(filepath, stat) if stat is not None else None
if index is None:
try:
index = self._build_file_index(document, filepath, cache_tree)
except Exception as e:
LOGGER.debug("Error parsing %s: %s", filepath, e)
self._discard_index_update(filepath, token)
return False
# Only cache results whose file did not change while being read.
if stat is not None and file_stat(filepath) == stat:
self.index_cache.put(filepath, stat, index)
if require_file_exists and not pathlib.Path(filepath).is_file():
self._discard_index_update(filepath, token)
@@ -673,12 +741,12 @@ class TclLanguageServer(LanguageServer):
with self._index_lock:
if self._index_tokens.get(filepath) != token:
return False
self.poco_completion[filepath] = list(collector.custom_functions)
self.proc_signatures[filepath] = dict(collector.proc_signatures)
self.proc_docs[filepath] = docs
self.class_indexes[filepath] = classes
self.navigation_indexes[filepath] = navigation_index
self.variable_indexes[filepath] = (document.version, variable_index)
self.poco_completion[filepath] = list(index.completion_items)
self.proc_signatures[filepath] = dict(index.proc_signatures)
self.proc_docs[filepath] = index.proc_docs
self.class_indexes[filepath] = index.classes
self.navigation_indexes[filepath] = index.navigation_index
self.variable_indexes[filepath] = (document.version, index.variable_index)
self._committed_index_versions[filepath] = document.version
self._invalidate_workspace_caches_locked()
return True