feat(indexing): add persistent index cache and incremental reparse
- Pass extension storage path to the server (client/ changes) so the server can persist a workspace index. - Introduce IndexCache (server/tools/index_cache.py) and load/save it on initialization and after background indexing. Index entries are stored only when the file's stat hasn't changed while being read. - Add incremental reparse logic (server/tools/incremental_parse.py) and use a per-file _last_parse cache in the language server to reparse only the top-level Tcl commands touched by an edit, falling back to a full parse when necessary. - Use a new _FileIndex dataclass and _build_file_index helper to unify what is stored/loaded for a file; update update_poco_completion_for_file to use the persistent cache for disk-read files (from_disk/source_stat). - Keep background indexing non-blocking and persist the index at the end of the run. Add basic unit tests for incremental parse and index cache. Before: edits and background work always required full parsing of files and no persistent cross-restart index. After: some edits reuse previous ASTs and files read from disk can use a persisted index to skip re-indexing across restarts.
This commit is contained in:
Christoph Brandau
@@ -0,0 +1,131 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
THIS_DIR = Path(__file__).parent
|
||||
SRC_DIR = THIS_DIR.parent.parent / "src"
|
||||
if str(SRC_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SRC_DIR))
|
||||
|
||||
from lsp_tclserver import TclLanguageServer # noqa: E402
|
||||
from pygls.workspace.text_document import TextDocument # noqa: E402
|
||||
from tclint.lexer import TclSyntaxError # noqa: E402
|
||||
from tclint.syntax_tree import Node # noqa: E402
|
||||
from tools.incremental_parse import reparse # noqa: E402
|
||||
from tools.parser import CustomParser # noqa: E402
|
||||
|
||||
SOURCE = """\
|
||||
# header comment
|
||||
set a 1; set b 2
|
||||
proc first {x} {
|
||||
global mom_pos
|
||||
if {$x > 0} {
|
||||
MOM_output_literal "first $x"
|
||||
}
|
||||
return [expr {$x + 1}]
|
||||
}
|
||||
|
||||
proc second {} {
|
||||
set list [list a b \\
|
||||
c d]
|
||||
return $list
|
||||
}
|
||||
lappend ::handlers {second}
|
||||
"""
|
||||
|
||||
|
||||
def _parse(text, pos=None):
|
||||
parser = CustomParser()
|
||||
tree = parser.parse(text, pos=pos)
|
||||
return tree, list(parser.violations)
|
||||
|
||||
|
||||
def _differences(a, b, path="root"):
|
||||
if type(a) is not type(b):
|
||||
return f"{path}: {type(a).__name__} != {type(b).__name__}"
|
||||
for key in a.__dict__.keys() | b.__dict__.keys():
|
||||
first, second = a.__dict__.get(key), b.__dict__.get(key)
|
||||
if isinstance(first, Node):
|
||||
difference = _differences(first, second, f"{path}.{key}")
|
||||
elif isinstance(first, (list, tuple)) and first and isinstance(first[0], Node):
|
||||
if len(first) != len(second):
|
||||
return f"{path}.{key}: {len(first)} != {len(second)}"
|
||||
difference = next(
|
||||
(d for i, (x, y) in enumerate(zip(first, second)) if (d := _differences(x, y, f"{path}.{key}[{i}]"))),
|
||||
None,
|
||||
)
|
||||
else:
|
||||
difference = None if first == second else f"{path}.{key}: {first!r} != {second!r}"
|
||||
if difference:
|
||||
return difference
|
||||
return None
|
||||
|
||||
|
||||
def _violations(violations):
|
||||
return [(str(v.id), v.message, v.start, v.end) for v in violations]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("old, new", [
|
||||
("MOM_output_literal \"first $x\"", "MOM_output_literal \"first $x\" extra"),
|
||||
(" return $list\n", " return $list\n puts done\n"),
|
||||
("proc second {} {", "\nproc second {} {"),
|
||||
("set a 1; set b 2\n", ""),
|
||||
("# header comment\n", "# header comment\nset inserted 0\n"),
|
||||
("lappend ::handlers {second}\n", "lappend ::handlers {second}\nproc third {} {}\n"),
|
||||
(" c d]", " c d e]"),
|
||||
("global mom_pos", "global mom_pos mom_out_angle_pos"),
|
||||
])
|
||||
def test_incremental_tree_matches_full_parse(old, new):
|
||||
edited = SOURCE.replace(old, new, 1)
|
||||
assert edited != SOURCE
|
||||
previous = (SOURCE, *_parse(SOURCE))
|
||||
result = reparse(*previous, edited, _parse)
|
||||
assert result is not None
|
||||
expected = _parse(edited)
|
||||
assert _differences(result[0], expected[0]) is None
|
||||
assert _violations(result[1]) == _violations(expected[1])
|
||||
|
||||
|
||||
def test_continuation_across_the_edit_forces_full_parse():
|
||||
edited = SOURCE.replace("set a 1; set b 2", "set a 1; set b 2 \\")
|
||||
assert reparse(SOURCE, *_parse(SOURCE), edited, _parse) is None
|
||||
|
||||
|
||||
def test_quote_closing_outside_the_edit_is_left_to_the_full_parse():
|
||||
edited = SOURCE.replace("set a 1; set b 2", 'set a "1; set b 2')
|
||||
try:
|
||||
result = reparse(SOURCE, *_parse(SOURCE), edited, _parse)
|
||||
except TclSyntaxError:
|
||||
result = None
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_unchanged_commands_are_reused_and_never_mutated():
|
||||
tree, violations = _parse(SOURCE)
|
||||
edited = SOURCE.replace("return $list", "return [lsort $list]")
|
||||
new_tree, _ = reparse(SOURCE, tree, violations, edited, _parse)
|
||||
assert new_tree.children[0] is tree.children[0]
|
||||
# Commands after the edit are shifted copies; the old tree stays valid.
|
||||
inserted = SOURCE.replace("proc first", "\n\nproc first")
|
||||
shifted_tree, _ = reparse(SOURCE, tree, violations, inserted, _parse)
|
||||
assert shifted_tree.children[-1] is not tree.children[-1]
|
||||
assert shifted_tree.children[-1].line == tree.children[-1].line + 2
|
||||
assert tree.children[-1].line == _parse(SOURCE)[0].children[-1].line
|
||||
|
||||
|
||||
def test_server_reparses_edits_incrementally(tmp_path, monkeypatch):
|
||||
server = TclLanguageServer(name="incremental-test", version="1", max_workers=1)
|
||||
uri = (tmp_path / "edit.tcl").as_uri()
|
||||
first = server.get_tree(TextDocument(uri=uri, source=SOURCE, version=1, language_id="tcl"))
|
||||
|
||||
parsed_sources = []
|
||||
parse_source = server._parse_source
|
||||
monkeypatch.setattr(server, "_parse_source", lambda text, pos=None: parsed_sources.append(text) or parse_source(text, pos))
|
||||
server.clear_cache_for_uri(uri)
|
||||
edited = SOURCE.replace("return $list", "return [lsort $list]")
|
||||
second = server.get_tree(TextDocument(uri=uri, source=edited, version=2, language_id="tcl"))
|
||||
|
||||
assert parsed_sources and all(text != edited for text in parsed_sources)
|
||||
assert second.children[0] is first.children[0]
|
||||
assert _differences(second, _parse(edited)[0]) is None
|
||||
@@ -0,0 +1,109 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
THIS_DIR = Path(__file__).parent
|
||||
SRC_DIR = THIS_DIR.parent.parent / "src"
|
||||
if str(SRC_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SRC_DIR))
|
||||
|
||||
import tools.index_cache as index_cache # noqa: E402
|
||||
from lsp_tclserver import TclLanguageServer # noqa: E402
|
||||
from pygls.workspace.text_document import TextDocument # noqa: E402
|
||||
from tools.index_cache import IndexCache # noqa: E402
|
||||
|
||||
|
||||
def _index_from_disk(server, path: Path) -> bool:
|
||||
return server.update_poco_completion_for_file(
|
||||
TextDocument(uri=path.as_uri(), language_id="tcl"),
|
||||
cache_tree=False,
|
||||
require_file_exists=True,
|
||||
from_disk=True,
|
||||
)
|
||||
|
||||
|
||||
def _warm_server(cache_dir: Path) -> TclLanguageServer:
|
||||
server = TclLanguageServer(name="cache-test", version="1", max_workers=1)
|
||||
server.index_cache = IndexCache.load(cache_dir)
|
||||
return server
|
||||
|
||||
|
||||
def _fail_build(*_args, **_kwargs):
|
||||
raise AssertionError("file was parsed although it is cached")
|
||||
|
||||
|
||||
def test_second_start_uses_cached_index(tmp_path: Path, monkeypatch):
|
||||
source = tmp_path / "post.tcl"
|
||||
source.write_text("proc cached_proc {a b} { return $a }\n", encoding="utf-8")
|
||||
cache_dir = tmp_path / "storage"
|
||||
|
||||
server = _warm_server(cache_dir)
|
||||
assert _index_from_disk(server, source)
|
||||
server.index_cache.save()
|
||||
|
||||
restarted = _warm_server(cache_dir)
|
||||
monkeypatch.setattr(restarted, "_build_file_index", _fail_build)
|
||||
assert _index_from_disk(restarted, source)
|
||||
assert "cached_proc" in restarted.custom_function_names_snapshot()
|
||||
assert restarted.proc_metadata_snapshot(str(source))[0]["cached_proc"] == ["a", "b"]
|
||||
|
||||
|
||||
def test_changed_file_is_parsed_again(tmp_path: Path):
|
||||
source = tmp_path / "post.tcl"
|
||||
source.write_text("proc old_proc {} {}\n", encoding="utf-8")
|
||||
cache_dir = tmp_path / "storage"
|
||||
server = _warm_server(cache_dir)
|
||||
assert _index_from_disk(server, source)
|
||||
server.index_cache.save()
|
||||
|
||||
source.write_text("proc new_proc {} {}\n", encoding="utf-8")
|
||||
stat = source.stat()
|
||||
os.utime(source, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000))
|
||||
restarted = _warm_server(cache_dir)
|
||||
assert _index_from_disk(restarted, source)
|
||||
names = restarted.custom_function_names_snapshot()
|
||||
assert "new_proc" in names and "old_proc" not in names
|
||||
|
||||
|
||||
def test_code_change_discards_the_cache(tmp_path: Path, monkeypatch):
|
||||
source = tmp_path / "post.tcl"
|
||||
source.write_text("proc cached_proc {} {}\n", encoding="utf-8")
|
||||
cache_dir = tmp_path / "storage"
|
||||
server = _warm_server(cache_dir)
|
||||
assert _index_from_disk(server, source)
|
||||
server.index_cache.save()
|
||||
|
||||
# E.g. an updated tclint: a different fingerprint must not load old entries.
|
||||
monkeypatch.setattr(index_cache, "code_fingerprint", lambda: "other tclint")
|
||||
restarted = _warm_server(cache_dir)
|
||||
built = []
|
||||
build = restarted._build_file_index
|
||||
monkeypatch.setattr(restarted, "_build_file_index", lambda *args: built.append(args) or build(*args))
|
||||
assert _index_from_disk(restarted, source)
|
||||
assert built
|
||||
|
||||
|
||||
def test_damaged_cache_is_ignored(tmp_path: Path, monkeypatch):
|
||||
cache_dir = tmp_path / "storage"
|
||||
cache_dir.mkdir()
|
||||
(cache_dir / index_cache.CACHE_FILE).write_bytes(b"not a cache")
|
||||
source = tmp_path / "post.tcl"
|
||||
source.write_text("proc fresh_proc {} {}\n", encoding="utf-8")
|
||||
|
||||
server = _warm_server(cache_dir)
|
||||
assert _index_from_disk(server, source)
|
||||
server.index_cache.save()
|
||||
|
||||
restarted = _warm_server(cache_dir)
|
||||
monkeypatch.setattr(restarted, "_build_file_index", _fail_build)
|
||||
assert _index_from_disk(restarted, source)
|
||||
|
||||
|
||||
def test_open_documents_never_touch_the_cache(tmp_path: Path):
|
||||
source = tmp_path / "post.tcl"
|
||||
source.write_text("proc on_disk {} {}\n", encoding="utf-8")
|
||||
server = _warm_server(tmp_path / "storage")
|
||||
unsaved = TextDocument(uri=source.as_uri(), source="proc unsaved {} {}\n", version=3, language_id="tcl")
|
||||
assert server.update_poco_completion_for_file(unsaved)
|
||||
server.index_cache.save()
|
||||
assert not (tmp_path / "storage" / index_cache.CACHE_FILE).exists()
|
||||
Reference in New Issue
Block a user