feat(indexing): add persistent index cache and incremental reparse
- Pass extension storage path to the server (client/ changes) so the server can persist a workspace index. - Introduce IndexCache (server/tools/index_cache.py) and load/save it on initialization and after background indexing. Index entries are stored only when the file's stat hasn't changed while being read. - Add incremental reparse logic (server/tools/incremental_parse.py) and use a per-file _last_parse cache in the language server to reparse only the top-level Tcl commands touched by an edit, falling back to a full parse when necessary. - Use a new _FileIndex dataclass and _build_file_index helper to unify what is stored/loaded for a file; update update_poco_completion_for_file to use the persistent cache for disk-read files (from_disk/source_stat). - Keep background indexing non-blocking and persist the index at the end of the run. Add basic unit tests for incremental parse and index cache. Before: edits and background work always required full parsing of files and no persistent cross-restart index. After: some edits reuse previous ASTs and files read from disk can use a persisted index to skip re-indexing across restarts.
This commit is contained in:
Christoph Brandau
@@ -1,5 +1,10 @@
|
||||
## Unreleased
|
||||
|
||||
- Reparse only the top-level TCL commands touched by an edit instead of the whole file
|
||||
- Cache the workspace index in the extension storage so restarts skip reparsing unchanged files; the cache is discarded automatically when the server or bundled tclint changes
|
||||
- Speed up TclOO completion, signature help, inlay hints, and Go to Definition by reusing the cached syntax tree and skipping files without classes
|
||||
- Speed up references, document highlights, and call hierarchy with cached definition lookups
|
||||
- Keep background indexing from blocking requests that need a fresh syntax tree
|
||||
- Add Go to Definition for TclOO classes, constructors, and resolved methods, including PSC library definitions
|
||||
- Index PSC layer scripts (including external paths and legacy Windows encoding) and share TclOO class metadata across files for completion, signatures, inlay hints, and highlighting
|
||||
- Add document-local TclOO method completion for `new`/`create` instances, `my`, and statically inferred return chains
|
||||
|
||||
@@ -22,7 +22,12 @@ import {
|
||||
import { getLSClientTraceLevel, getProjectRoot } from "./utilities"
|
||||
import { isVirtualWorkspace } from "./vscodeapi"
|
||||
|
||||
export type IInitOptions = { settings: ISettings[]; globalSettings: ISettings }
|
||||
export type IInitOptions = {
|
||||
settings: ISettings[]
|
||||
globalSettings: ISettings
|
||||
// Folder for the server's persistent index cache; omitted without a workspace.
|
||||
indexCachePath?: string
|
||||
}
|
||||
|
||||
let _disposables: Disposable[] = []
|
||||
|
||||
@@ -114,7 +119,8 @@ export async function restartServer(
|
||||
serverId: string,
|
||||
serverName: string,
|
||||
outputChannel: LogOutputChannel,
|
||||
lsClient?: LanguageClient
|
||||
lsClient?: LanguageClient,
|
||||
indexCachePath?: string
|
||||
): Promise<LanguageClient | undefined> {
|
||||
if (lsClient) {
|
||||
traceInfo(`Server: Stop requested`)
|
||||
@@ -132,7 +138,8 @@ export async function restartServer(
|
||||
outputChannel,
|
||||
{
|
||||
settings: await getExtensionSettings(serverId, true),
|
||||
globalSettings: await getGlobalSettings(serverId, false)
|
||||
globalSettings: await getGlobalSettings(serverId, false),
|
||||
indexCachePath
|
||||
}
|
||||
)
|
||||
traceInfo(`Server: Start requested.`)
|
||||
|
||||
+14
-2
@@ -73,7 +73,13 @@ export async function activate(context: vscode.ExtensionContext) {
|
||||
traceVerbose(
|
||||
`Using interpreter from ${serverInfo.module}.interpreter: ${interpreter.join(" ")}`
|
||||
)
|
||||
client = await restartServer(serverId, serverName, outputChannel, client)
|
||||
client = await restartServer(
|
||||
serverId,
|
||||
serverName,
|
||||
outputChannel,
|
||||
client,
|
||||
context.storageUri?.fsPath
|
||||
)
|
||||
}
|
||||
return
|
||||
}
|
||||
@@ -83,7 +89,13 @@ export async function activate(context: vscode.ExtensionContext) {
|
||||
traceVerbose(
|
||||
`Using interpreter from Python extension: ${interpreterDetails.path.join(" ")}`
|
||||
)
|
||||
client = await restartServer(serverId, serverName, outputChannel, client)
|
||||
client = await restartServer(
|
||||
serverId,
|
||||
serverName,
|
||||
outputChannel,
|
||||
client,
|
||||
context.storageUri?.fsPath
|
||||
)
|
||||
return
|
||||
}
|
||||
|
||||
|
||||
@@ -11,6 +11,7 @@ import pathlib
|
||||
import re
|
||||
import sys
|
||||
import threading
|
||||
import time
|
||||
from collections import ChainMap
|
||||
from functools import reduce
|
||||
from typing import Any, Optional
|
||||
@@ -52,6 +53,7 @@ from tools.completion_items import (
|
||||
ranked_completion_items,
|
||||
)
|
||||
from tools.folding_ranges import build_folding_ranges
|
||||
from tools.index_cache import IndexCache
|
||||
from tools.inlay_hint import (
|
||||
InlayHintGenerator,
|
||||
build_builtin_inlay_signatures,
|
||||
@@ -89,6 +91,8 @@ from tools.tcl_command_completion import (
|
||||
|
||||
WORKSPACE_SETTINGS = {}
|
||||
GLOBAL_SETTINGS = {}
|
||||
# Extension storage folder of the workspace; without it nothing is persisted.
|
||||
INDEX_CACHE_PATH: dict[str, str | None] = {}
|
||||
|
||||
|
||||
MAX_WORKERS = 5
|
||||
@@ -175,6 +179,7 @@ def _index_tcl_file_from_disk(uri: str) -> None:
|
||||
document,
|
||||
cache_tree=False,
|
||||
require_file_exists=True,
|
||||
from_disk=True,
|
||||
)
|
||||
except (OSError, UnicodeError) as error:
|
||||
log_warning(f"Could not re-index {path}: {error}")
|
||||
@@ -861,6 +866,7 @@ def initialize(params: lsp.InitializeParams) -> lsp.InitializeResult:
|
||||
log_to_output(f"sys.path used to run Server:\r\n {paths}")
|
||||
|
||||
GLOBAL_SETTINGS.update(**params.initialization_options.get("globalSettings", {}))
|
||||
INDEX_CACHE_PATH["path"] = params.initialization_options.get("indexCachePath")
|
||||
|
||||
settings = params.initialization_options["settings"]
|
||||
_update_workspace_settings(settings)
|
||||
@@ -899,6 +905,8 @@ def initialized(_params: lsp.InitializedParams):
|
||||
log_to_output("Background indexing skipped: no workspace folder is open.")
|
||||
return
|
||||
log_to_output("Background indexing started...")
|
||||
started = time.perf_counter()
|
||||
LSP_SERVER.index_cache = IndexCache.load(INDEX_CACHE_PATH.get("path"))
|
||||
root_path = pathlib.Path(root)
|
||||
skipped_directories = {
|
||||
".git",
|
||||
@@ -916,11 +924,13 @@ def initialized(_params: lsp.InitializedParams):
|
||||
document,
|
||||
cache_tree=False,
|
||||
require_file_exists=True,
|
||||
from_disk=True,
|
||||
)
|
||||
except Exception as error:
|
||||
log_to_output(f"Fehler beim Parsen von {filepath}: {error}")
|
||||
_refresh_psc_index()
|
||||
log_to_output("Background indexing completed.")
|
||||
LSP_SERVER.index_cache.save()
|
||||
log_to_output(f"Background indexing completed in {time.perf_counter() - started:.1f}s.")
|
||||
except Exception as e:
|
||||
log_to_output(f"Background indexing failed: {e}")
|
||||
|
||||
|
||||
+100
-32
@@ -3,6 +3,7 @@ import os
|
||||
import pathlib
|
||||
import threading
|
||||
import xml.etree.ElementTree as ET
|
||||
from dataclasses import dataclass
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import lsprotocol.types as lsp
|
||||
@@ -13,12 +14,13 @@ from pygls.workspace.text_document import TextDocument
|
||||
from tclint.format import FormatterOpts
|
||||
from tclint.lexer import TclSyntaxError
|
||||
from tclint.violations import Violation
|
||||
from tools import checks, parser
|
||||
from tools import checks, incremental_parse, parser
|
||||
from tools.completion_items import CompletionCollector
|
||||
from tools.tcloo_symbols import class_completion_items
|
||||
from tools.tcloo_completion import indexed_classes
|
||||
from tools.file_sourcing import get_all_psc_files, psc_script_files
|
||||
from tools.formatter import NxFormatter as Formatter
|
||||
from tools.index_cache import FileStat, IndexCache, file_stat
|
||||
from tools.inlay_hint import InlayHintSignature, build_custom_inlay_signatures
|
||||
from tools.navigation import (
|
||||
FileSymbolIndex,
|
||||
@@ -33,6 +35,18 @@ DIAGNOSTIC_SOURCE = "nx-post-support"
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class _FileIndex:
|
||||
"""Everything indexed for one file; also the persistent cache entry."""
|
||||
|
||||
completion_items: list[lsp.CompletionItem]
|
||||
proc_signatures: dict[str, list[str]]
|
||||
proc_docs: dict[str, str]
|
||||
classes: dict
|
||||
navigation_index: FileSymbolIndex
|
||||
variable_index: tuple[set[str], dict[str, set[str]], list[ProcRange]]
|
||||
|
||||
|
||||
class TclLanguageServer(LanguageServer):
|
||||
def __init__(self, *args, **kwargs):
|
||||
super().__init__(*args, **kwargs)
|
||||
@@ -60,6 +74,10 @@ class TclLanguageServer(LanguageServer):
|
||||
self._line_cache: dict[tuple[str, int | None], tuple[str, ...]] = {}
|
||||
self._parser_lock = threading.RLock()
|
||||
self._thread_parsers = threading.local()
|
||||
# uri -> (normalized source, tree, violations) of the last successful
|
||||
# parse; survives version changes so edits can be reparsed partially.
|
||||
self._last_parse: dict[str, tuple[str, object, list]] = {}
|
||||
self.index_cache = IndexCache()
|
||||
self._index_lock = threading.RLock()
|
||||
self._index_tokens: dict[str, int] = {}
|
||||
self._index_versions: dict[str, int | None] = {}
|
||||
@@ -87,11 +105,32 @@ class TclLanguageServer(LanguageServer):
|
||||
self._analysis_tokens: dict[str, int] = {}
|
||||
self._next_analysis_token = 0
|
||||
|
||||
def _parse_source(self, source: str):
|
||||
def _parse_source(self, source: str, pos=None):
|
||||
self.parser.violations = []
|
||||
tree = self.parser.parse(source)
|
||||
tree = self.parser.parse(source, pos=pos)
|
||||
return tree, list(self.parser.violations)
|
||||
|
||||
def _parse_document(self, document: TextDocument):
|
||||
"""Parse a document version, reusing unchanged parts of the last one.
|
||||
|
||||
Callers hold the parser lock.
|
||||
"""
|
||||
source = incremental_parse.normalize_newlines(document.source)
|
||||
previous = self._last_parse.get(document.uri)
|
||||
result = None
|
||||
if previous is not None:
|
||||
try:
|
||||
result = incremental_parse.reparse(
|
||||
*previous, source, self._parse_source
|
||||
)
|
||||
except TclSyntaxError:
|
||||
# E.g. a quote opened in the edit closes further down.
|
||||
result = None
|
||||
if result is None:
|
||||
result = self._parse_source(source)
|
||||
self._last_parse[document.uri] = (source, *result)
|
||||
return result
|
||||
|
||||
def parse_source(self, source: str):
|
||||
"""Parse without retaining an AST, on a parser owned by this thread.
|
||||
|
||||
@@ -111,7 +150,7 @@ class TclLanguageServer(LanguageServer):
|
||||
cached = self._ast_cache.get(key)
|
||||
if cached is not None:
|
||||
return cached[0]
|
||||
tree, violations = self._parse_source(document.source)
|
||||
tree, violations = self._parse_document(document)
|
||||
self._ast_cache[key] = (tree, violations)
|
||||
return tree
|
||||
|
||||
@@ -121,7 +160,7 @@ class TclLanguageServer(LanguageServer):
|
||||
cached = self._ast_cache.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
tree, violations = self._parse_source(document.source)
|
||||
tree, violations = self._parse_document(document)
|
||||
self._ast_cache[key] = (tree, violations)
|
||||
return tree, violations
|
||||
|
||||
@@ -250,7 +289,9 @@ class TclLanguageServer(LanguageServer):
|
||||
report(f"PSC script not found: {path}")
|
||||
continue
|
||||
try:
|
||||
source_stat = None
|
||||
if document is None:
|
||||
source_stat = file_stat(path_string)
|
||||
data = path.read_bytes()
|
||||
try:
|
||||
source = data.decode("utf-8-sig")
|
||||
@@ -258,7 +299,7 @@ class TclLanguageServer(LanguageServer):
|
||||
# Older Windows NX layers use the ANSI code page.
|
||||
source = data.decode("cp1252")
|
||||
document = TextDocument(uri=uri, source=source, language_id="tcl")
|
||||
if not self.update_poco_completion_for_file(document, cache_tree=False):
|
||||
if not self.update_poco_completion_for_file(document, cache_tree=False, source_stat=source_stat):
|
||||
report(f"Could not index PSC script: {path}")
|
||||
except (OSError, UnicodeError) as error:
|
||||
report(f"Could not read PSC script {path}: {error}")
|
||||
@@ -632,6 +673,33 @@ class TclLanguageServer(LanguageServer):
|
||||
if self._is_same_or_child(cached_path, target):
|
||||
self._ast_cache.pop(key, None)
|
||||
self._line_cache.pop(key, None)
|
||||
for uri in list(self._last_parse):
|
||||
try:
|
||||
parsed_path = pathlib.Path(uris.to_fs_path(uri))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if self._is_same_or_child(parsed_path, target):
|
||||
del self._last_parse[uri]
|
||||
|
||||
def _build_file_index(
|
||||
self, document: TextDocument, filepath: str, cache_tree: bool
|
||||
) -> "_FileIndex":
|
||||
tree = (
|
||||
self.get_tree(document)
|
||||
if cache_tree
|
||||
else self.parse_source(document.source)
|
||||
)
|
||||
collector = CompletionCollector()
|
||||
tree.accept(collector, recurse=True)
|
||||
collector.custom_functions.extend(class_completion_items(tree))
|
||||
return _FileIndex(
|
||||
completion_items=list(collector.custom_functions),
|
||||
proc_signatures=dict(collector.proc_signatures),
|
||||
proc_docs=build_proc_docs(tree, document.source),
|
||||
classes=indexed_classes(tree, document.uri, document.source),
|
||||
navigation_index=build_file_symbol_index(filepath, document.uri, tree),
|
||||
variable_index=build_variable_index(document.source, tree),
|
||||
)
|
||||
|
||||
def update_poco_completion_for_file(
|
||||
self,
|
||||
@@ -639,32 +707,32 @@ class TclLanguageServer(LanguageServer):
|
||||
*,
|
||||
cache_tree: bool = True,
|
||||
require_file_exists: bool = False,
|
||||
from_disk: bool = False,
|
||||
source_stat: FileStat | None = None,
|
||||
):
|
||||
"""Update poco_completion for a specific file when it changes"""
|
||||
"""Update poco_completion for a specific file when it changes.
|
||||
|
||||
`from_disk` marks documents that read their source from disk lazily;
|
||||
`source_stat` is the file's stat taken before a caller read it. Such
|
||||
results are served from and stored in the persistent index cache.
|
||||
"""
|
||||
filepath = str(pathlib.Path(uris.to_fs_path(document.uri)))
|
||||
token = self._begin_index_update(filepath, document.version)
|
||||
if token is None:
|
||||
return False
|
||||
|
||||
collector = CompletionCollector()
|
||||
try:
|
||||
tree = (
|
||||
self.get_tree(document)
|
||||
if cache_tree
|
||||
else self.parse_source(document.source)
|
||||
)
|
||||
tree.accept(collector, recurse=True)
|
||||
collector.custom_functions.extend(class_completion_items(tree))
|
||||
classes = indexed_classes(tree, document.uri, document.source)
|
||||
docs = build_proc_docs(tree, document.source)
|
||||
navigation_index = build_file_symbol_index(
|
||||
filepath, document.uri, tree
|
||||
)
|
||||
variable_index = build_variable_index(document.source, tree)
|
||||
except Exception as e:
|
||||
LOGGER.debug("Error parsing %s: %s", filepath, e)
|
||||
self._discard_index_update(filepath, token)
|
||||
return False
|
||||
stat = source_stat or (file_stat(filepath) if from_disk else None)
|
||||
index = self.index_cache.get(filepath, stat) if stat is not None else None
|
||||
if index is None:
|
||||
try:
|
||||
index = self._build_file_index(document, filepath, cache_tree)
|
||||
except Exception as e:
|
||||
LOGGER.debug("Error parsing %s: %s", filepath, e)
|
||||
self._discard_index_update(filepath, token)
|
||||
return False
|
||||
# Only cache results whose file did not change while being read.
|
||||
if stat is not None and file_stat(filepath) == stat:
|
||||
self.index_cache.put(filepath, stat, index)
|
||||
|
||||
if require_file_exists and not pathlib.Path(filepath).is_file():
|
||||
self._discard_index_update(filepath, token)
|
||||
@@ -673,12 +741,12 @@ class TclLanguageServer(LanguageServer):
|
||||
with self._index_lock:
|
||||
if self._index_tokens.get(filepath) != token:
|
||||
return False
|
||||
self.poco_completion[filepath] = list(collector.custom_functions)
|
||||
self.proc_signatures[filepath] = dict(collector.proc_signatures)
|
||||
self.proc_docs[filepath] = docs
|
||||
self.class_indexes[filepath] = classes
|
||||
self.navigation_indexes[filepath] = navigation_index
|
||||
self.variable_indexes[filepath] = (document.version, variable_index)
|
||||
self.poco_completion[filepath] = list(index.completion_items)
|
||||
self.proc_signatures[filepath] = dict(index.proc_signatures)
|
||||
self.proc_docs[filepath] = index.proc_docs
|
||||
self.class_indexes[filepath] = index.classes
|
||||
self.navigation_indexes[filepath] = index.navigation_index
|
||||
self.variable_indexes[filepath] = (document.version, index.variable_index)
|
||||
self._committed_index_versions[filepath] = document.version
|
||||
self._invalidate_workspace_caches_locked()
|
||||
return True
|
||||
|
||||
@@ -0,0 +1,156 @@
|
||||
"""Reparse only the top-level commands touched by an edit.
|
||||
|
||||
Tcl top-level commands are independent once the previous command ended on its
|
||||
own line: the parser keeps no state between them. An edit is therefore
|
||||
reparsed from the first to the last top-level command it touches, commands
|
||||
before it are reused as-is and commands after it are reused with their line
|
||||
numbers shifted. Whenever that assumption could break, the caller falls back
|
||||
to a full parse.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
from collections.abc import Callable
|
||||
|
||||
from tclint.syntax_tree import Node, Script
|
||||
from tclint.violations import Violation
|
||||
|
||||
ParseChunk = Callable[[str, tuple[int, int]], tuple[Script, list[Violation]]]
|
||||
|
||||
|
||||
def normalize_newlines(source: str) -> str:
|
||||
"""Match the universal newline handling of the tclint parser."""
|
||||
return source.replace("\r\n", "\n").replace("\r", "\n")
|
||||
|
||||
|
||||
def _shifted_tree(node: Node, delta: int) -> Node:
|
||||
"""Copy a subtree with its line numbers moved by `delta`.
|
||||
|
||||
Cached trees may still be read by other requests, so nodes are never
|
||||
mutated. Attributes such as `Command.routine` alias entries of `children`,
|
||||
so every reference is remapped to the same copy.
|
||||
"""
|
||||
copies: dict[int, Node] = {}
|
||||
|
||||
def shifted(original: Node) -> Node:
|
||||
existing = copies.get(id(original))
|
||||
if existing is not None:
|
||||
return existing
|
||||
clone = object.__new__(type(original))
|
||||
copies[id(original)] = clone
|
||||
state = dict(original.__dict__)
|
||||
if state.get("line") is not None:
|
||||
state["line"] += delta
|
||||
end = state.get("end_pos")
|
||||
if end is not None:
|
||||
state["end_pos"] = (end[0] + delta, end[1])
|
||||
for key, value in state.items():
|
||||
if isinstance(value, Node):
|
||||
state[key] = shifted(value)
|
||||
elif isinstance(value, (list, tuple)) and value and isinstance(value[0], Node):
|
||||
state[key] = type(value)(shifted(item) for item in value)
|
||||
clone.__dict__.update(state)
|
||||
return clone
|
||||
|
||||
return shifted(node)
|
||||
|
||||
|
||||
def _shifted_violation(violation: Violation, delta: int) -> Violation:
|
||||
clone = copy.copy(violation)
|
||||
clone.start = (violation.start[0] + delta, violation.start[1])
|
||||
clone.end = (violation.end[0] + delta, violation.end[1])
|
||||
return clone
|
||||
|
||||
|
||||
def reparse(
|
||||
old_source: str,
|
||||
old_tree: Script,
|
||||
old_violations: list[Violation],
|
||||
new_source: str,
|
||||
parse_chunk: ParseChunk,
|
||||
) -> tuple[Script, list[Violation]] | None:
|
||||
"""Return the tree of `new_source`, or None when a full parse is needed.
|
||||
|
||||
Both sources must already be newline-normalized. `parse_chunk` parses a
|
||||
top-level fragment starting at the given (line, column) and may raise
|
||||
TclSyntaxError, which the caller handles like any failed parse.
|
||||
"""
|
||||
if old_source == new_source:
|
||||
return old_tree, list(old_violations)
|
||||
|
||||
# Changed line range (1-indexed); lines outside it are identical. A pure
|
||||
# insertion leaves last_changed_old == first_changed_line - 1.
|
||||
old_lines = old_source.split("\n")
|
||||
new_lines = new_source.split("\n")
|
||||
limit = min(len(old_lines), len(new_lines))
|
||||
same_before = 0
|
||||
while same_before < limit and old_lines[same_before] == new_lines[same_before]:
|
||||
same_before += 1
|
||||
same_after = 0
|
||||
while (
|
||||
same_after < limit - same_before
|
||||
and old_lines[-1 - same_after] == new_lines[-1 - same_after]
|
||||
):
|
||||
same_after += 1
|
||||
|
||||
first_changed_line = same_before + 1
|
||||
last_changed_old = len(old_lines) - same_after
|
||||
delta = len(new_lines) - len(old_lines)
|
||||
|
||||
commands = old_tree.children
|
||||
if any(command.line is None or command.end_pos is None for command in commands):
|
||||
return None
|
||||
|
||||
# Commands overlapping the changed lines, widened so that no reused
|
||||
# command shares a line with the reparsed range.
|
||||
first = next(
|
||||
(index for index, command in enumerate(commands) if command.end_pos[0] >= first_changed_line),
|
||||
len(commands),
|
||||
)
|
||||
start_line = first_changed_line
|
||||
if first < len(commands):
|
||||
start_line = min(start_line, commands[first].line)
|
||||
while first > 0 and commands[first - 1].end_pos[0] >= start_line:
|
||||
first -= 1
|
||||
start_line = min(start_line, commands[first].line)
|
||||
|
||||
last = first - 1
|
||||
end_line_old = last_changed_old
|
||||
while last + 1 < len(commands) and commands[last + 1].line <= end_line_old:
|
||||
last += 1
|
||||
end_line_old = max(end_line_old, commands[last].end_pos[0])
|
||||
end_line_new = end_line_old + delta
|
||||
|
||||
if end_line_new < start_line - 1 or end_line_new > len(new_lines):
|
||||
return None
|
||||
# A trailing backslash joins a line with the next one across the boundary.
|
||||
if start_line > 1 and new_lines[start_line - 2].endswith("\\"):
|
||||
return None
|
||||
if end_line_new >= start_line and new_lines[end_line_new - 1].endswith("\\"):
|
||||
return None
|
||||
|
||||
chunk_commands: list[Node] = []
|
||||
chunk_violations: list[Violation] = []
|
||||
if end_line_new >= start_line:
|
||||
chunk = "\n".join(new_lines[start_line - 1 : end_line_new])
|
||||
chunk_tree, chunk_violations = parse_chunk(chunk, (start_line, 1))
|
||||
chunk_commands = chunk_tree.children
|
||||
|
||||
reused_after = [_shifted_tree(command, delta) for command in commands[last + 1 :]]
|
||||
tree = Script(
|
||||
*commands[:first],
|
||||
*chunk_commands,
|
||||
*reused_after,
|
||||
pos=(old_tree.line, old_tree.col),
|
||||
)
|
||||
tree.end_pos = (len(new_lines), len(new_lines[-1]) + 1)
|
||||
|
||||
violations = [violation for violation in old_violations if violation.start[0] < start_line]
|
||||
violations += chunk_violations
|
||||
violations += [
|
||||
_shifted_violation(violation, delta)
|
||||
for violation in old_violations
|
||||
if violation.start[0] > end_line_old
|
||||
]
|
||||
return tree, violations
|
||||
@@ -0,0 +1,128 @@
|
||||
"""Persist per-file index results across server restarts.
|
||||
|
||||
Entries are keyed by path and validated by the file's size and mtime. The
|
||||
whole cache is tied to a fingerprint of the code that produced it: the
|
||||
indexing sources of this server, the bundled tclint sources and the versions
|
||||
of all bundled libraries. Any change to them, including a tclint update or a
|
||||
local patch, discards the cache instead of loading stale results.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import hashlib
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import pickle
|
||||
import sys
|
||||
import tempfile
|
||||
import threading
|
||||
import zlib
|
||||
from typing import Any
|
||||
|
||||
LOGGER = logging.getLogger(__name__)
|
||||
|
||||
# Bump when the cached data layout changes without a source change above.
|
||||
CACHE_FORMAT = 1
|
||||
CACHE_FILE = "index-cache.pickle.z"
|
||||
|
||||
_SRC_DIR = pathlib.Path(__file__).resolve().parent.parent
|
||||
_LIBS_DIR = _SRC_DIR.parent / "libs"
|
||||
|
||||
FileStat = tuple[int, int]
|
||||
|
||||
|
||||
def code_fingerprint() -> str:
|
||||
digest = hashlib.sha256()
|
||||
digest.update(f"{CACHE_FORMAT}|{sys.version}".encode())
|
||||
sources = [
|
||||
_SRC_DIR / "lsp_tclserver.py",
|
||||
*sorted((_SRC_DIR / "tools").glob("*.py")),
|
||||
*sorted((_SRC_DIR / "plugins").glob("*.py")),
|
||||
*sorted((_LIBS_DIR / "tclint").rglob("*.py")),
|
||||
]
|
||||
for source in sources:
|
||||
digest.update(source.relative_to(_SRC_DIR.parent).as_posix().encode())
|
||||
digest.update(source.read_bytes())
|
||||
for dist_info in sorted(_LIBS_DIR.glob("*.dist-info")):
|
||||
digest.update(dist_info.name.encode())
|
||||
return digest.hexdigest()
|
||||
|
||||
|
||||
def file_stat(path: str) -> FileStat | None:
|
||||
try:
|
||||
stat = os.stat(path)
|
||||
except OSError:
|
||||
return None
|
||||
return stat.st_mtime_ns, stat.st_size
|
||||
|
||||
|
||||
class IndexCache:
|
||||
def __init__(self, directory: pathlib.Path | None = None, fingerprint: str = ""):
|
||||
self._path = directory / CACHE_FILE if directory is not None else None
|
||||
self._fingerprint = fingerprint
|
||||
self._entries: dict[str, tuple[FileStat, Any]] = {}
|
||||
self._used: set[str] = set()
|
||||
self._dirty = False
|
||||
self._lock = threading.Lock()
|
||||
|
||||
@classmethod
|
||||
def load(cls, directory: pathlib.Path | str | None) -> IndexCache:
|
||||
"""Open the cache in `directory`; without one, nothing is persisted."""
|
||||
if not directory:
|
||||
return cls()
|
||||
cache = cls(pathlib.Path(directory), code_fingerprint())
|
||||
try:
|
||||
with open(cache._path, "rb") as file:
|
||||
fingerprint, entries = pickle.loads(zlib.decompress(file.read()))
|
||||
except FileNotFoundError:
|
||||
return cache
|
||||
except Exception as error: # A damaged cache must never stop indexing.
|
||||
LOGGER.warning("Ignoring unreadable index cache %s: %s", cache._path, error)
|
||||
cache._dirty = True
|
||||
return cache
|
||||
if fingerprint == cache._fingerprint:
|
||||
cache._entries = entries
|
||||
else:
|
||||
cache._dirty = True
|
||||
return cache
|
||||
|
||||
def get(self, path: str, stat: FileStat) -> Any | None:
|
||||
with self._lock:
|
||||
entry = self._entries.get(path)
|
||||
if entry is None or entry[0] != stat:
|
||||
return None
|
||||
self._used.add(path)
|
||||
return entry[1]
|
||||
|
||||
def put(self, path: str, stat: FileStat, data: Any) -> None:
|
||||
if self._path is None:
|
||||
return
|
||||
with self._lock:
|
||||
self._entries[path] = (stat, data)
|
||||
self._used.add(path)
|
||||
self._dirty = True
|
||||
|
||||
def save(self) -> None:
|
||||
"""Write entries used in this session atomically; others are dropped."""
|
||||
if self._path is None:
|
||||
return
|
||||
with self._lock:
|
||||
if not self._dirty and self._used == self._entries.keys():
|
||||
return
|
||||
entries = {path: self._entries[path] for path in self._used if path in self._entries}
|
||||
self._entries = entries
|
||||
self._dirty = False
|
||||
try:
|
||||
self._path.parent.mkdir(parents=True, exist_ok=True)
|
||||
with tempfile.NamedTemporaryFile(dir=self._path.parent, delete=False) as file:
|
||||
data = pickle.dumps((self._fingerprint, entries), protocol=pickle.HIGHEST_PROTOCOL)
|
||||
# Pickled indexes are very repetitive; fast compression cuts ~90%.
|
||||
file.write(zlib.compress(data, 1))
|
||||
os.replace(file.name, self._path)
|
||||
except Exception as error:
|
||||
LOGGER.warning("Could not write index cache %s: %s", self._path, error)
|
||||
try:
|
||||
os.unlink(file.name)
|
||||
except (OSError, NameError):
|
||||
pass
|
||||
@@ -265,6 +265,12 @@ def build_file_symbol_index(
|
||||
filepath: str, uri: str, tree: Node
|
||||
) -> FileSymbolIndex:
|
||||
occurrences: list[SymbolOccurrence] = []
|
||||
# Most occurrences repeat a few identities; sharing one object per identity
|
||||
# keeps the index (and its persistent cache) small.
|
||||
identities: dict[SymbolIdentity, SymbolIdentity] = {}
|
||||
|
||||
def shared(identity: SymbolIdentity | None) -> SymbolIdentity | None:
|
||||
return None if identity is None else identities.setdefault(identity, identity)
|
||||
|
||||
def add_proc(
|
||||
node: Node,
|
||||
@@ -274,7 +280,7 @@ def build_file_symbol_index(
|
||||
is_definition: bool,
|
||||
declaration_range: lsp.Range | None = None,
|
||||
) -> None:
|
||||
identity = _proc_identity(raw_name, scope.namespace)
|
||||
identity = shared(_proc_identity(raw_name, scope.namespace))
|
||||
caller = None
|
||||
if not is_definition:
|
||||
caller = (
|
||||
@@ -288,14 +294,14 @@ def build_file_symbol_index(
|
||||
fallback_identity=(
|
||||
None
|
||||
if is_definition
|
||||
else _proc_fallback(raw_name, scope.namespace)
|
||||
else shared(_proc_fallback(raw_name, scope.namespace))
|
||||
),
|
||||
range=_name_range(node, raw_name),
|
||||
placeholder=_basename(raw_name),
|
||||
is_definition=is_definition,
|
||||
symbol_kind=lsp.SymbolKind.Function,
|
||||
container_name=_container_name(identity),
|
||||
caller=caller,
|
||||
caller=shared(caller),
|
||||
declaration_range=declaration_range,
|
||||
)
|
||||
)
|
||||
@@ -309,7 +315,7 @@ def build_file_symbol_index(
|
||||
variable_sub: bool = False,
|
||||
identity: SymbolIdentity | None = None,
|
||||
) -> None:
|
||||
symbol_identity = identity or _variable_identity(raw_name, scope)
|
||||
symbol_identity = shared(identity or _variable_identity(raw_name, scope))
|
||||
occurrences.append(
|
||||
SymbolOccurrence(
|
||||
identity=symbol_identity,
|
||||
|
||||
@@ -0,0 +1,131 @@
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
import pytest
|
||||
|
||||
THIS_DIR = Path(__file__).parent
|
||||
SRC_DIR = THIS_DIR.parent.parent / "src"
|
||||
if str(SRC_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SRC_DIR))
|
||||
|
||||
from lsp_tclserver import TclLanguageServer # noqa: E402
|
||||
from pygls.workspace.text_document import TextDocument # noqa: E402
|
||||
from tclint.lexer import TclSyntaxError # noqa: E402
|
||||
from tclint.syntax_tree import Node # noqa: E402
|
||||
from tools.incremental_parse import reparse # noqa: E402
|
||||
from tools.parser import CustomParser # noqa: E402
|
||||
|
||||
SOURCE = """\
|
||||
# header comment
|
||||
set a 1; set b 2
|
||||
proc first {x} {
|
||||
global mom_pos
|
||||
if {$x > 0} {
|
||||
MOM_output_literal "first $x"
|
||||
}
|
||||
return [expr {$x + 1}]
|
||||
}
|
||||
|
||||
proc second {} {
|
||||
set list [list a b \\
|
||||
c d]
|
||||
return $list
|
||||
}
|
||||
lappend ::handlers {second}
|
||||
"""
|
||||
|
||||
|
||||
def _parse(text, pos=None):
|
||||
parser = CustomParser()
|
||||
tree = parser.parse(text, pos=pos)
|
||||
return tree, list(parser.violations)
|
||||
|
||||
|
||||
def _differences(a, b, path="root"):
|
||||
if type(a) is not type(b):
|
||||
return f"{path}: {type(a).__name__} != {type(b).__name__}"
|
||||
for key in a.__dict__.keys() | b.__dict__.keys():
|
||||
first, second = a.__dict__.get(key), b.__dict__.get(key)
|
||||
if isinstance(first, Node):
|
||||
difference = _differences(first, second, f"{path}.{key}")
|
||||
elif isinstance(first, (list, tuple)) and first and isinstance(first[0], Node):
|
||||
if len(first) != len(second):
|
||||
return f"{path}.{key}: {len(first)} != {len(second)}"
|
||||
difference = next(
|
||||
(d for i, (x, y) in enumerate(zip(first, second)) if (d := _differences(x, y, f"{path}.{key}[{i}]"))),
|
||||
None,
|
||||
)
|
||||
else:
|
||||
difference = None if first == second else f"{path}.{key}: {first!r} != {second!r}"
|
||||
if difference:
|
||||
return difference
|
||||
return None
|
||||
|
||||
|
||||
def _violations(violations):
|
||||
return [(str(v.id), v.message, v.start, v.end) for v in violations]
|
||||
|
||||
|
||||
@pytest.mark.parametrize("old, new", [
|
||||
("MOM_output_literal \"first $x\"", "MOM_output_literal \"first $x\" extra"),
|
||||
(" return $list\n", " return $list\n puts done\n"),
|
||||
("proc second {} {", "\nproc second {} {"),
|
||||
("set a 1; set b 2\n", ""),
|
||||
("# header comment\n", "# header comment\nset inserted 0\n"),
|
||||
("lappend ::handlers {second}\n", "lappend ::handlers {second}\nproc third {} {}\n"),
|
||||
(" c d]", " c d e]"),
|
||||
("global mom_pos", "global mom_pos mom_out_angle_pos"),
|
||||
])
|
||||
def test_incremental_tree_matches_full_parse(old, new):
|
||||
edited = SOURCE.replace(old, new, 1)
|
||||
assert edited != SOURCE
|
||||
previous = (SOURCE, *_parse(SOURCE))
|
||||
result = reparse(*previous, edited, _parse)
|
||||
assert result is not None
|
||||
expected = _parse(edited)
|
||||
assert _differences(result[0], expected[0]) is None
|
||||
assert _violations(result[1]) == _violations(expected[1])
|
||||
|
||||
|
||||
def test_continuation_across_the_edit_forces_full_parse():
|
||||
edited = SOURCE.replace("set a 1; set b 2", "set a 1; set b 2 \\")
|
||||
assert reparse(SOURCE, *_parse(SOURCE), edited, _parse) is None
|
||||
|
||||
|
||||
def test_quote_closing_outside_the_edit_is_left_to_the_full_parse():
|
||||
edited = SOURCE.replace("set a 1; set b 2", 'set a "1; set b 2')
|
||||
try:
|
||||
result = reparse(SOURCE, *_parse(SOURCE), edited, _parse)
|
||||
except TclSyntaxError:
|
||||
result = None
|
||||
assert result is None
|
||||
|
||||
|
||||
def test_unchanged_commands_are_reused_and_never_mutated():
|
||||
tree, violations = _parse(SOURCE)
|
||||
edited = SOURCE.replace("return $list", "return [lsort $list]")
|
||||
new_tree, _ = reparse(SOURCE, tree, violations, edited, _parse)
|
||||
assert new_tree.children[0] is tree.children[0]
|
||||
# Commands after the edit are shifted copies; the old tree stays valid.
|
||||
inserted = SOURCE.replace("proc first", "\n\nproc first")
|
||||
shifted_tree, _ = reparse(SOURCE, tree, violations, inserted, _parse)
|
||||
assert shifted_tree.children[-1] is not tree.children[-1]
|
||||
assert shifted_tree.children[-1].line == tree.children[-1].line + 2
|
||||
assert tree.children[-1].line == _parse(SOURCE)[0].children[-1].line
|
||||
|
||||
|
||||
def test_server_reparses_edits_incrementally(tmp_path, monkeypatch):
|
||||
server = TclLanguageServer(name="incremental-test", version="1", max_workers=1)
|
||||
uri = (tmp_path / "edit.tcl").as_uri()
|
||||
first = server.get_tree(TextDocument(uri=uri, source=SOURCE, version=1, language_id="tcl"))
|
||||
|
||||
parsed_sources = []
|
||||
parse_source = server._parse_source
|
||||
monkeypatch.setattr(server, "_parse_source", lambda text, pos=None: parsed_sources.append(text) or parse_source(text, pos))
|
||||
server.clear_cache_for_uri(uri)
|
||||
edited = SOURCE.replace("return $list", "return [lsort $list]")
|
||||
second = server.get_tree(TextDocument(uri=uri, source=edited, version=2, language_id="tcl"))
|
||||
|
||||
assert parsed_sources and all(text != edited for text in parsed_sources)
|
||||
assert second.children[0] is first.children[0]
|
||||
assert _differences(second, _parse(edited)[0]) is None
|
||||
@@ -0,0 +1,109 @@
|
||||
import os
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
THIS_DIR = Path(__file__).parent
|
||||
SRC_DIR = THIS_DIR.parent.parent / "src"
|
||||
if str(SRC_DIR) not in sys.path:
|
||||
sys.path.insert(0, str(SRC_DIR))
|
||||
|
||||
import tools.index_cache as index_cache # noqa: E402
|
||||
from lsp_tclserver import TclLanguageServer # noqa: E402
|
||||
from pygls.workspace.text_document import TextDocument # noqa: E402
|
||||
from tools.index_cache import IndexCache # noqa: E402
|
||||
|
||||
|
||||
def _index_from_disk(server, path: Path) -> bool:
|
||||
return server.update_poco_completion_for_file(
|
||||
TextDocument(uri=path.as_uri(), language_id="tcl"),
|
||||
cache_tree=False,
|
||||
require_file_exists=True,
|
||||
from_disk=True,
|
||||
)
|
||||
|
||||
|
||||
def _warm_server(cache_dir: Path) -> TclLanguageServer:
|
||||
server = TclLanguageServer(name="cache-test", version="1", max_workers=1)
|
||||
server.index_cache = IndexCache.load(cache_dir)
|
||||
return server
|
||||
|
||||
|
||||
def _fail_build(*_args, **_kwargs):
|
||||
raise AssertionError("file was parsed although it is cached")
|
||||
|
||||
|
||||
def test_second_start_uses_cached_index(tmp_path: Path, monkeypatch):
|
||||
source = tmp_path / "post.tcl"
|
||||
source.write_text("proc cached_proc {a b} { return $a }\n", encoding="utf-8")
|
||||
cache_dir = tmp_path / "storage"
|
||||
|
||||
server = _warm_server(cache_dir)
|
||||
assert _index_from_disk(server, source)
|
||||
server.index_cache.save()
|
||||
|
||||
restarted = _warm_server(cache_dir)
|
||||
monkeypatch.setattr(restarted, "_build_file_index", _fail_build)
|
||||
assert _index_from_disk(restarted, source)
|
||||
assert "cached_proc" in restarted.custom_function_names_snapshot()
|
||||
assert restarted.proc_metadata_snapshot(str(source))[0]["cached_proc"] == ["a", "b"]
|
||||
|
||||
|
||||
def test_changed_file_is_parsed_again(tmp_path: Path):
|
||||
source = tmp_path / "post.tcl"
|
||||
source.write_text("proc old_proc {} {}\n", encoding="utf-8")
|
||||
cache_dir = tmp_path / "storage"
|
||||
server = _warm_server(cache_dir)
|
||||
assert _index_from_disk(server, source)
|
||||
server.index_cache.save()
|
||||
|
||||
source.write_text("proc new_proc {} {}\n", encoding="utf-8")
|
||||
stat = source.stat()
|
||||
os.utime(source, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000))
|
||||
restarted = _warm_server(cache_dir)
|
||||
assert _index_from_disk(restarted, source)
|
||||
names = restarted.custom_function_names_snapshot()
|
||||
assert "new_proc" in names and "old_proc" not in names
|
||||
|
||||
|
||||
def test_code_change_discards_the_cache(tmp_path: Path, monkeypatch):
|
||||
source = tmp_path / "post.tcl"
|
||||
source.write_text("proc cached_proc {} {}\n", encoding="utf-8")
|
||||
cache_dir = tmp_path / "storage"
|
||||
server = _warm_server(cache_dir)
|
||||
assert _index_from_disk(server, source)
|
||||
server.index_cache.save()
|
||||
|
||||
# E.g. an updated tclint: a different fingerprint must not load old entries.
|
||||
monkeypatch.setattr(index_cache, "code_fingerprint", lambda: "other tclint")
|
||||
restarted = _warm_server(cache_dir)
|
||||
built = []
|
||||
build = restarted._build_file_index
|
||||
monkeypatch.setattr(restarted, "_build_file_index", lambda *args: built.append(args) or build(*args))
|
||||
assert _index_from_disk(restarted, source)
|
||||
assert built
|
||||
|
||||
|
||||
def test_damaged_cache_is_ignored(tmp_path: Path, monkeypatch):
|
||||
cache_dir = tmp_path / "storage"
|
||||
cache_dir.mkdir()
|
||||
(cache_dir / index_cache.CACHE_FILE).write_bytes(b"not a cache")
|
||||
source = tmp_path / "post.tcl"
|
||||
source.write_text("proc fresh_proc {} {}\n", encoding="utf-8")
|
||||
|
||||
server = _warm_server(cache_dir)
|
||||
assert _index_from_disk(server, source)
|
||||
server.index_cache.save()
|
||||
|
||||
restarted = _warm_server(cache_dir)
|
||||
monkeypatch.setattr(restarted, "_build_file_index", _fail_build)
|
||||
assert _index_from_disk(restarted, source)
|
||||
|
||||
|
||||
def test_open_documents_never_touch_the_cache(tmp_path: Path):
|
||||
source = tmp_path / "post.tcl"
|
||||
source.write_text("proc on_disk {} {}\n", encoding="utf-8")
|
||||
server = _warm_server(tmp_path / "storage")
|
||||
unsaved = TextDocument(uri=source.as_uri(), source="proc unsaved {} {}\n", version=3, language_id="tcl")
|
||||
assert server.update_poco_completion_for_file(unsaved)
|
||||
server.index_cache.save()
|
||||
assert not (tmp_path / "storage" / index_cache.CACHE_FILE).exists()
|
||||
Reference in New Issue
Block a user