feat(lsp): add incremental indexing and file ops support
Adds a thread-safe incremental index and snapshot API for LSP. Introduces cache invalidation and file operation hooks for delete and rename. This keeps indices in sync with disk changes. Supports reindexing TCL files from disk when needed. - Adds workspace file change handlers to sync indices on delete/rename. - Introduces locking and snapshot helpers to safely access shared state. - Refactors to invalidate caches on edits and reindex TCL files.
This commit is contained in:
Christoph Brandau
+225
-59
@@ -1,16 +1,20 @@
|
||||
import logging
|
||||
import os
|
||||
import pathlib
|
||||
import threading
|
||||
from typing import List, Optional, Tuple
|
||||
|
||||
import lsprotocol.types as lsp
|
||||
from pygls.workspace.text_document import TextDocument
|
||||
from tclint.lexer import TclSyntaxError
|
||||
from tclint.format import FormatterOpts
|
||||
from tools.formatter import NxFormatter as Formatter
|
||||
from tclint.violations import Violation
|
||||
from plugins.poco_plugin import commands
|
||||
from tools import checks, parser
|
||||
from pygls import server, uris
|
||||
from tools.completion_items import completion, remove_existing_items, remove_shared_keys
|
||||
from pygls.workspace.text_document import TextDocument
|
||||
from tclint.format import FormatterOpts
|
||||
from tclint.lexer import TclSyntaxError
|
||||
from tclint.violations import Violation
|
||||
|
||||
from tools import checks, parser
|
||||
from tools.completion_items import CompletionCollector
|
||||
from tools.formatter import NxFormatter as Formatter
|
||||
from tools.proc_docs import build_proc_docs
|
||||
|
||||
|
||||
@@ -29,60 +33,201 @@ class TclLanguageServer(server.LanguageServer):
|
||||
self.proc_docs: dict = {}
|
||||
# Cache: (uri, version) -> (tree, violations)
|
||||
self._ast_cache = {}
|
||||
self._parser_lock = threading.RLock()
|
||||
self._index_lock = threading.RLock()
|
||||
self._index_tokens: dict[str, int] = {}
|
||||
self._index_versions: dict[str, int | None] = {}
|
||||
self._next_index_token = 0
|
||||
self._diagnostic_tokens: dict[str, int] = {}
|
||||
self._next_diagnostic_token = 0
|
||||
|
||||
def _parse_source(self, source: str):
|
||||
self.parser.violations = []
|
||||
tree = self.parser.parse(source)
|
||||
return tree, list(self.parser.violations)
|
||||
|
||||
def parse_source(self, source: str):
|
||||
"""Parse without retaining an AST, serialized around the shared parser."""
|
||||
with self._parser_lock:
|
||||
tree, _ = self._parse_source(source)
|
||||
return tree
|
||||
|
||||
def get_tree(self, document: TextDocument):
|
||||
key = (document.uri, document.version)
|
||||
cached = self._ast_cache.get(key)
|
||||
if cached:
|
||||
return cached[0]
|
||||
# Parse and cache
|
||||
self.parser.violations = []
|
||||
tree = self.parser.parse(document.source)
|
||||
violations = list(self.parser.violations)
|
||||
self._ast_cache[key] = (tree, violations)
|
||||
return tree
|
||||
with self._parser_lock:
|
||||
cached = self._ast_cache.get(key)
|
||||
if cached is not None:
|
||||
return cached[0]
|
||||
tree, violations = self._parse_source(document.source)
|
||||
self._ast_cache[key] = (tree, violations)
|
||||
return tree
|
||||
|
||||
def get_tree_and_violations(self, document: TextDocument):
|
||||
key = (document.uri, document.version)
|
||||
cached = self._ast_cache.get(key)
|
||||
if cached:
|
||||
return cached
|
||||
# Parse and cache
|
||||
self.parser.violations = []
|
||||
tree = self.parser.parse(document.source)
|
||||
violations = list(self.parser.violations)
|
||||
self._ast_cache[key] = (tree, violations)
|
||||
return tree, violations
|
||||
with self._parser_lock:
|
||||
cached = self._ast_cache.get(key)
|
||||
if cached is not None:
|
||||
return cached
|
||||
tree, violations = self._parse_source(document.source)
|
||||
self._ast_cache[key] = (tree, violations)
|
||||
return tree, violations
|
||||
|
||||
def clear_cache_for_uri(self, uri: str):
|
||||
to_delete = [k for k in self._ast_cache.keys() if k[0] == uri]
|
||||
for k in to_delete:
|
||||
del self._ast_cache[k]
|
||||
with self._parser_lock:
|
||||
to_delete = [key for key in self._ast_cache if key[0] == uri]
|
||||
for key in to_delete:
|
||||
del self._ast_cache[key]
|
||||
|
||||
def update_poco_completion_for_file(self, document: TextDocument):
|
||||
@staticmethod
|
||||
def _normalized_path(path: pathlib.Path | str) -> str:
|
||||
return os.path.normcase(os.path.abspath(os.fspath(path)))
|
||||
|
||||
@classmethod
|
||||
def _is_same_or_child(
|
||||
cls, candidate: pathlib.Path | str, parent: pathlib.Path | str
|
||||
) -> bool:
|
||||
candidate_path = cls._normalized_path(candidate)
|
||||
parent_path = cls._normalized_path(parent)
|
||||
try:
|
||||
return os.path.commonpath([candidate_path, parent_path]) == parent_path
|
||||
except ValueError:
|
||||
return False
|
||||
|
||||
@classmethod
|
||||
def paths_equal(
|
||||
cls, first: pathlib.Path | str, second: pathlib.Path | str
|
||||
) -> bool:
|
||||
return cls._normalized_path(first) == cls._normalized_path(second)
|
||||
|
||||
def index_snapshot(self) -> tuple[dict, dict, dict]:
|
||||
"""Return stable copies for request handlers running beside the indexer."""
|
||||
with self._index_lock:
|
||||
return (
|
||||
{path: list(items) for path, items in self.poco_completion.items()},
|
||||
{
|
||||
path: dict(signatures)
|
||||
for path, signatures in self.proc_signatures.items()
|
||||
},
|
||||
{path: dict(docs) for path, docs in self.proc_docs.items()},
|
||||
)
|
||||
|
||||
def diagnostic_snapshot(self, uri: str):
|
||||
with self._index_lock:
|
||||
return self.diagnostics.get(uri)
|
||||
|
||||
def _begin_index_update(self, filepath: str, version: int | None) -> int | None:
|
||||
with self._index_lock:
|
||||
indexed_version = self._index_versions.get(filepath)
|
||||
if version is None and indexed_version is not None:
|
||||
return None
|
||||
if (
|
||||
version is not None
|
||||
and indexed_version is not None
|
||||
and version < indexed_version
|
||||
):
|
||||
return None
|
||||
|
||||
self._next_index_token += 1
|
||||
token = self._next_index_token
|
||||
self._index_tokens[filepath] = token
|
||||
self._index_versions[filepath] = version
|
||||
return token
|
||||
|
||||
def _discard_index_update(self, filepath: str, token: int) -> None:
|
||||
with self._index_lock:
|
||||
if self._index_tokens.get(filepath) != token:
|
||||
return
|
||||
self.poco_completion.pop(filepath, None)
|
||||
self.proc_signatures.pop(filepath, None)
|
||||
self.proc_docs.pop(filepath, None)
|
||||
|
||||
def indexed_paths_under_uri(self, uri: str) -> list[pathlib.Path]:
|
||||
target = pathlib.Path(uris.to_fs_path(uri))
|
||||
with self._index_lock:
|
||||
indexed_paths = set(self.poco_completion)
|
||||
indexed_paths.update(self.proc_signatures)
|
||||
indexed_paths.update(self.proc_docs)
|
||||
indexed_paths.update(self._index_tokens)
|
||||
return [
|
||||
pathlib.Path(path)
|
||||
for path in indexed_paths
|
||||
if self._is_same_or_child(path, target)
|
||||
]
|
||||
|
||||
def remove_file_state(self, uri: str) -> None:
|
||||
"""Remove cached and indexed state for a file or a complete folder."""
|
||||
target = pathlib.Path(uris.to_fs_path(uri))
|
||||
|
||||
with self._index_lock:
|
||||
for store in (
|
||||
self.poco_completion,
|
||||
self.proc_signatures,
|
||||
self.proc_docs,
|
||||
self._index_tokens,
|
||||
self._index_versions,
|
||||
):
|
||||
for path in list(store):
|
||||
if self._is_same_or_child(path, target):
|
||||
del store[path]
|
||||
|
||||
diagnostic_uris = set(self.diagnostics)
|
||||
diagnostic_uris.update(self._diagnostic_tokens)
|
||||
for diagnostic_uri in diagnostic_uris:
|
||||
try:
|
||||
diagnostic_path = pathlib.Path(uris.to_fs_path(diagnostic_uri))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if self._is_same_or_child(diagnostic_path, target):
|
||||
self.diagnostics.pop(diagnostic_uri, None)
|
||||
self._diagnostic_tokens.pop(diagnostic_uri, None)
|
||||
|
||||
with self._parser_lock:
|
||||
for key in list(self._ast_cache):
|
||||
try:
|
||||
cached_path = pathlib.Path(uris.to_fs_path(key[0]))
|
||||
except (TypeError, ValueError):
|
||||
continue
|
||||
if self._is_same_or_child(cached_path, target):
|
||||
del self._ast_cache[key]
|
||||
|
||||
def update_poco_completion_for_file(
|
||||
self,
|
||||
document: TextDocument,
|
||||
*,
|
||||
cache_tree: bool = True,
|
||||
require_file_exists: bool = False,
|
||||
):
|
||||
"""Update poco_completion for a specific file when it changes"""
|
||||
filepath = str(pathlib.Path(uris.to_fs_path(document.uri)))
|
||||
token = self._begin_index_update(filepath, document.version)
|
||||
if token is None:
|
||||
return False
|
||||
|
||||
# Remove existing completion items for this file
|
||||
if filepath in self.poco_completion:
|
||||
del self.poco_completion[filepath]
|
||||
if filepath in self.proc_signatures:
|
||||
del self.proc_signatures[filepath]
|
||||
if filepath in self.proc_docs:
|
||||
del self.proc_docs[filepath]
|
||||
|
||||
# Parse and extract new completion items
|
||||
completion.reset()
|
||||
collector = CompletionCollector()
|
||||
try:
|
||||
tree = self.get_tree(document)
|
||||
tree.accept(completion, recurse=True)
|
||||
remove_existing_items(completion.custom_functions, self.poco_completion)
|
||||
self.poco_completion[filepath] = completion.custom_functions
|
||||
remove_shared_keys(self.proc_signatures, completion.proc_signatures)
|
||||
self.proc_signatures[filepath] = completion.proc_signatures
|
||||
self.proc_docs[filepath] = build_proc_docs(tree, document.source)
|
||||
tree = (
|
||||
self.get_tree(document)
|
||||
if cache_tree
|
||||
else self.parse_source(document.source)
|
||||
)
|
||||
tree.accept(collector, recurse=True)
|
||||
docs = build_proc_docs(tree, document.source)
|
||||
except Exception as e:
|
||||
logging.debug(f"Error parsing {filepath}: {e}")
|
||||
self._discard_index_update(filepath, token)
|
||||
return False
|
||||
|
||||
if require_file_exists and not pathlib.Path(filepath).is_file():
|
||||
self._discard_index_update(filepath, token)
|
||||
return False
|
||||
|
||||
with self._index_lock:
|
||||
if self._index_tokens.get(filepath) != token:
|
||||
return False
|
||||
self.poco_completion[filepath] = list(collector.custom_functions)
|
||||
self.proc_signatures[filepath] = dict(collector.proc_signatures)
|
||||
self.proc_docs[filepath] = docs
|
||||
return True
|
||||
|
||||
def format(
|
||||
self,
|
||||
@@ -107,17 +252,19 @@ class TclLanguageServer(server.LanguageServer):
|
||||
),
|
||||
)
|
||||
|
||||
if range is not None:
|
||||
start, end = range
|
||||
return formatter.format_partial(document.source[start:end], self.parser)
|
||||
with self._parser_lock:
|
||||
if range is not None:
|
||||
start, end = range
|
||||
return formatter.format_partial(document.source[start:end], self.parser)
|
||||
|
||||
return formatter.format_top(document.source, self.parser)
|
||||
return formatter.format_top(document.source, self.parser)
|
||||
|
||||
def linter(
|
||||
self,
|
||||
document: TextDocument,
|
||||
) -> List[Violation]:
|
||||
tree, violations = self.get_tree_and_violations(document)
|
||||
tree, cached_violations = self.get_tree_and_violations(document)
|
||||
violations = list(cached_violations)
|
||||
for checker in checks.get_checkers():
|
||||
violations += checker.check(document.source, tree)
|
||||
return violations
|
||||
@@ -144,8 +291,12 @@ class TclLanguageServer(server.LanguageServer):
|
||||
for violation in violations:
|
||||
message = violation.message
|
||||
severity = lsp.DiagnosticSeverity.Warning
|
||||
start = lsp.Position(line=violation.start[0] - 1, character=violation.start[1] - 1)
|
||||
end = lsp.Position(line=violation.end[0] - 1, character=violation.end[1] - 1)
|
||||
start = lsp.Position(
|
||||
line=violation.start[0] - 1, character=violation.start[1] - 1
|
||||
)
|
||||
end = lsp.Position(
|
||||
line=violation.end[0] - 1, character=violation.end[1] - 1
|
||||
)
|
||||
|
||||
diagnostics.append(
|
||||
lsp.Diagnostic(
|
||||
@@ -166,12 +317,27 @@ class TclLanguageServer(server.LanguageServer):
|
||||
return self.lint(document)
|
||||
|
||||
def compute_diagnostics(self, document: TextDocument):
|
||||
# `None` sentinel ensures that `diagnostics` gets updated if the URI is not
|
||||
# present.
|
||||
_, previous = self.diagnostics.get(document, (0, None))
|
||||
with self._index_lock:
|
||||
self._next_diagnostic_token += 1
|
||||
token = self._next_diagnostic_token
|
||||
self._diagnostic_tokens[document.uri] = token
|
||||
|
||||
diagnostics = self._compute_diagnostics(document)
|
||||
|
||||
# Only update if the list has changed
|
||||
if previous != diagnostics:
|
||||
self.diagnostics[document.uri] = (document.version, diagnostics)
|
||||
with self._index_lock:
|
||||
if self._diagnostic_tokens.get(document.uri) != token:
|
||||
return
|
||||
|
||||
current = self.diagnostics.get(document.uri)
|
||||
if current is not None:
|
||||
current_version, _ = current
|
||||
if (
|
||||
current_version is not None
|
||||
and document.version is not None
|
||||
and current_version > document.version
|
||||
):
|
||||
return
|
||||
|
||||
# Keep the result id in sync even when only the document version changed.
|
||||
if current != (document.version, diagnostics):
|
||||
self.diagnostics[document.uri] = (document.version, diagnostics)
|
||||
|
||||
Reference in New Issue
Block a user