diff --git a/.vscode/launch.json b/.vscode/launch.json index 3083a79..5f47196 100644 --- a/.vscode/launch.json +++ b/.vscode/launch.json @@ -39,6 +39,48 @@ "autoAttachChildProcesses": true, "preLaunchTask": "NX Post Support: Compile Debug" }, + { + // Linux workaround: js-debug's extensionHost attach probes localhost and [::1] + // in parallel and aborts when [::1] is refused. Start the dev host ourselves + // with a fixed inspector port and attach directly to 127.0.0.1. + "name": "Run Extension (Linux)", + "type": "node", + "request": "attach", + "address": "127.0.0.1", + "port": 9333, + "timeout": 30000, + "continueOnAttach": true, + "sourceMaps": true, + "outFiles": ["${workspaceFolder}/dist/**/*.js"], + "resolveSourceMapLocations": [ + "${workspaceFolder}/dist/**/*.js", + "!**/node_modules/**" + ], + "skipFiles": ["/**"], + "preLaunchTask": "NX Post Support: Start Dev Host (Linux)" + }, + { + "name": "Debug Extension (Linux, hidden)", + "type": "node", + "request": "attach", + "address": "127.0.0.1", + "port": 9333, + "timeout": 30000, + "continueOnAttach": true, + "sourceMaps": true, + "outFiles": ["${workspaceFolder}/dist/**/*.js"], + "resolveSourceMapLocations": [ + "${workspaceFolder}/dist/**/*.js", + "!**/node_modules/**" + ], + "skipFiles": ["/**"], + "preLaunchTask": "NX Post Support: Start Dev Host with debugpy (Linux)", + "presentation": { + "hidden": true, + "group": "", + "order": 4 + } + }, { "name": "Python Attach", "type": "debugpy", @@ -118,6 +160,16 @@ "group": "", "order": 1 } + }, + { + "name": "Debug Extension and Python (Linux)", + "configurations": ["Python debug server (hidden)", "Debug Extension (Linux, hidden)"], + "stopAll": true, + "presentation": { + "hidden": false, + "group": "", + "order": 2 + } } ] } diff --git a/.vscode/tasks.json b/.vscode/tasks.json index fb7f391..8d92b16 100644 --- a/.vscode/tasks.json +++ b/.vscode/tasks.json @@ -28,6 +28,47 @@ "panel": "dedicated", "clear": true } + }, + { + "label": "NX Post Support: Start Dev Host (Linux)", + "detail": "Opens the Extension Development Host with the inspector on 127.0.0.1:9333 (workaround for js-debug localhost/::1 attach bug).", + "type": "shell", + "command": "\"$(dirname \"${execPath}\")/bin/$(basename \"${execPath}\")\" --new-window --inspect-brk-extensions=9333 --extensionDevelopmentPath=\"${workspaceFolder}\" \"${workspaceFolder}\" \"${workspaceFolder}/test/postprocessor\"", + "options": { + "shell": { + "executable": "/bin/bash", + "args": ["-c"] + } + }, + "dependsOn": "NX Post Support: Compile Debug", + "problemMatcher": [], + "presentation": { + "reveal": "silent", + "panel": "dedicated" + } + }, + { + "label": "NX Post Support: Start Dev Host with debugpy (Linux)", + "detail": "Same as above, but the language server connects to the debugpy listener on 127.0.0.1:5678.", + "type": "shell", + "command": "\"$(dirname \"${execPath}\")/bin/$(basename \"${execPath}\")\" --new-window --inspect-brk-extensions=9333 --extensionDevelopmentPath=\"${workspaceFolder}\" \"${workspaceFolder}\" \"${workspaceFolder}/test/postprocessor\"", + "options": { + "shell": { + "executable": "/bin/bash", + "args": ["-c"] + }, + "env": { + "USE_DEBUGPY": "True", + "NXPS_DEBUG_HOST": "127.0.0.1", + "NXPS_DEBUG_PORT": "5678" + } + }, + "dependsOn": "NX Post Support: Compile Debug", + "problemMatcher": [], + "presentation": { + "reveal": "silent", + "panel": "dedicated" + } } ] } diff --git a/CHANGELOG.md b/CHANGELOG.md index bd31eb8..4214d81 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ ## Unreleased +- Reparse only the top-level TCL commands touched by an edit instead of the whole file +- Cache the workspace index in the extension storage so restarts skip reparsing unchanged files; the cache is discarded automatically when the server or bundled tclint changes +- Speed up TclOO completion, signature help, inlay hints, and Go to Definition by reusing the cached syntax tree and skipping files without classes +- Speed up references, document highlights, and call hierarchy with cached definition lookups +- Keep background indexing from blocking requests that need a fresh syntax tree - Add Go to Definition for TclOO classes, constructors, and resolved methods, including PSC library definitions - Index PSC layer scripts (including external paths and legacy Windows encoding) and share TclOO class metadata across files for completion, signatures, inlay hints, and highlighting - Add document-local TclOO method completion for `new`/`create` instances, `my`, and statically inferred return chains diff --git a/client/src/common/server.ts b/client/src/common/server.ts index 8b8300f..46d845b 100644 --- a/client/src/common/server.ts +++ b/client/src/common/server.ts @@ -22,7 +22,12 @@ import { import { getLSClientTraceLevel, getProjectRoot } from "./utilities" import { isVirtualWorkspace } from "./vscodeapi" -export type IInitOptions = { settings: ISettings[]; globalSettings: ISettings } +export type IInitOptions = { + settings: ISettings[] + globalSettings: ISettings + // Folder for the server's persistent index cache; omitted without a workspace. + indexCachePath?: string +} let _disposables: Disposable[] = [] @@ -86,7 +91,7 @@ async function createServer( } // Options to control the language client - const tclFileWatcher = workspace.createFileSystemWatcher("**/*.{tcl,psc}") + const tclFileWatcher = workspace.createFileSystemWatcher("**/*.{tcl,psc,def}") const clientOptions: LanguageClientOptions = { // Register the server for python documents documentSelector: isVirtualWorkspace() @@ -114,7 +119,8 @@ export async function restartServer( serverId: string, serverName: string, outputChannel: LogOutputChannel, - lsClient?: LanguageClient + lsClient?: LanguageClient, + indexCachePath?: string ): Promise { if (lsClient) { traceInfo(`Server: Stop requested`) @@ -132,7 +138,8 @@ export async function restartServer( outputChannel, { settings: await getExtensionSettings(serverId, true), - globalSettings: await getGlobalSettings(serverId, false) + globalSettings: await getGlobalSettings(serverId, false), + indexCachePath } ) traceInfo(`Server: Start requested.`) diff --git a/client/src/extension.ts b/client/src/extension.ts index bf27936..6876c00 100644 --- a/client/src/extension.ts +++ b/client/src/extension.ts @@ -73,7 +73,13 @@ export async function activate(context: vscode.ExtensionContext) { traceVerbose( `Using interpreter from ${serverInfo.module}.interpreter: ${interpreter.join(" ")}` ) - client = await restartServer(serverId, serverName, outputChannel, client) + client = await restartServer( + serverId, + serverName, + outputChannel, + client, + context.storageUri?.fsPath + ) } return } @@ -83,7 +89,13 @@ export async function activate(context: vscode.ExtensionContext) { traceVerbose( `Using interpreter from Python extension: ${interpreterDetails.path.join(" ")}` ) - client = await restartServer(serverId, serverName, outputChannel, client) + client = await restartServer( + serverId, + serverName, + outputChannel, + client, + context.storageUri?.fsPath + ) return } diff --git a/package-lock.json b/package-lock.json index 2678556..cd52e93 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,12 +1,12 @@ { "name": "nx-post-support", - "version": "2026.8.201", + "version": "2026.9.501", "lockfileVersion": 3, "requires": true, "packages": { "": { "name": "nx-post-support", - "version": "2026.8.201", + "version": "2026.9.501", "devDependencies": { "@types/vscode": "^1.96.0", "@vscode/vsce": "^3.2.1", diff --git a/server/src/_debug_server.py b/server/src/_debug_server.py index a583b81..140a51c 100644 --- a/server/src/_debug_server.py +++ b/server/src/_debug_server.py @@ -6,7 +6,7 @@ import os import pathlib import runpy import sys -import time +import threading def update_sys_path(path_to_add: str) -> None: @@ -27,21 +27,35 @@ def _debug_endpoint() -> tuple[str, int]: return host, port -def _connect_debugger(debugpy, host: str, port: int, timeout: float = 15.0) -> None: - deadline = time.monotonic() + timeout - last_error: OSError | None = None - while time.monotonic() < deadline: +def _connect_debugger(debugpy, host: str, port: int, timeout: float = 30.0) -> None: + errors: list[BaseException] = [] + + def attach() -> None: + # debugpy.connect() cannot be retried: after a refused connection a second + # call terminates the process silently. Connect exactly once. try: debugpy.connect((host, port)) debugpy.wait_for_client() - return - except (ConnectionRefusedError, OSError) as error: - last_error = error - time.sleep(0.25) + except BaseException as error: # pylint: disable=broad-exception-caught + errors.append(error) - raise RuntimeError( - f"Could not connect debugpy to {host}:{port} within {timeout:.0f} seconds" - ) from last_error + # A stale debugpy adapter from an earlier debug session can still own the port. + # It accepts the connection but never attaches, so bound the wait. + waiter = threading.Thread(target=attach, daemon=True) + waiter.start() + waiter.join(timeout) + if waiter.is_alive(): + raise RuntimeError( + f"Connected to {host}:{port}, but no VS Code debug session attached within " + f"{timeout:.0f} seconds. A stale debugpy adapter probably still owns the " + f"port; stop it (e.g. 'fuser -k {port}/tcp') and restart debugging." + ) + if errors: + raise RuntimeError( + f"No debugpy listener on {host}:{port}. Start the launch configuration " + "'Python debug server (hidden)' (e.g. via a 'Debug Extension and Python' " + "compound) before the language server." + ) from errors[0] def main() -> None: @@ -61,7 +75,12 @@ def main() -> None: host, port = _debug_endpoint() print(f"debugpy: waiting for VS Code at {host}:{port}", file=sys.stderr) - _connect_debugger(debugpy, host, port) + try: + _connect_debugger(debugpy, host, port) + except RuntimeError as error: + print(f"debugpy: {error}", file=sys.stderr, flush=True) + # debugpy's background threads can keep the interpreter alive; exit hard. + os._exit(1) print("debugpy: VS Code attached; starting language server", file=sys.stderr) server_path = os.fspath(pathlib.Path(__file__).parent / "lsp_server.py") diff --git a/server/src/lsp_server.py b/server/src/lsp_server.py index 4745b92..a321e08 100644 --- a/server/src/lsp_server.py +++ b/server/src/lsp_server.py @@ -11,6 +11,7 @@ import pathlib import re import sys import threading +import time from collections import ChainMap from functools import reduce from typing import Any, Optional @@ -52,6 +53,7 @@ from tools.completion_items import ( ranked_completion_items, ) from tools.folding_ranges import build_folding_ranges +from tools.index_cache import IndexCache from tools.inlay_hint import ( InlayHintGenerator, build_builtin_inlay_signatures, @@ -60,7 +62,6 @@ from tools.navigation import ( SymbolIdentity, call_hierarchy_identity, call_hierarchy_items, - definition_identities, document_highlights, incoming_call_hierarchy, matching_occurrences, @@ -77,7 +78,7 @@ from tools.semantic_tokens import ( from tools.signature_help import build_signature_help from tools.tcloo_arguments import method_signature_help from tclint.lexer import TclSyntaxError -from tools.tcloo_completion import parse_completion_source, tcloo_completions +from tools.tcloo_completion import may_contain_classes, parse_completion_source, tcloo_completions from tools.tcloo_symbols import class_completion_items from tools.tcloo_navigation import tcloo_definition from tools.tcl_command_completion import ( @@ -90,6 +91,8 @@ from tools.tcl_command_completion import ( WORKSPACE_SETTINGS = {} GLOBAL_SETTINGS = {} +# Extension storage folder of the workspace; without it nothing is persisted. +INDEX_CACHE_PATH: dict[str, str | None] = {} MAX_WORKERS = 5 @@ -176,6 +179,7 @@ def _index_tcl_file_from_disk(uri: str) -> None: document, cache_tree=False, require_file_exists=True, + from_disk=True, ) except (OSError, UnicodeError) as error: log_warning(f"Could not re-index {path}: {error}") @@ -215,21 +219,29 @@ def did_rename_files(params: lsp.RenameFilesParams) -> None: def did_change_watched_files(params: lsp.DidChangeWatchedFilesParams) -> None: """Keep indexes for closed Tcl files synchronized with disk changes.""" for change in params.changes: - if pathlib.Path(uris.to_fs_path(change.uri)).suffix.lower() == ".psc": + suffix = pathlib.Path(uris.to_fs_path(change.uri)).suffix.lower() + if suffix == ".psc": _refresh_psc_index() continue + if suffix == ".def": + LSP_SERVER.refresh_def_symbols(_workspace_roots(), report=log_warning) + continue if change.type == lsp.FileChangeType.Deleted: LSP_SERVER.remove_file_state(change.uri) else: _index_tcl_file_from_disk(change.uri) -def _refresh_psc_index(): +def _workspace_roots() -> list[pathlib.Path]: folders = LSP_SERVER.workspace.folders roots = [pathlib.Path(uris.to_fs_path(uri)) for uri in folders] if not roots and LSP_SERVER.workspace.root_path: roots = [pathlib.Path(LSP_SERVER.workspace.root_path)] - LSP_SERVER.refresh_psc_scripts(roots, report=log_warning) + return roots + + +def _refresh_psc_index(): + LSP_SERVER.refresh_psc_scripts(_workspace_roots(), report=log_warning) @LSP_SERVER.feature( @@ -270,7 +282,14 @@ def on_completion(params: lsp.CompletionParams) -> lsp.CompletionList: doc = LSP_SERVER.workspace.get_text_document(params.text_document.uri) position = params.position source_lines = LSP_SERVER.get_lines(doc) - oo_items = tcloo_completions(source_lines, position, LSP_SERVER.class_snapshot(doc.path)) + + def current_tree(): + try: + return LSP_SERVER.get_tree(doc) + except TclSyntaxError: + return None + + oo_items = tcloo_completions(source_lines, position, LSP_SERVER.class_snapshot(doc.path), current_tree) if oo_items is not None: return lsp.CompletionList(is_incomplete=False, items=oo_items) array_items = array_element_completions( @@ -382,6 +401,30 @@ def on_completion(params: lsp.CompletionParams) -> lsp.CompletionList: items = [*ranked_completion_items(static_candidates, CompletionContext.GENERAL), *items] return lsp.CompletionList(is_incomplete=False, items=items) + def_symbol_items = { + DynamicCompletionKind.BLOCK_TEMPLATE: LSP_SERVER.block_template_items, + DynamicCompletionKind.ADDRESS: LSP_SERVER.address_items, + DynamicCompletionKind.VALUE: list, + }.get(argument_completion.dynamic_kind) + if def_symbol_items is not None: + # MOM_do_template, MOM_force, ... take a .def name, a fixed value (e.g. + # Always|Once|Off) or a variable holding one. + def_symbols = [*static_candidates, *((0, item) for item in def_symbol_items())] + variable_candidates = list(candidates) + variable_candidates.extend((300, item) for item in standard_items.nx_variables) + variables = [] + for item in ranked_completion_items(variable_candidates, CompletionContext.VARIABLE): + # Without a typed "$" the variable must be substituted. + item.insert_text = f"${item.label}" + item.filter_text = item.label + item.text_edit = None + variables.append(item) + quoted = [ + _quoted_item(item, source_lines, position, argument_completion.active_prefix) + for item in ranked_completion_items(def_symbols, CompletionContext.GENERAL) + ] + return lsp.CompletionList(is_incomplete=False, items=[*quoted, *variables]) + if argument_completion.dynamic_kind == DynamicCompletionKind.PROCEDURE: procedure_kinds = { lsp.CompletionItemKind.Constructor, @@ -425,6 +468,35 @@ def on_completion(params: lsp.CompletionParams) -> lsp.CompletionList: return lsp.CompletionList(is_incomplete=False, items=items) +def _quoted_item( + item: lsp.CompletionItem, + source_lines, + position: lsp.Position, + active_prefix: str, +) -> lsp.CompletionItem: + """Insert ``item`` as a quoted word, replacing quotes the user already typed.""" + line = source_lines[position.line] if position.line < len(source_lines) else "" + utf16 = line.encode("utf-16-le") + start = position.character - len(active_prefix.encode("utf-16-le")) // 2 + end = position.character + opened = start > 0 and utf16[(start - 1) * 2 : start * 2].decode("utf-16-le") == '"' + if opened: + start -= 1 + # Also replace a closing quote the editor inserted automatically. + if utf16[end * 2 : (end + 1) * 2].decode("utf-16-le") == '"': + end += 1 + item.text_edit = lsp.TextEdit( + range=lsp.Range( + start=lsp.Position(line=position.line, character=start), + end=lsp.Position(line=position.line, character=end), + ), + new_text=f'"{item.label}"', + ) + item.insert_text = None + item.filter_text = f'"{item.label}' if opened else item.label + return item + + @LSP_SERVER.feature( lsp.TEXT_DOCUMENT_SIGNATURE_HELP, lsp.SignatureHelpOptions( @@ -434,15 +506,15 @@ def on_completion(params: lsp.CompletionParams) -> lsp.CompletionList: ) def signature_help(params: lsp.SignatureHelpParams) -> lsp.SignatureHelp | None: document = LSP_SERVER.workspace.get_text_document(params.text_document.uri) - method_help = method_signature_help(document.source, params.position, LSP_SERVER.class_snapshot(document.path)) - if method_help is not None: - return method_help try: tree = LSP_SERVER.get_tree(document) except TclSyntaxError: tree = parse_completion_source(document.source) if tree is None: return None + method_help = method_signature_help(document.source, params.position, LSP_SERVER.class_snapshot(document.path), tree) + if method_help is not None: + return method_help custom_signatures, custom_docs = LSP_SERVER.proc_metadata_snapshot(document.path) @@ -522,8 +594,9 @@ def semantic_tokens(params: lsp.SemanticTokensParams): # Reuse cached AST tree = LSP_SERVER.get_tree(document) classes = LSP_SERVER.class_snapshot(document.path) - hl.highlight_classes(tree, classes) - hl.highlight_methods(tree, document.source, document.uri, classes) + if may_contain_classes(document.source, classes): + hl.highlight_classes(tree, classes) + hl.highlight_methods(tree, document.source, document.uri, classes) tree.accept(hl, recurse=True) tokens = hl.tokens() @@ -619,8 +692,12 @@ def goto_definition(params: lsp.DefinitionParams): workspace = None if workspace is not None: document = workspace.get_text_document(params.text_document.uri) + try: + tree = LSP_SERVER.get_tree(document) + except TclSyntaxError: + tree = None # tcloo_definition repairs open delimiters itself. target = tcloo_definition(document.source, document.uri, params.position, - LSP_SERVER.class_snapshot(document.path)) + LSP_SERVER.class_snapshot(document.path), tree) if target is not None: return [target] context = _navigation_context(params.text_document.uri, params.position) @@ -633,18 +710,17 @@ def goto_definition(params: lsp.DefinitionParams): def _navigation_context(uri: str, position: lsp.Position): - indexes = LSP_SERVER.navigation_snapshot() + indexes, definitions = LSP_SERVER.navigation_state() filepath = str(pathlib.Path(uris.to_fs_path(uri))) index = indexes.get(filepath) if index is None or LSP_SERVER.index_update_pending(filepath): document = LSP_SERVER.workspace.get_text_document(uri) LSP_SERVER.update_poco_completion_for_file(document) - indexes = LSP_SERVER.navigation_snapshot() + indexes, definitions = LSP_SERVER.navigation_state() index = indexes.get(filepath) if index is None: return None - definitions = definition_identities(indexes) result = symbol_at_position(index, position, definitions) if result is None: return None @@ -786,12 +862,8 @@ def incoming_calls(params: lsp.CallHierarchyIncomingCallsParams): if identity is None: return [] - indexes = LSP_SERVER.navigation_snapshot() - return incoming_call_hierarchy( - identity, - indexes, - definition_identities(indexes), - ) + indexes, definitions = LSP_SERVER.navigation_state() + return incoming_call_hierarchy(identity, indexes, definitions) @LSP_SERVER.feature(lsp.CALL_HIERARCHY_OUTGOING_CALLS) @@ -800,12 +872,8 @@ def outgoing_calls(params: lsp.CallHierarchyOutgoingCallsParams): if identity is None: return [] - indexes = LSP_SERVER.navigation_snapshot() - return outgoing_call_hierarchy( - identity, - indexes, - definition_identities(indexes), - ) + indexes, definitions = LSP_SERVER.navigation_state() + return outgoing_call_hierarchy(identity, indexes, definitions) # ********************************************************** @@ -859,6 +927,7 @@ def initialize(params: lsp.InitializeParams) -> lsp.InitializeResult: log_to_output(f"sys.path used to run Server:\r\n {paths}") GLOBAL_SETTINGS.update(**params.initialization_options.get("globalSettings", {})) + INDEX_CACHE_PATH["path"] = params.initialization_options.get("indexCachePath") settings = params.initialization_options["settings"] _update_workspace_settings(settings) @@ -897,6 +966,8 @@ def initialized(_params: lsp.InitializedParams): log_to_output("Background indexing skipped: no workspace folder is open.") return log_to_output("Background indexing started...") + started = time.perf_counter() + LSP_SERVER.index_cache = IndexCache.load(INDEX_CACHE_PATH.get("path")) root_path = pathlib.Path(root) skipped_directories = { ".git", @@ -914,11 +985,13 @@ def initialized(_params: lsp.InitializedParams): document, cache_tree=False, require_file_exists=True, + from_disk=True, ) except Exception as error: log_to_output(f"Fehler beim Parsen von {filepath}: {error}") _refresh_psc_index() - log_to_output("Background indexing completed.") + LSP_SERVER.index_cache.save() + log_to_output(f"Background indexing completed in {time.perf_counter() - started:.1f}s.") except Exception as e: log_to_output(f"Background indexing failed: {e}") diff --git a/server/src/lsp_tclserver.py b/server/src/lsp_tclserver.py index 3b41f55..81bd5a8 100644 --- a/server/src/lsp_tclserver.py +++ b/server/src/lsp_tclserver.py @@ -3,6 +3,7 @@ import os import pathlib import threading import xml.etree.ElementTree as ET +from dataclasses import dataclass from typing import List, Optional, Tuple import lsprotocol.types as lsp @@ -13,14 +14,21 @@ from pygls.workspace.text_document import TextDocument from tclint.format import FormatterOpts from tclint.lexer import TclSyntaxError from tclint.violations import Violation -from tools import checks, parser +from tools import checks, incremental_parse, parser from tools.completion_items import CompletionCollector from tools.tcloo_symbols import class_completion_items from tools.tcloo_completion import indexed_classes -from tools.file_sourcing import get_all_psc_files, psc_script_files +from tools.def_symbols import DefSymbols, read_def_symbols +from tools.file_sourcing import get_all_psc_files, psc_defined_event_files, psc_script_files from tools.formatter import NxFormatter as Formatter +from tools.index_cache import FileStat, IndexCache, file_stat from tools.inlay_hint import InlayHintSignature, build_custom_inlay_signatures -from tools.navigation import FileSymbolIndex, build_file_symbol_index +from tools.navigation import ( + FileSymbolIndex, + SymbolIdentity, + build_file_symbol_index, + definition_identities, +) from tools.proc_docs import build_proc_docs from tools.variable_index import ProcRange, build_variable_index @@ -28,6 +36,18 @@ DIAGNOSTIC_SOURCE = "nx-post-support" LOGGER = logging.getLogger(__name__) +@dataclass(frozen=True) +class _FileIndex: + """Everything indexed for one file; also the persistent cache entry.""" + + completion_items: list[lsp.CompletionItem] + proc_signatures: dict[str, list[str]] + proc_docs: dict[str, str] + classes: dict + navigation_index: FileSymbolIndex + variable_index: tuple[set[str], dict[str, set[str]], list[ProcRange]] + + class TclLanguageServer(LanguageServer): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) @@ -42,6 +62,8 @@ class TclLanguageServer(LanguageServer): self.psc_script_paths: list[str] = [] self._psc_files: dict[str, list[pathlib.Path]] = {} self._psc_lock = threading.RLock() + # .def path -> BLOCK_TEMPLATE/ADDRESS names, in PSC DefinedEvents order. + self.def_symbols: dict[str, DefSymbols] = {} self.navigation_indexes: dict[str, FileSymbolIndex] = {} self.variable_indexes: dict[ str, @@ -54,6 +76,11 @@ class TclLanguageServer(LanguageServer): self._ast_cache = {} self._line_cache: dict[tuple[str, int | None], tuple[str, ...]] = {} self._parser_lock = threading.RLock() + self._thread_parsers = threading.local() + # uri -> (normalized source, tree, violations) of the last successful + # parse; survives version changes so edits can be reparsed partially. + self._last_parse: dict[str, tuple[str, object, list]] = {} + self.index_cache = IndexCache() self._index_lock = threading.RLock() self._index_tokens: dict[str, int] = {} self._index_versions: dict[str, int | None] = {} @@ -67,6 +94,9 @@ class TclLanguageServer(LanguageServer): -1, frozenset(), ) + self._definition_identities_cache: tuple[ + int, frozenset[SymbolIdentity] + ] = (-1, frozenset()) self._proc_metadata_cache: dict[ str, tuple[int, dict[str, list[str]], dict[str, str]] ] = {} @@ -78,16 +108,44 @@ class TclLanguageServer(LanguageServer): self._analysis_tokens: dict[str, int] = {} self._next_analysis_token = 0 - def _parse_source(self, source: str): + def _parse_source(self, source: str, pos=None): self.parser.violations = [] - tree = self.parser.parse(source) + tree = self.parser.parse(source, pos=pos) return tree, list(self.parser.violations) + def _parse_document(self, document: TextDocument): + """Parse a document version, reusing unchanged parts of the last one. + + Callers hold the parser lock. + """ + source = incremental_parse.normalize_newlines(document.source) + previous = self._last_parse.get(document.uri) + result = None + if previous is not None: + try: + result = incremental_parse.reparse( + *previous, source, self._parse_source + ) + except TclSyntaxError: + # E.g. a quote opened in the edit closes further down. + result = None + if result is None: + result = self._parse_source(source) + self._last_parse[document.uri] = (source, *result) + return result + def parse_source(self, source: str): - """Parse without retaining an AST, serialized around the shared parser.""" - with self._parser_lock: - tree, _ = self._parse_source(source) - return tree + """Parse without retaining an AST, on a parser owned by this thread. + + Background indexing must not hold the shared parser lock for whole + files while request handlers wait for their document's tree. + """ + local_parser = getattr(self._thread_parsers, "parser", None) + if local_parser is None: + # Plugin commands live in tclint's shared registry, see __init__. + local_parser = self._thread_parsers.parser = parser.CustomParser() + local_parser.violations = [] + return local_parser.parse(source) def get_tree(self, document: TextDocument): key = (document.uri, document.version) @@ -95,7 +153,7 @@ class TclLanguageServer(LanguageServer): cached = self._ast_cache.get(key) if cached is not None: return cached[0] - tree, violations = self._parse_source(document.source) + tree, violations = self._parse_document(document) self._ast_cache[key] = (tree, violations) return tree @@ -105,7 +163,7 @@ class TclLanguageServer(LanguageServer): cached = self._ast_cache.get(key) if cached is not None: return cached - tree, violations = self._parse_source(document.source) + tree, violations = self._parse_document(document) self._ast_cache[key] = (tree, violations) return tree, violations @@ -154,6 +212,7 @@ class TclLanguageServer(LanguageServer): self._index_generation += 1 self._workspace_completion_cache = (-1, ()) self._custom_function_names_cache = (-1, frozenset()) + self._definition_identities_cache = (-1, frozenset()) self._proc_metadata_cache.clear() self._custom_inlay_cache.clear() @@ -196,8 +255,48 @@ class TclLanguageServer(LanguageServer): classes.update(indexes.get(self._normalized_path(path), {})) return classes + def refresh_def_symbols(self, roots, report=LOGGER.warning): + """Read the block templates and addresses of all .def files listed as PSC DefinedEvents.""" + symbols: dict[str, DefSymbols] = {} + for root in roots: + for psc in get_all_psc_files(root): + try: + def_files = psc_defined_event_files(psc) + except (OSError, ET.ParseError) as error: + report(f"Could not read PSC {psc}: {error}") + continue + for def_file in def_files: + if str(def_file) in symbols: + continue + try: + symbols[str(def_file)] = read_def_symbols(def_file) + except OSError as error: + report(f"Could not read DEF file {def_file}: {error}") + with self._index_lock: + self.def_symbols = symbols + + def _def_symbol_items(self, attribute: str, kind, description: str) -> list[lsp.CompletionItem]: + with self._index_lock: + symbols = dict(self.def_symbols) + return [ + lsp.CompletionItem( + label=name, + kind=kind, + detail=f"{description} ({pathlib.Path(path).name})", + ) + for path, def_symbols in symbols.items() + for name in getattr(def_symbols, attribute) + ] + + def block_template_items(self) -> list[lsp.CompletionItem]: + return self._def_symbol_items("block_templates", lsp.CompletionItemKind.Struct, "Block template") + + def address_items(self) -> list[lsp.CompletionItem]: + return self._def_symbol_items("addresses", lsp.CompletionItemKind.Field, "Address") + def refresh_psc_scripts(self, roots, report=LOGGER.warning): """Index PSC dependencies through the same pipeline as workspace procs.""" + self.refresh_def_symbols(roots, report=report) with self._psc_lock: discovered = {} for root in roots: @@ -233,7 +332,9 @@ class TclLanguageServer(LanguageServer): report(f"PSC script not found: {path}") continue try: + source_stat = None if document is None: + source_stat = file_stat(path_string) data = path.read_bytes() try: source = data.decode("utf-8-sig") @@ -241,7 +342,7 @@ class TclLanguageServer(LanguageServer): # Older Windows NX layers use the ANSI code page. source = data.decode("cp1252") document = TextDocument(uri=uri, source=source, language_id="tcl") - if not self.update_poco_completion_for_file(document, cache_tree=False): + if not self.update_poco_completion_for_file(document, cache_tree=False, source_stat=source_stat): report(f"Could not index PSC script: {path}") except (OSError, UnicodeError) as error: report(f"Could not read PSC script {path}: {error}") @@ -505,6 +606,22 @@ class TclLanguageServer(LanguageServer): with self._index_lock: return dict(self.navigation_indexes) + def navigation_state( + self, + ) -> tuple[dict[str, FileSymbolIndex], frozenset[SymbolIdentity]]: + """Return indexes plus their definitions, cached by index generation.""" + with self._index_lock: + generation, definitions = self._definition_identities_cache + if generation != self._index_generation: + definitions = frozenset( + definition_identities(self.navigation_indexes) + ) + self._definition_identities_cache = ( + self._index_generation, + definitions, + ) + return dict(self.navigation_indexes), definitions + def _begin_index_update(self, filepath: str, version: int | None) -> int | None: with self._index_lock: indexed_version = self._index_versions.get(filepath) @@ -599,6 +716,33 @@ class TclLanguageServer(LanguageServer): if self._is_same_or_child(cached_path, target): self._ast_cache.pop(key, None) self._line_cache.pop(key, None) + for uri in list(self._last_parse): + try: + parsed_path = pathlib.Path(uris.to_fs_path(uri)) + except (TypeError, ValueError): + continue + if self._is_same_or_child(parsed_path, target): + del self._last_parse[uri] + + def _build_file_index( + self, document: TextDocument, filepath: str, cache_tree: bool + ) -> "_FileIndex": + tree = ( + self.get_tree(document) + if cache_tree + else self.parse_source(document.source) + ) + collector = CompletionCollector() + tree.accept(collector, recurse=True) + collector.custom_functions.extend(class_completion_items(tree)) + return _FileIndex( + completion_items=list(collector.custom_functions), + proc_signatures=dict(collector.proc_signatures), + proc_docs=build_proc_docs(tree, document.source), + classes=indexed_classes(tree, document.uri, document.source), + navigation_index=build_file_symbol_index(filepath, document.uri, tree), + variable_index=build_variable_index(document.source, tree), + ) def update_poco_completion_for_file( self, @@ -606,32 +750,32 @@ class TclLanguageServer(LanguageServer): *, cache_tree: bool = True, require_file_exists: bool = False, + from_disk: bool = False, + source_stat: FileStat | None = None, ): - """Update poco_completion for a specific file when it changes""" + """Update poco_completion for a specific file when it changes. + + `from_disk` marks documents that read their source from disk lazily; + `source_stat` is the file's stat taken before a caller read it. Such + results are served from and stored in the persistent index cache. + """ filepath = str(pathlib.Path(uris.to_fs_path(document.uri))) token = self._begin_index_update(filepath, document.version) if token is None: return False - collector = CompletionCollector() - try: - tree = ( - self.get_tree(document) - if cache_tree - else self.parse_source(document.source) - ) - tree.accept(collector, recurse=True) - collector.custom_functions.extend(class_completion_items(tree)) - classes = indexed_classes(tree, document.uri, document.source) - docs = build_proc_docs(tree, document.source) - navigation_index = build_file_symbol_index( - filepath, document.uri, tree - ) - variable_index = build_variable_index(document.source, tree) - except Exception as e: - LOGGER.debug("Error parsing %s: %s", filepath, e) - self._discard_index_update(filepath, token) - return False + stat = source_stat or (file_stat(filepath) if from_disk else None) + index = self.index_cache.get(filepath, stat) if stat is not None else None + if index is None: + try: + index = self._build_file_index(document, filepath, cache_tree) + except Exception as e: + LOGGER.debug("Error parsing %s: %s", filepath, e) + self._discard_index_update(filepath, token) + return False + # Only cache results whose file did not change while being read. + if stat is not None and file_stat(filepath) == stat: + self.index_cache.put(filepath, stat, index) if require_file_exists and not pathlib.Path(filepath).is_file(): self._discard_index_update(filepath, token) @@ -640,12 +784,12 @@ class TclLanguageServer(LanguageServer): with self._index_lock: if self._index_tokens.get(filepath) != token: return False - self.poco_completion[filepath] = list(collector.custom_functions) - self.proc_signatures[filepath] = dict(collector.proc_signatures) - self.proc_docs[filepath] = docs - self.class_indexes[filepath] = classes - self.navigation_indexes[filepath] = navigation_index - self.variable_indexes[filepath] = (document.version, variable_index) + self.poco_completion[filepath] = list(index.completion_items) + self.proc_signatures[filepath] = dict(index.proc_signatures) + self.proc_docs[filepath] = index.proc_docs + self.class_indexes[filepath] = index.classes + self.navigation_indexes[filepath] = index.navigation_index + self.variable_indexes[filepath] = (document.version, index.variable_index) self._committed_index_versions[filepath] = document.version self._invalidate_workspace_caches_locked() return True diff --git a/server/src/tools/def_symbols.py b/server/src/tools/def_symbols.py new file mode 100644 index 0000000..8e48b9c --- /dev/null +++ b/server/src/tools/def_symbols.py @@ -0,0 +1,36 @@ +"""Block templates and addresses declared in NX post definition (.def) files.""" + +import re +from dataclasses import dataclass, field +from pathlib import Path + +_BLOCK_TEMPLATE_RE = re.compile(r"^\s*BLOCK_TEMPLATE\s+([^\s{]+)", re.MULTILINE) +_ADDRESS_RE = re.compile(r"^\s*ADDRESS\s+([^\s{]+)", re.MULTILINE) + + +@dataclass(frozen=True) +class DefSymbols: + block_templates: tuple[str, ...] = () + addresses: tuple[str, ...] = () + + +def _names(pattern: re.Pattern, source: str) -> tuple[str, ...]: + return tuple(dict.fromkeys(pattern.findall(source))) + + +def parse_def_symbols(source: str) -> DefSymbols: + """Return the BLOCK_TEMPLATE and ADDRESS names of a .def source in declaration order.""" + return DefSymbols( + block_templates=_names(_BLOCK_TEMPLATE_RE, source), + addresses=_names(_ADDRESS_RE, source), + ) + + +def read_def_symbols(path: Path) -> DefSymbols: + data = path.read_bytes() + try: + source = data.decode("utf-8-sig") + except UnicodeDecodeError: + # Older Windows NX layers use the ANSI code page. + source = data.decode("cp1252") + return parse_def_symbols(source) diff --git a/server/src/tools/file_sourcing.py b/server/src/tools/file_sourcing.py index 936c30a..8643cae 100644 --- a/server/src/tools/file_sourcing.py +++ b/server/src/tools/file_sourcing.py @@ -1,6 +1,6 @@ import xml.etree.ElementTree as ET import os -from dataclasses import dataclass +from dataclasses import dataclass, field from typing import List, Optional from pathlib import Path @@ -10,6 +10,7 @@ class SourcedFile: layer_name: str subfolder: Optional[str] files: List[str] + defined_events: List[str] = field(default_factory=list) def read_psc_file(psc_file: Path) -> List[SourcedFile]: @@ -34,8 +35,22 @@ def read_psc_file(psc_file: Path) -> List[SourcedFile]: if name: script_names.append(name) + # DefinedEvents-Filenames (.def files) + defined_events = layer.find("DefinedEvents") + event_names = [] + if defined_events is not None: + for filename in defined_events.findall("Filename"): + name = filename.attrib.get("Name") + if name: + event_names.append(name) + layer_info_list.append( - SourcedFile(layer_name=layer_name, subfolder=subfolder, files=script_names) + SourcedFile( + layer_name=layer_name, + subfolder=subfolder, + files=script_names, + defined_events=event_names, + ) ) return layer_info_list @@ -44,20 +59,30 @@ def get_all_psc_files(root_path: Path) -> list[Path]: return sorted(root_path.rglob("*.psc"), key=lambda path: str(path).casefold()) -def psc_script_files(psc_file: Path) -> list[Path]: - """Resolve layer script paths relative to the PSC, preserving load order.""" - def expanded(value): - return Path(os.path.expandvars(value).replace("\\", "/")) +def _expanded(value) -> Path: + return Path(os.path.expandvars(value).replace("\\", "/")) + +def _layer_files(psc_file: Path, attribute: str, suffix: str) -> list[Path]: paths = [] for layer in read_psc_file(psc_file): folder = layer.subfolder or "." - base = psc_file.parent / expanded(os.environ.get(folder, folder)) - for name in layer.files: - filename = expanded(name) + base = psc_file.parent / _expanded(os.environ.get(folder, folder)) + for name in getattr(layer, attribute): + filename = _expanded(name) if not filename.suffix: - filename = filename.with_suffix(".tcl") + filename = filename.with_suffix(suffix) path = (base / filename).resolve() - if path.suffix.lower() == ".tcl": + if path.suffix.lower() == suffix: paths.append(path) return paths + + +def psc_script_files(psc_file: Path) -> list[Path]: + """Resolve layer script paths relative to the PSC, preserving load order.""" + return _layer_files(psc_file, "files", ".tcl") + + +def psc_defined_event_files(psc_file: Path) -> list[Path]: + """Resolve the layers' DefinedEvents (.def) paths relative to the PSC.""" + return _layer_files(psc_file, "defined_events", ".def") diff --git a/server/src/tools/incremental_parse.py b/server/src/tools/incremental_parse.py new file mode 100644 index 0000000..a0ab1de --- /dev/null +++ b/server/src/tools/incremental_parse.py @@ -0,0 +1,156 @@ +"""Reparse only the top-level commands touched by an edit. + +Tcl top-level commands are independent once the previous command ended on its +own line: the parser keeps no state between them. An edit is therefore +reparsed from the first to the last top-level command it touches, commands +before it are reused as-is and commands after it are reused with their line +numbers shifted. Whenever that assumption could break, the caller falls back +to a full parse. +""" + +from __future__ import annotations + +import copy +from collections.abc import Callable + +from tclint.syntax_tree import Node, Script +from tclint.violations import Violation + +ParseChunk = Callable[[str, tuple[int, int]], tuple[Script, list[Violation]]] + + +def normalize_newlines(source: str) -> str: + """Match the universal newline handling of the tclint parser.""" + return source.replace("\r\n", "\n").replace("\r", "\n") + + +def _shifted_tree(node: Node, delta: int) -> Node: + """Copy a subtree with its line numbers moved by `delta`. + + Cached trees may still be read by other requests, so nodes are never + mutated. Attributes such as `Command.routine` alias entries of `children`, + so every reference is remapped to the same copy. + """ + copies: dict[int, Node] = {} + + def shifted(original: Node) -> Node: + existing = copies.get(id(original)) + if existing is not None: + return existing + clone = object.__new__(type(original)) + copies[id(original)] = clone + state = dict(original.__dict__) + if state.get("line") is not None: + state["line"] += delta + end = state.get("end_pos") + if end is not None: + state["end_pos"] = (end[0] + delta, end[1]) + for key, value in state.items(): + if isinstance(value, Node): + state[key] = shifted(value) + elif isinstance(value, (list, tuple)) and value and isinstance(value[0], Node): + state[key] = type(value)(shifted(item) for item in value) + clone.__dict__.update(state) + return clone + + return shifted(node) + + +def _shifted_violation(violation: Violation, delta: int) -> Violation: + clone = copy.copy(violation) + clone.start = (violation.start[0] + delta, violation.start[1]) + clone.end = (violation.end[0] + delta, violation.end[1]) + return clone + + +def reparse( + old_source: str, + old_tree: Script, + old_violations: list[Violation], + new_source: str, + parse_chunk: ParseChunk, +) -> tuple[Script, list[Violation]] | None: + """Return the tree of `new_source`, or None when a full parse is needed. + + Both sources must already be newline-normalized. `parse_chunk` parses a + top-level fragment starting at the given (line, column) and may raise + TclSyntaxError, which the caller handles like any failed parse. + """ + if old_source == new_source: + return old_tree, list(old_violations) + + # Changed line range (1-indexed); lines outside it are identical. A pure + # insertion leaves last_changed_old == first_changed_line - 1. + old_lines = old_source.split("\n") + new_lines = new_source.split("\n") + limit = min(len(old_lines), len(new_lines)) + same_before = 0 + while same_before < limit and old_lines[same_before] == new_lines[same_before]: + same_before += 1 + same_after = 0 + while ( + same_after < limit - same_before + and old_lines[-1 - same_after] == new_lines[-1 - same_after] + ): + same_after += 1 + + first_changed_line = same_before + 1 + last_changed_old = len(old_lines) - same_after + delta = len(new_lines) - len(old_lines) + + commands = old_tree.children + if any(command.line is None or command.end_pos is None for command in commands): + return None + + # Commands overlapping the changed lines, widened so that no reused + # command shares a line with the reparsed range. + first = next( + (index for index, command in enumerate(commands) if command.end_pos[0] >= first_changed_line), + len(commands), + ) + start_line = first_changed_line + if first < len(commands): + start_line = min(start_line, commands[first].line) + while first > 0 and commands[first - 1].end_pos[0] >= start_line: + first -= 1 + start_line = min(start_line, commands[first].line) + + last = first - 1 + end_line_old = last_changed_old + while last + 1 < len(commands) and commands[last + 1].line <= end_line_old: + last += 1 + end_line_old = max(end_line_old, commands[last].end_pos[0]) + end_line_new = end_line_old + delta + + if end_line_new < start_line - 1 or end_line_new > len(new_lines): + return None + # A trailing backslash joins a line with the next one across the boundary. + if start_line > 1 and new_lines[start_line - 2].endswith("\\"): + return None + if end_line_new >= start_line and new_lines[end_line_new - 1].endswith("\\"): + return None + + chunk_commands: list[Node] = [] + chunk_violations: list[Violation] = [] + if end_line_new >= start_line: + chunk = "\n".join(new_lines[start_line - 1 : end_line_new]) + chunk_tree, chunk_violations = parse_chunk(chunk, (start_line, 1)) + chunk_commands = chunk_tree.children + + reused_after = [_shifted_tree(command, delta) for command in commands[last + 1 :]] + tree = Script( + *commands[:first], + *chunk_commands, + *reused_after, + pos=(old_tree.line, old_tree.col), + ) + tree.end_pos = (len(new_lines), len(new_lines[-1]) + 1) + + violations = [violation for violation in old_violations if violation.start[0] < start_line] + violations += chunk_violations + violations += [ + _shifted_violation(violation, delta) + for violation in old_violations + if violation.start[0] > end_line_old + ] + return tree, violations diff --git a/server/src/tools/index_cache.py b/server/src/tools/index_cache.py new file mode 100644 index 0000000..33e68cd --- /dev/null +++ b/server/src/tools/index_cache.py @@ -0,0 +1,128 @@ +"""Persist per-file index results across server restarts. + +Entries are keyed by path and validated by the file's size and mtime. The +whole cache is tied to a fingerprint of the code that produced it: the +indexing sources of this server, the bundled tclint sources and the versions +of all bundled libraries. Any change to them, including a tclint update or a +local patch, discards the cache instead of loading stale results. +""" + +from __future__ import annotations + +import hashlib +import logging +import os +import pathlib +import pickle +import sys +import tempfile +import threading +import zlib +from typing import Any + +LOGGER = logging.getLogger(__name__) + +# Bump when the cached data layout changes without a source change above. +CACHE_FORMAT = 1 +CACHE_FILE = "index-cache.pickle.z" + +_SRC_DIR = pathlib.Path(__file__).resolve().parent.parent +_LIBS_DIR = _SRC_DIR.parent / "libs" + +FileStat = tuple[int, int] + + +def code_fingerprint() -> str: + digest = hashlib.sha256() + digest.update(f"{CACHE_FORMAT}|{sys.version}".encode()) + sources = [ + _SRC_DIR / "lsp_tclserver.py", + *sorted((_SRC_DIR / "tools").glob("*.py")), + *sorted((_SRC_DIR / "plugins").glob("*.py")), + *sorted((_LIBS_DIR / "tclint").rglob("*.py")), + ] + for source in sources: + digest.update(source.relative_to(_SRC_DIR.parent).as_posix().encode()) + digest.update(source.read_bytes()) + for dist_info in sorted(_LIBS_DIR.glob("*.dist-info")): + digest.update(dist_info.name.encode()) + return digest.hexdigest() + + +def file_stat(path: str) -> FileStat | None: + try: + stat = os.stat(path) + except OSError: + return None + return stat.st_mtime_ns, stat.st_size + + +class IndexCache: + def __init__(self, directory: pathlib.Path | None = None, fingerprint: str = ""): + self._path = directory / CACHE_FILE if directory is not None else None + self._fingerprint = fingerprint + self._entries: dict[str, tuple[FileStat, Any]] = {} + self._used: set[str] = set() + self._dirty = False + self._lock = threading.Lock() + + @classmethod + def load(cls, directory: pathlib.Path | str | None) -> IndexCache: + """Open the cache in `directory`; without one, nothing is persisted.""" + if not directory: + return cls() + cache = cls(pathlib.Path(directory), code_fingerprint()) + try: + with open(cache._path, "rb") as file: + fingerprint, entries = pickle.loads(zlib.decompress(file.read())) + except FileNotFoundError: + return cache + except Exception as error: # A damaged cache must never stop indexing. + LOGGER.warning("Ignoring unreadable index cache %s: %s", cache._path, error) + cache._dirty = True + return cache + if fingerprint == cache._fingerprint: + cache._entries = entries + else: + cache._dirty = True + return cache + + def get(self, path: str, stat: FileStat) -> Any | None: + with self._lock: + entry = self._entries.get(path) + if entry is None or entry[0] != stat: + return None + self._used.add(path) + return entry[1] + + def put(self, path: str, stat: FileStat, data: Any) -> None: + if self._path is None: + return + with self._lock: + self._entries[path] = (stat, data) + self._used.add(path) + self._dirty = True + + def save(self) -> None: + """Write entries used in this session atomically; others are dropped.""" + if self._path is None: + return + with self._lock: + if not self._dirty and self._used == self._entries.keys(): + return + entries = {path: self._entries[path] for path in self._used if path in self._entries} + self._entries = entries + self._dirty = False + try: + self._path.parent.mkdir(parents=True, exist_ok=True) + with tempfile.NamedTemporaryFile(dir=self._path.parent, delete=False) as file: + data = pickle.dumps((self._fingerprint, entries), protocol=pickle.HIGHEST_PROTOCOL) + # Pickled indexes are very repetitive; fast compression cuts ~90%. + file.write(zlib.compress(data, 1)) + os.replace(file.name, self._path) + except Exception as error: + LOGGER.warning("Could not write index cache %s: %s", self._path, error) + try: + os.unlink(file.name) + except (OSError, NameError): + pass diff --git a/server/src/tools/inlay_hint.py b/server/src/tools/inlay_hint.py index a9063c2..e57bbd8 100644 --- a/server/src/tools/inlay_hint.py +++ b/server/src/tools/inlay_hint.py @@ -200,7 +200,7 @@ class InlayHintGenerator(Visitor): walk(tree) if self.parameter_names != "none": - for call in resolved_method_calls(self.source, self.external_classes): + for call in resolved_method_calls(self.source, self.external_classes, tree): if not self._node_intersects_requested_range(call.command): continue parameters = method_parameters(call.parameters) diff --git a/server/src/tools/navigation.py b/server/src/tools/navigation.py index 4410743..fb5fd52 100644 --- a/server/src/tools/navigation.py +++ b/server/src/tools/navigation.py @@ -265,6 +265,12 @@ def build_file_symbol_index( filepath: str, uri: str, tree: Node ) -> FileSymbolIndex: occurrences: list[SymbolOccurrence] = [] + # Most occurrences repeat a few identities; sharing one object per identity + # keeps the index (and its persistent cache) small. + identities: dict[SymbolIdentity, SymbolIdentity] = {} + + def shared(identity: SymbolIdentity | None) -> SymbolIdentity | None: + return None if identity is None else identities.setdefault(identity, identity) def add_proc( node: Node, @@ -274,7 +280,7 @@ def build_file_symbol_index( is_definition: bool, declaration_range: lsp.Range | None = None, ) -> None: - identity = _proc_identity(raw_name, scope.namespace) + identity = shared(_proc_identity(raw_name, scope.namespace)) caller = None if not is_definition: caller = ( @@ -288,14 +294,14 @@ def build_file_symbol_index( fallback_identity=( None if is_definition - else _proc_fallback(raw_name, scope.namespace) + else shared(_proc_fallback(raw_name, scope.namespace)) ), range=_name_range(node, raw_name), placeholder=_basename(raw_name), is_definition=is_definition, symbol_kind=lsp.SymbolKind.Function, container_name=_container_name(identity), - caller=caller, + caller=shared(caller), declaration_range=declaration_range, ) ) @@ -309,7 +315,7 @@ def build_file_symbol_index( variable_sub: bool = False, identity: SymbolIdentity | None = None, ) -> None: - symbol_identity = identity or _variable_identity(raw_name, scope) + symbol_identity = shared(identity or _variable_identity(raw_name, scope)) occurrences.append( SymbolOccurrence( identity=symbol_identity, @@ -506,6 +512,15 @@ def resolve_identity( return occurrence.identity +def _may_resolve_to(occurrence: SymbolOccurrence, identity: SymbolIdentity) -> bool: + """Cheap name pre-filter; resolve_identity only returns one of these two.""" + name = identity.name + fallback = occurrence.fallback_identity + return occurrence.identity.name == name or ( + fallback is not None and fallback.name == name + ) + + def symbol_at_position( index: FileSymbolIndex, position: lsp.Position, @@ -530,6 +545,8 @@ def matching_occurrences( matches = [] for index in indexes.values(): for occurrence in index.occurrences: + if not _may_resolve_to(occurrence, identity): + continue if resolve_identity(occurrence, definitions) == identity: matches.append((index, occurrence)) return matches @@ -543,6 +560,8 @@ def document_highlights( """Return all occurrences of one symbol in the active document.""" highlights = [] for occurrence in index.occurrences: + if not _may_resolve_to(occurrence, identity): + continue if resolve_identity(occurrence, definitions) != identity: continue diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py index c1f15e8..7363a4e 100644 --- a/server/src/tools/parser.py +++ b/server/src/tools/parser.py @@ -1,7 +1,9 @@ import io +import re from typing import Optional, Tuple from tclint.parser import Parser from tclint.commands import CommandArgError +from tclint.commands.checks import eval as eval_script_args from tclint.syntax_tree import ( BracedWord, BareWord, @@ -13,9 +15,32 @@ from tclint.syntax_tree import ( from tclint.lexer import TclSyntaxError, Lexer, TOK_EOF +_UPLEVEL_LEVEL_RE = re.compile(r"^#?\d+$") + + +def _uplevel(args, parser): + """uplevel ?level? arg ?arg ...?""" + # ref: https://www.tcl.tk/man/tcl/TclCmd/uplevel.html + if len(args) == 0: + raise CommandArgError("not enough args to 'uplevel': got 0, expected at least 1") + + # The level can only be omitted when the first arg doesn't look like one. + # A non-literal first arg (e.g. $level) is treated as a level as well. + level = [] + if len(args) > 1: + first = args[0].contents + if first is None or _UPLEVEL_LEVEL_RE.match(first): + level = args[0:1] + + return level + eval_script_args(args[len(level) :], parser, "uplevel") + + class CustomParser(Parser): def __init__(self, debug=False, command_plugins=None): super().__init__(debug, command_plugins) + # tclint only checks the arg count of uplevel; parse its body as a script + # so it gets formatted and linted like eval/namespace eval bodies. + self._commands = {**self._commands, "uplevel": _uplevel} # Used to normalize newlines consistently with open()'s universal newlines mode. self._decoder = io.IncrementalNewlineDecoder(None, True) diff --git a/server/src/tools/tcl_command_completion.py b/server/src/tools/tcl_command_completion.py index b560225..a3b0974 100644 --- a/server/src/tools/tcl_command_completion.py +++ b/server/src/tools/tcl_command_completion.py @@ -24,6 +24,10 @@ class DynamicCompletionKind(Enum): PROCEDURE = "procedure" NAMESPACE = "namespace" PATH = "path" + BLOCK_TEMPLATE = "block_template" + ADDRESS = "address" + # A variable substituted as a value, inserted with a leading "$". + VALUE = "value" @dataclass(frozen=True) @@ -424,6 +428,8 @@ for _string_class in STRING_CLASSES: VALUES_BY_POSITION: dict[tuple[tuple[str, ...], int], tuple[str, ...]] = { (("array", "names"), 3): ("-exact", "-glob", "-regexp"), + (("MOM_force",), 1): ("Always", "Once", "Off"), + (("MOM_suppress",), 1): ("Always", "Once", "Off"), (("close",), 2): ("read", "write"), (("open",), 2): ("r", "r+", "w", "w+", "a", "a+"), (("package", "prefer"), 2): ("latest", "stable"), @@ -468,6 +474,12 @@ DYNAMIC_COMPLETION_RULES = ( DynamicCompletionRule(("namespace", "origin"), frozenset({2}), DynamicCompletionKind.PROCEDURE), DynamicCompletionRule(("rename",), frozenset({1}), DynamicCompletionKind.PROCEDURE), # Namespace-taking commands. + DynamicCompletionRule(("MOM_do_template",), frozenset({1}), DynamicCompletionKind.BLOCK_TEMPLATE), + DynamicCompletionRule(("MOM_ask_address_value",), frozenset({1}), DynamicCompletionKind.ADDRESS), + DynamicCompletionRule(("MOM_force",), frozenset({1}), DynamicCompletionKind.VALUE), + DynamicCompletionRule(("MOM_force",), _REPEATED_SUBCOMMAND_ARGUMENTS, DynamicCompletionKind.ADDRESS), + DynamicCompletionRule(("MOM_suppress",), frozenset({1}), DynamicCompletionKind.VALUE), + DynamicCompletionRule(("MOM_suppress",), _REPEATED_SUBCOMMAND_ARGUMENTS, DynamicCompletionKind.ADDRESS), DynamicCompletionRule(("namespace", "children"), frozenset({2}), DynamicCompletionKind.NAMESPACE), DynamicCompletionRule( ("namespace", "delete"), @@ -685,13 +697,14 @@ def tcl_argument_completion(source_lines: Sequence[str], position: Position) -> for (path, argument_index), values in VALUES_BY_POSITION.items(): if active_index == argument_index and tuple(words[: len(path)]) == path: - return TclArgumentCompletion( - items=_completion_items( + return _merge_dynamic_completion( + _completion_items( values, CompletionItemKind.Value, f"{' '.join(path)} value", ), - active_prefix=active_prefix, + dynamic_completion, + active_prefix, ) for path in sorted(OPTIONS_BY_PATH, key=len, reverse=True): diff --git a/server/src/tools/tcloo_arguments.py b/server/src/tools/tcloo_arguments.py index 232d8ee..51bcbc2 100644 --- a/server/src/tools/tcloo_arguments.py +++ b/server/src/tools/tcloo_arguments.py @@ -38,14 +38,14 @@ def method_parameters(parameters: str) -> list[MethodParameter]: return [] -def method_signature_help(source: str, position: lsp.Position, external_classes=None) -> lsp.SignatureHelp | None: +def method_signature_help(source: str, position: lsp.Position, external_classes=None, tree=None) -> lsp.SignatureHelp | None: lines = source.split("\n") if position.line >= len(lines): return None # AST columns are codepoints; LSP columns are UTF-16 code units. prefix = lines[position.line].encode("utf-16-le")[:position.character * 2].decode("utf-16-le", errors="ignore") cursor = (position.line, len(prefix)) - candidates = [call for call in resolved_method_calls(source, external_classes) + candidates = [call for call in resolved_method_calls(source, external_classes, tree) if _contains_cursor(call.command, lines, cursor)] if not candidates: return None diff --git a/server/src/tools/tcloo_completion.py b/server/src/tools/tcloo_completion.py index 7caa1b8..fe420a3 100644 --- a/server/src/tools/tcloo_completion.py +++ b/server/src/tools/tcloo_completion.py @@ -1,6 +1,6 @@ """Static TclOO inference using local and indexed classes, without executing Tcl.""" -from collections.abc import Sequence +from collections.abc import Callable, Sequence from dataclasses import dataclass, field import re @@ -9,8 +9,30 @@ from tclint.lexer import TclSyntaxError from tclint.syntax_tree import BracedWord, Command, CommandSub, Script, VarSub from tools.parser import CustomParser +from tools.signature_help import _active_argument, _contains_cursor from tools.tcl_command_completion import line_prefix_at_position +_LEADING_RECEIVER = re.compile(r"\s*([A-Za-z_]\w*)\s+[\w:]*$") + + +def may_contain_classes(source, external_classes=None) -> bool: + """Cheap pre-check: without any class, TclOO analysis yields nothing.""" + return bool(external_classes) or "oo::class" in source + + +def _may_be_receiver(name, source_lines, external_classes) -> bool: + """Whether a bare command word can name a class, an object or `my`. + + Objects and local classes only come from `... create `, so a word + never created anywhere cannot resolve and needs no full-document parse. + """ + if name in {"my", "self", "next"}: + return True + if any(key.rsplit("::", 1)[-1] == name for key in external_classes or ()): + return True + created = re.compile(rf"\bcreate\s+[{{\"]?(?:[\w:]*::)?{re.escape(name)}\b") + return any("create" in line and created.search(line) for line in source_lines) + @dataclass class ClassInfo: @@ -74,10 +96,42 @@ def _qualified(name, namespace): return name if name.startswith("::") else f"{namespace}::{name}" +def _cursor_may_be_method_word(tree, source_lines, position) -> bool: + """Whether the innermost command at the cursor is at its first argument. + + The marker parse below can only succeed there, and the current document's + tree has the same structure apart from the marker. + """ + line = source_lines[position.line] + # AST columns are codepoints; LSP columns are UTF-16 code units. + prefix = line.encode("utf-16-le")[:position.character * 2].decode("utf-16-le", errors="ignore") + cursor = (position.line, len(prefix)) + lines = list(source_lines) + innermost = None + + def walk(node): + nonlocal innermost + start, end = getattr(node, "pos", None), getattr(node, "end_pos", None) + if start is not None and end is not None and not start[0] - 1 <= cursor[0] <= end[0] - 1: + return + if isinstance(node, Command) and _contains_cursor(node, lines, cursor): + innermost = node + for child in node.children: + walk(child) + + walk(tree) + return innermost is None or _active_argument(innermost, cursor) == 0 + + def tcloo_completions( source_lines: Sequence[str], position: lsp.Position, external_classes=None, + current_tree: Callable[[], Script | None] | None = None, ) -> list[lsp.CompletionItem] | None: - """Return receiver-specific methods, or None outside a known OO context.""" + """Return receiver-specific methods, or None outside a known OO context. + + `current_tree` lazily returns the parsed, unmodified document (or None) so + cursors that cannot hold a method name skip the full marker reparse. + """ prefix = line_prefix_at_position(source_lines, position) if prefix is None: return None @@ -87,6 +141,19 @@ def tcloo_completions( word_start = len(prefix) - len(typed) if word_start == 0 or prefix[word_start - 1] not in " \t": return None + if not external_classes and not any("oo::class" in line for line in source_lines): + return None + continued = position.line > 0 and source_lines[position.line - 1].endswith("\\") + if not continued: + # The first word is the command itself, never a method name. + if not prefix[:word_start].strip(): + return None + receiver = _LEADING_RECEIVER.match(prefix) + if receiver is not None and not _may_be_receiver(receiver.group(1), source_lines, external_classes): + return None + tree = current_tree() if current_tree is not None else None + if tree is not None and not _cursor_may_be_method_word(tree, source_lines, position): + return None marker = "__nx_tcloo_completion_cursor__" lines = list(source_lines) suffix = lines[position.line][len(prefix):] @@ -258,6 +325,9 @@ def _analyze(tree, typed="", marker="", external_classes=None, uri=None, source= return classes, result, calls -def resolved_method_calls(source, external_classes=None): - tree = parse_completion_source(source) +def resolved_method_calls(source, external_classes=None, tree=None): + if not may_contain_classes(source, external_classes): + return [] + if tree is None: + tree = parse_completion_source(source) return _analyze(tree, external_classes=external_classes)[2] if tree is not None else [] diff --git a/server/src/tools/tcloo_navigation.py b/server/src/tools/tcloo_navigation.py index 69b8bcb..86c48a4 100644 --- a/server/src/tools/tcloo_navigation.py +++ b/server/src/tools/tcloo_navigation.py @@ -1,11 +1,14 @@ """Definition targets for literal TclOO classes and resolved method calls.""" -from tools.tcloo_completion import _analyze, name_location, parse_completion_source +from tools.tcloo_completion import _analyze, may_contain_classes, name_location, parse_completion_source from tools.tcloo_symbols import class_symbols -def tcloo_definition(source, uri, position, external_classes=None): - tree = parse_completion_source(source) +def tcloo_definition(source, uri, position, external_classes=None, tree=None): + if not may_contain_classes(source, external_classes): + return None + if tree is None: + tree = parse_completion_source(source) if tree is None: return None classes, _, calls = _analyze(tree, external_classes=external_classes, uri=uri, source=source) diff --git a/server/tests/python_tests/test_debug_server.py b/server/tests/python_tests/test_debug_server.py index 4bc167b..6ec1285 100644 --- a/server/tests/python_tests/test_debug_server.py +++ b/server/tests/python_tests/test_debug_server.py @@ -1,4 +1,5 @@ import sys +import threading from pathlib import Path import pytest @@ -27,27 +28,46 @@ def test_debug_endpoint_rejects_invalid_port(monkeypatch, port): _debug_server._debug_endpoint() -def test_connect_debugger_retries_until_adapter_is_ready(monkeypatch): - class FakeDebugpy: - def __init__(self): - self.connect_calls = 0 - self.wait_calls = 0 +class FakeDebugpy: + def __init__(self, refuse=False, block_wait=False): + self.refuse = refuse + self.block_wait = block_wait + self.connect_calls = 0 + self.wait_calls = 0 - def connect(self, endpoint): - assert endpoint == ("127.0.0.1", 5678) - self.connect_calls += 1 - if self.connect_calls < 3: - raise ConnectionRefusedError("listener is starting") + def connect(self, endpoint): + assert endpoint == ("127.0.0.1", 5678) + self.connect_calls += 1 + if self.refuse: + raise ConnectionRefusedError("no listener") - def wait_for_client(self): - self.wait_calls += 1 + def wait_for_client(self): + self.wait_calls += 1 + if self.block_wait: + threading.Event().wait() + +def test_connect_debugger_connects_once_and_waits_for_client(): fake_debugpy = FakeDebugpy() - monkeypatch.setattr(_debug_server.time, "sleep", lambda _seconds: None) - _debug_server._connect_debugger( - fake_debugpy, "127.0.0.1", 5678, timeout=1.0 - ) + _debug_server._connect_debugger(fake_debugpy, "127.0.0.1", 5678, timeout=1.0) - assert fake_debugpy.connect_calls == 3 + assert fake_debugpy.connect_calls == 1 assert fake_debugpy.wait_calls == 1 + + +def test_connect_debugger_does_not_retry_refused_connection(): + # debugpy.connect() cannot be called a second time after a refused connection. + fake_debugpy = FakeDebugpy(refuse=True) + + with pytest.raises(RuntimeError, match="No debugpy listener"): + _debug_server._connect_debugger(fake_debugpy, "127.0.0.1", 5678, timeout=1.0) + + assert fake_debugpy.connect_calls == 1 + + +def test_connect_debugger_times_out_on_stale_adapter(): + fake_debugpy = FakeDebugpy(block_wait=True) + + with pytest.raises(RuntimeError, match="stale debugpy adapter"): + _debug_server._connect_debugger(fake_debugpy, "127.0.0.1", 5678, timeout=0.1) diff --git a/server/tests/python_tests/test_def_symbols.py b/server/tests/python_tests/test_def_symbols.py new file mode 100644 index 0000000..b7afc86 --- /dev/null +++ b/server/tests/python_tests/test_def_symbols.py @@ -0,0 +1,200 @@ +"""Completion of .def block templates and addresses from PSC DefinedEvents.""" + +import itertools +from pathlib import Path + +import lsprotocol.types as lsp # type: ignore +from pygls.workspace import Workspace + +import lsp_server +from lsp_tclserver import TclLanguageServer +from tools.def_symbols import parse_def_symbols +from tools.file_sourcing import psc_defined_event_files + +PSC = """ + + + + + + + + + +""" + +DEF = """MACHINE Default + +FORMATTING +{ + ADDRESS SPOS + { + FORMAT Coordinate + } + ADDRESS X { + } +\tBLOCK_TEMPLATE external_subprogram +\t{ +\t\tText[$lib_spf(value,subprogram_output_name)] +\t} +\tBLOCK_TEMPLATE steady_rest +\t{ +\t\tText[M60] +\t} +#\tBLOCK_TEMPLATE commented_out +} +""" + + +def _project(tmp_path: Path, monkeypatch): + (tmp_path / "service").mkdir() + psc = tmp_path / "post.psc" + psc.write_text(PSC, encoding="utf-8") + (tmp_path / "service" / "service.def").write_text(DEF, encoding="utf-8") + (tmp_path / "service" / "service.tcl").write_text("proc svc {} {}\n", encoding="utf-8") + + server = TclLanguageServer(name="block-template-test", version="1", max_workers=1) + server.protocol._workspace = Workspace( # pylint: disable=protected-access + root_uri=tmp_path.as_uri(), + sync_kind=lsp.TextDocumentSyncKind.Incremental, + workspace_folders=[lsp.WorkspaceFolder(uri=tmp_path.as_uri(), name="root")], + position_encoding=lsp.PositionEncodingKind.Utf16, + ) + monkeypatch.setattr(lsp_server, "LSP_SERVER", server) + server.refresh_psc_scripts([tmp_path]) + return server, psc + + +_versions = itertools.count(1) + + +def _complete(server, tmp_path: Path, source: str, trigger: str | None = None): + uri = (tmp_path / "caller.tcl").as_uri() + # A new version per request, the server caches lines by (uri, version). + server.workspace.put_text_document( + lsp.TextDocumentItem(uri=uri, language_id="tcl", version=next(_versions), text=source) + ) + lines = source.split("\n") + context = None + if trigger is not None: + context = lsp.CompletionContext( + trigger_kind=lsp.CompletionTriggerKind.TriggerCharacter, + trigger_character=trigger, + ) + result = lsp_server.on_completion( + lsp.CompletionParams( + text_document=lsp.TextDocumentIdentifier(uri=uri), + position=lsp.Position(line=len(lines) - 1, character=len(lines[-1])), + context=context, + ) + ) + return sorted(result.items, key=lambda item: item.sort_text or "") + + +def test_parse_def_symbols_ignores_comments(): + symbols = parse_def_symbols(DEF) + assert symbols.block_templates == ("external_subprogram", "steady_rest") + assert symbols.addresses == ("SPOS", "X") + + +def test_psc_defined_event_files_resolve_def_paths(tmp_path, monkeypatch): + _, psc = _project(tmp_path, monkeypatch) + assert psc_defined_event_files(psc) == [(tmp_path / "service" / "service.def").resolve()] + + +def test_mom_do_template_offers_templates_then_variables_on_space(tmp_path, monkeypatch): + server, _ = _project(tmp_path, monkeypatch) + items = _complete(server, tmp_path, "set globalValue 1\nMOM_do_template ", trigger=" ") + labels = [item.label for item in items] + assert labels[:2] == ["external_subprogram", "steady_rest"] + assert items[0].detail == "Block template (service.def)" + variable = next(item for item in items if item.label == "globalValue") + assert variable.insert_text == "$globalValue" + assert "svc" not in labels + + +def test_mom_do_template_ranks_templates_before_variables(tmp_path, monkeypatch): + server, _ = _project(tmp_path, monkeypatch) + items = _complete(server, tmp_path, "set globalValue 1\nMOM_do_template st") + labels = [item.label for item in items] + assert labels[:2] == ["external_subprogram", "steady_rest"] + assert "globalValue" in labels + + +def test_mom_do_template_dollar_still_completes_variables(tmp_path, monkeypatch): + server, _ = _project(tmp_path, monkeypatch) + items = _complete(server, tmp_path, "set globalValue 1\nMOM_do_template $glob") + labels = {item.label for item in items} + assert "globalValue" in labels + assert "steady_rest" not in labels + + +def test_def_change_refreshes_block_templates(tmp_path, monkeypatch): + server, _ = _project(tmp_path, monkeypatch) + def_file = tmp_path / "service" / "service.def" + def_file.write_text("BLOCK_TEMPLATE new_block\n{\n}\n", encoding="utf-8") + lsp_server.did_change_watched_files(lsp.DidChangeWatchedFilesParams(changes=[ + lsp.FileEvent(uri=def_file.as_uri(), type=lsp.FileChangeType.Changed)])) + assert [item.label for item in server.block_template_items()] == ["new_block"] + assert server.address_items() == [] + + +def test_mom_ask_address_value_offers_addresses_then_variables(tmp_path, monkeypatch): + server, _ = _project(tmp_path, monkeypatch) + items = _complete(server, tmp_path, "set globalValue 1\nMOM_ask_address_value ", trigger=" ") + labels = [item.label for item in items] + assert labels[:2] == ["SPOS", "X"] + assert items[0].detail == "Address (service.def)" + assert "globalValue" in labels + assert "steady_rest" not in labels + + +def test_mom_force_and_suppress_offer_mode_or_variable_then_addresses(tmp_path, monkeypatch): + server, _ = _project(tmp_path, monkeypatch) + for command in ("MOM_force", "MOM_suppress"): + items = _complete(server, tmp_path, f"set globalValue 1\n{command} ", trigger=" ") + labels = [item.label for item in items] + assert set(labels[:3]) == {"Always", "Once", "Off"} + variable = next(item for item in items if item.label == "globalValue") + assert variable.insert_text == "$globalValue" + assert "SPOS" not in labels + + for source in (f"{command} Once ", f"{command} Always SPOS "): + labels = [item.label for item in _complete(server, tmp_path, source, trigger=" ")] + assert labels[:2] == ["SPOS", "X"] + assert "Once" not in labels + + +def _edit(items, label): + item = next(item for item in items if item.label == label) + return item.text_edit.new_text, (item.text_edit.range.start.character, item.text_edit.range.end.character) + + +def test_def_symbols_and_modes_are_inserted_quoted(tmp_path, monkeypatch): + server, _ = _project(tmp_path, monkeypatch) + source = "MOM_do_template st" + assert _edit(_complete(server, tmp_path, source), "steady_rest") == ('"steady_rest"', (16, 18)) + source = "MOM_force " + assert _edit(_complete(server, tmp_path, source, trigger=" "), "Once") == ('"Once"', (10, 10)) + source = "MOM_force Once SP" + assert _edit(_complete(server, tmp_path, source), "SPOS") == ('"SPOS"', (15, 17)) + + +def test_typed_quotes_are_replaced_not_doubled(tmp_path, monkeypatch): + server, _ = _project(tmp_path, monkeypatch) + # Opening quote typed by the user. + source = 'MOM_ask_address_value "SP' + assert _edit(_complete(server, tmp_path, source), "SPOS") == ('"SPOS"', (22, 25)) + # Closing quote inserted by the editor after the cursor. + uri = (tmp_path / "caller.tcl").as_uri() + source = 'MOM_ask_address_value "SP"' + server.workspace.put_text_document( + lsp.TextDocumentItem(uri=uri, language_id="tcl", version=next(_versions), text=source) + ) + items = lsp_server.on_completion( + lsp.CompletionParams( + text_document=lsp.TextDocumentIdentifier(uri=uri), + position=lsp.Position(line=0, character=25), + ) + ).items + assert _edit(items, "SPOS") == ('"SPOS"', (22, 26)) diff --git a/server/tests/python_tests/test_format_uplevel.py b/server/tests/python_tests/test_format_uplevel.py new file mode 100644 index 0000000..3532b7c --- /dev/null +++ b/server/tests/python_tests/test_format_uplevel.py @@ -0,0 +1,33 @@ +"""Formatting of uplevel bodies.""" + +from tclint.format import FormatterOpts +from tools.formatter import NxFormatter +from tools.parser import CustomParser + + +def _format(source: str) -> str: + formatter = NxFormatter( + FormatterOpts( + indent="\t", + spaces_in_braces=False, + balanced_spaces_in_braces=False, + max_blank_lines=500, + indent_namespace_eval=True, + indent_mixed_tab_size=0, + emacs=False, + debug_whitespace=False, + ) + ) + return formatter.format_top(source, CustomParser()) + + +def test_uplevel_body_is_indented(): + source = "proc a {} {\n\tuplevel #0 {\n\tset x 1\n\t\tset y 2\n }\n}\n" + assert _format(source) == "proc a {} {\n\tuplevel #0 {\n\t\tset x 1\n\t\tset y 2\n\t}\n}\n" + + +def test_uplevel_without_level_and_with_variable_level(): + source = "uplevel {\nset x 1\n}\nuplevel $lvl {\nset y 2\n}\nuplevel set z 3\n" + assert _format(source) == ( + "uplevel {\n\tset x 1\n}\nuplevel $lvl {\n\tset y 2\n}\nuplevel set z 3\n" + ) diff --git a/server/tests/python_tests/test_incremental_parse.py b/server/tests/python_tests/test_incremental_parse.py new file mode 100644 index 0000000..1bed913 --- /dev/null +++ b/server/tests/python_tests/test_incremental_parse.py @@ -0,0 +1,131 @@ +import sys +from pathlib import Path + +import pytest + +THIS_DIR = Path(__file__).parent +SRC_DIR = THIS_DIR.parent.parent / "src" +if str(SRC_DIR) not in sys.path: + sys.path.insert(0, str(SRC_DIR)) + +from lsp_tclserver import TclLanguageServer # noqa: E402 +from pygls.workspace.text_document import TextDocument # noqa: E402 +from tclint.lexer import TclSyntaxError # noqa: E402 +from tclint.syntax_tree import Node # noqa: E402 +from tools.incremental_parse import reparse # noqa: E402 +from tools.parser import CustomParser # noqa: E402 + +SOURCE = """\ +# header comment +set a 1; set b 2 +proc first {x} { + global mom_pos + if {$x > 0} { + MOM_output_literal "first $x" + } + return [expr {$x + 1}] +} + +proc second {} { + set list [list a b \\ + c d] + return $list +} +lappend ::handlers {second} +""" + + +def _parse(text, pos=None): + parser = CustomParser() + tree = parser.parse(text, pos=pos) + return tree, list(parser.violations) + + +def _differences(a, b, path="root"): + if type(a) is not type(b): + return f"{path}: {type(a).__name__} != {type(b).__name__}" + for key in a.__dict__.keys() | b.__dict__.keys(): + first, second = a.__dict__.get(key), b.__dict__.get(key) + if isinstance(first, Node): + difference = _differences(first, second, f"{path}.{key}") + elif isinstance(first, (list, tuple)) and first and isinstance(first[0], Node): + if len(first) != len(second): + return f"{path}.{key}: {len(first)} != {len(second)}" + difference = next( + (d for i, (x, y) in enumerate(zip(first, second)) if (d := _differences(x, y, f"{path}.{key}[{i}]"))), + None, + ) + else: + difference = None if first == second else f"{path}.{key}: {first!r} != {second!r}" + if difference: + return difference + return None + + +def _violations(violations): + return [(str(v.id), v.message, v.start, v.end) for v in violations] + + +@pytest.mark.parametrize("old, new", [ + ("MOM_output_literal \"first $x\"", "MOM_output_literal \"first $x\" extra"), + (" return $list\n", " return $list\n puts done\n"), + ("proc second {} {", "\nproc second {} {"), + ("set a 1; set b 2\n", ""), + ("# header comment\n", "# header comment\nset inserted 0\n"), + ("lappend ::handlers {second}\n", "lappend ::handlers {second}\nproc third {} {}\n"), + (" c d]", " c d e]"), + ("global mom_pos", "global mom_pos mom_out_angle_pos"), +]) +def test_incremental_tree_matches_full_parse(old, new): + edited = SOURCE.replace(old, new, 1) + assert edited != SOURCE + previous = (SOURCE, *_parse(SOURCE)) + result = reparse(*previous, edited, _parse) + assert result is not None + expected = _parse(edited) + assert _differences(result[0], expected[0]) is None + assert _violations(result[1]) == _violations(expected[1]) + + +def test_continuation_across_the_edit_forces_full_parse(): + edited = SOURCE.replace("set a 1; set b 2", "set a 1; set b 2 \\") + assert reparse(SOURCE, *_parse(SOURCE), edited, _parse) is None + + +def test_quote_closing_outside_the_edit_is_left_to_the_full_parse(): + edited = SOURCE.replace("set a 1; set b 2", 'set a "1; set b 2') + try: + result = reparse(SOURCE, *_parse(SOURCE), edited, _parse) + except TclSyntaxError: + result = None + assert result is None + + +def test_unchanged_commands_are_reused_and_never_mutated(): + tree, violations = _parse(SOURCE) + edited = SOURCE.replace("return $list", "return [lsort $list]") + new_tree, _ = reparse(SOURCE, tree, violations, edited, _parse) + assert new_tree.children[0] is tree.children[0] + # Commands after the edit are shifted copies; the old tree stays valid. + inserted = SOURCE.replace("proc first", "\n\nproc first") + shifted_tree, _ = reparse(SOURCE, tree, violations, inserted, _parse) + assert shifted_tree.children[-1] is not tree.children[-1] + assert shifted_tree.children[-1].line == tree.children[-1].line + 2 + assert tree.children[-1].line == _parse(SOURCE)[0].children[-1].line + + +def test_server_reparses_edits_incrementally(tmp_path, monkeypatch): + server = TclLanguageServer(name="incremental-test", version="1", max_workers=1) + uri = (tmp_path / "edit.tcl").as_uri() + first = server.get_tree(TextDocument(uri=uri, source=SOURCE, version=1, language_id="tcl")) + + parsed_sources = [] + parse_source = server._parse_source + monkeypatch.setattr(server, "_parse_source", lambda text, pos=None: parsed_sources.append(text) or parse_source(text, pos)) + server.clear_cache_for_uri(uri) + edited = SOURCE.replace("return $list", "return [lsort $list]") + second = server.get_tree(TextDocument(uri=uri, source=edited, version=2, language_id="tcl")) + + assert parsed_sources and all(text != edited for text in parsed_sources) + assert second.children[0] is first.children[0] + assert _differences(second, _parse(edited)[0]) is None diff --git a/server/tests/python_tests/test_index_cache.py b/server/tests/python_tests/test_index_cache.py new file mode 100644 index 0000000..78b84f4 --- /dev/null +++ b/server/tests/python_tests/test_index_cache.py @@ -0,0 +1,109 @@ +import os +import sys +from pathlib import Path + +THIS_DIR = Path(__file__).parent +SRC_DIR = THIS_DIR.parent.parent / "src" +if str(SRC_DIR) not in sys.path: + sys.path.insert(0, str(SRC_DIR)) + +import tools.index_cache as index_cache # noqa: E402 +from lsp_tclserver import TclLanguageServer # noqa: E402 +from pygls.workspace.text_document import TextDocument # noqa: E402 +from tools.index_cache import IndexCache # noqa: E402 + + +def _index_from_disk(server, path: Path) -> bool: + return server.update_poco_completion_for_file( + TextDocument(uri=path.as_uri(), language_id="tcl"), + cache_tree=False, + require_file_exists=True, + from_disk=True, + ) + + +def _warm_server(cache_dir: Path) -> TclLanguageServer: + server = TclLanguageServer(name="cache-test", version="1", max_workers=1) + server.index_cache = IndexCache.load(cache_dir) + return server + + +def _fail_build(*_args, **_kwargs): + raise AssertionError("file was parsed although it is cached") + + +def test_second_start_uses_cached_index(tmp_path: Path, monkeypatch): + source = tmp_path / "post.tcl" + source.write_text("proc cached_proc {a b} { return $a }\n", encoding="utf-8") + cache_dir = tmp_path / "storage" + + server = _warm_server(cache_dir) + assert _index_from_disk(server, source) + server.index_cache.save() + + restarted = _warm_server(cache_dir) + monkeypatch.setattr(restarted, "_build_file_index", _fail_build) + assert _index_from_disk(restarted, source) + assert "cached_proc" in restarted.custom_function_names_snapshot() + assert restarted.proc_metadata_snapshot(str(source))[0]["cached_proc"] == ["a", "b"] + + +def test_changed_file_is_parsed_again(tmp_path: Path): + source = tmp_path / "post.tcl" + source.write_text("proc old_proc {} {}\n", encoding="utf-8") + cache_dir = tmp_path / "storage" + server = _warm_server(cache_dir) + assert _index_from_disk(server, source) + server.index_cache.save() + + source.write_text("proc new_proc {} {}\n", encoding="utf-8") + stat = source.stat() + os.utime(source, ns=(stat.st_atime_ns, stat.st_mtime_ns + 1_000_000_000)) + restarted = _warm_server(cache_dir) + assert _index_from_disk(restarted, source) + names = restarted.custom_function_names_snapshot() + assert "new_proc" in names and "old_proc" not in names + + +def test_code_change_discards_the_cache(tmp_path: Path, monkeypatch): + source = tmp_path / "post.tcl" + source.write_text("proc cached_proc {} {}\n", encoding="utf-8") + cache_dir = tmp_path / "storage" + server = _warm_server(cache_dir) + assert _index_from_disk(server, source) + server.index_cache.save() + + # E.g. an updated tclint: a different fingerprint must not load old entries. + monkeypatch.setattr(index_cache, "code_fingerprint", lambda: "other tclint") + restarted = _warm_server(cache_dir) + built = [] + build = restarted._build_file_index + monkeypatch.setattr(restarted, "_build_file_index", lambda *args: built.append(args) or build(*args)) + assert _index_from_disk(restarted, source) + assert built + + +def test_damaged_cache_is_ignored(tmp_path: Path, monkeypatch): + cache_dir = tmp_path / "storage" + cache_dir.mkdir() + (cache_dir / index_cache.CACHE_FILE).write_bytes(b"not a cache") + source = tmp_path / "post.tcl" + source.write_text("proc fresh_proc {} {}\n", encoding="utf-8") + + server = _warm_server(cache_dir) + assert _index_from_disk(server, source) + server.index_cache.save() + + restarted = _warm_server(cache_dir) + monkeypatch.setattr(restarted, "_build_file_index", _fail_build) + assert _index_from_disk(restarted, source) + + +def test_open_documents_never_touch_the_cache(tmp_path: Path): + source = tmp_path / "post.tcl" + source.write_text("proc on_disk {} {}\n", encoding="utf-8") + server = _warm_server(tmp_path / "storage") + unsaved = TextDocument(uri=source.as_uri(), source="proc unsaved {} {}\n", version=3, language_id="tcl") + assert server.update_poco_completion_for_file(unsaved) + server.index_cache.save() + assert not (tmp_path / "storage" / index_cache.CACHE_FILE).exists() diff --git a/server/tests/python_tests/test_index_stability.py b/server/tests/python_tests/test_index_stability.py index 384b653..3d88aa8 100644 --- a/server/tests/python_tests/test_index_stability.py +++ b/server/tests/python_tests/test_index_stability.py @@ -267,3 +267,35 @@ def test_variable_and_workspace_request_caches_are_reused(tmp_path: Path): assert first_completions is second_completions assert first_names is second_names assert "cached_proc" in first_names + + +def test_navigation_definitions_are_cached_until_the_index_changes(tmp_path: Path): + server = _server() + first = _document(tmp_path / "first.tcl", "proc first_proc {} {}") + assert server.update_poco_completion_for_file(first) + + indexes, definitions = server.navigation_state() + assert server.navigation_state()[1] is definitions + assert {identity.name for identity in definitions} >= {"::first_proc"} + + second = _document(tmp_path / "second.tcl", "proc second_proc {} {}") + assert server.update_poco_completion_for_file(second) + indexes, definitions = server.navigation_state() + assert second.path in indexes + assert {identity.name for identity in definitions} >= {"::first_proc", "::second_proc"} + + +def test_background_parse_does_not_wait_for_the_request_parser(tmp_path: Path): + server = _server() + document = _document(tmp_path / "background.tcl", "proc background_proc {} {}") + finished = Event() + + def index(): + assert server.update_poco_completion_for_file(document, cache_tree=False) + finished.set() + + with server._parser_lock: + with ThreadPoolExecutor(max_workers=1) as executor: + executor.submit(index) + assert finished.wait(timeout=5) + assert "background_proc" in server.custom_function_names_snapshot() diff --git a/server/tests/python_tests/test_tcloo_arguments.py b/server/tests/python_tests/test_tcloo_arguments.py index f9b269b..04fce87 100644 --- a/server/tests/python_tests/test_tcloo_arguments.py +++ b/server/tests/python_tests/test_tcloo_arguments.py @@ -98,3 +98,21 @@ def test_lsp_signature_help_with_unfinished_bracket(tmp_path, monkeypatch): )) assert result.active_parameter == 1 assert result.signatures[0].label == "::MCS initValue i value" + + +def test_cached_tree_gives_same_signature_without_reparsing(monkeypatch): + import tools.tcloo_completion as tcloo + + source = CLASS + "set mcs [MCS new test]\n$mcs initValue 0 " + position = lsp.Position(line=source.count("\n"), character=len(source.rsplit("\n", 1)[-1])) + expected = method_signature_help(source, position) + tree = CustomParser().parse(source) + parse_body = tcloo.parse_completion_source + + def parse(text, pos=None): + # Braced class bodies are still parsed on demand, the document is not. + assert text != source, "unexpected full-document parse" + return parse_body(text, pos) + + monkeypatch.setattr(tcloo, "parse_completion_source", parse) + assert method_signature_help(source, position, tree=tree) == expected diff --git a/server/tests/python_tests/test_tcloo_completion.py b/server/tests/python_tests/test_tcloo_completion.py index 61e5eb0..1cd0f28 100644 --- a/server/tests/python_tests/test_tcloo_completion.py +++ b/server/tests/python_tests/test_tcloo_completion.py @@ -1,7 +1,7 @@ import lsprotocol.types as lsp import pytest -from tools.tcloo_completion import tcloo_completions +from tools.tcloo_completion import ClassInfo, tcloo_completions CLASS = """oo::class create MCS { @@ -164,3 +164,68 @@ def test_namespaced_class_symbols_and_method_body_references(): assert set(declarations) == {"::geometry::MCS"} assert [node.contents for node in references] == ["MCS", "MCS", "geometry::MCS", "::geometry::MCS"] assert class_completion_items(tree)[0].label == "geometry::MCS" + + +@pytest.mark.parametrize("code", [ + "set mcs [MCS new]\nputs |", + "set mcs [MCS new]\n se|", + "set mcs [MCS new]\nMOM_output_literal |", +]) +def test_non_receivers_skip_the_completion_reparse(code, monkeypatch): + import tools.tcloo_completion as tcloo + + def fail(*_args, **_kwargs): + raise AssertionError("unexpected full-document parse") + + monkeypatch.setattr(tcloo, "parse_completion_source", fail) + assert complete(CLASS + code) is None + + +def test_documents_without_classes_skip_the_completion_reparse(monkeypatch): + import tools.tcloo_completion as tcloo + + monkeypatch.setattr(tcloo, "parse_completion_source", lambda *_: pytest.fail("parsed")) + assert complete("set value [expr 1]\n$value |") is None + + +def test_external_class_receiver_still_completes(): + source = "Logger |" + position = lsp.Position(line=0, character=len(source) - 1) + items = tcloo_completions([source.replace("|", "")], position, {"::Logger": ClassInfo()}) + assert {item.label for item in items} == {"new", "create"} + + +def complete_with_tree(source): + from tclint.lexer import TclSyntaxError + from tools.parser import CustomParser + + offset = source.index("|") + before = source[:offset] + position = lsp.Position(line=before.count("\n"), character=len(before.rsplit("\n", 1)[-1].encode("utf-16-le")) // 2) + text = source.replace("|", "") + try: + tree = CustomParser().parse(text) + except TclSyntaxError: + tree = None + return tcloo_completions(text.splitlines(), position, current_tree=lambda: tree) + + +@pytest.mark.parametrize("code", [ + "set mcs [MCS new]\n$mcs |", + "MCS create instance\ninstance |", + "set mcs [[MCS new] initOrg 1 2 3]\n$mcs |", + "proc run {} {set mcs [MCS new]; $mcs |}", + "set mcs [MCS new]\nputs [$mcs |]", + "set mcs [MCS new]\nputs [$mcs |", + "[MCS new] |", + "set mcs [MCS new]\n$mcs initV|alue", +]) +def test_cached_tree_keeps_method_completions(code): + assert "initValue" in {item.label for item in complete_with_tree(CLASS + code)} + + +def test_cached_tree_skips_reparse_outside_first_argument(monkeypatch): + import tools.tcloo_completion as tcloo + + monkeypatch.setattr(tcloo, "parse_completion_source", lambda *_: pytest.fail("parsed")) + assert complete_with_tree(CLASS + "set mcs [MCS new]\nif {$mcs ne {}} |{ puts 1 }") is None