Merge pull request 'Recognize stored COMMANDBLOCK procs and warn on unknown .def names' (#49) from enhancements into main

CHANGELOG.md

@@ -7,11 +7,13 @@ Versions correspond to the Git tags of this repository.
 
 ### Added
 
+- Procedures stored in PostConfigurator COMMANDBLOCK properties (`CONF_CTRL_tool set auto_preselect_last_template {custom_header}`) are highlighted as procedures and support Go to Definition, Find References, Rename, and the call hierarchy
 - Go to Definition from block template and address arguments in Tcl (`MOM_do_template`, `MOM_force`, `MOM_suppress`, `MOM_ask_address_value`, ...) to the `BLOCK_TEMPLATE`/`ADDRESS` declaration in the PSC `.def` files
 - Hover over block templates shows the template body; hover over addresses shows format, leader, trailer, min/max, and modality
 - Find References and Rename for block templates and addresses across Tcl and `.def` files, including addresses used inside block templates
 - Go to Definition, hover, references, and rename also work inside `.def` files
 - Hover and Go to Definition also recognize block template and address names that reach an NX command through a local variable (`set`, `lappend`, `list`, `foreach`) or through the parameter of a custom proc such as `LIB_SPF_call_cycle "absolute_mode"`, including nested wrapper procs; same-named strings without such a path are not recognized, and these derived names are not renamed
+- Warning for block template and address names in NX commands (`MOM_do_template stedy_rest`) that no loaded `.def` file declares; names built at runtime (`$var`, `"CYCLE_$x"`) are not checked, and the warnings update when a `.def` file changes
 - Completion items for block templates and addresses (`BLOCK_LIST`, `ADDR_LIST`, `MOM_do_template`, ...) show the same preview as the hover
 
 ### Documentation

server/src/lsp_tclserver.py

@@ -18,7 +18,7 @@ from tools import checks, incremental_parse, parser
 from tools.completion_items import CompletionCollector
 from tools.tcloo_symbols import class_completion_items
 from tools.tcloo_completion import indexed_classes
-from tools.def_flow import WrapperTable, build_wrapper_table
+from tools.def_flow import WrapperTable, build_wrapper_table, unknown_def_names
 from tools.def_symbols import ADDRESS, BLOCK_TEMPLATE, DefDocument, parse_def_document, read_def_source
 from tools.file_sourcing import get_all_psc_files, psc_defined_event_files, psc_script_files
 from tools.formatter import NxFormatter as Formatter
@@ -276,7 +276,20 @@ class TclLanguageServer(LanguageServer):
                     except OSError as error:
                         report(f"Could not read DEF file {def_file}: {error}")
         with self._index_lock:
+            changed = documents != self.def_documents
             self.def_documents = documents
+            if changed:
+                # Unknown template/address warnings depend on the .def files.
+                self.diagnostics.clear()
+        if changed:
+            self._request_diagnostic_refresh()
+
+    def _request_diagnostic_refresh(self) -> None:
+        # Set by the initialize request; absent before it and in tests.
+        capabilities = getattr(self.protocol, "client_capabilities", None)
+        diagnostics = getattr(getattr(capabilities, "workspace", None), "diagnostics", None)
+        if getattr(diagnostics, "refresh_support", False):
+            self.workspace_diagnostic_refresh(None)
 
     def def_documents_snapshot(self, current_path=None, current_source: str | None = None) -> dict[str, DefDocument]:
         """Return the PSC .def documents; ``current_source`` replaces the file being edited."""
@@ -898,8 +911,27 @@ class TclLanguageServer(LanguageServer):
                 )
             )
 
+        diagnostics.extend(self._def_diagnostics(document))
         return diagnostics
 
+    def _def_diagnostics(self, document: TextDocument) -> List[lsp.Diagnostic]:
+        documents = self.def_documents_snapshot()
+        declared = {
+            kind: frozenset(name for def_document in documents.values() for name in def_document.names(kind))
+            for kind in (BLOCK_TEMPLATE, ADDRESS)
+        }
+        labels = {BLOCK_TEMPLATE: "Block template", ADDRESS: "Address"}
+        return [
+            lsp.Diagnostic(
+                message=f"{labels[kind]} '{name}' is not declared in any loaded .def file",
+                severity=lsp.DiagnosticSeverity.Warning,
+                range=range_,
+                code=f"unknown-{kind.replace('_', '-')}",
+                source=DIAGNOSTIC_SOURCE,
+            )
+            for kind, name, range_ in unknown_def_names(self.get_tree(document), declared)
+        ]
+
     def _compute_diagnostics(self, document: TextDocument) -> List[lsp.Diagnostic]:
         return self.lint(document)
 

server/src/tools/def_flow.py

@@ -338,3 +338,27 @@ def derived_def_symbol(
         targets.extend(target for variable in variables for target in scope_targets.get(variable, ()))
     kinds = frozenset(kind for target in targets for kind in _target_kinds(target, table, namespace))
     return (kinds, *literal) if kinds else None
+
+
+def _all_commands(node: Node) -> Iterator[Command]:
+    for child in getattr(node, "children", []):
+        if isinstance(child, Command):
+            yield child
+        yield from _all_commands(child)
+
+
+def unknown_def_names(tree: Node, declared: dict[str, frozenset[str]]) -> list[tuple[str, str, lsp.Range]]:
+    """Literal NX command arguments naming a block template or address no .def file declares.
+
+    ``declared`` maps each kind to its declared names; kinds without any
+    declaration are not checked, since their .def file is not loaded.
+    """
+    unknown = []
+    for command in _all_commands(tree):
+        for node, kind in def_argument_kinds(command):
+            names = declared.get(kind)
+            name = def_name(node) if names else None
+            if name is not None and name not in names:
+                line, column = node.contents_pos
+                unknown.append((kind, name, _range(line, column, name)))
+    return unknown

server/src/tools/navigation.py

@@ -15,6 +15,7 @@ from tools.def_flow import (  # noqa: F401  (re-exported)
     proc_def_flows,
 )
 from tools.def_flow import def_name as _def_name
+from tools.stored_procs import stored_command_names
 from tools.variable_names import array_key_parts, variable_name
 
 ROOT_NAMESPACE = "::"
@@ -294,6 +295,7 @@ def build_file_symbol_index(
         *,
         is_definition: bool,
         declaration_range: lsp.Range | None = None,
+        name_range: lsp.Range | None = None,
     ) -> None:
         identity = shared(_proc_identity(raw_name, scope.namespace))
         caller = None
@@ -311,7 +313,7 @@ def build_file_symbol_index(
                     if is_definition
                     else shared(_proc_fallback(raw_name, scope.namespace))
                 ),
-                range=_name_range(node, raw_name),
+                range=name_range or _name_range(node, raw_name),
                 placeholder=_basename(raw_name),
                 is_definition=is_definition,
                 symbol_kind=lsp.SymbolKind.Function,
@@ -457,6 +459,18 @@ def build_file_symbol_index(
 
         if routine:
             add_proc(command.routine, routine, scope, is_definition=False)
+        for name, line, column in stored_command_names(command):
+            start = column - 1 + (name.rfind("::") + 2 if "::" in name else 0)
+            add_proc(
+                command.args[2],
+                name,
+                scope,
+                is_definition=False,
+                name_range=lsp.Range(
+                    start=lsp.Position(line=line - 1, character=start),
+                    end=lsp.Position(line=line - 1, character=column - 1 + len(name)),
+                ),
+            )
         if flow_facts:
             flow_facts[-1].extend(command_flow_facts(command))
 

server/src/tools/semantic_tokens.py

@@ -5,6 +5,7 @@ import attrs
 from common.load_data import standard_items
 from tclint.commands.plugins import PluginManager
 from tclint.syntax_tree import BareWord, BracedWord, Command, QuotedWord, Visitor
+from tools.stored_procs import stored_command_names
 from tools.variable_names import variable_name
 from tools.tcloo_symbols import class_symbols
 from tools.tcloo_completion import _analyze
@@ -182,6 +183,11 @@ class _Highlighter(Visitor):
                     line, col = argument.contents_pos
                     self._append_token((line - 1, col - 1), len(argument.contents), "function", [])
 
+        # Procedures stored in COMMANDBLOCK properties (CONF_x set prop {proc}).
+        for stored_name, line, col in stored_command_names(command):
+            if stored_name in self._custom_function_names or stored_name in _STANDARD_PROC_NAMES:
+                self._append_token((line - 1, col - 1), len(stored_name), "function", [])
+
         # Highlight functions (custom or standard) when used as the routine
         name = getattr(routine, "contents", None)
         if name:

server/src/tools/stored_procs.py

@@ -0,0 +1,80 @@
+"""Procedure names stored as data and called later.
+
+PostConfigurator COMMANDBLOCK properties hold a Tcl list whose elements are
+executed as commands (see LIB_CONF_do_prop_custom_proc), e.g.
+``CONF_CTRL_tool set auto_preselect_last_template {custom_header}`` or
+``CONF_CTRL_moves set return_safety_pos {{OEM_output arg}}``. The first word of
+each element is a command name; only braced values are considered.
+"""
+
+from __future__ import annotations
+
+import re
+
+from tclint.syntax_tree import BracedWord, Command, Node
+
+_CONF_OBJECT_RE = re.compile(r"^(::)?CONF_\w+$")
+_COMMAND_NAME_RE = re.compile(r"[A-Za-z_:][\w:]*")
+
+
+def _static(node: Node | None) -> str | None:
+    value = getattr(node, "contents", None)
+    return value if isinstance(value, str) else None
+
+
+def _list_element_spans(text: str) -> list[tuple[int, int]]:
+    """(start, end) offsets of the top-level elements of a Tcl list, braces stripped."""
+    spans = []
+    index = 0
+    while index < len(text):
+        if text[index].isspace():
+            index += 1
+            continue
+        if text[index] == "{":
+            depth, start = 0, index + 1
+            while index < len(text):
+                if text[index] == "\\":
+                    index += 2
+                    continue
+                depth += {"{": 1, "}": -1}.get(text[index], 0)
+                index += 1
+                if depth == 0:
+                    break
+            spans.append((start, index - 1))
+        elif text[index] == '"':
+            start = index + 1
+            index = text.find('"', start)
+            index = len(text) if index < 0 else index
+            spans.append((start, index))
+            index += 1
+        else:
+            start = index
+            while index < len(text) and not text[index].isspace():
+                index += 1
+            spans.append((start, index))
+    return spans
+
+
+def stored_command_names(command: Command) -> list[tuple[str, int, int]]:
+    """Command names stored in ``command`` as (name, line, column), 1-based."""
+    routine = _static(command.routine)
+    args = command.args
+    if not (routine and _CONF_OBJECT_RE.match(routine) and len(args) >= 3 and _static(args[0]) == "set"):
+        return []
+    value = args[2]
+    text = _static(value)
+    if not isinstance(value, BracedWord) or not text or value.contents_pos is None:
+        return []
+    line, column = value.contents_pos
+    names = []
+    for start, end in _list_element_spans(text):
+        element = text[start:end]
+        match = _COMMAND_NAME_RE.match(element, len(element) - len(element.lstrip()))
+        if match is None or (match.end() < len(element) and not element[match.end()].isspace()):
+            continue
+        offset = start + match.start()
+        before = text[:offset]
+        name_line = line + before.count("\n")
+        name_column = offset - before.rfind("\n") if "\n" in before else column + offset
+        names.append((match.group(), name_line, name_column))
+    return names

server/tests/python_tests/test_def_flow.py

@@ -180,3 +180,44 @@ def test_derived_names_are_not_renamed(tmp_path, monkeypatch):
         text_document=lsp.TextDocumentIdentifier(uri=caller.as_uri()), position=_position("absolute_mode")
     )
     assert lsp_server.prepare_rename(params) is None
+
+
+def _warnings(server, tmp_path: Path, source: str):
+    uri = (tmp_path / "check.tcl").as_uri()
+    server.workspace.put_text_document(lsp.TextDocumentItem(uri=uri, language_id="tcl", version=1, text=source))
+    diagnostics = server.lint(server.workspace.get_text_document(uri))
+    return [
+        (diagnostic.code, diagnostic.message, diagnostic.range.start.line, diagnostic.range.start.character, diagnostic.range.end.character)
+        for diagnostic in diagnostics
+        if diagnostic.code in {"unknown-block-template", "unknown-address"}
+    ]
+
+
+def test_unknown_template_and_address_are_warned(tmp_path, monkeypatch):
+    server, _ = _project(tmp_path, monkeypatch)
+    source = 'proc p {} {\n    MOM_do_template stedy_rest\n    MOM_force Once SPOS "SPSO"\n}\n'
+    assert _warnings(server, tmp_path, source) == [
+        ("unknown-block-template", "Block template 'stedy_rest' is not declared in any loaded .def file", 1, 20, 30),
+        ("unknown-address", "Address 'SPSO' is not declared in any loaded .def file", 2, 25, 29),
+    ]
+
+
+def test_known_and_dynamic_names_are_not_warned(tmp_path, monkeypatch):
+    server, _ = _project(tmp_path, monkeypatch)
+    source = 'MOM_do_template steady_rest CREATE\nMOM_do_template $name\nMOM_do_template "CYCLE_$x"\nMOM_force Once SPOS\n'
+    assert _warnings(server, tmp_path, source) == []
+
+
+def test_no_warnings_without_loaded_def_files(tmp_path, monkeypatch):
+    server, _ = _project(tmp_path, monkeypatch)
+    server.def_documents = {}
+    assert _warnings(server, tmp_path, "MOM_do_template stedy_rest\n") == []
+
+
+def test_def_change_invalidates_cached_diagnostics(tmp_path, monkeypatch):
+    server, caller = _project(tmp_path, monkeypatch)
+    server.compute_diagnostics(server.workspace.get_text_document(caller.as_uri()))
+    assert server.diagnostic_snapshot(caller.as_uri()) is not None
+    (tmp_path / "service" / "service.def").write_text(DEF.replace("absolute_mode", "incremental_mode"), encoding="utf-8")
+    server.refresh_def_symbols([tmp_path])
+    assert server.diagnostic_snapshot(caller.as_uri()) is None

server/tests/python_tests/test_stored_procs.py

@@ -0,0 +1,92 @@
+"""Procedures stored in PostConfigurator COMMANDBLOCK properties."""
+
+from pathlib import Path
+
+import lsprotocol.types as lsp  # type: ignore
+
+from lsp_server import LSP_SERVER, goto_definition, references
+from tools.parser import CustomParser
+from tools.semantic_tokens import _Highlighter
+from tools.stored_procs import stored_command_names
+
+SOURCE = """proc custom_header {} {}
+proc ::ns::output {args} {}
+CONF_CTRL_tool set auto_preselect_last_template {custom_header}
+CONF_CTRL_moves set return_safety_pos {{::ns::output 1} custom_header}
+CONF_CTRL_moves set return_end_of_pgm {4th5th}
+CONF_CTRL_tool set auto_preselect_template "custom_header"
+set x {custom_header}
+"""
+
+
+def _names(source: str) -> list[tuple[str, int, int]]:
+    tree = CustomParser().parse(source)
+    return [name for command in tree.children for name in stored_command_names(command)]
+
+
+def test_stored_command_names_take_the_first_word_of_braced_list_elements():
+    # Values like {4th5th} are options, not procedure names.
+    assert _names(SOURCE) == [
+        ("custom_header", 3, 50),
+        ("::ns::output", 4, 41),
+        ("custom_header", 4, 57),
+    ]
+
+
+def test_stored_command_names_span_lines_and_skip_non_names():
+    source = "CONF_x set p {\n\t{first 1}\n\t\"second\"\n\t{$var}\n\t{a-b}\n}\n"
+    assert _names(source) == [("first", 2, 3), ("second", 3, 3)]
+
+
+def _document(tmp_path: Path) -> str:
+    uri = (tmp_path / "stored.tcl").as_uri()
+    LSP_SERVER.workspace.put_text_document(lsp.TextDocumentItem(uri=uri, language_id="tcl", version=1, text=SOURCE))
+    LSP_SERVER.update_poco_completion_for_file(LSP_SERVER.workspace.get_text_document(uri))
+    return uri
+
+
+def _at(needle: str, occurrence: int = 0) -> lsp.Position:
+    index = -1
+    for _ in range(occurrence + 1):
+        index = SOURCE.index(needle, index + 1)
+    line = SOURCE.count("\n", 0, index)
+    return lsp.Position(line=line, character=index - (SOURCE.rfind("\n", 0, index) + 1) + 1)
+
+
+def test_goto_definition_from_stored_proc(tmp_path):
+    uri = _document(tmp_path)
+    for needle, occurrence in (("custom_header", 1), ("custom_header", 2), ("output", 1)):
+        [location] = goto_definition(
+            lsp.DefinitionParams(text_document=lsp.TextDocumentIdentifier(uri=uri), position=_at(needle, occurrence))
+        )
+        assert location.range.start.line == (0 if needle == "custom_header" else 1)
+
+
+def test_references_include_stored_procs_but_not_plain_strings(tmp_path):
+    uri = _document(tmp_path)
+    found = references(
+        lsp.ReferenceParams(
+            text_document=lsp.TextDocumentIdentifier(uri=uri),
+            position=_at("custom_header"),
+            context=lsp.ReferenceContext(include_declaration=False),
+        )
+    )
+    # The shared server also holds the files of other tests.
+    assert [(location.range.start.line, location.range.start.character) for location in found if location.uri == uri] == [(2, 49), (3, 56)]
+
+
+def test_stored_procs_are_highlighted_only_when_known():
+    tree = CustomParser().parse(SOURCE)
+    highlighter = _Highlighter([], {"file": [lsp.CompletionItem(label="custom_header")]})
+    tree.accept(highlighter, recurse=True)
+    line = column = 0
+    functions = []
+    for token in highlighter.tokens():
+        column = column + token.offset if token.line == 0 else token.offset
+        line += token.line
+        if token.tok_type == "function":
+            functions.append((line, SOURCE.splitlines()[line][column:column + token.length]))
+    assert (2, "custom_header") in functions
+    assert (3, "custom_header") in functions
+    assert not any(text == "4th5th" for _, text in functions)
+    assert not any(line in {5, 6} for line, _ in functions)