"""Command-aware completion data and cursor parsing for Tcl commands.""" from __future__ import annotations from collections.abc import Sequence from dataclasses import dataclass from enum import Enum from pathlib import Path from lsprotocol.types import ( CompletionItem, CompletionItemKind, InsertTextFormat, Position, Range, TextEdit, ) class DynamicCompletionKind(Enum): """Workspace- or filesystem-backed completion requested by the grammar.""" VARIABLE = "variable" PROCEDURE = "procedure" NAMESPACE = "namespace" PATH = "path" BLOCK_TEMPLATE = "block_template" ADDRESS = "address" # A variable substituted as a value, inserted with a leading "$". VALUE = "value" @dataclass(frozen=True) class TclArgumentCompletion: """Static suggestions plus an optional dynamic completion category.""" items: tuple[CompletionItem, ...] = () dynamic_kind: DynamicCompletionKind | None = None active_prefix: str = "" path_extensions: tuple[str, ...] = () @dataclass(frozen=True) class DynamicCompletionRule: path: tuple[str, ...] argument_indices: frozenset[int] kind: DynamicCompletionKind path_extensions: tuple[str, ...] = () @dataclass(frozen=True) class OptionSpec: """A command option and whether it consumes the following word.""" label: str takes_value: bool = False values: tuple[str, ...] = () STRING_CLASSES = ( "alnum", "alpha", "ascii", "boolean", "control", "digit", "double", "entier", "false", "graph", "integer", "list", "lower", "print", "punct", "space", "true", "upper", "wideinteger", "wordchar", "xdigit", ) # Keys are complete command paths. Their values are valid words immediately # following that path. Keeping this declarative makes Tcl-version additions easy. SUBCOMMANDS_BY_PATH: dict[tuple[str, ...], tuple[str, ...]] = { ("array",): ( "anymore", "donesearch", "exists", "get", "names", "nextelement", "set", "size", "startsearch", "statistics", "unset", ), ("binary",): ("decode", "encode", "format", "scan"), ("binary", "decode"): ("base64", "hex", "uuencode"), ("binary", "encode"): ("base64", "hex", "uuencode"), ("chan",): ( "blocked", "close", "configure", "copy", "create", "eof", "event", "flush", "gets", "names", "pending", "pipe", "pop", "postevent", "push", "puts", "read", "seek", "tell", "truncate", ), ("clock",): ( "add", "clicks", "format", "microseconds", "milliseconds", "scan", "seconds", ), ("dict",): ( "append", "create", "exists", "filter", "for", "get", "incr", "info", "keys", "lappend", "map", "merge", "remove", "replace", "set", "size", "unset", "update", "values", "with", ), ("dict", "filter"): ("key", "script", "value"), ("encoding",): ("convertfrom", "convertto", "dirs", "names", "system"), ("file",): ( "atime", "attributes", "channels", "copy", "delete", "dirname", "executable", "exists", "extension", "isdirectory", "isfile", "join", "link", "lstat", "mkdir", "mtime", "nativename", "normalize", "owned", "pathtype", "readable", "readlink", "rename", "rootname", "separator", "size", "split", "stat", "system", "tail", "tempfile", "type", "volumes", "writable", ), ("info",): ( "args", "body", "class", "cmdcount", "commands", "complete", "coroutine", "default", "errorstack", "exists", "frame", "functions", "globals", "hostname", "level", "library", "loaded", "locals", "nameofexecutable", "object", "patchlevel", "procs", "script", "sharedlibextension", "tclversion", "vars", ), ("namespace",): ( "children", "code", "current", "delete", "ensemble", "eval", "exists", "export", "forget", "import", "inscope", "origin", "parent", "path", "qualifiers", "tail", "unknown", "upvar", "which", ), ("namespace", "ensemble"): ("configure", "create", "exists"), ("package",): ( "forget", "ifneeded", "names", "prefer", "present", "provide", "require", "unknown", "vcompare", "versions", "vsatisfies", ), ("string",): ( "bytelength", "cat", "compare", "equal", "first", "index", "is", "last", "length", "map", "match", "range", "repeat", "replace", "reverse", "tolower", "totitle", "toupper", "trim", "trimleft", "trimright", "wordend", "wordstart", ), } def _options(*labels: str) -> tuple[OptionSpec, ...]: return tuple(OptionSpec(label) for label in labels) OPTIONS_BY_PATH: dict[tuple[str, ...], tuple[OptionSpec, ...]] = { ("unset",): _options("nocomplain"), ("binary", "decode", "base64"): (OptionSpec("-strict"),), ("binary", "encode", "base64"): ( OptionSpec("-maxlen", takes_value=True), OptionSpec("-wrapchar", takes_value=True), ), ("clock", "format"): ( OptionSpec("-format", takes_value=True), OptionSpec("-gmt", takes_value=True, values=("0", "1")), OptionSpec("-locale", takes_value=True), OptionSpec("-timezone", takes_value=True), ), ("clock", "scan"): ( OptionSpec("-base", takes_value=True), OptionSpec("-format", takes_value=True), OptionSpec("-gmt", takes_value=True, values=("0", "1")), OptionSpec("-locale", takes_value=True), OptionSpec("-timezone", takes_value=True), ), ("exec",): _options("-ignorestderr", "-keepnewline", "--"), ("file", "copy"): _options("-force", "--"), ("file", "delete"): _options("-force", "--"), ("file", "link"): _options("-symbolic", "-hard"), ("file", "rename"): _options("-force", "--"), ("glob",): ( OptionSpec("-directory", takes_value=True), OptionSpec("-join"), OptionSpec("-nocomplain"), OptionSpec("-path", takes_value=True), OptionSpec("-tails"), OptionSpec("-types", takes_value=True), OptionSpec("--"), ), ("lsearch",): ( OptionSpec("-all"), OptionSpec("-ascii"), OptionSpec("-bisect"), OptionSpec("-decreasing"), OptionSpec("-dictionary"), OptionSpec("-exact"), OptionSpec("-glob"), OptionSpec("-increasing"), OptionSpec("-index", takes_value=True), OptionSpec("-inline"), OptionSpec("-integer"), OptionSpec("-nocase"), OptionSpec("-not"), OptionSpec("-real"), OptionSpec("-regexp"), OptionSpec("-sorted"), OptionSpec("-start", takes_value=True), OptionSpec("-subindices"), ), ("lsort",): ( OptionSpec("-ascii"), OptionSpec("-command", takes_value=True), OptionSpec("-decreasing"), OptionSpec("-dictionary"), OptionSpec("-increasing"), OptionSpec("-index", takes_value=True), OptionSpec("-indices"), OptionSpec("-integer"), OptionSpec("-nocase"), OptionSpec("-real"), OptionSpec("-stride", takes_value=True), OptionSpec("-unique"), ), ("namespace", "which"): _options("-command", "-variable"), ("package", "present"): _options("-exact"), ("package", "require"): _options("-exact"), ("puts",): _options("-nonewline"), ("regexp",): ( OptionSpec("-about"), OptionSpec("-all"), OptionSpec("-expanded"), OptionSpec("-indices"), OptionSpec("-inline"), OptionSpec("-line"), OptionSpec("-lineanchor"), OptionSpec("-linestop"), OptionSpec("-nocase"), OptionSpec("-start", takes_value=True), OptionSpec("--"), ), ("regsub",): ( OptionSpec("-all"), OptionSpec("-command"), OptionSpec("-expanded"), OptionSpec("-line"), OptionSpec("-lineanchor"), OptionSpec("-linestop"), OptionSpec("-nocase"), OptionSpec("-start", takes_value=True), OptionSpec("--"), ), ("return",): ( OptionSpec( "-code", takes_value=True, values=("ok", "error", "return", "break", "continue"), ), OptionSpec("-errorcode", takes_value=True), OptionSpec("-errorinfo", takes_value=True), OptionSpec("-errorstack", takes_value=True), OptionSpec("-level", takes_value=True), OptionSpec("-options", takes_value=True), ), ("source",): (OptionSpec("-encoding", takes_value=True),), ("string", "compare"): ( OptionSpec("-nocase"), OptionSpec("-length", takes_value=True), ), ("string", "equal"): ( OptionSpec("-nocase"), OptionSpec("-length", takes_value=True), ), ("string", "map"): _options("-nocase"), ("string", "match"): _options("-nocase"), ("switch",): ( OptionSpec("-exact"), OptionSpec("-glob"), OptionSpec("-indexvar", takes_value=True), OptionSpec("-matchvar", takes_value=True), OptionSpec("-nocase"), OptionSpec("-regexp"), OptionSpec("--"), ), } # ``string is`` takes its class before its options, so each class is a concrete # command path for the generic option resolver below. for _string_class in STRING_CLASSES: OPTIONS_BY_PATH[("string", "is", _string_class)] = ( OptionSpec("-strict"), OptionSpec("-failindex", takes_value=True), ) VALUES_BY_POSITION: dict[tuple[tuple[str, ...], int], tuple[str, ...]] = { (("array", "names"), 3): ("-exact", "-glob", "-regexp"), (("MOM_force",), 1): ("Always", "Once", "Off"), (("MOM_suppress",), 1): ("Always", "Once", "Off"), (("close",), 2): ("read", "write"), (("open",), 2): ("r", "r+", "w", "w+", "a", "a+"), (("package", "prefer"), 2): ("latest", "stable"), (("seek",), 3): ("start", "current", "end"), (("string", "is"), 2): STRING_CLASSES, } _REPEATED_ARGUMENTS = frozenset(range(1, 33)) _REPEATED_SUBCOMMAND_ARGUMENTS = frozenset(range(2, 33)) DYNAMIC_COMPLETION_RULES = ( # Variable-taking commands. DynamicCompletionRule(("append",), frozenset({1}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("array", "exists"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("array", "get"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("array", "names"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("array", "set"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("array", "size"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("array", "statistics"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("array", "unset"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("catch",), frozenset({2, 3}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("dict", "append"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("dict", "incr"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("dict", "lappend"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("dict", "set"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("dict", "unset"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("dict", "update"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("dict", "with"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("global",), _REPEATED_ARGUMENTS, DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("incr",), frozenset({1}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("info", "exists"), frozenset({2}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("lappend",), frozenset({1}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("set",), frozenset({1}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("unset",), _REPEATED_ARGUMENTS, DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("variable",), frozenset({1}), DynamicCompletionKind.VARIABLE), DynamicCompletionRule(("vwait",), frozenset({1}), DynamicCompletionKind.VARIABLE), # Procedure-taking commands. DynamicCompletionRule(("info", "args"), frozenset({2}), DynamicCompletionKind.PROCEDURE), DynamicCompletionRule(("info", "body"), frozenset({2}), DynamicCompletionKind.PROCEDURE), DynamicCompletionRule(("info", "default"), frozenset({2}), DynamicCompletionKind.PROCEDURE), DynamicCompletionRule(("namespace", "origin"), frozenset({2}), DynamicCompletionKind.PROCEDURE), DynamicCompletionRule(("rename",), frozenset({1}), DynamicCompletionKind.PROCEDURE), # Namespace-taking commands. DynamicCompletionRule(("MOM_do_template",), frozenset({1}), DynamicCompletionKind.BLOCK_TEMPLATE), DynamicCompletionRule(("MOM_ask_address_value",), frozenset({1}), DynamicCompletionKind.ADDRESS), DynamicCompletionRule(("MOM_force",), frozenset({1}), DynamicCompletionKind.VALUE), DynamicCompletionRule(("MOM_force",), _REPEATED_SUBCOMMAND_ARGUMENTS, DynamicCompletionKind.ADDRESS), DynamicCompletionRule(("MOM_suppress",), frozenset({1}), DynamicCompletionKind.VALUE), DynamicCompletionRule(("MOM_suppress",), _REPEATED_SUBCOMMAND_ARGUMENTS, DynamicCompletionKind.ADDRESS), DynamicCompletionRule(("namespace", "children"), frozenset({2}), DynamicCompletionKind.NAMESPACE), DynamicCompletionRule( ("namespace", "delete"), _REPEATED_SUBCOMMAND_ARGUMENTS, DynamicCompletionKind.NAMESPACE, ), DynamicCompletionRule(("namespace", "eval"), frozenset({2}), DynamicCompletionKind.NAMESPACE), DynamicCompletionRule(("namespace", "exists"), frozenset({2}), DynamicCompletionKind.NAMESPACE), DynamicCompletionRule(("namespace", "parent"), frozenset({2}), DynamicCompletionKind.NAMESPACE), # Path-taking commands. Source files are narrowed to Tcl while directories # remain visible so users can continue navigating. DynamicCompletionRule(("cd",), frozenset({1}), DynamicCompletionKind.PATH), DynamicCompletionRule(("load",), frozenset({1}), DynamicCompletionKind.PATH, (".dll", ".so", ".dylib")), DynamicCompletionRule(("open",), frozenset({1}), DynamicCompletionKind.PATH), DynamicCompletionRule(("source",), frozenset({1, 3}), DynamicCompletionKind.PATH, (".tcl",)), *( DynamicCompletionRule( ("file", subcommand), _REPEATED_SUBCOMMAND_ARGUMENTS, DynamicCompletionKind.PATH, ) for subcommand in ("copy", "delete", "join", "link", "mkdir", "rename") ), *( DynamicCompletionRule( ("file", subcommand), frozenset({2}), DynamicCompletionKind.PATH, ) for subcommand in ( "atime", "attributes", "dirname", "executable", "exists", "extension", "isdirectory", "isfile", "lstat", "mtime", "nativename", "normalize", "owned", "pathtype", "readable", "readlink", "rootname", "separator", "size", "split", "stat", "system", "tail", "type", "writable", ) ), ) def _snippet_item(label: str, insert_text: str, detail: str) -> CompletionItem: return CompletionItem( label=label, kind=CompletionItemKind.Snippet, detail=detail, insert_text=insert_text, insert_text_format=InsertTextFormat.Snippet, ) TCL_COMMAND_SNIPPET_ITEMS = { "foreach": _snippet_item( "foreach", "foreach ${1:item} ${2:list} {\n\t${0}\n}", "Tcl foreach loop", ), "if": _snippet_item( "if", "if {${1:condition}} {\n\t${0}\n}", "Tcl if block", ), "proc": _snippet_item( "proc", "proc ${1:name} {${2:arguments}} {\n\t${0}\n}", "Tcl procedure", ), "switch": _snippet_item( "switch", "switch -- ${1:value} {\n\t${2:pattern} {\n\t\t${0}\n\t}\n}", "Tcl switch block", ), "try": _snippet_item( "try", "try {\n\t${1}\n} on error {${2:message} ${3:options}} {\n\t${0}\n}", "Tcl try/on error block", ), } ARGUMENT_SNIPPETS_BY_PATH = { ("dict", "for"): _snippet_item( "dict for loop", "{${1:key} ${2:value}} ${3:dictionary} {\n\t${0}\n}", "Arguments and body for dict for", ), ("foreach",): _snippet_item( "foreach loop", "${1:item} ${2:list} {\n\t${0}\n}", "Arguments and body for foreach", ), ("if",): _snippet_item( "if block", "{${1:condition}} {\n\t${0}\n}", "Condition and body for if", ), ("proc",): _snippet_item( "procedure", "${1:name} {${2:arguments}} {\n\t${0}\n}", "Name, arguments, and body for proc", ), ("switch",): _snippet_item( "switch block", "-- ${1:value} {\n\t${2:pattern} {\n\t\t${0}\n\t}\n}", "Value, patterns, and body for switch", ), ("try",): _snippet_item( "try/on error block", "{\n\t${1}\n} on error {${2:message} ${3:options}} {\n\t${0}\n}", "Body and error handler for try", ), } SUBCOMMAND_SNIPPET_ITEMS = { ("dict", "for"): _snippet_item( "for", "for {${1:key} ${2:value}} ${3:dictionary} {\n\t${0}\n}", "dict for loop", ) } TCL_COMMAND_NAMES = tuple( sorted({path[0] for path in SUBCOMMANDS_BY_PATH} | {path[0] for path in OPTIONS_BY_PATH} | set(TCL_COMMAND_SNIPPET_ITEMS) | {rule.path[0] for rule in DYNAMIC_COMPLETION_RULES}) ) TCL_COMMAND_ITEMS = tuple( TCL_COMMAND_SNIPPET_ITEMS.get( command, CompletionItem( label=command, kind=CompletionItemKind.Function, detail="Tcl command", insert_text=command, ), ) for command in TCL_COMMAND_NAMES ) def line_prefix_at_position(source_lines: Sequence[str], position: Position) -> str | None: """Return the current line before an LSP UTF-16 position.""" if position.line < 0 or position.line >= len(source_lines): return None line = source_lines[position.line] codepoint_offset = _codepoint_offset(line, position.character) if codepoint_offset is None: return None return line[:codepoint_offset] def tcl_argument_completion(source_lines: Sequence[str], position: Position) -> TclArgumentCompletion | None: """Describe static and dynamic argument completion at ``position``. ``None`` means that the cursor is not at a command-specific completion position and the caller should fall back to normal symbol completion. """ line_prefix = line_prefix_at_position(source_lines, position) if line_prefix is None: return None segment = _current_command_segment(line_prefix) if not segment.strip() or segment.lstrip().startswith("#"): return None words = _tokenize_command_segment(segment) if not words: return None words[0] = words[0].removeprefix("::") active_index = len(words) - 1 active_prefix = words[active_index] completed_path = tuple(words[:active_index]) dynamic_completion = _dynamic_completion(words, active_index, active_prefix) subcommands = SUBCOMMANDS_BY_PATH.get(completed_path) if subcommands is not None: items = tuple( SUBCOMMAND_SNIPPET_ITEMS.get( (*completed_path, label), CompletionItem( label=label, kind=CompletionItemKind.EnumMember, detail=f"{' '.join(completed_path)} subcommand", insert_text=label, ), ) for label in subcommands ) return TclArgumentCompletion(items=items, active_prefix=active_prefix) argument_snippet = ARGUMENT_SNIPPETS_BY_PATH.get(completed_path) argument_items = (argument_snippet,) if argument_snippet is not None else () for (path, argument_index), values in VALUES_BY_POSITION.items(): if active_index == argument_index and tuple(words[: len(path)]) == path: return _merge_dynamic_completion( _completion_items( values, CompletionItemKind.Value, f"{' '.join(path)} value", ), dynamic_completion, active_prefix, ) for path in sorted(OPTIONS_BY_PATH, key=len, reverse=True): if active_index < len(path) or tuple(words[: len(path)]) != path: continue option_completion = _option_completion( path, OPTIONS_BY_PATH[path], words[len(path) : active_index], active_prefix, ) if option_completion is not None: if active_prefix.startswith("-") or (path == ("unset",) and active_index == 1 and not active_prefix): return option_completion return _merge_dynamic_completion( (*argument_items, *option_completion.items), dynamic_completion, active_prefix, ) if argument_items: return _merge_dynamic_completion(argument_items, dynamic_completion, active_prefix) return dynamic_completion def _option_completion( path: tuple[str, ...], options: tuple[OptionSpec, ...], completed_arguments: Sequence[str], active_prefix: str, ) -> TclArgumentCompletion | None: option_by_label = {option.label: option for option in options} used_options: set[str] = set() argument_index = 0 while argument_index < len(completed_arguments): argument = completed_arguments[argument_index] if argument == "--": return None option = option_by_label.get(argument) if option is None: # Tcl options precede normal operands for the commands covered here. return None used_options.add(option.label) argument_index += 1 if not option.takes_value: continue if argument_index >= len(completed_arguments): if option.values: return TclArgumentCompletion( items=_completion_items( option.values, CompletionItemKind.Value, f"{option.label} value", ), active_prefix=active_prefix, ) return None argument_index += 1 if active_prefix and not active_prefix.startswith("-"): return None remaining_options = tuple(option.label for option in options if option.label not in used_options) return TclArgumentCompletion( items=_completion_items( remaining_options, CompletionItemKind.Keyword, f"{' '.join(path)} option", ), active_prefix=active_prefix, ) def _dynamic_completion(words: Sequence[str], active_index: int, active_prefix: str) -> TclArgumentCompletion | None: for rule in sorted(DYNAMIC_COMPLETION_RULES, key=lambda item: len(item.path), reverse=True): if active_index in rule.argument_indices and tuple(words[: len(rule.path)]) == rule.path: return TclArgumentCompletion( dynamic_kind=rule.kind, active_prefix=active_prefix, path_extensions=rule.path_extensions, ) return None def _merge_dynamic_completion( items: Sequence[CompletionItem], dynamic_completion: TclArgumentCompletion | None, active_prefix: str, ) -> TclArgumentCompletion: if dynamic_completion is None: return TclArgumentCompletion(items=tuple(items), active_prefix=active_prefix) return TclArgumentCompletion( items=tuple(items), dynamic_kind=dynamic_completion.dynamic_kind, active_prefix=active_prefix, path_extensions=dynamic_completion.path_extensions, ) def _completion_items(labels: Sequence[str], kind: CompletionItemKind, detail: str) -> tuple[CompletionItem, ...]: return tuple( CompletionItem( label=label, kind=kind, detail=detail, insert_text=label, ) for label in labels ) def path_completion_items( base_directory: Path, completion: TclArgumentCompletion, position: Position, *, limit: int = 200, ) -> tuple[CompletionItem, ...]: """Complete one filesystem path relative to the current Tcl document.""" raw_prefix = completion.active_prefix separator_index = max(raw_prefix.rfind("/"), raw_prefix.rfind("\\")) typed_directory = raw_prefix[: separator_index + 1] name_prefix = raw_prefix[separator_index + 1 :] normalized_directory = typed_directory.replace("\\", "/") filesystem_directory = Path(normalized_directory) if not filesystem_directory.is_absolute(): filesystem_directory = base_directory / filesystem_directory try: entries = sorted( filesystem_directory.iterdir(), key=lambda entry: (not entry.is_dir(), entry.name.casefold()), ) except (OSError, ValueError): return () allowed_extensions = {extension.casefold() for extension in completion.path_extensions} replace_start = max( 0, position.character - len(raw_prefix.encode("utf-16-le")) // 2, ) replace_range = Range( start=Position(line=position.line, character=replace_start), end=position, ) items: list[CompletionItem] = [] for entry in entries: if not entry.name.casefold().startswith(name_prefix.casefold()): continue try: is_directory = entry.is_dir() except OSError: continue if not is_directory and allowed_extensions and entry.suffix.casefold() not in allowed_extensions: continue escaped_name = "".join(f"\\{character}" if character.isspace() else character for character in entry.name) new_text = f"{normalized_directory}{escaped_name}" if is_directory: new_text += "/" items.append( CompletionItem( label=new_text, kind=(CompletionItemKind.Folder if is_directory else CompletionItemKind.File), detail="Directory" if is_directory else "File", text_edit=TextEdit(range=replace_range, new_text=new_text), ) ) if len(items) >= limit: break return tuple(items) def _current_command_segment(line_prefix: str) -> str: """Select the innermost unfinished command from a line prefix.""" brace_starts: list[int] = [] bracket_starts: list[int] = [] last_command_delimiter = -1 in_quote = False escaped = False for index, char in enumerate(line_prefix): if escaped: escaped = False continue if char == "\\": escaped = True continue if char == '"' and not brace_starts: in_quote = not in_quote continue if not in_quote: if char == "{": brace_starts.append(index) continue if char == "}" and brace_starts: brace_starts.pop() continue if not brace_starts: if char == "[": bracket_starts.append(index) continue if char == "]" and bracket_starts: bracket_starts.pop() continue if char == ";" and not in_quote: last_command_delimiter = index starts = [last_command_delimiter + 1] if brace_starts: starts.append(brace_starts[-1] + 1) if bracket_starts: starts.append(bracket_starts[-1] + 1) start = max(starts) if brace_starts and start == brace_starts[-1] + 1: # An unfinished braced body or expression can itself contain commands # (for example, ``if {[string compare ...``). Scan that inner context # independently, while keeping completed braced arguments opaque. return _current_command_segment(line_prefix[start:]) return line_prefix[start:] def _tokenize_command_segment(segment: str) -> list[str]: words: list[str] = [] current: list[str] = [] brace_depth = 0 in_quote = False escaped = False ended_with_separator = False word_started = False for char in segment: if escaped: current.append(char) escaped = False ended_with_separator = False word_started = True continue if char == "\\": current.append(char) escaped = True ended_with_separator = False word_started = True continue if char == '"' and brace_depth == 0: in_quote = not in_quote ended_with_separator = False word_started = True continue if not in_quote and char == "{": brace_depth += 1 ended_with_separator = False word_started = True continue if not in_quote and char == "}" and brace_depth: brace_depth -= 1 ended_with_separator = False continue if char.isspace() and not in_quote and brace_depth == 0: if current or word_started: words.append("".join(current)) current = [] word_started = False ended_with_separator = bool(words) continue current.append(char) ended_with_separator = False word_started = True if current or word_started: words.append("".join(current)) elif ended_with_separator: words.append("") return words def _codepoint_offset(text: str, utf16_offset: int) -> int | None: if utf16_offset < 0: return None consumed = 0 for index, char in enumerate(text): if consumed == utf16_offset: return index consumed += 2 if ord(char) > 0xFFFF else 1 if consumed > utf16_offset: return None if consumed == utf16_offset: return len(text) return None