From 528c7a6b9e8a1d48a19c60e63f19d81ae61f3523 Mon Sep 17 00:00:00 2001 From: christoph_xd Date: Sun, 3 Aug 2025 21:54:45 +0200 Subject: [PATCH 1/6] add more ignore files --- .vscodeignore | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.vscodeignore b/.vscodeignore index a94eca4..fea41e9 100644 --- a/.vscodeignore +++ b/.vscodeignore @@ -11,8 +11,11 @@ esbuild.js .gitea .venv/** .nox/** +server/.venv/** +server/.nox/** **/__pycache__/** **/requirements.txt **/requirements.in **/server/src/_debug_server.py -noxfile.py \ No newline at end of file +noxfile.py +client/** \ No newline at end of file -- 2.54.0 From 57c65221d32e0c8948ba2907e59b35e7e24b0d15 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Mon, 4 Aug 2025 09:17:30 +0200 Subject: [PATCH 2/6] add LIB_GE_command_buffer_edit_replace to plugins --- server/src/plugins/poco_plugin.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/server/src/plugins/poco_plugin.py b/server/src/plugins/poco_plugin.py index eac8dd3..4582443 100644 --- a/server/src/plugins/poco_plugin.py +++ b/server/src/plugins/poco_plugin.py @@ -36,6 +36,12 @@ def lib_ge_command_buffer_edit_prepend(args, parser): ) +def lib_ge_command_buffer_edit_replace(args, parser): + _lib_ge_command_buffer_edit( + args, parser, "LIB_GE_command_buffer_edit_replace", 2, 4 + ) + + def lib_ge_command_buffer_edit_insert(args, parser): _lib_ge_command_buffer_edit(args, parser, "LIB_GE_command_buffer_edit_insert", 2, 6) @@ -44,5 +50,6 @@ commands = [ {"LIB_GE_command_buffer_edit_append": lib_ge_command_buffer_edit_append}, {"LIB_GE_command_buffer_edit_prepend": lib_ge_command_buffer_edit_prepend}, {"LIB_GE_command_buffer_edit_insert": lib_ge_command_buffer_edit_insert}, + {"LIB_GE_command_buffer_edit_replace": lib_ge_command_buffer_edit_replace}, {"LIB_GE_command_buffer": _lib_ge_command_buffer}, ] -- 2.54.0 From fd5b1f2e71ba85d34fc4f4c0da1903911080dd34 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Mon, 4 Aug 2025 12:47:49 +0200 Subject: [PATCH 3/6] add poco replace plugin --- server/src/plugins/poco_plugin.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/plugins/poco_plugin.py b/server/src/plugins/poco_plugin.py index 4582443..f51125c 100644 --- a/server/src/plugins/poco_plugin.py +++ b/server/src/plugins/poco_plugin.py @@ -38,7 +38,7 @@ def lib_ge_command_buffer_edit_prepend(args, parser): def lib_ge_command_buffer_edit_replace(args, parser): _lib_ge_command_buffer_edit( - args, parser, "LIB_GE_command_buffer_edit_replace", 2, 4 + args, parser, "LIB_GE_command_buffer_edit_replace", 3, 5 ) -- 2.54.0 From c644ac9bd3fa95edb9e6142c5c4c939a082c9905 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Mon, 4 Aug 2025 12:48:02 +0200 Subject: [PATCH 4/6] add changes from tclint --- server/src/tools/parser.py | 364 +++++-------------------------------- 1 file changed, 45 insertions(+), 319 deletions(-) diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py index 408ab28..c1f15e8 100644 --- a/server/src/tools/parser.py +++ b/server/src/tools/parser.py @@ -1,3 +1,5 @@ +import io +from typing import Optional, Tuple from tclint.parser import Parser from tclint.commands import CommandArgError from tclint.syntax_tree import ( @@ -8,254 +10,17 @@ from tclint.syntax_tree import ( QuotedWord, Expression, ) - -import ply.lex as lex -from typing import Tuple - -TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE" -TOK_BACKSLASH_SUB = "BACKSLASH_SUB" -TOK_NEWLINE = "NEWLINE" -TOK_SEMI = "SEMI" -TOK_WS = "WS" -TOK_QUOTE = "QUOTE" -TOK_ARG_EXPANSION = "ARG_EXPANSION" -TOK_LBRACE = "LBRACE" -TOK_RBRACE = "RBRACE" -TOK_STAR = "STAR" -TOK_LBRACKET = "LBRACKET" -TOK_RBRACKET = "RBRACKET" -TOK_DOLLAR = "DOLLAR" -TOK_LPAREN = "LPAREN" -TOK_RPAREN = "RPAREN" -TOK_HASH = "HASH" -TOK_ALPHA_CHARS = "ALPHA_CHARS" -TOK_NUM_CHARS = "NUM_CHARS" -TOK_NAMESPACE_SEP = "NAMESPACE_SEP" -TOK_CHAR = "CHAR" -TOK_CONTENTS = "CONTENTS" -TOK_EOF = None - -STATE_BRACEDWORD = "bracedword" - - -class TclSyntaxError(Exception): - def __init__(self, message, start: Tuple[int, int], end: Tuple[int, int]): - super().__init__(message) - self.start = start - self.end = end - - -class _LexTable: - tokens = ( - TOK_BACKSLASH_NEWLINE, - TOK_BACKSLASH_SUB, - TOK_NEWLINE, - TOK_SEMI, - TOK_WS, - TOK_QUOTE, - TOK_ARG_EXPANSION, - TOK_LBRACE, - TOK_RBRACE, - TOK_STAR, - TOK_LBRACKET, - TOK_RBRACKET, - TOK_DOLLAR, - TOK_LPAREN, - TOK_RPAREN, - TOK_HASH, - TOK_ALPHA_CHARS, - TOK_NUM_CHARS, - TOK_NAMESPACE_SEP, - TOK_CHAR, - TOK_CONTENTS, - ) - - # This defines a conditional lexing state for parsing braced words. This is a - # performance optimization; since there are few special characters in this context, - # we can use a smaller set of tokens to parse them faster. This has a large impact - # since most Tcl programs have a large number of braced words. Any token with - # `bracedword` in its name is included in this state. Tokens that are included in - # this state and the default state also include `INITIAL` in their name. - states = ((STATE_BRACEDWORD, "exclusive"),) - - def _tok(self, t): - pos = (t.lexer.lineno, t.lexer.colno) - t.lexer.lineno += t.value.count("\n") - index = t.value.rfind("\n") - if index == -1: - t.lexer.colno += len(t.value) - else: - remaining = t.value[index + 1 :] - t.lexer.colno = len(remaining) + 1 - - t.value = (t.value, pos) - return t - - # Priority important - def t_bracedword_INITIAL_BACKSLASH_NEWLINE(self, t): - r"\\\r?\n" - return self._tok(t) - - # Priority important - def t_bracedword_INITIAL_BACKSLASH_SUB(self, t): - r"\\." - return self._tok(t) - - def t_NEWLINE(self, t): - r"\n" - return self._tok(t) - - def t_SEMI(self, t): - r";" - return self._tok(t) - - # TODO: should use \s? - def t_WS(self, t): - r"[\t\v\f\r ]+" - return self._tok(t) - - def t_QUOTE(self, t): - r'"' - return self._tok(t) - - # Must be higher priority than LBRACE - def t_ARG_EXPANSION(self, t): - r"\{\*\}" - return self._tok(t) - - def t_bracedword_INITIAL_LBRACE(self, t): - r"\{" - return self._tok(t) - - def t_bracedword_INITIAL_RBRACE(self, t): - r"\}" - return self._tok(t) - - def t_STAR(self, t): - r"\*" - return self._tok(t) - - def t_LBRACKET(self, t): - r"\[" - return self._tok(t) - - def t_RBRACKET(self, t): - r"\]" - return self._tok(t) - - def t_DOLLAR(self, t): - r"\$" - return self._tok(t) - - def t_LPAREN(self, t): - r"\(" - return self._tok(t) - - def t_RPAREN(self, t): - r"\)" - return self._tok(t) - - def t_HASH(self, t): - r"\#" - return self._tok(t) - - # Valid non-numeric chars in variable names - def t_ALPHA_CHARS(self, t): - r"[A-Za-z_]+" - return self._tok(t) - - # Valid numeric chars in variable names - # This is split up from the above to facilitate expression parsing, since - # e.g. 1eq1 can't be a single token. - def t_NUM_CHARS(self, t): - r"[0-9]+" - return self._tok(t) - - def t_NAMESPACE_SEP(self, t): - r"::+" - return self._tok(t) - - def t_bracedword_CONTENTS(self, t): - r"[^{}\\]+" - return self._tok(t) - - # Catch-all. TODO: inefficient, should probably munch multiple chars - def t_CHAR(self, t): - r"." - return self._tok(t) - - # Error handling rule - # TODO: do we need this? since we have a catch-all... - # there is a warning - def t_bracedword_INITIAL_error(self, t): - print("Illegal character '%s'" % t.value[0]) - t.lexer.skip(1) - - def __init__(self): - self.lexer = lex.lex(object=self) - self.lexer.lineno = 1 - self.lexer.colno = 1 - - def new_lexer(self, pos=None): - lexer = self.lexer.clone() - lexer.lineno = 1 - lexer.colno = 1 - - if pos is not None: - line, col = pos - lexer.lineno = line - lexer.colno = col - - return lexer - - -# Calling `lex.lex()` performs an expensive reflection process to generate the lexer. -# This singleton class holds a preinitialized lexer that can then be cloned to create -# individual instances. -LexTable = _LexTable() - - -class Lexer: - def __init__(self, pos=None): - self.lexer = LexTable.new_lexer(pos) - self.current = None - - def input(self, text): - self.lexer.input(text) - self.current = self.lexer.token() - - def type(self): - if self.current is None: - return TOK_EOF - return self.current.type - - def value(self): - if self.current is None: - return None - return self.current.value[0] - - def pos(self): - if self.current is None: - return (self.lexer.lineno, self.lexer.colno) - return self.current.value[1] - - def next(self): - self.current = self.lexer.token() - - def expect(self, *tokens, message, pos): - if self.type() not in tokens: - self.next() # munch another token to update position - raise TclSyntaxError(message, pos, self.pos()) - - self.next() - - def assert_(self, *tokens): - assert self.current.type in tokens - self.next() +from tclint.lexer import TclSyntaxError, Lexer, TOK_EOF class CustomParser(Parser): - def parse(self, script, pos=None): + def __init__(self, debug=False, command_plugins=None): + super().__init__(debug, command_plugins) + # Used to normalize newlines consistently with open()'s universal newlines mode. + self._decoder = io.IncrementalNewlineDecoder(None, True) + + def parse(self, script: str, pos: Optional[Tuple[int, int]] = None): + script = self._decoder.decode(script, True) lexer = Lexer(pos=pos) lexer.input(script) tree = self._parse_script(lexer, in_command_sub=False) @@ -265,83 +30,44 @@ class CustomParser(Parser): return tree - def parse_list(self, node): - """Parse contents of node as Tcl list. This is a distinct entry point - that doesn't get used when generating the main syntax tree, but is used - in command-specific argument parsing. - """ - if isinstance(node, List): - return node + def _parse_operator(self, ts): + pos = ts.pos() - if node.contents is None: - raise CommandArgError( - "expected braced word or word without substitutions in argument" - " interpreted as list" - ) + # hacky logic to handle parsing legal operators - ts = Lexer(pos=node.contents_pos) - ts.input(node.contents) - - DELIMITERS = {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE} - - list_node = List(pos=node.pos, end_pos=node.end_pos) - while ts.type() is not TOK_EOF: - while ts.type() in DELIMITERS: + if ts.value() in {"*", "&", "|"}: + # one or two of these characters are legal operators + operator = ts.value() + ts.next() + if ts.value() == operator: + operator += ts.value() ts.next() - - if ts.type() is TOK_EOF: - break - - if ts.type() == TOK_LBRACE: - # we can reuse parse_braced_word, since it doesn't use - # substitutions in any case - list_node.add(self.parse_braced_word(ts)) - elif ts.type() == TOK_QUOTE: - quote_word_pos = ts.pos() - - ts.assert_(TOK_QUOTE) - - bare_word_pos = ts.pos() - contents = "" - while ts.type() not in {TOK_QUOTE, TOK_EOF}: - contents += ts.value() - ts.next() - word = BareWord(contents, pos=bare_word_pos, end_pos=ts.pos()) - - ts.expect( - TOK_QUOTE, - message="reached EOF without finding match for quote", - pos=quote_word_pos, + elif ts.value() in {"<", ">"}: + operator = ts.value() + ts.next() + if ts.value() in {operator, "="}: + operator += ts.value() + ts.next() + elif ts.value() in {"=", "!"}: + operator = ts.value() + ts.next() + if ts.value() != "=": + raise TclSyntaxError( + f"invalid operator in expression: {operator}", pos, ts.pos() + ) + operator += ts.value() + ts.next() + elif ts.value() in {"*", "/", "%", "+", "-", "^", "eq", "ne", "in", "ni"}: + operator = ts.value() + ts.next() + else: + message = "invalid operator in expression: " + if ts.value() == "\\ ": + message += ( + "\\ (check for trailing whitespace if it's the end of the line)" ) - - list_node.add(QuotedWord(word, pos=quote_word_pos, end_pos=ts.pos())) else: - pos = ts.pos() - contents = "" - while ts.type() not in {*DELIMITERS, TOK_EOF}: - contents += ts.value() - ts.next() - list_node.add(BareWord(contents, pos=pos, end_pos=ts.pos())) + message += ts.value() + raise TclSyntaxError(message, pos, ts.pos()) - return list_node - - def parse_expression(self, node): - if node.contents is None: - raise CommandArgError( - "expected braced word or word without substitutions in argument" - " interpreted as expr" - ) - - ts = Lexer(pos=node.contents_pos) - ts.input(node.contents) - - contents = self._parse_expression(ts) - ts.expect( - TOK_EOF, - message=f"expected end of expression, got {ts.value()}", - pos=ts.pos(), - ) - if isinstance(node, BracedWord): - return BracedExpression(contents, pos=node.pos, end_pos=node.end_pos) - - return Expression(contents, pos=node.pos, end_pos=node.end_pos) + return BareWord(operator, pos=pos, end_pos=ts.pos()) -- 2.54.0 From 894a5b8b52b71b2ec6001cb5323691bc1ce15811 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Mon, 4 Aug 2025 12:48:21 +0200 Subject: [PATCH 5/6] add more test cases --- server/src/tools/semantic_tokens.py | 6 ------ test/test.tcl | 15 +++++++++++++-- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/server/src/tools/semantic_tokens.py b/server/src/tools/semantic_tokens.py index a5a0c68..dc1a6f5 100644 --- a/server/src/tools/semantic_tokens.py +++ b/server/src/tools/semantic_tokens.py @@ -76,8 +76,6 @@ class _Highlighter(Visitor): ) ) ) - pass - if routine.contents == "set" and command.args: first_arg = command.args[0] if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): @@ -140,7 +138,6 @@ class _Highlighter(Visitor): [TokenModifier.declaration], ) ) - if routine.contents == "namespace" and command.args: first_arg = command.args[1] if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): @@ -149,9 +146,6 @@ class _Highlighter(Visitor): (((line - 1, col - 1), len(first_arg.value), "class", [])) ) - def visit_var_sub(self, var_sub): - pass - def tokens(self) -> list[Token]: """Encode tokens as described in https://microsoft.github.io/language-server-protocol/specifications/lsp/3.17/specification/#textDocument_semanticTokens. diff --git a/test/test.tcl b/test/test.tcl index 9a65346..a958218 100644 --- a/test/test.tcl +++ b/test/test.tcl @@ -1,4 +1,10 @@ -set main 1 +# set ::custom_flag(from_move,$::mom_path_name) 1 +# set ::custom_flag(from_move,$::mom_path_name) 1 + +set te875st 11111 + +#set ::custom_flag(from_move,$::mom_path_name) 1 + if {$main == 1 && 1 == 1} { puts "main" } @@ -93,7 +99,7 @@ proc SERVICE_output_handling {handler} { #_________________________________________________________________________________________________ proc SERVICE_get_tool_data {} { global mom_tool_data -global mom_operation_info + global mom_operation_info set mom_tool_data(toollist) "" set operations $::mom_operation_name_list @@ -103,3 +109,8 @@ global mom_operation_info } } } + + +LIB_GE_command_buffer_edit_replace MOM_end_of_program_LIB END_OF_PROGRAM @END_OF_PROG { + MOM_do_template "end_of_program_rewind" +} EndOfProgramRewind -- 2.54.0 From 3613391c23fc50fb74bd92559c7e1981e1ec63a3 Mon Sep 17 00:00:00 2001 From: Christoph Brandau Date: Mon, 4 Aug 2025 12:57:01 +0200 Subject: [PATCH 6/6] fix semantics with var in array --- server/src/tools/semantic_tokens.py | 40 ++++++++++++++++++++++++++--- 1 file changed, 36 insertions(+), 4 deletions(-) diff --git a/server/src/tools/semantic_tokens.py b/server/src/tools/semantic_tokens.py index dc1a6f5..b2f762e 100644 --- a/server/src/tools/semantic_tokens.py +++ b/server/src/tools/semantic_tokens.py @@ -45,6 +45,37 @@ class _Highlighter(Visitor): self._tokens = [] self.log_to_output = log_to_output + def _get_token_info(self, node): + """Hilfsmethode um Token-Informationen aus verschiedenen Node-Typen zu extrahieren.""" + if not hasattr(node, "pos"): + return None + + # Einfacher Fall: Node hat direkten value + if hasattr(node, "value") and node.value is not None: + line, col = node.pos + return (line - 1, col - 1), len(node.value) + + # CompoundBareWord: versuche erstes Segment + if hasattr(node, "children") and node.children: + first_segment = node.children[0] + if ( + hasattr(first_segment, "value") + and first_segment.value is not None + and hasattr(first_segment, "pos") + ): + line, col = first_segment.pos + return (line - 1, col - 1), len(first_segment.value) + + # Fallback: Gesamtlänge aus Positionen berechnen + if hasattr(node, "end_pos"): + start_line, start_col = node.pos + end_line, end_col = node.end_pos + if start_line == end_line: + length = end_col - start_col + return (start_line - 1, start_col - 1), length + + return None + def visit_quoted_word(self, word: QuotedWord): if not word.contents: return @@ -78,13 +109,14 @@ class _Highlighter(Visitor): ) if routine.contents == "set" and command.args: first_arg = command.args[0] - if hasattr(first_arg, "pos") and hasattr(first_arg, "value"): - line, col = first_arg.pos + token_info = self._get_token_info(first_arg) + if token_info: + (line, col), length = token_info self._tokens.append( ( ( - (line - 1, col - 1), - len(first_arg.value), + (line, col), + length, "variable", [TokenModifier.declaration], ) -- 2.54.0