Merge branch 'tcl_language_support' of https://git.cbsk-tech.de/Christoph/nx_post_support into tcl_language_support

server/src/lsp_server.py

@@ -52,7 +52,7 @@ from tclint.format import Formatter, FormatterOpts
 from tclint.violations import Violation
 
 from plugins.poco_plugin import commands
-from tools import checks
+from tools import checks, parser
 from tools.semantic_tokens import _Highlighter, TOKEN_TYPES, TokenModifier
 from tools.completion_items import _Completion
 
@@ -62,7 +62,7 @@ DIAGNOSTIC_SOURCE = "nx-post-support"
 class TclLanguageServer(server.LanguageServer):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.parser = Parser()
+        self.parser = parser.CustomParser()  # Parser()
         for command in commands:
             self.parser._commands.update(command)
         self.diagnostics = {}
@@ -201,6 +201,8 @@ def did_open(params: lsp.DidOpenTextDocumentParams) -> None:
 def did_save(params: lsp.DidSaveTextDocumentParams) -> None:
     """LSP handler for textDocument/didSave request."""
     document = LSP_SERVER.workspace.get_text_document(params.text_document.uri)
+    tree = LSP_SERVER.parser.parse(document.source)
+    log_to_output(tree.pretty(2))
 
 
 @LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_CLOSE)

server/src/tools/parser.py

@@ -1,14 +1,225 @@
-from tclint.parser import Parser, _strip_ws
+from tclint.parser import (
+    Parser,
+    _is_bool_literal,
+    _is_float_literal,
+    _is_float_prefix,
+    _is_function,
+    _is_int_literal,
+    _is_int_prefix,
+)
 from tclint.lexer import (
-    TOK_WS,
+    STATE_BRACEDWORD,
     TOK_BACKSLASH_NEWLINE,
     TOK_EOF,
-    TOK_RPAREN,
+    TOK_LBRACE,
+    TOK_RBRACE,
+    TOK_WS,
     TOK_NEWLINE,
+    TOK_ALPHA_CHARS,
+    TOK_RPAREN,
+    TOK_LPAREN,
+    TOK_DOLLAR,
+    TOK_QUOTE,
+    TOK_LBRACKET,
+    TOK_NUM_CHARS,
     TclSyntaxError,
 )
-from tclint.syntax_tree import BareWord, TernaryOp, BinaryOp
+from tclint.syntax_tree import (
+    BracedWord,
+    BareWord,
+    BinaryOp,
+    TernaryOp,
+    ParenExpression,
+    UnaryOp,
+)
 
 
 class CustomParser(Parser):
-    pass
+    def _strip_ws(parse_func):
+        """Decorator used by expression parser for stripping whitespace around a node."""
+
+        def func(parser, ts):
+            while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}:
+                ts.next()
+
+            node = parse_func(parser, ts)
+
+            while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}:
+                ts.next()
+
+            return node
+
+        return func
+
+    @_strip_ws
+    def _parse_expression(self, ts):
+        op1 = self._parse_operand(ts)
+        expr = op1
+
+        # last condition is hack to break out of expression in case we're in ternary op
+        if ts.type() not in {TOK_EOF, TOK_RPAREN} and ts.value() not in {":", ","}:
+            if ts.value() == "?":
+                # weird hack to record operator
+                start = ts.pos()
+                ts.next()
+                q = BareWord("?", pos=start, end_pos=ts.pos())
+
+                op2 = self._parse_expression(ts)
+                if ts.value() != ":":
+                    start = ts.pos()
+                    ts.next()
+                    end = ts.pos()
+                    raise TclSyntaxError(
+                        "expected ':' to continue ternary expression", start, end
+                    )
+
+                # weird hack again
+                start = ts.pos()
+                ts.next()
+                colon = BareWord(":", pos=start, end_pos=ts.pos())
+
+                op3 = self._parse_expression(ts)
+                expr = TernaryOp(
+                    op1, q, op2, colon, op3, pos=op1.pos, end_pos=op3.end_pos
+                )
+            else:
+                operator = self._parse_operator(ts)
+                op2 = self._parse_expression(ts)
+                expr = BinaryOp(op1, operator, op2, pos=op1.pos, end_pos=op2.end_pos)
+
+            if ts.type() != TOK_RPAREN and ts.value() not in {":", ","}:
+                ts.expect(TOK_EOF, message="expected end of expression", pos=ts.pos())
+
+        return expr
+
+    @_strip_ws
+    def _parse_operand(self, ts):
+        if ts.type() == TOK_DOLLAR:
+            return self.parse_var_sub(ts)
+        if ts.type() == TOK_QUOTE:
+            return self.parse_quoted_word(ts)
+        if ts.type() == TOK_LBRACE:
+            return self.parse_braced_word(ts)
+        if ts.type() == TOK_LBRACKET:
+            return self.parse_command_sub(ts)
+        if ts.type() == TOK_LPAREN:
+            start = ts.pos()
+            ts.next()
+            expr = self._parse_expression(ts)
+            ts.expect(
+                TOK_RPAREN,
+                message="reached EOF without finding match for paren",
+                pos=expr.pos,
+            )
+            end = ts.pos()
+            return ParenExpression(expr, start, end)
+        if ts.value() in {"-", "+", "~", "!"}:
+            operator_val = ts.value()
+            operator_pos = ts.pos()
+            ts.next()
+            operator = BareWord(operator_val, pos=operator_pos, end_pos=ts.pos())
+            operand = self._parse_operand(ts)
+            # Since _parse_operand() munches whitespace after the operand, we
+            # set the end of the UnaryOp to the end of the operand rather than
+            # ts.pos(). Otherwise, the bounds of the UnaryOp would include all
+            # that whitespace.
+            return UnaryOp(operator, operand, pos=operator_pos, end_pos=operand.end_pos)
+
+        # If none of these, collect tokens that may comprise an operand
+        operand = ""
+        operand_pos = ts.pos()
+
+        # First, we want to check for numeric operands (either ints or numeric
+        # floats) by consuming tokens as long as they comprise the prefix of a
+        # numeric operand
+        while ts.type() != TOK_EOF and (
+            _is_int_prefix(operand + ts.value())
+            or _is_float_prefix(operand + ts.value())
+        ):
+            operand += ts.value()
+            ts.next()
+
+        # Next, we check if we've consumed an entire numeric literal. If so, we
+        # move on. If not, we keep consuming tokens that may correspond to a
+        # valid bareword (pretty much just alphanumeric chars).
+        if not (_is_int_literal(operand) or _is_float_literal(operand)):
+            while ts.type() in {TOK_ALPHA_CHARS, TOK_NUM_CHARS}:
+                operand += ts.value()
+                ts.next()
+
+        # The above method is a little hacky. Note that it doesn't parse things
+        # exactly the same as Tcl. E.g. if a script includes `expr {1foo}`,
+        # tclint will report an invalid operator "foo", whereas tclsh will
+        # report an invalid bareword "1foo". Despite reporting them differently
+        # both tools should still catch the same syntax errors, since there are
+        # no legal barewords that begin with a numeric literal prefix, and tclsh
+        # will stop parsing numeric operands if they're actually followed by a
+        # legal operator (e.g. `expr {1eq1}` will be handled properly).
+
+        is_func = _is_function(operand)
+
+        if not (
+            _is_int_literal(operand)
+            or _is_float_literal(operand)
+            or _is_bool_literal(operand)
+            or is_func
+        ):
+            raise TclSyntaxError(
+                f"invalid bareword in expression: {operand}", operand_pos, ts.pos()
+            )
+
+        node = BareWord(operand, pos=operand_pos, end_pos=ts.pos())
+
+        if is_func:
+            node = self._parse_function(ts, node)
+
+        return node
+
+    def parse_braced_word(self, ts):
+        self.debug(f"parse_braced_word({ts.current})")
+        pos = ts.pos()
+
+        ts.lexer.push_state(STATE_BRACEDWORD)
+
+        ts.assert_(TOK_LBRACE)
+
+        word = ""
+        expected_braces = [pos]  # Stack für geschachtelte Klammern
+
+        while True:
+            toktype = ts.type()
+            if toktype == TOK_EOF:
+                raise TclSyntaxError(
+                    "reached EOF without finding match for brace",
+                    expected_braces[-1],
+                    ts.pos(),
+                )
+
+            if toktype == TOK_BACKSLASH_NEWLINE:
+                # TCL-spezifisch: Zeilenumbruch mit Backslash ignorieren
+                ts.next()
+                continue
+
+            if toktype == TOK_LBRACE:
+                expected_braces.append(ts.pos())
+            elif toktype == TOK_RBRACE:
+                try:
+                    expected_braces.pop()
+                except IndexError:
+                    start = ts.pos()
+                    ts.next()
+                    end = ts.pos()
+                    raise TclSyntaxError(
+                        "found closing brace without matching open brace", start, end
+                    )
+
+                if len(expected_braces) == 0:
+                    ts.lexer.pop_state()
+                    ts.next()
+                    break
+
+            word += ts.value()
+            ts.next()
+
+        end_pos = ts.pos()
+        return BracedWord(word, pos=pos, end_pos=end_pos)

server/src/tools/semantic_tokens.py

@@ -3,8 +3,6 @@ from typing import List
 from tclint.syntax_tree import Visitor
 from tclint.commands import get_commands
 import attrs
-from common.load_data import standard_items
-import lsprotocol.types as lsp
 
 
 class TokenModifier(enum.IntFlag):
@@ -63,6 +61,34 @@ class _Highlighter(Visitor):
                     (((line - 1, col - 1), len(first_arg.value), "function"))
                 )
 
+            if len(command.args) >= 2:
+                param_list = command.args[1]
+
+                # BracedWord oder Liste erwartet
+                if hasattr(param_list, "children"):
+                    for child in param_list.children:
+                        # Parameter kann einfaches Wort sein
+                        if hasattr(child, "value") and child.value is not None:
+                            line, col = child.pos
+                            self._tokens.append(
+                                ((line - 1, col - 1), len(child.value), "variable")
+                            )
+
+                        # Parameter mit Default-Wert ist meist eine List (z. B. {arg default})
+                        elif hasattr(child, "children") and len(child.children) >= 1:
+                            name_node = child.children[0]
+                            if hasattr(name_node, "value") and hasattr(
+                                name_node, "pos"
+                            ):
+                                line, col = name_node.pos
+                                self._tokens.append(
+                                    (
+                                        (line - 1, col - 1),
+                                        len(name_node.value),
+                                        "variable",
+                                    )
+                                )
+
         if routine.contents == "namespace" and command.args:
             first_arg = command.args[1]
             if hasattr(first_arg, "pos") and hasattr(first_arg, "value"):

test/test.tcl

@@ -1,13 +1,6 @@
-proc myProc {arg {opt 1}} {
-    
-}
 
-set myVar 1
+set main 1
-
+if {$main == 1 \
-namespace eval myNameSpace {
+&& 1 == 1} {
-    proc namespaceProc {} {}
+    puts "main"
-}
+} {}
-
-set result [myNameSpace::namespaceProc]
-
-MOM_abort_program "Test"