use tclint as parser / formatter

server/src/tools/semantic_tokens.py

@@ -0,0 +1,76 @@
+from tclint.parser import Parser
+from tclint.syntax_tree import Visitor, BareWord, VarSub, Comment, Command, Function
+
+TOKEN_TYPES = {
+    "command": 0,
+    "variable": 1,
+    "function": 2,
+    "string": 3,
+    "number": 4,
+    "keyword": 5,
+    "comment": 6,
+}
+
+
+class SemanticTokenCollector(Visitor):
+    def __init__(self):
+        self.tokens = []
+
+    def _add_token(self, node, token_type):
+        if not node.pos or not node.end_pos:
+            return
+
+        line, col = node.pos
+        end_line, end_col = node.end_pos
+        length = (end_col - col) if line == end_line else 1
+
+        self.tokens.append((line - 1, col - 1, length, token_type, 0))
+
+    def visit_command(self, command: Command):
+        self._add_token(command.routine, "command")
+        for arg in command.args:
+            arg.accept(self, recurse=True)
+
+    def visit_comment(self, comment: Comment):
+        self._add_token(comment, "comment")
+
+    def visit_bare_word(self, word: BareWord):
+        if word.value.isdigit():
+            self._add_token(word, "number")
+        else:
+            self._add_token(word, "string")
+
+    def visit_var_sub(self, var_sub: VarSub):
+        self._add_token(var_sub, "variable")
+
+    def visit_function(self, function: Function):
+        self._add_token(function.name, "function")
+        for arg in function.args:
+            arg.accept(self, recurse=True)
+
+
+def collect_semantic_tokens(code: str):
+    parser = Parser()
+    tree = parser.parse(code)
+    visitor = SemanticTokenCollector()
+    tree.accept(visitor, recurse=True)
+    return visitor.tokens
+
+
+def encode_tokens(tokens):
+    tokens.sort()
+    encoded = []
+
+    last_line = 0
+    last_char = 0
+
+    for line, char, length, token_type, modifiers in tokens:
+        delta_line = line - last_line
+        delta_start = char - last_char if delta_line == 0 else char
+
+        encoded.extend([delta_line, delta_start, length, token_type, modifiers])
+
+        last_line = line
+        last_char = char if delta_line == 0 else 0
+
+    return encoded