2025-07-20 15:18:27 +00:00
1 changed files with 0 additions and 81 deletions
@@ -1,81 +0,0 @@
-# tokens.py
-from typing import List
-import attrs
-import enum
-from lark import Tree, Token
-
-# Legend must match the client
-SEMANTIC_TOKEN_TYPES = {
-    "keyword": 0,
-    "variable": 2,
-    "string": 3,
-}
-SEMANTIC_TOKEN_MODIFIERS = {
-    "declaration": 1 << 0,
-}
-
-
-class TokenModifier(enum.IntFlag):
-    deprecated = enum.auto()
-    readonly = enum.auto()
-    defaultLibrary = enum.auto()
-    definition = enum.auto()
-
-
-@attrs.define
-class TokenData:
-    line: int
-    offset: int
-    text: str
-
-    tok_type: str = ""
-    tok_modifiers: List[TokenModifier] = attrs.field(factory=list)
-
-
-TokenTypes = ["keyword", "variable", "function", "operator", "parameter", "type"]
-
-
-def collect_semantic_tokens(tree: Tree) -> list[int]:
-    """
-    Walk the parse tree and return a flat LSP semanticTokens/full array:
-      [line, char, length, tokenType, tokenModifiers, …]
-    """
-    data = []
-    for tok in tree.scan_values(lambda v: isinstance(v, Token)):
-        # 'set' keyword
-        if tok.type == "SET":
-            data.append(
-                [
-                    tok.line - 1,
-                    tok.column - 1,
-                    len(tok.value),
-                    SEMANTIC_TOKEN_TYPES["keyword"],
-                    0,
-                ]
-            )
-        # variable being declared
-        elif tok.type == "NAME":
-            data.append(
-                [
-                    tok.line - 1,
-                    tok.column - 1,
-                    len(tok.value),
-                    SEMANTIC_TOKEN_TYPES["variable"],
-                    SEMANTIC_TOKEN_MODIFIERS["declaration"],
-                ]
-            )
-        # string literal
-        elif tok.type == "STRING":
-            data.append(
-                [
-                    tok.line - 1,
-                    tok.column - 1,
-                    len(tok.value),
-                    SEMANTIC_TOKEN_TYPES["string"],
-                    0,
-                ]
-            )
-
-    # sort by position and flatten
-    data.sort(key=lambda x: (x[0], x[1]))
-    return [p for token in data for p in token]