del tokens

2025-07-20 17:17:38 +02:00
parent 9db2f0f7ac
commit 31cb644f85
1 changed files with 0 additions and 81 deletions
@@ -1,81 +0,0 @@
 # tokens.py
 from typing import List
 import attrs
 import enum
 from lark import Tree, Token
 # Legend must match the client
 SEMANTIC_TOKEN_TYPES = {
    "keyword": 0,
    "variable": 2,
    "string": 3,
 }
 SEMANTIC_TOKEN_MODIFIERS = {
    "declaration": 1 << 0,
 }
 class TokenModifier(enum.IntFlag):
    deprecated = enum.auto()
    readonly = enum.auto()
    defaultLibrary = enum.auto()
    definition = enum.auto()
@attrs.define
 class TokenData:
    line: int
    offset: int
    text: str
    tok_type: str = ""
    tok_modifiers: List[TokenModifier] = attrs.field(factory=list)
 TokenTypes = ["keyword", "variable", "function", "operator", "parameter", "type"]
 def collect_semantic_tokens(tree: Tree) -> list[int]:
    """
    Walk the parse tree and return a flat LSP semanticTokens/full array:
      [line, char, length, tokenType, tokenModifiers, …]
    """
    data = []
    for tok in tree.scan_values(lambda v: isinstance(v, Token)):
        # 'set' keyword
        if tok.type == "SET":
            data.append(
                [
                    tok.line - 1,
                    tok.column - 1,
                    len(tok.value),
                    SEMANTIC_TOKEN_TYPES["keyword"],
                    0,
                ]
            )
        # variable being declared
        elif tok.type == "NAME":
            data.append(
                [
                    tok.line - 1,
                    tok.column - 1,
                    len(tok.value),
                    SEMANTIC_TOKEN_TYPES["variable"],
                    SEMANTIC_TOKEN_MODIFIERS["declaration"],
                ]
            )
        # string literal
        elif tok.type == "STRING":
            data.append(
                [
                    tok.line - 1,
                    tok.column - 1,
                    len(tok.value),
                    SEMANTIC_TOKEN_TYPES["string"],
                    0,
                ]
            )
    # sort by position and flatten
    data.sort(key=lambda x: (x[0], x[1]))
    return [p for token in data for p in token]