From 5a611619923d3cb79a0c94b6645c03717d146986 Mon Sep 17 00:00:00 2001 From: christoph_xd Date: Mon, 25 Aug 2025 22:03:21 +0200 Subject: [PATCH] try rust parser --- server/src/tools/tcl_parser_client.py | 108 +++++++++++++ tcl_parser/Cargo.lock | 216 ++++++++++++++++++++++++++ tcl_parser/Cargo.toml | 16 ++ tcl_parser/src/lib.rs | 26 ++++ tcl_parser/src/main.rs | 59 +++++++ 5 files changed, 425 insertions(+) create mode 100644 server/src/tools/tcl_parser_client.py create mode 100644 tcl_parser/Cargo.lock create mode 100644 tcl_parser/Cargo.toml create mode 100644 tcl_parser/src/lib.rs create mode 100644 tcl_parser/src/main.rs diff --git a/server/src/tools/tcl_parser_client.py b/server/src/tools/tcl_parser_client.py new file mode 100644 index 0000000..afcb0d3 --- /dev/null +++ b/server/src/tools/tcl_parser_client.py @@ -0,0 +1,108 @@ +# tools/tcl_parser_client.py +import asyncio +import json +import uuid +from typing import Optional, Dict, Any + + +class ParserProcess: + """ + Async wrapper for the NDJSON stdin/stdout parser process. + Start once at server startup; call parse() concurrently. + """ + + def __init__(self, cmd): + self.cmd = cmd + self.proc: Optional[asyncio.Process] = None + self._write_lock = asyncio.Lock() + self._pending: Dict[str, asyncio.Future] = {} + self._reader_task: Optional[asyncio.Task] = None + + async def start(self): + if self.proc: + return + self.proc = await asyncio.create_subprocess_exec( + *self.cmd, + stdin=asyncio.subprocess.PIPE, + stdout=asyncio.subprocess.PIPE, + stderr=asyncio.subprocess.PIPE, + ) + self._reader_task = asyncio.create_task(self._reader_loop()) + asyncio.create_task(self._read_stderr()) + + async def stop(self): + if not self.proc: + return + try: + self.proc.terminate() + except Exception: + pass + await self.proc.wait() + if self._reader_task: + self._reader_task.cancel() + self.proc = None + + async def parse(self, tcl: str, timeout: float = 5.0) -> str: + """ + Send `tcl` to the parser and await the S-expression string result. + Raises on parse error or if the parser dies. + """ + if not self.proc: + await self.start() + + req_id = uuid.uuid4().hex + payload = json.dumps({"id": req_id, "tcl": tcl}, separators=(",", ":")) + + fut = asyncio.get_event_loop().create_future() + self._pending[req_id] = fut + + async with self._write_lock: + assert self.proc and self.proc.stdin + self.proc.stdin.write((payload + "\n").encode("utf-8")) + await self.proc.stdin.drain() + + try: + result = await asyncio.wait_for(fut, timeout) + return result + finally: + self._pending.pop(req_id, None) + + async def _reader_loop(self): + assert self.proc and self.proc.stdout + reader = self.proc.stdout + try: + while True: + line = await reader.readline() + if not line: + # parser process closed + for f in list(self._pending.values()): + if not f.done(): + f.set_exception(RuntimeError("parser process terminated")) + break + try: + obj = json.loads(line.decode("utf-8", "replace")) + except Exception as e: + # ignore malformed lines or log + continue + rid = obj.get("id") + fut = self._pending.get(rid) + if fut and not fut.done(): + if obj.get("ok"): + fut.set_result(obj.get("result")) + else: + fut.set_exception(RuntimeError(obj.get("error") or "parse error")) + except asyncio.CancelledError: + return + + async def _read_stderr(self): + # optional: log parser stderr + assert self.proc and self.proc.stderr + try: + while True: + line = await self.proc.stderr.readline() + if not line: + break + # Replace this with your LSP server logging + print("[tcl_parser stderr]", line.decode("utf-8", "replace").rstrip()) + except asyncio.CancelledError: + return diff --git a/tcl_parser/Cargo.lock b/tcl_parser/Cargo.lock new file mode 100644 index 0000000..df67a78 --- /dev/null +++ b/tcl_parser/Cargo.lock @@ -0,0 +1,216 @@ +# This file is automatically @generated by Cargo. +# It is not intended for manual editing. +version = 4 + +[[package]] +name = "aho-corasick" +version = "1.1.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916" +dependencies = [ + "memchr", +] + +[[package]] +name = "anyhow" +version = "1.0.99" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b0674a1ddeecb70197781e945de4b3b8ffb61fa939a5597bcf48503737663100" + +[[package]] +name = "cc" +version = "1.2.34" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "42bc4aea80032b7bf409b0bc7ccad88853858911b7713a8062fdc0623867bedc" +dependencies = [ + "shlex", +] + +[[package]] +name = "equivalent" +version = "1.0.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "877a4ace8713b0bcf2a4e7eec82529c029f1d0619886d18145fea96c3ffe5c0f" + +[[package]] +name = "hashbrown" +version = "0.15.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1" + +[[package]] +name = "indexmap" +version = "2.11.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f2481980430f9f78649238835720ddccc57e52df14ffce1c6f37391d61b563e9" +dependencies = [ + "equivalent", + "hashbrown", +] + +[[package]] +name = "itoa" +version = "1.0.15" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4a5f13b858c8d314ee3e8f639011f7ccefe71f97f96e50151fb991f267928e2c" + +[[package]] +name = "memchr" +version = "2.7.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0" + +[[package]] +name = "proc-macro2" +version = "1.0.101" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "89ae43fd86e4158d6db51ad8e2b80f313af9cc74f5c0e03ccb87de09998732de" +dependencies = [ + "unicode-ident", +] + +[[package]] +name = "quote" +version = "1.0.40" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1885c039570dc00dcb4ff087a89e185fd56bae234ddc7f056a945bf36467248d" +dependencies = [ + "proc-macro2", +] + +[[package]] +name = "regex" +version = "1.11.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "23d7fd106d8c02486a8d64e778353d1cffe08ce79ac2e82f540c86d0facf6912" +dependencies = [ + "aho-corasick", + "memchr", + "regex-automata", + "regex-syntax", +] + +[[package]] +name = "regex-automata" +version = "0.4.10" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6b9458fa0bfeeac22b5ca447c63aaf45f28439a709ccd244698632f9aa6394d6" +dependencies = [ + "aho-corasick", + "memchr", + "regex-syntax", +] + +[[package]] +name = "regex-syntax" +version = "0.8.6" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "caf4aa5b0f434c91fe5c7f1ecb6a5ece2130b02ad2a590589dda5146df959001" + +[[package]] +name = "ryu" +version = "1.0.20" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "28d3b2b1366ec20994f1fd18c3c594f05c5dd4bc44d8bb0c1c632c8d6829481f" + +[[package]] +name = "serde" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5f0e2c6ed6606019b4e29e69dbaba95b11854410e5347d525002456dbbb786b6" +dependencies = [ + "serde_derive", +] + +[[package]] +name = "serde_derive" +version = "1.0.219" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5b0276cf7f2c73365f7157c8123c21cd9a50fbbd844757af28ca1f5925fc2a00" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "serde_json" +version = "1.0.143" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d401abef1d108fbd9cbaebc3e46611f4b1021f714a0597a71f41ee463f5f4a5a" +dependencies = [ + "indexmap", + "itoa", + "memchr", + "ryu", + "serde", +] + +[[package]] +name = "shlex" +version = "1.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" + +[[package]] +name = "streaming-iterator" +version = "0.1.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2b2231b7c3057d5e4ad0156fb3dc807d900806020c5ffa3ee6ff2c8c76fb8520" + +[[package]] +name = "syn" +version = "2.0.106" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "ede7c438028d4436d71104916910f5bb611972c5cfd7f89b8300a8186e6fada6" +dependencies = [ + "proc-macro2", + "quote", + "unicode-ident", +] + +[[package]] +name = "tcl_parser" +version = "0.1.0" +dependencies = [ + "anyhow", + "serde", + "serde_json", + "tree-sitter", + "tree-sitter-tcl", +] + +[[package]] +name = "tree-sitter" +version = "0.25.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6d7b8994f367f16e6fa14b5aebbcb350de5d7cbea82dc5b00ae997dd71680dd2" +dependencies = [ + "cc", + "regex", + "regex-syntax", + "serde_json", + "streaming-iterator", + "tree-sitter-language", +] + +[[package]] +name = "tree-sitter-language" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c4013970217383f67b18aef68f6fb2e8d409bc5755227092d32efb0422ba24b8" + +[[package]] +name = "tree-sitter-tcl" +version = "1.1.0" +source = "git+https://github.com/tree-sitter-grammars/tree-sitter-tcl#8f11ac7206a54ed11210491cee1e0657e2962c47" +dependencies = [ + "cc", + "tree-sitter-language", +] + +[[package]] +name = "unicode-ident" +version = "1.0.18" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "5a5f39404a5da50712a4c1eecf25e90dd62b613502b7e925fd4e4d19b5c96512" diff --git a/tcl_parser/Cargo.toml b/tcl_parser/Cargo.toml new file mode 100644 index 0000000..f15515d --- /dev/null +++ b/tcl_parser/Cargo.toml @@ -0,0 +1,16 @@ +[package] +name = "tcl_parser" +version = "0.1.0" +edition = "2024" + +[dependencies] +tree-sitter = "0.25.8" +tree-sitter-tcl = { git = "https://github.com/tree-sitter-grammars/tree-sitter-tcl" } +serde = { version = "1.0", features = ["derive"] } +serde_json = "1.0" +anyhow = "1.0" + +[lib] +name = "tcl_parser" +path = "src/lib.rs" + diff --git a/tcl_parser/src/lib.rs b/tcl_parser/src/lib.rs new file mode 100644 index 0000000..2510368 --- /dev/null +++ b/tcl_parser/src/lib.rs @@ -0,0 +1,26 @@ +use tree_sitter::Parser; + +pub fn parse_to_sexp(src: &str) -> Result { + let lang = tree_sitter_tcl::LANGUAGE; + let mut parser = Parser::new(); + parser.set_language(&lang.into())?; + let tree = parser + .parse(src.as_bytes(), None) + .ok_or_else(|| anyhow::anyhow!("failed to parse input"))?; + Ok(tree.root_node().to_sexp()) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn parses_sample() { + let src = r#"proc ProcName {arg1 arg2 {arg3 3}} { + # Comment + set var1 100 +}"#; + let sexp = parse_to_sexp(src).expect("parse"); + assert!(sexp.contains("proc")); + } +} diff --git a/tcl_parser/src/main.rs b/tcl_parser/src/main.rs new file mode 100644 index 0000000..9962333 --- /dev/null +++ b/tcl_parser/src/main.rs @@ -0,0 +1,59 @@ +use std::io::{ self, BufRead }; +use serde::{ Deserialize, Serialize }; + +fn main() -> anyhow::Result<()> { + println!( + "tcl_parser server starting (NDJSON stdin). Send JSON lines with {{'id':..., 'tcl': '...'}}" + ); + let stdin = io::stdin(); + for line in stdin.lock().lines() { + let line = line?; + if line.trim().is_empty() { + continue; + } + let req: Request = match serde_json::from_str(&line) { + Ok(r) => r, + Err(e) => { + let resp = Response::error(None, format!("invalid request json: {}", e)); + println!("{}", serde_json::to_string(&resp)?); + continue; + } + }; + + let id = req.id.clone(); + match tcl_parser::parse_to_sexp(&req.tcl) { + Ok(sexp) => { + let resp = Response::ok(id, sexp); + println!("{}", serde_json::to_string(&resp)?); + } + Err(e) => { + let resp = Response::error(id, format!("parse error: {}", e)); + println!("{}", serde_json::to_string(&resp)?); + } + } + } + Ok(()) +} + +#[derive(Deserialize)] +struct Request { + id: Option, + tcl: String, +} + +#[derive(Serialize)] +struct Response { + id: Option, + ok: bool, + result: Option, + error: Option, +} + +impl Response { + fn ok(id: Option, result: String) -> Self { + Self { id, ok: true, result: Some(result), error: None } + } + fn error(id: Option, err: String) -> Self { + Self { id, ok: false, result: None, error: Some(err) } + } +}