python lsp

server/src/common/completion_list.json

@@ -0,0 +1,16 @@
+{
+    "tcl": {
+        "keywords": [
+            "proc",
+            "puts",
+            "if"
+        ]
+    },
+    "MOM_procs": [
+        "MOM_output_literal",
+        "MOM_ask_ude_info",
+        "MOM_suppress",
+        "MOM_force",
+        "MOM_enable_address"
+    ]
+}

server/src/lsp_jsonrpc.py

@@ -0,0 +1,254 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+"""Light-weight JSON-RPC over standard IO."""
+
+
+import atexit
+import contextlib
+import io
+import json
+import pathlib
+import subprocess
+import threading
+import uuid
+from concurrent.futures import ThreadPoolExecutor
+from typing import BinaryIO, Dict, Optional, Sequence, Union
+
+CONTENT_LENGTH = "Content-Length: "
+RUNNER_SCRIPT = str(pathlib.Path(__file__).parent / "lsp_runner.py")
+
+
+def to_str(text) -> str:
+    """Convert bytes to string as needed."""
+    return text.decode("utf-8") if isinstance(text, bytes) else text
+
+
+class StreamClosedException(Exception):
+    """JSON RPC stream is closed."""
+
+    pass  # pylint: disable=unnecessary-pass
+
+
+class JsonWriter:
+    """Manages writing JSON-RPC messages to the writer stream."""
+
+    def __init__(self, writer: io.TextIOWrapper):
+        self._writer = writer
+        self._lock = threading.Lock()
+
+    def close(self):
+        """Closes the underlying writer stream."""
+        with self._lock:
+            if not self._writer.closed:
+                self._writer.close()
+
+    def write(self, data):
+        """Writes given data to stream in JSON-RPC format."""
+        if self._writer.closed:
+            raise StreamClosedException()
+
+        with self._lock:
+            content = json.dumps(data)
+            length = len(content.encode("utf-8"))
+            self._writer.write(
+                f"{CONTENT_LENGTH}{length}\r\n\r\n{content}".encode("utf-8")
+            )
+            self._writer.flush()
+
+
+class JsonReader:
+    """Manages reading JSON-RPC messages from stream."""
+
+    def __init__(self, reader: io.TextIOWrapper):
+        self._reader = reader
+
+    def close(self):
+        """Closes the underlying reader stream."""
+        if not self._reader.closed:
+            self._reader.close()
+
+    def read(self):
+        """Reads data from the stream in JSON-RPC format."""
+        if self._reader.closed:
+            raise StreamClosedException
+        length = None
+        while not length:
+            line = to_str(self._readline())
+            if line.startswith(CONTENT_LENGTH):
+                length = int(line[len(CONTENT_LENGTH) :])
+
+        line = to_str(self._readline()).strip()
+        while line:
+            line = to_str(self._readline()).strip()
+
+        content = to_str(self._reader.read(length))
+        return json.loads(content)
+
+    def _readline(self):
+        line = self._reader.readline()
+        if not line:
+            raise EOFError
+        return line
+
+
+class JsonRpc:
+    """Manages sending and receiving data over JSON-RPC."""
+
+    def __init__(self, reader: io.TextIOWrapper, writer: io.TextIOWrapper):
+        self._reader = JsonReader(reader)
+        self._writer = JsonWriter(writer)
+
+    def close(self):
+        """Closes the underlying streams."""
+        with contextlib.suppress(Exception):
+            self._reader.close()
+        with contextlib.suppress(Exception):
+            self._writer.close()
+
+    def send_data(self, data):
+        """Send given data in JSON-RPC format."""
+        self._writer.write(data)
+
+    def receive_data(self):
+        """Receive data in JSON-RPC format."""
+        return self._reader.read()
+
+
+def create_json_rpc(readable: BinaryIO, writable: BinaryIO) -> JsonRpc:
+    """Creates JSON-RPC wrapper for the readable and writable streams."""
+    return JsonRpc(readable, writable)
+
+
+class ProcessManager:
+    """Manages sub-processes launched for running tools."""
+
+    def __init__(self):
+        self._args: Dict[str, Sequence[str]] = {}
+        self._processes: Dict[str, subprocess.Popen] = {}
+        self._rpc: Dict[str, JsonRpc] = {}
+        self._lock = threading.Lock()
+        self._thread_pool = ThreadPoolExecutor(10)
+
+    def stop_all_processes(self):
+        """Send exit command to all processes and shutdown transport."""
+        for i in self._rpc.values():
+            with contextlib.suppress(Exception):
+                i.send_data({"id": str(uuid.uuid4()), "method": "exit"})
+        self._thread_pool.shutdown(wait=False)
+
+    def start_process(self, workspace: str, args: Sequence[str], cwd: str) -> None:
+        """Starts a process and establishes JSON-RPC communication over stdio."""
+        # pylint: disable=consider-using-with
+        proc = subprocess.Popen(
+            args,
+            cwd=cwd,
+            stdout=subprocess.PIPE,
+            stdin=subprocess.PIPE,
+        )
+        self._processes[workspace] = proc
+        self._rpc[workspace] = create_json_rpc(proc.stdout, proc.stdin)
+
+        def _monitor_process():
+            proc.wait()
+            with self._lock:
+                try:
+                    del self._processes[workspace]
+                    rpc = self._rpc.pop(workspace)
+                    rpc.close()
+                except:  # pylint: disable=bare-except
+                    pass
+
+        self._thread_pool.submit(_monitor_process)
+
+    def get_json_rpc(self, workspace: str) -> JsonRpc:
+        """Gets the JSON-RPC wrapper for the a given id."""
+        with self._lock:
+            if workspace in self._rpc:
+                return self._rpc[workspace]
+        raise StreamClosedException()
+
+
+_process_manager = ProcessManager()
+atexit.register(_process_manager.stop_all_processes)
+
+
+def _get_json_rpc(workspace: str) -> Union[JsonRpc, None]:
+    try:
+        return _process_manager.get_json_rpc(workspace)
+    except StreamClosedException:
+        return None
+    except KeyError:
+        return None
+
+
+def get_or_start_json_rpc(
+    workspace: str, interpreter: Sequence[str], cwd: str
+) -> Union[JsonRpc, None]:
+    """Gets an existing JSON-RPC connection or starts one and return it."""
+    res = _get_json_rpc(workspace)
+    if not res:
+        args = [*interpreter, RUNNER_SCRIPT]
+        _process_manager.start_process(workspace, args, cwd)
+        res = _get_json_rpc(workspace)
+    return res
+
+
+class RpcRunResult:
+    """Object to hold result from running tool over RPC."""
+
+    def __init__(self, stdout: str, stderr: str, exception: Optional[str] = None):
+        self.stdout: str = stdout
+        self.stderr: str = stderr
+        self.exception: Optional[str] = exception
+
+
+# pylint: disable=too-many-arguments
+def run_over_json_rpc(
+    workspace: str,
+    interpreter: Sequence[str],
+    module: str,
+    argv: Sequence[str],
+    use_stdin: bool,
+    cwd: str,
+    source: str = None,
+) -> RpcRunResult:
+    """Uses JSON-RPC to execute a command."""
+    rpc: Union[JsonRpc, None] = get_or_start_json_rpc(workspace, interpreter, cwd)
+    if not rpc:
+        raise Exception("Failed to run over JSON-RPC.")
+
+    msg_id = str(uuid.uuid4())
+    msg = {
+        "id": msg_id,
+        "method": "run",
+        "module": module,
+        "argv": argv,
+        "useStdin": use_stdin,
+        "cwd": cwd,
+    }
+    if source:
+        msg["source"] = source
+
+    rpc.send_data(msg)
+
+    data = rpc.receive_data()
+
+    if data["id"] != msg_id:
+        return RpcRunResult(
+            "", f"Invalid result for request: {json.dumps(msg, indent=4)}"
+        )
+
+    result = data["result"] if "result" in data else ""
+    if "error" in data:
+        error = data["error"]
+
+        if data.get("exception", False):
+            return RpcRunResult(result, "", error)
+        return RpcRunResult(result, error)
+
+    return RpcRunResult(result, "")
+
+
+def shutdown_json_rpc():
+    """Shutdown all JSON-RPC processes."""
+    _process_manager.stop_all_processes()

server/src/lsp_runner.py

@@ -0,0 +1,77 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+"""
+Runner to use when running under a different interpreter.
+"""
+
+import os
+import pathlib
+import sys
+import traceback
+
+
+# **********************************************************
+# Update sys.path before importing any bundled libraries.
+# **********************************************************
+def update_sys_path(path_to_add: str, strategy: str) -> None:
+    """Add given path to `sys.path`."""
+    if path_to_add not in sys.path and os.path.isdir(path_to_add):
+        if strategy == "useBundled":
+            sys.path.insert(0, path_to_add)
+        elif strategy == "fromEnvironment":
+            sys.path.append(path_to_add)
+
+
+# Ensure that we can import LSP libraries, and other bundled libraries.
+update_sys_path(
+    os.fspath(pathlib.Path(__file__).parent.parent / "libs"),
+    os.getenv("LS_IMPORT_STRATEGY", "useBundled"),
+)
+
+
+# pylint: disable=wrong-import-position,import-error
+import lsp_jsonrpc as jsonrpc
+import lsp_utils as utils
+
+RPC = jsonrpc.create_json_rpc(sys.stdin.buffer, sys.stdout.buffer)
+
+EXIT_NOW = False
+while not EXIT_NOW:
+    msg = RPC.receive_data()
+
+    method = msg["method"]
+    if method == "exit":
+        EXIT_NOW = True
+        continue
+
+    if method == "run":
+        is_exception = False
+        # This is needed to preserve sys.path, pylint modifies
+        # sys.path and that might not work for this scenario
+        # next time around.
+        with utils.substitute_attr(sys, "path", sys.path[:]):
+            try:
+                # TODO: `utils.run_module` is equivalent to running `python -m <pytool-module>`.
+                # If your tool supports a programmatic API then replace the function below
+                # with code for your tool. You can also use `utils.run_api` helper, which
+                # handles changing working directories, managing io streams, etc.
+                # Also update `_run_tool_on_document` and `_run_tool` functions in `lsp_server.py`.
+                result = utils.run_module(
+                    module=msg["module"],
+                    argv=msg["argv"],
+                    use_stdin=msg["useStdin"],
+                    cwd=msg["cwd"],
+                    source=msg["source"] if "source" in msg else None,
+                )
+            except Exception:  # pylint: disable=broad-except
+                result = utils.RunResult("", traceback.format_exc(chain=True))
+                is_exception = True
+
+        response = {"id": msg["id"]}
+        if result.stderr:
+            response["error"] = result.stderr
+            response["exception"] = is_exception
+        elif result.stdout:
+            response["result"] = result.stdout
+
+        RPC.send_data(response)

server/src/lsp_server.py

@@ -1,3 +1,5 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
 """Implementation of tool support over LSP."""
 
 from __future__ import annotations
@@ -30,3 +32,564 @@ update_sys_path(
     os.fspath(pathlib.Path(__file__).parent.parent / "libs"),
     os.getenv("LS_IMPORT_STRATEGY", "useBundled"),
 )
+
+# **********************************************************
+# Imports needed for the language server goes below this.
+# **********************************************************
+# pylint: disable=wrong-import-position,import-error
+import lsp_jsonrpc as jsonrpc
+import lsp_utils as utils
+import lsprotocol.types as lsp
+from pygls import server, uris, workspace
+
+WORKSPACE_SETTINGS = {}
+GLOBAL_SETTINGS = {}
+RUNNER = pathlib.Path(__file__).parent / "lsp_runner.py"
+
+MAX_WORKERS = 5
+LSP_SERVER = server.LanguageServer(
+    name="NX Postprocessor Support", version="0.0.1", max_workers=MAX_WORKERS
+)
+
+KEYWORD_LIST = []
+
+# **********************************************************
+# Tool specific code goes below this.
+# **********************************************************
+TOOL_MODULE = "nx-post-support"
+TOOL_DISPLAY = "NX Postprocessor Support"
+TOOL_ARGS = []  # default arguments always passed to your tool.
+
+# Delete "Linting features" section if your tool is NOT a linter.
+# **********************************************************
+# Linting features start here
+# **********************************************************
+
+#  See `pylint` implementation for a full featured linter extension:
+#  Pylint: https://github.com/microsoft/vscode-pylint/blob/main/bundled/tool
+
+
+@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_OPEN)
+def did_open(params: lsp.DidOpenTextDocumentParams) -> None:
+    """LSP handler for textDocument/didOpen request."""
+    document = LSP_SERVER.workspace.get_document(params.text_document.uri)
+
+
+@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_SAVE)
+def did_save(params: lsp.DidSaveTextDocumentParams) -> None:
+    """LSP handler for textDocument/didSave request."""
+    document = LSP_SERVER.workspace.get_document(params.text_document.uri)
+
+
+@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_CLOSE)
+def did_close(params: lsp.DidCloseTextDocumentParams) -> None:
+    """LSP handler for textDocument/didClose request."""
+
+
+@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_DID_CHANGE)
+def did_change(params: lsp.DidChangeTextDocumentParams) -> None:
+    """LSP handler for textDocument/didChange request"""
+    log_to_output("Document has changed")
+
+
+@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_COMPLETION)
+def on_completion(params: lsp.CompletionParams) -> lsp.CompletionItem:
+    items = [
+        lsp.CompletionItem(
+            label="proc",
+            kind=lsp.CompletionItemKind.Keyword,
+            detail="TCL Procedure Definition",
+            documentation="Defines a new procedure in TCL",
+        ),
+        lsp.CompletionItem(
+            label="puts",
+            kind=lsp.CompletionItemKind.Function,
+            detail="Print to stdout",
+            documentation="Prints a string to the standard output in TCL",
+        ),
+    ]
+    return lsp.CompletionList(is_incomplete=False, items=items)
+
+
+# TODO: If your linter outputs in a known format like JSON, then parse
+# accordingly. But incase you need to parse the output using RegEx here
+# is a helper you can work with.
+# flake8 example:
+# If you use following format argument with flake8 you can use the regex below to parse it.
+# TOOL_ARGS += ["--format='%(row)d,%(col)d,%(code).1s,%(code)s:%(text)s'"]
+# DIAGNOSTIC_RE =
+#    r"(?P<line>\d+),(?P<column>-?\d+),(?P<type>\w+),(?P<code>\w+\d+):(?P<message>[^\r\n]*)"
+DIAGNOSTIC_RE = re.compile(r"")
+
+
+def _parse_output_using_regex(content: str) -> list[lsp.Diagnostic]:
+    lines: list[str] = content.splitlines()
+    diagnostics: list[lsp.Diagnostic] = []
+
+    # TODO: Determine if your linter reports line numbers starting at 1 (True) or 0 (False).
+    line_at_1 = True
+    # TODO: Determine if your linter reports column numbers starting at 1 (True) or 0 (False).
+    column_at_1 = True
+
+    line_offset = 1 if line_at_1 else 0
+    col_offset = 1 if column_at_1 else 0
+    for line in lines:
+        if line.startswith("'") and line.endswith("'"):
+            line = line[1:-1]
+        match = DIAGNOSTIC_RE.match(line)
+        if match:
+            data = match.groupdict()
+            position = lsp.Position(
+                line=max([int(data["line"]) - line_offset, 0]),
+                character=int(data["column"]) - col_offset,
+            )
+            diagnostic = lsp.Diagnostic(
+                range=lsp.Range(
+                    start=position,
+                    end=position,
+                ),
+                message=data.get("message"),
+                severity=_get_severity(data["code"], data["type"]),
+                code=data["code"],
+                source=TOOL_MODULE,
+            )
+            diagnostics.append(diagnostic)
+
+    return diagnostics
+
+
+# TODO: if you want to handle setting specific severity for your linter
+# in a user configurable way, then look at look at how it is implemented
+# for `pylint` extension from our team.
+# Pylint: https://github.com/microsoft/vscode-pylint
+# Follow the flow of severity from the settings in package.json to the server.
+def _get_severity(*_codes: list[str]) -> lsp.DiagnosticSeverity:
+    # TODO: All reported issues from linter are treated as warning.
+    # change it as appropriate for your linter.
+    return lsp.DiagnosticSeverity.Warning
+
+
+# **********************************************************
+# Linting features end here
+# **********************************************************
+
+# TODO: If your tool is a formatter then update this section.
+# Delete "Formatting features" section if your tool is NOT a
+# formatter.
+# **********************************************************
+# Formatting features start here
+# **********************************************************
+#  Sample implementations:
+#  Black: https://github.com/microsoft/vscode-black-formatter/blob/main/bundled/tool
+
+
+@LSP_SERVER.feature(lsp.TEXT_DOCUMENT_FORMATTING)
+def formatting(params: lsp.DocumentFormattingParams) -> list[lsp.TextEdit] | None:
+    """LSP handler for textDocument/formatting request."""
+    # If your tool is a formatter you can use this handler to provide
+    # formatting support on save. You have to return an array of lsp.TextEdit
+    # objects, to provide your formatted results.
+
+    document = LSP_SERVER.workspace.get_document(params.text_document.uri)
+    edits = _formatting_helper(document)
+    if edits:
+        return edits
+
+    # NOTE: If you provide [] array, VS Code will clear the file of all contents.
+    # To indicate no changes to file return None.
+    return None
+
+
+def _formatting_helper(document: workspace.Document) -> list[lsp.TextEdit] | None:
+    # TODO: For formatting on save support the formatter you use must support
+    # formatting via stdin.
+    # Read, and update_run_tool_on_document and _run_tool functions as needed
+    # for your formatter.
+    result = _run_tool_on_document(document, use_stdin=True)
+    if result.stdout:
+        new_source = _match_line_endings(document, result.stdout)
+        return [
+            lsp.TextEdit(
+                range=lsp.Range(
+                    start=lsp.Position(line=0, character=0),
+                    end=lsp.Position(line=len(document.lines), character=0),
+                ),
+                new_text=new_source,
+            )
+        ]
+    return None
+
+
+def _get_line_endings(lines: list[str]) -> str:
+    """Returns line endings used in the text."""
+    try:
+        if lines[0][-2:] == "\r\n":
+            return "\r\n"
+        return "\n"
+    except Exception:  # pylint: disable=broad-except
+        return None
+
+
+def _match_line_endings(document: workspace.Document, text: str) -> str:
+    """Ensures that the edited text line endings matches the document line endings."""
+    expected = _get_line_endings(document.source.splitlines(keepends=True))
+    actual = _get_line_endings(text.splitlines(keepends=True))
+    if actual == expected or actual is None or expected is None:
+        return text
+    return text.replace(actual, expected)
+
+
+# **********************************************************
+# Formatting features ends here
+# **********************************************************
+
+
+# **********************************************************
+# Required Language Server Initialization and Exit handlers.
+# **********************************************************
+@LSP_SERVER.feature(lsp.INITIALIZE)
+def initialize(params: lsp.InitializeParams) -> None:
+    """LSP handler for initialize request."""
+    log_to_output(f"CWD Server: {os.getcwd()}")
+
+    paths = "\r\n   ".join(sys.path)
+    log_to_output(f"sys.path used to run Server:\r\n   {paths}")
+
+    GLOBAL_SETTINGS.update(**params.initialization_options.get("globalSettings", {}))
+
+    settings = params.initialization_options["settings"]
+    _update_workspace_settings(settings)
+    log_to_output(
+        f"Settings used to run Server:\r\n{json.dumps(settings, indent=4, ensure_ascii=False)}\r\n"
+    )
+    log_to_output(
+        f"Global settings:\r\n{json.dumps(GLOBAL_SETTINGS, indent=4, ensure_ascii=False)}\r\n"
+    )
+
+
+@LSP_SERVER.feature(lsp.EXIT)
+def on_exit(_params: Optional[Any] = None) -> None:
+    """Handle clean up on exit."""
+    jsonrpc.shutdown_json_rpc()
+
+
+@LSP_SERVER.feature(lsp.SHUTDOWN)
+def on_shutdown(_params: Optional[Any] = None) -> None:
+    """Handle clean up on shutdown."""
+    jsonrpc.shutdown_json_rpc()
+
+
+def _get_global_defaults():
+    return {
+        "path": GLOBAL_SETTINGS.get("path", []),
+        "interpreter": GLOBAL_SETTINGS.get("interpreter", [sys.executable]),
+        "args": GLOBAL_SETTINGS.get("args", []),
+        "importStrategy": GLOBAL_SETTINGS.get("importStrategy", "useBundled"),
+        "showNotifications": GLOBAL_SETTINGS.get("showNotifications", "off"),
+    }
+
+
+def _update_workspace_settings(settings):
+    if not settings:
+        key = os.getcwd()
+        WORKSPACE_SETTINGS[key] = {
+            "cwd": key,
+            "workspaceFS": key,
+            "workspace": uris.from_fs_path(key),
+            **_get_global_defaults(),
+        }
+        return
+
+    for setting in settings:
+        key = uris.to_fs_path(setting["workspace"])
+        WORKSPACE_SETTINGS[key] = {
+            "cwd": key,
+            **setting,
+            "workspaceFS": key,
+        }
+
+
+def _get_settings_by_path(file_path: pathlib.Path):
+    workspaces = {s["workspaceFS"] for s in WORKSPACE_SETTINGS.values()}
+
+    while file_path != file_path.parent:
+        str_file_path = str(file_path)
+        if str_file_path in workspaces:
+            return WORKSPACE_SETTINGS[str_file_path]
+        file_path = file_path.parent
+
+    setting_values = list(WORKSPACE_SETTINGS.values())
+    return setting_values[0]
+
+
+def _get_document_key(document: workspace.Document):
+    if WORKSPACE_SETTINGS:
+        document_workspace = pathlib.Path(document.path)
+        workspaces = {s["workspaceFS"] for s in WORKSPACE_SETTINGS.values()}
+
+        # Find workspace settings for the given file.
+        while document_workspace != document_workspace.parent:
+            if str(document_workspace) in workspaces:
+                return str(document_workspace)
+            document_workspace = document_workspace.parent
+
+    return None
+
+
+def _get_settings_by_document(document: workspace.Document | None):
+    if document is None or document.path is None:
+        return list(WORKSPACE_SETTINGS.values())[0]
+
+    key = _get_document_key(document)
+    if key is None:
+        # This is either a non-workspace file or there is no workspace.
+        key = os.fspath(pathlib.Path(document.path).parent)
+        return {
+            "cwd": key,
+            "workspaceFS": key,
+            "workspace": uris.from_fs_path(key),
+            **_get_global_defaults(),
+        }
+
+    return WORKSPACE_SETTINGS[str(key)]
+
+
+# *****************************************************
+# Internal execution APIs.
+# *****************************************************
+def _run_tool_on_document(
+    document: workspace.Document,
+    use_stdin: bool = False,
+    extra_args: Optional[Sequence[str]] = None,
+) -> utils.RunResult | None:
+    """Runs tool on the given document.
+
+    if use_stdin is true then contents of the document is passed to the
+    tool via stdin.
+    """
+    if extra_args is None:
+        extra_args = []
+    if str(document.uri).startswith("vscode-notebook-cell"):
+        # TODO: Decide on if you want to skip notebook cells.
+        # Skip notebook cells
+        return None
+
+    if utils.is_stdlib_file(document.path):
+        # TODO: Decide on if you want to skip standard library files.
+        # Skip standard library python files.
+        return None
+
+    # deep copy here to prevent accidentally updating global settings.
+    settings = copy.deepcopy(_get_settings_by_document(document))
+
+    code_workspace = settings["workspaceFS"]
+    cwd = settings["cwd"]
+
+    use_path = False
+    use_rpc = False
+    if settings["path"]:
+        # 'path' setting takes priority over everything.
+        use_path = True
+        argv = settings["path"]
+    elif settings["interpreter"] and not utils.is_current_interpreter(
+        settings["interpreter"][0]
+    ):
+        # If there is a different interpreter set use JSON-RPC to the subprocess
+        # running under that interpreter.
+        argv = [TOOL_MODULE]
+        use_rpc = True
+    else:
+        # if the interpreter is same as the interpreter running this
+        # process then run as module.
+        argv = [TOOL_MODULE]
+
+    argv += TOOL_ARGS + settings["args"] + extra_args
+
+    if use_stdin:
+        # TODO: update these to pass the appropriate arguments to provide document contents
+        # to tool via stdin.
+        # For example, for pylint args for stdin looks like this:
+        #     pylint --from-stdin <path>
+        # Here `--from-stdin` path is used by pylint to make decisions on the file contents
+        # that are being processed. Like, applying exclusion rules.
+        # It should look like this when you pass it:
+        #     argv += ["--from-stdin", document.path]
+        # Read up on how your tool handles contents via stdin. If stdin is not supported use
+        # set use_stdin to False, or provide path, what ever is appropriate for your tool.
+        argv += []
+    else:
+        argv += [document.path]
+
+    if use_path:
+        # This mode is used when running executables.
+        log_to_output(" ".join(argv))
+        log_to_output(f"CWD Server: {cwd}")
+        result = utils.run_path(
+            argv=argv,
+            use_stdin=use_stdin,
+            cwd=cwd,
+            source=document.source.replace("\r\n", "\n"),
+        )
+        if result.stderr:
+            log_to_output(result.stderr)
+    elif use_rpc:
+        # This mode is used if the interpreter running this server is different from
+        # the interpreter used for running this server.
+        log_to_output(" ".join(settings["interpreter"] + ["-m"] + argv))
+        log_to_output(f"CWD Linter: {cwd}")
+
+        result = jsonrpc.run_over_json_rpc(
+            workspace=code_workspace,
+            interpreter=settings["interpreter"],
+            module=TOOL_MODULE,
+            argv=argv,
+            use_stdin=use_stdin,
+            cwd=cwd,
+            source=document.source,
+        )
+        if result.exception:
+            log_error(result.exception)
+            result = utils.RunResult(result.stdout, result.stderr)
+        elif result.stderr:
+            log_to_output(result.stderr)
+    else:
+        # In this mode the tool is run as a module in the same process as the language server.
+        log_to_output(" ".join([sys.executable, "-m"] + argv))
+        log_to_output(f"CWD Linter: {cwd}")
+        # This is needed to preserve sys.path, in cases where the tool modifies
+        # sys.path and that might not work for this scenario next time around.
+        with utils.substitute_attr(sys, "path", sys.path[:]):
+            try:
+                # TODO: `utils.run_module` is equivalent to running `python -m <pytool-module>`.
+                # If your tool supports a programmatic API then replace the function below
+                # with code for your tool. You can also use `utils.run_api` helper, which
+                # handles changing working directories, managing io streams, etc.
+                # Also update `_run_tool` function and `utils.run_module` in `lsp_runner.py`.
+                result = utils.run_module(
+                    module=TOOL_MODULE,
+                    argv=argv,
+                    use_stdin=use_stdin,
+                    cwd=cwd,
+                    source=document.source,
+                )
+            except Exception:
+                log_error(traceback.format_exc(chain=True))
+                raise
+        if result.stderr:
+            log_to_output(result.stderr)
+
+    log_to_output(f"{document.uri} :\r\n{result.stdout}")
+    return result
+
+
+def _run_tool(extra_args: Sequence[str]) -> utils.RunResult:
+    """Runs tool."""
+    # deep copy here to prevent accidentally updating global settings.
+    settings = copy.deepcopy(_get_settings_by_document(None))
+
+    code_workspace = settings["workspaceFS"]
+    cwd = settings["workspaceFS"]
+
+    use_path = False
+    use_rpc = False
+    if len(settings["path"]) > 0:
+        # 'path' setting takes priority over everything.
+        use_path = True
+        argv = settings["path"]
+    elif len(settings["interpreter"]) > 0 and not utils.is_current_interpreter(
+        settings["interpreter"][0]
+    ):
+        # If there is a different interpreter set use JSON-RPC to the subprocess
+        # running under that interpreter.
+        argv = [TOOL_MODULE]
+        use_rpc = True
+    else:
+        # if the interpreter is same as the interpreter running this
+        # process then run as module.
+        argv = [TOOL_MODULE]
+
+    argv += extra_args
+
+    if use_path:
+        # This mode is used when running executables.
+        log_to_output(" ".join(argv))
+        log_to_output(f"CWD Server: {cwd}")
+        result = utils.run_path(argv=argv, use_stdin=True, cwd=cwd)
+        if result.stderr:
+            log_to_output(result.stderr)
+    elif use_rpc:
+        # This mode is used if the interpreter running this server is different from
+        # the interpreter used for running this server.
+        log_to_output(" ".join(settings["interpreter"] + ["-m"] + argv))
+        log_to_output(f"CWD Linter: {cwd}")
+        result = jsonrpc.run_over_json_rpc(
+            workspace=code_workspace,
+            interpreter=settings["interpreter"],
+            module=TOOL_MODULE,
+            argv=argv,
+            use_stdin=True,
+            cwd=cwd,
+        )
+        if result.exception:
+            log_error(result.exception)
+            result = utils.RunResult(result.stdout, result.stderr)
+        elif result.stderr:
+            log_to_output(result.stderr)
+    else:
+        # In this mode the tool is run as a module in the same process as the language server.
+        log_to_output(" ".join([sys.executable, "-m"] + argv))
+        log_to_output(f"CWD Linter: {cwd}")
+        # This is needed to preserve sys.path, in cases where the tool modifies
+        # sys.path and that might not work for this scenario next time around.
+        with utils.substitute_attr(sys, "path", sys.path[:]):
+            try:
+                # TODO: `utils.run_module` is equivalent to running `python -m <pytool-module>`.
+                # If your tool supports a programmatic API then replace the function below
+                # with code for your tool. You can also use `utils.run_api` helper, which
+                # handles changing working directories, managing io streams, etc.
+                # Also update `_run_tool_on_document` function and `utils.run_module` in `lsp_runner.py`.
+                result = utils.run_module(
+                    module=TOOL_MODULE, argv=argv, use_stdin=True, cwd=cwd
+                )
+            except Exception:
+                log_error(traceback.format_exc(chain=True))
+                raise
+        if result.stderr:
+            log_to_output(result.stderr)
+
+    log_to_output(f"\r\n{result.stdout}\r\n")
+    return result
+
+
+# *****************************************************
+# Logging and notification.
+# *****************************************************
+def log_to_output(
+    message: str, msg_type: lsp.MessageType = lsp.MessageType.Log
+) -> None:
+    LSP_SERVER.show_message_log(message, msg_type)
+
+
+def log_error(message: str) -> None:
+    LSP_SERVER.show_message_log(message, lsp.MessageType.Error)
+    if os.getenv("LS_SHOW_NOTIFICATION", "off") in ["onError", "onWarning", "always"]:
+        LSP_SERVER.show_message(message, lsp.MessageType.Error)
+
+
+def log_warning(message: str) -> None:
+    LSP_SERVER.show_message_log(message, lsp.MessageType.Warning)
+    if os.getenv("LS_SHOW_NOTIFICATION", "off") in ["onWarning", "always"]:
+        LSP_SERVER.show_message(message, lsp.MessageType.Warning)
+
+
+def log_always(message: str) -> None:
+    LSP_SERVER.show_message_log(message, lsp.MessageType.Info)
+    if os.getenv("LS_SHOW_NOTIFICATION", "off") in ["always"]:
+        LSP_SERVER.show_message(message, lsp.MessageType.Info)
+
+
+# *****************************************************
+# Start the server.
+# *****************************************************
+if __name__ == "__main__":
+    LSP_SERVER.start_io()

server/src/lsp_utils.py

@@ -0,0 +1,207 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+"""Utility functions and classes for use with running tools over LSP."""
+from __future__ import annotations
+
+import contextlib
+import io
+import os
+import os.path
+import runpy
+import site
+import subprocess
+import sys
+import threading
+from typing import Any, Callable, List, Sequence, Tuple, Union
+
+# Save the working directory used when loading this module
+SERVER_CWD = os.getcwd()
+CWD_LOCK = threading.Lock()
+
+
+def as_list(content: Union[Any, List[Any], Tuple[Any]]) -> Union[List[Any], Tuple[Any]]:
+    """Ensures we always get a list"""
+    if isinstance(content, (list, tuple)):
+        return content
+    return [content]
+
+
+# pylint: disable-next=consider-using-generator
+_site_paths = tuple(
+    [
+        os.path.normcase(os.path.normpath(p))
+        for p in (as_list(site.getsitepackages()) + as_list(site.getusersitepackages()))
+    ]
+)
+
+
+def is_same_path(file_path1, file_path2) -> bool:
+    """Returns true if two paths are the same."""
+    return os.path.normcase(os.path.normpath(file_path1)) == os.path.normcase(
+        os.path.normpath(file_path2)
+    )
+
+
+def is_current_interpreter(executable) -> bool:
+    """Returns true if the executable path is same as the current interpreter."""
+    return is_same_path(executable, sys.executable)
+
+
+def is_stdlib_file(file_path) -> bool:
+    """Return True if the file belongs to standard library."""
+    return os.path.normcase(os.path.normpath(file_path)).startswith(_site_paths)
+
+
+# pylint: disable-next=too-few-public-methods
+class RunResult:
+    """Object to hold result from running tool."""
+
+    def __init__(self, stdout: str, stderr: str):
+        self.stdout: str = stdout
+        self.stderr: str = stderr
+
+
+class CustomIO(io.TextIOWrapper):
+    """Custom stream object to replace stdio."""
+
+    name = None
+
+    def __init__(self, name, encoding="utf-8", newline=None):
+        self._buffer = io.BytesIO()
+        self._buffer.name = name
+        super().__init__(self._buffer, encoding=encoding, newline=newline)
+
+    def close(self):
+        """Provide this close method which is used by some tools."""
+        # This is intentionally empty.
+
+    def get_value(self) -> str:
+        """Returns value from the buffer as string."""
+        self.seek(0)
+        return self.read()
+
+
+@contextlib.contextmanager
+def substitute_attr(obj: Any, attribute: str, new_value: Any):
+    """Manage object attributes context when using runpy.run_module()."""
+    old_value = getattr(obj, attribute)
+    setattr(obj, attribute, new_value)
+    yield
+    setattr(obj, attribute, old_value)
+
+
+@contextlib.contextmanager
+def redirect_io(stream: str, new_stream):
+    """Redirect stdio streams to a custom stream."""
+    old_stream = getattr(sys, stream)
+    setattr(sys, stream, new_stream)
+    yield
+    setattr(sys, stream, old_stream)
+
+
+@contextlib.contextmanager
+def change_cwd(new_cwd):
+    """Change working directory before running code."""
+    os.chdir(new_cwd)
+    yield
+    os.chdir(SERVER_CWD)
+
+
+def _run_module(
+    module: str, argv: Sequence[str], use_stdin: bool, source: str = None
+) -> RunResult:
+    """Runs as a module."""
+    str_output = CustomIO("<stdout>", encoding="utf-8")
+    str_error = CustomIO("<stderr>", encoding="utf-8")
+
+    with contextlib.suppress(SystemExit):
+        with substitute_attr(sys, "argv", argv):
+            with redirect_io("stdout", str_output):
+                with redirect_io("stderr", str_error):
+                    if use_stdin and source is not None:
+                        str_input = CustomIO("<stdin>", encoding="utf-8", newline="\n")
+                        with redirect_io("stdin", str_input):
+                            str_input.write(source)
+                            str_input.seek(0)
+                            runpy.run_module(module, run_name="__main__")
+                    else:
+                        runpy.run_module(module, run_name="__main__")
+
+    return RunResult(str_output.get_value(), str_error.get_value())
+
+
+def run_module(
+    module: str, argv: Sequence[str], use_stdin: bool, cwd: str, source: str = None
+) -> RunResult:
+    """Runs as a module."""
+    with CWD_LOCK:
+        if is_same_path(os.getcwd(), cwd):
+            return _run_module(module, argv, use_stdin, source)
+        with change_cwd(cwd):
+            return _run_module(module, argv, use_stdin, source)
+
+
+def run_path(
+    argv: Sequence[str], use_stdin: bool, cwd: str, source: str = None
+) -> RunResult:
+    """Runs as an executable."""
+    if use_stdin:
+        with subprocess.Popen(
+            argv,
+            encoding="utf-8",
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            stdin=subprocess.PIPE,
+            cwd=cwd,
+        ) as process:
+            return RunResult(*process.communicate(input=source))
+    else:
+        result = subprocess.run(
+            argv,
+            encoding="utf-8",
+            stdout=subprocess.PIPE,
+            stderr=subprocess.PIPE,
+            check=False,
+            cwd=cwd,
+        )
+        return RunResult(result.stdout, result.stderr)
+
+
+def run_api(
+    callback: Callable[[Sequence[str], CustomIO, CustomIO, CustomIO | None], None],
+    argv: Sequence[str],
+    use_stdin: bool,
+    cwd: str,
+    source: str = None,
+) -> RunResult:
+    """Run a API."""
+    with CWD_LOCK:
+        if is_same_path(os.getcwd(), cwd):
+            return _run_api(callback, argv, use_stdin, source)
+        with change_cwd(cwd):
+            return _run_api(callback, argv, use_stdin, source)
+
+
+def _run_api(
+    callback: Callable[[Sequence[str], CustomIO, CustomIO, CustomIO | None], None],
+    argv: Sequence[str],
+    use_stdin: bool,
+    source: str = None,
+) -> RunResult:
+    str_output = CustomIO("<stdout>", encoding="utf-8")
+    str_error = CustomIO("<stderr>", encoding="utf-8")
+
+    with contextlib.suppress(SystemExit):
+        with substitute_attr(sys, "argv", argv):
+            with redirect_io("stdout", str_output):
+                with redirect_io("stderr", str_error):
+                    if use_stdin and source is not None:
+                        str_input = CustomIO("<stdin>", encoding="utf-8", newline="\n")
+                        with redirect_io("stdin", str_input):
+                            str_input.write(source)
+                            str_input.seek(0)
+                            callback(argv, str_output, str_error, str_input)
+                    else:
+                        callback(argv, str_output, str_error)
+
+    return RunResult(str_output.get_value(), str_error.get_value())