From 666c537f077eba1695dc78b5dbd5aee77e075c05 Mon Sep 17 00:00:00 2001 From: christoph_xd Date: Thu, 24 Jul 2025 21:31:08 +0200 Subject: [PATCH] add tclint --- .../libs/voluptuous-0.15.2.dist-info/COPYING | 25 + .../voluptuous-0.15.2.dist-info/INSTALLER | 1 + .../libs/voluptuous-0.15.2.dist-info/METADATA | 743 ++++++++++ .../libs/voluptuous-0.15.2.dist-info/RECORD | 20 + .../voluptuous-0.15.2.dist-info/REQUESTED | 0 server/libs/voluptuous-0.15.2.dist-info/WHEEL | 5 + .../voluptuous-0.15.2.dist-info/top_level.txt | 1 + server/libs/voluptuous/__init__.py | 88 ++ server/libs/voluptuous/error.py | 219 +++ server/libs/voluptuous/humanize.py | 57 + server/libs/voluptuous/py.typed | 0 server/libs/voluptuous/schema_builder.py | 1315 +++++++++++++++++ server/libs/voluptuous/util.py | 149 ++ server/libs/voluptuous/validators.py | 1248 ++++++++++++++++ server/requirements.in | 3 +- server/requirements.txt | 4 + server/src/tools/__init__.py | 0 server/src/tools/checks.py | 425 +++--- server/src/tools/commands/__init__.py | 6 +- server/src/tools/commands/builtin.py | 1082 ++++++++++++++ server/src/tools/commands/checks.py | 240 +++ server/src/tools/commands/plugins.py | 86 ++ server/src/tools/commands/schema.py | 35 + server/src/tools/comments.py | 91 ++ server/src/tools/config.py | 429 ++++++ server/src/tools/format.py | 480 ++++++ server/src/tools/lexer.py | 133 +- server/src/tools/parser.py | 76 +- server/src/tools/syntax_tree.py | 14 +- server/src/tools/violations.py | 50 + test/test.tcl | 4 +- 31 files changed, 6721 insertions(+), 308 deletions(-) create mode 100644 server/libs/voluptuous-0.15.2.dist-info/COPYING create mode 100644 server/libs/voluptuous-0.15.2.dist-info/INSTALLER create mode 100644 server/libs/voluptuous-0.15.2.dist-info/METADATA create mode 100644 server/libs/voluptuous-0.15.2.dist-info/RECORD create mode 100644 server/libs/voluptuous-0.15.2.dist-info/REQUESTED create mode 100644 server/libs/voluptuous-0.15.2.dist-info/WHEEL create mode 100644 server/libs/voluptuous-0.15.2.dist-info/top_level.txt create mode 100644 server/libs/voluptuous/__init__.py create mode 100644 server/libs/voluptuous/error.py create mode 100644 server/libs/voluptuous/humanize.py create mode 100644 server/libs/voluptuous/py.typed create mode 100644 server/libs/voluptuous/schema_builder.py create mode 100644 server/libs/voluptuous/util.py create mode 100644 server/libs/voluptuous/validators.py create mode 100644 server/src/tools/__init__.py create mode 100644 server/src/tools/commands/builtin.py create mode 100644 server/src/tools/commands/checks.py create mode 100644 server/src/tools/commands/plugins.py create mode 100644 server/src/tools/commands/schema.py create mode 100644 server/src/tools/comments.py create mode 100644 server/src/tools/config.py create mode 100644 server/src/tools/format.py create mode 100644 server/src/tools/violations.py diff --git a/server/libs/voluptuous-0.15.2.dist-info/COPYING b/server/libs/voluptuous-0.15.2.dist-info/COPYING new file mode 100644 index 0000000..a19b705 --- /dev/null +++ b/server/libs/voluptuous-0.15.2.dist-info/COPYING @@ -0,0 +1,25 @@ +Copyright (c) 2010, Alec Thomas +All rights reserved. + +Redistribution and use in source and binary forms, with or without +modification, are permitted provided that the following conditions are met: + + - Redistributions of source code must retain the above copyright notice, this + list of conditions and the following disclaimer. + - Redistributions in binary form must reproduce the above copyright notice, + this list of conditions and the following disclaimer in the documentation + and/or other materials provided with the distribution. + - Neither the name of SwapOff.org nor the names of its contributors may + be used to endorse or promote products derived from this software without + specific prior written permission. + +THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND +ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED +WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE +DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE +FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL +DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR +SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER +CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, +OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. diff --git a/server/libs/voluptuous-0.15.2.dist-info/INSTALLER b/server/libs/voluptuous-0.15.2.dist-info/INSTALLER new file mode 100644 index 0000000..a1b589e --- /dev/null +++ b/server/libs/voluptuous-0.15.2.dist-info/INSTALLER @@ -0,0 +1 @@ +pip diff --git a/server/libs/voluptuous-0.15.2.dist-info/METADATA b/server/libs/voluptuous-0.15.2.dist-info/METADATA new file mode 100644 index 0000000..85d2ef1 --- /dev/null +++ b/server/libs/voluptuous-0.15.2.dist-info/METADATA @@ -0,0 +1,743 @@ +Metadata-Version: 2.1 +Name: voluptuous +Version: 0.15.2 +Summary: Python data validation library +Home-page: https://github.com/alecthomas/voluptuous +Download-URL: https://pypi.python.org/pypi/voluptuous +Author: Alec Thomas +Author-email: alec@swapoff.org +License: BSD-3-Clause +Platform: any +Classifier: Development Status :: 5 - Production/Stable +Classifier: Intended Audience :: Developers +Classifier: License :: OSI Approved :: BSD License +Classifier: Operating System :: OS Independent +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.9 +Classifier: Programming Language :: Python :: 3.10 +Classifier: Programming Language :: Python :: 3.11 +Classifier: Programming Language :: Python :: 3.12 +Requires-Python: >=3.9 +Description-Content-Type: text/markdown +License-File: COPYING + + +# CONTRIBUTIONS ONLY + +**What does this mean?** I do not have time to fix issues myself. The only way fixes or new features will be added is by people submitting PRs. + +**Current status:** Voluptuous is largely feature stable. There hasn't been a need to add new features in a while, but there are some bugs that should be fixed. + +**Why?** I no longer use Voluptuous personally (in fact I no longer regularly write Python code). Rather than leave the project in a limbo of people filing issues and wondering why they're not being worked on, I believe this notice will more clearly set expectations. + +# Voluptuous is a Python data validation library + +[![image](https://img.shields.io/pypi/v/voluptuous.svg)](https://python.org/pypi/voluptuous) +[![image](https://img.shields.io/pypi/l/voluptuous.svg)](https://python.org/pypi/voluptuous) +[![image](https://img.shields.io/pypi/pyversions/voluptuous.svg)](https://python.org/pypi/voluptuous) +[![Test status](https://github.com/alecthomas/voluptuous/actions/workflows/tests.yml/badge.svg)](https://github.com/alecthomas/voluptuous/actions/workflows/tests.yml) +[![Coverage status](https://coveralls.io/repos/github/alecthomas/voluptuous/badge.svg?branch=master)](https://coveralls.io/github/alecthomas/voluptuous?branch=master) +[![Gitter chat](https://badges.gitter.im/alecthomas.svg)](https://gitter.im/alecthomas/Lobby) + +Voluptuous, *despite* the name, is a Python data validation library. It +is primarily intended for validating data coming into Python as JSON, +YAML, etc. + +It has three goals: + +1. Simplicity. +2. Support for complex data structures. +3. Provide useful error messages. + +## Contact + +Voluptuous now has a mailing list! Send a mail to +[](mailto:voluptuous@librelist.com) to subscribe. Instructions +will follow. + +You can also contact me directly via [email](mailto:alec@swapoff.org) or +[Twitter](https://twitter.com/alecthomas). + +To file a bug, create a [new issue](https://github.com/alecthomas/voluptuous/issues/new) on GitHub with a short example of how to replicate the issue. + +## Documentation + +The documentation is provided [here](http://alecthomas.github.io/voluptuous/). + +## Contribution to Documentation + +Documentation is built using `Sphinx`. You can install it by + + pip install -r requirements.txt + +For building `sphinx-apidoc` from scratch you need to set PYTHONPATH to `voluptuous/voluptuous` repository. + +The documentation is provided [here.](http://alecthomas.github.io/voluptuous/) + +## Changelog + +See [CHANGELOG.md](https://github.com/alecthomas/voluptuous/blob/master/CHANGELOG.md). + +## Why use Voluptuous over another validation library? + +**Validators are simple callables:** +No need to subclass anything, just use a function. + +**Errors are simple exceptions:** +A validator can just `raise Invalid(msg)` and expect the user to get +useful messages. + +**Schemas are basic Python data structures:** +Should your data be a dictionary of integer keys to strings? +`{int: str}` does what you expect. List of integers, floats or +strings? `[int, float, str]`. + +**Designed from the ground up for validating more than just forms:** +Nested data structures are treated in the same way as any other +type. Need a list of dictionaries? `[{}]` + +**Consistency:** +Types in the schema are checked as types. Values are compared as +values. Callables are called to validate. Simple. + +## Show me an example + +Twitter's [user search API](https://dev.twitter.com/rest/reference/get/users/search) accepts +query URLs like: + +```bash +$ curl 'https://api.twitter.com/1.1/users/search.json?q=python&per_page=20&page=1' +``` + +To validate this we might use a schema like: + +```pycon +>>> from voluptuous import Schema +>>> schema = Schema({ +... 'q': str, +... 'per_page': int, +... 'page': int, +... }) +``` + +This schema very succinctly and roughly describes the data required by +the API, and will work fine. But it has a few problems. Firstly, it +doesn't fully express the constraints of the API. According to the API, +`per_page` should be restricted to at most 20, defaulting to 5, for +example. To describe the semantics of the API more accurately, our +schema will need to be more thoroughly defined: + +```pycon +>>> from voluptuous import Required, All, Length, Range +>>> schema = Schema({ +... Required('q'): All(str, Length(min=1)), +... Required('per_page', default=5): All(int, Range(min=1, max=20)), +... 'page': All(int, Range(min=0)), +... }) +``` + +This schema fully enforces the interface defined in Twitter's +documentation, and goes a little further for completeness. + +"q" is required: + +```pycon +>>> from voluptuous import MultipleInvalid, Invalid +>>> try: +... schema({}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "required key not provided @ data['q']" +True +``` + +...must be a string: + +```pycon +>>> try: +... schema({'q': 123}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "expected str for dictionary value @ data['q']" +True +``` + +...and must be at least one character in length: + +```pycon +>>> try: +... schema({'q': ''}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "length of value must be at least 1 for dictionary value @ data['q']" +True +>>> schema({'q': '#topic'}) == {'q': '#topic', 'per_page': 5} +True +``` + +"per\_page" is a positive integer no greater than 20: + +```pycon +>>> try: +... schema({'q': '#topic', 'per_page': 900}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "value must be at most 20 for dictionary value @ data['per_page']" +True +>>> try: +... schema({'q': '#topic', 'per_page': -10}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "value must be at least 1 for dictionary value @ data['per_page']" +True +``` + +"page" is an integer \>= 0: + +```pycon +>>> try: +... schema({'q': '#topic', 'per_page': 'one'}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) +"expected int for dictionary value @ data['per_page']" +>>> schema({'q': '#topic', 'page': 1}) == {'q': '#topic', 'page': 1, 'per_page': 5} +True +``` + +## Defining schemas + +Schemas are nested data structures consisting of dictionaries, lists, +scalars and *validators*. Each node in the input schema is pattern +matched against corresponding nodes in the input data. + +### Literals + +Literals in the schema are matched using normal equality checks: + +```pycon +>>> schema = Schema(1) +>>> schema(1) +1 +>>> schema = Schema('a string') +>>> schema('a string') +'a string' +``` + +### Types + +Types in the schema are matched by checking if the corresponding value +is an instance of the type: + +```pycon +>>> schema = Schema(int) +>>> schema(1) +1 +>>> try: +... schema('one') +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "expected int" +True +``` + +### URLs + +URLs in the schema are matched by using `urlparse` library. + +```pycon +>>> from voluptuous import Url +>>> schema = Schema(Url()) +>>> schema('http://w3.org') +'http://w3.org' +>>> try: +... schema('one') +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "expected a URL" +True +``` + +### Lists + +Lists in the schema are treated as a set of valid values. Each element +in the schema list is compared to each value in the input data: + +```pycon +>>> schema = Schema([1, 'a', 'string']) +>>> schema([1]) +[1] +>>> schema([1, 1, 1]) +[1, 1, 1] +>>> schema(['a', 1, 'string', 1, 'string']) +['a', 1, 'string', 1, 'string'] +``` + +However, an empty list (`[]`) is treated as is. If you want to specify a list that can +contain anything, specify it as `list`: + +```pycon +>>> schema = Schema([]) +>>> try: +... schema([1]) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "not a valid value @ data[1]" +True +>>> schema([]) +[] +>>> schema = Schema(list) +>>> schema([]) +[] +>>> schema([1, 2]) +[1, 2] +``` + +### Sets and frozensets + +Sets and frozensets are treated as a set of valid values. Each element +in the schema set is compared to each value in the input data: + +```pycon +>>> schema = Schema({42}) +>>> schema({42}) == {42} +True +>>> try: +... schema({43}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "invalid value in set" +True +>>> schema = Schema({int}) +>>> schema({1, 2, 3}) == {1, 2, 3} +True +>>> schema = Schema({int, str}) +>>> schema({1, 2, 'abc'}) == {1, 2, 'abc'} +True +>>> schema = Schema(frozenset([int])) +>>> try: +... schema({3}) +... raise AssertionError('Invalid not raised') +... except Invalid as e: +... exc = e +>>> str(exc) == 'expected a frozenset' +True +``` + +However, an empty set (`set()`) is treated as is. If you want to specify a set +that can contain anything, specify it as `set`: + +```pycon +>>> schema = Schema(set()) +>>> try: +... schema({1}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "invalid value in set" +True +>>> schema(set()) == set() +True +>>> schema = Schema(set) +>>> schema({1, 2}) == {1, 2} +True +``` + +### Validation functions + +Validators are simple callables that raise an `Invalid` exception when +they encounter invalid data. The criteria for determining validity is +entirely up to the implementation; it may check that a value is a valid +username with `pwd.getpwnam()`, it may check that a value is of a +specific type, and so on. + +The simplest kind of validator is a Python function that raises +ValueError when its argument is invalid. Conveniently, many builtin +Python functions have this property. Here's an example of a date +validator: + +```pycon +>>> from datetime import datetime +>>> def Date(fmt='%Y-%m-%d'): +... return lambda v: datetime.strptime(v, fmt) +``` + +```pycon +>>> schema = Schema(Date()) +>>> schema('2013-03-03') +datetime.datetime(2013, 3, 3, 0, 0) +>>> try: +... schema('2013-03') +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "not a valid value" +True +``` + +In addition to simply determining if a value is valid, validators may +mutate the value into a valid form. An example of this is the +`Coerce(type)` function, which returns a function that coerces its +argument to the given type: + +```python +def Coerce(type, msg=None): + """Coerce a value to a type. + + If the type constructor throws a ValueError, the value will be marked as + Invalid. + """ + def f(v): + try: + return type(v) + except ValueError: + raise Invalid(msg or ('expected %s' % type.__name__)) + return f +``` + +This example also shows a common idiom where an optional human-readable +message can be provided. This can vastly improve the usefulness of the +resulting error messages. + +### Dictionaries + +Each key-value pair in a schema dictionary is validated against each +key-value pair in the corresponding data dictionary: + +```pycon +>>> schema = Schema({1: 'one', 2: 'two'}) +>>> schema({1: 'one'}) +{1: 'one'} +``` + +#### Extra dictionary keys + +By default any additional keys in the data, not in the schema will +trigger exceptions: + +```pycon +>>> schema = Schema({2: 3}) +>>> try: +... schema({1: 2, 2: 3}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "extra keys not allowed @ data[1]" +True +``` + +This behaviour can be altered on a per-schema basis. To allow +additional keys use +`Schema(..., extra=ALLOW_EXTRA)`: + +```pycon +>>> from voluptuous import ALLOW_EXTRA +>>> schema = Schema({2: 3}, extra=ALLOW_EXTRA) +>>> schema({1: 2, 2: 3}) +{1: 2, 2: 3} +``` + +To remove additional keys use +`Schema(..., extra=REMOVE_EXTRA)`: + +```pycon +>>> from voluptuous import REMOVE_EXTRA +>>> schema = Schema({2: 3}, extra=REMOVE_EXTRA) +>>> schema({1: 2, 2: 3}) +{2: 3} +``` + +It can also be overridden per-dictionary by using the catch-all marker +token `extra` as a key: + +```pycon +>>> from voluptuous import Extra +>>> schema = Schema({1: {Extra: object}}) +>>> schema({1: {'foo': 'bar'}}) +{1: {'foo': 'bar'}} +``` + +#### Required dictionary keys + +By default, keys in the schema are not required to be in the data: + +```pycon +>>> schema = Schema({1: 2, 3: 4}) +>>> schema({3: 4}) +{3: 4} +``` + +Similarly to how extra\_ keys work, this behaviour can be overridden +per-schema: + +```pycon +>>> schema = Schema({1: 2, 3: 4}, required=True) +>>> try: +... schema({3: 4}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "required key not provided @ data[1]" +True +``` + +And per-key, with the marker token `Required(key)`: + +```pycon +>>> schema = Schema({Required(1): 2, 3: 4}) +>>> try: +... schema({3: 4}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "required key not provided @ data[1]" +True +>>> schema({1: 2}) +{1: 2} +``` + +#### Optional dictionary keys + +If a schema has `required=True`, keys may be individually marked as +optional using the marker token `Optional(key)`: + +```pycon +>>> from voluptuous import Optional +>>> schema = Schema({1: 2, Optional(3): 4}, required=True) +>>> try: +... schema({}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "required key not provided @ data[1]" +True +>>> schema({1: 2}) +{1: 2} +>>> try: +... schema({1: 2, 4: 5}) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "extra keys not allowed @ data[4]" +True +``` + +```pycon +>>> schema({1: 2, 3: 4}) +{1: 2, 3: 4} +``` + +### Recursive / nested schema + +You can use `voluptuous.Self` to define a nested schema: + +```pycon +>>> from voluptuous import Schema, Self +>>> recursive = Schema({"more": Self, "value": int}) +>>> recursive({"more": {"value": 42}, "value": 41}) == {'more': {'value': 42}, 'value': 41} +True +``` + +### Extending an existing Schema + +Often it comes handy to have a base `Schema` that is extended with more +requirements. In that case you can use `Schema.extend` to create a new +`Schema`: + +```pycon +>>> from voluptuous import Schema +>>> person = Schema({'name': str}) +>>> person_with_age = person.extend({'age': int}) +>>> sorted(list(person_with_age.schema.keys())) +['age', 'name'] +``` + +The original `Schema` remains unchanged. + +### Objects + +Each key-value pair in a schema dictionary is validated against each +attribute-value pair in the corresponding object: + +```pycon +>>> from voluptuous import Object +>>> class Structure(object): +... def __init__(self, q=None): +... self.q = q +... def __repr__(self): +... return ''.format(self) +... +>>> schema = Schema(Object({'q': 'one'}, cls=Structure)) +>>> schema(Structure(q='one')) + +``` + +### Allow None values + +To allow value to be None as well, use Any: + +```pycon +>>> from voluptuous import Any + +>>> schema = Schema(Any(None, int)) +>>> schema(None) +>>> schema(5) +5 +``` + +## Error reporting + +Validators must throw an `Invalid` exception if invalid data is passed +to them. All other exceptions are treated as errors in the validator and +will not be caught. + +Each `Invalid` exception has an associated `path` attribute representing +the path in the data structure to our currently validating value, as well +as an `error_message` attribute that contains the message of the original +exception. This is especially useful when you want to catch `Invalid` +exceptions and give some feedback to the user, for instance in the context of +an HTTP API. + +```pycon +>>> def validate_email(email): +... """Validate email.""" +... if not "@" in email: +... raise Invalid("This email is invalid.") +... return email +>>> schema = Schema({"email": validate_email}) +>>> exc = None +>>> try: +... schema({"email": "whatever"}) +... except MultipleInvalid as e: +... exc = e +>>> str(exc) +"This email is invalid. for dictionary value @ data['email']" +>>> exc.path +['email'] +>>> exc.msg +'This email is invalid.' +>>> exc.error_message +'This email is invalid.' +``` + +The `path` attribute is used during error reporting, but also during matching +to determine whether an error should be reported to the user or if the next +match should be attempted. This is determined by comparing the depth of the +path where the check is, to the depth of the path where the error occurred. If +the error is more than one level deeper, it is reported. + +The upshot of this is that *matching is depth-first and fail-fast*. + +To illustrate this, here is an example schema: + +```pycon +>>> schema = Schema([[2, 3], 6]) +``` + +Each value in the top-level list is matched depth-first in-order. Given +input data of `[[6]]`, the inner list will match the first element of +the schema, but the literal `6` will not match any of the elements of +that list. This error will be reported back to the user immediately. No +backtracking is attempted: + +```pycon +>>> try: +... schema([[6]]) +... raise AssertionError('MultipleInvalid not raised') +... except MultipleInvalid as e: +... exc = e +>>> str(exc) == "not a valid value @ data[0][0]" +True +``` + +If we pass the data `[6]`, the `6` is not a list type and so will not +recurse into the first element of the schema. Matching will continue on +to the second element in the schema, and succeed: + +```pycon +>>> schema([6]) +[6] +``` + +## Multi-field validation + +Validation rules that involve multiple fields can be implemented as +custom validators. It's recommended to use `All()` to do a two-pass +validation - the first pass checking the basic structure of the data, +and only after that, the second pass applying your cross-field +validator: + +```python +def passwords_must_match(passwords): + if passwords['password'] != passwords['password_again']: + raise Invalid('passwords must match') + return passwords + +schema = Schema(All( + # First "pass" for field types + {'password': str, 'password_again': str}, + # Follow up the first "pass" with your multi-field rules + passwords_must_match +)) + +# valid +schema({'password': '123', 'password_again': '123'}) + +# raises MultipleInvalid: passwords must match +schema({'password': '123', 'password_again': 'and now for something completely different'}) + +``` + +With this structure, your multi-field validator will run with +pre-validated data from the first "pass" and so will not have to do +its own type checking on its inputs. + +The flipside is that if the first "pass" of validation fails, your +cross-field validator will not run: + +```python +# raises Invalid because password_again is not a string +# passwords_must_match() will not run because first-pass validation already failed +schema({'password': '123', 'password_again': 1337}) +``` + +## Running tests + +Voluptuous is using `pytest`: + +```bash +$ pip install pytest +$ pytest +``` + +To also include a coverage report: + +```bash +$ pip install pytest pytest-cov coverage>=3.0 +$ pytest --cov=voluptuous voluptuous/tests/ +``` + +## Other libraries and inspirations + +Voluptuous is heavily inspired by +[Validino](http://code.google.com/p/validino/), and to a lesser extent, +[jsonvalidator](http://code.google.com/p/jsonvalidator/) and +[json\_schema](http://blog.sendapatch.se/category/json_schema.html). + +[pytest-voluptuous](https://github.com/F-Secure/pytest-voluptuous) is a +[pytest](https://github.com/pytest-dev/pytest) plugin that helps in +using voluptuous validators in `assert`s. + +I greatly prefer the light-weight style promoted by these libraries to +the complexity of libraries like FormEncode. + diff --git a/server/libs/voluptuous-0.15.2.dist-info/RECORD b/server/libs/voluptuous-0.15.2.dist-info/RECORD new file mode 100644 index 0000000..07b7692 --- /dev/null +++ b/server/libs/voluptuous-0.15.2.dist-info/RECORD @@ -0,0 +1,20 @@ +voluptuous-0.15.2.dist-info/COPYING,sha256=JHtJdren-k2J2Vh8qlCVVh60bcVFfyJ59ipitUUq3qk,1486 +voluptuous-0.15.2.dist-info/INSTALLER,sha256=zuuue4knoyJ-UwPPXg8fezS7VCrXJQrAP7zeNuwvFQg,4 +voluptuous-0.15.2.dist-info/METADATA,sha256=skO8Rp2Rq3VpxIPpE5LWhWiiWWXWHf9HL_-TFOkEz60,20641 +voluptuous-0.15.2.dist-info/RECORD,, +voluptuous-0.15.2.dist-info/REQUESTED,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +voluptuous-0.15.2.dist-info/WHEEL,sha256=GJ7t_kWBFywbagK5eo9IoUwLW6oyOeTKmQ-9iHFVNxQ,92 +voluptuous-0.15.2.dist-info/top_level.txt,sha256=TTdVb7M-vndb67UqTmAxuVjpAUakrlAWJYqvo3w4Iqc,11 +voluptuous/__init__.py,sha256=6_S65O_9lnoewl5dQSLIz_BKrsfxmOK-lG_i3Djd8Z8,2227 +voluptuous/__pycache__/__init__.cpython-311.pyc,, +voluptuous/__pycache__/error.cpython-311.pyc,, +voluptuous/__pycache__/humanize.cpython-311.pyc,, +voluptuous/__pycache__/schema_builder.cpython-311.pyc,, +voluptuous/__pycache__/util.cpython-311.pyc,, +voluptuous/__pycache__/validators.cpython-311.pyc,, +voluptuous/error.py,sha256=qipmadJhLycX4zIju6j8T8rjJHiiELVDv3CSoBCDnwM,4606 +voluptuous/humanize.py,sha256=CWBrrE6fK73iOM19w1CK9_f_Qrc92u2PQIjngG8-EC0,1905 +voluptuous/py.typed,sha256=47DEQpj8HBSa-_TImW-5JCeuQeRkm5NMpJWZG3hSuFU,0 +voluptuous/schema_builder.py,sha256=QDt5o1ZtLdqTtOd5IVzKczNBPftLKGk77Cz4UFJUD0g,43730 +voluptuous/util.py,sha256=BNxkVJZ6qbg8pDWY_TOMloLLgNgzixV1ZQ9rhTdbFgs,3174 +voluptuous/validators.py,sha256=wp3fmKr-KC7saw8aeUWw1CLOoxwrcj8YiteXJN9eUIQ,36501 diff --git a/server/libs/voluptuous-0.15.2.dist-info/REQUESTED b/server/libs/voluptuous-0.15.2.dist-info/REQUESTED new file mode 100644 index 0000000..e69de29 diff --git a/server/libs/voluptuous-0.15.2.dist-info/WHEEL b/server/libs/voluptuous-0.15.2.dist-info/WHEEL new file mode 100644 index 0000000..bab98d6 --- /dev/null +++ b/server/libs/voluptuous-0.15.2.dist-info/WHEEL @@ -0,0 +1,5 @@ +Wheel-Version: 1.0 +Generator: bdist_wheel (0.43.0) +Root-Is-Purelib: true +Tag: py3-none-any + diff --git a/server/libs/voluptuous-0.15.2.dist-info/top_level.txt b/server/libs/voluptuous-0.15.2.dist-info/top_level.txt new file mode 100644 index 0000000..55356d5 --- /dev/null +++ b/server/libs/voluptuous-0.15.2.dist-info/top_level.txt @@ -0,0 +1 @@ +voluptuous diff --git a/server/libs/voluptuous/__init__.py b/server/libs/voluptuous/__init__.py new file mode 100644 index 0000000..d030b35 --- /dev/null +++ b/server/libs/voluptuous/__init__.py @@ -0,0 +1,88 @@ +"""Schema validation for Python data structures. + +Given eg. a nested data structure like this: + + { + 'exclude': ['Users', 'Uptime'], + 'include': [], + 'set': { + 'snmp_community': 'public', + 'snmp_timeout': 15, + 'snmp_version': '2c', + }, + 'targets': { + 'localhost': { + 'exclude': ['Uptime'], + 'features': { + 'Uptime': { + 'retries': 3, + }, + 'Users': { + 'snmp_community': 'monkey', + 'snmp_port': 15, + }, + }, + 'include': ['Users'], + 'set': { + 'snmp_community': 'monkeys', + }, + }, + }, + } + +A schema like this: + + >>> settings = { + ... 'snmp_community': str, + ... 'retries': int, + ... 'snmp_version': All(Coerce(str), Any('3', '2c', '1')), + ... } + >>> features = ['Ping', 'Uptime', 'Http'] + >>> schema = Schema({ + ... 'exclude': features, + ... 'include': features, + ... 'set': settings, + ... 'targets': { + ... 'exclude': features, + ... 'include': features, + ... 'features': { + ... str: settings, + ... }, + ... }, + ... }) + +Validate like so: + + >>> schema({ + ... 'set': { + ... 'snmp_community': 'public', + ... 'snmp_version': '2c', + ... }, + ... 'targets': { + ... 'exclude': ['Ping'], + ... 'features': { + ... 'Uptime': {'retries': 3}, + ... 'Users': {'snmp_community': 'monkey'}, + ... }, + ... }, + ... }) == { + ... 'set': {'snmp_version': '2c', 'snmp_community': 'public'}, + ... 'targets': { + ... 'exclude': ['Ping'], + ... 'features': {'Uptime': {'retries': 3}, + ... 'Users': {'snmp_community': 'monkey'}}}} + True +""" + +# flake8: noqa +# fmt: off +from voluptuous.schema_builder import * +from voluptuous.util import * +from voluptuous.validators import * + +from voluptuous.error import * # isort: skip + +# fmt: on + +__version__ = '0.15.2' +__author__ = 'alecthomas' diff --git a/server/libs/voluptuous/error.py b/server/libs/voluptuous/error.py new file mode 100644 index 0000000..9dab943 --- /dev/null +++ b/server/libs/voluptuous/error.py @@ -0,0 +1,219 @@ +# fmt: off +import typing + +# fmt: on + + +class Error(Exception): + """Base validation exception.""" + + +class SchemaError(Error): + """An error was encountered in the schema.""" + + +class Invalid(Error): + """The data was invalid. + + :attr msg: The error message. + :attr path: The path to the error, as a list of keys in the source data. + :attr error_message: The actual error message that was raised, as a + string. + + """ + + def __init__( + self, + message: str, + path: typing.Optional[typing.List[typing.Hashable]] = None, + error_message: typing.Optional[str] = None, + error_type: typing.Optional[str] = None, + ) -> None: + Error.__init__(self, message) + self._path = path or [] + self._error_message = error_message or message + self.error_type = error_type + + @property + def msg(self) -> str: + return self.args[0] + + @property + def path(self) -> typing.List[typing.Hashable]: + return self._path + + @property + def error_message(self) -> str: + return self._error_message + + def __str__(self) -> str: + path = ' @ data[%s]' % ']['.join(map(repr, self.path)) if self.path else '' + output = Exception.__str__(self) + if self.error_type: + output += ' for ' + self.error_type + return output + path + + def prepend(self, path: typing.List[typing.Hashable]) -> None: + self._path = path + self.path + + +class MultipleInvalid(Invalid): + def __init__(self, errors: typing.Optional[typing.List[Invalid]] = None) -> None: + self.errors = errors[:] if errors else [] + + def __repr__(self) -> str: + return 'MultipleInvalid(%r)' % self.errors + + @property + def msg(self) -> str: + return self.errors[0].msg + + @property + def path(self) -> typing.List[typing.Hashable]: + return self.errors[0].path + + @property + def error_message(self) -> str: + return self.errors[0].error_message + + def add(self, error: Invalid) -> None: + self.errors.append(error) + + def __str__(self) -> str: + return str(self.errors[0]) + + def prepend(self, path: typing.List[typing.Hashable]) -> None: + for error in self.errors: + error.prepend(path) + + +class RequiredFieldInvalid(Invalid): + """Required field was missing.""" + + +class ObjectInvalid(Invalid): + """The value we found was not an object.""" + + +class DictInvalid(Invalid): + """The value found was not a dict.""" + + +class ExclusiveInvalid(Invalid): + """More than one value found in exclusion group.""" + + +class InclusiveInvalid(Invalid): + """Not all values found in inclusion group.""" + + +class SequenceTypeInvalid(Invalid): + """The type found is not a sequence type.""" + + +class TypeInvalid(Invalid): + """The value was not of required type.""" + + +class ValueInvalid(Invalid): + """The value was found invalid by evaluation function.""" + + +class ContainsInvalid(Invalid): + """List does not contain item""" + + +class ScalarInvalid(Invalid): + """Scalars did not match.""" + + +class CoerceInvalid(Invalid): + """Impossible to coerce value to type.""" + + +class AnyInvalid(Invalid): + """The value did not pass any validator.""" + + +class AllInvalid(Invalid): + """The value did not pass all validators.""" + + +class MatchInvalid(Invalid): + """The value does not match the given regular expression.""" + + +class RangeInvalid(Invalid): + """The value is not in given range.""" + + +class TrueInvalid(Invalid): + """The value is not True.""" + + +class FalseInvalid(Invalid): + """The value is not False.""" + + +class BooleanInvalid(Invalid): + """The value is not a boolean.""" + + +class UrlInvalid(Invalid): + """The value is not a URL.""" + + +class EmailInvalid(Invalid): + """The value is not an email address.""" + + +class FileInvalid(Invalid): + """The value is not a file.""" + + +class DirInvalid(Invalid): + """The value is not a directory.""" + + +class PathInvalid(Invalid): + """The value is not a path.""" + + +class LiteralInvalid(Invalid): + """The literal values do not match.""" + + +class LengthInvalid(Invalid): + pass + + +class DatetimeInvalid(Invalid): + """The value is not a formatted datetime string.""" + + +class DateInvalid(Invalid): + """The value is not a formatted date string.""" + + +class InInvalid(Invalid): + pass + + +class NotInInvalid(Invalid): + pass + + +class ExactSequenceInvalid(Invalid): + pass + + +class NotEnoughValid(Invalid): + """The value did not pass enough validations.""" + + pass + + +class TooManyValid(Invalid): + """The value passed more than expected validations.""" + + pass diff --git a/server/libs/voluptuous/humanize.py b/server/libs/voluptuous/humanize.py new file mode 100644 index 0000000..eabfd02 --- /dev/null +++ b/server/libs/voluptuous/humanize.py @@ -0,0 +1,57 @@ +# fmt: off +import typing + +from voluptuous import Invalid, MultipleInvalid +from voluptuous.error import Error +from voluptuous.schema_builder import Schema + +# fmt: on + +MAX_VALIDATION_ERROR_ITEM_LENGTH = 500 + + +def _nested_getitem( + data: typing.Any, path: typing.List[typing.Hashable] +) -> typing.Optional[typing.Any]: + for item_index in path: + try: + data = data[item_index] + except (KeyError, IndexError, TypeError): + # The index is not present in the dictionary, list or other + # indexable or data is not subscriptable + return None + return data + + +def humanize_error( + data, + validation_error: Invalid, + max_sub_error_length: int = MAX_VALIDATION_ERROR_ITEM_LENGTH, +) -> str: + """Provide a more helpful + complete validation error message than that provided automatically + Invalid and MultipleInvalid do not include the offending value in error messages, + and MultipleInvalid.__str__ only provides the first error. + """ + if isinstance(validation_error, MultipleInvalid): + return '\n'.join( + sorted( + humanize_error(data, sub_error, max_sub_error_length) + for sub_error in validation_error.errors + ) + ) + else: + offending_item_summary = repr(_nested_getitem(data, validation_error.path)) + if len(offending_item_summary) > max_sub_error_length: + offending_item_summary = ( + offending_item_summary[: max_sub_error_length - 3] + '...' + ) + return '%s. Got %s' % (validation_error, offending_item_summary) + + +def validate_with_humanized_errors( + data, schema: Schema, max_sub_error_length: int = MAX_VALIDATION_ERROR_ITEM_LENGTH +) -> typing.Any: + try: + return schema(data) + except (Invalid, MultipleInvalid) as e: + raise Error(humanize_error(data, e, max_sub_error_length)) diff --git a/server/libs/voluptuous/py.typed b/server/libs/voluptuous/py.typed new file mode 100644 index 0000000..e69de29 diff --git a/server/libs/voluptuous/schema_builder.py b/server/libs/voluptuous/schema_builder.py new file mode 100644 index 0000000..cdeb514 --- /dev/null +++ b/server/libs/voluptuous/schema_builder.py @@ -0,0 +1,1315 @@ +# fmt: off +from __future__ import annotations + +import collections +import inspect +import itertools +import re +import sys +import typing +from collections.abc import Generator +from contextlib import contextmanager +from functools import cache, wraps + +from voluptuous import error as er +from voluptuous.error import Error + +# fmt: on + +# options for extra keys +PREVENT_EXTRA = 0 # any extra key not in schema will raise an error +ALLOW_EXTRA = 1 # extra keys not in schema will be included in output +REMOVE_EXTRA = 2 # extra keys not in schema will be excluded from output + + +def _isnamedtuple(obj): + return isinstance(obj, tuple) and hasattr(obj, '_fields') + + +class Undefined(object): + def __nonzero__(self): + return False + + def __repr__(self): + return '...' + + +UNDEFINED = Undefined() + + +def Self() -> None: + raise er.SchemaError('"Self" should never be called') + + +DefaultFactory = typing.Union[Undefined, typing.Callable[[], typing.Any]] + + +def default_factory(value) -> DefaultFactory: + if value is UNDEFINED or callable(value): + return value + return lambda: value + + +@contextmanager +def raises( + exc, msg: typing.Optional[str] = None, regex: typing.Optional[re.Pattern] = None +) -> Generator[None, None, None]: + try: + yield + except exc as e: + if msg is not None: + assert str(e) == msg, '%r != %r' % (str(e), msg) + if regex is not None: + assert re.search(regex, str(e)), '%r does not match %r' % (str(e), regex) + else: + raise AssertionError(f"Did not raise exception {exc.__name__}") + + +def Extra(_) -> None: + """Allow keys in the data that are not present in the schema.""" + raise er.SchemaError('"Extra" should never be called') + + +# As extra() is never called there's no way to catch references to the +# deprecated object, so we just leave an alias here instead. +extra = Extra + +primitive_types = (bool, bytes, int, str, float, complex) + +# fmt: off +Schemable = typing.Union[ + 'Schema', 'Object', + collections.abc.Mapping, + list, tuple, frozenset, set, + bool, bytes, int, str, float, complex, + type, object, dict, None, typing.Callable +] +# fmt: on + + +class Schema(object): + """A validation schema. + + The schema is a Python tree-like structure where nodes are pattern + matched against corresponding trees of values. + + Nodes can be values, in which case a direct comparison is used, types, + in which case an isinstance() check is performed, or callables, which will + validate and optionally convert the value. + + We can equate schemas also. + + For Example: + + >>> v = Schema({Required('a'): str}) + >>> v1 = Schema({Required('a'): str}) + >>> v2 = Schema({Required('b'): str}) + >>> assert v == v1 + >>> assert v != v2 + + """ + + _extra_to_name = { + REMOVE_EXTRA: 'REMOVE_EXTRA', + ALLOW_EXTRA: 'ALLOW_EXTRA', + PREVENT_EXTRA: 'PREVENT_EXTRA', + } + + def __init__( + self, schema: Schemable, required: bool = False, extra: int = PREVENT_EXTRA + ) -> None: + """Create a new Schema. + + :param schema: Validation schema. See :module:`voluptuous` for details. + :param required: Keys defined in the schema must be in the data. + :param extra: Specify how extra keys in the data are treated: + - :const:`~voluptuous.PREVENT_EXTRA`: to disallow any undefined + extra keys (raise ``Invalid``). + - :const:`~voluptuous.ALLOW_EXTRA`: to include undefined extra + keys in the output. + - :const:`~voluptuous.REMOVE_EXTRA`: to exclude undefined extra keys + from the output. + - Any value other than the above defaults to + :const:`~voluptuous.PREVENT_EXTRA` + """ + self.schema: typing.Any = schema + self.required = required + self.extra = int(extra) # ensure the value is an integer + self._compiled = self._compile(schema) + + @classmethod + def infer(cls, data, **kwargs) -> Schema: + """Create a Schema from concrete data (e.g. an API response). + + For example, this will take a dict like: + + { + 'foo': 1, + 'bar': { + 'a': True, + 'b': False + }, + 'baz': ['purple', 'monkey', 'dishwasher'] + } + + And return a Schema: + + { + 'foo': int, + 'bar': { + 'a': bool, + 'b': bool + }, + 'baz': [str] + } + + Note: only very basic inference is supported. + """ + + def value_to_schema_type(value): + if isinstance(value, dict): + if len(value) == 0: + return dict + return {k: value_to_schema_type(v) for k, v in value.items()} + if isinstance(value, list): + if len(value) == 0: + return list + else: + return [value_to_schema_type(v) for v in value] + return type(value) + + return cls(value_to_schema_type(data), **kwargs) + + def __eq__(self, other): + if not isinstance(other, Schema): + return False + return other.schema == self.schema + + def __ne__(self, other): + return not (self == other) + + def __str__(self): + return str(self.schema) + + def __repr__(self): + return "" % ( + self.schema, + self._extra_to_name.get(self.extra, '??'), + self.required, + id(self), + ) + + def __call__(self, data): + """Validate data against this schema.""" + try: + return self._compiled([], data) + except er.MultipleInvalid: + raise + except er.Invalid as e: + raise er.MultipleInvalid([e]) + # return self.validate([], self.schema, data) + + def _compile(self, schema): + if schema is Extra: + return lambda _, v: v + if schema is Self: + return lambda p, v: self._compiled(p, v) + elif hasattr(schema, "__voluptuous_compile__"): + return schema.__voluptuous_compile__(self) + if isinstance(schema, Object): + return self._compile_object(schema) + if isinstance(schema, collections.abc.Mapping): + return self._compile_dict(schema) + elif isinstance(schema, list): + return self._compile_list(schema) + elif isinstance(schema, tuple): + return self._compile_tuple(schema) + elif isinstance(schema, (frozenset, set)): + return self._compile_set(schema) + type_ = type(schema) + if inspect.isclass(schema): + type_ = schema + if type_ in (*primitive_types, object, type(None)) or callable(schema): + return _compile_scalar(schema) + raise er.SchemaError('unsupported schema data type %r' % type(schema).__name__) + + def _compile_mapping(self, schema, invalid_msg=None): + """Create validator for given mapping.""" + invalid_msg = invalid_msg or 'mapping value' + + # Keys that may be required + all_required_keys = set( + key + for key in schema + if key is not Extra + and ( + (self.required and not isinstance(key, (Optional, Remove))) + or isinstance(key, Required) + ) + ) + + # Keys that may have defaults + all_default_keys = set( + key + for key in schema + if isinstance(key, Required) or isinstance(key, Optional) + ) + + _compiled_schema = {} + for skey, svalue in schema.items(): + new_key = self._compile(skey) + new_value = self._compile(svalue) + _compiled_schema[skey] = (new_key, new_value) + + candidates = list(_iterate_mapping_candidates(_compiled_schema)) + + # After we have the list of candidates in the correct order, we want to apply some optimization so that each + # key in the data being validated will be matched against the relevant schema keys only. + # No point in matching against different keys + additional_candidates = [] + candidates_by_key = {} + for skey, (ckey, cvalue) in candidates: + if type(skey) in primitive_types: + candidates_by_key.setdefault(skey, []).append((skey, (ckey, cvalue))) + elif isinstance(skey, Marker) and type(skey.schema) in primitive_types: + candidates_by_key.setdefault(skey.schema, []).append( + (skey, (ckey, cvalue)) + ) + else: + # These are wildcards such as 'int', 'str', 'Remove' and others which should be applied to all keys + additional_candidates.append((skey, (ckey, cvalue))) + + def validate_mapping(path, iterable, out): + required_keys = all_required_keys.copy() + + # Build a map of all provided key-value pairs. + # The type(out) is used to retain ordering in case a ordered + # map type is provided as input. + key_value_map = type(out)() + for key, value in iterable: + key_value_map[key] = value + + # Insert default values for non-existing keys. + for key in all_default_keys: + if ( + not isinstance(key.default, Undefined) + and key.schema not in key_value_map + ): + # A default value has been specified for this missing + # key, insert it. + key_value_map[key.schema] = key.default() + + errors = [] + for key, value in key_value_map.items(): + key_path = path + [key] + remove_key = False + + # Optimization. Validate against the matching key first, then fallback to the rest + relevant_candidates = itertools.chain( + candidates_by_key.get(key, []), additional_candidates + ) + + # compare each given key/value against all compiled key/values + # schema key, (compiled key, compiled value) + error = None + for skey, (ckey, cvalue) in relevant_candidates: + try: + new_key = ckey(key_path, key) + except er.Invalid as e: + if len(e.path) > len(key_path): + raise + if not error or len(e.path) > len(error.path): + error = e + continue + # Backtracking is not performed once a key is selected, so if + # the value is invalid we immediately throw an exception. + exception_errors = [] + # check if the key is marked for removal + is_remove = new_key is Remove + try: + cval = cvalue(key_path, value) + # include if it's not marked for removal + if not is_remove: + out[new_key] = cval + else: + remove_key = True + continue + except er.MultipleInvalid as e: + exception_errors.extend(e.errors) + except er.Invalid as e: + exception_errors.append(e) + + if exception_errors: + if is_remove or remove_key: + continue + for err in exception_errors: + if len(err.path) <= len(key_path): + err.error_type = invalid_msg + errors.append(err) + # If there is a validation error for a required + # key, this means that the key was provided. + # Discard the required key so it does not + # create an additional, noisy exception. + required_keys.discard(skey) + break + + # Key and value okay, mark as found in case it was + # a Required() field. + required_keys.discard(skey) + + break + else: + if remove_key: + # remove key + continue + elif self.extra == ALLOW_EXTRA: + out[key] = value + elif error: + errors.append(error) + elif self.extra != REMOVE_EXTRA: + errors.append(er.Invalid('extra keys not allowed', key_path)) + # else REMOVE_EXTRA: ignore the key so it's removed from output + + # for any required keys left that weren't found and don't have defaults: + for key in required_keys: + msg = ( + key.msg + if hasattr(key, 'msg') and key.msg + else 'required key not provided' + ) + errors.append(er.RequiredFieldInvalid(msg, path + [key])) + if errors: + raise er.MultipleInvalid(errors) + + return out + + return validate_mapping + + def _compile_object(self, schema): + """Validate an object. + + Has the same behavior as dictionary validator but work with object + attributes. + + For example: + + >>> class Structure(object): + ... def __init__(self, one=None, three=None): + ... self.one = one + ... self.three = three + ... + >>> validate = Schema(Object({'one': 'two', 'three': 'four'}, cls=Structure)) + >>> with raises(er.MultipleInvalid, "not a valid value for object value @ data['one']"): + ... validate(Structure(one='three')) + + """ + base_validate = self._compile_mapping(schema, invalid_msg='object value') + + def validate_object(path, data): + if schema.cls is not UNDEFINED and not isinstance(data, schema.cls): + raise er.ObjectInvalid('expected a {0!r}'.format(schema.cls), path) + iterable = _iterate_object(data) + iterable = filter(lambda item: item[1] is not None, iterable) + out = base_validate(path, iterable, {}) + return type(data)(**out) + + return validate_object + + def _compile_dict(self, schema): + """Validate a dictionary. + + A dictionary schema can contain a set of values, or at most one + validator function/type. + + A dictionary schema will only validate a dictionary: + + >>> validate = Schema({}) + >>> with raises(er.MultipleInvalid, 'expected a dictionary'): + ... validate([]) + + An invalid dictionary value: + + >>> validate = Schema({'one': 'two', 'three': 'four'}) + >>> with raises(er.MultipleInvalid, "not a valid value for dictionary value @ data['one']"): + ... validate({'one': 'three'}) + + An invalid key: + + >>> with raises(er.MultipleInvalid, "extra keys not allowed @ data['two']"): + ... validate({'two': 'three'}) + + + Validation function, in this case the "int" type: + + >>> validate = Schema({'one': 'two', 'three': 'four', int: str}) + + Valid integer input: + + >>> validate({10: 'twenty'}) + {10: 'twenty'} + + By default, a "type" in the schema (in this case "int") will be used + purely to validate that the corresponding value is of that type. It + will not Coerce the value: + + >>> with raises(er.MultipleInvalid, "extra keys not allowed @ data['10']"): + ... validate({'10': 'twenty'}) + + Wrap them in the Coerce() function to achieve this: + >>> from voluptuous import Coerce + >>> validate = Schema({'one': 'two', 'three': 'four', + ... Coerce(int): str}) + >>> validate({'10': 'twenty'}) + {10: 'twenty'} + + Custom message for required key + + >>> validate = Schema({Required('one', 'required'): 'two'}) + >>> with raises(er.MultipleInvalid, "required @ data['one']"): + ... validate({}) + + (This is to avoid unexpected surprises.) + + Multiple errors for nested field in a dict: + + >>> validate = Schema({ + ... 'adict': { + ... 'strfield': str, + ... 'intfield': int + ... } + ... }) + >>> try: + ... validate({ + ... 'adict': { + ... 'strfield': 123, + ... 'intfield': 'one' + ... } + ... }) + ... except er.MultipleInvalid as e: + ... print(sorted(str(i) for i in e.errors)) # doctest: +NORMALIZE_WHITESPACE + ["expected int for dictionary value @ data['adict']['intfield']", + "expected str for dictionary value @ data['adict']['strfield']"] + + """ + base_validate = self._compile_mapping(schema, invalid_msg='dictionary value') + + groups_of_exclusion = {} + groups_of_inclusion = {} + for node in schema: + if isinstance(node, Exclusive): + g = groups_of_exclusion.setdefault(node.group_of_exclusion, []) + g.append(node) + elif isinstance(node, Inclusive): + g = groups_of_inclusion.setdefault(node.group_of_inclusion, []) + g.append(node) + + def validate_dict(path, data): + if not isinstance(data, dict): + raise er.DictInvalid('expected a dictionary', path) + + errors = [] + for label, group in groups_of_exclusion.items(): + exists = False + for exclusive in group: + if exclusive.schema in data: + if exists: + msg = ( + exclusive.msg + if hasattr(exclusive, 'msg') and exclusive.msg + else "two or more values in the same group of exclusion '%s'" + % label + ) + next_path = path + [VirtualPathComponent(label)] + errors.append(er.ExclusiveInvalid(msg, next_path)) + break + exists = True + + if errors: + raise er.MultipleInvalid(errors) + + for label, group in groups_of_inclusion.items(): + included = [node.schema in data for node in group] + if any(included) and not all(included): + msg = ( + "some but not all values in the same group of inclusion '%s'" + % label + ) + for g in group: + if hasattr(g, 'msg') and g.msg: + msg = g.msg + break + next_path = path + [VirtualPathComponent(label)] + errors.append(er.InclusiveInvalid(msg, next_path)) + break + + if errors: + raise er.MultipleInvalid(errors) + + out = data.__class__() + return base_validate(path, data.items(), out) + + return validate_dict + + def _compile_sequence(self, schema, seq_type): + """Validate a sequence type. + + This is a sequence of valid values or validators tried in order. + + >>> validator = Schema(['one', 'two', int]) + >>> validator(['one']) + ['one'] + >>> with raises(er.MultipleInvalid, 'expected int @ data[0]'): + ... validator([3.5]) + >>> validator([1]) + [1] + """ + _compiled = [self._compile(s) for s in schema] + seq_type_name = seq_type.__name__ + + def validate_sequence(path, data): + if not isinstance(data, seq_type): + raise er.SequenceTypeInvalid('expected a %s' % seq_type_name, path) + + # Empty seq schema, reject any data. + if not schema: + if data: + raise er.MultipleInvalid( + [er.ValueInvalid('not a valid value', path if path else data)] + ) + return data + + out = [] + invalid = None + errors = [] + index_path = UNDEFINED + for i, value in enumerate(data): + index_path = path + [i] + invalid = None + for validate in _compiled: + try: + cval = validate(index_path, value) + if cval is not Remove: # do not include Remove values + out.append(cval) + break + except er.Invalid as e: + if len(e.path) > len(index_path): + raise + invalid = e + else: + errors.append(invalid) + if errors: + raise er.MultipleInvalid(errors) + + if _isnamedtuple(data): + return type(data)(*out) + else: + return type(data)(out) + + return validate_sequence + + def _compile_tuple(self, schema): + """Validate a tuple. + + A tuple is a sequence of valid values or validators tried in order. + + >>> validator = Schema(('one', 'two', int)) + >>> validator(('one',)) + ('one',) + >>> with raises(er.MultipleInvalid, 'expected int @ data[0]'): + ... validator((3.5,)) + >>> validator((1,)) + (1,) + """ + return self._compile_sequence(schema, tuple) + + def _compile_list(self, schema): + """Validate a list. + + A list is a sequence of valid values or validators tried in order. + + >>> validator = Schema(['one', 'two', int]) + >>> validator(['one']) + ['one'] + >>> with raises(er.MultipleInvalid, 'expected int @ data[0]'): + ... validator([3.5]) + >>> validator([1]) + [1] + """ + return self._compile_sequence(schema, list) + + def _compile_set(self, schema): + """Validate a set. + + A set is an unordered collection of unique elements. + + >>> validator = Schema({int}) + >>> validator(set([42])) == set([42]) + True + >>> with raises(er.Invalid, 'expected a set'): + ... validator(42) + >>> with raises(er.MultipleInvalid, 'invalid value in set'): + ... validator(set(['a'])) + """ + type_ = type(schema) + type_name = type_.__name__ + + def validate_set(path, data): + if not isinstance(data, type_): + raise er.Invalid('expected a %s' % type_name, path) + + _compiled = [self._compile(s) for s in schema] + errors = [] + for value in data: + for validate in _compiled: + try: + validate(path, value) + break + except er.Invalid: + pass + else: + invalid = er.Invalid('invalid value in %s' % type_name, path) + errors.append(invalid) + + if errors: + raise er.MultipleInvalid(errors) + + return data + + return validate_set + + def extend( + self, + schema: Schemable, + required: typing.Optional[bool] = None, + extra: typing.Optional[int] = None, + ) -> Schema: + """Create a new `Schema` by merging this and the provided `schema`. + + Neither this `Schema` nor the provided `schema` are modified. The + resulting `Schema` inherits the `required` and `extra` parameters of + this, unless overridden. + + Both schemas must be dictionary-based. + + :param schema: dictionary to extend this `Schema` with + :param required: if set, overrides `required` of this `Schema` + :param extra: if set, overrides `extra` of this `Schema` + """ + + assert isinstance(self.schema, dict) and isinstance( + schema, dict + ), 'Both schemas must be dictionary-based' + + result = self.schema.copy() + + # returns the key that may have been passed as an argument to Marker constructor + def key_literal(key): + return key.schema if isinstance(key, Marker) else key + + # build a map that takes the key literals to the needed objects + # literal -> Required|Optional|literal + result_key_map = dict((key_literal(key), key) for key in result) + + # for each item in the extension schema, replace duplicates + # or add new keys + for key, value in schema.items(): + # if the key is already in the dictionary, we need to replace it + # transform key to literal before checking presence + if key_literal(key) in result_key_map: + result_key = result_key_map[key_literal(key)] + result_value = result[result_key] + + # if both are dictionaries, we need to extend recursively + # create the new extended sub schema, then remove the old key and add the new one + if isinstance(result_value, dict) and isinstance(value, dict): + new_value = Schema(result_value).extend(value).schema + del result[result_key] + result[key] = new_value + # one or the other or both are not sub-schemas, simple replacement is fine + # remove old key and add new one + else: + del result[result_key] + result[key] = value + + # key is new and can simply be added + else: + result[key] = value + + # recompile and send old object + result_cls = type(self) + result_required = required if required is not None else self.required + result_extra = extra if extra is not None else self.extra + return result_cls(result, required=result_required, extra=result_extra) + + +def _compile_scalar(schema): + """A scalar value. + + The schema can either be a value or a type. + + >>> _compile_scalar(int)([], 1) + 1 + >>> with raises(er.Invalid, 'expected float'): + ... _compile_scalar(float)([], '1') + + Callables have + >>> _compile_scalar(lambda v: float(v))([], '1') + 1.0 + + As a convenience, ValueError's are trapped: + + >>> with raises(er.Invalid, 'not a valid value'): + ... _compile_scalar(lambda v: float(v))([], 'a') + """ + if inspect.isclass(schema): + + def validate_instance(path, data): + if isinstance(data, schema): + return data + else: + msg = 'expected %s' % schema.__name__ + raise er.TypeInvalid(msg, path) + + return validate_instance + + if callable(schema): + + def validate_callable(path, data): + try: + return schema(data) + except ValueError: + raise er.ValueInvalid('not a valid value', path) + except er.Invalid as e: + e.prepend(path) + raise + + return validate_callable + + def validate_value(path, data): + if data != schema: + raise er.ScalarInvalid('not a valid value', path) + return data + + return validate_value + + +def _compile_itemsort(): + '''return sort function of mappings''' + + def is_extra(key_): + return key_ is Extra + + def is_remove(key_): + return isinstance(key_, Remove) + + def is_marker(key_): + return isinstance(key_, Marker) + + def is_type(key_): + return inspect.isclass(key_) + + def is_callable(key_): + return callable(key_) + + # priority list for map sorting (in order of checking) + # We want Extra to match last, because it's a catch-all. On the other hand, + # Remove markers should match first (since invalid values will not + # raise an Error, instead the validator will check if other schemas match + # the same value). + priority = [ + (1, is_remove), # Remove highest priority after values + (2, is_marker), # then other Markers + (4, is_type), # types/classes lowest before Extra + (3, is_callable), # callables after markers + (5, is_extra), # Extra lowest priority + ] + + def item_priority(item_): + key_ = item_[0] + for i, check_ in priority: + if check_(key_): + return i + # values have highest priorities + return 0 + + return item_priority + + +_sort_item = _compile_itemsort() + + +def _iterate_mapping_candidates(schema): + """Iterate over schema in a meaningful order.""" + # Without this, Extra might appear first in the iterator, and fail to + # validate a key even though it's a Required that has its own validation, + # generating a false positive. + return sorted(schema.items(), key=_sort_item) + + +def _iterate_object(obj): + """Return iterator over object attributes. Respect objects with + defined __slots__. + + """ + d = {} + try: + d = vars(obj) + except TypeError: + # maybe we have named tuple here? + if hasattr(obj, '_asdict'): + d = obj._asdict() + for item in d.items(): + yield item + try: + slots = obj.__slots__ + except AttributeError: + pass + else: + for key in slots: + if key != '__dict__': + yield (key, getattr(obj, key)) + + +class Msg(object): + """Report a user-friendly message if a schema fails to validate. + + >>> validate = Schema( + ... Msg(['one', 'two', int], + ... 'should be one of "one", "two" or an integer')) + >>> with raises(er.MultipleInvalid, 'should be one of "one", "two" or an integer'): + ... validate(['three']) + + Messages are only applied to invalid direct descendants of the schema: + + >>> validate = Schema(Msg([['one', 'two', int]], 'not okay!')) + >>> with raises(er.MultipleInvalid, 'expected int @ data[0][0]'): + ... validate([['three']]) + + The type which is thrown can be overridden but needs to be a subclass of Invalid + + >>> with raises(er.SchemaError, 'Msg can only use subclases of Invalid as custom class'): + ... validate = Schema(Msg([int], 'should be int', cls=KeyError)) + + If you do use a subclass of Invalid, that error will be thrown (wrapped in a MultipleInvalid) + + >>> validate = Schema(Msg([['one', 'two', int]], 'not okay!', cls=er.RangeInvalid)) + >>> try: + ... validate(['three']) + ... except er.MultipleInvalid as e: + ... assert isinstance(e.errors[0], er.RangeInvalid) + """ + + def __init__( + self, + schema: Schemable, + msg: str, + cls: typing.Optional[typing.Type[Error]] = None, + ) -> None: + if cls and not issubclass(cls, er.Invalid): + raise er.SchemaError( + "Msg can only use subclases of Invalid as custom class" + ) + self._schema = schema + self.schema = Schema(schema) + self.msg = msg + self.cls = cls + + def __call__(self, v): + try: + return self.schema(v) + except er.Invalid as e: + if len(e.path) > 1: + raise e + else: + raise (self.cls or er.Invalid)(self.msg) + + def __repr__(self): + return 'Msg(%s, %s, cls=%s)' % (self._schema, self.msg, self.cls) + + +class Object(dict): + """Indicate that we should work with attributes, not keys.""" + + def __init__(self, schema: typing.Any, cls: object = UNDEFINED) -> None: + self.cls = cls + super(Object, self).__init__(schema) + + +class VirtualPathComponent(str): + def __str__(self): + return '<' + self + '>' + + def __repr__(self): + return self.__str__() + + +class Marker(object): + """Mark nodes for special treatment. + + `description` is an optional field, unused by Voluptuous itself, but can be + introspected by any external tool, for example to generate schema documentation. + """ + + __slots__ = ('schema', '_schema', 'msg', 'description', '__hash__') + + def __init__( + self, + schema_: Schemable, + msg: typing.Optional[str] = None, + description: typing.Any | None = None, + ) -> None: + self.schema: typing.Any = schema_ + self._schema = Schema(schema_) + self.msg = msg + self.description = description + self.__hash__ = cache(lambda: hash(schema_)) # type: ignore[method-assign] + + def __call__(self, v): + try: + return self._schema(v) + except er.Invalid as e: + if not self.msg or len(e.path) > 1: + raise + raise er.Invalid(self.msg) + + def __str__(self): + return str(self.schema) + + def __repr__(self): + return repr(self.schema) + + def __lt__(self, other): + if isinstance(other, Marker): + return self.schema < other.schema + return self.schema < other + + def __eq__(self, other): + return self.schema == other + + def __ne__(self, other): + return not (self.schema == other) + + +class Optional(Marker): + """Mark a node in the schema as optional, and optionally provide a default + + >>> schema = Schema({Optional('key'): str}) + >>> schema({}) + {} + >>> schema = Schema({Optional('key', default='value'): str}) + >>> schema({}) + {'key': 'value'} + >>> schema = Schema({Optional('key', default=list): list}) + >>> schema({}) + {'key': []} + + If 'required' flag is set for an entire schema, optional keys aren't required + + >>> schema = Schema({ + ... Optional('key'): str, + ... 'key2': str + ... }, required=True) + >>> schema({'key2':'value'}) + {'key2': 'value'} + """ + + def __init__( + self, + schema: Schemable, + msg: typing.Optional[str] = None, + default: typing.Any = UNDEFINED, + description: typing.Any | None = None, + ) -> None: + super(Optional, self).__init__(schema, msg=msg, description=description) + self.default = default_factory(default) + + +class Exclusive(Optional): + """Mark a node in the schema as exclusive. + + Exclusive keys inherited from Optional: + + >>> schema = Schema({Exclusive('alpha', 'angles'): int, Exclusive('beta', 'angles'): int}) + >>> schema({'alpha': 30}) + {'alpha': 30} + + Keys inside a same group of exclusion cannot be together, it only makes sense for dictionaries: + + >>> with raises(er.MultipleInvalid, "two or more values in the same group of exclusion 'angles' @ data[]"): + ... schema({'alpha': 30, 'beta': 45}) + + For example, API can provides multiple types of authentication, but only one works in the same time: + + >>> msg = 'Please, use only one type of authentication at the same time.' + >>> schema = Schema({ + ... Exclusive('classic', 'auth', msg=msg):{ + ... Required('email'): str, + ... Required('password'): str + ... }, + ... Exclusive('internal', 'auth', msg=msg):{ + ... Required('secret_key'): str + ... }, + ... Exclusive('social', 'auth', msg=msg):{ + ... Required('social_network'): str, + ... Required('token'): str + ... } + ... }) + + >>> with raises(er.MultipleInvalid, "Please, use only one type of authentication at the same time. @ data[]"): + ... schema({'classic': {'email': 'foo@example.com', 'password': 'bar'}, + ... 'social': {'social_network': 'barfoo', 'token': 'tEMp'}}) + """ + + def __init__( + self, + schema: Schemable, + group_of_exclusion: str, + msg: typing.Optional[str] = None, + description: typing.Any | None = None, + ) -> None: + super(Exclusive, self).__init__(schema, msg=msg, description=description) + self.group_of_exclusion = group_of_exclusion + + +class Inclusive(Optional): + """Mark a node in the schema as inclusive. + + Inclusive keys inherited from Optional: + + >>> schema = Schema({ + ... Inclusive('filename', 'file'): str, + ... Inclusive('mimetype', 'file'): str + ... }) + >>> data = {'filename': 'dog.jpg', 'mimetype': 'image/jpeg'} + >>> data == schema(data) + True + + Keys inside a same group of inclusive must exist together, it only makes sense for dictionaries: + + >>> with raises(er.MultipleInvalid, "some but not all values in the same group of inclusion 'file' @ data[]"): + ... schema({'filename': 'dog.jpg'}) + + If none of the keys in the group are present, it is accepted: + + >>> schema({}) + {} + + For example, API can return 'height' and 'width' together, but not separately. + + >>> msg = "Height and width must exist together" + >>> schema = Schema({ + ... Inclusive('height', 'size', msg=msg): int, + ... Inclusive('width', 'size', msg=msg): int + ... }) + + >>> with raises(er.MultipleInvalid, msg + " @ data[]"): + ... schema({'height': 100}) + + >>> with raises(er.MultipleInvalid, msg + " @ data[]"): + ... schema({'width': 100}) + + >>> data = {'height': 100, 'width': 100} + >>> data == schema(data) + True + """ + + def __init__( + self, + schema: Schemable, + group_of_inclusion: str, + msg: typing.Optional[str] = None, + description: typing.Any | None = None, + default: typing.Any = UNDEFINED, + ) -> None: + super(Inclusive, self).__init__( + schema, msg=msg, default=default, description=description + ) + self.group_of_inclusion = group_of_inclusion + + +class Required(Marker): + """Mark a node in the schema as being required, and optionally provide a default value. + + >>> schema = Schema({Required('key'): str}) + >>> with raises(er.MultipleInvalid, "required key not provided @ data['key']"): + ... schema({}) + + >>> schema = Schema({Required('key', default='value'): str}) + >>> schema({}) + {'key': 'value'} + >>> schema = Schema({Required('key', default=list): list}) + >>> schema({}) + {'key': []} + """ + + def __init__( + self, + schema: Schemable, + msg: typing.Optional[str] = None, + default: typing.Any = UNDEFINED, + description: typing.Any | None = None, + ) -> None: + super(Required, self).__init__(schema, msg=msg, description=description) + self.default = default_factory(default) + + +class Remove(Marker): + """Mark a node in the schema to be removed and excluded from the validated + output. Keys that fail validation will not raise ``Invalid``. Instead, these + keys will be treated as extras. + + >>> schema = Schema({str: int, Remove(int): str}) + >>> with raises(er.MultipleInvalid, "extra keys not allowed @ data[1]"): + ... schema({'keep': 1, 1: 1.0}) + >>> schema({1: 'red', 'red': 1, 2: 'green'}) + {'red': 1} + >>> schema = Schema([int, Remove(float), Extra]) + >>> schema([1, 2, 3, 4.0, 5, 6.0, '7']) + [1, 2, 3, 5, '7'] + """ + + def __init__( + self, + schema_: Schemable, + msg: typing.Optional[str] = None, + description: typing.Any | None = None, + ) -> None: + super().__init__(schema_, msg, description) + self.__hash__ = cache(lambda: object.__hash__(self)) # type: ignore[method-assign] + + def __call__(self, schema: Schemable): + super(Remove, self).__call__(schema) + return self.__class__ + + def __repr__(self): + return "Remove(%r)" % (self.schema,) + + +def message( + default: typing.Optional[str] = None, + cls: typing.Optional[typing.Type[Error]] = None, +) -> typing.Callable: + """Convenience decorator to allow functions to provide a message. + + Set a default message: + + >>> @message('not an integer') + ... def isint(v): + ... return int(v) + + >>> validate = Schema(isint()) + >>> with raises(er.MultipleInvalid, 'not an integer'): + ... validate('a') + + The message can be overridden on a per validator basis: + + >>> validate = Schema(isint('bad')) + >>> with raises(er.MultipleInvalid, 'bad'): + ... validate('a') + + The class thrown too: + + >>> class IntegerInvalid(er.Invalid): pass + >>> validate = Schema(isint('bad', clsoverride=IntegerInvalid)) + >>> try: + ... validate('a') + ... except er.MultipleInvalid as e: + ... assert isinstance(e.errors[0], IntegerInvalid) + """ + if cls and not issubclass(cls, er.Invalid): + raise er.SchemaError( + "message can only use subclases of Invalid as custom class" + ) + + def decorator(f): + @wraps(f) + def check(msg=None, clsoverride=None): + @wraps(f) + def wrapper(*args, **kwargs): + try: + return f(*args, **kwargs) + except ValueError: + raise (clsoverride or cls or er.ValueInvalid)( + msg or default or 'invalid value' + ) + + return wrapper + + return check + + return decorator + + +def _args_to_dict(func, args): + """Returns argument names as values as key-value pairs.""" + if sys.version_info >= (3, 0): + arg_count = func.__code__.co_argcount + arg_names = func.__code__.co_varnames[:arg_count] + else: + arg_count = func.func_code.co_argcount + arg_names = func.func_code.co_varnames[:arg_count] + + arg_value_list = list(args) + arguments = dict( + (arg_name, arg_value_list[i]) + for i, arg_name in enumerate(arg_names) + if i < len(arg_value_list) + ) + return arguments + + +def _merge_args_with_kwargs(args_dict, kwargs_dict): + """Merge args with kwargs.""" + ret = args_dict.copy() + ret.update(kwargs_dict) + return ret + + +def validate(*a, **kw) -> typing.Callable: + """Decorator for validating arguments of a function against a given schema. + + Set restrictions for arguments: + + >>> @validate(arg1=int, arg2=int) + ... def foo(arg1, arg2): + ... return arg1 * arg2 + + Set restriction for returned value: + + >>> @validate(arg=int, __return__=int) + ... def bar(arg1): + ... return arg1 * 2 + + """ + RETURNS_KEY = '__return__' + + def validate_schema_decorator(func): + returns_defined = False + returns = None + + schema_args_dict = _args_to_dict(func, a) + schema_arguments = _merge_args_with_kwargs(schema_args_dict, kw) + + if RETURNS_KEY in schema_arguments: + returns_defined = True + returns = schema_arguments[RETURNS_KEY] + del schema_arguments[RETURNS_KEY] + + input_schema = ( + Schema(schema_arguments, extra=ALLOW_EXTRA) + if len(schema_arguments) != 0 + else lambda x: x + ) + output_schema = Schema(returns) if returns_defined else lambda x: x + + @wraps(func) + def func_wrapper(*args, **kwargs): + args_dict = _args_to_dict(func, args) + arguments = _merge_args_with_kwargs(args_dict, kwargs) + validated_arguments = input_schema(arguments) + output = func(**validated_arguments) + return output_schema(output) + + return func_wrapper + + return validate_schema_decorator diff --git a/server/libs/voluptuous/util.py b/server/libs/voluptuous/util.py new file mode 100644 index 0000000..0bf9302 --- /dev/null +++ b/server/libs/voluptuous/util.py @@ -0,0 +1,149 @@ +# F401: "imported but unused" +# fmt: off +import typing + +from voluptuous import validators # noqa: F401 +from voluptuous.error import Invalid, LiteralInvalid, TypeInvalid # noqa: F401 +from voluptuous.schema_builder import DefaultFactory # noqa: F401 +from voluptuous.schema_builder import Schema, default_factory, raises # noqa: F401 + +# fmt: on + +__author__ = 'tusharmakkar08' + + +def Lower(v: str) -> str: + """Transform a string to lower case. + + >>> s = Schema(Lower) + >>> s('HI') + 'hi' + """ + return str(v).lower() + + +def Upper(v: str) -> str: + """Transform a string to upper case. + + >>> s = Schema(Upper) + >>> s('hi') + 'HI' + """ + return str(v).upper() + + +def Capitalize(v: str) -> str: + """Capitalise a string. + + >>> s = Schema(Capitalize) + >>> s('hello world') + 'Hello world' + """ + return str(v).capitalize() + + +def Title(v: str) -> str: + """Title case a string. + + >>> s = Schema(Title) + >>> s('hello world') + 'Hello World' + """ + return str(v).title() + + +def Strip(v: str) -> str: + """Strip whitespace from a string. + + >>> s = Schema(Strip) + >>> s(' hello world ') + 'hello world' + """ + return str(v).strip() + + +class DefaultTo(object): + """Sets a value to default_value if none provided. + + >>> s = Schema(DefaultTo(42)) + >>> s(None) + 42 + >>> s = Schema(DefaultTo(list)) + >>> s(None) + [] + """ + + def __init__(self, default_value, msg: typing.Optional[str] = None) -> None: + self.default_value = default_factory(default_value) + self.msg = msg + + def __call__(self, v): + if v is None: + v = self.default_value() + return v + + def __repr__(self): + return 'DefaultTo(%s)' % (self.default_value(),) + + +class SetTo(object): + """Set a value, ignoring any previous value. + + >>> s = Schema(validators.Any(int, SetTo(42))) + >>> s(2) + 2 + >>> s("foo") + 42 + """ + + def __init__(self, value) -> None: + self.value = default_factory(value) + + def __call__(self, v): + return self.value() + + def __repr__(self): + return 'SetTo(%s)' % (self.value(),) + + +class Set(object): + """Convert a list into a set. + + >>> s = Schema(Set()) + >>> s([]) == set([]) + True + >>> s([1, 2]) == set([1, 2]) + True + >>> with raises(Invalid, regex="^cannot be presented as set: "): + ... s([set([1, 2]), set([3, 4])]) + """ + + def __init__(self, msg: typing.Optional[str] = None) -> None: + self.msg = msg + + def __call__(self, v): + try: + set_v = set(v) + except Exception as e: + raise TypeInvalid(self.msg or 'cannot be presented as set: {0}'.format(e)) + return set_v + + def __repr__(self): + return 'Set()' + + +class Literal(object): + def __init__(self, lit) -> None: + self.lit = lit + + def __call__(self, value, msg: typing.Optional[str] = None): + if self.lit != value: + raise LiteralInvalid(msg or '%s not match for %s' % (value, self.lit)) + else: + return self.lit + + def __str__(self): + return str(self.lit) + + def __repr__(self): + return repr(self.lit) diff --git a/server/libs/voluptuous/validators.py b/server/libs/voluptuous/validators.py new file mode 100644 index 0000000..d385260 --- /dev/null +++ b/server/libs/voluptuous/validators.py @@ -0,0 +1,1248 @@ +# fmt: off +from __future__ import annotations + +import datetime +import os +import re +import sys +import typing +from decimal import Decimal, InvalidOperation +from functools import wraps + +from voluptuous.error import ( + AllInvalid, AnyInvalid, BooleanInvalid, CoerceInvalid, ContainsInvalid, DateInvalid, + DatetimeInvalid, DirInvalid, EmailInvalid, ExactSequenceInvalid, FalseInvalid, + FileInvalid, InInvalid, Invalid, LengthInvalid, MatchInvalid, MultipleInvalid, + NotEnoughValid, NotInInvalid, PathInvalid, RangeInvalid, TooManyValid, TrueInvalid, + TypeInvalid, UrlInvalid, +) + +# F401: flake8 complains about 'raises' not being used, but it is used in doctests +from voluptuous.schema_builder import Schema, Schemable, message, raises # noqa: F401 + +if typing.TYPE_CHECKING: + from _typeshed import SupportsAllComparisons + +# fmt: on + + +Enum: typing.Union[type, None] +try: + from enum import Enum +except ImportError: + Enum = None + + +if sys.version_info >= (3,): + import urllib.parse as urlparse + + basestring = str +else: + import urlparse + +# Taken from https://github.com/kvesteri/validators/blob/master/validators/email.py +# fmt: off +USER_REGEX = re.compile( + # start anchor, because fullmatch is not available in python 2.7 + "(?:" + # dot-atom + r"(^[-!#$%&'*+/=?^_`{}|~0-9A-Z]+" + r"(\.[-!#$%&'*+/=?^_`{}|~0-9A-Z]+)*$" + # quoted-string + r'|^"([\001-\010\013\014\016-\037!#-\[\]-\177]|' + r"""\\[\001-\011\013\014\016-\177])*"$)""" + # end anchor, because fullmatch is not available in python 2.7 + r")\Z", + re.IGNORECASE, +) +DOMAIN_REGEX = re.compile( + # start anchor, because fullmatch is not available in python 2.7 + "(?:" + # domain + r'(?:[A-Z0-9](?:[A-Z0-9-]{0,61}[A-Z0-9])?\.)+' + # tld + r'(?:[A-Z]{2,6}\.?|[A-Z0-9-]{2,}\.?$)' + # literal form, ipv4 address (SMTP 4.1.3) + r'|^\[(25[0-5]|2[0-4]\d|[0-1]?\d?\d)' + r'(\.(25[0-5]|2[0-4]\d|[0-1]?\d?\d)){3}\]$' + # end anchor, because fullmatch is not available in python 2.7 + r")\Z", + re.IGNORECASE, +) +# fmt: on + +__author__ = 'tusharmakkar08' + + +def truth(f: typing.Callable) -> typing.Callable: + """Convenience decorator to convert truth functions into validators. + + >>> @truth + ... def isdir(v): + ... return os.path.isdir(v) + >>> validate = Schema(isdir) + >>> validate('/') + '/' + >>> with raises(MultipleInvalid, 'not a valid value'): + ... validate('/notavaliddir') + """ + + @wraps(f) + def check(v): + t = f(v) + if not t: + raise ValueError + return v + + return check + + +class Coerce(object): + """Coerce a value to a type. + + If the type constructor throws a ValueError or TypeError, the value + will be marked as Invalid. + + Default behavior: + + >>> validate = Schema(Coerce(int)) + >>> with raises(MultipleInvalid, 'expected int'): + ... validate(None) + >>> with raises(MultipleInvalid, 'expected int'): + ... validate('foo') + + With custom message: + + >>> validate = Schema(Coerce(int, "moo")) + >>> with raises(MultipleInvalid, 'moo'): + ... validate('foo') + """ + + def __init__( + self, + type: typing.Union[type, typing.Callable], + msg: typing.Optional[str] = None, + ) -> None: + self.type = type + self.msg = msg + self.type_name = type.__name__ + + def __call__(self, v): + try: + return self.type(v) + except (ValueError, TypeError, InvalidOperation): + msg = self.msg or ('expected %s' % self.type_name) + if not self.msg and Enum and issubclass(self.type, Enum): + msg += " or one of %s" % str([e.value for e in self.type])[1:-1] + raise CoerceInvalid(msg) + + def __repr__(self): + return 'Coerce(%s, msg=%r)' % (self.type_name, self.msg) + + +@message('value was not true', cls=TrueInvalid) +@truth +def IsTrue(v): + """Assert that a value is true, in the Python sense. + + >>> validate = Schema(IsTrue()) + + "In the Python sense" means that implicitly false values, such as empty + lists, dictionaries, etc. are treated as "false": + + >>> with raises(MultipleInvalid, "value was not true"): + ... validate([]) + >>> validate([1]) + [1] + >>> with raises(MultipleInvalid, "value was not true"): + ... validate(False) + + ...and so on. + + >>> try: + ... validate([]) + ... except MultipleInvalid as e: + ... assert isinstance(e.errors[0], TrueInvalid) + """ + return v + + +@message('value was not false', cls=FalseInvalid) +def IsFalse(v): + """Assert that a value is false, in the Python sense. + + (see :func:`IsTrue` for more detail) + + >>> validate = Schema(IsFalse()) + >>> validate([]) + [] + >>> with raises(MultipleInvalid, "value was not false"): + ... validate(True) + + >>> try: + ... validate(True) + ... except MultipleInvalid as e: + ... assert isinstance(e.errors[0], FalseInvalid) + """ + if v: + raise ValueError + return v + + +@message('expected boolean', cls=BooleanInvalid) +def Boolean(v): + """Convert human-readable boolean values to a bool. + + Accepted values are 1, true, yes, on, enable, and their negatives. + Non-string values are cast to bool. + + >>> validate = Schema(Boolean()) + >>> validate(True) + True + >>> validate("1") + True + >>> validate("0") + False + >>> with raises(MultipleInvalid, "expected boolean"): + ... validate('moo') + >>> try: + ... validate('moo') + ... except MultipleInvalid as e: + ... assert isinstance(e.errors[0], BooleanInvalid) + """ + if isinstance(v, basestring): + v = v.lower() + if v in ('1', 'true', 'yes', 'on', 'enable'): + return True + if v in ('0', 'false', 'no', 'off', 'disable'): + return False + raise ValueError + return bool(v) + + +class _WithSubValidators(object): + """Base class for validators that use sub-validators. + + Special class to use as a parent class for validators using sub-validators. + This class provides the `__voluptuous_compile__` method so the + sub-validators are compiled by the parent `Schema`. + """ + + def __init__( + self, *validators, msg=None, required=False, discriminant=None, **kwargs + ) -> None: + self.validators = validators + self.msg = msg + self.required = required + self.discriminant = discriminant + + def __voluptuous_compile__(self, schema: Schema) -> typing.Callable: + self._compiled = [] + old_required = schema.required + self.schema = schema + for v in self.validators: + schema.required = self.required + self._compiled.append(schema._compile(v)) + schema.required = old_required + return self._run + + def _run(self, path: typing.List[typing.Hashable], value): + if self.discriminant is not None: + self._compiled = [ + self.schema._compile(v) + for v in self.discriminant(value, self.validators) + ] + + return self._exec(self._compiled, value, path) + + def __call__(self, v): + return self._exec((Schema(val) for val in self.validators), v) + + def __repr__(self): + return '%s(%s, msg=%r)' % ( + self.__class__.__name__, + ", ".join(repr(v) for v in self.validators), + self.msg, + ) + + def _exec( + self, + funcs: typing.Iterable, + v, + path: typing.Optional[typing.List[typing.Hashable]] = None, + ): + raise NotImplementedError() + + +class Any(_WithSubValidators): + """Use the first validated value. + + :param msg: Message to deliver to user if validation fails. + :param kwargs: All other keyword arguments are passed to the sub-schema constructors. + :returns: Return value of the first validator that passes. + + >>> validate = Schema(Any('true', 'false', + ... All(Any(int, bool), Coerce(bool)))) + >>> validate('true') + 'true' + >>> validate(1) + True + >>> with raises(MultipleInvalid, "not a valid value"): + ... validate('moo') + + msg argument is used + + >>> validate = Schema(Any(1, 2, 3, msg="Expected 1 2 or 3")) + >>> validate(1) + 1 + >>> with raises(MultipleInvalid, "Expected 1 2 or 3"): + ... validate(4) + """ + + def _exec(self, funcs, v, path=None): + error = None + for func in funcs: + try: + if path is None: + return func(v) + else: + return func(path, v) + except Invalid as e: + if error is None or len(e.path) > len(error.path): + error = e + else: + if error: + raise error if self.msg is None else AnyInvalid(self.msg, path=path) + raise AnyInvalid(self.msg or 'no valid value found', path=path) + + +# Convenience alias +Or = Any + + +class Union(_WithSubValidators): + """Use the first validated value among those selected by discriminant. + + :param msg: Message to deliver to user if validation fails. + :param discriminant(value, validators): Returns the filtered list of validators based on the value. + :param kwargs: All other keyword arguments are passed to the sub-schema constructors. + :returns: Return value of the first validator that passes. + + >>> validate = Schema(Union({'type':'a', 'a_val':'1'},{'type':'b', 'b_val':'2'}, + ... discriminant=lambda val, alt: filter( + ... lambda v : v['type'] == val['type'] , alt))) + >>> validate({'type':'a', 'a_val':'1'}) == {'type':'a', 'a_val':'1'} + True + >>> with raises(MultipleInvalid, "not a valid value for dictionary value @ data['b_val']"): + ... validate({'type':'b', 'b_val':'5'}) + + ```discriminant({'type':'b', 'a_val':'5'}, [{'type':'a', 'a_val':'1'},{'type':'b', 'b_val':'2'}])``` is invoked + + Without the discriminant, the exception would be "extra keys not allowed @ data['b_val']" + """ + + def _exec(self, funcs, v, path=None): + error = None + for func in funcs: + try: + if path is None: + return func(v) + else: + return func(path, v) + except Invalid as e: + if error is None or len(e.path) > len(error.path): + error = e + else: + if error: + raise error if self.msg is None else AnyInvalid(self.msg, path=path) + raise AnyInvalid(self.msg or 'no valid value found', path=path) + + +# Convenience alias +Switch = Union + + +class All(_WithSubValidators): + """Value must pass all validators. + + The output of each validator is passed as input to the next. + + :param msg: Message to deliver to user if validation fails. + :param kwargs: All other keyword arguments are passed to the sub-schema constructors. + + >>> validate = Schema(All('10', Coerce(int))) + >>> validate('10') + 10 + """ + + def _exec(self, funcs, v, path=None): + try: + for func in funcs: + if path is None: + v = func(v) + else: + v = func(path, v) + except Invalid as e: + raise e if self.msg is None else AllInvalid(self.msg, path=path) + return v + + +# Convenience alias +And = All + + +class Match(object): + """Value must be a string that matches the regular expression. + + >>> validate = Schema(Match(r'^0x[A-F0-9]+$')) + >>> validate('0x123EF4') + '0x123EF4' + >>> with raises(MultipleInvalid, 'does not match regular expression ^0x[A-F0-9]+$'): + ... validate('123EF4') + + >>> with raises(MultipleInvalid, 'expected string or buffer'): + ... validate(123) + + Pattern may also be a compiled regular expression: + + >>> validate = Schema(Match(re.compile(r'0x[A-F0-9]+', re.I))) + >>> validate('0x123ef4') + '0x123ef4' + """ + + def __init__( + self, pattern: typing.Union[re.Pattern, str], msg: typing.Optional[str] = None + ) -> None: + if isinstance(pattern, basestring): + pattern = re.compile(pattern) + self.pattern = pattern + self.msg = msg + + def __call__(self, v): + try: + match = self.pattern.match(v) + except TypeError: + raise MatchInvalid("expected string or buffer") + if not match: + raise MatchInvalid( + self.msg + or 'does not match regular expression {}'.format(self.pattern.pattern) + ) + return v + + def __repr__(self): + return 'Match(%r, msg=%r)' % (self.pattern.pattern, self.msg) + + +class Replace(object): + """Regex substitution. + + >>> validate = Schema(All(Replace('you', 'I'), + ... Replace('hello', 'goodbye'))) + >>> validate('you say hello') + 'I say goodbye' + """ + + def __init__( + self, + pattern: typing.Union[re.Pattern, str], + substitution: str, + msg: typing.Optional[str] = None, + ) -> None: + if isinstance(pattern, basestring): + pattern = re.compile(pattern) + self.pattern = pattern + self.substitution = substitution + self.msg = msg + + def __call__(self, v): + return self.pattern.sub(self.substitution, v) + + def __repr__(self): + return 'Replace(%r, %r, msg=%r)' % ( + self.pattern.pattern, + self.substitution, + self.msg, + ) + + +def _url_validation(v: str) -> urlparse.ParseResult: + parsed = urlparse.urlparse(v) + if not parsed.scheme or not parsed.netloc: + raise UrlInvalid("must have a URL scheme and host") + return parsed + + +@message('expected an email address', cls=EmailInvalid) +def Email(v): + """Verify that the value is an email address or not. + + >>> s = Schema(Email()) + >>> with raises(MultipleInvalid, 'expected an email address'): + ... s("a.com") + >>> with raises(MultipleInvalid, 'expected an email address'): + ... s("a@.com") + >>> with raises(MultipleInvalid, 'expected an email address'): + ... s("a@.com") + >>> s('t@x.com') + 't@x.com' + """ + try: + if not v or "@" not in v: + raise EmailInvalid("Invalid email address") + user_part, domain_part = v.rsplit('@', 1) + + if not (USER_REGEX.match(user_part) and DOMAIN_REGEX.match(domain_part)): + raise EmailInvalid("Invalid email address") + return v + except: # noqa: E722 + raise ValueError + + +@message('expected a fully qualified domain name URL', cls=UrlInvalid) +def FqdnUrl(v): + """Verify that the value is a fully qualified domain name URL. + + >>> s = Schema(FqdnUrl()) + >>> with raises(MultipleInvalid, 'expected a fully qualified domain name URL'): + ... s("http://localhost/") + >>> s('http://w3.org') + 'http://w3.org' + """ + try: + parsed_url = _url_validation(v) + if "." not in parsed_url.netloc: + raise UrlInvalid("must have a domain name in URL") + return v + except: # noqa: E722 + raise ValueError + + +@message('expected a URL', cls=UrlInvalid) +def Url(v): + """Verify that the value is a URL. + + >>> s = Schema(Url()) + >>> with raises(MultipleInvalid, 'expected a URL'): + ... s(1) + >>> s('http://w3.org') + 'http://w3.org' + """ + try: + _url_validation(v) + return v + except: # noqa: E722 + raise ValueError + + +@message('Not a file', cls=FileInvalid) +@truth +def IsFile(v): + """Verify the file exists. + + >>> os.path.basename(IsFile()(__file__)).startswith('validators.py') + True + >>> with raises(FileInvalid, 'Not a file'): + ... IsFile()("random_filename_goes_here.py") + >>> with raises(FileInvalid, 'Not a file'): + ... IsFile()(None) + """ + try: + if v: + v = str(v) + return os.path.isfile(v) + else: + raise FileInvalid('Not a file') + except TypeError: + raise FileInvalid('Not a file') + + +@message('Not a directory', cls=DirInvalid) +@truth +def IsDir(v): + """Verify the directory exists. + + >>> IsDir()('/') + '/' + >>> with raises(DirInvalid, 'Not a directory'): + ... IsDir()(None) + """ + try: + if v: + v = str(v) + return os.path.isdir(v) + else: + raise DirInvalid("Not a directory") + except TypeError: + raise DirInvalid("Not a directory") + + +@message('path does not exist', cls=PathInvalid) +@truth +def PathExists(v): + """Verify the path exists, regardless of its type. + + >>> os.path.basename(PathExists()(__file__)).startswith('validators.py') + True + >>> with raises(Invalid, 'path does not exist'): + ... PathExists()("random_filename_goes_here.py") + >>> with raises(PathInvalid, 'Not a Path'): + ... PathExists()(None) + """ + try: + if v: + v = str(v) + return os.path.exists(v) + else: + raise PathInvalid("Not a Path") + except TypeError: + raise PathInvalid("Not a Path") + + +def Maybe(validator: Schemable, msg: typing.Optional[str] = None): + """Validate that the object matches given validator or is None. + + :raises Invalid: If the value does not match the given validator and is not + None. + + >>> s = Schema(Maybe(int)) + >>> s(10) + 10 + >>> with raises(Invalid): + ... s("string") + + """ + return Any(None, validator, msg=msg) + + +class Range(object): + """Limit a value to a range. + + Either min or max may be omitted. + Either min or max can be excluded from the range of accepted values. + + :raises Invalid: If the value is outside the range. + + >>> s = Schema(Range(min=1, max=10, min_included=False)) + >>> s(5) + 5 + >>> s(10) + 10 + >>> with raises(MultipleInvalid, 'value must be at most 10'): + ... s(20) + >>> with raises(MultipleInvalid, 'value must be higher than 1'): + ... s(1) + >>> with raises(MultipleInvalid, 'value must be lower than 10'): + ... Schema(Range(max=10, max_included=False))(20) + """ + + def __init__( + self, + min: SupportsAllComparisons | None = None, + max: SupportsAllComparisons | None = None, + min_included: bool = True, + max_included: bool = True, + msg: typing.Optional[str] = None, + ) -> None: + self.min = min + self.max = max + self.min_included = min_included + self.max_included = max_included + self.msg = msg + + def __call__(self, v): + try: + if self.min_included: + if self.min is not None and not v >= self.min: + raise RangeInvalid( + self.msg or 'value must be at least %s' % self.min + ) + else: + if self.min is not None and not v > self.min: + raise RangeInvalid( + self.msg or 'value must be higher than %s' % self.min + ) + if self.max_included: + if self.max is not None and not v <= self.max: + raise RangeInvalid( + self.msg or 'value must be at most %s' % self.max + ) + else: + if self.max is not None and not v < self.max: + raise RangeInvalid( + self.msg or 'value must be lower than %s' % self.max + ) + + return v + + # Objects that lack a partial ordering, e.g. None or strings will raise TypeError + except TypeError: + raise RangeInvalid( + self.msg or 'invalid value or type (must have a partial ordering)' + ) + + def __repr__(self): + return 'Range(min=%r, max=%r, min_included=%r, max_included=%r, msg=%r)' % ( + self.min, + self.max, + self.min_included, + self.max_included, + self.msg, + ) + + +class Clamp(object): + """Clamp a value to a range. + + Either min or max may be omitted. + + >>> s = Schema(Clamp(min=0, max=1)) + >>> s(0.5) + 0.5 + >>> s(5) + 1 + >>> s(-1) + 0 + """ + + def __init__( + self, + min: SupportsAllComparisons | None = None, + max: SupportsAllComparisons | None = None, + msg: typing.Optional[str] = None, + ) -> None: + self.min = min + self.max = max + self.msg = msg + + def __call__(self, v): + try: + if self.min is not None and v < self.min: + v = self.min + if self.max is not None and v > self.max: + v = self.max + return v + + # Objects that lack a partial ordering, e.g. None or strings will raise TypeError + except TypeError: + raise RangeInvalid( + self.msg or 'invalid value or type (must have a partial ordering)' + ) + + def __repr__(self): + return 'Clamp(min=%s, max=%s)' % (self.min, self.max) + + +class Length(object): + """The length of a value must be in a certain range.""" + + def __init__( + self, + min: SupportsAllComparisons | None = None, + max: SupportsAllComparisons | None = None, + msg: typing.Optional[str] = None, + ) -> None: + self.min = min + self.max = max + self.msg = msg + + def __call__(self, v): + try: + if self.min is not None and len(v) < self.min: + raise LengthInvalid( + self.msg or 'length of value must be at least %s' % self.min + ) + if self.max is not None and len(v) > self.max: + raise LengthInvalid( + self.msg or 'length of value must be at most %s' % self.max + ) + return v + + # Objects that have no length e.g. None or strings will raise TypeError + except TypeError: + raise RangeInvalid(self.msg or 'invalid value or type') + + def __repr__(self): + return 'Length(min=%s, max=%s)' % (self.min, self.max) + + +class Datetime(object): + """Validate that the value matches the datetime format.""" + + DEFAULT_FORMAT = '%Y-%m-%dT%H:%M:%S.%fZ' + + def __init__( + self, format: typing.Optional[str] = None, msg: typing.Optional[str] = None + ) -> None: + self.format = format or self.DEFAULT_FORMAT + self.msg = msg + + def __call__(self, v): + try: + datetime.datetime.strptime(v, self.format) + except (TypeError, ValueError): + raise DatetimeInvalid( + self.msg or 'value does not match expected format %s' % self.format + ) + return v + + def __repr__(self): + return 'Datetime(format=%s)' % self.format + + +class Date(Datetime): + """Validate that the value matches the date format.""" + + DEFAULT_FORMAT = '%Y-%m-%d' + + def __call__(self, v): + try: + datetime.datetime.strptime(v, self.format) + except (TypeError, ValueError): + raise DateInvalid( + self.msg or 'value does not match expected format %s' % self.format + ) + return v + + def __repr__(self): + return 'Date(format=%s)' % self.format + + +class In(object): + """Validate that a value is in a collection.""" + + def __init__( + self, container: typing.Container, msg: typing.Optional[str] = None + ) -> None: + self.container = container + self.msg = msg + + def __call__(self, v): + try: + check = v not in self.container + except TypeError: + check = True + if check: + try: + raise InInvalid( + self.msg or f'value must be one of {sorted(self.container)}' + ) + except TypeError: + raise InInvalid( + self.msg + or f'value must be one of {sorted(self.container, key=str)}' + ) + return v + + def __repr__(self): + return 'In(%s)' % (self.container,) + + +class NotIn(object): + """Validate that a value is not in a collection.""" + + def __init__( + self, container: typing.Iterable, msg: typing.Optional[str] = None + ) -> None: + self.container = container + self.msg = msg + + def __call__(self, v): + try: + check = v in self.container + except TypeError: + check = True + if check: + try: + raise NotInInvalid( + self.msg or f'value must not be one of {sorted(self.container)}' + ) + except TypeError: + raise NotInInvalid( + self.msg + or f'value must not be one of {sorted(self.container, key=str)}' + ) + return v + + def __repr__(self): + return 'NotIn(%s)' % (self.container,) + + +class Contains(object): + """Validate that the given schema element is in the sequence being validated. + + >>> s = Contains(1) + >>> s([3, 2, 1]) + [3, 2, 1] + >>> with raises(ContainsInvalid, 'value is not allowed'): + ... s([3, 2]) + """ + + def __init__(self, item, msg: typing.Optional[str] = None) -> None: + self.item = item + self.msg = msg + + def __call__(self, v): + try: + check = self.item not in v + except TypeError: + check = True + if check: + raise ContainsInvalid(self.msg or 'value is not allowed') + return v + + def __repr__(self): + return 'Contains(%s)' % (self.item,) + + +class ExactSequence(object): + """Matches each element in a sequence against the corresponding element in + the validators. + + :param msg: Message to deliver to user if validation fails. + :param kwargs: All other keyword arguments are passed to the sub-schema + constructors. + + >>> from voluptuous import Schema, ExactSequence + >>> validate = Schema(ExactSequence([str, int, list, list])) + >>> validate(['hourly_report', 10, [], []]) + ['hourly_report', 10, [], []] + >>> validate(('hourly_report', 10, [], [])) + ('hourly_report', 10, [], []) + """ + + def __init__( + self, + validators: typing.Iterable[Schemable], + msg: typing.Optional[str] = None, + **kwargs, + ) -> None: + self.validators = validators + self.msg = msg + self._schemas = [Schema(val, **kwargs) for val in validators] + + def __call__(self, v): + if not isinstance(v, (list, tuple)) or len(v) != len(self._schemas): + raise ExactSequenceInvalid(self.msg) + try: + v = type(v)(schema(x) for x, schema in zip(v, self._schemas)) + except Invalid as e: + raise e if self.msg is None else ExactSequenceInvalid(self.msg) + return v + + def __repr__(self): + return 'ExactSequence([%s])' % ", ".join(repr(v) for v in self.validators) + + +class Unique(object): + """Ensure an iterable does not contain duplicate items. + + Only iterables convertible to a set are supported (native types and + objects with correct __eq__). + + JSON does not support set, so they need to be presented as arrays. + Unique allows ensuring that such array does not contain dupes. + + >>> s = Schema(Unique()) + >>> s([]) + [] + >>> s([1, 2]) + [1, 2] + >>> with raises(Invalid, 'contains duplicate items: [1]'): + ... s([1, 1, 2]) + >>> with raises(Invalid, "contains duplicate items: ['one']"): + ... s(['one', 'two', 'one']) + >>> with raises(Invalid, regex="^contains unhashable elements: "): + ... s([set([1, 2]), set([3, 4])]) + >>> s('abc') + 'abc' + >>> with raises(Invalid, regex="^contains duplicate items: "): + ... s('aabbc') + """ + + def __init__(self, msg: typing.Optional[str] = None) -> None: + self.msg = msg + + def __call__(self, v): + try: + set_v = set(v) + except TypeError as e: + raise TypeInvalid(self.msg or 'contains unhashable elements: {0}'.format(e)) + if len(set_v) != len(v): + seen = set() + dupes = list(set(x for x in v if x in seen or seen.add(x))) + raise Invalid(self.msg or 'contains duplicate items: {0}'.format(dupes)) + return v + + def __repr__(self): + return 'Unique()' + + +class Equal(object): + """Ensure that value matches target. + + >>> s = Schema(Equal(1)) + >>> s(1) + 1 + >>> with raises(Invalid): + ... s(2) + + Validators are not supported, match must be exact: + + >>> s = Schema(Equal(str)) + >>> with raises(Invalid): + ... s('foo') + """ + + def __init__(self, target, msg: typing.Optional[str] = None) -> None: + self.target = target + self.msg = msg + + def __call__(self, v): + if v != self.target: + raise Invalid( + self.msg + or 'Values are not equal: value:{} != target:{}'.format(v, self.target) + ) + return v + + def __repr__(self): + return 'Equal({})'.format(self.target) + + +class Unordered(object): + """Ensures sequence contains values in unspecified order. + + >>> s = Schema(Unordered([2, 1])) + >>> s([2, 1]) + [2, 1] + >>> s([1, 2]) + [1, 2] + >>> s = Schema(Unordered([str, int])) + >>> s(['foo', 1]) + ['foo', 1] + >>> s([1, 'foo']) + [1, 'foo'] + """ + + def __init__( + self, + validators: typing.Iterable[Schemable], + msg: typing.Optional[str] = None, + **kwargs, + ) -> None: + self.validators = validators + self.msg = msg + self._schemas = [Schema(val, **kwargs) for val in validators] + + def __call__(self, v): + if not isinstance(v, (list, tuple)): + raise Invalid(self.msg or 'Value {} is not sequence!'.format(v)) + + if len(v) != len(self._schemas): + raise Invalid( + self.msg + or 'List lengths differ, value:{} != target:{}'.format( + len(v), len(self._schemas) + ) + ) + + consumed = set() + missing = [] + for index, value in enumerate(v): + found = False + for i, s in enumerate(self._schemas): + if i in consumed: + continue + try: + s(value) + except Invalid: + pass + else: + found = True + consumed.add(i) + break + if not found: + missing.append((index, value)) + + if len(missing) == 1: + el = missing[0] + raise Invalid( + self.msg + or 'Element #{} ({}) is not valid against any validator'.format( + el[0], el[1] + ) + ) + elif missing: + raise MultipleInvalid( + [ + Invalid( + self.msg + or 'Element #{} ({}) is not valid against any validator'.format( + el[0], el[1] + ) + ) + for el in missing + ] + ) + return v + + def __repr__(self): + return 'Unordered([{}])'.format(", ".join(repr(v) for v in self.validators)) + + +class Number(object): + """ + Verify the number of digits that are present in the number(Precision), + and the decimal places(Scale). + + :raises Invalid: If the value does not match the provided Precision and Scale. + + >>> schema = Schema(Number(precision=6, scale=2)) + >>> schema('1234.01') + '1234.01' + >>> schema = Schema(Number(precision=6, scale=2, yield_decimal=True)) + >>> schema('1234.01') + Decimal('1234.01') + """ + + def __init__( + self, + precision: typing.Optional[int] = None, + scale: typing.Optional[int] = None, + msg: typing.Optional[str] = None, + yield_decimal: bool = False, + ) -> None: + self.precision = precision + self.scale = scale + self.msg = msg + self.yield_decimal = yield_decimal + + def __call__(self, v): + """ + :param v: is a number enclosed with string + :return: Decimal number + """ + precision, scale, decimal_num = self._get_precision_scale(v) + + if ( + self.precision is not None + and self.scale is not None + and precision != self.precision + and scale != self.scale + ): + raise Invalid( + self.msg + or "Precision must be equal to %s, and Scale must be equal to %s" + % (self.precision, self.scale) + ) + else: + if self.precision is not None and precision != self.precision: + raise Invalid( + self.msg or "Precision must be equal to %s" % self.precision + ) + + if self.scale is not None and scale != self.scale: + raise Invalid(self.msg or "Scale must be equal to %s" % self.scale) + + if self.yield_decimal: + return decimal_num + else: + return v + + def __repr__(self): + return 'Number(precision=%s, scale=%s, msg=%s)' % ( + self.precision, + self.scale, + self.msg, + ) + + def _get_precision_scale(self, number) -> typing.Tuple[int, int, Decimal]: + """ + :param number: + :return: tuple(precision, scale, decimal_number) + """ + try: + decimal_num = Decimal(number) + except InvalidOperation: + raise Invalid(self.msg or 'Value must be a number enclosed with string') + + exp = decimal_num.as_tuple().exponent + if isinstance(exp, int): + return (len(decimal_num.as_tuple().digits), -exp, decimal_num) + else: + # TODO: handle infinity and NaN + # raise Invalid(self.msg or 'Value has no precision') + raise TypeError("infinity and NaN have no precision") + + +class SomeOf(_WithSubValidators): + """Value must pass at least some validations, determined by the given parameter. + Optionally, number of passed validations can be capped. + + The output of each validator is passed as input to the next. + + :param min_valid: Minimum number of valid schemas. + :param validators: List of schemas or validators to match input against. + :param max_valid: Maximum number of valid schemas. + :param msg: Message to deliver to user if validation fails. + :param kwargs: All other keyword arguments are passed to the sub-schema constructors. + + :raises NotEnoughValid: If the minimum number of validations isn't met. + :raises TooManyValid: If the maximum number of validations is exceeded. + + >>> validate = Schema(SomeOf(min_valid=2, validators=[Range(1, 5), Any(float, int), 6.6])) + >>> validate(6.6) + 6.6 + >>> validate(3) + 3 + >>> with raises(MultipleInvalid, 'value must be at most 5, not a valid value'): + ... validate(6.2) + """ + + def __init__( + self, + validators: typing.List[Schemable], + min_valid: typing.Optional[int] = None, + max_valid: typing.Optional[int] = None, + **kwargs, + ) -> None: + assert min_valid is not None or max_valid is not None, ( + 'when using "%s" you should specify at least one of min_valid and max_valid' + % (type(self).__name__,) + ) + self.min_valid = min_valid or 0 + self.max_valid = max_valid or len(validators) + super(SomeOf, self).__init__(*validators, **kwargs) + + def _exec(self, funcs, v, path=None): + errors = [] + funcs = list(funcs) + for func in funcs: + try: + if path is None: + v = func(v) + else: + v = func(path, v) + except Invalid as e: + errors.append(e) + + passed_count = len(funcs) - len(errors) + if self.min_valid <= passed_count <= self.max_valid: + return v + + msg = self.msg + if not msg: + msg = ', '.join(map(str, errors)) + + if passed_count > self.max_valid: + raise TooManyValid(msg) + raise NotEnoughValid(msg) + + def __repr__(self): + return 'SomeOf(min_valid=%s, validators=[%s], max_valid=%s, msg=%r)' % ( + self.min_valid, + ", ".join(repr(v) for v in self.validators), + self.max_valid, + self.msg, + ) diff --git a/server/requirements.in b/server/requirements.in index 46312f1..beeb719 100644 --- a/server/requirements.in +++ b/server/requirements.in @@ -14,4 +14,5 @@ packaging # TODO: Add your tool here ply -lark \ No newline at end of file +lark +voluptuous \ No newline at end of file diff --git a/server/requirements.txt b/server/requirements.txt index 94f34c4..c585a64 100644 --- a/server/requirements.txt +++ b/server/requirements.txt @@ -40,3 +40,7 @@ typing-extensions==4.14.1 \ --hash=sha256:38b39f4aeeab64884ce9f74c94263ef78f3c22467c8724005483154c26648d36 \ --hash=sha256:d1e1e3b58374dc93031d6eda2420a48ea44a36c2b4766a4fdeb3710755731d76 # via cattrs +voluptuous==0.15.2 \ + --hash=sha256:016348bc7788a9af9520b1764ebd4de0df41fe2138ebe9e06fa036bf86a65566 \ + --hash=sha256:6ffcab32c4d3230b4d2af3a577c87e1908a714a11f6f95570456b1849b0279aa + # via -r ./requirements.in diff --git a/server/src/tools/__init__.py b/server/src/tools/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/server/src/tools/checks.py b/server/src/tools/checks.py index 08be911..cc36ae4 100644 --- a/server/src/tools/checks.py +++ b/server/src/tools/checks.py @@ -1,240 +1,227 @@ -"""Helpers for checking command arguments.""" +import re -from collections.abc import Callable -from typing import List, Optional, Union +from src.tools.commands import get_commands +from src.tools.violations import Rule, Violation -from syntax_tree import ArgExpansion, QuotedWord, BracedWord, BareWord, Node +from src.tools.syntax_tree import ( + Visitor, + BracedExpression, + Expression, + BracedWord, + QuotedWord, + CommandSub, +) -class CommandArgError(Exception): - pass +class LineLengthChecker: + """Ensures lines aren't too long. + Reports 'line-length' violations. + """ -def arg_count(args, parser): - # TODO: graceful handling of argsub going into things with recursive parsing. - # if the argsub happens to be "concrete", we can technically do the right - # thing (although this should probably be flagged as a readability issue...) - # otherwise, we should flag that the non-concrete argsub is not okay for - # these cases. however, I think its not okay-ness doesn't need to be absolute, e.g. - # I think we could allow: - # - # catch {puts "my script"} {*}$catchopts - # + # ref: https://github.com/eslint/eslint/blob/b29a16b22f234f6134475efb6c7be5ac946556ee/lib/rules/max-len.js#L101 # noqa: E501 + # ^ ironic lint waiver... + URL_RE = re.compile(r"[^:/?#]:\/\/[^?#]") - arg_count = 0 - has_arg_expansion = False - for arg in args: - if isinstance(arg, ArgExpansion): - if arg.contents is None: - has_arg_expansion = True + def check(self, input, _, config): + violations = [] + for i, line in enumerate(input.split("\n")): + if self.URL_RE.search(line) is not None: + # ignore URLs continue - arg_count += len(parser.parse_list(arg.contents)) - else: - arg_count += 1 - return arg_count, has_arg_expansion - - -def check_count(command, min=None, max=None, args_name="args"): - def check(args, parser): - if min is None and max is None: - return None - - count, has_arg_expansion = arg_count(args, parser) - - if not has_arg_expansion and min == max and count != min: - raise CommandArgError( - f"wrong # of {args_name} for {command}: got {count}, expected {min}" - ) - - if not has_arg_expansion and min is not None and count < min: - raise CommandArgError( - f"not enough {args_name} for {command}: got {count}, expected at least" - f" {min}" - ) - - if max is not None and count > max: - raise CommandArgError( - f"too many {args_name} for {command}: got {count}, expected no more" - f" than {max}" - ) - - return None - - return check - - -def eval(args, parser, command): - if len(args) > 1 and any(isinstance(arg, (QuotedWord, BracedWord)) for arg in args): - # Slightly odd restriction, but our syntax tree doesn't have a great way - # to handle this case. We require each command argument to correspond to - # one child node, but multiple quoted or braced word arguments can be - # combined into a single subcommand when interpreted eval-style. This - # requirement exists to facilitate style checking, if we had a separate - # CST for style checks and AST for logical checks we may be able to - # handle it. - - raise CommandArgError( - f"unable to parse multiple {command} arguments when one includes a braced" - " or quoted word" - ) - - # Construct the body of the eval taking whitespace into account to ensure we get - # style checking. - - eval_script = "" - prev_arg_end_pos = None - for arg in args: - contents = arg.contents - if contents is None: - # TODO: flag sort of eval-specific violation? Common patterns will - # often trigger this, and it seems useful to be able to turn it off - raise CommandArgError( - f"{command} received an argument with a substitution, unable to parse" - " its arguments" - ) - - if prev_arg_end_pos is not None: - if prev_arg_end_pos[0] != arg.line: - # If we have multiple args on the same line, we know there must be a - # backslash newline. Add it so the parsing works. - eval_script += "\\\n" * (arg.line - prev_arg_end_pos[0]) - eval_script += " " * (arg.col - 1) - else: - eval_script += " " * (arg.col - prev_arg_end_pos[1]) - eval_script += contents - - prev_arg_end_pos = arg.end_pos - - script = parser.parse(eval_script, pos=(args[0].pos)) - script.end_pos = args[-1].end_pos - - return [script] - - -def check_command( - command: str, args: List[Node], parser, command_spec: Union[Callable, dict, None] -) -> Optional[List[Node]]: - if command_spec is None: - return None - - if isinstance(command_spec, dict): - return check_arg_spec(command, args, parser, command_spec) - - return command_spec(args, parser) - - -def check_arg_spec( - command: str, args: List[Node], parser, arg_spec: dict -) -> Optional[List[Node]]: - if "subcommands" in arg_spec: - subcommands = arg_spec["subcommands"] - try: - subcommand = args[0].contents - except IndexError: - subcommand = None - - if subcommand in subcommands: - new_args = check_command( - f"{command} {subcommand}", args[1:], parser, subcommands[subcommand] - ) - if new_args is None: - return new_args - return args[0:1] + new_args - - if "" in subcommands: - return check_command(command, args, parser, subcommands[""]) - - if subcommand is not None: - msg = f"invalid subcommand for {command}: got {subcommand}" - else: - msg = f"no subcommand provided for {command}" - - raise CommandArgError(f"{msg}, expected one of {', '.join(subcommands.keys())}") - - switches = arg_spec["switches"] - args_allowed = set(switches) - args_required = {switch for switch in switches if switches[switch]["required"]} - positional_args = [] - - args = list(args) - while len(args) > 0: - arg = args.pop(0) - - # To facilitate better error messages, we expect that switches are always - # specified as BareWords that start with "-" or ">". This lets us throw an - # error when a switch-like thing doesn't match any supported arguments, - # rather than counting it towards the positional arguments (which usually - # ends up in a vague "too many arguments" error). To make tclint interpret a - # switch-like word as a positional argument, users should wrap it in "", and - # any switches should be BareWords. - contents = arg.contents - if not (isinstance(arg, BareWord) and contents and contents[0] in {"-", ">"}): - positional_args.append(arg) - continue - - # TODO check required arguments - if contents in args_allowed: - if switches[contents]["value"]: - try: - args.pop(0) - except IndexError: - raise CommandArgError( - f"invalid arguments for {command}: expected value after" - f" {contents}" + lineno = i + 1 + if len(line) > config.style_line_length: + start = (lineno, 1) + end = (lineno, len(line) + 1) + violations.append( + Violation( + Rule.LINE_LENGTH, + f"line length is {len(line)}, maximum allowed is" + f" {config.style_line_length}", + start, + end, ) - if not switches[contents]["repeated"]: - args_allowed.remove(contents) - if contents in args_required: - args_required.remove(contents) - elif contents in arg_spec: - raise CommandArgError(f"duplicate argument for {command}: {contents}") - else: - prefix_matches = [] - for switch in switches: - if switch.startswith(contents): - prefix_matches.append(switch) - - if len(prefix_matches) == 1: - raise CommandArgError( - f"shortened argument for {command}: expand {contents} to" - f" {prefix_matches[0]}" ) - if len(prefix_matches) > 1: - raise CommandArgError( - f"ambiguous argument for {command}: {contents} could be any of" - f" {', '.join(prefix_matches)}" + return violations + + +class TrailingWhitespaceChecker: + """Ensures lines don't include trailing whitespace. + + Reports 'trailing-whitespace' violations. + """ + + def check(self, input, _, config): + violations = [] + for i, line in enumerate(input.split("\n")): + lineno = i + 1 + + WHITESPACE = (" ", "\t") + if line.endswith(WHITESPACE): + start_col = len(line.rstrip("".join(WHITESPACE))) + start = (lineno, start_col + 1) + end = (lineno, len(line) + 1) + violations.append( + Violation( + Rule.TRAILING_WHITESPACE, + "line has trailing whitespace", + start, + end, + ) ) - raise CommandArgError(f"unrecognized argument for {command}: {contents}") + return violations - if len(args_required) > 1: - raise CommandArgError( - f"missing required arguments for {command}: {', '.join(args_required)}" - ) - elif len(args_required) == 1: - raise CommandArgError( - f"missing required argument for {command}: {args_required.pop()}" + +class RedefinedBuiltinChecker(Visitor): + """Ensures names of built-in commands aren't reused by proc definitions. + + Reports 'redefined-builtin' violations. + """ + + def check(self, _, tree, config): + self._violations = [] + + plugins = [config.commands] if config.commands is not None else [] + commands = get_commands(plugins) + self._commands = commands.keys() + + tree.accept(self, recurse=True) + + return self._violations + + def visit_command(self, command): + if command.routine.contents != "proc": + return + + if len(command.args) == 0: + # This is a syntax error, but should already be caught as a command-args + # error by the parser's `proc` command handling. + return + + name = command.args[0].contents + + if name in self._commands: + self._violations.append( + Violation( + Rule.REDEFINED_BUILTIN, + f"redefinition of built-in command '{name}'", + command.pos, + command.args[1].end_pos, + ) + ) + + +class UnbracedExprChecker(Visitor): + def check(self, _, tree, __): + self._violations = [] + tree.accept(self, recurse=True) + return self._violations + + def visit_command(self, command): + if command.routine.contents != "expr": + return + + if len(command.args) == 0: + # This is a syntax error, but should already be caught as a command-args + # error by the parser's `expr` command handling. + return + + if len(command.args) == 1 and isinstance( + command.args[0], (BracedExpression, Expression) + ): + return + + # If we got here, tclint had trouble parsing the expression due to one of the + # two following cases. + + for child in command.args: + if child.contents is None: + self._violations.append( + Violation( + Rule.UNBRACED_EXPR, + "expression with substitutions should be enclosed by braces", + command.args[0].pos, + command.args[-1].end_pos, + ) + ) + return + + for child in command.args: + if isinstance(child, (BracedWord, QuotedWord)): + self._violations.append( + Violation( + Rule.UNBRACED_EXPR, + "expression containing braced or quoted words should be" + " enclosed by braces", + command.args[0].pos, + command.args[-1].end_pos, + ) + ) + return + + # If we reach here, there's probably a bug in expr parsing logic. + assert False, ( + "Children of expr node were different than expected, please file a bug" + " report" ) - min_positionals = 0 - max_positionals: Optional[int] = 0 - for positional in arg_spec["positionals"]: - if positional["value"]["type"] == "variadic": - max_positionals = None - if positional["required"]: - min_positionals += 1 - if max_positionals is not None: - max_positionals += 1 +class RedundantExprChecker(Visitor): + def check(self, _, tree, __): + self._violations = [] + tree.accept(self, recurse=True) + return self._violations - check = check_count( - command, - min=min_positionals, - max=max_positionals, - args_name="positional args", + def _check_operand(self, operand): + if not isinstance(operand, CommandSub) or len(operand.children) != 1: + return + + command = operand.children[0] + if command.routine.contents == "expr": + self._violations.append( + Violation( + Rule.REDUNDANT_EXPR, + "unnecessary command substitution within expression", + operand.pos, + operand.end_pos, + ) + ) + + def visit_braced_expression(self, expression): + if len(expression.children) == 1: + self._check_operand(expression.children[0]) + + def visit_expression(self, expression): + if len(expression.children) == 1: + self._check_operand(expression.children[0]) + + def visit_unary_op(self, expr): + self._check_operand(expr.children[1]) + + def visit_binary_op(self, expr): + self._check_operand(expr.children[0]) + self._check_operand(expr.children[2]) + + def visit_ternary_op(self, expr): + self._check_operand(expr.children[0]) + self._check_operand(expr.children[2]) + self._check_operand(expr.children[4]) + + def visit_function(self, function): + for arg in function.children[1:]: + self._check_operand(arg) + + +def get_checkers(): + checkers = ( + RedefinedBuiltinChecker(), + UnbracedExprChecker(), + RedundantExprChecker(), + LineLengthChecker(), + TrailingWhitespaceChecker(), ) - check(positional_args, None) - return None + return checkers diff --git a/server/src/tools/commands/__init__.py b/server/src/tools/commands/__init__.py index 1202114..f69a211 100644 --- a/server/src/tools/commands/__init__.py +++ b/server/src/tools/commands/__init__.py @@ -1,11 +1,11 @@ import pathlib from typing import List, Dict, Union -from tools.commands import builtin as _builtin -from tools.commands.plugins import PluginManager +from src.tools.commands import builtin as _builtin +from src.tools.commands.plugins import PluginManager # import to expose in package -from tools.commands.checks import CommandArgError +from src.tools.commands.checks import CommandArgError __all__ = ["CommandArgError", "validate_command_plugins", "get_commands"] diff --git a/server/src/tools/commands/builtin.py b/server/src/tools/commands/builtin.py new file mode 100644 index 0000000..b0aec14 --- /dev/null +++ b/server/src/tools/commands/builtin.py @@ -0,0 +1,1082 @@ +"""Parse-time handling of Tcl's builtin commands. + +Based on Tcl 8.6, https://www.tcl-lang.org/man/tcl8.6/TclCmd/contents.htm. + +Note that the following commands are not currently supported. If support for any of +these would be helpful for your use case, please file an issue. + +- Anything related to TclOO: + - https://www.tcl.tk/man/tcl/TclCmd/my.html + - https://www.tcl.tk/man/tcl/TclCmd/next.html + - https://www.tcl.tk/man/tcl/TclCmd/class.html + - https://www.tcl.tk/man/tcl/TclCmd/copy.html + - https://www.tcl.tk/man/tcl/TclCmd/define.html + - https://www.tcl.tk/man/tcl/TclCmd/object.html + - https://www.tcl.tk/man/tcl/TclCmd/self.html + +- Things that are imported via `package require` + - https://www.tcl.tk/man/tcl/TclCmd/dde.html + - https://www.tcl.tk/man/tcl/TclCmd/http.html + - https://www.tcl.tk/man/tcl/TclCmd/msgcat.html + - https://www.tcl.tk/man/tcl/TclCmd/platform.html + - https://www.tcl.tk/man/tcl/TclCmd/platform_shell.html + - https://www.tcl.tk/man/tcl/TclCmd/transchan.html + - https://www.tcl.tk/man/tcl/TclCmd/tcltest.html + +- Tcl library commands: https://www.tcl.tk/man/tcl/TclCmd/library.html + +- The "unknown" command: https://www.tcl.tk/man/tcl/TclCmd/unknown.html + +- Math ops: + - https://www.tcl.tk/man/tcl/TclCmd/mathfunc.html + - https://www.tcl.tk/man/tcl/TclCmd/mathop.html +""" + +from src.tools.commands.checks import ( + CommandArgError, + check_count, + eval, +) +from src.tools.commands.schema import commands_schema +from src.tools.syntax_tree import BareWord + + +def _check_code(arg): + """Check 'code' argument used by return and try.""" + + val = arg.contents + if val is None: + return + + try: + int(val) + except ValueError: + pass + else: + return + + if val in {"ok", "error", "return", "break", "continue"}: + return + + raise CommandArgError( + f"got {val}, expected one of ok, error, return, break, continue, or an integer" + ) + + +def _after(args, parser): + """after ms [script...]""" + # ref: https://www.tcl.tk/man/tcl/TclCmd/after.html + + script_arg = [] + if len(args) > 1: + script_arg = eval(args[1:], parser, "after") + + return args[0:1] + script_arg + + +def _after_cancel(args, parser): + """after id|(script...)""" + # ref: https://www.tcl.tk/man/tcl/TclCmd/after.html + check_count("after cancel", 1, None) + + # TODO: raise warning about not checking code + + return None + + +def _after_idle(args, parser): + """after idle [script...]""" + # ref: https://www.tcl.tk/man/tcl/TclCmd/after.html + return eval(args, parser, "after idle") + + +def _apply(args, parser): + """apply func [arg...]""" + # ref: https://www.tcl.tk/man/tcl/TclCmd/apply.html + if len(args) < 1: + raise CommandArgError( + f"not enough args to apply: got {len(args)}, expected at least 1" + ) + + func_list = parser.parse_list(args[0]) + list_len = len(func_list.children) + if list_len < 2 or list_len > 3: + raise CommandArgError( + f"Invalid first argument to apply: got list of {list_len} elements," + " expected 2 or 3" + ) + + body = parser.parse_script(func_list.children[1]) + func_list.children[1] = body + + return [func_list] + args[1:] + + +_array = { + "subcommands": { + "anymore": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + {"name": "searchId", "value": {"type": "any"}, "required": True}, + ] + }, + "donesearch": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + {"name": "searchId", "value": {"type": "any"}, "required": True}, + ] + }, + "exists": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + ] + }, + "get": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + {"name": "pattern", "value": {"type": "any"}, "required": False}, + ] + }, + "names": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + {"name": "mode", "value": {"type": "any"}, "required": False}, + {"name": "pattern", "value": {"type": "any"}, "required": False}, + ] + }, + "nextelement": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + {"name": "searchId", "value": {"type": "any"}, "required": True}, + ] + }, + "set": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + {"name": "list", "value": {"type": "any"}, "required": True}, + ] + }, + "size": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + ] + }, + "startsearch": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + ] + }, + "statistics": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + ] + }, + "unset": { + "positionals": [ + {"name": "arrayName", "value": {"type": "any"}, "required": True}, + {"name": "pattern", "value": {"type": "any"}, "required": False}, + ] + }, + }, +} + + +def _catch(args, parser): + """catch script [resultVarName] [optionsVarName]""" + if len(args) < 1: + raise CommandArgError( + f"not enough args to catch: got {len(args)}, expected at least 1" + ) + if len(args) > 3: + raise CommandArgError( + f"too many args to catch: got {len(args)}, expected no more than 3" + ) + + return [parser.parse_script(args[0])] + args[1:] + + +_chan = { + "subcommands": { + "blocked": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + ] + }, + "close": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + {"name": "direction", "value": {"type": "any"}, "required": False}, + ] + }, + "configure": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + {"name": "options", "value": {"type": "variadic"}, "required": False}, + ], + }, + "copy": { + "positionals": [ + {"name": "inputChan", "value": {"type": "any"}, "required": True}, + {"name": "outputChan", "value": {"type": "any"}, "required": True}, + {"name": "options", "value": {"type": "variadic"}, "required": False}, + ], + }, + "create": { + "positionals": [ + {"name": "mode", "value": {"type": "any"}, "required": True}, + {"name": "cmdPrefix", "value": {"type": "any"}, "required": True}, + ] + }, + "eof": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + ] + }, + "event": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + {"name": "event", "value": {"type": "any"}, "required": True}, + # TODO: parse this as script + {"name": "script", "value": {"type": "any"}, "required": False}, + ] + }, + "flush": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + ] + }, + "gets": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + {"name": "varName", "value": {"type": "any"}, "required": False}, + ] + }, + "names": { + "positionals": [ + {"name": "pattern", "value": {"type": "any"}, "required": False}, + ] + }, + "pending": { + "positionals": [ + {"name": "mode", "value": {"type": "any"}, "required": True}, + {"name": "channelId", "value": {"type": "any"}, "required": True}, + ] + }, + "pipe": {}, + "pop": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + ] + }, + "postevent": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + {"name": "eventSpec", "value": {"type": "any"}, "required": True}, + ] + }, + "push": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + {"name": "cmdPrefix", "value": {"type": "any"}, "required": True}, + ] + }, + "puts": { + "positionals": [ + {"name": "-nonewline", "value": {"type": "any"}, "required": False}, + {"name": "channelId", "value": {"type": "any"}, "required": False}, + {"name": "string", "value": {"type": "any"}, "required": True}, + ], + }, + "read": { + "positionals": [ + {"name": "-nonewline", "value": {"type": "any"}, "required": False}, + {"name": "channelId", "value": {"type": "any"}, "required": True}, + {"name": "numChars", "value": {"type": "any"}, "required": False}, + ], + }, + "seek": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + {"name": "offset", "value": {"type": "any"}, "required": True}, + {"name": "origin", "value": {"type": "any"}, "required": False}, + ] + }, + "tell": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + ] + }, + "truncate": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + {"name": "length", "value": {"type": "any"}, "required": False}, + ] + }, + }, +} + + +def _dict_filter(args, parser): + """dict filter [arg...] + dict filter key [globPattern...] + dict filter value [globPattern...] + """ + + # ref: https://www.tcl.tk/man/tcl/TclCmd/dict.html#M8 + + if len(args) < 2: + raise CommandArgError( + f"not enough args to 'dict filter': got {len(args)}, expected at least 2" + ) + + if args[1].contents not in {"key", "script", "value"}: + raise CommandArgError( + "invalid argument to 'dict filter': expected filter type to be one of key," + " script, or value" + ) + + if args[1].contents == "script": + kv_pair = parser.parse_list(args[2]) + list_len = len(kv_pair.children) + if len(kv_pair.children) != 2: + raise CommandArgError( + "invalid argument to 'dict filter': expected list of 2 elements in" + f" second-to-last argument, got {list_len}" + ) + return args[0:2] + [kv_pair, parser.parse_script(args[3])] + + return None + + +def _dict_map_for(cmd): + def check(args, parser): + if len(args) != 3: + raise CommandArgError( + f"wrong # of args to '{cmd}': got {len(args)}, expected 3" + ) + + # TODO: might be worth checking that arg[0] is a pair? + + return args[0:2] + [parser.parse_script(args[2])] + + return check + + +def _dict_update(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/dict.html#M25 + + if len(args) < 4: + raise CommandArgError( + f"not enough args to 'dict update': got {len(args)}, expected at least 4" + ) + + if len(args) % 2 != 0: + raise CommandArgError( + "invalid # of args to 'dict update': expected an even number" + ) + + return args[0:-1] + [parser.parse_script(args[-1])] + + +def _dict_with(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/dict.html#M27 + + if len(args) < 2: + raise CommandArgError( + f"not enough args to 'dict with': got {len(args)}, expected at least 2" + ) + + return args[0:-1] + [parser.parse_script(args[-1])] + + +def _eval(args, parser): + return eval(args, parser, "eval") + + +def _expr(args, parser): + if len(args) == 0: + raise CommandArgError("not enough args to 'expr': got 0, expected at least 1") + + # Handle single argument consisting of BareWord, BracedWord, or concrete QuotedWord. + if len(args) == 1 and args[0].contents is not None: + # this method will handle the `node.contents is None` case fine, but + # will throw an error. We'll instead pass thru silently, since that error + # will be caught by a separate lint check. + return [parser.parse_expression(args[0])] + + # Handle multiple BareWord arguments. Non-BareWords are hard to handle in this case, + # since we need to pop the contents out of quoted or braced words, but then we have + # no way of storing the original info about these words in the syntax tree. + contents = "" + last_pos = args[0].pos + for arg in args: + if not isinstance(arg, BareWord): + return None + + if arg.pos[0] != last_pos[0]: + contents += "\n" * (arg.pos[0] - last_pos[0]) + contents += " " * (arg.pos[1] - 1) + else: + contents += " " * (arg.pos[1] - last_pos[1]) + contents += arg.contents + last_pos = arg.end_pos + + node = BareWord(contents, pos=args[0].pos, end_pos=args[-1].end_pos) + return [parser.parse_expression(node)] + + +def _fileevent(args, parser): + # ref: https://www.tcl.tk/man/tcl8.4/TclCmd/fileevent.html + # TODO: implement + raise CommandArgError( + "argument parsing for 'fileevent' not implemented, script argument will not be" + " checked for violations" + ) + + +def _for(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/for.html + if len(args) != 4: + raise CommandArgError(f"wrong # of args to for: got {len(args)}, expected 4") + + return [ + parser.parse_script(args[0]), + parser.parse_expression(args[1]), + parser.parse_script(args[2]), + parser.parse_script(args[3]), + ] + + +def _foreach(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/foreach.html + if len(args) < 3: + raise CommandArgError( + f"insufficient args to foreach: got {len(args)}, expected at least 3" + ) + + # last argument is script body + return args[0:-1] + [parser.parse_script(args[-1])] + + +def _if(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/if.html + # TODO: make arg checking strict + + new_args = [] + + new_args.append(parser.parse_expression(args[0])) + + while len(new_args) < len(args): + arg = args[len(new_args)] + + if arg.contents == "then" or arg.contents == "else": + new_args.append(arg) + continue + if arg.contents == "elseif": + new_args.append(arg) + new_args.append(parser.parse_expression(args[len(new_args)])) + continue + + arg = parser.parse_script(arg) + new_args.append(arg) + + return new_args + + +def _interp_eval(args, parser): + if len(args) < 2: + raise CommandArgError( + f"not enough args to 'interp eval': got {len(args)}, expected at least 2" + ) + return args[0:1] + eval(args[1:], parser, "interp eval") + + +def _lmap(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/lmap.html + if len(args) < 3: + raise CommandArgError( + f"not enough args to lmap: got {len(args)}, expected at least 3" + ) + + return args[:-1] + [parser.parse_script(args[-1])] + + +def _namespace_code(args, parser): + # ref: https://www.tcl.tk/man/tcl8.4/TclCmd/namespace.html#M6 + # TODO: seems like a possible pattern is to execute things in these scripts + # with additional args provided, so command-args checks within this might + # actually be false positive. will keep as-is for now though. + return [parser.parse_script(args[0])] + + +def _namespace_eval(args, parser): + if len(args) < 2: + raise CommandArgError( + f"not enough args to 'namespace eval': got {len(args)}, expected at least 2" + ) + return args[0:1] + eval(args[1:], parser, "namespace eval") + + +def _namespace_inscope(args, parser): + # ref: https://www.tcl.tk/man/tcl8.4/TclCmd/namespace.html#M14 + raise CommandArgError( + "'namespace inscope' is not meant to be called directly, consider using" + " 'namespace code' or 'namespace eval' instead" + ) + + +def _package_ifneeded(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/package.html + + # TODO: implement + + # one issue with this one - it seems like calls to package ifneeded are + # often generated by pkg_MkIndex and these calls won't lint clean. Probably + # need a special case to ensure that these don't generate violations + + raise CommandArgError( + "argument parsing for 'package ifneeded' not implemented, any script argument" + " will not be checked for violations" + ) + + +def _proc(args, parser): + if len(args) != 3: + raise CommandArgError(f"wrong # of args to proc: got {len(args)}, expected 3") + + # Parse args as list, then iterate over each item to parse arg specifier lists and + # do some validation. We don't store non-defaulted arguments as Lists so that they + # don't get formatted inside braces. + arg_list = parser.parse_list(args[1]) + for i, arg in enumerate(arg_list.children): + if isinstance(arg, BareWord): + continue + + arg_specifier = parser.parse_list(arg) + arg_specifier_len = len(arg_specifier.children) + + if arg_specifier_len == 2: + arg_list.children[i] = arg_specifier + elif arg_specifier_len != 1: + raise CommandArgError( + f"too many fields in argument specifier: got {arg_specifier_len}," + " expected no more than 2" + ) + + return args[0:1] + [arg_list, parser.parse_script(args[2])] + + +def _return(args, parser): + args = list(args) + while len(args) > 0: + option = args.pop(0).contents + + try: + if option == "-code": + arg = args.pop(0) + try: + _check_code(arg) + except CommandArgError as e: + raise CommandArgError(f"invalid value for return -code: {e}") + elif option == "-level": + val = args.pop(0).contents + + if val is None: + continue + + try: + if int(val) >= 0: + continue + except ValueError: + pass + + raise CommandArgError( + f"invalid value for return -level: got {val}, expected a" + " non-negative integer" + ) + elif option in {"-errorcode", "-errorinfo", "-errorstack", "-options"}: + args.pop(0) + else: + break + except IndexError: + raise CommandArgError( + f"insufficient args to return: expected value after {option}" + ) + + if len(args) > 0: + raise CommandArgError( + "too many arguments to return: expected no more than 1 argument after" + " explicit options. Provide -options argument if you intend to specify" + " additional return options." + ) + + return None + + +def _switch(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/switch.html + # This one's complicated... + + # TODO: better checking of malformed switch command + + arg_contents = [arg.contents for arg in args] + arg_i = 0 + + try: + arg_i = arg_contents.index("--") + 1 + except ValueError: + while True: + contents = args[arg_i].contents + if contents in {"-exact", "-glob", "-regexp", "-nocase"}: + arg_i += 1 + elif contents in {"-matchvar", "-indexvar"}: + arg_i += 2 + else: + break + + # accounts for string to be matched + arg_i += 1 + + new_args = args[0:arg_i] + + # one argument left => form where patterns and bodies are in list + last_arg_is_list = arg_i == len(args) - 1 + + if last_arg_is_list: + pattern_and_commands_list = parser.parse_list(args[arg_i]) + new_args.append(pattern_and_commands_list) + pattern_and_commands = pattern_and_commands_list.children + else: + pattern_and_commands = args[arg_i:] + + if len(pattern_and_commands) % 2 != 0: + raise CommandArgError("Expected even number of patterns and commands") + + parsed_patterns_and_commands = [] + for i, node in enumerate(pattern_and_commands): + if i % 2 == 0: + parsed_patterns_and_commands.append(node) + else: + parsed_patterns_and_commands.append(parser.parse_script(node)) + + if last_arg_is_list: + pattern_and_commands_list.children = parsed_patterns_and_commands + else: + new_args.extend(parsed_patterns_and_commands) + + return new_args + + +def _time(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/time.html + if len(args) < 1: + raise CommandArgError( + f"not enough args to time: got {len(args)}, expected at least 1" + ) + + if len(args) > 2: + raise CommandArgError( + f"too many args to time: got {len(args)}, expected no more than 2" + ) + + if len(args) == 2: + time = args[1].contents + if time is not None: + try: + int(time) + except ValueError: + raise CommandArgError( + "invalid argument to time: expected integer for last argument" + ) + + return [parser.parse_script(args[0])] + args[1:] + + +def _timerate(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/timerate.html + # timerate doesn't seem to be implemented in tclsh 8.6 for me - why? + + args = list(args) + new_args = [] + + while True: + try: + arg = args.pop(0) + except IndexError: + raise CommandArgError("invalid arguments to timerate: expected script body") + + if arg.contents in {"-direct", "-calibrate"}: + new_args.append(arg) + elif arg.contents in {"-overhead"}: + new_args.append(arg) + try: + val = args.pop(0) + if val.contents is not None: + float(val.contents) + except (ValueError, IndexError, TypeError): + raise CommandArgError( + "invalid argument to timerate: -overhead must be followed by a" + " double" + ) + new_args.append(val) + else: + break + + new_args.append(parser.parse_script(arg)) + + if len(args) > 2: + raise CommandArgError( + "too many arguments to timerate: expected no more than 2 arguments" + " following script body" + ) + + try: + [int(arg.contents) for arg in args] + except ValueError: + raise CommandArgError( + "invalid argument to timerate: expected one or two integers following" + " script body" + ) + + return new_args + args + + +def _try(args, parser): + # ref: https://www.tcl.tk/man/tcl/TclCmd/try.html + args = list(args) + new_args = [] + + while True: + try: + arg = args.pop(0) + except IndexError: + raise CommandArgError("invalid arguments to try: missing script body") + new_args.append(parser.parse_script(arg)) + + try: + arg = args.pop(0) + except IndexError: + break + + new_args.append(arg) + + if arg.contents == "on": + try: + code = args.pop(0) + try: + _check_code(code) + except CommandArgError as e: + raise CommandArgError( + f"invalid code argument to 'on' handler in try: {e}" + ) + new_args.append(code) + new_args.append(args.pop(0)) + except IndexError: + raise CommandArgError( + "invalid arguments to try: expected 3 arguments after 'on' handler" + ) + elif arg.contents == "trap": + try: + new_args.append(args.pop(0)) + new_args.append(args.pop(0)) + except IndexError: + raise CommandArgError( + "invalid arguments to try: expected 3 arguments after 'trap'" + " handler" + ) + elif arg.contents == "finally": + continue + else: + raise CommandArgError( + "invalid handler argument to try: expected one of 'on', 'trap', or" + " 'finally'" + ) + + return new_args + + +def _while(args, parser): + if len(args) != 2: + raise CommandArgError(f"wrong # of args to while: got {len(args)}, expected 2") + + return [ + parser.parse_expression(args[0]), + parser.parse_script(args[1]), + ] + + +commands = commands_schema( + { + "after": { + "subcommands": { + "cancel": _after_cancel, + "idle": _after_idle, + "info": { + "positionals": [ + {"name": "id", "value": {"type": "any"}, "required": False} + ] + }, + "": _after, + }, + }, + "append": { + "positionals": [ + {"name": "varname", "value": {"type": "any"}, "required": True}, + {"name": "value", "value": {"type": "variadic"}, "required": False}, + ] + }, + "apply": _apply, + "array": _array, + "binary": { + "subcommands": { + "decode": check_count("binary decode", 2, None), + "encode": check_count("binary encode", 2, None), + "format": check_count("binary format", 1, None), + "scan": check_count("binary scan", 2, None), + }, + }, + "break": check_count("break", 0, 0), + "catch": _catch, + "cd": { + "positionals": [ + {"name": "dirName", "value": {"type": "any"}, "required": False} + ], + }, + "chan": _chan, + # TODO: check subcommands + "clock": check_count("clock"), + "close": { + "positionals": [ + {"name": "channelId", "value": {"type": "any"}, "required": True}, + {"name": "read|write", "value": {"type": "any"}, "required": False}, + ], + }, + "concat": { + "positionals": [ + {"name": "arg", "value": {"type": "variadic"}, "required": True}, + ] + }, + "continue": {}, + "coroutine": { + "positionals": [ + {"name": "name", "value": {"type": "any"}, "required": True}, + {"name": "command", "value": {"type": "any"}, "required": True}, + {"name": "arg", "value": {"type": "variadic"}, "required": False}, + ] + }, + "dict": { + "subcommands": { + "append": check_count("dict append", 2, None), + "create": check_count("dict create"), + "exists": check_count("dict exists", 2, None), + "filter": _dict_filter, + "for": _dict_map_for("dict for"), + "get": check_count("dict get", 1, None), + "incr": check_count("dict incr", 2, 3), + "info": check_count("dict info", 1, 1), + "keys": check_count("dict keys", 1, 2), + "lappend": check_count("dict lappend", 2, None), + "map": _dict_map_for("dict map"), + "merge": check_count("dict merge"), + "remove": check_count("dict remove", 1, None), + "replace": check_count("dict replace", 1, None), + "set": check_count("dict set", 3, None), + "size": check_count("dict size", 1, 1), + "unset": check_count("dict unset", 2, None), + "update": _dict_update, + "values": check_count("dict values", 1, 2), + "with": _dict_with, + }, + }, + "encoding": { + "subcommands": { + "convertfrom": check_count("encoding convertfrom", 1, 2), + "convertto": check_count("encoding convertto", 1, 2), + "dirs": check_count("encoding dirs", 0, 1), + "names": check_count("encoding names", 0, 0), + "system": check_count("encoding system", 0, 1), + }, + }, + "eof": check_count("eof", 1, 1), + "error": check_count("error", 1, 3), + "eval": _eval, + "exec": check_count("exec", 1, None), + "exit": check_count("exit", 0, 1), + "expr": _expr, + "fblocked": check_count("fblocked", 1, 1), + "fconfigure": check_count("fconfigure", 1, None), + "fcopy": check_count("fcopy", 2, 6), + # TODO: check subcommands + "file": check_count("file", 1, None), + "fileevent": _fileevent, + "flush": check_count("flush", 1, 1), + "for": _for, + "foreach": _foreach, + "format": check_count("format", 1, None), + "gets": check_count("gets", 1, 2), + "glob": check_count("glob"), + "global": check_count("global"), + "history": check_count("history"), + "if": _if, + "incr": check_count("incr", 1, 2), + # TODO: check subcommands + "info": check_count("info", 1, None), + # TODO: check other subcommands + "interp": { + "subcommands": { + "eval": _interp_eval, + "": check_count("interp", 1, None), + }, + }, + "join": check_count("join", 1, 2), + "lappend": check_count("lappend", 1, None), + "lassign": check_count("lassign", 1, None), + "lindex": check_count("lindex", 1, None), + "linsert": check_count("linsert", 2, None), + "list": check_count("list", 0, None), + "llength": check_count("llength", 1, 1), + "lrepeat": check_count("lrepeat", 1, None), + "lreplace": check_count("lreplace", 3, None), + "lreverse": check_count("lreverse", 1, 1), + "lset": check_count("lset", 2, None), + "lsort": check_count("lsort", 1, None), + "lmap": _lmap, + "load": check_count("load", 1, 6), + "lrange": check_count("lrange", 3, 3), + "lsearch": check_count("lsearch", 2, None), + "memory": { + "subcommands": { + "active": check_count("memory active", 1, 1), + "break_on_malloc": check_count("memory break_on_malloc", 1, 1), + "info": check_count("memory info", 0, 0), + # just on or off + "init": check_count("memory init", 1, 1), + "objs": check_count("memory objs", 1, 1), + "onexit": check_count("memory onexit", 1, 1), + "tag": check_count("memory tag", 1, 1), + # just on or off + "trace": check_count("memory trace", 1, 1), + "trace_on_at_malloc": check_count("memory trace_on_at_malloc", 1, 1), + # just on or off + "validate": check_count("memory validate", 1, 1), + }, + }, + "namespace": { + "subcommands": { + "children": check_count("namespace children", 0, 2), + "code": _namespace_code, + "current": check_count("namespace current", 0, 0), + "delete": None, + "eval": _namespace_eval, + "exists": check_count("namespace exists", 1, 1), + "export": None, + "forget": None, + "import": None, + "inscope": _namespace_inscope, + "origin": check_count("namespace origin", 1, 1), + "parent": check_count("namespace parent", 0, 1), + "qualifiers": check_count("namespace qualifiers", 1, 1), + "tail": check_count("namespace tail", 1, 1), + "which": check_count("namespace which", 1, 2), + "ensemble": { + "subcommands": { + "create": None, + "configure": check_count( + "namespace ensemble configure", 1, None + ), + "exists": check_count("namespace ensemble exists", 1, 1), + }, + }, + }, + }, + "open": check_count("open", 1, 3), + "package": { + "subcommands": { + "forget": None, + "ifneeded": _package_ifneeded, + "names": check_count("package names", 0, 0), + "present": check_count("package present", 0, None), + "provide": check_count("package provide", 1, 2), + "require": check_count("package require", 1, None), + "unknown": check_count("package unknown", 1, None), + "vcompare": check_count("package vcompare", 2, 2), + "versions": check_count("package versions", 1, 1), + "vsatisfies": check_count("package vsatisfies", 2, None), + "prefer": check_count("package prefer", 1, 1), + }, + }, + "pid": check_count("pid", 0, 1), + "pkg::create": check_count("pkg::create", 2, None), + "pkg_mkIndex": check_count("pkg_mkIndex", 1, None), + "proc": _proc, + "puts": { + "positionals": [ + {"name": "-nonewline", "value": {"type": "any"}, "required": False}, + {"name": "channelId", "value": {"type": "any"}, "required": False}, + {"name": "string", "value": {"type": "any"}, "required": True}, + ], + }, + "pwd": check_count("pwd", 0, 0), + "read": check_count("read", 1, 2), + "regexp": check_count("regexp", 2, None), + "regsub": check_count("regsub", 3, None), + "rename": check_count("rename", 2, 2), + "return": _return, + # TODO: check subcommands + "safe": check_count("safe", 1, None), + "scan": check_count("scan", 2, None), + "seek": check_count("seek", 2, 3), + "set": check_count("set", 1, 2), + "socket": check_count("socket", 2, None), + "source": check_count("source", 1, 3), + "split": check_count("split", 1, 2), + # TODO: check subcommands + "string": check_count("string", 2, None), + "subst": check_count("subst", 1, 4), + "switch": _switch, + "tailcall": check_count("tailcall", 1, None), + "tcl::prefix": { + "subcommands": { + "all": check_count("tcl::prefix all", 2, 2), + "longest": check_count("tcl::prefix longest", 2, 2), + "match": check_count("tcl::prefix match", 2, None), + }, + }, + "tell": check_count("tell", 1, 1), + "throw": check_count("throw", 2, 2), + "time": _time, + "timerate": _timerate, + "tcl::tm::path": { + "subcommands": { + "add": check_count("tcl::tm::path add"), + "remove": check_count("tcl::tm::path remove"), + "list": check_count("tcl::tm::path list", 0, 0), + }, + }, + "tcl::tm::roots": check_count("tcl::tm::roots"), + # TODO: check subcommands + "trace": check_count("trace", 2, None), + "try": _try, + "unload": check_count("unload", 1, 6), + "unset": check_count("unset"), + "update": check_count("update", 0, 1), + "uplevel": check_count("uplevel", 1, None), + "upvar": check_count("upvar", 2, None), + "variable": check_count("variable", 1, None), + "vwait": check_count("vwait", 1, 1), + "while": _while, + "yield": { + "positionals": [ + {"name": "value", "value": {"type": "any"}, "required": False}, + ] + }, + "yieldto": { + "positionals": [ + {"name": "command", "value": {"type": "any"}, "required": True}, + {"name": "arg", "value": {"type": "variadic"}, "required": False}, + ] + }, + # TODO: check subcommands + "zlib": check_count("zlib", 3, None), + } +) diff --git a/server/src/tools/commands/checks.py b/server/src/tools/commands/checks.py new file mode 100644 index 0000000..a325f59 --- /dev/null +++ b/server/src/tools/commands/checks.py @@ -0,0 +1,240 @@ +"""Helpers for checking command arguments.""" + +from collections.abc import Callable +from typing import List, Optional, Union + +from src.tools.syntax_tree import ArgExpansion, QuotedWord, BracedWord, BareWord, Node + + +class CommandArgError(Exception): + pass + + +def arg_count(args, parser): + # TODO: graceful handling of argsub going into things with recursive parsing. + # if the argsub happens to be "concrete", we can technically do the right + # thing (although this should probably be flagged as a readability issue...) + # otherwise, we should flag that the non-concrete argsub is not okay for + # these cases. however, I think its not okay-ness doesn't need to be absolute, e.g. + # I think we could allow: + # + # catch {puts "my script"} {*}$catchopts + # + + arg_count = 0 + has_arg_expansion = False + for arg in args: + if isinstance(arg, ArgExpansion): + if arg.contents is None: + has_arg_expansion = True + continue + arg_count += len(parser.parse_list(arg.contents)) + else: + arg_count += 1 + + return arg_count, has_arg_expansion + + +def check_count(command, min=None, max=None, args_name="args"): + def check(args, parser): + if min is None and max is None: + return None + + count, has_arg_expansion = arg_count(args, parser) + + if not has_arg_expansion and min == max and count != min: + raise CommandArgError( + f"wrong # of {args_name} for {command}: got {count}, expected {min}" + ) + + if not has_arg_expansion and min is not None and count < min: + raise CommandArgError( + f"not enough {args_name} for {command}: got {count}, expected at least" + f" {min}" + ) + + if max is not None and count > max: + raise CommandArgError( + f"too many {args_name} for {command}: got {count}, expected no more" + f" than {max}" + ) + + return None + + return check + + +def eval(args, parser, command): + if len(args) > 1 and any(isinstance(arg, (QuotedWord, BracedWord)) for arg in args): + # Slightly odd restriction, but our syntax tree doesn't have a great way + # to handle this case. We require each command argument to correspond to + # one child node, but multiple quoted or braced word arguments can be + # combined into a single subcommand when interpreted eval-style. This + # requirement exists to facilitate style checking, if we had a separate + # CST for style checks and AST for logical checks we may be able to + # handle it. + + raise CommandArgError( + f"unable to parse multiple {command} arguments when one includes a braced" + " or quoted word" + ) + + # Construct the body of the eval taking whitespace into account to ensure we get + # style checking. + + eval_script = "" + prev_arg_end_pos = None + for arg in args: + contents = arg.contents + if contents is None: + # TODO: flag sort of eval-specific violation? Common patterns will + # often trigger this, and it seems useful to be able to turn it off + raise CommandArgError( + f"{command} received an argument with a substitution, unable to parse" + " its arguments" + ) + + if prev_arg_end_pos is not None: + if prev_arg_end_pos[0] != arg.line: + # If we have multiple args on the same line, we know there must be a + # backslash newline. Add it so the parsing works. + eval_script += "\\\n" * (arg.line - prev_arg_end_pos[0]) + eval_script += " " * (arg.col - 1) + else: + eval_script += " " * (arg.col - prev_arg_end_pos[1]) + eval_script += contents + + prev_arg_end_pos = arg.end_pos + + script = parser.parse(eval_script, pos=(args[0].pos)) + script.end_pos = args[-1].end_pos + + return [script] + + +def check_command( + command: str, args: List[Node], parser, command_spec: Union[Callable, dict, None] +) -> Optional[List[Node]]: + if command_spec is None: + return None + + if isinstance(command_spec, dict): + return check_arg_spec(command, args, parser, command_spec) + + return command_spec(args, parser) + + +def check_arg_spec( + command: str, args: List[Node], parser, arg_spec: dict +) -> Optional[List[Node]]: + if "subcommands" in arg_spec: + subcommands = arg_spec["subcommands"] + try: + subcommand = args[0].contents + except IndexError: + subcommand = None + + if subcommand in subcommands: + new_args = check_command( + f"{command} {subcommand}", args[1:], parser, subcommands[subcommand] + ) + if new_args is None: + return new_args + return args[0:1] + new_args + + if "" in subcommands: + return check_command(command, args, parser, subcommands[""]) + + if subcommand is not None: + msg = f"invalid subcommand for {command}: got {subcommand}" + else: + msg = f"no subcommand provided for {command}" + + raise CommandArgError(f"{msg}, expected one of {', '.join(subcommands.keys())}") + + switches = arg_spec["switches"] + args_allowed = set(switches) + args_required = {switch for switch in switches if switches[switch]["required"]} + positional_args = [] + + args = list(args) + while len(args) > 0: + arg = args.pop(0) + + # To facilitate better error messages, we expect that switches are always + # specified as BareWords that start with "-" or ">". This lets us throw an + # error when a switch-like thing doesn't match any supported arguments, + # rather than counting it towards the positional arguments (which usually + # ends up in a vague "too many arguments" error). To make tclint interpret a + # switch-like word as a positional argument, users should wrap it in "", and + # any switches should be BareWords. + contents = arg.contents + if not (isinstance(arg, BareWord) and contents and contents[0] in {"-", ">"}): + positional_args.append(arg) + continue + + # TODO check required arguments + if contents in args_allowed: + if switches[contents]["value"]: + try: + args.pop(0) + except IndexError: + raise CommandArgError( + f"invalid arguments for {command}: expected value after" + f" {contents}" + ) + if not switches[contents]["repeated"]: + args_allowed.remove(contents) + if contents in args_required: + args_required.remove(contents) + elif contents in arg_spec: + raise CommandArgError(f"duplicate argument for {command}: {contents}") + else: + prefix_matches = [] + for switch in switches: + if switch.startswith(contents): + prefix_matches.append(switch) + + if len(prefix_matches) == 1: + raise CommandArgError( + f"shortened argument for {command}: expand {contents} to" + f" {prefix_matches[0]}" + ) + + if len(prefix_matches) > 1: + raise CommandArgError( + f"ambiguous argument for {command}: {contents} could be any of" + f" {', '.join(prefix_matches)}" + ) + + raise CommandArgError(f"unrecognized argument for {command}: {contents}") + + if len(args_required) > 1: + raise CommandArgError( + f"missing required arguments for {command}: {', '.join(args_required)}" + ) + elif len(args_required) == 1: + raise CommandArgError( + f"missing required argument for {command}: {args_required.pop()}" + ) + + min_positionals = 0 + max_positionals: Optional[int] = 0 + for positional in arg_spec["positionals"]: + if positional["value"]["type"] == "variadic": + max_positionals = None + + if positional["required"]: + min_positionals += 1 + if max_positionals is not None: + max_positionals += 1 + + check = check_count( + command, + min=min_positionals, + max=max_positionals, + args_name="positional args", + ) + check(positional_args, None) + + return None diff --git a/server/src/tools/commands/plugins.py b/server/src/tools/commands/plugins.py new file mode 100644 index 0000000..9c41c5b --- /dev/null +++ b/server/src/tools/commands/plugins.py @@ -0,0 +1,86 @@ +from importlib.metadata import entry_points +import json +import pathlib +from typing import Dict, Optional +from types import ModuleType + +import voluptuous + +from src.tools.commands.schema import schema as command_schema + + +class _PluginManager: + def __init__(self): + self._loaded = {} + self._installed = {} + self._loaded_specs = {} + for plugin in entry_points(group="tclint.plugins"): + if plugin.name in self._installed: + print(f"Warning: found duplicate definitions for plugin {plugin.name}") + self._installed[plugin.name] = plugin + + def load(self, name: str) -> Optional[Dict]: + if name in self._loaded: + return self._loaded[name] + + mod = self._load(name) + self._loaded[name] = mod + return mod + + def load_from_spec(self, path: pathlib.Path) -> Optional[Dict]: + if path in self._loaded_specs: + return self._loaded_specs[path] + + spec = self._load_from_spec(path) + self._loaded_specs[path] = spec + return spec + + def _load_from_spec(self, path: pathlib.Path) -> Optional[Dict]: + try: + with open(path.expanduser(), "r") as f: + spec = json.load(f) + except (FileNotFoundError, RuntimeError): + print(f"Warning: command spec {path} not found, skipping...") + return None + + try: + # Apply defaults and validate the spec. + spec = command_schema(spec) + except voluptuous.Invalid as e: + print(f"Warning: invalid command spec {path}: {e}") + return None + + return spec["commands"] + + def get_mod(self, name: str) -> Optional[ModuleType]: + if name not in self._installed: + print(f"Warning: plugin {name} is not installed") + return None + + plugin = self._installed[name] + + try: + module = plugin.load() + except Exception as e: + print(f"Warning: error loading plugin {name}: {e}") + return None + + return module + + def _load(self, name: str): + module = self.get_mod(name) + if module is None: + print(f"Skipping requested plugin {name}") + return None + + if not hasattr(module, "commands"): + print(f"Warning: skipping plugin {name} since it does not define commands") + return None + + return getattr(module, "commands") + + +# TODO: we'll probably want to construct this in the tclint entry point and pass +# it around rather than using a singleton instance, but this made for an easier +# refactor. +PluginManager = _PluginManager() diff --git a/server/src/tools/commands/schema.py b/server/src/tools/commands/schema.py new file mode 100644 index 0000000..7662a10 --- /dev/null +++ b/server/src/tools/commands/schema.py @@ -0,0 +1,35 @@ +from collections.abc import Callable +from voluptuous import Schema, Optional, Or, Self + +# Need to define this as a Schema with required=True to ensure that this requirement +# persists through the Or in the main schema definition. +_command_args = Schema( + { + Optional("positionals", default=[]): [ + { + "name": str, + "required": bool, + "value": Or({"type": "any"}, {"type": "variadic"}), + } + ], + Optional("switches", default={}): { + Optional(str): { + "required": bool, + "repeated": bool, + "value": Or({"type": "any"}, None), + Optional("metavar"): str, + } + }, + }, + required=True, +) + +commands_schema = Schema( + {Optional(str): Or(_command_args, None, {"subcommands": Self}, Callable)}, + required=True, +) + +schema = Schema( + {"name": str, "commands": commands_schema}, + required=True, +) diff --git a/server/src/tools/comments.py b/server/src/tools/comments.py new file mode 100644 index 0000000..293a997 --- /dev/null +++ b/server/src/tools/comments.py @@ -0,0 +1,91 @@ +from collections import defaultdict + +from src.tools.syntax_tree import Visitor +from src.tools.violations import ALL_RULES, Rule + + +class CommentVisitor(Visitor): + """Scans the tree for lint waiver comments.""" + + def __init__(self): + # line -> [rule] + self.ignore_lines = defaultdict(set) + + self._disable_regions = { + # rule -> line + } + + def run(self, tree, path): + self._path = path + tree.accept(self, recurse=True) + + # resolve remaining disabled regions + last_line = tree.end_pos[0] + for rule, start_line in self._disable_regions.items(): + for line in range(start_line, last_line + 1): + self.ignore_lines[line].add(rule) + + return self.ignore_lines + + def visit_comment(self, comment): + contents = comment.value.strip() + + if not contents.startswith("tclint-"): + return + + split = contents.split(" ", 1) + + command = split[0] + + rule_strs = [] + if len(split) > 1: + rest = split[-1] + rule_strs = rest.split("--", 1)[0] + rule_strs = rule_strs.replace(" ", "") + rule_strs = rule_strs.split(",") + + rules = [] + if not rule_strs: + # default if no rules specified is all violation types + rules = ALL_RULES + else: + for rule in rule_strs: + try: + rules.append(Rule(rule)) + except ValueError: + self._warning( + f"unknown rule '{rule}' provided to '{command}'", comment.pos + ) + + if command == "tclint-disable": + for rule in rules: + # if in dictionary, already disabled - this has no effect + if rule not in self._disable_regions: + self._disable_regions[rule] = comment.line + elif command == "tclint-disable-line": + line = comment.line + self.ignore_lines[line].update(rules) + elif command == "tclint-disable-next-line": + line = comment.line + 1 + self.ignore_lines[line].update(rules) + elif command == "tclint-enable": + for rule in rules: + if rule in self._disable_regions: + disable_start_line = self._disable_regions[rule] + disable_end_line = comment.line + + for line in range(disable_start_line, disable_end_line + 1): + self.ignore_lines[line].add(rule) + + del self._disable_regions[rule] + else: + self._warning( + f"comment starts with '{command}', which looks like a tclint keyword." + " Is this a typo?", + comment.pos, + ) + + def _warning(self, message, pos): + # TODO: formal warning mechanism + prefix = self._path if self._path is not None else "(stdin)" + print(f"Warning: {prefix}:{pos[0]}:{pos[1]}: {message}") diff --git a/server/src/tools/config.py b/server/src/tools/config.py new file mode 100644 index 0000000..87cf234 --- /dev/null +++ b/server/src/tools/config.py @@ -0,0 +1,429 @@ +import argparse +import pathlib +from typing import Union, List +from typing import Optional as OptionalType +import dataclasses +import sys + +if sys.version_info >= (3, 11): + import tomllib +else: + import tomli as tomllib + +from voluptuous import Schema, Optional, And, Coerce, Invalid, Range + +from src.tools.violations import Rule + + +@dataclasses.dataclass +class Config: + """This dataclass defines the supported Config fields and their default + values. It provides an external interface for accessing config values. + + The type annotations defined here are fairly loose - more specific type + validation (and normalization) is defined by `validators` below. + """ + + exclude: List[str] = dataclasses.field(default_factory=list) + ignore: List[Rule] = dataclasses.field(default_factory=list) + commands: OptionalType[pathlib.Path] = dataclasses.field(default=None) + extensions: List[str] = dataclasses.field( + default_factory=lambda: ["tcl", "sdc", "xdc", "upf"] + ) + style_indent: OptionalType[Union[str, int]] = dataclasses.field(default=None) + style_line_length: int = dataclasses.field(default=100) + style_max_blank_lines: int = dataclasses.field(default=2) + style_indent_namespace_eval: bool = dataclasses.field(default=True) + style_spaces_in_braces: bool = dataclasses.field(default=False) + + def apply_cli_args(self, args): + args_dict = vars(args) + for field in dataclasses.fields(self): + if field.name in args_dict and args_dict[field.name] is not None: + setattr(self, field.name, args_dict[field.name]) + + # Special arguments that aren't handled automatically + if "extend_exclude" in args_dict and args_dict["extend_exclude"] is not None: + self.exclude.extend(args_dict["extend_exclude"]) + + if "extend_ignore" in args_dict and args_dict["extend_ignore"] is not None: + self.ignore.extend(args_dict["extend_ignore"]) + + def get_indent(self) -> str: + """Get indent setting as string. + + This helper does two things. One, it's a helpful utility to factor out the logic + required for calculating the indent. Two, it lets us ergonomically store if the + indentation is not set in style_indent, which the LSP relies on. + """ + if self.style_indent is None: + # Default indent + return " " * 4 + elif self.style_indent == "tab": + return "\t" + elif isinstance(self.style_indent, int): + return " " * self.style_indent + + # Should be unreachable, validated on ingestion of config + raise ValueError( + f"unexpected value for config.style_indent: {self.style_indent}" + ) + + +# Validators using `voluptuous` library that check and normalize config inputs. +# Used for checking both config files as well as config-related CLI args. + +# Using these for CLI args adds a constraint that all non-boolean validators +# need to be able to normalize a value from a string. This means one could put +# e.g. a string representation of a list into a .toml config file, but we shouldn't +# document this, since it won't be considered stable behavior. + + +def _str2list(s): + """Handles string-to-list normalization.""" + if isinstance(s, str): + if s == "": + return [] + return [v.strip() for v in s.split(",")] + return s + + +_VALIDATORS = { + # note: it's ok if paths don't exist - allows for generic + # configurations with directories like .git/ excluded + "exclude": _str2list, + "ignore": And( + _str2list, + [ + Coerce(Rule, msg="invalid rule ID"), + ], + ), + "commands": Coerce(pathlib.Path), + "extensions": _str2list, + "style_indent": Coerce( + lambda v: v if v == "tab" else int(v), msg="expected integer or 'tab'" + ), + "style_line_length": Coerce(int), + "style_max_blank_lines": And( + Coerce(int), + # we could technically support i >= 0, but I think 0 would be a weird + # setting and this lets us ignore pluralizing the violation message :) + Range(min=1), + ), + "style_indent_namespace_eval": bool, + "style_spaces_in_braces": bool, +} + + +def _validate_config(config): + """Validates dictionary read from TOML config file. Individual value validators + are implemented in the global dict, this defines the actual structure of the + schema.""" + + base_config = { + Optional("ignore"): _VALIDATORS["ignore"], + Optional("commands"): _VALIDATORS["commands"], + Optional("style"): { + Optional("indent"): _VALIDATORS["style_indent"], + Optional("line-length"): _VALIDATORS["style_line_length"], + Optional("max-blank-lines"): _VALIDATORS["style_max_blank_lines"], + Optional("indent-namespace-eval"): _VALIDATORS[ + "style_indent_namespace_eval" + ], + Optional("spaces-in-braces"): _VALIDATORS["style_spaces_in_braces"], + }, + } + + schema = Schema( + { + # exclude and extensions can only be used in global context + Optional("exclude"): _VALIDATORS["exclude"], + Optional("extensions"): _VALIDATORS["extensions"], + **base_config, + Optional("fileset"): Schema( + [{"paths": [Coerce(pathlib.Path)], **base_config}], required=True + ), + } + ) + + try: + return schema(config) + except Invalid as e: + if not e.path: + raise ConfigError(e.error_message) + + # Stringify error path to my own taste. + path = [] + for item in e.path: + if isinstance(item, int): + # Brackets around indices + if len(path) > 0: + path[-1] += f"[{item}]" + else: + path.append(f"[{item}]") + else: + path.append(str(item)) + + raise ConfigError(f"{e.error_message} ({'.'.join(path)})") + + +def _validator(key): + def func(s): + try: + return Schema(_VALIDATORS[key])(s) + except Invalid as e: + raise argparse.ArgumentTypeError(str(e)) + + return func + + +def _add_bool(group, parser, dest, yes_flag, no_flag): + mutex_group = group.add_mutually_exclusive_group(required=False) + mutex_group.add_argument(yes_flag, dest=dest, action="store_true") + mutex_group.add_argument(no_flag, dest=dest, action="store_false") + parser.set_defaults(**{dest: None}) + + +def setup_common_config_cli_args(config_group): + config_group.add_argument( + "--exclude", type=_validator("exclude"), metavar='"path1, path2, ..."' + ) + config_group.add_argument( + "--extend-exclude", type=_validator("exclude"), metavar='"path1, path2, ..."' + ) + config_group.add_argument( + "--extensions", type=_validator("extensions"), metavar='"tcl, xdc, ..."' + ) + config_group.add_argument( + "--commands", type=_validator("commands"), metavar="" + ) + + +def setup_config_cli_args(parser): + """This method defines config-related CLI arguments. + + The destvars of these switches should match the fields of Config. + """ + config_group = parser.add_argument_group("configuration arguments") + + config_group.add_argument( + "--ignore", type=_validator("ignore"), metavar='"rule1, rule2, ..."' + ) + config_group.add_argument( + "--extend-ignore", type=_validator("ignore"), metavar='"rule1, rule2, ..."' + ) + setup_common_config_cli_args(config_group) + config_group.add_argument( + "--style-line-length", + type=_validator("style_line_length"), + metavar="", + ) + + +def setup_tclfmt_config_cli_args(parser): + """This method defines the subset of config-related CLI arguments used by tclfmt. + + The destvars of these switches should match the fields of Config. + """ + config_group = parser.add_argument_group("configuration arguments") + + setup_common_config_cli_args(config_group) + + config_group.add_argument( + "--indent", + type=_validator("style_indent"), + metavar="", + dest="style_indent", + ) + config_group.add_argument( + "--max-blank-lines", + type=_validator("style_max_blank_lines"), + metavar="", + dest="style_max_blank_lines", + ) + _add_bool( + config_group, + parser, + "style_indent_namespace_eval", + "--indent-namespace-eval", + "--no-indent-namespace-eval", + ) + _add_bool( + config_group, + parser, + "style_spaces_in_braces", + "--spaces-in-braces", + "--no-spaces-in-braces", + ) + + +def _flatten(d, prefix=None): + """Flattens TOML config dictionary structure to match the flat set of fields + expected by Config dataclass.""" + if prefix is None: + prefix = [] + + flat = {} + for k, v in d.items(): + if isinstance(v, dict): + flat.update(_flatten(v, prefix=prefix + [k])) + else: + flat["_".join(prefix + [k]).replace("-", "_")] = v + + return flat + + +class RunConfig: + """Class that holds information about both global and fileset configs. User + code can get a Config object that applies to a particular file by calling + get_from_path() and supplying that file's path.""" + + def __init__(self, global_config=None, fileset_configs=None): + if global_config is not None: + self._global_config = global_config + else: + self._global_config = Config() + + self._fileset_configs = [ + # ([pathlib.Path...], Config]) + ] + if fileset_configs is not None: + self._fileset_configs = fileset_configs + + @property + def exclude(self): + return self._global_config.exclude + + @property + def extensions(self): + return self._global_config.extensions + + @classmethod + def from_dict(cls, config_dict: dict, root: pathlib.Path): + config_dict = _validate_config(config_dict) + try: + fileset_config_dicts = config_dict.pop("fileset") + except KeyError: + fileset_config_dicts = [] + + config_dict = _flatten(config_dict) + global_config = Config(**config_dict) + + fileset_configs = [] + for fileset_config in fileset_config_dicts: + paths = [] + for path in fileset_config.pop("paths"): + if not path.is_absolute(): + path = root / path + paths.append(path.resolve()) + + fileset_config = _flatten(fileset_config) + + # pull in default values from global config + full_fileset_config = config_dict.copy() + full_fileset_config.update(fileset_config) + + fileset_configs.append((paths, Config(**full_fileset_config))) + + return cls(global_config, fileset_configs) + + @classmethod + def from_path(cls, path: Union[str, pathlib.Path], root: pathlib.Path): + path = pathlib.Path(path) + + if not path.exists(): + raise FileNotFoundError + + with open(path, "rb") as f: + try: + data = tomllib.load(f) + except tomllib.TOMLDecodeError as e: + raise ConfigError(f"{path}: {e}") + + try: + return cls.from_dict(data, root) + except ConfigError as e: + raise ConfigError(f"{path}: {e}") + + @classmethod + def from_pyproject(cls, directory=None): + if directory is None: + directory = pathlib.Path(".") + else: + directory = pathlib.Path(directory) + + path = directory / "pyproject.toml" + + if not path.exists(): + raise FileNotFoundError + + with open(path, "rb") as f: + data = tomllib.load(f) + + tclint_config = data.get("tool", {})["tclint"] + + try: + return cls.from_dict(tclint_config, directory) + except ConfigError as e: + raise ConfigError(f"pyproject.toml: {e}") + + def get_for_path(self, path) -> Config: + if path is None: + return self._global_config + + path = path.resolve() + for fileset_paths, config in self._fileset_configs: + for fileset_path in fileset_paths: + if path.is_relative_to(fileset_path): + return config + + return self._global_config + + def apply_cli_args(self, args): + self._global_config.apply_cli_args(args) + for _, fileset_config in self._fileset_configs: + fileset_config.apply_cli_args(args) + + +class ConfigError(Exception): + pass + + +DEFAULT_CONFIGS = ("tclint.toml", ".tclint") + + +def get_config( + config_path: OptionalType[pathlib.Path], root: pathlib.Path +) -> OptionalType[RunConfig]: + """Loads a config file. + + If `config_path` is supplied, attempts to read config file from this path. If the + path can't be found, raises a ConfigError. + + Otherwise, attempts to read config from `root`/{tclint.toml, .tclint, + pyproject.toml} (in that order). If none of these files can be found, returns None. + + `root` is also used to resolve some relative paths in the config file. + """ + # user-supplied + if config_path is not None: + try: + return RunConfig.from_path(config_path, root) + except FileNotFoundError: + raise ConfigError(f"path {config_path} doesn't exist") + + for path in DEFAULT_CONFIGS: + try: + return RunConfig.from_path(root / path, root) + except FileNotFoundError: + pass + + try: + return RunConfig.from_pyproject(directory=root) + except ConfigError as e: + raise e + except (FileNotFoundError, tomllib.TOMLDecodeError, KeyError): + # just skip if file doesn't exist, contains TOML errors, or tclint key not found + pass + + return None diff --git a/server/src/tools/format.py b/server/src/tools/format.py new file mode 100644 index 0000000..66441e5 --- /dev/null +++ b/server/src/tools/format.py @@ -0,0 +1,480 @@ +import dataclasses +import itertools +import textwrap +from typing import List, Tuple, Union +import sys + +from src.tools.syntax_tree import ( + Node, + Script, + Command, + Comment, + CommandSub, + BareWord, + QuotedWord, + BracedWord, + CompoundBareWord, + VarSub, + ArgExpansion, + Expression, + BracedExpression, + ParenExpression, + UnaryOp, + BinaryOp, + TernaryOp, + Function, +) +from src.tools.parser import Parser +from src.tools.syntax_tree import List as ListNode + + +@dataclasses.dataclass +class LiteralBlock: + block: List[str] + pos: Tuple[int, int] + end_pos: Tuple[int, int] + + +@dataclasses.dataclass +class FormatterOpts: + indent: str + spaces_in_braces: bool + max_blank_lines: int + indent_namespace_eval: bool + + +class Formatter: + def __init__(self, opts: FormatterOpts): + self.opts = opts + + def _indent(self, lines: List[str], indent: str) -> List[str]: + indented = [] + for line in lines: + if line == "": + indented.append("") + else: + indented.append(indent + line) + + return indented + + def _brace(self, lines: List[str]) -> List[str]: + spaces_in_braces = " " if self.opts.spaces_in_braces else "" + if lines == [""]: + return ["{" + spaces_in_braces + "}"] + + braced_lines = lines[:] + braced_lines[0] = "{" + spaces_in_braces + lines[0] + braced_lines[-1] += spaces_in_braces + "}" + return braced_lines + + def format(self, *nodes: Union[Node, LiteralBlock]) -> List[str]: + formatted = [] + for node in nodes: + if isinstance(node, Script): + formatted += self.format_script(node) + elif isinstance(node, Command): + formatted += self.format_command(node) + elif isinstance(node, Comment): + formatted += self.format_comment(node) + elif isinstance(node, CommandSub): + formatted += self.format_command_sub(node) + elif isinstance(node, BareWord): + formatted += self.format_bare_word(node) + elif isinstance(node, QuotedWord): + formatted += self.format_quoted_word(node) + elif isinstance(node, BracedWord): + formatted += self.format_braced_word(node) + elif isinstance(node, CompoundBareWord): + formatted += self.format_compound_bare_word(node) + elif isinstance(node, VarSub): + formatted += self.format_var_sub(node) + elif isinstance(node, ArgExpansion): + formatted += self.format_arg_expansion(node) + elif isinstance(node, ListNode): + formatted += self.format_list(node) + elif isinstance(node, Expression): + formatted += self.format_expression(node) + elif isinstance(node, BracedExpression): + formatted += self.format_braced_expression(node) + elif isinstance(node, ParenExpression): + formatted += self.format_paren_expression(node) + elif isinstance(node, UnaryOp): + formatted += self.format_unary_op(node) + elif isinstance(node, BinaryOp): + formatted += self.format_binary_op(node) + elif isinstance(node, TernaryOp): + formatted += self.format_ternary_op(node) + elif isinstance(node, Function): + formatted += self.format_function(node) + elif isinstance(node, LiteralBlock): + formatted += node.block + else: + assert False, f"unrecognized node: {type(node)}" + + return formatted + + def format_top(self, script: str, parser: Parser) -> str: + tree = parser.parse(script) + self.script = script.split("\n") + return "\n".join(self.format_script_contents(tree)) + "\n" + + def format_partial(self, script: str, parser: Parser) -> str: + """Formats a partial Tcl script. + + This function formats a partial script according to the gofmt partial formatting + rules, "[preserving] leading indentation as well as leading and trailing spaces" + (ref: https://pkg.go.dev/cmd/gofmt#pkg-overview). Unlike Go, we have no way of + detecting if a given script is a program fragment, hence the distinct method + from `format_top` . + """ + leading = "".join(itertools.takewhile(str.isspace, script)) + try: + leading, indent = leading.rsplit("\n", 1) + leading += "\n" + except ValueError: + leading, indent = "", leading + trailing = "".join(itertools.takewhile(str.isspace, reversed(script)))[::-1] + + script = script.strip() + tree = parser.parse(script) + self.script = script.split("\n") + + formatted = "\n".join(self.format_script_contents(tree)) + + return leading + textwrap.indent(formatted, indent) + trailing + + def format_script_contents(self, script: Union[Script, CommandSub]) -> List[str]: + to_format = [] + skip_formatting_start = None + for child in script.children: + if skip_formatting_start is None: + to_format.append(child) + + if isinstance(child, Comment): + if child.value.strip() == "tclfmt-disable": + if skip_formatting_start is not None: + print( + "Warning: encountered 'tclint-disable' while formatting is" + " already disabled, ignoring...", + file=sys.stderr, + ) + else: + skip_formatting_start = child.pos[0] + elif child.value.strip() == "tclfmt-enable": + if skip_formatting_start is None: + print( + "Warning: encountered 'tclint-enable' while formatting is" + " already disabled, ignoring...", + file=sys.stderr, + ) + else: + skip_formatting_end = child.pos[0] + block = self.script[skip_formatting_start:skip_formatting_end] + to_format.append( + LiteralBlock( + block, + pos=(skip_formatting_start + 1, 1), + end_pos=(skip_formatting_end, 1), + ) + ) + skip_formatting_start = None + + if skip_formatting_start is not None: + print("Warning: missing 'tclint-enable'", file=sys.stderr) + to_format.append( + LiteralBlock( + self.script[skip_formatting_start:], + pos=(skip_formatting_start + 1, 1), + end_pos=script.end_pos, + ) + ) + + formatted = [""] + last_line = None + for child in to_format: + if last_line is not None: + if last_line == child.pos[0]: + if isinstance(child, Comment): + formatted[-1] += " ;" + else: + formatted[-1] += "; " + else: + newlines = child.pos[0] - last_line + newlines = min(newlines, self.opts.max_blank_lines + 1) + formatted.extend([""] * newlines) + last_line = child.end_pos[0] + + lines = self.format(child) + formatted[-1] += lines[0] + formatted.extend(lines[1:]) + + return formatted + + def format_script(self, script: Script, should_indent=True) -> List[str]: + lines = self.format_script_contents(script) + if script.pos[0] == script.end_pos[0]: + return self._brace(lines) + + # Usually, we enforce that multi-line scripts start on a new line after the open + # brace. However, if a comment was originally on the same line as the open brace + # we preserve it, since it's probably meant to be associated with this line + # (e.g. a tclint-disable-line). + open_brace = "{" + if ( + len(script.children) > 0 + and isinstance(script.children[0], Comment) + and script.pos[0] == script.children[0].pos[0] + ): + open_brace += " " + lines[0] + lines = lines[1:] + + if should_indent: + return [open_brace] + self._indent(lines, self.opts.indent) + ["}"] + else: + return [open_brace] + lines + ["}"] + + def format_command(self, command: Command) -> List[str]: + is_namespace_eval = ( + command.routine.contents == "namespace" + and len(command.args) > 0 + and command.args[0].contents == "eval" + ) + should_indent = not is_namespace_eval or self.opts.indent_namespace_eval + + hanging_indent = False + formatted = self.format(command.routine) + last_line = command.routine.end_pos[0] + for child in command.args: + if isinstance(child, Script): + child_lines = self.format_script(child, should_indent=should_indent) + else: + child_lines = self.format(child) + + if last_line == child.pos[0]: + formatted[-1] += " " + formatted[-1] += child_lines[0] + else: + formatted[-1] += " \\" + formatted.append(self.opts.indent + child_lines[0]) + hanging_indent = True + + if hanging_indent: + formatted.extend(self._indent(child_lines[1:], self.opts.indent)) + else: + formatted.extend(child_lines[1:]) + + last_line = child.end_pos[0] + + return formatted + + def format_comment(self, comment: Comment) -> List[str]: + return [f"#{comment.value}"] + + def format_command_sub(self, command_sub): + if len(command_sub.children) == 0: + return ["[]"] + + formatted = [] + contents = self.format_script_contents(command_sub) + if len(command_sub.children) > 1 and len(contents) > 1: + formatted.append("[") + formatted.extend(self._indent(contents, self.opts.indent)) + formatted.append("]") + else: + formatted.append("[" + contents[0]) + formatted.extend(contents[1:]) + formatted[-1] += "]" + + return formatted + + def format_bare_word(self, word) -> List[str]: + # Property enforced by parser + assert word.contents is not None + return [word.contents] + + def format_quoted_word(self, word) -> List[str]: + if word.contents is not None: + return [f'"{word.contents}"'] + + formatted = "" + for child in word.children: + formatted += "\n".join(self.format(child)) + + return [f'"{formatted}"'] + + def format_braced_word(self, word) -> List[str]: + assert word.contents is not None + return [f"{{{word.contents}}}"] + + def format_compound_bare_word(self, word) -> List[str]: + formatted = [""] + for child in word.children: + child_lines = self.format(child) + formatted[-1] += child_lines[0] + formatted.extend(child_lines[1:]) + + return formatted + + def format_var_sub(self, varsub) -> List[str]: + # We might be able to make the formatter infer whether braces are required, and + # remove them from the syntax tree. For now it's easier to just mimic the + # original format. + if varsub.braced: + formatted = [f"${{{varsub.value}}}"] + else: + formatted = [f"${varsub.value}"] + + if varsub.children: + # We just concatenate everything as is, since changes in whitespace are + # semantically meaningful in this context. Any newlines are captured by + # BareWords. + formatted[-1] += "(" + for child in varsub.children: + child_lines = self.format(child) + formatted[-1] += child_lines[0] + formatted.extend(child_lines[1:]) + formatted[-1] += ")" + + return formatted + + def format_arg_expansion(self, arg_expansion) -> List[str]: + lines = self.format(arg_expansion.list) + lines[0] = "{*}" + lines[0] + + return lines + + def format_list(self, list_node) -> List[str]: + # Similar to Script, but the contents are a bit more straightforward. + contents = [""] + last_line = None + for child in list_node.children: + if last_line is not None: + if last_line == child.pos[0]: + contents[-1] += " " + else: + newlines = child.pos[0] - last_line + newlines = min(newlines, 3) + contents.extend([""] * newlines) + + lines = self.format(child) + contents[-1] += lines[0] + contents.extend(lines[1:]) + + last_line = child.end_pos[0] + + if list_node.pos[0] == list_node.end_pos[0]: + return self._brace(contents) + + return ["{"] + self._indent(contents, self.opts.indent) + ["}"] + + def format_expression(self, expr) -> List[str]: + formatted = [""] + for child in expr.children: + lines = self.format(child) + formatted[-1] += lines[0] + for line in lines[1:]: + formatted[-1] += " \\" + formatted += self._indent([line], self.opts.indent) + + # Trick: we know there are quotes around the expression if the start of the + # expression is a different column than its first child. + quoted = expr.pos[1] != expr.children[0].pos[1] + if quoted: + formatted[0] = '"' + formatted[0] + formatted[-1] += '"' + + return formatted + + def format_braced_expression(self, expr) -> List[str]: + formatted = [""] + for child in expr.children: + lines = self.format(child) + formatted[-1] += lines[0] + formatted.extend(lines[1:]) + + if expr.pos[0] == expr.end_pos[0]: + return self._brace(formatted) + + return ["{"] + self._indent(formatted, self.opts.indent) + ["}"] + + def format_paren_expression(self, expr) -> List[str]: + body = expr.body + + formatted = ["("] + lines = self.format(body) + if expr.pos[0] != body.pos[0]: + formatted.extend(lines) + else: + formatted[-1] += lines[0] + formatted.extend(lines[1:]) + + formatted = formatted[0:1] + self._indent(formatted[1:], self.opts.indent) + + if expr.end_pos[0] != body.end_pos[0]: + formatted.append(")") + else: + formatted[-1] += ")" + + return formatted + + def format_unary_op(self, expr): + op = self.format(expr.operator) + assert len(op) == 1 + + lines = self.format(expr.operand) + lines[0] = op[0] + lines[0] + return lines + + def _format_op(self, expr) -> List[str]: + nodes = expr.children + formatted = self.format(nodes[0]) + + last = nodes[0] + for next in nodes[1:]: + lines = self.format(next) + if last.end_pos[0] != next.pos[0]: + formatted.extend(lines) + else: + formatted[-1] += " " + formatted[-1] += lines[0] + formatted.extend(lines[1:]) + last = next + + return formatted + + def format_binary_op(self, expr) -> List[str]: + return self._format_op(expr) + + def format_ternary_op(self, expr) -> List[str]: + return self._format_op(expr) + + def format_function(self, function): + name = self.format(function.name) + assert len(name) == 1 + name = name[0] + + formatted = [f"{name}("] + + last = function.name + for i, child in enumerate(function.args): + if i > 0: + formatted[-1] += "," + lines = self.format(child) + if last.end_pos[0] != child.pos[0]: + formatted.extend(lines) + else: + if i > 0: + formatted[-1] += " " + formatted[-1] += lines[0] + formatted.extend(lines[1:]) + last = child + + # indent any continuation lines, but we leave the closing paren dedented + formatted = formatted[0:1] + self._indent(formatted[1:], self.opts.indent) + + if last.end_pos[0] != function.end_pos[0]: + formatted.append(")") + else: + formatted[-1] += ")" + + return formatted diff --git a/server/src/tools/lexer.py b/server/src/tools/lexer.py index 1375c4b..dc18312 100644 --- a/server/src/tools/lexer.py +++ b/server/src/tools/lexer.py @@ -1,34 +1,30 @@ -from enum import Enum import ply.lex as lex from typing import Tuple - -class Tok(str, Enum): - TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE" - TOK_BACKSLASH_SUB = "BACKSLASH_SUB" - TOK_NEWLINE = "NEWLINE" - TOK_SEMI = "SEMI" - TOK_WS = "WS" - TOK_QUOTE = "QUOTE" - TOK_ARG_EXPANSION = "ARG_EXPANSION" - TOK_LBRACE = "LBRACE" - TOK_RBRACE = "RBRACE" - TOK_STAR = "STAR" - TOK_LBRACKET = "LBRACKET" - TOK_RBRACKET = "RBRACKET" - TOK_DOLLAR = "DOLLAR" - TOK_LPAREN = "LPAREN" - TOK_RPAREN = "RPAREN" - TOK_HASH = "HASH" - TOK_ALPHA_CHARS = "ALPHA_CHARS" - TOK_NUM_CHARS = "NUM_CHARS" - TOK_NAMESPACE_SEP = "NAMESPACE_SEP" - TOK_CHAR = "CHAR" - TOK_CONTENTS = "CONTENTS" - +TOK_BACKSLASH_NEWLINE = "BACKSLASH_NEWLINE" +TOK_BACKSLASH_SUB = "BACKSLASH_SUB" +TOK_NEWLINE = "NEWLINE" +TOK_SEMI = "SEMI" +TOK_WS = "WS" +TOK_QUOTE = "QUOTE" +TOK_ARG_EXPANSION = "ARG_EXPANSION" +TOK_LBRACE = "LBRACE" +TOK_RBRACE = "RBRACE" +TOK_STAR = "STAR" +TOK_LBRACKET = "LBRACKET" +TOK_RBRACKET = "RBRACKET" +TOK_DOLLAR = "DOLLAR" +TOK_LPAREN = "LPAREN" +TOK_RPAREN = "RPAREN" +TOK_HASH = "HASH" +TOK_ALPHA_CHARS = "ALPHA_CHARS" +TOK_NUM_CHARS = "NUM_CHARS" +TOK_NAMESPACE_SEP = "NAMESPACE_SEP" +TOK_CHAR = "CHAR" +TOK_CONTENTS = "CONTENTS" +TOK_EOF = None STATE_BRACEDWORD = "bracedword" -TOK_EOF = None class TclSyntaxError(Exception): @@ -39,27 +35,38 @@ class TclSyntaxError(Exception): class _LexTable: - tokens = tuple(t.value for t in Tok) + tokens = ( + TOK_BACKSLASH_NEWLINE, + TOK_BACKSLASH_SUB, + TOK_NEWLINE, + TOK_SEMI, + TOK_WS, + TOK_QUOTE, + TOK_ARG_EXPANSION, + TOK_LBRACE, + TOK_RBRACE, + TOK_STAR, + TOK_LBRACKET, + TOK_RBRACKET, + TOK_DOLLAR, + TOK_LPAREN, + TOK_RPAREN, + TOK_HASH, + TOK_ALPHA_CHARS, + TOK_NUM_CHARS, + TOK_NAMESPACE_SEP, + TOK_CHAR, + TOK_CONTENTS, + ) + # This defines a conditional lexing state for parsing braced words. This is a + # performance optimization; since there are few special characters in this context, + # we can use a smaller set of tokens to parse them faster. This has a large impact + # since most Tcl programs have a large number of braced words. Any token with + # `bracedword` in its name is included in this state. Tokens that are included in + # this state and the default state also include `INITIAL` in their name. states = ((STATE_BRACEDWORD, "exclusive"),) - def __init__(self): - self.lexer = lex.lex(object=self) - self.lexer.lineno = 1 - self.lexer.colno = 1 - - def new_lexer(self, pos=None): - lexer = self.lexer.clone() - lexer.lineno = 1 - lexer.colno = 1 - - if pos is not None: - line, col = pos - lexer.lineno = line - lexer.colno = col - - return lexer - def _tok(self, t): pos = (t.lexer.lineno, t.lexer.colno) t.lexer.lineno += t.value.count("\n") @@ -146,6 +153,9 @@ class _LexTable: r"[A-Za-z_]+" return self._tok(t) + # Valid numeric chars in variable names + # This is split up from the above to facilitate expression parsing, since + # e.g. 1eq1 can't be a single token. def t_NUM_CHARS(self, t): r"[0-9]+" return self._tok(t) @@ -170,6 +180,23 @@ class _LexTable: print("Illegal character '%s'" % t.value[0]) t.lexer.skip(1) + def __init__(self): + self.lexer = lex.lex(object=self) + self.lexer.lineno = 1 + self.lexer.colno = 1 + + def new_lexer(self, pos=None): + lexer = self.lexer.clone() + lexer.lineno = 1 + lexer.colno = 1 + + if pos is not None: + line, col = pos + lexer.lineno = line + lexer.colno = col + + return lexer + # Calling `lex.lex()` performs an expensive reflection process to generate the lexer. # This singleton class holds a preinitialized lexer that can then be cloned to create @@ -214,21 +241,3 @@ class Lexer: def assert_(self, *tokens): assert self.current.type in tokens self.next() - - -def dump_tokens(code): - lx = Lexer() - lx.input(code) - out = [] - while lx.type() is not TOK_EOF: - out.append((lx.type(), lx.value(), lx.pos())) - lx.next() - return out - - -if __name__ == "__main__": - code = ( - "set a 1\nputs $a\nnamespace eval test {}\n proc myProc {arg1 {optArg 10}} {}" - ) - for ttype, val, (ln, col) in dump_tokens(code): - print(f"{ttype:<18} {val!r:<10} @ ({ln},{col})") diff --git a/server/src/tools/parser.py b/server/src/tools/parser.py index abf6553..5819a75 100644 --- a/server/src/tools/parser.py +++ b/server/src/tools/parser.py @@ -1,19 +1,64 @@ -from tools.lexer import Lexer, TclSyntaxError, Tok, TOK_EOF -from tools import syntax_tree as st -from tools.commands import CommandArgError, get_commands -from tools.checks import check_command +import string +import re + +from src.tools.lexer import ( + Lexer, + TclSyntaxError, + STATE_BRACEDWORD, + TOK_BACKSLASH_NEWLINE, + TOK_NEWLINE, + TOK_SEMI, + TOK_WS, + TOK_QUOTE, + TOK_ARG_EXPANSION, + TOK_LBRACE, + TOK_RBRACE, + TOK_LBRACKET, + TOK_RBRACKET, + TOK_DOLLAR, + TOK_LPAREN, + TOK_RPAREN, + TOK_HASH, + TOK_ALPHA_CHARS, + TOK_NUM_CHARS, + TOK_NAMESPACE_SEP, + TOK_EOF, +) +from src.tools.syntax_tree import ( + Script, + Comment, + Command, + CommandSub, + ArgExpansion, + VarSub, + BareWord, + BracedWord, + QuotedWord, + CompoundBareWord, + List, + Expression, + BracedExpression, + ParenExpression, + UnaryOp, + BinaryOp, + TernaryOp, + Function, +) +from src.tools.commands import CommandArgError, get_commands +from src.tools.commands.checks import check_command +from src.tools.violations import Rule, Violation def _strip_ws(parse_func): """Decorator used by expression parser for stripping whitespace around a node.""" def func(parser, ts): - while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE, Tok.TOK_NEWLINE}: + while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: ts.next() node = parse_func(parser, ts) - while ts.type() in {Tok.TOK_WS, Tok.TOK_BACKSLASH_NEWLINE, Tok.TOK_NEWLINE}: + while ts.type() in {TOK_WS, TOK_BACKSLASH_NEWLINE, TOK_NEWLINE}: ts.next() return node @@ -22,6 +67,8 @@ def _strip_ws(parse_func): class _Word: + """Helper class for constructing Word nodes out of multiple segments.""" + def __init__(self): self.segments = [] self.current_segment = "" @@ -30,13 +77,12 @@ class _Word: def add_tok(self, tok): if self.current_start is None: self.current_start = tok.value[1] + self.current_segment += tok.value[0] def add_node(self, node): if self.current_segment != "": self.segments.append( - st.BareWord( - self.current_segment, pos=self.current_start, end_pos=node.pos - ) + BareWord(self.current_segment, pos=self.current_start, end_pos=node.pos) ) self.current_segment = "" self.current_start = None @@ -45,9 +91,7 @@ class _Word: def resolve(self, end_pos): if self.current_segment: self.segments.append( - st.BareWord( - self.current_segment, pos=self.current_start, end_pos=end_pos - ) + BareWord(self.current_segment, pos=self.current_start, end_pos=end_pos) ) return self.segments @@ -57,7 +101,9 @@ class Parser: def __init__(self, debug=False, command_plugins=None): self._debug = debug self._debug_indent = 0 + # TODO: better way to handle this? self.violations = [] + if command_plugins is None: command_plugins = [] self._commands = get_commands(command_plugins) @@ -726,7 +772,7 @@ class Parser: pos=name.pos, ) - delims = {Tok.TOK_RPAREN, TOK_EOF} + delims = {TOK_RPAREN, TOK_EOF} arguments = [] if ts.type() not in delims: @@ -745,11 +791,11 @@ class Parser: arguments.append(self._parse_expression(ts)) ts.expect( - Tok.TOK_RPAREN, + TOK_RPAREN, message="expected close paren after function arguments", pos=name.pos, ) - return st.Function(name, *arguments, pos=name.pos, end_pos=ts.pos()) + return Function(name, *arguments, pos=name.pos, end_pos=ts.pos()) def _all(_list, non_empty=False): diff --git a/server/src/tools/syntax_tree.py b/server/src/tools/syntax_tree.py index 97f646c..d11f6df 100644 --- a/server/src/tools/syntax_tree.py +++ b/server/src/tools/syntax_tree.py @@ -1,4 +1,4 @@ -"""Classes for representing and interacting with Tcl syntax trees.""" +"""Classes for representing and interacting with Tcl syntax trees. """ class Visitor: @@ -188,12 +188,12 @@ class Node: return lines if len(self.children) != len(other.children): - my_children = ",".join( - [child.__class__.__name__ for child in self.children] - ) - other_children = ",".join( - [child.__class__.__name__ for child in other.children] - ) + my_children = ",".join([ + child.__class__.__name__ for child in self.children + ]) + other_children = ",".join([ + child.__class__.__name__ for child in other.children + ]) lines += [f"{indent}-{my_cls}({my_children})"] lines += [f"{indent}+{other_cls}({other_children})"] diff --git a/server/src/tools/violations.py b/server/src/tools/violations.py new file mode 100644 index 0000000..8b08329 --- /dev/null +++ b/server/src/tools/violations.py @@ -0,0 +1,50 @@ +from enum import Enum +from typing import Tuple + + +class Rule(Enum): + """This enum serves a few purposes: + + 1) define symbols for rule IDs to be used in code + 2) map these symbols to names in the UI + 3) collect all rule IDs/provide validation for IDs + """ + + LINE_LENGTH = "line-length" + TRAILING_WHITESPACE = "trailing-whitespace" + COMMAND_ARGS = "command-args" + REDEFINED_BUILTIN = "redefined-builtin" + UNBRACED_EXPR = "unbraced-expr" + REDUNDANT_EXPR = "redundant-expr" + + def __str__(self): + return self.value + + +ALL_RULES = [rule for rule in Rule] + + +class Violation: + def __init__( + self, id: Rule, message: str, start: Tuple[int, int], end: Tuple[int, int] + ): + self.id = id + self.message = message + self.start = start + self.end = end + + def __lt__(self, other): + return self.start < other.start + + def __str__(self): + line, col = self.start + rule = str(self.id) + + return f"{line}:{col}: {self.message} [{rule}]" + + @classmethod + def create(cls, id): + def func(message: str, start: Tuple[int, int], end: Tuple[int, int]): + return cls(id, message, start, end) + + return func diff --git a/test/test.tcl b/test/test.tcl index 0163f2d..bbac630 100644 --- a/test/test.tcl +++ b/test/test.tcl @@ -1,3 +1,5 @@ proc myProc {arg {opt 1}} { + +} -} \ No newline at end of file +MOM_abort_program "Test" \ No newline at end of file