diff --git a/docs/spelling-wordlist.txt b/docs/spelling-wordlist.txt index cf9eaf0b5..380517a4f 100644 --- a/docs/spelling-wordlist.txt +++ b/docs/spelling-wordlist.txt @@ -7,6 +7,8 @@ ValidationError # 0th, sigh... th amongst +backreference +backreferences callables # non-codeblocked cls from autoapi cls @@ -26,6 +28,7 @@ iterable iteratively Javascript jsonschema +lookahead majorly metaschema online diff --git a/docs/validate.rst b/docs/validate.rst index 4e287e4a3..7a6ab3615 100644 --- a/docs/validate.rst +++ b/docs/validate.rst @@ -299,6 +299,15 @@ The JSON Schema specification `recommends (but does not require) ` that may be + in use. A pattern accepted by Python's :mod:`re` but rejected by a + custom engine — such as a backreference when using RE2 — will pass the + format check but fail at match time. See :ref:`regex-providers` for + details. + email ^^^^^ @@ -307,3 +316,140 @@ Since in most cases "validating" an email address is an attempt instead to confi The same applies to the ``idn-email`` format. If you indeed want a particular well-specified set of emails to be considered valid, you can use `FormatChecker.checks` to provide your specific definition. + + +.. _regex-providers: + +Regex Providers +--------------- + +``jsonschema`` uses regular expressions when validating schemas. +By default, these operations use Python's built-in :mod:`re` module. + +Python's :mod:`re` engine is vulnerable to catastrophic (or "pathological") backtracking, +which occurs when a regular expression and an input +cause the :mod:`re` module to run for exponentially long periods of time. +If you are validating user-submitted input against user-supplied schemas, +you may want to use a regular expression engine that isn't vulnerable. + + +The ``RegexProvider`` Protocol +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +A regex provider is any class that satisfies the +`jsonschema.protocols.RegexProvider` protocol. + +Two methods are required, as well as a class attribute named ``raises``: + +``raises: tuple[type[Exception], ...]`` + A tuple of exception classes that might be raised + when interacting with the regex provider. + +``compile(pattern: str)`` + Compile ``pattern`` and return the result. + + This is expected to be equivalent to the behavior of + the builtin ``re.compile()`` function. + +``search(pattern, text: str)`` + Search for ``text`` in the ``pattern``. + + ``pattern`` may be a string, or the result returned by ``compile()``, above. + + This is expected to be equivalent to the behavior of + the builtin ``re.search()`` function. + +Depending on your use case and the features or drawbacks of the regex engine you're using, +it may be beneficial to use caching. + + +Installing a Provider at the Class Level +~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + +Pass a regex provider instance to `jsonschema.validators.extend` +to derive a validator class that uses your regex provider: + +.. code-block:: python + + from jsonschema import validators + + SafeValidator = validators.extend( + validators.Draft202012Validator, + regex_provider=MyProvider(), + ) + +All instances of ``SafeValidator`` will use ``MyProvider`` by default. +The same argument is accepted by `jsonschema.validators.create` +when building a validator class from scratch. + + +Per-Instance Override +~~~~~~~~~~~~~~~~~~~~~ + +Pass a provider instance to the validator constructor +to override the default for a single validator instance: + +.. code-block:: python + + validator = Draft202012Validator(schema, regex_provider=MyProvider()) + + +Critical pitfalls +~~~~~~~~~~~~~~~~~ + +The ``{"format": "regex"}`` format checker in ``jsonschema`` +is hard-coded to use Python's :mod:`re` module +to determine whether a string is a valid regular expression. +It currently cannot access the regex provider you designate, +so its validity check may not align with what your regex engine supports. + +At the current time, the only solution is to override +the default ``"regex"`` format checker with a custom function. + +You cannot use the ``extend`` function's ``format_checker`` argument +because when ``Validator.check_schema`` is called +``jsonschema`` creates a new validator class +that will not honor that argument ``format_checker``. +Therefore, always override the class attribute ``FORMAT_CHECKER`` directly: + +.. code-block:: python + + def is_regex(instance: str) -> bool: + if not isinstance(instance, str): + return True + return bool(re2.compile(_instance)) + + MyValidator.FORMAT_CHECKER.checkers["regex"] = (is_regex, (MyException,)) + + +Example: ``google-re2`` +~~~~~~~~~~~~~~~~~~~~~~~ + +`Google RE2 `_ is a regex engine +that is not vulnerable to catastrophic backtracking +(it simply doesn't support features that require backtracking, +like backreferences or lookahead assertions). +It is available as the ``google-re2`` package on PyPI +and exposes a :mod:`re`-compatible Python API. + +.. code-block:: python + + import jsonschema + import re2 + + class Re2Provider(jsonschema.protocols.RegexProvider): + raises = (re2.error,) + compile = staticmethod(re2.compile) + search = staticmethod(re2.search) + + SafeValidator = jsonschema.validators.extend( + jsonschema.validators.Draft202012Validator, + regex_provider=Re2Provider(), + ) + + def is_regex(instance: str) -> bool: + if not isinstance(instance, str): + return True + return bool(re2.compile(instance)) + + SafeValidator.FORMAT_CHECKER.checkers["regex"] = (is_regex, Re2Provider.raises) diff --git a/jsonschema/_keywords.py b/jsonschema/_keywords.py index f30f95419..baadc6c1f 100644 --- a/jsonschema/_keywords.py +++ b/jsonschema/_keywords.py @@ -1,5 +1,4 @@ from fractions import Fraction -import re from jsonschema._utils import ( ensure_list, @@ -19,7 +18,7 @@ def patternProperties(validator, patternProperties, instance, schema): for pattern, subschema in patternProperties.items(): for k, v in instance.items(): - if re.search(pattern, k): + if validator.regex_provider.search(pattern, k): yield from validator.descend( v, subschema, path=k, schema_path=pattern, ) @@ -37,7 +36,7 @@ def additionalProperties(validator, aP, instance, schema): if not validator.is_type(instance, "object"): return - extras = set(find_additional_properties(instance, schema)) + extras = set(find_additional_properties(validator, instance, schema)) if validator.is_type(aP, "object"): for extra in extras: @@ -215,7 +214,7 @@ def uniqueItems(validator, uI, instance, schema): def pattern(validator, patrn, instance, schema): if ( validator.is_type(instance, "string") - and not re.search(patrn, instance) + and not validator.regex_provider.search(patrn, instance) ): yield ValidationError(f"{instance!r} does not match {patrn!r}") diff --git a/jsonschema/_legacy_keywords.py b/jsonschema/_legacy_keywords.py index a251571bc..9bc316acd 100644 --- a/jsonschema/_legacy_keywords.py +++ b/jsonschema/_legacy_keywords.py @@ -1,5 +1,3 @@ -import re - from referencing.jsonschema import lookup_recursive_ref from jsonschema import _utils @@ -380,7 +378,7 @@ def find_evaluated_property_keys_by_schema(validator, instance, schema): if "patternProperties" in schema: for property in instance: for pattern in schema["patternProperties"]: - if re.search(pattern, property): + if validator.regex_provider.search(pattern, property): evaluated_keys.append(property) if "dependentSchemas" in schema: diff --git a/jsonschema/_regex_provider.py b/jsonschema/_regex_provider.py new file mode 100644 index 000000000..7f3a4aae3 --- /dev/null +++ b/jsonschema/_regex_provider.py @@ -0,0 +1,9 @@ +import re + +from jsonschema.protocols import RegexProvider + + +class PythonRegexProvider(RegexProvider): + raises = (re.error,) + compile = staticmethod(re.compile) # type: ignore[assignment] + search = staticmethod(re.search) # type: ignore[assignment] diff --git a/jsonschema/_utils.py b/jsonschema/_utils.py index 3177caea3..9fdb2f1cc 100644 --- a/jsonschema/_utils.py +++ b/jsonschema/_utils.py @@ -1,6 +1,5 @@ from collections.abc import Mapping, MutableMapping, Sequence from urllib.parse import urlsplit -import re # Module-level sentinels so recursive `_uniq_key` calls produce comparable # keys for nested True/False (function-default sentinels would also work, but @@ -72,7 +71,7 @@ def format_as_index(container, indices): return f"{container}[{']['.join(repr(index) for index in indices)}]" -def find_additional_properties(instance, schema): +def find_additional_properties(validator, instance, schema): """ Return the set of additional properties for the given ``instance``. @@ -83,9 +82,10 @@ def find_additional_properties(instance, schema): """ properties = schema.get("properties", {}) patterns = "|".join(schema.get("patternProperties", {})) + _search = validator.regex_provider.search for property in instance: if property not in properties: - if patterns and re.search(patterns, property): + if patterns and _search(patterns, property): continue yield property @@ -370,7 +370,7 @@ def find_evaluated_property_keys_by_schema(validator, instance, schema): if "patternProperties" in schema: for property in instance: for pattern in schema["patternProperties"]: - if re.search(pattern, property): + if validator.regex_provider.search(pattern, property): evaluated_keys.append(property) if "dependentSchemas" in schema: diff --git a/jsonschema/protocols.py b/jsonschema/protocols.py index b6288dcc2..fc9a7c76e 100644 --- a/jsonschema/protocols.py +++ b/jsonschema/protocols.py @@ -75,6 +75,11 @@ class Validator(Protocol): its `extra (optional) dependencies ` when invoking ``pip``. + regex_provider: + + if provided, this is the `RegexProvider` that will be used + when validating regex-based constructs, like ``patternProperties``. + .. deprecated:: v4.12.0 Subclassing validator classes now explicitly warns this is not part of @@ -99,6 +104,10 @@ class Validator(Protocol): #: :kw:`format` keywords in JSON schemas. FORMAT_CHECKER: ClassVar[jsonschema.FormatChecker] + #: A `jsonschema.protocols.RegexProvider` that will be used + #: when handling + REGEX_PROVIDER: ClassVar[jsonschema.protocols.RegexProvider] + #: A function which given a schema returns its ID. ID_OF: _typing.id_of @@ -112,6 +121,7 @@ def __init__( format_checker: jsonschema.FormatChecker | None = None, *, registry: referencing.jsonschema.SchemaRegistry = ..., + regex_provider: jsonschema.protocols.RegexProvider = ..., ) -> None: ... @classmethod @@ -228,3 +238,35 @@ def evolve(self, **kwargs) -> Validator: ... ) Draft7Validator(schema=..., format_checker=None) """ + + +class Match(Protocol): + """Protocol for a regex match result.""" + + def __bool__(self) -> bool: ... + + +class Pattern(Protocol): + """Protocol for a compiled regex pattern.""" + + def __bool__(self) -> bool: ... + + +class RegexProvider(Protocol): + """Protocol for a regular expression engine provider.""" + + raises: ClassVar[tuple[type[Exception], ...]] + + def compile(self, pattern: str, /) -> Pattern: + """ + Compile the given pattern into a Pattern object. + + Any exception raised must be listed in the *raises* attribute. + """ + + def search(self, pattern: str | Pattern, text: str, /) -> Match | None: + """ + Find text in the given pattern, or return None. + + Any exception raised must be listed in the *raises* attribute. + """ diff --git a/jsonschema/tests/test_validators.py b/jsonschema/tests/test_validators.py index 7d8a4c5cd..a6bdefba5 100644 --- a/jsonschema/tests/test_validators.py +++ b/jsonschema/tests/test_validators.py @@ -9,6 +9,7 @@ from urllib.request import pathname2url import json import os +import re import sys import tempfile import warnings @@ -24,6 +25,7 @@ protocols, validators, ) +import jsonschema.protocols def fail(validator, errors, instance, schema): @@ -303,6 +305,140 @@ def test_extend_applicable_validators(self): self.assertTrue(Derived(schema).is_valid(37)) +class TestRegexProvider(TestCase): + """ + Test regex_provider customization. + + These tests exercise the regex provider code + and the code locations that use regular expressions. + """ + + class MatchNothingProvider(jsonschema.protocols.RegexProvider): + """A provider whose search() never matches.""" + + def compile(self, pattern): + raise NotImplementedError + + def search(self, pattern, text): + return None + + class MatchEverythingProvider(jsonschema.protocols.RegexProvider): + """A provider whose search() always matches.""" + + def compile(self, pattern): + raise NotImplementedError + + def search(self, pattern, text): + return re.search("", "") + + def test_custom_provider_can_be_set(self): + """create() must be able to set a custom regex provider.""" + + provider = self.MatchNothingProvider() + Validator = validators.create( + meta_schema={}, + regex_provider=provider, + ) + validator = Validator({}) + + self.assertIs(Validator.REGEX_PROVIDER, provider) + self.assertIs(validator.regex_provider, provider) + + def test_instance_attribute_can_be_overridden_at_construction(self): + """ + regex_provider can be overridden per-instance at construction time. + """ + + custom = self.MatchEverythingProvider() + Validator = validators.create(meta_schema={}) + validator = Validator({}, regex_provider=custom) + self.assertIs(validator.regex_provider, custom) + + def test_pattern_keyword_uses_provider(self): + """ + `pattern` must delegate to the regex provider. + + This is confirmed by using the MatchNothing regex provider + and verifying that an exact text match is rejected. + """ + provider = self.MatchNothingProvider() + schema = {"pattern": "^abc$"} + validator = validators.Draft202012Validator( + schema, regex_provider=provider, + ) + errors = list(validator.iter_errors("abc")) + self.assertEqual(len(errors), 1) + + def test_pattern_properties_keyword_uses_provider(self): + """ + `patternProperties` must delegate to the regex provider. + + This is confirmed by using the MatchEverything regex provider + and verifying that a non-matching field's value type is rejected. + """ + provider = self.MatchEverythingProvider() + schema = {"patternProperties": {"^foo$": {"type": "integer"}}} + errors = list( + validators.Draft202012Validator( + schema, regex_provider=provider, + ).iter_errors({"not-a-foo": "not-an-integer"}), + ) + self.assertGreater(len(errors), 0) + + def test_additional_properties_uses_provider(self): + """ + `additionalProperties` must use the regex provider + to determine which properties are covered by `patternProperties` + and therefore not "additional". + + Because the configured regex provider matches everything, + the obviously-additional property "bar" is not an additional property, + and the schema passes. + """ + provider = self.MatchEverythingProvider() + schema = { + "patternProperties": {"^foo$": {}}, + "additionalProperties": False, + } + validators.Draft202012Validator( + schema, regex_provider=provider, + ).validate({"bar": "anything"}) + + def test_unevaluated_properties_uses_provider(self): + """ + `unevaluatedProperties` must use the regex provider + to decide which properties have been evaluated by `patternProperties`. + + Because the configured regex provider matches everything, + the obviously-unevaluated property "bar" is not considered unevaluated, + and the schema passes. + """ + provider = self.MatchEverythingProvider() + schema = { + "patternProperties": {"^foo$": {}}, + "unevaluatedProperties": False, + } + validators.Draft202012Validator( + schema, regex_provider=provider, + ).validate({"bar": "anything"}) + + def test_legacy_unevaluated_properties_uses_provider(self): + """ + A legacy copy of `find_evaluated_property_keys_by_schema` exists, + which is used by at least the draft 2019-09 validator. + Like the test above, it must rely on the regex provider. + """ + + provider = self.MatchEverythingProvider() + schema = { + "patternProperties": {"^foo$": {}}, + "unevaluatedProperties": False, + } + validators.Draft201909Validator( + schema, regex_provider=provider, + ).validate({"bar": "anything"}) + + class TestValidationErrorMessages(TestCase): def message_for(self, instance, schema, *args, **kwargs): cls = kwargs.pop("cls", validators._LATEST_VERSION) diff --git a/jsonschema/validators.py b/jsonschema/validators.py index 98ddf6bfb..45eeb31f8 100644 --- a/jsonschema/validators.py +++ b/jsonschema/validators.py @@ -25,6 +25,7 @@ _format, _keywords, _legacy_keywords, + _regex_provider, _types, _typing, _utils, @@ -32,7 +33,7 @@ ) if TYPE_CHECKING: - from jsonschema.protocols import Validator + from jsonschema.protocols import RegexProvider, Validator _UNSET = _utils.Unset() @@ -146,6 +147,7 @@ def create( applicable_validators: _typing.ApplicableValidators = methodcaller( "items", ), + regex_provider: RegexProvider = _regex_provider.PythonRegexProvider(), ) -> type[Validator]: """ Create a new validator class. @@ -206,6 +208,15 @@ def create( implement similar behavior, you can typically ignore this argument and leave it at its default. + regex_provider: + + The regular expression provider to use. + + By default, this is backed by the Python ``re`` module, + but it is possible to replace this with an alternate engine. + For example, it may be desirable to use a regex engine + that isn't subject to pathological backtracking. + Returns: a new `jsonschema.protocols.Validator` class @@ -213,6 +224,8 @@ def create( """ # preemptively don't shadow the `Validator.format_checker` local format_checker_arg = format_checker + # preemptively don't shadow the `Validator.regex_provider` local + regex_provider_arg = regex_provider specification = referencing.jsonschema.specification_with( dialect_id=id_of(meta_schema) or "urn:unknown-dialect", @@ -227,6 +240,7 @@ class Validator: TYPE_CHECKER = type_checker FORMAT_CHECKER = format_checker_arg ID_OF = staticmethod(id_of) + REGEX_PROVIDER = regex_provider_arg _APPLICABLE_VALIDATORS = applicable_validators _validators = field(init=False, repr=False, eq=False) @@ -234,6 +248,10 @@ class Validator: schema: referencing.jsonschema.Schema = field(repr=reprlib.repr) _ref_resolver = field(default=None, repr=False, alias="resolver") format_checker: _format.FormatChecker | None = field(default=None) + regex_provider: RegexProvider = field( + default=regex_provider_arg, + repr=False, + ) # TODO: include new meta-schemas added at runtime _registry: referencing.jsonschema.SchemaRegistry = field( default=_REMOTE_WARNING_REGISTRY, @@ -519,6 +537,7 @@ def extend( version=None, type_checker=None, format_checker=None, + regex_provider: RegexProvider | None = None, ): """ Create a new validator class by extending an existing one. @@ -564,6 +583,13 @@ def extend( If unprovided, the format checker of the extended `jsonschema.protocols.Validator` will be carried along. + regex_provider: + + A regular expression provider. + + If not provided, the regex provider of the extended + `jsonschema.protocols.Validator` will be carried along. + Returns: a new `jsonschema.protocols.Validator` class extending the one @@ -587,6 +613,8 @@ def extend( type_checker = validator.TYPE_CHECKER if format_checker is None: format_checker = validator.FORMAT_CHECKER + if regex_provider is None: + regex_provider = validator.REGEX_PROVIDER return create( meta_schema=validator.META_SCHEMA, validators=all_validators, @@ -595,6 +623,7 @@ def extend( format_checker=format_checker, id_of=validator.ID_OF, applicable_validators=validator._APPLICABLE_VALIDATORS, + regex_provider=regex_provider, )