Implement nondeterminism

Author: jg-rp

jsonpath_rfc9535/__init__.py

@@ -1,3 +1,4 @@
+from .environment import JSONLikeData
 from .environment import JSONPathEnvironment
 from .exceptions import JSONPathError
 from .exceptions import JSONPathIndexError
@@ -13,6 +14,7 @@
 from .path import JSONPath
 
 __all__ = (
+    "JSONLikeData",
     "JSONPathEnvironment",
     "JSONPathError",
     "JSONPathIndexError",

jsonpath_rfc9535/environment.py

@@ -59,10 +59,12 @@ class JSONPathEnvironment:
             index. Defaults to `(2**53) - 1`.
         min_int_index (int): The minimum integer allowed when selecting array items by
             index. Defaults to `-(2**53) + 1`.
-        max_recursion_depth: The maximum number of dict/objects and/or arrays/lists the
-            recursive descent selector can visit before a `JSONPathRecursionError`
-            is thrown.
-        parser_class: The parser to use when parsing tokens from the lexer.
+        max_recursion_depth (int): The maximum number of dict/objects and/or
+            arrays/lists the recursive descent selector can visit before a
+            `JSONPathRecursionError` is thrown.
+        parser_class (Parser): The parser to use when parsing tokens from the lexer.
+        nondeterministic (bool): If `True`, enable nondeterminism when iterating objects
+            and visiting nodes with the recursive descent segment. Defaults to `False`.
     """
 
     parser_class: Type[Parser] = Parser
@@ -71,6 +73,8 @@ class JSONPathEnvironment:
     min_int_index = -(2**53) + 1
     max_recursion_depth = 100
 
+    nondeterministic = False
+
     def __init__(self) -> None:
         self.parser: Parser = self.parser_class(env=self)
         """The parser bound to this environment."""

jsonpath_rfc9535/segments.py

@@ -2,9 +2,12 @@
 
 from __future__ import annotations
 
+import random
 from abc import ABC
 from abc import abstractmethod
+from collections import deque
 from typing import TYPE_CHECKING
+from typing import Deque
 from typing import Iterable
 from typing import Tuple
 
@@ -66,12 +69,24 @@ class JSONPathRecursiveDescentSegment(JSONPathSegment):
 
     def resolve(self, nodes: Iterable[JSONPathNode]) -> Iterable[JSONPathNode]:
         """Select descendants of each node in _nodes_."""
+        # The nondeterministic visitor never generates a pre order traversal, so we
+        # still use the deterministic visitor 20% of the time, to cover all
+        # permutations.
+        #
+        # XXX: This feels like a bit of a hack.
+        visitor = (
+            self._nondeterministic_visit
+            if self.env.nondeterministic and random.random() < 0.2  # noqa: S311, PLR2004
+            else self._visit
+        )
+
         for node in nodes:
-            for _node in self._visit(node):
+            for _node in visitor(node):
                 for selector in self.selectors:
                     yield from selector.resolve(_node)
 
     def _visit(self, node: JSONPathNode, depth: int = 1) -> Iterable[JSONPathNode]:
+        """Pre order node traversal."""
         if depth > self.env.max_recursion_depth:
             raise JSONPathRecursionError("recursion limit exceeded", token=self.token)
 
@@ -96,6 +111,45 @@ def _visit(self, node: JSONPathNode, depth: int = 1) -> Iterable[JSONPathNode]:
                     )
                     yield from self._visit(_node, depth + 1)
 
+    def _nondeterministic_visit(
+        self,
+        root: JSONPathNode,
+        _: int = 1,
+    ) -> Iterable[JSONPathNode]:
+        def _children(node: JSONPathNode) -> Iterable[JSONPathNode]:
+            if isinstance(node.value, dict):
+                items = list(node.value.items())
+                random.shuffle(items)
+                for key, val in items:
+                    if isinstance(val, (dict, list)):
+                        yield JSONPathNode(
+                            value=val,
+                            parts=node.parts + (key,),
+                            root=node.root,
+                        )
+            elif isinstance(node.value, list):
+                for i, val in enumerate(node.value):
+                    if isinstance(val, (dict, list)):
+                        yield JSONPathNode(
+                            value=val,
+                            parts=node.parts + (i,),
+                            root=node.root,
+                        )
+
+        queue: Deque[JSONPathNode] = deque(_children(root))
+        yield root
+
+        while queue:
+            _node = queue.popleft()
+            yield _node
+            for child in _children(_node):
+                # Queue the child node or visit it now?
+                if random.choice([True, False]):  # noqa: S311
+                    queue.append(child)
+                else:
+                    yield child
+                    queue.extend(_children(child))
+
     def __str__(self) -> str:
         return f"..[{', '.join(str(itm) for itm in self.selectors)}]"
 

pyproject.toml

@@ -168,3 +168,4 @@ convention = "google"
 "jsonpath_rfc9535/__init__.py" = ["D104"]
 "tests/*" = ["D100", "D101", "D104", "D103"]
 "jsonpath_rfc9535/lex.py" = ["E741"]
+"scripts/nondeterministic_descent.py" = ["D103", "T201"]

scripts/__init__.py

@@ -0,0 +1,151 @@
+"""Utilities for exploring nondeterminism in the recursive descent segment."""
+
+from __future__ import annotations
+
+import os
+import random
+import sys
+from collections import deque
+from typing import TYPE_CHECKING
+from typing import Deque
+from typing import Iterable
+from typing import List
+from typing import Optional
+from typing import TextIO
+from typing import Tuple
+
+if TYPE_CHECKING:
+    from jsonpath_rfc9535.environment import JSONLikeData
+
+
+HORIZONTAL_SEP = "\N{BOX DRAWINGS LIGHT HORIZONTAL}" * 2
+VERTICAL_SEP = "\N{BOX DRAWINGS LIGHT VERTICAL}"
+BRANCH = "\N{BOX DRAWINGS LIGHT VERTICAL AND RIGHT}" + HORIZONTAL_SEP + " "
+TERMINAL_BRANCH = "\N{BOX DRAWINGS LIGHT UP AND RIGHT}" + HORIZONTAL_SEP + " "
+INDENT = VERTICAL_SEP + " " * 3
+TERMINAL_INDENT = " " * 4
+
+COLOR_CODES = [
+    ("\033[92m", "\033[0m"),
+    ("\033[93m", "\033[0m"),
+    ("\033[94m", "\033[0m"),
+    ("\033[95m", "\033[0m"),
+    ("\033[96m", "\033[0m"),
+    ("\033[91m", "\033[0m"),
+]
+
+
+class AuxNode:
+    def __init__(
+        self,
+        depth: int,
+        value: object,
+        children: Optional[List[AuxNode]] = None,
+    ) -> None:
+        self.value = value
+        self.children = children or []
+        self.depth = depth
+
+    def __str__(self) -> str:
+        c_start, c_stop = COLOR_CODES[self.depth % len(COLOR_CODES)]
+        return f"{c_start}{self.value}{c_stop}"
+
+    @staticmethod
+    def from_(data: JSONLikeData) -> AuxNode:
+        def _visit(node: AuxNode, depth: int = 0) -> None:
+            if isinstance(node.value, dict):
+                for val in node.value.values():
+                    _node = AuxNode(depth + 1, val)
+                    _visit(_node, depth + 1)
+                    node.children.append(_node)
+
+            elif isinstance(node.value, list):
+                for val in node.value:
+                    _node = AuxNode(depth + 1, val)
+                    _visit(_node, depth + 1)
+                    node.children.append(_node)
+
+        root = AuxNode(0, data)
+        _visit(root)
+        return root
+
+
+def pptree(
+    node: AuxNode,
+    indent: str = "",
+    buf: TextIO = sys.stdout,
+) -> None:
+    """Pretty print the tree rooted at `node`."""
+    # Pre-order tree traversal
+    buf.write(str(node) + os.linesep)
+
+    if node.children:
+        # Recursively call pptree for all but the last child of `node`.
+        for child in node.children[:-1]:
+            buf.write(indent + BRANCH)
+            pptree(child, indent=indent + INDENT, buf=buf)
+
+        # Terminal branch case for last, possibly only, child of `node`.
+        buf.write(indent + TERMINAL_BRANCH)
+        pptree(node.children[-1], indent=indent + TERMINAL_INDENT, buf=buf)
+
+    # Base case. No children.
+
+
+def pre_order_visit(node: AuxNode) -> Iterable[AuxNode]:
+    yield node
+
+    for child in node.children:
+        yield from pre_order_visit(child)
+
+
+def breadth_first_visit(node: AuxNode) -> Iterable[AuxNode]:
+    queue: Deque[AuxNode] = deque([node])
+
+    while queue:
+        _node = queue.popleft()
+        yield _node
+        queue.extend(_node.children)
+
+
+def nondeterministic_visit(root: AuxNode) -> Iterable[AuxNode]:
+    queue: Deque[AuxNode] = deque(root.children)
+    yield root
+
+    while queue:
+        _node = queue.popleft()
+        yield _node
+        for child in _node.children:
+            # Queue the child node or visit it now?
+            if random.choice([True, False]):  # noqa: S311
+                queue.append(child)
+            else:
+                yield child
+                queue.extend(child.children)
+
+
+def get_perms(root: AuxNode) -> List[Tuple[AuxNode, ...]]:
+    perms = {tuple(nondeterministic_visit(root)) for _ in range(1000)}
+    perms.add(tuple(pre_order_visit(root)))
+    return sorted(perms, key=lambda t: str(t))
+
+
+def pp_json_path_data(data: JSONLikeData) -> None:
+    aux_tree = AuxNode.from_(data)
+    pptree(aux_tree)
+
+    print("\nPre order\n")
+    print(", ".join(str(n) for n in pre_order_visit(aux_tree)))
+
+    print("\nLevel order\n")
+    print(", ".join(str(n) for n in breadth_first_visit(aux_tree)))
+
+    print("\nNondeterministic order\n")
+    for perm in get_perms(aux_tree):
+        print(", ".join(str(node) for node in perm))
+
+
+if __name__ == "__main__":
+    # basic, descendant segment, name shorthand
+    data = {"o": [{"a": "b"}, {"a": "c"}]}
+    pp_json_path_data(data)

scripts/nondeterministic_descent.py

@@ -0,0 +1,53 @@
+"""Test against the JSONPath Compliance Test Suite with nondeterminism enabled.
+
+The CTS is a submodule located in /tests/cts. After a git clone, run
+`git submodule update --init` from the root of the repository.
+"""
+
+import json
+import operator
+from dataclasses import dataclass
+from typing import Any
+from typing import List
+from typing import Optional
+
+import pytest
+
+from jsonpath_rfc9535 import JSONLikeData
+from jsonpath_rfc9535 import JSONPathEnvironment
+
+
+@dataclass
+class Case:
+    name: str
+    selector: str
+    document: JSONLikeData = None
+    result: Any = None
+    results: Optional[List[Any]] = None
+    invalid_selector: Optional[bool] = None
+
+
+def cases() -> List[Case]:
+    with open("tests/cts/cts.json", encoding="utf8") as fd:
+        data = json.load(fd)
+    return [Case(**case) for case in data["tests"]]
+
+
+def valid_cases() -> List[Case]:
+    return [case for case in cases() if not case.invalid_selector]
+
+
+class MockEnv(JSONPathEnvironment):
+    nondeterministic = True
+
+
+@pytest.mark.parametrize("case", valid_cases(), ids=operator.attrgetter("name"))
+def test_nondeterminism(case: Case) -> None:
+    assert case.document is not None
+    env = MockEnv()
+    rv = env.findall(case.selector, case.document)
+
+    if case.results is not None:
+        assert rv in case.results
+    else:
+        assert rv == case.result

tests/cts

@@ -11,18 +11,18 @@
 from typing import Dict
 from typing import List
 from typing import Optional
-from typing import Union
 
 import pytest
 
 import jsonpath_rfc9535 as jsonpath
+from jsonpath_rfc9535.environment import JSONLikeData
 
 
 @dataclass
 class Case:
     name: str
     selector: str
-    document: Union[Dict[str, Any], List[Any], None] = None
+    document: JSONLikeData = None
     result: Any = None
     results: Optional[List[Any]] = None
     invalid_selector: Optional[bool] = None

tests/nondeterminism.py

@@ -36,7 +36,11 @@ def test_non_singular_query_is_not_comparable(env: JSONPathEnvironment) -> None:
         env.compile("$[?@.* > 2]")
 
 
-def test_recursive_data(env: JSONPathEnvironment) -> None:
+def test_recursive_data() -> None:
+    class MockEnv(JSONPathEnvironment):
+        nondeterministic = False
+
+    env = MockEnv()
     query = "$..a"
     arr: List[Any] = []
     data: Any = {"foo": arr}
@@ -46,11 +50,10 @@ def test_recursive_data(env: JSONPathEnvironment) -> None:
         env.query(query, data)
 
 
-class MockEnv(JSONPathEnvironment):
-    max_recursion_depth = 3
-
-
 def test_low_recursion_limit() -> None:
+    class MockEnv(JSONPathEnvironment):
+        max_recursion_depth = 3
+
     env = MockEnv()
     query = "$..a"
     data = {"foo": [{"bar": [1, 2, 3]}]}