Merge pull request #5 from bogdandm/optimization

Optimization

Author: bogdandm

rest_client_gen/dynamic_typing/__init__.py

@@ -1,5 +1,5 @@
 from .base import (
-    BaseType, ImportPathList, MetaData, NoneType, Unknown, UnknownType
+    BaseType, ImportPathList, MetaData, NoneType, Unknown, UnknownType, get_hash_string
 )
 from .complex import ComplexType, DList, DOptional, DTuple, DUnion, SingleType
 from .models_meta import AbsoluteModelRef, ModelMeta, ModelPtr

rest_client_gen/dynamic_typing/base.py

@@ -1,3 +1,4 @@
+from inspect import isclass
 from typing import Iterable, List, Tuple, Union
 
 ImportPathList = List[Tuple[str, Union[Iterable[str], str, None]]]
@@ -28,6 +29,27 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]:
         """
         raise NotImplementedError()
 
+    def to_hash_string(self) -> str:
+        """
+        Return unique string that can be used to generate hash of type instance.
+        Caches hash value by default. If subclass can mutate (by default it always can)
+        then it should define setters to safely invalidate cached value.
+
+        :return: hash string
+        """
+        # NOTE: Do not override __hash__ function because BaseType instances isn't immutable
+        if not self._hash:
+            self._hash = self._to_hash_string()
+        return self._hash
+
+    def _to_hash_string(self) -> str:
+        """
+        Hash getter method to override
+
+        :return:
+        """
+        raise NotImplementedError()
+
 
 class UnknownType(BaseType):
     __slots__ = []
@@ -44,7 +66,19 @@ def replace(self, t: 'MetaData', **kwargs) -> 'UnknownType':
     def to_typing_code(self) -> Tuple[ImportPathList, str]:
         return ([('typing', 'Any')], 'Any')
 
+    def to_hash_string(self) -> str:
+        return "Unknown"
+
 
 Unknown = UnknownType()
 NoneType = type(None)
 MetaData = Union[type, dict, BaseType]
+
+
+def get_hash_string(t: MetaData):
+    if isinstance(t, dict):
+        return str(hash(tuple((k, get_hash_string(v)) for k, v in t.items())))
+    elif isclass(t):
+        return str(t)
+    elif isinstance(t, BaseType):
+        return t.to_hash_string()

rest_client_gen/dynamic_typing/complex.py

@@ -1,38 +1,53 @@
 from itertools import chain
 from typing import Iterable, List, Tuple, Union
 
-from .base import BaseType, ImportPathList, MetaData
+from .base import BaseType, ImportPathList, MetaData, get_hash_string
 from .typing import metadata_to_typing
 
 
 class SingleType(BaseType):
-    __slots__ = ["type"]
+    __slots__ = ["_type", "_hash"]
 
     def __init__(self, t: MetaData):
-        self.type = t
+        self._type = t
+        self._hash = None
+
+    @property
+    def type(self):
+        return self._type
+
+    @type.setter
+    def type(self, t: MetaData):
+        self._type = t
+        self._hash = None
 
     def __str__(self):
-        return f"{self.__class__.__name__}[{self.type}]"
+        return f"{type(self).__name__}[{self.type}]"
 
     def __repr__(self):
-        return f"<{self.__class__.__name__} [{self.type}]>"
+        return f"<{type(self).__name__} [{self.type}]>"
 
     def __iter__(self) -> Iterable['MetaData']:
         yield self.type
 
     def __eq__(self, other):
-        return isinstance(other, self.__class__) and self.type == other.type
+        return type(other) is type(self) and self.type == other.type
 
     def replace(self, t: 'MetaData', **kwargs) -> 'SingleType':
         self.type = t
         return self
 
+    def _to_hash_string(self) -> str:
+        return f"{type(self).__name__}/{get_hash_string(self.type)}"
+
 
 class ComplexType(BaseType):
-    __slots__ = ["_types"]
+    __slots__ = ["_types", "_sorted", "_hash"]
 
     def __init__(self, *types: MetaData):
         self._types = list(types)
+        self._sorted = None
+        self._hash = None
 
     @property
     def types(self):
@@ -42,6 +57,7 @@ def types(self):
     def types(self, value):
         self._types = value
         self._sorted = None
+        self._hash = None
 
     @property
     def sorted(self):
@@ -62,17 +78,17 @@ def _sort_key(self, item):
 
     def __str__(self):
         items = ', '.join(map(str, self.types))
-        return f"{self.__class__.__name__}[{items}]"
+        return f"{type(self).__name__}[{items}]"
 
     def __repr__(self):
         items = ', '.join(map(str, self.types))
-        return f"<{self.__class__.__name__} [{items}]>"
+        return f"<{type(self).__name__} [{items}]>"
 
     def __iter__(self) -> Iterable['MetaData']:
         yield from self.types
 
     def __eq__(self, other):
-        return isinstance(other, self.__class__) and self.sorted == other.sorted
+        return type(other) is type(self) and self.sorted == other.sorted
 
     def __len__(self):
         return len(self.types)
@@ -97,6 +113,9 @@ def to_typing_code(self) -> Tuple[ImportPathList, str]:
             f"[{nested}]"
         )
 
+    def _to_hash_string(self) -> str:
+        return type(self).__name__ + "/" + ",".join(map(get_hash_string, self.types))
+
 
 class DOptional(SingleType):
     """
@@ -117,16 +136,22 @@ class DUnion(ComplexType):
     """
 
     def __init__(self, *types: Union[type, BaseType, dict]):
+        hashes = set()
         unique_types = []
+        # Ensure that types in union are unique
         for t in types:
             if isinstance(t, DUnion):
                 # Merging nested DUnions
                 for t2 in list(t._extract_nested_types()):
-                    if t2 not in unique_types:
+                    h = get_hash_string(t2)
+                    if h not in hashes:
                         unique_types.append(t2)
-            elif t not in unique_types:
-                # Ensure that types in union are unique
-                unique_types.append(t)
+                        hashes.add(h)
+            else:
+                h = get_hash_string(t)
+                if h not in hashes:
+                    hashes.add(h)
+                    unique_types.append(t)
         super().__init__(*unique_types)
 
     def _extract_nested_types(self):

rest_client_gen/dynamic_typing/models_meta.py

@@ -121,6 +121,7 @@ def replace(self, t: ModelMeta, **kwargs) -> 'ModelPtr':
         return self
 
     def replace_parent(self, t: ModelMeta, **kwargs) -> 'ModelPtr':
+        self._hash = None
         self.parent.remove_child_ref(self)
         self.parent = t
         self.parent.add_child_ref(self)
@@ -129,6 +130,9 @@ def replace_parent(self, t: ModelMeta, **kwargs) -> 'ModelPtr':
     def to_typing_code(self) -> Tuple[ImportPathList, str]:
         return AbsoluteModelRef(self.type).to_typing_code()
 
+    def _to_hash_string(self) -> str:
+        return f"{type(self).__name__}_#{self.type.index}"
+
 
 ContextInjectionType = Dict[ModelMeta, Union[ModelMeta, str]]
 
@@ -150,6 +154,11 @@ class NestedModel:
     This information is only available at the models code generation stage
     while typing code is generated from raw metadata and passing this absolute path as argument
     to each ModelPtr would be annoying.
+
+    Usage:
+
+    with AbsoluteModelRef.inject({TestModel: "ParentModelName"}):
+        <some code generation>
     """
 
     class Context:

test/test_dynamic_typing/test_dynamic_typing.py

@@ -1,9 +1,11 @@
+from builtins import complex
+
 import pytest
 
-from rest_client_gen.dynamic_typing import DUnion
+from rest_client_gen.dynamic_typing import DUnion, get_hash_string
 
 # *args | MetaData
-test_dunion= [
+test_dunion = [
     pytest.param(
         [int, int],
         DUnion(int),
@@ -21,7 +23,21 @@
     )
 ]
 
+
 @pytest.mark.parametrize("value,expected", test_dunion)
 def test_dunion_creation(value, expected):
     result = DUnion(*value)
-    assert result == expected
+    assert result == expected
+
+
+def test_hash_string():
+    a = {'a': int}
+    b = {'b': int}
+    c = {'a': float}
+    assert len(set(map(get_hash_string, (a, b, c)))) == 3
+
+    union = DUnion(str, float)
+    h1 = union.to_hash_string()
+    union.replace(complex, index=0)
+    h2 = union.to_hash_string()
+    assert h1 != h2, f"{h1}, {h2}"

testing_tools/pprint_meta_data.py

@@ -37,11 +37,11 @@ def _pprint_gen(value, key=None, lvl=0, empty_line=True, ignore_ptr=False):
             yield " <empty>"
 
     elif isinstance(value, SingleType):
-        yield f"{value.__class__.__name__}:"
+        yield f"{type(value).__name__}:"
         yield from _pprint_gen(value.type, lvl=lvl, empty_line=False, ignore_ptr=ignore_ptr)
 
     elif isinstance(value, ComplexType):
-        yield f"{value.__class__.__name__}:"
+        yield f"{type(value).__name__}:"
 
         for t in value.types:
             yield from _pprint_gen(t, lvl=lvl + 1, ignore_ptr=ignore_ptr)

testing_tools/real_apis/pathofexile.py

@@ -1,6 +1,8 @@
 """
 Path of Exile API http://www.pathofexile.com/developer/docs/api-resource-public-stash-tabs
 """
+from datetime import datetime
+
 import requests
 
 from rest_client_gen.generator import MetadataGenerator
@@ -23,6 +25,7 @@ def main():
     tabs = tabs['stashes']
 
     print(f"Start model generation (data len = {len(tabs)})")
+    start_t = datetime.now()
     gen = MetadataGenerator()
     reg = ModelRegistry()
     fields = gen.generate(*tabs)
@@ -38,6 +41,7 @@ def main():
     print("=" * 20)
 
     print(generate_code(structure, AttrsModelCodeGenerator))
+    print(f"{(datetime.now() - start_t).total_seconds():.4f} seconds")
 
 
 if __name__ == '__main__':