Merge pull request #11 from bogdandm/features-27.11.18

Add --dict-keys-regex and --dict-keys-fields arguments;

Author: bogdandm

.travis.yml

@@ -4,6 +4,7 @@ language: python
 cache: pip
 
 install:
+  - python setup.py sdist
   - python setup.py install
   - pip install pytest pytest-cov requests coveralls codacy-coverage
 
@@ -22,11 +23,23 @@ matrix:
     - python: 3.8-dev
 
 deploy:
-  provider: pypi
-  user: "bogdandm"
-  password:
-    secure: "bfIiVsBVMRAjsZYT/UtvNFiY5RU9+2b8QyveKXB3JhZCC/+S8nU/ZJkM9/3Q4V9sUVTVd+NR4Iq1OWkgFwYhJXK+eYlqWJDwfzcxTY5sXAcIxtmXGpNKJaqOF8hyCRdeusGFXR9O1RqzZVQ1ioLCj43eOkjvgFbOw2pIRtmG8Sip6oSJUmY9MSYHFzz4EoY6s+Jv9g30P7bSGY9bekOySFi7LmnX2IM+2T7AMuRvRyd4gHlLVy7pxZxj3S/jz9IiwkVq55HUl5ITWCZVrk6RnYiQhWn5D1AllrOilJGCNQ6HmKQWhDu/fNXEg24BFUTs87OymPKAFNRFniaH9PGu5v2yoN0BWN6+23c2Zc/G9QpQzqm+eVIpW5pkp25EiQw8Cz1pjDmLAVzSZuPXxJnpmkihLUAmSoktI4Zo6+QeLaBoqxds//aLqupMPhO3kzUZe1O5CrogPXTCwpHUGKzMxtjaaXdLo3z7EVT8kRDRtggpdE9KD1shDRUMBrakmuOFA+Tms+iKZSBrW5xhC2g/lFnZluZidj2ir8hyJ9lPMUmGxn/OkIQIBcMkEKCsDFC3wPD39MY/eDbkBvmK++Uhah9T0ljLOR+j1n2ZoJ+N3zD/UpucW5681dGA/sDpgjyDc9ESj14IHjTPLI7sw3IXaial4X2h0ZdfXf6NesLDraI="
-  on:
-    tags: true
-    all_branches: false
-  skip_existing: true
+  - provider: pypi
+    user: bogdandm
+    password:
+      secure: bfIiVsBVMRAjsZYT/UtvNFiY5RU9+2b8QyveKXB3JhZCC/+S8nU/ZJkM9/3Q4V9sUVTVd+NR4Iq1OWkgFwYhJXK+eYlqWJDwfzcxTY5sXAcIxtmXGpNKJaqOF8hyCRdeusGFXR9O1RqzZVQ1ioLCj43eOkjvgFbOw2pIRtmG8Sip6oSJUmY9MSYHFzz4EoY6s+Jv9g30P7bSGY9bekOySFi7LmnX2IM+2T7AMuRvRyd4gHlLVy7pxZxj3S/jz9IiwkVq55HUl5ITWCZVrk6RnYiQhWn5D1AllrOilJGCNQ6HmKQWhDu/fNXEg24BFUTs87OymPKAFNRFniaH9PGu5v2yoN0BWN6+23c2Zc/G9QpQzqm+eVIpW5pkp25EiQw8Cz1pjDmLAVzSZuPXxJnpmkihLUAmSoktI4Zo6+QeLaBoqxds//aLqupMPhO3kzUZe1O5CrogPXTCwpHUGKzMxtjaaXdLo3z7EVT8kRDRtggpdE9KD1shDRUMBrakmuOFA+Tms+iKZSBrW5xhC2g/lFnZluZidj2ir8hyJ9lPMUmGxn/OkIQIBcMkEKCsDFC3wPD39MY/eDbkBvmK++Uhah9T0ljLOR+j1n2ZoJ+N3zD/UpucW5681dGA/sDpgjyDc9ESj14IHjTPLI7sw3IXaial4X2h0ZdfXf6NesLDraI=
+    on:
+      tags: true
+      all_branches: false
+
+  - provider: releases
+    api_key:
+      secure: AalzD76UPcpYdW0EqIbIi0prnxj1rsyRI6S4EBx91XGFY+BRJV+wJ99MiCE36A4q+cerDCxLmosiAQdzYkJ6/1ageezytJYD0FiC8GxvXtWoutySi0Ka1UTxvIQY22680lh5Xj4vki8tmQAMiLc9gTtwkk+e1J6bPgbQVNQtd5HD3697fdbeBrK70kzHUt93nW1x3N6Ers+WuLtZzV8ft1QzKGQYqgEGigV3rfCM6BThhOXq8B5DnzwwRHg/4TbHvtjyqKchPSzj9+zxGrInJ4dMAut8sGbvFRypDS8Y/I0ODveEBbbmi1SQkoRTH7OuP76T1qAPurmYR2nhq1owxQ4OLjDLgtSiFyzLTEXTjFAOENa8vlZcwC75iCyun/jj2lBVFvW/1rAIiogHxwLsKELbRZQzfY8pTJK5D5KPLNMtNDYecrxI0Bg0z7RPRjk5cpT3gAIyRHuVqCeXZUh+bP87Ev632jtK0thilF67vJNfSamHxLJZH6hEQNKOMjztcOdRDlY+vNbIVwl/RScrhSi+dQCH6FAINXNL9a6TENkwxYjgHYhB9xxNQXq6p+miSID7V9ptBTDVsNIGcv8xLzaChVWExzkFR/BflObjccKmiN9hTlzHSEUaRBpw3EX/L9YNKtPHvPSo/1KxZPt+AB/z0OgvWqSdNaVVtopooQg=
+    file_glob: true
+    file: dist/*
+    skip_cleanup: true
+    overwrite: true
+    draft: true
+    on:
+      repo: bogdandm/json2python-models
+      tags: true
+      all_branches: false

json_to_models/__init__.py

@@ -1,4 +1,4 @@
 from pkg_resources import parse_version
 
-__version__ = "0.1a1"
+__version__ = "0.1b1"
 VERSION = parse_version(__version__)

json_to_models/cli.py

@@ -3,6 +3,7 @@
 import itertools
 import json
 import os.path
+import re
 from collections import defaultdict
 from datetime import datetime
 from pathlib import Path
@@ -79,26 +80,21 @@ def parse_args(self, args: List[str] = None):
         framework = namespace.framework
         code_generator = namespace.code_generator
         code_generator_kwargs_raw: List[str] = namespace.code_generator_kwargs
-        code_generator_kwargs = {}
-        if code_generator_kwargs_raw:
-            for item in code_generator_kwargs_raw:
-                if item[0] == '"':
-                    item = item[1:]
-                if item[-1] == '"':
-                    item = item[:-1]
-                name, value = item.split("=", 1)
-                code_generator_kwargs[name] = value
+        dict_keys_regex: List[str] = namespace.dict_keys_regex
+        dict_keys_fields: List[str] = namespace.dict_keys_fields
 
         self.validate(models, models_lists, merge_policy, framework, code_generator)
         self.setup_models_data(models, models_lists)
-        self.set_args(merge_policy, structure, framework, code_generator, code_generator_kwargs)
+        self.set_args(merge_policy, structure, framework, code_generator, code_generator_kwargs_raw,
+                      dict_keys_regex, dict_keys_fields)
 
     def run(self):
         if self.enable_datetime:
             register_datetime_classes()
-
-        # TODO: Inject dict_keys_regex and dict_keys_fields
-        generator = MetadataGenerator()
+        generator = MetadataGenerator(
+            dict_keys_regex=self.dict_keys_regex,
+            dict_keys_fields=self.dict_keys_fields
+        )
         registry = ModelRegistry(*self.merge_policy)
         for name, data in self.models_data.items():
             meta = generator.generate(*data)
@@ -152,7 +148,8 @@ def setup_models_data(self, models: Iterable[Tuple[str, Iterable[Path]]],
         }
 
     def set_args(self, merge_policy: List[Union[List[str], str]],
-                 structure: str, framework: str, code_generator: str, code_generator_kwargs: dict):
+                 structure: str, framework: str, code_generator: str, code_generator_kwargs_raw: List[str],
+                 dict_keys_regex: List[str], dict_keys_fields: List[str]):
         """
         Convert CLI args to python representation and set them to appropriate object attributes
         """
@@ -175,8 +172,18 @@ def set_args(self, merge_policy: List[Union[List[str], str]],
             m = importlib.import_module(module)
             self.model_generator = getattr(m, cls)
 
-        if code_generator_kwargs:
-            self.model_generator_kwargs = code_generator_kwargs
+        self.model_generator_kwargs = {}
+        if code_generator_kwargs_raw:
+            for item in code_generator_kwargs_raw:
+                if item[0] == '"':
+                    item = item[1:]
+                if item[-1] == '"':
+                    item = item[:-1]
+                name, value = item.split("=", 1)
+                self.model_generator_kwargs[name] = value
+
+        self.dict_keys_regex = [re.compile(rf"^{r}$") for r in dict_keys_regex] if dict_keys_regex else ()
+        self.dict_keys_fields = dict_keys_fields or ()
 
         self.initialize = True
 
@@ -185,7 +192,6 @@ def _create_argparser() -> argparse.ArgumentParser:
         """
         ArgParser factory
         """
-        # TODO: dict_keys_regex and dict_keys_fields arguments
         parser = argparse.ArgumentParser(
             formatter_class=argparse.RawTextHelpFormatter,
             description="Convert given json files into Python models."
@@ -215,6 +221,21 @@ def _create_argparser() -> argparse.ArgumentParser:
                  "Warn.: This can lead to 6-7 times slowdown on large datasets.\n"
                  "       Be sure that you really need this option.\n\n"
         )
+        parser.add_argument(
+            "--dict-keys-regex", "--dkr",
+            nargs="+", metavar="RegEx",
+            help="List of regular expressions (Python syntax).\n"
+                 "If all keys of some dict are match one of them\n"
+                 "then this dict will be marked as dict field but not nested model.\n"
+                 "Note: ^ and $ tokens will be added automatically but you have to\n"
+                 "escape other special characters manually.\n"
+        )
+        parser.add_argument(
+            "--dict-keys-fields", "--dkf",
+            nargs="+", metavar="FIELD NAME",
+            help="List of model fields names that will be marked as dict fields\n\n"
+        )
+
         default_percent = f"{ModelFieldsPercentMatch.DEFAULT * 100:.0f}"
         default_number = f"{ModelFieldsNumberMatch.DEFAULT:.0f}"
         parser.add_argument(

json_to_models/generator.py

@@ -182,6 +182,12 @@ def optimize_type(self, meta: MetaData, process_model_ptr=False) -> MetaData:
         elif isinstance(meta, DUnion):
             return self._optimize_union(meta)
 
+        elif isinstance(meta, DOptional):
+            t = self.optimize_type(meta.type)
+            if isinstance(t, DOptional):
+                t = t.type
+            return meta.replace(t)
+
         elif isinstance(meta, SingleType) and (process_model_ptr or not isinstance(meta, ModelPtr)):
             # Optimize nested type
             return meta.replace(self.optimize_type(meta.type))
@@ -193,21 +199,27 @@ def optimize_type(self, meta: MetaData, process_model_ptr=False) -> MetaData:
 
     def _optimize_union(self, t: DUnion):
         # Replace DUnion of 1 element with this element
-        if len(t) == 1:
-            return t.types[0]
+        # if len(t) == 1:
+        #     return t.types[0]
 
         # Split nested types into categories
         str_types: List[Union[type, StringSerializable]] = []
         types_to_merge: List[dict] = []
         list_types: List[DList] = []
+        dict_types: List[DList] = []
         other_types: List[MetaData] = []
         for item in t.types:
+            if isinstance(item, DOptional):
+                item = item.type
+                other_types.append(NoneType)
             if isinstance(item, dict):
                 types_to_merge.append(item)
             elif item in self.str_types_registry or item is str:
                 str_types.append(item)
             elif isinstance(item, DList):
                 list_types.append(item)
+            elif isinstance(item, DDict):
+                dict_types.append(item)
             else:
                 other_types.append(item)
 
@@ -217,10 +229,11 @@ def _optimize_union(self, t: DUnion):
         if types_to_merge:
             other_types.append(self.merge_field_sets(types_to_merge))
 
-        if list_types:
-            other_types.append(DList(DUnion(*(
-                t.type for t in list_types
-            ))))
+        for cls, iterable_types in ((DList, list_types), (DDict, dict_types)):
+            if iterable_types:
+                other_types.append(cls(DUnion(*(
+                    t.type for t in iterable_types
+                ))))
 
         if str in str_types:
             other_types.append(str)
@@ -234,14 +247,20 @@ def _optimize_union(self, t: DUnion):
         if Unknown in types:
             types.remove(Unknown)
 
-        if len(types) > 1:
-            if NoneType in types:
+        optional = False
+        if NoneType in types:
+            optional = True
+            while NoneType in types:
                 types.remove(NoneType)
-                if len(types) > 1:
-                    return DOptional(DUnion(*types))
-                else:
-                    return DOptional(types[0])
-            return DUnion(*types)
 
+        if len(types) > 1:
+            meta_type = DUnion(*types)
+            if len(meta_type.types) == 1:
+                meta_type = meta_type.types[0]
+        else:
+            meta_type = types[0]
+
+        if optional:
+            return DOptional(meta_type)
         else:
-            return types[0]
+            return meta_type

json_to_models/registry.py

@@ -162,10 +162,16 @@ def merge_models(self, generator, strict=False) -> List[Tuple[ModelMeta, Set[Mod
                 groups = new_groups
 
         replaces = []
+        replaces_ids = set()
         for group in groups:
             model_meta = self._merge(generator, *group)
             generator.optimize_type(model_meta)
+            replaces_ids.add(model_meta.index)
             replaces.append((model_meta, group))
+
+        for model_meta in self.models:
+            if model_meta.index not in replaces_ids:
+                generator.optimize_type(model_meta)
         return replaces
 
     def _merge(self, generator, *models: ModelMeta):

test/test_cli/test_script.py

@@ -62,11 +62,11 @@ def test_help():
                                   -l User - "{test_data_path / 'users.json'}" """,
                  id="list1_list2"),
 
-    pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" """, id="gists"),
-    pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --datetime""", id="gists_datetime"),
-    pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --merge percent number_10""",
+    pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files""", id="gists"),
+    pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files --datetime""", id="gists_datetime"),
+    pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files --merge percent number_10""",
                  id="gists_merge_policy"),
-    pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --merge exact""",
+    pytest.param(f"""{executable} -m Gist "{tmp_path / '*.gist'}" --dkf files --merge exact""",
                  id="gists_no_merge"),
 ]
 

test/test_generator/test_optimize_type.py

@@ -1,7 +1,7 @@
 import pytest
 
-from json_to_models.dynamic_typing import (BooleanString, DList, DOptional, DTuple, DUnion, FloatString, IntString,
-                                           NoneType, Unknown)
+from json_to_models.dynamic_typing import (BooleanString, DDict, DList, DOptional, DTuple, DUnion, FloatString,
+                                           IntString, NoneType, Unknown)
 from json_to_models.generator import MetadataGenerator
 
 # MetaData | Optimized MetaData
@@ -106,6 +106,26 @@
         DList(DUnion(str, int)),
         id="union_of_str_int_FloatString"
     ),
+    pytest.param(
+        DOptional(DUnion(DOptional(str), str)),
+        DOptional(str),
+        id="optional_union_nested"
+    ),
+    pytest.param(
+        DUnion(NoneType, str, NoneType),
+        DOptional(str),
+        id="optional_str"
+    ),
+    pytest.param(
+        DUnion(DDict(str), DDict(str), DDict(str)),
+        DDict(str),
+        id="dict_union"
+    ),
+    pytest.param(
+        DUnion(DDict(str), DDict(int), DDict(str)),
+        DDict(DUnion(str, int)),
+        id="dict_union_2"
+    ),
 ]