Merge pull request #17 from bogdandm/decode-unicode-keys

Decode unicode and remove non-words characters in models fields names

Author: bogdandm

TODO.md

@@ -22,7 +22,6 @@
             - [ ] dataclasses
             - [ ] Decorator to mark class as exclude from models merge
         - Other features
-            - [ ] Decode unicode in keys
             - [ ] Nesting models generation
                 - [X] Cascade (default)
                 - [X] Flat
@@ -38,6 +37,7 @@
                 - [X] ISO time
                 - [X] ISO datetime
             - [X] Don't create metadata (J2M_ORIGINAL_FIELD) if original_field == generated_field
+            - [X] Decode unicode in keys
     - [X] Cli tool
 
 - Testing

json_to_models/generator.py

@@ -1,4 +1,3 @@
-import keyword
 import re
 from collections import OrderedDict
 from typing import Any, Callable, List, Optional, Pattern, Union
@@ -8,7 +7,6 @@
 from .dynamic_typing import (ComplexType, DDict, DList, DOptional, DUnion, MetaData, ModelPtr, Null, SingleType,
                              StringSerializable, StringSerializableRegistry, Unknown, registry)
 
-keywords_set = set(keyword.kwlist)
 _static_types = {float, bool, int}
 
 class MetadataGenerator:
@@ -51,8 +49,6 @@ def _convert(self, data: dict):
             #   Crash does not produce any useful logs and can occur any time after bad string was processed
             #   It can be reproduced on real_apis tests (openlibrary API)
             convert_dict = key not in self.dict_keys_fields
-            if key in keywords_set:
-                key += "_"
             fields[key] = self._detect_type(value if not isinstance(value, str) else unidecode(value), convert_dict)
         return fields
 

json_to_models/models/base.py

@@ -1,7 +1,10 @@
+import keyword
+import re
 from typing import Iterable, List, Tuple, Type
 
 import inflection
 from jinja2 import Template
+from unidecode import unidecode
 
 from . import INDENT, ModelsStructureType, OBJECTS_DELIMITER, indent, sort_fields
 from ..dynamic_typing import AbsoluteModelRef, ImportPathList, MetaData, ModelMeta, compile_imports, metadata_to_typing
@@ -12,6 +15,8 @@
                   "{% if not loop.last %}, {% endif %}" \
                   "{% endfor %}"
 
+keywords_set = set(keyword.kwlist)
+ones = ['', 'one', 'two', 'three', 'four', 'five', 'six', 'seven', 'eight', 'nine']
 
 def template(pattern: str, indent: str = INDENT) -> Template:
     """
@@ -80,6 +85,17 @@ def decorators(self) -> List[str]:
         """
         return []
 
+    @classmethod
+    def convert_field_name(cls, name):
+        if name in keywords_set:
+            name += "_"
+        name = unidecode(name)
+        name = re.sub(r"\W", "", name)
+        if not ('a' <= name[0].lower() <= 'z'):
+            if '0' <= name[0] <= '9':
+                name = ones[int(name[0])] + "_" + name[1:]
+        return inflection.underscore(name)
+
     def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportPathList, dict]:
         """
         Form field data for template
@@ -90,8 +106,9 @@ def field_data(self, name: str, meta: MetaData, optional: bool) -> Tuple[ImportP
         :return: imports, field data
         """
         imports, typing = metadata_to_typing(meta)
+
         data = {
-            "name": inflection.underscore(name),
+            "name": self.convert_field_name(name),
             "type": typing
         }
         return imports, data

test/test_code_generation/test_attrs_generation.py

@@ -88,38 +88,56 @@ class Test:
             "bar": DOptional(IntString),
             "qwerty": FloatString,
             "asdfg": DOptional(int),
-            "dict": DDict(int)
+            "dict": DDict(int),
+            "not": bool,
+            "1day": int,
+            "день_недели": str,
         }),
         "fields_data": {
             "foo": {
                 "name": "foo",
                 "type": "int",
-                "body": f"attr.ib()"
+                "body": "attr.ib()"
             },
             "baz": {
                 "name": "baz",
                 "type": "Optional[List[List[str]]]",
-                "body": f"attr.ib(factory=list)"
+                "body": "attr.ib(factory=list)"
             },
             "bar": {
                 "name": "bar",
                 "type": "Optional[IntString]",
-                "body": f"attr.ib(default=None, converter=optional(IntString))"
+                "body": "attr.ib(default=None, converter=optional(IntString))"
             },
             "qwerty": {
                 "name": "qwerty",
                 "type": "FloatString",
-                "body": f"attr.ib(converter=FloatString)"
+                "body": "attr.ib(converter=FloatString)"
             },
             "asdfg": {
                 "name": "asdfg",
                 "type": "Optional[int]",
-                "body": f"attr.ib(default=None)"
+                "body": "attr.ib(default=None)"
             },
             "dict": {
                 "name": "dict",
                 "type": "Dict[str, int]",
-                "body": f"attr.ib()"
+                "body": "attr.ib()"
+            },
+            "not": {
+                "name": "not_",
+                "type": "bool",
+                "body": f"attr.ib({field_meta('not')})"
+            },
+            "1day": {
+                "name": "one_day",
+                "type": "int",
+                "body": f"attr.ib({field_meta('1day')})"
+            },
+            "день_недели": {
+                "name": "den_nedeli",
+                "type": "str",
+                "body": f"attr.ib({field_meta('день_недели')})"
             }
         },
         "generated": trim(f"""
@@ -134,6 +152,9 @@ class Test:
             foo: int = attr.ib()
             qwerty: FloatString = attr.ib(converter=FloatString)
             dict: Dict[str, int] = attr.ib()
+            not_: bool = attr.ib({field_meta('not')})
+            one_day: int = attr.ib({field_meta('1day')})
+            den_nedeli: str = attr.ib({field_meta('день_недели')})
             baz: Optional[List[List[str]]] = attr.ib(factory=list)
             bar: Optional[IntString] = attr.ib(default=None, converter=optional(IntString))
             asdfg: Optional[int] = attr.ib(default=None)

test/test_generator/test_detect_type.py

@@ -53,5 +53,5 @@ def test_convert(models_generator: MetadataGenerator):
         "another_dict_field_2": DDict(int),
         "another_dict_field_3": DDict(int),
         "int_field": int,
-        "not_": bool
+        "not": bool
     }