Skip to content

Commit da1f350

Browse files
authored
Merge pull request #7 from bogdandm/datetime_parsing
Datetime parsing
2 parents 194fc4c + 44007a4 commit da1f350

File tree

11 files changed

+288
-12
lines changed

11 files changed

+288
-12
lines changed

TODO.md

Lines changed: 8 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -32,10 +32,10 @@
3232
- [ ] Complex python types annotations
3333
- [ ] Decorator to specify field metatype
3434
- [ ] Specify metatype in attr/dataclass argument (if dataclasses has such)
35-
- [ ] String based types
36-
- [ ] ISO date
37-
- [ ] ISO time
38-
- [ ] ISO datetime
35+
- [ ] String based types (Warning: 6 times slow down)
36+
- [X] ISO date
37+
- [X] ISO time
38+
- [X] ISO datetime
3939
- API Layer
4040
- [ ] Route object
4141
- [ ] Register model as route in/out data spec
@@ -68,6 +68,10 @@
6868
- Generate OpenAPI spec
6969
- [ ] Meta-model -> OpenAPI model converter
7070
- [ ] Route -> OpenAPI converter
71+
- [ ] String based types
72+
- [ ] ISO date
73+
- [ ] ISO time
74+
- [ ] ISO datetime
7175

7276
- Build, Deploy, CI
7377
- [ ] setup.py

rest_client_gen/dynamic_typing/__init__.py

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
)
44
from .complex import ComplexType, DList, DOptional, DTuple, DUnion, SingleType
55
from .models_meta import AbsoluteModelRef, ModelMeta, ModelPtr
6+
from .string_datetime import IsoDateString, IsoDatetimeString, IsoTimeString, register_datetime_classes
67
from .string_serializable import (
78
BooleanString, FloatString, IntString, StringSerializable, StringSerializableRegistry, registry
89
)
Lines changed: 137 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,137 @@
1+
import operator
2+
from datetime import date, datetime, time
3+
from typing import Any, Optional, Type, Union
4+
5+
import dateutil.parser
6+
7+
from .string_serializable import StringSerializable, StringSerializableRegistry, registry
8+
9+
_dt_args_getter = operator.attrgetter('year', 'month', 'day', 'hour', 'minute', 'second', 'microsecond', 'tzinfo')
10+
_d_args_getter = operator.attrgetter('year', 'month', 'day')
11+
_t_args_getter = operator.attrgetter('hour', 'minute', 'second', 'microsecond', 'tzinfo')
12+
13+
14+
def extend_datetime(d: Union[date, time, datetime], cls: Union[Type[date], Type[time], Type[datetime]]) -> Any:
15+
"""
16+
Wrap datetime object into datetime subclass
17+
18+
:param d: date/time/datetime instance
19+
:param cls: datetime subclass
20+
:return:
21+
"""
22+
if isinstance(d, datetime):
23+
args = _dt_args_getter
24+
elif isinstance(d, time):
25+
args = _t_args_getter
26+
else:
27+
args = _d_args_getter
28+
return cls(*args(d))
29+
30+
31+
_check_values_date = (
32+
datetime(2018, 1, 2, 0, 4, 5, 678, tzinfo=None),
33+
datetime(2018, 1, 2, 9, 4, 5, 678, tzinfo=None)
34+
)
35+
36+
37+
def is_date(s: str) -> Optional[date]:
38+
"""
39+
Return date instance if given string is a date and None otherwise
40+
41+
:param s: string
42+
:return: date or None
43+
"""
44+
# dateutil.parser.parse replaces missing parts of datetime with values from default value
45+
# so if there is hour part in given string then d1 and d2 would be equal and string is not pure date
46+
d1 = dateutil.parser.parse(s, default=_check_values_date[0])
47+
d2 = dateutil.parser.parse(s, default=_check_values_date[1])
48+
return None if d1 == d2 else d1.date()
49+
50+
51+
_check_values_time = (
52+
datetime(2018, 10, 11),
53+
datetime(2018, 12, 30)
54+
)
55+
56+
57+
def is_time(s: str) -> Optional[time]:
58+
"""
59+
Return time instance if given string is a time and None otherwise
60+
61+
:param s: string
62+
:return: time or None
63+
"""
64+
d1 = dateutil.parser.parse(s, default=_check_values_time[0])
65+
d2 = dateutil.parser.parse(s, default=_check_values_time[1])
66+
return None if d1 == d2 else d1.time()
67+
68+
69+
class IsoDateString(StringSerializable, date):
70+
"""
71+
Parse date using dateutil.parser.isoparse. Representation format always is ``YYYY-MM-DD``.
72+
You can override to_representation method to customize it. Just don't forget to call registry.remove(IsoDateString)
73+
"""
74+
75+
@classmethod
76+
def to_internal_value(cls, value: str) -> 'IsoDateString':
77+
if not is_date(value):
78+
raise ValueError(f"'{value}' is not valid date")
79+
dt = dateutil.parser.isoparse(value)
80+
return extend_datetime(dt.date(), cls)
81+
82+
def to_representation(self):
83+
return self.isoformat()
84+
85+
def replace(self, *args, **kwargs) -> 'IsoDateString':
86+
# noinspection PyTypeChecker
87+
return date.replace(self, *args, **kwargs)
88+
89+
90+
class IsoTimeString(StringSerializable, time):
91+
"""
92+
Parse time using dateutil.parser.parse. Representation format always is ``hh:mm:ss.ms``.
93+
You can override to_representation method to customize it.
94+
"""
95+
96+
@classmethod
97+
def to_internal_value(cls, value: str) -> 'IsoTimeString':
98+
t = is_time(value)
99+
if not t:
100+
raise ValueError(f"'{value}' is not valid time")
101+
return extend_datetime(t, cls)
102+
103+
def to_representation(self):
104+
return self.isoformat()
105+
106+
def replace(self, *args, **kwargs) -> 'IsoTimeString':
107+
# noinspection PyTypeChecker
108+
return time.replace(self, *args, **kwargs)
109+
110+
111+
class IsoDatetimeString(StringSerializable, datetime):
112+
"""
113+
Parse datetime using dateutil.parser.isoparse.
114+
Representation format always is ``YYYY-MM-DDThh:mm:ss.ms`` (datetime.isoformat method).
115+
"""
116+
117+
@classmethod
118+
def to_internal_value(cls, value: str) -> 'IsoDatetimeString':
119+
dt = dateutil.parser.isoparse(value)
120+
return extend_datetime(dt, cls)
121+
122+
def to_representation(self):
123+
return self.isoformat()
124+
125+
def replace(self, *args, **kwargs) -> 'IsoDatetimeString':
126+
# noinspection PyTypeChecker
127+
return datetime.replace(self, *args, **kwargs)
128+
129+
130+
def register_datetime_classes(registry: StringSerializableRegistry = registry):
131+
"""
132+
Register datetime classes in given registry (using default registry if no arguments is passed).
133+
Date parsing is expensive operation so this classes are disabled by default
134+
"""
135+
registry.add(cls=IsoDateString)
136+
registry.add(cls=IsoTimeString)
137+
registry.add(cls=IsoDatetimeString)

rest_client_gen/dynamic_typing/string_serializable.py

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,7 @@ def to_typing_code(cls) -> Tuple[ImportPathList, str]:
3434
as a metadata instance but contains actual data
3535
"""
3636
cls_name = cls.__name__
37-
return [('rest_client_gen.dynamic_typing.string_serializable', cls_name)], cls_name
37+
return [('rest_client_gen.dynamic_typing', cls_name)], cls_name
3838

3939

4040
T_StringSerializable = Type[StringSerializable]
@@ -71,6 +71,17 @@ def decorator(cls):
7171

7272
return decorator
7373

74+
def remove(self, cls: type):
75+
"""
76+
Unregister given class
77+
78+
:param cls: StringSerializable class
79+
"""
80+
self.types.remove(cls)
81+
for base, replace in list(self.replaces):
82+
if replace is cls or base is cls:
83+
self.replaces.remove((base, replace))
84+
7485
def resolve(self, *types: T_StringSerializable) -> Collection[T_StringSerializable]:
7586
"""
7687
Return set of StringSerializable classes which can represent all classes from types argument.

test/test_code_generation/test_attrs_generation.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ class Test:
119119
"generated": trim(f"""
120120
import attr
121121
from attr.converter import optional
122-
from rest_client_gen.dynamic_typing.string_serializable import FloatString, IntString
122+
from rest_client_gen.dynamic_typing import FloatString, IntString
123123
from typing import List, Optional
124124
125125

test/test_code_generation/test_models_code_generator.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -118,7 +118,7 @@ class Test:
118118
}
119119
},
120120
"fields": {
121-
"imports": "from rest_client_gen.dynamic_typing.string_serializable import IntString\n"
121+
"imports": "from rest_client_gen.dynamic_typing import IntString\n"
122122
"from typing import List, Optional",
123123
"fields": [
124124
"foo: int",
@@ -127,7 +127,7 @@ class Test:
127127
]
128128
},
129129
"generated": trim("""
130-
from rest_client_gen.dynamic_typing.string_serializable import IntString
130+
from rest_client_gen.dynamic_typing import IntString
131131
from typing import List, Optional
132132
133133

test/test_code_generation/test_typing.py

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -104,12 +104,12 @@ class TestModel:
104104
),
105105
pytest.param(
106106
FloatString,
107-
('from rest_client_gen.dynamic_typing.string_serializable import FloatString', FloatString),
107+
('from rest_client_gen.dynamic_typing import FloatString', FloatString),
108108
id="string_serializable"
109109
),
110110
pytest.param(
111111
DOptional(IntString),
112-
('from rest_client_gen.dynamic_typing.string_serializable import IntString\n'
112+
('from rest_client_gen.dynamic_typing import IntString\n'
113113
'from typing import Optional', Optional[IntString]),
114114
id="complex_string_serializable"
115115
),
Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,94 @@
1+
import datetime
2+
3+
import pytest
4+
5+
from rest_client_gen.dynamic_typing import (
6+
FloatString, IntString, IsoDateString, IsoDatetimeString, IsoTimeString, register_datetime_classes
7+
)
8+
from rest_client_gen.generator import MetadataGenerator
9+
10+
register_datetime_classes()
11+
12+
test_detect_type_data = [
13+
# Check that string datetime doesn't break default string types
14+
pytest.param(
15+
"1",
16+
IntString,
17+
id="default_check_int"
18+
),
19+
pytest.param(
20+
"1.5",
21+
FloatString,
22+
id="default_check_float"
23+
),
24+
pytest.param(
25+
"2018-12-31",
26+
IsoDateString,
27+
id="date"
28+
),
29+
pytest.param(
30+
"12:58",
31+
IsoTimeString,
32+
id="time"
33+
),
34+
pytest.param(
35+
"2018-12-31T12:58:12Z",
36+
IsoDatetimeString,
37+
id="datetime"
38+
)
39+
]
40+
41+
42+
@pytest.mark.parametrize("value,expected", test_detect_type_data)
43+
def test_detect_type(models_generator: MetadataGenerator, value, expected):
44+
result = models_generator._detect_type(value)
45+
assert result == expected
46+
47+
48+
test_parse_data = [
49+
pytest.param(
50+
"2018-12-31",
51+
IsoDateString(2018, 12, 31),
52+
id="date"
53+
),
54+
pytest.param(
55+
"12:13",
56+
IsoTimeString(12, 13),
57+
id="time"
58+
),
59+
pytest.param(
60+
"04:15:34",
61+
IsoTimeString(4, 15, 34),
62+
id="time_seconds"
63+
),
64+
pytest.param(
65+
"04:15:34.034",
66+
IsoTimeString(4, 15, 34, 34000),
67+
id="time_ms"
68+
),
69+
pytest.param(
70+
"2018-12-04T04:15:34.034000+00:00",
71+
IsoDatetimeString(2018, 12, 4, 4, 15, 34, 34000, tzinfo=datetime.timezone.utc),
72+
id="datetime_full"
73+
),
74+
pytest.param(
75+
"2018-12-04T04:15",
76+
IsoDatetimeString(2018, 12, 4, 4, 15),
77+
id="datetime_partial"
78+
)
79+
]
80+
81+
82+
@pytest.mark.parametrize("value,expected", test_parse_data)
83+
def test_parse(models_generator: MetadataGenerator, value, expected):
84+
cls = models_generator._detect_type(value)
85+
result = cls.to_internal_value(value)
86+
assert result == expected
87+
assert value in result.to_representation()
88+
89+
90+
def test_replace():
91+
assert IsoTimeString(14, 12, 57).replace(minute=58, second=32) == IsoTimeString(14, 58, 32)
92+
assert IsoDateString(2014, 12, 5).replace(day=4, month=5) == IsoDateString(2014, 5, 4)
93+
assert IsoDatetimeString(2014, 12, 5, 14, 12, 57).replace(minute=58, second=32, day=4, month=5) \
94+
== IsoDatetimeString(2014, 5, 4, 14, 58, 32)

test/test_dynamic_typing/test_string_serializable_registry.py

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,9 @@
11
import pytest
22

3-
from rest_client_gen.dynamic_typing.string_serializable import StringSerializable, StringSerializableRegistry
3+
from rest_client_gen.dynamic_typing import IsoTimeString
4+
from rest_client_gen.dynamic_typing.string_serializable import (FloatString, IntString, StringSerializable,
5+
StringSerializableRegistry)
6+
from rest_client_gen.generator import MetadataGenerator
47

58
r = StringSerializableRegistry()
69

@@ -54,7 +57,30 @@ class Y(StringSerializable):
5457
pytest.param((X, B), {B, X}),
5558
]
5659

60+
5761
@pytest.mark.parametrize("value,expected", test_data)
5862
def test_string_serializable_registry(value, expected):
5963
result = r.resolve(*value)
60-
assert result == expected
64+
assert result == expected
65+
66+
67+
r2 = StringSerializableRegistry()
68+
gen = MetadataGenerator(r2)
69+
70+
r2.add(cls=IsoTimeString)
71+
r2.add(cls=IntString)
72+
r2.add(replace_types=(IntString,), cls=FloatString)
73+
74+
75+
@pytest.mark.xfail()
76+
def test_without_remove():
77+
assert gen._detect_type("12") == IntString
78+
79+
80+
r2.remove(IsoTimeString)
81+
r2.add(cls=IsoTimeString)
82+
83+
84+
def test_remove():
85+
assert gen._detect_type("12") == IntString
86+
assert gen._detect_type("12:14") == IsoTimeString

testing_tools/real_apis/f1.py

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import inflection
66
import requests
77

8+
from rest_client_gen.dynamic_typing import register_datetime_classes
89
from rest_client_gen.generator import MetadataGenerator
910
from rest_client_gen.models import compose_models
1011
from rest_client_gen.models.attr import AttrsModelCodeGenerator
@@ -43,6 +44,7 @@ def main():
4344
dump_response("f1", "driver_standings", driver_standings_data)
4445
driver_standings_data = ("driver_standings", driver_standings_data)
4546

47+
register_datetime_classes()
4648
gen = MetadataGenerator()
4749
reg = ModelRegistry()
4850
for name, data in (results_data, drivers_data, driver_standings_data):

0 commit comments

Comments
 (0)