From f029d7b372cf398d305f6182140fabb835009040 Mon Sep 17 00:00:00 2001 From: Youness El idrissi Date: Tue, 13 Apr 2021 09:56:56 +0200 Subject: [PATCH 1/4] Add support for an async defined retry and callback_error_retry --- tenacity/_asyncio.py | 60 ++++++++++++++++++++++++++++++++-- tenacity/tests/test_asyncio.py | 44 +++++++++++++++++++++++-- 2 files changed, 99 insertions(+), 5 deletions(-) diff --git a/tenacity/_asyncio.py b/tenacity/_asyncio.py index 979b6544..8f615b78 100644 --- a/tenacity/_asyncio.py +++ b/tenacity/_asyncio.py @@ -15,10 +15,17 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. +import six + +try: + from inspect import iscoroutinefunction +except ImportError: + iscoroutinefunction = None + import sys from asyncio import sleep -from tenacity import AttemptManager +from tenacity import AttemptManager, TryAgain, RetryAction from tenacity import BaseRetrying from tenacity import DoAttempt from tenacity import DoSleep @@ -30,12 +37,58 @@ def __init__(self, sleep=sleep, **kwargs): super(AsyncRetrying, self).__init__(**kwargs) self.sleep = sleep + async def iter(self, retry_state): # noqa + fut = retry_state.outcome + if fut is None: + if self.before is not None: + self.before(retry_state) + return DoAttempt() + + is_explicit_retry = retry_state.outcome.failed and isinstance( + retry_state.outcome.exception(), TryAgain + ) + if iscoroutinefunction(self.retry): + should_retry = await self.retry(retry_state=retry_state) + else: + should_retry = self.retry(retry_state=retry_state) + if not (is_explicit_retry or should_retry): + return fut.result() + + if self.after is not None: + self.after(retry_state=retry_state) + + self.statistics["delay_since_first_attempt"] = retry_state.seconds_since_start + if self.stop(retry_state=retry_state): + if self.retry_error_callback: + if iscoroutinefunction(self.retry_error_callback): + return await self.retry_error_callback(retry_state=retry_state) + else: + return self.retry_error_callback(retry_state=retry_state) + retry_exc = self.retry_error_cls(fut) + if self.reraise: + raise retry_exc.reraise() + six.raise_from(retry_exc, fut.exception()) + + if self.wait: + iteration_sleep = self.wait(retry_state=retry_state) + else: + iteration_sleep = 0.0 + retry_state.next_action = RetryAction(iteration_sleep) + retry_state.idle_for += iteration_sleep + self.statistics["idle_for"] += iteration_sleep + self.statistics["attempt_number"] += 1 + + if self.before_sleep is not None: + self.before_sleep(retry_state=retry_state) + + return DoSleep(iteration_sleep) + async def __call__(self, fn, *args, **kwargs): self.begin(fn) retry_state = RetryCallState(retry_object=self, fn=fn, args=args, kwargs=kwargs) while True: - do = self.iter(retry_state=retry_state) + do = await self.iter(retry_state=retry_state) if isinstance(do, DoAttempt): try: result = await fn(*args, **kwargs) @@ -56,7 +109,7 @@ def __aiter__(self): async def __anext__(self): while True: - do = self.iter(retry_state=self._retry_state) + do = await self.iter(retry_state=self._retry_state) if do is None: raise StopAsyncIteration elif isinstance(do, DoAttempt): @@ -69,6 +122,7 @@ async def __anext__(self): def wraps(self, fn): fn = super().wraps(fn) + # Ensure wrapper is recognized as a coroutine function. async def async_wrapped(*args, **kwargs): diff --git a/tenacity/tests/test_asyncio.py b/tenacity/tests/test_asyncio.py index 2057fd2d..c477c999 100644 --- a/tenacity/tests/test_asyncio.py +++ b/tenacity/tests/test_asyncio.py @@ -124,6 +124,46 @@ def after(retry_state): assert len(set(things)) == 1 assert list(attempt_nos2) == [1, 2, 3] + @asynctest + async def test_async_retry(self): + attempts = [] + + async def async_retry(retry_state): + if retry_state.outcome.failed: + attempts.append((retry_state.outcome, retry_state.attempt_number)) + return True + else: + attempts.append((retry_state.outcome, retry_state.attempt_number)) + return False + + thing = NoIOErrorAfterCount(2) + + await _retryable_coroutine.retry_with(retry=async_retry)(thing) + + things, attempt_numbers = zip(*attempts) + assert len(attempts) == 3 + + for thing in things[:-1]: + with pytest.raises(IOError): + thing.result() + + assert things[-1].result() is True + + @asynctest + async def test_async_callback_error_retry(self): + async def async_return_text(retry_state): + await asyncio.sleep(0.00001) + return "Calling %s keeps raising errors after %s attempts" % ( + retry_state.fn.__name__, + retry_state.attempt_number, + ) + + thing = NoIOErrorAfterCount(3) + + result = await _retryable_coroutine_with_2_attempts.retry_with(retry_error_callback=async_return_text)(thing) + + assert result == "Calling _retryable_coroutine_with_2_attempts keeps raising errors after 2 attempts" + class TestContextManager(unittest.TestCase): @asynctest @@ -147,7 +187,7 @@ class CustomError(Exception): try: async for attempt in tasyncio.AsyncRetrying( - stop=stop_after_attempt(1), reraise=True + stop=stop_after_attempt(1), reraise=True ): with attempt: raise CustomError() @@ -161,7 +201,7 @@ async def test_sleeps(self): start = current_time_ms() try: async for attempt in tasyncio.AsyncRetrying( - stop=stop_after_attempt(1), wait=wait_fixed(1) + stop=stop_after_attempt(1), wait=wait_fixed(1) ): with attempt: raise Exception() From f412b4126bbc1c36cd24ed911c2a04bb47441f86 Mon Sep 17 00:00:00 2001 From: Youness El idrissi Date: Thu, 15 Apr 2021 11:44:46 +0200 Subject: [PATCH 2/4] Fix pep 8 - BLK100 error - import order errors --- tenacity/_asyncio.py | 6 +++--- tenacity/tests/test_asyncio.py | 10 +++++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/tenacity/_asyncio.py b/tenacity/_asyncio.py index 8f615b78..6991621d 100644 --- a/tenacity/_asyncio.py +++ b/tenacity/_asyncio.py @@ -15,8 +15,6 @@ # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. -import six - try: from inspect import iscoroutinefunction except ImportError: @@ -25,7 +23,9 @@ import sys from asyncio import sleep -from tenacity import AttemptManager, TryAgain, RetryAction +import six + +from tenacity import AttemptManager, RetryAction, TryAgain from tenacity import BaseRetrying from tenacity import DoAttempt from tenacity import DoSleep diff --git a/tenacity/tests/test_asyncio.py b/tenacity/tests/test_asyncio.py index c477c999..c6d33cc8 100644 --- a/tenacity/tests/test_asyncio.py +++ b/tenacity/tests/test_asyncio.py @@ -152,7 +152,9 @@ async def async_retry(retry_state): @asynctest async def test_async_callback_error_retry(self): async def async_return_text(retry_state): + await asyncio.sleep(0.00001) + return "Calling %s keeps raising errors after %s attempts" % ( retry_state.fn.__name__, retry_state.attempt_number, @@ -160,9 +162,11 @@ async def async_return_text(retry_state): thing = NoIOErrorAfterCount(3) - result = await _retryable_coroutine_with_2_attempts.retry_with(retry_error_callback=async_return_text)(thing) - - assert result == "Calling _retryable_coroutine_with_2_attempts keeps raising errors after 2 attempts" + result = await _retryable_coroutine_with_2_attempts.retry_with( + retry_error_callback=async_return_text + )(thing) + message = "Calling _retryable_coroutine_with_2_attempts keeps raising errors after 2 attempts" + assert result == message class TestContextManager(unittest.TestCase): From f4f678644780dbf8742010f33b39d35b38d5df0b Mon Sep 17 00:00:00 2001 From: Youness El idrissi Date: Thu, 15 Apr 2021 11:57:28 +0200 Subject: [PATCH 3/4] release notes --- ...defined_retry_callbacks-254eecdc313c52f8.yaml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) create mode 100644 releasenotes/notes/asynchronous_defined_retry_callbacks-254eecdc313c52f8.yaml diff --git a/releasenotes/notes/asynchronous_defined_retry_callbacks-254eecdc313c52f8.yaml b/releasenotes/notes/asynchronous_defined_retry_callbacks-254eecdc313c52f8.yaml new file mode 100644 index 00000000..908ad38e --- /dev/null +++ b/releasenotes/notes/asynchronous_defined_retry_callbacks-254eecdc313c52f8.yaml @@ -0,0 +1,16 @@ +--- +prelude: > + Example use cases: + - if we get logged out from the server + - if authentication tokens expire + We want to be able to automatically refresh our session by calling a specific + function. This function can be asynchronously defined. +features: + - | + Asynchronous defined retry callbacks: + - retry + - retry_error_callback +issues: + - | + #249 + From 0a7e2757a24761e402da83941179c0adb98ccb19 Mon Sep 17 00:00:00 2001 From: Youness El idrissi Date: Thu, 15 Apr 2021 12:02:27 +0200 Subject: [PATCH 4/4] Fix pep8 --- tenacity/tests/test_asyncio.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/tenacity/tests/test_asyncio.py b/tenacity/tests/test_asyncio.py index c6d33cc8..f954ec1d 100644 --- a/tenacity/tests/test_asyncio.py +++ b/tenacity/tests/test_asyncio.py @@ -152,7 +152,6 @@ async def async_retry(retry_state): @asynctest async def test_async_callback_error_retry(self): async def async_return_text(retry_state): - await asyncio.sleep(0.00001) return "Calling %s keeps raising errors after %s attempts" % ( @@ -191,7 +190,7 @@ class CustomError(Exception): try: async for attempt in tasyncio.AsyncRetrying( - stop=stop_after_attempt(1), reraise=True + stop=stop_after_attempt(1), reraise=True ): with attempt: raise CustomError() @@ -205,7 +204,7 @@ async def test_sleeps(self): start = current_time_ms() try: async for attempt in tasyncio.AsyncRetrying( - stop=stop_after_attempt(1), wait=wait_fixed(1) + stop=stop_after_attempt(1), wait=wait_fixed(1) ): with attempt: raise Exception()