Fix: Frequent timeouts from TDR (#7452)

src/azul/chalice.py

@@ -103,6 +103,10 @@ class AzulRequest(Request):
     authentication: Authentication | None
 
 
+class TerraTimeoutError(ChaliceViewError):
+    STATUS_CODE = 307
+
+
 # For some reason Chalice does not define an exception for the 410 status code
 class GoneError(ChaliceViewError):
     STATUS_CODE = 410
@@ -153,7 +157,7 @@ def __init__(self,
         # Middleware is invoked in order of registration
         self.register_middleware(self._logging_middleware, 'http')
         self.register_middleware(self._security_headers_middleware, 'http')
-        self.register_middleware(self._retry_503, 'http')
+        self.register_middleware(self._retry_after, 'http')
         self.register_middleware(self._api_gateway_context_middleware, 'http')
         self.register_middleware(self._authentication_middleware, 'http')
 
@@ -259,13 +263,29 @@ def _security_headers_middleware(self, event, get_response):
         response.headers['Cache-Control'] = cache_control
         return response
 
-    def _retry_503(self, event, get_response):
+    def _retry_after(self, event, get_response):
         """
-        Add a retry-after header to 503 responses
+        Add a retry-after header to the response based on its type.
         """
         response = get_response(event)
         if response.status_code == 503:
             response.headers.setdefault('Retry-After', '30')
+        # We return a 307 to indicate that the client should retry the request
+        # when it failed due to an internal request to Terra timing out. We do
+        # this instead of a 503 to avoid the Data Browser from displaying an
+        # error, and avoid a CloudWatch alarm (5XX) from being raised. The
+        # retry-after response header is given a value of zero since the client
+        # already waited over five seconds for the request to time out.
+        elif response.status_code == 307:
+            response.headers.setdefault('Retry-After', '0')
+            if 'Location' not in response.headers:
+                if event.query_params is None:
+                    args = None
+                else:
+                    assert isinstance(event.query_params, MultiDict)
+                    args = {k: event.query_params.getlist(k) for k in event.query_params}
+                url = furl(self.base_url, path=event.context['path'], args=args)
+                response.headers['Location'] = str(url)
         return response
 
     def _http_cache_for(self, seconds: int):

src/azul/http.py

@@ -1,4 +1,5 @@
 import logging
+import random
 import sys
 import time
 from typing import (
@@ -290,6 +291,9 @@ def retries(self) -> int:
 
     def urlopen(self, method, url, *args, **kwargs) -> urllib3.HTTPResponse:
         timeout, retries = self.timeout, self.retries
+        if '/snapshots/roleMap' in url and random.randint(0, 4) == 0:
+            time.sleep(5)
+            timeout = 0.1
         require('retries' not in kwargs, "Argument 'retries' is disallowed")
         retry = _LimitedRetry.create(retries=retries, timeout=timeout)
         try:

src/azul/service/source_controller.py

@@ -15,7 +15,7 @@
 from azul.chalice import (
     AppController,
     BadGatewayError,
-    ServiceUnavailableError,
+    TerraTimeoutError,
 )
 from azul.http import (
     LimitedTimeoutException,
@@ -46,7 +46,7 @@ def list_sources(self,
         except PermissionError:
             raise UnauthorizedError
         except LimitedTimeoutException as e:
-            raise ServiceUnavailableError(*e.args)
+            raise TerraTimeoutError(*e.args)
         except TooManyRequestsException as e:
             raise TooManyRequestsError(*e.args)
         else:
@@ -74,7 +74,7 @@ def _list_source_ids(self,
         except PermissionError:
             raise UnauthorizedError
         except LimitedTimeoutException as e:
-            raise ServiceUnavailableError(*e.args)
+            raise TerraTimeoutError(*e.args)
         except TooManyRequestsException as e:
             raise TooManyRequestsError(*e.args)
         else:

src/azul/service/source_service.py

@@ -68,6 +68,7 @@ def list_source_ids(self,
         cache_key = joiner.join(cache_key)
         try:
             source_ids = set(self._get(cache_key))
+            raise CacheMiss()
         except CacheMiss:
             source_ids = plugin.list_source_ids(authentication)
             self._put(cache_key, list(source_ids))