Use moto's server mode for unit tests

[hannes-ucsc]

A few years ago, when botocore switched from Requests to using urllib3 directly, Moto responded to that change by switching from Responses to patching botocore directly. These patches did not cover signed URLs so Requests and Responses had to be kept around.

Since then Moto added Server mode, a less invasive and more elegant means of emulating the AWS API. It requires no patching and supports signed URLs natively.

Here's a patch for a single test case in Azul. It is very crude and incomplete but it proves the concept.

Index: test/service/test_storage_service.py
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/test/service/test_storage_service.py b/test/service/test_storage_service.py
--- a/test/service/test_storage_service.py	(revision fcc42a78247b2c0dd32c1cb79850839577a71240)
+++ b/test/service/test_storage_service.py	(date 1766969499911)
@@ -3,16 +3,23 @@
     timedelta,
     timezone,
 )
+import logging
 import tempfile
 from unittest.mock import (
     patch,
 )
 
-import requests
+from moto.moto_server.threaded_moto_server import (
+    ThreadedMotoServer,
+)
+from urllib3 import HTTPResponse
 
 from azul import (
     false,
 )
+from azul.http import (
+    http_client,
+)
 from azul.logging import (
     configure_test_logging,
 )
@@ -26,6 +33,8 @@
     StorageServiceTestCase,
 )
 
+log = logging.getLogger(__name__)
+
 
 # noinspection PyPep8Naming
 def setUpModule():
@@ -37,6 +46,17 @@
     Functional Test for Storage Service
     """
 
+    @classmethod
+    def setUpClass(cls):
+        super().setUpClass()
+        cls.moto_server = ThreadedMotoServer()
+        cls.moto_server.start()
+
+    @classmethod
+    def tearDownClass(cls):
+        cls.moto_server.stop()
+        super().tearDownClass()
+
     def test_upload_tags(self):
         object_key = 'test_file'
         with tempfile.NamedTemporaryFile('w') as f:
@@ -72,12 +92,13 @@
 
     def test_presigned_url(self):
         object_key, object_content = 'foo-presigned-url', b'{"a": 1}'
-        service = self.storage_service
+        service, client = self.storage_service, http_client(log)
         service.put_object(object_key=object_key, data=object_content)
         for file_name in None, 'foo.json':
             with self.subTest(file_name=file_name):
                 url = service.get_presigned_url(object_key, file_name=file_name)
-                response = requests.get(url)
+                response = client.request('GET', url)
+                assert isinstance(response, HTTPResponse)
                 if file_name is None:
                     self.assertNotIn('Content-Disposition', response.headers)
                 else:
@@ -88,7 +109,7 @@
                         # section Request Parameters).
                         self.assertEqual(response.headers['Content-Disposition'],
                                          f'attachment;filename="{file_name}"')
-                self.assertEqual(object_content, response.content)
+                self.assertEqual(object_content, response.data)
 
     def test_time_until_object_expires(self):
         test_data = [(1, False), (0, False), (-1, True)]
Index: src/azul/deployment.py
IDEA additional info:
Subsystem: com.intellij.openapi.diff.impl.patch.CharsetEP
<+>UTF-8
===================================================================
diff --git a/src/azul/deployment.py b/src/azul/deployment.py
--- a/src/azul/deployment.py	(revision fcc42a78247b2c0dd32c1cb79850839577a71240)
+++ b/src/azul/deployment.py	(date 1766968458043)
@@ -530,7 +530,9 @@
                              enable logging of request bodies, which could
                              contain sensitive or secret information.
         """
-        client = self.boto3_session.client(service_name, region_name=region_name)
+        client = self.boto3_session.client(service_name,
+                                           region_name=region_name,
+                                           endpoint_url='http://localhost:5000')
         log.info('Allocated new Boto3 client for %r with ID %r',
                  service_name, id(client))
         if azul_logging:
@@ -588,7 +590,8 @@
         Same as `self.client()` but for `boto3.resource()` instead of
         `boto3.client()`.
         """
-        resource = self.boto3_session.resource(*args, **kwargs)
+        resource = self.boto3_session.resource(*args, **kwargs,
+                                               endpoint_url='http://localhost:5000')
         if azul_logging:
             events = resource.meta.client.meta.events
             events.register_last(self._request_event_name, self._log_client_request)

The endpoint override in deployment.py should of course only be active during unit tests. And ideally, we would like to ensure that accidental use of a boto3 or botocore client with no endpoint override during unit tests does not modify any resources, or use real credentials. There are probably also complications with resetting Moto's state.