VER: Release 0.52.0

.github/workflows/test.yml

@@ -32,11 +32,11 @@ jobs:
       - name: Test (release)
         timeout-minutes: 3
         if: ${{ github.ref == 'refs/heads/main' }}
-        run: scripts/test.sh --release
+        run: scripts/test.sh -vvv --release
         shell: bash
 
       - name: Test
         timeout-minutes: 3
         if: ${{ github.ref != 'refs/heads/main' }}
-        run: scripts/test.sh
+        run: scripts/test.sh -vvv
         shell: bash

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 0.52.0 - 2025-04-15
+
+#### Enhancements
+- Added new optional `id` field to `SubcriptionRequest` class which will be used for improved error messages
+- Upgraded `databento-dbn` to 0.32.0
+  - Fixed `RType` variant names in Python to match `Schema`
+  - Added missing Python type declarations for `RType` variants
+  - Fixed issue with Python `_hidden_fields` definition that caused `KeyError: _reserved1_00`
+    with `CMBP1Msg` and other records with `ConsolidatedBidAskPair`
+
 ## 0.51.0 - 2025-04-08
 
 #### Enhancements

README.md

@@ -32,7 +32,7 @@ The library is fully compatible with the latest distribution of Anaconda 3.9 and
 The minimum dependencies as found in the `pyproject.toml` are also listed below:
 - python = "^3.9"
 - aiohttp = "^3.8.3"
-- databento-dbn = "0.31.0"
+- databento-dbn = "0.32.0"
 - numpy= ">=1.23.5"
 - pandas = ">=1.5.3"
 - pip-system-certs = ">=4.0" (Windows only)

databento/common/dbnstore.py

@@ -1335,7 +1335,7 @@ def __init__(
         count: int | None = None,
     ) -> None:
         self._reader = reader
-        self._dtype: np.typing.DTypeLike = np.dtype(dtype)
+        self._dtype: np.dtype[Any] = np.dtype(dtype)
         self._offset = offset
         self._count = count
         self._close_on_next = False

databento/live/gateway.py

@@ -132,6 +132,7 @@ class SubscriptionRequest(GatewayControl):
     symbols: str
     start: int | None = None
     snapshot: int = 0
+    id: int | None = None
 
 
 @dataclasses.dataclass

databento/live/protocol.py

@@ -284,6 +284,7 @@ def subscribe(
         stype_in: SType | str = SType.RAW_SYMBOL,
         start: str | int | None = None,
         snapshot: bool = False,
+        subscription_id: int | None = None,
     ) -> list[SubscriptionRequest]:
         """
         Send a SubscriptionRequest to the gateway. Returns a list of all
@@ -302,6 +303,8 @@ def subscribe(
             within 24 hours.
         snapshot: bool, default to 'False'
             Request subscription with snapshot. The `start` parameter must be `None`.
+        subscription_id : int, optional
+            A numerical identifier to associate with this subscription.
 
         Returns
         -------
@@ -329,6 +332,7 @@ def subscribe(
                 symbols=batch_str,
                 start=optional_datetime_to_unix_nanoseconds(start),
                 snapshot=int(snapshot),
+                id=subscription_id,
             )
             subscriptions.append(message)
 

databento/live/session.py

@@ -332,6 +332,7 @@ def __init__(
         self._transport: asyncio.Transport | None = None
         self._session_id: str | None = None
 
+        self._subscription_counter = 0
         self._subscriptions: list[SubscriptionRequest] = []
         self._reconnect_policy = ReconnectPolicy(reconnect_policy)
         self._reconnect_task: asyncio.Task[None] | None = None
@@ -499,13 +500,15 @@ def subscribe(
             if self._protocol is None:
                 self._connect(dataset=dataset)
 
+            self._subscription_counter += 1
             self._subscriptions.extend(
                 self._protocol.subscribe(
                     schema=schema,
                     symbols=symbols,
                     stype_in=stype_in,
                     start=start,
                     snapshot=snapshot,
+                    subscription_id=self._subscription_counter,
                 ),
             )
 
@@ -672,6 +675,7 @@ async def _reconnect(self) -> None:
                             stype_in=sub.stype_in,
                             snapshot=bool(sub.snapshot),
                             start=None,
+                            subscription_id=sub.id,
                         )
 
                     if should_restart:

databento/version.py

@@ -1 +1 @@
-__version__ = "0.51.0"
+__version__ = "0.52.0"

pyproject.toml

@@ -1,6 +1,6 @@
 [tool.poetry]
 name = "databento"
-version = "0.51.0"
+version = "0.52.0"
 description = "Official Python client library for Databento"
 authors = [
     "Databento <support@databento.com>",
@@ -32,7 +32,7 @@ aiohttp = [
     {version = "^3.8.3", python = "<3.12"},
     {version = "^3.9.0", python = "^3.12"}
 ]
-databento-dbn = "0.31.0"
+databento-dbn = "0.32.0"
 numpy = [
     {version = ">=1.23.5", python = "<3.12"},
     {version = ">=1.26.0", python = "^3.12"}

tests/data/generator.py

@@ -55,21 +55,24 @@
         (Schema.IMBALANCE, ["NVDA"], "2020-12-28"),
     ],
     Dataset.OPRA_PILLAR: [
-        (Schema.MBP_1, ["AAPL  230331C00157500"], "2023-03-28"),
-        (Schema.TBBO, ["AAPL  230331C00157500"], "2023-03-28"),
-        (Schema.TRADES, ["AAPL  230331C00157500"], "2023-03-28"),
-        (Schema.OHLCV_1S, ["AAPL  230331C00157500"], "2023-03-28"),
-        (Schema.OHLCV_1M, ["AAPL  230331C00157500"], "2023-03-28"),
-        (Schema.OHLCV_1H, ["AAPL  230331C00157500"], "2023-03-28"),
-        (Schema.OHLCV_1D, ["AAPL  230331C00157500"], "2023-03-28"),
-        (Schema.DEFINITION, ["AAPL  230331C00157500"], "2023-03-28"),
-        (Schema.STATISTICS, ["AAPL  230331C00157500"], "2023-03-28"),
+        (Schema.MBP_1, ["AAPL  250221C00250000"], "2025-02-20"),
+        (Schema.TBBO, ["AAPL  250221C00250000"], "2025-02-20"),
+        (Schema.TRADES, ["AAPL  250221C00250000"], "2025-02-20"),
+        (Schema.CBBO_1S, ["AAPL  250221C00250000"], "2025-02-20"),
+        (Schema.CBBO_1M, ["AAPL  250221C00250000"], "2025-02-20"),
+        (Schema.OHLCV_1S, ["AAPL  250221C00250000"], "2025-02-20"),
+        (Schema.OHLCV_1M, ["AAPL  250221C00250000"], "2025-02-20"),
+        (Schema.OHLCV_1H, ["AAPL  250221C00250000"], "2025-02-20"),
+        (Schema.OHLCV_1D, ["AAPL  250221C00250000"], "2025-02-20"),
+        (Schema.DEFINITION, ["AAPL  250221C00250000"], "2025-02-20"),
+        (Schema.STATISTICS, ["AAPL  250221C00250000"], "2025-02-20"),
     ],
-    Dataset.DBEQ_BASIC: [
+    Dataset.EQUS_MINI: [
         (Schema.MBP_1, ["QQQ"], "2023-03-28"),
-        (Schema.MBP_10, ["QQQ"], "2023-03-28"),
         (Schema.TBBO, ["QQQ"], "2023-03-28"),
         (Schema.TRADES, ["QQQ"], "2023-03-28"),
+        (Schema.BBO_1S, ["QQQ"], "2023-03-28"),
+        (Schema.BBO_1M, ["QQQ"], "2023-03-28"),
         (Schema.OHLCV_1S, ["QQQ"], "2023-03-28"),
         (Schema.OHLCV_1M, ["QQQ"], "2023-03-28"),
         (Schema.OHLCV_1H, ["QQQ"], "2023-03-28"),
@@ -80,6 +83,8 @@
         (Schema.MBO, ["BRN FMJ0024!"], "2024-01-18"),
         (Schema.MBP_1, ["BRN FMJ0024!"], "2024-01-18"),
         (Schema.MBP_10, ["BRN FMJ0024!"], "2024-01-18"),
+        (Schema.BBO_1S, ["BRN FMJ0024!"], "2024-01-18"),
+        (Schema.BBO_1M, ["BRN FMJ0024!"], "2024-01-18"),
         (Schema.TBBO, ["BRN FMJ0024!"], "2024-01-18"),
         (Schema.TRADES, ["BRN FMJ0024!"], "2024-01-18"),
         (Schema.OHLCV_1S, ["BRN FMJ0024!"], "2024-01-18"),
@@ -93,6 +98,8 @@
         (Schema.MBO, ["TFM FMH0024!"], "2024-01-18"),
         (Schema.MBP_1, ["TFM FMH0024!"], "2024-01-18"),
         (Schema.MBP_10, ["TFM FMH0024!"], "2024-01-18"),
+        (Schema.BBO_1S, ["TFM FMH0024!"], "2024-01-18"),
+        (Schema.BBO_1M, ["TFM FMH0024!"], "2024-01-18"),
         (Schema.TBBO, ["TFM FMH0024!"], "2024-01-18"),
         (Schema.TRADES, ["TFM FMH0024!"], "2024-01-18"),
         (Schema.OHLCV_1S, ["TFM FMH0024!"], "2024-01-18"),

tests/test_bento_data_source.py

@@ -14,7 +14,7 @@
         Dataset.GLBX_MDP3,
         Dataset.XNAS_ITCH,
         Dataset.OPRA_PILLAR,
-        Dataset.DBEQ_BASIC,
+        Dataset.EQUS_MINI,
         Dataset.IFEU_IMPACT,
         Dataset.NDEX_IMPACT,
     ],
@@ -43,7 +43,7 @@ def test_memory_data_source(
         Dataset.GLBX_MDP3,
         Dataset.XNAS_ITCH,
         Dataset.OPRA_PILLAR,
-        Dataset.DBEQ_BASIC,
+        Dataset.EQUS_MINI,
         Dataset.IFEU_IMPACT,
         Dataset.NDEX_IMPACT,
     ],

tests/test_common_symbology.py

@@ -854,7 +854,7 @@ def test_instrument_map_ignore_duplicate(
         Dataset.GLBX_MDP3,
         Dataset.XNAS_ITCH,
         Dataset.OPRA_PILLAR,
-        Dataset.DBEQ_BASIC,
+        Dataset.EQUS_MINI,
         Dataset.IFEU_IMPACT,
         Dataset.NDEX_IMPACT,
     ],
@@ -914,7 +914,7 @@ def test_instrument_map_symbols_csv(
         Dataset.GLBX_MDP3,
         Dataset.XNAS_ITCH,
         Dataset.OPRA_PILLAR,
-        Dataset.DBEQ_BASIC,
+        Dataset.EQUS_MINI,
     ],
 )
 @pytest.mark.parametrize(

tests/test_historical_bento.py

@@ -665,7 +665,7 @@ def test_to_df_with_price_type_invalid(
         Dataset.GLBX_MDP3,
         Dataset.XNAS_ITCH,
         Dataset.OPRA_PILLAR,
-        Dataset.DBEQ_BASIC,
+        Dataset.EQUS_MINI,
         Dataset.IFEU_IMPACT,
         Dataset.NDEX_IMPACT,
     ],

tests/test_live_client.py

@@ -1151,7 +1151,7 @@ def callback(record: DBNRecord) -> None:
         Dataset.GLBX_MDP3,
         Dataset.XNAS_ITCH,
         Dataset.OPRA_PILLAR,
-        Dataset.DBEQ_BASIC,
+        Dataset.EQUS_MINI,
         Dataset.IFEU_IMPACT,
         Dataset.NDEX_IMPACT,
     ],
@@ -1204,7 +1204,7 @@ async def test_live_stream_to_dbn(
         Dataset.GLBX_MDP3,
         Dataset.XNAS_ITCH,
         Dataset.OPRA_PILLAR,
-        Dataset.DBEQ_BASIC,
+        Dataset.EQUS_MINI,
         Dataset.IFEU_IMPACT,
         Dataset.NDEX_IMPACT,
     ],

tests/test_live_gateway_messages.py

@@ -299,15 +299,15 @@ def test_serialize_session_start(
     "line, expected",
     [
         pytest.param(
-            "schema=trades|" "stype_in=instrument_id|" "symbols=1,2,3\n",
-            ("trades", "instrument_id", "1,2,3", None),
+            "schema=trades|" "stype_in=instrument_id|" "symbols=1,2,3|" "id=23\n",
+            ("trades", "instrument_id", "1,2,3", None, "23"),
         ),
         pytest.param(
             "schema=trades|"
             "stype_in=instrument_id|"
             "symbols=1,2,3|"
             "start=1671717080706865759\n",
-            ("trades", "instrument_id", "1,2,3", "1671717080706865759"),
+            ("trades", "instrument_id", "1,2,3", "1671717080706865759", None),
         ),
         pytest.param(
             "schema=trades|" "stype_in=instrument_id|" "symbols=1,2,3",
@@ -336,6 +336,7 @@ def test_parse_subscription_request(
             msg.stype_in,
             msg.symbols,
             msg.start,
+            msg.id,
         ) == expected
     else:
         with pytest.raises(expected):
@@ -374,8 +375,13 @@ def test_parse_subscription_request(
                 symbols="1234,5678,90",
                 start=None,
                 snapshot=1,
+                id=5,
             ),
-            b"schema=mbo|" b"stype_in=instrument_id|" b"symbols=1234,5678,90|" b"snapshot=1\n",
+            b"schema=mbo|"
+            b"stype_in=instrument_id|"
+            b"symbols=1234,5678,90|"
+            b"snapshot=1|"
+            b"id=5\n",
         ),
     ],
 )

tests/test_live_protocol.py

@@ -16,7 +16,7 @@
         Dataset.GLBX_MDP3,
         Dataset.XNAS_ITCH,
         Dataset.OPRA_PILLAR,
-        Dataset.DBEQ_BASIC,
+        Dataset.EQUS_MINI,
         Dataset.IFEU_IMPACT,
         Dataset.NDEX_IMPACT,
     ],
@@ -52,7 +52,7 @@ async def test_protocol_connection(
         Dataset.GLBX_MDP3,
         Dataset.XNAS_ITCH,
         Dataset.OPRA_PILLAR,
-        Dataset.DBEQ_BASIC,
+        Dataset.EQUS_MINI,
         Dataset.IFEU_IMPACT,
         Dataset.NDEX_IMPACT,
     ],