diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 22d4c019..93239656 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -30,13 +30,13 @@ jobs: shell: bash - name: Test (release) - timeout-minutes: 3 + timeout-minutes: 5 if: ${{ github.ref == 'refs/heads/main' }} run: scripts/test.sh -vvv --release shell: bash - name: Test - timeout-minutes: 3 + timeout-minutes: 5 if: ${{ github.ref != 'refs/heads/main' }} run: scripts/test.sh -vvv shell: bash diff --git a/CHANGELOG.md b/CHANGELOG.md index 20c513e9..302546a6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## 0.55.0 - 2025-05-29 + +#### Enhancements +- Added `exchanges` parameter to `Reference.corporate_actions.get_range(...)` +- Added `is_last` field to live subscription requests which will be used to improve + the handling of split subscription requests +- Upgraded `databento-dbn` to 0.35.0 + - This version delivers DBN version 3 (DBNv3), which is the new default + - Improved the performance of the Python `DBNDecoder` + +#### Bug fixes +- Fixed an issue where `JSONDecodeError` would not be caught when using `simplejson` with `requests` (credit: @xuanqing94) + ## 0.54.0 - 2025-05-13 #### Enhancements diff --git a/README.md b/README.md index c0eb42bc..1587f8f5 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ The library is fully compatible with the latest distribution of Anaconda 3.9 and The minimum dependencies as found in the `pyproject.toml` are also listed below: - python = "^3.9" - aiohttp = "^3.8.3" -- databento-dbn = "0.34.0" +- databento-dbn = "0.35.0" - numpy= ">=1.23.5" - pandas = ">=1.5.3" - pip-system-certs = ">=4.0" (Windows only) diff --git a/databento/common/dbnstore.py b/databento/common/dbnstore.py index 6cc0d9c8..dc8cf802 100644 --- a/databento/common/dbnstore.py +++ b/databento/common/dbnstore.py @@ -380,7 +380,7 @@ def __init__(self, data_source: DataSource) -> None: def __iter__(self) -> Generator[DBNRecord, None, None]: reader = self.reader decoder = DBNDecoder( - upgrade_policy=VersionUpgradePolicy.UPGRADE_TO_V2, + upgrade_policy=VersionUpgradePolicy.UPGRADE_TO_V3, ) while True: raw = reader.read(DBNStore.DBN_READ_SIZE) @@ -394,6 +394,9 @@ def __iter__(self) -> Generator[DBNRecord, None, None]: self._instrument_map.insert_symbol_mapping_msg(record) yield record else: + # This call to decode is required to seek past the decoded records + # This behavior will be fixed in the next version of databento_dbn + _ = decoder.decode() if len(decoder.buffer()) > 0: warnings.warn( BentoWarning("DBN file is truncated or contains an incomplete record"), diff --git a/databento/common/http.py b/databento/common/http.py index 5908e1bf..629cd2a3 100644 --- a/databento/common/http.py +++ b/databento/common/http.py @@ -5,7 +5,6 @@ from collections.abc import Iterable from collections.abc import Mapping from io import BytesIO -from json.decoder import JSONDecodeError from os import PathLike from typing import IO from typing import Any @@ -15,6 +14,7 @@ import requests from aiohttp import ClientResponse from aiohttp import ContentTypeError +from requests import JSONDecodeError from requests import Response from requests.auth import HTTPBasicAuth diff --git a/databento/common/types.py b/databento/common/types.py index a34fbc43..55214c58 100644 --- a/databento/common/types.py +++ b/databento/common/types.py @@ -21,8 +21,9 @@ databento_dbn.ImbalanceMsg, databento_dbn.InstrumentDefMsg, databento_dbn.InstrumentDefMsgV1, - databento_dbn.InstrumentDefMsgV3, + databento_dbn.InstrumentDefMsgV2, databento_dbn.StatMsg, + databento_dbn.StatMsgV1, databento_dbn.StatusMsg, databento_dbn.SymbolMappingMsg, databento_dbn.SymbolMappingMsgV1, diff --git a/databento/historical/api/metadata.py b/databento/historical/api/metadata.py index deaf4c4b..573663f6 100644 --- a/databento/historical/api/metadata.py +++ b/databento/historical/api/metadata.py @@ -407,7 +407,8 @@ def get_cost( ) -> float: """ Request the cost in US dollars for historical streaming or batched - files from Databento. + files from Databento. This cost respects any discounts provided by flat + rate plans. Makes a `GET /metadata.get_cost` HTTP request. diff --git a/databento/live/gateway.py b/databento/live/gateway.py index 307afcb8..414683bb 100644 --- a/databento/live/gateway.py +++ b/databento/live/gateway.py @@ -133,6 +133,7 @@ class SubscriptionRequest(GatewayControl): start: int | None = None snapshot: int = 0 id: int | None = None + is_last: int = 1 @dataclasses.dataclass diff --git a/databento/live/protocol.py b/databento/live/protocol.py index 0458acbe..e36db93e 100644 --- a/databento/live/protocol.py +++ b/databento/live/protocol.py @@ -82,7 +82,7 @@ def __init__( self._heartbeat_interval_s = heartbeat_interval_s self._dbn_decoder = databento_dbn.DBNDecoder( - upgrade_policy=VersionUpgradePolicy.UPGRADE_TO_V2, + upgrade_policy=VersionUpgradePolicy.UPGRADE_TO_V3, ) self._gateway_decoder = GatewayDecoder() @@ -175,7 +175,7 @@ def connection_made(self, transport: asyncio.BaseTransport) -> None: See Also -------- - asycnio.BufferedProtocol.connection_made + asyncio.BufferedProtocol.connection_made """ logger.debug("established connection to gateway") @@ -190,7 +190,7 @@ def connection_lost(self, exc: Exception | None) -> None: See Also -------- - asycnio.BufferedProtocol.connection_lost + asyncio.BufferedProtocol.connection_lost """ super().connection_lost(exc) @@ -216,7 +216,7 @@ def eof_received(self) -> bool | None: See Also -------- - asycnio.BufferedProtocol.eof_received + asyncio.BufferedProtocol.eof_received """ logger.info("received EOF from remote") @@ -228,7 +228,7 @@ def get_buffer(self, sizehint: int) -> bytearray: See Also -------- - asycnio.BufferedProtocol.get_buffer + asyncio.BufferedProtocol.get_buffer """ if len(self.__buffer) < sizehint: @@ -241,7 +241,7 @@ def buffer_updated(self, nbytes: int) -> None: See Also -------- - asycnio.BufferedProtocol.buffer_updated + asyncio.BufferedProtocol.buffer_updated """ logger.debug("read %d bytes from remote gateway", nbytes) @@ -325,7 +325,8 @@ def subscribe( subscriptions: list[SubscriptionRequest] = [] chunked_symbols = list(chunk(symbols_list, SYMBOL_LIST_BATCH_SIZE)) - for batch in chunked_symbols: + last_chunk_idx = len(chunked_symbols) - 1 + for i, batch in enumerate(chunked_symbols): batch_str = ",".join(batch) message = SubscriptionRequest( schema=validate_enum(schema, Schema, "schema"), @@ -334,6 +335,7 @@ def subscribe( start=optional_datetime_to_unix_nanoseconds(start), snapshot=int(snapshot), id=subscription_id, + is_last=int(i == last_chunk_idx), ) subscriptions.append(message) diff --git a/databento/reference/api/corporate.py b/databento/reference/api/corporate.py index df348693..57d1c9ac 100644 --- a/databento/reference/api/corporate.py +++ b/databento/reference/api/corporate.py @@ -39,6 +39,7 @@ def get_range( stype_in: SType | str = "raw_symbol", events: Iterable[str] | str | None = None, countries: Iterable[str] | str | None = None, + exchanges: Iterable[str] | str | None = None, security_types: Iterable[str] | str | None = None, flatten: bool = True, pit: bool = False, @@ -84,6 +85,11 @@ def get_range( Takes any number of two letter ISO 3166-1 alpha-2 country codes per request. If not specified then will select **all** listing countries by default. See [CNTRY](https://databento.com/docs/standards-and-conventions/reference-data-enums#cntry) enum. + exchanges : Iterable[str] or str, optional + The (listing) exchanges to filter for. + Takes any number of exchanges per request. + If not specified then will select **all** exchanges by default. + See [EXCHANGE](https://databento.com/docs/standards-and-conventions/reference-data-enums#exchange) enum. security_types : Iterable[str] or str, optional The security types to filter for. Takes any number of security types per request. @@ -108,6 +114,7 @@ def get_range( symbols_list = optional_symbols_list_to_list(symbols, SType.RAW_SYMBOL) events = optional_string_to_list(events) countries = optional_string_to_list(countries) + exchanges = optional_string_to_list(exchanges) security_types = optional_string_to_list(security_types) data: dict[str, object | None] = { @@ -122,6 +129,10 @@ def get_range( "compression": str(Compression.ZSTD), # Always request zstd } + # Only add the `exchanges` param if it is supplied, for compatibility + if exchanges: + data["exchanges"] = ",".join(exchanges) + response = self._post( url=self._base_url + ".get_range", data=data, diff --git a/databento/version.py b/databento/version.py index 450ee122..8b19221c 100644 --- a/databento/version.py +++ b/databento/version.py @@ -1 +1 @@ -__version__ = "0.54.0" +__version__ = "0.55.0" diff --git a/pyproject.toml b/pyproject.toml index 46e19db2..a91ee9a2 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,6 +1,6 @@ [tool.poetry] name = "databento" -version = "0.54.0" +version = "0.55.0" description = "Official Python client library for Databento" authors = [ "Databento ", @@ -32,7 +32,7 @@ aiohttp = [ {version = "^3.8.3", python = "<3.12"}, {version = "^3.9.0", python = "^3.12"} ] -databento-dbn = "0.34.0" +databento-dbn = "0.35.0" numpy = [ {version = ">=1.23.5", python = "<3.12"}, {version = ">=1.26.0", python = "^3.12"} diff --git a/tests/data/EQUS.MINI/test_data.bbo-1m.dbn.zst b/tests/data/EQUS.MINI/test_data.bbo-1m.dbn.zst index c40910b2..32a3fad7 100644 Binary files a/tests/data/EQUS.MINI/test_data.bbo-1m.dbn.zst and b/tests/data/EQUS.MINI/test_data.bbo-1m.dbn.zst differ diff --git a/tests/data/EQUS.MINI/test_data.bbo-1s.dbn.zst b/tests/data/EQUS.MINI/test_data.bbo-1s.dbn.zst index f12e1097..c0b6749b 100644 Binary files a/tests/data/EQUS.MINI/test_data.bbo-1s.dbn.zst and b/tests/data/EQUS.MINI/test_data.bbo-1s.dbn.zst differ diff --git a/tests/data/EQUS.MINI/test_data.definition.dbn.zst b/tests/data/EQUS.MINI/test_data.definition.dbn.zst index c43ff562..7c6e8bb0 100644 Binary files a/tests/data/EQUS.MINI/test_data.definition.dbn.zst and b/tests/data/EQUS.MINI/test_data.definition.dbn.zst differ diff --git a/tests/data/EQUS.MINI/test_data.mbp-1.dbn.zst b/tests/data/EQUS.MINI/test_data.mbp-1.dbn.zst index 4e4c22d8..c7a83e9e 100644 Binary files a/tests/data/EQUS.MINI/test_data.mbp-1.dbn.zst and b/tests/data/EQUS.MINI/test_data.mbp-1.dbn.zst differ diff --git a/tests/data/EQUS.MINI/test_data.ohlcv-1d.dbn.zst b/tests/data/EQUS.MINI/test_data.ohlcv-1d.dbn.zst index c8764ca0..c609a39f 100644 Binary files a/tests/data/EQUS.MINI/test_data.ohlcv-1d.dbn.zst and b/tests/data/EQUS.MINI/test_data.ohlcv-1d.dbn.zst differ diff --git a/tests/data/EQUS.MINI/test_data.ohlcv-1h.dbn.zst b/tests/data/EQUS.MINI/test_data.ohlcv-1h.dbn.zst index 43a6b549..e9ae8bd5 100644 Binary files a/tests/data/EQUS.MINI/test_data.ohlcv-1h.dbn.zst and b/tests/data/EQUS.MINI/test_data.ohlcv-1h.dbn.zst differ diff --git a/tests/data/EQUS.MINI/test_data.ohlcv-1m.dbn.zst b/tests/data/EQUS.MINI/test_data.ohlcv-1m.dbn.zst index c2e00e76..5e1737fd 100644 Binary files a/tests/data/EQUS.MINI/test_data.ohlcv-1m.dbn.zst and b/tests/data/EQUS.MINI/test_data.ohlcv-1m.dbn.zst differ diff --git a/tests/data/EQUS.MINI/test_data.ohlcv-1s.dbn.zst b/tests/data/EQUS.MINI/test_data.ohlcv-1s.dbn.zst index 0d4d05c4..5e0d3e94 100644 Binary files a/tests/data/EQUS.MINI/test_data.ohlcv-1s.dbn.zst and b/tests/data/EQUS.MINI/test_data.ohlcv-1s.dbn.zst differ diff --git a/tests/data/EQUS.MINI/test_data.tbbo.dbn.zst b/tests/data/EQUS.MINI/test_data.tbbo.dbn.zst index c6cb743e..53d3c567 100644 Binary files a/tests/data/EQUS.MINI/test_data.tbbo.dbn.zst and b/tests/data/EQUS.MINI/test_data.tbbo.dbn.zst differ diff --git a/tests/data/EQUS.MINI/test_data.trades.dbn.zst b/tests/data/EQUS.MINI/test_data.trades.dbn.zst index 29351936..bc315c88 100644 Binary files a/tests/data/EQUS.MINI/test_data.trades.dbn.zst and b/tests/data/EQUS.MINI/test_data.trades.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.bbo-1m.dbn.zst b/tests/data/GLBX.MDP3/test_data.bbo-1m.dbn.zst index abd773f2..fd5d96db 100644 Binary files a/tests/data/GLBX.MDP3/test_data.bbo-1m.dbn.zst and b/tests/data/GLBX.MDP3/test_data.bbo-1m.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.bbo-1s.dbn.zst b/tests/data/GLBX.MDP3/test_data.bbo-1s.dbn.zst index 040d758d..586823bc 100644 Binary files a/tests/data/GLBX.MDP3/test_data.bbo-1s.dbn.zst and b/tests/data/GLBX.MDP3/test_data.bbo-1s.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.definition.dbn.zst b/tests/data/GLBX.MDP3/test_data.definition.dbn.zst index 235ea8a6..d75a6426 100644 Binary files a/tests/data/GLBX.MDP3/test_data.definition.dbn.zst and b/tests/data/GLBX.MDP3/test_data.definition.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.mbo.dbn.zst b/tests/data/GLBX.MDP3/test_data.mbo.dbn.zst index e836c74e..7d3851a6 100644 Binary files a/tests/data/GLBX.MDP3/test_data.mbo.dbn.zst and b/tests/data/GLBX.MDP3/test_data.mbo.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.mbp-1.dbn.zst b/tests/data/GLBX.MDP3/test_data.mbp-1.dbn.zst index df17a18a..a33daaf5 100644 Binary files a/tests/data/GLBX.MDP3/test_data.mbp-1.dbn.zst and b/tests/data/GLBX.MDP3/test_data.mbp-1.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.mbp-10.dbn.zst b/tests/data/GLBX.MDP3/test_data.mbp-10.dbn.zst index 4c086005..39fa5c4a 100644 Binary files a/tests/data/GLBX.MDP3/test_data.mbp-10.dbn.zst and b/tests/data/GLBX.MDP3/test_data.mbp-10.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.ohlcv-1d.dbn.zst b/tests/data/GLBX.MDP3/test_data.ohlcv-1d.dbn.zst index 3369763f..6368e6c6 100644 Binary files a/tests/data/GLBX.MDP3/test_data.ohlcv-1d.dbn.zst and b/tests/data/GLBX.MDP3/test_data.ohlcv-1d.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.ohlcv-1h.dbn.zst b/tests/data/GLBX.MDP3/test_data.ohlcv-1h.dbn.zst index f8828a33..937aefcd 100644 Binary files a/tests/data/GLBX.MDP3/test_data.ohlcv-1h.dbn.zst and b/tests/data/GLBX.MDP3/test_data.ohlcv-1h.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.ohlcv-1m.dbn.zst b/tests/data/GLBX.MDP3/test_data.ohlcv-1m.dbn.zst index 1d3db3ae..41d63fbb 100644 Binary files a/tests/data/GLBX.MDP3/test_data.ohlcv-1m.dbn.zst and b/tests/data/GLBX.MDP3/test_data.ohlcv-1m.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.ohlcv-1s.dbn.zst b/tests/data/GLBX.MDP3/test_data.ohlcv-1s.dbn.zst index 7e960659..037ca385 100644 Binary files a/tests/data/GLBX.MDP3/test_data.ohlcv-1s.dbn.zst and b/tests/data/GLBX.MDP3/test_data.ohlcv-1s.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.statistics.dbn.zst b/tests/data/GLBX.MDP3/test_data.statistics.dbn.zst index 0bb2c812..7ea129ab 100644 Binary files a/tests/data/GLBX.MDP3/test_data.statistics.dbn.zst and b/tests/data/GLBX.MDP3/test_data.statistics.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.status.dbn.zst b/tests/data/GLBX.MDP3/test_data.status.dbn.zst index 1ef5d7ad..b1c7d5df 100644 Binary files a/tests/data/GLBX.MDP3/test_data.status.dbn.zst and b/tests/data/GLBX.MDP3/test_data.status.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.tbbo.dbn.zst b/tests/data/GLBX.MDP3/test_data.tbbo.dbn.zst index a870bfea..0aa25768 100644 Binary files a/tests/data/GLBX.MDP3/test_data.tbbo.dbn.zst and b/tests/data/GLBX.MDP3/test_data.tbbo.dbn.zst differ diff --git a/tests/data/GLBX.MDP3/test_data.trades.dbn.zst b/tests/data/GLBX.MDP3/test_data.trades.dbn.zst index 49a9eeb9..8107cf30 100644 Binary files a/tests/data/GLBX.MDP3/test_data.trades.dbn.zst and b/tests/data/GLBX.MDP3/test_data.trades.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.bbo-1m.dbn.zst b/tests/data/IFEU.IMPACT/test_data.bbo-1m.dbn.zst index b20fe4b4..518af749 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.bbo-1m.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.bbo-1m.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.bbo-1s.dbn.zst b/tests/data/IFEU.IMPACT/test_data.bbo-1s.dbn.zst index d324eec6..5020f9ce 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.bbo-1s.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.bbo-1s.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.definition.dbn.zst b/tests/data/IFEU.IMPACT/test_data.definition.dbn.zst index bcfdadaa..a4bb4f9f 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.definition.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.definition.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.mbo.dbn.zst b/tests/data/IFEU.IMPACT/test_data.mbo.dbn.zst index f0f009f1..472f6efa 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.mbo.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.mbo.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.mbp-1.dbn.zst b/tests/data/IFEU.IMPACT/test_data.mbp-1.dbn.zst index bfe53e16..b9cf92f7 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.mbp-1.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.mbp-1.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.mbp-10.dbn.zst b/tests/data/IFEU.IMPACT/test_data.mbp-10.dbn.zst index e44499e1..9ff7c17b 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.mbp-10.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.mbp-10.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.ohlcv-1d.dbn.zst b/tests/data/IFEU.IMPACT/test_data.ohlcv-1d.dbn.zst index 85c7d55c..cfbbc9be 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.ohlcv-1d.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.ohlcv-1d.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.ohlcv-1h.dbn.zst b/tests/data/IFEU.IMPACT/test_data.ohlcv-1h.dbn.zst index b9673c37..96c3ba78 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.ohlcv-1h.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.ohlcv-1h.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.ohlcv-1m.dbn.zst b/tests/data/IFEU.IMPACT/test_data.ohlcv-1m.dbn.zst index eac6cac9..4b6b68a4 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.ohlcv-1m.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.ohlcv-1m.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.ohlcv-1s.dbn.zst b/tests/data/IFEU.IMPACT/test_data.ohlcv-1s.dbn.zst index 250b8132..9731edf0 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.ohlcv-1s.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.ohlcv-1s.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.statistics.dbn.zst b/tests/data/IFEU.IMPACT/test_data.statistics.dbn.zst index 08ae05ab..136e6e6a 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.statistics.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.statistics.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.tbbo.dbn.zst b/tests/data/IFEU.IMPACT/test_data.tbbo.dbn.zst index c08410f6..4cf68fae 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.tbbo.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.tbbo.dbn.zst differ diff --git a/tests/data/IFEU.IMPACT/test_data.trades.dbn.zst b/tests/data/IFEU.IMPACT/test_data.trades.dbn.zst index 8c079864..75a7bc2a 100644 Binary files a/tests/data/IFEU.IMPACT/test_data.trades.dbn.zst and b/tests/data/IFEU.IMPACT/test_data.trades.dbn.zst differ diff --git a/tests/data/LIVE/test_data.live.dbn.zst b/tests/data/LIVE/test_data.live.dbn.zst index da35bb9b..240ff257 100644 Binary files a/tests/data/LIVE/test_data.live.dbn.zst and b/tests/data/LIVE/test_data.live.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.bbo-1m.dbn.zst b/tests/data/NDEX.IMPACT/test_data.bbo-1m.dbn.zst index 2ebf6b7f..38a7c07a 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.bbo-1m.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.bbo-1m.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.bbo-1s.dbn.zst b/tests/data/NDEX.IMPACT/test_data.bbo-1s.dbn.zst index aec1a465..754384cf 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.bbo-1s.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.bbo-1s.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.definition.dbn.zst b/tests/data/NDEX.IMPACT/test_data.definition.dbn.zst index 40187ad1..9728a529 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.definition.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.definition.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.mbo.dbn.zst b/tests/data/NDEX.IMPACT/test_data.mbo.dbn.zst index d333ecea..36dde4d1 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.mbo.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.mbo.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.mbp-1.dbn.zst b/tests/data/NDEX.IMPACT/test_data.mbp-1.dbn.zst index 58eb2d3e..5be0cb55 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.mbp-1.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.mbp-1.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.mbp-10.dbn.zst b/tests/data/NDEX.IMPACT/test_data.mbp-10.dbn.zst index 30fcd4a6..b21ce472 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.mbp-10.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.mbp-10.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.ohlcv-1d.dbn.zst b/tests/data/NDEX.IMPACT/test_data.ohlcv-1d.dbn.zst index acfd8927..da99a496 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.ohlcv-1d.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.ohlcv-1d.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.ohlcv-1h.dbn.zst b/tests/data/NDEX.IMPACT/test_data.ohlcv-1h.dbn.zst index d91570c1..b3c7b276 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.ohlcv-1h.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.ohlcv-1h.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.ohlcv-1m.dbn.zst b/tests/data/NDEX.IMPACT/test_data.ohlcv-1m.dbn.zst index 90012ee1..713fc0a1 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.ohlcv-1m.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.ohlcv-1m.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.ohlcv-1s.dbn.zst b/tests/data/NDEX.IMPACT/test_data.ohlcv-1s.dbn.zst index b1124e1d..98469920 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.ohlcv-1s.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.ohlcv-1s.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.statistics.dbn.zst b/tests/data/NDEX.IMPACT/test_data.statistics.dbn.zst index 9ae5f341..925fee72 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.statistics.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.statistics.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.tbbo.dbn.zst b/tests/data/NDEX.IMPACT/test_data.tbbo.dbn.zst index 2530efde..1044113c 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.tbbo.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.tbbo.dbn.zst differ diff --git a/tests/data/NDEX.IMPACT/test_data.trades.dbn.zst b/tests/data/NDEX.IMPACT/test_data.trades.dbn.zst index c7e44c48..a98900ab 100644 Binary files a/tests/data/NDEX.IMPACT/test_data.trades.dbn.zst and b/tests/data/NDEX.IMPACT/test_data.trades.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.cbbo-1m.dbn.zst b/tests/data/OPRA.PILLAR/test_data.cbbo-1m.dbn.zst index 0d492edc..8cc98373 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.cbbo-1m.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.cbbo-1m.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.cbbo-1s.dbn.zst b/tests/data/OPRA.PILLAR/test_data.cbbo-1s.dbn.zst index ea783c5e..3318fef6 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.cbbo-1s.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.cbbo-1s.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.definition.dbn.zst b/tests/data/OPRA.PILLAR/test_data.definition.dbn.zst index cfb96844..40c55992 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.definition.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.definition.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.mbp-1.dbn.zst b/tests/data/OPRA.PILLAR/test_data.mbp-1.dbn.zst index f0301d71..4cb524f6 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.mbp-1.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.mbp-1.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.ohlcv-1d.dbn.zst b/tests/data/OPRA.PILLAR/test_data.ohlcv-1d.dbn.zst index 76cb6d9f..45f54022 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.ohlcv-1d.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.ohlcv-1d.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.ohlcv-1h.dbn.zst b/tests/data/OPRA.PILLAR/test_data.ohlcv-1h.dbn.zst index 114a1c71..07a3d94d 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.ohlcv-1h.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.ohlcv-1h.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.ohlcv-1m.dbn.zst b/tests/data/OPRA.PILLAR/test_data.ohlcv-1m.dbn.zst index c442da21..ba637621 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.ohlcv-1m.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.ohlcv-1m.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.ohlcv-1s.dbn.zst b/tests/data/OPRA.PILLAR/test_data.ohlcv-1s.dbn.zst index 91f0a342..d8f1544b 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.ohlcv-1s.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.ohlcv-1s.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.statistics.dbn.zst b/tests/data/OPRA.PILLAR/test_data.statistics.dbn.zst index a0dd2071..dc1674ed 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.statistics.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.statistics.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.tbbo.dbn.zst b/tests/data/OPRA.PILLAR/test_data.tbbo.dbn.zst index a847cdd0..44e6504b 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.tbbo.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.tbbo.dbn.zst differ diff --git a/tests/data/OPRA.PILLAR/test_data.trades.dbn.zst b/tests/data/OPRA.PILLAR/test_data.trades.dbn.zst index 51e822eb..ab93ac25 100644 Binary files a/tests/data/OPRA.PILLAR/test_data.trades.dbn.zst and b/tests/data/OPRA.PILLAR/test_data.trades.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.definition.dbn.zst b/tests/data/XNAS.ITCH/test_data.definition.dbn.zst index 72940b55..2ad81413 100644 Binary files a/tests/data/XNAS.ITCH/test_data.definition.dbn.zst and b/tests/data/XNAS.ITCH/test_data.definition.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.imbalance.dbn.zst b/tests/data/XNAS.ITCH/test_data.imbalance.dbn.zst index 8d47c7f5..270082a3 100644 Binary files a/tests/data/XNAS.ITCH/test_data.imbalance.dbn.zst and b/tests/data/XNAS.ITCH/test_data.imbalance.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.mbo.dbn.zst b/tests/data/XNAS.ITCH/test_data.mbo.dbn.zst index 57530950..fca315d8 100644 Binary files a/tests/data/XNAS.ITCH/test_data.mbo.dbn.zst and b/tests/data/XNAS.ITCH/test_data.mbo.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.mbp-1.dbn.zst b/tests/data/XNAS.ITCH/test_data.mbp-1.dbn.zst index 10e37031..70b77bb9 100644 Binary files a/tests/data/XNAS.ITCH/test_data.mbp-1.dbn.zst and b/tests/data/XNAS.ITCH/test_data.mbp-1.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.mbp-10.dbn.zst b/tests/data/XNAS.ITCH/test_data.mbp-10.dbn.zst index bc7d230a..e2cb6d1c 100644 Binary files a/tests/data/XNAS.ITCH/test_data.mbp-10.dbn.zst and b/tests/data/XNAS.ITCH/test_data.mbp-10.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.ohlcv-1d.dbn.zst b/tests/data/XNAS.ITCH/test_data.ohlcv-1d.dbn.zst index 07f8ea6a..71d490c5 100644 Binary files a/tests/data/XNAS.ITCH/test_data.ohlcv-1d.dbn.zst and b/tests/data/XNAS.ITCH/test_data.ohlcv-1d.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.ohlcv-1h.dbn.zst b/tests/data/XNAS.ITCH/test_data.ohlcv-1h.dbn.zst index d4037e39..40d6609c 100644 Binary files a/tests/data/XNAS.ITCH/test_data.ohlcv-1h.dbn.zst and b/tests/data/XNAS.ITCH/test_data.ohlcv-1h.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.ohlcv-1m.dbn.zst b/tests/data/XNAS.ITCH/test_data.ohlcv-1m.dbn.zst index fcb1e387..f13c0458 100644 Binary files a/tests/data/XNAS.ITCH/test_data.ohlcv-1m.dbn.zst and b/tests/data/XNAS.ITCH/test_data.ohlcv-1m.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.ohlcv-1s.dbn.zst b/tests/data/XNAS.ITCH/test_data.ohlcv-1s.dbn.zst index 1dbdefb8..9dca6c51 100644 Binary files a/tests/data/XNAS.ITCH/test_data.ohlcv-1s.dbn.zst and b/tests/data/XNAS.ITCH/test_data.ohlcv-1s.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.tbbo.dbn.zst b/tests/data/XNAS.ITCH/test_data.tbbo.dbn.zst index 5d3c7bf2..3014c05e 100644 Binary files a/tests/data/XNAS.ITCH/test_data.tbbo.dbn.zst and b/tests/data/XNAS.ITCH/test_data.tbbo.dbn.zst differ diff --git a/tests/data/XNAS.ITCH/test_data.trades.dbn.zst b/tests/data/XNAS.ITCH/test_data.trades.dbn.zst index 1465304c..d85d8971 100644 Binary files a/tests/data/XNAS.ITCH/test_data.trades.dbn.zst and b/tests/data/XNAS.ITCH/test_data.trades.dbn.zst differ diff --git a/tests/test_historical_bento.py b/tests/test_historical_bento.py index 2a8af1af..f9ab3b0b 100644 --- a/tests/test_historical_bento.py +++ b/tests/test_historical_bento.py @@ -90,7 +90,7 @@ def test_sources_metadata_returns_expected_json_as_dict( dbnstore = DBNStore.from_bytes(data=stub_data) # Assert - assert dbnstore.metadata.version == 2 + assert dbnstore.metadata.version == 3 assert dbnstore.metadata.dataset == "GLBX.MDP3" assert dbnstore.metadata.schema == Schema.MBO assert dbnstore.metadata.stype_in == SType.RAW_SYMBOL @@ -123,7 +123,7 @@ def test_dbnstore_given_initial_nbytes_returns_expected_metadata( dbnstore = DBNStore.from_bytes(data=stub_data) # Assert - assert dbnstore.nbytes == 189 + assert dbnstore.nbytes == 191 assert dbnstore.dataset == "GLBX.MDP3" assert dbnstore.schema == Schema.MBO assert dbnstore.symbols == ["ESH1"] @@ -171,14 +171,14 @@ def test_file_dbnstore_given_valid_path_initialized_expected_data( # Assert assert dbnstore.dataset == "GLBX.MDP3" - assert dbnstore.nbytes == 189 + assert dbnstore.nbytes == 191 @pytest.mark.parametrize( "schema,expected_size", [ - (Schema.MBO, 189), - (Schema.DEFINITION, 290), + (Schema.MBO, 191), + (Schema.DEFINITION, 288), ], ) def test_to_file_persists_to_disk( @@ -215,14 +215,14 @@ def test_to_file_overwrite( dbnstore = DBNStore.from_bytes(data=stub_data) dbn_path = tmp_path / "my_test.dbn" dbnstore.to_file(path=dbn_path) - assert dbn_path.stat().st_size == 189 + assert dbn_path.stat().st_size == 191 # Act dbnstore.to_file(path=dbn_path) # Assert assert dbn_path.exists() - assert dbn_path.stat().st_size == 189 + assert dbn_path.stat().st_size == 191 def test_to_file_exclusive( diff --git a/tests/test_live_client.py b/tests/test_live_client.py index cc7af676..dc22c15a 100644 --- a/tests/test_live_client.py +++ b/tests/test_live_client.py @@ -358,8 +358,9 @@ async def test_live_start( assert message.start_session -def test_live_start_twice( +async def test_live_start_twice( live_client: client.Live, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test that calling start() twice raises a ValueError. @@ -373,6 +374,10 @@ def test_live_start_twice( # Act live_client.start() + _ = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SessionStart, + ) + # Assert with pytest.raises(ValueError): live_client.start() @@ -389,8 +394,9 @@ def test_live_start_before_subscribe( live_client.start() -def test_live_iteration_after_start( +async def test_live_iteration_after_start( live_client: client.Live, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test that iterating the Live client after it is started raises a @@ -405,13 +411,18 @@ def test_live_iteration_after_start( # Act live_client.start() + _ = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SessionStart, + ) + # Assert with pytest.raises(ValueError): iter(live_client) -def test_live_async_iteration_after_start( +async def test_live_async_iteration_after_start( live_client: client.Live, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test that async-iterating the Live client after it is started raises a @@ -426,6 +437,10 @@ def test_live_async_iteration_after_start( # Act live_client.start() + _ = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SessionStart, + ) + # Assert with pytest.raises(ValueError): live_client.__aiter__() @@ -581,10 +596,11 @@ async def test_live_subscribe_large_symbol_list( ) reconstructed: list[str] = [] - for _ in range(8): + for i in range(8): message = await mock_live_server.wait_for_message_of_type( message_type=gateway.SubscriptionRequest, ) + assert int(message.is_last) == int(i == 7) reconstructed.extend(message.symbols.split(",")) # Assert @@ -758,9 +774,10 @@ def test_live_block_for_close( assert not live_client.is_connected() -def test_live_block_for_close_timeout( +async def test_live_block_for_close_timeout( live_client: client.Live, monkeypatch: pytest.MonkeyPatch, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test that block_for_close terminates the session when the timeout is @@ -777,15 +794,20 @@ def test_live_block_for_close_timeout( ) # Act, Assert + _ = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SubscriptionRequest, + ) + live_client.block_for_close(timeout=0) live_client.terminate.assert_called_once() # type: ignore @pytest.mark.usefixtures("mock_live_server") -def test_live_block_for_close_timeout_stream( +async def test_live_block_for_close_timeout_stream( live_client: client.Live, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test that block_for_close flushes user streams on timeout. @@ -804,6 +826,10 @@ def test_live_block_for_close_timeout_stream( live_client.add_stream(stream) # Act, Assert + _ = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SubscriptionRequest, + ) + live_client.block_for_close(timeout=0) stream.flush.assert_called() # type: ignore [attr-defined] @@ -836,6 +862,7 @@ async def test_live_wait_for_close( async def test_live_wait_for_close_timeout( live_client: client.Live, monkeypatch: pytest.MonkeyPatch, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test that wait_for_close terminates the session when the timeout is @@ -852,6 +879,11 @@ async def test_live_wait_for_close_timeout( symbols="ALL_SYMBOLS", start=None, ) + + _ = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SubscriptionRequest, + ) + await live_client.wait_for_close(timeout=0) # Assert @@ -863,6 +895,7 @@ async def test_live_wait_for_close_timeout_stream( live_client: client.Live, monkeypatch: pytest.MonkeyPatch, tmp_path: pathlib.Path, + mock_live_server: MockLiveServerInterface, ) -> None: """ Test that wait_for_close flushes user streams on timeout. @@ -882,6 +915,10 @@ async def test_live_wait_for_close_timeout_stream( live_client.add_stream(stream) # Act + _ = await mock_live_server.wait_for_message_of_type( + message_type=gateway.SubscriptionRequest, + ) + await live_client.wait_for_close(timeout=0) # Assert diff --git a/tests/test_live_gateway_messages.py b/tests/test_live_gateway_messages.py index 3deaadae..6183520a 100644 --- a/tests/test_live_gateway_messages.py +++ b/tests/test_live_gateway_messages.py @@ -352,7 +352,11 @@ def test_parse_subscription_request( stype_in=SType.INSTRUMENT_ID, symbols="1234,5678,90", ), - b"schema=mbo|" b"stype_in=instrument_id|" b"symbols=1234,5678,90|" b"snapshot=0\n", + b"schema=mbo|" + b"stype_in=instrument_id|" + b"symbols=1234,5678,90|" + b"snapshot=0|" + b"is_last=1\n", ), pytest.param( SubscriptionRequest( @@ -361,12 +365,14 @@ def test_parse_subscription_request( symbols="UNI,TTE,ST", start=1671717080706865759, snapshot=0, + is_last=0, ), b"schema=mbo|" b"stype_in=raw_symbol|" b"symbols=UNI,TTE,ST|" b"start=1671717080706865759|" - b"snapshot=0\n", + b"snapshot=0|" + b"is_last=0\n", ), pytest.param( SubscriptionRequest( @@ -381,7 +387,8 @@ def test_parse_subscription_request( b"stype_in=instrument_id|" b"symbols=1234,5678,90|" b"snapshot=1|" - b"id=5\n", + b"id=5|" + b"is_last=1\n", ), ], )