diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index bc0fdc3e..d439a873 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -24,6 +24,10 @@ jobs: --network container:dp3_api dp3_interpreter python -m unittest discover -s tests/test_api -v + - name: Dump Logs on Failure + if: failure() + run: docker compose logs + - name: Check worker errors run: docker compose logs worker | grep "WARNING\|ERROR\|exception" | grep -v "RabbitMQ\|it's\ OK\ now,\ we're\ successfully\ connected" || true diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 51b97842..ea10d842 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: - id: black - repo: https://github.com/charliermarsh/ruff-pre-commit - rev: 'v0.6.1' + rev: 'v0.7.4' hooks: - id: ruff args: [ "--fix", "." ] diff --git a/docs/api.md b/docs/api.md index 3998c08a..191690a2 100644 --- a/docs/api.md +++ b/docs/api.md @@ -150,20 +150,20 @@ More details depends on the particular type of the attribute. Can be represented using both **plain** attributes and **observations**. The difference will be only in time specification. Two examples using observations: -**no data - `link`**: just the eid is sent +**no data - `link`**: Sent as a dictionary with a single `"eid"` key. ```json { "type": "ip", "id": "192.168.0.1", "attr": "mac_addrs", - "v": "AA:AA:AA:AA:AA", + "v": {"eid": "AA:AA:AA:AA:AA"}, "t1": "2022-08-01T12:00:00", "t2": "2022-08-01T12:10:00" } ``` -**with additional data - `link`**: The eid and the data are sent as a dictionary. +**with additional data - `link`**: Sent as a dictionary with `"eid"` and `"data"` keys. ```json { @@ -198,11 +198,27 @@ v -> some_embedded_dict_field ## List entities -List latest snapshots of all ids present in database under entity type. +List latest snapshots of all ids present in database under entity type, +filtered by `generic_filter` and `fulltext_filters`. +Contains only the latest snapshot per entity. -Contains only latest snapshot. +Uses pagination, default limit is 20, setting to 0 will return all results. -Uses pagination. +Fulltext filters are interpreted as regular expressions. +Only string values may be filtered this way. There's no validation that queried attribute +can be fulltext filtered. +Only plain and observation attributes with string-based data types can be queried. +Array and set data types are supported as well as long as they are not multi value +at the same time. +If you need to filter EIDs, use attribute `eid`. + +Generic filter allows filtering using generic MongoDB query (including `$and`, `$or`,`$lt`, etc.). +For querying non-JSON-native types, you can use the following magic strings, +as are defined by the search & replace [`magic`][dp3.database.magic] module. + +There are no attribute name checks (may be added in the future). + +Generic and fulltext filters are merged - fulltext overrides conflicting keys. ### Request @@ -212,6 +228,8 @@ Uses pagination. - skip: how many entities to skip (default: 0) - limit: how many entities to return (default: 20) +- fulltext_filters: dictionary of fulltext filters (default: no filters) +- generic_filter: dictionary of generic filters (default: no filters) ### Response @@ -410,6 +428,7 @@ Returns dictionary containing all entity types configured -- their simplified co { "": { "id": "", + "id_data_type": "", "name": "", "attribs": "", "eid_estimate_count": "" diff --git a/docs/configuration/db_entities.md b/docs/configuration/db_entities.md index eeeb8bc8..1dc7d6a1 100644 --- a/docs/configuration/db_entities.md +++ b/docs/configuration/db_entities.md @@ -86,10 +86,23 @@ Entity is described simply by: | Parameter | Data-type | Default value | Description | |----------------|---------------------|---------------------|-------------------------------------------------------------------------------------------------------------------------------------------| | **`id`** | string (identifier) | *(mandatory)* | Short string identifying the entity type, it's machine name (must match regex `[a-zA-Z_][a-zA-Z0-9_-]*`). Lower-case only is recommended. | +| `id_data_type` | string | "string" | Data type of the entity id (`eid`) value, see [Supported eid data types](#supported-entity-id-data-types). | | **`name`** | string | *(mandatory)* | Attribute name for humans. May contain any symbols. | | **`snapshot`** | bool | *(mandatory)* | Whether to create snapshots of the entity. See [Architecture](../architecture.md#data-flow) for more details. | | `lifetime` | `Lifetime Spec` | `Immortal Lifetime` | Defines the lifetime of the entitiy, entities are never deleted by default. See the [Entity Lifetimes](lifetimes.md) for details. | +### Supported entity id data types + +Only a subset of [primitive data types](#primitive-types) is supported for entity ids. The supported data types are: + +- `string` (default) +- `int`: 32-bit signed integer (range from -2147483648 to +2147483647) +- `ipv4`: IPv4 address, represented as [IPv4Address][ipaddress.IPv4Address] (passed as dotted-decimal string) +- `ipv6`: IPv6 address, represented as [IPv6Address][ipaddress.IPv6Address] (passed as string in short or full format) +- `mac`: MAC address, represented as [MACAddress][dp3.common.mac_address.MACAddress] (passed as string) + +Whenever writing a piece of code independent of a specific configuration, +the [`AnyEidT`][dp3.common.datatype.AnyEidT] type alias should be used. ## Attributes @@ -196,9 +209,9 @@ List of supported values for parameter `data_type`: - `int64`: 64-bit signed integer (use when the range of normal `int` is not sufficent) - `float` - `time`: Timestamp in `YYYY-MM-DD[T]HH:MM[:SS[.ffffff]][Z or [±]HH[:]MM]` format or timestamp since 1.1.1970 in seconds or milliseconds. -- `ip4`: IPv4 address (passed as dotted-decimal string) -- `ip6`: IPv6 address (passed as string in short or full format) -- `mac`: MAC address (passed as string) +- `ipv4`: IPv4 address, represented as [IPv4Address][ipaddress.IPv4Address] (passed as dotted-decimal string) +- `ipv6`: IPv6 address, represented as [IPv6Address][ipaddress.IPv6Address] (passed as string in short or full format) +- `mac`: MAC address, represented as [MACAddress][dp3.common.mac_address.MACAddress] (passed as string) - `json`: Any JSON object can be stored, all processing is handled by user's code. This is here for special cases which can't be mapped to any other data type. #### Composite types diff --git a/docs/modules.md b/docs/modules.md index 5303baca..a5d646b3 100644 --- a/docs/modules.md +++ b/docs/modules.md @@ -122,6 +122,19 @@ and [`on_new_attr`](#attribute-hooks) callbacks, and you can enable it by passing the `refresh` keyword argument to the callback registration. See the Callbacks section for more details. +## Type of `eid` + +!!! tip "Specifying the `eid` type" + + At runtime, the `eid` will be exactly the type as specified in the entity specification. + +All the examples on this page will show the `eid` as a string, as that is the default type. +The type of the `eid` is can be configured in the entity specification, as is +detailed [here](configuration//db_entities.md#entity). + +The typehint of the `eid` used in callback registration definitions is the [ +`AnyEidT`][dp3.common.datatype.AnyEidT] type, which is a type alias of Union of all the allowed +types of the `eid` in the entity specification. ## Callbacks @@ -184,10 +197,13 @@ registrar.register_task_hook("on_task_start", task_hook) Receives eid and Task, may prevent entity record creation (by returning False). The callback is registered using the [`register_allow_entity_creation_hook`][dp3.common.callback_registrar.CallbackRegistrar.register_allow_entity_creation_hook] method. -Required signature is `Callable[[str, DataPointTask], bool]`. +Required signature is `Callable[[AnyEidT, DataPointTask], bool]`. ```python -def entity_creation(eid: str, task: DataPointTask) -> bool: +def entity_creation( + eid: str, # (1)! + task: DataPointTask, +) -> bool: return eid.startswith("1") registrar.register_allow_entity_creation_hook( @@ -195,16 +211,22 @@ registrar.register_allow_entity_creation_hook( ) ``` +1. `eid` may not be string, depending on the entity configuration, see [Type of + `eid`](#type-of-eid). + #### Entity `on_entity_creation` hook Receives eid and Task, may return new DataPointTasks. Callbacks which are called once when an entity is created are registered using the [`register_on_entity_creation_hook`][dp3.common.callback_registrar.CallbackRegistrar.register_on_entity_creation_hook] method. -Required signature is `Callable[[str, DataPointTask], list[DataPointTask]]`. +Required signature is `Callable[[AnyEidT, DataPointTask], list[DataPointTask]]`. ```python -def processing_function(eid: str, task: DataPointTask) -> list[DataPointTask]: +def processing_function( + eid: str, # (1)! + task: DataPointTask +) -> list[DataPointTask]: output = does_work(task) return [DataPointTask( model_spec=task.model_spec, @@ -224,6 +246,9 @@ registrar.register_on_entity_creation_hook( ) ``` +1. `eid` may not be string, depending on the entity configuration, see [Type of + `eid`](#type-of-eid). + The `register_on_entity_creation_hook` method also allows for refreshing of values derived by the registered hook. This can be done using the `refresh` keyword argument, (expecting a [`SharedFlag`][dp3.common.state.SharedFlag] object, which is created by default for all modules) and the `may_change` keyword argument, which lists all the attributes that the hook may change. @@ -243,10 +268,13 @@ registrar.register_on_entity_creation_hook( Callbacks that are called on every incoming datapoint of an attribute are registered using the [`register_on_new_attr_hook`][dp3.common.callback_registrar.CallbackRegistrar.register_on_new_attr_hook] method. The callback allways receives eid, attribute and Task, and may return new DataPointTasks. -The required signature is `Callable[[str, DataPointBase], Union[None, list[DataPointTask]]]`. +The required signature is `Callable[[AnyEidT, DataPointBase], Union[None, list[DataPointTask]]]`. ```python -def attr_hook(eid: str, dp: DataPointBase) -> list[DataPointTask]: +def attr_hook( + eid: str, # (1)! + dp: DataPointBase, +) -> list[DataPointTask]: ... return [] @@ -255,6 +283,9 @@ registrar.register_on_new_attr_hook( ) ``` +1. `eid` may not be string, depending on the entity configuration, see [Type of + `eid`](#type-of-eid). + This hook can be refreshed on configuration changes if you feel like the attribute value may change too slowly to catch up naturally. This can be done using the `refresh` keyword argument, (expecting a [`SharedFlag`][dp3.common.state.SharedFlag] object, which is created by default for all modules) @@ -290,7 +321,6 @@ def timeseries_hook( ) -> list[DataPointTask]: ... return [] - registrar.register_timeseries_hook( timeseries_hook, "test_entity_type", "test_attr_type", @@ -371,8 +401,8 @@ Learn more about the updater module in the [updater configuration](configuration #### Periodic Update Hook The [`register_periodic_update_hook`][dp3.common.callback_registrar.CallbackRegistrar.register_periodic_update_hook] -method expects a callable with the following signature: -`Callable[[str, str, dict], list[DataPointTask]]`, where the arguments are the entity type, +method expects a callable with the following signature: +`Callable[[str, AnyEidT, dict], list[DataPointTask]]`, where the arguments are the entity type, entity ID and master record. The callable should return a list of DataPointTask objects to perform (possibly empty). @@ -382,7 +412,11 @@ The following example shows how to register a periodic update hook for an entity The hook will be called for all entities of this type every day. ```python -def periodic_update_hook(entity_type: str, eid: str, record: dict) -> list[DataPointTask]: +def periodic_update_hook( + entity_type: str, + eid: str, # (1)! + record: dict, +) -> list[DataPointTask]: ... return [] @@ -391,6 +425,9 @@ registrar.register_periodic_update_hook( ) ``` +1. `eid` may not be string, depending on the entity configuration, see [Type of + `eid`](#type-of-eid). + !!! warning "Set a Realistic Update Period" Try to configure the period to match the real execution time of the registered hooks, @@ -405,12 +442,15 @@ This hook is useful when the entity record is not needed for the update, meaning The [`register_periodic_eid_update_hook`][dp3.common.callback_registrar.CallbackRegistrar.register_periodic_eid_update_hook] method expects a callable with the following signature: -`Callable[[str, str], list[DataPointTask]]`, where the first argument is the entity type and the second is the entity ID. +`Callable[[str, AnyEidT], list[DataPointTask]]`, where the first argument is the entity type and the second is the entity ID. The callable should return a list of DataPointTask objects to perform (possibly empty). All other arguments are the same as for the [periodic update hook](#periodic-update-hook). ```python -def periodic_eid_update_hook(entity_type: str, eid: str) -> list[DataPointTask]: +def periodic_eid_update_hook( + entity_type: str, + eid: str, # (1)! +) -> list[DataPointTask]: ... return [] @@ -419,6 +459,9 @@ registrar.register_periodic_eid_update_hook( ) ``` +1. `eid` may not be string, depending on the entity configuration, see [Type of + `eid`](#type-of-eid). + ## Running module code in a separate thread The module is free to run its own code in separate threads or processes. diff --git a/dp3/api/internal/config.py b/dp3/api/internal/config.py index 95b90e98..49b6821e 100644 --- a/dp3/api/internal/config.py +++ b/dp3/api/internal/config.py @@ -2,7 +2,11 @@ import sys from enum import Enum -from pydantic import BaseModel, ValidationError, field_validator +from pydantic import ( + BaseModel, + ValidationError, + field_validator, +) from dp3.api.internal.dp_logger import DPLogger from dp3.common.config import ModelSpec, read_config_dir diff --git a/dp3/api/internal/entity_response_models.py b/dp3/api/internal/entity_response_models.py index 7845d364..728f7701 100644 --- a/dp3/api/internal/entity_response_models.py +++ b/dp3/api/internal/entity_response_models.py @@ -1,9 +1,10 @@ from datetime import datetime -from typing import Annotated, Any, Optional +from typing import Annotated, Any, Optional, Union -from pydantic import BaseModel, Field, NonNegativeInt, RootModel +from pydantic import BaseModel, Field, NonNegativeInt, PlainSerializer from dp3.common.attrspec import AttrSpecType, AttrType +from dp3.common.datapoint import to_json_friendly class EntityState(BaseModel): @@ -14,11 +15,28 @@ class EntityState(BaseModel): """ id: str + id_data_type: str name: str attribs: dict[str, Annotated[AttrSpecType, Field(discriminator="type")]] eid_estimate_count: NonNegativeInt +# This is necessary to allow for non-JSON-serializable types in the model +JsonVal = Annotated[Any, PlainSerializer(to_json_friendly, when_used="json")] + +LinkVal = dict[str, JsonVal] +PlainVal = Union[LinkVal, JsonVal] +MultiVal = list[PlainVal] +HistoryVal = list[dict[str, PlainVal]] + +Dp3Val = Union[HistoryVal, MultiVal, PlainVal] + +EntityEidMasterRecord = dict[str, Dp3Val] + +SnapshotType = dict[str, Dp3Val] +EntityEidSnapshots = list[SnapshotType] + + class EntityEidList(BaseModel): """List of entity eids and their data based on latest snapshot @@ -31,12 +49,7 @@ class EntityEidList(BaseModel): time_created: Optional[datetime] = None count: int total_count: int - data: list[dict] - - -EntityEidMasterRecord = RootModel[dict] - -EntityEidSnapshots = RootModel[list[dict]] + data: EntityEidSnapshots class EntityEidData(BaseModel): @@ -48,8 +61,8 @@ class EntityEidData(BaseModel): """ empty: bool - master_record: dict - snapshots: list[dict] + master_record: EntityEidMasterRecord + snapshots: EntityEidSnapshots class EntityEidAttrValueOrHistory(BaseModel): @@ -62,8 +75,8 @@ class EntityEidAttrValueOrHistory(BaseModel): """ attr_type: AttrType - current_value: Any = None - history: list[dict] = [] + current_value: Dp3Val = None + history: HistoryVal = [] class EntityEidAttrValue(BaseModel): @@ -72,4 +85,4 @@ class EntityEidAttrValue(BaseModel): The value is fetched from master record. """ - value: Any = None + value: JsonVal = None diff --git a/dp3/api/internal/models.py b/dp3/api/internal/models.py index 47500a3b..b60133b3 100644 --- a/dp3/api/internal/models.py +++ b/dp3/api/internal/models.py @@ -1,8 +1,9 @@ from datetime import datetime -from typing import Annotated, Any, Optional +from typing import Annotated, Any, Literal, Optional, Union -from pydantic import BaseModel, Field, model_validator +from pydantic import BaseModel, Field, TypeAdapter, create_model, model_validator +from dp3.api.internal.config import MODEL_SPEC from dp3.api.internal.helpers import api_to_dp3_datapoint from dp3.common.types import T2Datetime @@ -23,7 +24,7 @@ class DataPoint(BaseModel): """ type: str - id: str + id: Any attr: str v: Any t1: Optional[datetime] = None @@ -40,3 +41,31 @@ def validate_against_attribute(self): raise ValueError(f"Missing key: {e}") from e return self + + +class EntityId(BaseModel): + """Dummy model for entity id + + Attributes: + type: Entity type + id: Entity ID + """ + + type: Literal["entity_type"] + id: Any + + +entity_id_models = [] +for entity_type, entity_spec in MODEL_SPEC.entities.items(): + dtype = entity_spec.eid_type + entity_id_models.append( + create_model( + f"EntityId{{{entity_type}}}", + __base__=BaseModel, + type=(Literal[entity_type], Field(..., alias="etype")), + id=(dtype, Field(..., alias="eid")), + ) + ) + +EntityId = Annotated[Union[tuple(entity_id_models)], Field(discriminator="type")] # noqa: F811 +EntityIdAdapter = TypeAdapter(EntityId) diff --git a/dp3/api/routers/entity.py b/dp3/api/routers/entity.py index 57cb759c..7130455e 100644 --- a/dp3/api/routers/entity.py +++ b/dp3/api/routers/entity.py @@ -1,5 +1,5 @@ from datetime import datetime -from typing import Any, Optional +from typing import Annotated, Any, Optional from fastapi import APIRouter, Depends, HTTPException, Query, Request from pydantic import Json, NonNegativeInt, ValidationError @@ -12,11 +12,10 @@ EntityEidList, EntityEidMasterRecord, EntityEidSnapshots, + JsonVal, ) from dp3.api.internal.helpers import api_to_dp3_datapoint -from dp3.api.internal.models import ( - DataPoint, -) +from dp3.api.internal.models import DataPoint, EntityId, EntityIdAdapter from dp3.api.internal.response_models import ErrorResponse, RequestValidationError, SuccessResponse from dp3.common.attrspec import AttrType from dp3.common.task import DataPointTask, task_context @@ -30,35 +29,46 @@ async def check_etype(etype: str): return etype +async def parse_eid(etype: str, eid: str): + """Middleware to parse EID""" + try: + return EntityIdAdapter.validate_python({"etype": etype, "eid": eid}) + except ValidationError as e: + raise RequestValidationError(["path", "eid"], e.errors()[0]["msg"]) from e + + +ParsedEid = Annotated[EntityId, Depends(parse_eid)] + + def get_eid_master_record_handler( - etype: str, eid: str, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None + e: EntityId, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None ): """Handler for getting master record of EID""" # TODO: This is probably not the most efficient way. Maybe gather only # plain data from master record and then call `get_timeseries_history` # for timeseries. master_record = DB.get_master_record( - etype, eid, projection={"_id": False, "#hash": False, "#min_t2s": False} + e.type, e.id, projection={"_id": False, "#hash": False, "#min_t2s": False} ) - entity_attribs = MODEL_SPEC.attribs(etype) + entity_attribs = MODEL_SPEC.attribs(e.type) # Get filtered timeseries data for attr in master_record: # Check for no longer existing attributes if attr in entity_attribs and entity_attribs[attr].t == AttrType.TIMESERIES: master_record[attr] = DB.get_timeseries_history( - etype, attr, eid, t1=date_from, t2=date_to + e.type, attr, e.id, t1=date_from, t2=date_to ) return master_record def get_eid_snapshots_handler( - etype: str, eid: str, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None -): + e: EntityId, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None +) -> list[dict[str, Any]]: """Handler for getting snapshots of EID""" - snapshots = list(DB.get_snapshots(etype, eid, t1=date_from, t2=date_to)) + snapshots = list(DB.snapshots.get_by_eid(e.type, e.id, t1=date_from, t2=date_to)) return snapshots @@ -100,6 +110,61 @@ async def list_entity_type_eids( Generic filter allows filtering using generic MongoDB query (including `$and`, `$or`, `$lt`, etc.). + For querying non-JSON-native types, you can use the following magic strings: + + - `"$$IPv4{}"` - converts to IPv4Address object + - `"$$IPv6{}"` - converts to IPv6Address object + - `"$$int{}"` - may be necessary for filtering when `eid` data type is int + - `"$$Date{}"` - converts specified UTC date to UTC datetime object + - `"$$DateTs{}"` - converts POSIX timestamp (int/float) to UTC datetime object + - `"$$MAC{}"` - converts to MACAddress object + + To query an IP prefix, use the following magic strings: + + - `"$$IPv4Prefix{/}"` - matches prefix + - `"$$IPv6Prefix{/}"` - matches prefix + + To query a binary `_id`s of non-string snapshot buckets, + use the following magic string: + + - `"$$Binary_ID{}"` + + - converts to filter the exact EID object snapshots, only EID valid types are supported + + There are no attribute name checks (may be added in the future). + + Generic filter examples: + + - `{"attr1": {"$gt": 10}, "attr2": {"$lt": 20}}` + - `{"ip_attr": "$$IPv4{127.0.0.1}"}` - converts to IPv4Address object, exact match + - `{"ip_attr": "$$IPv4Prefix{127.0.0.0/24}"}` + - converts to `{"ip_attr": {"$gte": "$$IPv4{127.0.0.0}", + "$lte": "$$IPv4{127.0.0.255}"}}` + + - `{"ip_attr": "$$IPv6{::1}"}` - converts to IPv6Address object, exact match + - `{"ip_attr": "$$IPv6Prefix{::1/64}"}` + - converts to `{"ip_attr": {"$gte": "$$IPv6{::1}", + "$lte": "$$IPv6{::1:ffff:ffff:ffff:ffff}"}}` + + - `{"_id": "$$Binary_ID{$$IPv4{127.0.0.1}}"}` + - converts to `{"_id": {"$gte": Binary(), + "$lt": Binary()}}` + + - `{"id": "$$Binary_ID{$$IPv4Prefix{127.0.0.0/24}}"}` + - converts to `{"_id": {"$gte": Binary(), + "$lte": Binary()}}` + + - `{"_time_created": {"$gte": "$$Date{2024-11-07T00:00:00Z}"}}` + - converts to `{"_time_created": datetime(2024, 11, 7, 0, 0, 0, tzinfo=timezone.utc)}` + + - `{"_time_created": {"$gte": "$$DateTs{1609459200}"}}` + - converts to `{"_time_created": datetime(2021, 1, 1, 0, 0, 0, tzinfo=timezone.utc)}` + + - `{"attr": "$$MAC{00:11:22:33:44:55}"}` - converts to MACAddress object, exact match + - `{"_id": "$$Binary_ID{$$MAC{Ab-cD-Ef-12-34-56}}"}` + - converts to `{"_id": {"$gte": Binary(), + "$lt": Binary()}}` + There are no attribute name checks (may be added in the future). Generic and fulltext filters are merged - fulltext overrides conflicting keys. @@ -124,10 +189,10 @@ async def list_entity_type_eids( fulltext_filters["eid"] = eid_filter try: - cursor, total_count = DB.get_latest_snapshots(etype, fulltext_filters, generic_filter) + cursor, total_count = DB.snapshots.get_latest(etype, fulltext_filters, generic_filter) cursor_page = cursor.skip(skip).limit(limit) except DatabaseError as e: - raise HTTPException(status_code=400, detail="Query is invalid") from e + raise HTTPException(status_code=400, detail=str(e)) from e time_created = None @@ -144,7 +209,7 @@ async def list_entity_type_eids( @router.get("/{etype}/{eid}") async def get_eid_data( - etype: str, eid: str, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None + e: ParsedEid, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None ) -> EntityEidData: """Get data of `etype`'s `eid`. @@ -153,8 +218,8 @@ async def get_eid_data( Combines function of `/{etype}/{eid}/master` and `/{etype}/{eid}/snapshots`. """ - master_record = get_eid_master_record_handler(etype, eid, date_from, date_to) - snapshots = get_eid_snapshots_handler(etype, eid, date_from, date_to) + master_record = get_eid_master_record_handler(e, date_from, date_to) + snapshots = get_eid_snapshots_handler(e, date_from, date_to) # Whether this eid contains any data empty = not master_record and len(snapshots) == 0 @@ -164,24 +229,23 @@ async def get_eid_data( @router.get("/{etype}/{eid}/master") async def get_eid_master_record( - etype: str, eid: str, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None + e: ParsedEid, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None ) -> EntityEidMasterRecord: """Get master record of `etype`'s `eid`.""" - return get_eid_master_record_handler(etype, eid, date_from, date_to) + return get_eid_master_record_handler(e, date_from, date_to) @router.get("/{etype}/{eid}/snapshots") async def get_eid_snapshots( - etype: str, eid: str, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None + e: ParsedEid, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None ) -> EntityEidSnapshots: """Get snapshots of `etype`'s `eid`.""" - return get_eid_snapshots_handler(etype, eid, date_from, date_to) + return get_eid_snapshots_handler(e, date_from, date_to) @router.get("/{etype}/{eid}/get/{attr}") async def get_eid_attr_value( - etype: str, - eid: str, + e: ParsedEid, attr: str, date_from: Optional[datetime] = None, date_to: Optional[datetime] = None, @@ -194,12 +258,14 @@ async def get_eid_attr_value( - history: in case of timeseries attribute """ # Check if attribute exists - if attr not in MODEL_SPEC.attribs(etype): + if attr not in MODEL_SPEC.attribs(e.type): raise RequestValidationError(["path", "attr"], f"Attribute '{attr}' doesn't exist") - value_or_history = DB.get_value_or_history(etype, attr, eid, t1=date_from, t2=date_to) + value_or_history = DB.get_value_or_history(e.type, attr, e.id, t1=date_from, t2=date_to) - return EntityEidAttrValueOrHistory(attr_type=MODEL_SPEC.attr(etype, attr).t, **value_or_history) + return EntityEidAttrValueOrHistory( + attr_type=MODEL_SPEC.attr(e.type, attr).t, **value_or_history + ) @router.post("/{etype}/{eid}/set/{attr}") @@ -248,7 +314,7 @@ async def set_eid_attr_value( "/{etype}/_/distinct/{attr}", responses={400: {"description": "Query can't be processed", "model": ErrorResponse}}, ) -async def get_distinct_attribute_values(etype: str, attr: str) -> dict[Any, int]: +async def get_distinct_attribute_values(etype: str, attr: str) -> dict[JsonVal, int]: """Gets distinct attribute values and their counts based on latest snapshots Useful for displaying `