diff --git a/README.md b/README.md index 88da561..e1503ef 100644 --- a/README.md +++ b/README.md @@ -33,14 +33,18 @@ - **Business logic**: the rules that determine how the domain behaves; validations, decisions, orchestration of use-cases, state transitions, etc. - **Plumbing**: transport/infrastructure glue (HTTP routing, request parsing, wiring dependencies) that carries inputs to the correct business logic and returns the result. - **Feature**: a cohesive capability (feedback reporting, player scraping, proxy rotation, etc.) that owns its domain rules, ~~DTOs~~ structs, ports, and integrations. Each feature lives inside a component so it can be reused by multiple bases/projects without duplication. -- **Models** = SQLAlchemy ORM classes mapped to concrete tables (live under `components/bot_detector/database/**/models`). Only the persistence layer (repositories/adapters that talk to storage) should touch them. -- **Structs** = Pydantic data shapes (requests/responses/contracts) shared across components/bases; they live under `components/bot_detector/structs` and replace the old “DTO” term. +- **Models** = SQLAlchemy ORM classes mapped to concrete tables (each feature owns its models under `components/bot_detector//database*.py`). Only the persistence layer (repositories/adapters that talk to storage) should touch them. +- **Structs** = Pydantic data shapes (requests/responses/contracts) shared across components/bases; each feature owns its structs (e.g., `components/bot_detector/player/structs.py`) and shared ones live under `bot_detector.core.structs`. They replace the old “DTO” term. +- **Service**: the domain façade exposed by a component (e.g., `PlayerService`). A service orchestrates one cohesive use-case—validation, repositories, messaging, caching—and is what bases call once they finish plumbing work. +- **Repository**: a persistence/adapter layer focused solely on talking to infrastructure (SQLAlchemy, Kafka, S3, HTTP APIs, etc.). Repositories are consumed by services and do not contain orchestration or HTTP-specific logic. +- **Manager**: an infrastructure helper that owns shared resources (Kafka producers, sessions, caches). Managers live alongside plumbing or support libraries and keep long-lived connections healthy. +- **Cache**: lightweight, in-process memoization layers (like `SimpleALRUCache`) that services can use for hot-path data. Caches never contain business logic; they simply store/retrieve entities to reduce load on repositories. ## Working Standards -- **Components** encapsulate each feature’s business logic plus adapters, and may depend on other components/libraries only. -- **Bases** expose public APIs and handle plumbing only (routing, request parsing, dependency wiring) before delegating to components. +- **Components** encapsulate each feature’s business logic plus adapters, and may depend on other components/libraries only. Services may call repositories/adapters but never import FastAPI or base code. +- **Bases** expose public APIs and must stay thin: they handle routing, validation, dependency wiring, and immediately delegate to services. Bases never import ORM models or implement business rules. - **Projects** only compose bricks + libraries into deployable artifacts; they hold wiring/config, never feature code. -- **Shared structs** (DTOs, interfaces) belong in reusable components like `components/bot_detector/structs` so every base/project can import them without circular dependencies. +- **Shared structs** (DTOs, interfaces) live in reusable modules such as `bot_detector.core.structs`, while feature-specific structs stay within their feature packages. Use the `structs` naming everywhere (no `schemas` leftovers) and suffix types consistently (`FooInput`, `FooResponse`, etc.). - **Tests** live under the workspace-level `test/` directory via `[tool.polylith.test]`, so base/component fixtures and contract tests should be added there rather than inside each brick folder. Add per-base `resources/` directories only when a base needs static assets or config that isn’t shared elsewhere. # The Polylith Architecture @@ -125,4 +129,4 @@ find . -type f -name "pyproject.toml" -not -path "*/.venv/*" -execdir sh -c 'ech # syncing in all directories, so uv cache is setup ```sh find . -type f -name "pyproject.toml" -not -path "*/.venv/*" -execdir sh -c 'echo "🔄 syncing in $(pwd)"; uv sync' \; -``` \ No newline at end of file +``` diff --git a/bases/bot_detector/api_public/src/__init__.py b/bases/bot_detector/api_public/__init__.py similarity index 100% rename from bases/bot_detector/api_public/src/__init__.py rename to bases/bot_detector/api_public/__init__.py diff --git a/bases/bot_detector/api_public/src/app/.gitkeep b/bases/bot_detector/api_public/core/.gitkeep similarity index 100% rename from bases/bot_detector/api_public/src/app/.gitkeep rename to bases/bot_detector/api_public/core/.gitkeep diff --git a/bases/bot_detector/api_public/src/core/__init__.py b/bases/bot_detector/api_public/core/__init__.py similarity index 100% rename from bases/bot_detector/api_public/src/core/__init__.py rename to bases/bot_detector/api_public/core/__init__.py diff --git a/bases/bot_detector/api_public/src/core/_cache.py b/bases/bot_detector/api_public/core/_cache.py similarity index 100% rename from bases/bot_detector/api_public/src/core/_cache.py rename to bases/bot_detector/api_public/core/_cache.py diff --git a/bases/bot_detector/api_public/src/core/config.py b/bases/bot_detector/api_public/core/config.py similarity index 100% rename from bases/bot_detector/api_public/src/core/config.py rename to bases/bot_detector/api_public/core/config.py diff --git a/bases/bot_detector/api_public/src/core/fastapi/dependencies/kafka.py b/bases/bot_detector/api_public/core/fastapi/dependencies/kafka.py similarity index 100% rename from bases/bot_detector/api_public/src/core/fastapi/dependencies/kafka.py rename to bases/bot_detector/api_public/core/fastapi/dependencies/kafka.py diff --git a/bases/bot_detector/api_public/src/core/fastapi/dependencies/session.py b/bases/bot_detector/api_public/core/fastapi/dependencies/session.py similarity index 73% rename from bases/bot_detector/api_public/src/core/fastapi/dependencies/session.py rename to bases/bot_detector/api_public/core/fastapi/dependencies/session.py index 8bad32d..0772815 100644 --- a/bases/bot_detector/api_public/src/core/fastapi/dependencies/session.py +++ b/bases/bot_detector/api_public/core/fastapi/dependencies/session.py @@ -1,6 +1,5 @@ -from bot_detector.api_public.src.core.config import DB_SEMAPHORE, settings -from bot_detector.database import Settings as DBSettings -from bot_detector.database import get_session_factory +from bot_detector.api_public.core.config import DB_SEMAPHORE, settings +from bot_detector.core.database import Settings as DBSettings, get_session_factory from sqlalchemy.ext.asyncio import AsyncSession # Reuse the shared database component instead of maintaining copy diff --git a/bases/bot_detector/api_public/src/core/fastapi/dependencies/to_jagex_name.py b/bases/bot_detector/api_public/core/fastapi/dependencies/to_jagex_name.py similarity index 100% rename from bases/bot_detector/api_public/src/core/fastapi/dependencies/to_jagex_name.py rename to bases/bot_detector/api_public/core/fastapi/dependencies/to_jagex_name.py diff --git a/bases/bot_detector/api_public/src/core/fastapi/middleware/__init__.py b/bases/bot_detector/api_public/core/fastapi/middleware/__init__.py similarity index 100% rename from bases/bot_detector/api_public/src/core/fastapi/middleware/__init__.py rename to bases/bot_detector/api_public/core/fastapi/middleware/__init__.py diff --git a/bases/bot_detector/api_public/src/core/fastapi/middleware/logging.py b/bases/bot_detector/api_public/core/fastapi/middleware/logging.py similarity index 100% rename from bases/bot_detector/api_public/src/core/fastapi/middleware/logging.py rename to bases/bot_detector/api_public/core/fastapi/middleware/logging.py diff --git a/bases/bot_detector/api_public/src/core/fastapi/middleware/metrics.py b/bases/bot_detector/api_public/core/fastapi/middleware/metrics.py similarity index 100% rename from bases/bot_detector/api_public/src/core/fastapi/middleware/metrics.py rename to bases/bot_detector/api_public/core/fastapi/middleware/metrics.py diff --git a/bases/bot_detector/api_public/src/core/logging.py b/bases/bot_detector/api_public/core/logging.py similarity index 100% rename from bases/bot_detector/api_public/src/core/logging.py rename to bases/bot_detector/api_public/core/logging.py diff --git a/bases/bot_detector/api_public/src/core/server.py b/bases/bot_detector/api_public/core/server.py similarity index 85% rename from bases/bot_detector/api_public/src/core/server.py rename to bases/bot_detector/api_public/core/server.py index 073f065..056be97 100644 --- a/bases/bot_detector/api_public/src/core/server.py +++ b/bases/bot_detector/api_public/core/server.py @@ -1,79 +1,80 @@ -import logging -from contextlib import asynccontextmanager - -from bot_detector.api_public.src import api -from bot_detector.api_public.src.core.fastapi.dependencies.kafka import kafka_manager -from bot_detector.api_public.src.core.fastapi.middleware import ( - LoggingMiddleware, - PrometheusMiddleware, -) -from bot_detector.kafka import Settings as KafkaSettings -from bot_detector.kafka.repositories import RepoReportsToInsertProducer -from fastapi import FastAPI -from fastapi.middleware import Middleware -from fastapi.middleware.cors import CORSMiddleware -from prometheus_client import start_http_server - -logger = logging.getLogger(__name__) - - -def init_routers(_app: FastAPI) -> None: - _app.include_router(api.router) - - -def make_middleware() -> list[Middleware]: - middleware = [ - Middleware( - CORSMiddleware, - allow_origins=[ - "http://osrsbotdetector.com/", - "https://osrsbotdetector.com/", - "http://localhost", - "http://localhost:8080", - ], - allow_credentials=True, - allow_methods=["*"], - allow_headers=["*"], - ), - Middleware(LoggingMiddleware), - Middleware(PrometheusMiddleware), - ] - return middleware - - -@asynccontextmanager -async def lifespan(app: FastAPI): - logger.info("startup initiated") - kafka_manager.set_producer( - key="reports_to_insert", - producer=RepoReportsToInsertProducer( - bootstrap_servers=KafkaSettings().KAFKA_BOOTSTRAP_SERVERS - ), - ) - producer = kafka_manager.get_producer(key="reports_to_insert") - await producer.start() - yield - await producer.stop() - logger.info("shutdown completed") - - -def create_app() -> FastAPI: - _app = FastAPI( - title="Bot-Detector-API", - description="Bot-Detector-API", - middleware=make_middleware(), - lifespan=lifespan, - ) - init_routers(_app=_app) - return _app - - -app = create_app() - - -start_http_server(8000) - - -@app.get("/") -async def root(): - return {"message": "Hello World"} +import logging +from contextlib import asynccontextmanager + +from bot_detector.kafka import Settings as KafkaSettings +from bot_detector.kafka.repositories import RepoReportsToInsertProducer +from fastapi import FastAPI +from fastapi.middleware import Middleware +from fastapi.middleware.cors import CORSMiddleware +from prometheus_client import start_http_server + +from bases.bot_detector.api_public import routes +from bases.bot_detector.api_public.core.fastapi.dependencies.kafka import kafka_manager +from bases.bot_detector.api_public.core.fastapi.middleware import ( + LoggingMiddleware, + PrometheusMiddleware, +) + +logger = logging.getLogger(__name__) + + +def init_routers(_app: FastAPI) -> None: + _app.include_router(routes.router) + + +def make_middleware() -> list[Middleware]: + middleware = [ + Middleware( + CORSMiddleware, + allow_origins=[ + "http://osrsbotdetector.com/", + "https://osrsbotdetector.com/", + "http://localhost", + "http://localhost:8080", + ], + allow_credentials=True, + allow_methods=["*"], + allow_headers=["*"], + ), + Middleware(LoggingMiddleware), + Middleware(PrometheusMiddleware), + ] + return middleware + + +@asynccontextmanager +async def lifespan(app: FastAPI): + logger.info("startup initiated") + kafka_manager.set_producer( + key="reports_to_insert", + producer=RepoReportsToInsertProducer( + bootstrap_servers=KafkaSettings().KAFKA_BOOTSTRAP_SERVERS + ), + ) + producer = kafka_manager.get_producer(key="reports_to_insert") + await producer.start() + yield + await producer.stop() + logger.info("shutdown completed") + + +def create_app() -> FastAPI: + _app = FastAPI( + title="Bot-Detector-API", + description="Bot-Detector-API", + middleware=make_middleware(), + lifespan=lifespan, + ) + init_routers(_app=_app) + return _app + + +app = create_app() + + +start_http_server(8000) + + +@app.get("/") +async def root(): + return {"message": "Hello World"} diff --git a/bases/bot_detector/api_public/routes/__init__.py b/bases/bot_detector/api_public/routes/__init__.py new file mode 100644 index 0000000..dc15a3a --- /dev/null +++ b/bases/bot_detector/api_public/routes/__init__.py @@ -0,0 +1,24 @@ +from fastapi import APIRouter + +from bases.bot_detector.api_public.routes.v2.player import router as player_router +from bases.bot_detector.api_public.routes.v2.reports import router as reports_router +from bases.bot_detector.api_public.routes.v2.feedback import router as feedback_router +from bases.bot_detector.api_public.routes.v2.labels import router as labels_router + + +def _build_v2() -> APIRouter: + api_router = APIRouter(prefix="/v2") + for feature_router in ( + player_router, + reports_router, + feedback_router, + labels_router, + ): + api_router.include_router(feature_router) + return api_router + + +router = APIRouter() +router.include_router(_build_v2()) + +__all__ = ["router"] diff --git a/bases/bot_detector/api_public/routes/v2/feedback.py b/bases/bot_detector/api_public/routes/v2/feedback.py new file mode 100644 index 0000000..4c625db --- /dev/null +++ b/bases/bot_detector/api_public/routes/v2/feedback.py @@ -0,0 +1,24 @@ +import logging + +from bot_detector.feedback.services import FeedbackService +from bot_detector.feedback.structs import FeedbackInput +from bot_detector.core.structs.responses import Ok +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session +from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import to_jagex_name +from fastapi import APIRouter, Depends, HTTPException, status + +router = APIRouter(tags=["Feedback"]) +logger = logging.getLogger(__name__) + + +@router.post("/feedback", response_model=Ok, status_code=status.HTTP_201_CREATED) +async def post_feedback( + feedback: FeedbackInput, + session=Depends(get_session), +): + repo = FeedbackService(session) + feedback.player_name = await to_jagex_name(feedback.player_name) + success, detail = await repo.insert_feedback(feedback=feedback) + if not success: + raise HTTPException(status_code=422, detail=detail) + return Ok(detail=detail) diff --git a/bases/bot_detector/api_public/routes/v2/labels.py b/bases/bot_detector/api_public/routes/v2/labels.py new file mode 100644 index 0000000..7696d06 --- /dev/null +++ b/bases/bot_detector/api_public/routes/v2/labels.py @@ -0,0 +1,32 @@ +import logging + +from bot_detector.labels.services import LabelService +from bot_detector.labels.structs import LabelResponse +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session +from fastapi import APIRouter, Depends, status + +router = APIRouter(tags=["Labels"]) +logger = logging.getLogger(__name__) + + +@router.get("/labels", response_model=list[LabelResponse], status_code=status.HTTP_200_OK) +async def get_labels(session=Depends(get_session)): + repo = LabelService(session) + labels = await repo.get_labels() + _labels = [] + for label in labels: + res = LabelResponse(**label.__dict__) + res.label = res.label.lower() + _labels.append(res) + return _labels + + +@router.get("/labels/{label_id}", response_model=LabelResponse | None) +async def get_label_by_id(label_id: int, session=Depends(get_session)) -> LabelResponse | None: + repo = LabelService(session) + label = await repo.get_label_by_id(label_id=label_id) + if label is None: + return None + res = LabelResponse(**label.__dict__) + res.label = res.label.lower() + return res diff --git a/bases/bot_detector/api_public/src/api/v2/player.py b/bases/bot_detector/api_public/routes/v2/player.py similarity index 50% rename from bases/bot_detector/api_public/src/api/v2/player.py rename to bases/bot_detector/api_public/routes/v2/player.py index 0f4bfd4..5d08fab 100644 --- a/bases/bot_detector/api_public/src/api/v2/player.py +++ b/bases/bot_detector/api_public/routes/v2/player.py @@ -1,107 +1,71 @@ -import asyncio -import logging -from typing import Annotated - -from bot_detector.api_public.src.app.repositories.player import Player as repoPlayer -from bot_detector.api_public.src.app.views.response.feedback_score import ( - FeedbackScoreResponse, -) -from bot_detector.api_public.src.app.views.response.prediction import PredictionResponse -from bot_detector.api_public.src.app.views.response.report_score import ( - ReportScoreResponse, -) -from bot_detector.api_public.src.core.fastapi.dependencies.session import get_session -from bot_detector.api_public.src.core.fastapi.dependencies.to_jagex_name import ( - to_jagex_name, -) -from fastapi import APIRouter, Depends, HTTPException, Query, status -from pydantic.fields import Field - -router = APIRouter(tags=["Player"]) -logger = logging.getLogger(__name__) - - -@router.get("/player/report/score", response_model=list[ReportScoreResponse]) -async def get_players_kc( - name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( - ..., - min_length=1, - description="Name of the player", - examples=["Player1", "Player2"], - ), - session=Depends(get_session), -): - """ - Get the report score for one or multiple players. - - Args: - name (str): can be provided multiple times - - Returns: - list[ReportScoreResponse]: A list of dictionaries containing KC data for each player. - """ - repo = repoPlayer(session) - names = await asyncio.gather(*[to_jagex_name(n) for n in name]) - data = await repo.get_report_score(player_names=tuple(names)) - return data - - -@router.get("/player/feedback/score", response_model=list[FeedbackScoreResponse]) -async def get_feedback_score( - name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( - ..., - min_length=1, - description="Name of the player", - examples=["Player1", "Player2"], - ), - session=Depends(get_session), -): - """ - Get the feedback score for one or multiple players. - - Args: - name (str): can be provided multiple times - - Returns: - list[FeedbackScoreResponse]: A list of dictionaries containing KC data for each player. - """ - repo = repoPlayer(session) - names = await asyncio.gather(*[to_jagex_name(n) for n in name]) - data = await repo.get_feedback_score(player_names=names) - return data - - -@router.get("/player/prediction", response_model=list[PredictionResponse]) -async def get_prediction( - name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( - ..., - min_length=1, - max_length=5, - description="Name of the player", - examples=["Player1", "Player2"], - ), - breakdown: bool = Query(...), - session=Depends(get_session), -): - """ - Get prediction data for one or multiple users. - - Args: - name (str): The username of the user for whom predictions are requested. - breakdown (bool): A flag indicating whether to include a breakdown of predictions. - - Returns: - List[PredictionResponse]: A list of PredictionResponse objects containing prediction data. - - Raises: - HTTPException: Returns a 404 error with the message "Player not found" if no data is found for the user. - - """ - repo = repoPlayer(session) - names = await asyncio.gather(*[to_jagex_name(n) for n in name]) - data = await repo.get_prediction(player_names=names) - if not data: - raise HTTPException( - status_code=status.HTTP_404_NOT_FOUND, detail="Player not found" - ) - return [PredictionResponse.from_data(d, breakdown) for d in data] +import asyncio +import logging +from typing import Annotated + +from bot_detector.player.services import PlayerService +from bot_detector.player.structs import ( + FeedbackScoreResponse, + PredictionResponse, + ReportScoreResponse, +) +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session +from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import ( + to_jagex_name, +) +from fastapi import APIRouter, Depends, HTTPException, Query, status +from pydantic.fields import Field + +router = APIRouter(tags=["Player"]) +logger = logging.getLogger(__name__) + + +@router.get("/player/report/score", response_model=list[ReportScoreResponse]) +async def get_players_kc( + name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( + ..., + min_length=1, + description="Name of the player", + examples=["Player1", "Player2"], + ), + session=Depends(get_session), +): + repo = PlayerService(session) + names = await asyncio.gather(*[to_jagex_name(n) for n in name]) + return await repo.get_report_score(player_names=tuple(names)) + + +@router.get("/player/feedback/score", response_model=list[FeedbackScoreResponse]) +async def get_feedback_score( + name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( + ..., + min_length=1, + description="Name of the player", + examples=["Player1", "Player2"], + ), + session=Depends(get_session), +): + repo = PlayerService(session) + names = await asyncio.gather(*[to_jagex_name(n) for n in name]) + return await repo.get_feedback_score(player_names=names) + + +@router.get("/player/prediction", response_model=list[PredictionResponse]) +async def get_prediction( + name: list[Annotated[str, Field(..., min_length=1, max_length=13)]] = Query( + ..., + min_length=1, + max_length=5, + description="Name of the player", + examples=["Player1", "Player2"], + ), + breakdown: bool = Query(...), + session=Depends(get_session), +): + repo = PlayerService(session) + names = await asyncio.gather(*[to_jagex_name(n) for n in name]) + data = await repo.get_prediction(player_names=names) + if not data: + raise HTTPException( + status_code=status.HTTP_404_NOT_FOUND, detail="Player not found" + ) + return [PredictionResponse.from_data(d, breakdown) for d in data] diff --git a/bases/bot_detector/api_public/src/api/v2/report.py b/bases/bot_detector/api_public/routes/v2/reports.py similarity index 60% rename from bases/bot_detector/api_public/src/api/v2/report.py rename to bases/bot_detector/api_public/routes/v2/reports.py index dad6a5f..0683a23 100644 --- a/bases/bot_detector/api_public/src/api/v2/report.py +++ b/bases/bot_detector/api_public/routes/v2/reports.py @@ -1,70 +1,65 @@ -import logging - -from bot_detector.api_public.src.app.repositories.player import Player -from bot_detector.api_public.src.app.repositories.report import CustomError, Report -from bot_detector.api_public.src.app.views.response.ok import Ok -from bot_detector.api_public.src.core._cache import SimpleALRUCache -from bot_detector.api_public.src.core.fastapi.dependencies.session import get_session -from bot_detector.structs import Detection, ParsedDetection -from fastapi import APIRouter, Depends, status -from fastapi.exceptions import HTTPException -from sqlalchemy.ext.asyncio import AsyncSession - -logger = logging.getLogger(__name__) -router = APIRouter(tags=["Report"]) - -player_cache = SimpleALRUCache(max_size=100_000) - - -@router.post("/report", status_code=status.HTTP_201_CREATED, response_model=Ok) -async def post_reports( - detections: list[Detection], - session: AsyncSession = Depends(get_session), -): - global player_cache - report_repo = Report() - player_repo = Player(session=session, cache=player_cache) - - data, error = await report_repo.parse_data(detections) - if error: - raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=error) - - logger.debug(f"Received: {len(data)}, Reporter: {data[0].reporter}") - - # get unique list of names - player_names = list(set([d.reported for d in data] + [d.reporter for d in data])) - players = [await player_repo.get_or_insert(player_name=p) for p in player_names] - players = {p.name: p.id for p in players if p} - - _data = [] - for d in data: - _d = d.model_dump() - # get reported_id from name - reported = player_repo.sanitize_name(_d.pop("reported")) - reported_id = players.get(reported) - - # get reporter_id from name - reporter = player_repo.sanitize_name(_d.pop("reporter")) - reporter_id = players.get(reporter) - - # some validation - if reporter_id is None or reported_id is None: - logger.warning(msg=f"{reported_id=}, {reporter_id=}, {d}") - raise HTTPException( - status_code=status.HTTP_400_BAD_REQUEST, - detail="something went wrong", - ) - _d["reported_id"] = reported_id - _d["reporter_id"] = reporter_id - - _data.append(ParsedDetection(**_d)) - - # print(_data) - try: - await report_repo.send_to_kafka(data=_data) - except CustomError: - raise HTTPException( - status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, - detail="Internal error", - ) - return Ok() +import logging + +from bot_detector.core.cache import SimpleALRUCache +from bot_detector.core.structs.responses import Ok +from bot_detector.player.services import PlayerService +from bot_detector.report.services.reports import CustomError, ReportsService +from bot_detector.report.structs import ( + Detection, + ParsedDetection, +) +from fastapi import APIRouter, Depends, status +from fastapi.exceptions import HTTPException +from sqlalchemy.ext.asyncio import AsyncSession + +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session + +router = APIRouter(tags=["Report"]) +logger = logging.getLogger(__name__) +player_cache = SimpleALRUCache(max_size=100_000) + + +@router.post("/report", status_code=status.HTTP_201_CREATED, response_model=Ok) +async def post_reports( + detections: list[Detection], + session: AsyncSession = Depends(get_session), +): + global player_cache + report_repo = ReportsService() + player_repo = PlayerService(session=session, cache=player_cache) + + data, error = await report_repo.parse_data(detections) + if error: + raise HTTPException(status.HTTP_400_BAD_REQUEST, detail=error) + + logger.debug(f"Received: {len(data)}, Reporter: {data[0].reporter}") + + player_names = list(set([d.reported for d in data] + [d.reporter for d in data])) + players = [await player_repo.get_or_insert(player_name=p) for p in player_names] + players = {p.name: p.id for p in players if p} + + _data = [] + for d in data: + payload = d.model_dump() + reported = player_repo.sanitize_name(payload.pop("reported")) + reported_id = players.get(reported) + reporter = player_repo.sanitize_name(payload.pop("reporter")) + reporter_id = players.get(reporter) + if reporter_id is None or reported_id is None: + logger.warning(msg=f"{reported_id=}, {reporter_id=}, {d}") + raise HTTPException( + status_code=status.HTTP_400_BAD_REQUEST, + detail="something went wrong", + ) + payload["reported_id"] = reported_id + payload["reporter_id"] = reporter_id + _data.append(ParsedDetection(**payload)) + + try: + await report_repo.send_to_kafka(data=_data) + except CustomError: + raise HTTPException( + status_code=status.HTTP_500_INTERNAL_SERVER_ERROR, + detail="Internal error", + ) + return Ok() diff --git a/bases/bot_detector/api_public/src/api/__init__.py b/bases/bot_detector/api_public/src/api/__init__.py deleted file mode 100644 index a121193..0000000 --- a/bases/bot_detector/api_public/src/api/__init__.py +++ /dev/null @@ -1,6 +0,0 @@ -from fastapi import APIRouter - -from . import v2 - -router = APIRouter() -router.include_router(v2.router, prefix="/v2") diff --git a/bases/bot_detector/api_public/src/api/readme.md b/bases/bot_detector/api_public/src/api/readme.md deleted file mode 100644 index 0ff400d..0000000 --- a/bases/bot_detector/api_public/src/api/readme.md +++ /dev/null @@ -1 +0,0 @@ -the api folder can be considered the controller in the MVC approach \ No newline at end of file diff --git a/bases/bot_detector/api_public/src/api/v2/__init__.py b/bases/bot_detector/api_public/src/api/v2/__init__.py deleted file mode 100644 index cc967e7..0000000 --- a/bases/bot_detector/api_public/src/api/v2/__init__.py +++ /dev/null @@ -1,9 +0,0 @@ -from fastapi import APIRouter - -from . import feedback, labels, player, report - -router = APIRouter() -router.include_router(player.router) -router.include_router(report.router) -router.include_router(feedback.router) -router.include_router(labels.router) diff --git a/bases/bot_detector/api_public/src/api/v2/feedback.py b/bases/bot_detector/api_public/src/api/v2/feedback.py deleted file mode 100644 index d9ef4f4..0000000 --- a/bases/bot_detector/api_public/src/api/v2/feedback.py +++ /dev/null @@ -1,29 +0,0 @@ -import logging - -from bot_detector.api_public.src.app.repositories.feedback import Feedback -from bot_detector.api_public.src.app.views.input.feedback import FeedbackInput -from bot_detector.api_public.src.app.views.response.ok import Ok -from bot_detector.api_public.src.core.fastapi.dependencies.session import get_session -from bot_detector.api_public.src.core.fastapi.dependencies.to_jagex_name import ( - to_jagex_name, -) -from fastapi import APIRouter, Depends, HTTPException, status - -router = APIRouter(tags=["Feedback"]) -logger = logging.getLogger(__name__) - - -@router.post("/feedback", response_model=Ok, status_code=status.HTTP_201_CREATED) -async def post_feedback( - feedback: FeedbackInput, - session=Depends(get_session), -): - """ """ - _feedback = Feedback(session) - - feedback.player_name = await to_jagex_name(feedback.player_name) - - success, detail = await _feedback.insert_feedback(feedback=feedback) - if not success: - raise HTTPException(status_code=422, detail=detail) - return Ok(detail=detail) diff --git a/bases/bot_detector/api_public/src/api/v2/labels.py b/bases/bot_detector/api_public/src/api/v2/labels.py deleted file mode 100644 index 93c70a9..0000000 --- a/bases/bot_detector/api_public/src/api/v2/labels.py +++ /dev/null @@ -1,44 +0,0 @@ -import logging - -from bot_detector.api_public.src.app.repositories.labels import LabelRepository -from bot_detector.api_public.src.app.views.response.label import LabelResponse -from bot_detector.api_public.src.core.fastapi.dependencies.session import get_session -from fastapi import APIRouter, Depends, status - -router = APIRouter(tags=["Labels"]) -logger = logging.getLogger(__name__) - - -@router.get( - "/labels", - response_model=list[LabelResponse], - status_code=status.HTTP_200_OK, -) -async def get_labels(session=Depends(get_session)): - _label_repo = LabelRepository(session) - labels = await _label_repo.get_labels() - - _labels = [] - for label in labels: - _label = LabelResponse(**label.__dict__) - _label.label = _label.label.lower() - _labels.append(_label) - return _labels - - -@router.get( - "/labels/{label_id}", - response_model=LabelResponse | None, - status_code=status.HTTP_200_OK, -) -async def get_label_by_id( - label_id: int, session=Depends(get_session) -) -> LabelResponse | None: - _label_repo = LabelRepository(session) - label = await _label_repo.get_label_by_id(label_id=label_id) - - if label is None: - return None - _label = LabelResponse(**label.__dict__) - _label.label = _label.label.lower() - return _label diff --git a/bases/bot_detector/api_public/src/app/readme.md b/bases/bot_detector/api_public/src/app/readme.md deleted file mode 100644 index 64b97ea..0000000 --- a/bases/bot_detector/api_public/src/app/readme.md +++ /dev/null @@ -1,6 +0,0 @@ -the model is responsible for all the data handeling -- getting data from the database -- handles data logic - -the view is responsible for the data representation -- return format etc \ No newline at end of file diff --git a/bases/bot_detector/api_public/src/app/repositories/.gitkeep b/bases/bot_detector/api_public/src/app/repositories/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/bases/bot_detector/api_public/src/app/repositories/__init__.py b/bases/bot_detector/api_public/src/app/repositories/__init__.py deleted file mode 100644 index e69de29..0000000 diff --git a/bases/bot_detector/api_public/src/app/views/input/_metadata.py b/bases/bot_detector/api_public/src/app/views/input/_metadata.py deleted file mode 100644 index bd87da8..0000000 --- a/bases/bot_detector/api_public/src/app/views/input/_metadata.py +++ /dev/null @@ -1,5 +0,0 @@ -from pydantic import BaseModel - - -class Metadata(BaseModel): - version: str diff --git a/bases/bot_detector/api_public/src/app/views/player.py b/bases/bot_detector/api_public/src/app/views/player.py deleted file mode 100644 index 482b971..0000000 --- a/bases/bot_detector/api_public/src/app/views/player.py +++ /dev/null @@ -1,48 +0,0 @@ -from datetime import datetime -from typing import Optional - -from pydantic import BaseModel, field_validator - - -class PlayerCreate(BaseModel): - name: str - possible_ban: Optional[bool] = 0 - confirmed_ban: Optional[bool] = 0 - confirmed_player: Optional[bool] = 0 - label_id: Optional[int] = 0 - label_jagex: Optional[int] = 0 - ironman: Optional[int] = None - hardcore_ironman: Optional[int] = None - ultimate_ironman: Optional[int] = None - normalized_name: Optional[str] = None - - -class PlayerUpdate(BaseModel): - name: Optional[str] = None - possible_ban: Optional[bool] = None - confirmed_ban: Optional[bool] = None - confirmed_player: Optional[bool] = None - label_id: Optional[int] = None - label_jagex: Optional[int] = None - ironman: Optional[int] = None - hardcore_ironman: Optional[int] = None - ultimate_ironman: Optional[int] = None - normalized_name: Optional[str] = None - - -class PlayerInDB(PlayerCreate): - id: int - created_at: datetime - updated_at: datetime | None - - @field_validator("created_at", mode="before") - def parse_created_at(cls, value): - if isinstance(value, str): - return datetime.fromisoformat(value) - if value is None: - raise ValueError("created_at cannot be None") - return value - - -class Player(PlayerInDB): - pass diff --git a/bases/bot_detector/api_public/src/app/views/response/feedback.py b/bases/bot_detector/api_public/src/app/views/response/feedback.py deleted file mode 100644 index f0bf81f..0000000 --- a/bases/bot_detector/api_public/src/app/views/response/feedback.py +++ /dev/null @@ -1,11 +0,0 @@ -from typing import Optional - -from pydantic import BaseModel, Field - - -class FeedbackScore(BaseModel): - count: int - possible_ban: bool - confirmed_ban: bool - confirmed_player: bool - vote: Optional[int] = Field(None, ge=-1, le=1) diff --git a/bases/bot_detector/api_public/src/app/views/response/feedback_score.py b/bases/bot_detector/api_public/src/app/views/response/feedback_score.py deleted file mode 100644 index 0107c64..0000000 --- a/bases/bot_detector/api_public/src/app/views/response/feedback_score.py +++ /dev/null @@ -1,8 +0,0 @@ -from pydantic import BaseModel - - -class FeedbackScoreResponse(BaseModel): - count: int - possible_ban: bool - confirmed_ban: bool - confirmed_player: bool diff --git a/bases/bot_detector/api_public/src/app/views/response/player.py b/bases/bot_detector/api_public/src/app/views/response/player.py deleted file mode 100644 index 89d7bfd..0000000 --- a/bases/bot_detector/api_public/src/app/views/response/player.py +++ /dev/null @@ -1,19 +0,0 @@ -from datetime import datetime - -from pydantic import BaseModel - - -class PlayerResponse(BaseModel): - id: int - name: str - created_at: datetime - updated_at: datetime - possible_ban: bool - confirmed_ban: bool - confirmed_player: bool - label_id: int - label_jagex: int - ironman: bool - hardcore_ironman: bool - ultimate_ironman: bool - normalized_name: str diff --git a/bases/bot_detector/api_public/src/app/views/response/prediction.py b/bases/bot_detector/api_public/src/app/views/response/prediction.py deleted file mode 100644 index 2ef02d5..0000000 --- a/bases/bot_detector/api_public/src/app/views/response/prediction.py +++ /dev/null @@ -1,27 +0,0 @@ -from datetime import datetime - -from pydantic import BaseModel - - -class PredictionResponse(BaseModel): - player_id: int - player_name: str - prediction_label: str - prediction_confidence: float - created: datetime - predictions_breakdown: dict - - @classmethod - def from_data(self, data: dict, breakdown: bool): - # Create the player data dictionary with only the relevant fields - prediction_data: dict = data.pop("predictions", {}) - player_data = { - "player_id": data.pop("player_id"), - "player_name": data.pop("name"), - "created": data.pop("created_at"), - "prediction_label": data.pop("prediction").lower(), - "prediction_confidence": data.pop("confidence"), - "predictions_breakdown": prediction_data if breakdown else {}, - } - - return self(**player_data) diff --git a/bases/bot_detector/api_public/src/app/views/response/report_score.py b/bases/bot_detector/api_public/src/app/views/response/report_score.py deleted file mode 100644 index bcbcfcc..0000000 --- a/bases/bot_detector/api_public/src/app/views/response/report_score.py +++ /dev/null @@ -1,9 +0,0 @@ -from pydantic import BaseModel - - -class ReportScoreResponse(BaseModel): - count: int - possible_ban: bool - confirmed_ban: bool - confirmed_player: bool - manual_detect: bool diff --git a/bases/bot_detector/api_public/src/core/.gitkeep b/bases/bot_detector/api_public/src/core/.gitkeep deleted file mode 100644 index e69de29..0000000 diff --git a/bases/bot_detector/hiscore_scraper/core.py b/bases/bot_detector/hiscore_scraper/core.py index 97d335a..2be7b63 100644 --- a/bases/bot_detector/hiscore_scraper/core.py +++ b/bases/bot_detector/hiscore_scraper/core.py @@ -7,6 +7,8 @@ import aiohttp from aiohttp import ClientSession +from bot_detector.core.structs import MetaData, NotFoundStruct, ScrapedStruct +from bot_detector.highscore.structs import HighscoreBaseStruct from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayerScrapedProducer, @@ -14,14 +16,9 @@ RepoPlayersToScrapeConsumer, RepoPlayersToScrapeProducer, ) +from bot_detector.player.structs import PlayerStruct from bot_detector.proxy_manager import ProxyManager from bot_detector.proxy_manager import Settings as ProxySettings -from bot_detector.structs import ( - HighscoreBaseStruct, - MetaData, - PlayerStruct, -) -from bot_detector.structs.kafka import NotFoundStruct, ScrapedStruct from osrs.asyncio import Hiscore, HSMode from osrs.asyncio.osrs.hiscores import PlayerStats from osrs.exceptions import PlayerDoesNotExist, UnexpectedRedirection diff --git a/bases/bot_detector/job_hs_migration_v3/core.py b/bases/bot_detector/job_hs_migration_v3/core.py index 5d71131..ff3d6f6 100644 --- a/bases/bot_detector/job_hs_migration_v3/core.py +++ b/bases/bot_detector/job_hs_migration_v3/core.py @@ -5,18 +5,15 @@ from datetime import timedelta import sqlalchemy as sqla -from bot_detector.database import Settings as DBSettings -from bot_detector.database import get_session_factory +from bot_detector.core.database import Settings as DBSettings +from bot_detector.core.database import get_session_factory +from bot_detector.core.structs import MetaData, ScrapedStruct +from bot_detector.highscore.structs import HighscoreBaseStruct from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayerScrapedProducer, ) -from bot_detector.structs import ( - HighscoreBaseStruct, - MetaData, - PlayerStruct, - ScrapedStruct, -) +from bot_detector.player.structs import PlayerStruct from pydantic_settings import BaseSettings from sqlalchemy import TextClause from sqlalchemy.exc import OperationalError diff --git a/bases/bot_detector/job_prune_hs_data/core.py b/bases/bot_detector/job_prune_hs_data/core.py index 4d692aa..41befff 100644 --- a/bases/bot_detector/job_prune_hs_data/core.py +++ b/bases/bot_detector/job_prune_hs_data/core.py @@ -2,8 +2,7 @@ import logging import sqlalchemy as sqla -from bot_detector.database import Settings as DBSettings -from bot_detector.database import get_session_factory +from bot_detector.core.database import Settings as DBSettings, get_session_factory from pydantic_settings import BaseSettings from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker diff --git a/bases/bot_detector/runemetrics_scraper/core.py b/bases/bot_detector/runemetrics_scraper/core.py index dba5e1b..e69476d 100644 --- a/bases/bot_detector/runemetrics_scraper/core.py +++ b/bases/bot_detector/runemetrics_scraper/core.py @@ -15,7 +15,8 @@ from bot_detector.proxy_manager import Settings as ProxySettings from bot_detector.runemetrics_api import RuneMetrics, RuneMetricsResponse from bot_detector.runemetrics_api.exceptions import UnexpectedRedirection -from bot_detector.structs import MetaData, PlayerStruct, ScrapedStruct +from bot_detector.core.structs import MetaData, ScrapedStruct +from bot_detector.player.structs import PlayerStruct from osrs.utils import RateLimiter from prometheus_client import Counter, Histogram, start_http_server from pydantic import ValidationError diff --git a/bases/bot_detector/scrape_task_producer/core.py b/bases/bot_detector/scrape_task_producer/core.py index 74c01bb..35a6dd9 100644 --- a/bases/bot_detector/scrape_task_producer/core.py +++ b/bases/bot_detector/scrape_task_producer/core.py @@ -3,15 +3,15 @@ from dataclasses import asdict, dataclass from datetime import date, datetime, time, timedelta -from bot_detector.database import Settings as DBSettings -from bot_detector.database import get_session_factory -from bot_detector.database.player import PlayerRepo +from bot_detector.core.database import Settings as DBSettings, get_session_factory +from bot_detector.player.database.repository import PlayerRepo from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayersToScrapeConsumer, RepoPlayersToScrapeProducer, ) -from bot_detector.structs import MetaData, PlayerStruct, ToScrapeStruct +from bot_detector.core.structs import MetaData, ToScrapeStruct +from bot_detector.player.structs import PlayerStruct from pydantic_settings import BaseSettings from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker from typing_extensions import Literal diff --git a/bases/bot_detector/worker_hiscore/core.py b/bases/bot_detector/worker_hiscore/core.py index 7a27f31..50eef8b 100644 --- a/bases/bot_detector/worker_hiscore/core.py +++ b/bases/bot_detector/worker_hiscore/core.py @@ -2,16 +2,16 @@ import logging import traceback -from bot_detector import database as db -from bot_detector.database import Settings as DBSettings -from bot_detector.database.hiscore import HighscoreDataRepo -from bot_detector.database.player import PlayerRepo +from bot_detector.core.database import Settings as DBSettings +from bot_detector.core.database import get_session_factory +from bot_detector.core.structs import ScrapedStruct +from bot_detector.highscore.database.repository import HighscoreDataRepo from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayerScrapedConsumer, RepoPlayerScrapedProducer, ) -from bot_detector.structs import ScrapedStruct +from bot_detector.player.database.repository import PlayerRepo from pydantic_settings import BaseSettings from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -108,7 +108,7 @@ async def consume_many_task( async def main(): - session_factory, async_engine = db.get_session_factory(SETTINGS=DBSettings()) + session_factory, async_engine = get_session_factory(SETTINGS=DBSettings()) player_repo = PlayerRepo() highscore_repo = HighscoreDataRepo() diff --git a/bases/bot_detector/worker_ml/core.py b/bases/bot_detector/worker_ml/core.py index 4b3cba7..9be4822 100644 --- a/bases/bot_detector/worker_ml/core.py +++ b/bases/bot_detector/worker_ml/core.py @@ -3,9 +3,11 @@ import traceback import aiohttp -from bot_detector.database import Settings as DBSettings -from bot_detector.database import get_session_factory -from bot_detector.database.prediction import PredictionLatestRepo, PredictionRepo +from bot_detector.core.database import Settings as DBSettings, get_session_factory +from bot_detector.prediction.database.repository import ( + PredictionLatestRepo, + PredictionRepo, +) from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoPlayerScrapedConsumer, @@ -13,7 +15,8 @@ ) from bot_detector.ml_api.core import MLApiClient from bot_detector.ml_api.structs import InputData, Prediction -from bot_detector.structs import PredictionCreate, ScrapedStruct +from bot_detector.prediction.structs import PredictionCreate +from bot_detector.core.structs import ScrapedStruct from bot_detector.worker_ml.settings import Settings from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker diff --git a/bases/bot_detector/worker_report/main.py b/bases/bot_detector/worker_report/main.py index 5d66e1b..1d526da 100644 --- a/bases/bot_detector/worker_report/main.py +++ b/bases/bot_detector/worker_report/main.py @@ -3,15 +3,15 @@ import traceback from asyncio import Queue -from bot_detector import database as db -from bot_detector.database import Settings as DBSettings -from bot_detector.database.report import ReportRepo +from bot_detector.core.database import Settings as DBSettings, get_session_factory +from bot_detector.report.database.repository import ReportRepo from bot_detector.kafka import Settings as KafkaSettings from bot_detector.kafka.repositories import ( RepoReportsToInsertConsumer, RepoReportsToInsertProducer, ) -from bot_detector.structs import ParsedDetection, ReportsToInsertStruct +from bot_detector.report.structs import ParsedDetection +from bot_detector.core.structs import ReportsToInsertStruct from sqlalchemy.exc import OperationalError from sqlalchemy.ext.asyncio import AsyncSession, async_sessionmaker @@ -120,7 +120,7 @@ async def error_task(error_queue: Queue, report_producer: RepoReportsToInsertPro async def main(): - session_factory, async_engine = db.get_session_factory(SETTINGS=DBSettings()) + session_factory, async_engine = get_session_factory(SETTINGS=DBSettings()) report_repo = ReportRepo() MAX_BATCH_SIZE = 10_000 # TODO: env variable? MAX_INTERVAL_MS = 1_000 # TODO: env variable? diff --git a/components/bot_detector/core/cache.py b/components/bot_detector/core/cache.py new file mode 100644 index 0000000..1ff5b68 --- /dev/null +++ b/components/bot_detector/core/cache.py @@ -0,0 +1,37 @@ +import asyncio +import logging +from collections import OrderedDict + +logger = logging.getLogger(__name__) + + +class SimpleALRUCache: + def __init__(self, max_size: int = 10_000): + self.cache = OrderedDict() + self.max_size = max_size + self.lock = asyncio.Lock() + self.hits: int = 0 + self.misses: int = 0 + + async def get(self, key): + async with self.lock: + if key in self.cache: + self.cache.move_to_end(key) + self.hits += 1 + return self.cache[key] + self.misses += 1 + return None + + async def put(self, key, value): + async with self.lock: + if key in self.cache: + self.cache.move_to_end(key) + self.cache[key] = value + else: + if len(self.cache) >= self.max_size: + self.cache.popitem(last=False) + self.cache[key] = value + + async def clear(self): + async with self.lock: + self.cache.clear() diff --git a/components/bot_detector/database/core.py b/components/bot_detector/core/database.py similarity index 94% rename from components/bot_detector/database/core.py rename to components/bot_detector/core/database.py index bb1ddb2..7c6712f 100644 --- a/components/bot_detector/database/core.py +++ b/components/bot_detector/core/database.py @@ -17,7 +17,7 @@ class Settings(BaseSettings): class Base(MappedAsDataclass, DeclarativeBase): - """subclasses will be converted to dataclasses""" + """Subclasses will be converted to dataclasses.""" def get_session_factory( diff --git a/components/bot_detector/core/structs/__init__.py b/components/bot_detector/core/structs/__init__.py new file mode 100644 index 0000000..b216e06 --- /dev/null +++ b/components/bot_detector/core/structs/__init__.py @@ -0,0 +1,10 @@ +from ._metadata import MetaData +from .kafka import NotFoundStruct, ReportsToInsertStruct, ScrapedStruct, ToScrapeStruct + +__all__ = [ + "MetaData", + "NotFoundStruct", + "ToScrapeStruct", + "ScrapedStruct", + "ReportsToInsertStruct", +] diff --git a/components/bot_detector/structs/_metadata.py b/components/bot_detector/core/structs/_metadata.py similarity index 100% rename from components/bot_detector/structs/_metadata.py rename to components/bot_detector/core/structs/_metadata.py diff --git a/components/bot_detector/structs/kafka/__init__.py b/components/bot_detector/core/structs/kafka/__init__.py similarity index 100% rename from components/bot_detector/structs/kafka/__init__.py rename to components/bot_detector/core/structs/kafka/__init__.py diff --git a/components/bot_detector/structs/kafka/not_found.py b/components/bot_detector/core/structs/kafka/not_found.py similarity index 53% rename from components/bot_detector/structs/kafka/not_found.py rename to components/bot_detector/core/structs/kafka/not_found.py index e8996ee..e551f91 100644 --- a/components/bot_detector/structs/kafka/not_found.py +++ b/components/bot_detector/core/structs/kafka/not_found.py @@ -1,5 +1,5 @@ -from bot_detector.structs._metadata import MetaData -from bot_detector.structs.player import PlayerStruct +from .._metadata import MetaData +from bot_detector.player.structs import PlayerStruct from pydantic import BaseModel diff --git a/components/bot_detector/structs/kafka/reports_to_insert.py b/components/bot_detector/core/structs/kafka/reports_to_insert.py similarity index 53% rename from components/bot_detector/structs/kafka/reports_to_insert.py rename to components/bot_detector/core/structs/kafka/reports_to_insert.py index 82898aa..85acb9a 100644 --- a/components/bot_detector/structs/kafka/reports_to_insert.py +++ b/components/bot_detector/core/structs/kafka/reports_to_insert.py @@ -1,5 +1,5 @@ -from bot_detector.structs._metadata import MetaData -from bot_detector.structs.reports import ParsedDetection +from .._metadata import MetaData +from bot_detector.report.structs import ParsedDetection from pydantic import BaseModel diff --git a/components/bot_detector/core/structs/kafka/scraped.py b/components/bot_detector/core/structs/kafka/scraped.py new file mode 100644 index 0000000..518c98a --- /dev/null +++ b/components/bot_detector/core/structs/kafka/scraped.py @@ -0,0 +1,11 @@ +from bot_detector.highscore.structs import HighscoreBaseStruct +from bot_detector.player.structs import PlayerStruct +from pydantic import BaseModel + +from .._metadata import MetaData + + +class ScrapedStruct(BaseModel): + metadata: MetaData + player_data: PlayerStruct + highscore_data: HighscoreBaseStruct | None diff --git a/components/bot_detector/structs/kafka/to_scrape.py b/components/bot_detector/core/structs/kafka/to_scrape.py similarity index 53% rename from components/bot_detector/structs/kafka/to_scrape.py rename to components/bot_detector/core/structs/kafka/to_scrape.py index 4015fa1..2feac79 100644 --- a/components/bot_detector/structs/kafka/to_scrape.py +++ b/components/bot_detector/core/structs/kafka/to_scrape.py @@ -1,5 +1,5 @@ -from bot_detector.structs._metadata import MetaData -from bot_detector.structs.player import PlayerStruct +from .._metadata import MetaData +from bot_detector.player.structs import PlayerStruct from pydantic import BaseModel diff --git a/bases/bot_detector/api_public/src/app/views/response/ok.py b/components/bot_detector/core/structs/responses.py similarity index 93% rename from bases/bot_detector/api_public/src/app/views/response/ok.py rename to components/bot_detector/core/structs/responses.py index 85be632..59896f5 100644 --- a/bases/bot_detector/api_public/src/app/views/response/ok.py +++ b/components/bot_detector/core/structs/responses.py @@ -1,5 +1,5 @@ -from pydantic import BaseModel - - -class Ok(BaseModel): - detail: str = "ok" +from pydantic import BaseModel + + +class Ok(BaseModel): + detail: str = "ok" diff --git a/components/bot_detector/database/__init__.py b/components/bot_detector/database/__init__.py deleted file mode 100644 index 17db9e8..0000000 --- a/components/bot_detector/database/__init__.py +++ /dev/null @@ -1,3 +0,0 @@ -from bot_detector.database.core import Base, Settings, get_session_factory - -__all__ = ["Base", "Settings", "get_session_factory"] diff --git a/components/bot_detector/database/api_public/__init__.py b/components/bot_detector/database/api_public/__init__.py deleted file mode 100644 index dcb16b7..0000000 --- a/components/bot_detector/database/api_public/__init__.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -API Public specific database models and helpers. -""" - -from .feedback import PredictionFeedback -from .label import Label -from .player import Player -from .prediction import Prediction_v1, Prediction_v2 -from .report import Report - -__all__ = [ - "Player", - "Prediction_v1", - "Prediction_v2", - "PredictionFeedback", - "Report", - "Label", -] diff --git a/components/bot_detector/database/api_public/player.py b/components/bot_detector/database/api_public/player.py deleted file mode 100644 index de772dd..0000000 --- a/components/bot_detector/database/api_public/player.py +++ /dev/null @@ -1,20 +0,0 @@ -from bot_detector.database import Base -from sqlalchemy import Boolean, Column, DateTime, Integer, Text - - -class Player(Base): - __tablename__ = "Players" - - id = Column(Integer, primary_key=True, autoincrement=True) - name = Column(Text) - created_at = Column(DateTime) - updated_at = Column(DateTime) - possible_ban = Column(Boolean) - confirmed_ban = Column(Boolean) - confirmed_player = Column(Boolean) - label_id = Column(Integer) - label_jagex = Column(Integer) - ironman = Column(Boolean) - hardcore_ironman = Column(Boolean) - ultimate_ironman = Column(Boolean) - normalized_name = Column(Text) diff --git a/components/bot_detector/database/hiscore/__init__.py b/components/bot_detector/database/hiscore/__init__.py deleted file mode 100644 index 2f7510f..0000000 --- a/components/bot_detector/database/hiscore/__init__.py +++ /dev/null @@ -1,35 +0,0 @@ -from .interface import ( - HighscoreDataDailyInterface, - HighscoreDataLatestInterface, - HighscoreDataMonthlyInterface, - HighscoreDataWeeklyInterface, -) -from .repository import ( - HighscoreDataDailyRepo, - HighscoreDataLatestRepo, - HighscoreDataMonthlyRepo, - HighscoreDataRepo, - HighscoreDataWeeklyRepo, -) -from .structs import ( - HighscoreDataDailyTableStruct, - HighscoreDataLatestTableStruct, - HighscoreDataMonthlyTableStruct, - HighscoreDataWeeklyTableStruct, -) - -__all__ = [ - "HighscoreDataDailyInterface", - "HighscoreDataWeeklyInterface", - "HighscoreDataMonthlyInterface", - "HighscoreDataLatestInterface", - "HighscoreDataDailyRepo", - "HighscoreDataWeeklyRepo", - "HighscoreDataMonthlyRepo", - "HighscoreDataRepo", - "HighscoreDataLatestRepo", - "HighscoreDataDailyTableStruct", - "HighscoreDataWeeklyTableStruct", - "HighscoreDataMonthlyTableStruct", - "HighscoreDataLatestTableStruct", -] diff --git a/components/bot_detector/database/player/__init__.py b/components/bot_detector/database/player/__init__.py deleted file mode 100644 index 3bbcf41..0000000 --- a/components/bot_detector/database/player/__init__.py +++ /dev/null @@ -1,5 +0,0 @@ -from .interface import playerInterface -from .repository import PlayerRepo -from .structs import PlayersTableStruct - -__all__ = ["playerInterface", "PlayerRepo", "PlayersTableStruct"] diff --git a/components/bot_detector/database/prediction/__init__.py b/components/bot_detector/database/prediction/__init__.py deleted file mode 100644 index e68f32e..0000000 --- a/components/bot_detector/database/prediction/__init__.py +++ /dev/null @@ -1,12 +0,0 @@ -from .interface import PredictionInterface, PredictionLatestInterface -from .repository import PredictionLatestRepo, PredictionRepo -from .structs import PredictionLatestStruct, PredictionStruct - -__all__ = [ - "PredictionInterface", - "PredictionLatestInterface", - "PredictionRepo", - "PredictionLatestRepo", - "PredictionStruct", - "PredictionLatestStruct", -] diff --git a/components/bot_detector/database/report/__init__.py b/components/bot_detector/database/report/__init__.py deleted file mode 100644 index 15cda4d..0000000 --- a/components/bot_detector/database/report/__init__.py +++ /dev/null @@ -1,4 +0,0 @@ -from .interface import ReportInterface -from .repository import ReportRepo - -__all__ = ["ReportInterface", "ReportRepo"] diff --git a/components/bot_detector/database/api_public/feedback.py b/components/bot_detector/feedback/database.py similarity index 95% rename from components/bot_detector/database/api_public/feedback.py rename to components/bot_detector/feedback/database.py index 8c06d63..3003576 100644 --- a/components/bot_detector/database/api_public/feedback.py +++ b/components/bot_detector/feedback/database.py @@ -1,4 +1,4 @@ -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import ( TIMESTAMP, Column, diff --git a/bases/bot_detector/api_public/src/app/repositories/feedback.py b/components/bot_detector/feedback/services.py similarity index 90% rename from bases/bot_detector/api_public/src/app/repositories/feedback.py rename to components/bot_detector/feedback/services.py index 9177c23..54bebd5 100644 --- a/bases/bot_detector/api_public/src/app/repositories/feedback.py +++ b/components/bot_detector/feedback/services.py @@ -1,10 +1,8 @@ import logging -from bot_detector.api_public.src.app.views.input.feedback import FeedbackInput -from bot_detector.database.api_public import ( - Player as dbPlayer, - PredictionFeedback as dbFeedback, -) +from bot_detector.feedback.structs import FeedbackInput +from bot_detector.player.database.structs import PlayersTableStruct as dbPlayer +from bot_detector.feedback.database import PredictionFeedback as dbFeedback from sqlalchemy import and_, insert, select from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession from sqlalchemy.sql.expression import Insert, Select @@ -12,7 +10,7 @@ logger = logging.getLogger(__name__) -class Feedback: +class FeedbackService: def __init__(self, session: AsyncSession) -> None: self.session = session diff --git a/bases/bot_detector/api_public/src/app/views/input/feedback.py b/components/bot_detector/feedback/structs.py similarity index 88% rename from bases/bot_detector/api_public/src/app/views/input/feedback.py rename to components/bot_detector/feedback/structs.py index 3741944..9aa46a8 100644 --- a/bases/bot_detector/api_public/src/app/views/input/feedback.py +++ b/components/bot_detector/feedback/structs.py @@ -43,3 +43,11 @@ def player_name_validator(cls, value: str): return value case _: raise ValueError("Invalid format for player_name") + + +class FeedbackScore(BaseModel): + count: int + possible_ban: bool + confirmed_ban: bool + confirmed_player: bool + vote: Optional[int] = Field(None, ge=-1, le=1) diff --git a/components/bot_detector/highscore/database/__init__.py b/components/bot_detector/highscore/database/__init__.py new file mode 100644 index 0000000..57afd1f --- /dev/null +++ b/components/bot_detector/highscore/database/__init__.py @@ -0,0 +1,17 @@ +"""Highscore worker persistence helpers.""" + +from .repository import HighscoreDataRepo +from .structs import ( + HighscoreDataDailyTableStruct, + HighscoreDataLatestTableStruct, + HighscoreDataMonthlyTableStruct, + HighscoreDataWeeklyTableStruct, +) + +__all__ = [ + "HighscoreDataRepo", + "HighscoreDataDailyTableStruct", + "HighscoreDataWeeklyTableStruct", + "HighscoreDataMonthlyTableStruct", + "HighscoreDataLatestTableStruct", +] diff --git a/components/bot_detector/database/hiscore/interface.py b/components/bot_detector/highscore/database/interface.py similarity index 97% rename from components/bot_detector/database/hiscore/interface.py rename to components/bot_detector/highscore/database/interface.py index 619e1d2..0178273 100644 --- a/components/bot_detector/database/hiscore/interface.py +++ b/components/bot_detector/highscore/database/interface.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from bot_detector.structs import ( +from bot_detector.highscore.structs import ( HighscoreDataDailyStruct, HighscoreDataLatestStruct, HighscoreDataMonthlyStruct, diff --git a/components/bot_detector/database/hiscore/repository.py b/components/bot_detector/highscore/database/repository.py similarity index 99% rename from components/bot_detector/database/hiscore/repository.py rename to components/bot_detector/highscore/database/repository.py index 628e256..23ed97f 100644 --- a/components/bot_detector/database/hiscore/repository.py +++ b/components/bot_detector/highscore/database/repository.py @@ -3,6 +3,10 @@ import sqlalchemy as sqla import sqlalchemy.dialects.mysql as sqla_mysql +from bot_detector.highscore.structs import HighscoreBaseStruct +from sqlalchemy import TextClause, func +from sqlalchemy.ext.asyncio import AsyncSession + from .interface import ( HighscoreDataDailyInterface, HighscoreDataLatestInterface, @@ -15,9 +19,6 @@ HighscoreDataMonthlyTableStruct, HighscoreDataWeeklyTableStruct, ) -from bot_detector.structs import HighscoreBaseStruct -from sqlalchemy import TextClause, func -from sqlalchemy.ext.asyncio import AsyncSession logger = logging.getLogger(__name__) diff --git a/components/bot_detector/database/hiscore/structs.py b/components/bot_detector/highscore/database/structs.py similarity index 98% rename from components/bot_detector/database/hiscore/structs.py rename to components/bot_detector/highscore/database/structs.py index 85432b7..7bb764f 100644 --- a/components/bot_detector/database/hiscore/structs.py +++ b/components/bot_detector/highscore/database/structs.py @@ -1,7 +1,7 @@ from datetime import date from typing import Optional -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import ( JSON, Computed, diff --git a/components/bot_detector/structs/hiscore.py b/components/bot_detector/highscore/structs.py similarity index 76% rename from components/bot_detector/structs/hiscore.py rename to components/bot_detector/highscore/structs.py index b6ffd3c..9d5d453 100644 --- a/components/bot_detector/structs/hiscore.py +++ b/components/bot_detector/highscore/structs.py @@ -32,3 +32,13 @@ class HighscoreDataWeeklyStruct(HighscoreDataBaseStruct): class HighscoreDataMonthlyStruct(HighscoreDataBaseStruct): pass + + +__all__ = [ + "HighscoreBaseStruct", + "HighscoreDataBaseStruct", + "HighscoreDataDailyStruct", + "HighscoreDataWeeklyStruct", + "HighscoreDataMonthlyStruct", + "HighscoreDataLatestStruct", +] diff --git a/components/bot_detector/kafka/__init__.py b/components/bot_detector/kafka/__init__.py index 825e652..f430919 100644 --- a/components/bot_detector/kafka/__init__.py +++ b/components/bot_detector/kafka/__init__.py @@ -4,11 +4,14 @@ DataToPredictProducer, DataToPredictStruct, ) +from .manager import KafkaManager, kafka_manager __all__ = [ "Settings", "ConsumerInterface", "ProducerInterface", + "KafkaManager", + "kafka_manager", "DataToPredictConsumer", "DataToPredictProducer", "DataToPredictStruct", diff --git a/components/bot_detector/kafka/manager.py b/components/bot_detector/kafka/manager.py new file mode 100644 index 0000000..1a48f27 --- /dev/null +++ b/components/bot_detector/kafka/manager.py @@ -0,0 +1,57 @@ +import logging + +from bot_detector.kafka.interface import ConsumerInterface, ProducerInterface + +logger = logging.getLogger(__name__) + + +class KafkaManager: + def __init__(self): + self.producers = {} + self.consumers = {} + logger.debug("KafkaManager initialized.") + + def set_producer(self, key: str, producer: ProducerInterface) -> None: + self.producers[key] = producer + logger.debug(f"Producer set for key: {key}") + + def get_producer(self, key: str | None) -> ProducerInterface | None: + logger.debug(f"Retrieving producer for key: {key}") + + if key is None: + return self.producers + + producer = self.producers.get(key) + + if not producer: + logger.debug(self.producers) + logger.warning(f"Producer not found for key: {key}") + return None + + logger.debug(f"Producer retrieved for key: {key}") + return producer + + def set_consumer(self, key: str, consumer: ConsumerInterface) -> None: + self.consumers[key] = consumer + logger.debug(f"Consumer set for key: {key}") + + def get_consumer(self, key: str | None) -> ConsumerInterface | None: + if key is None: + return self.consumers + + consumer = self.consumers.get(key) + + if not consumer: + logger.debug(self.consumers) + logger.warning(f"Consumer not found for key: {key}") + return None + + logger.debug(f"Consumer retrieved for key: {key}") + return consumer + + +# Create a global KafkaManager instance +kafka_manager = KafkaManager() + +# memory location +# print(id(kafka_manager)) diff --git a/components/bot_detector/kafka/repositories/players_not_found.py b/components/bot_detector/kafka/repositories/players_not_found.py index 4e77fc2..3c94908 100644 --- a/components/bot_detector/kafka/repositories/players_not_found.py +++ b/components/bot_detector/kafka/repositories/players_not_found.py @@ -6,7 +6,7 @@ ConsumerInterface, ProducerInterface, ) -from bot_detector.structs import NotFoundStruct +from bot_detector.core.structs import NotFoundStruct logger = logging.getLogger(__name__) diff --git a/components/bot_detector/kafka/repositories/players_scraped.py b/components/bot_detector/kafka/repositories/players_scraped.py index eb68ce2..6844f9a 100644 --- a/components/bot_detector/kafka/repositories/players_scraped.py +++ b/components/bot_detector/kafka/repositories/players_scraped.py @@ -7,7 +7,7 @@ ConsumerInterface, ProducerInterface, ) -from bot_detector.structs import ScrapedStruct +from bot_detector.core.structs import ScrapedStruct logger = logging.getLogger(__name__) diff --git a/components/bot_detector/kafka/repositories/players_to_scrape.py b/components/bot_detector/kafka/repositories/players_to_scrape.py index bf4df72..8f331cc 100644 --- a/components/bot_detector/kafka/repositories/players_to_scrape.py +++ b/components/bot_detector/kafka/repositories/players_to_scrape.py @@ -6,7 +6,7 @@ ConsumerInterface, ProducerInterface, ) -from bot_detector.structs import ToScrapeStruct +from bot_detector.core.structs import ToScrapeStruct logger = logging.getLogger(__name__) diff --git a/components/bot_detector/kafka/repositories/reports_to_insert.py b/components/bot_detector/kafka/repositories/reports_to_insert.py index f138225..f45621e 100644 --- a/components/bot_detector/kafka/repositories/reports_to_insert.py +++ b/components/bot_detector/kafka/repositories/reports_to_insert.py @@ -7,7 +7,7 @@ ConsumerInterface, ProducerInterface, ) -from bot_detector.structs import ReportsToInsertStruct +from bot_detector.core.structs import ReportsToInsertStruct logger = logging.getLogger(__name__) diff --git a/components/bot_detector/database/api_public/label.py b/components/bot_detector/labels/database.py similarity index 80% rename from components/bot_detector/database/api_public/label.py rename to components/bot_detector/labels/database.py index fdb9229..f008d2d 100644 --- a/components/bot_detector/database/api_public/label.py +++ b/components/bot_detector/labels/database.py @@ -1,4 +1,4 @@ -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import Column, Integer, Text diff --git a/bases/bot_detector/api_public/src/app/repositories/labels.py b/components/bot_detector/labels/services.py similarity index 92% rename from bases/bot_detector/api_public/src/app/repositories/labels.py rename to components/bot_detector/labels/services.py index 90b347d..d8abe5d 100644 --- a/bases/bot_detector/api_public/src/app/repositories/labels.py +++ b/components/bot_detector/labels/services.py @@ -1,6 +1,6 @@ import logging -from bot_detector.database.api_public import Label as dbLabel +from bot_detector.labels.database import Label as dbLabel from sqlalchemy import select from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession from sqlalchemy.sql.expression import Select @@ -8,7 +8,7 @@ logger = logging.getLogger(__name__) -class LabelRepository: +class LabelService: def __init__(self, session: AsyncSession) -> None: self.session = session diff --git a/bases/bot_detector/api_public/src/app/views/response/label.py b/components/bot_detector/labels/structs.py similarity index 100% rename from bases/bot_detector/api_public/src/app/views/response/label.py rename to components/bot_detector/labels/structs.py diff --git a/components/bot_detector/logfmt/core.py b/components/bot_detector/logfmt/core.py index 5b28b98..75b4dee 100644 --- a/components/bot_detector/logfmt/core.py +++ b/components/bot_detector/logfmt/core.py @@ -2,12 +2,25 @@ import json import logging +from pydantic import field_validator from pydantic_settings import BaseSettings class Settings(BaseSettings): DEBUG: bool = False + @field_validator("DEBUG", mode="before") + @classmethod + def _coerce_debug(cls, value): + if isinstance(value, str): + lowered = value.lower() + if lowered in {"1", "true", "yes", "on", "debug"}: + return True + if lowered in {"0", "false", "no", "off"}: + return False + return False + return value + def can_convert_to_json(s: str) -> dict | None: try: diff --git a/components/bot_detector/player/__init__.py b/components/bot_detector/player/__init__.py new file mode 100644 index 0000000..1bc6f6e --- /dev/null +++ b/components/bot_detector/player/__init__.py @@ -0,0 +1,3 @@ +"""Player feature package.""" + +__all__ = ["database", "services", "structs"] diff --git a/components/bot_detector/player/database/__init__.py b/components/bot_detector/player/database/__init__.py new file mode 100644 index 0000000..c81d000 --- /dev/null +++ b/components/bot_detector/player/database/__init__.py @@ -0,0 +1,6 @@ +"""Player database helpers.""" + +from .repository import PlayerRepo +from .structs import PlayersTableStruct + +__all__ = ["PlayerRepo", "PlayersTableStruct"] diff --git a/components/bot_detector/database/player/interface.py b/components/bot_detector/player/database/interface.py similarity index 94% rename from components/bot_detector/database/player/interface.py rename to components/bot_detector/player/database/interface.py index 4bc99c1..44edac4 100644 --- a/components/bot_detector/database/player/interface.py +++ b/components/bot_detector/player/database/interface.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from bot_detector.structs import PlayerStruct +from bot_detector.player.structs import PlayerStruct from sqlalchemy.ext.asyncio import AsyncSession diff --git a/components/bot_detector/database/player/repository.py b/components/bot_detector/player/database/repository.py similarity index 99% rename from components/bot_detector/database/player/repository.py rename to components/bot_detector/player/database/repository.py index f301dca..4f5d974 100644 --- a/components/bot_detector/database/player/repository.py +++ b/components/bot_detector/player/database/repository.py @@ -5,7 +5,7 @@ import sqlalchemy as sqla from .interface import playerInterface from .structs import PlayersTableStruct -from bot_detector.structs import PlayerStruct +from bot_detector.player.structs import PlayerStruct from sqlalchemy import TextClause from sqlalchemy.ext.asyncio import AsyncSession diff --git a/components/bot_detector/database/player/structs.py b/components/bot_detector/player/database/structs.py similarity index 95% rename from components/bot_detector/database/player/structs.py rename to components/bot_detector/player/database/structs.py index 98c50c6..3bd4c76 100644 --- a/components/bot_detector/database/player/structs.py +++ b/components/bot_detector/player/database/structs.py @@ -1,6 +1,6 @@ from datetime import datetime -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import Boolean, DateTime, Integer, Text from sqlalchemy.orm import Mapped, mapped_column diff --git a/bases/bot_detector/api_public/src/app/repositories/player.py b/components/bot_detector/player/services.py similarity index 91% rename from bases/bot_detector/api_public/src/app/repositories/player.py rename to components/bot_detector/player/services.py index f272aa1..d7bd0c0 100644 --- a/bases/bot_detector/api_public/src/app/repositories/player.py +++ b/components/bot_detector/player/services.py @@ -1,161 +1,162 @@ -import logging - -import sqlalchemy as sqla -from bot_detector.api_public.src.app.views.player import PlayerCreate, PlayerInDB -from bot_detector.api_public.src.core._cache import SimpleALRUCache -from bot_detector.database.api_public import ( - Player as dbPlayer, - PredictionFeedback as dbFeedback, - Prediction_v2 as dbPrediction, -) -from fastapi.encoders import jsonable_encoder -from pydantic import ValidationError -from sqlalchemy import func, select -from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession -from sqlalchemy.orm import aliased -from sqlalchemy.sql.expression import Select - -# from bot_detector.database.api_public import Report as dbReport - -logger = logging.getLogger(__name__) - - -def model_to_dict(model): - """Converts an SQLAlchemy model instance to a dictionary.""" - return {c.name: getattr(model, c.name) for c in model.__table__.columns} - - -class Player: - def __init__( - self, - session: AsyncSession, - cache: SimpleALRUCache = SimpleALRUCache(), - ) -> None: - self.session = session - self.cache = cache - - def sanitize_name(self, player_name: str) -> str: - return player_name.lower().replace("_", " ").replace("-", " ").strip() - - async def update_session(self, session: AsyncSession): - self.session = session - - async def get_report_score(self, player_names: tuple[str, ...]): - if not isinstance(player_names, tuple): - raise Exception() - sql_select = """ - select - count(rs.reporting_id) as count, - subject.confirmed_ban, - subject.possible_ban, - subject.confirmed_player, - rs.manual_detect - from report_sighting rs - join Players voter ON rs.reporting_id = voter.id - join Players subject ON rs.reported_id = subject.id - WHERE voter.name in :name - GROUP BY - subject.confirmed_ban, - subject.possible_ban, - subject.confirmed_player, - rs.manual_detect - """ - params = {"name": player_names} - data = await self.session.execute(sqla.text(sql_select), params=params) - result = data.mappings().all() - return result - - async def get_feedback_score(self, player_names: list[str]): - # dbFeedback - fb_voter: dbPlayer = aliased(dbPlayer, name="feedback_voter") - fb_subject: dbPlayer = aliased(dbPlayer, name="feedback_subject") - - query: Select = select( - func.count(func.distinct(fb_subject.id)).label("count"), - fb_subject.possible_ban, - fb_subject.confirmed_ban, - fb_subject.confirmed_player, - ) - query = query.select_from(dbFeedback) - query = query.join(fb_voter, dbFeedback.voter_id == fb_voter.id) - query = query.join(fb_subject, dbFeedback.subject_id == fb_subject.id) - query = query.where(fb_voter.name.in_(player_names)) - query = query.group_by( - fb_subject.possible_ban, - fb_subject.confirmed_ban, - fb_subject.confirmed_player, - ) - - result: AsyncResult = await self.session.execute(query) - await self.session.commit() - return tuple(result.mappings()) - - async def get_prediction(self, player_names: list[str]): - query: Select = select( - dbPlayer.id.label("player_id"), - dbPlayer.name, - dbPrediction.created_at, - dbPrediction.model_name, - dbPrediction.prediction, - dbPrediction.confidence, - dbPrediction.predictions, - ) - query = query.select_from(dbPrediction) - query = query.join(dbPlayer, dbPrediction.player_id == dbPlayer.id) - query = query.where(dbPlayer.name.in_(player_names)) - - result: AsyncResult = await self.session.execute(query) - result = result.mappings().all() - return jsonable_encoder(result) - - async def get(self, player_name: str) -> PlayerInDB: - assert isinstance(player_name, str) - player_name = self.sanitize_name(player_name) - - sql = sqla.select(dbPlayer).where(dbPlayer.name == player_name) - - result = await self.session.execute(sql) - data = result.scalars().all() - try: - if len(data) == 0: - return None - player_in_db = PlayerInDB(**model_to_dict(data[0])) - except ValidationError as e: - logger.error(f"Validation error: {e.json()}") - return None - return player_in_db - - async def get_cache(self, player_name: str) -> PlayerInDB: - player_name = self.sanitize_name(player_name) - player = await self.cache.get(key=player_name) - - if isinstance(player, PlayerInDB): - if self.cache.hits % 100 == 0 and self.cache.hits > 0: - logger.info(f"hits: {self.cache.hits}, misses: {self.cache.misses}") - return player - - player = await self.get(player_name=player_name) - - if isinstance(player, PlayerInDB): - await self.cache.put(key=player_name, value=player) - return player - - async def insert(self, player: PlayerCreate) -> PlayerInDB: - player.name = self.sanitize_name(player.name) - sql = sqla.insert(dbPlayer).values(player.model_dump()).prefix_with("IGNORE") - await self.session.execute(sql) - await self.session.commit() - return await self.get(player_name=player.name) - - async def get_or_insert(self, player_name: str, cached=True) -> PlayerInDB: - player_name = self.sanitize_name(player_name) - - if cached: - player = await self.get_cache(player_name=player_name) - else: - player = await self.get(player_name=player_name) - - if player is None: - player = await self.insert(PlayerCreate(name=player_name)) - - return player +import logging + +import sqlalchemy as sqla +from bot_detector.core.cache import SimpleALRUCache +from bot_detector.player.database.structs import PlayersTableStruct as dbPlayer +from bot_detector.prediction.database.prediction import ( + Prediction_v2 as dbPrediction, +) +from bot_detector.feedback.database import PredictionFeedback as dbFeedback +from bot_detector.player.structs import PlayerCreate, PlayerInDB +from fastapi.encoders import jsonable_encoder +from pydantic import ValidationError +from sqlalchemy import func, select +from sqlalchemy.ext.asyncio import AsyncResult, AsyncSession +from sqlalchemy.orm import aliased +from sqlalchemy.sql.expression import Select + +logger = logging.getLogger(__name__) + + +def model_to_dict(model): + """Converts an SQLAlchemy model instance to a dictionary.""" + return {c.name: getattr(model, c.name) for c in model.__table__.columns} + + +class PlayerService: + def __init__( + self, + session: AsyncSession, + cache: SimpleALRUCache = SimpleALRUCache(), + ) -> None: + self.session = session + self.cache = cache + + def sanitize_name(self, player_name: str) -> str: + return player_name.lower().replace("_", " ").replace("-", " ").strip() + + async def update_session(self, session: AsyncSession): + self.session = session + + async def get_report_score(self, player_names: tuple[str, ...]): + if not isinstance(player_names, tuple): + raise Exception() + sql_select = """ + select + count(rs.reporting_id) as count, + subject.confirmed_ban, + subject.possible_ban, + subject.confirmed_player, + rs.manual_detect + from report_sighting rs + join Players voter ON rs.reporting_id = voter.id + join Players subject ON rs.reported_id = subject.id + WHERE voter.name in :name + GROUP BY + subject.confirmed_ban, + subject.possible_ban, + subject.confirmed_player, + rs.manual_detect + """ + params = {"name": player_names} + data = await self.session.execute(sqla.text(sql_select), params=params) + result = data.mappings().all() + return result + + async def get_feedback_score(self, player_names: list[str]): + # dbFeedback + fb_voter: dbPlayer = aliased(dbPlayer, name="feedback_voter") + fb_subject: dbPlayer = aliased(dbPlayer, name="feedback_subject") + + query: Select = select( + func.count(func.distinct(fb_subject.id)).label("count"), + fb_subject.possible_ban, + fb_subject.confirmed_ban, + fb_subject.confirmed_player, + ) + query = query.select_from(dbFeedback) + query = query.join(fb_voter, dbFeedback.voter_id == fb_voter.id) + query = query.join(fb_subject, dbFeedback.subject_id == fb_subject.id) + query = query.where(fb_voter.name.in_(player_names)) + query = query.group_by( + fb_subject.possible_ban, + fb_subject.confirmed_ban, + fb_subject.confirmed_player, + ) + + result: AsyncResult = await self.session.execute(query) + await self.session.commit() + return tuple(result.mappings()) + + async def get_prediction(self, player_names: list[str]): + query: Select = select( + dbPlayer.id.label("player_id"), + dbPlayer.name, + dbPrediction.created_at, + dbPrediction.model_name, + dbPrediction.prediction, + dbPrediction.confidence, + dbPrediction.predictions, + ) + query = query.select_from(dbPrediction) + query = query.join(dbPlayer, dbPrediction.player_id == dbPlayer.id) + query = query.where(dbPlayer.name.in_(player_names)) + + result: AsyncResult = await self.session.execute(query) + result = result.mappings().all() + return jsonable_encoder(result) + + async def get(self, player_name: str) -> PlayerInDB: + assert isinstance(player_name, str) + player_name = self.sanitize_name(player_name) + + sql = sqla.select(dbPlayer).where(dbPlayer.name == player_name) + + result = await self.session.execute(sql) + data = result.scalars().all() + try: + if len(data) == 0: + return None + player_in_db = PlayerInDB(**model_to_dict(data[0])) + except ValidationError as e: + logger.error(f"Validation error: {e.json()}") + return None + return player_in_db + + async def get_cache(self, player_name: str) -> PlayerInDB: + player_name = self.sanitize_name(player_name) + player = await self.cache.get(key=player_name) + + if isinstance(player, PlayerInDB): + if self.cache.hits % 100 == 0 and self.cache.hits > 0: + logger.info(f"hits: {self.cache.hits}, misses: {self.cache.misses}") + return player + + player = await self.get(player_name=player_name) + + if isinstance(player, PlayerInDB): + await self.cache.put(key=player_name, value=player) + return player + + async def insert(self, player: PlayerCreate) -> PlayerInDB: + player.name = self.sanitize_name(player.name) + sql = sqla.insert(dbPlayer).values(player.model_dump()).prefix_with("IGNORE") + await self.session.execute(sql) + await self.session.commit() + return await self.get(player_name=player.name) + + async def get_or_insert(self, player_name: str, cached=True) -> PlayerInDB: + player_name = self.sanitize_name(player_name) + + if cached: + player = await self.get_cache(player_name=player_name) + else: + player = await self.get(player_name=player_name) + + if player is None: + player = await self.insert(PlayerCreate(name=player_name)) + + return player + + +__all__ = ["PlayerService"] diff --git a/components/bot_detector/player/structs.py b/components/bot_detector/player/structs.py new file mode 100644 index 0000000..8beef8c --- /dev/null +++ b/components/bot_detector/player/structs.py @@ -0,0 +1,130 @@ +from datetime import datetime +from typing import Optional + +from pydantic import BaseModel, Field, field_validator + + +class PlayerCreate(BaseModel): + name: str + possible_ban: Optional[bool] = 0 + confirmed_ban: Optional[bool] = 0 + confirmed_player: Optional[bool] = 0 + label_id: Optional[int] = 0 + label_jagex: Optional[int] = 0 + ironman: Optional[int] = None + hardcore_ironman: Optional[int] = None + ultimate_ironman: Optional[int] = None + normalized_name: Optional[str] = None + + +class PlayerUpdate(BaseModel): + name: Optional[str] = None + possible_ban: Optional[bool] = None + confirmed_ban: Optional[bool] = None + confirmed_player: Optional[bool] = None + label_id: Optional[int] = None + label_jagex: Optional[int] = None + ironman: Optional[int] = None + hardcore_ironman: Optional[int] = None + ultimate_ironman: Optional[int] = None + normalized_name: Optional[str] = None + + +class PlayerInDB(PlayerCreate): + id: int + created_at: datetime + updated_at: datetime | None + + @field_validator("created_at", mode="before") + def parse_created_at(cls, value): + if isinstance(value, str): + return datetime.fromisoformat(value) + if value is None: + raise ValueError("created_at cannot be None") + return value + + +class Player(PlayerInDB): + pass + + +class PlayerResponse(BaseModel): + id: int + name: str + created_at: datetime + updated_at: datetime + possible_ban: bool + confirmed_ban: bool + confirmed_player: bool + label_id: int + label_jagex: int + ironman: bool + hardcore_ironman: bool + ultimate_ironman: bool + normalized_name: str + + +class ReportScoreResponse(BaseModel): + count: int + possible_ban: bool + confirmed_ban: bool + confirmed_player: bool + manual_detect: bool + + +class FeedbackScoreResponse(BaseModel): + count: int + possible_ban: bool + confirmed_ban: bool + confirmed_player: bool + + +class PredictionResponse(BaseModel): + player_id: int + player_name: str + prediction_label: str + prediction_confidence: float + created: datetime + predictions_breakdown: dict + + @classmethod + def from_data(cls, data: dict, breakdown: bool): + prediction_data: dict = data.pop("predictions", {}) + player_data = { + "player_id": data.pop("player_id"), + "player_name": data.pop("name"), + "created": data.pop("created_at"), + "prediction_label": data.pop("prediction").lower(), + "prediction_confidence": data.pop("confidence"), + "predictions_breakdown": prediction_data if breakdown else {}, + } + return cls(**player_data) + + +class PlayerStruct(BaseModel): + id: int + name: str + created_at: datetime + updated_at: Optional[datetime] = None + possible_ban: bool = False + confirmed_ban: bool = False + confirmed_player: bool = False + label_id: int = 0 + label_jagex: int = 0 + ironman: Optional[bool] = None + hardcore_ironman: Optional[bool] = None + ultimate_ironman: Optional[bool] = None + normalized_name: Optional[str] = None + + +__all__ = [ + "PlayerCreate", + "PlayerUpdate", + "PlayerInDB", + "Player", + "PlayerResponse", + "ReportScoreResponse", + "FeedbackScoreResponse", + "PredictionResponse", + "PlayerStruct", +] diff --git a/components/bot_detector/database/prediction/interface.py b/components/bot_detector/prediction/database/interface.py similarity index 95% rename from components/bot_detector/database/prediction/interface.py rename to components/bot_detector/prediction/database/interface.py index a80eda7..a1bddfb 100644 --- a/components/bot_detector/database/prediction/interface.py +++ b/components/bot_detector/prediction/database/interface.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from bot_detector.structs import ( +from bot_detector.prediction.structs import ( PredictionCreate, PredictionLatestRead, PredictionRead, diff --git a/components/bot_detector/database/api_public/prediction.py b/components/bot_detector/prediction/database/prediction.py similarity index 97% rename from components/bot_detector/database/api_public/prediction.py rename to components/bot_detector/prediction/database/prediction.py index 09e583f..bef96ca 100644 --- a/components/bot_detector/database/api_public/prediction.py +++ b/components/bot_detector/prediction/database/prediction.py @@ -1,4 +1,4 @@ -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import DECIMAL, JSON, TIMESTAMP, Column, Integer, String diff --git a/components/bot_detector/database/prediction/repository.py b/components/bot_detector/prediction/database/repository.py similarity index 96% rename from components/bot_detector/database/prediction/repository.py rename to components/bot_detector/prediction/database/repository.py index 4b5a1ad..2624720 100644 --- a/components/bot_detector/database/prediction/repository.py +++ b/components/bot_detector/prediction/database/repository.py @@ -2,10 +2,10 @@ from dataclasses import asdict import sqlalchemy as sqla -from bot_detector.database.player.structs import PlayersTableStruct +from bot_detector.player.database.structs import PlayersTableStruct from .interface import PredictionInterface, PredictionLatestInterface from .structs import PredictionLatestStruct, PredictionStruct -from bot_detector.structs import ( +from bot_detector.prediction.structs import ( PredictionCreate, PredictionLatestRead, PredictionRead, diff --git a/components/bot_detector/database/prediction/structs.py b/components/bot_detector/prediction/database/structs.py similarity index 96% rename from components/bot_detector/database/prediction/structs.py rename to components/bot_detector/prediction/database/structs.py index a2e62d9..b69ebeb 100644 --- a/components/bot_detector/database/prediction/structs.py +++ b/components/bot_detector/prediction/database/structs.py @@ -1,7 +1,7 @@ from datetime import datetime from typing import Any, Optional -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import DECIMAL, JSON, DateTime, ForeignKey, Integer, String from sqlalchemy.orm import Mapped, mapped_column diff --git a/components/bot_detector/structs/prediction.py b/components/bot_detector/prediction/structs.py similarity index 85% rename from components/bot_detector/structs/prediction.py rename to components/bot_detector/prediction/structs.py index a598ecf..502bfe5 100644 --- a/components/bot_detector/structs/prediction.py +++ b/components/bot_detector/prediction/structs.py @@ -30,3 +30,11 @@ class PredictionLatestRead(PredictionBase): class Config: from_attributes = True + + +__all__ = [ + "PredictionBase", + "PredictionCreate", + "PredictionRead", + "PredictionLatestRead", +] diff --git a/components/bot_detector/database/report/interface.py b/components/bot_detector/report/database/interface.py similarity index 92% rename from components/bot_detector/database/report/interface.py rename to components/bot_detector/report/database/interface.py index 644eac4..5f102ac 100644 --- a/components/bot_detector/database/report/interface.py +++ b/components/bot_detector/report/database/interface.py @@ -1,6 +1,6 @@ from abc import ABC, abstractmethod -from bot_detector.structs import ParsedDetection +from bot_detector.report.structs import ParsedDetection from sqlalchemy.ext.asyncio import AsyncSession diff --git a/components/bot_detector/database/api_public/report.py b/components/bot_detector/report/database/report.py similarity index 95% rename from components/bot_detector/database/api_public/report.py rename to components/bot_detector/report/database/report.py index ffbd0bf..b1644b7 100644 --- a/components/bot_detector/database/api_public/report.py +++ b/components/bot_detector/report/database/report.py @@ -1,4 +1,4 @@ -from bot_detector.database import Base +from bot_detector.core.database import Base from sqlalchemy import TIMESTAMP, BigInteger, Column, Integer, SmallInteger diff --git a/components/bot_detector/database/report/repository.py b/components/bot_detector/report/database/repository.py similarity index 99% rename from components/bot_detector/database/report/repository.py rename to components/bot_detector/report/database/repository.py index 3d9a3c7..d1360e1 100644 --- a/components/bot_detector/database/report/repository.py +++ b/components/bot_detector/report/database/repository.py @@ -3,7 +3,7 @@ import sqlalchemy as sqla from .interface import ReportInterface -from bot_detector.structs import ParsedDetection +from bot_detector.report.structs import ParsedDetection from sqlalchemy import TextClause from sqlalchemy.ext.asyncio import AsyncSession diff --git a/bases/bot_detector/api_public/src/app/repositories/report.py b/components/bot_detector/report/services/reports.py similarity index 91% rename from bases/bot_detector/api_public/src/app/repositories/report.py rename to components/bot_detector/report/services/reports.py index 2a3f0e8..1af8905 100644 --- a/bases/bot_detector/api_public/src/app/repositories/report.py +++ b/components/bot_detector/report/services/reports.py @@ -1,112 +1,108 @@ -import asyncio -import logging -import time - -from bot_detector.api_public.src.core.fastapi.dependencies.kafka import kafka_manager -from bot_detector.kafka.repositories.reports_to_insert import ( - RepoReportsToInsertProducer, -) -from bot_detector.structs import ( - Detection, - MetaData, - ParsedDetection, - ReportsToInsertStruct, -) -from pydantic import ValidationError - -logger = logging.getLogger(__name__) - - -class CustomError(Exception): ... - - -class Report: - def __init__(self) -> None: - pass - - def _check_data_size(self, data: list[Detection]) -> list[Detection] | None: - return None if len(data) > 5000 else data - - def _filter_valid_time(self, data: list[Detection]) -> list[Detection]: - current_time = int(time.time()) - min_ts = current_time - 25200 # 7 hours ago - max_ts = current_time + 3600 # 1 hour in the future - # [d for d in data if min_ts < d.ts < max_ts] - output = [] - - for d in data: - if d.ts <= min_ts: - logger.info( - f"invalid: {d.ts} <= {min_ts}, now={current_time}, {d.reporter}" - ) - continue - if d.ts >= max_ts: - logger.info( - f"invalid: {d.ts} >= {max_ts}, now={current_time}, {d.reporter}" - ) - continue - output.append(d) - - return output - - def _check_unique_reporter(self, data: list[Detection]) -> list[Detection] | None: - return None if len(set(d.reporter for d in data)) > 1 else data - - async def parse_data(self, data: list[Detection]) -> tuple[list[Detection], None]: - """ - Parse and validate a list of detection data. - """ - data = self._check_data_size(data) - if not data: - error = "invalid data size" - logger.warning(error) - return None, error - - data = self._filter_valid_time(data) - if not data: - error = "invalid time" - logger.warning(error) - return None, error - - data = self._check_unique_reporter(data) - if not data: - error = "invalid unique reporter" - logger.warning(error) - return None, error - return data, None - - def _transform_detection( - self, data: list[ParsedDetection] - ) -> tuple[list[ReportsToInsertStruct], list | None]: - reports = [] - errors = [] - - for d in data: - metadata = MetaData(version=1, source="api_public") - try: - report = ReportsToInsertStruct(metadata=metadata, report=d.model_dump()) - reports.append(report) - except ValidationError as e: - error = f"Validation error: {e.json()}" - errors.append(error) - return reports, errors - - async def send_to_kafka(self, data: list[ParsedDetection]) -> None: - producer = kafka_manager.get_producer(key="reports_to_insert") - producer: RepoReportsToInsertProducer | None - - if not producer: - raise CustomError("Producer not found") - - tasks = [] - - # Transform data to ReportsToInsertStruct - reports, error = self._transform_detection(data) - - tasks = [producer.produce_one(report=report) for report in reports] - await asyncio.gather(*tasks) - - if len(error) > 0: - error_msg = f"Received {len(error)} validation errors like this: {error[0]}" - logger.error(error_msg) - raise CustomError(error_msg) +import asyncio +import logging +import time + +from bot_detector.kafka import kafka_manager +from bot_detector.kafka.repositories.reports_to_insert import ( + RepoReportsToInsertProducer, +) +from bot_detector.report.structs import Detection, ParsedDetection +from bot_detector.core.structs import MetaData, ReportsToInsertStruct +from pydantic import ValidationError + +logger = logging.getLogger(__name__) + + +class CustomError(Exception): ... + + +class ReportsService: + def __init__(self) -> None: + pass + + def _check_data_size(self, data: list[Detection]) -> list[Detection] | None: + return None if len(data) > 5000 else data + + def _filter_valid_time(self, data: list[Detection]) -> list[Detection]: + current_time = int(time.time()) + min_ts = current_time - 25200 # 7 hours ago + max_ts = current_time + 3600 # 1 hour in the future + # [d for d in data if min_ts < d.ts < max_ts] + output = [] + + for d in data: + if d.ts <= min_ts: + logger.info( + f"invalid: {d.ts} <= {min_ts}, now={current_time}, {d.reporter}" + ) + continue + if d.ts >= max_ts: + logger.info( + f"invalid: {d.ts} >= {max_ts}, now={current_time}, {d.reporter}" + ) + continue + output.append(d) + + return output + + def _check_unique_reporter(self, data: list[Detection]) -> list[Detection] | None: + return None if len(set(d.reporter for d in data)) > 1 else data + + async def parse_data(self, data: list[Detection]) -> tuple[list[Detection], None]: + """ + Parse and validate a list of detection data. + """ + data = self._check_data_size(data) + if not data: + error = "invalid data size" + logger.warning(error) + return None, error + + data = self._filter_valid_time(data) + if not data: + error = "invalid time" + logger.warning(error) + return None, error + + data = self._check_unique_reporter(data) + if not data: + error = "invalid unique reporter" + logger.warning(error) + return None, error + return data, None + + def _transform_detection( + self, data: list[ParsedDetection] + ) -> tuple[list[ReportsToInsertStruct], list | None]: + reports = [] + errors = [] + + for d in data: + metadata = MetaData(version=1, source="api_public") + try: + report = ReportsToInsertStruct(metadata=metadata, report=d.model_dump()) + reports.append(report) + except ValidationError as e: + error = f"Validation error: {e.json()}" + errors.append(error) + return reports, errors + + async def send_to_kafka(self, data: list[ParsedDetection]) -> None: + producer = kafka_manager.get_producer(key="reports_to_insert") + producer: RepoReportsToInsertProducer | None + + if not producer: + raise CustomError("Producer not found") + + tasks = [] + + # Transform data to ReportsToInsertStruct + reports, error = self._transform_detection(data) + + tasks = [producer.produce_one(report=report) for report in reports] + await asyncio.gather(*tasks) + + if len(error) > 0: + error_msg = f"Received {len(error)} validation errors like this: {error[0]}" + logger.error(error_msg) + raise CustomError(error_msg) diff --git a/bases/bot_detector/api_public/src/app/views/input/report.py b/components/bot_detector/report/structs.py similarity index 87% rename from bases/bot_detector/api_public/src/app/views/input/report.py rename to components/bot_detector/report/structs.py index 2d897c8..6d78af9 100644 --- a/bases/bot_detector/api_public/src/app/views/input/report.py +++ b/components/bot_detector/report/structs.py @@ -1,54 +1,67 @@ -import time -from typing import Optional - -from bot_detector.api_public.src.app.views.input._metadata import Metadata -from pydantic import BaseModel -from pydantic.fields import Field - - -class Equipment(BaseModel): - equip_head_id: Optional[int] = Field(None, ge=0) - equip_amulet_id: Optional[int] = Field(None, ge=0) - equip_torso_id: Optional[int] = Field(None, ge=0) - equip_legs_id: Optional[int] = Field(None, ge=0) - equip_boots_id: Optional[int] = Field(None, ge=0) - equip_cape_id: Optional[int] = Field(None, ge=0) - equip_hands_id: Optional[int] = Field(None, ge=0) - equip_weapon_id: Optional[int] = Field(None, ge=0) - equip_shield_id: Optional[int] = Field(None, ge=0) - - -class BaseDetection(BaseModel): - region_id: int = Field(0, ge=0, le=100_000) - x_coord: int = Field(0, ge=0) - y_coord: int = Field(0, ge=0) - z_coord: int = Field(0, ge=0) - ts: int = Field(int(time.time()), ge=0) - manual_detect: int = Field(0, ge=0, le=1) - on_members_world: int = Field(0, ge=0, le=1) - on_pvp_world: int = Field(0, ge=0, le=1) - world_number: int = Field(0, ge=300, le=1_000) - equipment: Equipment - equip_ge_value: int = Field(0, ge=0) - - -class Detection(BaseDetection): - reporter: str = Field(..., min_length=1, max_length=13) - reported: str = Field(..., min_length=1, max_length=12) - - -class ParsedDetection(BaseDetection): - reporter_id: int = Field(..., ge=0) - reported_id: int = Field(..., ge=0) - - -class KafkaDetectionV1(BaseDetection): - metadata: Metadata = Metadata(version="v1.0.0") - reporter: str = Field(..., min_length=1, max_length=13) - reported: str = Field(..., min_length=1, max_length=12) - - -class KafkaDetectionV2(BaseDetection): - metadata: Metadata = Metadata(version="v2.0.0") - reporter_id: int = Field(..., ge=0) - reported_id: int = Field(..., ge=0) +import time +from typing import Optional + +from pydantic import BaseModel, Field + + +class Metadata(BaseModel): + version: str + + +class Equipment(BaseModel): + equip_head_id: Optional[int] = Field(None, ge=0) + equip_amulet_id: Optional[int] = Field(None, ge=0) + equip_torso_id: Optional[int] = Field(None, ge=0) + equip_legs_id: Optional[int] = Field(None, ge=0) + equip_boots_id: Optional[int] = Field(None, ge=0) + equip_cape_id: Optional[int] = Field(None, ge=0) + equip_hands_id: Optional[int] = Field(None, ge=0) + equip_weapon_id: Optional[int] = Field(None, ge=0) + equip_shield_id: Optional[int] = Field(None, ge=0) + + +class BaseDetection(BaseModel): + region_id: int = Field(0, ge=0, le=100_000) + x_coord: int = Field(0, ge=0) + y_coord: int = Field(0, ge=0) + z_coord: int = Field(0, ge=0) + ts: int = Field(int(time.time()), ge=0) + manual_detect: int = Field(0, ge=0, le=1) + on_members_world: int = Field(0, ge=0, le=1) + on_pvp_world: int = Field(0, ge=0, le=1) + world_number: int = Field(0, ge=300, le=1_000) + equipment: Equipment + equip_ge_value: int = Field(0, ge=0) + + +class Detection(BaseDetection): + reporter: str = Field(..., min_length=1, max_length=13) + reported: str = Field(..., min_length=1, max_length=12) + + +class ParsedDetection(BaseDetection): + reporter_id: int = Field(..., ge=0) + reported_id: int = Field(..., ge=0) + + +class KafkaDetectionV1(BaseDetection): + metadata: Metadata = Metadata(version="v1.0.0") + reporter: str = Field(..., min_length=1, max_length=13) + reported: str = Field(..., min_length=1, max_length=12) + + +class KafkaDetectionV2(BaseDetection): + metadata: Metadata = Metadata(version="v2.0.0") + reporter_id: int = Field(..., ge=0) + reported_id: int = Field(..., ge=0) + + +__all__ = [ + "Metadata", + "Equipment", + "BaseDetection", + "Detection", + "ParsedDetection", + "KafkaDetectionV1", + "KafkaDetectionV2", +] diff --git a/components/bot_detector/structs/__init__.py b/components/bot_detector/structs/__init__.py deleted file mode 100644 index ae410de..0000000 --- a/components/bot_detector/structs/__init__.py +++ /dev/null @@ -1,40 +0,0 @@ -from ._metadata import MetaData -from .hiscore import ( - HighscoreBaseStruct, - HighscoreDataBaseStruct, - HighscoreDataDailyStruct, - HighscoreDataLatestStruct, - HighscoreDataMonthlyStruct, - HighscoreDataWeeklyStruct, -) -from .kafka import NotFoundStruct, ReportsToInsertStruct, ScrapedStruct, ToScrapeStruct -from .player import PlayerStruct -from .prediction import ( - PredictionBase, - PredictionCreate, - PredictionLatestRead, - PredictionRead, -) -from .reports import Detection, Equipment, ParsedDetection - -__all__ = [ - "MetaData", - "HighscoreBaseStruct", - "HighscoreDataLatestStruct", - "HighscoreDataBaseStruct", - "HighscoreDataDailyStruct", - "HighscoreDataWeeklyStruct", - "HighscoreDataMonthlyStruct", - "NotFoundStruct", - "PlayerStruct", - "ToScrapeStruct", - "ScrapedStruct", - "ReportsToInsertStruct", - "Detection", - "ParsedDetection", - "Equipment", - "PredictionLatestRead", - "PredictionBase", - "PredictionCreate", - "PredictionRead", -] diff --git a/components/bot_detector/structs/kafka/scraped.py b/components/bot_detector/structs/kafka/scraped.py deleted file mode 100644 index ac113e8..0000000 --- a/components/bot_detector/structs/kafka/scraped.py +++ /dev/null @@ -1,10 +0,0 @@ -from bot_detector.structs._metadata import MetaData -from bot_detector.structs.hiscore import HighscoreBaseStruct -from bot_detector.structs.player import PlayerStruct -from pydantic import BaseModel - - -class ScrapedStruct(BaseModel): - metadata: MetaData - player_data: PlayerStruct - highscore_data: HighscoreBaseStruct | None diff --git a/components/bot_detector/structs/player.py b/components/bot_detector/structs/player.py deleted file mode 100644 index b1d0ac2..0000000 --- a/components/bot_detector/structs/player.py +++ /dev/null @@ -1,20 +0,0 @@ -from datetime import datetime -from typing import Optional - -from pydantic import BaseModel - - -class PlayerStruct(BaseModel): - id: int - name: str - created_at: datetime - updated_at: Optional[datetime] = None - possible_ban: bool = False - confirmed_ban: bool = False - confirmed_player: bool = False - label_id: int = 0 - label_jagex: int = 0 - ironman: Optional[bool] = None - hardcore_ironman: Optional[bool] = None - ultimate_ironman: Optional[bool] = None - normalized_name: Optional[str] = None diff --git a/components/bot_detector/structs/reports.py b/components/bot_detector/structs/reports.py deleted file mode 100644 index 90a1d53..0000000 --- a/components/bot_detector/structs/reports.py +++ /dev/null @@ -1,41 +0,0 @@ -import time -from typing import Optional - -from pydantic import BaseModel -from pydantic.fields import Field - - -class Equipment(BaseModel): - equip_head_id: Optional[int] = Field(None, ge=0) - equip_amulet_id: Optional[int] = Field(None, ge=0) - equip_torso_id: Optional[int] = Field(None, ge=0) - equip_legs_id: Optional[int] = Field(None, ge=0) - equip_boots_id: Optional[int] = Field(None, ge=0) - equip_cape_id: Optional[int] = Field(None, ge=0) - equip_hands_id: Optional[int] = Field(None, ge=0) - equip_weapon_id: Optional[int] = Field(None, ge=0) - equip_shield_id: Optional[int] = Field(None, ge=0) - - -class BaseDetection(BaseModel): - region_id: int = Field(0, ge=0, le=100_000) - x_coord: int = Field(0, ge=0) - y_coord: int = Field(0, ge=0) - z_coord: int = Field(0, ge=0) - ts: int = Field(int(time.time()), ge=0) - manual_detect: int = Field(0, ge=0, le=1) - on_members_world: int = Field(0, ge=0, le=1) - on_pvp_world: int = Field(0, ge=0, le=1) - world_number: int = Field(0, ge=300, le=1_000) - equipment: Equipment - equip_ge_value: int = Field(0, ge=0) - - -class Detection(BaseDetection): - reporter: str = Field(..., min_length=1, max_length=13) - reported: str = Field(..., min_length=1, max_length=12) - - -class ParsedDetection(BaseDetection): - reporter_id: int = Field(..., ge=0) - reported_id: int = Field(..., ge=0) diff --git a/docker-compose-dev.yml b/docker-compose-dev.yml index 82bdf87..1325dce 100644 --- a/docker-compose-dev.yml +++ b/docker-compose-dev.yml @@ -121,7 +121,7 @@ services: context: ./ dockerfile: Dockerfile # command: ["uv", "run", "bases/bot_detector/hiscore_scraper/core.py"] - # command: uv run uvicorn bases.bot_detector.api_public.src.core.server:app --host 0.0.0.0 --reload --port 5000 + # command: uv run uvicorn bases.bot_detector.api_public.core.server:app --host 0.0.0.0 --reload --port 5000 # command: uv run uvicorn bases.bot_detector.website.core.server:app --host 0.0.0.0 --reload --port 5000 # command: uv run uvicorn bases.bot_detector.api_ml.core.server:app --host 0.0.0.0 --reload --port 5000 ports: @@ -146,4 +146,4 @@ networks: botdetector-network: name: bd-network volumes: - uv_cache: \ No newline at end of file + uv_cache: diff --git a/docker-compose.yml b/docker-compose.yml index ea124cd..c1825f0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -137,7 +137,7 @@ services: context: ./ dockerfile: ./projects/hiscore_scraper/Dockerfile target: production - # command: ["uv", "run", "uvicorn", "bot_detector.api_public.src.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning", "--reload", "--reload-dir", "/app/bot_detector/api_public/"] + # command: ["uv", "run", "uvicorn", "bases.bot_detector.api_public.core.server:app", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning", "--reload", "--reload-dir", "/app/bot_detector/api_public/"] networks: - botdetector-network env_file: @@ -283,7 +283,7 @@ services: target: builder # command: ["sleep", "infinity"] command: > - sh -c "cd ../.. && projects/api_public/.venv/bin/uvicorn bases.bot_detector.api_public.src.core.server:app --host 0.0.0.0 --port 5000 --log-level warning --reload --reload-dir /app" + sh -c "cd ../.. && projects/api_public/.venv/bin/uvicorn bases.bot_detector.api_public.core.server:app --host 0.0.0.0 --port 5000 --log-level warning --reload --reload-dir /app" env_file: - .env environment: @@ -395,4 +395,4 @@ networks: # alpine sh -c "cp -r /source/* /data/" volumes: uv_cache: - external: true \ No newline at end of file + external: true diff --git a/projects/api_public/Dockerfile b/projects/api_public/Dockerfile index c7b3656..c71fc01 100644 --- a/projects/api_public/Dockerfile +++ b/projects/api_public/Dockerfile @@ -27,4 +27,4 @@ COPY --from=builder --chown=appuser /app/projects/api_public/.venv /app/projects USER appuser -CMD [".venv/bin/uvicorn", "bot_detector.api_public.src.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] \ No newline at end of file +CMD [".venv/bin/uvicorn", "bases.bot_detector.api_public.core.server:app", "--proxy-headers", "--host", "0.0.0.0", "--port", "5000", "--log-level", "warning"] diff --git a/projects/api_public/pyproject.toml b/projects/api_public/pyproject.toml index ed7ff76..633dffc 100644 --- a/projects/api_public/pyproject.toml +++ b/projects/api_public/pyproject.toml @@ -20,14 +20,19 @@ dependencies = [ ] [project.scripts] -scrape_task_producer = "bot_detector.api_public.core.server:run" +scrape_task_producer = "bases.bot_detector.api_public.core.server:run" [tool.hatch.build.hooks.polylith-bricks] packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/api_public" = "bot_detector/api_public" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" "../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/player" = "bot_detector/player" +"../../components/bot_detector/feedback" = "bot_detector/feedback" +"../../components/bot_detector/labels" = "bot_detector/labels" +"../../components/bot_detector/report" = "bot_detector/report" +"../../components/bot_detector/prediction" = "bot_detector/prediction" +"../../components/bot_detector/highscore_worker" = "bot_detector/highscore_worker" diff --git a/projects/hiscore_scraper/pyproject.toml b/projects/hiscore_scraper/pyproject.toml index 7582ce7..4d9771c 100644 --- a/projects/hiscore_scraper/pyproject.toml +++ b/projects/hiscore_scraper/pyproject.toml @@ -29,5 +29,6 @@ packages = ["bot_detector"] "../../bases/bot_detector/hiscore_scraper" = "bot_detector/hiscore_scraper" "../../components/bot_detector/proxy_manager" = "bot_detector/proxy_manager" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/core" = "bot_detector/core" +"../../components/bot_detector/player" = "bot_detector/player" +"../../components/bot_detector/highscore_worker" = "bot_detector/highscore_worker" diff --git a/projects/job_hs_migration_v3/pyproject.toml b/projects/job_hs_migration_v3/pyproject.toml index 38989a4..d283f0a 100644 --- a/projects/job_hs_migration_v3/pyproject.toml +++ b/projects/job_hs_migration_v3/pyproject.toml @@ -28,7 +28,7 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/job_hs_migration_v3" = "bot_detector/job_hs_migration_v3" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" \ No newline at end of file +"../../components/bot_detector/player" = "bot_detector/player" +"../../components/bot_detector/highscore_worker" = "bot_detector/highscore_worker" diff --git a/projects/job_prune_hs_data/pyproject.toml b/projects/job_prune_hs_data/pyproject.toml index 8dfdaca..42353e0 100644 --- a/projects/job_prune_hs_data/pyproject.toml +++ b/projects/job_prune_hs_data/pyproject.toml @@ -26,5 +26,4 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/job_prune_hs_data" = "bot_detector/job_prune_hs_data" -"../../components/bot_detector/database" = "bot_detector/database" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" \ No newline at end of file +"../../components/bot_detector/core" = "bot_detector/core" diff --git a/projects/runemetrics_scraper/pyproject.toml b/projects/runemetrics_scraper/pyproject.toml index 13bb0ce..1a86a75 100644 --- a/projects/runemetrics_scraper/pyproject.toml +++ b/projects/runemetrics_scraper/pyproject.toml @@ -30,5 +30,5 @@ packages = ["bot_detector"] "../../components/bot_detector/runemetrics_api" = "bot_detector/runemetrics_api" "../../components/bot_detector/proxy_manager" = "bot_detector/proxy_manager" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/core" = "bot_detector/core" +"../../components/bot_detector/player" = "bot_detector/player" diff --git a/projects/scrape_task_producer/pyproject.toml b/projects/scrape_task_producer/pyproject.toml index 1385e6e..36013a3 100644 --- a/projects/scrape_task_producer/pyproject.toml +++ b/projects/scrape_task_producer/pyproject.toml @@ -28,7 +28,6 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/scrape_task_producer" = "bot_detector/scrape_task_producer" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/player" = "bot_detector/player" diff --git a/projects/worker_hiscore/pyproject.toml b/projects/worker_hiscore/pyproject.toml index eb2f991..3b6fb62 100644 --- a/projects/worker_hiscore/pyproject.toml +++ b/projects/worker_hiscore/pyproject.toml @@ -28,7 +28,7 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/worker_hiscore" = "bot_detector/worker_hiscore" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/player" = "bot_detector/player" +"../../components/bot_detector/highscore_worker" = "bot_detector/highscore_worker" diff --git a/projects/worker_ml/pyproject.toml b/projects/worker_ml/pyproject.toml index de05fd9..ef351df 100644 --- a/projects/worker_ml/pyproject.toml +++ b/projects/worker_ml/pyproject.toml @@ -30,7 +30,6 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/worker_ml" = "bot_detector/worker_ml" "../../components/bot_detector/ml_api" = "bot_detector/ml_api" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/prediction" = "bot_detector/prediction" diff --git a/projects/worker_report/pyproject.toml b/projects/worker_report/pyproject.toml index 2d842fe..5d94116 100644 --- a/projects/worker_report/pyproject.toml +++ b/projects/worker_report/pyproject.toml @@ -28,7 +28,6 @@ packages = ["bot_detector"] [tool.polylith.bricks] "../../bases/bot_detector/worker_report" = "bot_detector/worker_report" -"../../components/bot_detector/database" = "bot_detector/database" +"../../components/bot_detector/core" = "bot_detector/core" "../../components/bot_detector/kafka" = "bot_detector/kafka" -"../../components/bot_detector/structs" = "bot_detector/structs" -"../../components/bot_detector/logfmt" = "bot_detector/logfmt" +"../../components/bot_detector/report" = "bot_detector/report" diff --git a/test/bases/bot_detector/api_public/test_core.py b/test/bases/bot_detector/api_public/test_core.py index e69de29..85ca40f 100644 --- a/test/bases/bot_detector/api_public/test_core.py +++ b/test/bases/bot_detector/api_public/test_core.py @@ -0,0 +1,64 @@ +import asyncio +from unittest.mock import AsyncMock, MagicMock + +import pytest +from fastapi import FastAPI + +from bases.bot_detector.api_public.core import server +from bases.bot_detector.api_public.core.fastapi.dependencies import session as session_dep +from bases.bot_detector.api_public.core.fastapi.dependencies import kafka as kafka_dep + + +def test_create_app_wires_routes_and_middleware(): + app = server.create_app() + assert isinstance(app, FastAPI) + paths = {route.path for route in app.routes} + assert "/v2/player/prediction" in paths + middleware_names = {mw.cls.__name__ for mw in app.user_middleware} + assert {"LoggingMiddleware", "PrometheusMiddleware"}.issubset(middleware_names) + + +@pytest.mark.asyncio() +async def test_get_session_yields_session(monkeypatch): + fake_session = AsyncMock() + + class _Factory: + async def __aenter__(self): + return fake_session + + async def __aexit__(self, exc_type, exc, tb): + pass + + async def _session_factory(): + return _Factory() + + monkeypatch.setattr( + session_dep, + "SessionFactory", + MagicMock(return_value=_Factory()), + ) + + gen = session_dep.get_session() + session = await gen.__anext__() + assert session is fake_session + with pytest.raises(StopAsyncIteration): + await gen.__anext__() + + +@pytest.mark.asyncio() +async def test_lifespan_starts_and_stops_producer(monkeypatch): + fake_producer = AsyncMock() + monkeypatch.setattr( + kafka_dep.kafka_manager, + "set_producer", + lambda key, producer: None, + ) + monkeypatch.setattr( + kafka_dep.kafka_manager, + "get_producer", + lambda key: fake_producer, + ) + + async with server.lifespan(server.create_app()): + fake_producer.start.assert_awaited_once() + fake_producer.stop.assert_awaited_once() diff --git a/test/bases/bot_detector/api_public/test_endpoints.py b/test/bases/bot_detector/api_public/test_endpoints.py new file mode 100644 index 0000000..2faea97 --- /dev/null +++ b/test/bases/bot_detector/api_public/test_endpoints.py @@ -0,0 +1,276 @@ +from fastapi.testclient import TestClient + +from bases.bot_detector.api_public.core import server +from bases.bot_detector.api_public.core.fastapi.dependencies.session import get_session +from bot_detector.player.services import PlayerService +from bot_detector.feedback.services import FeedbackService +from bot_detector.report.services.reports import CustomError, ReportsService + + +async def _dummy_session(): + yield object() + + +def _client(monkeypatch): + app = server.create_app() + app.dependency_overrides[get_session] = _dummy_session + return TestClient(app) + + +def test_player_prediction_not_found(monkeypatch): + async def fake_prediction(*args, **kwargs): + return [] + + monkeypatch.setattr(PlayerService, "get_prediction", fake_prediction) + client = _client(monkeypatch) + resp = client.get( + "/v2/player/prediction", + params={"name": ["abc"], "breakdown": "false"}, + ) + assert resp.status_code == 404 + assert resp.json()["detail"] == "Player not found" + + +def test_feedback_duplicate_returns_422(monkeypatch): + async def fake_insert(*args, **kwargs): + return False, "duplicate_record" + + monkeypatch.setattr(FeedbackService, "insert_feedback", fake_insert) + client = _client(monkeypatch) + payload = { + "player_name": "abc", + "vote": 1, + "prediction": "bot", + "confidence": 0.5, + "subject_id": 1, + } + resp = client.post("/v2/feedback", json=payload) + assert resp.status_code == 422 + assert resp.json()["detail"] == "duplicate_record" + + +def test_report_validation_error(monkeypatch): + async def fake_parse(self, data): + return None, "invalid data size" + + monkeypatch.setattr(ReportsService, "parse_data", fake_parse) + client = _client(monkeypatch) + resp = client.post("/v2/report", json=[]) + assert resp.status_code == 400 + assert resp.json()["detail"] == "invalid data size" + + +def test_report_producer_failure(monkeypatch): + async def fake_parse(self, data): + class _Detection: + reporter = "a" + reported = "b" + + def model_dump(self): + return { + "reporter": "a", + "reported": "b", + "region_id": 1, + "x_coord": 1, + "y_coord": 1, + "z_coord": 0, + "ts": 1, + "manual_detect": 0, + "on_members_world": 0, + "on_pvp_world": 0, + "world_number": 301, + "equipment": { + "equip_head_id": 1, + "equip_amulet_id": 1, + "equip_torso_id": 1, + "equip_legs_id": 1, + "equip_boots_id": 1, + "equip_cape_id": 1, + "equip_hands_id": 1, + "equip_weapon_id": 1, + "equip_shield_id": 1, + }, + "equip_ge_value": 1, + } + + return [_Detection()], None + + class _Player: + def __init__(self, name, id_): + self.name = name + self.id = id_ + + async def fake_get_or_insert(self, player_name, **kwargs): + return _Player(player_name, 1 if player_name == "a" else 2) + + async def fake_send(self, *args, **kwargs): + raise CustomError("boom") + + monkeypatch.setattr(ReportsService, "parse_data", fake_parse) + monkeypatch.setattr(PlayerService, "get_or_insert", fake_get_or_insert) + monkeypatch.setattr(PlayerService, "sanitize_name", lambda self, n: n) + monkeypatch.setattr(ReportsService, "send_to_kafka", fake_send) + + client = _client(monkeypatch) + resp = client.post( + "/v2/report", + json=[ + { + "reporter": "a", + "reported": "b", + "region_id": 1, + "x_coord": 1, + "y_coord": 1, + "z_coord": 0, + "ts": 1, + "manual_detect": 0, + "on_members_world": 0, + "on_pvp_world": 0, + "world_number": 301, + "equipment": { + "equip_head_id": 1, + "equip_amulet_id": 1, + "equip_torso_id": 1, + "equip_legs_id": 1, + "equip_boots_id": 1, + "equip_cape_id": 1, + "equip_hands_id": 1, + "equip_weapon_id": 1, + "equip_shield_id": 1, + }, + "equip_ge_value": 1, + } + ], + ) + assert resp.status_code == 500 + assert resp.json()["detail"] == "Internal error" + + +def test_player_prediction_success(monkeypatch): + async def fake_prediction(*args, **kwargs): + return [ + { + "player_id": 1, + "name": "abc", + "created_at": "2024-01-01T00:00:00", + "model_name": "m", + "prediction": "bot", + "confidence": 0.9, + "predictions": {"bot": 0.9}, + } + ] + + monkeypatch.setattr(PlayerService, "get_prediction", fake_prediction) + client = _client(monkeypatch) + resp = client.get( + "/v2/player/prediction", + params={"name": ["abc"], "breakdown": "true"}, + ) + assert resp.status_code == 200 + body = resp.json() + assert body[0]["player_name"] == "abc" + assert body[0]["predictions_breakdown"] == {"bot": 0.9} + + +def test_feedback_success(monkeypatch): + async def fake_insert(*args, **kwargs): + return True, "success" + + monkeypatch.setattr(FeedbackService, "insert_feedback", fake_insert) + client = _client(monkeypatch) + payload = { + "player_name": "abc", + "vote": 1, + "prediction": "bot", + "confidence": 0.5, + "subject_id": 1, + } + resp = client.post("/v2/feedback", json=payload) + assert resp.status_code == 201 + assert resp.json()["detail"] == "success" + + +def test_report_success(monkeypatch): + async def fake_parse(self, data): + class _Detection: + reporter = "a" + reported = "b" + + def model_dump(self): + return { + "reporter": "a", + "reported": "b", + "region_id": 1, + "x_coord": 1, + "y_coord": 1, + "z_coord": 0, + "ts": 1, + "manual_detect": 0, + "on_members_world": 0, + "on_pvp_world": 0, + "world_number": 301, + "equipment": { + "equip_head_id": 1, + "equip_amulet_id": 1, + "equip_torso_id": 1, + "equip_legs_id": 1, + "equip_boots_id": 1, + "equip_cape_id": 1, + "equip_hands_id": 1, + "equip_weapon_id": 1, + "equip_shield_id": 1, + }, + "equip_ge_value": 1, + } + + return [_Detection()], None + + class _Player: + def __init__(self, name, id_): + self.name = name + self.id = id_ + + async def fake_get_or_insert(self, player_name, **kwargs): + return _Player(player_name, 1 if player_name == "a" else 2) + + async def fake_send(self, *args, **kwargs): + return None + + monkeypatch.setattr(ReportsService, "parse_data", fake_parse) + monkeypatch.setattr(PlayerService, "get_or_insert", fake_get_or_insert) + monkeypatch.setattr(PlayerService, "sanitize_name", lambda self, n: n) + monkeypatch.setattr(ReportsService, "send_to_kafka", fake_send) + + client = _client(monkeypatch) + resp = client.post( + "/v2/report", + json=[ + { + "reporter": "a", + "reported": "b", + "region_id": 1, + "x_coord": 1, + "y_coord": 1, + "z_coord": 0, + "ts": 1, + "manual_detect": 0, + "on_members_world": 0, + "on_pvp_world": 0, + "world_number": 301, + "equipment": { + "equip_head_id": 1, + "equip_amulet_id": 1, + "equip_torso_id": 1, + "equip_legs_id": 1, + "equip_boots_id": 1, + "equip_cape_id": 1, + "equip_hands_id": 1, + "equip_weapon_id": 1, + "equip_shield_id": 1, + }, + "equip_ge_value": 1, + } + ], + ) + assert resp.status_code == 201 + assert resp.json()["detail"] == "ok" diff --git a/test/bases/bot_detector/api_public/test_routes.py b/test/bases/bot_detector/api_public/test_routes.py new file mode 100644 index 0000000..8ca5a89 --- /dev/null +++ b/test/bases/bot_detector/api_public/test_routes.py @@ -0,0 +1,31 @@ +from fastapi.routing import APIRoute + +from bases.bot_detector.api_public import routes + + +def _api_paths() -> set[str]: + return { + route.path + for route in routes.router.routes + if isinstance(route, APIRoute) + } + + +def test_router_registers_all_feature_paths(): + expected = { + "/v2/player/report/score", + "/v2/player/feedback/score", + "/v2/player/prediction", + "/v2/report", + "/v2/feedback", + "/v2/labels", + "/v2/labels/{label_id}", + } + paths = _api_paths() + assert expected.issubset(paths) + + +def test_router_applies_v2_prefix_once(): + paths = _api_paths() + assert paths, "router should expose routes" + assert all(path.startswith("/v2/") for path in paths) diff --git a/test/bases/bot_detector/api_public/test_schemas.py b/test/bases/bot_detector/api_public/test_schemas.py new file mode 100644 index 0000000..1ec248a --- /dev/null +++ b/test/bases/bot_detector/api_public/test_schemas.py @@ -0,0 +1,25 @@ +import pytest + +from bot_detector.feedback.structs import FeedbackInput + + +def _base_feedback_kwargs() -> dict: + return { + "player_name": "Some_Player", + "vote": 1, + "prediction": "bot", + "confidence": 0.5, + "subject_id": 123, + } + + +def test_feedback_input_accepts_valid_osrs_name(): + data = FeedbackInput(**_base_feedback_kwargs()) + assert data.player_name == "Some_Player" + + +def test_feedback_input_rejects_invalid_name(): + bad_kwargs = _base_feedback_kwargs() + bad_kwargs["player_name"] = "this-name-is-way-too-long" + with pytest.raises(ValueError): + FeedbackInput(**bad_kwargs) diff --git a/test/bases/bot_detector/api_public/test_utils.py b/test/bases/bot_detector/api_public/test_utils.py new file mode 100644 index 0000000..d99e030 --- /dev/null +++ b/test/bases/bot_detector/api_public/test_utils.py @@ -0,0 +1,25 @@ +import asyncio + +import pytest +from bot_detector.core.cache import SimpleALRUCache +from bot_detector.player.services import PlayerService + +from bases.bot_detector.api_public.core.fastapi.dependencies.to_jagex_name import ( + to_jagex_name, +) + + +class _DummySession: + async def execute(self, *args, **kwargs): # pragma: no cover - not used here + raise AssertionError("execute should not be called") + + +@pytest.mark.asyncio() +async def test_to_jagex_name_normalizes_variants(): + assert await to_jagex_name("Some_Name") == "some name" + assert await to_jagex_name("AlreadyClean") == "alreadyclean" + + +def test_player_sanitize_name_is_consistent(): + repo = PlayerService(session=_DummySession(), cache=SimpleALRUCache()) + assert repo.sanitize_name("My_Name-Here ") == "my name here" diff --git a/test/bases/bot_detector/runemetrics_scraper/test_core.py b/test/bases/bot_detector/runemetrics_scraper/test_core.py index 28f0da0..efd8281 100644 --- a/test/bases/bot_detector/runemetrics_scraper/test_core.py +++ b/test/bases/bot_detector/runemetrics_scraper/test_core.py @@ -7,7 +7,7 @@ RuneMetricsResponse, ) from bot_detector.runemetrics_scraper import core -from bot_detector.structs import PlayerStruct +from bot_detector.player.structs import PlayerStruct from pydantic import BaseModel os.environ["ENVIRONMENT"] = "test" diff --git a/test/bases/bot_detector/scrape_task_producer/test_core.py b/test/bases/bot_detector/scrape_task_producer/test_core.py index 879ef88..dcb829b 100644 --- a/test/bases/bot_detector/scrape_task_producer/test_core.py +++ b/test/bases/bot_detector/scrape_task_producer/test_core.py @@ -2,7 +2,7 @@ import pytest from bot_detector.scrape_task_producer.core import FetchParams, determine_fetch_params -from bot_detector.structs import PlayerStruct +from bot_detector.player.structs import PlayerStruct def make_fetch_params(**overrides) -> FetchParams: diff --git a/test/components/bot_detector/api_public/test_reports_service.py b/test/components/bot_detector/api_public/test_reports_service.py new file mode 100644 index 0000000..35c19f3 --- /dev/null +++ b/test/components/bot_detector/api_public/test_reports_service.py @@ -0,0 +1,151 @@ +import asyncio +import time + +import pytest + +from bot_detector.report.services.reports import CustomError, ReportsService +from bot_detector.report.structs import Detection, Equipment, ParsedDetection +from bot_detector.core.structs import ReportsToInsertStruct, MetaData + + +def _make_detection(ts: int | None = None, reporter: str = "tester", reported: str = "target") -> Detection: + equipment = Equipment( + equip_head_id=1, + equip_amulet_id=1, + equip_torso_id=1, + equip_legs_id=1, + equip_boots_id=1, + equip_cape_id=1, + equip_hands_id=1, + equip_weapon_id=1, + equip_shield_id=1, + ) + return Detection( + reporter=reporter, + reported=reported, + region_id=1, + x_coord=1, + y_coord=1, + z_coord=0, + ts=ts if ts is not None else int(time.time()), + manual_detect=0, + on_members_world=0, + on_pvp_world=0, + world_number=301, + equipment=equipment, + equip_ge_value=1, + ) + + +@pytest.mark.asyncio() +async def test_parse_data_rejects_large_payload(): + service = ReportsService() + detections = [_make_detection(reported=f"target-{i}") for i in range(5001)] + + data, error = await service.parse_data(detections) + + assert data is None + assert error == "invalid data size" + + +@pytest.mark.asyncio() +async def test_parse_data_detects_invalid_unique_reporter(): + service = ReportsService() + detections = [ + _make_detection(reporter="alpha", reported="x"), + _make_detection(reporter="beta", reported="y"), + ] + + data, error = await service.parse_data(detections) + + assert data is None + assert error == "invalid unique reporter" + + +@pytest.mark.asyncio() +async def test_transform_detection_builds_structs(): + service = ReportsService() + parsed = ParsedDetection( + reporter_id=1, + reported_id=2, + region_id=1, + x_coord=1, + y_coord=1, + z_coord=0, + ts=1, + manual_detect=0, + on_members_world=0, + on_pvp_world=0, + world_number=301, + equip_ge_value=1, + equipment=Equipment( + equip_head_id=1, + equip_amulet_id=1, + equip_torso_id=1, + equip_legs_id=1, + equip_boots_id=1, + equip_cape_id=1, + equip_hands_id=1, + equip_weapon_id=1, + equip_shield_id=1, + ), + ) + + reports, errors = service._transform_detection([parsed]) + + assert not errors + assert isinstance(reports[0], ReportsToInsertStruct) + assert reports[0].report.reporter_id == 1 + + +@pytest.mark.asyncio() +async def test_send_to_kafka_raises_if_producer_missing(monkeypatch): + service = ReportsService() + monkeypatch.setattr(service, "_transform_detection", lambda data: ([], [])) + monkeypatch.setattr("bot_detector.kafka.kafka_manager.get_producer", lambda key: None) + + with pytest.raises(CustomError): + await service.send_to_kafka([]) + + +@pytest.mark.asyncio() +async def test_send_to_kafka_happy_path(monkeypatch): + service = ReportsService() + class _Producer: + def __init__(self): + self.calls = [] + + async def produce_one(self, report): + self.calls.append(report) + + producer = _Producer() + monkeypatch.setattr("bot_detector.kafka.kafka_manager.get_producer", lambda key: producer) + parsed = ParsedDetection( + reporter_id=1, + reported_id=2, + region_id=1, + x_coord=1, + y_coord=1, + z_coord=0, + ts=1, + manual_detect=0, + on_members_world=0, + on_pvp_world=0, + world_number=301, + equip_ge_value=1, + equipment=Equipment( + equip_head_id=1, + equip_amulet_id=1, + equip_torso_id=1, + equip_legs_id=1, + equip_boots_id=1, + equip_cape_id=1, + equip_hands_id=1, + equip_weapon_id=1, + equip_shield_id=1, + ), + ) + + await service.send_to_kafka([parsed]) + + assert producer.calls, "producer should be invoked" diff --git a/test/components/bot_detector/database/test_core.py b/test/components/bot_detector/database/test_core.py deleted file mode 100644 index b4e83c8..0000000 --- a/test/components/bot_detector/database/test_core.py +++ /dev/null @@ -1,5 +0,0 @@ -from bot_detector.database import core - - -def test_sample(): - assert core is not None diff --git a/test/components/bot_detector/database/test_report_repository.py b/test/components/bot_detector/database/test_report_repository.py index c179905..3353b87 100644 --- a/test/components/bot_detector/database/test_report_repository.py +++ b/test/components/bot_detector/database/test_report_repository.py @@ -1,7 +1,7 @@ from datetime import datetime -from bot_detector.database.report import ReportRepo -from bot_detector.structs import Equipment, ParsedDetection +from bot_detector.report.database.repository import ReportRepo +from bot_detector.report.structs import Equipment, ParsedDetection def _sample_detection( diff --git a/test/components/bot_detector/report/test_report_repo.py b/test/components/bot_detector/report/test_report_repo.py new file mode 100644 index 0000000..89862f3 --- /dev/null +++ b/test/components/bot_detector/report/test_report_repo.py @@ -0,0 +1,51 @@ +import datetime + +from bot_detector.report.database.repository import ReportRepo +from bot_detector.report.structs import Equipment, ParsedDetection + + +def _parsed_detection() -> ParsedDetection: + return ParsedDetection( + reporter_id=1, + reported_id=2, + region_id=1, + x_coord=10, + y_coord=20, + z_coord=0, + ts=1, + manual_detect=0, + on_members_world=0, + on_pvp_world=0, + world_number=301, + equip_ge_value=123, + equipment=Equipment( + equip_head_id=1, + equip_amulet_id=2, + equip_torso_id=3, + equip_legs_id=4, + equip_boots_id=5, + equip_cape_id=6, + equip_hands_id=7, + equip_weapon_id=8, + equip_shield_id=9, + ), + ) + + +def test_parse_reports_flattens_and_converts(): + repo = ReportRepo() + result = repo._parse_reports([_parsed_detection()]) + + assert len(result) == 1 + flattened = result[0] + assert flattened["reporter_id"] == 1 + assert flattened["equip_head_id"] == 1 + assert isinstance(flattened["timestamp"], datetime.datetime) + + +def test_parse_reports_skips_invalid_entries(): + repo = ReportRepo() + + result = repo._parse_reports(["not-a-detection"]) + + assert result == []