diff --git a/CHANGELOG.md b/CHANGELOG.md index c57999c1..ce88d94f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,9 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and ## Unreleased +- {pull}`739` closes file descriptors for the capture manager between CLI runs and + disposes stale database engines to prevent hitting OS file descriptor limits in + large test runs. - {pull}`725` fixes the pickle node hash test by accounting for Python 3.14's default pickle protocol. - {pull}`726` adapts the interactive debugger integration to Python 3.14's diff --git a/src/_pytask/capture.py b/src/_pytask/capture.py index c4201b1f..8d947710 100644 --- a/src/_pytask/capture.py +++ b/src/_pytask/capture.py @@ -57,6 +57,7 @@ from types import TracebackType from _pytask.node_protocols import PTask + from _pytask.session import Session @hookimpl @@ -109,6 +110,14 @@ def pytask_post_parse(config: dict[str, Any]) -> None: capman.suspend() +@hookimpl +def pytask_unconfigure(session: Session) -> None: + """Stop capturing and release file descriptors.""" + capman = session.config["pm"].get_plugin("capturemanager") + if isinstance(capman, CaptureManager): + capman.stop_capturing() + + # Copied from pytest with slightly modified docstrings. diff --git a/src/_pytask/database_utils.py b/src/_pytask/database_utils.py index 77be06c5..63810343 100644 --- a/src/_pytask/database_utils.py +++ b/src/_pytask/database_utils.py @@ -14,6 +14,8 @@ from _pytask.dag_utils import node_and_neighbors if TYPE_CHECKING: + from sqlalchemy.engine import Engine + from _pytask.node_protocols import PNode from _pytask.node_protocols import PTask from _pytask.session import Session @@ -29,6 +31,7 @@ DatabaseSession = sessionmaker() +_ENGINE: Engine | None = None class BaseTable(DeclarativeBase): @@ -47,9 +50,12 @@ class State(BaseTable): def create_database(url: str) -> None: """Create the database.""" - engine = create_engine(url) - BaseTable.metadata.create_all(bind=engine) - DatabaseSession.configure(bind=engine) + global _ENGINE # noqa: PLW0603 + if _ENGINE is not None: + _ENGINE.dispose() + _ENGINE = create_engine(url) + BaseTable.metadata.create_all(bind=_ENGINE) + DatabaseSession.configure(bind=_ENGINE) def _create_or_update_state(first_key: str, second_key: str, hash_: str) -> None: