Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
11 changes: 9 additions & 2 deletions .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -43,7 +43,7 @@ jobs:
fail-fast: false
matrix:
os: ['ubuntu-latest', 'macos-latest', 'windows-latest']
python-version: ['3.10', '3.11', '3.12', '3.13']
python-version: ['3.10', '3.11', '3.12', '3.13', '3.14']

steps:
- uses: actions/checkout@v6
Expand All @@ -60,9 +60,16 @@ jobs:
sudo apt-get install graphviz graphviz-dev

- name: Run tests, doctests, and notebook tests
if: matrix.os != 'windows-latest' || matrix.python-version != '3.14'
shell: bash -l {0}
run: just test-cov

# pywin32 has no wheels for Python 3.14 yet, so skip notebook tests
- name: Run tests without notebook tests (Windows + Python 3.14)
if: matrix.os == 'windows-latest' && matrix.python-version == '3.14'
shell: bash -l {0}
run: uv run --group test pytest --cov=src --cov=tests --cov-report=xml -n auto

- name: Upload test coverage reports to Codecov with GitHub Action
uses: codecov/codecov-action@v5

Expand All @@ -71,5 +78,5 @@ jobs:
run: just test-lowest

- name: Run tests with highest resolution
if: matrix.python-version == '3.13' && matrix.os == 'ubuntu-latest'
if: matrix.python-version == '3.14' && matrix.os == 'ubuntu-latest'
run: just test-highest
2 changes: 1 addition & 1 deletion .python-version
Original file line number Diff line number Diff line change
@@ -1 +1 @@
3.12
3.14
25 changes: 13 additions & 12 deletions .readthedocs.yaml
Original file line number Diff line number Diff line change
@@ -1,18 +1,19 @@
version: 2

sphinx:
configuration: docs/source/conf.py
fail_on_warning: true

build:
os: ubuntu-24.04
tools:
python: "3.12"
python: "3.13"
jobs:
create_environment:
- asdf plugin add uv
- asdf install uv latest
- asdf global uv latest
- UV_PROJECT_ENVIRONMENT=$READTHEDOCS_VIRTUALENV_PATH uv sync --group docs
install:
- "true"

sphinx:
configuration: docs/source/conf.py
fail_on_warning: true
pre_create_environment:
- asdf plugin add uv
- asdf install uv latest
- asdf global uv latest
create_environment:
- uv venv "${READTHEDOCS_VIRTUALENV_PATH}"
install:
- UV_PROJECT_ENVIRONMENT="${READTHEDOCS_VIRTUALENV_PATH}" uv sync --frozen --group docs
3 changes: 2 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,8 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and
- {pull}`706` disables syntax highlighting for platform version information in session header.
- {pull}`707` drops support for Python 3.9 as it has reached end of life.
- {pull}`708` updates mypy and fixes type issues.
- {pull}`709` add uv pre-commit check.
- {pull}`709` adds uv pre-commit check.
- {pull}`710` adds support for Python 3.14.
- {pull}`713` removes uv as a test dependency. Closes {issue}`712`. Thanks to {user}`erooke`!
- {pull}`718` fixes {issue}`717` by properly parsing the `pdbcls` configuration option from config files. Thanks to {user}`MImmesberger` for the report!
- {pull}`719` fixes repeated tasks with the same function name in the programmatic interface to ensure all tasks execute correctly.
Expand Down
2 changes: 1 addition & 1 deletion justfile
Original file line number Diff line number Diff line change
Expand Up @@ -35,4 +35,4 @@ test-lowest:

# Run tests with highest dependency resolution (like CI)
test-highest:
uv run --python 3.13 --group test --resolution highest pytest --nbmake -n auto
uv run --python 3.14 --group test --resolution highest pytest --nbmake -n auto
19 changes: 16 additions & 3 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ classifiers = [
"Programming Language :: Python :: 3.11",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Topic :: Scientific/Engineering",
"Topic :: Software Development :: Build Tools",
]
Expand Down Expand Up @@ -59,20 +60,21 @@ docs = [
"sphinx-design>=0.3",
"sphinx-toolbox>=4.0.0",
"sphinxext-opengraph>=0.10.0",
"sphinx-autobuild>=2024.10.3",
]
docs-live = ["sphinx-autobuild>=2024.10.3"]
plugin-list = ["httpx>=0.27.0", "tabulate[widechars]>=0.9.0", "tqdm>=4.66.3"]
test = [
"cloudpickle>=3.0.0",
"deepdiff>=7.0.0",
"nbmake>=1.5.5",
# nbmake requires pywin32 on Windows, which has no wheels for Python 3.14 yet
"nbmake>=1.5.5; platform_system != 'Windows' or python_version < '3.14'",
"pygments>=2.18.0",
"pexpect>=4.9.0",
"pytest>=8.4.0",
"pytest-cov>=5.0.0",
"pytest-xdist>=3.6.1",
"syrupy>=4.5.0",
"aiohttp>=3.11.0", # For HTTPPath tests.
"aiohttp>=3.11.0", # For HTTPPath tests.
"coiled>=1.42.0",
"pygraphviz>=1.12;platform_system=='Linux'",
]
Expand Down Expand Up @@ -165,11 +167,22 @@ markers = [
filterwarnings = [
"ignore:'@pytask.mark.*. is deprecated:FutureWarning",
"ignore:The --rsyncdir command line argument:DeprecationWarning",
"ignore:'asyncio\\..*' is deprecated:DeprecationWarning",
]

[tool.ty.rules]
unused-ignore-comment = "error"

[[tool.ty.overrides]]
include = [
"src/_pytask/_version.py",
"src/_pytask/click.py",
"tests/test_dag_command.py",
]

[tool.ty.overrides.rules]
unused-ignore-comment = "ignore"

[tool.ty.src]
exclude = ["src/_pytask/_hashlib.py"]

Expand Down
136 changes: 135 additions & 1 deletion src/_pytask/_inspect.py
Original file line number Diff line number Diff line change
@@ -1,6 +1,140 @@
from __future__ import annotations

import ast
import inspect
import sys
from inspect import get_annotations as _get_annotations_from_inspect
from typing import TYPE_CHECKING
from typing import Any
from typing import cast

if TYPE_CHECKING:
from collections.abc import Callable

__all__ = ["get_annotations"]


from inspect import get_annotations
def get_annotations(
obj: Callable[..., Any],
*,
globals: dict[str, Any] | None = None, # noqa: A002
locals: dict[str, Any] | None = None, # noqa: A002
eval_str: bool = False,
) -> dict[str, Any]:
"""Return evaluated annotations with better support for deferred evaluation.

Context
-------
* PEP 649 introduces deferred annotations which are only evaluated when explicitly
requested. See https://peps.python.org/pep-0649/ for background and why locals can
disappear between definition and evaluation time.
* Python 3.14 ships :mod:`annotationlib` which exposes the raw annotation source and
provides the building blocks we reuse here. The module doc explains the available
formats: https://docs.python.org/3/library/annotationlib.html
* Other projects run into the same constraints. Pydantic tracks their work in
https://github.com/pydantic/pydantic/issues/12080; we might copy improvements from
there once they settle on a stable strategy.

Rationale
---------
When annotations refer to loop variables inside task generators, the locals that
existed during decoration have vanished by the time pytask evaluates annotations
while collecting tasks. Using :func:`inspect.get_annotations` would therefore yield
the same product path for every repeated task. By asking :mod:`annotationlib` for
string representations and re-evaluating them with reconstructed locals (globals,
default arguments, and the frame locals captured via ``@task`` at decoration time)
we recover the correct per-task values. The frame locals capture is essential for
cases where loop variables are only referenced in annotations (not in the function
body or closure). If any of these ingredients are missing—for example on Python
versions without :mod:`annotationlib` - we fall back to the stdlib implementation,
so behaviour on 3.10-3.13 remains unchanged.
"""
if not eval_str or not hasattr(obj, "__globals__"):
return _get_annotations_from_inspect(
obj, globals=globals, locals=locals, eval_str=eval_str
)

if sys.version_info < (3, 14):
raw_annotations = _get_annotations_from_inspect(
obj, globals=globals, locals=locals, eval_str=False
)
evaluation_globals = cast(
"dict[str, Any]", obj.__globals__ if globals is None else globals
)
evaluation_locals = evaluation_globals if locals is None else locals
evaluated_annotations = {}
for name, expression in raw_annotations.items():
evaluated_annotations[name] = _evaluate_annotation_expression(
expression, evaluation_globals, evaluation_locals
)
return evaluated_annotations

import annotationlib # noqa: PLC0415

raw_annotations = annotationlib.get_annotations(
obj, globals=globals, locals=locals, format=annotationlib.Format.STRING
)

evaluation_globals = obj.__globals__ if globals is None else globals
evaluation_locals = _build_evaluation_locals(obj, locals)

evaluated_annotations = {}
for name, expression in raw_annotations.items():
evaluated_annotations[name] = _evaluate_annotation_expression(
expression, evaluation_globals, evaluation_locals
)

return evaluated_annotations


def _build_evaluation_locals(
obj: Callable[..., Any], provided_locals: dict[str, Any] | None
) -> dict[str, Any]:
# Order matters: later updates override earlier ones.
# Default arguments are lowest priority (fallbacks), then provided_locals,
# then snapshot_locals (captured loop variables) have highest priority.
evaluation_locals: dict[str, Any] = {}
evaluation_locals.update(_get_default_argument_locals(obj))
if provided_locals:
evaluation_locals.update(provided_locals)
evaluation_locals.update(_get_snapshot_locals(obj))
return evaluation_locals


def _get_snapshot_locals(obj: Callable[..., Any]) -> dict[str, Any]:
metadata = getattr(obj, "pytask_meta", None)
snapshot = getattr(metadata, "annotation_locals", None)
return dict(snapshot) if snapshot else {}


def _get_default_argument_locals(obj: Callable[..., Any]) -> dict[str, Any]:
try:
parameters = inspect.signature(obj).parameters.values()
except (TypeError, ValueError):
return {}

defaults = {}
for parameter in parameters:
if parameter.default is not inspect.Parameter.empty:
defaults[parameter.name] = parameter.default
return defaults


def _evaluate_annotation_expression(
expression: Any, globals_: dict[str, Any] | None, locals_: dict[str, Any]
) -> Any:
if not isinstance(expression, str):
return expression
evaluation_globals = globals_ if globals_ is not None else {}
evaluated = eval(expression, evaluation_globals, locals_) # noqa: S307
if isinstance(evaluated, str):
try:
literal = ast.literal_eval(expression)
except (SyntaxError, ValueError):
return evaluated
if isinstance(literal, str):
try:
return eval(literal, evaluation_globals, locals_) # noqa: S307
except Exception: # noqa: BLE001
return evaluated
return evaluated
6 changes: 4 additions & 2 deletions src/_pytask/click.py
Original file line number Diff line number Diff line change
Expand Up @@ -35,9 +35,11 @@


if importlib.metadata.version("click") < "8.2":
from click.parser import split_opt
from click.parser import split_opt as _split_opt
else:
from click.parser import _split_opt as split_opt # ty: ignore[unresolved-import]
from click.parser import _split_opt # ty: ignore[unresolved-import]

split_opt = _split_opt


class EnumChoice(Choice):
Expand Down
4 changes: 4 additions & 0 deletions src/_pytask/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -38,6 +38,9 @@ class CollectionMetadata:
kwargs
A dictionary containing keyword arguments which are passed to the task when it
is executed.
annotation_locals
A snapshot of local variables captured during decoration which helps evaluate
deferred annotations later on.
markers
A list of markers that are attached to the task.
name
Expand All @@ -51,6 +54,7 @@ class CollectionMetadata:

after: str | list[Callable[..., Any]] = field(factory=list)
attributes: dict[str, Any] = field(factory=dict)
annotation_locals: dict[str, Any] | None = None
is_generator: bool = False
id_: str | None = None
kwargs: dict[str, Any] = field(factory=dict)
Expand Down
Loading