Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
5 changes: 4 additions & 1 deletion .github/workflows/main.yml
Original file line number Diff line number Diff line change
Expand Up @@ -28,8 +28,11 @@ jobs:
enable-cache: true
- name: Install just
uses: extractions/setup-just@v3
- name: Install graphviz
run: |
sudo apt-get update
sudo apt-get install graphviz graphviz-dev
- run: just typing
- run: just typing-nb

run-tests:

Expand Down
5 changes: 3 additions & 2 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,10 @@ releases are available on [PyPI](https://pypi.org/project/pytask) and

- {pull}`725` fixes the pickle node hash test by accounting for Python 3.14's
default pickle protocol.
- {pull}`???` adapts the interactive debugger integration to Python 3.14's
- {pull}`726` adapts the interactive debugger integration to Python 3.14's
updated `pdb` behaviour and keeps pytest-style capturing intact.
- {pull}`???` updates the comparison to other tools documentation and adds a section on
- {pull}`734` migrates from mypy to ty for type checking.
- {pull}`736` updates the comparison to other tools documentation and adds a section on
the Common Workflow Language (CWL) and WorkflowHub.

## 0.5.7 - 2025-11-22
Expand Down
10 changes: 1 addition & 9 deletions justfile
Original file line number Diff line number Diff line change
Expand Up @@ -10,17 +10,9 @@ test *FLAGS:
test-cov *FLAGS:
uv run --group test pytest --nbmake --cov=src --cov=tests --cov-report=xml -n auto {{FLAGS}}

# Run tests with notebook validation
test-nb:
uv run --group test pytest --nbmake -n auto

# Run type checking
typing:
uv run --group typing --no-dev --isolated mypy

# Run type checking on notebooks
typing-nb:
uv run --group typing --no-dev --isolated nbqa mypy --ignore-missing-imports .
uv run --group typing --group test ty check src/ tests/

# Run linting
lint:
Expand Down
43 changes: 12 additions & 31 deletions pyproject.toml
Original file line number Diff line number Diff line change
Expand Up @@ -46,7 +46,6 @@ name = "Tobias Raabe"
email = "raabe@posteo.de"

[dependency-groups]
dev = ["pygraphviz>=1.12;platform_system=='Linux'"]
docs = [
"furo>=2024.8.6",
"ipython>=8.13.2",
Expand All @@ -64,6 +63,7 @@ docs = [
]
plugin-list = ["httpx>=0.27.0", "tabulate[widechars]>=0.9.0", "tqdm>=4.66.3"]
test = [
"cloudpickle>=3.0.0",
"deepdiff>=7.0.0",
"nbmake>=1.5.5",
"pygments>=2.18.0",
Expand All @@ -72,11 +72,11 @@ test = [
"pytest-cov>=5.0.0",
"pytest-xdist>=3.6.1",
"syrupy>=4.5.0",
"aiohttp>=3.11.0", # For HTTPPath tests.
"aiohttp>=3.11.0", # For HTTPPath tests.
"coiled>=1.42.0",
"cloudpickle>=3.0.0",
"pygraphviz>=1.12;platform_system=='Linux'",
]
typing = ["mypy>=1.11.0", "nbqa>=1.8.5"]
typing = ["ty>=0.0.7"]

[project.urls]
Changelog = "https://pytask-dev.readthedocs.io/en/stable/changes.html"
Expand Down Expand Up @@ -167,33 +167,14 @@ filterwarnings = [
"ignore:The --rsyncdir command line argument:DeprecationWarning",
]

[tool.mypy]
files = ["src", "tests"]
check_untyped_defs = true
disallow_any_generics = true
disallow_incomplete_defs = true
disallow_untyped_defs = true
no_implicit_optional = true
warn_redundant_casts = true
warn_unused_ignores = true
disable_error_code = ["import-untyped"]

[[tool.mypy.overrides]]
module = "tests.*"
disallow_untyped_defs = false
ignore_errors = true

[[tool.mypy.overrides]]
module = ["click_default_group", "networkx"]
ignore_missing_imports = true

[[tool.mypy.overrides]]
module = ["_pytask.coiled_utils"]
disable_error_code = ["import-not-found"]

[[tool.mypy.overrides]]
module = ["_pytask.hookspecs"]
disable_error_code = ["empty-body"]
[tool.ty.rules]
unused-ignore-comment = "error"

[tool.ty.src]
exclude = ["src/_pytask/_hashlib.py"]

[tool.ty.terminal]
error-on-warning = true

[tool.coverage.report]
exclude_also = [
Expand Down
19 changes: 16 additions & 3 deletions src/_pytask/build.py
Original file line number Diff line number Diff line change
Expand Up @@ -9,6 +9,7 @@
from typing import TYPE_CHECKING
from typing import Any
from typing import Literal
from typing import cast

import click

Expand Down Expand Up @@ -65,7 +66,7 @@ def pytask_unconfigure(session: Session) -> None:
path.write_text(json.dumps(HashPathCache._cache))


def build( # noqa: C901, PLR0912, PLR0913
def build( # noqa: C901, PLR0912, PLR0913, PLR0915
*,
capture: Literal["fd", "no", "sys", "tee-sys"] | CaptureMethod = CaptureMethod.FD,
check_casing_of_paths: bool = True,
Expand Down Expand Up @@ -230,10 +231,22 @@ def build( # noqa: C901, PLR0912, PLR0913

raw_config = {**DEFAULTS_FROM_CLI, **raw_config}

raw_config["paths"] = parse_paths(raw_config["paths"])
paths_value = raw_config["paths"]
# Convert tuple to list since parse_paths expects Path | list[Path]
if isinstance(paths_value, tuple):
paths_value = list(paths_value)
if not isinstance(paths_value, (Path, list)):
msg = f"paths must be Path or list, got {type(paths_value)}"
raise TypeError(msg) # noqa: TRY301
# Cast is justified - we validated at runtime
raw_config["paths"] = parse_paths(cast("Path | list[Path]", paths_value))

if raw_config["config"] is not None:
raw_config["config"] = Path(raw_config["config"]).resolve()
config_value = raw_config["config"]
if not isinstance(config_value, (str, Path)):
msg = f"config must be str or Path, got {type(config_value)}"
raise TypeError(msg) # noqa: TRY301
raw_config["config"] = Path(config_value).resolve()
raw_config["root"] = raw_config["config"].parent
else:
(
Expand Down
27 changes: 23 additions & 4 deletions src/_pytask/cache.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,6 +8,9 @@
from inspect import FullArgSpec
from typing import TYPE_CHECKING
from typing import Any
from typing import ParamSpec
from typing import Protocol
from typing import TypeVar

from attrs import define
from attrs import field
Expand All @@ -16,6 +19,20 @@

if TYPE_CHECKING:
from collections.abc import Callable
from typing import TypeAlias

from ty_extensions import Intersection

Memoized: TypeAlias = "Intersection[Callable[P, R], HasCache]"

P = ParamSpec("P")
R = TypeVar("R")


class HasCache(Protocol):
"""Protocol for objects that have a cache attribute."""

cache: Cache


@define
Expand All @@ -30,12 +47,14 @@ class Cache:
_sentinel: Any = field(factory=object)
cache_info: CacheInfo = field(factory=CacheInfo)

def memoize(self, func: Callable[..., Any]) -> Callable[..., Any]:
prefix = f"{func.__module__}.{func.__name__}:"
def memoize(self, func: Callable[P, R]) -> Memoized[P, R]:
func_module = getattr(func, "__module__", "")
func_name = getattr(func, "__name__", "")
prefix = f"{func_module}.{func_name}:"
argspec = inspect.getfullargspec(func)

@functools.wraps(func)
def wrapped(*args: Any, **kwargs: Any) -> Callable[..., Any]:
def wrapped(*args: P.args, **kwargs: P.kwargs) -> R:
key = _make_memoize_key(
args, kwargs, typed=False, argspec=argspec, prefix=prefix
)
Expand All @@ -50,7 +69,7 @@ def wrapped(*args: Any, **kwargs: Any) -> Callable[..., Any]:

return value

wrapped.cache = self # type: ignore[attr-defined]
wrapped.cache = self # ty: ignore[unresolved-attribute]

return wrapped

Expand Down
7 changes: 4 additions & 3 deletions src/_pytask/capture.py
Original file line number Diff line number Diff line change
Expand Up @@ -129,7 +129,8 @@ def mode(self) -> str:
# TextIOWrapper doesn't expose a mode, but at least some of our
# tests check it.
assert hasattr(self.buffer, "mode")
return cast("str", self.buffer.mode.replace("b", ""))
mode_value = cast("str", self.buffer.mode)
return mode_value.replace("b", "")


class CaptureIO(io.TextIOWrapper):
Expand All @@ -146,7 +147,7 @@ def __init__(self, other: TextIO) -> None:
self._other = other
super().__init__()

def write(self, s: str) -> int:
def write(self, s: str) -> int: # ty: ignore[invalid-method-override]
super().write(s)
return self._other.write(s)

Expand Down Expand Up @@ -209,7 +210,7 @@ def truncate(self, size: int | None = None) -> int: # noqa: ARG002
msg = "Cannot truncate stdin."
raise UnsupportedOperation(msg)

def write(self, data: str) -> int: # noqa: ARG002
def write(self, data: str) -> int: # noqa: ARG002 # ty: ignore[invalid-method-override]
msg = "Cannot write to stdin."
raise UnsupportedOperation(msg)

Expand Down
15 changes: 7 additions & 8 deletions src/_pytask/click.py
Original file line number Diff line number Diff line change
Expand Up @@ -37,12 +37,10 @@
if importlib.metadata.version("click") < "8.2":
from click.parser import split_opt
else:
from click.parser import ( # type: ignore[attr-defined, no-redef, unused-ignore]
_split_opt as split_opt,
)
from click.parser import _split_opt as split_opt # ty: ignore[unresolved-import]


class EnumChoice(Choice): # type: ignore[type-arg, unused-ignore]
class EnumChoice(Choice):
"""An enum-based choice type.

The implementation is copied from https://github.com/pallets/click/pull/2210 and
Expand Down Expand Up @@ -75,7 +73,7 @@ def convert(self, value: Any, param: Parameter | None, ctx: Context | None) -> A
class _OptionHighlighter(RegexHighlighter):
"""A highlighter for help texts."""

highlights: ClassVar = [ # type: ignore[misc]
highlights: ClassVar = [
r"(?P<switch>\-\w)\b",
r"(?P<option>\-\-[\w\-]+)",
r"\-\-[\w\-]+(?P<metavar>[ |=][\w\.:]+)",
Expand Down Expand Up @@ -114,7 +112,7 @@ def format_help(
else:
formatted_name = Text(command_name, style="command")

commands_table.add_row(formatted_name, highlighter(command.help))
commands_table.add_row(formatted_name, highlighter(command.help or ""))

console.print(
Panel(
Expand Down Expand Up @@ -177,12 +175,13 @@ def parse_args(self, ctx: Context, args: list[str]) -> list[str]:
_value, args = param.handle_parse_result(ctx, opts, args)

if args and not ctx.allow_extra_args and not ctx.resilient_parsing:
args_list = list(args) if not isinstance(args, list) else args
ctx.fail(
ngettext(
"Got unexpected extra argument ({args})",
"Got unexpected extra arguments ({args})",
len(args),
).format(args=" ".join(map(str, args)))
).format(args=" ".join(str(arg) for arg in args_list))
)

ctx.args = args
Expand Down Expand Up @@ -328,7 +327,7 @@ def _format_help_text( # noqa: C901, PLR0912, PLR0915
elif param.is_bool_flag and param.secondary_opts: # type: ignore[attr-defined]
# For boolean flags that have distinct True/False opts,
# use the opt without prefix instead of the value.
default_string = split_opt( # type: ignore[operator, unused-ignore]
default_string = split_opt(
(param.opts if param.default else param.secondary_opts)[0]
)[1]
elif (
Expand Down
10 changes: 5 additions & 5 deletions src/_pytask/coiled_utils.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,7 +13,7 @@
except ImportError:

@define
class Function: # type: ignore[no-redef]
class Function:
cluster_kwargs: dict[str, Any]
environ: dict[str, Any]
function: Callable[..., Any] | None
Expand All @@ -26,9 +26,9 @@ class Function: # type: ignore[no-redef]
def extract_coiled_function_kwargs(func: Function) -> dict[str, Any]:
"""Extract the kwargs for a coiled function."""
return {
"cluster_kwargs": func._cluster_kwargs,
"cluster_kwargs": func._cluster_kwargs, # ty: ignore[possibly-missing-attribute]
"keepalive": func.keepalive,
"environ": func._environ,
"local": func._local,
"name": func._name,
"environ": func._environ, # ty: ignore[possibly-missing-attribute]
"local": func._local, # ty: ignore[possibly-missing-attribute]
"name": func._name, # ty: ignore[possibly-missing-attribute]
}
5 changes: 3 additions & 2 deletions src/_pytask/collect.py
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,7 @@
from _pytask.task_utils import COLLECTED_TASKS
from _pytask.task_utils import parse_collected_tasks_with_task_marker
from _pytask.task_utils import task as task_decorator
from _pytask.typing import TaskFunction
from _pytask.typing import is_task_function

if TYPE_CHECKING:
Expand Down Expand Up @@ -115,7 +116,7 @@ def _collect_from_tasks(session: Session) -> None:

for raw_task in to_list(session.config.get("tasks", ())):
if is_task_function(raw_task):
if not hasattr(raw_task, "pytask_meta"):
if not isinstance(raw_task, TaskFunction):
raw_task = task_decorator()(raw_task) # noqa: PLW2901

path = get_file(raw_task)
Expand Down Expand Up @@ -339,7 +340,7 @@ def pytask_collect_task(

markers = get_all_marks(obj)

if hasattr(obj, "pytask_meta"):
if isinstance(obj, TaskFunction):
attributes = {
**obj.pytask_meta.attributes,
"collection_id": obj.pytask_meta._id,
Expand Down
16 changes: 6 additions & 10 deletions src/_pytask/collect_command.py
Original file line number Diff line number Diff line change
Expand Up @@ -124,15 +124,11 @@ def _find_common_ancestor_of_all_nodes(
for task in tasks:
all_paths.append(task.path)
if show_nodes:
all_paths.extend( # type: ignore[var-annotated]
x.path
for x in tree_leaves(task.depends_on) # type: ignore[arg-type]
if isinstance(x, PPathNode)
all_paths.extend(
x.path for x in tree_leaves(task.depends_on) if isinstance(x, PPathNode)
)
all_paths.extend( # type: ignore[var-annotated]
x.path
for x in tree_leaves(task.produces) # type: ignore[arg-type]
if isinstance(x, PPathNode)
all_paths.extend(
x.path for x in tree_leaves(task.produces) if isinstance(x, PPathNode)
)

return find_common_ancestor(*all_paths, *paths)
Expand Down Expand Up @@ -200,7 +196,7 @@ def _print_collected_tasks(
)

if show_nodes:
deps: list[Any] = list(tree_leaves(task.depends_on)) # type: ignore[arg-type]
deps: list[Any] = list(tree_leaves(task.depends_on))
for node in sorted(
deps,
key=(
Expand All @@ -212,7 +208,7 @@ def _print_collected_tasks(
text = format_node_name(node, (common_ancestor,))
task_branch.add(Text.assemble(FILE_ICON, "<Dependency ", text, ">"))

products: list[Any] = list(tree_leaves(task.produces)) # type: ignore[arg-type]
products: list[Any] = list(tree_leaves(task.produces))
for node in sorted(
products,
key=lambda x: x.path.as_posix()
Expand Down
Loading