Skip to content

Commit b000a9a

Browse files
authored
Add an example of reading CSV file with pandas, with tests (#33)
Also fix some cosmetic issues
1 parent f9aa49d commit b000a9a

File tree

5 files changed

+75
-14
lines changed

5 files changed

+75
-14
lines changed

README.md

Lines changed: 11 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -235,17 +235,17 @@ configuration. If both are done, the test level configuration takes precedence o
235235
If multiple `conftest.py` files register a reader for the same file extension, the closest one from the current test
236236
becomes effective.
237237

238-
Here are some of the common readers you could use:
239-
240-
| File type | Reader | Notes |
241-
|-----------|-----------------------------------------|---------------------------------------------------|
242-
| .json | `json.load` | The plugin automatically uses this by default |
243-
| .csv | `csv.reader`, `csv.DictReader` | |
244-
| .yml | `yaml.safe_load`, `yaml.safe_load_all` | Requires `PyYAML` |
245-
| .xml | `xml.etree.ElementTree.parse` | |
246-
| .toml | `tomllib.load` | `tomli.load` for Python <3.11 (Requires `tomli`) |
247-
| .ini | `configparser.ConfigParser().read_file` | |
248-
| .pdf | `pypdf.PdfReader` | Requires `pypdf` |
238+
As examples, here are some of the common readers you could use:
239+
240+
| File type | Reader | Notes |
241+
|-----------|---------------------------------------------------|---------------------------------------------------|
242+
| .json | `json.load` | The plugin automatically uses this by default |
243+
| .csv | `csv.reader`, `csv.DictReader`, `pandas.csv_read` | `pandas.csv_read` requires `pandas` |
244+
| .yml | `yaml.safe_load`, `yaml.safe_load_all` | Requires `PyYAML` |
245+
| .xml | `xml.etree.ElementTree.parse` | |
246+
| .toml | `tomllib.load` | `tomli.load` for Python <3.11 (Requires `tomli`) |
247+
| .ini | `configparser.ConfigParser().read_file` | |
248+
| .pdf | `pypdf.PdfReader` | Requires `pypdf` |
249249

250250
Here are some examples of loading a CSV file using the built-in CSV readers with file read options:
251251

pyproject.toml

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -44,9 +44,11 @@ test = [
4444
"pytest-mock>=3.0.0,<4",
4545
"pytest-smoke",
4646
"pytest-xdist[psutil]>=2.3.0,<4",
47+
# for testing with 3rd party file readers
48+
"pandas>=2.0.0",
49+
"pypdf>=6.1.0",
4750
"pyyaml>=6.0.0",
4851
'tomli>=2.3.0; python_version < "3.11"',
49-
"pypdf>=6.1.0"
5052
]
5153
dev = [
5254
"tox>=4.0.0,<5",

tests/tests_loader/tests_reader/test_csv_reader.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -75,7 +75,7 @@ def test_parametrize_csv_file_with_reader(request: FixtureRequest, file_path: Pa
7575

7676

7777
@parametrize(("file_path", "data"), PATH_CSV_FILE, file_reader=csv.DictReader, **read_options)
78-
def test_parametrize_csv_file_with_dict_reader(request: FixtureRequest, file_path: Path, data: Any) -> None:
78+
def test_parametrize_csv_file_with_dict_reader(request: FixtureRequest, file_path: Path, data: dict[str, str]) -> None:
7979
"""Test @parametrize loader with CSV DictReader reader"""
8080
assert isinstance(data, dict)
8181
expected_data = get_expected_data(file_path, read_options)
Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
from collections.abc import Hashable
2+
from pathlib import Path
3+
4+
import pandas
5+
import pytest
6+
from pytest import FixtureRequest
7+
8+
from pytest_data_loader import load, parametrize, parametrize_dir
9+
from tests.tests_loader.helper import PATH_CSV_FILE, PATH_CSV_FILE_DIR, get_parametrized_test_idx
10+
11+
pytestmark = pytest.mark.readers
12+
13+
14+
DELIMITER = {"comma": ",", "semicolon": ","}
15+
16+
17+
@load(("file_path", "df"), PATH_CSV_FILE, file_reader=pandas.read_csv)
18+
def test_load_csv_file_with_pandas(file_path: Path, df: pandas.DataFrame) -> None:
19+
"""Test @load loader with the CSV reader from pandas"""
20+
assert isinstance(df, pandas.DataFrame)
21+
expected_data = get_expected_data(file_path)
22+
for i, row in df.iterrows():
23+
assert isinstance(row, pandas.Series)
24+
assert DELIMITER[file_path.stem].join(row.to_list()) == expected_data[i + 1]
25+
26+
27+
@parametrize(
28+
("file_path", "idx_and_row"), PATH_CSV_FILE, file_reader=pandas.read_csv, parametrizer_func=lambda df: df.iterrows()
29+
)
30+
def test_parametrize_csv_file_with_pandas(
31+
request: FixtureRequest, file_path: Path, idx_and_row: tuple[Hashable, pandas.Series]
32+
) -> None:
33+
"""Test @parametrize loader with the CSV reader from pands"""
34+
assert isinstance(idx_and_row, tuple)
35+
i, row = idx_and_row
36+
expected_data = get_expected_data(file_path)
37+
assert i == get_parametrized_test_idx(request, arg_name="idx_and_row")
38+
assert DELIMITER[file_path.stem].join(row.to_list()) == expected_data[i + 1]
39+
40+
41+
@parametrize_dir(
42+
("file_path", "df"),
43+
PATH_CSV_FILE_DIR,
44+
file_reader_func=lambda p: (lambda f: pandas.read_csv(f, delimiter=DELIMITER[p.stem])),
45+
)
46+
def test_parametrize_dir_with_pandas(file_path: Path, df: pandas.DataFrame) -> None:
47+
"""Test @parametrize_dir loader with the CSV reader from pands"""
48+
assert isinstance(df, pandas.DataFrame)
49+
expected_data = get_expected_data(file_path)
50+
rows: list[pandas.Series] = []
51+
for i, row in df.iterrows():
52+
assert DELIMITER[file_path.stem].join(row.to_list()) == expected_data[i + 1]
53+
rows.append(row)
54+
assert len(rows) + 1 == len(expected_data)
55+
56+
57+
def get_expected_data(file_path: Path) -> list[str]:
58+
with open(file_path) as f:
59+
return f.read().splitlines()

tests/tests_loader/tests_reader/test_text_reader.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,7 +11,7 @@
1111

1212

1313
# NOTE: These tests don't make much sense in reality since open() already handles the same things.
14-
# These tests just make sures that the file_reader option works as low level
14+
# These tests just make sure that the file_reader option works as low level
1515

1616

1717
@load(

0 commit comments

Comments
 (0)