Update tests for new status_msg

Author: ConorMacBride

tests/subtests/helpers.py

@@ -53,7 +53,7 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra
 
         # Swap the baseline and result hashes in the summary
         # for the corresponding hashes in each hash library
-        if baseline_hash_library:
+        if baseline_hash_library and test in baseline_hash_library:
             baseline_summary = replace_hash(baseline_summary, 'baseline_hash',
                                             baseline_hash_library[test])
         if result_hash_library:
@@ -114,7 +114,11 @@ def diff_dict_item(baseline, result, error=''):
     if isinstance(baseline, (bool, type(None))) and baseline is result:
         return
 
-    # Handle str, int and float
+    # Handle float
+    if isinstance(baseline, float) and abs(baseline - result) < 1e-4:
+        return
+
+    # Handle str and int
     if baseline == result:
         return
 

tests/subtests/test_subtest.py

@@ -6,8 +6,8 @@
 
 import matplotlib
 import matplotlib.ft2font
-from packaging.version import Version
 import pytest
+from packaging.version import Version
 
 from .helpers import assert_existence, diff_summary, patch_summary
 
@@ -20,7 +20,8 @@
 HASH_LIBRARY_FLAG = rf'--mpl-hash-library={HASH_LIBRARY}'
 FULL_BASELINE_PATH = Path(__file__).parent / 'baseline'
 
-BASELINE_IMAGES_FLAG = '--mpl-baseline-path=baseline'
+BASELINE_IMAGES_FLAG_REL = ['--mpl-baseline-path=baseline', '--mpl-baseline-relative']
+BASELINE_IMAGES_FLAG_ABS = rf'--mpl-baseline-path={FULL_BASELINE_PATH}'
 
 TEST_FILE = Path(__file__).parent / 'subtest.py'
 
@@ -121,19 +122,19 @@ def test_hash(tmp_path):
 
 @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
 def test_hybrid(tmp_path):
-    run_subtest('test_hybrid', tmp_path, [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG])
+    run_subtest('test_hybrid', tmp_path, [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS])
 
 
 @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
 def test_results_always(tmp_path):
     run_subtest('test_results_always', tmp_path,
-                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG, '--mpl-results-always'])
+                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always'])
 
 
 @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
 def test_html(tmp_path):
     run_subtest('test_results_always', tmp_path,
-                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG], summaries=['html'])
+                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'])
     assert (tmp_path / 'results' / 'fig_comparison.html').exists()
     assert (tmp_path / 'results' / 'extra.js').exists()
     assert (tmp_path / 'results' / 'styles.css').exists()
@@ -157,5 +158,5 @@ def test_html_images_only(tmp_path):
 @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
 def test_basic_html(tmp_path):
     run_subtest('test_results_always', tmp_path,
-                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG], summaries=['basic-html'])
+                [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'])
     assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists()

tests/test_pytest_mpl.py

@@ -291,7 +291,7 @@ def test_hash_fails(tmpdir):
     # If we use --mpl, it should detect that the figure is wrong
     output = assert_pytest_fails_with(['--mpl', test_file], "doesn't match hash FAIL in library")
     # We didn't specify a baseline dir so we shouldn't attempt to find one
-    assert "Unable to find baseline image" not in output, output
+    assert "Image file not found for comparison test" not in output, output
 
     # Check that the summary path is printed and that it exists.
     output = assert_pytest_fails_with(['--mpl', test_file, '--mpl-generate-summary=html'],
@@ -337,7 +337,7 @@ def test_hash_fail_hybrid(tmpdir):
     output = assert_pytest_fails_with(['--mpl', test_file,
                                        '--mpl-baseline-path=/not/a/path'],
                                       "doesn't match hash FAIL in library")
-    assert "Unable to find baseline image" in output, output
+    assert "Image file not found for comparison test" in output, output
 
     # Assert reports image comparison succeeds
     output = assert_pytest_fails_with(['--mpl', test_file,

tox.ini

@@ -33,7 +33,8 @@ extras =
 commands =
     pip freeze
     # Make sure the tests pass with and without --mpl
-    pytest '{toxinidir}' {posargs}
+    # Use -m so pytest skips "subtests" which always apply --mpl
+    pytest '{toxinidir}' -m "mpl_image_compare" {posargs}
     pytest '{toxinidir}' --mpl --cov pytest_mpl {posargs}
 
 [testenv:codestyle]