Save a result hash library to results dir

Author: ConorMacBride

README.rst

@@ -229,6 +229,12 @@ test (based on the hash library) also shown in the generated
 summary. This option is applied automatically when generating
 a HTML summary.
 
+When the ``--mpl-results-always`` option is active, and some hash
+comparison tests are performed, a hash library containing all the
+result hashes will also be saved to the root of the results directory.
+The filename will be extracted from ``--mpl-generate-hash-library``,
+``--mpl-hash-library`` or ``hash_library=`` in that order.
+
 Base style
 ^^^^^^^^^^
 

pytest_mpl/plugin.py

@@ -283,6 +283,12 @@ def __init__(self,
             self.results_dir = Path(tempfile.mkdtemp(dir=self.results_dir))
         self.results_dir.mkdir(parents=True, exist_ok=True)
 
+        # Decide what to call the downloadable results hash library
+        if self.hash_library is not None:
+            self.results_hash_library_name = self.hash_library.name
+        else:  # Use the first filename encountered in a `hash_library=` kwarg
+            self.results_hash_library_name = None
+
         # We need global state to store all the hashes generated over the run
         self._generated_hash_library = {}
         self._test_results = {}
@@ -499,6 +505,10 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None):
         compare = self.get_compare(item)
         savefig_kwargs = compare.kwargs.get('savefig_kwargs', {})
 
+        if not self.results_hash_library_name:
+            # Use hash library name of current test as results hash library name
+            self.results_hash_library_name = Path(compare.kwargs.get("hash_library", "")).name
+
         hash_library_filename = self.hash_library or compare.kwargs.get('hash_library', None)
         hash_library_filename = (Path(item.fspath).parent / hash_library_filename).absolute()
 
@@ -674,11 +684,21 @@ def pytest_unconfigure(self, config):
         """
         Save out the hash library at the end of the run.
         """
+        result_hash_library = self.results_dir / (self.results_hash_library_name or "temp.json")
         if self.generate_hash_library is not None:
             hash_library_path = Path(config.rootdir) / self.generate_hash_library
             hash_library_path.parent.mkdir(parents=True, exist_ok=True)
             with open(hash_library_path, "w") as fp:
                 json.dump(self._generated_hash_library, fp, indent=2)
+            if self.results_always:  # Make accessible in results directory
+                result_hash_library.name = hash_library_path.name  # use same name as generated
+                shutil.copy(hash_library_path, result_hash_library)
+        elif self.results_always and self.results_hash_library_name:
+            result_hashes = {k: v['result_hash'] for k, v in self._test_results.items()
+                             if v['result_hash']}
+            if len(result_hashes) > 0:  # At least one hash comparison test
+                with open(result_hash_library, "w") as fp:
+                    json.dump(result_hashes, fp, indent=2)
 
         if self.generate_summary:
             if 'json' in self.generate_summary:

tests/subtests/test_subtest.py

@@ -34,6 +34,7 @@
 
 
 def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=True,
+                has_result_hashes=False,
                 update_baseline=UPDATE_BASELINE, update_summary=UPDATE_SUMMARY):
     """ Run pytest (within pytest) and check JSON summary report.
 
@@ -49,6 +50,9 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru
         Summaries to generate in addition to `json`.
     xfail : bool, optional, default=True
         Whether the overall pytest run should fail.
+    has_result_hashes : bool or str, optional, default=False
+        Whether a hash library is expected to exist in the results directory.
+        If a string, this is the name of the expected results file.
     """
     # Parse arguments
     if summaries is None:
@@ -110,6 +114,24 @@ def run_subtest(baseline_summary_name, tmp_path, args, summaries=None, xfail=Tru
     # Ensure reported images exist
     assert_existence(result_summary, path=results_path)
 
+    # Get expected name for the hash library saved to the results directory
+    if isinstance(has_result_hashes, str):
+        result_hash_file = tmp_path / 'results' / has_result_hashes
+        has_result_hashes = True  # convert to bool after processing str
+    else:
+        result_hash_file = tmp_path / 'results' / HASH_LIBRARY.name
+
+    # Compare the generated hash library to the expected hash library
+    if has_result_hashes:
+        assert result_hash_file.exists()
+        with open(RESULT_LIBRARY, "r") as f:
+            baseline = json.load(f)
+        with open(result_hash_file, "r") as f:
+            result = json.load(f)
+        diff_summary({'a': baseline}, {'a': result})
+    else:
+        assert not result_hash_file.exists()
+
 
 def test_default(tmp_path):
     run_subtest('test_default', tmp_path, [])
@@ -128,21 +150,24 @@ def test_hybrid(tmp_path):
 @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
 def test_results_always(tmp_path):
     run_subtest('test_results_always', tmp_path,
-                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always'])
+                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS, '--mpl-results-always'],
+                has_result_hashes=True)
 
 
 @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
 def test_html(tmp_path):
     run_subtest('test_results_always', tmp_path,
-                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'])
+                [HASH_LIBRARY_FLAG, BASELINE_IMAGES_FLAG_ABS], summaries=['html'],
+                has_result_hashes=True)
     assert (tmp_path / 'results' / 'fig_comparison.html').exists()
     assert (tmp_path / 'results' / 'extra.js').exists()
     assert (tmp_path / 'results' / 'styles.css').exists()
 
 
 @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
 def test_html_hashes_only(tmp_path):
-    run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html'])
+    run_subtest('test_html_hashes_only', tmp_path, [HASH_LIBRARY_FLAG], summaries=['html'],
+                has_result_hashes=True)
     assert (tmp_path / 'results' / 'fig_comparison.html').exists()
     assert (tmp_path / 'results' / 'extra.js').exists()
     assert (tmp_path / 'results' / 'styles.css').exists()
@@ -158,5 +183,6 @@ def test_html_images_only(tmp_path):
 @pytest.mark.skipif(not HASH_LIBRARY.exists(), reason="No hash library for this mpl version")
 def test_basic_html(tmp_path):
     run_subtest('test_results_always', tmp_path,
-                [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'])
+                [HASH_LIBRARY_FLAG, *BASELINE_IMAGES_FLAG_REL], summaries=['basic-html'],
+                has_result_hashes=True)
     assert (tmp_path / 'results' / 'fig_comparison_basic.html').exists()