Refactor image_status and hash_status to plugin.py

Include specific results from the image and hash test in the JSON summaries instead of infering them from RMS etc.

Author: ConorMacBride

pytest_mpl/plugin.py

@@ -442,6 +442,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None):
 
         if not os.path.exists(baseline_image_ref):
             summary['status'] = 'failed'
+            summary['image_status'] = 'missing'
             error_message = ("Image file not found for comparison test in: \n\t"
                              f"{self.get_baseline_directory(item)}\n"
                              "(This is expected for new tests.)\n"
@@ -463,6 +464,7 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None):
         actual_shape = imread(str(test_image)).shape[:2]
         if expected_shape != actual_shape:
             summary['status'] = 'failed'
+            summary['image_status'] = 'diff'
             error_message = SHAPE_MISMATCH_ERROR.format(expected_path=baseline_image,
                                                         expected_shape=expected_shape,
                                                         actual_path=test_image,
@@ -474,10 +476,12 @@ def compare_image_to_baseline(self, item, fig, result_dir, summary=None):
         summary['tolerance'] = tolerance
         if results is None:
             summary['status'] = 'passed'
+            summary['image_status'] = 'match'
             summary['status_msg'] = 'Image comparison passed.'
             return None
         else:
             summary['status'] = 'failed'
+            summary['image_status'] = 'diff'
             summary['rms'] = results['rms']
             diff_image = (result_dir / 'result-failed-diff.png').absolute()
             summary['diff_image'] = diff_image.relative_to(self.results_dir).as_posix()
@@ -523,14 +527,17 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None):
 
         if baseline_hash is None:  # hash-missing
             summary['status'] = 'failed'
+            summary['hash_status'] = 'missing'
             summary['status_msg'] = (f"Hash for test '{hash_name}' not found in {hash_library_filename}. "
                                      f"Generated hash is {test_hash}.")
         elif test_hash == baseline_hash:  # hash-match
             hash_comparison_pass = True
             summary['status'] = 'passed'
+            summary['hash_status'] = 'match'
             summary['status_msg'] = 'Test hash matches baseline hash.'
         else:  # hash-diff
             summary['status'] = 'failed'
+            summary['hash_status'] = 'diff'
             summary['status_msg'] = (f"Hash {test_hash} doesn't match hash "
                                      f"{baseline_hash} in library "
                                      f"{hash_library_filename} for test {hash_name}.")
@@ -555,7 +562,8 @@ def compare_image_to_hash_library(self, item, fig, result_dir, summary=None):
             except Exception as baseline_error:  # Append to test error later
                 baseline_comparison = str(baseline_error)
             else:  # Update main summary
-                for k in ['baseline_image', 'diff_image', 'rms', 'tolerance', 'result_image']:
+                for k in ['image_status', 'baseline_image', 'diff_image',
+                          'rms', 'tolerance', 'result_image']:
                     summary[k] = summary[k] or baseline_summary.get(k)
 
             # Append the log from image comparison
@@ -612,6 +620,8 @@ def item_function_wrapper(*args, **kwargs):
 
                 summary = {
                     'status': None,
+                    'image_status': None,
+                    'hash_status': None,
                     'status_msg': None,
                     'baseline_image': None,
                     'diff_image': None,
@@ -626,6 +636,7 @@ def item_function_wrapper(*args, **kwargs):
                 # reference images or simply running the test.
                 if self.generate_dir is not None:
                     summary['status'] = 'skipped'
+                    summary['image_status'] = 'generated'
                     summary['status_msg'] = 'Skipped test, since generating image.'
                     generate_image = self.generate_baseline_image(item, fig)
                     if self.results_always:  # Make baseline image available in HTML
@@ -635,6 +646,7 @@ def item_function_wrapper(*args, **kwargs):
                             result_image.relative_to(self.results_dir).as_posix()
 
                 if self.generate_hash_library is not None:
+                    summary['hash_status'] = 'generated'
                     image_hash = self.generate_image_hash(item, fig)
                     self._generated_hash_library[test_name] = image_hash
                     summary['baseline_hash'] = image_hash

pytest_mpl/summary/html.py

@@ -107,32 +107,6 @@ def __init__(self, name, item, id, warn_missing):
             ('hash', self.hash_status),
         ]]
 
-    @cached_property
-    def image_status(self):
-        """Status of the image comparison test."""
-        if self.rms is None and self.tolerance is not None:
-            return 'match'
-        elif self.rms is not None:
-            return 'diff'
-        elif self.baseline_image is None:
-            return 'missing'
-        else:
-            return 'generated'
-
-    @cached_property
-    def hash_status(self):
-        """Status of the hash comparison test."""
-        if self.baseline_hash is not None or self.result_hash is not None:
-            if self.baseline_hash is None:
-                return 'missing'
-            elif self.result_hash is None:
-                return 'generated'
-            elif self.baseline_hash == self.result_hash:
-                return 'match'
-            else:
-                return 'diff'
-        return None
-
     @cached_property
     def indexes(self):
         """Dictionary with strings optimized for sorting."""

tests/subtests/summaries/test_default.json

@@ -1,6 +1,8 @@
 {
   "subtests.subtest.test_hmatch_imatch": {
     "status": "passed",
+    "image_status": "match",
+    "hash_status": null,
     "status_msg": "Image comparison passed.",
     "baseline_image": null,
     "diff_image": null,
@@ -12,6 +14,8 @@
   },
   "subtests.subtest.test_hmatch_idiff": {
     "status": "failed",
+    "image_status": "diff",
+    "hash_status": null,
     "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 11\\.1[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
     "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png",
     "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png",
@@ -23,6 +27,8 @@
   },
   "subtests.subtest.test_hmatch_imissing": {
     "status": "failed",
+    "image_status": "missing",
+    "hash_status": null,
     "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
     "baseline_image": null,
     "diff_image": null,
@@ -34,6 +40,8 @@
   },
   "subtests.subtest.test_hdiff_imatch": {
     "status": "passed",
+    "image_status": "match",
+    "hash_status": null,
     "status_msg": "Image comparison passed.",
     "baseline_image": null,
     "diff_image": null,
@@ -45,6 +53,8 @@
   },
   "subtests.subtest.test_hdiff_idiff": {
     "status": "failed",
+    "image_status": "diff",
+    "hash_status": null,
     "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 11\\.1[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
     "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png",
     "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png",
@@ -56,6 +66,8 @@
   },
   "subtests.subtest.test_hdiff_imissing": {
     "status": "failed",
+    "image_status": "missing",
+    "hash_status": null,
     "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
     "baseline_image": null,
     "diff_image": null,
@@ -67,6 +79,8 @@
   },
   "subtests.subtest.test_hmissing_imatch": {
     "status": "passed",
+    "image_status": "match",
+    "hash_status": null,
     "status_msg": "Image comparison passed.",
     "baseline_image": null,
     "diff_image": null,
@@ -78,6 +92,8 @@
   },
   "subtests.subtest.test_hmissing_idiff": {
     "status": "failed",
+    "image_status": "diff",
+    "hash_status": null,
     "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 12\\.1[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
     "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png",
     "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png",
@@ -89,6 +105,8 @@
   },
   "subtests.subtest.test_hmissing_imissing": {
     "status": "failed",
+    "image_status": "missing",
+    "hash_status": null,
     "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
     "baseline_image": null,
     "diff_image": null,
@@ -100,6 +118,8 @@
   },
   "subtests.subtest.test_hdiff_imatch_tolerance": {
     "status": "passed",
+    "image_status": "match",
+    "hash_status": null,
     "status_msg": "Image comparison passed.",
     "baseline_image": null,
     "diff_image": null,
@@ -111,6 +131,8 @@
   },
   "subtests.subtest.test_hdiff_idiff_tolerance": {
     "status": "failed",
+    "image_status": "diff",
+    "hash_status": null,
     "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 29\\.2[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    3",
     "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png",
     "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png",
@@ -122,6 +144,8 @@
   },
   "subtests.subtest.test_hdiff_imatch_savefig": {
     "status": "passed",
+    "image_status": "match",
+    "hash_status": null,
     "status_msg": "Image comparison passed.",
     "baseline_image": null,
     "diff_image": null,
@@ -133,6 +157,8 @@
   },
   "subtests.subtest.test_hdiff_imatch_style": {
     "status": "passed",
+    "image_status": "match",
+    "hash_status": null,
     "status_msg": "Image comparison passed.",
     "baseline_image": null,
     "diff_image": null,
@@ -144,6 +170,8 @@
   },
   "subtests.subtest.test_hdiff_imatch_removetext": {
     "status": "passed",
+    "image_status": "match",
+    "hash_status": null,
     "status_msg": "Image comparison passed.",
     "baseline_image": null,
     "diff_image": null,

tests/subtests/summaries/test_generate.json

@@ -1,6 +1,8 @@
 {
   "subtests.subtest.test_hmatch_imatch": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -12,6 +14,8 @@
   },
   "subtests.subtest.test_hmatch_idiff": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -23,6 +27,8 @@
   },
   "subtests.subtest.test_hmatch_imissing": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -34,6 +40,8 @@
   },
   "subtests.subtest.test_hdiff_imatch": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -45,6 +53,8 @@
   },
   "subtests.subtest.test_hdiff_idiff": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -56,6 +66,8 @@
   },
   "subtests.subtest.test_hdiff_imissing": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -67,6 +79,8 @@
   },
   "subtests.subtest.test_hmissing_imatch": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -78,6 +92,8 @@
   },
   "subtests.subtest.test_hmissing_idiff": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -89,6 +105,8 @@
   },
   "subtests.subtest.test_hmissing_imissing": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -100,6 +118,8 @@
   },
   "subtests.subtest.test_hdiff_imatch_tolerance": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -111,6 +131,8 @@
   },
   "subtests.subtest.test_hdiff_idiff_tolerance": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -122,6 +144,8 @@
   },
   "subtests.subtest.test_hdiff_imatch_savefig": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -133,6 +157,8 @@
   },
   "subtests.subtest.test_hdiff_imatch_style": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,
@@ -144,6 +170,8 @@
   },
   "subtests.subtest.test_hdiff_imatch_removetext": {
     "status": "skipped",
+    "image_status": "generated",
+    "hash_status": "generated",
     "status_msg": "Skipped test, since generating image.",
     "baseline_image": null,
     "diff_image": null,