Convert status_msg to regex

Author: ConorMacBride

tests/subtests/helpers.py

@@ -2,7 +2,7 @@
 import json
 from pathlib import Path
 
-__all__ = ['diff_summary', 'assert_existence', 'patch_summary']
+__all__ = ['diff_summary', 'assert_existence', 'patch_summary', 'apply_regex']
 
 
 class MatchError(Exception):
@@ -31,10 +31,14 @@ def diff_summary(baseline, result, baseline_hash_library=None, result_hash_libra
         # Load "correct" baseline hashes
         with open(baseline_hash_library, 'r') as f:
             baseline_hash_library = json.load(f)
+    else:
+        baseline_hash_library = {}
     if result_hash_library and result_hash_library.exists():
         # Load "correct" result hashes
         with open(result_hash_library, 'r') as f:
             result_hash_library = json.load(f)
+    else:
+        result_hash_library = {}
 
     # Get test names
     baseline_tests = set(baseline.keys())
@@ -176,3 +180,71 @@ def assert_existence(summary, items=('baseline_image', 'diff_image', 'result_ima
         for item in items:
             if test[item] is not None:
                 assert (Path(path) / test[item]).exists()
+
+
+def _escape_regex(msg):
+    if not msg.startswith('REGEX:'):
+        msg = msg.replace('.', r'\.').replace('(', r'\(').replace(')', r'\)')
+        msg = 'REGEX:' + msg
+    return msg
+
+
+def _escape_path(msg, path):
+    pattern = (rf"({path}[A-Za-z0-9_\-\/.\\]*)" +
+               r"(baseline\\.png|result-failed-diff\\.png|result\\.png|\\.json)")
+    msg = re.sub(pattern, r".*\2", msg)
+    pattern = rf"({path}[A-Za-z0-9_\-\/.\\]*)"
+    msg = re.sub(pattern, r".*", msg)
+    return msg
+
+
+def _escape_float(msg, key):
+    pattern = rf"({key}[0-9]+\\\.[0-9]{{1}})([0-9]+)"
+    msg = re.sub(pattern, r"\1[0-9]*", msg)
+    return msg
+
+
+def apply_regex(file, regex_paths, regex_strs):
+    """Convert all `status_msg` entries in JSON summary file to regex.
+
+    Use in your own script to assist with updating baseline summaries.
+
+    Parameters
+    ----------
+    file : Path
+        JSON summary file to convert `status_msg` to regex in. Overwritten.
+    regex_paths : list of str
+        List of path beginnings to identify paths that need to be converted to regex.
+        E.g. `['/home/user/']`
+        Does: `aaa /home/user/pytest/tmp/result\\.png bbb` -> `aaa .*result\\.png bbb`
+    regex_strs : list of str
+        List of keys to convert following floats to 1 d.p.
+        E.g. ['RMS Value: ']
+        Does: `aaa RMS Value: 12\\.432644 bbb` -> `aaa RMS Value: 12\\.4[0-9]* bbb`
+    """
+
+    with open(file, 'r') as f:
+        summary = json.load(f)
+
+    for test in summary.keys():
+
+        msg = summary[test]['status_msg']
+
+        for signal in [*regex_paths, *regex_strs]:
+            if signal in msg:
+                msg = _escape_regex(msg)
+        if not msg.startswith('REGEX:'):
+            continue
+
+        for signal in regex_paths:
+            if signal in msg:
+                msg = _escape_path(msg, path=signal)
+
+        for signal in regex_strs:
+            if signal in msg:
+                msg = _escape_float(msg, key=signal)
+
+        summary[test]['status_msg'] = msg
+
+    with open(file, 'w') as f:
+        json.dump(summary, f, indent=2)

tests/subtests/summaries/test_default.json

@@ -12,7 +12,7 @@
   },
   "subtests.subtest.test_hmatch_idiff": {
     "status": "failed",
-    "status_msg": "Error: Image files did not match.\n  RMS Value: 11.100353848213828\n  Expected:  \n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmatch_idiff/baseline.png\n  Actual:    \n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmatch_idiff/result.png\n  Difference:\n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmatch_idiff/result-failed-diff.png\n  Tolerance: \n    2",
+    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 11\\.1[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
     "baseline_image": "subtests.subtest.test_hmatch_idiff/baseline.png",
     "diff_image": "subtests.subtest.test_hmatch_idiff/result-failed-diff.png",
     "rms": 11.100353848213828,
@@ -23,7 +23,7 @@
   },
   "subtests.subtest.test_hmatch_imissing": {
     "status": "failed",
-    "status_msg": "Image file not found for comparison test in: \n\t/Users/conor/Documents/pytest-mpl/tests/subtests/baseline\n(This is expected for new tests.)\nGenerated Image: \n\t/private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmatch_imissing/result.png",
+    "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -45,7 +45,7 @@
   },
   "subtests.subtest.test_hdiff_idiff": {
     "status": "failed",
-    "status_msg": "Error: Image files did not match.\n  RMS Value: 11.182677079602481\n  Expected:  \n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff/baseline.png\n  Actual:    \n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff/result.png\n  Difference:\n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff/result-failed-diff.png\n  Tolerance: \n    2",
+    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 11\\.1[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
     "baseline_image": "subtests.subtest.test_hdiff_idiff/baseline.png",
     "diff_image": "subtests.subtest.test_hdiff_idiff/result-failed-diff.png",
     "rms": 11.182677079602481,
@@ -56,7 +56,7 @@
   },
   "subtests.subtest.test_hdiff_imissing": {
     "status": "failed",
-    "status_msg": "Image file not found for comparison test in: \n\t/Users/conor/Documents/pytest-mpl/tests/subtests/baseline\n(This is expected for new tests.)\nGenerated Image: \n\t/private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_imissing/result.png",
+    "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -78,7 +78,7 @@
   },
   "subtests.subtest.test_hmissing_idiff": {
     "status": "failed",
-    "status_msg": "Error: Image files did not match.\n  RMS Value: 12.12938597648977\n  Expected:  \n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmissing_idiff/baseline.png\n  Actual:    \n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmissing_idiff/result.png\n  Difference:\n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmissing_idiff/result-failed-diff.png\n  Tolerance: \n    2",
+    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 12\\.1[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    2",
     "baseline_image": "subtests.subtest.test_hmissing_idiff/baseline.png",
     "diff_image": "subtests.subtest.test_hmissing_idiff/result-failed-diff.png",
     "rms": 12.12938597648977,
@@ -89,7 +89,7 @@
   },
   "subtests.subtest.test_hmissing_imissing": {
     "status": "failed",
-    "status_msg": "Image file not found for comparison test in: \n\t/Users/conor/Documents/pytest-mpl/tests/subtests/baseline\n(This is expected for new tests.)\nGenerated Image: \n\t/private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hmissing_imissing/result.png",
+    "status_msg": "REGEX:Image file not found for comparison test in: \n\t.*\n\\(This is expected for new tests\\.\\)\nGenerated Image: \n\t.*result\\.png",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -111,7 +111,7 @@
   },
   "subtests.subtest.test_hdiff_idiff_tolerance": {
     "status": "failed",
-    "status_msg": "Error: Image files did not match.\n  RMS Value: 29.260332173249314\n  Expected:  \n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff_tolerance/baseline.png\n  Actual:    \n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff_tolerance/result.png\n  Difference:\n    /private/var/folders/28/h4xnr2f56qbcdlrntr3z54f80000gp/T/pytest-of-conor/pytest-124/test_default0/results/subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png\n  Tolerance: \n    3",
+    "status_msg": "REGEX:Error: Image files did not match\\.\n  RMS Value: 29\\.2[0-9]*\n  Expected:  \n    .*baseline\\.png\n  Actual:    \n    .*result\\.png\n  Difference:\n    .*result-failed-diff\\.png\n  Tolerance: \n    3",
     "baseline_image": "subtests.subtest.test_hdiff_idiff_tolerance/baseline.png",
     "diff_image": "subtests.subtest.test_hdiff_idiff_tolerance/result-failed-diff.png",
     "rms": 29.260332173249314,
@@ -153,4 +153,4 @@
     "baseline_hash": null,
     "result_hash": null
   }
-}
+}

tests/subtests/summaries/test_hash.json

@@ -34,7 +34,7 @@
   },
   "subtests.subtest.test_hdiff_imatch": {
     "status": "failed",
-    "status_msg": "Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imatch.",
+    "status_msg": "REGEX:Hash 6e2fdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 doesn't match hash d1ffdde5a6682dc6abba7121f5df702c3664b1ce09593534fc0d7c3514eb07e1 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -45,7 +45,7 @@
   },
   "subtests.subtest.test_hdiff_idiff": {
     "status": "failed",
-    "status_msg": "Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_idiff.",
+    "status_msg": "REGEX:Hash 443361bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 doesn't match hash d1ff61bdd0efd1cdd343eabf73af6f20439d4834ab5503a574ac7ec28e0c2b43 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -56,7 +56,7 @@
   },
   "subtests.subtest.test_hdiff_imissing": {
     "status": "failed",
-    "status_msg": "Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imissing.",
+    "status_msg": "REGEX:Hash 301e63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e doesn't match hash d1ff63d656d7a586cc4e498bc32b970f8cb7c7c47bbd2fec33b931219fc0690e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imissing\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -67,7 +67,7 @@
   },
   "subtests.subtest.test_hmissing_imatch": {
     "status": "failed",
-    "status_msg": "Hash for test 'subtests.subtest.test_hmissing_imatch' not found in /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824.",
+    "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imatch' not found in .*\\.json\\. Generated hash is eabd8a2e22afd88682990bfb8e4a0700a942fe68e5114e8da4ab6bd93c47b824\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -78,7 +78,7 @@
   },
   "subtests.subtest.test_hmissing_idiff": {
     "status": "failed",
-    "status_msg": "Hash for test 'subtests.subtest.test_hmissing_idiff' not found in /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab.",
+    "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_idiff' not found in .*\\.json\\. Generated hash is e69570c4a70b2cc88ddee0f0a82312cae4f394b7f62e5760245feda1364c03ab\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -89,7 +89,7 @@
   },
   "subtests.subtest.test_hmissing_imissing": {
     "status": "failed",
-    "status_msg": "Hash for test 'subtests.subtest.test_hmissing_imissing' not found in /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b.",
+    "status_msg": "REGEX:Hash for test 'subtests\\.subtest\\.test_hmissing_imissing' not found in .*\\.json\\. Generated hash is 5c8a9c7412e4e098f6f2683ee247c08bd50805a93df4d4b6d8fccf3579b4c56b\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -100,7 +100,7 @@
   },
   "subtests.subtest.test_hdiff_imatch_tolerance": {
     "status": "failed",
-    "status_msg": "Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imatch_tolerance.",
+    "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_tolerance\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -111,7 +111,7 @@
   },
   "subtests.subtest.test_hdiff_idiff_tolerance": {
     "status": "failed",
-    "status_msg": "Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_idiff_tolerance.",
+    "status_msg": "REGEX:Hash aaf4e85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 doesn't match hash d1ffe85fda98298347c274adae98ca7728f9bb2444ca8a49295145b0727b8c96 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_idiff_tolerance\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -122,7 +122,7 @@
   },
   "subtests.subtest.test_hdiff_imatch_savefig": {
     "status": "failed",
-    "status_msg": "Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imatch_savefig.",
+    "status_msg": "REGEX:Hash 5dc1c2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 doesn't match hash d1ffc2c68c2d34c03a89ab394e3c11349b76594d0c8837374daef299ac227568 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_savefig\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -133,7 +133,7 @@
   },
   "subtests.subtest.test_hdiff_imatch_style": {
     "status": "failed",
-    "status_msg": "Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imatch_style.",
+    "status_msg": "REGEX:Hash 185ed1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 doesn't match hash d1ffd1b702c7bbd810370b12e46ecea4b9c9eb87b743397f1d4a50177e7ba7f7 in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_style\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -144,7 +144,7 @@
   },
   "subtests.subtest.test_hdiff_imatch_removetext": {
     "status": "failed",
-    "status_msg": "Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library /Users/conor/Documents/pytest-mpl/tests/subtests/hashes/mpl34_ft261.json for test subtests.subtest.test_hdiff_imatch_removetext.",
+    "status_msg": "REGEX:Hash be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e doesn't match hash d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e in library .*\\.json for test subtests\\.subtest\\.test_hdiff_imatch_removetext\\.",
     "baseline_image": null,
     "diff_image": null,
     "rms": null,
@@ -153,4 +153,4 @@
     "baseline_hash": "d1fff83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e",
     "result_hash": "be5af83a43cb89f5e13923f532fe5c9bedbf7d13585533efef2f4051c4968b5e"
   }
-}
+}