Fix tests on Windows

Author: astrofrog

tests/test_pytest_mpl.py

@@ -22,16 +22,21 @@
 
 WIN = sys.platform.startswith('win')
 
+# In some cases, the fonts on Windows can be quite different
+DEFAULT_TOLERANCE = 10 if WIN else None
 
-@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local)
+
+@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local,
+                               tolerance=DEFAULT_TOLERANCE)
 def test_succeeds():
     fig = plt.figure()
     ax = fig.add_subplot(1, 1, 1)
     ax.plot([1, 2, 3])
     return fig
 
 
-@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_remote)
+@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_remote,
+                               tolerance=DEFAULT_TOLERANCE)
 def test_succeeds_remote():
     fig = plt.figure()
     ax = fig.add_subplot(1, 1, 1)
@@ -42,7 +47,8 @@ def test_succeeds_remote():
 # The following tries an invalid URL first (or at least a URL where the baseline
 # image won't exist), but should succeed with the second mirror.
 @pytest.mark.mpl_image_compare(baseline_dir='http://www.python.org,' + baseline_dir_remote,
-                               filename='test_succeeds_remote.png')
+                               filename='test_succeeds_remote.png',
+                               tolerance=DEFAULT_TOLERANCE)
 def test_succeeds_faulty_mirror():
     fig = plt.figure()
     ax = fig.add_subplot(1, 1, 1)
@@ -52,15 +58,18 @@ def test_succeeds_faulty_mirror():
 
 class TestClass(object):
 
-    @pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local)
+    @pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local,
+                                   tolerance=DEFAULT_TOLERANCE)
     def test_succeeds(self):
         fig = plt.figure()
         ax = fig.add_subplot(1, 1, 1)
         ax.plot([1, 2, 3])
         return fig
 
 
-@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local, savefig_kwargs={'dpi': 30})
+@pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local,
+                               savefig_kwargs={'dpi': 30},
+                               tolerance=DEFAULT_TOLERANCE)
 def test_dpi():
     fig = plt.figure()
     ax = fig.add_subplot(1, 1, 1)
@@ -87,11 +96,11 @@ def test_fails(tmpdir):
         f.write(TEST_FAILING)
 
     # If we use --mpl, it should detect that the figure is wrong
-    code = subprocess.call('{0} -m pytest --mpl {1}'.format(sys.executable, test_file), shell=True)
+    code = subprocess.call([sys.executable, '-m', 'pytest', '--mpl', test_file])
     assert code != 0
 
     # If we don't use --mpl option, the test should succeed
-    code = subprocess.call('{0} -m pytest {1}'.format(sys.executable, test_file), shell=True)
+    code = subprocess.call([sys.executable, '-m', 'pytest', test_file])
     assert code == 0
 
 
@@ -114,9 +123,9 @@ def test_output_dir(tmpdir):
 
     # When we run the test, we should get output images where we specify
     output_dir = tmpdir.join('test_output_dir').strpath
-    code = subprocess.call('{0} -m pytest --mpl-results-path={1} --mpl {2}'
-                           .format(sys.executable, output_dir, test_file),
-                           shell=True)
+    code = subprocess.call([sys.executable, '-m', 'pytest',
+                            '--mpl-results-path={0}'.format(output_dir),
+                            '--mpl', test_file])
 
     assert code != 0
     assert os.path.exists(output_dir)
@@ -138,12 +147,6 @@ def test_gen():
 """
 
 
-# TODO: We skip the following test on Windows since the first subprocess calls.
-# This should be fixed in the long term, but is not critical since we already
-# test this on Linux.
-
-
-@pytest.mark.skipif("WIN")
 def test_generate(tmpdir):
 
     test_file = tmpdir.join('test.py').strpath
@@ -153,14 +156,15 @@ def test_generate(tmpdir):
     gen_dir = tmpdir.mkdir('spam').mkdir('egg').strpath
 
     # If we don't generate, the test will fail
-    p = subprocess.Popen('{0} -m pytest --mpl {1}'.format(sys.executable, test_file), shell=True,
-                         stdout=subprocess.PIPE, stderr=subprocess.PIPE)
-    p.wait()
-    assert b'Image file not found for comparison test' in p.stdout.read()
+    try:
+        subprocess.check_output([sys.executable, '-m', 'pytest', '--mpl', test_file])
+    except subprocess.CalledProcessError as exc:
+        assert b'Image file not found for comparison test' in exc.output
 
     # If we do generate, the test should succeed and a new file will appear
-    code = subprocess.call('{0} -m pytest --mpl-generate-path={1} {2}'
-                           .format(sys.executable, gen_dir, test_file), shell=True)
+    code = subprocess.call([sys.executable, '-m', 'pytest',
+                           '--mpl-generate-path={0}'.format(gen_dir),
+                           test_file])
     assert code == 0
     assert os.path.exists(os.path.join(gen_dir, 'test_gen.png'))
 
@@ -180,7 +184,8 @@ def test_nofigure():
 @pytest.mark.skipif(MPL_LT_2, reason="the fivethirtyeight style is only available "
                                      "in Matplotlib 2.0 and later")
 @pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local,
-                               style='fivethirtyeight')
+                               style='fivethirtyeight',
+                               tolerance=DEFAULT_TOLERANCE)
 def test_base_style():
     fig = plt.figure()
     ax = fig.add_subplot(1, 1, 1)
@@ -215,7 +220,8 @@ def setup_method(self, method):
         self.x = [1, 2, 3]
 
     @pytest.mark.mpl_image_compare(baseline_dir=baseline_dir_local,
-                                   filename='test_succeeds.png')
+                                   filename='test_succeeds.png',
+                                   tolerance=DEFAULT_TOLERANCE)
     def test_succeeds(self):
         fig = plt.figure()
         ax = fig.add_subplot(1, 1, 1)

tox.ini

@@ -27,8 +27,8 @@ extras =
 commands =
     pip freeze
     # Make sure the tests pass with and without --mpl
-    !minimal: pytest {toxinidir}/tests --cov pytest_mpl --cov-config={toxinidir}/setup.cfg {posargs}
-    !minimal: pytest {toxinidir}/tests --mpl --cov pytest_mpl --cov-config={toxinidir}/setup.cfg {posargs}
+    !minimal: pytest '{toxinidir}' {posargs}
+    !minimal: pytest '{toxinidir}' --mpl --cov pytest_mpl {posargs}
     # Make sure that the plugin won't crash pytest if Matplotlib isn't installed
     minimal: python -c 'import pytest_mpl.plugin'