Ensure state test test against intended fork only

The statetest fixtures have different definitions
for each fork and we must ensure to only run
test against against the intended fork (e.g run
Constantinople state tests against Constantinople VM).

We can not rely on `pytest -k` matching for that
as that matches against an identification string that
includes the path and name of the test file which in
some cases also contains fork names. A test file may
be named "ConstantinopleSomething.json" but still
contains individual definitions per fork.

Author: cburgdorf

eth/tools/fixtures/loading.py

@@ -77,7 +77,7 @@ def load_fixture(fixture_path: str,
 def filter_fixtures(all_fixtures: Iterable[Any],
                     fixtures_base_dir: str,
                     mark_fn: Callable[[str, str], bool]=None,
-                    ignore_fn: Callable[[str, str], bool]=None) -> Any:
+                    ignore_fn: Callable[..., bool]=None) -> Any:
     """
     Helper function for filtering test fixtures.
 

tests/conftest.py

@@ -186,3 +186,7 @@ def chain_without_block_validation(
     }
     chain = klass.from_genesis(base_db, genesis_params, genesis_state)
     return chain
+
+
+def pytest_addoption(parser):
+    parser.addoption("--fork", type=str, required=False)

tests/json-fixtures/test_state.py

@@ -162,12 +162,33 @@ def mark_statetest_fixtures(fixture_path, fixture_key, fixture_fork, fixture_ind
         return pytest.mark.xfail(reason="Listed in INCORRECT_UPSTREAM_TESTS.")
 
 
+def generate_ignore_fn_for_fork(metafunc):
+    """
+    The statetest fixtures have different definitions for each fork and we must ensure to only run
+    test against against the intended fork (e.g run Constantinople state tests against
+    Constantinople VM).
+    We can not rely on `pytest -k` matching for that as that matches against an identification
+    string that includes the path and name of the test which in some cases also contains fork
+    fork names. A test file may be named "ConstantinopleSomething.json" but still contains
+    individual definitions per fork.
+    """
+    passed_fork = metafunc.config.getoption('fork')
+    if passed_fork:
+        passed_fork = passed_fork.lower()
+
+        def ignore_fn(fixture_path, fixture_key, fixture_fork, post_state_index):
+            return fixture_fork.lower() != passed_fork
+
+        return ignore_fn
+
+
 def pytest_generate_tests(metafunc):
     generate_fixture_tests(
         metafunc=metafunc,
         base_fixture_path=BASE_FIXTURE_PATH,
         preprocess_fn=expand_fixtures_forks,
         filter_fn=filter_fixtures(
+            ignore_fn=generate_ignore_fn_for_fork(metafunc),
             fixtures_base_dir=BASE_FIXTURE_PATH,
             mark_fn=mark_statetest_fixtures,
         ),

tox.ini

@@ -36,13 +36,13 @@ commands=
     transactions: pytest {posargs:tests/json-fixtures/test_transactions.py}
     vm: pytest {posargs:tests/json-fixtures/test_virtual_machine.py}
     native-blockchain: pytest {posargs:tests/json-fixtures/test_blockchain.py}
-    native-state-frontier: pytest {posargs:tests/json-fixtures/test_state.py -k Frontier}
-    native-state-homestead: pytest {posargs:tests/json-fixtures/test_state.py -k Homestead}
-    native-state-eip150: pytest {posargs:tests/json-fixtures/test_state.py -k EIP150}
-    native-state-eip158: pytest {posargs:tests/json-fixtures/test_state.py -k EIP158}
-    native-state-byzantium: pytest {posargs:tests/json-fixtures/test_state.py -k Byzantium}
-    native-state-constantinople: pytest {posargs:tests/json-fixtures/test_state.py -k Constantinople}
-    native-state-metropolis: pytest {posargs:tests/json-fixtures/test_state.py -k Metropolis}
+    native-state-frontier: pytest {posargs:tests/json-fixtures/test_state.py --fork Frontier}
+    native-state-homestead: pytest {posargs:tests/json-fixtures/test_state.py --fork Homestead}
+    native-state-eip150: pytest {posargs:tests/json-fixtures/test_state.py --fork EIP150}
+    native-state-eip158: pytest {posargs:tests/json-fixtures/test_state.py --fork EIP158}
+    native-state-byzantium: pytest {posargs:tests/json-fixtures/test_state.py --fork Byzantium}
+    native-state-constantinople: pytest {posargs:tests/json-fixtures/test_state.py --fork Constantinople}
+    native-state-metropolis: pytest {posargs:tests/json-fixtures/test_state.py --fork Metropolis}
     lightchain_integration: pytest --integration {posargs:tests/trinity/integration/test_lightchain_integration.py}
 
 deps = .[p2p,trinity,eth-extra,test]