attempt to update ci for output tests

Author: gregsdennis

bin/jsonschema_suite

@@ -30,11 +30,13 @@ else:
 
 ROOT_DIR = Path(__file__).parent.parent
 SUITE_ROOT_DIR = ROOT_DIR / "tests"
+OUTPUT_ROOT_DIR = ROOT_DIR / "output-tests"
 
 REMOTES_DIR = ROOT_DIR / "remotes"
 REMOTES_BASE_URL = "http://localhost:1234/"
 
 TESTSUITE_SCHEMA = json.loads((ROOT_DIR / "test-schema.json").read_text())
+OUTPUTTESTSUITE_SCHEMA = json.loads((ROOT_DIR / "output-test-schema.json").read_text())
 
 
 def files(paths):
@@ -88,12 +90,17 @@ class SanityTests(unittest.TestCase):
     @classmethod
     def setUpClass(cls):
         print(f"Looking for tests in {SUITE_ROOT_DIR}")
+        print(f"Looking for output tests in {OUTPUT_ROOT_DIR}")
         print(f"Looking for remotes in {REMOTES_DIR}")
 
         cls.test_files = list(collect(SUITE_ROOT_DIR))
         assert cls.test_files, "Didn't find the test files!"
         print(f"Found {len(cls.test_files)} test files")
 
+        cls.output_test_files = list(collect(OUTPUT_ROOT_DIR))
+        assert cls.remote_files, "Didn't find the output test files!"
+        print(f"Found {len(cls.remote_files)} output test files")
+
         cls.remote_files = list(collect(REMOTES_DIR))
         assert cls.remote_files, "Didn't find the remote files!"
         print(f"Found {len(cls.remote_files)} remote files")
@@ -142,6 +149,17 @@ class SanityTests(unittest.TestCase):
                 except ValueError as error:
                     self.fail(f"{path} contains invalid JSON ({error})")
 
+    def test_all_output_test_files_are_valid_json(self):
+        """
+        All test files contain valid JSON.
+        """
+        for path in self.output_test_files:
+            with self.subTest(path=path):
+                try:
+                    json.loads(path.read_text())
+                except ValueError as error:
+                    self.fail(f"{path} contains invalid JSON ({error})")
+
     def test_all_remote_files_are_valid_json(self):
         """
         All remote files contain valid JSON.
@@ -157,7 +175,7 @@ class SanityTests(unittest.TestCase):
         """
         All cases have reasonably long descriptions.
         """
-        for case in cases(self.test_files):
+        for case in cases(self.test_files + self.output_test_files):
             with self.subTest(description=case["description"]):
                 self.assertLess(
                     len(case["description"]),
@@ -169,7 +187,7 @@ class SanityTests(unittest.TestCase):
         """
         All tests have reasonably long descriptions.
         """
-        for count, test in enumerate(tests(self.test_files)):
+        for count, test in enumerate(tests(self.test_files + self.output_test_files)):
             with self.subTest(description=test["description"]):
                 self.assertLess(
                     len(test["description"]),
@@ -182,28 +200,28 @@ class SanityTests(unittest.TestCase):
         """
         All cases have unique descriptions in their files.
         """
-        for path, cases in files(self.test_files):
+        for path, cases in files(self.test_files + self.output_test_files):
             with self.subTest(path=path):
                 self.assertUnique(case["description"] for case in cases)
 
     def test_all_test_descriptions_are_unique(self):
         """
         All test cases have unique test descriptions in their tests.
         """
-        for count, case in enumerate(cases(self.test_files)):
+        for count, case in enumerate(cases(self.test_files + self.output_test_files)):
             with self.subTest(description=case["description"]):
                 self.assertUnique(
                     test["description"] for test in case["tests"]
                 )
         print(f"Found {count} test cases.")
 
     def test_case_descriptions_do_not_use_modal_verbs(self):
-        for case in cases(self.test_files):
+        for case in cases(self.test_files + self.output_test_files):
             with self.subTest(description=case["description"]):
                 self.assertFollowsDescriptionStyle(case["description"])
 
     def test_test_descriptions_do_not_use_modal_verbs(self):
-        for test in tests(self.test_files):
+        for test in tests(self.test_files + self.output_test_files):
             with self.subTest(description=test["description"]):
                 self.assertFollowsDescriptionStyle(test["description"])
 
@@ -244,6 +262,17 @@ class SanityTests(unittest.TestCase):
                     validator.validate(cases)
                 except jsonschema.ValidationError as error:
                     self.fail(str(error))
+        """
+        All output test files are valid under output-test-schema.json.
+        """
+        Validator = jsonschema.validators.validator_for(OUTPUTTESTSUITE_SCHEMA)
+        validator = Validator(OUTPUTTESTSUITE_SCHEMA)
+        for path, cases in files(self.output_test_files):
+            with self.subTest(path=path):
+                try:
+                    validator.validate(cases)
+                except jsonschema.ValidationError as error:
+                    self.fail(str(error))
 
 
 def main(arguments):

output-test-schema.json

@@ -0,0 +1,36 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://json-schema.org/tests/output-test-schema",
+    "description": "A schema for files contained within this suite",
+
+    "$ref": "test-schema",
+
+    "$defs": {
+        "test": {
+            "$dynamicAnchor": "foo",
+            "description": "A single output test",
+
+            "allOf": [
+                { "$ref": "test-schema#/$defs/test" },
+                {
+                    "properties": {
+                        "output": {
+                            "description": "schemas that are used to verify output",
+                            "type": "object",
+                            "properties": {
+                                "flag": { "$ref": "https://json-schema.org/draft/2020-12/schema" },
+                                "basic": { "$ref": "https://json-schema.org/draft/2020-12/schema" },
+                                "detailed": { "$ref": "https://json-schema.org/draft/2020-12/schema" },
+                                "verbose": { "$ref": "https://json-schema.org/draft/2020-12/schema" },
+                                "list": { "$ref": "https://json-schema.org/draft/2020-12/schema" },
+                                "hierarchy": { "$ref": "https://json-schema.org/draft/2020-12/schema" }
+                            },
+                            "minProperties": 1,
+                            "additionalProperties": false
+                        }
+                    }
+                }
+            ]
+        }
+    }
+}

test-schema.json

@@ -1,5 +1,6 @@
 {
     "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://json-schema.org/tests/test-schema",
     "description": "A schema for files contained within this suite",
 
     "type": "array",
@@ -24,7 +25,10 @@
             "tests": {
                 "description": "A set of related tests all using the same schema",
                 "type": "array",
-                "items": { "$ref": "#/$defs/test" },
+                "items": {
+                    "$dynamicRef": "#foo",
+                    "unevaluatedProperties": false
+                },
                 "minItems": 1
             }
         },
@@ -33,6 +37,7 @@
 
     "$defs": {
         "test": {
+            "$dynamicAnchor": "foo",
             "description": "A single test",
 
             "type": "object",
@@ -53,8 +58,7 @@
                     "description": "Whether the validation process of this instance should consider the instance valid or not",
                     "type": "boolean"
                 }
-            },
-            "additionalProperties": false
+            }
         }
     }
 }