csv parser now gets format column

Author: andrewelamb

synapseclient/extensions/curator/schema_generation.py

@@ -114,6 +114,27 @@ class ListColumnType(ColumnType):
 ]
 
 
+class Format(Enum):
+    """
+    Allowed formats by the JSON Schema validator used by Synapse: https://github.com/everit-org/json-schema#format-validators
+    For descriptions see: https://json-schema.org/understanding-json-schema/reference/type#format
+    """
+
+    DATE_TIME = "date-time"
+    EMAIL = "email"
+    HOSTNAME = "hostname"
+    IPV4 = "ipv4"
+    IPV6 = "ipv6"
+    URI = "uri"
+    URI_REFERENCE = "uri-reference"
+    URI_TEMPLATE = "uri-template"
+    JSON_POINTER = "json-pointer"
+    DATE = "date"
+    TIME = "time"
+    REGEX = "regex"
+    RELATIVE_JSON_POINTER = "relative-json-pointer"
+
+
 # Translates list types to their atomic type
 LIST_TYPE_DICT = {
     ListColumnType.STRING_LIST: AtomicColumnType.STRING,
@@ -636,6 +657,7 @@ def gather_csv_attributes_relationships(
         # get attributes from Attribute column
         attributes = model_df.to_dict("records")
         model_includes_column_type = "columnType" in model_df.columns
+        model_includes_format = "Format" in model_df.columns
 
         # Build attribute/relationship dictionary
         relationship_types = self.required_headers
@@ -659,6 +681,9 @@ def gather_csv_attributes_relationships(
                 attr_rel_dictionary[attribute_name]["Relationships"].update(
                     column_type_dict
                 )
+            if model_includes_format:
+                format_dict = self.parse_format(attr)
+                attr_rel_dictionary[attribute_name]["Relationships"].update(format_dict)
         return attr_rel_dictionary
 
     def parse_column_type(self, attr: dict) -> dict:
@@ -691,6 +716,34 @@ def parse_column_type(self, attr: dict) -> dict:
 
         return {"ColumnType": column_type}
 
+    def parse_format(self, attr: dict) -> dict[str, str]:
+        """Parse the format for a given attribute.
+
+        Args:
+            attr: The attribute dictionary.
+
+        Returns:
+            A dictionary containing the parsed column type information if present
+            else an empty dict
+        """
+        from pandas import isna
+
+        format_value = attr.get("Format")
+
+        if isna(format_value):
+            return {}
+
+        format_string = str(format_value).strip().lower()
+
+        check_allowed_values(
+            self.dmr,
+            entry_id=attr["Format"],
+            value=format_string,
+            relationship="format",
+        )
+
+        return {"Format": format_string}
+
     def parse_csv_model(
         self,
         path_to_data_model: str,
@@ -1708,6 +1761,37 @@ def get_node_column_type(
                 raise ValueError(msg)
         return column_type
 
+    def get_node_format(
+        self, node_label: Optional[str] = None, node_display_name: Optional[str] = None
+    ) -> Optional[ColumnType]:
+        """Gets the format of the node
+
+        Args:
+            node_label: The label of the node to get the format from
+            node_display_name: The display name of the node to get the format from
+
+        Raises:
+            ValueError: If the value from the node is not allowed
+
+        Returns:
+            The format of the node if it has one, otherwise None
+        """
+        node_label = self._get_node_label(node_label, node_display_name)
+        rel_node_label = self.dmr.get_relationship_value("format", "node_label")
+        format_value = self.graph.nodes[node_label][rel_node_label]
+        if format_value is None:
+            return format_value
+        format_string = str(format_value).lower()
+        try:
+            column_type = Format(format_string)
+        except ValueError as exc:
+            msg = (
+                f"Node: '{node_label}' had illegal format value: '{format_value}'. "
+                f"Allowed values are: [{[member.value for member in Format]}]"
+            )
+            raise ValueError(msg) from exc
+        return column_type
+
     def _get_node_label(
         self, node_label: Optional[str] = None, node_display_name: Optional[str] = None
     ) -> str:
@@ -2826,6 +2910,16 @@ def define_data_model_relationships(self) -> dict:
                 "node_attr_dict": {"default": None},
                 "allowed_values": ALL_COLUMN_TYPE_VALUES,
             },
+            "format": {
+                "jsonld_key": "sms:format",
+                "csv_header": "Format",
+                "node_label": "format",
+                "type": str,
+                "required_header": False,
+                "edge_rel": False,
+                "node_attr_dict": {"default": None},
+                "allowed_values": [member.value for member in Format],
+            },
         }
 
         return map_data_model_relationships

tests/unit/synapseclient/extensions/conftest.py

@@ -1,12 +1,13 @@
 import os
-import sys
 from unittest.mock import Mock
 
 import pytest
 
 from synapseclient.extensions.curator.schema_generation import (
+    DataModelCSVParser,
     DataModelGraph,
     DataModelGraphExplorer,
+    DataModelJSONLDParser,
     DataModelParser,
     DataModelRelationships,
 )
@@ -68,3 +69,33 @@ def DMGE(helpers: Helpers) -> DataModelGraphExplorer:
 def fixture_dmr():
     "Returns a DataModelRelationships instance"
     return DataModelRelationships()
+
+
+@pytest.fixture(name="dmp")
+def fixture_dmp(helpers: Helpers) -> DataModelParser:
+    "Returns a DataModelParser using the csv data model"
+    data_model_path = helpers.get_schema_file_path("example.model.csv")
+    dmp = DataModelParser(data_model_path, logger=Mock())
+    return dmp
+
+
+@pytest.fixture(name="dmp_jsonld")
+def fixture_dmp_json_ld(helpers: Helpers) -> DataModelParser:
+    "Returns a DataModelParser using the jsonld data model"
+    data_model_path = helpers.get_schema_file_path("example.model.jsonld")
+    dmp = DataModelParser(data_model_path, logger=Mock())
+    return dmp
+
+
+@pytest.fixture(name="csv_dmp")
+def fixture_csv_dmp() -> DataModelCSVParser:
+    "Returns a DataModelCSVParser"
+    dmp = DataModelCSVParser(logger=Mock())
+    return dmp
+
+
+@pytest.fixture(name="jsonld_dmp")
+def fixture_jsonld_dmp() -> DataModelCSVParser:
+    "Returns a DataModelJSONLDParser"
+    dmp = DataModelJSONLDParser()
+    return dmp

tests/unit/synapseclient/extensions/schema_files/example.model.csv

@@ -1,84 +1,84 @@
-Attribute,Description,Valid Values,DependsOn,Properties,Required,Parent,DependsOn Component,Source,Validation Rules,columnType
-Component,,,,,TRUE,,,,,
-Patient,,,"Patient ID, Sex, Year of Birth, Diagnosis, Component",,FALSE,DataType,,,,
-Patient ID,,,,,TRUE,DataProperty,,,#Patient unique warning^^#Biospecimen unique error,
-Sex,,"Female, Male, Other",,,TRUE,DataProperty,,,,
-Year of Birth,,,,,FALSE,DataProperty,,,,
-Diagnosis,,"Healthy, Cancer",,,TRUE,DataProperty,,,,
-Cancer,,,"Cancer Type, Family History",,FALSE,ValidValue,,,,
-Cancer Type,,"Breast, Colorectal, Lung, Prostate, Skin",,,TRUE,DataProperty,,,,
-Family History,,"Breast, Colorectal, Lung, Prostate, Skin",,,TRUE,DataProperty,,,list strict,
-Biospecimen,,,"Sample ID, Patient ID, Tissue Status, Component",,FALSE,DataType,Patient,,,
-Sample ID,,,,,TRUE,DataProperty,,,,
-Tissue Status,,"Healthy, Malignant, None",,,TRUE,DataProperty,,,,
-Bulk RNA-seq Assay,,,"Filename, Sample ID, File Format, Component",,FALSE,DataType,Biospecimen,,,
-Filename,,,,,TRUE,DataProperty,,,#MockFilename filenameExists^^,
-File Format,,"FASTQ, BAM, CRAM, CSV/TSV",,,TRUE,DataProperty,,,,
-BAM,,,Genome Build,,FALSE,ValidValue,,,,
-CRAM,,,"Genome Build, Genome FASTA",,FALSE,ValidValue,,,,
-CSV/TSV,,,Genome Build,,FALSE,ValidValue,,,,
-Genome Build,,"GRCh37, GRCh38, GRCm38, GRCm39",,,TRUE,DataProperty,,,,
-Genome FASTA,,,,,TRUE,DataProperty,,,,
-MockComponent,Component to hold mock attributes for testing all validation rules,,"Component, Check List, Check List Enum, Check List Like, Check List Like Enum, Check List Strict, Check List Enum Strict, Check Regex List, Check Regex List Like, Check Regex List Strict, Check Regex Single, Check Regex Format, Check Regex Integer, Check Num, Check Float, Check Int, Check String, Check URL,Check Match at Least, Check Match at Least values, Check Match Exactly, Check Match Exactly values, Check Match None, Check Match None values, Check Recommended, Check Ages, Check Unique, Check Range, Check Date, Check NA",,FALSE,DataType,,,,
-Check List,,,,,TRUE,DataProperty,,,list,
-Check List Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list,
-Check List Like,,,,,TRUE,DataProperty,,,list like,
-Check List Like Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list like,
-Check List Strict,,,,,TRUE,DataProperty,,,list strict,
-Check List Enum Strict,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list strict,
-Check Regex List,,,,,TRUE,DataProperty,,,list::regex match [a-f],
-Check Regex List Strict,,,,,TRUE,DataProperty,,,list strict::regex match [a-f],
-Check Regex List Like,,,,,TRUE,DataProperty,,,list like::regex match [a-f],
-Check Regex Single,,,,,TRUE,DataProperty,,,regex search [a-f],
-Check Regex Format,,,,,TRUE,DataProperty,,,regex match [a-f],
-Check Regex Integer,,,,,TRUE,DataProperty,,,regex search ^\d+$,
-Check Num,,,,,TRUE,DataProperty,,,num error,
-Check Float,,,,,TRUE,DataProperty,,,float error,
-Check Int,,,,,TRUE,DataProperty,,,int error,
-Check String,,,,,TRUE,DataProperty,,,str error,
-Check URL,,,,,TRUE,DataProperty,,,url,
-Check Match at Least,,,,,TRUE,DataProperty,,,matchAtLeastOne Patient.PatientID set,
-Check Match Exactly,,,,,TRUE,DataProperty,,,matchExactlyOne MockComponent.checkMatchExactly set,
-Check Match None,,,,,TRUE,DataProperty,,,matchNone MockComponent.checkMatchNone set error,
-Check Match at Least values,,,,,TRUE,DataProperty,,,matchAtLeastOne MockComponent.checkMatchatLeastvalues value,
-Check Match Exactly values,,,,,TRUE,DataProperty,,,matchExactlyOne MockComponent.checkMatchExactlyvalues value,
-Check Match None values,,,,,TRUE,DataProperty,,,matchNone MockComponent.checkMatchNonevalues value error,
-Check Recommended,,,,,FALSE,DataProperty,,,recommended,
-Check Ages,,,,,TRUE,DataProperty,,,protectAges,
-Check Unique,,,,,TRUE,DataProperty,,,unique error,
-Check Range,,,,,TRUE,DataProperty,,,inRange 50 100 error,
-Check Date,,,,,TRUE,DataProperty,,,date,
-Check NA,,,,,TRUE,DataProperty,,,int::IsNA,
-MockRDB,,,"Component, MockRDB_id, SourceManifest",,FALSE,DataType,,,,
-MockRDB_id,,,,,TRUE,DataProperty,,,int,
-SourceManifest,,,,,TRUE,DataProperty,,,,
-MockFilename,,,"Component, Filename",,FALSE,DataType,,,,
-JSONSchemaComponent,Component to hold attributes for testing JSON Schemas,,"Component, No Rules, No Rules Not Required, String, String Not Required, Enum, Enum Not Required, Date, URL, InRange, Regex, List, List Not Required, List Enum, List Enum Not Required, List Boolean, List, Integer, List InRange",,FALSE,DataType,,,,
-No Rules,,,,,TRUE,DataProperty,,,,
-No Rules Not Required,,,,,FALSE,DataProperty,,,,
-String,,,,,TRUE,DataProperty,,,,string
-String Not Required,,,,,FALSE,DataProperty,,,,string
-Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,,string
-Enum Not Required,,"ab, cd, ef, gh",,,FALSE,DataProperty,,,,string
-Date,,,,,TRUE,DataProperty,,,date,string
-URL,,,,,TRUE,DataProperty,,,url,string
-InRange,,,,,TRUE,DataProperty,,,inRange 50 100,number
-Regex,,,,,TRUE,DataProperty,,,regex search [a-f],string
-List,,,,,TRUE,DataProperty,,,,string_list
-List Not Required,,,,,FALSE,DataProperty,,,,string_list
-List Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,,string_list
-List Enum Not Required,,"ab, cd, ef, gh",,,FALSE,DataProperty,,,,string_list
-List Boolean,,,,,TRUE,DataProperty,,,,boolean_list
-List Integer,,,,,TRUE,DataProperty,,,,integer_list
-List InRange,,,,,TRUE,DataProperty,,,inRange 50 100,integer_list
-TypeDefinitionComponent,Component to check type specification,,"Component, String type, String type caps, Int type, Int type caps, Num type, Num type caps, Nan type, Missing type, Boolean type, Boolean type caps",,FALSE,DataType,,,,
-String type,,,,,TRUE,DataProperty,,,,string
-String type caps,,,,,TRUE,DataProperty,,,,STRING
-Int type,,,,,TRUE,DataProperty,,,,integer
-Int type caps,,,,,TRUE,DataProperty,,,,INTEGER
-Num type,,,,,TRUE,DataProperty,,,,number
-Num type caps,,,,,TRUE,DataProperty,,,,NUMBER
-Nan type,,,,,TRUE,DataProperty,,,,nan
-Missing type,,,,,TRUE,DataProperty,,,,
-Boolean type,,,,,TRUE,DataProperty,,,,boolean
-Boolean type caps,,,,,TRUE,DataProperty,,,,BOOLEAN
+Attribute,Description,Valid Values,DependsOn,Properties,Required,Parent,DependsOn Component,Source,Validation Rules,columnType,Format
+Component,,,,,TRUE,,,,,,
+Patient,,,"Patient ID, Sex, Year of Birth, Diagnosis, Component",,FALSE,DataType,,,,,
+Patient ID,,,,,TRUE,DataProperty,,,#Patient unique warning^^#Biospecimen unique error,,
+Sex,,"Female, Male, Other",,,TRUE,DataProperty,,,,,
+Year of Birth,,,,,FALSE,DataProperty,,,,,
+Diagnosis,,"Healthy, Cancer",,,TRUE,DataProperty,,,,,
+Cancer,,,"Cancer Type, Family History",,FALSE,ValidValue,,,,,
+Cancer Type,,"Breast, Colorectal, Lung, Prostate, Skin",,,TRUE,DataProperty,,,,,
+Family History,,"Breast, Colorectal, Lung, Prostate, Skin",,,TRUE,DataProperty,,,list strict,,
+Biospecimen,,,"Sample ID, Patient ID, Tissue Status, Component",,FALSE,DataType,Patient,,,,
+Sample ID,,,,,TRUE,DataProperty,,,,,
+Tissue Status,,"Healthy, Malignant, None",,,TRUE,DataProperty,,,,,
+Bulk RNA-seq Assay,,,"Filename, Sample ID, File Format, Component",,FALSE,DataType,Biospecimen,,,,
+Filename,,,,,TRUE,DataProperty,,,#MockFilename filenameExists^^,,
+File Format,,"FASTQ, BAM, CRAM, CSV/TSV",,,TRUE,DataProperty,,,,,
+BAM,,,Genome Build,,FALSE,ValidValue,,,,,
+CRAM,,,"Genome Build, Genome FASTA",,FALSE,ValidValue,,,,,
+CSV/TSV,,,Genome Build,,FALSE,ValidValue,,,,,
+Genome Build,,"GRCh37, GRCh38, GRCm38, GRCm39",,,TRUE,DataProperty,,,,,
+Genome FASTA,,,,,TRUE,DataProperty,,,,,
+MockComponent,Component to hold mock attributes for testing all validation rules,,"Component, Check List, Check List Enum, Check List Like, Check List Like Enum, Check List Strict, Check List Enum Strict, Check Regex List, Check Regex List Like, Check Regex List Strict, Check Regex Single, Check Regex Format, Check Regex Integer, Check Num, Check Float, Check Int, Check String, Check URL,Check Match at Least, Check Match at Least values, Check Match Exactly, Check Match Exactly values, Check Match None, Check Match None values, Check Recommended, Check Ages, Check Unique, Check Range, Check Date, Check NA",,FALSE,DataType,,,,,
+Check List,,,,,TRUE,DataProperty,,,list,,
+Check List Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list,,
+Check List Like,,,,,TRUE,DataProperty,,,list like,,
+Check List Like Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list like,,
+Check List Strict,,,,,TRUE,DataProperty,,,list strict,,
+Check List Enum Strict,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,list strict,,
+Check Regex List,,,,,TRUE,DataProperty,,,list::regex match [a-f],,
+Check Regex List Strict,,,,,TRUE,DataProperty,,,list strict::regex match [a-f],,
+Check Regex List Like,,,,,TRUE,DataProperty,,,list like::regex match [a-f],,
+Check Regex Single,,,,,TRUE,DataProperty,,,regex search [a-f],,
+Check Regex Format,,,,,TRUE,DataProperty,,,regex match [a-f],,
+Check Regex Integer,,,,,TRUE,DataProperty,,,regex search ^\d+$,,
+Check Num,,,,,TRUE,DataProperty,,,num error,,
+Check Float,,,,,TRUE,DataProperty,,,float error,,
+Check Int,,,,,TRUE,DataProperty,,,int error,,
+Check String,,,,,TRUE,DataProperty,,,str error,,
+Check URL,,,,,TRUE,DataProperty,,,url,string,uri
+Check Match at Least,,,,,TRUE,DataProperty,,,matchAtLeastOne Patient.PatientID set,,
+Check Match Exactly,,,,,TRUE,DataProperty,,,matchExactlyOne MockComponent.checkMatchExactly set,,
+Check Match None,,,,,TRUE,DataProperty,,,matchNone MockComponent.checkMatchNone set error,,
+Check Match at Least values,,,,,TRUE,DataProperty,,,matchAtLeastOne MockComponent.checkMatchatLeastvalues value,,
+Check Match Exactly values,,,,,TRUE,DataProperty,,,matchExactlyOne MockComponent.checkMatchExactlyvalues value,,
+Check Match None values,,,,,TRUE,DataProperty,,,matchNone MockComponent.checkMatchNonevalues value error,,
+Check Recommended,,,,,FALSE,DataProperty,,,recommended,,
+Check Ages,,,,,TRUE,DataProperty,,,protectAges,,
+Check Unique,,,,,TRUE,DataProperty,,,unique error,,
+Check Range,,,,,TRUE,DataProperty,,,inRange 50 100 error,,
+Check Date,,,,,TRUE,DataProperty,,,date,string,date
+Check NA,,,,,TRUE,DataProperty,,,int::IsNA,,
+MockRDB,,,"Component, MockRDB_id, SourceManifest",,FALSE,DataType,,,,,
+MockRDB_id,,,,,TRUE,DataProperty,,,int,,
+SourceManifest,,,,,TRUE,DataProperty,,,,,
+MockFilename,,,"Component, Filename",,FALSE,DataType,,,,,
+JSONSchemaComponent,Component to hold attributes for testing JSON Schemas,,"Component, No Rules, No Rules Not Required, String, String Not Required, Enum, Enum Not Required, Date, URL, InRange, Regex, List, List Not Required, List Enum, List Enum Not Required, List Boolean, List, Integer, List InRange",,FALSE,DataType,,,,,
+No Rules,,,,,TRUE,DataProperty,,,,,
+No Rules Not Required,,,,,FALSE,DataProperty,,,,,
+String,,,,,TRUE,DataProperty,,,,string,
+String Not Required,,,,,FALSE,DataProperty,,,,string,
+Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,,string,
+Enum Not Required,,"ab, cd, ef, gh",,,FALSE,DataProperty,,,,string,
+Date,,,,,TRUE,DataProperty,,,date,string,date
+URL,,,,,TRUE,DataProperty,,,url,string,uri
+InRange,,,,,TRUE,DataProperty,,,inRange 50 100,number,
+Regex,,,,,TRUE,DataProperty,,,regex search [a-f],string,
+List,,,,,TRUE,DataProperty,,,,string_list,
+List Not Required,,,,,FALSE,DataProperty,,,,string_list,
+List Enum,,"ab, cd, ef, gh",,,TRUE,DataProperty,,,,string_list,
+List Enum Not Required,,"ab, cd, ef, gh",,,FALSE,DataProperty,,,,string_list,
+List Boolean,,,,,TRUE,DataProperty,,,,boolean_list,
+List Integer,,,,,TRUE,DataProperty,,,,integer_list,
+List InRange,,,,,TRUE,DataProperty,,,inRange 50 100,integer_list,
+TypeDefinitionComponent,Component to check type specification,,"Component, String type, String type caps, Int type, Int type caps, Num type, Num type caps, Nan type, Missing type, Boolean type, Boolean type caps",,FALSE,DataType,,,,,
+String type,,,,,TRUE,DataProperty,,,,string,
+String type caps,,,,,TRUE,DataProperty,,,,STRING,
+Int type,,,,,TRUE,DataProperty,,,,integer,
+Int type caps,,,,,TRUE,DataProperty,,,,INTEGER,
+Num type,,,,,TRUE,DataProperty,,,,number,
+Num type caps,,,,,TRUE,DataProperty,,,,NUMBER,
+Nan type,,,,,TRUE,DataProperty,,,,nan,
+Missing type,,,,,TRUE,DataProperty,,,,,
+Boolean type,,,,,TRUE,DataProperty,,,,boolean,
+Boolean type caps,,,,,TRUE,DataProperty,,,,BOOLEAN,

tests/unit/synapseclient/extensions/unit_test_data_model_graph_explorer.py

@@ -6,6 +6,7 @@
     AtomicColumnType,
     ColumnType,
     DataModelGraphExplorer,
+    Format,
     ListColumnType,
 )
 
@@ -66,3 +67,13 @@ def test_get_node_column_type(
     dmge: DataModelGraphExplorer, node_label: str, column_type: ColumnType
 ) -> None:
     assert dmge.get_node_column_type(node_label) == column_type
+
+
+@pytest.mark.parametrize(
+    "node_label, column_type",
+    [("String", None), ("Date", Format.DATE), ("URL", Format.URI)],
+)
+def test_get_node_format(
+    dmge: DataModelGraphExplorer, node_label: str, column_type: Format
+) -> None:
+    assert dmge.get_node_format(node_label) == column_type