Merge pull request #1276 from Sage-Bionetworks/SYNPY-1679

[SYNPY-1679] Add format column to csv data model.

Author: andrewelamb

synapseclient/extensions/curator/schema_generation.py

@@ -122,13 +122,37 @@ class ListColumnType(ColumnType):
 }
 
 
+class JSONSchemaFormat(Enum):
+    """
+    Allowed formats by the JSON Schema validator used by Synapse: https://github.com/everit-org/json-schema#format-validators
+    For descriptions see: https://json-schema.org/understanding-json-schema/reference/type#format
+    """
+
+    DATE_TIME = "date-time"
+    EMAIL = "email"
+    HOSTNAME = "hostname"
+    IPV4 = "ipv4"
+    IPV6 = "ipv6"
+    URI = "uri"
+    URI_REFERENCE = "uri-reference"
+    URI_TEMPLATE = "uri-template"
+    JSON_POINTER = "json-pointer"
+    DATE = "date"
+    TIME = "time"
+    REGEX = "regex"
+    RELATIVE_JSON_POINTER = "relative-json-pointer"
+
+
 class ValidationRuleName(Enum):
     """Names of validation rules that are used to create JSON Schema"""
 
     # list validation rule is been deprecated for use in deciding type
     # TODO: remove list:
     # https://sagebionetworks.jira.com/browse/SYNPY-1692
     LIST = "list"
+    # url and date rules are deprecated for adding format keyword
+    # TODO: remove url and date
+    # https://sagebionetworks.jira.com/browse/SYNPY-1685
     DATE = "date"
     URL = "url"
     REGEX = "regex"
@@ -166,6 +190,9 @@ class ValidationRule:
         name=ValidationRuleName.LIST,
         incompatible_rules=[],
     ),
+    # url and date rules are deprecated for adding format keyword
+    # TODO: remove url and date
+    # https://sagebionetworks.jira.com/browse/SYNPY-1685
     "date": ValidationRule(
         name=ValidationRuleName.DATE,
         incompatible_rules=[
@@ -215,13 +242,6 @@ def __post_init__(self) -> None:
         self.display_name = str(self.fields["displayName"])
 
 
-class JSONSchemaFormat(Enum):
-    """This enum is the currently supported JSON Schema formats"""
-
-    DATE = "date"
-    URI = "uri"
-
-
 def load_json(file_path: str) -> Any:
     """Load json document from file path or url
 
@@ -635,7 +655,10 @@ def gather_csv_attributes_relationships(
 
         # get attributes from Attribute column
         attributes = model_df.to_dict("records")
+
+        # Check for presence of optional columns
         model_includes_column_type = "columnType" in model_df.columns
+        model_includes_format = "Format" in model_df.columns
 
         # Build attribute/relationship dictionary
         relationship_types = self.required_headers
@@ -659,6 +682,9 @@ def gather_csv_attributes_relationships(
                 attr_rel_dictionary[attribute_name]["Relationships"].update(
                     column_type_dict
                 )
+            if model_includes_format:
+                format_dict = self.parse_format(attr)
+                attr_rel_dictionary[attribute_name]["Relationships"].update(format_dict)
         return attr_rel_dictionary
 
     def parse_column_type(self, attr: dict) -> dict:
@@ -691,6 +717,34 @@ def parse_column_type(self, attr: dict) -> dict:
 
         return {"ColumnType": column_type}
 
+    def parse_format(self, attribute_dict: dict) -> dict[str, str]:
+        """Finds the format value if it exists and returns it as a dictionary.
+
+        Args:
+            attribute_dict: The attribute dictionary.
+
+        Returns:
+            A dictionary containing the format value if it exists
+            else an empty dict
+        """
+        from pandas import isna
+
+        format_value = attribute_dict.get("Format")
+
+        if isna(format_value):
+            return {}
+
+        format_string = str(format_value).strip().lower()
+
+        check_allowed_values(
+            self.dmr,
+            entry_id=attribute_dict["Format"],
+            value=format_string,
+            relationship="format",
+        )
+
+        return {"Format": format_string}
+
     def parse_csv_model(
         self,
         path_to_data_model: str,
@@ -1708,6 +1762,37 @@ def get_node_column_type(
                 raise ValueError(msg)
         return column_type
 
+    def get_node_format(
+        self, node_label: Optional[str] = None, node_display_name: Optional[str] = None
+    ) -> Optional[JSONSchemaFormat]:
+        """Gets the format of the node
+
+        Args:
+            node_label: The label of the node to get the format from
+            node_display_name: The display name of the node to get the format from
+
+        Raises:
+            ValueError: If the value from the node is not allowed
+
+        Returns:
+            The format of the node if it has one, otherwise None
+        """
+        node_label = self._get_node_label(node_label, node_display_name)
+        rel_node_label = self.dmr.get_relationship_value("format", "node_label")
+        format_value = self.graph.nodes[node_label][rel_node_label]
+        if format_value is None:
+            return format_value
+        format_string = str(format_value).lower()
+        try:
+            column_type = JSONSchemaFormat(format_string)
+        except ValueError as exc:
+            msg = (
+                f"Node: '{node_label}' had illegal format value: '{format_value}'. "
+                f"Allowed values are: [{[member.value for member in JSONSchemaFormat]}]"
+            )
+            raise ValueError(msg) from exc
+        return column_type
+
     def _get_node_label(
         self, node_label: Optional[str] = None, node_display_name: Optional[str] = None
     ) -> str:
@@ -2826,6 +2911,16 @@ def define_data_model_relationships(self) -> dict:
                 "node_attr_dict": {"default": None},
                 "allowed_values": ALL_COLUMN_TYPE_VALUES,
             },
+            "format": {
+                "jsonld_key": "sms:format",
+                "csv_header": "Format",
+                "node_label": "format",
+                "type": str,
+                "required_header": False,
+                "edge_rel": False,
+                "node_attr_dict": {"default": None},
+                "allowed_values": [member.value for member in JSONSchemaFormat],
+            },
         }
 
         return map_data_model_relationships
@@ -4290,6 +4385,7 @@ def _get_rules_by_names(names: list[str]) -> list[ValidationRule]:
 def _get_validation_rule_based_fields(
     validation_rules: list[str],
     explicit_is_array: Optional[bool],
+    explicit_format: Optional[JSONSchemaFormat],
     name: str,
     column_type: Optional[ColumnType],
     logger: Logger,
@@ -4334,7 +4430,7 @@ def _get_validation_rule_based_fields(
         - js_pattern: If the type is string the JSON Schema pattern
     """
     js_is_array = False
-    js_format = None
+    js_format = explicit_format
     js_minimum = None
     js_maximum = None
     js_pattern = None
@@ -4398,10 +4494,51 @@ def _get_validation_rule_based_fields(
                 )
                 logger.warning(msg)
 
-        if ValidationRuleName.URL in validation_rule_names:
-            js_format = JSONSchemaFormat.URI
-        elif ValidationRuleName.DATE in validation_rule_names:
-            js_format = JSONSchemaFormat.DATE
+        # url and date rules are deprecated for adding format keyword
+        # TODO: remove the if/else block below
+        # https://sagebionetworks.jira.com/browse/SYNPY-1685
+
+        if explicit_format:
+            if (
+                ValidationRuleName.DATE in validation_rule_names
+                and explicit_format == JSONSchemaFormat.URI
+            ):
+                msg = (
+                    f"For property: {name}, the format is uri, "
+                    "but the validation rule date is present. "
+                    "The format will be set to uri."
+                )
+                logger.warning(msg)
+            elif (
+                ValidationRuleName.URL in validation_rule_names
+                and explicit_format == JSONSchemaFormat.DATE
+            ):
+                msg = (
+                    f"For property: {name}, the format is date, "
+                    "but the validation rule url is present. "
+                    "The format will be set to date."
+                )
+                logger.warning(msg)
+
+        else:
+            if ValidationRuleName.URL in validation_rule_names:
+                js_format = JSONSchemaFormat.URI
+                msg = (
+                    f"A url validation rule is set for property: {name}, but the format is not set. "
+                    "The format will be set to uri, but this behavior is deprecated and validation "
+                    "rules will no longer be used in the future."
+                    "Please explicitly set the format to uri in the data model."
+                )
+                logger.warning(msg)
+            elif ValidationRuleName.DATE in validation_rule_names:
+                js_format = JSONSchemaFormat.DATE
+                msg = (
+                    f"A date validation rule is set for property: {name}, but the format is not set. "
+                    "The format will be set to date, but this behavior is deprecated and validation "
+                    "rules will no longer be used in the future."
+                    "Please explicitly set the format to uri in the data model."
+                )
+                logger.warning(msg)
 
         in_range_rule = get_rule_from_inputted_rules(
             ValidationRuleName.IN_RANGE, validation_rules
@@ -4417,7 +4554,6 @@ def _get_validation_rule_based_fields(
         if regex_rule:
             js_pattern = get_regex_parameters_from_inputted_rule(regex_rule)
 
-    print(js_is_array)
     return (
         js_is_array,
         js_format,
@@ -4496,7 +4632,6 @@ def __post_init__(self) -> None:
         column_type = self.dmge.get_node_column_type(
             node_display_name=self.display_name
         )
-
         # list validation rule is been deprecated for use in deciding type
         # TODO: set self.is_array here instead of return from _get_validation_rule_based_fields
         # https://sagebionetworks.jira.com/browse/SYNPY-1692
@@ -4509,6 +4644,22 @@ def __post_init__(self) -> None:
         else:
             self.type = None
             explicit_is_array = None
+
+        # url and date rules are deprecated for adding format keyword
+        # TODO: set self.format here instead of passing it to get_validation_rule_based_fields
+        # https://sagebionetworks.jira.com/browse/SYNPY-1685
+        explicit_format = self.dmge.get_node_format(node_display_name=self.display_name)
+        if explicit_format:
+            if column_type not in (ListColumnType.STRING_LIST, AtomicColumnType.STRING):
+                msg = (
+                    f"A format value (current value: {explicit_format.value}) "
+                    f"is set for property: {self.name}, but columnType is not a string type "
+                    f"(current value: {column_type.value}). "
+                    "To use a format value the columnType must be set to one of: "
+                    "[string, string_list] "
+                )
+                raise ValueError(msg)
+
         (
             self.is_array,
             self.format,
@@ -4518,6 +4669,7 @@ def __post_init__(self) -> None:
         ) = _get_validation_rule_based_fields(
             validation_rules=validation_rules,
             explicit_is_array=explicit_is_array,
+            explicit_format=explicit_format,
             name=self.name,
             column_type=self.type,
             logger=self.logger,
@@ -4896,7 +5048,7 @@ def _set_conditional_dependencies(
 
 
 def _create_enum_array_property(
-    node: Node, use_valid_value_display_names: bool = True
+    node: TraversalNode, use_valid_value_display_names: bool = True
 ) -> Property:
     """
     Creates a JSON Schema property array with enum items
@@ -4930,7 +5082,7 @@ def _create_enum_array_property(
     return array_property
 
 
-def _create_array_property(node: Node) -> Property:
+def _create_array_property(node: TraversalNode) -> Property:
     """
     Creates a JSON Schema property array
 
@@ -4962,7 +5114,7 @@ def _create_array_property(node: Node) -> Property:
 
 
 def _create_enum_property(
-    node: Node, use_valid_value_display_names: bool = True
+    node: TraversalNode, use_valid_value_display_names: bool = True
 ) -> Property:
     """
     Creates a JSON Schema property enum
@@ -4995,7 +5147,7 @@ def _create_enum_property(
     return enum_property
 
 
-def _create_simple_property(node: Node) -> Property:
+def _create_simple_property(node: TraversalNode) -> Property:
     """
     Creates a JSON Schema property
 
@@ -5031,7 +5183,7 @@ def _create_simple_property(node: Node) -> Property:
     return prop
 
 
-def _set_type_specific_keywords(schema: dict[str, Any], node: Node) -> None:
+def _set_type_specific_keywords(schema: dict[str, Any], node: TraversalNode) -> None:
     """Sets JSON Schema keywords that are allowed if type has been set
 
     Arguments:
@@ -5049,7 +5201,7 @@ def _set_type_specific_keywords(schema: dict[str, Any], node: Node) -> None:
 
 def _set_property(
     json_schema: JSONSchema,
-    node: Node,
+    node: TraversalNode,
     use_property_display_names: bool = True,
     use_valid_value_display_names: bool = True,
 ) -> None: