Merge pull request #120 from WDGPH/bug/replace-unspecified

Refactor preprocessing logic to replace unspecified agents and update…

Author: kassyray

pipeline/orchestrator.py

@@ -236,7 +236,7 @@ def run_step_2_preprocess(
 
     # Build preprocessing result
     result = preprocess.build_preprocess_result(
-        df, language, vaccine_reference, preprocess.IGNORE_AGENTS
+        df, language, vaccine_reference, preprocess.REPLACE_UNSPECIFIED
     )
 
     # Write artifact

pipeline/preprocess.py

@@ -70,7 +70,7 @@
 
 _FORMATTER = Formatter()
 
-IGNORE_AGENTS = [
+REPLACE_UNSPECIFIED = [
     "-unspecified",
     "unspecified",
     "Not Specified",
@@ -613,7 +613,7 @@ def process_vaccines_due(vaccines_due: Any, language: str) -> str:
 
 
 def process_received_agents(
-    received_agents: Any, ignore_agents: List[str]
+    received_agents: Any, replace_unspecified: List[str]
 ) -> List[Dict[str, Any]]:
     """Extract and normalize vaccination history from received_agents string."""
     if not isinstance(received_agents, str) or not received_agents.strip():
@@ -626,7 +626,7 @@ def process_received_agents(
     for match in matches:
         date_str, vaccine = match.split(" - ", maxsplit=1)
         vaccine = vaccine.strip()
-        if vaccine in ignore_agents:
+        if vaccine in replace_unspecified:
             continue
         date_iso = convert_date_iso(date_str.strip())
         rows.append({"date_given": date_iso, "vaccine": vaccine})
@@ -716,7 +716,7 @@ def build_preprocess_result(
     df: pd.DataFrame,
     language: str,
     vaccine_reference: Dict[str, Any],
-    ignore_agents: List[str],
+    replace_unspecified: List[str],
 ) -> PreprocessResult:
     """Process and normalize client data into structured artifact.
 
@@ -788,7 +788,7 @@ def build_preprocess_result(
         vaccines_due_list = [
             item.strip() for item in vaccines_due.split(",") if item.strip()
         ]
-        received_grouped = process_received_agents(row.IMMS_GIVEN, ignore_agents)  # type: ignore[attr-defined]
+        received_grouped = process_received_agents(row.IMMS_GIVEN, replace_unspecified)  # type: ignore[attr-defined]
         received = enrich_grouped_records(
             received_grouped, vaccine_reference, language, chart_diseases_header
         )

templates/en_template.py

@@ -49,7 +49,7 @@
 
 #v(0.2cm)
 
-#conf.header_info_cim("__LOGO_PATH__", black, 16pt, "Request for Immunization Record")
+#conf.header_info_cim("__LOGO_PATH__", 6cm, black, 16pt, "Request for Immunization Record")
 
 #v(0.2cm)
 

templates/fr_template.py

@@ -50,7 +50,7 @@
 
 #v(0.2cm)
 
-#conf.header_info_cim("__LOGO_PATH__", black, 16pt, "Demande de dossier d'immunisation")
+#conf.header_info_cim("__LOGO_PATH__", 6cm, black, 16pt, "Demande de dossier d'immunisation")
 
 #v(0.2cm)
 

tests/conftest.py

@@ -148,7 +148,7 @@ def default_config(tmp_output_structure: Dict[str, Path]) -> Dict[str, Any]:
             "Mumps",
             "Rubella",
         ],
-        "ignore_agents": [],
+        "replace_unspecified": [],
         "typst": {
             "bin": "typst",
         },

tests/fixtures/conftest.py

@@ -144,7 +144,7 @@ def default_config(tmp_output_structure: Dict[str, Path]) -> Dict[str, Any]:
             "Mumps",
             "Rubella",
         ],
-        "ignore_agents": [],
+        "replace_unspecified": [],
     }
 
 

tests/integration/test_config_driven_behavior.py

@@ -106,15 +106,15 @@ def test_chart_diseases_header_configuration(
         assert isinstance(default_config["chart_diseases_header"], list)
         assert len(default_config["chart_diseases_header"]) > 0
 
-    def test_ignore_agents_configuration(self, default_config: Dict[str, Any]) -> None:
-        """Verify ignore_agents list is configurable.
+    def test_replace_unspecified_configuration(self, default_config: Dict[str, Any]) -> None:
+        """Verify replace_unspecified list is configurable.
 
         Real-world significance:
         - Some agents (staff) should not receive notices
         - Config allows filtering out specific agent types
         """
-        assert "ignore_agents" in default_config
-        assert isinstance(default_config["ignore_agents"], list)
+        assert "replace_unspecified" in default_config
+        assert isinstance(default_config["replace_unspecified"], list)
 
 
 @pytest.mark.integration

tests/unit/test_preprocess.py

@@ -483,7 +483,7 @@ def test_build_result_generates_clients_with_sequences(
             normalized,
             language="en",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=[],
+            replace_unspecified=[],
         )
 
         assert len(result.clients) == 3
@@ -508,14 +508,14 @@ def test_build_result_sorts_clients_deterministically(
             normalized,
             language="en",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=[],
+            replace_unspecified=[],
         )
 
         result2 = preprocess.build_preprocess_result(
             normalized,
             language="en",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=[],
+            replace_unspecified=[],
         )
 
         ids1 = [c.client_id for c in result1.clients]
@@ -569,7 +569,7 @@ def test_build_result_sorts_by_school_then_name(
             normalized,
             language="en",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=[],
+            replace_unspecified=[],
         )
 
         # Expected order: Apple/Chloe/Jones, Apple/Diana/Jones, Zebra/Alice/Smith, Zebra/Bob/Smith
@@ -595,7 +595,7 @@ def test_build_result_maps_vaccines_correctly(
             normalized,
             language="en",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=[],
+            replace_unspecified=[],
         )
 
         # Should have DTaP expanded to component diseases
@@ -637,7 +637,7 @@ def test_build_result_handles_missing_board_name_with_warning(
             normalized,
             language="en",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=[],
+            replace_unspecified=[],
         )
 
         # Should still process - at least one client
@@ -660,16 +660,16 @@ def test_build_result_french_language_support(
             normalized,
             language="fr",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=[],
+            replace_unspecified=[],
         )
 
         assert len(result.clients) == 1
         assert result.clients[0].language == "fr"
 
-    def test_build_result_handles_ignore_agents(
+    def test_build_result_handles_replace_unspecified(
         self, default_vaccine_reference
     ) -> None:
-        """Verify ignore_agents filters out unspecified vaccines.
+        """Verify replace_unspecified filters out unspecified vaccines.
 
         Real-world significance:
         - Input may contain "Not Specified" vaccine agents
@@ -682,7 +682,7 @@ def test_build_result_handles_ignore_agents(
             normalized,
             language="en",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=["Not Specified", "unspecified"],
+            replace_unspecified=["Not Specified", "unspecified"],
         )
 
         assert len(result.clients) == 1
@@ -708,7 +708,7 @@ def test_build_result_detects_duplicate_client_ids(
             normalized,
             language="en",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=[],
+            replace_unspecified=[],
         )
 
         # Should have 2 clients (no deduplication)
@@ -744,7 +744,7 @@ def test_build_result_detects_multiple_duplicate_client_ids(
             normalized,
             language="en",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=[],
+            replace_unspecified=[],
         )
 
         # Should have 5 clients (no deduplication)
@@ -776,7 +776,7 @@ def test_build_result_no_warning_for_unique_client_ids(
             normalized,
             language="en",
             vaccine_reference=default_vaccine_reference,
-            ignore_agents=[],
+            replace_unspecified=[],
         )
 
         # Should have 3 unique clients