ColumnTypeInterpreter understands numeric

Author: frederikhoengaard

Pipfile

@@ -7,6 +7,7 @@ verify_ssl = true
 loguru = "==0.6.*"
 pandas = "==1.5.*"
 scikit-learn = "*"
+tqdm = "*"
 
 [dev-packages]
 black = "==23.*"

Pipfile.lock

@@ -1,4 +1,6 @@
-from pipeline.pipeline import IngestionPipeline, PipelineStep
+from ingestion.ingestion_pipeline_steps.data_parser_step import DataSourceParser  # noqa
+from ingestion.ingestion_pipeline_steps.interpreter_step import ColumnTypeInterpreter # noqa
+from pipeline.pipeline import IngestionPipeline
 
 
 class Ingestion:
@@ -11,7 +13,7 @@ def run(self, data):
 
         pipeline.add(DataSourceParser(data))
 
-        pipeline.add(ColumnInterpreter())
+        pipeline.add(ColumnTypeInterpreter())
 
         pipeline.run()
 

python/src/lazylearn/ingestion/ingestion_pipeline.py

@@ -0,0 +1,41 @@
+from pandas import Series
+from pipeline.pipeline import IngestionPipeline
+from tqdm import tqdm
+
+
+class ColumnTypeInterpreter:
+    def apply(self, pipeline: IngestionPipeline):
+        """
+
+        :param pipeline: parent IngestionPipeline
+        :return:
+        """
+        columns = pipeline.df.columns
+        column_types = {}
+
+        for column_name in tqdm(columns):
+            column_types[column_name] = self.analyze_column(pipeline.df[column_name])  # noqa
+
+        pipeline.column_type_map = column_types
+
+    def analyze_column(self, column: Series):
+        # is it numeric?
+        values = set(column)
+        types = set([type(value) for value in values])
+
+        if self.numeric_test(types):
+            return "numeric"
+
+        return "object"
+
+    @staticmethod
+    def numeric_test(types: set):
+        return all([item == float or item == int for item in types])
+
+    @staticmethod
+    def string_test(types: set):
+        raise NotImplementedError
+
+    @staticmethod
+    def date_check(types: set):
+        raise NotImplementedError

python/src/lazylearn/ingestion/ingestion_pipeline_steps/interpreter_step.py

@@ -27,6 +27,7 @@ def __init__(self):
         super().__init__()
         self.raw_data = None
         self.df: DataFrame = None
+        self.column_type_map: dict = None
 
     def response(self):
         return Dataset

python/src/lazylearn/pipeline/pipeline.py

@@ -0,0 +1,17 @@
+from ingestion.ingestion_pipeline_steps.interpreter_step import ColumnTypeInterpreter
+from pipeline.pipeline import IngestionPipeline
+from sklearn.datasets import load_iris
+
+
+def test_iris_okay():
+    pipeline = IngestionPipeline()
+    pipeline.df = load_iris(return_X_y=True, as_frame=True)[0]
+    pipeline.add(ColumnTypeInterpreter())
+    pipeline.run()
+
+    assert pipeline.column_type_map == {
+        "sepal length (cm)": "numeric",
+        "sepal width (cm)": "numeric",
+        "petal length (cm)": "numeric",
+        "petal width (cm)": "numeric",
+    }