Remove pd & numpy, bulk insert volg ppl

Author: c-simpson

src/server/api/file_uploader.py

@@ -1,4 +1,4 @@
-import pandas as pd
+# import pandas as pd
 from config import engine
 from donations_importer import validate_import_sfd
 from flask import current_app

src/server/api/internal_api.py

@@ -5,7 +5,7 @@
 
 from api.API_ingest import ingest_sources_from_api, salesforce_contacts
 from api.api import internal_api
-from rfm_funcs.create_scores import create_scores
+# from rfm_funcs.create_scores import create_scores
 from api.API_ingest import updated_data
 
 logger = structlog.get_logger()
@@ -37,12 +37,12 @@ def ingest_raw_data():
     return jsonify({'outcome': 'OK'}), 200
 
 
-@internal_api.route("/api/internal/create_scores", methods=["GET"])
-def hit_create_scores():
-    logger.info("Hitting create_scores() ")
-    tuple_count = create_scores()
-    logger.info("create_scores()  processed %s scores",  str(tuple_count) )
-    return jsonify(200)
+# @internal_api.route("/api/internal/create_scores", methods=["GET"])
+# def hit_create_scores():
+#     logger.info("Hitting create_scores() ")
+#     tuple_count = create_scores()
+#     logger.info("create_scores()  processed %s scores",  str(tuple_count) )
+#     return jsonify(200)
 
 
 @internal_api.route("/api/internal/get_updated_data", methods=["GET"])

src/server/models.py

@@ -2,7 +2,7 @@
 import re
 from itertools import combinations
 
-import pandas as pd
+# import pandas as pd
 import sqlalchemy as sa
 from sqlalchemy import (
     Boolean,
@@ -296,35 +296,35 @@ class Volgistics(Base):
     json = Column(JSONB)
     created_date = Column(DateTime, default=datetime.datetime.utcnow)
 
-    @classmethod
-    def insert_from_file(cls, xl_file, conn):
-        df = pd.read_excel(xl_file, sheet_name="Master")
-
-        column_translation = get_source_column_translation(cls)
-        df = df[column_translation.keys()]
-        df = df.rename(columns=column_translation)
-
-        df["home"] = df["home"].apply(normalize_phone_number)
-        df["work"] = df["work"].apply(normalize_phone_number)
-        df["cell"] = df["cell"].apply(normalize_phone_number)
-
-        dedup_on = [col for col in cls.__table__.columns if col.name in df.columns]
-        df["created_date"] = datetime.datetime.utcnow()
-        df.to_sql(
-            cls.__tablename__,
-            conn,
-            if_exists="append",
-            index=False,
-        )
-        conn.execute(
-            dedup_consecutive(
-                cls.__table__,
-                unique_id=cls._id,
-                id=cls.number,
-                order_by=cls.created_date,
-                dedup_on=tuple_(*dedup_on),
-            )
-        )
+    # @classmethod
+    # def insert_from_file(cls, xl_file, conn):
+    #     df = pd.read_excel(xl_file, sheet_name="Master")
+
+    #     column_translation = get_source_column_translation(cls)
+    #     df = df[column_translation.keys()]
+    #     df = df.rename(columns=column_translation)
+
+    #     df["home"] = df["home"].apply(normalize_phone_number)
+    #     df["work"] = df["work"].apply(normalize_phone_number)
+    #     df["cell"] = df["cell"].apply(normalize_phone_number)
+
+    #     dedup_on = [col for col in cls.__table__.columns if col.name in df.columns]
+    #     df["created_date"] = datetime.datetime.utcnow()
+    #     df.to_sql(
+    #         cls.__tablename__,
+    #         conn,
+    #         if_exists="append",
+    #         index=False,
+    #     )
+    #     conn.execute(
+    #         dedup_consecutive(
+    #             cls.__table__,
+    #             unique_id=cls._id,
+    #             id=cls.number,
+    #             order_by=cls.created_date,
+    #             dedup_on=tuple_(*dedup_on),
+    #         )
+    #     )
 
     @classmethod
     def insert_into_pdp_contacts(cls):
@@ -356,29 +356,29 @@ class ManualMatches(Base):
     source_type_2 = Column(String, primary_key=True)
     source_id_2 = Column(String, primary_key=True)
 
-    @classmethod
-    def insert_from_df(cls, df, conn):
-        # Our input csv has columns like "salesforcecontacts," "volgistics," and
-        # "shelterluvpeople," where two columns are non-null if there is an
-        # association between those two ids. We massage this table into one that
-        # is easier to join on.
+    # @classmethod
+    # def insert_from_df(cls, df, conn):
+    #     # Our input csv has columns like "salesforcecontacts," "volgistics," and
+    #     # "shelterluvpeople," where two columns are non-null if there is an
+    #     # association between those two ids. We massage this table into one that
+    #     # is easier to join on.
 
-        match_dicts = df.to_dict(orient="records")
-
-        matched_pairs = []
-        for match in match_dicts:
-            non_nulls = {k: v for (k, v) in match.items() if not pd.isna(v)}
-            for ((st1, sid1), (st2, sid2)) in combinations(non_nulls.items(), 2):
-                matched_pairs.append(
-                    {
-                        "source_type_1": st1,
-                        "source_id_1": sid1,
-                        "source_type_2": st2,
-                        "source_id_2": sid2,
-                    }
-                )
-
-        conn.execute(insert(cls).values(matched_pairs).on_conflict_do_nothing())
+    #     match_dicts = df.to_dict(orient="records")
+
+    #     matched_pairs = []
+    #     for match in match_dicts:
+    #         non_nulls = {k: v for (k, v) in match.items() if not pd.isna(v)}
+    #         for ((st1, sid1), (st2, sid2)) in combinations(non_nulls.items(), 2):
+    #             matched_pairs.append(
+    #                 {
+    #                     "source_type_1": st1,
+    #                     "source_id_1": sid1,
+    #                     "source_type_2": st2,
+    #                     "source_id_2": sid2,
+    #                 }
+    #             )
+
+    #     conn.execute(insert(cls).values(matched_pairs).on_conflict_do_nothing())
 
 class SalesforceDonations(Base):
     __tablename__ = "salesforcedonations"

src/server/shifts_importer.py

@@ -7,6 +7,8 @@
 from config import  engine
 
 import structlog
+
+from api.API_ingest.volgistics import insert_volgistics_people
 logger = structlog.get_logger()
 
 
@@ -93,6 +95,10 @@ def validate_import_vs(workbook, conn):
         row_count = 0
         missing_volgistics_id = 0
 
+
+        #TODO: Perform bulk insert as for people_insert
+
+
         for row in ws.values:        
             if seen_header: 
                 row_count += 1
@@ -166,40 +172,52 @@ def volgistics_people_import(workbook,conn):
     metadata = MetaData()
     volg_table = Table("volgistics", metadata, autoload=True, autoload_with=engine)
 
-
-    # Cells are addressed as ws[row][col] with row being 1-based and col being 0-based
+    # Worksheet cells are addressed as ws[row][col] with row being 1-based and col being 0-based
 
     insert_list = []
 
-    #TODO: Create a dict from header row so can reference r["number"] instead of r[15]
-
-
-    for r in ws.iter_rows(min_row=2, max_col=42,values_only=True):
-        insert_list.append(
-            {
-                "number": r[15],
-                "last_name": r[3],
-                "first_name": r[4],
-                "middle_name": r[5],
-                "complete_address": r[16],
-                "street_1": r[17],
-                "street_2": r[18],
-                "street_3": r[19],
-                "city": r[20],
-                "state": r[21],
-                "zip": r[22],
-                "all_phone_numbers": r[27],
-                "home": r[28],
-                "work": r[30],
-                "cell": r[32],
-                "email": r[41]
-            }
-        )
-
-
-    ret = session.execute(volg_table.insert(insert_list))
-
-    session.commit()  # Commit all inserted rows
-    session.close()
-
-    logger.debug('%d rows inserted', ret.rowcount)
+    # Create a dict from header row so can reference columns by name
+    # e.g., r[col['Number']] instead of r[15]
+    header = ws[1]
+    col = {};
+    idx = 0
+    for cell in header:
+        col[cell.value] = idx
+        idx += 1
+
+
+
+    time_stamp = datetime.utcnow()
+
+    try:
+        for r in ws.iter_rows(min_row=2, max_col=42,values_only=True):
+            insert_list.append(
+                {
+                    "number": r[col['Number']],
+                    "last_name": r[col['Last name']],
+                    "first_name": r[col['First name']],
+                    "middle_name": r[col['Middle name']],
+                    "complete_address": r[col['Complete address']],
+                    "street_1": r[col['Street 1']],
+                    "street_2": r[col['Street 2']],
+                    "street_3": r[col['Street 3']],
+                    "city": r[col['City']],
+                    "state": r[col['State']],
+                    "zip": r[col['Zip']],
+                    "all_phone_numbers": r[col['All phone numbers']],
+                    "home": r[col['Home']],
+                    "work": r[col['Work']],
+                    "cell": r[col['Cell']],
+                    "email": r[col['Email']],
+                    "created_date" : time_stamp
+                }
+            )
+    except KeyError as e:
+        logger.error("Volgistics source XLSX file 'Master' tab missing expected column (see following)  - cannot import")
+        logger.exception(e)
+
+
+
+    rows = insert_volgistics_people(insert_list)
+
+    logger.debug('Inserted %d Volgistics people rows', rows)