last-update logging #538

Author: c-simpson

src/server/api/API_ingest/ingest_sources_from_api.py

@@ -1,5 +1,7 @@
 from api.API_ingest import shelterluv_people, salesforce_contacts, sl_animal_events
 import structlog
+
+from pipeline.log_db import log_shelterluv_update
 logger = structlog.get_logger()
 
 def start():
@@ -16,6 +18,7 @@ def start():
     logger.debug("   Fetching Shelterluv events")
     sle_count = sl_animal_events.store_all_animals_and_events()
     logger.debug("   Finished fetching Shelterluv events - %d records" , sle_count)
+    log_shelterluv_update()
 
     logger.debug("Finished fetching raw data from different API sources")
 

src/server/api/admin_api.py

@@ -231,6 +231,26 @@ def start_job():
         return job_id
 
 
+@admin_api.route("/api/get_last_runs", methods=["GET"])
+#@jwt_ops.admin_required
+def get_run_logs():
+    """ Get the timestamps of the last update runs"""
+
+    with engine.connect() as connection:
+        q = text("""select keycol,valcol from kv_unique where keycol like '%_update'; """)
+        result = connection.execute(q)
+
+        if result.rowcount > 0:
+           rows = result.fetchall()
+
+        row_list = []
+
+        for row in rows:
+            row_dict = row._mapping 
+            row_list.append({row_dict['keycol'] : row_dict['valcol']})
+
+        return jsonify(row_list)
+
 
 def insert_rfm_scores(score_list):
     """Take a list of (matching_id, score) and insert into the

src/server/api/file_uploader.py

@@ -2,9 +2,9 @@
 from donations_importer import validate_import_sfd
 from flask import current_app
 from models import ManualMatches, SalesForceContacts, ShelterluvPeople, Volgistics
+from pipeline.log_db import log_volgistics_update
 from volgistics_importer import open_volgistics, validate_import_vs, volgistics_people_import
 from werkzeug.utils import secure_filename
-
 import structlog
 logger = structlog.get_logger()
 
@@ -35,6 +35,7 @@ def determine_upload_type(file, file_extension, conn):
             validate_import_vs(workbook)
             volgistics_people_import(workbook)
             workbook.close()
+            log_volgistics_update()
         return
 
     logger.error("Don't know how to process file: %s",  file.filename)

src/server/pipeline/log_db.py

@@ -2,6 +2,7 @@
 import json
 from sqlalchemy.sql import text
 from flask import  current_app
+import time
 
 from sqlalchemy.dialects.postgresql import insert
 from sqlalchemy import Table, Column, Integer, String, MetaData, ForeignKey, exc, select
@@ -13,6 +14,8 @@
 metadata = MetaData()
 
 ex_stat = Table("execution_status", metadata, autoload=True, autoload_with=engine)
+kvt = Table("kv_unique", metadata, autoload=True, autoload_with=engine)
+
 
 # Alembic version bfb1262d3195
 
@@ -52,3 +55,70 @@ def log_exec_status(job_id: str, exec_stage: str, exec_status: str, job_details:
             logger.error(e)
 
 
+def log_volgistics_update():
+    """Log Volgistics data update"""
+
+    timestamp = datetime.now().ctime()
+
+    with engine.connect() as connection:
+        ins_stmt = insert(kvt).values(               
+            keycol = 'last_volgistics_update',
+            valcol = timestamp,
+            )
+        # If key already present in DB, do update instead
+        upsert = ins_stmt.on_conflict_do_update(
+                constraint='kv_unique_keycol_key',
+                set_=dict(valcol=timestamp)
+                )
+
+        try:
+            connection.execute(upsert)
+        except Exception as e:
+            logger.error("Insert/Update failed on Volgistics stats")
+            logger.error(e)
+
+
+def log_shelterluv_update():
+    """Log Shelterluv data update"""
+
+    timestamp =  datetime.now().ctime()
+
+    with engine.connect() as connection:
+        ins_stmt = insert(kvt).values(               
+            keycol = 'last_shelterluv_update',
+            valcol = timestamp,
+            )
+        # If key already present in DB, do update instead
+        upsert = ins_stmt.on_conflict_do_update(
+                constraint='kv_unique_keycol_key',
+                set_=dict(valcol=timestamp)
+                )
+
+        try:
+            connection.execute(upsert)
+        except Exception as e:
+            logger.error("Insert/Update failed on Shelterluv stats")
+            logger.error(e)
+
+
+def log_salesforce_update():
+    """Log SalesForce data update"""
+
+    timestamp = datetime.now().ctime()
+
+    with engine.connect() as connection:
+        ins_stmt = insert(kvt).values(               
+            keycol = 'last_salesforce_update',
+            valcol = timestamp,
+            )
+        # If key already present in DB, do update instead
+        upsert = ins_stmt.on_conflict_do_update(
+                constraint='kv_unique_keycol_key',
+                set_=dict(valcol=timestamp)
+                )
+
+        try:
+            connection.execute(upsert)
+        except Exception as e:
+            logger.error("Insert/Update failed on SalseForce stats")
+            logger.error(e)