First pass at SL events pull

Author: cris_gk

src/server/alembic/versions/9687db7928ee_shelterluv_animals.py

@@ -1,4 +1,4 @@
-"""empty message
+"""Create SL_animals table
 
 Revision ID: 9687db7928ee
 Revises: a3ba63dee8f4
@@ -11,7 +11,7 @@
 
 # revision identifiers, used by Alembic.
 revision = '9687db7928ee'
-down_revision = 'a3ba63dee8f4'
+down_revision = 'fc7325372396'
 branch_labels = None
 depends_on = None
 

src/server/api/API_ingest/shelterluv_db.py

@@ -51,17 +51,88 @@ def insert_animals(animal_list):
 def truncate_animals():
     """Truncate the shelterluv_animals table"""
 
-
-    Session = sessionmaker(engine) 
-    session =  Session()   
+    Session = sessionmaker(engine)
+    session = Session()
     metadata = MetaData()
     sla = Table("shelterluv_animals", metadata, autoload=True, autoload_with=engine)
 
-
     truncate = "TRUNCATE table shelterluv_animals;"
     result = session.execute(truncate)
 
-    session.commit()   # Commit all inserted rows
+    session.commit()  # Commit all inserted rows
+    session.close()
+
+    return 0
+
+
+def truncate_events():
+    """Truncate the shelterluv_events table"""
+
+    Session = sessionmaker(engine)
+    session = Session()
+    metadata = MetaData()
+    sla = Table("sl_animal_events", metadata, autoload=True, autoload_with=engine)
+
+    truncate = "TRUNCATE table sl_animal_events;"
+    result = session.execute(truncate)
+
+    session.commit()  # Commit all inserted rows
     session.close()
 
     return 0
+
+
+def insert_events(event_list):
+    """Insert event records into sl_animal_events table and return row count. """
+
+    # Always a clean insert
+    truncate_events()
+
+    Session = sessionmaker(engine)
+    session = Session()
+    metadata = MetaData()
+    sla = Table("sl_animal_events", metadata, autoload=True, autoload_with=engine)
+
+    # TODO: Pull from DB
+    event_map = {
+        "Outcome.Adoption": 1,
+        "Outcome.Foster": 2,
+        "Outcome.ReturnToOwner": 3,
+        "Intake.AdoptionReturn": 4,
+    }
+
+    # Event record:    [ AssociatedRecords[Type = Person]["Id"]',
+    #                    AssociatedRecords[Type = Animal]["Id"]',
+    #                     "Type",
+    #                     "Time"
+    #                     ]
+    #
+    #  In db:           ['id',
+    #                     'person_id',
+    #                     'animal_id',
+    #                     'event_type',
+    #                     'time']
+
+    ins_list = []  # Create a list of per-row dicts
+    for rec in event_list:
+        ins_list.append(
+            {
+                "person_id": next(
+                    filter(lambda x: x["Type"] == "Person", rec["AssociatedRecords"])
+                )["Id"],
+                "animal_id": next(
+                    filter(lambda x: x["Type"] == "Animal", rec["AssociatedRecords"])
+                )["Id"],
+                "event_type": event_map[rec["Type"]],
+                "time": rec["Time"],
+            }
+        )
+
+    # TODO: Wrap with try/catch
+    ret = session.execute(sla.insert(ins_list))
+
+    session.commit()  # Commit all inserted rows
+    session.close()
+
+    return ret.rowcount
+

src/server/api/API_ingest/sl_animal_events.py

@@ -0,0 +1,161 @@
+import os, time, json
+import posixpath as path
+
+import requests
+
+from api.API_ingest import shelterluv_db
+from server.api.API_ingest.shelterluv_db import insert_animals
+
+# There are a number of different record types. These are the ones we care about.
+keep_record_types = [
+    "Outcome.Adoption",
+    "Outcome.Foster",
+    "Outcome.ReturnToOwner",
+    "Intake.AdoptionReturn",
+]
+
+# from config import engine
+# from flask import current_app
+# from sqlalchemy.sql import text
+
+BASE_URL = "http://shelterluv.com/api/"
+MAX_COUNT = 100  # Max records the API will return for one call
+
+# Get the API key
+try:
+    from secrets_dict import SHELTERLUV_SECRET_TOKEN
+except ImportError:
+    # Not running locally
+    from os import environ
+
+    try:
+        SHELTERLUV_SECRET_TOKEN = environ["SHELTERLUV_SECRET_TOKEN"]
+    except KeyError:
+        # Not in environment
+        # You're SOL for now
+        print("Couldn't get SHELTERLUV_SECRET_TOKEN from file or environment")
+
+
+headers = {"Accept": "application/json", "X-API-Key": SHELTERLUV_SECRET_TOKEN}
+
+logger = print  # print to console for testing
+
+
+# Sample response from events request:
+
+# {
+#     "success": 1,
+#     "events": [
+#         {
+#             "Type": "Outcome.Adoption",
+#             "Subtype": "PAC",
+#             "Time": "1656536900",
+#             "User": "phlp_mxxxx",
+#             "AssociatedRecords": [
+#                 {
+#                     "Type": "Animal",
+#                     "Id": "5276xxxx"
+#                 },
+#                 {
+#                     "Type": "Person",
+#                     "Id": "5633xxxx"
+#                 }
+#             ]
+#         },
+#         {...}
+#     ],
+#     "has_more": true,
+#     "total_count": 67467
+# }
+
+
+def get_event_count():
+    """Test that server is operational and get total event count."""
+    events = "v1/events&offset=0&limit=1"
+    URL = path.join(BASE_URL, events)
+
+    try:
+        response = requests.request("GET", URL, headers=headers)
+    except Exception as e:
+        logger("get_event_count failed with ", e)
+        return -2
+
+    if response.status_code != 200:
+        logger("get_event_count ", response.status_code, "code")
+        return -3
+
+    try:
+        decoded = json.loads(response.text)
+    except json.decoder.JSONDecodeError as e:
+        logger("get_event_count JSON decode failed with", e)
+        return -4
+
+    if decoded["success"]:
+        return decoded["total_count"]
+    else:
+        return -5  # AFAICT, this means URL was bad
+
+
+def get_events_bulk():
+    """Pull all event records from SL """
+
+    # Interesting API design - event record 0 is the newest. But since we pull all records each time it doesn't
+    # really matter which direction we go. Simplest to count up, and we can pull until 'has_more' goes false.
+    # Good news, the API is robust and won't blow up if you request past the end.
+    # At 100 per request, API returns about 5000 records/minute
+
+    event_records = []
+
+    raw_url = path.join(BASE_URL, "v1/events&offset={0}&limit={1}")
+    offset = 0
+    limit = MAX_COUNT
+    more_records = True
+
+    while more_records:
+
+        url = raw_url.format(offset, limit)
+
+        try:
+            response = requests.request("GET", url, headers=headers)
+        except Exception as e:
+            logger("get_events failed with ", e)
+            return -2
+
+        if response.status_code != 200:
+            logger("get_event_count ", response.status_code, "code")
+            return -3
+
+        try:
+            decoded = json.loads(response.text)
+        except json.decoder.JSONDecodeError as e:
+            logger("get_event_count JSON decode failed with", e)
+            return -4
+
+        if decoded["success"]:
+            for evrec in decoded["events"]:
+                if evrec["Type"] in keep_record_types:
+                    event_records.append(evrec)
+
+            more_records = decoded["has_more"]  # if so, we'll make another pass
+            offset += limit
+            if offset % 1000 == 0:
+                print("Reading offset ", str(offset))
+
+        else:
+            return -5  # AFAICT, this means URL was bad
+
+    return event_records
+
+
+def slae_test():
+    total_count = get_event_count()
+    print("Total events:", total_count)
+
+    b = get_events_bulk()
+    print("Records:", len(b))
+
+    # f = filter_events(b)
+    # print(f)
+
+    count = shelterluv_db.insert_events(b)
+    return count

src/server/api/admin_api.py

@@ -388,18 +388,17 @@ def generate_dummy_rfm_scores():
     return count
 
 
+# ########### Test API endpoints   
+# TODO: Remove for production
 
-
-# Use this as a way to trigger functions for testing
-# TODO: Remove when not needed
+# trigger rfm scoring process
 @admin_api.route("/api/admin/test_endpoint_gdrs", methods=["GET"])
 def hit_gdrs():
     num_scores = generate_dummy_rfm_scores()
     return jsonify({"scores added" : num_scores})
 
 
-
-
+# trigger pull of SL animals
 @admin_api.route("/api/admin/test_sla", methods=["GET"])
 def trigger_sla_pull():
 
@@ -408,7 +407,7 @@ def trigger_sla_pull():
     num_rows = api.API_ingest.shelterluv_animals.sla_test()
     return jsonify({"rows added" : num_rows})
 
-
+# trigger pull of SL people
 @admin_api.route("/api/admin/test_slp", methods=["GET"])
 def trigger_slp_pull():
 
@@ -417,6 +416,16 @@ def trigger_slp_pull():
     num_rows = api.API_ingest.shelterluv_api_handler.store_shelterluv_people_all()
     return jsonify({"rows added" : num_rows})
 
+# trigger pull of SL animal events
+@admin_api.route("/api/admin/test_slae", methods=["GET"])
+def trigger_slae_pull():
+
+    import api.API_ingest.sl_animal_events
+
+    num_rows = api.API_ingest.sl_animal_events.slae_test()
+    return jsonify({"rows added" : num_rows})
+
+