Adjust ingest endpoint (internal)

Add TEST_MODE check on import

Author: c-simpson

.gitignore

@@ -23,3 +23,4 @@ start_env.sh
 .mypy_cache/
 *secrets*
 *kustomization*
+src/.venv/

src/server/api/API_ingest/ingest_sources_from_api.py

@@ -1,7 +1,19 @@
-from api.API_ingest import shelterluv_api_handler
+from api.API_ingest import shelterluv_api_handler, sl_animal_events
 
 def start(conn):
-    print("Start Fetching raw data from different API sources")
+    print("Start fetching raw data from different API sources")
+
+    print("   Fetching Shelterluv people")
+    #Run each source to store the output in dropbox and in the container as a CSV
+    slp_count = shelterluv_api_handler.store_shelterluv_people_all(conn)
+    print("   Finished fetching Shelterluv people - %d records" % slp_count)
+
+    print("   Fetching Shelterluv events")
     #Run each source to store the output in dropbox and in the container as a CSV
-    shelterluv_api_handler.store_shelterluv_people_all(conn)
-    print("Finish Fetching raw data from different API sources")
+    sle_count = sl_animal_events.slae_test()
+    print("   Finished fetching Shelterluv events - %d records" % sle_count)
+
+    print("Finished fetching raw data from different API sources")
+
+
+    #TODO:   Return object with count for each data source?

src/server/api/API_ingest/shelterluv_api_handler.py

@@ -8,6 +8,9 @@
 from constants import RAW_DATA_PATH
 from models import ShelterluvPeople
 
+
+TEST_MODE = os.getenv("TEST_MODE")
+
 try:
     from secrets_dict import SHELTERLUV_SECRET_TOKEN
 except ImportError:
@@ -78,6 +81,13 @@ def store_shelterluv_people_all(conn):
         has_more = response["has_more"]
         offset += 100
 
+        if offset % 1000 == 0:
+            print("Reading offset ", str(offset))
+            if TEST_MODE and offset > 1000:
+                has_more=False  # Break out early 
+
+
+
     print("Finish getting shelterluv contacts from people table")
 
     print("Start storing latest shelterluvpeople results to container")
@@ -98,3 +108,5 @@ def store_shelterluv_people_all(conn):
 
     print("Uploading shelterluvpeople csv to database")
     ShelterluvPeople.insert_from_df(pd.read_csv(file_path, dtype="string"), conn)
+
+    return offset

src/server/api/API_ingest/sl_animal_events.py

@@ -38,6 +38,8 @@
         print("Couldn't get SHELTERLUV_SECRET_TOKEN from file or environment")
 
 
+TEST_MODE=os.getenv("TEST_MODE")  # if not present, has value None
+
 headers = {"Accept": "application/json", "X-API-Key": SHELTERLUV_SECRET_TOKEN}
 
 logger = print  # print to console for testing
@@ -143,6 +145,8 @@ def get_events_bulk():
             offset += limit
             if offset % 1000 == 0:
                 print("Reading offset ", str(offset))
+                if TEST_MODE and offset > 1000:
+                    more_records=False  # Break out early 
 
         else:
             return -5  # AFAICT, this means URL was bad
@@ -155,7 +159,7 @@ def slae_test():
     print("Total events:", total_count)
 
     b = get_events_bulk()
-    print("Records:", len(b))
+    print("Strored records:", len(b))
 
     # f = filter_events(b)
     # print(f)

src/server/api/internal_api.py

@@ -22,7 +22,7 @@ def user_test2():
     return jsonify(("OK from INTERNAL test/test  @ " + str(datetime.now())))
 
 
-@internal_api.route("/api/ingestRawData", methods=["GET"])
+@internal_api.route("/api/internal/ingestRawData", methods=["GET"])
 def ingest_raw_data():
     try:
         with engine.begin() as conn: