test endpoint and DB inserts

Author: c-simpson

src/server/api/API_ingest/shelterluv_animals.py

@@ -0,0 +1,151 @@
+import os, time, json
+import posixpath as path
+
+import requests
+
+from api.API_ingest import shelterluv_db
+
+
+# from config import engine
+# from flask import current_app
+# from sqlalchemy.sql import text
+
+BASE_URL = 'http://shelterluv.com/api/'
+
+
+try:
+    from secrets_dict import SHELTERLUV_SECRET_TOKEN
+except ImportError:
+    # Not running locally
+    from os import environ
+
+    try:
+        SHELTERLUV_SECRET_TOKEN = environ['SHELTERLUV_SECRET_TOKEN']
+    except KeyError:
+        # Not in environment
+        # You're SOL for now
+        print("Couldn't get SHELTERLUV_SECRET_TOKEN from file or environment")
+
+
+
+headers = {
+    "Accept": "application/json",
+    "X-API-Key": SHELTERLUV_SECRET_TOKEN
+}
+
+logger = print
+
+def get_animal_count():
+    """Test that server is operational and get total animal count."""
+    animals = 'v1/animals&offset=0&limit=1'
+    URL = path.join(BASE_URL,animals)
+
+    try:
+        response = requests.request("GET",URL, headers=headers)
+    except Exception as e:
+        logger('get_animal_count failed with ', e)
+        return -2
+
+    if response.status_code != 200:
+        logger("get_animal_count ", response.status_code, "code")
+        return -3
+
+    try:
+        decoded = json.loads(response.text)
+    except json.decoder.JSONDecodeError as e:
+        logger("get_animal_count JSON decode failed with", e)
+        return -4
+
+    if decoded['success']:
+        return decoded['total_count']
+    else:
+        return -5   # AFAICT, this means URL was bad
+
+
+def filter_animals(raw_list):
+    """Given a list of animal records as returned by SL, return a list of records with only the fields we care about."""
+
+    good_keys = ['ID', 'Internal-ID', 'Name', 'Type', 'DOBUnixTime', 'CoverPhoto','LastUpdatedUnixTime']
+
+    filtered = []
+
+    for r in raw_list:
+        f = {}
+        for k in good_keys:
+            f[k] = r[k]
+        filtered.append(f)
+
+    return filtered
+
+
+
+
+def get_animals_bulk(total_count):
+    """Pull all animal records from SL """
+
+    MAX_COUNT = 100  # Max records the API will return for one call
+
+    # 'Great' API design - animal record 0 is the newest, so we need to start at the end, 
+    # back up MAX_COUNT rows, make our request, then keep backing up. We need to keep checking
+    # the total records to ensure one wasn't added in the middle of the process.
+    # Good news, the API is robust and won't blow up if you request past the end.
+
+    raw_url = path.join(BASE_URL, 'v1/animals&offset={0}&limit={1}')
+
+    start_record = total_count -1
+    offset = (start_record - MAX_COUNT)  if  (start_record - MAX_COUNT) > -1  else 0
+
+    while offset > -1 :
+
+        url = raw_url.format(offset,MAX_COUNT)
+
+        try:
+            response = requests.request("GET",url, headers=headers)
+        except Exception as e:
+            logger('get_animals failed with ', e)
+            return -2
+
+        if response.status_code != 200:
+            logger("get_animal_count ", response.status_code, "code")
+            return -3
+
+        try:
+            decoded = json.loads(response.text)
+        except json.decoder.JSONDecodeError as e:
+            logger("get_animal_count JSON decode failed with", e)
+            return -4
+
+        if decoded['success']:
+            return decoded['animals']
+        else:
+            return -5   # AFAICT, this means URL was bad
+
+
+
+
+def sla_test():
+    total_count = get_animal_count()
+    print('Total animals:',total_count)
+
+    b = get_animals_bulk(200)
+    print(len(b))
+
+    f = filter_animals(b)
+    print(f)
+
+    count = shelterluv_db.insert_animals(f)
+    return count
+
+
+# if __name__ == '__main__' :    
+
+#     total_count = get_animal_count()
+#     print('Total animals:',total_count)
+
+#     b = get_animals_bulk(9)
+#     print(len(b))
+
+#     f = filter_animals(b)
+#     print(f)
+
+#     count = shelterluv_db.insert_animals(f)

src/server/api/API_ingest/shelterluv_db.py

@@ -0,0 +1,48 @@
+from api.api import common_api
+from config import engine
+from flask import jsonify, current_app
+from sqlalchemy.sql import text
+import requests
+import time
+from datetime import datetime
+
+from sqlalchemy.dialects.postgresql import insert
+from sqlalchemy import Table, MetaData
+from pipeline import flow_script
+from config import engine
+from flask import request, redirect, jsonify, current_app
+from api.file_uploader import validate_and_arrange_upload
+from sqlalchemy.orm import Session, sessionmaker
+
+
+def insert_animals(animal_list):
+    """Insert animal records into shelterluv_animals table and return row count. """
+
+    Session = sessionmaker(engine)
+    session = Session()
+    metadata = MetaData()
+    sla = Table("shelterluv_animals", metadata, autoload=True, autoload_with=engine)
+
+    # From Shelterluv: ['ID',       'Internal-ID', 'Name', 'Type', 'DOBUnixTime', 'CoverPhoto', 'LastUpdatedUnixTime']
+    # In db:           ['local_id', 'id' (PK),     'name', 'type', 'dob',         'photo',      'updatestamp']
+
+    ins_list = []  # Create a list of per-row dicts
+    for rec in animal_list:
+        ins_list.append(
+            {
+                "id": rec["Internal-ID"],
+                "local_id": rec["ID"] if rec["ID"] else 0,  # Sometimes there's no local id
+                "name": rec["Name"],
+                "type": rec["Type"],
+                "dob": rec["DOBUnixTime"],
+                "updatestamp": rec["LastUpdatedUnixTime"],
+                "photo": rec["CoverPhoto"],
+            }
+        )
+
+    ret = session.execute(sla.insert(ins_list))
+
+    session.commit()  # Commit all inserted rows
+    session.close()
+
+    return ret.rowcount

src/server/api/admin_api.py

@@ -398,6 +398,17 @@ def hit_gdrs():
     return jsonify({"scores added" : num_scores})
 
 
+
+
+@admin_api.route("/api/admin/test_sla", methods=["GET"])
+def trigger_sla_pull():
+
+    import api.API_ingest.shelterluv_animals
+
+    num_rows = api.API_ingest.shelterluv_animals.sla_test()
+    return jsonify({"rows added" : num_rows})
+
+
 # def pdfr():
 #     dlist = pull_donations_for_rfm()
 #     print("Returned " + str(len(dlist)) + " rows")