Merge branch '494-logging' of github.com:CodeForPhilly/paws-data-pipeline into 494-logging

Author: c-simpson

.gitignore

@@ -15,7 +15,6 @@ src/server/api/__pycache__
 /.idea
 start_env.sh
 *.DS_Store
-
 /src/server/venv/
 /src/local_files/
 /src/server/secrets_dict.py

src/server/Dockerfile

@@ -23,7 +23,7 @@ RUN chmod 666 /usr/lib/uwsgi/plugins/python38_plugin.so
 
 COPY . .
 
-RUN mkdir /app/static \
+RUN mkdir -p /app/static \
     /app/static/raw_data \
     /app/static/logs \
     /app/static/zipped

src/server/api/API_ingest/updated_data.py

@@ -0,0 +1,80 @@
+
+from sqlalchemy.orm import  sessionmaker
+from simple_salesforce import Salesforce
+from config import engine
+
+import structlog
+logger = structlog.get_logger()
+
+
+def get_updated_contact_data():
+    Session = sessionmaker(engine)
+
+    qry = """ -- Collect latest foster/volunteer dates
+        with ev_dates as 
+        (select 
+            person_id, 
+            max(case when event_type=1 then time else null end) adopt,
+            max(case when event_type=2 then time else null end) foster_out,
+           --  max(case when event_type=3 then time else null end) rto,
+            max(case when event_type=5 then time else null end) foster_return
+           
+        from 
+            sl_animal_events sla
+            left join sl_event_types sle on sle.id = sla.event_type
+            
+        where sle.id in (1,2,5)
+        group by person_id
+        order by person_id
+        )
+  
+
+    select json_agg (upd)  as "cd" from (
+            select
+            slsf.source_id as "contactId" ,  -- long salesforce string
+            slp.id   as  "personId" ,           -- short PAWS-local shelterluv id
+        
+            case
+                when 
+                    (extract(epoch from now())::bigint - foster_out < 365*86400)  -- foster out in last year 
+                    or (extract(epoch from now())::bigint - foster_return < 365*86400) -- foster return 
+                then 'Active'
+                else 'Inactive'
+            end  as "updatedFosterStatus" ,
+        
+            (to_timestamp(foster_out ) at time zone  'America/New_York')::date   as "updatedFosterStartDate",
+            (to_timestamp(foster_return ) at time zone 'America/New_York')::date  as "updatedFosterEndDate",
+
+            min(vs.from_date) as "updatedFirstVolunteerDate",
+            max(vs.from_date) as "updatedLastVolunteerDate",
+            vc.source_id as   "volgisticsId"     
+
+            
+        from 
+            ev_dates
+            left join pdp_contacts slc on slc.source_id = person_id::text and slc.source_type = 'shelterluvpeople'
+            left join pdp_contacts slsf on slsf.matching_id = slc.matching_id and slsf.source_type = 'salesforcecontacts'
+            left join shelterluvpeople slp on slp.internal_id = person_id::text
+            left join pdp_contacts vc on vc.matching_id = slc.matching_id and vc.source_type = 'volgistics'
+            left join volgisticsshifts vs on vs.volg_id::text = vc.source_id 
+
+    where 
+        slsf.source_id is not null
+
+        group by
+            slsf.source_id,
+            slp.id,
+            vc.source_id,
+            foster_out ,
+            foster_return
+        
+    ) upd ;
+
+
+    """
+
+    with Session() as session:
+        result = session.execute(qry)
+        sfdata = result.fetchone()[0]
+        logger.debug("Query for Salesforce update returned %d records", len(sfdata))
+        return sfdata

src/server/api/internal_api.py

@@ -3,9 +3,10 @@
 import structlog
 from flask import jsonify
 
-from api.API_ingest import ingest_sources_from_api
+from api.API_ingest import ingest_sources_from_api, salesforce_contacts
 from api.api import internal_api
 from rfm_funcs.create_scores import create_scores
+from api.API_ingest import updated_data
 
 logger = structlog.get_logger()
 
@@ -42,3 +43,11 @@ def hit_create_scores():
     tuple_count = create_scores()
     logger.info("create_scores()  processed %s scores",  str(tuple_count) )
     return jsonify(200)
+
+
+@internal_api.route("/api/internal/get_updated_data", methods=["GET"])
+def get_contact_data():
+    logger.debug("Calling  get_updated_contact_data()")
+    contact_json =  updated_data.get_updated_contact_data()
+    logger.debug("Returning %d contact records",  len(contact_json) )
+    return jsonify(contact_json), 200

src/server/bin/startServer.sh

@@ -11,7 +11,4 @@ echo "SLEEPING.. WAITING FOR DB"; sleep 5; echo "WAKING"; alembic upgrade head;
 
 # --no-reload prevents Flask restart, which usually happens in middle of create_base_users()
 #TODO: SECURITY - ensure we are not running in debug mode in production
-UWSGI_LOG_CONFIG="bin/uwsgi_log_config.json"
-# Remove newlines and extra whitespace from log formatting
-LOG_FORMAT=$(cat $UWSGI_LOG_CONFIG | tr -d \\n | tr -s ' ')
-uwsgi --logformat "$LOG_FORMAT" --http-socket :5000 --plugin python38 --module wsgi:app --chdir /app --pythonpath . --processes 2 --threads 4 --master
+uwsgi bin/uwsgi.ini

src/server/bin/uwsgi.ini

@@ -0,0 +1,14 @@
+[uwsgi]
+http-socket = :5000
+plugin = python38
+module = wsgi:app
+chdir = /app
+pythonpath = .
+processes = 2
+threads = 4
+log-4xx = true
+log-5xx = true
+disable-logging = true
+logdate = %%Y-%%m-%%d-T%%H:%%M:%%SZ
+logformat = {"timestamp": "%(ftime)", "address": "%(addr)", "method": "%(method)", "protocol": "%(proto)", "resp_size": "%(size)", "request_body_size": "%(cl)", "response_status": "%(status)", "response_time": "%(secs)", "uri": "%(uri)"}
+logformat-strftime = true