Added/tweaked logging

Author: c-simpson

.gitignore

@@ -24,3 +24,5 @@ start_env.sh
 *secrets*
 *kustomization*
 src/.venv/
+src/server/secrets_dict.py
+

src/server/api/.optic/.gitignore

@@ -10,8 +10,8 @@ def start():
     logger.debug("   Finished fetching Salesforce contacts")
 
     logger.debug("   Fetching Shelterluv people")
-    shelterluv_people.store_shelterluv_people_all()
-    logger.debug("   Finished fetching Shelterluv people")
+    slp_count = shelterluv_people.store_shelterluv_people_all()
+    logger.debug("   Finished fetching Shelterluv people - %d records" , slp_count)
 
     logger.debug("   Fetching Shelterluv events")
     sle_count = sl_animal_events.store_all_animals_and_events()

src/server/api/API_ingest/ingest_sources_from_api.py

@@ -18,6 +18,8 @@ def store_contacts_all():
         logger.debug("retrieving the latest salesforce contacts data")
         sf = Salesforce(domain=os.getenv('SALESFORCE_DOMAIN'), password=os.getenv('SALESFORCE_PW'), username=os.getenv('SALESFORCE_USERNAME'), security_token=os.getenv('SALESFORCE_SECURITY_TOKEN'))
         results = sf.query("SELECT Contact_ID_18__c, FirstName, LastName, Contact.Account.Name, MailingCountry, MailingStreet, MailingCity, MailingState, MailingPostalCode, Phone, MobilePhone, Email FROM Contact")
+        logger.debug("Query returned %d Salesforce contact records", len(results['records']) )
+        
         done = False
         while not done:
             for row in results['records']:

src/server/api/API_ingest/salesforce_contacts.py

@@ -70,7 +70,10 @@ def store_shelterluv_people_all():
                                       animal_ids=person["Animal_ids"]))
             offset += LIMIT
             has_more = response["has_more"] if not TEST_MODE else response["has_more"] and offset < 1000
+            if offset % 1000 == 0:
+                logger.debug("Reading offset %s", str(offset))
         session.commit()
 
-    logger.debug("Finish getting shelterluv contacts from people table")
+    logger.debug("Finished getting shelterluv contacts from people table")
+    return offset
 

src/server/api/API_ingest/shelterluv_people.py

@@ -224,10 +224,10 @@ def start_job():
 
     if running_job :
         # There was a running job already
-        logger.warn("Request to start job, but job_id " + str(running_job) + " already executing")
+        logger.warn("Request to start job, but job_id %s already executing", str(running_job))
         return None
     else:
-        logger.info("Assigned job_id  %s" + str(job_id ) )
+        logger.info("Assigned job_id  %s" ,  str(job_id ) )
         return job_id
 
 

src/server/api/admin_api.py

@@ -67,22 +67,26 @@ def start_flow():
             # 6. Update each row in pdp_contacts to give it a match id
             #    corresponding to its connected componenet.
 
-            logger.info("Clearing pdp_contacts to prepare for match")
+            logger.debug("Clearing pdp_contacts to prepare for match")
             reset_pdp_contacts_with_unmatched(conn)
 
-            logger.info("Computing matches")
+            logger.debug("Computing automatic matches")
             automatic_matches = get_automatic_matches(conn)
+            logger.debug("Computing manual matches")
             manual_matches = get_manual_matches(conn)
 
             match_graph = Graph()
+            logger.debug("Adding automatic matches to graph")
             match_graph.add_edges_from(automatic_matches)
+            logger.debug("Adding manual matches to graph")
             match_graph.add_edges_from(manual_matches)
+            logger.debug("Processing graph")
             match_groups = connected_components(match_graph)
 
-            logger.info("Updating pdp_contacts with match ids")
+            logger.debug("Updating pdp_contacts with match ids")
             update_matching_ids(match_groups, conn)
 
-            logger.info("Finished flow script run")
+            logger.debug("Finished flow script run")
             job_outcome = "completed"
             log_db.log_exec_status(job_id, "flow", "complete", "")
 

src/server/pipeline/flow_script.py

@@ -130,7 +130,7 @@ def validate_import_vs(filename, conn):
         # parent will commit. Don't commit here!
 
 
-        logger.info("Total rows: %s  Dupes: %s Missing volgistics id: ",  str(row_count), str(dupes), str(missing_volgistics_id)  )
+        logger.info("Total rows: %s  Dupes: %s Missing volgistics id: %s",  str(row_count), str(dupes), str(missing_volgistics_id)  )
         logger.info("Other integrity exceptions: %s  Other exceptions: %s",  str(other_exceptions),  str(other_integrity) )
         wb.close()
         return { True : "File imported" }