code for downloading salesforce data with test endpoint in the admin api

Author: sposerina

src/server/api/API_ingest/salesforce_api_handler.py

@@ -0,0 +1,58 @@
+import os
+
+from sqlalchemy.orm import  sessionmaker
+from simple_salesforce import Salesforce
+from config import engine
+from models import SalesForceContacts, SalesforceDonations
+
+def ingest_data():
+
+    Session = sessionmaker(engine)
+
+    session = Session()
+    session.execute("TRUNCATE TABLE salesforcecontacts")
+
+    sf = Salesforce(domain=os.getenv('SALESFORCE_DOMAIN'), password=os.getenv('SALESFORCE_PW'), username=os.getenv('SALESFORCE_USERNAME'), organizationId=os.getenv('SALESFORCE_ORGANIZATION_ID'), security_token=os.getenv('SALESFORCE_SECURITY_TOKEN'))
+    results = sf.query("SELECT Contact_ID_18__c, FirstName, LastName, Contact.Account.Name, MailingCountry, MailingStreet, MailingCity, MailingState, MailingPostalCode, Phone, MobilePhone, Email FROM Contact")
+    done = False
+    while not done:
+        for row in results['records']:
+            account_name = row['Account']['Name'] if row['Account'] is not None else None
+            contact = SalesForceContacts(contact_id=row['Contact_ID_18__c'],
+                                         first_name=row['FirstName'],
+                                         last_name=row['LastName'],
+                                         account_name=account_name,
+                                         mailing_country=row['MailingCountry'],
+                                         mailing_street=row['MailingStreet'],
+                                         mailing_city=row['MailingCity'],
+                                         mailing_state_province=row['MailingState'],
+                                         mailing_zip_postal_code=row['MailingPostalCode'],
+                                         phone=row['Phone'],
+                                         mobile=row['MobilePhone'],
+                                         email=['Email'])
+            session.add(contact)
+            done = results['done']
+            if not done:
+                results = sf.query_more(results['nextRecordsUrl'])
+
+
+
+    session.execute("TRUNCATE TABLE salesforcedonations")
+    results = sf.query("SELECT Opportunity_ID_18__c, npe03__Recurring_Donation__c, Opportunity.Account.Name, Contact_ID_18__c, Amount, CloseDate, Type, Campaign.Name FROM Opportunity")
+    done = False
+    while not done:
+        for row in results['records']:
+            account_name = row['Account']['Name'] if row['Account'] is not None else None
+            donation = SalesforceDonations(opp_id=row['Opportunity_ID_18__c'],
+                                           recurring_donor=False if row['npe03__Recurring_Donation__c'] is None else True,
+                                           primary_contact= account_name,
+                                           contact_id=row['Contact_ID_18__c'],
+                                           amount=row['Amount'],
+                                           close_date=row['CloseDate'],
+                                           donation_type=row['Type'])
+            session.add(donation)
+        done = results['done']
+        if not done:
+            results = sf.query_more(results['nextRecordsUrl'])
+
+    session.commit()

src/server/api/admin_api.py

@@ -15,6 +15,7 @@
 
 from api import jwt_ops
 from config import RAW_DATA_PATH
+from api.API_ingest.salesforce_api_handler import ingest_data
 
 ALLOWED_EXTENSIONS = {"csv", "xlsx"}
 
@@ -397,6 +398,12 @@ def hit_gdrs():
     num_scores = generate_dummy_rfm_scores()
     return jsonify({"scores added" : num_scores})
 
+@admin_api.route("/api/admin/test-salesforce-ingest")
+def ingest_salesforce():
+    current_app.logger.info("Getting latest Salesforce data")
+    ingest_data()
+    return jsonify(200)
+
 
 # def pdfr():
 #     dlist = pull_donations_for_rfm()

src/server/bin/export_secrets.sh

@@ -1,3 +1,3 @@
 set -o allexport
-source secrets_dict.py
+source bin/secrets_dict.py
 set +o allexport

src/server/models.py

@@ -379,3 +379,17 @@ def insert_from_df(cls, df, conn):
                 )
 
         conn.execute(insert(cls).values(matched_pairs).on_conflict_do_nothing())
+
+class SalesforceDonations(Base):
+    __tablename__ = "salesforcedonations"
+
+    _id = sa.Column(sa.Integer, primary_key=True)
+    opp_id = sa.Column(sa.String)
+    recurring_donor = sa.Column(sa.Boolean)
+    primary_contact = sa.Column(sa.String)
+    contact_id = sa.Column(sa.String)
+    amount = sa.Column(sa.Numeric)
+    close_date = sa.Column(sa.Date)
+    donation_type = sa.Column(sa.String)
+    primary_campaign_source = sa.Column(sa.String)
+

src/server/requirements.txt

@@ -15,4 +15,5 @@ jellyfish
 networkx
 jinja2<3.1.0
 itsdangerous==2.0.1 # TODO: upgrade flask so we don't have to do this
+simple-salesforce
 werkzeug==2.0.3