delimited names, lowercase emails

Author: carlos-dominguez

src/server/alembic/versions/45a668fa6325_postgres_matching.py

@@ -85,19 +85,19 @@ def upgrade():
     sa.PrimaryKeyConstraint('_id')
     )
     op.create_index('idx_pdp_contacts_source_type_and_id', 'pdp_contacts', ['source_type', 'source_id'], unique=False)
-    op.create_index(op.f('ix_pdp_contacts_email'), 'pdp_contacts', ['email'], unique=False)
     op.create_index(op.f('ix_pdp_contacts_mobile'), 'pdp_contacts', ['mobile'], unique=False)
     op.create_index(op.f('idx_pdp_contacts_lower_first_name'), 'pdp_contacts', [sa.text('lower(first_name)')], unique=False)
     op.create_index(op.f('idx_pdp_contacts_lower_last_name'), 'pdp_contacts', [sa.text('lower(last_name)')], unique=False)
+    op.create_index(op.f('idx_pdp_contacts_lower_email'), 'pdp_contacts', [sa.text('lower(email)')], unique=False)
     # ### end Alembic commands ###
 
 
 def downgrade():
     # ### commands auto generated by Alembic - please adjust! ###
+    op.drop_index(op.f('ix_pdp_contacts_lower_email'), table_name='pdp_contacts')
     op.drop_index(op.f('ix_pdp_contacts_lower_last_name'), table_name='pdp_contacts')
     op.drop_index(op.f('ix_pdp_contacts_lower_first_name'), table_name='pdp_contacts')
     op.drop_index(op.f('ix_pdp_contacts_mobile'), table_name='pdp_contacts')
-    op.drop_index(op.f('ix_pdp_contacts_email'), table_name='pdp_contacts')
     op.drop_index('idx_pdp_contacts_source_type_and_id', table_name='pdp_contacts')
     op.drop_table('volgistics')
     op.drop_table('shelterluvpeople')

src/server/models.py

@@ -113,6 +113,7 @@ class PdpContacts(Base):
     __table_args__ = (
         Index("idx_pdp_contacts_lower_first_name", text("lower(first_name)")),
         Index("idx_pdp_contacts_lower_last_name", text("lower(last_name)")),
+        Index("idx_pdp_contacts_lower_email", text("lower(email)")),
         Index("idx_pdp_contacts_source_type_and_id", "source_type", "source_id"),
     )
 

src/server/pipeline/flow_script.py

@@ -23,6 +23,7 @@
     insert,
     or_,
     select,
+    text,
     update,
 )
 
@@ -112,6 +113,9 @@ def reset_pdp_contacts_with_unmatched(conn):
     conn.execute(Volgistics.insert_into_pdp_contacts())
     conn.execute(ShelterluvPeople.insert_into_pdp_contacts())
 
+def compare_names(n1, n2):
+    delims = text("'( and | & |, | )'")
+    return func.regexp_split_to_array(func.lower(n1), delims).bool_op("&&")(func.regexp_split_to_array(func.lower(n2), delims))
 
 def get_automatic_matches(conn):
     pc1 = PdpContacts.__table__.alias()
@@ -121,16 +125,16 @@ def get_automatic_matches(conn):
         and_(
             or_(
                 and_(
-                    func.lower(pc1.c.first_name) == func.lower(pc2.c.first_name),
-                    func.lower(pc1.c.last_name) == func.lower(pc2.c.last_name),
+                    compare_names(pc1.c.first_name, pc2.c.first_name),
+                    compare_names(pc1.c.last_name, pc2.c.last_name),
                 ),
                 and_(
-                    func.lower(pc1.c.first_name) == func.lower(pc2.c.last_name),
-                    func.lower(pc1.c.last_name) == func.lower(pc2.c.first_name),
+                    compare_names(pc1.c.first_name, pc2.c.last_name),
+                    compare_names(pc1.c.last_name, pc2.c.first_name),
                 ),
             ),
             or_(
-                pc1.c.email == pc2.c.email,
+                func.lower(pc1.c.email) == func.lower(pc2.c.email),
                 pc1.c.mobile == pc2.c.mobile,
             ),
             # This ensures we don't get e.g. every row matching itself