diff --git a/apps/assisted_tagging/tests/test_query.py b/apps/assisted_tagging/tests/test_query.py index e00711a0a6..debe151b2a 100644 --- a/apps/assisted_tagging/tests/test_query.py +++ b/apps/assisted_tagging/tests/test_query.py @@ -1,19 +1,11 @@ -from unittest.mock import patch -from snapshottest.django import TestCase as SnapShotTextCase - from utils.graphene.tests import GraphQLTestCase -from deep.tests import TestCase from assisted_tagging.models import ( AssistedTaggingPrediction, ) -from deepl_integration.handlers import AssistedTaggingDraftEntryHandler -from assisted_tagging.tasks import sync_tags_with_deepl from assisted_tagging.models import ( - AssistedTaggingModel, AssistedTaggingModelVersion, - AssistedTaggingModelPredictionTag, DraftEntry, ) @@ -28,7 +20,6 @@ AssistedTaggingModelVersionFactory, DraftEntryFactory, AssistedTaggingPredictionFactory, - MissingPredictionReviewFactory, ) @@ -179,1095 +170,6 @@ def _query_check(**kwargs): )) -class AssistedTaggingCallbackApiTest(TestCase, SnapShotTextCase): - factories_used = [ - UserFactory, - ProjectFactory, - LeadFactory, - AssistedTaggingModelFactory, - AssistedTaggingModelPredictionTagFactory, - AssistedTaggingModelVersionFactory, - DraftEntryFactory, - AssistedTaggingPredictionFactory, - MissingPredictionReviewFactory, - ] - - DEEPL_CALLBACK_MOCK_DATA = { - "client_id": "random-client-id", - "model_tags": { - "1": { - "101": { - "prediction": 0.002, - "threshold": 0.14, - "is_selected": False - }, - "102": { - "prediction": 0.648, - "threshold": 0.17, - "is_selected": True - }, - "103": { - "prediction": 0.027, - "threshold": 0.1, - "is_selected": False - }, - "104": { - "prediction": 0.062, - "threshold": 0.14, - "is_selected": False - } - }, - "3": { - "301": { - "prediction": 0.001, - "threshold": 0.01, - "is_selected": False - }, - "302": { - "prediction": 0.001, - "threshold": 0.11, - "is_selected": False - }, - "303": { - "prediction": 0.083, - "threshold": 0.38, - "is_selected": False - }, - "304": { - "prediction": 0.086, - "threshold": 0.01, - "is_selected": True - }, - "315": { - "prediction": 0.003, - "threshold": 0.45, - "is_selected": False - }, - "316": { - "prediction": 0.001, - "threshold": 0.06, - "is_selected": False - }, - "317": { - "prediction": 0.004, - "threshold": 0.28, - "is_selected": False - }, - "318": { - "prediction": 0.0, - "threshold": 0.13, - "is_selected": False - } - }, - "2": { - "219": { - "prediction": 0.003, - "threshold": 0.13, - "is_selected": False - }, - "217": { - "prediction": 0.001, - "threshold": 0.04, - "is_selected": False - }, - "218": { - "prediction": 0.004, - "threshold": 0.09, - "is_selected": False - }, - "204": { - "prediction": 0.007, - "threshold": 0.14, - "is_selected": False - }, - "216": { - "prediction": 0.003, - "threshold": 0.13, - "is_selected": False - }, - "214": { - "prediction": 0.001, - "threshold": 0.09, - "is_selected": False - }, - "209": { - "prediction": 0.458, - "threshold": 0.05, - "is_selected": True - } - }, - "6": { - "601": { - "prediction": 0.0, - "threshold": 0.06, - "is_selected": False - }, - "602": { - "prediction": 0.001, - "threshold": 0.48, - "is_selected": False - }, - "603": { - "prediction": 0.022, - "threshold": 0.34, - "is_selected": False - }, - "604": { - "prediction": 0.0, - "threshold": 0.16, - "is_selected": False - } - }, - "5": { - "501": { - "prediction": 0.0, - "threshold": 0.45, - "is_selected": False - }, - "502": { - "prediction": 0.0, - "threshold": 0.48, - "is_selected": False - } - }, - "8": { - "801": { - "prediction": 0.0, - "threshold": 0.66, - "is_selected": False - }, - "802": { - "prediction": 0.0, - "threshold": 0.3, - "is_selected": False - } - }, - "4": { - "401": { - "prediction": 0.001, - "threshold": 0.29, - "is_selected": False - }, - "402": { - "prediction": 0.001, - "threshold": 0.45, - "is_selected": False - }, - "407": { - "prediction": 0.0, - "threshold": 0.07, - "is_selected": False - }, - "408": { - "prediction": 0.001, - "threshold": 0.11, - "is_selected": False - }, - "412": { - "prediction": 0.0, - "threshold": 0.36, - "is_selected": False - } - }, - "7": { - "701": { - "prediction": 0.008, - "threshold": 0.27, - "is_selected": False - } - }, - "9": { - "904": { - "prediction": -1, - "threshold": -1, - "is_selected": False - }, - "905": { - "prediction": -1, - "threshold": -1, - "is_selected": False - }, - "907": { - "prediction": -1, - "threshold": -1, - "is_selected": False - } - } - }, - "geolocations": [ - "New York" - ], - "model_info": { - "id": "all_tags_model", - "version": "1.0.0" - }, - "prediction_status": True - } - DEEPL_TAGS_MOCK_RESPONSE = { - '101': { - 'label': 'Agriculture', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, - '102': { - 'label': 'Cross', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, - '103': { - 'label': 'Education', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, - '104': { - 'label': 'Food Security', - 'group': 'Sectors', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '1', - }, - '201': { - 'label': 'Environment', - 'group': 'Context', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '202': { - 'label': 'Socio Cultural', - 'group': 'Context', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '203': { - 'label': 'Economy', - 'group': 'Context', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '204': { - 'label': 'Demography', - 'group': 'Context', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '205': { - 'label': 'Legal & Policy', - 'group': 'Context', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '206': { - 'label': 'Security & Stability', - 'group': 'Context', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '207': { - 'label': 'Politics', - 'group': 'Context', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '208': { - 'label': 'Type And Characteristics', - 'group': 'Shock/Event', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '209': { - 'label': 'Underlying/Aggravating Factors', - 'group': 'Shock/Event', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '210': { - 'label': 'Hazard & Threats', - 'group': 'Shock/Event', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '212': { - 'label': 'Type/Numbers/Movements', - 'group': 'Displacement', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '213': { - 'label': 'Push Factors', - 'group': 'Displacement', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '214': { - 'label': 'Pull Factors', - 'group': 'Displacement', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '215': { - 'label': 'Intentions', - 'group': 'Displacement', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '216': { - 'label': 'Local Integration', - 'group': 'Displacement', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '217': { - 'label': 'Injured', - 'group': 'Casualties', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '218': { - 'label': 'Missing', - 'group': 'Casualties', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '219': { - 'label': 'Dead', - 'group': 'Casualties', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '220': { - 'label': 'Relief To Population', - 'group': 'Humanitarian Access', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '221': { - 'label': 'Population To Relief', - 'group': 'Humanitarian Access', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '222': { - 'label': 'Physical Constraints', - 'group': 'Humanitarian Access', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '223': { - 'label': 'Number Of People Facing Humanitarian Access Constraints/Humanitarian Access Gaps', - 'group': 'Humanitarian Access', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '224': { - 'label': 'Communication Means And Preferences', - 'group': 'Information And Communication', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '225': { - 'label': 'Information Challenges And Barriers', - 'group': 'Information And Communication', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '226': { - 'label': 'Knowledge And Info Gaps (Pop)', - 'group': 'Information And Communication', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '227': { - 'label': 'Knowledge And Info Gaps (Hum)', - 'group': 'Information And Communication', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '228': { - 'label': 'Cases', - 'group': 'Covid-19', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '229': { - 'label': 'Contact Tracing', - 'group': 'Covid-19', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '230': { - 'label': 'Deaths', - 'group': 'Covid-19', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '231': { - 'label': 'Hospitalization & Care', - 'group': 'Covid-19', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '232': { - 'label': 'Restriction Measures', - 'group': 'Covid-19', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '233': { - 'label': 'Testing', - 'group': 'Covid-19', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '234': { - 'label': 'Vaccination', - 'group': 'Covid-19', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '2', - }, - '301': { - 'label': 'Number Of People At Risk', - 'group': 'At Risk', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '302': { - 'label': 'Risk And Vulnerabilities', - 'group': 'At Risk', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '303': { - 'label': 'International Response', - 'group': 'Capacities & Response', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '304': { - 'label': 'Local Response', - 'group': 'Capacities & Response', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '305': { - 'label': 'National Response', - 'group': 'Capacities & Response', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '306': { - 'label': 'Number Of People Reached/Response Gaps', - 'group': 'Capacities & Response', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '307': { - 'label': 'Coping Mechanisms', - 'group': 'Humanitarian Conditions', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '308': { - 'label': 'Living Standards', - 'group': 'Humanitarian Conditions', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '309': { - 'label': 'Number Of People In Need', - 'group': 'Humanitarian Conditions', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '310': { - 'label': 'Physical And Mental Well Being', - 'group': 'Humanitarian Conditions', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '311': { - 'label': 'Driver/Aggravating Factors', - 'group': 'Impact', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '312': { - 'label': 'Impact On People', - 'group': 'Impact', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '313': { - 'label': 'Impact On Systems, Services And Networks', - 'group': 'Impact', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '314': { - 'label': 'Number Of People Affected', - 'group': 'Impact', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '315': { - 'label': 'Expressed By Humanitarian Staff', - 'group': 'Priority Interventions', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '316': { - 'label': 'Expressed By Population', - 'group': 'Priority Interventions', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '317': { - 'label': 'Expressed By Humanitarian Staff', - 'group': 'Priority Needs', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '318': { - 'label': 'Expressed By Population', - 'group': 'Priority Needs', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '3', - }, - '401': { - 'label': 'Child Head of Household', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '402': { - 'label': 'Chronically Ill', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '403': { - 'label': 'Elderly Head of Household', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '404': { - 'label': 'Female Head of Household', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '405': { - 'label': 'GBV survivors', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '406': { - 'label': 'Indigenous people', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '407': { - 'label': 'LGBTQI+', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '408': { - 'label': 'Minorities', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '409': { - 'label': 'Persons with Disability', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '410': { - 'label': 'Pregnant or Lactating Women', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '411': { - 'label': 'Single Women (including Widows)', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '412': { - 'label': 'Unaccompanied or Separated Children', - 'group': 'Specific Needs Group', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '4', - }, - '901': { - 'label': 'Infants/Toddlers (<5 years old) ', - 'group': 'Demographic Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '9', - }, - '902': { - 'label': 'Female Children/Youth (5 to 17 years old)', - 'group': 'Demographic Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '9', - }, - '903': { - 'label': 'Male Children/Youth (5 to 17 years old)', - 'group': 'Demographic Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '9', - }, - '904': { - 'label': 'Female Adult (18 to 59 years old)', - 'group': 'Demographic Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '9', - }, - '905': { - 'label': 'Male Adult (18 to 59 years old)', - 'group': 'Demographic Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '9', - }, - '906': { - 'label': 'Female Older Persons (60+ years old)', - 'group': 'Demographic Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '9', - }, - '907': { - 'label': 'Male Older Persons (60+ years old)', - 'group': 'Demographic Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '9', - }, - '701': { - 'label': 'Critical', - 'group': 'Severity', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '7', - }, - '702': { - 'label': 'Major', - 'group': 'Severity', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '7', - }, - '703': { - 'label': 'Minor Problem', - 'group': 'Severity', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '7', - }, - '704': { - 'label': 'No problem', - 'group': 'Severity', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '7', - }, - '705': { - 'label': 'Of Concern', - 'group': 'Severity', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '7', - }, - '801': { - 'label': 'Asylum Seekers', - 'group': 'Affected Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '8', - }, - '802': { - 'label': 'Host', - 'group': 'Affected Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '8', - }, - '803': { - 'label': 'IDP', - 'group': 'Affected Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '8', - }, - '804': { - 'label': 'Migrants', - 'group': 'Affected Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '8', - }, - '805': { - 'label': 'Refugees', - 'group': 'Affected Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '8', - }, - '806': { - 'label': 'Returnees', - 'group': 'Affected Groups', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '8', - }, - '1001': { - 'label': 'Completely reliable', - 'group': 'Reliability', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '10', - }, - '1002': { - 'label': 'Usually reliable', - 'group': 'Reliability', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '10', - }, - '1003': { - 'label': 'Fairly Reliable', - 'group': 'Reliability', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '10', - }, - '1004': { - 'label': 'Unreliable', - 'group': 'Reliability', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '10', - }, - '501': { - 'label': 'Female', - 'group': 'Gender', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '5', - }, - '502': { - 'label': 'Male', - 'group': 'Gender', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '5', - }, - '601': { - 'label': 'Adult (18 to 59 years old)', - 'group': 'Age', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '6', - }, - '602': { - 'label': 'Children/Youth (5 to 17 years old)', - 'group': 'Age', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '6', - }, - '603': { - 'label': 'Infants/Toddlers (<5 years old)', - 'group': 'Age', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '6', - }, - '604': { - 'label': 'Older Persons (60+ years old)', - 'group': 'Age', - 'hide_in_analysis_framework_mapping': False, - 'is_category': False, - 'parent_id': '6', - }, - '1': { - 'label': 'sectors', - 'is_category': True, - 'hide_in_analysis_framework_mapping': True - }, - '2': { - 'label': 'subpillars_1d', - 'is_category': True, - 'hide_in_analysis_framework_mapping': True - }, - '3': { - 'label': 'subpillars_2d', - 'is_category': True, - 'hide_in_analysis_framework_mapping': True - }, - '6': { - 'label': 'age', - 'is_category': True, - 'hide_in_analysis_framework_mapping': True - }, - '5': { - 'label': 'gender', - 'is_category': True, - 'hide_in_analysis_framework_mapping': True - }, - '9': { - 'label': 'demographic_group', - 'is_category': True, - 'hide_in_analysis_framework_mapping': True - }, - '8': { - 'label': 'affected_groups', - 'is_category': True, - 'hide_in_analysis_framework_mapping': True - }, - '4': { - 'label': 'specific_needs_groups', - 'is_category': True, - 'hide_in_analysis_framework_mapping': True - }, - '7': { - 'label': 'severity', - 'is_category': True, - 'hide_in_analysis_framework_mapping': True - }, - '10': { - 'label': 'reliability', - 'is_category': True, - 'hide_in_analysis_framework_mapping': True - }, - } - - def setUp(self): - super().setUp() - self.sync_request_mock = patch('assisted_tagging.tasks.requests') - mock = self.sync_request_mock.start() - mock.get.return_value.status_code = 200 - mock.get.return_value.json.return_value = self.DEEPL_TAGS_MOCK_RESPONSE - - def tearDown(self): - self.sync_request_mock.stop() - super().tearDown() - - def test_save_draft_entry(self): - def _check_draft_entry_status(draft_entry, status): - draft_entry.refresh_from_db() - self.assertEqual(draft_entry.prediction_status, status) - - def _get_current_model_stats(): - return dict( - model_count=AssistedTaggingModel.objects.count(), - model_version_count=AssistedTaggingModelVersion.objects.count(), - tag_count=AssistedTaggingModelPredictionTag.objects.count(), - models=list( - AssistedTaggingModel.objects.values('model_id', 'name').order_by('model_id') - ), - model_versions=list( - AssistedTaggingModelVersion.objects.values('model__model_id', 'version').order_by('model__model_id') - ), - tags=list( - AssistedTaggingModelPredictionTag.objects.values('name', 'tag_id', 'is_deprecated').order_by('tag_id') - ), - ) - - def _get_current_prediction_stats(): - return dict( - prediction_count=AssistedTaggingPrediction.objects.count(), - predictions=list( - AssistedTaggingPrediction.objects.values( - 'data_type', - 'model_version__model__model_id', - 'draft_entry__excerpt', - 'value', - 'category__tag_id', - 'tag__tag_id', - 'prediction', - 'threshold', - 'is_selected', - ).order_by( - 'data_type', - 'model_version__model__model_id', - 'draft_entry__excerpt', - 'value', - 'category__tag_id', - 'tag__tag_id', - 'prediction', - 'threshold', - 'is_selected', - ) - ) - ) - - url = '/api/v1/callback/assisted-tagging-draft-entry-prediction/' - project = ProjectFactory.create() - lead = LeadFactory.create(project=project) - draft_args = dict( - project=project, - lead=lead, - prediction_status=DraftEntry.PredictionStatus.STARTED, - ) - draft_entry1 = DraftEntryFactory.create( - **draft_args, - excerpt='sample excerpt 101', - ) - draft_entry2 = DraftEntryFactory.create( - **draft_args, - excerpt='sample excerpt 102', - ) - - # ------ Invalid entry_id - data = { - **self.DEEPL_CALLBACK_MOCK_DATA, - 'client_id': 'invalid-id', - } - - response = self.client.post(url, data) - self.assert_400(response) - _check_draft_entry_status(draft_entry1, DraftEntry.PredictionStatus.STARTED) - - # ----- Valid entry_id - data = { - **self.DEEPL_CALLBACK_MOCK_DATA, - 'client_id': AssistedTaggingDraftEntryHandler.get_client_id(draft_entry1), - } - - self.maxDiff = None - current_model_stats = _get_current_model_stats() - current_prediction_stats = _get_current_prediction_stats() - response = self.client.post(url, data) - self.assert_200(response) - self.assertNotEqual(current_model_stats, _get_current_model_stats()) - self.assertNotEqual(current_prediction_stats, _get_current_prediction_stats()) - _check_draft_entry_status(draft_entry1, DraftEntry.PredictionStatus.DONE) - - current_model_stats = _get_current_model_stats() - current_prediction_stats = _get_current_prediction_stats() - response = self.client.post(url, data) - _check_draft_entry_status(draft_entry1, DraftEntry.PredictionStatus.DONE) - self.assertEqual(current_model_stats, _get_current_model_stats()) - self.assertEqual(current_prediction_stats, _get_current_prediction_stats()) - - # ----- Valid entry_id send with same type of data - data = { - **self.DEEPL_CALLBACK_MOCK_DATA, - 'client_id': AssistedTaggingDraftEntryHandler.get_client_id(draft_entry2), - } - - current_model_stats = _get_current_model_stats() - response = self.client.post(url, data) - _check_draft_entry_status(draft_entry2, DraftEntry.PredictionStatus.DONE) - self.assertEqual(current_model_stats, _get_current_model_stats()) - self.assertNotEqual(current_prediction_stats, _get_current_prediction_stats()) - - current_model_stats = _get_current_model_stats() - current_prediction_stats = _get_current_prediction_stats() - self.assertMatchSnapshot(current_model_stats, 'final-current-model-stats') - self.assertMatchSnapshot(current_prediction_stats, 'final-current-prediction-stats') - - def test_tags_sync(self): - def _get_current_tags(): - return list( - AssistedTaggingModelPredictionTag.objects.values( - 'name', - 'group', - 'tag_id', - 'is_deprecated', - 'is_category', - 'hide_in_analysis_framework_mapping', - 'parent_tag__tag_id', - ).order_by('tag_id') - ) - - self.maxDiff = None - self.assertEqual(len(_get_current_tags()), 0) - sync_tags_with_deepl() - self.assertNotEqual(len(_get_current_tags()), 0) - self.assertMatchSnapshot(_get_current_tags(), 'sync-tags') - - class TestAssistedTaggingModules(GraphQLTestCase): def test_assisted_tagging_model_version_latest_model_fetch(self): diff --git a/apps/deepl_integration/handlers.py b/apps/deepl_integration/handlers.py index 7731c5ab11..4995e0fd6c 100644 --- a/apps/deepl_integration/handlers.py +++ b/apps/deepl_integration/handlers.py @@ -25,8 +25,6 @@ DraftEntry, AssistedTaggingModel, AssistedTaggingModelVersion, - AssistedTaggingModelPredictionTag, - AssistedTaggingPrediction, LLMAssistedTaggingPredication, ) from unified_connector.models import ( @@ -161,407 +159,6 @@ def save_data(cls, *_): raise Exception('Not implemented yet.') -class AssistedTaggingDraftEntryHandler(BaseHandler): - model = DraftEntry - callback_url_name = 'assisted_tagging_draft_entry_prediction_callback' - - @classmethod - def send_trigger_request_to_extractor(cls, draft_entry): - source_organization = draft_entry.lead.source - author_organizations = [ - author.data.title - for author in draft_entry.lead.authors.all() - ] - payload = { - 'entries': [ - { - 'client_id': cls.get_client_id(draft_entry), - 'entry': draft_entry.excerpt, - } - ], - 'lead': draft_entry.lead_id, - 'project': draft_entry.project_id, - 'publishing_organization': source_organization and source_organization.data.title, - 'authoring_organization': author_organizations, - 'callback_url': cls.get_callback_url(), - } - response_content = None - try: - response = requests.post( - DeeplServiceEndpoint.ASSISTED_TAGGING_ENTRY_PREDICT_ENDPOINT, - headers=cls.REQUEST_HEADERS, - json=payload - ) - response_content = response.content - if response.status_code == 202: - return True - except Exception: - logger.error('Assisted tagging send failed, Exception occurred!!', exc_info=True) - draft_entry.prediction_status = DraftEntry.PredictionStatus.SEND_FAILED - draft_entry.save(update_fields=('prediction_status',)) - logger.error( - 'Assisted tagging send failed!!', - extra={ - 'data': { - 'payload': payload, - 'response': response_content, - }, - }, - ) - - # --- Callback logics - @staticmethod - def _get_or_create_models_version(models_data): - def get_versions_map(): - return { - (model_version.model.model_id, model_version.version): model_version - for model_version in AssistedTaggingModelVersion.objects.filter( - reduce( - lambda acc, item: acc | item, - [ - models.Q( - model__model_id=model_data['id'], - version=model_data['version'], - ) - for model_data in models_data - ], - ) - ).select_related('model').all() - } - - existing_model_versions = get_versions_map() - new_model_versions = [ - model_data - for model_data in models_data - if (model_data['id'], model_data['version']) not in existing_model_versions - ] - - if new_model_versions: - AssistedTaggingModelVersion.objects.bulk_create([ - AssistedTaggingModelVersion( - model=AssistedTaggingModel.objects.get_or_create( - model_id=model_data['id'], - defaults=dict( - name=model_data['id'], - ), - )[0], - version=model_data['version'], - ) - for model_data in models_data - ]) - existing_model_versions = get_versions_map() - return existing_model_versions - - @classmethod - def _get_or_create_tags_map(cls, tags): - from assisted_tagging.tasks import sync_tags_with_deepl_task - - def get_tags_map(): - return { - tag_id: _id - for _id, tag_id in AssistedTaggingModelPredictionTag.objects.values_list('id', 'tag_id') - } - - current_tags_map = get_tags_map() - # Check if new tags needs to be created - new_tags = [ - tag - for tag in tags - if tag not in current_tags_map - ] - if new_tags: - # Create new tags - AssistedTaggingModelPredictionTag.objects.bulk_create([ - AssistedTaggingModelPredictionTag( - name=new_tag, - tag_id=new_tag, - ) - for new_tag in new_tags - ]) - # Refetch - current_tags_map = get_tags_map() - sync_tags_with_deepl_task.delay() - return current_tags_map - - @classmethod - def _process_model_preds(cls, model_version, current_tags_map, draft_entry, model_prediction): - prediction_status = model_prediction['prediction_status'] - if not prediction_status: # If False no tags are provided - return - - tags = model_prediction.get('model_tags', {}) # NLP TagId - values = model_prediction.get('values', []) # Raw value - - common_attrs = dict( - model_version=model_version, - draft_entry_id=draft_entry.id, - ) - new_predictions = [] - for category_tag, tags in tags.items(): - for tag, prediction_data in tags.items(): - prediction_value = prediction_data.get('prediction') - threshold_value = prediction_data.get('threshold') - is_selected = prediction_data.get('is_selected', False) - new_predictions.append( - AssistedTaggingPrediction( - **common_attrs, - data_type=AssistedTaggingPrediction.DataType.TAG, - category_id=current_tags_map[category_tag], - tag_id=current_tags_map[tag], - prediction=prediction_value, - threshold=threshold_value, - is_selected=is_selected, - ) - ) - - for value in set(values): - new_predictions.append( - AssistedTaggingPrediction( - **common_attrs, - data_type=AssistedTaggingPrediction.DataType.RAW, - value=value, - is_selected=True, - ) - ) - AssistedTaggingPrediction.objects.bulk_create(new_predictions) - - @classmethod - def save_data(cls, draft_entry, data): - model_preds = data - # Save if new tags are provided - current_tags_map = cls._get_or_create_tags_map([ - tag - for category_tag, tags in model_preds['model_tags'].items() - for tag in [ - category_tag, - *tags.keys(), - ] - ]) - models_version_map = cls._get_or_create_models_version( - [ - model_preds['model_info'] - ] - ) - with transaction.atomic(): - draft_entry.clear_data() # Clear old data if exists - draft_entry.calculated_at = timezone.now() - model_version = models_version_map[(model_preds['model_info']['id'], model_preds['model_info']['version'])] - cls._process_model_preds(model_version, current_tags_map, draft_entry, model_preds) - draft_entry.prediction_status = DraftEntry.PredictionStatus.DONE - draft_entry.save_geo_data() - draft_entry.save() - return draft_entry - - -class AutoAssistedTaggingDraftEntryHandler(BaseHandler): - # TODO: Fix N+1 issues here. Try to do bulk_update for each models. - # Or do this Async - model = Lead - callback_url_name = 'auto-assisted_tagging_draft_entry_prediction_callback' - - @classmethod - def auto_trigger_request_to_extractor(cls, lead): - lead_preview = LeadPreview.objects.get(lead=lead) - payload = { - "documents": [ - { - "client_id": cls.get_client_id(lead), - "text_extraction_id": str(lead_preview.text_extraction_id), - } - ], - "callback_url": cls.get_callback_url() - } - response_content = None - try: - response = requests.post( - url=DeeplServiceEndpoint.ENTRY_EXTRACTION_CLASSIFICATION, - headers=cls.REQUEST_HEADERS, - json=payload - ) - response_content = response.content - if response.status_code == 202: - lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.PENDING - lead.save(update_fields=('auto_entry_extraction_status',)) - return True - - except Exception: - logger.error('Entry Extraction send failed, Exception occurred!!', exc_info=True) - lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.FAILED - lead.save(update_fields=('auto_entry_extraction_status',)) - logger.error( - 'Entry Extraction send failed!!', - extra={ - 'data': { - 'payload': payload, - 'response': response_content, - }, - }, - ) - - # --- Callback logics - @staticmethod - def _get_or_create_models_version(models_data): - def get_versions_map(): - return { - (model_version.model.model_id, model_version.version): model_version - for model_version in AssistedTaggingModelVersion.objects.filter( - reduce( - lambda acc, item: acc | item, - [ - models.Q( - model__model_id=model_data['name'], - version=model_data['version'], - ) - for model_data in models_data - ], - ) - ).select_related('model').all() - } - - existing_model_versions = get_versions_map() - new_model_versions = [ - model_data - for model_data in models_data - if (model_data['name'], model_data['version']) not in existing_model_versions - ] - - if new_model_versions: - AssistedTaggingModelVersion.objects.bulk_create([ - AssistedTaggingModelVersion( - model=AssistedTaggingModel.objects.get_or_create( - model_id=model_data['name'], - defaults=dict( - name=model_data['name'], - ), - )[0], - version=model_data['version'], - ) - for model_data in models_data - ]) - existing_model_versions = get_versions_map() - return existing_model_versions - - @classmethod - def _get_or_create_tags_map(cls, tags): - from assisted_tagging.tasks import sync_tags_with_deepl_task - - def get_tags_map(): - return { - tag_id: _id - for _id, tag_id in AssistedTaggingModelPredictionTag.objects.values_list('id', 'tag_id') - } - - current_tags_map = get_tags_map() - # Check if new tags needs to be created - new_tags = [ - tag - for tag in tags - if tag not in current_tags_map - ] - if new_tags: - # Create new tags - AssistedTaggingModelPredictionTag.objects.bulk_create([ - AssistedTaggingModelPredictionTag( - name=new_tag, - tag_id=new_tag, - ) - for new_tag in new_tags - ]) - # Refetch - current_tags_map = get_tags_map() - sync_tags_with_deepl_task.delay() - return current_tags_map - - @classmethod - def _process_model_preds(cls, model_version, current_tags_map, draft_entry, model_prediction): - prediction_status = model_prediction['prediction_status'] - if not prediction_status: # If False no tags are provided - return - - tags = model_prediction.get('classification', {}) # NLP TagId - values = model_prediction.get('values', []) # Raw value - - common_attrs = dict( - model_version=model_version, - draft_entry_id=draft_entry.id, - ) - new_predictions = [] - for category_tag, tags in tags.items(): - for tag, prediction_data in tags.items(): - prediction_value = prediction_data.get('prediction') - threshold_value = prediction_data.get('threshold') - is_selected = prediction_data.get('is_selected', False) - new_predictions.append( - AssistedTaggingPrediction( - **common_attrs, - data_type=AssistedTaggingPrediction.DataType.TAG, - category_id=current_tags_map[category_tag], - tag_id=current_tags_map[tag], - prediction=prediction_value, - threshold=threshold_value, - is_selected=is_selected, - ) - ) - - for value in set(values): - new_predictions.append( - AssistedTaggingPrediction( - **common_attrs, - data_type=AssistedTaggingPrediction.DataType.RAW, - value=value, - is_selected=True, - ) - ) - AssistedTaggingPrediction.objects.bulk_create(new_predictions) - - @classmethod - @transaction.atomic - def save_data(cls, lead, data_url): - # NOTE: Schema defined here - # - https://docs.google.com/document/d/1NmjOO5sOrhJU6b4QXJBrGAVk57_NW87mLJ9wzeY_NZI/edit#heading=h.t3u7vdbps5pt - data = RequestHelper(url=data_url, ignore_error=True).json() - draft_entry_qs = DraftEntry.objects.filter(lead=lead, type=DraftEntry.Type.AUTO) - if draft_entry_qs.exists(): - raise serializers.ValidationError('Draft entries already exit') - for model_preds in data['blocks']: - if not model_preds['relevant']: - continue - classification = model_preds['classification'] - current_tags_map = cls._get_or_create_tags_map([ - tag - for category_tag, tags in classification.items() - for tag in [ - category_tag, - *tags.keys(), - ] - ]) - models_version_map = cls._get_or_create_models_version([ - data['classification_model_info'] - ]) - draft = DraftEntry.objects.create( - page=model_preds['page'], - text_order=model_preds['textOrder'], - project=lead.project, - lead=lead, - excerpt=model_preds['text'], - prediction_status=DraftEntry.PredictionStatus.DONE, - type=DraftEntry.Type.AUTO - ) - if model_preds['geolocations']: - geo_areas_qs = GeoAreaGqlFilterSet( - data={'titles': [geo['entity'] for geo in model_preds['geolocations']]}, - queryset=GeoArea.get_for_project(lead.project) - ).qs.distinct('title') - draft.related_geoareas.set(geo_areas_qs) - model_version = models_version_map[ - (data['classification_model_info']['name'], data['classification_model_info']['version']) - ] - cls._process_model_preds(model_version, current_tags_map, draft, model_preds) - lead.auto_entry_extraction_status = Lead.AutoExtractionStatus.SUCCESS - lead.save(update_fields=('auto_entry_extraction_status',)) - return lead - - class LeadExtractionHandler(BaseHandler): model = Lead callback_url_name = 'lead_extract_callback' diff --git a/apps/deepl_integration/serializers.py b/apps/deepl_integration/serializers.py index 6ecdf98f19..b7f7363d79 100644 --- a/apps/deepl_integration/serializers.py +++ b/apps/deepl_integration/serializers.py @@ -6,7 +6,6 @@ from deepl_integration.handlers import ( BaseHandler, - AssistedTaggingDraftEntryHandler, LeadExtractionHandler, LlmAssistedTaggingDraftEntryHandler, UnifiedConnectorLeadHandler, @@ -14,13 +13,11 @@ AnalysisAutomaticSummaryHandler, AnalyticalStatementNGramHandler, AnalyticalStatementGeoHandler, - AutoAssistedTaggingDraftEntryHandler, LLMAutoAssistedTaggingDraftEntryHandler, ) from deduplication.tasks.indexing import index_lead_and_calculate_duplicates from assisted_tagging.models import ( - AssistedTaggingPrediction, DraftEntry, ) from unified_connector.models import ConnectorLead @@ -190,79 +187,6 @@ def create(self, data): return connector_lead -# --- AssistedTagging -class AssistedTaggingModelPredictionCallbackSerializer(serializers.Serializer): - - class ModelPredictionCallbackSerializerTagValue(serializers.Serializer): - prediction = serializers.DecimalField( - # From apps/assisted_tagging/models.py::AssistedTaggingPrediction::prediction - max_digits=AssistedTaggingPrediction.prediction.field.max_digits, - decimal_places=AssistedTaggingPrediction.prediction.field.decimal_places, - required=False, - ) - threshold = serializers.DecimalField( - # From apps/assisted_tagging/models.py::AssistedTaggingPrediction::threshold - max_digits=AssistedTaggingPrediction.threshold.field.max_digits, - decimal_places=AssistedTaggingPrediction.threshold.field.decimal_places, - required=False, - ) - is_selected = serializers.BooleanField() - - # model_info = ModelInfoCallbackSerializer() removed from the DEEPL TODO Use different api for model information - values = serializers.ListSerializer( - child=serializers.CharField(), - required=False, - ) - tags = serializers.DictField( - child=serializers.DictField( - child=ModelPredictionCallbackSerializerTagValue(), - ), - required=False, - ) - - -class AutoAssistedTaggingModelPredicationCallBackSerializer(serializers.Serializer): - class ModelPredictionCallbackSerializerTagValue(serializers.Serializer): - predication = serializers.DecimalField( - max_digits=AssistedTaggingPrediction.prediction.field.max_digits, - decimal_places=AssistedTaggingPrediction.prediction.field.decimal_places, - required=False, - ) - threshold = serializers.DecimalField( - # From apps/assisted_tagging/models.py::AssistedTaggingPrediction::threshold - max_digits=AssistedTaggingPrediction.threshold.field.max_digits, - decimal_places=AssistedTaggingPrediction.threshold.field.decimal_places, - required=False, - ) - is_selected = serializers.BooleanField() - values = serializers.ListSerializer( - child=serializers.CharField(), - required=False, - ) - tags = serializers.DictField( - child=serializers.DictField( - child=ModelPredictionCallbackSerializerTagValue(), - ), - required=False, - ) - - -class AssistedTaggingDraftEntryPredictionCallbackSerializer(BaseCallbackSerializer): - model_tags = serializers.DictField(child=serializers.DictField()) - prediction_status = serializers.BooleanField() - model_info = serializers.DictField() - nlp_handler = AssistedTaggingDraftEntryHandler - - def create(self, validated_data): - draft_entry = validated_data['object'] - if draft_entry.prediction_status == DraftEntry.PredictionStatus.DONE: - return draft_entry - return self.nlp_handler.save_data( - draft_entry, - validated_data, - ) - - class LlmAssistedTaggingDraftEntryPredictionCallbackSerializer(BaseCallbackSerializer): model_tags = serializers.DictField(child=serializers.DictField()) prediction_status = serializers.BooleanField() @@ -279,30 +203,6 @@ def create(self, validated_data): ) -class AutoAssistedBlockPredicationCallbackSerializer(serializers.Serializer): - page = serializers.IntegerField() - textOrder = serializers.IntegerField() - text = serializers.CharField() - relevant = serializers.BooleanField() - prediction_status = serializers.BooleanField() - classification = serializers.DictField(child=serializers.DictField()) - geolocations = serializers.DictField(child=serializers.DictField()) - - -class AutoAssistedTaggingDraftEntryCallbackSerializer(BaseCallbackSerializer): - entry_extraction_classification_path = serializers.URLField(required=True) - text_extraction_id = serializers.CharField(required=True) - status = serializers.IntegerField() - nlp_handler = AutoAssistedTaggingDraftEntryHandler - - def create(self, validated_data): - obj = validated_data['object'] - return self.nlp_handler.save_data( - obj, - validated_data['entry_extraction_classification_path'], - ) - - class AutoLLMAssistedTaggingDraftEntryCallbackSerializer(BaseCallbackSerializer): entry_extraction_classification_path = serializers.URLField(required=True) text_extraction_id = serializers.CharField(required=True) diff --git a/apps/deepl_integration/views.py b/apps/deepl_integration/views.py index 890c794f6f..982e907371 100644 --- a/apps/deepl_integration/views.py +++ b/apps/deepl_integration/views.py @@ -8,7 +8,6 @@ ) from .serializers import ( - AssistedTaggingDraftEntryPredictionCallbackSerializer, AutoLLMAssistedTaggingDraftEntryCallbackSerializer, LlmAssistedTaggingDraftEntryPredictionCallbackSerializer, LeadExtractCallbackSerializer, @@ -17,7 +16,6 @@ AnalysisAutomaticSummaryCallbackSerializer, AnalyticalStatementNGramCallbackSerializer, AnalyticalStatementGeoCallbackSerializer, - AutoAssistedTaggingDraftEntryCallbackSerializer, ) @@ -32,18 +30,10 @@ def post(self, request, **_): return response.Response("Request successfully completed", status=status.HTTP_200_OK) -class AssistedTaggingDraftEntryPredictionCallbackView(BaseCallbackView): - serializer = AssistedTaggingDraftEntryPredictionCallbackSerializer - - class LlmAssistedTaggingDraftEntryPredictionCallbackView(BaseCallbackView): serializer = LlmAssistedTaggingDraftEntryPredictionCallbackSerializer -class AutoTaggingDraftEntryPredictionCallbackView(BaseCallbackView): - serializer = AutoAssistedTaggingDraftEntryCallbackSerializer - - class AutoLLMTaggingDraftEntryPredictionCallbackView(BaseCallbackView): serializer = AutoLLMAssistedTaggingDraftEntryCallbackSerializer diff --git a/apps/lead/tests/test_apis.py b/apps/lead/tests/test_apis.py index 11e6aae21a..a313be163e 100644 --- a/apps/lead/tests/test_apis.py +++ b/apps/lead/tests/test_apis.py @@ -1,6 +1,5 @@ import logging from datetime import date -import uuid from django.db.models import Q from django.core.files.uploadedfile import SimpleUploadedFile @@ -26,7 +25,7 @@ from organization.serializers import SimpleOrganizationSerializer from lead.filter_set import LeadFilterSet from lead.serializers import SimpleLeadGroupSerializer -from deepl_integration.handlers import AutoAssistedTaggingDraftEntryHandler, LeadExtractionHandler +from deepl_integration.handlers import LeadExtractionHandler from deepl_integration.serializers import DeeplServerBaseCallbackSerializer from entry.models import ( Entry, @@ -44,7 +43,7 @@ ) from user_group.models import UserGroup, GroupMembership from ary.models import Assessment -from lead.factories import LeadFactory, LeadPreviewFactory +from lead.factories import LeadFactory from unittest import mock @@ -1873,85 +1872,3 @@ def test_client_id_generator(self): LeadExtractionHandler.get_object_using_client_id(client_id) else: assert LeadExtractionHandler.get_object_using_client_id(client_id) == lead - - -class AutoEntryExtractionTestCase(TestCase): - def setUp(self): - super().setUp() - self.lead = LeadFactory.create() - self.lead_preview = LeadPreviewFactory.create(lead=self.lead, text_extraction_id=str(uuid.uuid1())) - - @mock.patch('deepl_integration.handlers.RequestHelper.json') - def test_entry_extraction_callback_url(self, get_json_mock): - url = '/api/v1/callback/auto-assisted-tagging-draft-entry-prediction/' - self.authenticate() - SAMPLE_AUTO_ASSISTED_TAGGING = { - "metadata": {"total_pages": 10, "total_words_count": 5876}, - "blocks": [ - { - "type": "text", - "page": 1, - "text": "Entry-Text-1", - "textOrder": 1, - "relevant": True, - "prediction_status": True, - "geolocations": [ - { - "entity": "Somalia", - "meta": {"offset_start": 88, "offset_end": 94, "latitude": -10, "longitude": -55}, - } - ], - "classification": {"1": {"101": {"prediction": 2.70270529e-05, "threshold": 0.14, "is_selected": True}}}, - }, - { - "type": "text", - "page": 2, - "text": "Entry-Text-2", - "textOrder": 2, - "relevant": True, - "prediction_status": True, - "geolocations": [ - { - "entity": "Nigeria", - "meta": {"offset_start": 183, "offset_end": 191, "latitude": None, "longitude": None}, - } - ], - "classification": { - "1": {"101": {"prediction": 2.0000270270529, "threshold": 0.14, "is_selected": True}} - }, - }, - { - "type": "text", - "page": 3, - "text": "This is a non-relevant text", - "textOrder": 3, - "relevant": False, - "prediction_status": False, - "geolocations": [], - "classification": {}, - }, - ], - "classification_model_info": {"name": "all_tags_model", "version": "1.0.0"}, - "client_id": AutoAssistedTaggingDraftEntryHandler.get_client_id(self.lead), - "entry_extraction_id": "73f9ca13-deb2-4f39-8e86-a856490bfc0d", - "text_extraction_id": str(self.lead_preview.text_extraction_id), - } - get_json_mock.return_value = SAMPLE_AUTO_ASSISTED_TAGGING - - # Invalid clientId - data = { - 'client_id': 'invalid-client-id', - 'entry_extraction_classification_path': 'https://random-domain.com/random-url.json', - 'text_extraction_id': str(self.lead_preview.text_extraction_id), - 'status': 1 - } - response = self.client.post(url, data) - self.assert_400(response) - - # valid ClientID - data['client_id'] = AutoAssistedTaggingDraftEntryHandler.get_client_id(self.lead) - response = self.client.post(url, data) - self.assert_200(response) - self.lead.refresh_from_db() - self.assertEqual(str(LeadPreview.objects.get(lead=self.lead).text_extraction_id), data['text_extraction_id']) - self.assertEqual(self.lead.auto_entry_extraction_status, Lead.AutoExtractionStatus.SUCCESS) diff --git a/deep/urls.py b/deep/urls.py index 3f14a5db8b..6b8ff74ba6 100644 --- a/deep/urls.py +++ b/deep/urls.py @@ -149,9 +149,7 @@ ExportViewSet, ) from deepl_integration.views import ( - AssistedTaggingDraftEntryPredictionCallbackView, LlmAssistedTaggingDraftEntryPredictionCallbackView, - AutoTaggingDraftEntryPredictionCallbackView, AutoLLMTaggingDraftEntryPredictionCallbackView, LeadExtractCallbackView, UnifiedConnectorLeadExtractCallbackView, @@ -571,16 +569,6 @@ def get_api_path(path): UnifiedConnectorLeadExtractCallbackView.as_view(), name='unified_connector_lead_extract_callback', ), - re_path( - get_api_path(r'callback/assisted-tagging-draft-entry-prediction/$'), - AssistedTaggingDraftEntryPredictionCallbackView.as_view(), - name='assisted_tagging_draft_entry_prediction_callback', - ), - re_path( - get_api_path(r'callback/auto-assisted-tagging-draft-entry-prediction/$'), - AutoTaggingDraftEntryPredictionCallbackView.as_view(), - name='auto-assisted_tagging_draft_entry_prediction_callback', - ), re_path( get_api_path(r'callback/llm-assisted-tagging-draft-entry-prediction/$'),