refactoring for new way to page results

Author: mustberuss

src/main.py

@@ -159,34 +159,38 @@ def print_roundtrip(response, *args, **kwargs):
 
 
 def get_patent(patent_number, fields=None):
-    patent_query = '{"patent_number":"%s"}' % patent_number
-    fields = ('["patent_number","patent_title","patent_abstract","patent_date","patent_year",'
-              '"cpc_current.cpc_subsection_id","cpc_current","patent_kind","patent_type",'
-              '"assignees_at_grant.country","assignees_at_grant.assignee_id",'
-              '"assignees_at_grant.organization"]')
+    patent_query = '{"patent_id":"%s"}' % patent_number
+    fields = ('["patent_id","patent_title","patent_abstract","patent_date","patent_year",'
+              '"cpc_current.cpc_group_id","cpc_current","patent_kind","patent_type",'
+              '"assignees.assignee_country","assignees.assignee_assignee_id",'
+              '"assignees.assignee_organization",,"assignees.assignee_name_first","assignees.assignee_name_last"]')
     return patentsview_post_request(patent_search_endpoint, patent_query, fields)
 
 
 def get_all_company_patents(company, beginning_year=None, end_year=None, verbose=False):
+
     first_page = get_one_page_of_company_patents(company, beginning_year, end_year, verbose=verbose)
     patents = first_page["patents"]
     number_of_pages = 1
+
     # API change, attribute was total_patent_count, now total_hits
     if first_page["total_hits"] > first_page["count"]:
-        number_of_pages = first_page["total_hits"] // 25
-        if first_page["total_hits"] % 25:
+        after = patents[-1]["patent_id"]  
+        number_of_pages = first_page["total_hits"] // REQUEST_SIZE
+        if first_page["total_hits"] % REQUEST_SIZE:
             number_of_pages += 1
-    for page_number in range(2, number_of_pages + 1):
-        page_results = get_one_page_of_company_patents(company, beginning_year, end_year, page_number, verbose=verbose)
-        if page_results["patents"]:
-            patents += page_results["patents"]
+        for page_number in range(1, number_of_pages):
+            page_results = get_one_page_of_company_patents(company, beginning_year, end_year, verbose=verbose, after=after)
+            if page_results["patents"]:
+                patents += page_results["patents"]
+                after = patents[-1]["patent_id"] 
     # TODO see if it is better to yield instead of to return
     return patents
 
 
-def get_one_page_of_company_patents(company, beginning_year=None, end_year=None, page=1, perpage=25, verbose=False):
-    print("Requesting PatentsView: %s, page %d" % (company, page))
-    company_query = '{"%s":{"assignees_at_grant.organization":"%s"}}' % (COMPANY_SEARCH_CRITERIA, company)
+def get_one_page_of_company_patents(company, beginning_year=None, end_year=None, perpage=REQUEST_SIZE, verbose=False, after=""):
+    print("Requesting PatentsView: %s" % (company))
+    company_query = '{"%s":{"assignees.assignee_organization":"%s"}}' % (COMPANY_SEARCH_CRITERIA, company)
     date_range = None
 
     if beginning_year is not None and end_year is not None:
@@ -267,7 +271,7 @@ def patentsview_post_request(endpoint, query_param, format_param=None, options_p
 
         raise Exception("Status code: %s\r\n%s\n" % (r.status_code, r.text, extra))
 
-    return r.text
+    return json.loads(r.text)
 
 
 def insert_alternate_names(primary_id, alternate_names, commit_after_insert=True):
@@ -339,84 +343,32 @@ def add_cited_patents(limit=REQUEST_SIZE, verbose=False):
     for patents in fetch_patents_by_number(patent_search_endpoint, cited_patents_to_add, results_format, limit=limit, verbose=verbose):
         add_patents(patents)
 
-def add_cited_patent_numbers(patents_list, limit=25, verbose=False):
-    results_format = '["patent_number","cited_patent_number"]'
+def add_cited_patent_numbers(patents_list, limit=REQUEST_SIZE, verbose=False):
+    results_format = '["patent_id","citation_patent_id"]'
     # now we only get the citations from the citation endpoint
-    cited_patents = fetch_patents_by_number(citation_search_endpoint, patents_list, results_format, limit=limit, verbose=verbose)
-
-    # we have to call to get the details of the cited patents and then add them to the database
-    # TODO refactor to make this a constant etc
-    patents_format =  ('["patent_number","patent_date","patent_year",'
-                      '"assignees_at_grant.organization","cpc_current",'
-                      '"patent_title","assignees_at_grant.name_first","assignees_at_grant.name_last"]'
-                      )
 
-    for patents in fetch_patents_by_number(patent_search_endpoint, cited_patents, patents_format, limit=limit, verbose=verbose):
-        add_cited_patent_numbers_to_db(patents)
+    for citations in fetch_patents_by_number(citation_search_endpoint, patents_list, results_format, limit=limit, verbose=verbose):
+        add_cited_patent_numbers_to_db(citations)
 
+def fetch_patents_by_number(search_endpoint, patents_list, results_format, limit=REQUEST_SIZE, verbose=False):
 
-def fetch_patents_by_number(search_endpoint, patents_list, results_format, limit=25, verbose=False):
-    q_list = ['"%s"' % patent_number for patent_number in patents_list]
-    q_str = '{"patent_number":[%s]}' % ",".join(q_list)
+    # We'll post requests in batches of limit number of patents at a time
+    # so we won't have to page 
+    number_of_batches = (len(patents_list) + limit - 1) // limit
 
-    # ** the api does accept POST requests
-    # PatentsView only accepts GET requests; the endpoints for GET requests have a max length of 2000 characters.
-    # As such if the length of the endpoint exceeds the maximum allowed length, a '414 URI Too Long' error is returned.
-    # (for an explanation see: https://stackoverflow.com/a/50018203/6288413)
-    # To circumvent the issue, we have to break up the query into chunks
-    patents = []
-    endpoint_length = len(patent_search_endpoint) + len('&q=') + len(q_str) + len('&f=') + len(results_format)
-
-    # TODO: rework this
-    # ** the options parameter needs to be set to retrieve move than 25 patents at a time
-    # With the new API the default is increased to 100
-    # ** options parameter "size": 1000  is max request size now, originally "per_page": 10000 
-
-    # The PatentsView API apparently only allows 25 patents to be looked up at a time, hence the need for limit
-    # TODO: investigate why this is and if there is a way to change it
-
-    # we're calling either the patent endpoint or the new patent_citation endpoint
-    # the nested entity we need will either be patents or patent_citations
+    # Currently we're calling either the patent endpoint or patent_citation endpoint
+    # the returned entity will either be patents or patent_citations
     m = re.search(r'/([^/]*)/$', search_endpoint)
     return_entity = m.group(1) + "s"
-    print("return entity: {}".format(return_entity))
-
-    if endpoint_length < 2000:
-        response = patentsview_post_request(search_endpoint, q_str, results_format, verbose=verbose)
-        results = json.loads(response)
-        # 'patents' if search_endpoint end in /patent/ 
-        patents = results[return_entity]
-        yield patents
-    else:
-        if limit and ((endpoint_length // 2000) < (endpoint_length // limit)):
-            number_of_chunks = endpoint_length // limit + 1
-        else:
-            number_of_chunks = endpoint_length // 2000 + 1
-
-        interval = max(len(q_list) // number_of_chunks, limit)
-        num_intervals = range(len(q_list) // interval + 2)
-
-        for i in num_intervals:
-            start_index = i * interval
-            end_index = (i + 1) * interval
-            print("start index: {} end_index: {}".format(start_index, end_index))
-
-            q_str = '{"patent_number":[%s]}' % ",".join(q_list[start_index:end_index])
-            response = patentsview_post_request(search_endpoint, q_str, results_format, verbose=verbose)
-
-            results = json.loads(response)
-            if verbose:
-                print(results)
-            if results[return_entity]:
-                patents += results[return_entity]
-
-            # This is to potentially avoid a "Segmentation Fault (core dumped)" error
-
-            # TODO change this to an implementation that is more programmatic
-            if len(patents) >= 1000:
-                yield patents
-                patents = []
-        yield patents
+    options_param =  '{"size":%d}' % (limit) 
+
+    for i in range(number_of_batches):
+        start_index = i * limit
+        end_index = (i + 1) * limit
+        q_str = '{"patent_id":[%s]}' % ",".join(patents_list[start_index:end_index])
+        results = patentsview_post_request(search_endpoint, q_str, results_format, options_param=options_param, verbose=verbose)
+        entity = results[return_entity]
+        yield entity
 
 
 def add_cited_patent_numbers_to_db(citations: List) -> None: