Fixes

conjurer_librarian/search_bot.py

@@ -68,7 +68,7 @@ def producer(out_q, control_q, filename, _logger):
 
 
 # IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
-def consumer(in_q, control_q, doi, live_results, result_list, _logger, ):
+def consumer(in_q, control_q, doi, live_results, result_list, control_dict, _logger):
     """
     Consumes items from an input queue and checks if DOI exists in the live results list.
 
@@ -79,7 +79,6 @@ def consumer(in_q, control_q, doi, live_results, result_list, _logger, ):
         live_results (list): List to store the search results.
         _logger: Logger object for logging.
     """
-    sentinels = 0
     for item in doi:
         result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
     while True:
@@ -87,7 +86,7 @@ def consumer(in_q, control_q, doi, live_results, result_list, _logger, ):
         data = in_q.get()
         if data is _sentinel:
             _logger.info("Worker finished")
-            sentinels += 1
+            control_dict["sentinels"] += 1
         else:
             for item in result_list:
                 if item["DOI"] in data and not item["exists"]:
@@ -98,7 +97,7 @@ def consumer(in_q, control_q, doi, live_results, result_list, _logger, ):
                 done_check = done_check and item["exists"]
             if done_check:
                 control_q.put(_sentinel)
-        if sentinels >= MAXTHREADS:
+        if control_dict["sentinels"] >= MAXTHREADS:
             _logger.info("All workers finished")
             break
 
@@ -112,15 +111,17 @@ def search_for_doi(doi, live_results, _logger):
         live_results (list): List to store the search results.
         _logger: Logger object for logging.
     """
+    control_dict = {"sentinels":0}
     result_list = []
     threads = []
     work_q = Queue(maxsize=WORK_Q_SIZE)
     control_q = Queue()
-    t_cons = Thread(
-        target=consumer, args=(work_q, control_q, doi, live_results, result_list, _logger)
-    )
-    _logger.info("Consumer thread created")
-    threads.append(t_cons)
+    for _ in range (1, len(doi)//1000+1):
+        t_cons = Thread(
+            target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, _logger)
+        )
+        _logger.info("Consumer thread created")
+        threads.append(t_cons)
     for i in range(0, MAXTHREADS):
         # TEST DATA
         # filename = "test" + str(i) + CHUNk