Optimization bug fix #1

conjurer_librarian/search_bot.py

@@ -35,7 +35,7 @@ DATABASE_PATH = r"C:\\Database\\chunks\\"
 ENCODING = "utf-8"
 CHUNK = "_chunk.txt"
 _sentinel = object()
-WORK_Q_SIZE = 15500000
+WORK_Q_SIZE = 35500000
 
 
 def producer(out_q, control_q, filename, _logger):
@@ -83,7 +83,8 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, _log
         result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
     while True:
         done_check = True
-        data = in_q.get()
+        try:
+            data = in_q.get(block=False)
             if data is _sentinel:
                 _logger.info("Worker finished")
                 control_dict["sentinels"] += 1
@@ -97,11 +98,14 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, _log
                     done_check = done_check and item["exists"]
                 if done_check:
                     control_q.put(_sentinel)
+        except Empty:
+            pass
         if control_dict["sentinels"] >= MAXTHREADS:
             _logger.info("All workers finished")
             break
 
 
+
 def search_for_doi(doi, live_results, _logger):
     """
     Search for DOI in live_results using _logger for logging.