Optimization bug fix #1

conjurer_librarian/search_bot.py

@@ -35,7 +35,7 @@ DATABASE_PATH = r"C:\\Database\\chunks\\"
 ENCODING = "utf-8"
 CHUNK = "_chunk.txt"
 _sentinel = object()
-WORK_Q_SIZE = 15500000
+WORK_Q_SIZE = 35500000
 
 
 def producer(out_q, control_q, filename, _logger):
@@ -83,25 +83,29 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, _log
         result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
     while True:
         done_check = True
-        data = in_q.get()
+        try:
-        if data is _sentinel:
+            data = in_q.get(block=False)
-            _logger.info("Worker finished")
+            if data is _sentinel:
-            control_dict["sentinels"] += 1
+                _logger.info("Worker finished")
-        else:
+                control_dict["sentinels"] += 1
-            for item in result_list:
+            else:
-                if item["DOI"] in data and not item["exists"]:
+                for item in result_list:
-                    _logger.info(data)
+                    if item["DOI"] in data and not item["exists"]:
-                    _logger.info("HIT")
+                        _logger.info(data)
-                    item["exists"] = True
+                        _logger.info("HIT")
-                    live_results.append(item)
+                        item["exists"] = True
-                done_check = done_check and item["exists"]
+                        live_results.append(item)
-            if done_check:
+                    done_check = done_check and item["exists"]
-                control_q.put(_sentinel)
+                if done_check:
+                    control_q.put(_sentinel)
+        except Empty:
+            pass
         if control_dict["sentinels"] >= MAXTHREADS:
             _logger.info("All workers finished")
             break
 
 
+
 def search_for_doi(doi, live_results, _logger):
     """
     Search for DOI in live_results using _logger for logging.