Fixing librarian result (#6)

* Formatting and additional logging for bugfix

* Formatting and additional logging for bugfix

* Change of log location

* fixing_librarian_result

* Hopefully final fix that will be needed

* Temporary

* Glebokie gardlo

* Fix in scrape_bot

* Fix that was needed.

* Scraper settings

* Small fix in scraping

* FIxing bad  naming convention

* Log message fix

* Test of deep search

* Fix in deep search

* Refactoring

* WHat and idiot coded that....
Ooops. That was me. A month ago.

* Refactoring continued

* Bugfix

* FIx

* Not needed after fix

conjurer_librarian/search_bot.py

@@ -1,26 +1,55 @@
-#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+"""
+This module contains functions for searching for DOI (Digital Object Identifier)
+in a list of live results.
+It includes a producer-consumer pattern implementation using threads and queues.
 
-from queue import Queue, Empty
+Functions:
+- producer: Reads lines from a file and puts them into an output queue.
+- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
+- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
+
+Global Variables:
+- MAXTHREADS: Maximum number of worker threads.
+- DATABASE_PATH: Path to the database files.
+- ENCODING: Encoding of the database files.
+- CHUNK: File name pattern for the database files.
+- _sentinel: Sentinel object used to signal termination.
+- result_list: List to store the search results.
+- WORK_Q_SIZE: Maximum size of the work queue.
+"""
+
+# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+from queue import Empty, Queue
 from threading import Thread
+
 q = Queue()
 
-#TODO: DATA FOR TEST ONLY
-#MAXTHREADS = 5
-#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+# DATA FOR TEST ONLY
+# MAXTHREADS = 5
+# DATABASE_PATH = r'C:\\Database\\chunks_1\\'
 
-#TODO: DEPLOYMENT DATA
+# DEPLOYMENT DATA
 MAXTHREADS = 41
-DATABASE_PATH = r'C:\\Database\\chunks\\'
+DATABASE_PATH = r"C:\\Database\\chunks\\"
 
 ENCODING = "utf-8"
-CHUNk = "_chunk.txt"
+CHUNK = "_chunk.txt"
 _sentinel = object()
-result_list = []
 WORK_Q_SIZE = 15500000
 
-def producer(out_q, control_q, filename, logger):
+
+def producer(out_q, control_q, filename, _logger):
+    """
+    Produces items from the output queue and puts them into the control queue.
+
+    Args:
+        out_q (Queue): Output queue.
+        control_q (Queue): Control queue.
+        filename (str): Name of the file.
+        _logger: Logger object for logging.
+    """
     with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
-        logger.info(f"Worker {filename} ")
+        _logger.info(f"Worker {filename} ")
         while True:
             line = operated_file.readline()
             if not line:
@@ -30,77 +59,107 @@ def producer(out_q, control_q, filename, logger):
                 check = control_q.get(block=False)
             except Empty:
                 check = False
-                pass
             if check is _sentinel:
-                logger.info("TERM signal received")
+                _logger.info("TERM signal received")
                 control_q.put(check)
                 break
-        logger.info(f"Worker finished: {filename}")
+        _logger.info(f"Worker finished: {filename}")
         out_q.put(_sentinel)
- 
-#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
-#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
-def consumer(in_q, control_q, doi, live_results, logger):
+
+
+# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+def consumer(in_q, control_q, doi, live_results, result_list, _logger, ):
+    """
+    Consumes items from an input queue and checks if DOI exists in the live results list.
+
+    Args:
+        in_q (Queue): Input queue.
+        control_q (Queue): Control queue.
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
     sentinels = 0
     for item in doi:
-        result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
+        result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
     while True:
         done_check = True
         data = in_q.get()
         if data is _sentinel:
-            logger.info("Worker finished")
+            _logger.info("Worker finished")
             sentinels += 1
         else:
             for item in result_list:
                 if item["DOI"] in data and not item["exists"]:
-                    logger.info(data)
-                    logger.info("HIT")
+                    _logger.info(data)
+                    _logger.info("HIT")
                     item["exists"] = True
                     live_results.append(item)
                 done_check = done_check and item["exists"]
             if done_check:
                 control_q.put(_sentinel)
         if sentinels >= MAXTHREADS:
-            logger.info("All workers finished")
+            _logger.info("All workers finished")
             break
 
 
-def search_for_doi(doi, live_results, logger):
+def search_for_doi(doi, live_results, _logger):
+    """
+    Search for DOI in live_results using _logger for logging.
+
+    Args:
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    result_list = []
     threads = []
     work_q = Queue(maxsize=WORK_Q_SIZE)
     control_q = Queue()
-    t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
-    logger.info("Consumer thread created")
+    t_cons = Thread(
+        target=consumer, args=(work_q, control_q, doi, live_results, result_list, _logger)
+    )
+    _logger.info("Consumer thread created")
     threads.append(t_cons)
-    for i in range(0,MAXTHREADS):
-        #TODO: TEST DATA
-        #filename = "test" + str(i) + CHUNk
-        #TODO: DEPLOYMENT DATA
-        filename = str(i) + CHUNk
-        logger.info(f"Creating worker thread no: {i}")
-        threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger)))
+    for i in range(0, MAXTHREADS):
+        # TEST DATA
+        # filename = "test" + str(i) + CHUNk
+        # DEPLOYMENT DATA
+        filename = str(i) + CHUNK
+        _logger.info(f"Creating worker thread no: {i}")
+        threads.append(
+            Thread(target=producer, args=(work_q, control_q, filename, _logger))
+        )
     for worker in threads:
         worker.start()
     for worker in threads:
         worker.join()
     return result_list
 
+
 if __name__ == "__main__":
     import logging
+
     logger = logging.getLogger()
     logger.setLevel("DEBUG")
     h1 = logging.StreamHandler()
     logger.addHandler(h1)
     logger.info("TEST RUN")
     live_result = []
-    logger.info(search_for_doi([
-        ("10.1002/9781118786352.wbieg0998.pub2","DATA"),
-        ("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
-        ("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
-        ("10.2165/00128415-200309690-00017","DATA"),
-        ("10.5772/48313","DATA"),
-        ("10.15803/ijnc.7.2_419","DATA"),
-        ("10.1051/0004-6361/201321596e","DATA"),
-        ("10.2307/40835941","DATA"),
-        ], live_result, logger))
+    logger.info(
+        search_for_doi(
+            [
+                ("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
+                ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
+                ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
+                ("10.2165/00128415-200309690-00017", "DATA"),
+                ("10.5772/48313", "DATA"),
+                ("10.15803/ijnc.7.2_419", "DATA"),
+                ("10.1051/0004-6361/201321596e", "DATA"),
+                ("10.2307/40835941", "DATA"),
+            ],
+            live_result,
+            logger,
+        )
+    )
     logger.info(live_result)