""" This module contains functions for searching for DOI (Digital Object Identifier) in a list of live results. It includes a producer-consumer pattern implementation using threads and queues. Functions: - producer: Reads lines from a file and puts them into an output queue. - consumer: Consumes items from an input queue and checks if DOI exists in the live results list. - search_for_doi: Searches for DOI in live results using the producer-consumer pattern. Global Variables: - MAXTHREADS: Maximum number of worker threads. - DATABASE_PATH: Path to the database files. - ENCODING: Encoding of the database files. - CHUNK: File name pattern for the database files. - _sentinel: Sentinel object used to signal termination. - result_list: List to store the search results. - WORK_Q_SIZE: Maximum size of the work queue. """ # TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry from multiprocessing import Manager, Process, Queue from queue import Empty, Full import time # DATA FOR TEST ONLY #MAXTHREADS = 6 #DATABASE_PATH = r'C:\\Database\\chunks_1\\' # DEPLOYMENT DATA MAXTHREADS = 41 DATABASE_PATH = r"C:\\Database\\chunks\\" ENCODING = "utf-8" CHUNK = "_chunk.txt" _sentinel = None WORK_Q_SIZE = 355000 def producer(out_q, control_q, filename, _logger): """ Produces items from the output queue and puts them into the control queue. Args: out_q (Queue): Output queue. control_q (Queue): Control queue. filename (str): Name of the file. _logger: Logger object for logging. """ with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file: print(f"Worker {filename} ") line_no = 0 while True: line = operated_file.readline() line_no += 1 print(f"\t \t \t W{filename}{line_no}\r", end="") if not line: print(f"EOF {filename}") break try: out_q.put([line, line_no, filename], block = True, timeout = 60) except Full: print("queue fulll") try: check = control_q.get(block=False) except Empty: check = False if check is None: print(f"TERM signal received {filename}") control_q.put(check) control_q.close() break print(f"Worker finished: {filename}") out_q.put(_sentinel) out_q.close() # IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger): """ Consumes items from an input queue and checks if DOI exists in the live results list. Args: in_q (Queue): Input queue. control_q (Queue): Control queue. doi (list): List of DOI to search for. live_results (list): List to store the search results. _logger: Logger object for logging. """ print(f"Consumer thread started: {no} no") empty_counter = 0 alive_no = 0 while True: done_check = True try: data = in_q.get(block=True, timeout = 1) empty_counter = 0 if data is None: control_dict["sentinels"] += 1 print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}") else: empty_counter = 0 if alive_no > 9: alive_no = 0 alive_no += 1 print(f"C{no}{alive_no}\r", end="") for item in result_list: if item["DOI"] in data[0] and not item["exists"]: print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}") item["exists"] = True live_results.append(item) done_check = done_check and item["exists"] if done_check: control_q.put(_sentinel) except Empty: empty_counter += 1 time.sleep(1) print(f"Consumer {no} empty") if empty_counter > 5: print("Consumer %s empty lvl 2", no) time.sleep(2) elif empty_counter > 10: print(f"Consumer thread finished {no}") break if control_dict["sentinels"] >= MAXTHREADS: print(f"All workers finished killing {no}") break print("Consumer thread finished %s",no ) control_q.close() def search_for_doi(doi, live_results, _logger): """ Search for DOI in live_results using _logger for logging. Args: doi (list): List of DOI to search for. live_results (list): List to store the search results. _logger: Logger object for logging. """ with Manager() as manager: control_dict = manager.dict() control_dict["sentinels"] = 0 result_list = [] for item in doi: result_list.append({"DOI": item[0], "exists": False, "data": item[1]}) result_list_proxy = manager.list(result_list) threads = [] work_q = Queue(maxsize=WORK_Q_SIZE) control_q = Queue() for i in range(0, MAXTHREADS): # TEST DATA #filename = "test" + str(i) + CHUNK # DEPLOYMENT DATA filename = str(i) + CHUNK print(f"Creating worker thread no: {i}") threads.append( Process(target=producer, args=(work_q, control_q, filename, _logger)) ) for i in range (0, (len(doi)//1000)+5): t_cons = Process( target=consumer, args=(work_q, control_q, doi, live_results, result_list_proxy, control_dict, i, _logger) ) print("Consumer thread created") threads.append(t_cons) for worker in threads: worker.start() for worker in threads: worker.join() result_list = list(result_list_proxy) for worker in threads: worker.close() return result_list if __name__ == "__main__": import logging logger = logging.getLogger() logger.setLevel("DEBUG") h1 = logging.StreamHandler() logger.addHandler(h1) print("TEST RUN") live_result = [] print( search_for_doi( [ ("10.1002/9781118786352.wbieg0998.pub2", "DATA"), ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"), ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"), ("10.2165/00128415-200309690-00017", "DATA"), ("10.5772/48313", "DATA"), ("10.15803/ijnc.7.2_419", "DATA"), ("10.1051/0004-6361/201321596e", "DATA"), ("10.2307/40835941", "DATA"), ("NOT_ON_LIST", "DATA"), ], live_result, logger, ) ) print(live_result)