diff --git a/conjurer_librarian/search_bot.py b/conjurer_librarian/search_bot.py index 8dc5590..4ef563e 100644 --- a/conjurer_librarian/search_bot.py +++ b/conjurer_librarian/search_bot.py @@ -1,21 +1,22 @@ +#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry + from queue import Queue, Empty from threading import Thread q = Queue() -#TODO: TO w klasie jako parametry wybierane czy robimy pelen run czy testowy -#MAXTHREADS = 3 -MAXTHREADS = 1 +#TODO: DATA FOR TEST ONLY +MAXTHREADS = 5 +DATABASE_PATH = r'C:\\Database\\chunks_1\\' -#DATABASE_PATH = r'C:\\Database\\chunks_1\\' -DATABASE_PATH = r'C:\\Database\\chunks\\' +#TODO: DEPLOYMENT DATA +#MAXTHREADS = 40 +#DATABASE_PATH = r'C:\\Database\\chunks\\' ENCODING = "utf-8" CHUNk = "_chunk.txt" _sentinel = object() result_list = [] -#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry - def producer(out_q, control_q, filename): with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file: lines = operated_file.readlines() @@ -40,10 +41,6 @@ def consumer(in_q, control_q, doi, live_results, logger): for item in doi: result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]}) while True: - #if in_q.empty(): - # print("Queue empty!") - #if in_q.full(): - # print("Queue full!") done_check = True data = in_q.get() if data is _sentinel: @@ -72,8 +69,9 @@ def search_for_doi(doi, live_results, logger): logger.info("Consumer thread created") threads.append(t_cons) for i in range(0,MAXTHREADS): - #filename = "test" + str(i) + CHUNk - filename = "40" + CHUNk + #TODO: TEST DATA + filename = "test" + str(i) + CHUNk + #TODO: DEPLOYMENT DATA #filename = str(i) + CHUNk logger.info(f"Creating worker thread no: {i}") threads.append(Thread(target = producer, args = (work_q, control_q, filename))) @@ -83,8 +81,6 @@ def search_for_doi(doi, live_results, logger): worker.join() return result_list - - if __name__ == "__main__": import logging logger = logging.getLogger()