gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 21:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Preparation for deployment

Browse Source
This commit is contained in:
Michal Tuszowski
2024-04-10 19:12:44 +02:00
parent 8b1950823b
commit b08642dd88
1 changed files with 11 additions and 15 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/search_bot.py
+11 -15
View File
@@ -1,21 +1,22 @@
#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
from queue import Queue, Empty from queue import Queue, Empty
from threading import Thread from threading import Thread
q = Queue() q = Queue()
#TODO: TO w klasie jako parametry wybierane czy robimy pelen run czy testowy #TODO: DATA FOR TEST ONLY
#MAXTHREADS = 3 MAXTHREADS = 5
MAXTHREADS = 1 DATABASE_PATH = r'C:\\Database\\chunks_1\\'
#DATABASE_PATH = r'C:\\Database\\chunks_1\\' #TODO: DEPLOYMENT DATA
DATABASE_PATH = r'C:\\Database\\chunks\\' #MAXTHREADS = 40
#DATABASE_PATH = r'C:\\Database\\chunks\\'
ENCODING = "utf-8" ENCODING = "utf-8"
CHUNk = "_chunk.txt" CHUNk = "_chunk.txt"
_sentinel = object() _sentinel = object()
result_list = [] result_list = []
#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
def producer(out_q, control_q, filename): def producer(out_q, control_q, filename):
with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file: with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
lines = operated_file.readlines() lines = operated_file.readlines()
@@ -40,10 +41,6 @@ def consumer(in_q, control_q, doi, live_results, logger):
for item in doi: for item in doi:
result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]}) result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
while True: while True:
#if in_q.empty():
# print("Queue empty!")
#if in_q.full():
# print("Queue full!")
done_check = True done_check = True
data = in_q.get() data = in_q.get()
if data is _sentinel: if data is _sentinel:
@@ -72,8 +69,9 @@ def search_for_doi(doi, live_results, logger):
logger.info("Consumer thread created") logger.info("Consumer thread created")
threads.append(t_cons) threads.append(t_cons)
for i in range(0,MAXTHREADS): for i in range(0,MAXTHREADS):
#filename = "test" + str(i) + CHUNk #TODO: TEST DATA
filename = "40" + CHUNk filename = "test" + str(i) + CHUNk
#TODO: DEPLOYMENT DATA
#filename = str(i) + CHUNk #filename = str(i) + CHUNk
logger.info(f"Creating worker thread no: {i}") logger.info(f"Creating worker thread no: {i}")
threads.append(Thread(target = producer, args = (work_q, control_q, filename))) threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
@@ -83,8 +81,6 @@ def search_for_doi(doi, live_results, logger):
worker.join() worker.join()
return result_list return result_list
if __name__ == "__main__": if __name__ == "__main__":
import logging import logging
logger = logging.getLogger() logger = logging.getLogger()