gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 21:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Search webservice (#2)

Browse Source 
* Deployable script for windows file service

* Deploy!

* Server side work
This commit is contained in:
migatu
2024-04-12 23:21:48 +02:00
committed by GitHub
parent d912d9bc1b
commit 0ac9006665
5 changed files with 152 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/search_bot.py
+19 -15
View File
@@ -5,22 +5,26 @@ from threading import Thread
q = Queue()
#TODO: DATA FOR TEST ONLY
MAXTHREADS = 5
DATABASE_PATH = r'C:\\Database\\chunks_1\\'
#MAXTHREADS = 5
#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
#TODO: DEPLOYMENT DATA
#MAXTHREADS = 40
#DATABASE_PATH = r'C:\\Database\\chunks\\'
MAXTHREADS = 41
DATABASE_PATH = r'C:\\Database\\chunks\\'
ENCODING = "utf-8"
CHUNk = "_chunk.txt"
_sentinel = object()
result_list = []
WORK_Q_SIZE = 15500000
def producer(out_q, control_q, filename):
def producer(out_q, control_q, filename, logger):
with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
lines = operated_file.readlines()
for line in lines:
logger.info(f"Worker {filename} ")
while True:
line = operated_file.readline()
if not line:
break
out_q.put(line)
try:
check = control_q.get(block=False)
@@ -28,11 +32,11 @@ def producer(out_q, control_q, filename):
check = False
pass
if check is _sentinel:
print("TERM signal received")
logger.info("TERM signal received")
control_q.put(check)
break
print(f"Worker finished: {filename}")
out_q.put(_sentinel)
logger.info(f"Worker finished: {filename}")
out_q.put(_sentinel)
#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
@@ -48,7 +52,7 @@ def consumer(in_q, control_q, doi, live_results, logger):
sentinels += 1
else:
for item in result_list:
if item["DOI"] in data:
if item["DOI"] in data and not item["exists"]:
logger.info(data)
logger.info("HIT")
item["exists"] = True
@@ -63,18 +67,18 @@ def consumer(in_q, control_q, doi, live_results, logger):
def search_for_doi(doi, live_results, logger):
threads = []
work_q = Queue(maxsize=200000)
work_q = Queue(maxsize=WORK_Q_SIZE)
control_q = Queue()
t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
logger.info("Consumer thread created")
threads.append(t_cons)
for i in range(0,MAXTHREADS):
#TODO: TEST DATA
filename = "test" + str(i) + CHUNk
#filename = "test" + str(i) + CHUNk
#TODO: DEPLOYMENT DATA
#filename = str(i) + CHUNk
filename = str(i) + CHUNk
logger.info(f"Creating worker thread no: {i}")
threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger)))
for worker in threads:
worker.start()
for worker in threads: