gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 21:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Music tunneling (#7)

Browse Source 
* New requirement

* SMall fixes for tunnel

* First step towards cogization

* Tmp chng

* Testing of new functions in conjurer_musician.py and radio_conjurer.liq

* Fixes!

* Fix fix

* Mostly working ?

* Requests

* Fixes

* Fix

* Radio requests

* Radio hardkor

* Fix

* Fixes

* Change log to print

* Clearing fetch failed

* Small fix

* sx

* Fix fix fix

* test

* el fixo stupido

* test

* Maybe fix ?

* FIX!

* Fix fix fix

* FIx of volume
This commit is contained in:
migatu
2024-04-23 12:59:33 +02:00
committed by GitHub
parent 6fefede271
commit 082b25c89c
8 changed files with 312 additions and 94 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/search_bot.py
+10 -9
View File
@@ -68,7 +68,7 @@ def producer(out_q, control_q, filename, _logger):
# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
def consumer(in_q, control_q, doi, live_results, result_list, _logger, ):
def consumer(in_q, control_q, doi, live_results, result_list, control_dict, _logger):
"""
Consumes items from an input queue and checks if DOI exists in the live results list.
@@ -79,7 +79,6 @@ def consumer(in_q, control_q, doi, live_results, result_list, _logger, ):
live_results (list): List to store the search results.
_logger: Logger object for logging.
"""
sentinels = 0
for item in doi:
result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
while True:
@@ -87,7 +86,7 @@ def consumer(in_q, control_q, doi, live_results, result_list, _logger, ):
data = in_q.get()
if data is _sentinel:
_logger.info("Worker finished")
sentinels += 1
control_dict["sentinels"] += 1
else:
for item in result_list:
if item["DOI"] in data and not item["exists"]:
@@ -98,7 +97,7 @@ def consumer(in_q, control_q, doi, live_results, result_list, _logger, ):
done_check = done_check and item["exists"]
if done_check:
control_q.put(_sentinel)
if sentinels >= MAXTHREADS:
if control_dict["sentinels"] >= MAXTHREADS:
_logger.info("All workers finished")
break
@@ -112,15 +111,17 @@ def search_for_doi(doi, live_results, _logger):
live_results (list): List to store the search results.
_logger: Logger object for logging.
"""
control_dict = {"sentinels":0}
result_list = []
threads = []
work_q = Queue(maxsize=WORK_Q_SIZE)
control_q = Queue()
t_cons = Thread(
target=consumer, args=(work_q, control_q, doi, live_results, result_list, _logger)
)
_logger.info("Consumer thread created")
threads.append(t_cons)
for _ in range (1, len(doi)//1000+1):
t_cons = Thread(
target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, _logger)
)
_logger.info("Consumer thread created")
threads.append(t_cons)
for i in range(0, MAXTHREADS):
# TEST DATA
# filename = "test" + str(i) + CHUNk