gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-18 07:42:09 +00:00
Code Issues Packages Projects Releases Wiki Activity

Librarian side fix

Browse Source
This commit is contained in:
Michal Tuszowski
2024-04-08 18:31:25 +02:00
parent 6d3225ad00
commit 1a14b9b520
2 changed files with 81 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/conjurer_librarian.py
+51 -39
View File
@@ -7,12 +7,10 @@ import netrc
import json import json
import asyncio import asyncio
from json.decoder import JSONDecodeError from json.decoder import JSONDecodeError
from platform import uname
from sys import platform
from urllib.request import urlopen from urllib.request import urlopen
from habanero import Crossref from habanero import Crossref
from waitress import serve from waitress import serve
from queue import Queue
import search_bot import search_bot
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
@@ -21,33 +19,41 @@ HOST_ADDRESS = "192.168.1.192"
PORT_ADDRESS = 5001 PORT_ADDRESS = 5001
DATABASE_PATH = r'C:\\Database\\chunks' DATABASE_PATH = r'C:\\Database\\chunks'
ENCODING = "utf-8" ENCODING = "utf-8"
app = Flask(__name__)
librarian_queue = Queue()
class Librarian(object): class Librarian(object):
def __init__(self) -> None: def __init__(self, app, query, uuid) -> None:
netrc_mod = netrc.netrc(NETRC_FILE) netrc_mod = netrc.netrc(NETRC_FILE)
authTokens = netrc_mod.authenticators("crossref") authTokens = netrc_mod.authenticators("crossref")
self.cr = Crossref(mailto=authTokens[0]) self.cr = Crossref(mailto=authTokens[0])
self.query = query
self.uuid = uuid
self.limit = 1000 self.limit = 1000
self.fetched = 0 self.fetched = 0
self.hit = 0 self.hit = 0
self.total = 0 self.total = 0
self.app = app
async def search_crossref(self, query, uuid): async def search_crossref(self, query, uuid):
self.app.logger.info("STARTED SEARCH")
search_result_from_cr = self.cr.works(query = query, limit = self.limit) search_result_from_cr = self.cr.works(query = query, limit = self.limit)
self.app.logger.info("CROSSREF DONE")
self.total = search_result_from_cr['message']['total-results'] self.total = search_result_from_cr['message']['total-results']
self.fetched = len(search_result_from_cr['message']['items']) self.fetched = len(search_result_from_cr['message']['items'])
#add to sql with UUID #add to sql with UUID
#sql row - title, doi, author, uuid, last retrieved, is_available #sql row - title, doi, author, uuid, last retrieved, is_available
with open("results.json", "r+", encoding="UTF-8") as data_file: with open("results.json", "r+", encoding="UTF-8") as data_file:
# First we load existing data into a dict. # First we load existing data into a dict.
try: try:
file_data = json.load(data_file) file_data = json.load(data_file)
except JSONDecodeError: except JSONDecodeError:
file_data = [] file_data = []
# Join new_data with file_data inside emp_details
summarized_results = [] summarized_results = []
self.app.logger.info("REFINE: Removing all derived works from the list")
for item in search_result_from_cr['message']['items']: for item in search_result_from_cr['message']['items']:
container = "NO" container = "NO"
if "container-title" in item: if "container-title" in item:
@@ -55,9 +61,8 @@ class Librarian(object):
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container}) summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'], } result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'], }
file_data.append(result) file_data.append(result)
print("Saving")
data_file.seek(0) data_file.seek(0)
# convert back to json. self.app.logger.info("REFINE: Dumping to file")
json.dump(file_data, data_file, indent=4) json.dump(file_data, data_file, indent=4)
return result return result
#TODO: check if exists odpala wątki dla kolejnych plików, z każdego wątku dostaje zwrotki - trafienia na bieżąco oraz sygnał że plik się skończył i pora na następny #TODO: check if exists odpala wątki dla kolejnych plików, z każdego wątku dostaje zwrotki - trafienia na bieżąco oraz sygnał że plik się skończył i pora na następny
@@ -70,13 +75,13 @@ class Librarian(object):
async def check_dois_in_local_db(self, doi): async def check_dois_in_local_db(self, doi):
#odwolanie do search_bota self.app.logger.info("REFINE: Running search in the backend app")
coro = asyncio.to_thread(search_bot.search_for_doi, doi) coro = asyncio.to_thread(search_bot.search_for_doi, doi)
result = await coro result = await coro
return result return result
async def check_if_exists_brute_force(self, page_url): async def check_if_exists_brute_force(self, page_url):
self.app.logger.error("REFINE: Brute force search!")
if not page_url.startswith(("http:", "https:")): if not page_url.startswith(("http:", "https:")):
raise ValueError("URL must start with 'http:' or 'https:'") raise ValueError("URL must start with 'http:' or 'https:'")
# trunk-ignore(bandit/B310) # trunk-ignore(bandit/B310)
@@ -92,39 +97,41 @@ class Librarian(object):
return True return True
return False return False
def provide_sci_hub_link(self, page_url):
if self.check_if_exists(page_url):
return page_url
return None
async def check_if_exists(self, doi, brute_force = False): async def check_if_exists(self, doi, brute_force = False):
result = []
result = await self.check_dois_in_local_db(doi) result = await self.check_dois_in_local_db(doi)
if brute_force and not result: if brute_force and not result:
item_link = "https://sci-hub.se/" + doi for item in doi:
result = await self.check_if_exists_brute_force(item_link) item_link = "https://sci-hub.se/" + item[0]
result.append(await self.check_if_exists_brute_force(item_link))
return result return result
async def refine_query(self, unrefined_reqult, brute_force = False): async def refine_query(self, unrefined_reqult, brute_force = False):
print("Refine result") print("Refine result")
refined_result = [] refined_result = []
temp = []
for item in unrefined_reqult["summary"]: for item in unrefined_reqult["summary"]:
if item["container"] == "NO" and item["title"]: if item["container"] == "NO" and item["title"]:
result = await self.check_if_exists(str(item["DOI"])) refined_result.append[item]
if result: for item in refined_result:
refined_result.append(item) temp.append[(item["DOI"], item)]
print ("HIT")
print(item["title"]) refined_result = self.check_if_exists(temp)
time.sleep(0.01) temp = []
print("Existing hits") for item in refined_result:
if item["exists"]:
temp.append(item["data"])
refined_result = temp
self.hit = len(refined_result) self.hit = len(refined_result)
return refined_result return refined_result
async def answer_query(self, query, uuid): async def answer_query(self):
#Search sql for UUID #Search sql for UUID
#SQL SELECT * FROM RESULTS WHERE UUID = uuid #SQL SELECT * FROM RESULTS WHERE UUID = uuid
print("Answer result")
uuid_found = False uuid_found = False
database = None database = None
query = self.query
uuid = self.uuid
with open("results.json", "r", encoding="UTF-8") as new_file: with open("results.json", "r", encoding="UTF-8") as new_file:
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
#ale tymczasowo plik #ale tymczasowo plik
@@ -146,14 +153,7 @@ class Librarian(object):
print("Refined") print("Refined")
return await self.refine_query(answer) return await self.refine_query(answer)
#example not exists #============================= FLASK INTERNALS===============================
#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
#sample usage to download
#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
#print(cl.download(sample_existing_page_url))
#cl.search_crossref("BDSM")
app = Flask(__name__)
def flask_debug(): def flask_debug():
""" """
The `flask_debug` function starts a Flask application in debug mode without using the reloader. The `flask_debug` function starts a Flask application in debug mode without using the reloader.
@@ -169,24 +169,26 @@ def waitress_run():
""" """
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS) serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
# TUTAJ TYLKO LOGUJEMY ŻE PRZYSZŁO, TWORZYMY LIBRARIAN class dla danego zapytania (po UUID)
# ODPALAMY SZUKANIE I POZWALAMYSIĘ JUŻ DALEJ KRĘCIĆ
#==================================SERVER ROUTES==========================================
@app.route("/query", methods=["POST"]) @app.route("/query", methods=["POST"])
async def query_database(): async def query_database():
record = json.loads(request.data) record = json.loads(request.data)
app.logger.info(record) app.logger.info(record)
app.logger.info(record["query"]) app.logger.info(record["query"])
app.logger.info(record["UUID"]) app.logger.info(record["UUID"])
cl = Librarian() cl = Librarian(app, record["query"], record["UUID"])
answer_data = await cl.answer_query(record["query"], record["UUID"]) librarian_queue.put(cl)
app.logger.info(answer_data)
summary = (cl.hit,cl.fetched, cl.total, record["UUID"]) answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
answer_data = (summary,answer_data)
return_data = ( return_data = (
jsonify(isError=False, message="Success", statusCode=200, data=answer_data), jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
200, 200,
) )
return return_data return return_data
#=======================================MAIN===================================================
if __name__ == "__main__": if __name__ == "__main__":
logger = logging.getLogger('conjurer_musician') logger = logging.getLogger('conjurer_musician')
logger.setLevel(logging.DEBUG) logger.setLevel(logging.DEBUG)
@@ -198,3 +200,13 @@ if __name__ == "__main__":
worker.start() worker.start()
for worker in threads: for worker in threads:
worker.join() worker.join()
#example not exists
#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
#sample usage to download
#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
#print(cl.download(sample_existing_page_url))
#cl.search_crossref("BDSM")
conjurer_librarian/search_bot.py
+30 -24
View File
@@ -2,13 +2,13 @@ from queue import Queue, Empty
from threading import Thread from threading import Thread
q = Queue() q = Queue()
MAXTHREADS = 39 #MAXTHREADS = 39
#MAXTHREADS = 4 MAXTHREADS = 4
DATABASE_PATH = r'C:\\Database\\chunks_1\\' DATABASE_PATH = r'C:\\Database\\chunks_1\\'
ENCODING = "utf-8" ENCODING = "utf-8"
CHUNk = "_chunk.txt" CHUNk = "_chunk.txt"
_sentinel = object() _sentinel = object()
result_list = []
#TODO: Doisy wchodzą jako lista krotek (doi, False) - po znalezieniu zaznaczamy na True i potem już nie porównujemy #TODO: Doisy wchodzą jako lista krotek (doi, False) - po znalezieniu zaznaczamy na True i potem już nie porównujemy
#TODO: lista krotek - jeśli wszystkie mają True dajemy terma #TODO: lista krotek - jeśli wszystkie mają True dajemy terma
@@ -35,15 +35,23 @@ def producer(out_q, control_q, filename):
#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True) #TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
def consumer(in_q, control_q, doi): def consumer(in_q, control_q, doi):
sentinels = 0 sentinels = 0
for item in doi:
result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
while True: while True:
done_check = True
data = in_q.get() data = in_q.get()
if data is _sentinel: if data is _sentinel:
print("Worker finished") print("Worker finished")
sentinels += 1 sentinels += 1
elif doi in data: else:
control_q.put(_sentinel) for item in result_list:
print(data) if item["DOI"] in data:
print("HIT") print(data)
print("HIT")
item["exists"] = True
done_check = done_check and item["exists"]
if done_check:
control_q.put(_sentinel)
if sentinels >= MAXTHREADS: if sentinels >= MAXTHREADS:
print("All workers finished") print("All workers finished")
break break
@@ -51,31 +59,29 @@ def consumer(in_q, control_q, doi):
def search_for_doi(doi): def search_for_doi(doi):
threads = [] threads = []
work_q = Queue(maxsize=10000000) work_q = Queue(maxsize=100)
control_q = Queue() control_q = Queue()
t_cons = Thread(target = consumer, args = (work_q, control_q, doi)) t_cons = Thread(target = consumer, args = (work_q, control_q, doi))
threads.append(t_cons) threads.append(t_cons)
for i in range(0,MAXTHREADS): for i in range(0,MAXTHREADS):
#filename = "test" + str(i) + CHUNk filename = "test" + str(i) + CHUNk
filename = str(i) + CHUNk filename = str(i) + CHUNk
print(filename) #print(filename)
threads.append(Thread(target = producer, args = (work_q, control_q, filename))) threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
for worker in threads: for worker in threads:
worker.start() worker.start()
for worker in threads: for worker in threads:
worker.join() worker.join()
return result_list
try:
check = control_q.get(block=False)
control_q.put(check)
return True
except Empty:
return False
if __name__ == "__main__": if __name__ == "__main__":
if search_for_doi("10.1002/9781118786352.wbieg0998.pub2"): print(search_for_doi([
#if search_for_doi("10.1108/gm-02-2016-0025xx"): ("10.1002/9781118786352.wbieg0998.pub2","DATA"),
print("found") ("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
exit() ("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
else: ("10.2165/00128415-200309690-00017","DATA"),
print("Not found") ("10.5772/48313","DATA"),
("10.15803/ijnc.7.2_419","DATA"),
("10.1051/0004-6361/201321596e","DATA"),
("10.2307/40835941","DATA"),
]))