gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 21:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Librarian side fix

Browse Source
This commit is contained in:
Michal Tuszowski
2024-04-08 18:31:25 +02:00
parent 6d3225ad00
commit 1a14b9b520
2 changed files with 81 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/conjurer_librarian.py
+51 -39
View File
@@ -7,12 +7,10 @@ import netrc
import json
import asyncio
from json.decoder import JSONDecodeError
from platform import uname
from sys import platform
from urllib.request import urlopen
from habanero import Crossref
from waitress import serve
from queue import Queue
import search_bot
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
@@ -21,33 +19,41 @@ HOST_ADDRESS = "192.168.1.192"
PORT_ADDRESS = 5001
DATABASE_PATH = r'C:\\Database\\chunks'
ENCODING = "utf-8"
app = Flask(__name__)
librarian_queue = Queue()
class Librarian(object):
def __init__(self) -> None:
def __init__(self, app, query, uuid) -> None:
netrc_mod = netrc.netrc(NETRC_FILE)
authTokens = netrc_mod.authenticators("crossref")
self.cr = Crossref(mailto=authTokens[0])
self.query = query
self.uuid = uuid
self.limit = 1000
self.fetched = 0
self.hit = 0
self.total = 0
self.app = app
async def search_crossref(self, query, uuid):
self.app.logger.info("STARTED SEARCH")
search_result_from_cr = self.cr.works(query = query, limit = self.limit)
self.app.logger.info("CROSSREF DONE")
self.total = search_result_from_cr['message']['total-results']
self.fetched = len(search_result_from_cr['message']['items'])
#add to sql with UUID
#sql row - title, doi, author, uuid, last retrieved, is_available
with open("results.json", "r+", encoding="UTF-8") as data_file:
# First we load existing data into a dict.
try:
file_data = json.load(data_file)
except JSONDecodeError:
file_data = []
# Join new_data with file_data inside emp_details
summarized_results = []
self.app.logger.info("REFINE: Removing all derived works from the list")
for item in search_result_from_cr['message']['items']:
container = "NO"
if "container-title" in item:
@@ -55,9 +61,8 @@ class Librarian(object):
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'], }
file_data.append(result)
print("Saving")
data_file.seek(0)
# convert back to json.
self.app.logger.info("REFINE: Dumping to file")
json.dump(file_data, data_file, indent=4)
return result
#TODO: check if exists odpala wątki dla kolejnych plików, z każdego wątku dostaje zwrotki - trafienia na bieżąco oraz sygnał że plik się skończył i pora na następny
@@ -70,13 +75,13 @@ class Librarian(object):
async def check_dois_in_local_db(self, doi):
#odwolanie do search_bota
self.app.logger.info("REFINE: Running search in the backend app")
coro = asyncio.to_thread(search_bot.search_for_doi, doi)
result = await coro
return result
async def check_if_exists_brute_force(self, page_url):
self.app.logger.error("REFINE: Brute force search!")
if not page_url.startswith(("http:", "https:")):
raise ValueError("URL must start with 'http:' or 'https:'")
# trunk-ignore(bandit/B310)
@@ -92,39 +97,41 @@ class Librarian(object):
return True
return False
def provide_sci_hub_link(self, page_url):
if self.check_if_exists(page_url):
return page_url
return None
async def check_if_exists(self, doi, brute_force = False):
result = []
result = await self.check_dois_in_local_db(doi)
if brute_force and not result:
item_link = "https://sci-hub.se/" + doi
result = await self.check_if_exists_brute_force(item_link)
for item in doi:
item_link = "https://sci-hub.se/" + item[0]
result.append(await self.check_if_exists_brute_force(item_link))
return result
async def refine_query(self, unrefined_reqult, brute_force = False):
print("Refine result")
refined_result = []
temp = []
for item in unrefined_reqult["summary"]:
if item["container"] == "NO" and item["title"]:
result = await self.check_if_exists(str(item["DOI"]))
if result:
refined_result.append(item)
print ("HIT")
print(item["title"])
time.sleep(0.01)
print("Existing hits")
refined_result.append[item]
for item in refined_result:
temp.append[(item["DOI"], item)]
refined_result = self.check_if_exists(temp)
temp = []
for item in refined_result:
if item["exists"]:
temp.append(item["data"])
refined_result = temp
self.hit = len(refined_result)
return refined_result
async def answer_query(self, query, uuid):
async def answer_query(self):
#Search sql for UUID
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
print("Answer result")
uuid_found = False
database = None
query = self.query
uuid = self.uuid
with open("results.json", "r", encoding="UTF-8") as new_file:
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem
#ale tymczasowo plik
@@ -146,14 +153,7 @@ class Librarian(object):
print("Refined")
return await self.refine_query(answer)
#example not exists
#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
#sample usage to download
#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
#print(cl.download(sample_existing_page_url))
#cl.search_crossref("BDSM")
app = Flask(__name__)
#============================= FLASK INTERNALS===============================
def flask_debug():
"""
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
@@ -169,24 +169,26 @@ def waitress_run():
"""
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
# TUTAJ TYLKO LOGUJEMY ŻE PRZYSZŁO, TWORZYMY LIBRARIAN class dla danego zapytania (po UUID)
# ODPALAMY SZUKANIE I POZWALAMYSIĘ JUŻ DALEJ KRĘCIĆ
#==================================SERVER ROUTES==========================================
@app.route("/query", methods=["POST"])
async def query_database():
record = json.loads(request.data)
app.logger.info(record)
app.logger.info(record["query"])
app.logger.info(record["UUID"])
cl = Librarian()
answer_data = await cl.answer_query(record["query"], record["UUID"])
cl = Librarian(app, record["query"], record["UUID"])
librarian_queue.put(cl)
app.logger.info(answer_data)
summary = (cl.hit,cl.fetched, cl.total, record["UUID"])
answer_data = (summary,answer_data)
answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
return_data = (
jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
200,
)
return return_data
#=======================================MAIN===================================================
if __name__ == "__main__":
logger = logging.getLogger('conjurer_musician')
logger.setLevel(logging.DEBUG)
@@ -198,3 +200,13 @@ if __name__ == "__main__":
worker.start()
for worker in threads:
worker.join()
#example not exists
#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
#sample usage to download
#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
#print(cl.download(sample_existing_page_url))
#cl.search_crossref("BDSM")
conjurer_librarian/search_bot.py
+30 -24
View File
@@ -2,13 +2,13 @@ from queue import Queue, Empty
from threading import Thread
q = Queue()
MAXTHREADS = 39
#MAXTHREADS = 4
#MAXTHREADS = 39
MAXTHREADS = 4
DATABASE_PATH = r'C:\\Database\\chunks_1\\'
ENCODING = "utf-8"
CHUNk = "_chunk.txt"
_sentinel = object()
result_list = []
#TODO: Doisy wchodzą jako lista krotek (doi, False) - po znalezieniu zaznaczamy na True i potem już nie porównujemy
#TODO: lista krotek - jeśli wszystkie mają True dajemy terma
@@ -35,15 +35,23 @@ def producer(out_q, control_q, filename):
#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
def consumer(in_q, control_q, doi):
sentinels = 0
for item in doi:
result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
while True:
done_check = True
data = in_q.get()
if data is _sentinel:
print("Worker finished")
sentinels += 1
elif doi in data:
control_q.put(_sentinel)
print(data)
print("HIT")
else:
for item in result_list:
if item["DOI"] in data:
print(data)
print("HIT")
item["exists"] = True
done_check = done_check and item["exists"]
if done_check:
control_q.put(_sentinel)
if sentinels >= MAXTHREADS:
print("All workers finished")
break
@@ -51,31 +59,29 @@ def consumer(in_q, control_q, doi):
def search_for_doi(doi):
threads = []
work_q = Queue(maxsize=10000000)
work_q = Queue(maxsize=100)
control_q = Queue()
t_cons = Thread(target = consumer, args = (work_q, control_q, doi))
threads.append(t_cons)
for i in range(0,MAXTHREADS):
#filename = "test" + str(i) + CHUNk
filename = "test" + str(i) + CHUNk
filename = str(i) + CHUNk
print(filename)
#print(filename)
threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
for worker in threads:
worker.start()
for worker in threads:
worker.join()
try:
check = control_q.get(block=False)
control_q.put(check)
return True
except Empty:
return False
return result_list
if __name__ == "__main__":
if search_for_doi("10.1002/9781118786352.wbieg0998.pub2"):
#if search_for_doi("10.1108/gm-02-2016-0025xx"):
print("found")
exit()
else:
print("Not found")
print(search_for_doi([
("10.1002/9781118786352.wbieg0998.pub2","DATA"),
("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
("10.2165/00128415-200309690-00017","DATA"),
("10.5772/48313","DATA"),
("10.15803/ijnc.7.2_419","DATA"),
("10.1051/0004-6361/201321596e","DATA"),
("10.2307/40835941","DATA"),
]))