From 1a14b9b520d894820bcba33052f3166c133d25ab Mon Sep 17 00:00:00 2001 From: Michal Tuszowski Date: Mon, 8 Apr 2024 18:31:25 +0200 Subject: [PATCH] Librarian side fix --- conjurer_librarian/conjurer_librarian.py | 90 ++++++++++++++---------- conjurer_librarian/search_bot.py | 54 +++++++------- 2 files changed, 81 insertions(+), 63 deletions(-) diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index b9d3400..33f272b 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -7,12 +7,10 @@ import netrc import json import asyncio from json.decoder import JSONDecodeError -from platform import uname -from sys import platform from urllib.request import urlopen from habanero import Crossref from waitress import serve - +from queue import Queue import search_bot NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" @@ -21,33 +19,41 @@ HOST_ADDRESS = "192.168.1.192" PORT_ADDRESS = 5001 DATABASE_PATH = r'C:\\Database\\chunks' ENCODING = "utf-8" +app = Flask(__name__) +librarian_queue = Queue() class Librarian(object): - def __init__(self) -> None: + def __init__(self, app, query, uuid) -> None: netrc_mod = netrc.netrc(NETRC_FILE) authTokens = netrc_mod.authenticators("crossref") self.cr = Crossref(mailto=authTokens[0]) + self.query = query + self.uuid = uuid self.limit = 1000 self.fetched = 0 self.hit = 0 self.total = 0 + self.app = app async def search_crossref(self, query, uuid): + self.app.logger.info("STARTED SEARCH") search_result_from_cr = self.cr.works(query = query, limit = self.limit) + self.app.logger.info("CROSSREF DONE") self.total = search_result_from_cr['message']['total-results'] self.fetched = len(search_result_from_cr['message']['items']) #add to sql with UUID #sql row - title, doi, author, uuid, last retrieved, is_available + with open("results.json", "r+", encoding="UTF-8") as data_file: # First we load existing data into a dict. try: file_data = json.load(data_file) except JSONDecodeError: file_data = [] - # Join new_data with file_data inside emp_details summarized_results = [] + self.app.logger.info("REFINE: Removing all derived works from the list") for item in search_result_from_cr['message']['items']: container = "NO" if "container-title" in item: @@ -55,9 +61,8 @@ class Librarian(object): summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container}) result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'], } file_data.append(result) - print("Saving") data_file.seek(0) - # convert back to json. + self.app.logger.info("REFINE: Dumping to file") json.dump(file_data, data_file, indent=4) return result #TODO: check if exists odpala wątki dla kolejnych plików, z każdego wątku dostaje zwrotki - trafienia na bieżąco oraz sygnał że plik się skończył i pora na następny @@ -70,13 +75,13 @@ class Librarian(object): async def check_dois_in_local_db(self, doi): - #odwolanie do search_bota + self.app.logger.info("REFINE: Running search in the backend app") coro = asyncio.to_thread(search_bot.search_for_doi, doi) result = await coro return result - async def check_if_exists_brute_force(self, page_url): + self.app.logger.error("REFINE: Brute force search!") if not page_url.startswith(("http:", "https:")): raise ValueError("URL must start with 'http:' or 'https:'") # trunk-ignore(bandit/B310) @@ -92,39 +97,41 @@ class Librarian(object): return True return False - def provide_sci_hub_link(self, page_url): - if self.check_if_exists(page_url): - return page_url - return None - async def check_if_exists(self, doi, brute_force = False): + result = [] result = await self.check_dois_in_local_db(doi) if brute_force and not result: - item_link = "https://sci-hub.se/" + doi - result = await self.check_if_exists_brute_force(item_link) + for item in doi: + item_link = "https://sci-hub.se/" + item[0] + result.append(await self.check_if_exists_brute_force(item_link)) return result async def refine_query(self, unrefined_reqult, brute_force = False): print("Refine result") refined_result = [] + temp = [] for item in unrefined_reqult["summary"]: if item["container"] == "NO" and item["title"]: - result = await self.check_if_exists(str(item["DOI"])) - if result: - refined_result.append(item) - print ("HIT") - print(item["title"]) - time.sleep(0.01) - print("Existing hits") + refined_result.append[item] + for item in refined_result: + temp.append[(item["DOI"], item)] + + refined_result = self.check_if_exists(temp) + temp = [] + for item in refined_result: + if item["exists"]: + temp.append(item["data"]) + refined_result = temp self.hit = len(refined_result) return refined_result - async def answer_query(self, query, uuid): + async def answer_query(self): #Search sql for UUID #SQL SELECT * FROM RESULTS WHERE UUID = uuid - print("Answer result") uuid_found = False database = None + query = self.query + uuid = self.uuid with open("results.json", "r", encoding="UTF-8") as new_file: #tu logika komunikacji z SQL i cała logika związana z wyszukaniem #ale tymczasowo plik @@ -146,14 +153,7 @@ class Librarian(object): print("Refined") return await self.refine_query(answer) -#example not exists -#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237" -#sample usage to download -#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026" -#print(cl.download(sample_existing_page_url)) -#cl.search_crossref("BDSM") - -app = Flask(__name__) +#============================= FLASK INTERNALS=============================== def flask_debug(): """ The `flask_debug` function starts a Flask application in debug mode without using the reloader. @@ -169,24 +169,26 @@ def waitress_run(): """ serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS) +# TUTAJ TYLKO LOGUJEMY ŻE PRZYSZŁO, TWORZYMY LIBRARIAN class dla danego zapytania (po UUID) +# ODPALAMY SZUKANIE I POZWALAMYSIĘ JUŻ DALEJ KRĘCIĆ +#==================================SERVER ROUTES========================================== @app.route("/query", methods=["POST"]) async def query_database(): record = json.loads(request.data) app.logger.info(record) app.logger.info(record["query"]) app.logger.info(record["UUID"]) - cl = Librarian() - answer_data = await cl.answer_query(record["query"], record["UUID"]) + cl = Librarian(app, record["query"], record["UUID"]) + librarian_queue.put(cl) - app.logger.info(answer_data) - summary = (cl.hit,cl.fetched, cl.total, record["UUID"]) - answer_data = (summary,answer_data) + + answer_data = (record["query"], record["UUID"], librarian_queue.qsize()) return_data = ( jsonify(isError=False, message="Success", statusCode=200, data=answer_data), 200, ) return return_data - +#=======================================MAIN=================================================== if __name__ == "__main__": logger = logging.getLogger('conjurer_musician') logger.setLevel(logging.DEBUG) @@ -198,3 +200,13 @@ if __name__ == "__main__": worker.start() for worker in threads: worker.join() + + + + +#example not exists +#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237" +#sample usage to download +#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026" +#print(cl.download(sample_existing_page_url)) +#cl.search_crossref("BDSM") diff --git a/conjurer_librarian/search_bot.py b/conjurer_librarian/search_bot.py index 7406f4c..a9220a2 100644 --- a/conjurer_librarian/search_bot.py +++ b/conjurer_librarian/search_bot.py @@ -2,13 +2,13 @@ from queue import Queue, Empty from threading import Thread q = Queue() -MAXTHREADS = 39 -#MAXTHREADS = 4 +#MAXTHREADS = 39 +MAXTHREADS = 4 DATABASE_PATH = r'C:\\Database\\chunks_1\\' ENCODING = "utf-8" CHUNk = "_chunk.txt" _sentinel = object() - +result_list = [] #TODO: Doisy wchodzą jako lista krotek (doi, False) - po znalezieniu zaznaczamy na True i potem już nie porównujemy #TODO: lista krotek - jeśli wszystkie mają True dajemy terma @@ -35,15 +35,23 @@ def producer(out_q, control_q, filename): #TODO: Change doi to tuple list and only put term when all doi are found => (doi, True) def consumer(in_q, control_q, doi): sentinels = 0 + for item in doi: + result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]}) while True: + done_check = True data = in_q.get() if data is _sentinel: print("Worker finished") sentinels += 1 - elif doi in data: - control_q.put(_sentinel) - print(data) - print("HIT") + else: + for item in result_list: + if item["DOI"] in data: + print(data) + print("HIT") + item["exists"] = True + done_check = done_check and item["exists"] + if done_check: + control_q.put(_sentinel) if sentinels >= MAXTHREADS: print("All workers finished") break @@ -51,31 +59,29 @@ def consumer(in_q, control_q, doi): def search_for_doi(doi): threads = [] - work_q = Queue(maxsize=10000000) + work_q = Queue(maxsize=100) control_q = Queue() t_cons = Thread(target = consumer, args = (work_q, control_q, doi)) threads.append(t_cons) for i in range(0,MAXTHREADS): - #filename = "test" + str(i) + CHUNk + filename = "test" + str(i) + CHUNk filename = str(i) + CHUNk - print(filename) + #print(filename) threads.append(Thread(target = producer, args = (work_q, control_q, filename))) for worker in threads: worker.start() for worker in threads: worker.join() - - try: - check = control_q.get(block=False) - control_q.put(check) - return True - except Empty: - return False - + return result_list if __name__ == "__main__": - if search_for_doi("10.1002/9781118786352.wbieg0998.pub2"): - #if search_for_doi("10.1108/gm-02-2016-0025xx"): - print("found") - exit() - else: - print("Not found") \ No newline at end of file + print(search_for_doi([ + ("10.1002/9781118786352.wbieg0998.pub2","DATA"), + ("10.1002/j.2050-0416.2002.tb00563.x","DATA"), + ("10.1111/j.1365-2958.1994.tb00448.x","DATA"), + ("10.2165/00128415-200309690-00017","DATA"), + ("10.5772/48313","DATA"), + ("10.15803/ijnc.7.2_419","DATA"), + ("10.1051/0004-6361/201321596e","DATA"), + ("10.2307/40835941","DATA"), + ])) +