diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index b1b627a..bf194cf 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -5,6 +5,7 @@ import time from flask import Flask, jsonify, request import netrc import json +import asyncio from json.decoder import JSONDecodeError from platform import uname from sys import platform @@ -12,14 +13,9 @@ from urllib.request import urlopen from habanero import Crossref from waitress import serve -if platform in ("linux", "linux2"): - SEPARATOR_FILE_PATH = "/" - if "microsoft-standard" in uname().release: - NETRC_FILE = "/home/mtuszowski/.netrc" - else: - NETRC_FILE = "/home/pi/.netrc" -elif platform == "win32": - NETRC_FILE = "C:\\Users\\mtusz\\.netrc" +import search_bot + +NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" #HOST_ADDRESS = "192.168.1.191" HOST_ADDRESS = "0.0.0.0" @@ -39,7 +35,7 @@ class Librarian(object): self.hit = 0 self.total = 0 - def search_crossref(self, query, uuid): + async def search_crossref(self, query, uuid): search_result_from_cr = self.cr.works(query = query, limit = self.limit) self.total = search_result_from_cr['message']['total-results'] self.fetched = len(search_result_from_cr['message']['items']) @@ -74,14 +70,14 @@ class Librarian(object): #TODO: Paginacja - najpierw hity z pliku, potem na końcu to co nie zwróciło żadnych hitów - i to dopiero można crawlerem sprawdzić - def check_dois_in_local_db(self, dois): + async def check_dois_in_local_db(self, doi): #odwolanie do search_bota - - result = False + coro = asyncio.to_thread(search_bot.search_for_doi, doi) + result = await coro return result - def check_if_exists_brute_force(self, page_url): + async def check_if_exists_brute_force(self, page_url): if not page_url.startswith(("http:", "https:")): raise ValueError("URL must start with 'http:' or 'https:'") # trunk-ignore(bandit/B310) @@ -102,19 +98,20 @@ class Librarian(object): return page_url return None - def check_if_exists(self, doi, brute_force = False): - self.check_dois_in_local_db(doi) - if brute_force: + async def check_if_exists(self, doi, brute_force = False): + result = await self.check_dois_in_local_db(doi) + if brute_force and not result: item_link = "https://sci-hub.se/" + doi - self.check_if_exists_brute_force(item_link) - return False + result = await self.check_if_exists_brute_force(item_link) + return result - def refine_query(self, unrefined_reqult, brute_force = False): + async def refine_query(self, unrefined_reqult, brute_force = False): print("Refine result") refined_result = [] for item in unrefined_reqult["summary"]: if item["container"] == "NO" and item["title"]: - if self.check_if_exists(str(item["DOI"])): + result = await self.check_if_exists(str(item["DOI"])) + if result: refined_result.append(item) print ("HIT") print(item["title"]) @@ -123,7 +120,7 @@ class Librarian(object): self.hit = len(refined_result) return refined_result - def answer_query(self, query, uuid): + async def answer_query(self, query, uuid): #Search sql for UUID #SQL SELECT * FROM RESULTS WHERE UUID = uuid print("Answer result") @@ -142,11 +139,13 @@ class Librarian(object): print (item["UUID"]) uuid_found = True #TODO: THREAD IT - return self.refine_query(item) + result = await self.refine_query(item) + return result if not uuid_found: - answer = self.search_crossref(query=query, uuid=uuid) - return self.refine_query(answer) + answer = await self.search_crossref(query=query, uuid=uuid) + print("Refined") + return await self.refine_query(answer) #example not exists #sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237" @@ -172,13 +171,14 @@ def waitress_run(): serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS) @app.route("/query", methods=["POST"]) -def query_database(): +async def query_database(): record = json.loads(request.data) app.logger.info(record) app.logger.info(record["query"]) app.logger.info(record["UUID"]) cl = Librarian() - answer_data = cl.answer_query(record["query"], record["UUID"]) + answer_data = await cl.answer_query(record["query"], record["UUID"]) + app.logger.info(answer_data) summary = (cl.hit,cl.fetched, cl.total, record["UUID"]) answer_data = (summary,answer_data) diff --git a/conjurer_librarian/search_bot.py b/conjurer_librarian/search_bot.py index 9fc9fad..22dacea 100644 --- a/conjurer_librarian/search_bot.py +++ b/conjurer_librarian/search_bot.py @@ -3,6 +3,7 @@ from threading import Thread q = Queue() MAXTHREADS = 39 +#MAXTHREADS = 4 DATABASE_PATH = r'C:\\Database\\chunks_1\\' ENCODING = "utf-8" CHUNk = "_chunk.txt" @@ -46,7 +47,7 @@ def consumer(in_q, control_q, doi): def search_for_doi(doi): threads = [] - work_q = Queue() + work_q = Queue(maxsize=10000000) control_q = Queue() t_cons = Thread(target = consumer, args = (work_q, control_q, doi)) threads.append(t_cons) @@ -68,7 +69,8 @@ def search_for_doi(doi): return False if __name__ == "__main__": - if search_for_doi("10.1145/2559206.2567832"): + if search_for_doi("10.1002/9781118786352.wbieg0998.pub2"): + #if search_for_doi("10.1108/gm-02-2016-0025xx"): print("found") exit() else: