Librarian side work

conjurer_librarian/conjurer_librarian.py

@@ -5,6 +5,7 @@ import time
 from flask import Flask, jsonify, request
 import netrc
 import json
+import asyncio
 from json.decoder import JSONDecodeError
 from platform import uname
 from sys import platform
@@ -12,14 +13,9 @@ from urllib.request import urlopen
 from habanero import Crossref
 from waitress import serve
 
-if platform in ("linux", "linux2"):
+import search_bot
-    SEPARATOR_FILE_PATH = "/"
+
-    if "microsoft-standard" in uname().release:
+NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
-        NETRC_FILE = "/home/mtuszowski/.netrc"
-    else:
-        NETRC_FILE = "/home/pi/.netrc"
-elif platform == "win32":
-    NETRC_FILE = "C:\\Users\\mtusz\\.netrc"
 
 #HOST_ADDRESS = "192.168.1.191"
 HOST_ADDRESS = "0.0.0.0"
@@ -39,7 +35,7 @@ class Librarian(object):
         self.hit = 0
         self.total = 0
 
-    def search_crossref(self, query, uuid):
+    async def search_crossref(self, query, uuid):
         search_result_from_cr = self.cr.works(query = query, limit = self.limit)
         self.total = search_result_from_cr['message']['total-results']
         self.fetched = len(search_result_from_cr['message']['items'])
@@ -74,14 +70,14 @@ class Librarian(object):
         #TODO: Paginacja - najpierw hity z pliku, potem na końcu to co nie zwróciło żadnych hitów - i to dopiero można crawlerem sprawdzić
 
 
-    def check_dois_in_local_db(self, dois):
+    async def check_dois_in_local_db(self, doi):
         #odwolanie do search_bota
-
+        coro = asyncio.to_thread(search_bot.search_for_doi, doi)
-        result = False
+        result = await coro
         return result
 
 
-    def check_if_exists_brute_force(self, page_url):
+    async def check_if_exists_brute_force(self, page_url):
         if not page_url.startswith(("http:", "https:")):
             raise ValueError("URL must start with 'http:' or 'https:'")
         # trunk-ignore(bandit/B310)
@@ -102,19 +98,20 @@ class Librarian(object):
             return page_url
         return None
 
-    def check_if_exists(self, doi, brute_force = False):
+    async def check_if_exists(self, doi, brute_force = False):
-        self.check_dois_in_local_db(doi)
+        result = await self.check_dois_in_local_db(doi)
-        if brute_force:
+        if brute_force and not result:
             item_link = "https://sci-hub.se/" + doi
-            self.check_if_exists_brute_force(item_link)
+            result = await self.check_if_exists_brute_force(item_link)
-        return False
+        return result
 
-    def refine_query(self, unrefined_reqult, brute_force = False):
+    async def refine_query(self, unrefined_reqult, brute_force = False):
         print("Refine result")
         refined_result = []
         for item in unrefined_reqult["summary"]:
             if item["container"] == "NO" and item["title"]:
-                if self.check_if_exists(str(item["DOI"])):
+                result = await self.check_if_exists(str(item["DOI"]))
+                if result:
                     refined_result.append(item)
                     print ("HIT")
                     print(item["title"])
@@ -123,7 +120,7 @@ class Librarian(object):
         self.hit = len(refined_result)
         return refined_result
 
-    def answer_query(self, query, uuid):
+    async def answer_query(self, query, uuid):
         #Search sql for UUID
         #SQL SELECT * FROM RESULTS WHERE UUID = uuid
         print("Answer result")
@@ -142,11 +139,13 @@ class Librarian(object):
                     print (item["UUID"])
                     uuid_found = True
                     #TODO: THREAD IT
-                    return self.refine_query(item)
+                    result = await self.refine_query(item)
+                    return result
 
         if not uuid_found:
-            answer = self.search_crossref(query=query, uuid=uuid)
+            answer = await self.search_crossref(query=query, uuid=uuid)
-            return self.refine_query(answer)
+            print("Refined")
+            return await self.refine_query(answer)
 
 #example not exists
 #sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
@@ -172,13 +171,14 @@ def waitress_run():
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
 @app.route("/query", methods=["POST"])
-def query_database():
+async def query_database():
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["query"])
     app.logger.info(record["UUID"])
     cl = Librarian()
-    answer_data = cl.answer_query(record["query"], record["UUID"])
+    answer_data = await cl.answer_query(record["query"], record["UUID"])
+
     app.logger.info(answer_data)
     summary = (cl.hit,cl.fetched, cl.total, record["UUID"])
     answer_data = (summary,answer_data)

conjurer_librarian/search_bot.py

@@ -3,6 +3,7 @@ from threading import Thread
 q = Queue()
 
 MAXTHREADS = 39
+#MAXTHREADS = 4
 DATABASE_PATH = r'C:\\Database\\chunks_1\\'
 ENCODING = "utf-8"
 CHUNk = "_chunk.txt"
@@ -46,7 +47,7 @@ def consumer(in_q, control_q, doi):
 
 def search_for_doi(doi):
     threads = []
-    work_q = Queue()
+    work_q = Queue(maxsize=10000000)
     control_q = Queue()
     t_cons = Thread(target = consumer, args = (work_q, control_q, doi))
     threads.append(t_cons)
@@ -68,7 +69,8 @@ def search_for_doi(doi):
         return False
 
 if __name__ == "__main__":
-    if search_for_doi("10.1145/2559206.2567832"):
+    if search_for_doi("10.1002/9781118786352.wbieg0998.pub2"):
+    #if search_for_doi("10.1108/gm-02-2016-0025xx"):        
         print("found")
         exit()
     else: