Merge branch 'main' of https://github.com/migatu/conjurer

conjurer_librarian/conjurer_librarian.py

@@ -7,12 +7,10 @@ import netrc
 import json
 import asyncio
 from json.decoder import JSONDecodeError
-from platform import uname
-from sys import platform
 from urllib.request import urlopen
 from habanero import Crossref
 from waitress import serve
-
+from queue import Queue
 import search_bot
 
 NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
@@ -21,33 +19,41 @@ HOST_ADDRESS = "192.168.1.192"
 PORT_ADDRESS = 5001
 DATABASE_PATH = r'C:\\Database\\chunks'
 ENCODING = "utf-8"
+app = Flask(__name__)
 
+librarian_queue = Queue()
 
 class Librarian(object):
 
-    def __init__(self) -> None:
+    def __init__(self, app, query, uuid) -> None:
         netrc_mod = netrc.netrc(NETRC_FILE)
         authTokens = netrc_mod.authenticators("crossref")
         self.cr = Crossref(mailto=authTokens[0])
+        self.query = query
+        self.uuid = uuid
         self.limit = 1000
         self.fetched = 0
         self.hit = 0
         self.total = 0
+        self.app = app
 
     async def search_crossref(self, query, uuid):
+        self.app.logger.info("STARTED SEARCH")
         search_result_from_cr = self.cr.works(query = query, limit = self.limit)
+        self.app.logger.info("CROSSREF DONE")
         self.total = search_result_from_cr['message']['total-results']
         self.fetched = len(search_result_from_cr['message']['items'])
         #add to sql with UUID
         #sql row - title, doi, author, uuid, last retrieved, is_available
+        
         with open("results.json", "r+", encoding="UTF-8") as data_file:
             # First we load existing data into a dict.
             try:
                 file_data = json.load(data_file)
             except JSONDecodeError:
                 file_data = []
-            # Join new_data with file_data inside emp_details
             summarized_results = []
+            self.app.logger.info("REFINE: Removing all derived works from the list")
             for item in search_result_from_cr['message']['items']:
                 container = "NO"
                 if "container-title" in item:
@@ -55,9 +61,8 @@ class Librarian(object):
                 summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
             result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'],  }
             file_data.append(result)
-            print("Saving")
             data_file.seek(0)
-            # convert back to json.
+            self.app.logger.info("REFINE: Dumping to file")
             json.dump(file_data, data_file, indent=4)
             return result
         #TODO: check if exists odpala wątki dla kolejnych plików, z każdego wątku dostaje zwrotki - trafienia na bieżąco oraz sygnał że plik się skończył i pora na następny
@@ -70,13 +75,13 @@ class Librarian(object):
 
 
     async def check_dois_in_local_db(self, doi):
-        #odwolanie do search_bota
+        self.app.logger.info("REFINE: Running search in the backend app")
         coro = asyncio.to_thread(search_bot.search_for_doi, doi)
         result = await coro
         return result
 
-
     async def check_if_exists_brute_force(self, page_url):
+        self.app.logger.error("REFINE: Brute force search!")
         if not page_url.startswith(("http:", "https:")):
             raise ValueError("URL must start with 'http:' or 'https:'")
         # trunk-ignore(bandit/B310)
@@ -92,39 +97,41 @@ class Librarian(object):
                     return True
         return False
 
-    def provide_sci_hub_link(self, page_url):
-        if self.check_if_exists(page_url):
-            return page_url
-        return None
-
     async def check_if_exists(self, doi, brute_force = False):
+        result = []
         result = await self.check_dois_in_local_db(doi)
         if brute_force and not result:
-            item_link = "https://sci-hub.se/" + doi
+            for item in doi:
-            result = await self.check_if_exists_brute_force(item_link)
+                item_link = "https://sci-hub.se/" + item[0]
+                result.append(await self.check_if_exists_brute_force(item_link))
         return result
 
     async def refine_query(self, unrefined_reqult, brute_force = False):
         print("Refine result")
         refined_result = []
+        temp = []
         for item in unrefined_reqult["summary"]:
             if item["container"] == "NO" and item["title"]:
-                result = await self.check_if_exists(str(item["DOI"]))
+                refined_result.append[item]
-                if result:
+        for item in refined_result:
-                    refined_result.append(item)
+            temp.append[(item["DOI"], item)]
-                    print ("HIT")
+
-                    print(item["title"])
+        refined_result = self.check_if_exists(temp)
-                time.sleep(0.01)
+        temp = []
-        print("Existing hits")
+        for item in refined_result:
+            if item["exists"]:
+                temp.append(item["data"])
+        refined_result = temp
         self.hit = len(refined_result)
         return refined_result
 
-    async def answer_query(self, query, uuid):
+    async def answer_query(self):
         #Search sql for UUID
         #SQL SELECT * FROM RESULTS WHERE UUID = uuid
-        print("Answer result")
         uuid_found = False
         database = None
+        query = self.query
+        uuid = self.uuid
         with open("results.json", "r", encoding="UTF-8") as new_file:
             #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
             #ale tymczasowo plik
@@ -146,14 +153,7 @@ class Librarian(object):
             print("Refined")
             return await self.refine_query(answer)
 
-#example not exists
+#============================= FLASK INTERNALS===============================
-#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
-#sample usage to download
-#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
-#print(cl.download(sample_existing_page_url))
-#cl.search_crossref("BDSM")
-
-app = Flask(__name__)
 def flask_debug():
     """
     The `flask_debug` function starts a Flask application in debug mode without using the reloader.
@@ -169,24 +169,26 @@ def waitress_run():
     """
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
+# TUTAJ TYLKO LOGUJEMY ŻE PRZYSZŁO, TWORZYMY LIBRARIAN class dla danego zapytania (po UUID)
+# ODPALAMY SZUKANIE I POZWALAMYSIĘ JUŻ DALEJ KRĘCIĆ
+#==================================SERVER ROUTES==========================================
 @app.route("/query", methods=["POST"])
 async def query_database():
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["query"])
     app.logger.info(record["UUID"])
-    cl = Librarian()
+    cl = Librarian(app, record["query"], record["UUID"])
-    answer_data = await cl.answer_query(record["query"], record["UUID"])
+    librarian_queue.put(cl)
 
-    app.logger.info(answer_data)
+
-    summary = (cl.hit,cl.fetched, cl.total, record["UUID"])
+    answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
-    answer_data = (summary,answer_data)
     return_data = (
         jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
         200,
     )
     return return_data
-
+#=======================================MAIN===================================================
 if __name__ == "__main__":
     logger = logging.getLogger('conjurer_musician')
     logger.setLevel(logging.DEBUG)
@@ -198,3 +200,13 @@ if __name__ == "__main__":
         worker.start()
     for worker in threads:
         worker.join()
+
+
+
+
+#example not exists
+#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
+#sample usage to download
+#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
+#print(cl.download(sample_existing_page_url))
+#cl.search_crossref("BDSM")

conjurer_librarian/search_bot.py

@@ -2,13 +2,13 @@ from queue import Queue, Empty
 from threading import Thread
 q = Queue()
 
-MAXTHREADS = 39
+#MAXTHREADS = 39
-#MAXTHREADS = 4
+MAXTHREADS = 4
 DATABASE_PATH = r'C:\\Database\\chunks_1\\'
 ENCODING = "utf-8"
 CHUNk = "_chunk.txt"
 _sentinel = object()
-
+result_list = []
 #TODO: Doisy wchodzą jako lista krotek (doi, False) - po znalezieniu zaznaczamy na True i potem już nie porównujemy
 #TODO: lista krotek - jeśli wszystkie mają True dajemy terma
 
@@ -35,15 +35,23 @@ def producer(out_q, control_q, filename):
 #TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
 def consumer(in_q, control_q, doi):
     sentinels = 0
+    for item in doi:
+        result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
     while True:
+        done_check = True
         data = in_q.get()
         if data is _sentinel:
             print("Worker finished")
             sentinels += 1
-        elif doi in data:
+        else:
-            control_q.put(_sentinel)
+            for item in result_list:
-            print(data)
+                if item["DOI"] in data:
-            print("HIT")
+                    print(data)
+                    print("HIT")
+                    item["exists"] = True
+                done_check = done_check and item["exists"]
+            if done_check:
+                control_q.put(_sentinel)
         if sentinels >= MAXTHREADS:
             print("All workers finished")
             break
@@ -51,31 +59,29 @@ def consumer(in_q, control_q, doi):
 
 def search_for_doi(doi):
     threads = []
-    work_q = Queue(maxsize=10000000)
+    work_q = Queue(maxsize=100)
     control_q = Queue()
     t_cons = Thread(target = consumer, args = (work_q, control_q, doi))
     threads.append(t_cons)
     for i in range(0,MAXTHREADS):
-        #filename = "test" + str(i) + CHUNk
+        filename = "test" + str(i) + CHUNk
         filename = str(i) + CHUNk
-        print(filename)
+        #print(filename)
         threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
     for worker in threads:
         worker.start()
     for worker in threads:
         worker.join()
-
+    return result_list
-    try:
-        check = control_q.get(block=False)
-        control_q.put(check)
-        return True
-    except Empty:
-        return False
-
 if __name__ == "__main__":
-    if search_for_doi("10.1002/9781118786352.wbieg0998.pub2"):
+    print(search_for_doi([
-    #if search_for_doi("10.1108/gm-02-2016-0025xx"):        
+        ("10.1002/9781118786352.wbieg0998.pub2","DATA"),
-        print("found")
+        ("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
-        exit()
+        ("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
-    else:
+        ("10.2165/00128415-200309690-00017","DATA"),
-        print("Not found")
+        ("10.5772/48313","DATA"),
+        ("10.15803/ijnc.7.2_419","DATA"),
+        ("10.1051/0004-6361/201321596e","DATA"),
+        ("10.2307/40835941","DATA"),
+        ]))
+