Serverside search.

conjurer_librarian/conjurer_librarian.py

@@ -12,16 +12,22 @@ from habanero import Crossref
 from waitress import serve
 from queue import Queue
 import search_bot
-
+import requests
+from typing import Callable
+from uuid import UUID
 NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
 
 HOST_ADDRESS = "192.168.1.192"
 PORT_ADDRESS = 5001
-DATABASE_PATH = r'C:\\Database\\chunks'
+MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
+SEND_RESULTS = "/conjurer"
+BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
+
 ENCODING = "utf-8"
 app = Flask(__name__)
 
 librarian_queue = Queue()
+librarian_list = []
 
 class Librarian(object):
 
@@ -30,53 +36,67 @@ class Librarian(object):
         authTokens = netrc_mod.authenticators("crossref")
         self.cr = Crossref(mailto=authTokens[0])
         self.query = query
-        self.uuid = uuid
+        self.uuid = str(uuid)
-        self.limit = 1000
+        self.limit = 3000
         self.fetched = 0
         self.hit = 0
         self.total = 0
         self.app = app
+        self.logger = app.logger
+        self.live_results = []
+        self.search_result_from_cr = {}
+        self.done = False
 
-    async def search_crossref(self, query, uuid):
+    async def search_crossref(self, query):
         self.app.logger.info("STARTED SEARCH")
-        search_result_from_cr = self.cr.works(query = query, limit = self.limit)
+        result = self.cr.works(query = query, limit = 1000)
+        self.search_result_from_cr.update(result)
+        self.total = result['message']['total-results']
+        self.fetched += len(result['message']['items'])
+        self.logger.info (self.total)
+        self.logger.info(self.fetched)
+        while self.total > self.fetched and self.limit > self.fetched:
+            result = self.cr.works(query = query, limit = 1000)
+            self.search_result_from_cr.update(result)
+            self.total = result['message']['total-results']
+            self.fetched += len(result['message']['items'])
+            self.logger.info (self.total)
+            self.logger.info(self.fetched)
+
+            time.sleep(1)
         self.app.logger.info("CROSSREF DONE")
-        self.total = search_result_from_cr['message']['total-results']
+        #TODO: add to sql with UUID
-        self.fetched = len(search_result_from_cr['message']['items'])
-        #add to sql with UUID
         #sql row - title, doi, author, uuid, last retrieved, is_available
         
-        with open("results.json", "r+", encoding="UTF-8") as data_file:
+        with open("rr_results.json", "r+", encoding="utf-8") as data_file:
             # First we load existing data into a dict.
             try:
                 file_data = json.load(data_file)
             except JSONDecodeError:
-                file_data = []
+                file_data = {}
             summarized_results = []
             self.app.logger.info("REFINE: Removing all derived works from the list")
-            for item in search_result_from_cr['message']['items']:
+            for item in self.search_result_from_cr['message']['items']:
                 container = "NO"
                 if "container-title" in item:
                     container = "YES"
-                summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
+                else:
-            result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'],  }
+                    summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
-            file_data.append(result)
+            result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
-            data_file.seek(0)
+            if file_data:
+                file_data.update(result)
+            else:
+                file_data = result
+            
             self.app.logger.info("REFINE: Dumping to file")
+            data_file.truncate(0)
+            data_file.seek(0)
             json.dump(file_data, data_file, indent=4)
             return result
-        #TODO: check if exists odpala wątki dla kolejnych plików, z każdego wątku dostaje zwrotki - trafienia na bieżąco oraz sygnał że plik się skończył i pora na następny
-        #TODO: paczki danych są upychane po 10 w tablicy, POSTy ze strony klienta mówią tylko nazwę tablicy (poprzez UUID) oraz którą stronę tejże
-        #TODO: jeśli UUID nie istnieje to budujemy całość dla nowego uuid
-        #TODO: kiedy wszystkie wątki pilokowe się zakończą - zamykamy merger i dajemy znać do procedury obsługi odpowiadania na posty że dla tego uuid już nic więcej nie znajdziemy
-        #TODO: po 60 minutach storeujemy wyniki zapytania pod query w sql i kasujemy UUID
-        #TODO: Później robimy tak że query jest najpierw sprawdzane w sql - i dawane jako "zapisana lokalna kopia"
-        #TODO: Paginacja - najpierw hity z pliku, potem na końcu to co nie zwróciło żadnych hitów - i to dopiero można crawlerem sprawdzić
-
 
     async def check_dois_in_local_db(self, doi):
         self.app.logger.info("REFINE: Running search in the backend app")
-        coro = asyncio.to_thread(search_bot.search_for_doi, doi)
+        coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
         result = await coro
         return result
 
@@ -90,33 +110,38 @@ class Librarian(object):
             text = data.decode("utf-8")
             for line in text.splitlines():
                 if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
-                    return True
+                    return False
                 if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
                     direct_download_link = "https:" + str(m.group(1))
-                    print (direct_download_link)
+                    self.app.logger.info (direct_download_link)
                     return True
         return False
 
     async def check_if_exists(self, doi, brute_force = False):
-        result = []
+        result = {}
         result = await self.check_dois_in_local_db(doi)
         if brute_force and not result:
             for item in doi:
                 item_link = "https://sci-hub.se/" + item[0]
-                result.append(await self.check_if_exists_brute_force(item_link))
+                tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
+                result.update(tmp)
         return result
 
     async def refine_query(self, unrefined_reqult, brute_force = False):
-        print("Refine result")
-        refined_result = []
         temp = []
-        for item in unrefined_reqult["summary"]:
+        refined_result = []
-            if item["container"] == "NO" and item["title"]:
+        for key in unrefined_reqult:
-                refined_result.append[item]
+            self.logger.info("KEY:")
-        for item in refined_result:
+            self.logger.info(key)
-            temp.append[(item["DOI"], item)]
 
-        refined_result = self.check_if_exists(temp)
+        for item in unrefined_reqult[self.uuid]["summary"]:
+            if item["container"] == "NO" and item["title"]:                
+                refined_result.append(item)
+
+        for item in refined_result:
+            temp.append([item["DOI"], item])
+
+        refined_result = await self.check_if_exists(temp)
         temp = []
         for item in refined_result:
             if item["exists"]:
@@ -124,34 +149,45 @@ class Librarian(object):
         refined_result = temp
         self.hit = len(refined_result)
         return refined_result
-
+    #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
     async def answer_query(self):
+        #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
         #Search sql for UUID
         #SQL SELECT * FROM RESULTS WHERE UUID = uuid
+        self.logger.info(f"Search started {self.uuid}")
         uuid_found = False
         database = None
         query = self.query
-        uuid = self.uuid
+        with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
-        with open("results.json", "r", encoding="UTF-8") as new_file:
             #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
             #ale tymczasowo plik
             try:
-                database =  json.load(new_file)
+                database =  json.load(cr_file)
             except JSONDecodeError:
                 pass
-        if database:
-            for item in database:
-                if item["UUID"] == uuid:
-                    print (item["UUID"])
-                    uuid_found = True
-                    #TODO: THREAD IT
-                    result = await self.refine_query(item)
-                    return result
 
-        if not uuid_found:
+            if database:
-            answer = await self.search_crossref(query=query, uuid=uuid)
+                for item in database.keys():
-            print("Refined")
+                    if item in self.uuid:
-            return await self.refine_query(answer)
+                        self.result = await self.refine_query(database)
+                        return self.result
+                answer = await self.search_crossref(query=query)
+                database.update(answer)
+                cr_file.truncate(0)
+                cr_file.seek(0)
+                json.dump(database,cr_file)
+                self.result = await self.refine_query(answer)
+                return self.result
+            else:
+                answer = await self.search_crossref(query=query)
+                database = answer
+                cr_file.truncate(0)
+                cr_file.seek(0)
+                json.dump(database,cr_file)
+                self.result = await self.refine_query(answer)
+                self.app.logger.info("Refined")
+                self.done = True
+                return self.result
 
 #============================= FLASK INTERNALS===============================
 def flask_debug():
@@ -169,8 +205,47 @@ def waitress_run():
     """
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
-# TUTAJ TYLKO LOGUJEMY ŻE PRZYSZŁO, TWORZYMY LIBRARIAN class dla danego zapytania (po UUID)
+class BackgroundTaskSearch(threading.Thread):
-# ODPALAMY SZUKANIE I POZWALAMYSIĘ JUŻ DALEJ KRĘCIĆ
+    def run(self):
+        loop = asyncio.new_event_loop()  # loop = asyncio.get_event_loop()
+        loop.run_until_complete(self._run())
+        loop.close()
+
+    async def _run(self):
+        while True:
+            database = None
+            librarian = librarian_queue.get()
+            self.app.logger.info("STARTED")
+            result = await librarian.answer_query()
+            result = {librarian.uuid: result}
+            self.app.logger.info("Saving to file")
+            with open("s_results.json", "r+", encoding="utf-8") as s_file:
+                try:
+                    database =  json.load(s_file)
+                except JSONDecodeError:
+                    pass
+                if database:
+                    database.extend(result)
+                else:
+                    database = result
+                self.app.logger.info("DUMPING DATA")
+                s_file.truncate(0)
+                s_file.seek(0)
+                json.dump(database, s_file)
+            self.app.logger.info("FINISHED")
+
+            self.app.logger.info(result)
+            coroutine = asyncio.to_thread(
+            requests.post,
+            f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
+            json=result,
+            timeout=360,
+    )       
+            self.app.logger.info("SENT")
+            _ = await coroutine
+            self.app.logger.info("SEND CONFIRMED")
+            time.sleep(1)
+            
 #==================================SERVER ROUTES==========================================
 @app.route("/query", methods=["POST"])
 async def query_database():
@@ -178,16 +253,35 @@ async def query_database():
     app.logger.info(record)
     app.logger.info(record["query"])
     app.logger.info(record["UUID"])
-    cl = Librarian(app, record["query"], record["UUID"])
+    uuid = record["UUID"]
+    #TODO: TESTING ONLY======================
+    uuid = UUID('{12345678-1234-5678-1234-567812345678}')
+
+    cl = Librarian(app, record["query"], uuid)
     librarian_queue.put(cl)
-
+    librarian_list.append(cl)
-
     answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
     return_data = (
         jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
         200,
     )
     return return_data
+
+@app.route("/get_partial_result", methods=["POST"])
+async def get_partial():
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["UUID"])
+    for lib in librarian_list:
+        if lib.uuid == record["UUID"]:
+            answer_data = lib.live_results
+            break
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
+        200,
+    )
+    return return_data
+
 #=======================================MAIN===================================================
 if __name__ == "__main__":
     logger = logging.getLogger('conjurer_musician')
@@ -195,18 +289,17 @@ if __name__ == "__main__":
     threads = []
     #threads.append(threading.Thread(target=waitress_run))
     threads.append(threading.Thread(target=flask_debug))
-
+    bgtask = BackgroundTaskSearch()
+    bgtask.app = app
+    threads.append(bgtask)
+    i = 0
     for worker in threads:
-        worker.start()
+        try:
+            logger.info(f"App number: {i}")
+            i+=1
+            worker.start()
+        except RuntimeError as e:
+            logger.error("Exploded")
+            print(str(e))
     for worker in threads:
         worker.join()
-
-
-
-
-#example not exists
-#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
-#sample usage to download
-#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
-#print(cl.download(sample_existing_page_url))
-#cl.search_crossref("BDSM")

conjurer_librarian/search_bot.py

@@ -2,15 +2,19 @@ from queue import Queue, Empty
 from threading import Thread
 q = Queue()
 
-#MAXTHREADS = 39
+#TODO: TO w klasie jako parametry wybierane czy robimy pelen run czy testowy
-MAXTHREADS = 4
+#MAXTHREADS = 3
-DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+MAXTHREADS = 1
+
+#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+DATABASE_PATH = r'C:\\Database\\chunks\\'
+
 ENCODING = "utf-8"
 CHUNk = "_chunk.txt"
 _sentinel = object()
 result_list = []
-#TODO: Doisy wchodzą jako lista krotek (doi, False) - po znalezieniu zaznaczamy na True i potem już nie porównujemy
+
-#TODO: lista krotek - jeśli wszystkie mają True dajemy terma
+#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
 
 def producer(out_q, control_q, filename):
     with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
@@ -26,55 +30,70 @@ def producer(out_q, control_q, filename):
                 print("TERM signal received")
                 control_q.put(check)
                 break
-    print("Worker finished")
+    print(f"Worker finished: {filename}")
     out_q.put(_sentinel)
  
-
-
 #IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
 #TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
-def consumer(in_q, control_q, doi):
+def consumer(in_q, control_q, doi, live_results, logger):
     sentinels = 0
     for item in doi:
         result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
     while True:
+        #if in_q.empty():
+        #    print("Queue empty!")
+        #if in_q.full():
+        #    print("Queue full!")
         done_check = True
         data = in_q.get()
         if data is _sentinel:
-            print("Worker finished")
+            logger.info("Worker finished")
             sentinels += 1
         else:
             for item in result_list:
                 if item["DOI"] in data:
-                    print(data)
+                    logger.info(data)
-                    print("HIT")
+                    logger.info("HIT")
                     item["exists"] = True
+                    live_results.append(item)
                 done_check = done_check and item["exists"]
             if done_check:
                 control_q.put(_sentinel)
         if sentinels >= MAXTHREADS:
-            print("All workers finished")
+            logger.info("All workers finished")
             break
 
 
-def search_for_doi(doi):
+def search_for_doi(doi, live_results, logger):
     threads = []
-    work_q = Queue(maxsize=100)
+    work_q = Queue(maxsize=200000)
     control_q = Queue()
-    t_cons = Thread(target = consumer, args = (work_q, control_q, doi))
+    t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
+    logger.info("Consumer thread created")
     threads.append(t_cons)
     for i in range(0,MAXTHREADS):
-        filename = "test" + str(i) + CHUNk
+        #filename = "test" + str(i) + CHUNk
-        filename = str(i) + CHUNk
+        filename = "40" + CHUNk
-        #print(filename)
+        #filename = str(i) + CHUNk
+        logger.info(f"Creating worker thread no: {i}")
         threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
     for worker in threads:
         worker.start()
     for worker in threads:
         worker.join()
     return result_list
+
+
+
 if __name__ == "__main__":
-    print(search_for_doi([
+    import logging
+    logger = logging.getLogger()
+    logger.setLevel("DEBUG")
+    h1 = logging.StreamHandler()
+    logger.addHandler(h1)
+    logger.info("TEST RUN")
+    live_result = []
+    logger.info(search_for_doi([
         ("10.1002/9781118786352.wbieg0998.pub2","DATA"),
         ("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
         ("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
@@ -83,5 +102,5 @@ if __name__ == "__main__":
         ("10.15803/ijnc.7.2_419","DATA"),
         ("10.1051/0004-6361/201321596e","DATA"),
         ("10.2307/40835941","DATA"),
-        ]))
+        ], live_result, logger))
-
+    logger.info(live_result)

s_results.json

@@ -0,0 +1 @@
+{"12345678-1234-5678-1234-567812345678": [{"DOI": "10.1057/9781137435026", "title": ["Queer BDSM Intimacies"], "type": "book", "container": "NO"}]}