From 499d9c5715fe888a04c37e2ff1246f81b508b407 Mon Sep 17 00:00:00 2001
From: Migatu <mtuszowski@gmail.com>
Date: Thu, 18 Apr 2024 01:29:27 +0200
Subject: [PATCH] Reformatting of conjurer_librarian

---
 conjurer_librarian/conjurer_librarian.py | 116 +++++++++++++++--------
 1 file changed, 79 insertions(+), 37 deletions(-)

diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py
index 8065661..e3595e4 100644
--- a/conjurer_librarian/conjurer_librarian.py
+++ b/conjurer_librarian/conjurer_librarian.py
@@ -10,23 +10,24 @@ Functions:
 - BackgroundTaskSearch: Represents a background task for running the Librarian object asynchronously.
 """
 
-import netrc
-import json
 import asyncio
-from json.decoder import JSONDecodeError
-from urllib.request import urlopen
-from queue import Queue
-from logging import handlers
-import re
+import json
 import logging
+import netrc
+import re
 import threading
 import time
+from json.decoder import JSONDecodeError
+from logging import handlers
+from queue import Queue
+from urllib.request import urlopen
+
+import requests
+import scrape_bot
+import search_bot
 from flask import Flask, jsonify, request
 from habanero import Crossref
 from waitress import serve
-import search_bot
-import scrape_bot
-import requests
 
 # Constants
 NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
@@ -34,7 +35,7 @@ HOST_ADDRESS = "192.168.1.192"
 PORT_ADDRESS = 5001
 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
 SEND_RESULTS = "/conjurer"
-BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
+BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
 MAX_CR_RESULTS = 150000
 ENCODING = "utf-8"
 
@@ -43,6 +44,7 @@ app = Flask(__name__)
 librarian_queue = Queue()
 librarian_list = []
 
+
 class Librarian(object):
     def __init__(self, _app, query, uuid) -> None:
         """
@@ -100,15 +102,15 @@ class Librarian(object):
         self.app.logger.info("STARTED SEARCH")
         result = self.cr.works(query=query, limit=1000)
         self.search_result_from_cr.update(result)
-        self.total = result['message']['total-results']
-        self.fetched += len(result['message']['items'])
+        self.total = result["message"]["total-results"]
+        self.fetched += len(result["message"]["items"])
         self.app.logger.info(self.total)
         self.app.logger.info(self.fetched)
         while self.total > self.fetched and self.limit > self.fetched:
             result = self.cr.works(query=query, limit=1000)
             self.search_result_from_cr.update(result)
-            self.total = result['message']['total-results']
-            self.fetched += len(result['message']['items'])
+            self.total = result["message"]["total-results"]
+            self.fetched += len(result["message"]["items"])
             self.app.logger.info(self.total)
             self.app.logger.info(self.fetched)
             time.sleep(0.1)
@@ -124,13 +126,29 @@ class Librarian(object):
                 file_data = {}
             summarized_results = []
             self.app.logger.info("REFINE: Removing all derived works from the list")
-            for item in self.search_result_from_cr['message']['items']:
+            for item in self.search_result_from_cr["message"]["items"]:
                 container = "NO"
                 if "container-title" in item:
                     container = "YES"
                 else:
-                    summarized_results.append({"DOI": item['DOI'], "title": item['title'] if 'title' in item else None, "type": item['type'] if 'type' in item else None, "container": container})
-            result = {self.uuid: {"total_results": self.search_result_from_cr['message']['total-results'], "on_page": 1, "summary": summarized_results, "results": self.search_result_from_cr['message']['items']}}
+                    summarized_results.append(
+                        {
+                            "DOI": item["DOI"],
+                            "title": item["title"] if "title" in item else None,
+                            "type": item["type"] if "type" in item else None,
+                            "container": container,
+                        }
+                    )
+            result = {
+                self.uuid: {
+                    "total_results": self.search_result_from_cr["message"][
+                        "total-results"
+                    ],
+                    "on_page": 1,
+                    "summary": summarized_results,
+                    "results": self.search_result_from_cr["message"]["items"],
+                }
+            }
             if file_data:
                 file_data.update(result)
             else:
@@ -156,7 +174,9 @@ class Librarian(object):
         - None.
         """
         self.app.logger.info("REFINE: Running search in the backend app")
-        coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
+        coro = asyncio.to_thread(
+            search_bot.search_for_doi, doi, self.live_results, self.app.logger
+        )
         result = await coro
         return result
 
@@ -181,9 +201,14 @@ class Librarian(object):
             data = response.read()
             text = data.decode("utf-8")
             for line in text.splitlines():
-                if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
+                if re.match(
+                    r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
+                    line,
+                ):
                     return False
-                if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line):
+                if m := re.match(
+                    r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
+                ):
                     direct_download_link = "https:" + str(m.group(1))
                     self.app.logger.info(direct_download_link)
                     return True
@@ -208,7 +233,11 @@ class Librarian(object):
         if brute_force and not result:
             for item in doi:
                 item_link = "https://sci-hub.se/" + item[0]
-                tmp = {"DOI": item["DOI"], "exists": await self.check_if_exists_brute_force(item_link), "data": item}
+                tmp = {
+                    "DOI": item["DOI"],
+                    "exists": await self.check_if_exists_brute_force(item_link),
+                    "data": item,
+                }
                 result.update(tmp)
         return result
 
@@ -276,11 +305,17 @@ class Librarian(object):
                         self.result = await self.refine_query(database)
                         temp = {}
                         for item in self.result:
-                            temp[item['DOI']] = {"Title": item['title'], "type": item['type']}
+                            temp[item["DOI"]] = {
+                                "Title": item["title"],
+                                "type": item["type"],
+                            }
                         self.result = temp
                         temp2 = {}
                         for item in self.not_in_db:
-                            temp2[item['DOI']] = {"Title": item['title'], "type": item['type']}
+                            temp2[item["DOI"]] = {
+                                "Title": item["title"],
+                                "type": item["type"],
+                            }
                         self.not_in_db = temp2
                         return self.result
                 answer = await self.search_crossref(query=query)
@@ -300,15 +335,17 @@ class Librarian(object):
                 self.done = True
             temp = {}
             for item in self.result:
-                temp[item['DOI']] = {"Title": item['title'], "type": item['type']}
+                temp[item["DOI"]] = {"Title": item["title"], "type": item["type"]}
             self.result = temp
             temp2 = {}
             for item in self.not_in_db:
-                temp2[item['DOI']] = {"Title": item['title'], "type": item['type']}
+                temp2[item["DOI"]] = {"Title": item["title"], "type": item["type"]}
                 self.not_in_db = temp2
             return self.result
 
-#============================= FLASK INTERNALS===============================
+
+# ============================= FLASK INTERNALS===============================
+
 
 def flask_debug():
     """
@@ -326,6 +363,7 @@ def flask_debug():
     # trunk-ignore(bandit/B201)
     app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
+
 def waitress_run():
     """
     Serves the Flask application using the Waitress WSGI server.
@@ -341,6 +379,7 @@ def waitress_run():
     """
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
+
 class BackgroundTaskSearch(threading.Thread):
     """
     A background task for searching and saving results to files.
@@ -428,7 +467,8 @@ class BackgroundTaskSearch(threading.Thread):
             self.app.logger.info("SEND CONFIRMED")
             time.sleep(1)
 
-#==================================SERVER ROUTES==========================================
+
+# ==================================SERVER ROUTES==========================================
 @app.route("/query", methods=["POST"])
 async def query_database():
     """
@@ -456,6 +496,7 @@ async def query_database():
     )
     return return_data
 
+
 @app.route("/get_partial_result", methods=["POST"])
 async def get_partial():
     """
@@ -477,30 +518,31 @@ async def get_partial():
     )
     return return_data
 
-#=======================================MAIN===================================================
+
+# =======================================MAIN===================================================
 if __name__ == "__main__":
     app.logger.setLevel(logging.DEBUG)
     h1 = handlers.RotatingFileHandler(
-    filename="E:\\logs\\librarian.log",
-    encoding="utf-8",
-    mode="a",
-    maxBytes=6 * 1024 * 1024,
-    backupCount=6,
+        filename="E:\\logs\\librarian.log",
+        encoding="utf-8",
+        mode="a",
+        maxBytes=6 * 1024 * 1024,
+        backupCount=6,
     )
 
     app.logger.addHandler(h1)
     threads = []
     threads.append(threading.Thread(target=waitress_run))
-    #threads.append(threading.Thread(target=flask_debug))
+    # threads.append(threading.Thread(target=flask_debug))
     bgtask = BackgroundTaskSearch()
     bgtask.app = app
     threads.append(bgtask)
-    threads.append(threading.Thread(target = scrape_bot.scraper, args=(app.logger,)))
+    threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,)))
     i = 0
     for worker in threads:
         try:
             app.logger.info("App number: %s", i)
-            i+=1
+            i += 1
             worker.start()
         except RuntimeError as e:
             app.logger.error("Exploded")