Work in progress

conjurer_librarian/conjurer_librarian.py

@@ -9,6 +9,9 @@ from json.decoder import JSONDecodeError
 from platform import uname
 from sys import platform
 from urllib.request import urlopen
+import asyncio
+from queue import Queue
+from threading import Thread
 from habanero import Crossref
 from waitress import serve
 
@@ -24,6 +27,9 @@ elif platform == "win32":
 #HOST_ADDRESS = "192.168.1.191"
 HOST_ADDRESS = "0.0.0.0"
 PORT_ADDRESS = 5001
+DATABASE_PATH = r'C:\\Database\\chunks'
+ENCODING = "utf-8"
+
 
 class Librarian(object):
 
@@ -70,13 +76,29 @@ class Librarian(object):
         #TODO: Później robimy tak że query jest najpierw sprawdzane w sql - i dawane jako "zapisana lokalna kopia"
         #TODO: Paginacja - najpierw hity z pliku, potem na końcu to co nie zwróciło żadnych hitów - i to dopiero można crawlerem sprawdzić
 
+    def check_dois_in_file(self, file_name, dois, out_queue):
+        with open(f'{DATABASE_PATH}{file_name}', "r+", encoding=ENCODING) as file:
+            while True:
+                # Get next line from file
+                line = file.readline()
+                if line == dois:
+                    return True
+                # if line is empty
+                # end of file is reached
+                if not line:
+                    return False
+
+    def check_dois_in_local_db(self, dois):
+        _sentinel = object()
+        chunk_list = []
+        q = Queue()
+
+        for file in chunk_list:
+            t = Thread(target = self.check_dois_in_file, args = (file ,dois, q))
+            t.start()
+        q.put(_sentinel)
+
     def check_if_exists(self, page_url):
-
-        
-
-        return None
-
-        in_scihub_db = True
         if not page_url.startswith(("http:", "https:")):
             raise ValueError("URL must start with 'http:' or 'https:'")
         # trunk-ignore(bandit/B310)
@@ -86,13 +108,6 @@ class Librarian(object):
             for line in text.splitlines():
                 if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
                     return None
-        if in_scihub_db:
-            return text
-        return None
-
-    def download(self, page_url):
-        if response := self.check_if_exists(page_url):
-            for line in response.splitlines():
                 if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
                     return "https:" + str(m.group(1))
         return None