Fixes!

conjurer_librarian/conjurer_librarian.py

@@ -9,9 +9,6 @@ from json.decoder import JSONDecodeError
 from platform import uname
 from sys import platform
 from urllib.request import urlopen
-import asyncio
-from queue import Queue
-from threading import Thread
 from habanero import Crossref
 from waitress import serve
 
@@ -76,29 +73,15 @@ class Librarian(object):
         #TODO: Później robimy tak że query jest najpierw sprawdzane w sql - i dawane jako "zapisana lokalna kopia"
         #TODO: Paginacja - najpierw hity z pliku, potem na końcu to co nie zwróciło żadnych hitów - i to dopiero można crawlerem sprawdzić
 
-    def check_dois_in_file(self, file_name, dois, out_queue):
-        with open(f'{DATABASE_PATH}{file_name}', "r+", encoding=ENCODING) as file:
-            while True:
-                # Get next line from file
-                line = file.readline()
-                if line == dois:
-                    return True
-                # if line is empty
-                # end of file is reached
-                if not line:
-                    return False
 
     def check_dois_in_local_db(self, dois):
-        _sentinel = object()
+        #odwolanie do search_bota
-        chunk_list = []
-        q = Queue()
 
-        for file in chunk_list:
+        result = False
-            t = Thread(target = self.check_dois_in_file, args = (file ,dois, q))
+        return result
-            t.start()
-        q.put(_sentinel)
 
-    def check_if_exists(self, page_url):
+
+    def check_if_exists_brute_force(self, page_url):
         if not page_url.startswith(("http:", "https:")):
             raise ValueError("URL must start with 'http:' or 'https:'")
         # trunk-ignore(bandit/B310)
@@ -107,23 +90,31 @@ class Librarian(object):
             text = data.decode("utf-8")
             for line in text.splitlines():
                 if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
-                    return None
+                    return True
                 if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
-                    return "https:" + str(m.group(1))
+                    direct_download_link = "https:" + str(m.group(1))
-        return None
+                    print (direct_download_link)
+                    return True
+        return False
 
     def provide_sci_hub_link(self, page_url):
         if self.check_if_exists(page_url):
             return page_url
         return None
 
-    def refine_query(self, unrefined_reqult):
+    def check_if_exists(self, doi, brute_force = False):
+        self.check_dois_in_local_db(doi)
+        if brute_force:
+            item_link = "https://sci-hub.se/" + doi
+            self.check_if_exists_brute_force(item_link)
+        return False
+
+    def refine_query(self, unrefined_reqult, brute_force = False):
         print("Refine result")
         refined_result = []
         for item in unrefined_reqult["summary"]:
             if item["container"] == "NO" and item["title"]:
-                item_link = "https://sci-hub.se/" + str(item['DOI'])
+                if self.check_if_exists(str(item["DOI"])):
-                if self.check_if_exists(item_link):
                     refined_result.append(item)
                     print ("HIT")
                     print(item["title"])
@@ -150,6 +141,7 @@ class Librarian(object):
                 if item["UUID"] == uuid:
                     print (item["UUID"])
                     uuid_found = True
+                    #TODO: THREAD IT
                     return self.refine_query(item)
 
         if not uuid_found:

conjurer_librarian/search_bot.py

@@ -0,0 +1,75 @@
+from queue import Queue, Empty
+from threading import Thread
+q = Queue()
+
+MAXTHREADS = 39
+DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+ENCODING = "utf-8"
+CHUNk = "_chunk.txt"
+_sentinel = object()
+
+def producer(out_q, control_q, filename):
+    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
+        lines = operated_file.readlines()
+        for line in lines:
+            out_q.put(line)
+            try:
+                check = control_q.get(block=False)
+            except Empty:
+                check = False
+                pass
+            if check is _sentinel:
+                print("TERM signal received")
+                control_q.put(check)
+                break
+    print("Worker finished")
+    out_q.put(_sentinel)
+ 
+
+
+#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+def consumer(in_q, control_q, doi):
+    sentinels = 0
+    while True:
+        data = in_q.get()
+        if data is _sentinel:
+            print("Worker finished")
+            sentinels += 1
+        elif doi in data:
+            control_q.put(_sentinel)
+            print(data)
+            print("HIT")
+        if sentinels >= MAXTHREADS:
+            print("All workers finished")
+            break
+
+
+def search_for_doi(doi):
+    threads = []
+    work_q = Queue()
+    control_q = Queue()
+    t_cons = Thread(target = consumer, args = (work_q, control_q, doi))
+    threads.append(t_cons)
+    for i in range(0,MAXTHREADS):
+        #filename = "test" + str(i) + CHUNk
+        filename = str(i) + CHUNk
+        print(filename)
+        threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
+    for worker in threads:
+        worker.start()
+    for worker in threads:
+        worker.join()
+
+    try:
+        check = control_q.get(block=False)
+        control_q.put(check)
+        return True
+    except Empty:
+        return False
+
+if __name__ == "__main__":
+    if search_for_doi("10.1145/2559206.2567832"):
+        print("found")
+        exit()
+    else:
+        print("Not found")

utils/cut_large_file.py

@@ -39,9 +39,12 @@ def smart_write(lines, file):
     :return: The function `smart_write` returns the size of the encoded bytes of the input `lines` in
     bytes.
     """
-    encoded_bytes = lines.encode("utf-8")
+    try:
-    size_in_bytes = len(encoded_bytes)
+        encoded_bytes = lines.encode("utf-8")
-    file.write(lines)
+        size_in_bytes = len(encoded_bytes)
+        file.write(lines)
+    except:
+        return 0
 
     return size_in_bytes
 
@@ -65,8 +68,9 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
     with open(source, encoding="utf-8") as infile:
         output_size = 0
         output_no = 0
-
+        lineno = 0
         for line in infile:
+            lineno += 1
             if output_size > FILE_SIZE_LIMIT:
                 output_no += 1
                 output_size = 0
@@ -76,7 +80,12 @@ def cut_my_life_into_pieces(source, output_folder, output_file):
                 "a",
                 encoding="utf-8",
             ) as op_file:
-                output_size += smart_write(line, op_file)
+                tmp =  smart_write(line, op_file)
+                if tmp > 0:
+                    output_size += smart_write(line, op_file)
+                else:
+                    print (f'Error at {lineno}')
+                    print(line)
 
 if __name__ == "__main__":
     cut_my_life_into_pieces(SOURCE,OUTPUT_FOLDER,OUTPUT_FILE_SCHEMA)