Search webservice (#2)

* Deployable script for windows file service

* Deploy!

* Server side work

conjurer_librarian/conjurer_librarian.py

@@ -12,9 +12,8 @@ from habanero import Crossref
 from waitress import serve
 from queue import Queue
 import search_bot
+import scrape_bot
 import requests
-from typing import Callable
-from uuid import UUID
 NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
 
 HOST_ADDRESS = "192.168.1.192"
@@ -22,7 +21,7 @@ PORT_ADDRESS = 5001
 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
 SEND_RESULTS = "/conjurer"
 BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
-
+MAX_CR_RESULTS = 150000
 ENCODING = "utf-8"
 app = Flask(__name__)
 
@@ -37,13 +36,15 @@ class Librarian(object):
         self.cr = Crossref(mailto=authTokens[0])
         self.query = query
         self.uuid = str(uuid)
-        self.limit = 3000
+        self.limit = MAX_CR_RESULTS
         self.fetched = 0
         self.hit = 0
         self.total = 0
         self.app = app
         self.logger = app.logger
         self.live_results = []
+        self.final_result = []
+        self.not_in_db = []
         self.search_result_from_cr = {}
         self.done = False
 
@@ -62,8 +63,7 @@ class Librarian(object):
             self.fetched += len(result['message']['items'])
             self.logger.info (self.total)
             self.logger.info(self.fetched)
-
+            time.sleep(0.1)
-            time.sleep(1)
         self.app.logger.info("CROSSREF DONE")
         #TODO: add to sql with UUID
         #sql row - title, doi, author, uuid, last retrieved, is_available
@@ -142,20 +142,20 @@ class Librarian(object):
             temp.append([item["DOI"], item])
 
         refined_result = await self.check_if_exists(temp)
-        temp = []
+
         for item in refined_result:
             if item["exists"]:
-                temp.append(item["data"])
+                self.final_result.append(item["data"])
-        refined_result = temp
+            else:
-        self.hit = len(refined_result)
+                self.not_in_db.append(item["data"])
-        return refined_result
+        self.hit = len(self.final_result)
+        return self.final_result
     #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
     async def answer_query(self):
         #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
         #Search sql for UUID
         #SQL SELECT * FROM RESULTS WHERE UUID = uuid
         self.logger.info(f"Search started {self.uuid}")
-        uuid_found = False
         database = None
         query = self.query
         with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
@@ -165,7 +165,7 @@ class Librarian(object):
                 database =  json.load(cr_file)
             except JSONDecodeError:
                 pass
-
+            #TODO: Zwracać i final_result i not_in_db. Not in db_do pliku - i do ręcznego przeglądania potem.
             if database:
                 for item in database.keys():
                     if item in self.uuid:
@@ -174,6 +174,10 @@ class Librarian(object):
                         for item in self.result:
                             temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
                         self.result = temp
+                        temp2 = {}
+                        for item in self.not_in_db:
+                            temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
+                        self.not_in_db = temp2
                         return self.result
                 answer = await self.search_crossref(query=query)
                 database.update(answer)
@@ -194,6 +198,10 @@ class Librarian(object):
             for item in self.result:
                 temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
             self.result = temp
+            temp2 = {}
+            for item in self.not_in_db:
+                temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
+                self.not_in_db = temp2
             return self.result
 
 #============================= FLASK INTERNALS===============================
@@ -221,11 +229,25 @@ class BackgroundTaskSearch(threading.Thread):
     async def _run(self):
         while True:
             database = None
+            ndb_database = None
             librarian = librarian_queue.get()
             self.app.logger.info("STARTED")
             result = await librarian.answer_query()
             result = {librarian.uuid: result}
             self.app.logger.info("Saving to file")
+            with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
+                try:
+                    ndb_database =  json.load(ndb_file)
+                except JSONDecodeError:
+                    pass
+                if ndb_database:
+                    ndb_database.update(librarian.not_in_db)
+                else:
+                    ndb_database = librarian.not_in_db
+                ndb_file.truncate(0)
+                ndb_file.seek(0)
+                json.dump(ndb_database, ndb_file)
+
             with open("s_results.json", "r+", encoding="utf-8") as s_file:
                 try:
                     database =  json.load(s_file)
@@ -291,14 +313,15 @@ async def get_partial():
 
 #=======================================MAIN===================================================
 if __name__ == "__main__":
-    logger = logging.getLogger('conjurer_musician')
+    logger = app.logger
     logger.setLevel(logging.DEBUG)
     threads = []
-    #threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=waitress_run))
-    threads.append(threading.Thread(target=flask_debug))
+    #threads.append(threading.Thread(target=flask_debug))
     bgtask = BackgroundTaskSearch()
     bgtask.app = app
     threads.append(bgtask)
+    threads.append(threading.Thread(target = scrape_bot.scraper, args=(logger,)))
     i = 0
     for worker in threads:
         try:

conjurer_librarian/run_server.bat

@@ -0,0 +1 @@
+c:\conjurer\env\Scripts\activate.bat && c:\conjurer\conjurer_librarian\conjurer_librarian.py

conjurer_librarian/scrape_bot.py

@@ -0,0 +1,93 @@
+import json
+from json.decoder import JSONDecodeError
+from urllib.request import urlopen
+from requests import Timeout,ConnectTimeout, ConnectionError
+from queue import Queue
+import re
+import time
+import random
+import logging
+from threading import Thread
+SCR_DATABASE_PATH = r'C:\\Database\\chunks\\'
+SCR_FILENAME = "40_chunk.txt"
+SCR_ENCODING = "utf-8"
+
+WORK_Q = Queue()
+random.seed()
+def load_ndb_to_q(logger):
+    logger.info("Loader started")
+    while True:    
+        with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
+            try:
+                ndb_database =  json.load(ndb_file)
+                for item in ndb_database.keys():
+                    logger.info(item)
+                    url = f"https://sci-hub.se/{item}"
+                    WORK_Q.put([item, url, False])
+                ndb_file.truncate(0)
+            except JSONDecodeError:
+                time.sleep(60*60*2)
+                pass
+        time.sleep(180)
+
+def check_if_exists_brute_force(logger):
+    while True:
+        logger.info("Scraper tick")
+        item = WORK_Q.get()
+        page_url =item[1] 
+        logger.error("REFINE: Brute force search!")
+        if not page_url.startswith(("http:", "https:")):
+            raise ValueError("URL must start with 'http:' or 'https:'")
+        blocked = True
+        try:
+        # trunk-ignore(bandit/B310)
+            with urlopen(page_url) as response:
+                data = response.read()
+                text = data.decode("utf-8")
+                if item[2]:
+                    logger.info("Already found")
+                    time.sleep(180)
+                    blocked = False
+                else:
+                    for line in text.splitlines():
+                        if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
+                            blocked = False
+                            logger.info("Not found")
+                            item[2] = False
+                        if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
+                            blocked = False
+                            direct_download_link = "https:" + str(m.group(1))
+                            logger.info (direct_download_link)
+                            with open(SCR_DATABASE_PATH + SCR_FILENAME, "a", encoding=SCR_ENCODING) as operated_file:
+                                operated_file.write("\n")
+                                operated_file.write(item[0])
+                                item[2] = True
+        except (Timeout,ConnectTimeout, ConnectionRefusedError, ConnectionError):
+            pass
+        if blocked:
+            logger.info(item)
+            logger.info(text)
+            logger.error("Got blocked. Fuck.")
+            time.sleep(60*60*72)
+        # trunk-ignore(bandit/B311)
+        rand = random.randint(1,100)
+        logger.info(f"Sleeping for {3*rand} minutes")
+        time.sleep(180*rand)
+
+def scraper(logger = None):
+    if not logger:
+        logger = logging.getLogger()
+        logger.setLevel("DEBUG")
+        h1 = logging.StreamHandler()
+        logger.addHandler(h1)
+        logger.info("No logger set. Test run")
+  
+    loader = Thread(target = load_ndb_to_q, args = (logger,))
+    scraper = Thread(target = check_if_exists_brute_force, args = (logger,))
+    loader.start()
+    scraper.start()
+    loader.join()
+    scraper.join()
+
+if __name__ == "__main__":
+    scraper()

conjurer_librarian/search_bot.py

@@ -5,22 +5,26 @@ from threading import Thread
 q = Queue()
 
 #TODO: DATA FOR TEST ONLY
-MAXTHREADS = 5
+#MAXTHREADS = 5
-DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
 
 #TODO: DEPLOYMENT DATA
-#MAXTHREADS = 40
+MAXTHREADS = 41
-#DATABASE_PATH = r'C:\\Database\\chunks\\'
+DATABASE_PATH = r'C:\\Database\\chunks\\'
 
 ENCODING = "utf-8"
 CHUNk = "_chunk.txt"
 _sentinel = object()
 result_list = []
+WORK_Q_SIZE = 15500000
 
-def producer(out_q, control_q, filename):
+def producer(out_q, control_q, filename, logger):
     with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
-        lines = operated_file.readlines()
+        logger.info(f"Worker {filename} ")
-        for line in lines:
+        while True:
+            line = operated_file.readline()
+            if not line:
+                break
             out_q.put(line)
             try:
                 check = control_q.get(block=False)
@@ -28,11 +32,11 @@ def producer(out_q, control_q, filename):
                 check = False
                 pass
             if check is _sentinel:
-                print("TERM signal received")
+                logger.info("TERM signal received")
                 control_q.put(check)
                 break
-    print(f"Worker finished: {filename}")
+        logger.info(f"Worker finished: {filename}")
-    out_q.put(_sentinel)
+        out_q.put(_sentinel)
  
 #IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
 #TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
@@ -48,7 +52,7 @@ def consumer(in_q, control_q, doi, live_results, logger):
             sentinels += 1
         else:
             for item in result_list:
-                if item["DOI"] in data:
+                if item["DOI"] in data and not item["exists"]:
                     logger.info(data)
                     logger.info("HIT")
                     item["exists"] = True
@@ -63,18 +67,18 @@ def consumer(in_q, control_q, doi, live_results, logger):
 
 def search_for_doi(doi, live_results, logger):
     threads = []
-    work_q = Queue(maxsize=200000)
+    work_q = Queue(maxsize=WORK_Q_SIZE)
     control_q = Queue()
     t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
     logger.info("Consumer thread created")
     threads.append(t_cons)
     for i in range(0,MAXTHREADS):
         #TODO: TEST DATA
-        filename = "test" + str(i) + CHUNk
+        #filename = "test" + str(i) + CHUNk
         #TODO: DEPLOYMENT DATA
-        #filename = str(i) + CHUNk
+        filename = str(i) + CHUNk
         logger.info(f"Creating worker thread no: {i}")
-        threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
+        threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger)))
     for worker in threads:
         worker.start()
     for worker in threads: