Search webservice (#2)

* Deployable script for windows file service * Deploy! * Server side work
2026-07-19 08:12:09 +00:00 · 2024-04-12 23:21:48 +02:00
parent d912d9bc1b
commit 0ac9006665
5 changed files with 152 additions and 31 deletions
@@ -0,0 +1,93 @@
+import json
+from json.decoder import JSONDecodeError
+from urllib.request import urlopen
+from requests import Timeout,ConnectTimeout, ConnectionError
+from queue import Queue
+import re
+import time
+import random
+import logging
+from threading import Thread
+SCR_DATABASE_PATH = r'C:\\Database\\chunks\\'
+SCR_FILENAME = "40_chunk.txt"
+SCR_ENCODING = "utf-8"
+
+WORK_Q = Queue()
+random.seed()
+def load_ndb_to_q(logger):
+    logger.info("Loader started")
+    while True:    
+        with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
+            try:
+                ndb_database =  json.load(ndb_file)
+                for item in ndb_database.keys():
+                    logger.info(item)
+                    url = f"https://sci-hub.se/{item}"
+                    WORK_Q.put([item, url, False])
+                ndb_file.truncate(0)
+            except JSONDecodeError:
+                time.sleep(60*60*2)
+                pass
+        time.sleep(180)
+
+def check_if_exists_brute_force(logger):
+    while True:
+        logger.info("Scraper tick")
+        item = WORK_Q.get()
+        page_url =item[1] 
+        logger.error("REFINE: Brute force search!")
+        if not page_url.startswith(("http:", "https:")):
+            raise ValueError("URL must start with 'http:' or 'https:'")
+        blocked = True
+        try:
+        # trunk-ignore(bandit/B310)
+            with urlopen(page_url) as response:
+                data = response.read()
+                text = data.decode("utf-8")
+                if item[2]:
+                    logger.info("Already found")
+                    time.sleep(180)
+                    blocked = False
+                else:
+                    for line in text.splitlines():
+                        if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
+                            blocked = False
+                            logger.info("Not found")
+                            item[2] = False
+                        if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
+                            blocked = False
+                            direct_download_link = "https:" + str(m.group(1))
+                            logger.info (direct_download_link)
+                            with open(SCR_DATABASE_PATH + SCR_FILENAME, "a", encoding=SCR_ENCODING) as operated_file:
+                                operated_file.write("\n")
+                                operated_file.write(item[0])
+                                item[2] = True
+        except (Timeout,ConnectTimeout, ConnectionRefusedError, ConnectionError):
+            pass
+        if blocked:
+            logger.info(item)
+            logger.info(text)
+            logger.error("Got blocked. Fuck.")
+            time.sleep(60*60*72)
+        # trunk-ignore(bandit/B311)
+        rand = random.randint(1,100)
+        logger.info(f"Sleeping for {3*rand} minutes")
+        time.sleep(180*rand)
+
+def scraper(logger = None):
+    if not logger:
+        logger = logging.getLogger()
+        logger.setLevel("DEBUG")
+        h1 = logging.StreamHandler()
+        logger.addHandler(h1)
+        logger.info("No logger set. Test run")
+  
+    loader = Thread(target = load_ndb_to_q, args = (logger,))
+    scraper = Thread(target = check_if_exists_brute_force, args = (logger,))
+    loader.start()
+    scraper.start()
+    loader.join()
+    scraper.join()
+
+if __name__ == "__main__":
+    scraper()