mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
93 lines
3.4 KiB
Python
93 lines
3.4 KiB
Python
import json
|
|
from json.decoder import JSONDecodeError
|
|
from urllib.request import urlopen
|
|
from requests import Timeout,ConnectTimeout, ConnectionError
|
|
from queue import Queue
|
|
import re
|
|
import time
|
|
import random
|
|
import logging
|
|
from threading import Thread
|
|
SCR_DATABASE_PATH = r'C:\\Database\\chunks\\'
|
|
SCR_FILENAME = "40_chunk.txt"
|
|
SCR_ENCODING = "utf-8"
|
|
|
|
WORK_Q = Queue()
|
|
random.seed()
|
|
def load_ndb_to_q(logger):
|
|
logger.info("Loader started")
|
|
while True:
|
|
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
|
|
try:
|
|
ndb_database = json.load(ndb_file)
|
|
for item in ndb_database.keys():
|
|
logger.info(item)
|
|
url = f"https://sci-hub.se/{item}"
|
|
WORK_Q.put([item, url, False])
|
|
ndb_file.truncate(0)
|
|
except JSONDecodeError:
|
|
time.sleep(60*60*2)
|
|
pass
|
|
time.sleep(180)
|
|
|
|
def check_if_exists_brute_force(logger):
|
|
while True:
|
|
logger.info("Scraper tick")
|
|
item = WORK_Q.get()
|
|
page_url =item[1]
|
|
logger.error("REFINE: Brute force search!")
|
|
if not page_url.startswith(("http:", "https:")):
|
|
raise ValueError("URL must start with 'http:' or 'https:'")
|
|
blocked = True
|
|
try:
|
|
# trunk-ignore(bandit/B310)
|
|
with urlopen(page_url) as response:
|
|
data = response.read()
|
|
text = data.decode("utf-8")
|
|
if item[2]:
|
|
logger.info("Already found")
|
|
time.sleep(180)
|
|
blocked = False
|
|
else:
|
|
for line in text.splitlines():
|
|
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
|
|
blocked = False
|
|
logger.info("Not found")
|
|
item[2] = False
|
|
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
|
|
blocked = False
|
|
direct_download_link = "https:" + str(m.group(1))
|
|
logger.info (direct_download_link)
|
|
with open(SCR_DATABASE_PATH + SCR_FILENAME, "a", encoding=SCR_ENCODING) as operated_file:
|
|
operated_file.write("\n")
|
|
operated_file.write(item[0])
|
|
item[2] = True
|
|
except (Timeout,ConnectTimeout, ConnectionRefusedError, ConnectionError):
|
|
pass
|
|
if blocked:
|
|
logger.info(item)
|
|
logger.info(text)
|
|
logger.error("Got blocked. Fuck.")
|
|
time.sleep(60*60*72)
|
|
# trunk-ignore(bandit/B311)
|
|
rand = random.randint(1,100)
|
|
logger.info(f"Sleeping for {3*rand} minutes")
|
|
time.sleep(180*rand)
|
|
|
|
def scraper(logger = None):
|
|
if not logger:
|
|
logger = logging.getLogger()
|
|
logger.setLevel("DEBUG")
|
|
h1 = logging.StreamHandler()
|
|
logger.addHandler(h1)
|
|
logger.info("No logger set. Test run")
|
|
|
|
loader = Thread(target = load_ndb_to_q, args = (logger,))
|
|
scraper = Thread(target = check_if_exists_brute_force, args = (logger,))
|
|
loader.start()
|
|
scraper.start()
|
|
loader.join()
|
|
scraper.join()
|
|
|
|
if __name__ == "__main__":
|
|
scraper() |