mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
Fixing librarian result (#6)
* Formatting and additional logging for bugfix * Formatting and additional logging for bugfix * Change of log location * fixing_librarian_result * Hopefully final fix that will be needed * Temporary * Glebokie gardlo * Fix in scrape_bot * Fix that was needed. * Scraper settings * Small fix in scraping * FIxing bad naming convention * Log message fix * Test of deep search * Fix in deep search * Refactoring * WHat and idiot coded that.... Ooops. That was me. A month ago. * Refactoring continued * Bugfix * FIx * Not needed after fix
This commit is contained in:
GitHub
@@ -1,46 +1,67 @@
|
||||
"""
|
||||
This module contains the code for the scrape bot.
|
||||
"""
|
||||
|
||||
import json
|
||||
from json.decoder import JSONDecodeError
|
||||
from urllib.request import urlopen
|
||||
from requests import Timeout,ConnectTimeout, ConnectionError
|
||||
from queue import Queue
|
||||
import logging
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
import random
|
||||
import logging
|
||||
from json import JSONDecodeError
|
||||
from queue import Queue
|
||||
from threading import Thread
|
||||
SCR_DATABASE_PATH = r'C:\\Database\\chunks\\'
|
||||
from urllib.request import urlopen
|
||||
|
||||
from requests import ConnectionError as RequestsConnectionError
|
||||
from requests import ConnectTimeout, Timeout
|
||||
|
||||
SCR_DATABASE_PATH = r"C:\\Database\\chunks\\"
|
||||
SCR_FILENAME = "40_chunk.txt"
|
||||
SCR_ENCODING = "utf-8"
|
||||
|
||||
WORK_Q = Queue()
|
||||
random.seed()
|
||||
|
||||
|
||||
def load_ndb_to_q(logger):
|
||||
"""
|
||||
This function loads items from the not_in_db.json file into the work queue.
|
||||
"""
|
||||
logger.info("Loader started")
|
||||
while True:
|
||||
while True:
|
||||
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
|
||||
try:
|
||||
ndb_database = json.load(ndb_file)
|
||||
for item in ndb_database.keys():
|
||||
logger.info(item)
|
||||
url = f"https://sci-hub.se/{item}"
|
||||
ndb_database = json.load(ndb_file)
|
||||
for _ in range (1,10):
|
||||
key = next(iter(ndb_database))
|
||||
item = ndb_database.pop(key)
|
||||
logger.info(key)
|
||||
url = f"https://sci-hub.se/{key}"
|
||||
WORK_Q.put([item, url, False])
|
||||
|
||||
ndb_file.truncate(0)
|
||||
ndb_file.seek(0)
|
||||
json.dump(ndb_database, ndb_file, indent=4)
|
||||
except JSONDecodeError:
|
||||
time.sleep(60*60*2)
|
||||
pass
|
||||
time.sleep(180)
|
||||
time.sleep(60 * 60 * 3)
|
||||
time.sleep(60*60*3)
|
||||
|
||||
|
||||
def check_if_exists_brute_force(logger):
|
||||
"""
|
||||
This function checks if a document exists using brute force search.
|
||||
"""
|
||||
# Function code here
|
||||
while True:
|
||||
logger.info("Scraper tick")
|
||||
item = WORK_Q.get()
|
||||
page_url =item[1]
|
||||
page_url = item[1]
|
||||
logger.error("REFINE: Brute force search!")
|
||||
if not page_url.startswith(("http:", "https:")):
|
||||
raise ValueError("URL must start with 'http:' or 'https:'")
|
||||
blocked = True
|
||||
try:
|
||||
# trunk-ignore(bandit/B310)
|
||||
# trunk-ignore(bandit/B310)
|
||||
with urlopen(page_url) as response:
|
||||
data = response.read()
|
||||
text = data.decode("utf-8")
|
||||
@@ -50,44 +71,64 @@ def check_if_exists_brute_force(logger):
|
||||
blocked = False
|
||||
else:
|
||||
for line in text.splitlines():
|
||||
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
|
||||
if re.match(
|
||||
r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
|
||||
line,
|
||||
):
|
||||
blocked = False
|
||||
logger.info("Not found")
|
||||
item[2] = False
|
||||
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
|
||||
if m := re.match(
|
||||
r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
|
||||
):
|
||||
blocked = False
|
||||
direct_download_link = "https:" + str(m.group(1))
|
||||
logger.info (direct_download_link)
|
||||
with open(SCR_DATABASE_PATH + SCR_FILENAME, "a", encoding=SCR_ENCODING) as operated_file:
|
||||
logger.info(direct_download_link)
|
||||
with open(
|
||||
SCR_DATABASE_PATH + SCR_FILENAME,
|
||||
"a",
|
||||
encoding=SCR_ENCODING,
|
||||
) as operated_file:
|
||||
operated_file.write("\n")
|
||||
operated_file.write(item[0])
|
||||
item[2] = True
|
||||
except (Timeout,ConnectTimeout, ConnectionRefusedError, ConnectionError):
|
||||
except (
|
||||
Timeout,
|
||||
ConnectTimeout,
|
||||
ConnectionRefusedError,
|
||||
ConnectionError,
|
||||
RequestsConnectionError,
|
||||
):
|
||||
pass
|
||||
if blocked:
|
||||
logger.info(item)
|
||||
logger.info(text)
|
||||
logger.error("Got blocked. Fuck.")
|
||||
time.sleep(60*60*72)
|
||||
time.sleep(60 * 60 * 72)
|
||||
# trunk-ignore(bandit/B311)
|
||||
rand = random.randint(1,100)
|
||||
logger.info(f"Sleeping for {3*rand} minutes")
|
||||
time.sleep(180*rand)
|
||||
rand = random.randint(1, 60)
|
||||
logger.info(f"Sleeping for {2*rand} minutes")
|
||||
time.sleep(120 * rand)
|
||||
|
||||
def scraper(logger = None):
|
||||
|
||||
def scraper(logger=None):
|
||||
"""
|
||||
This function is responsible for scraping data.
|
||||
"""
|
||||
if not logger:
|
||||
logger = logging.getLogger()
|
||||
logger.setLevel("DEBUG")
|
||||
h1 = logging.StreamHandler()
|
||||
logger.addHandler(h1)
|
||||
logger.info("No logger set. Test run")
|
||||
|
||||
loader = Thread(target = load_ndb_to_q, args = (logger,))
|
||||
scraper = Thread(target = check_if_exists_brute_force, args = (logger,))
|
||||
|
||||
loader = Thread(target=load_ndb_to_q, args=(logger,))
|
||||
_scraper = Thread(target=check_if_exists_brute_force, args=(logger,))
|
||||
loader.start()
|
||||
scraper.start()
|
||||
_scraper.start()
|
||||
loader.join()
|
||||
scraper.join()
|
||||
_scraper.join()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
scraper()
|
||||
scraper()
|
||||
|
||||
Reference in New Issue
Block a user