import json
from json.decoder import JSONDecodeError
from urllib.request import urlopen
from requests import Timeout,ConnectTimeout, ConnectionError
from queue import Queue
import re
import time
import random
import logging
from threading import Thread
SCR_DATABASE_PATH = r'C:\\Database\\chunks\\'
SCR_FILENAME = "40_chunk.txt"
SCR_ENCODING = "utf-8"

WORK_Q = Queue()
random.seed()
def load_ndb_to_q(logger):
    logger.info("Loader started")
    while True:    
        with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
            try:
                ndb_database =  json.load(ndb_file)
                for item in ndb_database.keys():
                    logger.info(item)
                    url = f"https://sci-hub.se/{item}"
                    WORK_Q.put([item, url, False])
                ndb_file.truncate(0)
            except JSONDecodeError:
                time.sleep(60*60*2)
                pass
        time.sleep(180)

def check_if_exists_brute_force(logger):
    while True:
        logger.info("Scraper tick")
        item = WORK_Q.get()
        page_url =item[1] 
        logger.error("REFINE: Brute force search!")
        if not page_url.startswith(("http:", "https:")):
            raise ValueError("URL must start with 'http:' or 'https:'")
        blocked = True
        try:
        # trunk-ignore(bandit/B310)
            with urlopen(page_url) as response:
                data = response.read()
                text = data.decode("utf-8")
                if item[2]:
                    logger.info("Already found")
                    time.sleep(180)
                    blocked = False
                else:
                    for line in text.splitlines():
                        if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
                            blocked = False
                            logger.info("Not found")
                            item[2] = False
                        if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
                            blocked = False
                            direct_download_link = "https:" + str(m.group(1))
                            logger.info (direct_download_link)
                            with open(SCR_DATABASE_PATH + SCR_FILENAME, "a", encoding=SCR_ENCODING) as operated_file:
                                operated_file.write("\n")
                                operated_file.write(item[0])
                                item[2] = True
        except (Timeout,ConnectTimeout, ConnectionRefusedError, ConnectionError):
            pass
        if blocked:
            logger.info(item)
            logger.info(text)
            logger.error("Got blocked. Fuck.")
            time.sleep(60*60*72)
        # trunk-ignore(bandit/B311)
        rand = random.randint(1,100)
        logger.info(f"Sleeping for {3*rand} minutes")
        time.sleep(180*rand)

def scraper(logger = None):
    if not logger:
        logger = logging.getLogger()
        logger.setLevel("DEBUG")
        h1 = logging.StreamHandler()
        logger.addHandler(h1)
        logger.info("No logger set. Test run")
  
    loader = Thread(target = load_ndb_to_q, args = (logger,))
    scraper = Thread(target = check_if_exists_brute_force, args = (logger,))
    loader.start()
    scraper.start()
    loader.join()
    scraper.join()

if __name__ == "__main__":
    scraper()