diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index a7be7fd..9098cf6 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -12,9 +12,8 @@ from habanero import Crossref from waitress import serve from queue import Queue import search_bot +import scrape_bot import requests -from typing import Callable -from uuid import UUID NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" HOST_ADDRESS = "192.168.1.192" @@ -22,7 +21,7 @@ PORT_ADDRESS = 5001 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" SEND_RESULTS = "/conjurer" BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147' - +MAX_CR_RESULTS = 150000 ENCODING = "utf-8" app = Flask(__name__) @@ -37,13 +36,15 @@ class Librarian(object): self.cr = Crossref(mailto=authTokens[0]) self.query = query self.uuid = str(uuid) - self.limit = 3000 + self.limit = MAX_CR_RESULTS self.fetched = 0 self.hit = 0 self.total = 0 self.app = app self.logger = app.logger self.live_results = [] + self.final_result = [] + self.not_in_db = [] self.search_result_from_cr = {} self.done = False @@ -62,8 +63,7 @@ class Librarian(object): self.fetched += len(result['message']['items']) self.logger.info (self.total) self.logger.info(self.fetched) - - time.sleep(1) + time.sleep(0.1) self.app.logger.info("CROSSREF DONE") #TODO: add to sql with UUID #sql row - title, doi, author, uuid, last retrieved, is_available @@ -142,20 +142,20 @@ class Librarian(object): temp.append([item["DOI"], item]) refined_result = await self.check_if_exists(temp) - temp = [] + for item in refined_result: if item["exists"]: - temp.append(item["data"]) - refined_result = temp - self.hit = len(refined_result) - return refined_result + self.final_result.append(item["data"]) + else: + self.not_in_db.append(item["data"]) + self.hit = len(self.final_result) + return self.final_result #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku async def answer_query(self): #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany #Search sql for UUID #SQL SELECT * FROM RESULTS WHERE UUID = uuid self.logger.info(f"Search started {self.uuid}") - uuid_found = False database = None query = self.query with open("cr_results.json", "r+", encoding="utf-8") as cr_file: @@ -165,7 +165,7 @@ class Librarian(object): database = json.load(cr_file) except JSONDecodeError: pass - + #TODO: Zwracać i final_result i not_in_db. Not in db_do pliku - i do ręcznego przeglądania potem. if database: for item in database.keys(): if item in self.uuid: @@ -174,6 +174,10 @@ class Librarian(object): for item in self.result: temp[item['DOI']] = {"Title" : item['title'], "type": item['type']} self.result = temp + temp2 = {} + for item in self.not_in_db: + temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']} + self.not_in_db = temp2 return self.result answer = await self.search_crossref(query=query) database.update(answer) @@ -194,6 +198,10 @@ class Librarian(object): for item in self.result: temp[item['DOI']] = {"Title" : item['title'], "type": item['type']} self.result = temp + temp2 = {} + for item in self.not_in_db: + temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']} + self.not_in_db = temp2 return self.result #============================= FLASK INTERNALS=============================== @@ -221,11 +229,25 @@ class BackgroundTaskSearch(threading.Thread): async def _run(self): while True: database = None + ndb_database = None librarian = librarian_queue.get() self.app.logger.info("STARTED") result = await librarian.answer_query() result = {librarian.uuid: result} self.app.logger.info("Saving to file") + with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file: + try: + ndb_database = json.load(ndb_file) + except JSONDecodeError: + pass + if ndb_database: + ndb_database.update(librarian.not_in_db) + else: + ndb_database = librarian.not_in_db + ndb_file.truncate(0) + ndb_file.seek(0) + json.dump(ndb_database, ndb_file) + with open("s_results.json", "r+", encoding="utf-8") as s_file: try: database = json.load(s_file) @@ -291,14 +313,15 @@ async def get_partial(): #=======================================MAIN=================================================== if __name__ == "__main__": - logger = logging.getLogger('conjurer_musician') + logger = app.logger logger.setLevel(logging.DEBUG) threads = [] - #threads.append(threading.Thread(target=waitress_run)) - threads.append(threading.Thread(target=flask_debug)) + threads.append(threading.Thread(target=waitress_run)) + #threads.append(threading.Thread(target=flask_debug)) bgtask = BackgroundTaskSearch() bgtask.app = app threads.append(bgtask) + threads.append(threading.Thread(target = scrape_bot.scraper, args=(logger,))) i = 0 for worker in threads: try: diff --git a/conjurer_librarian/run_server.bat b/conjurer_librarian/run_server.bat new file mode 100644 index 0000000..563e30a --- /dev/null +++ b/conjurer_librarian/run_server.bat @@ -0,0 +1 @@ +c:\conjurer\env\Scripts\activate.bat && c:\conjurer\conjurer_librarian\conjurer_librarian.py diff --git a/conjurer_librarian/scrape_bot.py b/conjurer_librarian/scrape_bot.py new file mode 100644 index 0000000..bdf0a3b --- /dev/null +++ b/conjurer_librarian/scrape_bot.py @@ -0,0 +1,93 @@ +import json +from json.decoder import JSONDecodeError +from urllib.request import urlopen +from requests import Timeout,ConnectTimeout, ConnectionError +from queue import Queue +import re +import time +import random +import logging +from threading import Thread +SCR_DATABASE_PATH = r'C:\\Database\\chunks\\' +SCR_FILENAME = "40_chunk.txt" +SCR_ENCODING = "utf-8" + +WORK_Q = Queue() +random.seed() +def load_ndb_to_q(logger): + logger.info("Loader started") + while True: + with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file: + try: + ndb_database = json.load(ndb_file) + for item in ndb_database.keys(): + logger.info(item) + url = f"https://sci-hub.se/{item}" + WORK_Q.put([item, url, False]) + ndb_file.truncate(0) + except JSONDecodeError: + time.sleep(60*60*2) + pass + time.sleep(180) + +def check_if_exists_brute_force(logger): + while True: + logger.info("Scraper tick") + item = WORK_Q.get() + page_url =item[1] + logger.error("REFINE: Brute force search!") + if not page_url.startswith(("http:", "https:")): + raise ValueError("URL must start with 'http:' or 'https:'") + blocked = True + try: + # trunk-ignore(bandit/B310) + with urlopen(page_url) as response: + data = response.read() + text = data.decode("utf-8") + if item[2]: + logger.info("Already found") + time.sleep(180) + blocked = False + else: + for line in text.splitlines(): + if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): + blocked = False + logger.info("Not found") + item[2] = False + if m := re.match(r".* (doi, True) @@ -48,7 +52,7 @@ def consumer(in_q, control_q, doi, live_results, logger): sentinels += 1 else: for item in result_list: - if item["DOI"] in data: + if item["DOI"] in data and not item["exists"]: logger.info(data) logger.info("HIT") item["exists"] = True @@ -63,18 +67,18 @@ def consumer(in_q, control_q, doi, live_results, logger): def search_for_doi(doi, live_results, logger): threads = [] - work_q = Queue(maxsize=200000) + work_q = Queue(maxsize=WORK_Q_SIZE) control_q = Queue() t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger)) logger.info("Consumer thread created") threads.append(t_cons) for i in range(0,MAXTHREADS): #TODO: TEST DATA - filename = "test" + str(i) + CHUNk + #filename = "test" + str(i) + CHUNk #TODO: DEPLOYMENT DATA - #filename = str(i) + CHUNk + filename = str(i) + CHUNk logger.info(f"Creating worker thread no: {i}") - threads.append(Thread(target = producer, args = (work_q, control_q, filename))) + threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger))) for worker in threads: worker.start() for worker in threads: diff --git a/not_in_db.json b/not_in_db.json new file mode 100644 index 0000000..e69de29