conjurer/conjurer_librarian/conjurer_librarian.py

import re
import logging
import threading
import time
from flask import Flask, jsonify, request
import netrc
import json
import asyncio
from json.decoder import JSONDecodeError
from urllib.request import urlopen
from habanero import Crossref
from waitress import serve
from queue import Queue
import search_bot

NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"

HOST_ADDRESS = "192.168.1.192"
PORT_ADDRESS = 5001
DATABASE_PATH = r'C:\\Database\\chunks'
ENCODING = "utf-8"
app = Flask(__name__)

librarian_queue = Queue()

class Librarian(object):

    def __init__(self, app, query, uuid) -> None:
        netrc_mod = netrc.netrc(NETRC_FILE)
        authTokens = netrc_mod.authenticators("crossref")
        self.cr = Crossref(mailto=authTokens[0])
        self.query = query
        self.uuid = uuid
        self.limit = 1000
        self.fetched = 0
        self.hit = 0
        self.total = 0
        self.app = app

    async def search_crossref(self, query, uuid):
        self.app.logger.info("STARTED SEARCH")
        search_result_from_cr = self.cr.works(query = query, limit = self.limit)
        self.app.logger.info("CROSSREF DONE")
        self.total = search_result_from_cr['message']['total-results']
        self.fetched = len(search_result_from_cr['message']['items'])
        #add to sql with UUID
        #sql row - title, doi, author, uuid, last retrieved, is_available
        
        with open("results.json", "r+", encoding="UTF-8") as data_file:
            # First we load existing data into a dict.
            try:
                file_data = json.load(data_file)
            except JSONDecodeError:
                file_data = []
            summarized_results = []
            self.app.logger.info("REFINE: Removing all derived works from the list")
            for item in search_result_from_cr['message']['items']:
                container = "NO"
                if "container-title" in item:
                    container = "YES"
                summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
            result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'],  }
            file_data.append(result)
            data_file.seek(0)
            self.app.logger.info("REFINE: Dumping to file")
            json.dump(file_data, data_file, indent=4)
            return result
        #TODO: check if exists odpala wątki dla kolejnych plików, z każdego wątku dostaje zwrotki - trafienia na bieżąco oraz sygnał że plik się skończył i pora na następny
        #TODO: paczki danych są upychane po 10 w tablicy, POSTy ze strony klienta mówią tylko nazwę tablicy (poprzez UUID) oraz którą stronę tejże
        #TODO: jeśli UUID nie istnieje to budujemy całość dla nowego uuid
        #TODO: kiedy wszystkie wątki pilokowe się zakończą - zamykamy merger i dajemy znać do procedury obsługi odpowiadania na posty że dla tego uuid już nic więcej nie znajdziemy
        #TODO: po 60 minutach storeujemy wyniki zapytania pod query w sql i kasujemy UUID
        #TODO: Później robimy tak że query jest najpierw sprawdzane w sql - i dawane jako "zapisana lokalna kopia"
        #TODO: Paginacja - najpierw hity z pliku, potem na końcu to co nie zwróciło żadnych hitów - i to dopiero można crawlerem sprawdzić


    async def check_dois_in_local_db(self, doi):
        self.app.logger.info("REFINE: Running search in the backend app")
        coro = asyncio.to_thread(search_bot.search_for_doi, doi)
        result = await coro
        return result

    async def check_if_exists_brute_force(self, page_url):
        self.app.logger.error("REFINE: Brute force search!")
        if not page_url.startswith(("http:", "https:")):
            raise ValueError("URL must start with 'http:' or 'https:'")
        # trunk-ignore(bandit/B310)
        with urlopen(page_url) as response:
            data = response.read()
            text = data.decode("utf-8")
            for line in text.splitlines():
                if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
                    return True
                if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
                    direct_download_link = "https:" + str(m.group(1))
                    print (direct_download_link)
                    return True
        return False

    async def check_if_exists(self, doi, brute_force = False):
        result = []
        result = await self.check_dois_in_local_db(doi)
        if brute_force and not result:
            for item in doi:
                item_link = "https://sci-hub.se/" + item[0]
                result.append(await self.check_if_exists_brute_force(item_link))
        return result

    async def refine_query(self, unrefined_reqult, brute_force = False):
        print("Refine result")
        refined_result = []
        temp = []
        for item in unrefined_reqult["summary"]:
            if item["container"] == "NO" and item["title"]:
                refined_result.append[item]
        for item in refined_result:
            temp.append[(item["DOI"], item)]

        refined_result = self.check_if_exists(temp)
        temp = []
        for item in refined_result:
            if item["exists"]:
                temp.append(item["data"])
        refined_result = temp
        self.hit = len(refined_result)
        return refined_result

    async def answer_query(self):
        #Search sql for UUID
        #SQL SELECT * FROM RESULTS WHERE UUID = uuid
        uuid_found = False
        database = None
        query = self.query
        uuid = self.uuid
        with open("results.json", "r", encoding="UTF-8") as new_file:
            #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
            #ale tymczasowo plik
            try:
                database =  json.load(new_file)
            except JSONDecodeError:
                pass
        if database:
            for item in database:
                if item["UUID"] == uuid:
                    print (item["UUID"])
                    uuid_found = True
                    #TODO: THREAD IT
                    result = await self.refine_query(item)
                    return result

        if not uuid_found:
            answer = await self.search_crossref(query=query, uuid=uuid)
            print("Refined")
            return await self.refine_query(answer)

#============================= FLASK INTERNALS===============================
def flask_debug():
    """
    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
    Do not use for production for fucks sake.
    """
    # trunk-ignore(bandit/B201)
    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)

def waitress_run():
    """
    The `waitress_run` function serves the `app` on host "0.0.0.0"
    and port 5000 using the Waitress WSGI server.
    """
    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)

# TUTAJ TYLKO LOGUJEMY ŻE PRZYSZŁO, TWORZYMY LIBRARIAN class dla danego zapytania (po UUID)
# ODPALAMY SZUKANIE I POZWALAMYSIĘ JUŻ DALEJ KRĘCIĆ
#==================================SERVER ROUTES==========================================
@app.route("/query", methods=["POST"])
async def query_database():
    record = json.loads(request.data)
    app.logger.info(record)
    app.logger.info(record["query"])
    app.logger.info(record["UUID"])
    cl = Librarian(app, record["query"], record["UUID"])
    librarian_queue.put(cl)


    answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
    return_data = (
        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
        200,
    )
    return return_data
#=======================================MAIN===================================================
if __name__ == "__main__":
    logger = logging.getLogger('conjurer_musician')
    logger.setLevel(logging.DEBUG)
    threads = []
    #threads.append(threading.Thread(target=waitress_run))
    threads.append(threading.Thread(target=flask_debug))

    for worker in threads:
        worker.start()
    for worker in threads:
        worker.join()




#example not exists
#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
#sample usage to download
#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
#print(cl.download(sample_existing_page_url))
#cl.search_crossref("BDSM")