conjurer/conjurer_librarian/conjurer_librarian.py

import re
import logging
import threading
import time
from flask import Flask, jsonify, request
import netrc
import json
from json.decoder import JSONDecodeError
from platform import uname
from sys import platform
from urllib.request import urlopen
import asyncio
from queue import Queue
from threading import Thread
from habanero import Crossref
from waitress import serve

if platform in ("linux", "linux2"):
    SEPARATOR_FILE_PATH = "/"
    if "microsoft-standard" in uname().release:
        NETRC_FILE = "/home/mtuszowski/.netrc"
    else:
        NETRC_FILE = "/home/pi/.netrc"
elif platform == "win32":
    NETRC_FILE = "C:\\Users\\mtusz\\.netrc"

#HOST_ADDRESS = "192.168.1.191"
HOST_ADDRESS = "0.0.0.0"
PORT_ADDRESS = 5001
DATABASE_PATH = r'C:\\Database\\chunks'
ENCODING = "utf-8"


class Librarian(object):

    def __init__(self) -> None:
        netrc_mod = netrc.netrc(NETRC_FILE)
        authTokens = netrc_mod.authenticators("crossref")
        self.cr = Crossref(mailto=authTokens[0])
        self.limit = 1000
        self.fetched = 0
        self.hit = 0
        self.total = 0

    def search_crossref(self, query, uuid):
        search_result_from_cr = self.cr.works(query = query, limit = self.limit)
        self.total = search_result_from_cr['message']['total-results']
        self.fetched = len(search_result_from_cr['message']['items'])
        #add to sql with UUID
        #sql row - title, doi, author, uuid, last retrieved, is_available
        with open("results.json", "r+", encoding="UTF-8") as data_file:
            # First we load existing data into a dict.
            try:
                file_data = json.load(data_file)
            except JSONDecodeError:
                file_data = []
            # Join new_data with file_data inside emp_details
            summarized_results = []
            for item in search_result_from_cr['message']['items']:
                container = "NO"
                if "container-title" in item:
                    container = "YES"
                summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
            result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'],  }
            file_data.append(result)
            print("Saving")
            data_file.seek(0)
            # convert back to json.
            json.dump(file_data, data_file, indent=4)
            return result
        #TODO: check if exists odpala wątki dla kolejnych plików, z każdego wątku dostaje zwrotki - trafienia na bieżąco oraz sygnał że plik się skończył i pora na następny
        #TODO: paczki danych są upychane po 10 w tablicy, POSTy ze strony klienta mówią tylko nazwę tablicy (poprzez UUID) oraz którą stronę tejże
        #TODO: jeśli UUID nie istnieje to budujemy całość dla nowego uuid
        #TODO: kiedy wszystkie wątki pilokowe się zakończą - zamykamy merger i dajemy znać do procedury obsługi odpowiadania na posty że dla tego uuid już nic więcej nie znajdziemy
        #TODO: po 60 minutach storeujemy wyniki zapytania pod query w sql i kasujemy UUID
        #TODO: Później robimy tak że query jest najpierw sprawdzane w sql - i dawane jako "zapisana lokalna kopia"
        #TODO: Paginacja - najpierw hity z pliku, potem na końcu to co nie zwróciło żadnych hitów - i to dopiero można crawlerem sprawdzić

    def check_dois_in_file(self, file_name, dois, out_queue):
        with open(f'{DATABASE_PATH}{file_name}', "r+", encoding=ENCODING) as file:
            while True:
                # Get next line from file
                line = file.readline()
                if line == dois:
                    return True
                # if line is empty
                # end of file is reached
                if not line:
                    return False

    def check_dois_in_local_db(self, dois):
        _sentinel = object()
        chunk_list = []
        q = Queue()

        for file in chunk_list:
            t = Thread(target = self.check_dois_in_file, args = (file ,dois, q))
            t.start()
        q.put(_sentinel)

    def check_if_exists(self, page_url):
        if not page_url.startswith(("http:", "https:")):
            raise ValueError("URL must start with 'http:' or 'https:'")
        # trunk-ignore(bandit/B310)
        with urlopen(page_url) as response:
            data = response.read()
            text = data.decode("utf-8")
            for line in text.splitlines():
                if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
                    return None
                if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
                    return "https:" + str(m.group(1))
        return None

    def provide_sci_hub_link(self, page_url):
        if self.check_if_exists(page_url):
            return page_url
        return None

    def refine_query(self, unrefined_reqult):
        print("Refine result")
        refined_result = []
        for item in unrefined_reqult["summary"]:
            if item["container"] == "NO" and item["title"]:
                item_link = "https://sci-hub.se/" + str(item['DOI'])
                if self.check_if_exists(item_link):
                    refined_result.append(item)
                    print ("HIT")
                    print(item["title"])
                time.sleep(0.01)
        print("Existing hits")
        self.hit = len(refined_result)
        return refined_result

    def answer_query(self, query, uuid):
        #Search sql for UUID
        #SQL SELECT * FROM RESULTS WHERE UUID = uuid
        print("Answer result")
        uuid_found = False
        database = None
        with open("results.json", "r", encoding="UTF-8") as new_file:
            #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
            #ale tymczasowo plik
            try:
                database =  json.load(new_file)
            except JSONDecodeError:
                pass
        if database:
            for item in database:
                if item["UUID"] == uuid:
                    print (item["UUID"])
                    uuid_found = True
                    return self.refine_query(item)

        if not uuid_found:
            answer = self.search_crossref(query=query, uuid=uuid)
            return self.refine_query(answer)

#example not exists
#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
#sample usage to download
#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
#print(cl.download(sample_existing_page_url))
#cl.search_crossref("BDSM")

app = Flask(__name__)
def flask_debug():
    """
    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
    Do not use for production for fucks sake.
    """
    # trunk-ignore(bandit/B201)
    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)

def waitress_run():
    """
    The `waitress_run` function serves the `app` on host "0.0.0.0"
    and port 5000 using the Waitress WSGI server.
    """
    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)

@app.route("/query", methods=["POST"])
def query_database():
    record = json.loads(request.data)
    app.logger.info(record)
    app.logger.info(record["query"])
    app.logger.info(record["UUID"])
    cl = Librarian()
    answer_data = cl.answer_query(record["query"], record["UUID"])
    app.logger.info(answer_data)
    summary = (cl.hit,cl.fetched, cl.total, record["UUID"])
    answer_data = (summary,answer_data)
    return_data = (
        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
        200,
    )
    return return_data

if __name__ == "__main__":
    logger = logging.getLogger('conjurer_musician')
    logger.setLevel(logging.DEBUG)
    threads = []
    #threads.append(threading.Thread(target=waitress_run))
    threads.append(threading.Thread(target=flask_debug))

    for worker in threads:
        worker.start()
    for worker in threads:
        worker.join()

#1. robi liste wyników i ją wyświetla - ale tylko tytuły niezależnych dzieł - jeśli jest coś w containerze to trza pogrupować
#2. Tworzy przyciski pozwalające sprawdzić czy jest dostępne - i jeśli jest zwraca link
#3. Jest też przycisk "more" - wyszukujemy po 100, wyświetlamy po 20. Po 100 jak jeszcze to dopiero odpytujemy crossref przyciski strzałek przesuwają po obecnej porcji danych