conjurer/conjurer_librarian/conjurer_librarian.py

import re
import logging
import threading
from flask import Flask, jsonify, request
import netrc
import json
from platform import uname
from sys import platform
from urllib.request import urlopen
from habanero import Crossref
from waitress import serve

if platform in ("linux", "linux2"):
    SEPARATOR_FILE_PATH = "/"
    if "microsoft-standard" in uname().release:
        NETRC_FILE = "/home/mtuszowski/.netrc"
    else:
        NETRC_FILE = "/home/pi/.netrc"
elif platform == "win32":
    NETRC_FILE = "C:\\Users\\mtusz\\.netrc"

#HOST_ADDRESS = "192.168.1.191"
HOST_ADDRESS = "0.0.0.0"
PORT_ADDRESS = 5001

class Librarian(object):

    def __init__(self) -> None:
        netrc_mod = netrc.netrc(NETRC_FILE)
        authTokens = netrc_mod.authenticators("crossref")
        self.cr = Crossref(mailto=authTokens[0])
        self.last_search = ""
        self.limit = 1000

    def search_crossref(self, query, uuid):
        self.last_search = self.cr.works(query = query, limit = self.limit)
        #add to sql with UUID
        #sql row - title, doi, author, uuid, last retrieved, is_available
        with open("results.json", "r+", encoding="UTF-8") as data_file:
            # First we load existing data into a dict.
            file_data = json.load(data_file)
            # Join new_data with file_data inside emp_details
            summarized_results = []
            for item in self.last_search['message']['items']:
                container = "NO"
                if "container-title" in item:
                    container = "YES"
                summarized_results.append({"DOI" : item['DOI'], "title": item['title'], "type": item['type'], "container": container})
            result = {"UUID": uuid, "total_results": self.last_search['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.last_search['message']['items'],  }
            file_data.append(result)
            data_file.seek(0)
            # convert back to json.
            json.dump(file_data, data_file, indent=4)


    def check_if_exists(self,page_url):
        in_scihub_db = True
        if not page_url.startswith(("http:", "https:")):
            raise ValueError("URL must start with 'http:' or 'https:'")
        # trunk-ignore(bandit/B310)
        with urlopen(page_url) as response:
            info = response.info()
            print(info.get_content_type())
            data = response.read()
            text = data.decode("utf-8")
            for line in text.splitlines():
                if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
                    return None
        if in_scihub_db:
            return text
        return None

    def download(self, page_url):
        if response := self.check_if_exists(page_url):
            for line in response.splitlines():
                if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
                    return "https:" + str(m.group(1))
        return None

    def provide_sci_hub_link(self, page_url):
        if self.check_if_exists(page_url):
            return page_url
        return None

    def answer_query(self, uuid, query):
        #Search sql for UUID
        #SQL SELECT * FROM RESULTS WHERE UUID = uuid
        uuid_found = False
        with open("results.json", "r", encoding="UTF-8") as new_file:
            #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
            #ale tymczasowo plik
            database =  json.load(new_file)
        for item in database:
            if item["UUID"]:
                print item["UUID"]
                uuid_found = True
        if uuid_found:
            return self.last_search
        self.search_crossref(query=query, uuid=uuid)

#example not exists
#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
#sample usage to download
#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
#print(cl.download(sample_existing_page_url))
#cl.search_crossref("BDSM")

app = Flask(__name__)
def flask_debug():
    """
    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
    Do not use for production for fucks sake.
    """
    # trunk-ignore(bandit/B201)
    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)

def waitress_run():
    """
    The `waitress_run` function serves the `app` on host "0.0.0.0"
    and port 5000 using the Waitress WSGI server.
    """
    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)

@app.route("/query", methods=["POST"])
def query_database():
    """
    The function `update_music_list` receives a POST request with a JSON payload, logs the received
    item, adds it to a music file list, and returns a success message along with the updated record.
    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
    with values indicating the success of the operation and the data that was
    received and added to the `music_file_list`.
    The status code returned is 200, indicating a successful response.
    """
    record = json.loads(request.data)
    app.logger.info(record)
    app.logger.info(record["query"])
    app.logger.info(record["UUID"])

    cl = Librarian()

    answer_data = cl.last_search
    return_data = (
        jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
        200,
    )
    return return_data

if __name__ == "__main__":
    logger = logging.getLogger('conjurer_musician')
    logger.setLevel(logging.DEBUG)
    threads = []
    #threads.append(threading.Thread(target=waitress_run))
    threads.append(threading.Thread(target=flask_debug))

    for worker in threads:
        worker.start()
    for worker in threads:
        worker.join()

#1. robi liste wyników i ją wyświetla - ale tylko tytuły niezależnych dzieł - jeśli jest coś w containerze to trza pogrupować
#2. Tworzy przyciski pozwalające sprawdzić czy jest dostępne - i jeśli jest zwraca link
#3. Jest też przycisk "more" - wyszukujemy po 100, wyświetlamy po 20. Po 100 jak jeszcze to dopiero odpytujemy crossref przyciski strzałek przesuwają po obecnej porcji danych