diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py new file mode 100644 index 0000000..b0942aa --- /dev/null +++ b/conjurer_librarian/conjurer_librarian.py @@ -0,0 +1,167 @@ +import re +import logging +import threading +from flask import Flask, jsonify, request +import netrc +import json +from json.decoder import JSONDecodeError +from platform import uname +from sys import platform +from urllib.request import urlopen +from habanero import Crossref +from waitress import serve + +if platform in ("linux", "linux2"): + SEPARATOR_FILE_PATH = "/" + if "microsoft-standard" in uname().release: + NETRC_FILE = "/home/mtuszowski/.netrc" + else: + NETRC_FILE = "/home/pi/.netrc" +elif platform == "win32": + NETRC_FILE = "C:\\Users\\mtusz\\.netrc" + +#HOST_ADDRESS = "192.168.1.191" +HOST_ADDRESS = "0.0.0.0" +PORT_ADDRESS = 5001 + +class Librarian(object): + + def __init__(self) -> None: + netrc_mod = netrc.netrc(NETRC_FILE) + authTokens = netrc_mod.authenticators("crossref") + self.cr = Crossref(mailto=authTokens[0]) + self.last_search = "" + self.limit = 20 + + def search_crossref(self, query, uuid): + self.last_search = self.cr.works(query = query, limit = self.limit) + #add to sql with UUID + #sql row - title, doi, author, uuid, last retrieved, is_available + with open("results.json", "r+", encoding="UTF-8") as data_file: + # First we load existing data into a dict. + try: + file_data = json.load(data_file) + except JSONDecodeError: + file_data = [] + # Join new_data with file_data inside emp_details + summarized_results = [] + for item in self.last_search['message']['items']: + container = "NO" + if "container-title" in item: + container = "YES" + print(item['title']) + print(item['DOI']) + + summarized_results.append({"DOI" : item['DOI'], "title": item['title'], "type": item['type'], "container": container}) + result = {"UUID": uuid, "total_results": self.last_search['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.last_search['message']['items'], } + file_data.append(result) + print("Saving") + data_file.seek(0) + # convert back to json. + json.dump(file_data, data_file, indent=4) + + + def check_if_exists(self,page_url): + in_scihub_db = True + if not page_url.startswith(("http:", "https:")): + raise ValueError("URL must start with 'http:' or 'https:'") + # trunk-ignore(bandit/B310) + with urlopen(page_url) as response: + info = response.info() + print(info.get_content_type()) + data = response.read() + text = data.decode("utf-8") + for line in text.splitlines(): + if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): + return None + if in_scihub_db: + return text + return None + + def download(self, page_url): + if response := self.check_if_exists(page_url): + for line in response.splitlines(): + if m := re.match(r".*