import re import logging import threading from flask import Flask, jsonify, request import netrc import json from json.decoder import JSONDecodeError from platform import uname from sys import platform from urllib.request import urlopen from habanero import Crossref from waitress import serve if platform in ("linux", "linux2"): SEPARATOR_FILE_PATH = "/" if "microsoft-standard" in uname().release: NETRC_FILE = "/home/mtuszowski/.netrc" else: NETRC_FILE = "/home/pi/.netrc" elif platform == "win32": NETRC_FILE = "C:\\Users\\mtusz\\.netrc" #HOST_ADDRESS = "192.168.1.191" HOST_ADDRESS = "0.0.0.0" PORT_ADDRESS = 5001 class Librarian(object): def __init__(self) -> None: netrc_mod = netrc.netrc(NETRC_FILE) authTokens = netrc_mod.authenticators("crossref") self.cr = Crossref(mailto=authTokens[0]) self.last_search = "" self.limit = 20 def search_crossref(self, query, uuid): self.last_search = self.cr.works(query = query, limit = self.limit) #add to sql with UUID #sql row - title, doi, author, uuid, last retrieved, is_available with open("results.json", "r+", encoding="UTF-8") as data_file: # First we load existing data into a dict. try: file_data = json.load(data_file) except JSONDecodeError: file_data = [] # Join new_data with file_data inside emp_details summarized_results = [] for item in self.last_search['message']['items']: container = "NO" if "container-title" in item: container = "YES" print(item['title']) print(item['DOI']) summarized_results.append({"DOI" : item['DOI'], "title": item['title'], "type": item['type'], "container": container}) result = {"UUID": uuid, "total_results": self.last_search['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.last_search['message']['items'], } file_data.append(result) print("Saving") data_file.seek(0) # convert back to json. json.dump(file_data, data_file, indent=4) def check_if_exists(self,page_url): in_scihub_db = True if not page_url.startswith(("http:", "https:")): raise ValueError("URL must start with 'http:' or 'https:'") # trunk-ignore(bandit/B310) with urlopen(page_url) as response: info = response.info() print(info.get_content_type()) data = response.read() text = data.decode("utf-8") for line in text.splitlines(): if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): return None if in_scihub_db: return text return None def download(self, page_url): if response := self.check_if_exists(page_url): for line in response.splitlines(): if m := re.match(r".*