import re import logging import threading import time from flask import Flask, jsonify, request import netrc import json from json.decoder import JSONDecodeError from platform import uname from sys import platform from urllib.request import urlopen from habanero import Crossref from waitress import serve if platform in ("linux", "linux2"): SEPARATOR_FILE_PATH = "/" if "microsoft-standard" in uname().release: NETRC_FILE = "/home/mtuszowski/.netrc" else: NETRC_FILE = "/home/pi/.netrc" elif platform == "win32": NETRC_FILE = "C:\\Users\\mtusz\\.netrc" #HOST_ADDRESS = "192.168.1.191" HOST_ADDRESS = "0.0.0.0" PORT_ADDRESS = 5001 class Librarian(object): def __init__(self) -> None: netrc_mod = netrc.netrc(NETRC_FILE) authTokens = netrc_mod.authenticators("crossref") self.cr = Crossref(mailto=authTokens[0]) self.limit = 1000 self.fetched = 0 self.hit = 0 self.total = 0 def search_crossref(self, query, uuid): search_result_from_cr = self.cr.works(query = query, limit = self.limit) self.total = search_result_from_cr['message']['total-results'] self.fetched = len(search_result_from_cr['message']['items']) #add to sql with UUID #sql row - title, doi, author, uuid, last retrieved, is_available with open("results.json", "r+", encoding="UTF-8") as data_file: # First we load existing data into a dict. try: file_data = json.load(data_file) except JSONDecodeError: file_data = [] # Join new_data with file_data inside emp_details summarized_results = [] for item in search_result_from_cr['message']['items']: container = "NO" if "container-title" in item: container = "YES" summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container}) result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'], } file_data.append(result) print("Saving") data_file.seek(0) # convert back to json. json.dump(file_data, data_file, indent=4) return result def check_if_exists(self,page_url): in_scihub_db = True if not page_url.startswith(("http:", "https:")): raise ValueError("URL must start with 'http:' or 'https:'") # trunk-ignore(bandit/B310) with urlopen(page_url) as response: data = response.read() text = data.decode("utf-8") for line in text.splitlines(): if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): return None if in_scihub_db: return text return None def download(self, page_url): if response := self.check_if_exists(page_url): for line in response.splitlines(): if m := re.match(r".*