diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index 9920c8d..27649ee 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -1,27 +1,29 @@ -import re +import asyncio +import json import logging +import netrc +import re import threading import time -from flask import Flask, jsonify, request -import netrc -import json -import asyncio from json.decoder import JSONDecodeError +from queue import Queue +from typing import Callable from urllib.request import urlopen +from uuid import UUID + +import requests +import search_bot +from flask import Flask, jsonify, request from habanero import Crossref from waitress import serve -from queue import Queue -import search_bot -import requests -from typing import Callable -from uuid import UUID + NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" HOST_ADDRESS = "192.168.1.192" PORT_ADDRESS = 5001 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" SEND_RESULTS = "/conjurer" -BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147' +BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147" ENCODING = "utf-8" app = Flask(__name__) @@ -29,6 +31,7 @@ app = Flask(__name__) librarian_queue = Queue() librarian_list = [] + class Librarian(object): def __init__(self, app, query, uuid) -> None: @@ -49,25 +52,25 @@ class Librarian(object): async def search_crossref(self, query): self.app.logger.info("STARTED SEARCH") - result = self.cr.works(query = query, limit = 1000) + result = self.cr.works(query=query, limit=1000) self.search_result_from_cr.update(result) - self.total = result['message']['total-results'] - self.fetched += len(result['message']['items']) - self.logger.info (self.total) + self.total = result["message"]["total-results"] + self.fetched += len(result["message"]["items"]) + self.logger.info(self.total) self.logger.info(self.fetched) while self.total > self.fetched and self.limit > self.fetched: - result = self.cr.works(query = query, limit = 1000) + result = self.cr.works(query=query, limit=1000) self.search_result_from_cr.update(result) - self.total = result['message']['total-results'] - self.fetched += len(result['message']['items']) - self.logger.info (self.total) + self.total = result["message"]["total-results"] + self.fetched += len(result["message"]["items"]) + self.logger.info(self.total) self.logger.info(self.fetched) time.sleep(1) self.app.logger.info("CROSSREF DONE") - #TODO: add to sql with UUID - #sql row - title, doi, author, uuid, last retrieved, is_available - + # TODO: add to sql with UUID + # sql row - title, doi, author, uuid, last retrieved, is_available + with open("rr_results.json", "r+", encoding="utf-8") as data_file: # First we load existing data into a dict. try: @@ -76,18 +79,34 @@ class Librarian(object): file_data = {} summarized_results = [] self.app.logger.info("REFINE: Removing all derived works from the list") - for item in self.search_result_from_cr['message']['items']: + for item in self.search_result_from_cr["message"]["items"]: container = "NO" if "container-title" in item: container = "YES" else: - summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container}) - result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}} + summarized_results.append( + { + "DOI": item["DOI"], + "title": item["title"] if "title" in item else None, + "type": item["type"] if "type" in item else None, + "container": container, + } + ) + result = { + self.uuid: { + "total_results": self.search_result_from_cr["message"][ + "total-results" + ], + "on_page": 1, + "summary": summarized_results, + "results": self.search_result_from_cr["message"]["items"], + } + } if file_data: file_data.update(result) else: file_data = result - + self.app.logger.info("REFINE: Dumping to file") data_file.truncate(0) data_file.seek(0) @@ -96,7 +115,9 @@ class Librarian(object): async def check_dois_in_local_db(self, doi): self.app.logger.info("REFINE: Running search in the backend app") - coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger) + coro = asyncio.to_thread( + search_bot.search_for_doi, doi, self.live_results, self.app.logger + ) result = await coro return result @@ -109,25 +130,34 @@ class Librarian(object): data = response.read() text = data.decode("utf-8") for line in text.splitlines(): - if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): + if re.match( + r".*Unfortunately, Sci-Hub doesn't have the requested document.*", + line, + ): return False - if m := re.match(r".*