diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index 9740009..8065661 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -1,22 +1,35 @@ +""" +This module contains the implementation of the Librarian class and related functions for searching and refining queries. + +Classes: +- Librarian: Represents a librarian object that performs search and refinement operations on queries. + +Functions: +- flask_debug: Starts a Flask application in debug mode without using the reloader. +- waitress_run: Serves the Flask application using the Waitress WSGI server. +- BackgroundTaskSearch: Represents a background task for running the Librarian object asynchronously. +""" + +import netrc +import json +import asyncio +from json.decoder import JSONDecodeError +from urllib.request import urlopen +from queue import Queue from logging import handlers import re import logging import threading import time from flask import Flask, jsonify, request -import netrc -import json -import asyncio -from json.decoder import JSONDecodeError -from urllib.request import urlopen from habanero import Crossref from waitress import serve -from queue import Queue import search_bot import scrape_bot import requests -NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" +# Constants +NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" HOST_ADDRESS = "192.168.1.192" PORT_ADDRESS = 5001 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" @@ -24,14 +37,37 @@ SEND_RESULTS = "/conjurer" BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147' MAX_CR_RESULTS = 150000 ENCODING = "utf-8" + app = Flask(__name__) librarian_queue = Queue() librarian_list = [] class Librarian(object): + def __init__(self, _app, query, uuid) -> None: + """ + Initializes a Librarian object. - def __init__(self, app, query, uuid) -> None: + Args: + - _app: The Flask application object. + - query: The query to be searched. + - uuid: The unique identifier for the search. + + Attributes: + - cr: The Crossref object for performing the search. + - query: The query to be searched. + - uuid: The unique identifier for the search. + - limit: The maximum number of search results to fetch. + - fetched: The number of search results fetched so far. + - hit: The number of search results that match the refinement criteria. + - total: The total number of search results. + - app: The Flask application object. + - live_results: A list to store live search results. + - final_result: A list to store the final refined search results. + - not_in_db: A list to store search results that are not in the local database. + - search_result_from_cr: A dictionary to store the search results from Crossref. + - done: A flag indicating if the search is done. + """ netrc_mod = netrc.netrc(NETRC_FILE) authTokens = netrc_mod.authenticators("crossref") self.cr = Crossref(mailto=authTokens[0]) @@ -41,7 +77,7 @@ class Librarian(object): self.fetched = 0 self.hit = 0 self.total = 0 - self.app = app + self.app = _app self.live_results = [] self.final_result = [] self.not_in_db = [] @@ -49,25 +85,37 @@ class Librarian(object): self.done = False async def search_crossref(self, query): + """ + Performs a search on Crossref for the given query. + + Args: + - query: The query to be searched. + + Returns: + - result: The search result from Crossref. + + Raises: + - None. + """ self.app.logger.info("STARTED SEARCH") - result = self.cr.works(query = query, limit = 1000) + result = self.cr.works(query=query, limit=1000) self.search_result_from_cr.update(result) self.total = result['message']['total-results'] self.fetched += len(result['message']['items']) - self.app.logger.info (self.total) + self.app.logger.info(self.total) self.app.logger.info(self.fetched) while self.total > self.fetched and self.limit > self.fetched: - result = self.cr.works(query = query, limit = 1000) + result = self.cr.works(query=query, limit=1000) self.search_result_from_cr.update(result) self.total = result['message']['total-results'] self.fetched += len(result['message']['items']) - self.app.logger.info (self.total) + self.app.logger.info(self.total) self.app.logger.info(self.fetched) time.sleep(0.1) self.app.logger.info("CROSSREF DONE") - #TODO: add to sql with UUID - #sql row - title, doi, author, uuid, last retrieved, is_available - + # TODO: add to sql with UUID + # sql row - title, doi, author, uuid, last retrieved, is_available + with open("rr_results.json", "r+", encoding="utf-8") as data_file: # First we load existing data into a dict. try: @@ -81,13 +129,13 @@ class Librarian(object): if "container-title" in item: container = "YES" else: - summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container}) - result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}} + summarized_results.append({"DOI": item['DOI'], "title": item['title'] if 'title' in item else None, "type": item['type'] if 'type' in item else None, "container": container}) + result = {self.uuid: {"total_results": self.search_result_from_cr['message']['total-results'], "on_page": 1, "summary": summarized_results, "results": self.search_result_from_cr['message']['items']}} if file_data: file_data.update(result) else: file_data = result - + self.app.logger.info("REFINE: Dumping to file") data_file.truncate(0) data_file.seek(0) @@ -95,12 +143,36 @@ class Librarian(object): return result async def check_dois_in_local_db(self, doi): + """ + Checks if the given DOI exists in the local database. + + Args: + - doi: The DOI to be checked. + + Returns: + - result: The search result from the backend app. + + Raises: + - None. + """ self.app.logger.info("REFINE: Running search in the backend app") coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger) result = await coro return result async def check_if_exists_brute_force(self, page_url): + """ + Checks if the given page URL exists using brute force method. + + Args: + - page_url: The URL of the page to be checked. + + Returns: + - exists: A boolean indicating if the page exists. + + Raises: + - ValueError: If the URL does not start with 'http:' or 'https:'. + """ self.app.logger.error("REFINE: Brute force search!") if not page_url.startswith(("http:", "https:")): raise ValueError("URL must start with 'http:' or 'https:'") @@ -111,31 +183,57 @@ class Librarian(object): for line in text.splitlines(): if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): return False - if m := re.match(r".*