mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
Documentation of conjurer_librarian
This commit is contained in:
@@ -1,22 +1,35 @@
|
||||
"""
|
||||
This module contains the implementation of the Librarian class and related functions for searching and refining queries.
|
||||
|
||||
Classes:
|
||||
- Librarian: Represents a librarian object that performs search and refinement operations on queries.
|
||||
|
||||
Functions:
|
||||
- flask_debug: Starts a Flask application in debug mode without using the reloader.
|
||||
- waitress_run: Serves the Flask application using the Waitress WSGI server.
|
||||
- BackgroundTaskSearch: Represents a background task for running the Librarian object asynchronously.
|
||||
"""
|
||||
|
||||
import netrc
|
||||
import json
|
||||
import asyncio
|
||||
from json.decoder import JSONDecodeError
|
||||
from urllib.request import urlopen
|
||||
from queue import Queue
|
||||
from logging import handlers
|
||||
import re
|
||||
import logging
|
||||
import threading
|
||||
import time
|
||||
from flask import Flask, jsonify, request
|
||||
import netrc
|
||||
import json
|
||||
import asyncio
|
||||
from json.decoder import JSONDecodeError
|
||||
from urllib.request import urlopen
|
||||
from habanero import Crossref
|
||||
from waitress import serve
|
||||
from queue import Queue
|
||||
import search_bot
|
||||
import scrape_bot
|
||||
import requests
|
||||
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
|
||||
|
||||
# Constants
|
||||
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
|
||||
HOST_ADDRESS = "192.168.1.192"
|
||||
PORT_ADDRESS = 5001
|
||||
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
|
||||
@@ -24,14 +37,37 @@ SEND_RESULTS = "/conjurer"
|
||||
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
|
||||
MAX_CR_RESULTS = 150000
|
||||
ENCODING = "utf-8"
|
||||
|
||||
app = Flask(__name__)
|
||||
|
||||
librarian_queue = Queue()
|
||||
librarian_list = []
|
||||
|
||||
class Librarian(object):
|
||||
def __init__(self, _app, query, uuid) -> None:
|
||||
"""
|
||||
Initializes a Librarian object.
|
||||
|
||||
def __init__(self, app, query, uuid) -> None:
|
||||
Args:
|
||||
- _app: The Flask application object.
|
||||
- query: The query to be searched.
|
||||
- uuid: The unique identifier for the search.
|
||||
|
||||
Attributes:
|
||||
- cr: The Crossref object for performing the search.
|
||||
- query: The query to be searched.
|
||||
- uuid: The unique identifier for the search.
|
||||
- limit: The maximum number of search results to fetch.
|
||||
- fetched: The number of search results fetched so far.
|
||||
- hit: The number of search results that match the refinement criteria.
|
||||
- total: The total number of search results.
|
||||
- app: The Flask application object.
|
||||
- live_results: A list to store live search results.
|
||||
- final_result: A list to store the final refined search results.
|
||||
- not_in_db: A list to store search results that are not in the local database.
|
||||
- search_result_from_cr: A dictionary to store the search results from Crossref.
|
||||
- done: A flag indicating if the search is done.
|
||||
"""
|
||||
netrc_mod = netrc.netrc(NETRC_FILE)
|
||||
authTokens = netrc_mod.authenticators("crossref")
|
||||
self.cr = Crossref(mailto=authTokens[0])
|
||||
@@ -41,7 +77,7 @@ class Librarian(object):
|
||||
self.fetched = 0
|
||||
self.hit = 0
|
||||
self.total = 0
|
||||
self.app = app
|
||||
self.app = _app
|
||||
self.live_results = []
|
||||
self.final_result = []
|
||||
self.not_in_db = []
|
||||
@@ -49,24 +85,36 @@ class Librarian(object):
|
||||
self.done = False
|
||||
|
||||
async def search_crossref(self, query):
|
||||
"""
|
||||
Performs a search on Crossref for the given query.
|
||||
|
||||
Args:
|
||||
- query: The query to be searched.
|
||||
|
||||
Returns:
|
||||
- result: The search result from Crossref.
|
||||
|
||||
Raises:
|
||||
- None.
|
||||
"""
|
||||
self.app.logger.info("STARTED SEARCH")
|
||||
result = self.cr.works(query = query, limit = 1000)
|
||||
result = self.cr.works(query=query, limit=1000)
|
||||
self.search_result_from_cr.update(result)
|
||||
self.total = result['message']['total-results']
|
||||
self.fetched += len(result['message']['items'])
|
||||
self.app.logger.info (self.total)
|
||||
self.app.logger.info(self.total)
|
||||
self.app.logger.info(self.fetched)
|
||||
while self.total > self.fetched and self.limit > self.fetched:
|
||||
result = self.cr.works(query = query, limit = 1000)
|
||||
result = self.cr.works(query=query, limit=1000)
|
||||
self.search_result_from_cr.update(result)
|
||||
self.total = result['message']['total-results']
|
||||
self.fetched += len(result['message']['items'])
|
||||
self.app.logger.info (self.total)
|
||||
self.app.logger.info(self.total)
|
||||
self.app.logger.info(self.fetched)
|
||||
time.sleep(0.1)
|
||||
self.app.logger.info("CROSSREF DONE")
|
||||
#TODO: add to sql with UUID
|
||||
#sql row - title, doi, author, uuid, last retrieved, is_available
|
||||
# TODO: add to sql with UUID
|
||||
# sql row - title, doi, author, uuid, last retrieved, is_available
|
||||
|
||||
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
|
||||
# First we load existing data into a dict.
|
||||
@@ -81,8 +129,8 @@ class Librarian(object):
|
||||
if "container-title" in item:
|
||||
container = "YES"
|
||||
else:
|
||||
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
|
||||
result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
|
||||
summarized_results.append({"DOI": item['DOI'], "title": item['title'] if 'title' in item else None, "type": item['type'] if 'type' in item else None, "container": container})
|
||||
result = {self.uuid: {"total_results": self.search_result_from_cr['message']['total-results'], "on_page": 1, "summary": summarized_results, "results": self.search_result_from_cr['message']['items']}}
|
||||
if file_data:
|
||||
file_data.update(result)
|
||||
else:
|
||||
@@ -95,12 +143,36 @@ class Librarian(object):
|
||||
return result
|
||||
|
||||
async def check_dois_in_local_db(self, doi):
|
||||
"""
|
||||
Checks if the given DOI exists in the local database.
|
||||
|
||||
Args:
|
||||
- doi: The DOI to be checked.
|
||||
|
||||
Returns:
|
||||
- result: The search result from the backend app.
|
||||
|
||||
Raises:
|
||||
- None.
|
||||
"""
|
||||
self.app.logger.info("REFINE: Running search in the backend app")
|
||||
coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
|
||||
result = await coro
|
||||
return result
|
||||
|
||||
async def check_if_exists_brute_force(self, page_url):
|
||||
"""
|
||||
Checks if the given page URL exists using brute force method.
|
||||
|
||||
Args:
|
||||
- page_url: The URL of the page to be checked.
|
||||
|
||||
Returns:
|
||||
- exists: A boolean indicating if the page exists.
|
||||
|
||||
Raises:
|
||||
- ValueError: If the URL does not start with 'http:' or 'https:'.
|
||||
"""
|
||||
self.app.logger.error("REFINE: Brute force search!")
|
||||
if not page_url.startswith(("http:", "https:")):
|
||||
raise ValueError("URL must start with 'http:' or 'https:'")
|
||||
@@ -111,30 +183,56 @@ class Librarian(object):
|
||||
for line in text.splitlines():
|
||||
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
|
||||
return False
|
||||
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
|
||||
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line):
|
||||
direct_download_link = "https:" + str(m.group(1))
|
||||
self.app.logger.info (direct_download_link)
|
||||
self.app.logger.info(direct_download_link)
|
||||
return True
|
||||
return False
|
||||
|
||||
async def check_if_exists(self, doi, brute_force = False):
|
||||
async def check_if_exists(self, doi, brute_force=False):
|
||||
"""
|
||||
Checks if the given DOI exists.
|
||||
|
||||
Args:
|
||||
- doi: The DOI to be checked.
|
||||
- brute_force: A flag indicating if brute force method should be used.
|
||||
|
||||
Returns:
|
||||
- result: The search result.
|
||||
|
||||
Raises:
|
||||
- None.
|
||||
"""
|
||||
result = {}
|
||||
result = await self.check_dois_in_local_db(doi)
|
||||
if brute_force and not result:
|
||||
for item in doi:
|
||||
item_link = "https://sci-hub.se/" + item[0]
|
||||
tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
|
||||
tmp = {"DOI": item["DOI"], "exists": await self.check_if_exists_brute_force(item_link), "data": item}
|
||||
result.update(tmp)
|
||||
return result
|
||||
|
||||
async def refine_query(self, unrefined_reqult, brute_force = False):
|
||||
async def refine_query(self, unrefined_result):
|
||||
"""
|
||||
Refines the search query based on the unrefined search result.
|
||||
|
||||
Args:
|
||||
- unrefined_result: The unrefined search result.
|
||||
- brute_force: A flag indicating if brute force method should be used.
|
||||
|
||||
Returns:
|
||||
- final_result: The final refined search result.
|
||||
|
||||
Raises:
|
||||
- None.
|
||||
"""
|
||||
temp = []
|
||||
refined_result = []
|
||||
for key in unrefined_reqult:
|
||||
for key in unrefined_result:
|
||||
self.app.logger.info("KEY:")
|
||||
self.app.logger.info(key)
|
||||
|
||||
for item in unrefined_reqult[self.uuid]["summary"]:
|
||||
for item in unrefined_result[self.uuid]["summary"]:
|
||||
if item["container"] == "NO" and item["title"]:
|
||||
refined_result.append(item)
|
||||
|
||||
@@ -150,83 +248,132 @@ class Librarian(object):
|
||||
self.not_in_db.append(item["data"])
|
||||
self.hit = len(self.final_result)
|
||||
return self.final_result
|
||||
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
|
||||
|
||||
async def answer_query(self):
|
||||
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
|
||||
#Search sql for UUID
|
||||
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
|
||||
"""
|
||||
Answers the search query.
|
||||
|
||||
Args:
|
||||
- None.
|
||||
|
||||
Returns:
|
||||
- result: The search result.
|
||||
|
||||
Raises:
|
||||
- None.
|
||||
"""
|
||||
self.app.logger.info(f"Search started {self.uuid}")
|
||||
database = None
|
||||
query = self.query
|
||||
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
|
||||
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem
|
||||
#ale tymczasowo plik
|
||||
try:
|
||||
database = json.load(cr_file)
|
||||
except JSONDecodeError:
|
||||
pass
|
||||
#TODO: Zwracać i final_result i not_in_db. Not in db_do pliku - i do ręcznego przeglądania potem.
|
||||
if database:
|
||||
for item in database.keys():
|
||||
if item in self.uuid:
|
||||
self.result = await self.refine_query(database)
|
||||
temp = {}
|
||||
for item in self.result:
|
||||
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
|
||||
temp[item['DOI']] = {"Title": item['title'], "type": item['type']}
|
||||
self.result = temp
|
||||
temp2 = {}
|
||||
for item in self.not_in_db:
|
||||
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
|
||||
temp2[item['DOI']] = {"Title": item['title'], "type": item['type']}
|
||||
self.not_in_db = temp2
|
||||
return self.result
|
||||
answer = await self.search_crossref(query=query)
|
||||
database.update(answer)
|
||||
cr_file.truncate(0)
|
||||
cr_file.seek(0)
|
||||
json.dump(database,cr_file)
|
||||
json.dump(database, cr_file)
|
||||
self.result = await self.refine_query(answer)
|
||||
else:
|
||||
answer = await self.search_crossref(query=query)
|
||||
database = answer
|
||||
cr_file.truncate(0)
|
||||
cr_file.seek(0)
|
||||
json.dump(database,cr_file)
|
||||
json.dump(database, cr_file)
|
||||
self.result = await self.refine_query(answer)
|
||||
self.app.logger.info("Refined")
|
||||
self.done = True
|
||||
temp = {}
|
||||
for item in self.result:
|
||||
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
|
||||
temp[item['DOI']] = {"Title": item['title'], "type": item['type']}
|
||||
self.result = temp
|
||||
temp2 = {}
|
||||
for item in self.not_in_db:
|
||||
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
|
||||
temp2[item['DOI']] = {"Title": item['title'], "type": item['type']}
|
||||
self.not_in_db = temp2
|
||||
return self.result
|
||||
|
||||
#============================= FLASK INTERNALS===============================
|
||||
|
||||
def flask_debug():
|
||||
"""
|
||||
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
|
||||
Do not use for production for fucks sake.
|
||||
Starts a Flask application in debug mode without using the reloader.
|
||||
|
||||
Args:
|
||||
- None.
|
||||
|
||||
Returns:
|
||||
- None.
|
||||
|
||||
Raises:
|
||||
- None.
|
||||
"""
|
||||
# trunk-ignore(bandit/B201)
|
||||
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||
|
||||
def waitress_run():
|
||||
"""
|
||||
The `waitress_run` function serves the `app` on host "0.0.0.0"
|
||||
and port 5000 using the Waitress WSGI server.
|
||||
Serves the Flask application using the Waitress WSGI server.
|
||||
|
||||
Args:
|
||||
- None.
|
||||
|
||||
Returns:
|
||||
- None.
|
||||
|
||||
Raises:
|
||||
- None.
|
||||
"""
|
||||
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||
|
||||
class BackgroundTaskSearch(threading.Thread):
|
||||
"""
|
||||
A background task for searching and saving results to files.
|
||||
|
||||
This class extends the `threading.Thread` class and is responsible for running
|
||||
the search task in the background. It retrieves queries from a queue, performs
|
||||
the search, and saves the results to files.
|
||||
|
||||
Attributes:
|
||||
app (App): The application instance.
|
||||
"""
|
||||
|
||||
def run(self):
|
||||
loop = asyncio.new_event_loop() # loop = asyncio.get_event_loop()
|
||||
"""
|
||||
Run the background task.
|
||||
|
||||
This method is called when the thread is started. It creates a new event loop,
|
||||
runs the `_run` method, and closes the event loop.
|
||||
"""
|
||||
loop = asyncio.new_event_loop()
|
||||
loop.run_until_complete(self._run())
|
||||
loop.close()
|
||||
|
||||
async def _run(self):
|
||||
"""
|
||||
Perform the search task.
|
||||
|
||||
This method is an asynchronous coroutine that runs in a loop. It retrieves a
|
||||
librarian from the queue, answers the query, and saves the results to files.
|
||||
It also sends the results to a remote server.
|
||||
|
||||
The search task continues running indefinitely until the thread is stopped.
|
||||
"""
|
||||
while True:
|
||||
database = None
|
||||
ndb_database = None
|
||||
@@ -235,6 +382,8 @@ class BackgroundTaskSearch(threading.Thread):
|
||||
result = await librarian.answer_query()
|
||||
result = {librarian.uuid: result}
|
||||
self.app.logger.info("Saving to file")
|
||||
|
||||
# Save results to "not_in_db.json" file
|
||||
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
|
||||
try:
|
||||
ndb_database = json.load(ndb_file)
|
||||
@@ -248,6 +397,7 @@ class BackgroundTaskSearch(threading.Thread):
|
||||
ndb_file.seek(0)
|
||||
json.dump(ndb_database, ndb_file)
|
||||
|
||||
# Save results to "s_results.json" file
|
||||
with open("s_results.json", "r+", encoding="utf-8") as s_file:
|
||||
try:
|
||||
database = json.load(s_file)
|
||||
@@ -281,6 +431,16 @@ class BackgroundTaskSearch(threading.Thread):
|
||||
#==================================SERVER ROUTES==========================================
|
||||
@app.route("/query", methods=["POST"])
|
||||
async def query_database():
|
||||
"""
|
||||
Endpoint for querying the database.
|
||||
|
||||
This function receives a POST request containing a JSON payload with a query and a UUID.
|
||||
It creates a Librarian object with the query and UUID, and adds it to the librarian_queue and librarian_list.
|
||||
Finally, it returns a JSON response indicating the success of the operation, along with the query, UUID, and the current size of the librarian_queue.
|
||||
|
||||
Returns:
|
||||
tuple: A tuple containing a JSON response and a status code.
|
||||
"""
|
||||
record = json.loads(request.data)
|
||||
app.logger.info(record)
|
||||
app.logger.info(record["query"])
|
||||
@@ -298,6 +458,12 @@ async def query_database():
|
||||
|
||||
@app.route("/get_partial_result", methods=["POST"])
|
||||
async def get_partial():
|
||||
"""
|
||||
Retrieves the partial result for a given UUID.
|
||||
|
||||
Returns:
|
||||
A JSON response containing the partial result.
|
||||
"""
|
||||
record = json.loads(request.data)
|
||||
app.logger.info(record)
|
||||
app.logger.info(record["UUID"])
|
||||
@@ -333,7 +499,7 @@ if __name__ == "__main__":
|
||||
i = 0
|
||||
for worker in threads:
|
||||
try:
|
||||
app.logger.info(f"App number: {i}")
|
||||
app.logger.info("App number: %s", i)
|
||||
i+=1
|
||||
worker.start()
|
||||
except RuntimeError as e:
|
||||
|
||||
Reference in New Issue
Block a user