gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 21:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Documentation of conjurer_librarian

Browse Source
This commit is contained in:
Migatu
2024-04-18 01:28:26 +02:00
parent 8f360417a9
commit 70959da16a
1 changed files with 221 additions and 55 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/conjurer_librarian.py
+221 -55
View File
@@ -1,22 +1,35 @@
"""
This module contains the implementation of the Librarian class and related functions for searching and refining queries.
Classes:
- Librarian: Represents a librarian object that performs search and refinement operations on queries.
Functions:
- flask_debug: Starts a Flask application in debug mode without using the reloader.
- waitress_run: Serves the Flask application using the Waitress WSGI server.
- BackgroundTaskSearch: Represents a background task for running the Librarian object asynchronously.
"""
import netrc
import json
import asyncio
from json.decoder import JSONDecodeError
from urllib.request import urlopen
from queue import Queue
from logging import handlers
import re
import logging
import threading
import time
from flask import Flask, jsonify, request
import netrc
import json
import asyncio
from json.decoder import JSONDecodeError
from urllib.request import urlopen
from habanero import Crossref
from waitress import serve
from queue import Queue
import search_bot
import scrape_bot
import requests
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
# Constants
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
HOST_ADDRESS = "192.168.1.192"
PORT_ADDRESS = 5001
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
@@ -24,14 +37,37 @@ SEND_RESULTS = "/conjurer"
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
MAX_CR_RESULTS = 150000
ENCODING = "utf-8"
app = Flask(__name__)
librarian_queue = Queue()
librarian_list = []
class Librarian(object):
def __init__(self, _app, query, uuid) -> None:
"""
Initializes a Librarian object.
def __init__(self, app, query, uuid) -> None:
Args:
- _app: The Flask application object.
- query: The query to be searched.
- uuid: The unique identifier for the search.
Attributes:
- cr: The Crossref object for performing the search.
- query: The query to be searched.
- uuid: The unique identifier for the search.
- limit: The maximum number of search results to fetch.
- fetched: The number of search results fetched so far.
- hit: The number of search results that match the refinement criteria.
- total: The total number of search results.
- app: The Flask application object.
- live_results: A list to store live search results.
- final_result: A list to store the final refined search results.
- not_in_db: A list to store search results that are not in the local database.
- search_result_from_cr: A dictionary to store the search results from Crossref.
- done: A flag indicating if the search is done.
"""
netrc_mod = netrc.netrc(NETRC_FILE)
authTokens = netrc_mod.authenticators("crossref")
self.cr = Crossref(mailto=authTokens[0])
@@ -41,7 +77,7 @@ class Librarian(object):
self.fetched = 0
self.hit = 0
self.total = 0
self.app = app
self.app = _app
self.live_results = []
self.final_result = []
self.not_in_db = []
@@ -49,25 +85,37 @@ class Librarian(object):
self.done = False
async def search_crossref(self, query):
"""
Performs a search on Crossref for the given query.
Args:
- query: The query to be searched.
Returns:
- result: The search result from Crossref.
Raises:
- None.
"""
self.app.logger.info("STARTED SEARCH")
result = self.cr.works(query = query, limit = 1000)
result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(result)
self.total = result['message']['total-results']
self.fetched += len(result['message']['items'])
self.app.logger.info (self.total)
self.app.logger.info(self.total)
self.app.logger.info(self.fetched)
while self.total > self.fetched and self.limit > self.fetched:
result = self.cr.works(query = query, limit = 1000)
result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(result)
self.total = result['message']['total-results']
self.fetched += len(result['message']['items'])
self.app.logger.info (self.total)
self.app.logger.info(self.total)
self.app.logger.info(self.fetched)
time.sleep(0.1)
self.app.logger.info("CROSSREF DONE")
#TODO: add to sql with UUID
#sql row - title, doi, author, uuid, last retrieved, is_available
# TODO: add to sql with UUID
# sql row - title, doi, author, uuid, last retrieved, is_available
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
# First we load existing data into a dict.
try:
@@ -81,13 +129,13 @@ class Librarian(object):
if "container-title" in item:
container = "YES"
else:
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
summarized_results.append({"DOI": item['DOI'], "title": item['title'] if 'title' in item else None, "type": item['type'] if 'type' in item else None, "container": container})
result = {self.uuid: {"total_results": self.search_result_from_cr['message']['total-results'], "on_page": 1, "summary": summarized_results, "results": self.search_result_from_cr['message']['items']}}
if file_data:
file_data.update(result)
else:
file_data = result
self.app.logger.info("REFINE: Dumping to file")
data_file.truncate(0)
data_file.seek(0)
@@ -95,12 +143,36 @@ class Librarian(object):
return result
async def check_dois_in_local_db(self, doi):
"""
Checks if the given DOI exists in the local database.
Args:
- doi: The DOI to be checked.
Returns:
- result: The search result from the backend app.
Raises:
- None.
"""
self.app.logger.info("REFINE: Running search in the backend app")
coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
result = await coro
return result
async def check_if_exists_brute_force(self, page_url):
"""
Checks if the given page URL exists using brute force method.
Args:
- page_url: The URL of the page to be checked.
Returns:
- exists: A boolean indicating if the page exists.
Raises:
- ValueError: If the URL does not start with 'http:' or 'https:'.
"""
self.app.logger.error("REFINE: Brute force search!")
if not page_url.startswith(("http:", "https:")):
raise ValueError("URL must start with 'http:' or 'https:'")
@@ -111,31 +183,57 @@ class Librarian(object):
for line in text.splitlines():
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
return False
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line):
direct_download_link = "https:" + str(m.group(1))
self.app.logger.info (direct_download_link)
self.app.logger.info(direct_download_link)
return True
return False
async def check_if_exists(self, doi, brute_force = False):
async def check_if_exists(self, doi, brute_force=False):
"""
Checks if the given DOI exists.
Args:
- doi: The DOI to be checked.
- brute_force: A flag indicating if brute force method should be used.
Returns:
- result: The search result.
Raises:
- None.
"""
result = {}
result = await self.check_dois_in_local_db(doi)
if brute_force and not result:
for item in doi:
item_link = "https://sci-hub.se/" + item[0]
tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
tmp = {"DOI": item["DOI"], "exists": await self.check_if_exists_brute_force(item_link), "data": item}
result.update(tmp)
return result
async def refine_query(self, unrefined_reqult, brute_force = False):
async def refine_query(self, unrefined_result):
"""
Refines the search query based on the unrefined search result.
Args:
- unrefined_result: The unrefined search result.
- brute_force: A flag indicating if brute force method should be used.
Returns:
- final_result: The final refined search result.
Raises:
- None.
"""
temp = []
refined_result = []
for key in unrefined_reqult:
for key in unrefined_result:
self.app.logger.info("KEY:")
self.app.logger.info(key)
for item in unrefined_reqult[self.uuid]["summary"]:
if item["container"] == "NO" and item["title"]:
for item in unrefined_result[self.uuid]["summary"]:
if item["container"] == "NO" and item["title"]:
refined_result.append(item)
for item in refined_result:
@@ -150,83 +248,132 @@ class Librarian(object):
self.not_in_db.append(item["data"])
self.hit = len(self.final_result)
return self.final_result
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
async def answer_query(self):
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
#Search sql for UUID
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
"""
Answers the search query.
Args:
- None.
Returns:
- result: The search result.
Raises:
- None.
"""
self.app.logger.info(f"Search started {self.uuid}")
database = None
query = self.query
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem
#ale tymczasowo plik
try:
database = json.load(cr_file)
database = json.load(cr_file)
except JSONDecodeError:
pass
#TODO: Zwracać i final_result i not_in_db. Not in db_do pliku - i do ręcznego przeglądania potem.
if database:
for item in database.keys():
if item in self.uuid:
self.result = await self.refine_query(database)
temp = {}
for item in self.result:
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
temp[item['DOI']] = {"Title": item['title'], "type": item['type']}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
temp2[item['DOI']] = {"Title": item['title'], "type": item['type']}
self.not_in_db = temp2
return self.result
answer = await self.search_crossref(query=query)
database.update(answer)
cr_file.truncate(0)
cr_file.seek(0)
json.dump(database,cr_file)
json.dump(database, cr_file)
self.result = await self.refine_query(answer)
else:
answer = await self.search_crossref(query=query)
database = answer
cr_file.truncate(0)
cr_file.seek(0)
json.dump(database,cr_file)
json.dump(database, cr_file)
self.result = await self.refine_query(answer)
self.app.logger.info("Refined")
self.done = True
temp = {}
for item in self.result:
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
temp[item['DOI']] = {"Title": item['title'], "type": item['type']}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
temp2[item['DOI']] = {"Title": item['title'], "type": item['type']}
self.not_in_db = temp2
return self.result
#============================= FLASK INTERNALS===============================
def flask_debug():
"""
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
Do not use for production for fucks sake.
Starts a Flask application in debug mode without using the reloader.
Args:
- None.
Returns:
- None.
Raises:
- None.
"""
# trunk-ignore(bandit/B201)
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
def waitress_run():
"""
The `waitress_run` function serves the `app` on host "0.0.0.0"
and port 5000 using the Waitress WSGI server.
Serves the Flask application using the Waitress WSGI server.
Args:
- None.
Returns:
- None.
Raises:
- None.
"""
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
class BackgroundTaskSearch(threading.Thread):
"""
A background task for searching and saving results to files.
This class extends the `threading.Thread` class and is responsible for running
the search task in the background. It retrieves queries from a queue, performs
the search, and saves the results to files.
Attributes:
app (App): The application instance.
"""
def run(self):
loop = asyncio.new_event_loop() # loop = asyncio.get_event_loop()
"""
Run the background task.
This method is called when the thread is started. It creates a new event loop,
runs the `_run` method, and closes the event loop.
"""
loop = asyncio.new_event_loop()
loop.run_until_complete(self._run())
loop.close()
async def _run(self):
"""
Perform the search task.
This method is an asynchronous coroutine that runs in a loop. It retrieves a
librarian from the queue, answers the query, and saves the results to files.
It also sends the results to a remote server.
The search task continues running indefinitely until the thread is stopped.
"""
while True:
database = None
ndb_database = None
@@ -235,9 +382,11 @@ class BackgroundTaskSearch(threading.Thread):
result = await librarian.answer_query()
result = {librarian.uuid: result}
self.app.logger.info("Saving to file")
# Save results to "not_in_db.json" file
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
try:
ndb_database = json.load(ndb_file)
ndb_database = json.load(ndb_file)
except JSONDecodeError:
pass
if ndb_database:
@@ -248,9 +397,10 @@ class BackgroundTaskSearch(threading.Thread):
ndb_file.seek(0)
json.dump(ndb_database, ndb_file)
# Save results to "s_results.json" file
with open("s_results.json", "r+", encoding="utf-8") as s_file:
try:
database = json.load(s_file)
database = json.load(s_file)
except JSONDecodeError:
pass
if database:
@@ -267,20 +417,30 @@ class BackgroundTaskSearch(threading.Thread):
self.app.logger.info(result)
coroutine = asyncio.to_thread(
requests.post,
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
json=result,
timeout=360,
)
requests.post,
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
json=result,
timeout=360,
)
self.app.logger.info("SENT")
result = await coroutine
self.app.logger.info(result.status_code)
self.app.logger.info("SEND CONFIRMED")
time.sleep(1)
#==================================SERVER ROUTES==========================================
@app.route("/query", methods=["POST"])
async def query_database():
"""
Endpoint for querying the database.
This function receives a POST request containing a JSON payload with a query and a UUID.
It creates a Librarian object with the query and UUID, and adds it to the librarian_queue and librarian_list.
Finally, it returns a JSON response indicating the success of the operation, along with the query, UUID, and the current size of the librarian_queue.
Returns:
tuple: A tuple containing a JSON response and a status code.
"""
record = json.loads(request.data)
app.logger.info(record)
app.logger.info(record["query"])
@@ -298,6 +458,12 @@ async def query_database():
@app.route("/get_partial_result", methods=["POST"])
async def get_partial():
"""
Retrieves the partial result for a given UUID.
Returns:
A JSON response containing the partial result.
"""
record = json.loads(request.data)
app.logger.info(record)
app.logger.info(record["UUID"])
@@ -333,7 +499,7 @@ if __name__ == "__main__":
i = 0
for worker in threads:
try:
app.logger.info(f"App number: {i}")
app.logger.info("App number: %s", i)
i+=1
worker.start()
except RuntimeError as e: