gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 21:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reformatting of conjurer_librarian

Browse Source
This commit is contained in:
Migatu
2024-04-18 01:29:27 +02:00
parent 70959da16a
commit 499d9c5715
1 changed files with 79 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/conjurer_librarian.py
+79 -37
View File
@@ -10,23 +10,24 @@ Functions:
- BackgroundTaskSearch: Represents a background task for running the Librarian object asynchronously. - BackgroundTaskSearch: Represents a background task for running the Librarian object asynchronously.
""" """
import netrc
import json
import asyncio import asyncio
from json.decoder import JSONDecodeError import json
from urllib.request import urlopen
from queue import Queue
from logging import handlers
import re
import logging import logging
import netrc
import re
import threading import threading
import time import time
from json.decoder import JSONDecodeError
from logging import handlers
from queue import Queue
from urllib.request import urlopen
import requests
import scrape_bot
import search_bot
from flask import Flask, jsonify, request from flask import Flask, jsonify, request
from habanero import Crossref from habanero import Crossref
from waitress import serve from waitress import serve
import search_bot
import scrape_bot
import requests
# Constants # Constants
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
@@ -34,7 +35,7 @@ HOST_ADDRESS = "192.168.1.192"
PORT_ADDRESS = 5001 PORT_ADDRESS = 5001
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer" SEND_RESULTS = "/conjurer"
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147' BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
MAX_CR_RESULTS = 150000 MAX_CR_RESULTS = 150000
ENCODING = "utf-8" ENCODING = "utf-8"
@@ -43,6 +44,7 @@ app = Flask(__name__)
librarian_queue = Queue() librarian_queue = Queue()
librarian_list = [] librarian_list = []
class Librarian(object): class Librarian(object):
def __init__(self, _app, query, uuid) -> None: def __init__(self, _app, query, uuid) -> None:
""" """
@@ -100,15 +102,15 @@ class Librarian(object):
self.app.logger.info("STARTED SEARCH") self.app.logger.info("STARTED SEARCH")
result = self.cr.works(query=query, limit=1000) result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(result) self.search_result_from_cr.update(result)
self.total = result['message']['total-results'] self.total = result["message"]["total-results"]
self.fetched += len(result['message']['items']) self.fetched += len(result["message"]["items"])
self.app.logger.info(self.total) self.app.logger.info(self.total)
self.app.logger.info(self.fetched) self.app.logger.info(self.fetched)
while self.total > self.fetched and self.limit > self.fetched: while self.total > self.fetched and self.limit > self.fetched:
result = self.cr.works(query=query, limit=1000) result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(result) self.search_result_from_cr.update(result)
self.total = result['message']['total-results'] self.total = result["message"]["total-results"]
self.fetched += len(result['message']['items']) self.fetched += len(result["message"]["items"])
self.app.logger.info(self.total) self.app.logger.info(self.total)
self.app.logger.info(self.fetched) self.app.logger.info(self.fetched)
time.sleep(0.1) time.sleep(0.1)
@@ -124,13 +126,29 @@ class Librarian(object):
file_data = {} file_data = {}
summarized_results = [] summarized_results = []
self.app.logger.info("REFINE: Removing all derived works from the list") self.app.logger.info("REFINE: Removing all derived works from the list")
for item in self.search_result_from_cr['message']['items']: for item in self.search_result_from_cr["message"]["items"]:
container = "NO" container = "NO"
if "container-title" in item: if "container-title" in item:
container = "YES" container = "YES"
else: else:
summarized_results.append({"DOI": item['DOI'], "title": item['title'] if 'title' in item else None, "type": item['type'] if 'type' in item else None, "container": container}) summarized_results.append(
result = {self.uuid: {"total_results": self.search_result_from_cr['message']['total-results'], "on_page": 1, "summary": summarized_results, "results": self.search_result_from_cr['message']['items']}} {
"DOI": item["DOI"],
"title": item["title"] if "title" in item else None,
"type": item["type"] if "type" in item else None,
"container": container,
}
)
result = {
self.uuid: {
"total_results": self.search_result_from_cr["message"][
"total-results"
],
"on_page": 1,
"summary": summarized_results,
"results": self.search_result_from_cr["message"]["items"],
}
}
if file_data: if file_data:
file_data.update(result) file_data.update(result)
else: else:
@@ -156,7 +174,9 @@ class Librarian(object):
- None. - None.
""" """
self.app.logger.info("REFINE: Running search in the backend app") self.app.logger.info("REFINE: Running search in the backend app")
coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger) coro = asyncio.to_thread(
search_bot.search_for_doi, doi, self.live_results, self.app.logger
)
result = await coro result = await coro
return result return result
@@ -181,9 +201,14 @@ class Librarian(object):
data = response.read() data = response.read()
text = data.decode("utf-8") text = data.decode("utf-8")
for line in text.splitlines(): for line in text.splitlines():
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): if re.match(
r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
line,
):
return False return False
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line): if m := re.match(
r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
):
direct_download_link = "https:" + str(m.group(1)) direct_download_link = "https:" + str(m.group(1))
self.app.logger.info(direct_download_link) self.app.logger.info(direct_download_link)
return True return True
@@ -208,7 +233,11 @@ class Librarian(object):
if brute_force and not result: if brute_force and not result:
for item in doi: for item in doi:
item_link = "https://sci-hub.se/" + item[0] item_link = "https://sci-hub.se/" + item[0]
tmp = {"DOI": item["DOI"], "exists": await self.check_if_exists_brute_force(item_link), "data": item} tmp = {
"DOI": item["DOI"],
"exists": await self.check_if_exists_brute_force(item_link),
"data": item,
}
result.update(tmp) result.update(tmp)
return result return result
@@ -276,11 +305,17 @@ class Librarian(object):
self.result = await self.refine_query(database) self.result = await self.refine_query(database)
temp = {} temp = {}
for item in self.result: for item in self.result:
temp[item['DOI']] = {"Title": item['title'], "type": item['type']} temp[item["DOI"]] = {
"Title": item["title"],
"type": item["type"],
}
self.result = temp self.result = temp
temp2 = {} temp2 = {}
for item in self.not_in_db: for item in self.not_in_db:
temp2[item['DOI']] = {"Title": item['title'], "type": item['type']} temp2[item["DOI"]] = {
"Title": item["title"],
"type": item["type"],
}
self.not_in_db = temp2 self.not_in_db = temp2
return self.result return self.result
answer = await self.search_crossref(query=query) answer = await self.search_crossref(query=query)
@@ -300,15 +335,17 @@ class Librarian(object):
self.done = True self.done = True
temp = {} temp = {}
for item in self.result: for item in self.result:
temp[item['DOI']] = {"Title": item['title'], "type": item['type']} temp[item["DOI"]] = {"Title": item["title"], "type": item["type"]}
self.result = temp self.result = temp
temp2 = {} temp2 = {}
for item in self.not_in_db: for item in self.not_in_db:
temp2[item['DOI']] = {"Title": item['title'], "type": item['type']} temp2[item["DOI"]] = {"Title": item["title"], "type": item["type"]}
self.not_in_db = temp2 self.not_in_db = temp2
return self.result return self.result
#============================= FLASK INTERNALS===============================
# ============================= FLASK INTERNALS===============================
def flask_debug(): def flask_debug():
""" """
@@ -326,6 +363,7 @@ def flask_debug():
# trunk-ignore(bandit/B201) # trunk-ignore(bandit/B201)
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS) app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
def waitress_run(): def waitress_run():
""" """
Serves the Flask application using the Waitress WSGI server. Serves the Flask application using the Waitress WSGI server.
@@ -341,6 +379,7 @@ def waitress_run():
""" """
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS) serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
class BackgroundTaskSearch(threading.Thread): class BackgroundTaskSearch(threading.Thread):
""" """
A background task for searching and saving results to files. A background task for searching and saving results to files.
@@ -428,7 +467,8 @@ class BackgroundTaskSearch(threading.Thread):
self.app.logger.info("SEND CONFIRMED") self.app.logger.info("SEND CONFIRMED")
time.sleep(1) time.sleep(1)
#==================================SERVER ROUTES==========================================
# ==================================SERVER ROUTES==========================================
@app.route("/query", methods=["POST"]) @app.route("/query", methods=["POST"])
async def query_database(): async def query_database():
""" """
@@ -456,6 +496,7 @@ async def query_database():
) )
return return_data return return_data
@app.route("/get_partial_result", methods=["POST"]) @app.route("/get_partial_result", methods=["POST"])
async def get_partial(): async def get_partial():
""" """
@@ -477,30 +518,31 @@ async def get_partial():
) )
return return_data return return_data
#=======================================MAIN===================================================
# =======================================MAIN===================================================
if __name__ == "__main__": if __name__ == "__main__":
app.logger.setLevel(logging.DEBUG) app.logger.setLevel(logging.DEBUG)
h1 = handlers.RotatingFileHandler( h1 = handlers.RotatingFileHandler(
filename="E:\\logs\\librarian.log", filename="E:\\logs\\librarian.log",
encoding="utf-8", encoding="utf-8",
mode="a", mode="a",
maxBytes=6 * 1024 * 1024, maxBytes=6 * 1024 * 1024,
backupCount=6, backupCount=6,
) )
app.logger.addHandler(h1) app.logger.addHandler(h1)
threads = [] threads = []
threads.append(threading.Thread(target=waitress_run)) threads.append(threading.Thread(target=waitress_run))
#threads.append(threading.Thread(target=flask_debug)) # threads.append(threading.Thread(target=flask_debug))
bgtask = BackgroundTaskSearch() bgtask = BackgroundTaskSearch()
bgtask.app = app bgtask.app = app
threads.append(bgtask) threads.append(bgtask)
threads.append(threading.Thread(target = scrape_bot.scraper, args=(app.logger,))) threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,)))
i = 0 i = 0
for worker in threads: for worker in threads:
try: try:
app.logger.info("App number: %s", i) app.logger.info("App number: %s", i)
i+=1 i += 1
worker.start() worker.start()
except RuntimeError as e: except RuntimeError as e:
app.logger.error("Exploded") app.logger.error("Exploded")