gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 13:34:40 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reformatting of conjurer_librarian

Browse Source
This commit is contained in:
Migatu
2024-04-18 01:29:27 +02:00
parent 70959da16a
commit 499d9c5715
1 changed files with 79 additions and 37 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/conjurer_librarian.py
+79 -37
View File
@@ -10,23 +10,24 @@ Functions:
- BackgroundTaskSearch: Represents a background task for running the Librarian object asynchronously.
"""
import netrc
import json
import asyncio
from json.decoder import JSONDecodeError
from urllib.request import urlopen
from queue import Queue
from logging import handlers
import re
import json
import logging
import netrc
import re
import threading
import time
from json.decoder import JSONDecodeError
from logging import handlers
from queue import Queue
from urllib.request import urlopen
import requests
import scrape_bot
import search_bot
from flask import Flask, jsonify, request
from habanero import Crossref
from waitress import serve
import search_bot
import scrape_bot
import requests
# Constants
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
@@ -34,7 +35,7 @@ HOST_ADDRESS = "192.168.1.192"
PORT_ADDRESS = 5001
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer"
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
MAX_CR_RESULTS = 150000
ENCODING = "utf-8"
@@ -43,6 +44,7 @@ app = Flask(__name__)
librarian_queue = Queue()
librarian_list = []
class Librarian(object):
def __init__(self, _app, query, uuid) -> None:
"""
@@ -100,15 +102,15 @@ class Librarian(object):
self.app.logger.info("STARTED SEARCH")
result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(result)
self.total = result['message']['total-results']
self.fetched += len(result['message']['items'])
self.total = result["message"]["total-results"]
self.fetched += len(result["message"]["items"])
self.app.logger.info(self.total)
self.app.logger.info(self.fetched)
while self.total > self.fetched and self.limit > self.fetched:
result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(result)
self.total = result['message']['total-results']
self.fetched += len(result['message']['items'])
self.total = result["message"]["total-results"]
self.fetched += len(result["message"]["items"])
self.app.logger.info(self.total)
self.app.logger.info(self.fetched)
time.sleep(0.1)
@@ -124,13 +126,29 @@ class Librarian(object):
file_data = {}
summarized_results = []
self.app.logger.info("REFINE: Removing all derived works from the list")
for item in self.search_result_from_cr['message']['items']:
for item in self.search_result_from_cr["message"]["items"]:
container = "NO"
if "container-title" in item:
container = "YES"
else:
summarized_results.append({"DOI": item['DOI'], "title": item['title'] if 'title' in item else None, "type": item['type'] if 'type' in item else None, "container": container})
result = {self.uuid: {"total_results": self.search_result_from_cr['message']['total-results'], "on_page": 1, "summary": summarized_results, "results": self.search_result_from_cr['message']['items']}}
summarized_results.append(
{
"DOI": item["DOI"],
"title": item["title"] if "title" in item else None,
"type": item["type"] if "type" in item else None,
"container": container,
}
)
result = {
self.uuid: {
"total_results": self.search_result_from_cr["message"][
"total-results"
],
"on_page": 1,
"summary": summarized_results,
"results": self.search_result_from_cr["message"]["items"],
}
}
if file_data:
file_data.update(result)
else:
@@ -156,7 +174,9 @@ class Librarian(object):
- None.
"""
self.app.logger.info("REFINE: Running search in the backend app")
coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
coro = asyncio.to_thread(
search_bot.search_for_doi, doi, self.live_results, self.app.logger
)
result = await coro
return result
@@ -181,9 +201,14 @@ class Librarian(object):
data = response.read()
text = data.decode("utf-8")
for line in text.splitlines():
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
if re.match(
r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
line,
):
return False
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line):
if m := re.match(
r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
):
direct_download_link = "https:" + str(m.group(1))
self.app.logger.info(direct_download_link)
return True
@@ -208,7 +233,11 @@ class Librarian(object):
if brute_force and not result:
for item in doi:
item_link = "https://sci-hub.se/" + item[0]
tmp = {"DOI": item["DOI"], "exists": await self.check_if_exists_brute_force(item_link), "data": item}
tmp = {
"DOI": item["DOI"],
"exists": await self.check_if_exists_brute_force(item_link),
"data": item,
}
result.update(tmp)
return result
@@ -276,11 +305,17 @@ class Librarian(object):
self.result = await self.refine_query(database)
temp = {}
for item in self.result:
temp[item['DOI']] = {"Title": item['title'], "type": item['type']}
temp[item["DOI"]] = {
"Title": item["title"],
"type": item["type"],
}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title": item['title'], "type": item['type']}
temp2[item["DOI"]] = {
"Title": item["title"],
"type": item["type"],
}
self.not_in_db = temp2
return self.result
answer = await self.search_crossref(query=query)
@@ -300,15 +335,17 @@ class Librarian(object):
self.done = True
temp = {}
for item in self.result:
temp[item['DOI']] = {"Title": item['title'], "type": item['type']}
temp[item["DOI"]] = {"Title": item["title"], "type": item["type"]}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title": item['title'], "type": item['type']}
temp2[item["DOI"]] = {"Title": item["title"], "type": item["type"]}
self.not_in_db = temp2
return self.result
#============================= FLASK INTERNALS===============================
# ============================= FLASK INTERNALS===============================
def flask_debug():
"""
@@ -326,6 +363,7 @@ def flask_debug():
# trunk-ignore(bandit/B201)
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
def waitress_run():
"""
Serves the Flask application using the Waitress WSGI server.
@@ -341,6 +379,7 @@ def waitress_run():
"""
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
class BackgroundTaskSearch(threading.Thread):
"""
A background task for searching and saving results to files.
@@ -428,7 +467,8 @@ class BackgroundTaskSearch(threading.Thread):
self.app.logger.info("SEND CONFIRMED")
time.sleep(1)
#==================================SERVER ROUTES==========================================
# ==================================SERVER ROUTES==========================================
@app.route("/query", methods=["POST"])
async def query_database():
"""
@@ -456,6 +496,7 @@ async def query_database():
)
return return_data
@app.route("/get_partial_result", methods=["POST"])
async def get_partial():
"""
@@ -477,30 +518,31 @@ async def get_partial():
)
return return_data
#=======================================MAIN===================================================
# =======================================MAIN===================================================
if __name__ == "__main__":
app.logger.setLevel(logging.DEBUG)
h1 = handlers.RotatingFileHandler(
filename="E:\\logs\\librarian.log",
encoding="utf-8",
mode="a",
maxBytes=6 * 1024 * 1024,
backupCount=6,
filename="E:\\logs\\librarian.log",
encoding="utf-8",
mode="a",
maxBytes=6 * 1024 * 1024,
backupCount=6,
)
app.logger.addHandler(h1)
threads = []
threads.append(threading.Thread(target=waitress_run))
#threads.append(threading.Thread(target=flask_debug))
# threads.append(threading.Thread(target=flask_debug))
bgtask = BackgroundTaskSearch()
bgtask.app = app
threads.append(bgtask)
threads.append(threading.Thread(target = scrape_bot.scraper, args=(app.logger,)))
threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,)))
i = 0
for worker in threads:
try:
app.logger.info("App number: %s", i)
i+=1
i += 1
worker.start()
except RuntimeError as e:
app.logger.error("Exploded")