mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
Serverside work of search
This commit is contained in:
@@ -5,72 +5,103 @@ import time
|
||||
from flask import Flask, jsonify, request
|
||||
import netrc
|
||||
import json
|
||||
import asyncio
|
||||
from json.decoder import JSONDecodeError
|
||||
from platform import uname
|
||||
from sys import platform
|
||||
from urllib.request import urlopen
|
||||
from habanero import Crossref
|
||||
from waitress import serve
|
||||
from queue import Queue
|
||||
import search_bot
|
||||
import requests
|
||||
from typing import Callable
|
||||
from uuid import UUID
|
||||
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
|
||||
|
||||
if platform in ("linux", "linux2"):
|
||||
SEPARATOR_FILE_PATH = "/"
|
||||
if "microsoft-standard" in uname().release:
|
||||
NETRC_FILE = "/home/mtuszowski/.netrc"
|
||||
else:
|
||||
NETRC_FILE = "/home/pi/.netrc"
|
||||
elif platform == "win32":
|
||||
NETRC_FILE = "C:\\Users\\mtusz\\.netrc"
|
||||
|
||||
#HOST_ADDRESS = "192.168.1.191"
|
||||
HOST_ADDRESS = "0.0.0.0"
|
||||
HOST_ADDRESS = "192.168.1.192"
|
||||
PORT_ADDRESS = 5001
|
||||
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
|
||||
SEND_RESULTS = "/conjurer"
|
||||
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
|
||||
|
||||
ENCODING = "utf-8"
|
||||
app = Flask(__name__)
|
||||
|
||||
librarian_queue = Queue()
|
||||
librarian_list = []
|
||||
|
||||
class Librarian(object):
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, app, query, uuid) -> None:
|
||||
netrc_mod = netrc.netrc(NETRC_FILE)
|
||||
authTokens = netrc_mod.authenticators("crossref")
|
||||
self.cr = Crossref(mailto=authTokens[0])
|
||||
self.limit = 1000
|
||||
self.query = query
|
||||
self.uuid = str(uuid)
|
||||
self.limit = 3000
|
||||
self.fetched = 0
|
||||
self.hit = 0
|
||||
self.total = 0
|
||||
self.app = app
|
||||
self.logger = app.logger
|
||||
self.live_results = []
|
||||
self.search_result_from_cr = {}
|
||||
self.done = False
|
||||
|
||||
def search_crossref(self, query, uuid):
|
||||
search_result_from_cr = self.cr.works(query = query, limit = self.limit)
|
||||
self.total = search_result_from_cr['message']['total-results']
|
||||
self.fetched = len(search_result_from_cr['message']['items'])
|
||||
#add to sql with UUID
|
||||
async def search_crossref(self, query):
|
||||
self.app.logger.info("STARTED SEARCH")
|
||||
result = self.cr.works(query = query, limit = 1000)
|
||||
self.search_result_from_cr.update(result)
|
||||
self.total = result['message']['total-results']
|
||||
self.fetched += len(result['message']['items'])
|
||||
self.logger.info (self.total)
|
||||
self.logger.info(self.fetched)
|
||||
while self.total > self.fetched and self.limit > self.fetched:
|
||||
result = self.cr.works(query = query, limit = 1000)
|
||||
self.search_result_from_cr.update(result)
|
||||
self.total = result['message']['total-results']
|
||||
self.fetched += len(result['message']['items'])
|
||||
self.logger.info (self.total)
|
||||
self.logger.info(self.fetched)
|
||||
|
||||
time.sleep(1)
|
||||
self.app.logger.info("CROSSREF DONE")
|
||||
#TODO: add to sql with UUID
|
||||
#sql row - title, doi, author, uuid, last retrieved, is_available
|
||||
with open("results.json", "r+", encoding="UTF-8") as data_file:
|
||||
|
||||
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
|
||||
# First we load existing data into a dict.
|
||||
try:
|
||||
file_data = json.load(data_file)
|
||||
except JSONDecodeError:
|
||||
file_data = []
|
||||
# Join new_data with file_data inside emp_details
|
||||
file_data = {}
|
||||
summarized_results = []
|
||||
for item in search_result_from_cr['message']['items']:
|
||||
self.app.logger.info("REFINE: Removing all derived works from the list")
|
||||
for item in self.search_result_from_cr['message']['items']:
|
||||
container = "NO"
|
||||
if "container-title" in item:
|
||||
container = "YES"
|
||||
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
|
||||
result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'], }
|
||||
file_data.append(result)
|
||||
print("Saving")
|
||||
else:
|
||||
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
|
||||
result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
|
||||
if file_data:
|
||||
file_data.update(result)
|
||||
else:
|
||||
file_data = result
|
||||
|
||||
self.app.logger.info("REFINE: Dumping to file")
|
||||
data_file.truncate(0)
|
||||
data_file.seek(0)
|
||||
# convert back to json.
|
||||
json.dump(file_data, data_file, indent=4)
|
||||
return result
|
||||
|
||||
async def check_dois_in_local_db(self, doi):
|
||||
self.app.logger.info("REFINE: Running search in the backend app")
|
||||
coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
|
||||
result = await coro
|
||||
return result
|
||||
|
||||
def check_if_exists(self,page_url):
|
||||
|
||||
|
||||
|
||||
return None
|
||||
|
||||
in_scihub_db = True
|
||||
async def check_if_exists_brute_force(self, page_url):
|
||||
self.app.logger.error("REFINE: Brute force search!")
|
||||
if not page_url.startswith(("http:", "https:")):
|
||||
raise ValueError("URL must start with 'http:' or 'https:'")
|
||||
# trunk-ignore(bandit/B310)
|
||||
@@ -79,70 +110,86 @@ class Librarian(object):
|
||||
text = data.decode("utf-8")
|
||||
for line in text.splitlines():
|
||||
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
|
||||
return None
|
||||
if in_scihub_db:
|
||||
return text
|
||||
return None
|
||||
|
||||
def download(self, page_url):
|
||||
if response := self.check_if_exists(page_url):
|
||||
for line in response.splitlines():
|
||||
return False
|
||||
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
|
||||
return "https:" + str(m.group(1))
|
||||
return None
|
||||
direct_download_link = "https:" + str(m.group(1))
|
||||
self.app.logger.info (direct_download_link)
|
||||
return True
|
||||
return False
|
||||
|
||||
def provide_sci_hub_link(self, page_url):
|
||||
if self.check_if_exists(page_url):
|
||||
return page_url
|
||||
return None
|
||||
async def check_if_exists(self, doi, brute_force = False):
|
||||
result = {}
|
||||
result = await self.check_dois_in_local_db(doi)
|
||||
if brute_force and not result:
|
||||
for item in doi:
|
||||
item_link = "https://sci-hub.se/" + item[0]
|
||||
tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
|
||||
result.update(tmp)
|
||||
return result
|
||||
|
||||
def refine_query(self, unrefined_reqult):
|
||||
print("Refine result")
|
||||
async def refine_query(self, unrefined_reqult, brute_force = False):
|
||||
temp = []
|
||||
refined_result = []
|
||||
for item in unrefined_reqult["summary"]:
|
||||
if item["container"] == "NO" and item["title"]:
|
||||
item_link = "https://sci-hub.se/" + str(item['DOI'])
|
||||
if self.check_if_exists(item_link):
|
||||
refined_result.append(item)
|
||||
print ("HIT")
|
||||
print(item["title"])
|
||||
time.sleep(0.01)
|
||||
print("Existing hits")
|
||||
for key in unrefined_reqult:
|
||||
self.logger.info("KEY:")
|
||||
self.logger.info(key)
|
||||
|
||||
for item in unrefined_reqult[self.uuid]["summary"]:
|
||||
if item["container"] == "NO" and item["title"]:
|
||||
refined_result.append(item)
|
||||
|
||||
for item in refined_result:
|
||||
temp.append([item["DOI"], item])
|
||||
|
||||
refined_result = await self.check_if_exists(temp)
|
||||
temp = []
|
||||
for item in refined_result:
|
||||
if item["exists"]:
|
||||
temp.append(item["data"])
|
||||
refined_result = temp
|
||||
self.hit = len(refined_result)
|
||||
return refined_result
|
||||
|
||||
def answer_query(self, query, uuid):
|
||||
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
|
||||
async def answer_query(self):
|
||||
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
|
||||
#Search sql for UUID
|
||||
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
|
||||
print("Answer result")
|
||||
self.logger.info(f"Search started {self.uuid}")
|
||||
uuid_found = False
|
||||
database = None
|
||||
with open("results.json", "r", encoding="UTF-8") as new_file:
|
||||
query = self.query
|
||||
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
|
||||
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem
|
||||
#ale tymczasowo plik
|
||||
try:
|
||||
database = json.load(new_file)
|
||||
database = json.load(cr_file)
|
||||
except JSONDecodeError:
|
||||
pass
|
||||
if database:
|
||||
for item in database:
|
||||
if item["UUID"] == uuid:
|
||||
print (item["UUID"])
|
||||
uuid_found = True
|
||||
return self.refine_query(item)
|
||||
|
||||
if not uuid_found:
|
||||
answer = self.search_crossref(query=query, uuid=uuid)
|
||||
return self.refine_query(answer)
|
||||
if database:
|
||||
for item in database.keys():
|
||||
if item in self.uuid:
|
||||
self.result = await self.refine_query(database)
|
||||
return self.result
|
||||
answer = await self.search_crossref(query=query)
|
||||
database.update(answer)
|
||||
cr_file.truncate(0)
|
||||
cr_file.seek(0)
|
||||
json.dump(database,cr_file)
|
||||
self.result = await self.refine_query(answer)
|
||||
return self.result
|
||||
else:
|
||||
answer = await self.search_crossref(query=query)
|
||||
database = answer
|
||||
cr_file.truncate(0)
|
||||
cr_file.seek(0)
|
||||
json.dump(database,cr_file)
|
||||
self.result = await self.refine_query(answer)
|
||||
self.app.logger.info("Refined")
|
||||
self.done = True
|
||||
return self.result
|
||||
|
||||
#example not exists
|
||||
#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
|
||||
#sample usage to download
|
||||
#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
|
||||
#print(cl.download(sample_existing_page_url))
|
||||
#cl.search_crossref("BDSM")
|
||||
|
||||
app = Flask(__name__)
|
||||
#============================= FLASK INTERNALS===============================
|
||||
def flask_debug():
|
||||
"""
|
||||
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
|
||||
@@ -158,35 +205,101 @@ def waitress_run():
|
||||
"""
|
||||
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||
|
||||
class BackgroundTaskSearch(threading.Thread):
|
||||
def run(self):
|
||||
loop = asyncio.new_event_loop() # loop = asyncio.get_event_loop()
|
||||
loop.run_until_complete(self._run())
|
||||
loop.close()
|
||||
|
||||
async def _run(self):
|
||||
while True:
|
||||
database = None
|
||||
librarian = librarian_queue.get()
|
||||
self.app.logger.info("STARTED")
|
||||
result = await librarian.answer_query()
|
||||
result = {librarian.uuid: result}
|
||||
self.app.logger.info("Saving to file")
|
||||
with open("s_results.json", "r+", encoding="utf-8") as s_file:
|
||||
try:
|
||||
database = json.load(s_file)
|
||||
except JSONDecodeError:
|
||||
pass
|
||||
if database:
|
||||
database.extend(result)
|
||||
else:
|
||||
database = result
|
||||
self.app.logger.info("DUMPING DATA")
|
||||
s_file.truncate(0)
|
||||
s_file.seek(0)
|
||||
json.dump(database, s_file)
|
||||
self.app.logger.info("FINISHED")
|
||||
|
||||
self.app.logger.info(result)
|
||||
coroutine = asyncio.to_thread(
|
||||
requests.post,
|
||||
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
|
||||
json=result,
|
||||
timeout=360,
|
||||
)
|
||||
self.app.logger.info("SENT")
|
||||
_ = await coroutine
|
||||
self.app.logger.info("SEND CONFIRMED")
|
||||
time.sleep(1)
|
||||
|
||||
#==================================SERVER ROUTES==========================================
|
||||
@app.route("/query", methods=["POST"])
|
||||
def query_database():
|
||||
async def query_database():
|
||||
record = json.loads(request.data)
|
||||
app.logger.info(record)
|
||||
app.logger.info(record["query"])
|
||||
app.logger.info(record["UUID"])
|
||||
cl = Librarian()
|
||||
answer_data = cl.answer_query(record["query"], record["UUID"])
|
||||
app.logger.info(answer_data)
|
||||
summary = (cl.hit,cl.fetched, cl.total, record["UUID"])
|
||||
answer_data = (summary,answer_data)
|
||||
uuid = record["UUID"]
|
||||
#TODO: TESTING ONLY======================
|
||||
uuid = UUID('{12345678-1234-5678-1234-567812345678}')
|
||||
|
||||
cl = Librarian(app, record["query"], uuid)
|
||||
librarian_queue.put(cl)
|
||||
librarian_list.append(cl)
|
||||
answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
|
||||
return_data = (
|
||||
jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
|
||||
200,
|
||||
)
|
||||
return return_data
|
||||
|
||||
@app.route("/get_partial_result", methods=["POST"])
|
||||
async def get_partial():
|
||||
record = json.loads(request.data)
|
||||
app.logger.info(record)
|
||||
app.logger.info(record["UUID"])
|
||||
for lib in librarian_list:
|
||||
if lib.uuid == record["UUID"]:
|
||||
answer_data = lib.live_results
|
||||
break
|
||||
return_data = (
|
||||
jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
|
||||
200,
|
||||
)
|
||||
return return_data
|
||||
|
||||
#=======================================MAIN===================================================
|
||||
if __name__ == "__main__":
|
||||
logger = logging.getLogger('conjurer_musician')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
threads = []
|
||||
#threads.append(threading.Thread(target=waitress_run))
|
||||
threads.append(threading.Thread(target=flask_debug))
|
||||
|
||||
bgtask = BackgroundTaskSearch()
|
||||
bgtask.app = app
|
||||
threads.append(bgtask)
|
||||
i = 0
|
||||
for worker in threads:
|
||||
worker.start()
|
||||
try:
|
||||
logger.info(f"App number: {i}")
|
||||
i+=1
|
||||
worker.start()
|
||||
except RuntimeError as e:
|
||||
logger.error("Exploded")
|
||||
print(str(e))
|
||||
for worker in threads:
|
||||
worker.join()
|
||||
|
||||
#1. robi liste wyników i ją wyświetla - ale tylko tytuły niezależnych dzieł - jeśli jest coś w containerze to trza pogrupować
|
||||
#2. Tworzy przyciski pozwalające sprawdzić czy jest dostępne - i jeśli jest zwraca link
|
||||
#3. Jest też przycisk "more" - wyszukujemy po 100, wyświetlamy po 20. Po 100 jak jeszcze to dopiero odpytujemy crossref przyciski strzałek przesuwają po obecnej porcji danych
|
||||
|
||||
Reference in New Issue
Block a user