mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
Merge branch 'main' of https://github.com/migatu/conjurer
This commit is contained in:
@@ -7,12 +7,10 @@ import netrc
|
||||
import json
|
||||
import asyncio
|
||||
from json.decoder import JSONDecodeError
|
||||
from platform import uname
|
||||
from sys import platform
|
||||
from urllib.request import urlopen
|
||||
from habanero import Crossref
|
||||
from waitress import serve
|
||||
|
||||
from queue import Queue
|
||||
import search_bot
|
||||
|
||||
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
|
||||
@@ -21,33 +19,41 @@ HOST_ADDRESS = "192.168.1.192"
|
||||
PORT_ADDRESS = 5001
|
||||
DATABASE_PATH = r'C:\\Database\\chunks'
|
||||
ENCODING = "utf-8"
|
||||
app = Flask(__name__)
|
||||
|
||||
librarian_queue = Queue()
|
||||
|
||||
class Librarian(object):
|
||||
|
||||
def __init__(self) -> None:
|
||||
def __init__(self, app, query, uuid) -> None:
|
||||
netrc_mod = netrc.netrc(NETRC_FILE)
|
||||
authTokens = netrc_mod.authenticators("crossref")
|
||||
self.cr = Crossref(mailto=authTokens[0])
|
||||
self.query = query
|
||||
self.uuid = uuid
|
||||
self.limit = 1000
|
||||
self.fetched = 0
|
||||
self.hit = 0
|
||||
self.total = 0
|
||||
self.app = app
|
||||
|
||||
async def search_crossref(self, query, uuid):
|
||||
self.app.logger.info("STARTED SEARCH")
|
||||
search_result_from_cr = self.cr.works(query = query, limit = self.limit)
|
||||
self.app.logger.info("CROSSREF DONE")
|
||||
self.total = search_result_from_cr['message']['total-results']
|
||||
self.fetched = len(search_result_from_cr['message']['items'])
|
||||
#add to sql with UUID
|
||||
#sql row - title, doi, author, uuid, last retrieved, is_available
|
||||
|
||||
with open("results.json", "r+", encoding="UTF-8") as data_file:
|
||||
# First we load existing data into a dict.
|
||||
try:
|
||||
file_data = json.load(data_file)
|
||||
except JSONDecodeError:
|
||||
file_data = []
|
||||
# Join new_data with file_data inside emp_details
|
||||
summarized_results = []
|
||||
self.app.logger.info("REFINE: Removing all derived works from the list")
|
||||
for item in search_result_from_cr['message']['items']:
|
||||
container = "NO"
|
||||
if "container-title" in item:
|
||||
@@ -55,9 +61,8 @@ class Librarian(object):
|
||||
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
|
||||
result = {"UUID": uuid, "total_results": search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":search_result_from_cr['message']['items'], }
|
||||
file_data.append(result)
|
||||
print("Saving")
|
||||
data_file.seek(0)
|
||||
# convert back to json.
|
||||
self.app.logger.info("REFINE: Dumping to file")
|
||||
json.dump(file_data, data_file, indent=4)
|
||||
return result
|
||||
#TODO: check if exists odpala wątki dla kolejnych plików, z każdego wątku dostaje zwrotki - trafienia na bieżąco oraz sygnał że plik się skończył i pora na następny
|
||||
@@ -70,13 +75,13 @@ class Librarian(object):
|
||||
|
||||
|
||||
async def check_dois_in_local_db(self, doi):
|
||||
#odwolanie do search_bota
|
||||
self.app.logger.info("REFINE: Running search in the backend app")
|
||||
coro = asyncio.to_thread(search_bot.search_for_doi, doi)
|
||||
result = await coro
|
||||
return result
|
||||
|
||||
|
||||
async def check_if_exists_brute_force(self, page_url):
|
||||
self.app.logger.error("REFINE: Brute force search!")
|
||||
if not page_url.startswith(("http:", "https:")):
|
||||
raise ValueError("URL must start with 'http:' or 'https:'")
|
||||
# trunk-ignore(bandit/B310)
|
||||
@@ -92,39 +97,41 @@ class Librarian(object):
|
||||
return True
|
||||
return False
|
||||
|
||||
def provide_sci_hub_link(self, page_url):
|
||||
if self.check_if_exists(page_url):
|
||||
return page_url
|
||||
return None
|
||||
|
||||
async def check_if_exists(self, doi, brute_force = False):
|
||||
result = []
|
||||
result = await self.check_dois_in_local_db(doi)
|
||||
if brute_force and not result:
|
||||
item_link = "https://sci-hub.se/" + doi
|
||||
result = await self.check_if_exists_brute_force(item_link)
|
||||
for item in doi:
|
||||
item_link = "https://sci-hub.se/" + item[0]
|
||||
result.append(await self.check_if_exists_brute_force(item_link))
|
||||
return result
|
||||
|
||||
async def refine_query(self, unrefined_reqult, brute_force = False):
|
||||
print("Refine result")
|
||||
refined_result = []
|
||||
temp = []
|
||||
for item in unrefined_reqult["summary"]:
|
||||
if item["container"] == "NO" and item["title"]:
|
||||
result = await self.check_if_exists(str(item["DOI"]))
|
||||
if result:
|
||||
refined_result.append(item)
|
||||
print ("HIT")
|
||||
print(item["title"])
|
||||
time.sleep(0.01)
|
||||
print("Existing hits")
|
||||
refined_result.append[item]
|
||||
for item in refined_result:
|
||||
temp.append[(item["DOI"], item)]
|
||||
|
||||
refined_result = self.check_if_exists(temp)
|
||||
temp = []
|
||||
for item in refined_result:
|
||||
if item["exists"]:
|
||||
temp.append(item["data"])
|
||||
refined_result = temp
|
||||
self.hit = len(refined_result)
|
||||
return refined_result
|
||||
|
||||
async def answer_query(self, query, uuid):
|
||||
async def answer_query(self):
|
||||
#Search sql for UUID
|
||||
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
|
||||
print("Answer result")
|
||||
uuid_found = False
|
||||
database = None
|
||||
query = self.query
|
||||
uuid = self.uuid
|
||||
with open("results.json", "r", encoding="UTF-8") as new_file:
|
||||
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem
|
||||
#ale tymczasowo plik
|
||||
@@ -146,14 +153,7 @@ class Librarian(object):
|
||||
print("Refined")
|
||||
return await self.refine_query(answer)
|
||||
|
||||
#example not exists
|
||||
#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
|
||||
#sample usage to download
|
||||
#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
|
||||
#print(cl.download(sample_existing_page_url))
|
||||
#cl.search_crossref("BDSM")
|
||||
|
||||
app = Flask(__name__)
|
||||
#============================= FLASK INTERNALS===============================
|
||||
def flask_debug():
|
||||
"""
|
||||
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
|
||||
@@ -169,24 +169,26 @@ def waitress_run():
|
||||
"""
|
||||
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
|
||||
|
||||
# TUTAJ TYLKO LOGUJEMY ŻE PRZYSZŁO, TWORZYMY LIBRARIAN class dla danego zapytania (po UUID)
|
||||
# ODPALAMY SZUKANIE I POZWALAMYSIĘ JUŻ DALEJ KRĘCIĆ
|
||||
#==================================SERVER ROUTES==========================================
|
||||
@app.route("/query", methods=["POST"])
|
||||
async def query_database():
|
||||
record = json.loads(request.data)
|
||||
app.logger.info(record)
|
||||
app.logger.info(record["query"])
|
||||
app.logger.info(record["UUID"])
|
||||
cl = Librarian()
|
||||
answer_data = await cl.answer_query(record["query"], record["UUID"])
|
||||
cl = Librarian(app, record["query"], record["UUID"])
|
||||
librarian_queue.put(cl)
|
||||
|
||||
app.logger.info(answer_data)
|
||||
summary = (cl.hit,cl.fetched, cl.total, record["UUID"])
|
||||
answer_data = (summary,answer_data)
|
||||
|
||||
answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
|
||||
return_data = (
|
||||
jsonify(isError=False, message="Success", statusCode=200, data=answer_data),
|
||||
200,
|
||||
)
|
||||
return return_data
|
||||
|
||||
#=======================================MAIN===================================================
|
||||
if __name__ == "__main__":
|
||||
logger = logging.getLogger('conjurer_musician')
|
||||
logger.setLevel(logging.DEBUG)
|
||||
@@ -198,3 +200,13 @@ if __name__ == "__main__":
|
||||
worker.start()
|
||||
for worker in threads:
|
||||
worker.join()
|
||||
|
||||
|
||||
|
||||
|
||||
#example not exists
|
||||
#sample_non_existing_page_url = "https://sci-hub.se/10.4324/9781003006237"
|
||||
#sample usage to download
|
||||
#sample_existing_page_url = "https://sci-hub.se/10.1057/9781137435026"
|
||||
#print(cl.download(sample_existing_page_url))
|
||||
#cl.search_crossref("BDSM")
|
||||
|
||||
@@ -2,13 +2,13 @@ from queue import Queue, Empty
|
||||
from threading import Thread
|
||||
q = Queue()
|
||||
|
||||
MAXTHREADS = 39
|
||||
#MAXTHREADS = 4
|
||||
#MAXTHREADS = 39
|
||||
MAXTHREADS = 4
|
||||
DATABASE_PATH = r'C:\\Database\\chunks_1\\'
|
||||
ENCODING = "utf-8"
|
||||
CHUNk = "_chunk.txt"
|
||||
_sentinel = object()
|
||||
|
||||
result_list = []
|
||||
#TODO: Doisy wchodzą jako lista krotek (doi, False) - po znalezieniu zaznaczamy na True i potem już nie porównujemy
|
||||
#TODO: lista krotek - jeśli wszystkie mają True dajemy terma
|
||||
|
||||
@@ -35,15 +35,23 @@ def producer(out_q, control_q, filename):
|
||||
#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
|
||||
def consumer(in_q, control_q, doi):
|
||||
sentinels = 0
|
||||
for item in doi:
|
||||
result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
|
||||
while True:
|
||||
done_check = True
|
||||
data = in_q.get()
|
||||
if data is _sentinel:
|
||||
print("Worker finished")
|
||||
sentinels += 1
|
||||
elif doi in data:
|
||||
control_q.put(_sentinel)
|
||||
print(data)
|
||||
print("HIT")
|
||||
else:
|
||||
for item in result_list:
|
||||
if item["DOI"] in data:
|
||||
print(data)
|
||||
print("HIT")
|
||||
item["exists"] = True
|
||||
done_check = done_check and item["exists"]
|
||||
if done_check:
|
||||
control_q.put(_sentinel)
|
||||
if sentinels >= MAXTHREADS:
|
||||
print("All workers finished")
|
||||
break
|
||||
@@ -51,31 +59,29 @@ def consumer(in_q, control_q, doi):
|
||||
|
||||
def search_for_doi(doi):
|
||||
threads = []
|
||||
work_q = Queue(maxsize=10000000)
|
||||
work_q = Queue(maxsize=100)
|
||||
control_q = Queue()
|
||||
t_cons = Thread(target = consumer, args = (work_q, control_q, doi))
|
||||
threads.append(t_cons)
|
||||
for i in range(0,MAXTHREADS):
|
||||
#filename = "test" + str(i) + CHUNk
|
||||
filename = "test" + str(i) + CHUNk
|
||||
filename = str(i) + CHUNk
|
||||
print(filename)
|
||||
#print(filename)
|
||||
threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
|
||||
for worker in threads:
|
||||
worker.start()
|
||||
for worker in threads:
|
||||
worker.join()
|
||||
|
||||
try:
|
||||
check = control_q.get(block=False)
|
||||
control_q.put(check)
|
||||
return True
|
||||
except Empty:
|
||||
return False
|
||||
|
||||
return result_list
|
||||
if __name__ == "__main__":
|
||||
if search_for_doi("10.1002/9781118786352.wbieg0998.pub2"):
|
||||
#if search_for_doi("10.1108/gm-02-2016-0025xx"):
|
||||
print("found")
|
||||
exit()
|
||||
else:
|
||||
print("Not found")
|
||||
print(search_for_doi([
|
||||
("10.1002/9781118786352.wbieg0998.pub2","DATA"),
|
||||
("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
|
||||
("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
|
||||
("10.2165/00128415-200309690-00017","DATA"),
|
||||
("10.5772/48313","DATA"),
|
||||
("10.15803/ijnc.7.2_419","DATA"),
|
||||
("10.1051/0004-6361/201321596e","DATA"),
|
||||
("10.2307/40835941","DATA"),
|
||||
]))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user