gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-15 22:32:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Fixing the bug in the conjurer_musician.py file.

Browse Source 
Fx

Fix

fx

Fix ?

Revert "Fix"

This reverts commit 4e185f7b34dfede04ed36ab9813bd9d1f7fb8b35.

Adding deploy for music bot
This commit is contained in:
The Hammer
2024-04-17 12:32:54 +01:00
committed by migatu
parent 0375b0a892
commit ca6896ebcf
7 changed files with 229 additions and 100 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/conjurer_librarian.py
+95 -58
View File
@@ -1,27 +1,29 @@
import re
import asyncio
import json
import logging
import netrc
import re
import threading
import time
from flask import Flask, jsonify, request
import netrc
import json
import asyncio
from json.decoder import JSONDecodeError
from queue import Queue
from typing import Callable
from urllib.request import urlopen
from uuid import UUID
import requests
import search_bot
from flask import Flask, jsonify, request
from habanero import Crossref
from waitress import serve
from queue import Queue
import search_bot
import requests
from typing import Callable
from uuid import UUID
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
HOST_ADDRESS = "192.168.1.192"
PORT_ADDRESS = 5001
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer"
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
ENCODING = "utf-8"
app = Flask(__name__)
@@ -29,6 +31,7 @@ app = Flask(__name__)
librarian_queue = Queue()
librarian_list = []
class Librarian(object):
def __init__(self, app, query, uuid) -> None:
@@ -49,25 +52,25 @@ class Librarian(object):
async def search_crossref(self, query):
self.app.logger.info("STARTED SEARCH")
result = self.cr.works(query = query, limit = 1000)
result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(result)
self.total = result['message']['total-results']
self.fetched += len(result['message']['items'])
self.logger.info (self.total)
self.total = result["message"]["total-results"]
self.fetched += len(result["message"]["items"])
self.logger.info(self.total)
self.logger.info(self.fetched)
while self.total > self.fetched and self.limit > self.fetched:
result = self.cr.works(query = query, limit = 1000)
result = self.cr.works(query=query, limit=1000)
self.search_result_from_cr.update(result)
self.total = result['message']['total-results']
self.fetched += len(result['message']['items'])
self.logger.info (self.total)
self.total = result["message"]["total-results"]
self.fetched += len(result["message"]["items"])
self.logger.info(self.total)
self.logger.info(self.fetched)
time.sleep(1)
self.app.logger.info("CROSSREF DONE")
#TODO: add to sql with UUID
#sql row - title, doi, author, uuid, last retrieved, is_available
# TODO: add to sql with UUID
# sql row - title, doi, author, uuid, last retrieved, is_available
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
# First we load existing data into a dict.
try:
@@ -76,18 +79,34 @@ class Librarian(object):
file_data = {}
summarized_results = []
self.app.logger.info("REFINE: Removing all derived works from the list")
for item in self.search_result_from_cr['message']['items']:
for item in self.search_result_from_cr["message"]["items"]:
container = "NO"
if "container-title" in item:
container = "YES"
else:
summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
summarized_results.append(
{
"DOI": item["DOI"],
"title": item["title"] if "title" in item else None,
"type": item["type"] if "type" in item else None,
"container": container,
}
)
result = {
self.uuid: {
"total_results": self.search_result_from_cr["message"][
"total-results"
],
"on_page": 1,
"summary": summarized_results,
"results": self.search_result_from_cr["message"]["items"],
}
}
if file_data:
file_data.update(result)
else:
file_data = result
self.app.logger.info("REFINE: Dumping to file")
data_file.truncate(0)
data_file.seek(0)
@@ -96,7 +115,9 @@ class Librarian(object):
async def check_dois_in_local_db(self, doi):
self.app.logger.info("REFINE: Running search in the backend app")
coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
coro = asyncio.to_thread(
search_bot.search_for_doi, doi, self.live_results, self.app.logger
)
result = await coro
return result
@@ -109,25 +130,34 @@ class Librarian(object):
data = response.read()
text = data.decode("utf-8")
for line in text.splitlines():
if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
if re.match(
r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
line,
):
return False
if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
if m := re.match(
r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
):
direct_download_link = "https:" + str(m.group(1))
self.app.logger.info (direct_download_link)
self.app.logger.info(direct_download_link)
return True
return False
async def check_if_exists(self, doi, brute_force = False):
async def check_if_exists(self, doi, brute_force=False):
result = {}
result = await self.check_dois_in_local_db(doi)
if brute_force and not result:
for item in doi:
item_link = "https://sci-hub.se/" + item[0]
tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
tmp = {
"DOI": item["DOI"],
"exists": await self.check_if_exists_brute_force(item_link),
"data": item,
}
result.update(tmp)
return result
async def refine_query(self, unrefined_reqult, brute_force = False):
async def refine_query(self, unrefined_reqult, brute_force=False):
temp = []
refined_result = []
for key in unrefined_reqult:
@@ -135,7 +165,7 @@ class Librarian(object):
self.logger.info(key)
for item in unrefined_reqult[self.uuid]["summary"]:
if item["container"] == "NO" and item["title"]:
if item["container"] == "NO" and item["title"]:
refined_result.append(item)
for item in refined_result:
@@ -149,20 +179,21 @@ class Librarian(object):
refined_result = temp
self.hit = len(refined_result)
return refined_result
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
# TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
async def answer_query(self):
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
#Search sql for UUID
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
# TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
# Search sql for UUID
# SQL SELECT * FROM RESULTS WHERE UUID = uuid
self.logger.info(f"Search started {self.uuid}")
uuid_found = False
database = None
query = self.query
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
#tu logika komunikacji z SQL i cała logika związana z wyszukaniem
#ale tymczasowo plik
# tu logika komunikacji z SQL i cała logika związana z wyszukaniem
# ale tymczasowo plik
try:
database = json.load(cr_file)
database = json.load(cr_file)
except JSONDecodeError:
pass
@@ -175,7 +206,7 @@ class Librarian(object):
database.update(answer)
cr_file.truncate(0)
cr_file.seek(0)
json.dump(database,cr_file)
json.dump(database, cr_file)
self.result = await self.refine_query(answer)
return self.result
else:
@@ -183,13 +214,14 @@ class Librarian(object):
database = answer
cr_file.truncate(0)
cr_file.seek(0)
json.dump(database,cr_file)
json.dump(database, cr_file)
self.result = await self.refine_query(answer)
self.app.logger.info("Refined")
self.done = True
return self.result
#============================= FLASK INTERNALS===============================
# ============================= FLASK INTERNALS===============================
def flask_debug():
"""
The `flask_debug` function starts a Flask application in debug mode without using the reloader.
@@ -198,6 +230,7 @@ def flask_debug():
# trunk-ignore(bandit/B201)
app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
def waitress_run():
"""
The `waitress_run` function serves the `app` on host "0.0.0.0"
@@ -205,6 +238,7 @@ def waitress_run():
"""
serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
class BackgroundTaskSearch(threading.Thread):
def run(self):
loop = asyncio.new_event_loop() # loop = asyncio.get_event_loop()
@@ -221,7 +255,7 @@ class BackgroundTaskSearch(threading.Thread):
self.app.logger.info("Saving to file")
with open("s_results.json", "r+", encoding="utf-8") as s_file:
try:
database = json.load(s_file)
database = json.load(s_file)
except JSONDecodeError:
pass
if database:
@@ -236,17 +270,18 @@ class BackgroundTaskSearch(threading.Thread):
self.app.logger.info(result)
coroutine = asyncio.to_thread(
requests.post,
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
json=result,
timeout=360,
)
requests.post,
f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
json=result,
timeout=360,
)
self.app.logger.info("SENT")
_ = await coroutine
self.app.logger.info("SEND CONFIRMED")
time.sleep(1)
#==================================SERVER ROUTES==========================================
# ==================================SERVER ROUTES==========================================
@app.route("/query", methods=["POST"])
async def query_database():
record = json.loads(request.data)
@@ -254,8 +289,8 @@ async def query_database():
app.logger.info(record["query"])
app.logger.info(record["UUID"])
uuid = record["UUID"]
#TODO: TESTING ONLY======================
uuid = UUID('{12345678-1234-5678-1234-567812345678}')
# TODO: TESTING ONLY======================
uuid = UUID("{12345678-1234-5678-1234-567812345678}")
cl = Librarian(app, record["query"], uuid)
librarian_queue.put(cl)
@@ -267,6 +302,7 @@ async def query_database():
)
return return_data
@app.route("/get_partial_result", methods=["POST"])
async def get_partial():
record = json.loads(request.data)
@@ -282,12 +318,13 @@ async def get_partial():
)
return return_data
#=======================================MAIN===================================================
# =======================================MAIN===================================================
if __name__ == "__main__":
logger = logging.getLogger('conjurer_musician')
logger = logging.getLogger("conjurer_musician")
logger.setLevel(logging.DEBUG)
threads = []
#threads.append(threading.Thread(target=waitress_run))
# threads.append(threading.Thread(target=waitress_run))
threads.append(threading.Thread(target=flask_debug))
bgtask = BackgroundTaskSearch()
bgtask.app = app
@@ -296,7 +333,7 @@ if __name__ == "__main__":
for worker in threads:
try:
logger.info(f"App number: {i}")
i+=1
i += 1
worker.start()
except RuntimeError as e:
logger.error("Exploded")