gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-19 16:22:10 +00:00
Code Issues Packages Projects Releases Wiki Activity

Tag: 1.3

Browse Source 
Intermediate commits (oldest → newest):
- Almost deployment versio - for full deployment change parameteres in search_bot
- Preparation for deployment
- Main bot (#1)
- Search webservice (#2)
- Music service (#3)
- FIx
This commit is contained in:
Polish Hammer
2025-10-30 16:58:58 +01:00
parent 2988bd8205
commit 04638c9141
15 changed files with 463 additions and 161870 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/conjurer_librarian.py
+53 -23
View File
@@ -12,9 +12,8 @@ from habanero import Crossref
from waitress import serve
from queue import Queue
import search_bot
import scrape_bot
import requests
from typing import Callable
from uuid import UUID
NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
HOST_ADDRESS = "192.168.1.192"
@@ -22,7 +21,7 @@ PORT_ADDRESS = 5001
MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
SEND_RESULTS = "/conjurer"
BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
MAX_CR_RESULTS = 150000
ENCODING = "utf-8"
app = Flask(__name__)
@@ -37,13 +36,15 @@ class Librarian(object):
self.cr = Crossref(mailto=authTokens[0])
self.query = query
self.uuid = str(uuid)
self.limit = 3000
self.limit = MAX_CR_RESULTS
self.fetched = 0
self.hit = 0
self.total = 0
self.app = app
self.logger = app.logger
self.live_results = []
self.final_result = []
self.not_in_db = []
self.search_result_from_cr = {}
self.done = False
@@ -62,8 +63,7 @@ class Librarian(object):
self.fetched += len(result['message']['items'])
self.logger.info (self.total)
self.logger.info(self.fetched)
time.sleep(1)
time.sleep(0.1)
self.app.logger.info("CROSSREF DONE")
#TODO: add to sql with UUID
#sql row - title, doi, author, uuid, last retrieved, is_available
@@ -142,20 +142,20 @@ class Librarian(object):
temp.append([item["DOI"], item])
refined_result = await self.check_if_exists(temp)
temp = []
for item in refined_result:
if item["exists"]:
temp.append(item["data"])
refined_result = temp
self.hit = len(refined_result)
return refined_result
self.final_result.append(item["data"])
else:
self.not_in_db.append(item["data"])
self.hit = len(self.final_result)
return self.final_result
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
async def answer_query(self):
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
#Search sql for UUID
#SQL SELECT * FROM RESULTS WHERE UUID = uuid
self.logger.info(f"Search started {self.uuid}")
uuid_found = False
database = None
query = self.query
with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
@@ -165,11 +165,19 @@ class Librarian(object):
database = json.load(cr_file)
except JSONDecodeError:
pass
#TODO: Zwracać i final_result i not_in_db. Not in db_do pliku - i do ręcznego przeglądania potem.
if database:
for item in database.keys():
if item in self.uuid:
self.result = await self.refine_query(database)
temp = {}
for item in self.result:
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.not_in_db = temp2
return self.result
answer = await self.search_crossref(query=query)
database.update(answer)
@@ -177,7 +185,6 @@ class Librarian(object):
cr_file.seek(0)
json.dump(database,cr_file)
self.result = await self.refine_query(answer)
return self.result
else:
answer = await self.search_crossref(query=query)
database = answer
@@ -187,7 +194,15 @@ class Librarian(object):
self.result = await self.refine_query(answer)
self.app.logger.info("Refined")
self.done = True
return self.result
temp = {}
for item in self.result:
temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.result = temp
temp2 = {}
for item in self.not_in_db:
temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
self.not_in_db = temp2
return self.result
#============================= FLASK INTERNALS===============================
def flask_debug():
@@ -214,18 +229,34 @@ class BackgroundTaskSearch(threading.Thread):
async def _run(self):
while True:
database = None
ndb_database = None
librarian = librarian_queue.get()
self.app.logger.info("STARTED")
result = await librarian.answer_query()
result = {librarian.uuid: result}
self.app.logger.info("Saving to file")
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
try:
ndb_database = json.load(ndb_file)
except JSONDecodeError:
pass
if ndb_database:
ndb_database.update(librarian.not_in_db)
else:
ndb_database = librarian.not_in_db
ndb_file.truncate(0)
ndb_file.seek(0)
json.dump(ndb_database, ndb_file)
with open("s_results.json", "r+", encoding="utf-8") as s_file:
try:
database = json.load(s_file)
except JSONDecodeError:
pass
if database:
database.extend(result)
self.app.logger.info(database)
self.app.logger.info(result)
database.update(result)
else:
database = result
self.app.logger.info("DUMPING DATA")
@@ -242,7 +273,8 @@ class BackgroundTaskSearch(threading.Thread):
timeout=360,
)
self.app.logger.info("SENT")
_ = await coroutine
result = await coroutine
self.app.logger.info(result.status_code)
self.app.logger.info("SEND CONFIRMED")
time.sleep(1)
@@ -254,9 +286,6 @@ async def query_database():
app.logger.info(record["query"])
app.logger.info(record["UUID"])
uuid = record["UUID"]
#TODO: TESTING ONLY======================
uuid = UUID('{12345678-1234-5678-1234-567812345678}')
cl = Librarian(app, record["query"], uuid)
librarian_queue.put(cl)
librarian_list.append(cl)
@@ -284,14 +313,15 @@ async def get_partial():
#=======================================MAIN===================================================
if __name__ == "__main__":
logger = logging.getLogger('conjurer_musician')
logger = app.logger
logger.setLevel(logging.DEBUG)
threads = []
#threads.append(threading.Thread(target=waitress_run))
threads.append(threading.Thread(target=flask_debug))
threads.append(threading.Thread(target=waitress_run))
#threads.append(threading.Thread(target=flask_debug))
bgtask = BackgroundTaskSearch()
bgtask.app = app
threads.append(bgtask)
threads.append(threading.Thread(target = scrape_bot.scraper, args=(logger,)))
i = 0
for worker in threads:
try: