gitea/conjurer
1
0
Fork 0
mirror of https://github.com/migatu/conjurer.git synced 2026-07-14 21:38:38 +00:00
Code Issues Packages Projects Releases Wiki Activity

Logging fix on conjurer_librarian

Browse Source
This commit is contained in:
Michal Tuszowski
2024-04-17 23:55:01 +02:00
parent 548496e0a6
commit e121c6dc39
1 changed files with 11 additions and 13 deletions
Download Patch File Download Diff File Expand all files Collapse all files
conjurer_librarian/conjurer_librarian.py
+11 -13
View File
@@ -41,7 +41,6 @@ class Librarian(object):
self.hit = 0 self.hit = 0
self.total = 0 self.total = 0
self.app = app self.app = app
self.logger = app.logger
self.live_results = [] self.live_results = []
self.final_result = [] self.final_result = []
self.not_in_db = [] self.not_in_db = []
@@ -54,15 +53,15 @@ class Librarian(object):
self.search_result_from_cr.update(result) self.search_result_from_cr.update(result)
self.total = result['message']['total-results'] self.total = result['message']['total-results']
self.fetched += len(result['message']['items']) self.fetched += len(result['message']['items'])
self.logger.info (self.total) self.app.logger.info (self.total)
self.logger.info(self.fetched) self.app.logger.info(self.fetched)
while self.total > self.fetched and self.limit > self.fetched: while self.total > self.fetched and self.limit > self.fetched:
result = self.cr.works(query = query, limit = 1000) result = self.cr.works(query = query, limit = 1000)
self.search_result_from_cr.update(result) self.search_result_from_cr.update(result)
self.total = result['message']['total-results'] self.total = result['message']['total-results']
self.fetched += len(result['message']['items']) self.fetched += len(result['message']['items'])
self.logger.info (self.total) self.app.logger.info (self.total)
self.logger.info(self.fetched) self.app.logger.info(self.fetched)
time.sleep(0.1) time.sleep(0.1)
self.app.logger.info("CROSSREF DONE") self.app.logger.info("CROSSREF DONE")
#TODO: add to sql with UUID #TODO: add to sql with UUID
@@ -131,8 +130,8 @@ class Librarian(object):
temp = [] temp = []
refined_result = [] refined_result = []
for key in unrefined_reqult: for key in unrefined_reqult:
self.logger.info("KEY:") self.app.logger.info("KEY:")
self.logger.info(key) self.app.logger.info(key)
for item in unrefined_reqult[self.uuid]["summary"]: for item in unrefined_reqult[self.uuid]["summary"]:
if item["container"] == "NO" and item["title"]: if item["container"] == "NO" and item["title"]:
@@ -155,7 +154,7 @@ class Librarian(object):
#TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
#Search sql for UUID #Search sql for UUID
#SQL SELECT * FROM RESULTS WHERE UUID = uuid #SQL SELECT * FROM RESULTS WHERE UUID = uuid
self.logger.info(f"Search started {self.uuid}") self.app.logger.info(f"Search started {self.uuid}")
database = None database = None
query = self.query query = self.query
with open("cr_results.json", "r+", encoding="utf-8") as cr_file: with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
@@ -313,23 +312,22 @@ async def get_partial():
#=======================================MAIN=================================================== #=======================================MAIN===================================================
if __name__ == "__main__": if __name__ == "__main__":
logger = app.logger app.logger.setLevel(logging.DEBUG)
logger.setLevel(logging.DEBUG)
threads = [] threads = []
threads.append(threading.Thread(target=waitress_run)) threads.append(threading.Thread(target=waitress_run))
#threads.append(threading.Thread(target=flask_debug)) #threads.append(threading.Thread(target=flask_debug))
bgtask = BackgroundTaskSearch() bgtask = BackgroundTaskSearch()
bgtask.app = app bgtask.app = app
threads.append(bgtask) threads.append(bgtask)
threads.append(threading.Thread(target = scrape_bot.scraper, args=(logger,))) threads.append(threading.Thread(target = scrape_bot.scraper, args=(app.logger,)))
i = 0 i = 0
for worker in threads: for worker in threads:
try: try:
logger.info(f"App number: {i}") app.logger.info(f"App number: {i}")
i+=1 i+=1
worker.start() worker.start()
except RuntimeError as e: except RuntimeError as e:
logger.error("Exploded") app.logger.error("Exploded")
print(str(e)) print(str(e))
for worker in threads: for worker in threads:
worker.join() worker.join()