mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
librarian: mount + seed runtime JSON state under /lib_temp_files
(Re-applied cleanly on top of main: PR #18 was rebase-merged, so the branch that became #19 conflicted on the already-landed commits. This carries ONLY the librarian state fix - the sole content difference between that branch and main - so nothing else is touched or lost.) The worker threads open cr_results/rr_results/not_in_db/s_results.json in place ('r+'), crashing with FileNotFoundError in a fresh container. New conjurer_librarian/lib_paths.py resolves all four under a persistent dir (CONJURER_LIBRARIAN_STATE_DIR, default /lib_temp_files) and seeds missing ones with '{}' on import; shared by conjurer_librarian.py and scrape_bot.py (no circular import). ndb_database/database initialised to {} before load so a corrupt persisted file degrades to empty instead of NameError. search_bot /search_bot2 open the DOI DB 'r' not 'r+' so /doi can stay read-only. Docker: STATE_DIR env + VOLUME, compose mounts /srv/librarian/state:/lib_temp_files. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
@@ -25,6 +25,7 @@ from queue import Queue
|
||||
from typing import Dict, Optional
|
||||
|
||||
import requests
|
||||
import lib_paths
|
||||
import scrape_bot
|
||||
import search_bot
|
||||
# import search_bot2 as search_bot
|
||||
@@ -194,7 +195,7 @@ class Librarian(object):
|
||||
self.app.logger.info("CROSSREF DONE")
|
||||
|
||||
self.app.logger.info("CROSSREF DONE")
|
||||
with open("cr_results.json", "r+", encoding="utf-8") as data_file:
|
||||
with open(lib_paths.CR_RESULTS, "r+", encoding="utf-8") as data_file:
|
||||
# First we load existing data into a dict.
|
||||
try:
|
||||
file_data = json.load(data_file)
|
||||
@@ -258,7 +259,7 @@ class Librarian(object):
|
||||
|
||||
for item in temp:
|
||||
refined_result[item["DOI"]]= item
|
||||
with open("rr_results.json", "r+", encoding="utf-8") as data_file:
|
||||
with open(lib_paths.RR_RESULTS, "r+", encoding="utf-8") as data_file:
|
||||
# First we load existing data into a dict.
|
||||
try:
|
||||
file_data = json.load(data_file)
|
||||
@@ -417,7 +418,8 @@ class BackgroundTaskSearch(threading.Thread):
|
||||
self.app.logger.info("Saving to file")
|
||||
|
||||
# Save results to "not_in_db.json" file
|
||||
with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
|
||||
with open(lib_paths.NOT_IN_DB, "r+", encoding="utf-8") as ndb_file:
|
||||
ndb_database = {}
|
||||
try:
|
||||
ndb_database = json.load(ndb_file)
|
||||
except JSONDecodeError:
|
||||
@@ -431,7 +433,8 @@ class BackgroundTaskSearch(threading.Thread):
|
||||
json.dump(ndb_database, ndb_file)
|
||||
|
||||
# Save results to "s_results.json" file
|
||||
with open("s_results.json", "r+", encoding="utf-8") as s_file:
|
||||
with open(lib_paths.S_RESULTS, "r+", encoding="utf-8") as s_file:
|
||||
database = {}
|
||||
try:
|
||||
database = json.load(s_file)
|
||||
except JSONDecodeError:
|
||||
|
||||
Reference in New Issue
Block a user