diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index 51ca627..3dacfd1 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -25,6 +25,7 @@ from queue import Queue from typing import Dict, Optional import requests +import lib_paths import scrape_bot import search_bot # import search_bot2 as search_bot @@ -194,7 +195,7 @@ class Librarian(object): self.app.logger.info("CROSSREF DONE") self.app.logger.info("CROSSREF DONE") - with open("cr_results.json", "r+", encoding="utf-8") as data_file: + with open(lib_paths.CR_RESULTS, "r+", encoding="utf-8") as data_file: # First we load existing data into a dict. try: file_data = json.load(data_file) @@ -258,7 +259,7 @@ class Librarian(object): for item in temp: refined_result[item["DOI"]]= item - with open("rr_results.json", "r+", encoding="utf-8") as data_file: + with open(lib_paths.RR_RESULTS, "r+", encoding="utf-8") as data_file: # First we load existing data into a dict. try: file_data = json.load(data_file) @@ -417,7 +418,8 @@ class BackgroundTaskSearch(threading.Thread): self.app.logger.info("Saving to file") # Save results to "not_in_db.json" file - with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file: + with open(lib_paths.NOT_IN_DB, "r+", encoding="utf-8") as ndb_file: + ndb_database = {} try: ndb_database = json.load(ndb_file) except JSONDecodeError: @@ -431,7 +433,8 @@ class BackgroundTaskSearch(threading.Thread): json.dump(ndb_database, ndb_file) # Save results to "s_results.json" file - with open("s_results.json", "r+", encoding="utf-8") as s_file: + with open(lib_paths.S_RESULTS, "r+", encoding="utf-8") as s_file: + database = {} try: database = json.load(s_file) except JSONDecodeError: diff --git a/conjurer_librarian/lib_paths.py b/conjurer_librarian/lib_paths.py new file mode 100644 index 0000000..61de1d4 --- /dev/null +++ b/conjurer_librarian/lib_paths.py @@ -0,0 +1,41 @@ +"""Resolved, seeded paths for the librarian's runtime JSON state. + +``conjurer_librarian.py`` and ``scrape_bot.py`` open these files in place with +mode ``r+``, which requires them to already exist - in a fresh container the +working directory has none of them, so the worker threads crashed with +FileNotFoundError. + +They now live in a single mounted, persistent directory +(``CONJURER_LIBRARIAN_STATE_DIR``, default ``/lib_temp_files``) and are seeded +with an empty JSON object on import, so a fresh container/volume never crashes +and the accumulated results survive restarts. +""" +import os + +STATE_DIR = os.getenv("CONJURER_LIBRARIAN_STATE_DIR", "/lib_temp_files") + +CR_RESULTS = os.path.join(STATE_DIR, "cr_results.json") # Crossref raw hits +RR_RESULTS = os.path.join(STATE_DIR, "rr_results.json") # refined results +NOT_IN_DB = os.path.join(STATE_DIR, "not_in_db.json") # DOIs to scrape +S_RESULTS = os.path.join(STATE_DIR, "s_results.json") # final send results + +_ALL = (CR_RESULTS, RR_RESULTS, NOT_IN_DB, S_RESULTS) + + +def ensure_state_files(): + """Create the state dir and seed any missing file with an empty JSON dict.""" + try: + os.makedirs(STATE_DIR, exist_ok=True) + except OSError as exc: # pragma: no cover - surfaced in logs, not fatal + print(f"lib_paths: cannot create {STATE_DIR}: {exc}") + return + for path in _ALL: + if not os.path.exists(path): + try: + with open(path, "w", encoding="utf-8") as handle: + handle.write("{}") + except OSError as exc: # pragma: no cover + print(f"lib_paths: cannot seed {path}: {exc}") + + +ensure_state_files() diff --git a/conjurer_librarian/scrape_bot.py b/conjurer_librarian/scrape_bot.py index e25241f..73e8e9e 100644 --- a/conjurer_librarian/scrape_bot.py +++ b/conjurer_librarian/scrape_bot.py @@ -16,6 +16,8 @@ from urllib.request import urlopen from requests import ConnectionError as RequestsConnectionError from requests import ConnectTimeout, Timeout +import lib_paths + SCR_DATABASE_PATH = os.getenv("CONJURER_LIBRARIAN_DB_PATH", r"C:\\Database\\chunks\\") SCR_FILENAME = os.getenv("CONJURER_LIBRARIAN_SCRAPE_CHUNK", "40_chunk.txt") SCR_ENCODING = os.getenv("CONJURER_ENCODING", "utf-8") @@ -30,7 +32,7 @@ def load_ndb_to_q(logger): """ logger.info("Loader started") while True: - with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file: + with open(lib_paths.NOT_IN_DB, "r+", encoding="utf-8") as ndb_file: try: ndb_database = json.load(ndb_file) for _ in range (1,10): diff --git a/conjurer_librarian/search_bot.py b/conjurer_librarian/search_bot.py index dfa0f8c..ca6999e 100644 --- a/conjurer_librarian/search_bot.py +++ b/conjurer_librarian/search_bot.py @@ -47,7 +47,7 @@ def producer(out_q, control_q, filename, _logger): filename (str): Name of the file. _logger: Logger object for logging. """ - with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file: + with open(DATABASE_PATH + filename, "r", encoding=ENCODING) as operated_file: print(f"Worker {filename} ") line_no = 0 while True: diff --git a/conjurer_librarian/search_bot2.py b/conjurer_librarian/search_bot2.py index 26c7504..fa6dd69 100644 --- a/conjurer_librarian/search_bot2.py +++ b/conjurer_librarian/search_bot2.py @@ -47,7 +47,7 @@ def producer(out_q, control_q, filename, _logger): filename (str): Name of the file. _logger: Logger object for logging. """ - with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file: + with open(DATABASE_PATH + filename, "r", encoding=ENCODING) as operated_file: print(f"Worker {filename} ") line_no = 0 while True: diff --git a/docker/Dockerfile.librarian b/docker/Dockerfile.librarian index a5e9204..327532b 100644 --- a/docker/Dockerfile.librarian +++ b/docker/Dockerfile.librarian @@ -18,10 +18,13 @@ COPY conjurer_librarian/ ./ ENV PYTHONUNBUFFERED=1 \ CONJURER_LIBRARIAN_HOST=0.0.0.0 \ CONJURER_LIBRARIAN_PORT=5001 \ - CONJURER_LIBRARIAN_DB_PATH=/doi/ + CONJURER_LIBRARIAN_DB_PATH=/doi/ \ + CONJURER_LIBRARIAN_STATE_DIR=/lib_temp_files -# The local DOI chunk database (large) is mounted here. -VOLUME ["/doi"] +# /doi = the local DOI chunk database (large, read-only). +# /lib_temp_files = runtime JSON state (cr_results/rr_results/not_in_db/ +# s_results) - seeded on first run, persisted across restarts. +VOLUME ["/doi", "/lib_temp_files"] EXPOSE 5001 CMD ["python", "conjurer_librarian.py"] diff --git a/docker/compose.librarian.yaml b/docker/compose.librarian.yaml index 42d4fcb..07b2c6f 100644 --- a/docker/compose.librarian.yaml +++ b/docker/compose.librarian.yaml @@ -14,7 +14,11 @@ services: ports: - "5001:5001" volumes: - # Local DOI chunk database (0_chunk.txt ... N_chunk.txt). + # Local DOI chunk database (0_chunk.txt ... N_chunk.txt). Read-only: + # the search workers only read it (open mode "r"). - /srv/librarian/doi:/doi:ro + # Runtime JSON state (cr_results/rr_results/not_in_db/s_results) - + # seeded on first run, persisted here across restarts. + - /srv/librarian/state:/lib_temp_files # Optional: netrc holding Crossref credentials (or use CONJURER_CROSSREF_MAILTO). - /srv/librarian/secrets/.netrc:/secrets/.netrc:ro diff --git a/docker/env/librarian.env.example b/docker/env/librarian.env.example index d481960..dbc68fe 100644 --- a/docker/env/librarian.env.example +++ b/docker/env/librarian.env.example @@ -17,5 +17,9 @@ CONJURER_LIBRARIAN_DB_PATH=/doi/ CONJURER_LIBRARIAN_MAXTHREADS=41 CONJURER_LIBRARIAN_CHUNK=_chunk.txt +# Runtime JSON state dir (mounted, persistent): cr_results/rr_results/ +# not_in_db/s_results are seeded here on first run. +CONJURER_LIBRARIAN_STATE_DIR=/lib_temp_files + # Optional netrc (for Crossref credentials) CONJURER_NETRC_FILE=/secrets/.netrc