librarian: mount + seed runtime JSON state under /lib_temp_files

(Re-applied cleanly on top of main: PR #18 was rebase-merged, so the branch
that became #19 conflicted on the already-landed commits. This carries ONLY
the librarian state fix - the sole content difference between that branch
and main - so nothing else is touched or lost.)

The worker threads open cr_results/rr_results/not_in_db/s_results.json in
place ('r+'), crashing with FileNotFoundError in a fresh container. New
conjurer_librarian/lib_paths.py resolves all four under a persistent dir
(CONJURER_LIBRARIAN_STATE_DIR, default /lib_temp_files) and seeds missing
ones with '{}' on import; shared by conjurer_librarian.py and scrape_bot.py
(no circular import). ndb_database/database initialised to {} before load so
a corrupt persisted file degrades to empty instead of NameError. search_bot
/search_bot2 open the DOI DB 'r' not 'r+' so /doi can stay read-only. Docker:
STATE_DIR env + VOLUME, compose mounts /srv/librarian/state:/lib_temp_files.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

conjurer_librarian/conjurer_librarian.py

@@ -25,6 +25,7 @@ from queue import Queue
 from typing import Dict, Optional
 
 import requests
+import lib_paths
 import scrape_bot
 import search_bot
 # import search_bot2 as search_bot
@@ -194,7 +195,7 @@ class Librarian(object):
             self.app.logger.info("CROSSREF DONE")
 
             self.app.logger.info("CROSSREF DONE")
-        with open("cr_results.json", "r+", encoding="utf-8") as data_file:
+        with open(lib_paths.CR_RESULTS, "r+", encoding="utf-8") as data_file:
             # First we load existing data into a dict.
             try:
                 file_data = json.load(data_file)
@@ -258,7 +259,7 @@ class Librarian(object):
 
         for item in temp:
             refined_result[item["DOI"]]=  item
-        with open("rr_results.json", "r+", encoding="utf-8") as data_file:
+        with open(lib_paths.RR_RESULTS, "r+", encoding="utf-8") as data_file:
             # First we load existing data into a dict.
             try:
                 file_data = json.load(data_file)
@@ -417,7 +418,8 @@ class BackgroundTaskSearch(threading.Thread):
             self.app.logger.info("Saving to file")
 
             # Save results to "not_in_db.json" file
-            with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
+            with open(lib_paths.NOT_IN_DB, "r+", encoding="utf-8") as ndb_file:
+                ndb_database = {}
                 try:
                     ndb_database = json.load(ndb_file)
                 except JSONDecodeError:
@@ -431,7 +433,8 @@ class BackgroundTaskSearch(threading.Thread):
                 json.dump(ndb_database, ndb_file)
 
             # Save results to "s_results.json" file
-            with open("s_results.json", "r+", encoding="utf-8") as s_file:
+            with open(lib_paths.S_RESULTS, "r+", encoding="utf-8") as s_file:
+                database = {}
                 try:
                     database = json.load(s_file)
                 except JSONDecodeError:

conjurer_librarian/lib_paths.py

@@ -0,0 +1,41 @@
+"""Resolved, seeded paths for the librarian's runtime JSON state.
+
+``conjurer_librarian.py`` and ``scrape_bot.py`` open these files in place with
+mode ``r+``, which requires them to already exist - in a fresh container the
+working directory has none of them, so the worker threads crashed with
+FileNotFoundError.
+
+They now live in a single mounted, persistent directory
+(``CONJURER_LIBRARIAN_STATE_DIR``, default ``/lib_temp_files``) and are seeded
+with an empty JSON object on import, so a fresh container/volume never crashes
+and the accumulated results survive restarts.
+"""
+import os
+
+STATE_DIR = os.getenv("CONJURER_LIBRARIAN_STATE_DIR", "/lib_temp_files")
+
+CR_RESULTS = os.path.join(STATE_DIR, "cr_results.json")   # Crossref raw hits
+RR_RESULTS = os.path.join(STATE_DIR, "rr_results.json")   # refined results
+NOT_IN_DB = os.path.join(STATE_DIR, "not_in_db.json")     # DOIs to scrape
+S_RESULTS = os.path.join(STATE_DIR, "s_results.json")     # final send results
+
+_ALL = (CR_RESULTS, RR_RESULTS, NOT_IN_DB, S_RESULTS)
+
+
+def ensure_state_files():
+    """Create the state dir and seed any missing file with an empty JSON dict."""
+    try:
+        os.makedirs(STATE_DIR, exist_ok=True)
+    except OSError as exc:  # pragma: no cover - surfaced in logs, not fatal
+        print(f"lib_paths: cannot create {STATE_DIR}: {exc}")
+        return
+    for path in _ALL:
+        if not os.path.exists(path):
+            try:
+                with open(path, "w", encoding="utf-8") as handle:
+                    handle.write("{}")
+            except OSError as exc:  # pragma: no cover
+                print(f"lib_paths: cannot seed {path}: {exc}")
+
+
+ensure_state_files()

conjurer_librarian/scrape_bot.py

@@ -16,6 +16,8 @@ from urllib.request import urlopen
 from requests import ConnectionError as RequestsConnectionError
 from requests import ConnectTimeout, Timeout
 
+import lib_paths
+
 SCR_DATABASE_PATH = os.getenv("CONJURER_LIBRARIAN_DB_PATH", r"C:\\Database\\chunks\\")
 SCR_FILENAME = os.getenv("CONJURER_LIBRARIAN_SCRAPE_CHUNK", "40_chunk.txt")
 SCR_ENCODING = os.getenv("CONJURER_ENCODING", "utf-8")
@@ -30,7 +32,7 @@ def load_ndb_to_q(logger):
     """
     logger.info("Loader started")
     while True:
-        with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
+        with open(lib_paths.NOT_IN_DB, "r+", encoding="utf-8") as ndb_file:
             try:
                 ndb_database = json.load(ndb_file)
                 for _ in range (1,10):

conjurer_librarian/search_bot.py

@@ -47,7 +47,7 @@ def producer(out_q, control_q, filename, _logger):
         filename (str): Name of the file.
         _logger: Logger object for logging.
     """
-    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
+    with open(DATABASE_PATH + filename, "r", encoding=ENCODING) as operated_file:
         print(f"Worker {filename} ")
         line_no = 0
         while True:

conjurer_librarian/search_bot2.py

@@ -47,7 +47,7 @@ def producer(out_q, control_q, filename, _logger):
         filename (str): Name of the file.
         _logger: Logger object for logging.
     """
-    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
+    with open(DATABASE_PATH + filename, "r", encoding=ENCODING) as operated_file:
         print(f"Worker {filename} ")
         line_no = 0
         while True:

docker/Dockerfile.librarian

@@ -18,10 +18,13 @@ COPY conjurer_librarian/ ./
 ENV PYTHONUNBUFFERED=1 \
     CONJURER_LIBRARIAN_HOST=0.0.0.0 \
     CONJURER_LIBRARIAN_PORT=5001 \
-    CONJURER_LIBRARIAN_DB_PATH=/doi/
+    CONJURER_LIBRARIAN_DB_PATH=/doi/ \
+    CONJURER_LIBRARIAN_STATE_DIR=/lib_temp_files
 
-# The local DOI chunk database (large) is mounted here.
-VOLUME ["/doi"]
+# /doi          = the local DOI chunk database (large, read-only).
+# /lib_temp_files = runtime JSON state (cr_results/rr_results/not_in_db/
+#                 s_results) - seeded on first run, persisted across restarts.
+VOLUME ["/doi", "/lib_temp_files"]
 EXPOSE 5001
 
 CMD ["python", "conjurer_librarian.py"]

docker/compose.librarian.yaml

@@ -14,7 +14,11 @@ services:
     ports:
       - "5001:5001"
     volumes:
-      # Local DOI chunk database (0_chunk.txt ... N_chunk.txt).
+      # Local DOI chunk database (0_chunk.txt ... N_chunk.txt). Read-only:
+      # the search workers only read it (open mode "r").
       - /srv/librarian/doi:/doi:ro
+      # Runtime JSON state (cr_results/rr_results/not_in_db/s_results) -
+      # seeded on first run, persisted here across restarts.
+      - /srv/librarian/state:/lib_temp_files
       # Optional: netrc holding Crossref credentials (or use CONJURER_CROSSREF_MAILTO).
       - /srv/librarian/secrets/.netrc:/secrets/.netrc:ro

docker/env/librarian.env.example

@@ -17,5 +17,9 @@ CONJURER_LIBRARIAN_DB_PATH=/doi/
 CONJURER_LIBRARIAN_MAXTHREADS=41
 CONJURER_LIBRARIAN_CHUNK=_chunk.txt
 
+# Runtime JSON state dir (mounted, persistent): cr_results/rr_results/
+# not_in_db/s_results are seeded here on first run.
+CONJURER_LIBRARIAN_STATE_DIR=/lib_temp_files
+
 # Optional netrc (for Crossref credentials)
 CONJURER_NETRC_FILE=/secrets/.netrc