Librarian: drop write-only result dumps + tame search logging

Two hygiene fixes on top of the work-queue OOM bound:

Result dumps: cr_results / rr_results / s_results.json were write-only
(nothing reads them) yet accumulated EVERY search forever and json.load'd
the whole growing file on each write - unbounded RAM and PVC growth, and
for a deep search the raw cr_results dump is hundreds of MB. They are now
off by default (CONJURER_LIBRARIAN_DEBUG_DUMPS) and, when enabled, are
overwritten with just the latest search - never loaded or accumulated.
not_in_db.json is untouched: it's a real queue the scraper drains.

Search logging: search_bot logged via print(), including a per-line
carriage-return progress line that flooded stdout / the log file with
millions of entries - fine for a desktop app, unreadable and bloating in
a container. All of it is now proper logging at DEBUG (with coarse
per-500k-line progress), so a normal run is quiet. The librarian log
level is configurable (CONJURER_LIBRARIAN_LOG_LEVEL, default INFO) and a
stdout handler is added so  stays useful now that the
search no longer prints straight to stdout. Set DEBUG for full verbosity.

Also: make test_result_delivery_contract hermetic (point the durable
spool at a temp dir so it can't pollute or be poisoned by the real
result_inbox/ between runs) and gitignore the runtime spool dirs.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

conjurer_librarian/conjurer_librarian.py

@@ -76,6 +76,30 @@ RESULT_SEND_BACKOFF = float(_env("CONJURER_RESULT_SEND_BACKOFF", "2"))
 OUTBOX_RESEND_SECONDS = int(_env("CONJURER_OUTBOX_RESEND_SECONDS", "60"))
 _outbox = DiskQueue(OUTBOX_DIR)
 
+# cr_results/rr_results/s_results.json are write-only debug dumps (nothing reads
+# them). They used to accumulate EVERY search forever AND json.load the whole
+# growing file on each write - unbounded RAM + disk, and for a deep search the
+# raw dump is hundreds of MB. Off by default now; when explicitly enabled they
+# are overwritten with just the latest search (no load, no accumulation).
+DEBUG_DUMPS = _env("CONJURER_LIBRARIAN_DEBUG_DUMPS", "0").lower() in ("1", "true", "yes")
+# Log level: INFO keeps normal runs readable (the desktop-era per-line/per-file
+# chatter is now DEBUG); set DEBUG to get the full verbosity back.
+LOG_LEVEL = _env("CONJURER_LIBRARIAN_LOG_LEVEL", "INFO").upper()
+
+
+def _dump_debug(path, uuid, data) -> None:
+    """Optionally dump the latest search's data for debugging.
+
+    Overwrites (never accumulates) and does nothing unless DEBUG_DUMPS is on, so
+    it can't grow RAM or the state volume in normal operation."""
+    if not DEBUG_DUMPS:
+        return
+    try:
+        with open(path, "w", encoding="utf-8") as handle:
+            json.dump({uuid: data}, handle)
+    except OSError as exc:
+        logging.getLogger("conjurer_librarian").warning("Debug dump to %s failed: %s", path, exc)
+
 app = Flask(__name__)
 
 librarian_queue = Queue()
@@ -311,20 +335,7 @@ class Librarian(object):
             self.app.logger.info("CROSSREF DONE")
 
             self.app.logger.info("CROSSREF DONE")
-        with open(lib_paths.CR_RESULTS, "r+", encoding="utf-8") as data_file:
-            # First we load existing data into a dict.
-            try:
-                file_data = json.load(data_file)
-            except JSONDecodeError:
-                file_data = {}
-            data_file.truncate(0)
-            data_file.seek(0)
-            tmp = {self.uuid : self.search_result_from_cr}
-            if file_data:
-                file_data.update(tmp)
-            else:
-                file_data = tmp
-            json.dump(file_data, data_file, indent=4)
+        _dump_debug(lib_paths.CR_RESULTS, self.uuid, self.search_result_from_cr)
         return cr_result
 
 
@@ -375,20 +386,7 @@ class Librarian(object):
 
         for item in temp:
             refined_result[item["DOI"]]=  item
-        with open(lib_paths.RR_RESULTS, "r+", encoding="utf-8") as data_file:
-            # First we load existing data into a dict.
-            try:
-                file_data = json.load(data_file)
-            except JSONDecodeError:
-                file_data = {}
-            data_file.truncate(0)
-            data_file.seek(0)
-            tmp = {self.uuid: refined_result}
-            if file_data:
-                file_data.update(tmp)
-            else:
-                file_data = tmp
-            json.dump(file_data, data_file, indent=4)
+        _dump_debug(lib_paths.RR_RESULTS, self.uuid, refined_result)
         return refined_result
 
     async def check_if_exists(self, refined_result):
@@ -568,24 +566,9 @@ class BackgroundTaskSearch(threading.Thread):
                     ndb_file.seek(0)
                     json.dump(ndb_database, ndb_file)
 
-                # Save results to "s_results.json" file
-                with open(lib_paths.S_RESULTS, "r+", encoding="utf-8") as s_file:
-                    database = {}
-                    try:
-                        database = json.load(s_file)
-                    except JSONDecodeError:
-                        pass
-                    if database:
-                        self.app.logger.info(database)
-                        self.app.logger.info(result)
-                        database.update(result)
-                    else:
-                        database = result
-                    self.app.logger.info("DUMPING DATA")
-                    s_file.truncate(0)
-                    s_file.seek(0)
-                    json.dump(database, s_file)
-                self.app.logger.info("FINISHED")
+                # Optional debug dump of the final result (off by default).
+                _dump_debug(lib_paths.S_RESULTS, librarian.uuid, result[librarian.uuid])
+                self.app.logger.info("Search %s finished", librarian.uuid)
 
                 # Persist the result to the durable OUTBOX FIRST, then try to
                 # deliver it. Writing to disk before sending is the whole point:
@@ -741,7 +724,10 @@ async def get_partial():
 
 # =======================================MAIN===================================================
 if __name__ == "__main__":
-    app.logger.setLevel(logging.DEBUG)
+    # Default INFO (readable). Set CONJURER_LIBRARIAN_LOG_LEVEL=DEBUG for the
+    # full per-file / per-line search chatter.
+    app.logger.setLevel(LOG_LEVEL)
+    _fmt = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
     LOGFILE_PATH.parent.mkdir(parents=True, exist_ok=True)
     h1 = handlers.RotatingFileHandler(
         filename=str(LOGFILE_PATH),
@@ -750,8 +736,13 @@ if __name__ == "__main__":
         maxBytes=6 * 1024 * 1024,
         backupCount=6,
     )
-
+    h1.setFormatter(_fmt)
     app.logger.addHandler(h1)
+    # Console handler so `kubectl logs` shows what's happening (k8s reads stdout);
+    # the search internals no longer print() straight to stdout.
+    _console = logging.StreamHandler()
+    _console.setFormatter(_fmt)
+    app.logger.addHandler(_console)
     threads = []
     threads.append(threading.Thread(target=waitress_run, daemon=True))
     # threads.append(threading.Thread(target=flask_debug))