Durable result delivery: OUTBOX + idempotent INBOX so results never die

An 8h search result must survive a transient bot outage, an api/address
misroute, or a restart of either side. Make the librarian->bot result
path durably at-least-once with idempotent rendering:

Shared: durable_queue.DiskQueue - a dependency-free, atomically-written,
one-file-per-key disk queue (unit-tested), shared by both images
(added to Dockerfile.librarian; the bot already COPYs *.py).

Librarian (sender): finished results go to a persistent OUTBOX before
sending; delivery retries with backoff; an entry is removed only on a
positive ACK; a resender thread keeps flushing the OUTBOX, so a result
survives a bot outage AND a librarian restart (OUTBOX is on the state
volume) - it simply keeps trying until acked.

Bot (receiver): /conjurer is now idempotent and durable - each result is
persisted to an INBOX before acking and only queued if its uuid was not
already delivered (dropped as a duplicate) or already pending. Once the
cog actually renders it, mark_delivered() records the uuid and clears the
inbox, so the librarian's resends become no-ops. On startup the bot
replays any accepted-but-unrendered result from the INBOX, so a bot crash
mid-flight doesn't lose it. Pongs stay ephemeral.

Together: the librarian keeps a result until the bot confirms it; the bot
keeps it until it is on screen; duplicates never double-render. Combined
with the deploy return-path fix, an expensive result no longer vanishes.

Tests: unit test_durable_queue; integration test_librarian_outbox
(retry/backoff, resend survives outage) and test_result_durable_delivery
(persist, dedup pending, dedup delivered, replay, pong not persisted).
Suite: 55 unit + 39 integration green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

conjurer_librarian/conjurer_librarian.py

@@ -18,6 +18,7 @@ import json
 import logging
 import os
 import threading
+import time
 from json.decoder import JSONDecodeError
 from logging import handlers
 from pathlib import Path
@@ -29,6 +30,7 @@ import lib_paths
 import scrape_bot
 import search_bot
 # import search_bot2 as search_bot
+from durable_queue import DiskQueue
 from flask import Flask, jsonify, request, abort
 from habanero import Crossref
 from waitress import serve
@@ -64,6 +66,16 @@ LOGFILE_PATH = _env_path(
     "CONJURER_LIBRARIAN_LOG", str(BASE_DIR / "librarian.log")
 )
 
+# Durable OUTBOX for finished results. A result is expensive (hours of compute),
+# so it is written here and only removed once the bot ACKs it (HTTP 200). Lives
+# on the librarian's persistent state volume, so it survives a librarian restart
+# and a transient bot outage; the resender thread keeps retrying until delivered.
+OUTBOX_DIR = _env("CONJURER_LIBRARIAN_OUTBOX", os.path.join(lib_paths.STATE_DIR, "outbox"))
+RESULT_SEND_ATTEMPTS = int(_env("CONJURER_RESULT_SEND_ATTEMPTS", "3"))
+RESULT_SEND_BACKOFF = float(_env("CONJURER_RESULT_SEND_BACKOFF", "2"))
+OUTBOX_RESEND_SECONDS = int(_env("CONJURER_OUTBOX_RESEND_SECONDS", "60"))
+_outbox = DiskQueue(OUTBOX_DIR)
+
 app = Flask(__name__)
 
 librarian_queue = Queue()
@@ -113,6 +125,63 @@ def _post_pong(app_logger, ping_uuid) -> None:
         app_logger.warning("PING pong send failed for %s: %s", ping_uuid, exc)
 
 
+def _deliver_result(uuid, payload, app_logger, attempts=RESULT_SEND_ATTEMPTS) -> bool:
+    """POST one result to the bot, retrying with backoff. True only on HTTP 200.
+
+    The bot's /conjurer is idempotent (dedups by uuid), so re-POSTing a result
+    it already has is safe - it just answers 200 again. That is what lets the
+    OUTBOX keep retrying until the result is truly acknowledged, without ever
+    double-delivering to the user.
+    """
+    target = f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}"
+    for attempt in range(1, max(1, attempts) + 1):
+        try:
+            response = requests.post(
+                target, json=payload, headers=_service_headers(), timeout=60
+            )
+            if response.status_code == 200:
+                app_logger.info("Result %s delivered (HTTP 200) on attempt %d", uuid, attempt)
+                return True
+            app_logger.warning(
+                "Result %s: bot returned HTTP %s (attempt %d/%d): %s",
+                uuid, response.status_code, attempt, attempts, response.text[:300],
+            )
+        except requests.exceptions.RequestException as exc:
+            app_logger.warning(
+                "Result %s delivery failed (attempt %d/%d): %s", uuid, attempt, attempts, exc
+            )
+        if attempt < attempts:
+            time.sleep(RESULT_SEND_BACKOFF * attempt)
+    return False
+
+
+def _resend_once(app_logger) -> None:
+    """One sweep of the OUTBOX: try to deliver every un-acked result, once each.
+
+    Removes each entry only after a positive ACK, so nothing is dropped until
+    the bot has it. Corrupt/unreadable entries are skipped by DiskQueue.items().
+    """
+    for uuid, payload, _ts in _outbox.items():
+        if _deliver_result(uuid, payload, app_logger, attempts=1):
+            _outbox.remove(uuid)
+
+
+def outbox_resender(app_logger) -> None:
+    """Background loop: periodically flush the OUTBOX until the bot is reachable.
+
+    This is what makes an expensive result survive a transient bot outage or a
+    librarian restart - on restart the persisted OUTBOX is simply resent."""
+    pending = len(_outbox)
+    if pending:
+        app_logger.info("OUTBOX has %d un-acked result(s) on startup - will resend", pending)
+    while True:
+        try:
+            _resend_once(app_logger)
+        except Exception as exc:  # pylint: disable=broad-exception-caught
+            app_logger.exception("OUTBOX resend sweep failed: %s", exc)
+        time.sleep(OUTBOX_RESEND_SECONDS)
+
+
 # trunk-ignore(pylint/R0902)
 class Librarian(object):
     """
@@ -503,47 +572,24 @@ class BackgroundTaskSearch(threading.Thread):
                     json.dump(database, s_file)
                 self.app.logger.info("FINISHED")
 
-                # Send the result back to the bot. Log EXACTLY what goes out
-                # (target, uuid, how many DOIs and which) so the librarian log
-                # makes it plain a result was sent and what was in it.
+                # Persist the result to the durable OUTBOX FIRST, then try to
+                # deliver it. Writing to disk before sending is the whole point:
+                # an expensive (hours-long) result now survives a failed send, a
+                # bot outage, or a librarian restart - the resender keeps
+                # retrying until the bot ACKs, and only then is it removed.
                 payload = result  # shape: {uuid: {DOI: {"Title": ..., "type": ...}}}
+                uuid = str(librarian.uuid)
                 hits = payload.get(librarian.uuid, {}) if isinstance(payload, dict) else {}
-                target = f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}"
+                _outbox.put(uuid, payload)
                 self.app.logger.info(
-                    "SENDING result for %s to %s: %d DOI(s): %s",
-                    librarian.uuid,
-                    target,
-                    len(hits),
-                    list(hits.keys()),
+                    "SENDING result for %s: %d DOI(s): %s (queued to OUTBOX)",
+                    uuid, len(hits), list(hits.keys()),
                 )
-                # A failed send must NOT kill this worker - otherwise a bot that
-                # is momentarily down stalls every future query until the
-                # librarian is restarted. Log and carry on to the next search.
-                try:
-                    response = await asyncio.to_thread(
-                        requests.post,
-                        target,
-                        json=payload,
-                        headers=_service_headers(),
-                        timeout=360,
-                    )
-                    if response.status_code == 200:
-                        self.app.logger.info(
-                            "SENT result for %s -> HTTP 200 (bot accepted)", librarian.uuid
-                        )
-                    else:
-                        self.app.logger.warning(
-                            "SENT result for %s but bot returned HTTP %s: %s",
-                            librarian.uuid,
-                            response.status_code,
-                            response.text[:500],
-                        )
-                except requests.exceptions.RequestException as exc:
-                    self.app.logger.error(
-                        "FAILED to send result for %s to %s: %s",
-                        librarian.uuid,
-                        target,
-                        exc,
+                if await asyncio.to_thread(_deliver_result, uuid, payload, self.app.logger):
+                    _outbox.remove(uuid)
+                else:
+                    self.app.logger.warning(
+                        "Result %s not acked yet - left in OUTBOX for the resender", uuid
                     )
             except Exception as exc:  # pylint: disable=broad-exception-caught
                 # A crashing search must not kill the worker thread (which would
@@ -703,6 +749,11 @@ if __name__ == "__main__":
             target=scrape_bot.scraper, args=(app.logger,), daemon=True
         )
     )
+    # Durable delivery: keep flushing the OUTBOX so any result not yet acked by
+    # the bot (transient outage, or left over from before a restart) is resent.
+    threads.append(
+        threading.Thread(target=outbox_resender, args=(app.logger,), daemon=True)
+    )
     i = 0
     try:
         for worker in threads: