Librarian: graceful shutdown with resumable search state

A restart of the librarian used to throw away an in-flight search (and any
searches still queued). Now search state survives a restart:

* Resumable DB scan (search_bot): each producer records a tell()-cookie
  watermark per chunk file as it goes (safe because search_for_doi drains
  the work queue before returning), and can seek back to it. search_for_doi
  now takes stop_event + resume and returns (result_list, positions,
  interrupted).

* Persisted requests: /query writes the accepted request to a disk queue
  before enqueuing; replay_requests re-enqueues unfinished ones on startup.
  So even a search still waiting in the queue survives a restart.

* Checkpoints: when a graceful shutdown interrupts a scan, the librarian
  writes {dois, found-so-far, per-file offsets}. On restart answer_query
  loads it, skips the (already done) Crossref+refine, and continues the
  scan from the saved offsets with the found DOIs pre-marked - no line is
  read twice and none is missed. A finished or crashed search forgets its
  request+checkpoint (no poison-pill replay).

* Graceful shutdown: SIGTERM/SIGINT set a shutdown event; the running scan
  checkpoints and the worker stops. The main thread then exits within a
  BOUNDED window (CONJURER_LIBRARIAN_GRACEFUL_TIMEOUT, default 45s) so the
  pod can never become an un-killable zombie. Needs terminationGracePeriod
  >= that in the deploy (separate PR).

Tests: search_bot resume correctness (seek past scanned, don't miss/re-scan;
stop_event -> interrupted) and librarian state mechanics (request replay,
forget, checkpoint round-trip, poison-pill drop). Suite: 58 unit + 49
integration green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/integration/test_librarian_resume_state.py

@@ -0,0 +1,90 @@
+"""Integration: the librarian's persisted search state (requests + checkpoints).
+
+These pin the durable-state mechanics that let a search survive a restart:
+* accepted requests are replayed (re-enqueued) after a restart,
+* a finished/abandoned search is forgotten (request + checkpoint dropped),
+* a checkpoint round-trips through disk intact.
+
+The RESUME correctness itself (seek past scanned, don't miss, don't re-scan)
+lives in tests/unit/test_search_bot.py.
+"""
+import logging
+import sys
+import types
+
+import pytest
+
+if "habanero" not in sys.modules:
+    _habanero = types.ModuleType("habanero")
+    _habanero.Crossref = object
+    sys.modules["habanero"] = _habanero
+
+import conjurer_librarian as lib  # noqa: E402
+from durable_queue import DiskQueue  # noqa: E402
+
+_LOG = logging.getLogger("test-resume-state")
+_LOG.addHandler(logging.NullHandler())
+
+
+class _DummyCrossref:
+    def __init__(self, **kwargs):
+        self.kwargs = kwargs
+
+
+@pytest.fixture
+def state(tmp_path, monkeypatch):
+    monkeypatch.setattr(lib, "_requests", DiskQueue(str(tmp_path / "req")))
+    monkeypatch.setattr(lib, "_checkpoints", DiskQueue(str(tmp_path / "cp")))
+    monkeypatch.setattr(lib, "Crossref", _DummyCrossref)
+    monkeypatch.setenv("CONJURER_CROSSREF_MAILTO", "test@example.com")
+    monkeypatch.setattr(lib, "NETRC_FILE", "/nonexistent/conjurer/.netrc")
+    while not lib.librarian_queue.empty():
+        lib.librarian_queue.get()
+    lib.librarian_list.clear()
+    with lib._active_lock:
+        lib.active_queries.clear()
+    return None
+
+
+def test_accepted_request_is_replayed_after_restart(state):
+    lib._requests.put("u1", {"query": "kwas foliowy", "deep_search": False})
+
+    lib.replay_requests(_LOG)
+
+    item = lib.librarian_queue.get_nowait()
+    assert isinstance(item, lib.Librarian)
+    assert item.uuid == "u1"
+    assert item.query == "kwas foliowy"
+    assert lib.active_queries["u1"] == "queued"  # known again to the watchdog
+
+
+def test_forget_search_drops_request_and_checkpoint(state):
+    lib._requests.put("u2", {"query": "x", "deep_search": False})
+    lib._checkpoints.put("u2", {"dois": {}, "found": [], "positions": {}})
+
+    lib._forget_search("u2")
+
+    assert not lib._requests.contains("u2")
+    assert not lib._checkpoints.contains("u2")
+
+
+def test_checkpoint_round_trips_through_disk(state):
+    checkpoint = {
+        "dois": {"10.1/x": {"DOI": "10.1/x", "title": ["T"], "type": "article"}},
+        "found": ["10.1/already"],
+        "positions": {"0_chunk.txt": 4096},
+    }
+    lib._checkpoints.put("u3", checkpoint)
+    assert lib._checkpoints.get("u3") == checkpoint
+
+
+def test_unreadable_replayed_request_is_dropped_not_looped(state, monkeypatch):
+    # A request that can't be reconstructed (e.g. missing Crossref contact) must
+    # be dropped, not retried forever.
+    lib._requests.put("u4", {"query": "x", "deep_search": False})
+    monkeypatch.delenv("CONJURER_CROSSREF_MAILTO", raising=False)
+
+    lib.replay_requests(_LOG)
+
+    assert lib.librarian_queue.empty()
+    assert not lib._requests.contains("u4")  # forgotten, not left to loop

tests/unit/test_search_bot.py

@@ -27,17 +27,32 @@ def _write_chunks(directory, count, target=None, target_index=None):
         (directory / f"{n}_chunk.txt").write_text("".join(lines), encoding="utf-8")
 
 
-def _run_bounded(dois, timeout=20):
-    """Run search_for_doi in a thread; return (finished_in_time, result)."""
+def _run_full(dois, timeout=20, stop_event=None, resume=None):
+    """Run search_for_doi in a thread; return the whole result box.
+
+    search_for_doi now returns (result_list, positions, interrupted); the box
+    exposes all three (plus 'finished' and 'live') for the resume tests.
+    """
     box = {}
     live = []
-    worker = threading.Thread(
-        target=lambda: box.update(result=search_bot.search_for_doi(dois, live, _LOG)),
-        daemon=True,
-    )
+
+    def _run():
+        result_list, positions, interrupted = search_bot.search_for_doi(
+            dois, live, _LOG, stop_event=stop_event, resume=resume
+        )
+        box.update(result=result_list, positions=positions, interrupted=interrupted, live=live)
+
+    worker = threading.Thread(target=_run, daemon=True)
     worker.start()
     worker.join(timeout)
-    return (not worker.is_alive()), box.get("result")
+    box["finished"] = not worker.is_alive()
+    return box
+
+
+def _run_bounded(dois, timeout=20, stop_event=None, resume=None):
+    """Back-compat wrapper: return (finished_in_time, result_list)."""
+    box = _run_full(dois, timeout, stop_event, resume)
+    return box.get("finished"), box.get("result")
 
 
 def test_finds_doi_in_trailing_chunk(tmp_path, monkeypatch):
@@ -133,6 +148,55 @@ def test_discover_chunk_files_sorted_numerically(tmp_path, monkeypatch):
     assert found == ["0_chunk.txt", "1_chunk.txt", "2_chunk.txt", "10_chunk.txt"]
 
 
+def _offset_after(path, marker):
+    """Byte-cookie (tell) just past the line equal to `marker` in `path`."""
+    with open(path, "r", encoding="utf-8") as handle:
+        while True:
+            line = handle.readline()
+            if not line:
+                raise AssertionError(f"marker {marker!r} not found")
+            if line.strip() == marker:
+                return handle.tell()
+
+
+def test_resume_seeks_past_scanned_part_and_continues(tmp_path, monkeypatch):
+    # Chunk: early | first-half decoy | MIDDLE | late. Resume from just past
+    # MIDDLE with 'early' pre-found. The scan must: keep 'early' (pre-marked),
+    # find 'late' (after the resume point), and NOT find the first-half decoy
+    # (proving it seeked past it instead of re-reading from the top).
+    monkeypatch.setattr(search_bot, "DATABASE_PATH", str(tmp_path) + "/")
+    path = tmp_path / "0_chunk.txt"
+    path.write_text(
+        "10.1/early\n10.1/only-first-half\nMIDDLE\n10.1/late\n", encoding="utf-8"
+    )
+    offset = _offset_after(str(path), "MIDDLE")
+
+    resume = {"found": ["10.1/early"], "positions": {"0_chunk.txt": offset}}
+    box = _run_full(
+        [("10.1/early", "D"), ("10.1/late", "D"), ("10.1/only-first-half", "D")],
+        resume=resume,
+    )
+
+    assert box["finished"]
+    by_doi = {r["DOI"]: r["exists"] for r in box["result"]}
+    assert by_doi["10.1/early"] is True      # carried over from the checkpoint
+    assert by_doi["10.1/late"] is True       # found after the resume offset
+    assert by_doi["10.1/only-first-half"] is False  # skipped - not re-scanned
+
+
+def test_stop_event_interrupts_and_reports_positions(tmp_path, monkeypatch):
+    monkeypatch.setattr(search_bot, "DATABASE_PATH", str(tmp_path) + "/")
+    _write_chunks(tmp_path, count=2)
+    stop = __import__("threading").Event()
+    stop.set()  # already asked to stop before it starts
+
+    box = _run_full([("10.0000/decoy-0-a", "D")], stop_event=stop)
+
+    assert box["finished"], "an already-set stop must not hang the search"
+    assert box["interrupted"] is True
+    assert isinstance(box["positions"], dict)
+
+
 def test_bounded_queue_does_not_deadlock_on_early_termination(tmp_path, monkeypatch):
     # The OOM fix bounds the work queue. That means a producer can block on a
     # FULL queue - and if the consumers have already finished (all DOIs found)