Librarian: graceful shutdown with resumable search state

A restart of the librarian used to throw away an in-flight search (and any
searches still queued). Now search state survives a restart:

* Resumable DB scan (search_bot): each producer records a tell()-cookie
  watermark per chunk file as it goes (safe because search_for_doi drains
  the work queue before returning), and can seek back to it. search_for_doi
  now takes stop_event + resume and returns (result_list, positions,
  interrupted).

* Persisted requests: /query writes the accepted request to a disk queue
  before enqueuing; replay_requests re-enqueues unfinished ones on startup.
  So even a search still waiting in the queue survives a restart.

* Checkpoints: when a graceful shutdown interrupts a scan, the librarian
  writes {dois, found-so-far, per-file offsets}. On restart answer_query
  loads it, skips the (already done) Crossref+refine, and continues the
  scan from the saved offsets with the found DOIs pre-marked - no line is
  read twice and none is missed. A finished or crashed search forgets its
  request+checkpoint (no poison-pill replay).

* Graceful shutdown: SIGTERM/SIGINT set a shutdown event; the running scan
  checkpoints and the worker stops. The main thread then exits within a
  BOUNDED window (CONJURER_LIBRARIAN_GRACEFUL_TIMEOUT, default 45s) so the
  pod can never become an un-killable zombie. Needs terminationGracePeriod
  >= that in the deploy (separate PR).

Tests: search_bot resume correctness (seek past scanned, don't miss/re-scan;
stop_event -> interrupted) and librarian state mechanics (request replay,
forget, checkpoint round-trip, poison-pill drop). Suite: 58 unit + 49
integration green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/unit/test_search_bot.py

@@ -27,17 +27,32 @@ def _write_chunks(directory, count, target=None, target_index=None):
         (directory / f"{n}_chunk.txt").write_text("".join(lines), encoding="utf-8")
 
 
-def _run_bounded(dois, timeout=20):
-    """Run search_for_doi in a thread; return (finished_in_time, result)."""
+def _run_full(dois, timeout=20, stop_event=None, resume=None):
+    """Run search_for_doi in a thread; return the whole result box.
+
+    search_for_doi now returns (result_list, positions, interrupted); the box
+    exposes all three (plus 'finished' and 'live') for the resume tests.
+    """
     box = {}
     live = []
-    worker = threading.Thread(
-        target=lambda: box.update(result=search_bot.search_for_doi(dois, live, _LOG)),
-        daemon=True,
-    )
+
+    def _run():
+        result_list, positions, interrupted = search_bot.search_for_doi(
+            dois, live, _LOG, stop_event=stop_event, resume=resume
+        )
+        box.update(result=result_list, positions=positions, interrupted=interrupted, live=live)
+
+    worker = threading.Thread(target=_run, daemon=True)
     worker.start()
     worker.join(timeout)
-    return (not worker.is_alive()), box.get("result")
+    box["finished"] = not worker.is_alive()
+    return box
+
+
+def _run_bounded(dois, timeout=20, stop_event=None, resume=None):
+    """Back-compat wrapper: return (finished_in_time, result_list)."""
+    box = _run_full(dois, timeout, stop_event, resume)
+    return box.get("finished"), box.get("result")
 
 
 def test_finds_doi_in_trailing_chunk(tmp_path, monkeypatch):
@@ -133,6 +148,55 @@ def test_discover_chunk_files_sorted_numerically(tmp_path, monkeypatch):
     assert found == ["0_chunk.txt", "1_chunk.txt", "2_chunk.txt", "10_chunk.txt"]
 
 
+def _offset_after(path, marker):
+    """Byte-cookie (tell) just past the line equal to `marker` in `path`."""
+    with open(path, "r", encoding="utf-8") as handle:
+        while True:
+            line = handle.readline()
+            if not line:
+                raise AssertionError(f"marker {marker!r} not found")
+            if line.strip() == marker:
+                return handle.tell()
+
+
+def test_resume_seeks_past_scanned_part_and_continues(tmp_path, monkeypatch):
+    # Chunk: early | first-half decoy | MIDDLE | late. Resume from just past
+    # MIDDLE with 'early' pre-found. The scan must: keep 'early' (pre-marked),
+    # find 'late' (after the resume point), and NOT find the first-half decoy
+    # (proving it seeked past it instead of re-reading from the top).
+    monkeypatch.setattr(search_bot, "DATABASE_PATH", str(tmp_path) + "/")
+    path = tmp_path / "0_chunk.txt"
+    path.write_text(
+        "10.1/early\n10.1/only-first-half\nMIDDLE\n10.1/late\n", encoding="utf-8"
+    )
+    offset = _offset_after(str(path), "MIDDLE")
+
+    resume = {"found": ["10.1/early"], "positions": {"0_chunk.txt": offset}}
+    box = _run_full(
+        [("10.1/early", "D"), ("10.1/late", "D"), ("10.1/only-first-half", "D")],
+        resume=resume,
+    )
+
+    assert box["finished"]
+    by_doi = {r["DOI"]: r["exists"] for r in box["result"]}
+    assert by_doi["10.1/early"] is True      # carried over from the checkpoint
+    assert by_doi["10.1/late"] is True       # found after the resume offset
+    assert by_doi["10.1/only-first-half"] is False  # skipped - not re-scanned
+
+
+def test_stop_event_interrupts_and_reports_positions(tmp_path, monkeypatch):
+    monkeypatch.setattr(search_bot, "DATABASE_PATH", str(tmp_path) + "/")
+    _write_chunks(tmp_path, count=2)
+    stop = __import__("threading").Event()
+    stop.set()  # already asked to stop before it starts
+
+    box = _run_full([("10.0000/decoy-0-a", "D")], stop_event=stop)
+
+    assert box["finished"], "an already-set stop must not hang the search"
+    assert box["interrupted"] is True
+    assert isinstance(box["positions"], dict)
+
+
 def test_bounded_queue_does_not_deadlock_on_early_termination(tmp_path, monkeypatch):
     # The OOM fix bounds the work queue. That means a producer can block on a
     # FULL queue - and if the consumers have already finished (all DOIs found)