Librarian: answer each query back to the bot that sent it

So one librarian can serve several bots (test + deploy) instead of firing
every result/pong at a single static CONJURER_MAIN_BOT.

* The bot includes its own callback address (CONJURER_SELF_CALLBACK) in
  every /query and /ping.
* The librarian stores that callback with the query (persisted with the
  request, so a replay after restart still answers the right bot) and, for
  results, in the OUTBOX entry ({target, payload}) so the resender delivers
  to the origin bot even across a librarian restart.
* Pongs go back to the pinging bot too - otherwise a second bot's health
  check would be ponged to the first and always time out, so it could
  never enable its librarian cog.
* Empty callback falls back to MAIN_BOT_ADDRESS, and a legacy OUTBOX entry
  (raw payload, pre-callback) is still delivered to the default bot, so the
  upgrade is seamless.

Tests: per-origin result delivery + legacy-shape fallback (outbox),
busy/idle pong routed to the callback bot vs default (lifecycle). Suite:
58 unit + 52 integration green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

tests/integration/test_librarian_outbox.py

@@ -46,16 +46,24 @@ def outbox(tmp_path, monkeypatch):
 
 
 def test_deliver_succeeds_first_try(monkeypatch):
-    calls = []
-    monkeypatch.setattr(lib.requests, "post", lambda *a, **k: calls.append(1) or _Resp(200))
-    assert lib._deliver_result("u1", {"u1": {}}, _LOG, attempts=3) is True
-    assert len(calls) == 1  # no needless retries after a 200
+    urls = []
+    monkeypatch.setattr(lib.requests, "post", lambda url, **k: urls.append(url) or _Resp(200))
+    assert lib._deliver_result("http://bot-a:5000", "u1", {"u1": {}}, _LOG, attempts=3) is True
+    assert len(urls) == 1  # no needless retries after a 200
+    assert urls[0] == "http://bot-a:5000" + lib.SEND_RESULTS  # to the origin bot
+
+
+def test_deliver_falls_back_to_main_bot_when_no_target(monkeypatch):
+    urls = []
+    monkeypatch.setattr(lib.requests, "post", lambda url, **k: urls.append(url) or _Resp(200))
+    assert lib._deliver_result("", "u1b", {"u1b": {}}, _LOG, attempts=1) is True
+    assert urls[0] == lib.MAIN_BOT_ADDRESS + lib.SEND_RESULTS  # empty target -> default
 
 
 def test_deliver_retries_then_succeeds(monkeypatch):
     responses = iter([_Resp(503), _Resp(500), _Resp(200)])
     monkeypatch.setattr(lib.requests, "post", lambda *a, **k: next(responses))
-    assert lib._deliver_result("u2", {"u2": {}}, _LOG, attempts=3) is True
+    assert lib._deliver_result("http://bot", "u2", {"u2": {}}, _LOG, attempts=3) is True
 
 
 def test_deliver_returns_false_when_all_attempts_fail(monkeypatch):
@@ -63,27 +71,47 @@ def test_deliver_returns_false_when_all_attempts_fail(monkeypatch):
         raise lib.requests.exceptions.RequestException("bot down")
 
     monkeypatch.setattr(lib.requests, "post", boom)
-    assert lib._deliver_result("u3", {"u3": {}}, _LOG, attempts=2) is False
+    assert lib._deliver_result("http://bot", "u3", {"u3": {}}, _LOG, attempts=2) is False
 
 
-def test_resend_once_removes_only_acked_entries(outbox, monkeypatch):
-    outbox.put("ok", {"ok": {}})
-    outbox.put("bad", {"bad": {}})
+def test_resend_delivers_each_result_to_its_own_origin_bot(outbox, monkeypatch):
+    outbox.put("ok", {"target": "http://bot-a:5000", "payload": {"ok": {}}})
+    outbox.put("bad", {"target": "http://bot-b:5000", "payload": {"bad": {}}})
+    seen = []
 
-    def fake_deliver(query_uuid, _payload, _logger, attempts=1):
+    def fake_deliver(target, query_uuid, _payload, _logger, attempts=1):
+        seen.append((target, query_uuid))
         return query_uuid == "ok"
 
     monkeypatch.setattr(lib, "_deliver_result", fake_deliver)
     lib._resend_once(_LOG)
 
+    assert ("http://bot-a:5000", "ok") in seen   # delivered to A's address
+    assert ("http://bot-b:5000", "bad") in seen  # attempted to B's address
     assert not outbox.contains("ok")   # acked -> dropped
     assert outbox.contains("bad")      # not acked -> kept for the next sweep
 
 
+def test_resend_handles_legacy_entry_shape(outbox, monkeypatch):
+    # An OUTBOX entry from before per-origin callbacks (raw payload, no target)
+    # must still be delivered - to the default bot.
+    outbox.put("old", {"old": {"10.1/x": {"Title": ["P"], "type": "a"}}})
+    seen = []
+
+    def fake_deliver(target, query_uuid, _payload, _logger, attempts=1):
+        seen.append((target, query_uuid))
+        return True
+
+    monkeypatch.setattr(lib, "_deliver_result", fake_deliver)
+    lib._resend_once(_LOG)
+    assert seen == [("", "old")]  # empty target -> _deliver_result uses MAIN_BOT
+    assert not outbox.contains("old")
+
+
 def test_resend_keeps_result_until_bot_recovers(outbox, monkeypatch):
     # Simulate: bot down for the first sweep, up for the second. The result must
     # survive the outage and be delivered on recovery.
-    outbox.put("u9", {"u9": {"10.1/x": {"Title": ["P"], "type": "article"}}})
+    outbox.put("u9", {"target": "http://bot", "payload": {"u9": {"10.1/x": {"Title": ["P"], "type": "article"}}}})
     state = {"up": False}
 
     def flaky_post(*_a, **_k):

tests/integration/test_librarian_query_lifecycle.py

@@ -58,15 +58,19 @@ def test_ping_idle_routes_through_internal_queue(monkeypatch):
     monkeypatch.setattr(lib.requests, "post", lambda *a, **k: posted.append((a, k)))
     client = _client()
 
-    resp = client.post("/ping", json={"UUID": "ping-idle"})
+    resp = client.post("/ping", json={"UUID": "ping-idle", "callback": "http://bot-a:5000"})
 
     assert resp.status_code == 200
-    # Idle => it went onto the internal queue for the worker, NOT posted directly.
+    # Idle => it went onto the internal queue for the worker (carrying the
+    # callback so the worker pongs the right bot), NOT posted directly.
     assert posted == []
-    assert lib.librarian_queue.get_nowait() == {"__ping__": "ping-idle"}
+    assert lib.librarian_queue.get_nowait() == {
+        "__ping__": "ping-idle",
+        "callback": "http://bot-a:5000",
+    }
 
 
-def test_ping_while_busy_pongs_directly_without_queue(monkeypatch):
+def test_ping_while_busy_pongs_directly_to_the_pinging_bot(monkeypatch):
     _reset()
     lib.worker_busy.set()  # a search is grinding
     posted = []
@@ -81,15 +85,33 @@ def test_ping_while_busy_pongs_directly_without_queue(monkeypatch):
     monkeypatch.setattr(lib.requests, "post", fake_post)
     client = _client()
 
-    resp = client.post("/ping", json={"UUID": "ping-busy"})
+    resp = client.post("/ping", json={"UUID": "ping-busy", "callback": "http://bot-b:5000"})
 
     assert resp.status_code == 200
-    # Busy => direct pong back to the bot, and NOTHING queued (it would only wait
-    # behind the long search).
+    # Busy => direct pong, NOTHING queued, and it goes to the CALLBACK bot (not
+    # the static default) so a shared librarian health-checks each bot correctly.
     assert lib.librarian_queue.empty()
     assert len(posted) == 1
     assert posted[0]["json"] == {"__pong__": "ping-busy"}
-    assert posted[0]["url"].endswith(lib.SEND_RESULTS)
+    assert posted[0]["url"] == "http://bot-b:5000" + lib.SEND_RESULTS
+
+
+def test_ping_without_callback_pongs_to_default_bot(monkeypatch):
+    _reset()
+    lib.worker_busy.set()
+    posted = []
+
+    class _Resp:
+        status_code = 200
+
+    monkeypatch.setattr(
+        lib.requests, "post",
+        lambda url, **k: posted.append(url) or _Resp(),
+    )
+    client = _client()
+    client.post("/ping", json={"UUID": "ping-nocb"})  # no callback
+
+    assert posted == [lib.MAIN_BOT_ADDRESS + lib.SEND_RESULTS]
 
 
 def test_query_status_enforces_api_key():