gitea
cb2c55b791
An 8h search result must survive a transient bot outage, an api/address misroute, or a restart of either side. Make the librarian->bot result path durably at-least-once with idempotent rendering: Shared: durable_queue.DiskQueue - a dependency-free, atomically-written, one-file-per-key disk queue (unit-tested), shared by both images (added to Dockerfile.librarian; the bot already COPYs *.py). Librarian (sender): finished results go to a persistent OUTBOX before sending; delivery retries with backoff; an entry is removed only on a positive ACK; a resender thread keeps flushing the OUTBOX, so a result survives a bot outage AND a librarian restart (OUTBOX is on the state volume) - it simply keeps trying until acked. Bot (receiver): /conjurer is now idempotent and durable - each result is persisted to an INBOX before acking and only queued if its uuid was not already delivered (dropped as a duplicate) or already pending. Once the cog actually renders it, mark_delivered() records the uuid and clears the inbox, so the librarian's resends become no-ops. On startup the bot replays any accepted-but-unrendered result from the INBOX, so a bot crash mid-flight doesn't lose it. Pongs stay ephemeral. Together: the librarian keeps a result until the bot confirms it; the bot keeps it until it is on screen; duplicates never double-render. Combined with the deploy return-path fix, an expensive result no longer vanishes. Tests: unit test_durable_queue; integration test_librarian_outbox (retry/backoff, resend survives outage) and test_result_durable_delivery (persist, dedup pending, dedup delivered, replay, pong not persisted). Suite: 55 unit + 39 integration green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
100 lines
3.4 KiB
Python
100 lines
3.4 KiB
Python
"""Integration: the librarian's durable result OUTBOX + retrying delivery.
|
|
|
|
An 8-hour search result must not be lost to a transient bot outage. The result
|
|
is written to the OUTBOX before sending; delivery retries with backoff; the
|
|
entry is removed only on a positive ACK; and the resender keeps flushing the
|
|
OUTBOX (across restarts, since it is on the persistent state volume).
|
|
"""
|
|
import logging
|
|
import sys
|
|
import types
|
|
|
|
import pytest
|
|
|
|
# conjurer_librarian imports `from habanero import Crossref` at import time; the
|
|
# integration job doesn't install habanero. Stub it (we never build a real
|
|
# Librarian here).
|
|
if "habanero" not in sys.modules:
|
|
_habanero = types.ModuleType("habanero")
|
|
_habanero.Crossref = object
|
|
sys.modules["habanero"] = _habanero
|
|
|
|
import conjurer_librarian as lib # noqa: E402
|
|
from durable_queue import DiskQueue # noqa: E402
|
|
|
|
_LOG = logging.getLogger("test-outbox")
|
|
_LOG.addHandler(logging.NullHandler())
|
|
|
|
|
|
class _Resp:
|
|
def __init__(self, status_code, text=""):
|
|
self.status_code = status_code
|
|
self.text = text
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _no_backoff(monkeypatch):
|
|
# Never actually sleep during retry backoff in tests.
|
|
monkeypatch.setattr(lib.time, "sleep", lambda _s: None)
|
|
|
|
|
|
@pytest.fixture
|
|
def outbox(tmp_path, monkeypatch):
|
|
box = DiskQueue(str(tmp_path / "outbox"))
|
|
monkeypatch.setattr(lib, "_outbox", box)
|
|
return box
|
|
|
|
|
|
def test_deliver_succeeds_first_try(monkeypatch):
|
|
calls = []
|
|
monkeypatch.setattr(lib.requests, "post", lambda *a, **k: calls.append(1) or _Resp(200))
|
|
assert lib._deliver_result("u1", {"u1": {}}, _LOG, attempts=3) is True
|
|
assert len(calls) == 1 # no needless retries after a 200
|
|
|
|
|
|
def test_deliver_retries_then_succeeds(monkeypatch):
|
|
responses = iter([_Resp(503), _Resp(500), _Resp(200)])
|
|
monkeypatch.setattr(lib.requests, "post", lambda *a, **k: next(responses))
|
|
assert lib._deliver_result("u2", {"u2": {}}, _LOG, attempts=3) is True
|
|
|
|
|
|
def test_deliver_returns_false_when_all_attempts_fail(monkeypatch):
|
|
def boom(*_a, **_k):
|
|
raise lib.requests.exceptions.RequestException("bot down")
|
|
|
|
monkeypatch.setattr(lib.requests, "post", boom)
|
|
assert lib._deliver_result("u3", {"u3": {}}, _LOG, attempts=2) is False
|
|
|
|
|
|
def test_resend_once_removes_only_acked_entries(outbox, monkeypatch):
|
|
outbox.put("ok", {"ok": {}})
|
|
outbox.put("bad", {"bad": {}})
|
|
|
|
def fake_deliver(query_uuid, _payload, _logger, attempts=1):
|
|
return query_uuid == "ok"
|
|
|
|
monkeypatch.setattr(lib, "_deliver_result", fake_deliver)
|
|
lib._resend_once(_LOG)
|
|
|
|
assert not outbox.contains("ok") # acked -> dropped
|
|
assert outbox.contains("bad") # not acked -> kept for the next sweep
|
|
|
|
|
|
def test_resend_keeps_result_until_bot_recovers(outbox, monkeypatch):
|
|
# Simulate: bot down for the first sweep, up for the second. The result must
|
|
# survive the outage and be delivered on recovery.
|
|
outbox.put("u9", {"u9": {"10.1/x": {"Title": ["P"], "type": "article"}}})
|
|
state = {"up": False}
|
|
|
|
def flaky_post(*_a, **_k):
|
|
return _Resp(200) if state["up"] else _Resp(502)
|
|
|
|
monkeypatch.setattr(lib.requests, "post", flaky_post)
|
|
|
|
lib._resend_once(_LOG) # bot down
|
|
assert outbox.contains("u9") # preserved, not lost
|
|
|
|
state["up"] = True
|
|
lib._resend_once(_LOG) # bot recovered
|
|
assert not outbox.contains("u9") # now delivered and cleared
|