conjurer/tests/integration/test_librarian_outbox.py

"""Integration: the librarian's durable result OUTBOX + retrying delivery.

An 8-hour search result must not be lost to a transient bot outage. The result
is written to the OUTBOX before sending; delivery retries with backoff; the
entry is removed only on a positive ACK; and the resender keeps flushing the
OUTBOX (across restarts, since it is on the persistent state volume).
"""
import logging
import sys
import types

import pytest

# conjurer_librarian imports `from habanero import Crossref` at import time; the
# integration job doesn't install habanero. Stub it (we never build a real
# Librarian here).
if "habanero" not in sys.modules:
    _habanero = types.ModuleType("habanero")
    _habanero.Crossref = object
    sys.modules["habanero"] = _habanero

import conjurer_librarian as lib  # noqa: E402
from durable_queue import DiskQueue  # noqa: E402

_LOG = logging.getLogger("test-outbox")
_LOG.addHandler(logging.NullHandler())


class _Resp:
    def __init__(self, status_code, text=""):
        self.status_code = status_code
        self.text = text


@pytest.fixture(autouse=True)
def _no_backoff(monkeypatch):
    # Never actually sleep during retry backoff in tests.
    monkeypatch.setattr(lib.time, "sleep", lambda _s: None)


@pytest.fixture
def outbox(tmp_path, monkeypatch):
    box = DiskQueue(str(tmp_path / "outbox"))
    monkeypatch.setattr(lib, "_outbox", box)
    return box


def test_deliver_succeeds_first_try(monkeypatch):
    urls = []
    monkeypatch.setattr(lib.requests, "post", lambda url, **k: urls.append(url) or _Resp(200))
    assert lib._deliver_result("http://bot-a:5000", "u1", {"u1": {}}, _LOG, attempts=3) is True
    assert len(urls) == 1  # no needless retries after a 200
    assert urls[0] == "http://bot-a:5000" + lib.SEND_RESULTS  # to the origin bot


def test_deliver_falls_back_to_main_bot_when_no_target(monkeypatch):
    urls = []
    monkeypatch.setattr(lib.requests, "post", lambda url, **k: urls.append(url) or _Resp(200))
    assert lib._deliver_result("", "u1b", {"u1b": {}}, _LOG, attempts=1) is True
    assert urls[0] == lib.MAIN_BOT_ADDRESS + lib.SEND_RESULTS  # empty target -> default


def test_deliver_retries_then_succeeds(monkeypatch):
    responses = iter([_Resp(503), _Resp(500), _Resp(200)])
    monkeypatch.setattr(lib.requests, "post", lambda *a, **k: next(responses))
    assert lib._deliver_result("http://bot", "u2", {"u2": {}}, _LOG, attempts=3) is True


def test_deliver_returns_false_when_all_attempts_fail(monkeypatch):
    def boom(*_a, **_k):
        raise lib.requests.exceptions.RequestException("bot down")

    monkeypatch.setattr(lib.requests, "post", boom)
    assert lib._deliver_result("http://bot", "u3", {"u3": {}}, _LOG, attempts=2) is False


def test_resend_delivers_each_result_to_its_own_origin_bot(outbox, monkeypatch):
    outbox.put("ok", {"target": "http://bot-a:5000", "payload": {"ok": {}}})
    outbox.put("bad", {"target": "http://bot-b:5000", "payload": {"bad": {}}})
    seen = []

    def fake_deliver(target, query_uuid, _payload, _logger, attempts=1):
        seen.append((target, query_uuid))
        return query_uuid == "ok"

    monkeypatch.setattr(lib, "_deliver_result", fake_deliver)
    lib._resend_once(_LOG)

    assert ("http://bot-a:5000", "ok") in seen   # delivered to A's address
    assert ("http://bot-b:5000", "bad") in seen  # attempted to B's address
    assert not outbox.contains("ok")   # acked -> dropped
    assert outbox.contains("bad")      # not acked -> kept for the next sweep


def test_resend_handles_legacy_entry_shape(outbox, monkeypatch):
    # An OUTBOX entry from before per-origin callbacks (raw payload, no target)
    # must still be delivered - to the default bot.
    outbox.put("old", {"old": {"10.1/x": {"Title": ["P"], "type": "a"}}})
    seen = []

    def fake_deliver(target, query_uuid, _payload, _logger, attempts=1):
        seen.append((target, query_uuid))
        return True

    monkeypatch.setattr(lib, "_deliver_result", fake_deliver)
    lib._resend_once(_LOG)
    assert seen == [("", "old")]  # empty target -> _deliver_result uses MAIN_BOT
    assert not outbox.contains("old")


def test_resend_keeps_result_until_bot_recovers(outbox, monkeypatch):
    # Simulate: bot down for the first sweep, up for the second. The result must
    # survive the outage and be delivered on recovery.
    outbox.put("u9", {"target": "http://bot", "payload": {"u9": {"10.1/x": {"Title": ["P"], "type": "article"}}}})
    state = {"up": False}

    def flaky_post(*_a, **_k):
        return _Resp(200) if state["up"] else _Resp(502)

    monkeypatch.setattr(lib.requests, "post", flaky_post)

    lib._resend_once(_LOG)               # bot down
    assert outbox.contains("u9")         # preserved, not lost

    state["up"] = True
    lib._resend_once(_LOG)               # bot recovered
    assert not outbox.contains("u9")     # now delivered and cleared