gitea
44b7298a15
An 8h search result must survive a transient bot outage, an api/address misroute, or a restart of either side. Make the librarian->bot result path durably at-least-once with idempotent rendering: Shared: durable_queue.DiskQueue - a dependency-free, atomically-written, one-file-per-key disk queue (unit-tested), shared by both images (added to Dockerfile.librarian; the bot already COPYs *.py). Librarian (sender): finished results go to a persistent OUTBOX before sending; delivery retries with backoff; an entry is removed only on a positive ACK; a resender thread keeps flushing the OUTBOX, so a result survives a bot outage AND a librarian restart (OUTBOX is on the state volume) - it simply keeps trying until acked. Bot (receiver): /conjurer is now idempotent and durable - each result is persisted to an INBOX before acking and only queued if its uuid was not already delivered (dropped as a duplicate) or already pending. Once the cog actually renders it, mark_delivered() records the uuid and clears the inbox, so the librarian's resends become no-ops. On startup the bot replays any accepted-but-unrendered result from the INBOX, so a bot crash mid-flight doesn't lose it. Pongs stay ephemeral. Together: the librarian keeps a result until the bot confirms it; the bot keeps it until it is on screen; duplicates never double-render. Combined with the deploy return-path fix, an expensive result no longer vanishes. Tests: unit test_durable_queue; integration test_librarian_outbox (retry/backoff, resend survives outage) and test_result_durable_delivery (persist, dedup pending, dedup delivered, replay, pong not persisted). Suite: 55 unit + 39 integration green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
77 lines
2.4 KiB
Python
77 lines
2.4 KiB
Python
"""Unit tests for the disk-backed durable queue used by result delivery."""
|
|
import json
|
|
|
|
from durable_queue import DiskQueue
|
|
|
|
|
|
def test_put_contains_remove(tmp_path):
|
|
q = DiskQueue(str(tmp_path / "q"))
|
|
assert not q.contains("a")
|
|
q.put("a", {"hello": 1})
|
|
assert q.contains("a")
|
|
q.remove("a")
|
|
assert not q.contains("a")
|
|
q.remove("a") # idempotent - no error on missing
|
|
|
|
|
|
def test_put_overwrites_and_roundtrips_payload(tmp_path):
|
|
q = DiskQueue(str(tmp_path / "q"))
|
|
q.put("uuid-1", {"uuid-1": {"10.1/x": {"Title": ["P"], "type": "article"}}})
|
|
q.put("uuid-1", {"uuid-1": {"changed": True}})
|
|
items = q.items()
|
|
assert len(items) == 1
|
|
key, payload, _ts = items[0]
|
|
assert key == "uuid-1"
|
|
assert payload == {"uuid-1": {"changed": True}}
|
|
|
|
|
|
def test_items_sorted_oldest_first(tmp_path, monkeypatch):
|
|
q = DiskQueue(str(tmp_path / "q"))
|
|
import durable_queue
|
|
|
|
times = iter([100.0, 200.0, 300.0])
|
|
monkeypatch.setattr(durable_queue.time, "time", lambda: next(times))
|
|
q.put("c", {})
|
|
q.put("a", {})
|
|
q.put("b", {})
|
|
assert [k for k, _p, _ts in q.items()] == ["c", "a", "b"]
|
|
|
|
|
|
def test_corrupt_file_is_skipped_not_fatal(tmp_path):
|
|
directory = tmp_path / "q"
|
|
q = DiskQueue(str(directory))
|
|
q.put("good", {"ok": 1})
|
|
(directory / "broken.json").write_text("{ this is not json", encoding="utf-8")
|
|
keys = q.keys()
|
|
assert keys == ["good"] # broken file skipped, good one survives
|
|
|
|
|
|
def test_prune_keeps_newest(tmp_path, monkeypatch):
|
|
q = DiskQueue(str(tmp_path / "q"))
|
|
import durable_queue
|
|
|
|
times = iter([1.0, 2.0, 3.0, 4.0, 5.0])
|
|
monkeypatch.setattr(durable_queue.time, "time", lambda: next(times))
|
|
for key in ("k1", "k2", "k3", "k4", "k5"):
|
|
q.put(key, {})
|
|
dropped = q.prune(2)
|
|
assert dropped == 3
|
|
assert set(q.keys()) == {"k4", "k5"}
|
|
|
|
|
|
def test_atomic_write_leaves_no_tmp_files(tmp_path):
|
|
directory = tmp_path / "q"
|
|
q = DiskQueue(str(directory))
|
|
q.put("a", {"x": 1})
|
|
leftover = [p.name for p in directory.iterdir() if p.suffix == ".tmp"]
|
|
assert leftover == []
|
|
|
|
|
|
def test_key_with_slashes_is_sanitised(tmp_path):
|
|
q = DiskQueue(str(tmp_path / "q"))
|
|
q.put("../../etc/passwd", {"evil": 1})
|
|
# Stays inside the directory (no traversal), and round-trips by key.
|
|
files = list((tmp_path / "q").iterdir())
|
|
assert all(f.parent == tmp_path / "q" for f in files)
|
|
assert q.items()[0][1] == {"evil": 1}
|