gitea
031c1f8aea
CI / compile (pull_request) Successful in 10s
CI / unit (pull_request) Successful in 20s
CI / integration (pull_request) Successful in 15s
CI / compile (push) Successful in 29s
CI / unit (push) Successful in 30s
CI / integration (push) Successful in 21s
build / build (push) Failing after 7s
Two field-reported robustness gaps on top of the hang fix. 1) A stray non-UTF-8 byte in a chunk (0x96 in the report) raised UnicodeDecodeError from readline() - which is a ValueError, so the earlier `except OSError` did NOT catch it. The finally-sentinel meant no hang, but the producer died mid-file with a loud traceback and every DOI after the bad byte went unsearched. Now chunks are opened with errors="replace" (bad bytes become U+FFFD; DOIs are ASCII so a match is never affected) so the read runs to EOF, and the producer's except is broadened from OSError to Exception so no per-file error can ever crash the thread - it's logged and the sentinel still fires. 2) The result-send back to the bot (BackgroundTaskSearch._run) now logs exactly what goes out - target URL, uuid, DOI count and the DOI list - so the librarian log plainly shows a result was sent and what was in it. And a failed POST is no longer fatal: a RequestException used to propagate out of the worker loop and kill the thread, stalling every future query until restart; it's now caught and logged, and a non-200 from the bot is logged as a warning. Verified: tests/unit/test_search_bot.py gains a case writing a chunk with a 0x96 byte before a valid DOI and asserting that DOI is still found (file read to completion, not aborted). All 5 search_bot unit tests pass. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
107 lines
4.5 KiB
Python
107 lines
4.5 KiB
Python
"""Unit tests for the librarian DOI search - specifically that it always
|
|
terminates.
|
|
|
|
The producer/consumer search used to hang whenever the number of chunk files it
|
|
was told to read (MAXTHREADS) did not exactly match the files on disk: too few
|
|
and it silently skipped trailing chunks, too many and a producer pointed at a
|
|
missing file crashed before emitting its sentinel, starving the consumers'
|
|
termination count forever. These tests pin down the fix: chunk files are
|
|
auto-discovered, the sentinel threshold equals the number of producers actually
|
|
started, and every producer emits its sentinel even on error.
|
|
"""
|
|
import logging
|
|
import threading
|
|
|
|
import search_bot
|
|
|
|
_LOG = logging.getLogger("test-search-bot")
|
|
_LOG.addHandler(logging.NullHandler())
|
|
|
|
|
|
def _write_chunks(directory, count, target=None, target_index=None):
|
|
"""Create <n>_chunk.txt files; optionally drop `target` into one of them."""
|
|
for n in range(count):
|
|
lines = [f"10.0000/decoy-{n}-a\n", f"10.0000/decoy-{n}-b\n"]
|
|
if target is not None and n == target_index:
|
|
lines.append(target + "\n")
|
|
(directory / f"{n}_chunk.txt").write_text("".join(lines), encoding="utf-8")
|
|
|
|
|
|
def _run_bounded(dois, timeout=20):
|
|
"""Run search_for_doi in a thread; return (finished_in_time, result)."""
|
|
box = {}
|
|
live = []
|
|
worker = threading.Thread(
|
|
target=lambda: box.update(result=search_bot.search_for_doi(dois, live, _LOG)),
|
|
daemon=True,
|
|
)
|
|
worker.start()
|
|
worker.join(timeout)
|
|
return (not worker.is_alive()), box.get("result")
|
|
|
|
|
|
def test_finds_doi_in_trailing_chunk(tmp_path, monkeypatch):
|
|
# Target lives in the LAST chunk - the one the old MAXTHREADS=N-too-low would
|
|
# never have read. Auto-discovery must read every chunk present.
|
|
monkeypatch.setattr(search_bot, "DATABASE_PATH", str(tmp_path) + "/")
|
|
target = "10.1234/target.in.trailing.chunk"
|
|
_write_chunks(tmp_path, count=6, target=target, target_index=5)
|
|
|
|
finished, result = _run_bounded([(target, "DATA"), ("10.9999/absent", "DATA")])
|
|
|
|
assert finished, "search hung instead of terminating"
|
|
hit = [r for r in result if r["DOI"] == target and r["exists"]]
|
|
assert hit, "DOI in the trailing chunk was not found"
|
|
|
|
|
|
def test_terminates_when_a_chunk_is_unreadable(tmp_path, monkeypatch):
|
|
# A chunk that exists at discovery time but cannot be opened (here: it is a
|
|
# directory) makes its producer raise. The finally-sentinel must still fire
|
|
# so the consumers' count completes and the search does not deadlock.
|
|
monkeypatch.setattr(search_bot, "DATABASE_PATH", str(tmp_path) + "/")
|
|
_write_chunks(tmp_path, count=3)
|
|
(tmp_path / "9_chunk.txt").mkdir() # discovered as a chunk, un-openable
|
|
|
|
finished, _ = _run_bounded([("10.0000/decoy-0-a", "DATA")])
|
|
|
|
assert finished, "an unreadable chunk deadlocked the search"
|
|
|
|
|
|
def test_no_chunks_returns_immediately(tmp_path, monkeypatch):
|
|
# Empty database dir: return an (all-not-found) result at once, never hang.
|
|
monkeypatch.setattr(search_bot, "DATABASE_PATH", str(tmp_path) + "/")
|
|
|
|
finished, result = _run_bounded([("10.0/x", "DATA")], timeout=10)
|
|
|
|
assert finished
|
|
assert result == [{"DOI": "10.0/x", "exists": False, "data": "DATA"}]
|
|
|
|
|
|
def test_survives_invalid_utf8_byte_and_still_finds_later_doi(tmp_path, monkeypatch):
|
|
# A chunk with a stray non-UTF-8 byte (0x96, the one from the field report)
|
|
# must not crash the producer or abort the file mid-read: DOIs AFTER the bad
|
|
# byte still have to be found.
|
|
monkeypatch.setattr(search_bot, "DATABASE_PATH", str(tmp_path) + "/")
|
|
target = "10.1234/after.the.bad.byte"
|
|
(tmp_path / "0_chunk.txt").write_bytes(
|
|
b"10.0000/before\n" + b"\x96 broken \x96 line \x96\n" + target.encode() + b"\n"
|
|
)
|
|
|
|
finished, result = _run_bounded([(target, "DATA")])
|
|
|
|
assert finished, "an invalid UTF-8 byte hung or crashed the search"
|
|
hit = [r for r in result if r["DOI"] == target and r["exists"]]
|
|
assert hit, "DOI after the bad byte was not found - the file was aborted mid-read"
|
|
|
|
|
|
def test_discover_chunk_files_sorted_numerically(tmp_path, monkeypatch):
|
|
monkeypatch.setattr(search_bot, "DATABASE_PATH", str(tmp_path) + "/")
|
|
for n in (0, 2, 10, 1):
|
|
(tmp_path / f"{n}_chunk.txt").write_text("x\n", encoding="utf-8")
|
|
(tmp_path / "notes.txt").write_text("ignore me\n", encoding="utf-8")
|
|
|
|
found = search_bot.discover_chunk_files(_LOG)
|
|
|
|
# Numeric order (10 after 2, not lexicographic), and non-chunk files ignored.
|
|
assert found == ["0_chunk.txt", "1_chunk.txt", "2_chunk.txt", "10_chunk.txt"]
|