gitea
9f22dbf94b
CI / compile (pull_request) Successful in 18s
CI / unit (pull_request) Successful in 39s
CI / integration (pull_request) Failing after 1m2s
CI / compile (push) Successful in 14s
CI / unit (push) Successful in 34s
CI / integration (push) Successful in 37s
build / build (push) Successful in 33s
A deep scan runs for hours with nothing in the log between start and finish, so it's impossible to tell a working search from a wedged one. Every CONJURER_LIBRARIAN_HEARTBEAT_SECONDS (default 1200 = 20 min) a running search now logs that it is still going, with its uuid, the search phrase, hits so far, elapsed minutes, and a rough how-far-along. The estimate is deliberately cheap: the producers ALREADY record a byte offset per chunk file (the resume watermarks), and the total size is stat()'d once per search when the chunk list is discovered. A reading is then just a sum over ~40 ints - nothing extra happens per line, and no cycles are spent estimating how many cycles are left. search_for_doi takes an optional progress dict it fills with the live positions dict + total_bytes; the librarian publishes the running search (uuid/query/progress/live hits) while the scan runs and clears it in finally. Nothing running => the heartbeat stays quiet. Tests: percentage maths incl. unknown-total and >100% clamping, the register/clear round-trip, and an end-to-end check that a real scan fills progress so the offsets cover the chunk files on disk. Suite: 58 unit + 65 integration green. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
85 lines
3.1 KiB
Python
85 lines
3.1 KiB
Python
"""Integration: the 'still searching' heartbeat and its cheap progress estimate.
|
|
|
|
The estimate must stay free: producers already record a byte offset per chunk
|
|
file and the total is stat()'d once, so a reading is just a sum over ~40 ints.
|
|
"""
|
|
import logging
|
|
import sys
|
|
import types
|
|
|
|
import pytest
|
|
|
|
if "habanero" not in sys.modules:
|
|
_habanero = types.ModuleType("habanero")
|
|
_habanero.Crossref = object
|
|
sys.modules["habanero"] = _habanero
|
|
|
|
import conjurer_librarian as lib # noqa: E402
|
|
import search_bot # noqa: E402
|
|
|
|
_LOG = logging.getLogger("test-heartbeat")
|
|
_LOG.addHandler(logging.NullHandler())
|
|
|
|
|
|
def test_progress_summary_percentages():
|
|
progress = {"positions": {"0_chunk.txt": 250, "1_chunk.txt": 250}, "total_bytes": 1000}
|
|
done, total, percent = lib._progress_summary(progress)
|
|
assert (done, total) == (500, 1000)
|
|
assert percent == pytest.approx(50.0)
|
|
|
|
|
|
def test_progress_summary_unknown_total_is_zero_percent():
|
|
done, total, percent = lib._progress_summary({"positions": {"a": 10}})
|
|
assert (done, total, percent) == (10, 0, 0.0)
|
|
|
|
|
|
def test_progress_summary_handles_empty_and_none():
|
|
assert lib._progress_summary(None) == (0, 0, 0.0)
|
|
assert lib._progress_summary({}) == (0, 0, 0.0)
|
|
|
|
|
|
def test_progress_summary_is_clamped_to_100():
|
|
# A partially-buffered tail can push the summed offsets past the total.
|
|
_done, _total, percent = lib._progress_summary(
|
|
{"positions": {"a": 1500}, "total_bytes": 1000}
|
|
)
|
|
assert percent == pytest.approx(100.0)
|
|
|
|
|
|
def test_current_search_registration_round_trip():
|
|
progress = {"positions": {"a": 5}, "total_bytes": 10}
|
|
live = [{"DOI": "10.1/x"}]
|
|
lib._set_current_search("uuid-1", "kwas foliowy", progress, live)
|
|
with lib._current_lock:
|
|
snapshot = dict(lib._current_search)
|
|
assert snapshot["uuid"] == "uuid-1"
|
|
assert snapshot["query"] == "kwas foliowy"
|
|
assert lib._progress_summary(snapshot["progress"])[2] == pytest.approx(50.0)
|
|
lib._clear_current_search()
|
|
with lib._current_lock:
|
|
assert not lib._current_search
|
|
|
|
|
|
def test_search_fills_progress_with_live_positions_and_total(tmp_path, monkeypatch):
|
|
# End to end against the real scan: total_bytes matches the chunk files on
|
|
# disk, and once finished the recorded offsets cover them.
|
|
monkeypatch.setattr(search_bot, "DATABASE_PATH", str(tmp_path) + "/")
|
|
(tmp_path / "0_chunk.txt").write_text("10.1/a\n10.1/b\n", encoding="utf-8")
|
|
(tmp_path / "1_chunk.txt").write_text("10.1/c\n", encoding="utf-8")
|
|
expected_total = sum(
|
|
(tmp_path / name).stat().st_size for name in ("0_chunk.txt", "1_chunk.txt")
|
|
)
|
|
|
|
progress = {}
|
|
result, _positions, _interrupted = search_bot.search_for_doi(
|
|
[("10.1/c", "DATA")], [], _LOG, progress=progress
|
|
)
|
|
|
|
assert progress["total_bytes"] == expected_total
|
|
assert progress["chunk_files"] == 2
|
|
done, total, percent = lib._progress_summary(progress)
|
|
assert total == expected_total
|
|
assert done == expected_total # whole DB scanned
|
|
assert percent == pytest.approx(100.0)
|
|
assert [r for r in result if r["DOI"] == "10.1/c" and r["exists"]]
|