Librarian: 'still searching' heartbeat every 20 min

A deep scan runs for hours with nothing in the log between start and
finish, so it's impossible to tell a working search from a wedged one.
Every CONJURER_LIBRARIAN_HEARTBEAT_SECONDS (default 1200 = 20 min) a
running search now logs that it is still going, with its uuid, the search
phrase, hits so far, elapsed minutes, and a rough how-far-along.

The estimate is deliberately cheap: the producers ALREADY record a byte
offset per chunk file (the resume watermarks), and the total size is
stat()'d once per search when the chunk list is discovered. A reading is
then just a sum over ~40 ints - nothing extra happens per line, and no
cycles are spent estimating how many cycles are left.

search_for_doi takes an optional progress dict it fills with the live
positions dict + total_bytes; the librarian publishes the running search
(uuid/query/progress/live hits) while the scan runs and clears it in
finally. Nothing running => the heartbeat stays quiet.

Tests: percentage maths incl. unknown-total and >100% clamping, the
register/clear round-trip, and an end-to-end check that a real scan fills
progress so the offsets cover the chunk files on disk. Suite: 58 unit +
65 integration green.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>

conjurer_librarian/conjurer_librarian.py

@@ -173,6 +173,65 @@ def _cache_put(query, deep_search, final_result) -> None:
     _cache.prune(CACHE_MAX_ENTRIES)
 
 
+# ---- "Still alive" heartbeat for the running search ------------------------
+# A deep scan runs for hours with nothing in the log between start and finish.
+# Every HEARTBEAT_SECONDS the running search says it is still going, with its
+# uuid, the phrase, and a ROUGH how-far-along. The estimate is deliberately
+# cheap: producers already record a byte offset per chunk file, and the total
+# size is stat()'d once at search start - so it costs a sum over ~40 ints.
+HEARTBEAT_SECONDS = int(_env("CONJURER_LIBRARIAN_HEARTBEAT_SECONDS", "1200"))  # 20 min
+_current_search: Dict[str, object] = {}
+_current_lock = threading.Lock()
+
+
+def _set_current_search(uuid, query, progress, live_results) -> None:
+    with _current_lock:
+        _current_search.clear()
+        _current_search.update({
+            "uuid": str(uuid), "query": str(query), "started": time.monotonic(),
+            "progress": progress, "live": live_results,
+        })
+
+
+def _clear_current_search() -> None:
+    with _current_lock:
+        _current_search.clear()
+
+
+def _progress_summary(progress):
+    """(done_bytes, total_bytes, percent) from a live progress dict. Cheap: a
+    sum over one int per chunk file. Percent is 0.0 when the total is unknown."""
+    progress = progress or {}
+    positions = progress.get("positions") or {}
+    total = progress.get("total_bytes") or 0
+    done = sum(positions.values())
+    if total > 0:
+        done = min(done, total)  # a partially-buffered tail can nudge past 100%
+        return done, total, 100.0 * done / total
+    return done, total, 0.0
+
+
+def search_heartbeat(app_logger) -> None:
+    """Log a 'still searching' line every HEARTBEAT_SECONDS while one runs."""
+    while not SHUTDOWN_EVENT.wait(HEARTBEAT_SECONDS):
+        try:
+            with _current_lock:
+                snapshot = dict(_current_search) if _current_search else None
+            if not snapshot:
+                continue  # nothing running - stay quiet
+            done, total, percent = _progress_summary(snapshot.get("progress"))
+            app_logger.info(
+                "SEARCH ALIVE %s | '%s' | ~%.1f%% przeskanowane (%.2f/%.2f GB, "
+                "~%.2f GB do końca) | %d trafień | %.0f min",
+                snapshot["uuid"], snapshot["query"], percent,
+                done / 1e9, total / 1e9, max(0, total - done) / 1e9,
+                len(snapshot.get("live") or []),
+                (time.monotonic() - snapshot["started"]) / 60.0,
+            )
+        except Exception as exc:  # pylint: disable=broad-exception-caught
+            app_logger.warning("Heartbeat tick failed: %s", exc)
+
+
 def _forget_search(uuid) -> None:
     """A search is fully done (or abandoned): drop its persisted request and any
     checkpoint so it is never replayed or resumed again."""
@@ -516,10 +575,18 @@ class Librarian(object):
         dois = []
         for item, value in refined_result.items():
             dois.append([item, value])
-        result, positions, interrupted = await asyncio.to_thread(
-            search_bot.search_for_doi,
-            dois, self.live_results, self.app.logger, SHUTDOWN_EVENT, resume,
-        )
+        # Publish this scan as "the running search" so the heartbeat can report
+        # it; cleared in finally so a finished/crashed scan never lingers there.
+        progress = {}
+        _set_current_search(self.uuid, self.query, progress, self.live_results)
+        try:
+            result, positions, interrupted = await asyncio.to_thread(
+                search_bot.search_for_doi,
+                dois, self.live_results, self.app.logger, SHUTDOWN_EVENT, resume,
+                progress,
+            )
+        finally:
+            _clear_current_search()
         if interrupted:
             # Graceful shutdown hit mid-scan: checkpoint found-so-far + per-file
             # resume offsets + the DOI list, so a restart continues instead of
@@ -951,6 +1018,10 @@ if __name__ == "__main__":
     threads.append(
         threading.Thread(target=outbox_resender, args=(app.logger,), daemon=True)
     )
+    # "Still searching" heartbeat, so an hours-long scan isn't radio silence.
+    threads.append(
+        threading.Thread(target=search_heartbeat, args=(app.logger,), daemon=True)
+    )
     for worker in threads:
         worker.start()
     # Re-enqueue searches that were accepted/in-progress before the last stop.