Compare commits

..

1 Commits

Author SHA1 Message Date
gitea 41050fa879 ci: promote bot to a 'deploy' image channel only on [deploy] trigger
CI / compile (pull_request) Successful in 8s
CI / unit (pull_request) Successful in 26s
CI / integration (pull_request) Successful in 28s
The production bot tracks a separate image, conjurer-bot-deploy, so it
updates only when a commit message contains [deploy] - the test bot and
librarian keep updating on every build. On a [deploy] commit the already-
built conjurer-bot:<sha> is re-tagged (same bytes, no rebuild) and pushed
as conjurer-bot-deploy:<sha>; the deploy repo's image-updater tracks that.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-08-03 20:10:21 +02:00
2 changed files with 0 additions and 118 deletions
-48
View File
@@ -14,7 +14,6 @@ Functions:
"""
import asyncio
import hashlib
import json
import logging
import os
@@ -136,42 +135,6 @@ CHECKPOINT_DIR = _env("CONJURER_LIBRARIAN_CHECKPOINTS", os.path.join(lib_paths.S
_requests = DiskQueue(REQUESTS_DIR)
_checkpoints = DiskQueue(CHECKPOINT_DIR)
# Simple result cache: a repeat of the same query (normalised) skips the whole
# Crossref + DB-scan and returns the stored hits. Disk-backed so it survives a
# restart, TTL'd, and size-bounded. Set CONJURER_LIBRARIAN_CACHE_TTL=0 to disable.
CACHE_DIR = _env("CONJURER_LIBRARIAN_CACHE", os.path.join(lib_paths.STATE_DIR, "cache"))
CACHE_TTL_SECONDS = int(_env("CONJURER_LIBRARIAN_CACHE_TTL", str(7 * 24 * 3600)))
CACHE_MAX_ENTRIES = int(_env("CONJURER_LIBRARIAN_CACHE_MAX", "500"))
_cache = DiskQueue(CACHE_DIR)
def _cache_key(query, deep_search) -> str:
"""Stable key for a query: whitespace-normalised, case-insensitive, and
scoped by deep vs shallow (they return different result sets)."""
normalised = " ".join(str(query).lower().split())
return hashlib.sha256(f"{int(bool(deep_search))}:{normalised}".encode("utf-8")).hexdigest()
def _cache_get(query, deep_search):
"""Return the cached final_result for this query, or None on miss/expiry."""
if CACHE_TTL_SECONDS <= 0:
return None
entry = _cache.get(_cache_key(query, deep_search))
if not entry or entry.get("expires", 0) < time.time():
return None
return entry.get("final_result")
def _cache_put(query, deep_search, final_result) -> None:
"""Store a completed search's hits, with a TTL, and bound the cache size."""
if CACHE_TTL_SECONDS <= 0:
return
_cache.put(
_cache_key(query, deep_search),
{"query": str(query), "final_result": final_result, "expires": time.time() + CACHE_TTL_SECONDS},
)
_cache.prune(CACHE_MAX_ENTRIES)
def _forget_search(uuid) -> None:
"""A search is fully done (or abandoned): drop its persisted request and any
@@ -573,15 +536,6 @@ class Librarian(object):
"positions": checkpoint.get("positions", {}),
}
else:
# Cache hit: an identical (normalised) query ran recently - return its
# stored hits and skip Crossref + the whole DB scan entirely.
cached = _cache_get(self.query, deep_search)
if cached is not None:
self.final_result = cached
self.app.logger.info(
"Cache HIT for %s (%d hits): %s", self.uuid, len(cached), self.query
)
return self.final_result
self.app.logger.info(f"Search started {self.uuid}")
cr_result = await self.search_crossref(query=self.query, deep_search=deep_search)
refined_result = await self.refine_search(cr_result)
@@ -598,8 +552,6 @@ class Librarian(object):
for item in negative_answer:
self.not_in_db[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
self.app.logger.info("Search %s produced %d hits", self.uuid, len(self.final_result))
# Cache the completed result so a repeat of this query is instant.
_cache_put(self.query, deep_search, self.final_result)
return self.final_result
# ============================= FLASK INTERNALS===============================
-70
View File
@@ -1,70 +0,0 @@
"""Integration: the librarian's simple result cache.
A repeat of the same query (normalised) returns stored hits and skips the whole
Crossref + DB scan. Nothing fancy: whitespace/case-insensitive exact match,
disk-backed, TTL'd, size-bounded, deep/shallow kept separate.
"""
import sys
import types
import pytest
if "habanero" not in sys.modules:
_habanero = types.ModuleType("habanero")
_habanero.Crossref = object
sys.modules["habanero"] = _habanero
import conjurer_librarian as lib # noqa: E402
from durable_queue import DiskQueue # noqa: E402
@pytest.fixture
def cache(tmp_path, monkeypatch):
box = DiskQueue(str(tmp_path / "cache"))
monkeypatch.setattr(lib, "_cache", box)
monkeypatch.setattr(lib, "CACHE_TTL_SECONDS", 3600)
monkeypatch.setattr(lib, "CACHE_MAX_ENTRIES", 500)
return box
_HITS = {"10.1000/x": {"Title": ["A Paper"], "type": "journal-article"}}
def test_put_then_get_is_a_hit(cache):
lib._cache_put("kwas foliowy", False, _HITS)
assert lib._cache_get("kwas foliowy", False) == _HITS
def test_different_query_misses(cache):
lib._cache_put("kwas foliowy", False, _HITS)
assert lib._cache_get("witamina c", False) is None
def test_normalised_case_and_whitespace_hit_same_entry(cache):
lib._cache_put(" Kwas Foliowy ", False, _HITS)
assert lib._cache_get("kwas foliowy", False) == _HITS
def test_deep_and_shallow_are_cached_separately(cache):
lib._cache_put("q", False, _HITS)
assert lib._cache_get("q", True) is None # a deep search is a different key
assert lib._cache_get("q", False) == _HITS
def test_expired_entry_is_a_miss(cache):
cache.put(lib._cache_key("q", False), {"query": "q", "final_result": _HITS, "expires": 0})
assert lib._cache_get("q", False) is None
def test_ttl_zero_disables_the_cache(cache, monkeypatch):
monkeypatch.setattr(lib, "CACHE_TTL_SECONDS", 0)
lib._cache_put("q", False, _HITS) # no-op when disabled
assert len(cache) == 0
assert lib._cache_get("q", False) is None
def test_prune_bounds_the_cache(cache, monkeypatch):
monkeypatch.setattr(lib, "CACHE_MAX_ENTRIES", 3)
for i in range(6):
lib._cache_put(f"query-{i}", False, _HITS)
assert len(cache) <= 3