"""Dostawcy LLM (LOG-31) — bez wołania jakiegokolwiek prawdziwego modelu. Transport podstawiamy przez httpx.MockTransport, więc testy są szybkie, deterministyczne i nic nie wychodzi na zewnątrz. """ import json import httpx import pytest from app.llm import factory from app.llm.base import Completion, LLMError from app.llm.providers import AnthropicProvider, ChatCompletionsProvider def _mock_client(handler): """Podmienia httpx.Client na wersję z transportem testowym.""" class _C(httpx.Client): def __init__(self, *a, **kw): kw["transport"] = httpx.MockTransport(handler) super().__init__(*a, **kw) return _C @pytest.fixture def chat_ok(monkeypatch): def handler(request): assert request.url.path.endswith("/chat/completions") return httpx.Response(200, json={ "model": "test-model", "choices": [{"message": {"content": "Horoskop testowy."}}], "usage": {"prompt_tokens": 100, "completion_tokens": 50}, }) monkeypatch.setattr(httpx, "Client", _mock_client(handler)) # ------------------------------------------------------- protokół chat/completions def test_local_provider_generates(chat_ok): p = ChatCompletionsProvider("local", "http://localhost:11434/v1", "m", leaves_lan=False) out = p.generate("prompt", 500) assert isinstance(out, Completion) assert out.text == "Horoskop testowy." assert out.usage["completion_tokens"] == 50 def test_local_provider_does_not_leave_lan(chat_ok): p = ChatCompletionsProvider("local", "http://localhost:11434/v1", "m", leaves_lan=False) assert p.generate("prompt", 100).leaves_lan is False def test_cloud_provider_marks_leaving_lan(chat_ok): p = ChatCompletionsProvider("openai", "https://api.openai.com/v1", "m", "klucz", leaves_lan=True) assert p.generate("prompt", 100).leaves_lan is True def test_api_key_sent_only_when_set(monkeypatch): seen = {} def handler(request): seen["auth"] = request.headers.get("authorization") return httpx.Response(200, json={"choices": [{"message": {"content": "x"}}]}) monkeypatch.setattr(httpx, "Client", _mock_client(handler)) ChatCompletionsProvider("local", "http://x/v1", "m").generate("p", 10) assert seen["auth"] is None ChatCompletionsProvider("openai", "http://x/v1", "m", "tajny").generate("p", 10) assert seen["auth"] == "Bearer tajny" def test_http_error_becomes_readable_message(monkeypatch): monkeypatch.setattr(httpx, "Client", _mock_client(lambda r: httpx.Response(400, text="zly model"))) with pytest.raises(LLMError, match="400"): ChatCompletionsProvider("local", "http://x/v1", "m").generate("p", 10) def test_malformed_response_reported(monkeypatch): monkeypatch.setattr(httpx, "Client", _mock_client(lambda r: httpx.Response(200, json={"nonsens": 1}))) with pytest.raises(LLMError, match="kształt"): ChatCompletionsProvider("local", "http://x/v1", "m").generate("p", 10) def test_retries_then_succeeds(monkeypatch): calls = {"n": 0} def handler(request): calls["n"] += 1 if calls["n"] < 3: return httpx.Response(429, text="za duzo") return httpx.Response(200, json={"choices": [{"message": {"content": "ok"}}]}) monkeypatch.setattr(httpx, "Client", _mock_client(handler)) monkeypatch.setattr("app.llm.providers.time.sleep", lambda s: None) # bez czekania assert ChatCompletionsProvider("local", "http://x/v1", "m").generate("p", 10).text == "ok" assert calls["n"] == 3 # ------------------------------------------------------------------- Anthropic def test_anthropic_generates(monkeypatch): def handler(request): assert request.url.path.endswith("/v1/messages") assert request.headers.get("x-api-key") == "klucz" assert request.headers.get("anthropic-version") return httpx.Response(200, json={ "model": "claude-x", "content": [{"type": "text", "text": "Prognoza."}], "usage": {"input_tokens": 10, "output_tokens": 5}, }) monkeypatch.setattr(httpx, "Client", _mock_client(handler)) out = AnthropicProvider("https://api.anthropic.com", "claude-x", "klucz").generate("p", 100) assert out.text == "Prognoza." and out.leaves_lan is True def test_anthropic_requires_key(): with pytest.raises(LLMError, match="ANTHROPIC_API_KEY"): AnthropicProvider("https://api.anthropic.com", "m", "").generate("p", 10) # --------------------------------------------------------------------- fabryka def test_default_provider_is_local(monkeypatch): monkeypatch.delenv("LLM_PROVIDER", raising=False) monkeypatch.delenv("LLM_MODEL", raising=False) monkeypatch.delenv("LLM_BASE_URL", raising=False) p = factory.build_provider() assert p.name == "local" # domyślnie NIC nie opuszcza sieci — prompt niesie opisy z baz (LOG-32) assert p.leaves_lan is False def test_openai_requires_key(monkeypatch): monkeypatch.delenv("LLM_API_KEY", raising=False) monkeypatch.delenv("OPENAI_API_KEY", raising=False) # komunikat wskazuje ZMIENNĄ DO USTAWIENIA dla tego dostawcy, nie ogólne LLM_API_KEY with pytest.raises(LLMError, match="OPENAI_API_KEY"): factory.build_provider("openai") def test_unknown_provider_rejected(): with pytest.raises(LLMError, match="Nieznany dostawca"): factory.build_provider("bzdura") def test_env_overrides_model_and_url(monkeypatch): monkeypatch.setenv("LLM_MODEL", "moj-model") monkeypatch.setenv("LLM_BASE_URL", "http://serwer:8000/v1") p = factory.build_provider("local") assert p.model == "moj-model" and p.base_url == "http://serwer:8000/v1" # ---------------------------------------- konfiguracja per dostawca (regresja LOG-31) # UI pozwala przelaczac dostawce przy kazdym zadaniu, wiec ustawienia JEDNEGO nie moga # przeciekac na pozostalych. Wczesniej wspolne LLM_BASE_URL/LLM_MODEL kierowaly zadania # do OpenAI na adres lokalnej Ollamy i prosily Anthropic o model llama. def _clear(monkeypatch): for v in ("LLM_PROVIDER", "LLM_MODEL", "LLM_BASE_URL", "LLM_API_KEY", "LOCAL_MODEL", "LOCAL_BASE_URL", "LOCAL_API_KEY", "OPENAI_MODEL", "OPENAI_BASE_URL", "OPENAI_API_KEY", "ANTHROPIC_MODEL", "ANTHROPIC_BASE_URL", "ANTHROPIC_API_KEY"): monkeypatch.delenv(v, raising=False) def test_local_config_does_not_leak_to_cloud(monkeypatch): """Sedno bledu: skonfigurowany model lokalny przejmowal zadania do chmury.""" _clear(monkeypatch) monkeypatch.setenv("LLM_PROVIDER", "local") monkeypatch.setenv("LLM_BASE_URL", "http://ollama:11434/v1") # konfiguracja lokalnego monkeypatch.setenv("LLM_MODEL", "llama3.1:8b") monkeypatch.setenv("OPENAI_API_KEY", "sk-test") local = factory.build_provider("local") assert local.base_url == "http://ollama:11434/v1" and local.model == "llama3.1:8b" openai = factory.build_provider("openai") assert openai.base_url == "https://api.openai.com/v1", "zadanie do OpenAI poszloby do Ollamy" assert openai.model == "gpt-4o-mini", "OpenAI dostalby nazwe modelu llama" def test_provider_specific_settings_win(monkeypatch): _clear(monkeypatch) monkeypatch.setenv("LLM_PROVIDER", "local") monkeypatch.setenv("ANTHROPIC_API_KEY", "sk-ant") monkeypatch.setenv("ANTHROPIC_MODEL", "claude-opus-4-8") p = factory.build_provider("anthropic") assert p.model == "claude-opus-4-8" and p.api_key == "sk-ant" def test_generic_vars_apply_only_to_default_provider(monkeypatch): """Zgodnosc wstecz: wspolne LLM_* konfiguruja dostawce domyslnego i tylko jego.""" _clear(monkeypatch) monkeypatch.setenv("LLM_PROVIDER", "openai") monkeypatch.setenv("LLM_API_KEY", "sk-generic") monkeypatch.setenv("LLM_MODEL", "gpt-4o") assert factory.build_provider("openai").model == "gpt-4o" assert factory.build_provider("local").model == "llama3.1:8b" # nie dziedziczy def test_cloud_without_key_is_rejected_clearly(monkeypatch): _clear(monkeypatch) monkeypatch.setenv("LLM_PROVIDER", "local") for name in ("openai", "anthropic"): with pytest.raises(LLMError, match=f"{name.upper()}_API_KEY"): factory.build_provider(name) def test_local_needs_no_key(monkeypatch): _clear(monkeypatch) assert factory.build_provider("local").api_key == "" # ------------------------------------------- pusta odpowiedz modelu (cicha awaria) # Regresja: model potrafi oddac pusta tresc (prompt zjadl caly kontekst -> # finish_reason=length, completion_tokens=0). Wczesniej generate() zwracalo pusty # tekst BEZ bledu, widok nic nie renderowal i uzytkownik dostawal pusta strone # bez zadnego wyjasnienia. Pusta odpowiedz MUSI byc bledem. def test_empty_completion_raises_instead_of_silent_blank(monkeypatch): monkeypatch.setattr(httpx, "Client", _mock_client(lambda r: httpx.Response(200, json={ "model": "llama3.1:8b", "choices": [{"message": {"content": ""}, "finish_reason": "length"}], "usage": {"prompt_tokens": 8000, "completion_tokens": 0}, }))) with pytest.raises(LLMError, match="nie zwrócił żadnej treści"): ChatCompletionsProvider("local", "http://x/v1", "m").generate("p", 2000) def test_empty_completion_explains_context_window(monkeypatch): """Komunikat ma prowadzic do przyczyny, a nie tylko stwierdzac fakt.""" monkeypatch.setattr(httpx, "Client", _mock_client(lambda r: httpx.Response(200, json={ "choices": [{"message": {"content": " "}, "finish_reason": "length"}], "usage": {"prompt_tokens": 8000, "completion_tokens": 0}, }))) with pytest.raises(LLMError) as ei: ChatCompletionsProvider("local", "http://x/v1", "m").generate("p", 2000) msg = str(ei.value) assert "kontekstu" in msg and "budżet" in msg assert "powód zakończenia: length" in msg # diagnostyka w tresci bledu def test_whitespace_only_is_treated_as_empty(monkeypatch): monkeypatch.setattr(httpx, "Client", _mock_client(lambda r: httpx.Response(200, json={ "choices": [{"message": {"content": "\n\n \t "}}], }))) with pytest.raises(LLMError, match="nie zwrócił żadnej treści"): ChatCompletionsProvider("local", "http://x/v1", "m").generate("p", 100) def test_anthropic_empty_completion_raises(monkeypatch): monkeypatch.setattr(httpx, "Client", _mock_client(lambda r: httpx.Response(200, json={ "model": "claude-x", "content": [], "stop_reason": "max_tokens", "usage": {"input_tokens": 9000, "output_tokens": 0}, }))) with pytest.raises(LLMError, match="nie zwrócił żadnej treści"): AnthropicProvider("https://api.anthropic.com", "claude-x", "klucz").generate("p", 100) def test_normal_response_still_passes(monkeypatch): """Straznik nie moze psuc poprawnej odpowiedzi.""" monkeypatch.setattr(httpx, "Client", _mock_client(lambda r: httpx.Response(200, json={ "choices": [{"message": {"content": "Horoskop."}, "finish_reason": "stop"}], "usage": {"prompt_tokens": 100, "completion_tokens": 20}, }))) assert ChatCompletionsProvider("local", "http://x/v1", "m").generate("p", 100).text == "Horoskop." # ------------------------------------- kontynuacja: horoskop MA powstac zawsze # Sedno wymagania: niezaleznie od objetosci promptu i limitu wyjscia, pelna tresc # ma wrocic do uzytkownika. Model urwany na max_tokens jest proszony o dokonczenie # w ramach tej samej rozmowy, a kawalki sa sklejane. def _scripted(responses): """Transport oddajacy kolejne odpowiedzi z listy (po jednej na ture).""" seq = list(responses) seen = [] def handler(request): seen.append(request) return httpx.Response(200, json=seq.pop(0) if seq else seq_last) seq_last = responses[-1] return handler, seen def _chat(text, finish): return {"choices": [{"message": {"content": text}, "finish_reason": finish}], "usage": {"completion_tokens": 10}} def test_truncated_answer_is_continued_and_joined(monkeypatch): handler, seen = _scripted([ _chat("Czesc pierwsza.", "length"), _chat("Czesc druga. KONIEC", "stop"), ]) monkeypatch.setattr(httpx, "Client", _mock_client(handler)) out = ChatCompletionsProvider("local", "http://x/v1", "m").generate("prompt", 40000) assert "Czesc pierwsza." in out.text and "Czesc druga." in out.text assert "KONIEC" not in out.text # znacznik nie trafia do horoskopu assert out.usage["turns"] == 2 def test_continuation_asks_in_same_conversation(monkeypatch): """Kontynuacja musi isc jako kolejna tura rozmowy, a ostatnia wiadomosc MUSI byc od uzytkownika — Claude odrzuca prefill w turze asystenta (400).""" handler, seen = _scripted([_chat("Poczatek", "length"), _chat("Reszta", "stop")]) monkeypatch.setattr(httpx, "Client", _mock_client(handler)) ChatCompletionsProvider("local", "http://x/v1", "m").generate("prompt", 40000) msgs = json.loads(seen[1].content)["messages"] assert msgs[-1]["role"] == "user", "ostatnia wiadomosc nie moze byc prefillem asystenta" assert msgs[1]["role"] == "assistant" and "Poczatek" in msgs[1]["content"] def test_anthropic_thinking_only_turn_is_continued(monkeypatch): """DOKLADNIE zgloszony objaw: cala tura poszla na myslenie, tekst pusty. Wczesniej konczylo sie to pusta strona; teraz pytamy o tresc dalej.""" seq = [ {"content": [{"type": "thinking", "thinking": ""}], "stop_reason": "max_tokens", "usage": {"output_tokens": 2000}}, {"content": [{"type": "text", "text": "Horoskop urodzeniowy..."}], "stop_reason": "end_turn", "usage": {"output_tokens": 500}}, ] def handler(request): return httpx.Response(200, json=seq.pop(0) if seq else seq[-1]) monkeypatch.setattr(httpx, "Client", _mock_client(handler)) out = AnthropicProvider("https://api.anthropic.com", "claude-opus-4-8", "k").generate("p", 40000) assert out.text == "Horoskop urodzeniowy..." assert out.usage["turns"] == 2 def test_anthropic_sends_thinking_config(monkeypatch): """Bez jawnego `thinking` Sonnet 5 wlacza myslenie sam — konfigurujemy to wprost.""" seen = [] def handler(request): seen.append(json.loads(request.content)) return httpx.Response(200, json={"content": [{"type": "text", "text": "ok"}], "stop_reason": "end_turn"}) monkeypatch.setattr(httpx, "Client", _mock_client(handler)) monkeypatch.delenv("ANTHROPIC_THINKING", raising=False) AnthropicProvider("https://api.anthropic.com", "claude-opus-4-8", "k").generate("p", 5000) assert seen[0]["thinking"] == {"type": "adaptive"} seen.clear() monkeypatch.setenv("ANTHROPIC_THINKING", "off") AnthropicProvider("https://api.anthropic.com", "claude-opus-4-8", "k").generate("p", 5000) assert seen[0]["thinking"] == {"type": "disabled"} def test_complete_answer_does_not_loop(monkeypatch): """Straznik nie moze mnozyc zapytan, gdy model skonczyl normalnie.""" calls = {"n": 0} def handler(request): calls["n"] += 1 return httpx.Response(200, json=_chat("Gotowe.", "stop")) monkeypatch.setattr(httpx, "Client", _mock_client(handler)) out = ChatCompletionsProvider("local", "http://x/v1", "m").generate("p", 40000) assert out.text == "Gotowe." and calls["n"] == 1 # ------------------------------------------- wybor modelu przez uzytkownika (UI) def test_model_from_request_wins_over_config(monkeypatch): _clear(monkeypatch) monkeypatch.setenv("ANTHROPIC_MODEL", "claude-opus-4-8") monkeypatch.setenv("ANTHROPIC_API_KEY", "k") p = factory.build_provider("anthropic", "claude-fable-5") assert p.model == "claude-fable-5", "wybor z UI musi wygrac nad konfiguracja" def test_blank_model_falls_back_to_configured_default(monkeypatch): _clear(monkeypatch) monkeypatch.setenv("ANTHROPIC_MODEL", "claude-sonnet-5") monkeypatch.setenv("ANTHROPIC_API_KEY", "k") assert factory.build_provider("anthropic", " ").model == "claude-sonnet-5" def test_resolve_model_needs_no_api_key(monkeypatch): """Budzet promptu zalezy od okna kontekstu modelu — nie moze wymagac klucza. Wczesniej liczenie budzetu szlo przez build_provider(), ktory bez klucza rzuca bledem, wiec „maksymalny kontekst" cicho spadal do wartosci zapasowej. """ _clear(monkeypatch) provider, model = factory.resolve_model("anthropic", "claude-haiku-4-5") assert (provider, model) == ("anthropic", "claude-haiku-4-5") with pytest.raises(LLMError): # samo zbudowanie nadal wymaga klucza factory.build_provider("anthropic", "claude-haiku-4-5") def test_max_budget_differs_between_models(monkeypatch): """Sedno funkcji: wieksze okno = wiekszy budzet promptu.""" from app.llm.limits import prompt_token_budget _clear(monkeypatch) opus = prompt_token_budget(*factory.resolve_model("anthropic", "claude-opus-4-8")) haiku = prompt_token_budget(*factory.resolve_model("anthropic", "claude-haiku-4-5")) local = prompt_token_budget(*factory.resolve_model("local", "llama3.1:8b")) assert opus > haiku > local > 0