From a6c20a0054915032ace7b042c1b17356ef92374b Mon Sep 17 00:00:00 2001 From: Michal Tuszowski Date: Mon, 29 Jun 2026 11:57:40 +0200 Subject: [PATCH] ci: replace broken default workflows with compile/unit/integration CI MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The two scaffold workflows (Python application / Python package) failed on every PR: they installed deps from a non-existent requirements.txt, ran flake8/pytest over the vendored yt_dlp fork (new syntax under the 3.8/3.9 matrix), and collected ad-hoc root scripts — notably test_ai.py, which is an invalid pasted object dump (not Python). - Remove python-app.yml / python-package.yml and the junk root scripts (test.py, test_ai.py, test_time.py) - Add .github/workflows/ci.yml with three PR-check jobs: * compile — py_compile every first-party .py (no deps) * unit — pytest on pure logic (conanjurer_functions, constants) * integration — boot the Flask services and assert the X-Conjurer-Api-Key auth contract (communication_subroutine + conjurer_musician) - Add tests/ suite, pytest.ini (testpaths=tests) and conftest.py (sys.path) Fixes surfaced by the compile gate / needed for the integration job: - conjurer_librarian/search_bot.py + search_bot2.py: f-string reused the same quote ({item["exists"]}) -> SyntaxError on Python < 3.12 - conjurer_musician/media_search_functions.py: made import-safe (env-overridable paths, lazy DB load / mkdir) so the service can be imported and tested off the Pi Co-Authored-By: Claude Opus 4.8 --- .github/workflows/ci.yml | 64 +++++++++++++++++++++ .github/workflows/python-app.yml | 39 ------------- .github/workflows/python-package.yml | 40 ------------- conftest.py | 14 +++++ conjurer_librarian/search_bot.py | 2 +- conjurer_librarian/search_bot2.py | 2 +- conjurer_musician/media_search_functions.py | 52 ++++++++++++----- pytest.ini | 5 ++ test.py | 20 ------- test_ai.py | 13 ----- test_time.py | 8 --- tests/integration/test_comm_auth.py | 56 ++++++++++++++++++ tests/integration/test_musician_auth.py | 34 +++++++++++ tests/unit/test_conanjurer_functions.py | 63 ++++++++++++++++++++ tests/unit/test_constants.py | 35 +++++++++++ 15 files changed, 310 insertions(+), 137 deletions(-) create mode 100644 .github/workflows/ci.yml delete mode 100644 .github/workflows/python-app.yml delete mode 100644 .github/workflows/python-package.yml create mode 100644 conftest.py create mode 100644 pytest.ini delete mode 100644 test.py delete mode 100644 test_ai.py delete mode 100644 test_time.py create mode 100644 tests/integration/test_comm_auth.py create mode 100644 tests/integration/test_musician_auth.py create mode 100644 tests/unit/test_conanjurer_functions.py create mode 100644 tests/unit/test_constants.py diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml new file mode 100644 index 0000000..f9938ba --- /dev/null +++ b/.github/workflows/ci.yml @@ -0,0 +1,64 @@ +name: CI + +on: + push: + branches: ["main"] + pull_request: + branches: ["main"] + +permissions: + contents: read + +jobs: + # ---------------------------------------------------------------- compile + # Byte-compiles every first-party Python file. Catches syntax errors (e.g. + # a misplaced `from __future__` import) without needing any dependencies. + compile: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: py_compile first-party sources + run: | + # Vendored forks (yt_dlp, spotify_dl) are excluded — they ship their + # own validity and use syntax not meant for this gate. + git ls-files '*.py' \ + | grep -vE '^(yt_dlp|spotify_dl)/' \ + | xargs python -m py_compile + echo "All first-party sources compile." + + # ------------------------------------------------------------------- unit + # Pure-logic tests. The modules under test guard their optional/heavy + # dependencies, so only pytest is required. + unit: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install test deps + run: | + python -m pip install --upgrade pip + pip install pytest + - name: Run unit tests + run: pytest tests/unit -v + + # ------------------------------------------------------------ integration + # Boots the Flask services (no Discord/network) and exercises their HTTP + # surface — primarily the X-Conjurer-Api-Key auth contract end to end. + integration: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - name: Install service deps + run: | + python -m pip install --upgrade pip + pip install pytest flask waitress requests + - name: Run integration tests + run: pytest tests/integration -v diff --git a/.github/workflows/python-app.yml b/.github/workflows/python-app.yml deleted file mode 100644 index f3d4fca..0000000 --- a/.github/workflows/python-app.yml +++ /dev/null @@ -1,39 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a single version of Python -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - -name: Python application - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -permissions: - contents: read - -jobs: - build: - - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v3 - - name: Set up Python 3.10 - uses: actions/setup-python@v3 - with: - python-version: "3.10" - - name: Install dependencies - run: | - python -m pip install --upgrade pip - pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest diff --git a/.github/workflows/python-package.yml b/.github/workflows/python-package.yml deleted file mode 100644 index f6f35d7..0000000 --- a/.github/workflows/python-package.yml +++ /dev/null @@ -1,40 +0,0 @@ -# This workflow will install Python dependencies, run tests and lint with a variety of Python versions -# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python - -name: Python package - -on: - push: - branches: [ "main" ] - pull_request: - branches: [ "main" ] - -jobs: - build: - - runs-on: ubuntu-latest - strategy: - fail-fast: false - matrix: - python-version: ["3.8", "3.9", "3.10"] - - steps: - - uses: actions/checkout@v3 - - name: Set up Python ${{ matrix.python-version }} - uses: actions/setup-python@v3 - with: - python-version: ${{ matrix.python-version }} - - name: Install dependencies - run: | - python -m pip install --upgrade pip - python -m pip install flake8 pytest - if [ -f requirements.txt ]; then pip install -r requirements.txt; fi - - name: Lint with flake8 - run: | - # stop the build if there are Python syntax errors or undefined names - flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics - # exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide - flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics - - name: Test with pytest - run: | - pytest diff --git a/conftest.py b/conftest.py new file mode 100644 index 0000000..2f4e170 --- /dev/null +++ b/conftest.py @@ -0,0 +1,14 @@ +"""Pytest bootstrap: make first-party modules importable from the tests. + +The bot modules live at the repository root and the musician service lives in +``conjurer_musician/``; neither is an installable package, so we put both on +``sys.path`` here. +""" +import os +import sys + +_ROOT = os.path.dirname(os.path.abspath(__file__)) + +for _path in (_ROOT, os.path.join(_ROOT, "conjurer_musician")): + if _path not in sys.path: + sys.path.insert(0, _path) diff --git a/conjurer_librarian/search_bot.py b/conjurer_librarian/search_bot.py index 7a2ccaf..ca4f5f1 100644 --- a/conjurer_librarian/search_bot.py +++ b/conjurer_librarian/search_bot.py @@ -107,7 +107,7 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, for item in result_list: if item["DOI"] in data and not item["exists"]: - print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}") + print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item['exists']}") _logger.info(data) _logger.info("HIT") item["exists"] = True diff --git a/conjurer_librarian/search_bot2.py b/conjurer_librarian/search_bot2.py index 407b626..26c7504 100644 --- a/conjurer_librarian/search_bot2.py +++ b/conjurer_librarian/search_bot2.py @@ -108,7 +108,7 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, print(f"C{no}{alive_no}\r", end="") for item in result_list: if item["DOI"] in data[0] and not item["exists"]: - print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}") + print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item['exists']}") item["exists"] = True live_results.append(item) done_check = done_check and item["exists"] diff --git a/conjurer_musician/media_search_functions.py b/conjurer_musician/media_search_functions.py index 87a2712..1dc1009 100644 --- a/conjurer_musician/media_search_functions.py +++ b/conjurer_musician/media_search_functions.py @@ -1,24 +1,44 @@ #!/usr/bin/env python3 -import argparse +"""Share-list search/publish helpers for the musician service. + +Paths are environment-overridable and the share database / directory are +accessed lazily, so importing this module has no side effects (the previous +version ran ``SHARE_DIR.mkdir()`` and read the JSON DB at import time, which +crashed on any host without the Pi's ``/var/www`` / ``/var/log`` layout — and +made the service untestable). +""" import json import os -import sys import uuid from pathlib import Path -# CONFIGURATION -JSON_DB = '/var/log/share_scan.json' -SHARE_DIR = Path('/var/www/html/share') -BASE_URL = 'https://czernobog.pl/share' +# CONFIGURATION (env-overridable) +JSON_DB = os.getenv("CONJURER_SHARE_DB", "/var/log/share_scan.json") +SHARE_DIR = Path(os.getenv("CONJURER_SHARE_DIR", "/var/www/html/share")) +BASE_URL = os.getenv("CONJURER_SHARE_BASE_URL", "https://czernobog.pl/share") + +_entries_cache = None + + +def _ensure_share_dir(): + SHARE_DIR.mkdir(parents=True, exist_ok=True) -# Ensure share directory exists -SHARE_DIR.mkdir(parents=True, exist_ok=True) def load_db(): - with open(JSON_DB) as f: - return json.load(f)['entries'] + """Load share entries, returning [] when the DB is missing/corrupt.""" + try: + with open(JSON_DB) as handle: + return json.load(handle).get("entries", []) + except (FileNotFoundError, json.JSONDecodeError): + return [] + + +def _entries(): + global _entries_cache + if _entries_cache is None: + _entries_cache = load_db() + return _entries_cache -ENTRIES = load_db() def relevancy(path, keywords): score = 0 @@ -28,17 +48,20 @@ def relevancy(path, keywords): score += low.count(kw.lower()) return score + def find_matches(count, keywords): scored = [] - for e in ENTRIES: - score = relevancy(e['path'], keywords) + for entry in _entries(): + score = relevancy(entry["path"], keywords) if score > 0: - scored.append((score, e['path'])) + scored.append((score, entry["path"])) scored.sort(reverse=True, key=lambda x: x[0]) result = [p for _, p in scored] return result[:count] + def publish(paths): + _ensure_share_dir() urls = [] for path in paths: token = uuid.uuid4().hex @@ -49,4 +72,3 @@ def publish(paths): pass urls.append(f"{BASE_URL}/{token}") return urls - diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 0000000..5494680 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,5 @@ +[pytest] +testpaths = tests +python_files = test_*.py +python_functions = test_* +addopts = -ra diff --git a/test.py b/test.py deleted file mode 100644 index 357bb3f..0000000 --- a/test.py +++ /dev/null @@ -1,20 +0,0 @@ -import logging -from logging import handlers - -logger = logging.getLogger("discord") -logger.setLevel(logging.DEBUG) -handler = handlers.RotatingFileHandler( - filename="test.log", - encoding="utf-8", - mode="a", - maxBytes=6 * 1024 * 1024, - backupCount=6, -) -formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") -handler.setFormatter(formatter) -logger.addHandler(handler) - - -logger2 = logging.getLogger("discord") -for item in logger2.handlers: - print(item) diff --git a/test_ai.py b/test_ai.py deleted file mode 100644 index 8621cfc..0000000 --- a/test_ai.py +++ /dev/null @@ -1,13 +0,0 @@ -AsyncCursorPage[Message]( - data=[Message(id='msg_3eWSdgbcU8sbCmJK2momOgQQ', - assistant_id='asst_06eZiwvYNK3MR34suFP60gvg', - attachments=[], - completed_at=None, - content=[TextContentBlock(text=Text(annotations=[], value='Cześć! Oto coś do przemyślenia: \n\n„Sukces to suma niewielkich wysiłków powtarzanych dzień po dniu.” — Robert Collier\n\nTo przypomina nam, że często to nie wielkie działania, ale konsekwentne, małe kroki prowadzą do osiągnięcia celu. Jak mogę Ci dzisiaj pomóc?'), type='text')], created_at=1731620112, incomplete_at=None, incomplete_details=None, metadata={}, object='thread.message', role='assistant', run_id='run_JGjWQTCEkDcEYpyCJnrkZU8Q', status=None, thread_id='thread_dDEjGbGm6ICfG75u0KKpoVxD'), Message(id='msg_GFcnfCFgAthGm2D3oE5d0ZkQ', assistant_id=None, attachments=[], completed_at=None, content=[TextContentBlock(text=Text(annotations=[], value='Cześć! Powiedz coś mądrego'), type='text')], created_at=1731620110, incomplete_at=None, incomplete_details=None, metadata={}, object='thread.message', role='user', run_id=None, status=None, thread_id='thread_dDEjGbGm6ICfG75u0KKpoVxD')], object='list', first_id='msg_3eWSdgbcU8sbCmJK2momOgQQ', last_id='msg_GFcnfCFgAthGm2D3oE5d0ZkQ', has_more=False) - - -[TextContentBlock( - text=Text(annotations=[], - value='Cześć! Oto coś do rozważenia: "Największą przeszkodą w naszym życiu jest brak odwagi do wprowadzenia zmian." Niezależnie od tego, jakie masz cele czy marzenia, odwaga do działania i przystosowania się do nowych sytuacji jest kluczem do osiągnięcia sukcesu. Jakie masz przemyślenia na ten temat?'), - type='text') -] diff --git a/test_time.py b/test_time.py deleted file mode 100644 index 0c71465..0000000 --- a/test_time.py +++ /dev/null @@ -1,8 +0,0 @@ -import time - -first_time = time.time_ns() - -time.sleep(1) -time_diff = time.time_ns() - first_time -print(time_diff) -# 2000149433 diff --git a/tests/integration/test_comm_auth.py b/tests/integration/test_comm_auth.py new file mode 100644 index 0000000..435be4e --- /dev/null +++ b/tests/integration/test_comm_auth.py @@ -0,0 +1,56 @@ +"""Integration: the bot's communication Flask layer enforces the shared key, +and the key the bot would *send* (constants.service_headers) is accepted. +""" +import constants +import communication_subroutine as cs + + +def _client(key="test-secret"): + cs.API_KEY = key + return cs.app.test_client() + + +def test_prepped_tracks_rejected_without_key(): + client = _client() + resp = client.post( + "/prepped_tracks", data='["all", "x"]', content_type="application/json" + ) + assert resp.status_code == 401 + + +def test_prepped_tracks_accepted_with_key(): + client = _client() + resp = client.post( + "/prepped_tracks", + data='["all", "x"]', + headers={"X-Conjurer-Api-Key": "test-secret"}, + content_type="application/json", + ) + assert resp.status_code == 200 + + +def test_conjurer_get_is_open(): + client = _client() + assert client.get("/conjurer").status_code == 200 + + +def test_open_when_key_unset(): + client = _client(key=None) + resp = client.post( + "/prepped_tracks", data='["all", "x"]', content_type="application/json" + ) + assert resp.status_code == 200 + + +def test_bot_service_headers_accepted_by_service(monkeypatch): + # End-to-end auth contract: the header constants.service_headers() produces + # is exactly what communication_subroutine._authorize_request() expects. + monkeypatch.setattr(constants, "API_SHARED_KEY", "shared-xyz") + cs.API_KEY = "shared-xyz" + resp = cs.app.test_client().post( + "/prepped_tracks", + data='["all", "x"]', + headers=constants.service_headers(), + content_type="application/json", + ) + assert resp.status_code == 200 diff --git a/tests/integration/test_musician_auth.py b/tests/integration/test_musician_auth.py new file mode 100644 index 0000000..7e70401 --- /dev/null +++ b/tests/integration/test_musician_auth.py @@ -0,0 +1,34 @@ +"""Integration: the musician Flask service enforces the shared key on its +authenticated endpoints while leaving the open ones reachable. +""" +import conjurer_musician as m + + +def _client(key="test-secret"): + m.API_KEY = key + return m.app.test_client() + + +def test_clear_pr_pls_rejected_without_key(): + client = _client() + assert client.get("/clear_pr_pls").status_code == 401 + + +def test_clear_pr_pls_accepted_with_key(): + client = _client() + resp = client.get( + "/clear_pr_pls", headers={"X-Conjurer-Api-Key": "test-secret"} + ) + assert resp.status_code == 200 + + +def test_mp3_list_is_open(): + client = _client() + resp = client.get("/mp3") + assert resp.status_code == 200 + assert "music_file_list" in resp.get_json() + + +def test_open_when_key_unset(): + client = _client(key=None) + assert client.get("/clear_pr_pls").status_code == 200 diff --git a/tests/unit/test_conanjurer_functions.py b/tests/unit/test_conanjurer_functions.py new file mode 100644 index 0000000..e2c652b --- /dev/null +++ b/tests/unit/test_conanjurer_functions.py @@ -0,0 +1,63 @@ +"""Unit tests for the Conan Exiles bridge helpers (no Discord/RCON needed).""" +import conanjurer_functions as cf + + +def test_parse_players_basic(): + out = ( + "Idx | Char name | Player name | User ID\n" + "0 | Conan | SteamGuy | 1\n" + "1 | Khasar | OtherGuy | 2\n" + "--- | --- | --- | ---" + ) + assert cf.parse_players(out) == {"Conan", "Khasar"} + + +def test_parse_players_empty_inputs(): + assert cf.parse_players("No players connected.") == set() + assert cf.parse_players("") == set() + + +def test_parse_line_login(): + event = cf.parse_line("SomeGuy joined the server") + assert event is not None + assert event.kind == "login" + assert "SomeGuy" in event.text + + +def test_parse_line_chat(): + event = cf.parse_line("Chat: Bob: hello there") + assert event is not None + assert event.kind == "chat" + assert "Bob" in event.text and "hello" in event.text + + +def test_parse_line_unrecognised_is_ignored(): + assert cf.parse_line("random server noise") is None + + +def test_conanconfig_disabled_when_unconfigured(): + cfg = cf.ConanConfig("", 25575, "", "local", "", "", 22, "", "") + assert cfg.rcon_enabled is False + assert cfg.log_enabled is False + + +def test_conanconfig_local_log_enabled(): + cfg = cf.ConanConfig("", 0, "", "local", "/tmp/conan.log", "", 22, "", "") + assert cfg.log_enabled is True + + +def test_conanconfig_rcon_enabled_requires_lib(monkeypatch): + # Without aiomcrcon installed, rcon stays disabled even when host+pw are set. + monkeypatch.setattr(cf, "_Rcon", None) + cfg = cf.ConanConfig("1.2.3.4", 25575, "pw", "local", "", "", 22, "", "") + assert cfg.rcon_enabled is False + # With the lib present, it enables. + monkeypatch.setattr(cf, "_Rcon", object) + cfg2 = cf.ConanConfig("1.2.3.4", 25575, "pw", "local", "", "", 22, "", "") + assert cfg2.rcon_enabled is True + + +def test_conanconfig_sftp_log_requires_asyncssh(monkeypatch): + monkeypatch.setattr(cf, "asyncssh", None) + cfg = cf.ConanConfig("", 0, "", "sftp", "/log", "sftp.host", 22, "u", "p") + assert cfg.log_enabled is False diff --git a/tests/unit/test_constants.py b/tests/unit/test_constants.py new file mode 100644 index 0000000..1cf9fe2 --- /dev/null +++ b/tests/unit/test_constants.py @@ -0,0 +1,35 @@ +"""Unit tests for constants helpers (defensive config / auth headers).""" +import constants + + +def test_service_headers_empty_when_no_key(monkeypatch): + monkeypatch.setattr(constants, "API_SHARED_KEY", "") + assert constants.service_headers() == {} + + +def test_service_headers_with_key(monkeypatch): + monkeypatch.setattr(constants, "API_SHARED_KEY", "s3cr3t") + assert constants.service_headers() == {"X-Conjurer-Api-Key": "s3cr3t"} + + +def test_load_json_missing_returns_fallback(tmp_path): + missing = tmp_path / "nope.json" + assert constants._load_json(str(missing), {"fallback": 1}) == {"fallback": 1} + + +def test_load_json_valid(tmp_path): + good = tmp_path / "ok.json" + good.write_text('{"a": 2}', encoding="utf-8") + assert constants._load_json(str(good), {}) == {"a": 2} + + +def test_load_json_corrupt_returns_fallback(tmp_path): + bad = tmp_path / "bad.json" + bad.write_text("{ not valid json", encoding="utf-8") + assert constants._load_json(str(bad), []) == [] + + +def test_conan_defaults_present(): + # Feature toggles default to "off" so the bridge stays dormant. + assert constants.CONAN_JOIN_CHANNEL_ID == 0 + assert constants.CONAN_RCON_HOST == ""