mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
Compare commits
14 Commits
| Author | SHA1 | Date | |
|---|---|---|---|
| da8260d164 | |||
| b266db5bf3 | |||
| 0a29c9950d | |||
| 8ba954176a | |||
| 759c04a48f | |||
| 4ff0427b64 | |||
| 934e7a6240 | |||
| 597bc004fc | |||
| 9c8384b10c | |||
| a34a2a3299 | |||
| bd82369006 | |||
| a6c20a0054 | |||
| a32bbdd03c | |||
| 5adeb1b384 |
@@ -0,0 +1,17 @@
|
||||
# Keep build contexts lean. Component dirs (conjurer_librarian, conjurer_musician)
|
||||
# are intentionally NOT ignored — their images copy them from this same context.
|
||||
.git
|
||||
.github
|
||||
.trunk
|
||||
.vscode
|
||||
**/__pycache__/
|
||||
*.pyc
|
||||
*.pyo
|
||||
*.log
|
||||
*.mp3
|
||||
docs/
|
||||
tests/
|
||||
conftest.py
|
||||
pytest.ini
|
||||
docker/env/*.env
|
||||
secrets/
|
||||
@@ -0,0 +1,54 @@
|
||||
name: CI
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: ["main"]
|
||||
pull_request:
|
||||
branches: ["main"]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
compile:
|
||||
# Byte-compile every first-party .py to catch syntax errors. No deps.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
- name: py_compile first-party sources
|
||||
run: |
|
||||
git ls-files '*.py' | grep -vE '^(yt_dlp|spotify_dl)/' | xargs python -m py_compile
|
||||
echo "All first-party sources compile."
|
||||
|
||||
unit:
|
||||
# Pure-logic tests; tested modules guard heavy deps, so only pytest needed.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
- name: Install test deps
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pytest
|
||||
- name: Run unit tests
|
||||
run: pytest tests/unit -v
|
||||
|
||||
integration:
|
||||
# Boot the Flask services and assert the X-Conjurer-Api-Key auth contract.
|
||||
runs-on: ubuntu-latest
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
- uses: actions/setup-python@v5
|
||||
with:
|
||||
python-version: "3.11"
|
||||
- name: Install service deps
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install pytest flask waitress requests
|
||||
- name: Run integration tests
|
||||
run: pytest tests/integration -v
|
||||
@@ -1,39 +0,0 @@
|
||||
# This workflow will install Python dependencies, run tests and lint with a single version of Python
|
||||
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
|
||||
|
||||
name: Python application
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
|
||||
permissions:
|
||||
contents: read
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python 3.10
|
||||
uses: actions/setup-python@v3
|
||||
with:
|
||||
python-version: "3.10"
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
pip install flake8 pytest
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
# stop the build if there are Python syntax errors or undefined names
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
pytest
|
||||
@@ -1,40 +0,0 @@
|
||||
# This workflow will install Python dependencies, run tests and lint with a variety of Python versions
|
||||
# For more information see: https://docs.github.com/en/actions/automating-builds-and-tests/building-and-testing-python
|
||||
|
||||
name: Python package
|
||||
|
||||
on:
|
||||
push:
|
||||
branches: [ "main" ]
|
||||
pull_request:
|
||||
branches: [ "main" ]
|
||||
|
||||
jobs:
|
||||
build:
|
||||
|
||||
runs-on: ubuntu-latest
|
||||
strategy:
|
||||
fail-fast: false
|
||||
matrix:
|
||||
python-version: ["3.8", "3.9", "3.10"]
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v3
|
||||
- name: Set up Python ${{ matrix.python-version }}
|
||||
uses: actions/setup-python@v3
|
||||
with:
|
||||
python-version: ${{ matrix.python-version }}
|
||||
- name: Install dependencies
|
||||
run: |
|
||||
python -m pip install --upgrade pip
|
||||
python -m pip install flake8 pytest
|
||||
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
|
||||
- name: Lint with flake8
|
||||
run: |
|
||||
# stop the build if there are Python syntax errors or undefined names
|
||||
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
|
||||
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
|
||||
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
|
||||
- name: Test with pytest
|
||||
run: |
|
||||
pytest
|
||||
@@ -4,9 +4,21 @@
|
||||
# pylint: disable=too-many-lines
|
||||
"""
|
||||
Module of a python bot named Conjurer - used to work on BDSM discord servers.
|
||||
|
||||
Startup is defensive by design:
|
||||
|
||||
* every cog is loaded independently - one broken/missing dependency disables
|
||||
that cog only, never the whole bot,
|
||||
* cogs that need a sibling service (musician / librarian) are only enabled
|
||||
after a positive health check; a watchdog keeps re-checking and enables them
|
||||
the moment the service comes alive,
|
||||
* fatal misconfiguration (missing Discord token) exits loudly on stderr with a
|
||||
clear message instead of dying silently into a log file.
|
||||
"""
|
||||
import asyncio
|
||||
import logging
|
||||
import os
|
||||
import sys
|
||||
|
||||
# *=========================================== Standard Library Imports
|
||||
import random
|
||||
@@ -15,12 +27,26 @@ from logging import handlers
|
||||
|
||||
# *==============Imported libraries
|
||||
import discord
|
||||
import requests
|
||||
from discord.ext import commands
|
||||
|
||||
from communication_subroutine import comm_subroutine
|
||||
from constants import ENCODING, LOGFILE, TOKEN
|
||||
from constants import (
|
||||
ENCODING,
|
||||
FILE_SERVICE_ADDRESS,
|
||||
GET_MP3,
|
||||
LIBRARIAN_SERVICE_ADDRESS,
|
||||
LOGFILE,
|
||||
TOKEN,
|
||||
)
|
||||
|
||||
logger = logging.getLogger("discord")
|
||||
logger.setLevel(logging.INFO)
|
||||
logger.setLevel(logging.DEBUG)
|
||||
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
|
||||
# File log (rotated). constants ensures the directory exists; this is belt and
|
||||
# braces for exotic overrides.
|
||||
os.makedirs(os.path.dirname(LOGFILE) or ".", exist_ok=True)
|
||||
handler = handlers.RotatingFileHandler(
|
||||
filename=LOGFILE,
|
||||
encoding=ENCODING,
|
||||
@@ -28,10 +54,18 @@ handler = handlers.RotatingFileHandler(
|
||||
maxBytes=6 * 1024 * 1024,
|
||||
backupCount=6,
|
||||
)
|
||||
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
# Console log so `docker logs` / journalctl actually show what happened.
|
||||
console_handler = logging.StreamHandler()
|
||||
console_handler.setLevel(logging.INFO)
|
||||
console_handler.setFormatter(formatter)
|
||||
logger.addHandler(console_handler)
|
||||
# Some dependency calls logging.basicConfig(), adding a root handler; without
|
||||
# this the 'discord' logger's records get printed twice (our format + root's).
|
||||
logger.propagate = False
|
||||
|
||||
# *=========================================== Initializations
|
||||
intents = discord.Intents.default()
|
||||
intents.message_content = True
|
||||
@@ -48,37 +82,129 @@ intents.moderation = True
|
||||
random.seed()
|
||||
client = commands.Bot(intents=intents, command_prefix="$")
|
||||
|
||||
# *=========================================== Extension groups
|
||||
# Core cogs depend on nothing but the bot itself (broken ones are skipped
|
||||
# individually). Service groups are gated on a health check of the service
|
||||
# they talk to and enabled later by the watchdog when the service appears.
|
||||
CORE_EXTENSIONS = [
|
||||
"administration_commands",
|
||||
"ai_commands",
|
||||
"other_commands",
|
||||
"latex_commands",
|
||||
"voice_recognition_commands",
|
||||
"conanjurer_commands",
|
||||
]
|
||||
|
||||
SERVICE_EXTENSION_GROUPS = {
|
||||
# musician (file service): music download/search, radio control, file shares
|
||||
"musician": {
|
||||
"health_url": f"{FILE_SERVICE_ADDRESS}{GET_MP3}",
|
||||
"extensions": ["music_commands", "radio_commands", "file_search_commands"],
|
||||
},
|
||||
# librarian: DOI / Crossref search
|
||||
"librarian": {
|
||||
"health_url": f"{LIBRARIAN_SERVICE_ADDRESS}/",
|
||||
"extensions": ["librarian_commands"],
|
||||
},
|
||||
}
|
||||
|
||||
SERVICE_RECHECK_SECONDS = 300
|
||||
|
||||
|
||||
def _service_alive(url: str) -> bool:
|
||||
"""True when the service answers HTTP at all (any status code counts)."""
|
||||
try:
|
||||
requests.get(url, timeout=3)
|
||||
return True
|
||||
except requests.exceptions.RequestException:
|
||||
return False
|
||||
|
||||
|
||||
async def _load_extension_safe(name: str) -> bool:
|
||||
"""Load one extension; log and continue instead of killing startup."""
|
||||
if name in client.extensions:
|
||||
return True
|
||||
try:
|
||||
await client.load_extension(name)
|
||||
logger.info("Extension loaded: %s", name)
|
||||
return True
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.exception("Extension FAILED (disabled, bot continues): %s", name)
|
||||
return False
|
||||
|
||||
|
||||
async def _load_service_groups() -> bool:
|
||||
"""Health-check each service group and load its cogs when alive.
|
||||
|
||||
Returns True when anything new was loaded (caller may want to re-sync).
|
||||
"""
|
||||
loaded_any = False
|
||||
for service, group in SERVICE_EXTENSION_GROUPS.items():
|
||||
missing = [e for e in group["extensions"] if e not in client.extensions]
|
||||
if not missing:
|
||||
continue
|
||||
alive = await asyncio.to_thread(_service_alive, group["health_url"])
|
||||
if not alive:
|
||||
logger.warning(
|
||||
"Service '%s' unreachable (%s) - cogs stay disabled: %s",
|
||||
service,
|
||||
group["health_url"],
|
||||
", ".join(missing),
|
||||
)
|
||||
continue
|
||||
logger.info("Service '%s' is alive - enabling: %s", service, ", ".join(missing))
|
||||
for extension in missing:
|
||||
if await _load_extension_safe(extension):
|
||||
loaded_any = True
|
||||
return loaded_any
|
||||
|
||||
|
||||
async def _sync_tree() -> None:
|
||||
try:
|
||||
await client.tree.sync()
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.exception("Slash-command tree sync failed (commands may lag)")
|
||||
|
||||
|
||||
async def _service_watchdog() -> None:
|
||||
"""Periodically retry offline services and enable their cogs when up."""
|
||||
while not client.is_closed():
|
||||
await asyncio.sleep(SERVICE_RECHECK_SECONDS)
|
||||
try:
|
||||
if await _load_service_groups():
|
||||
await _sync_tree()
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.exception("Service watchdog tick failed")
|
||||
|
||||
|
||||
_STARTUP_DONE = False
|
||||
|
||||
|
||||
# *=========================================== Define Events
|
||||
@client.event
|
||||
async def on_ready():
|
||||
"""Metoda wywoływana przy połączeniu do serwera."""
|
||||
logger = logging.getLogger("discord")
|
||||
logger.debug("SAMPLE DEBUG LOG")
|
||||
global _STARTUP_DONE # pylint: disable=global-statement
|
||||
logger.info("%s has connected to Discord!", client.user)
|
||||
# TODO: load vs reload
|
||||
if _STARTUP_DONE:
|
||||
logger.info("Reconnected - extensions already loaded")
|
||||
return
|
||||
_STARTUP_DONE = True
|
||||
logger.info("Reactor: online")
|
||||
|
||||
await client.load_extension("administration_commands")
|
||||
for extension in CORE_EXTENSIONS:
|
||||
await _load_extension_safe(extension)
|
||||
|
||||
await client.load_extension("librarian_commands")
|
||||
await client.load_extension("music_commands")
|
||||
await client.load_extension("radio_commands")
|
||||
|
||||
await client.load_extension("ai_commands")
|
||||
|
||||
await client.load_extension("other_commands")
|
||||
await client.load_extension("voice_recognition_commands")
|
||||
await client.load_extension("file_search_commands")
|
||||
await client.load_extension("latex_commands")
|
||||
await client.load_extension("conanjurer_commands")
|
||||
await _load_service_groups()
|
||||
logger.info("Sensors: online")
|
||||
|
||||
logger.info(client.cogs)
|
||||
await client.tree.sync()
|
||||
await _sync_tree()
|
||||
for com in client.commands:
|
||||
logger.info("Command %s is awejleble", com.qualified_name)
|
||||
|
||||
asyncio.create_task(_service_watchdog())
|
||||
|
||||
logger.info("Logged in as ----> %s", client.user)
|
||||
logger.info("ID:%s ", client.user.id)
|
||||
logger.info("All systems: operational")
|
||||
@@ -93,22 +219,64 @@ async def on_ready():
|
||||
|
||||
|
||||
async def _run_comm_subroutine(stop_event: threading.Event) -> None:
|
||||
"""Run the blocking comm subroutine in a worker thread."""
|
||||
await asyncio.to_thread(comm_subroutine, stop_event)
|
||||
"""Run the blocking comm subroutine in a worker thread.
|
||||
|
||||
A crash here takes down the internal HTTP endpoints only - the Discord
|
||||
side keeps running, so log loudly and swallow.
|
||||
"""
|
||||
try:
|
||||
await asyncio.to_thread(comm_subroutine, stop_event)
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.exception(
|
||||
"Comm layer crashed - internal HTTP endpoints are down "
|
||||
"(musician/librarian callbacks will not arrive); bot continues"
|
||||
)
|
||||
|
||||
|
||||
async def _run_bot(token: str, shutdown_event: asyncio.Event) -> None:
|
||||
"""Start the Discord client and flag shutdown when it returns."""
|
||||
"""Start the Discord client; log WHY it died before flagging shutdown.
|
||||
|
||||
Exceptions were previously swallowed by ``gather(return_exceptions=True)``
|
||||
which made a failed login look like a clean exit (silent crash-loop in
|
||||
docker). Now every death is diagnosed on the console first.
|
||||
"""
|
||||
try:
|
||||
await client.start(token, log_handler=None)
|
||||
# NOTE: log_handler is a Client.run()-only kwarg (run() configures
|
||||
# logging, then calls start()); start() takes just the token. We set
|
||||
# up our own handlers above, so nothing is lost.
|
||||
await client.start(token)
|
||||
except discord.LoginFailure:
|
||||
logger.critical(
|
||||
"FATAL: Discord REJECTED the token (Improper token). Fix the "
|
||||
"'discord' entry in the netrc mounted at CONJURER_NETRC_FILE or "
|
||||
"the DISCORD_TOKEN env var. If the token leaked/reset, generate a "
|
||||
"new one in the Discord Developer Portal -> Bot -> Reset Token."
|
||||
)
|
||||
raise
|
||||
except discord.PrivilegedIntentsRequired:
|
||||
logger.critical(
|
||||
"FATAL: this bot application does not have the Privileged Gateway "
|
||||
"Intents enabled. Open Discord Developer Portal -> your app -> "
|
||||
"Bot -> enable 'Presence', 'Server Members' and 'Message Content' "
|
||||
"Intents, then restart."
|
||||
)
|
||||
raise
|
||||
except Exception: # pylint: disable=broad-exception-caught
|
||||
logger.exception("FATAL: Discord client crashed at startup/runtime")
|
||||
raise
|
||||
finally:
|
||||
shutdown_event.set()
|
||||
|
||||
|
||||
async def main() -> None:
|
||||
if not TOKEN:
|
||||
logger.error("Discord token missing - set DISCORD_TOKEN or configure netrc")
|
||||
return
|
||||
async def main() -> int:
|
||||
"""Run both halves; return a process exit code (0 = clean shutdown)."""
|
||||
if TOKEN:
|
||||
token_source = (
|
||||
"env DISCORD_TOKEN" if os.getenv("DISCORD_TOKEN") else "netrc file"
|
||||
)
|
||||
logger.info(
|
||||
"Discord token: loaded from %s (length %d)", token_source, len(TOKEN)
|
||||
)
|
||||
|
||||
logger.info("Starting discord bot")
|
||||
shutdown_event = asyncio.Event()
|
||||
@@ -117,6 +285,7 @@ async def main() -> None:
|
||||
comm_task = asyncio.create_task(_run_comm_subroutine(comm_stop_event))
|
||||
bot_task = asyncio.create_task(_run_bot(TOKEN, shutdown_event))
|
||||
|
||||
exit_code = 0
|
||||
try:
|
||||
await shutdown_event.wait()
|
||||
except (KeyboardInterrupt, asyncio.CancelledError):
|
||||
@@ -127,9 +296,31 @@ async def main() -> None:
|
||||
comm_stop_event.set()
|
||||
if not client.is_closed():
|
||||
await client.close()
|
||||
await asyncio.gather(bot_task, comm_task, return_exceptions=True)
|
||||
results = await asyncio.gather(bot_task, comm_task, return_exceptions=True)
|
||||
for result in results:
|
||||
if isinstance(result, BaseException) and not isinstance(
|
||||
result, asyncio.CancelledError
|
||||
):
|
||||
# Already logged with full traceback inside the task; repeat
|
||||
# the one-liner so it is the LAST thing in `docker logs`.
|
||||
logger.critical("Task died: %r", result)
|
||||
exit_code = 1
|
||||
return exit_code
|
||||
|
||||
|
||||
# *================================== Run
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(main())
|
||||
if not TOKEN:
|
||||
# Loud, unmissable and in `docker logs`: this is THE most common cause
|
||||
# of a silent container crash-loop.
|
||||
MSG = (
|
||||
"FATAL: Discord token missing.\n"
|
||||
"Provide it via the DISCORD_TOKEN environment variable or a netrc "
|
||||
"file (machine 'discord') at the path in CONJURER_NETRC_FILE.\n"
|
||||
"Docker: check that your secrets mount exists, e.g.\n"
|
||||
" -v /srv/conjurer/secrets/.netrc:/secrets/.netrc:ro\n"
|
||||
" CONJURER_NETRC_FILE=/secrets/.netrc"
|
||||
)
|
||||
logger.critical(MSG)
|
||||
sys.exit(MSG)
|
||||
sys.exit(asyncio.run(main()))
|
||||
|
||||
@@ -11,9 +11,9 @@ from urllib import request as urequest
|
||||
from flask import Flask, abort, jsonify, request
|
||||
from waitress import serve
|
||||
|
||||
HOST_ADDRESS = os.getenv("CONJURER_DISCORD_HOST", "192.168.1.31")
|
||||
HOST_ADDRESS = os.getenv("CONJURER_DISCORD_HOST", "192.168.1.92")
|
||||
PORT_ADDRESS = int(os.getenv("CONJURER_DISCORD_PORT", "5000"))
|
||||
ICECAST_ADDRESS = os.getenv("CONJURER_ICECAST", "http://192.168.1.15:8000")
|
||||
ICECAST_ADDRESS = os.getenv("CONJURER_ICECAST", "http://192.168.1.12:8000")
|
||||
API_KEY = os.getenv("CONJURER_API_KEY")
|
||||
OUT_COMM_Q = Queue()
|
||||
IN_COMM_Q = Queue()
|
||||
@@ -234,10 +234,13 @@ def comm_subroutine(stop_event: Optional[threading.Event] = None):
|
||||
:param stop_event: optional :class:`threading.Event` shared with the caller
|
||||
to coordinate a cooperative shutdown.
|
||||
"""
|
||||
# logger.setLevel(logging.DEBUG)
|
||||
logger = logging.getLogger("discord")
|
||||
logger.setLevel(logging.DEBUG)
|
||||
logger.info("Started comms")
|
||||
threads = []
|
||||
# NOTE: flask_debug is the dev server bound to the SAME host:port as
|
||||
# waitress - running both kills the comm layer with 'address in use'.
|
||||
# Enable it only INSTEAD of waitress_run, never alongside.
|
||||
# threads.append(threading.Thread(target=flask_debug))
|
||||
threads.append(threading.Thread(target=waitress_run, daemon=True))
|
||||
threads.append(
|
||||
|
||||
+14
@@ -0,0 +1,14 @@
|
||||
"""Pytest bootstrap: make first-party modules importable from the tests.
|
||||
|
||||
The bot modules live at the repository root and the musician service lives in
|
||||
``conjurer_musician/``; neither is an installable package, so we put both on
|
||||
``sys.path`` here.
|
||||
"""
|
||||
import os
|
||||
import sys
|
||||
|
||||
_ROOT = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
for _path in (_ROOT, os.path.join(_ROOT, "conjurer_musician")):
|
||||
if _path not in sys.path:
|
||||
sys.path.insert(0, _path)
|
||||
@@ -4,6 +4,7 @@ This module contains the code for the scrape bot.
|
||||
|
||||
import json
|
||||
import logging
|
||||
import os
|
||||
import random
|
||||
import re
|
||||
import time
|
||||
@@ -15,9 +16,9 @@ from urllib.request import urlopen
|
||||
from requests import ConnectionError as RequestsConnectionError
|
||||
from requests import ConnectTimeout, Timeout
|
||||
|
||||
SCR_DATABASE_PATH = r"C:\\Database\\chunks\\"
|
||||
SCR_FILENAME = "40_chunk.txt"
|
||||
SCR_ENCODING = "utf-8"
|
||||
SCR_DATABASE_PATH = os.getenv("CONJURER_LIBRARIAN_DB_PATH", r"C:\\Database\\chunks\\")
|
||||
SCR_FILENAME = os.getenv("CONJURER_LIBRARIAN_SCRAPE_CHUNK", "40_chunk.txt")
|
||||
SCR_ENCODING = os.getenv("CONJURER_ENCODING", "utf-8")
|
||||
|
||||
WORK_Q = Queue()
|
||||
random.seed()
|
||||
|
||||
@@ -19,22 +19,20 @@ Global Variables:
|
||||
"""
|
||||
|
||||
# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
|
||||
import os
|
||||
from queue import Empty, Queue
|
||||
from threading import Thread
|
||||
import time
|
||||
q = Queue()
|
||||
#TODO: Count number of lines in files and print to approximate on which part of the file search is
|
||||
|
||||
# DATA FOR TEST ONLY
|
||||
# MAXTHREADS = 5
|
||||
# DATABASE_PATH = r'C:\\Database\\chunks_1\\'
|
||||
# Deployment data is environment-overridable so the local DOI database can live
|
||||
# on a mounted volume (Docker/Linux) instead of the hardcoded Windows path.
|
||||
MAXTHREADS = int(os.getenv("CONJURER_LIBRARIAN_MAXTHREADS", "41"))
|
||||
DATABASE_PATH = os.getenv("CONJURER_LIBRARIAN_DB_PATH", r"C:\\Database\\chunks\\")
|
||||
|
||||
# DEPLOYMENT DATA
|
||||
MAXTHREADS = 41
|
||||
DATABASE_PATH = r"C:\\Database\\chunks\\"
|
||||
|
||||
ENCODING = "utf-8"
|
||||
CHUNK = "_chunk.txt"
|
||||
ENCODING = os.getenv("CONJURER_ENCODING", "utf-8")
|
||||
CHUNK = os.getenv("CONJURER_LIBRARIAN_CHUNK", "_chunk.txt")
|
||||
_sentinel = object()
|
||||
WORK_Q_SIZE = 35500000
|
||||
|
||||
@@ -107,7 +105,7 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no,
|
||||
|
||||
for item in result_list:
|
||||
if item["DOI"] in data and not item["exists"]:
|
||||
print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
|
||||
print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item['exists']}")
|
||||
_logger.info(data)
|
||||
_logger.info("HIT")
|
||||
item["exists"] = True
|
||||
|
||||
@@ -108,7 +108,7 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no,
|
||||
print(f"C{no}{alive_no}\r", end="")
|
||||
for item in result_list:
|
||||
if item["DOI"] in data[0] and not item["exists"]:
|
||||
print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
|
||||
print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item['exists']}")
|
||||
item["exists"] = True
|
||||
live_results.append(item)
|
||||
done_check = done_check and item["exists"]
|
||||
|
||||
@@ -1,24 +1,44 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
"""Share-list search/publish helpers for the musician service.
|
||||
|
||||
Paths are environment-overridable and the share database / directory are
|
||||
accessed lazily, so importing this module has no side effects (the previous
|
||||
version ran ``SHARE_DIR.mkdir()`` and read the JSON DB at import time, which
|
||||
crashed on any host without the Pi's ``/var/www`` / ``/var/log`` layout — and
|
||||
made the service untestable).
|
||||
"""
|
||||
import json
|
||||
import os
|
||||
import sys
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
|
||||
# CONFIGURATION
|
||||
JSON_DB = '/var/log/share_scan.json'
|
||||
SHARE_DIR = Path('/var/www/html/share')
|
||||
BASE_URL = 'https://czernobog.pl/share'
|
||||
# CONFIGURATION (env-overridable)
|
||||
JSON_DB = os.getenv("CONJURER_SHARE_DB", "/var/log/share_scan.json")
|
||||
SHARE_DIR = Path(os.getenv("CONJURER_SHARE_DIR", "/var/www/html/share"))
|
||||
BASE_URL = os.getenv("CONJURER_SHARE_BASE_URL", "https://czernobog.pl/share")
|
||||
|
||||
_entries_cache = None
|
||||
|
||||
|
||||
def _ensure_share_dir():
|
||||
SHARE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
# Ensure share directory exists
|
||||
SHARE_DIR.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
def load_db():
|
||||
with open(JSON_DB) as f:
|
||||
return json.load(f)['entries']
|
||||
"""Load share entries, returning [] when the DB is missing/corrupt."""
|
||||
try:
|
||||
with open(JSON_DB) as handle:
|
||||
return json.load(handle).get("entries", [])
|
||||
except (FileNotFoundError, json.JSONDecodeError):
|
||||
return []
|
||||
|
||||
|
||||
def _entries():
|
||||
global _entries_cache
|
||||
if _entries_cache is None:
|
||||
_entries_cache = load_db()
|
||||
return _entries_cache
|
||||
|
||||
ENTRIES = load_db()
|
||||
|
||||
def relevancy(path, keywords):
|
||||
score = 0
|
||||
@@ -28,17 +48,20 @@ def relevancy(path, keywords):
|
||||
score += low.count(kw.lower())
|
||||
return score
|
||||
|
||||
|
||||
def find_matches(count, keywords):
|
||||
scored = []
|
||||
for e in ENTRIES:
|
||||
score = relevancy(e['path'], keywords)
|
||||
for entry in _entries():
|
||||
score = relevancy(entry["path"], keywords)
|
||||
if score > 0:
|
||||
scored.append((score, e['path']))
|
||||
scored.append((score, entry["path"]))
|
||||
scored.sort(reverse=True, key=lambda x: x[0])
|
||||
result = [p for _, p in scored]
|
||||
return result[:count]
|
||||
|
||||
|
||||
def publish(paths):
|
||||
_ensure_share_dir()
|
||||
urls = []
|
||||
for path in paths:
|
||||
token = uuid.uuid4().hex
|
||||
@@ -49,4 +72,3 @@ def publish(paths):
|
||||
pass
|
||||
urls.append(f"{BASE_URL}/{token}")
|
||||
return urls
|
||||
|
||||
|
||||
+98
-11
@@ -107,18 +107,18 @@ if platform in ("linux", "linux2"):
|
||||
DIR_PATH_SADOX = "/mnt/c/Users/mtusz/OneDrive/Dokumenty/Fansadox"
|
||||
|
||||
else:
|
||||
LOGFILE = "/home/pi/Conjurer/discord.log"
|
||||
MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
|
||||
SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
|
||||
MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
|
||||
MUSIC_FOLDER = "/home/pi/MediaShare/mp3/"
|
||||
SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
|
||||
NETRC_FILE = "/home/pi/.netrc"
|
||||
LOGSTORE = "/home/pi/MediaShara/logs/"
|
||||
ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
|
||||
LOGFILE = "./discord.log"
|
||||
MEMORY_FIVE_SIARA = "./pamiec.json"
|
||||
SYSTEM_GPT_SETTINGS = "./system_gpt_settings.json"
|
||||
MEMORY_FIVE_MUZYKA = "./pamiec_muzyki.json"
|
||||
MUSIC_FOLDER = "./"
|
||||
SETTINGS_FILE = "./settings.json"
|
||||
NETRC_FILE = "/srv/conjurer/secrets/.netrc"
|
||||
LOGSTORE = "./logs/"
|
||||
ACCIDENT_LOG = "./accident_log.json"
|
||||
ENCODING = "utf-8"
|
||||
GRAPHICS_PATH = "/home/pi/MediaShare/Conjurer_graphics/"
|
||||
DIR_PATH_SADOX = "/home/pi/MediaShare/Fansadox/"
|
||||
GRAPHICS_PATH = "./Conjurer_graphics/"
|
||||
DIR_PATH_SADOX = "./Fansadox/"
|
||||
|
||||
|
||||
elif platform == "win32":
|
||||
@@ -158,6 +158,23 @@ else:
|
||||
# Values stay as plain strings so existing ``PATH + filename`` concatenation in
|
||||
# the command modules keeps working unchanged.
|
||||
|
||||
# Container-friendly shortcut: point CONJURER_DATA_DIR at a single mounted
|
||||
# volume and every writable data file/dir is rooted under it. The per-variable
|
||||
# CONJURER_* overrides below still take precedence, so granular control remains
|
||||
# possible and the native (Pi) deployment is unaffected when it is unset.
|
||||
_DATA_DIR = os.getenv("CONJURER_DATA_DIR")
|
||||
if _DATA_DIR:
|
||||
LOGFILE = os.path.join(_DATA_DIR, "discord.log")
|
||||
SETTINGS_FILE = os.path.join(_DATA_DIR, "settings.json")
|
||||
MEMORY_FIVE_SIARA = os.path.join(_DATA_DIR, "pamiec.json")
|
||||
MEMORY_FIVE_MUZYKA = os.path.join(_DATA_DIR, "pamiec_muzyki.json")
|
||||
SYSTEM_GPT_SETTINGS = os.path.join(_DATA_DIR, "system_gpt_settings.json")
|
||||
ACCIDENT_LOG = os.path.join(_DATA_DIR, "accident_log.json")
|
||||
LOGSTORE = os.path.join(_DATA_DIR, "logs") + os.sep
|
||||
GRAPHICS_PATH = os.path.join(_DATA_DIR, "Conjurer_graphics") + os.sep
|
||||
MUSIC_FOLDER = os.path.join(_DATA_DIR, "music") + os.sep
|
||||
DIR_PATH_SADOX = os.path.join(_DATA_DIR, "Fansadox") + os.sep
|
||||
|
||||
LOGFILE = os.getenv("CONJURER_LOG_FILE", LOGFILE)
|
||||
NETRC_FILE = os.getenv("CONJURER_NETRC_FILE", NETRC_FILE)
|
||||
SETTINGS_FILE = os.getenv("CONJURER_SETTINGS_FILE", SETTINGS_FILE)
|
||||
@@ -172,6 +189,13 @@ DIR_PATH_SADOX = os.getenv("CONJURER_SADOX_DIR", DIR_PATH_SADOX)
|
||||
ENCODING = os.getenv("CONJURER_ENCODING", ENCODING)
|
||||
SEPARATOR_FILE_PATH = os.getenv("CONJURER_PATH_SEPARATOR", SEPARATOR_FILE_PATH)
|
||||
|
||||
# Voice-recognition transcript dumps. Defaults next to the log file (so on the
|
||||
# Pi it lands in /home/pi/Conjurer/transcripts/, in docker under /data).
|
||||
TRANSCRIPTS_PATH = os.getenv(
|
||||
"CONJURER_TRANSCRIPTS_PATH",
|
||||
os.path.join(os.path.dirname(LOGFILE) or ".", "transcripts") + os.sep,
|
||||
)
|
||||
|
||||
FILE_SERVICE_ADDRESS = os.getenv("CONJURER_FILE_SERVICE", "http://192.168.1.15:5000")
|
||||
RADIO_HARBOR_ADDRESS = os.getenv("CONJURER_RADIO_HARBOR", "http://192.168.1.15:54321")
|
||||
SKIP_TRACK = os.getenv("CONJURER_SKIP_ENDPOINT", "/skip")
|
||||
@@ -185,6 +209,66 @@ PORT_ADDRESS = int(os.getenv("CONJURER_DISCORD_PORT", "5000"))
|
||||
API_SHARED_KEY = os.getenv("CONJURER_API_KEY", "")
|
||||
|
||||
|
||||
# *=========================================== Self-healing runtime layout
|
||||
# A fresh host/volume must never kill the bot at import time. Missing
|
||||
# directories are created and missing state files are seeded - first from the
|
||||
# templates shipped alongside this file (repo checkout / docker image), then
|
||||
# from a safe empty structure. Existing files are never touched, so preserved
|
||||
# history always wins.
|
||||
|
||||
_TEMPLATE_DIR = os.path.dirname(os.path.abspath(__file__))
|
||||
|
||||
|
||||
def _ensure_dir(path: str) -> None:
|
||||
if not path:
|
||||
return
|
||||
try:
|
||||
os.makedirs(path, exist_ok=True)
|
||||
except OSError as exc:
|
||||
logger.warning("Cannot create directory %s: %s", path, exc)
|
||||
|
||||
|
||||
def _seed_file(path: str, template_name: str, empty_content: str) -> None:
|
||||
"""Create *path* from the repo template (or *empty_content*) if missing."""
|
||||
if not path or os.path.exists(path):
|
||||
return
|
||||
_ensure_dir(os.path.dirname(path) or ".")
|
||||
template = os.path.join(_TEMPLATE_DIR, template_name)
|
||||
try:
|
||||
if os.path.exists(template) and os.path.abspath(template) != os.path.abspath(path):
|
||||
import shutil
|
||||
|
||||
shutil.copyfile(template, path)
|
||||
logger.warning("Seeded missing %s from repo template", path)
|
||||
else:
|
||||
with open(path, "w", encoding=ENCODING) as handle:
|
||||
handle.write(empty_content)
|
||||
logger.warning("Created missing %s as empty state", path)
|
||||
except OSError as exc:
|
||||
logger.warning("Cannot seed %s: %s", path, exc)
|
||||
|
||||
|
||||
def _ensure_runtime_layout() -> None:
|
||||
for directory in (
|
||||
os.path.dirname(LOGFILE) or ".",
|
||||
LOGSTORE,
|
||||
GRAPHICS_PATH,
|
||||
MUSIC_FOLDER,
|
||||
TRANSCRIPTS_PATH,
|
||||
):
|
||||
_ensure_dir(directory)
|
||||
|
||||
# (target path, template shipped next to this file, empty fallback)
|
||||
_seed_file(SETTINGS_FILE, "settings.json", "{}")
|
||||
_seed_file(SYSTEM_GPT_SETTINGS, "system_gpt_settings.json", "{}")
|
||||
_seed_file(MEMORY_FIVE_SIARA, "pamiec.json", "[]")
|
||||
_seed_file(MEMORY_FIVE_MUZYKA, "pamiec_muzyki.json", "[]")
|
||||
_seed_file(ACCIDENT_LOG, "accident_log.json", "[]")
|
||||
|
||||
|
||||
_ensure_runtime_layout()
|
||||
|
||||
|
||||
# *=========================================== Defensive state loading
|
||||
def _load_json(path: str, fallback):
|
||||
"""Load JSON from *path*, falling back gracefully on missing/corrupt files."""
|
||||
@@ -251,6 +335,9 @@ else:
|
||||
|
||||
TOKEN = _resolve_token("discord", "DISCORD_TOKEN")
|
||||
|
||||
# Voice recognition (AssemblyAI). None = the voice cog reports and disables.
|
||||
ASSEMBLYAI_API_KEY = _resolve_token("assemblyai", "ASSEMBLYAI_API_KEY")
|
||||
|
||||
if spotipy:
|
||||
_spotify_creds = _load_netrc_credentials("spotipy")
|
||||
if _spotify_creds and SpotifyClientCredentials:
|
||||
|
||||
@@ -0,0 +1,64 @@
|
||||
# Conjurer main Discord bot.
|
||||
# Build from the repository root:
|
||||
# docker build -f docker/Dockerfile.bot -t conjurer-bot .
|
||||
FROM python:3.13-trixie
|
||||
|
||||
# System dependencies:
|
||||
# ffmpeg - audio download/convert (yt_dlp) and Discord voice
|
||||
# libopus0 - Discord voice (PyNaCl / discord-ext-voice-recv)
|
||||
# poppler-utils - pdf2image (librarian/latex previews)
|
||||
# git - required by the git+https entry in requirements
|
||||
# build-essential - native wheels (PyNaCl etc.)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
libopus0 \
|
||||
poppler-utils \
|
||||
git \
|
||||
build-essential \
|
||||
curl \
|
||||
ca-certificates \
|
||||
python3-dev \
|
||||
build-essential \
|
||||
portaudio19-dev \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Optional: Tectonic for the $latex command. Remove this layer if unused.
|
||||
RUN curl -fsSL https://drop-sh.fullyjustified.net | sh \
|
||||
&& mv tectonic /usr/local/bin/tectonic \
|
||||
|| echo "tectonic not installed - the LaTeX feature will be disabled"
|
||||
|
||||
COPY requirements_bot.txt requirements_conan.txt ./
|
||||
RUN pip install --no-cache-dir --upgrade pip \
|
||||
&& pip install --no-cache-dir -r requirements_bot.txt
|
||||
# Best-effort Conan bridge extras: aiomcrcon supports Python <= 3.11 only, so
|
||||
# on this 3.13 image the install fails harmlessly and the conanjurer cog stays
|
||||
# dormant (its imports are guarded).
|
||||
RUN pip install --no-cache-dir -r requirements_conan.txt \
|
||||
|| echo "conan extras skipped - conanjurer cog will stay dormant"
|
||||
|
||||
# Vendored forks take import precedence over the pip packages of the same name,
|
||||
# because /app (the script dir) is first on sys.path when running `python bot.py`.
|
||||
COPY yt_dlp ./yt_dlp
|
||||
COPY spotify_dl ./spotify_dl
|
||||
|
||||
# Bot sources + default config/asset templates. Runtime state (conversation
|
||||
# history etc.) is read from the mounted /data volume via CONJURER_DATA_DIR,
|
||||
# so these committed copies only act as first-run fallbacks.
|
||||
COPY *.py ./
|
||||
COPY settings.json system_gpt_settings.json accident_log.json pamiec.json pamiec_muzyki.json ./
|
||||
COPY fuckery.jpg willowisp.png wod_beacon.jpg ./
|
||||
COPY docker/entrypoint.bot.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
CONJURER_DATA_DIR=/data \
|
||||
CONJURER_DISCORD_HOST=0.0.0.0 \
|
||||
CONJURER_DISCORD_PORT=5000
|
||||
|
||||
VOLUME ["/data"]
|
||||
EXPOSE 5000
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
CMD ["python", "bot.py"]
|
||||
@@ -0,0 +1,27 @@
|
||||
# Conjurer librarian (Crossref search + local DOI database lookup).
|
||||
# Build from the repository root:
|
||||
# docker build -f docker/Dockerfile.librarian -t conjurer-librarian .
|
||||
FROM python:3.11-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY conjurer_librarian/requirements_librarian.txt ./
|
||||
RUN pip install --no-cache-dir --upgrade pip \
|
||||
&& pip install --no-cache-dir -r requirements_librarian.txt requests
|
||||
|
||||
COPY conjurer_librarian/ ./
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
CONJURER_LIBRARIAN_HOST=0.0.0.0 \
|
||||
CONJURER_LIBRARIAN_PORT=5001 \
|
||||
CONJURER_LIBRARIAN_DB_PATH=/doi/
|
||||
|
||||
# The local DOI chunk database (large) is mounted here.
|
||||
VOLUME ["/doi"]
|
||||
EXPOSE 5001
|
||||
|
||||
CMD ["python", "conjurer_librarian.py"]
|
||||
@@ -0,0 +1,35 @@
|
||||
# Conjurer musician (Flask file/playlist service backing the radio).
|
||||
# Build from the repository root:
|
||||
# docker build -f docker/Dockerfile.musician -t conjurer-musician .
|
||||
#
|
||||
# NOTE: this containerises the musician *web service* only. The Liquidsoap
|
||||
# radio (radio_conjurer.liq) and any Samba/NFS share tooling run separately.
|
||||
FROM python:3.11-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY conjurer_musician/requirements_musician.txt ./
|
||||
RUN pip install --no-cache-dir --upgrade pip \
|
||||
&& pip install --no-cache-dir -r requirements_musician.txt
|
||||
|
||||
COPY conjurer_musician/ ./
|
||||
COPY docker/entrypoint.musician.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
CONJURER_MUSICIAN_HOST=0.0.0.0 \
|
||||
CONJURER_MUSICIAN_PORT=5000 \
|
||||
CONJURER_MUSIC_FOLDER=/music \
|
||||
CONJURER_MUSICIAN_BASE=/data \
|
||||
CONJURER_STREAM_TEMPLATE=/app/stream.html
|
||||
|
||||
# /music = the mp3 library (read-only ok); /data = writable playlists + logs.
|
||||
VOLUME ["/music", "/data"]
|
||||
EXPOSE 5000
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
CMD ["python", "conjurer_musician.py"]
|
||||
@@ -0,0 +1,22 @@
|
||||
# Main bot VM. Run from the repository root:
|
||||
# cp docker/env/bot.env.example docker/env/bot.env # then edit
|
||||
# docker compose -f docker/compose.bot.yaml up -d --build
|
||||
services:
|
||||
conjurer-bot:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile.bot
|
||||
image: conjurer-bot:latest
|
||||
container_name: conjurer-bot
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- env/bot.env
|
||||
ports:
|
||||
# Flask comm layer — musician/librarian POST results here (/prepped_tracks, /conjurer).
|
||||
- "5000:5000"
|
||||
volumes:
|
||||
# Persistent state (conversation history, settings, logs). Populate this
|
||||
# host dir with your existing pamiec.json etc. to preserve command history.
|
||||
- /srv/conjurer/data:/data
|
||||
# Tokens: a read-only netrc covers discord/openai/spotipy/youtube in one file.
|
||||
- /srv/conjurer/secrets/.netrc:/secrets/.netrc:ro
|
||||
@@ -0,0 +1,20 @@
|
||||
# Librarian VM. Run from the repository root:
|
||||
# cp docker/env/librarian.env.example docker/env/librarian.env # then edit
|
||||
# docker compose -f docker/compose.librarian.yaml up -d --build
|
||||
services:
|
||||
conjurer-librarian:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile.librarian
|
||||
image: conjurer-librarian:latest
|
||||
container_name: conjurer-librarian
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- env/librarian.env
|
||||
ports:
|
||||
- "5001:5001"
|
||||
volumes:
|
||||
# Local DOI chunk database (0_chunk.txt ... N_chunk.txt).
|
||||
- /srv/librarian/doi:/doi:ro
|
||||
# Optional: netrc holding Crossref credentials (or use CONJURER_CROSSREF_MAILTO).
|
||||
- /srv/librarian/secrets/.netrc:/secrets/.netrc:ro
|
||||
@@ -0,0 +1,21 @@
|
||||
# Musician VM (optional — you plan to adapt/implement this yourself).
|
||||
# Run from the repository root:
|
||||
# cp docker/env/musician.env.example docker/env/musician.env # then edit
|
||||
# docker compose -f docker/compose.musician.yaml up -d --build
|
||||
services:
|
||||
conjurer-musician:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile.musician
|
||||
image: conjurer-musician:latest
|
||||
container_name: conjurer-musician
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- env/musician.env
|
||||
ports:
|
||||
- "5000:5000"
|
||||
volumes:
|
||||
# The mp3 library the service indexes and serves.
|
||||
- /srv/musician/music:/music
|
||||
# Runtime playlists/logs the service writes.
|
||||
- /srv/musician/data:/data
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
# Seed the data volume with the baked-in default JSON state on first run only.
|
||||
# Existing files (e.g. your preserved pamiec.json history) are never overwritten.
|
||||
set -e
|
||||
|
||||
DATA="${CONJURER_DATA_DIR:-/data}"
|
||||
mkdir -p "$DATA"
|
||||
|
||||
for f in settings.json system_gpt_settings.json pamiec.json pamiec_muzyki.json accident_log.json; do
|
||||
if [ ! -e "$DATA/$f" ] && [ -e "/app/$f" ]; then
|
||||
cp "/app/$f" "$DATA/$f"
|
||||
echo "entrypoint: seeded $f into $DATA"
|
||||
fi
|
||||
done
|
||||
|
||||
exec "$@"
|
||||
Executable
+23
@@ -0,0 +1,23 @@
|
||||
#!/bin/sh
|
||||
# Prepare the musician's writable volume so the web service starts cleanly on a
|
||||
# fresh container. Existing files are never overwritten (preserves your data).
|
||||
set -e
|
||||
|
||||
DATA="${CONJURER_MUSICIAN_BASE:-/data}"
|
||||
MUSIC="${CONJURER_MUSIC_FOLDER:-/music}"
|
||||
mkdir -p "$DATA" "$DATA/logs" "$MUSIC"
|
||||
|
||||
# The track-forwarding thread tails the Liquidsoap radio logs. When the radio
|
||||
# runs separately (or hasn't started yet) these files may not exist; create
|
||||
# them empty so the tailer waits instead of crashing.
|
||||
for f in radio_log.log persistence.log; do
|
||||
[ -e "$DATA/$f" ] || : > "$DATA/$f"
|
||||
done
|
||||
|
||||
# Ensure the managed playlists exist (routes/rescan also create them; this just
|
||||
# avoids a first-tick race before the initial scan).
|
||||
for f in all_playlist.playlist hit.playlist request.playlist priority_queue.playlist; do
|
||||
[ -e "$DATA/$f" ] || : > "$DATA/$f"
|
||||
done
|
||||
|
||||
exec "$@"
|
||||
Vendored
+39
@@ -0,0 +1,39 @@
|
||||
# Copy to docker/env/bot.env and fill in. Do NOT commit the real file.
|
||||
|
||||
# --- Secrets ------------------------------------------------------------
|
||||
# Option A: mount a netrc (recommended — covers discord/openai/spotipy/youtube).
|
||||
CONJURER_NETRC_FILE=/secrets/.netrc
|
||||
# Option B: pass tokens directly (these take precedence over netrc).
|
||||
# DISCORD_TOKEN=
|
||||
# OPENAI_API_KEY=
|
||||
# ASSEMBLYAI_API_KEY= # voice recognition; netrc machine 'assemblyai' works too
|
||||
# YOUTUBE_USERNAME=
|
||||
# YOUTUBE_PASSWORD=
|
||||
|
||||
# --- Data ---------------------------------------------------------------
|
||||
# Single mounted volume; all writable state is rooted here.
|
||||
CONJURER_DATA_DIR=/data
|
||||
|
||||
# --- Flask comm layer (inbound from musician/librarian) -----------------
|
||||
CONJURER_DISCORD_HOST=0.0.0.0
|
||||
CONJURER_DISCORD_PORT=5000
|
||||
|
||||
# --- Internal service auth ----------------------------------------------
|
||||
# Set the SAME value on bot + musician + librarian. Empty = auth disabled.
|
||||
CONJURER_API_KEY=
|
||||
|
||||
# --- Where the bot reaches the other services (other Proxmox VMs) --------
|
||||
CONJURER_FILE_SERVICE=http://MUSICIAN_VM_IP:5000
|
||||
CONJURER_RADIO_HARBOR=http://MUSICIAN_VM_IP:54321
|
||||
CONJURER_LIBRARIAN_SERVICE=http://LIBRARIAN_VM_IP:5001
|
||||
|
||||
# --- Conan Exiles bridge (optional; empty/0 = disabled) -----------------
|
||||
# CONAN_GM_ROLE_ID=0
|
||||
# CONAN_RCON_HOST=
|
||||
# CONAN_RCON_PORT=25575
|
||||
# CONAN_RCON_PASSWORD=
|
||||
# CONAN_CHAT_CHANNEL_ID=0
|
||||
# CONAN_EVENTS_CHANNEL_ID=0
|
||||
# CONAN_JOIN_CHANNEL_ID=0
|
||||
# CONAN_LOG_MODE=local
|
||||
# CONAN_LOG_PATH=
|
||||
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
# Copy to docker/env/librarian.env and fill in.
|
||||
|
||||
CONJURER_LIBRARIAN_HOST=0.0.0.0
|
||||
CONJURER_LIBRARIAN_PORT=5001
|
||||
|
||||
# Same shared secret as the bot (empty = auth disabled).
|
||||
CONJURER_API_KEY=
|
||||
|
||||
# Where to POST search results back to (the bot's comm layer).
|
||||
CONJURER_MAIN_BOT=http://BOT_VM_IP:5000
|
||||
|
||||
# Crossref polite-pool contact (or put credentials in netrc under "crossref").
|
||||
CONJURER_CROSSREF_MAILTO=you@example.com
|
||||
|
||||
# Local DOI chunk database (mounted volume): expects 0_chunk.txt .. N_chunk.txt
|
||||
CONJURER_LIBRARIAN_DB_PATH=/doi/
|
||||
CONJURER_LIBRARIAN_MAXTHREADS=41
|
||||
CONJURER_LIBRARIAN_CHUNK=_chunk.txt
|
||||
|
||||
# Optional netrc (for Crossref credentials)
|
||||
CONJURER_NETRC_FILE=/secrets/.netrc
|
||||
Vendored
+17
@@ -0,0 +1,17 @@
|
||||
# Copy to docker/env/musician.env and fill in.
|
||||
|
||||
CONJURER_MUSICIAN_HOST=0.0.0.0
|
||||
CONJURER_MUSICIAN_PORT=5000
|
||||
|
||||
# Same shared secret as the bot (empty = auth disabled).
|
||||
CONJURER_API_KEY=
|
||||
|
||||
# Where to POST "now playing" / prepped-track updates (the bot's comm layer).
|
||||
CONJURER_MAIN_BOT=http://BOT_VM_IP:5000
|
||||
|
||||
# The mp3 library (mounted volume).
|
||||
CONJURER_MUSIC_FOLDER=/music
|
||||
|
||||
# Runtime paths (mounted volume) — playlists/logs the service writes.
|
||||
CONJURER_MUSICIAN_BASE=/data
|
||||
CONJURER_LOGSTORE=/data/logs
|
||||
@@ -0,0 +1,292 @@
|
||||
# Conjurer on Docker / Proxmox
|
||||
|
||||
Runbook for running Conjurer as Docker containers across Proxmox VMs:
|
||||
|
||||
| Component | VM | Container | Port | Image |
|
||||
|-----------|----|-----------|------|-------|
|
||||
| **Main bot** | VM-bot | `conjurer-bot` | 5000 (Flask comm) | `docker/Dockerfile.bot` |
|
||||
| **Librarian** | VM-librarian | `conjurer-librarian` | 5001 | `docker/Dockerfile.librarian` |
|
||||
| **Musician** | VM-musician | `conjurer-musician` | 5000 (+ radio) | `docker/Dockerfile.musician` |
|
||||
|
||||
The three talk to each other over HTTP on the Proxmox LAN. Direction of calls:
|
||||
|
||||
```
|
||||
bot --(/mp3,/get_music,/add_to_priority,...)--> musician
|
||||
bot --(/query)--------------------------------> librarian
|
||||
musician --(/prepped_tracks)--------------------> bot
|
||||
librarian --(/conjurer results)-----------------> bot
|
||||
```
|
||||
|
||||
Everything is configured through `CONJURER_*` environment variables (see the
|
||||
`docker/env/*.env.example` files). Nothing is hardcoded to a host path anymore.
|
||||
|
||||
---
|
||||
|
||||
## 0. Prerequisites (per VM)
|
||||
|
||||
Create a small Linux VM in Proxmox (Debian 12 / Ubuntu 22.04+ is fine), then
|
||||
install Docker:
|
||||
|
||||
```bash
|
||||
sudo apt-get update && sudo apt-get install -y ca-certificates curl git
|
||||
curl -fsSL https://get.docker.com | sh
|
||||
sudo usermod -aG docker "$USER" # log out/in afterwards
|
||||
```
|
||||
|
||||
Clone the repo on each VM (they build from it):
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /opt && cd /opt
|
||||
git clone https://github.com/migatu/conjurer.git
|
||||
cd conjurer
|
||||
```
|
||||
|
||||
> All `docker compose` commands below are run **from the repo root** (`/opt/conjurer`),
|
||||
> because the compose files use `context: ..` relative to `docker/`.
|
||||
|
||||
---
|
||||
|
||||
## 1. Main bot (VM-bot)
|
||||
|
||||
### 1a. Prepare host directories
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /srv/conjurer/data /srv/conjurer/secrets
|
||||
```
|
||||
|
||||
### 1b. Preserve existing command history ⭐
|
||||
|
||||
The bot's conversation memory and settings live in JSON files. Copy them from
|
||||
your current deployment (e.g. the Pi's `/home/pi/Conjurer/`) into the data
|
||||
volume so the history carries over:
|
||||
|
||||
```bash
|
||||
# run on the Pi, or scp the files across, then place them here:
|
||||
sudo cp pamiec.json /srv/conjurer/data/ # AI conversation history
|
||||
sudo cp pamiec_muzyki.json /srv/conjurer/data/ # music-DJ memory
|
||||
sudo cp settings.json /srv/conjurer/data/ # word/cyclic reactions
|
||||
sudo cp system_gpt_settings.json /srv/conjurer/data/
|
||||
sudo cp accident_log.json /srv/conjurer/data/ # if present
|
||||
```
|
||||
|
||||
If you skip this, the container starts with the (empty) template files baked
|
||||
into the image and history begins fresh.
|
||||
|
||||
### 1c. Tokens
|
||||
|
||||
Drop your existing netrc (the one with `discord`, `openai`, `spotipy`,
|
||||
`youtube` entries) into the secrets dir:
|
||||
|
||||
```bash
|
||||
sudo cp ~/.netrc /srv/conjurer/secrets/.netrc
|
||||
sudo chmod 600 /srv/conjurer/secrets/.netrc
|
||||
```
|
||||
|
||||
(Alternatively skip netrc and set `DISCORD_TOKEN` / `OPENAI_API_KEY` in the env
|
||||
file — those take precedence.)
|
||||
|
||||
### 1d. Configure and launch
|
||||
|
||||
```bash
|
||||
cp docker/env/bot.env.example docker/env/bot.env
|
||||
# edit docker/env/bot.env: set CONJURER_FILE_SERVICE / _LIBRARIAN_SERVICE to the
|
||||
# other VMs' IPs, and CONJURER_API_KEY (same value on all three) if you want auth.
|
||||
|
||||
docker compose -f docker/compose.bot.yaml up -d --build
|
||||
docker logs -f conjurer-bot
|
||||
```
|
||||
|
||||
Look for `Extension loaded: …` for each cog and `All systems: operational`.
|
||||
|
||||
### 1e. Startup model: core cogs vs service-gated cogs
|
||||
|
||||
The bot **always** starts with the cogs that depend on nothing but itself
|
||||
(administration, AI, other, latex, voice, conanjurer). Cogs that need a
|
||||
sibling service are **health-gated**:
|
||||
|
||||
| Group | Cogs | Enabled when |
|
||||
|-------|------|--------------|
|
||||
| musician | `music_commands`, `radio_commands`, `file_search_commands` | `GET {FILE_SERVICE}/mp3` answers |
|
||||
| librarian | `librarian_commands` | librarian answers HTTP at all |
|
||||
|
||||
When a service is down its cogs stay disabled (commands simply don't exist)
|
||||
and the log says so. A watchdog re-checks every 5 minutes and enables the
|
||||
cogs the moment the service starts answering — no bot restart needed.
|
||||
A single broken cog (missing pip package, bad import) is skipped with a full
|
||||
traceback in the log; it never takes the whole bot down.
|
||||
|
||||
### 1f. Troubleshooting a crash-looping container
|
||||
|
||||
`docker logs conjurer-bot` now shows the real reason (the bot logs to stdout
|
||||
as well as the rotating file). The most common cases:
|
||||
|
||||
- **`FATAL: Discord token missing`** — the secrets mount is missing/empty or
|
||||
`CONJURER_NETRC_FILE` points elsewhere. Check:
|
||||
`docker inspect -f '{{json .Mounts}}' conjurer-bot | jq` and
|
||||
`docker exec conjurer-bot ls -la /secrets/` (after a manual
|
||||
`docker run … sleep infinity` if it crash-loops too fast).
|
||||
- **Missing state files** — not fatal anymore: missing dirs are created and
|
||||
missing JSON state is seeded from the repo templates baked into the image
|
||||
(existing files are never overwritten). Fix the mount at your leisure.
|
||||
|
||||
**About files "disappearing" from `/srv/conjurer/...`:** nothing in this stack
|
||||
deletes host files — the entrypoint and the bot only ever *create* missing
|
||||
files. With a bind mount, `/srv/conjurer/data` **is** the live state (not an
|
||||
installation staging area): don't delete it after a successful install.
|
||||
If files vanished, the usual suspects are `docker compose down -v` (only
|
||||
affects *named* volumes, not binds), a re-provisioned VM, or copying the files
|
||||
to a different path than the one in the compose `volumes:` line — verify with
|
||||
`docker inspect -f '{{json .Mounts}}' conjurer-bot`.
|
||||
|
||||
---
|
||||
|
||||
## 2. Librarian (VM-librarian)
|
||||
|
||||
### 2a. Mount the DOI database
|
||||
|
||||
The librarian checks keyword hits from Crossref against a local database of
|
||||
DOI chunk files (`0_chunk.txt … N_chunk.txt`). Put that database on the VM and
|
||||
point the volume at it:
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /srv/librarian/doi /srv/librarian/secrets
|
||||
# copy/replicate your chunk files into /srv/librarian/doi/
|
||||
```
|
||||
|
||||
> The old Windows path `C:\Database\chunks\` is now `CONJURER_LIBRARIAN_DB_PATH`
|
||||
> (defaults to `/doi/` in the container). `CONJURER_LIBRARIAN_MAXTHREADS` (41)
|
||||
> and `CONJURER_LIBRARIAN_CHUNK` (`_chunk.txt`) are configurable too.
|
||||
|
||||
### 2b. Configure and launch
|
||||
|
||||
```bash
|
||||
cp docker/env/librarian.env.example docker/env/librarian.env
|
||||
# edit: CONJURER_MAIN_BOT=http://BOT_VM_IP:5000, CONJURER_CROSSREF_MAILTO, CONJURER_API_KEY
|
||||
|
||||
docker compose -f docker/compose.librarian.yaml up -d --build
|
||||
docker logs -f conjurer-librarian
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Musician (VM-musician)
|
||||
|
||||
The web service is container-ready (`docker/Dockerfile.musician` +
|
||||
`compose.musician.yaml`). It containerises the **Flask file/playlist service
|
||||
only** — the Liquidsoap radio (`radio_conjurer.liq`), `script.params` and any
|
||||
Samba/NFS share tooling are separate and typically stay on the host or a
|
||||
dedicated setup (you'll adapt those yourself).
|
||||
|
||||
### 3a. Two volumes: the library and the writable state
|
||||
|
||||
| Mount | Container path | Holds |
|
||||
|-------|---------------|-------|
|
||||
| `/srv/musician/music` | `/music` (`CONJURER_MUSIC_FOLDER`) | your mp3 library (indexed/served) |
|
||||
| `/srv/musician/data` | `/data` (`CONJURER_MUSICIAN_BASE`) | playlists, logs, `radio_log.log`/`persistence.log` |
|
||||
|
||||
```bash
|
||||
sudo mkdir -p /srv/musician/music /srv/musician/data
|
||||
# point /srv/musician/music at (or copy in) your mp3s
|
||||
```
|
||||
|
||||
### 3b. Preserve existing playlists/state (optional)
|
||||
|
||||
If you already run the musician, copy its working playlists into the data
|
||||
volume so nothing is regenerated from scratch:
|
||||
|
||||
```bash
|
||||
sudo cp all_playlist.playlist hit.playlist request.playlist \
|
||||
priority_queue.playlist playlist.json /srv/musician/data/ 2>/dev/null || true
|
||||
```
|
||||
|
||||
The container's entrypoint (`docker/entrypoint.musician.sh`) creates any
|
||||
missing playlists and touches `radio_log.log` / `persistence.log` empty so the
|
||||
track-forwarding thread waits instead of crashing when the radio runs
|
||||
elsewhere. It never overwrites files you copied in.
|
||||
|
||||
### 3c. Configure and launch
|
||||
|
||||
```bash
|
||||
cp docker/env/musician.env.example docker/env/musician.env
|
||||
# edit: CONJURER_MAIN_BOT=http://BOT_VM_IP:5000 and CONJURER_API_KEY (match the bot)
|
||||
|
||||
docker compose -f docker/compose.musician.yaml up -d --build
|
||||
docker logs -f conjurer-musician
|
||||
```
|
||||
|
||||
### 3d. Radio coupling (if you keep Liquidsoap separate)
|
||||
|
||||
The musician only forwards "now playing" to the bot by tailing the radio's
|
||||
`radio_log.log` / `persistence.log`. To wire them up, have Liquidsoap write
|
||||
those two files into the same `/srv/musician/data` directory (or set
|
||||
`CONJURER_RADIO_LOG` / `CONJURER_PERSISTENCE_LOG` to wherever it writes). The
|
||||
`/stream` page template is served from the baked-in `/app/stream.html`
|
||||
(override with `CONJURER_STREAM_TEMPLATE` if you customise it).
|
||||
|
||||
---
|
||||
|
||||
## 4. Networking & auth
|
||||
|
||||
- Open the ports between VMs on the Proxmox LAN: **bot 5000**, **librarian 5001**,
|
||||
**musician 5000** (+ radio harbor 54321 if used). A simple `ufw allow from
|
||||
<lan-subnet>` per port is enough; do not expose them to the internet.
|
||||
- **Auth:** set the same `CONJURER_API_KEY` in all three `*.env` files. Then
|
||||
every internal call carries `X-Conjurer-Api-Key` and each service rejects
|
||||
requests without it (HTTP 401). Leave it empty everywhere to disable auth
|
||||
(fully backward compatible). ⚠️ Setting it on only one side breaks the link.
|
||||
- The addresses point at each other by VM IP (or a DNS name). Set:
|
||||
- bot: `CONJURER_FILE_SERVICE`, `CONJURER_RADIO_HARBOR`, `CONJURER_LIBRARIAN_SERVICE`
|
||||
- librarian & musician: `CONJURER_MAIN_BOT`
|
||||
|
||||
---
|
||||
|
||||
## 5. Verify
|
||||
|
||||
```bash
|
||||
# bot is up and reachable from another VM:
|
||||
curl http://BOT_VM_IP:5000/conjurer # -> "ALIVE"
|
||||
# librarian answers:
|
||||
curl http://LIBRARIAN_VM_IP:5001/ -I # service reachable
|
||||
docker ps # all containers "Up"
|
||||
docker logs conjurer-bot --tail 50
|
||||
```
|
||||
|
||||
In Discord, exercise a command that round-trips through a service (e.g. a music
|
||||
search that hits the musician, or a librarian query) to confirm the wiring and
|
||||
the API key.
|
||||
|
||||
---
|
||||
|
||||
## 6. Updates
|
||||
|
||||
```bash
|
||||
cd /opt/conjurer && git pull
|
||||
docker compose -f docker/compose.bot.yaml up -d --build # rebuild + restart
|
||||
```
|
||||
|
||||
Data in `/srv/.../data` and `/doi` / `/music` volumes survives rebuilds, so
|
||||
history and databases persist across updates.
|
||||
|
||||
---
|
||||
|
||||
## 7. Rollback / coexistence
|
||||
|
||||
- The native (Raspberry Pi / systemd) deployment is unaffected — none of the
|
||||
defaults changed; the container behaviour is opt-in via `CONJURER_DATA_DIR`
|
||||
and the other env vars. You can run both during migration.
|
||||
- To roll back a VM: `docker compose -f docker/compose.<svc>.yaml down` and
|
||||
restart the previous deployment. The JSON state in `/srv/.../data` is plain
|
||||
files you can copy back to the Pi if needed.
|
||||
|
||||
---
|
||||
|
||||
## Notes on the images
|
||||
|
||||
- **Bot** uses the vendored `yt_dlp/` and `spotify_dl/` forks (they win over the
|
||||
pip packages because `/app` is first on `sys.path`), so your patches stay
|
||||
active without the old `sed` hacks from `install_main_bot.sh`.
|
||||
- **Tectonic** (LaTeX `$latex` command) is installed best-effort; if the build
|
||||
step fails the bot still runs, just without LaTeX. Remove that layer from
|
||||
`Dockerfile.bot` if you don't need it.
|
||||
- Voice needs `ffmpeg` + `libopus0` (both in the image). No microphone/pyaudio
|
||||
is required — voice is received over Discord and transcribed via AssemblyAI.
|
||||
@@ -0,0 +1,231 @@
|
||||
# Migracja: „working copy" → „prototype"
|
||||
|
||||
Runbook upgrade'u działającego deploymentu bota (Raspberry Pi) z wersji
|
||||
**working copy** (kod w roocie repo, bez auth między usługami, bez integracji
|
||||
Conan) na wersję **prototype** (konfiguracja przez zmienne środowiskowe,
|
||||
opcjonalna autoryzacja wewnętrznych wywołań HTTP, integracja Conan Exiles).
|
||||
|
||||
> **Najważniejsze:** upgrade jest **wstecznie kompatybilny**. Bez ustawiania
|
||||
> żadnych nowych zmiennych bot działa tak jak dotąd — istniejące tokeny z
|
||||
> `~/.netrc` i domyślne ścieżki są zachowane. Wszystkie nowe funkcje
|
||||
> (autoryzacja API, most Conan, powiadomienia o graczach) są **opt-in**.
|
||||
|
||||
---
|
||||
|
||||
## 0. Status / warunek wstępny
|
||||
|
||||
Pełny kod prototype znajduje się obecnie na branchu **`proto-improvements`**.
|
||||
Na `main` jest na razie sama restrukturyzacja (working copy w roocie). Zanim
|
||||
zmigrujesz produkcję z `main`, zmerguj prototype do `main`
|
||||
(`proto-improvements` → `main`) albo deployuj bezpośrednio z brancha
|
||||
`proto-improvements`. Dalsza część zakłada, że prototype jest już dostępny pod
|
||||
refem, który checkoutujesz w kroku 2.
|
||||
|
||||
**Układ deploymentu (bez zmian):**
|
||||
|
||||
| | Ścieżka |
|
||||
|---|---|
|
||||
| Klon repo | `/home/pi/conjurer` |
|
||||
| Runtime bota | `/home/pi/Conjurer` |
|
||||
| Virtualenv | `/home/pi/Conjurer/env` (używany przez `conjurer.service`) |
|
||||
| Usługa systemd | `conjurer.service` → `ExecStart … /home/pi/Conjurer/bot.py` |
|
||||
| Deploy | `deploy.sh` (kopiuje pliki z repo do runtime i restartuje usługę) |
|
||||
|
||||
---
|
||||
|
||||
## 1. Backup i punkt powrotu
|
||||
|
||||
```bash
|
||||
# zapamiętaj aktualny commit (do ewentualnego rollbacku)
|
||||
git -C /home/pi/conjurer rev-parse HEAD > /home/pi/conjurer_rollback_commit.txt
|
||||
|
||||
# snapshot runtime (config + dane)
|
||||
sudo cp -a /home/pi/Conjurer /home/pi/Conjurer.bak.$(date +%Y%m%d_%H%M%S)
|
||||
```
|
||||
|
||||
Sekrety nadal pochodzą z `~/.netrc` (Discord/OpenAI/Spotify/YouTube) — upewnij
|
||||
się, że ten plik istnieje i jest aktualny. Migracja go nie dotyka.
|
||||
|
||||
---
|
||||
|
||||
## 2. Pobranie kodu prototype
|
||||
|
||||
```bash
|
||||
cd /home/pi/conjurer
|
||||
git fetch --all
|
||||
git checkout main && git pull # gdy prototype jest już w main
|
||||
# albo, do czasu merge: git checkout proto-improvements && git pull
|
||||
```
|
||||
|
||||
---
|
||||
|
||||
## 3. Instalacja nowych zależności
|
||||
|
||||
Wersja prototype dodaje do `requirements_bot.txt` dwa pakiety używane przez most
|
||||
Conan: **`aiomcrcon`** i **`asyncssh`**.
|
||||
|
||||
```bash
|
||||
source /home/pi/Conjurer/env/bin/activate
|
||||
python3 -m pip install -r /home/pi/conjurer/requirements_bot.txt
|
||||
deactivate
|
||||
```
|
||||
|
||||
> Nawet bez tych pakietów bot się uruchomi — importy w module Conan są osłonięte
|
||||
> (`try/except ImportError`), a integracja pozostaje uśpiona. Instalacja jest
|
||||
> potrzebna tylko jeśli faktycznie używasz mostu Conan.
|
||||
|
||||
---
|
||||
|
||||
## 4. (Opcjonalnie) Konfiguracja zmiennych środowiskowych
|
||||
|
||||
Wersja prototype czyta konfigurację ze zmiennych środowiskowych z fallbackiem na
|
||||
dotychczasowe wartości. **Pomiń ten krok dla zwykłego upgrade'u** — domyślne
|
||||
ścieżki i `~/.netrc` wystarczą. Ustaw zmienne tylko gdy włączasz nową funkcję.
|
||||
|
||||
### 4a. Plik środowiskowy dla systemd
|
||||
|
||||
```bash
|
||||
sudo tee /home/pi/Conjurer/conjurer.env >/dev/null <<'EOF'
|
||||
# --- Autoryzacja wewnętrznych wywołań HTTP (opcjonalne) ---
|
||||
# Ten sam klucz MUSI być ustawiony na bocie, musicianie i librarianie.
|
||||
CONJURER_API_KEY=
|
||||
|
||||
# --- Adresy usług wewnętrznych (domyślne wartości jak dotąd) ---
|
||||
#CONJURER_FILE_SERVICE=http://192.168.1.15:5000
|
||||
#CONJURER_RADIO_HARBOR=http://192.168.1.15:54321
|
||||
#CONJURER_LIBRARIAN_SERVICE=http://192.168.1.192:5001
|
||||
|
||||
# --- Most Conan Exiles (opcjonalne; puste = wyłączone) ---
|
||||
#CONAN_GM_ROLE_ID=0
|
||||
#CONAN_RCON_HOST=
|
||||
#CONAN_RCON_PORT=25575
|
||||
#CONAN_RCON_PASSWORD=
|
||||
#CONAN_CHAT_CHANNEL_ID=0
|
||||
#CONAN_EVENTS_CHANNEL_ID=0
|
||||
# Powiadomienia o wejściu gracza — ustaw id kanału, by włączyć:
|
||||
#CONAN_JOIN_CHANNEL_ID=0
|
||||
#CONAN_PLAYER_POLL_SECONDS=60
|
||||
#CONAN_LOG_MODE=local
|
||||
#CONAN_LOG_PATH=
|
||||
EOF
|
||||
sudo chmod 600 /home/pi/Conjurer/conjurer.env
|
||||
```
|
||||
|
||||
### 4b. Podłączenie pliku do usługi
|
||||
|
||||
Dodaj `EnvironmentFile` do sekcji `[Service]` w `conjurer.service`
|
||||
(`/etc/systemd/system/conjurer.service`):
|
||||
|
||||
```ini
|
||||
[Service]
|
||||
Type=simple
|
||||
EnvironmentFile=/home/pi/Conjurer/conjurer.env
|
||||
ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/bot.py
|
||||
Restart=on-abort
|
||||
```
|
||||
|
||||
```bash
|
||||
sudo systemctl daemon-reload
|
||||
```
|
||||
|
||||
> `conjurer.service` w repo również warto zaktualizować o tę linię, ale `deploy.sh`
|
||||
> **nie** nadpisuje jednostki systemd przy każdym deployu — wpis robisz raz, ręcznie.
|
||||
|
||||
---
|
||||
|
||||
## 5. Deploy i restart
|
||||
|
||||
```bash
|
||||
cd /home/pi/conjurer
|
||||
./deploy.sh # kopiuje m.in. bot.py oraz conanjurer_commands/_functions.py do runtime
|
||||
```
|
||||
|
||||
`deploy.sh` na końcu sam wykonuje `systemctl restart conjurer.service`. Jeśli
|
||||
zmieniałeś jednostkę systemd w kroku 4b, wcześniej zrób `daemon-reload`.
|
||||
|
||||
---
|
||||
|
||||
## 6. Weryfikacja
|
||||
|
||||
```bash
|
||||
journalctl -u conjurer.service -f
|
||||
```
|
||||
|
||||
Czego szukać w logu (`/home/pi/Conjurer` → plik logu również):
|
||||
|
||||
- `Loading … module done` dla kolejnych rozszerzeń, w tym **`Loading conanjurer commands module done`**
|
||||
- Gdy most Conan nieskonfigurowany: `Conan: log watch disabled (not configured)`
|
||||
oraz `Conan: player-join notifications disabled (no channel or RCON)` — to
|
||||
oczekiwane, integracja jest uśpiona
|
||||
- `All systems: operational`
|
||||
|
||||
Szybki test funkcjonalny: bot łączy się z Discordem, dotychczasowe komendy
|
||||
działają, radio/biblioteka odpowiadają jak wcześniej.
|
||||
|
||||
---
|
||||
|
||||
## 7. Zmiany zachowania, o których warto wiedzieć
|
||||
|
||||
| Obszar | Working copy | Prototype |
|
||||
|---|---|---|
|
||||
| Konfiguracja | zahardkodowana per-platforma | env-vary z fallbackiem na stare wartości |
|
||||
| Tokeny | tylko `~/.netrc` | env-var → fallback `~/.netrc` (stare działa) |
|
||||
| Wewnętrzne HTTP (music/radio/librarian) | bez nagłówków | wysyła `X-Conjurer-Api-Key`, **gdy** `CONJURER_API_KEY` ustawione |
|
||||
| Endpointy przychodzące bota | bez weryfikacji | `_authorize_request()` zwraca 401 przy złym kluczu (no-op gdy klucz pusty) |
|
||||
| Pętla bota | wątki + `join()` | pojedyncza pętla asyncio z czystym shutdownem |
|
||||
| Moduł Conan | obecny, **nieładowany** (miał SyntaxError) | naprawiony i ładowany; uśpiony bez konfiguracji |
|
||||
|
||||
---
|
||||
|
||||
## 8. Włączanie funkcji opcjonalnych
|
||||
|
||||
### 8a. Autoryzacja wewnętrznych wywołań HTTP
|
||||
Ustaw **ten sam** `CONJURER_API_KEY` na **wszystkich** usługach: bocie,
|
||||
musicianie (`conjurer_musician`) i librarianie. Po ustawieniu:
|
||||
- bot dokleja nagłówek do wywołań do file-service/radia/biblioteki,
|
||||
- usługi odrzucają (401) żądania bez poprawnego klucza.
|
||||
|
||||
⚠️ Ustawienie klucza tylko po jednej stronie zepsuje komunikację (401). Albo
|
||||
wszędzie, albo nigdzie.
|
||||
|
||||
### 8b. Most Conan Exiles
|
||||
Ustaw `CONAN_RCON_HOST` + `CONAN_RCON_PASSWORD` (komendy GM `say/players/kick/
|
||||
rcon/ogłoś`) oraz, dla mirrorowania czatu/zdarzeń, `CONAN_LOG_PATH`
|
||||
(+ `CONAN_CHAT_CHANNEL_ID`/`CONAN_EVENTS_CHANNEL_ID`).
|
||||
|
||||
### 8c. Powiadomienia o wejściu gracza
|
||||
Ustaw **`CONAN_JOIN_CHANNEL_ID`** na id kanału Discord. Funkcja odpytuje RCON
|
||||
`listplayers` co `CONAN_PLAYER_POLL_SECONDS` i ogłasza nowych graczy. Pozostaw
|
||||
`0`, aby trzymać ją wyłączoną.
|
||||
|
||||
---
|
||||
|
||||
## 9. Rollback
|
||||
|
||||
```bash
|
||||
cd /home/pi/conjurer
|
||||
git checkout "$(cat /home/pi/conjurer_rollback_commit.txt)"
|
||||
./deploy.sh
|
||||
sudo systemctl restart conjurer.service
|
||||
```
|
||||
|
||||
W razie potrzeby przywróć snapshot runtime z `/home/pi/Conjurer.bak.*`. Nowe
|
||||
zależności (`aiomcrcon`, `asyncssh`) mogą zostać w venv — nie przeszkadzają
|
||||
starszej wersji.
|
||||
|
||||
---
|
||||
|
||||
## 10. Migracja usługi musician (jeśli prowadzisz radio)
|
||||
|
||||
Stabilny wariant `musician_old` (zahardkodowane adresy) został zastąpiony przez
|
||||
`conjurer_musician` (env-driven, domyślnie `127.0.0.1`). Jeśli bot i musician są
|
||||
na **różnych** hostach, ustaw na musicianie:
|
||||
|
||||
```bash
|
||||
CONJURER_MAIN_BOT=http://<ip-bota>:5000 # gdzie bot serwuje /prepped_tracks
|
||||
CONJURER_API_KEY=<ten sam sekret co bot> # jeśli włączasz auth (8a)
|
||||
CONJURER_MUSIC_FOLDER=/home/pi/MediaShare/mp3
|
||||
```
|
||||
|
||||
oraz na bocie `CONJURER_FILE_SERVICE=http://<ip-musiciana>:5000`. Szczegóły
|
||||
auth/kontraktu — patrz opis usługi `conjurer_musician`.
|
||||
@@ -0,0 +1,5 @@
|
||||
[pytest]
|
||||
testpaths = tests
|
||||
python_files = test_*.py
|
||||
python_functions = test_*
|
||||
addopts = -ra
|
||||
@@ -17,6 +17,5 @@ PyMuPDF
|
||||
waitress
|
||||
assemblyai[extras]
|
||||
SpeechRecognition
|
||||
aiomcrcon
|
||||
asyncssh
|
||||
git+https://github.com/imayhaveborkedit/discord-ext-voice-recv
|
||||
|
||||
@@ -0,0 +1,6 @@
|
||||
# Optional extras for the Conan Exiles bridge (conanjurer cog).
|
||||
#
|
||||
# aiomcrcon currently installs only on Python <= 3.11, while the main image
|
||||
# runs 3.13. Install is therefore best-effort: when it fails the conanjurer
|
||||
# cog simply stays dormant (its imports are guarded) and the bot runs fine.
|
||||
aiomcrcon
|
||||
@@ -1,20 +0,0 @@
|
||||
import logging
|
||||
from logging import handlers
|
||||
|
||||
logger = logging.getLogger("discord")
|
||||
logger.setLevel(logging.DEBUG)
|
||||
handler = handlers.RotatingFileHandler(
|
||||
filename="test.log",
|
||||
encoding="utf-8",
|
||||
mode="a",
|
||||
maxBytes=6 * 1024 * 1024,
|
||||
backupCount=6,
|
||||
)
|
||||
formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
|
||||
handler.setFormatter(formatter)
|
||||
logger.addHandler(handler)
|
||||
|
||||
|
||||
logger2 = logging.getLogger("discord")
|
||||
for item in logger2.handlers:
|
||||
print(item)
|
||||
-13
@@ -1,13 +0,0 @@
|
||||
AsyncCursorPage[Message](
|
||||
data=[Message(id='msg_3eWSdgbcU8sbCmJK2momOgQQ',
|
||||
assistant_id='asst_06eZiwvYNK3MR34suFP60gvg',
|
||||
attachments=[],
|
||||
completed_at=None,
|
||||
content=[TextContentBlock(text=Text(annotations=[], value='Cześć! Oto coś do przemyślenia: \n\n„Sukces to suma niewielkich wysiłków powtarzanych dzień po dniu.” — Robert Collier\n\nTo przypomina nam, że często to nie wielkie działania, ale konsekwentne, małe kroki prowadzą do osiągnięcia celu. Jak mogę Ci dzisiaj pomóc?'), type='text')], created_at=1731620112, incomplete_at=None, incomplete_details=None, metadata={}, object='thread.message', role='assistant', run_id='run_JGjWQTCEkDcEYpyCJnrkZU8Q', status=None, thread_id='thread_dDEjGbGm6ICfG75u0KKpoVxD'), Message(id='msg_GFcnfCFgAthGm2D3oE5d0ZkQ', assistant_id=None, attachments=[], completed_at=None, content=[TextContentBlock(text=Text(annotations=[], value='Cześć! Powiedz coś mądrego'), type='text')], created_at=1731620110, incomplete_at=None, incomplete_details=None, metadata={}, object='thread.message', role='user', run_id=None, status=None, thread_id='thread_dDEjGbGm6ICfG75u0KKpoVxD')], object='list', first_id='msg_3eWSdgbcU8sbCmJK2momOgQQ', last_id='msg_GFcnfCFgAthGm2D3oE5d0ZkQ', has_more=False)
|
||||
|
||||
|
||||
[TextContentBlock(
|
||||
text=Text(annotations=[],
|
||||
value='Cześć! Oto coś do rozważenia: "Największą przeszkodą w naszym życiu jest brak odwagi do wprowadzenia zmian." Niezależnie od tego, jakie masz cele czy marzenia, odwaga do działania i przystosowania się do nowych sytuacji jest kluczem do osiągnięcia sukcesu. Jakie masz przemyślenia na ten temat?'),
|
||||
type='text')
|
||||
]
|
||||
@@ -1,8 +0,0 @@
|
||||
import time
|
||||
|
||||
first_time = time.time_ns()
|
||||
|
||||
time.sleep(1)
|
||||
time_diff = time.time_ns() - first_time
|
||||
print(time_diff)
|
||||
# 2000149433
|
||||
@@ -0,0 +1,56 @@
|
||||
"""Integration: the bot's communication Flask layer enforces the shared key,
|
||||
and the key the bot would *send* (constants.service_headers) is accepted.
|
||||
"""
|
||||
import constants
|
||||
import communication_subroutine as cs
|
||||
|
||||
|
||||
def _client(key="test-secret"):
|
||||
cs.API_KEY = key
|
||||
return cs.app.test_client()
|
||||
|
||||
|
||||
def test_prepped_tracks_rejected_without_key():
|
||||
client = _client()
|
||||
resp = client.post(
|
||||
"/prepped_tracks", data='["all", "x"]', content_type="application/json"
|
||||
)
|
||||
assert resp.status_code == 401
|
||||
|
||||
|
||||
def test_prepped_tracks_accepted_with_key():
|
||||
client = _client()
|
||||
resp = client.post(
|
||||
"/prepped_tracks",
|
||||
data='["all", "x"]',
|
||||
headers={"X-Conjurer-Api-Key": "test-secret"},
|
||||
content_type="application/json",
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
def test_conjurer_get_is_open():
|
||||
client = _client()
|
||||
assert client.get("/conjurer").status_code == 200
|
||||
|
||||
|
||||
def test_open_when_key_unset():
|
||||
client = _client(key=None)
|
||||
resp = client.post(
|
||||
"/prepped_tracks", data='["all", "x"]', content_type="application/json"
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
def test_bot_service_headers_accepted_by_service(monkeypatch):
|
||||
# End-to-end auth contract: the header constants.service_headers() produces
|
||||
# is exactly what communication_subroutine._authorize_request() expects.
|
||||
monkeypatch.setattr(constants, "API_SHARED_KEY", "shared-xyz")
|
||||
cs.API_KEY = "shared-xyz"
|
||||
resp = cs.app.test_client().post(
|
||||
"/prepped_tracks",
|
||||
data='["all", "x"]',
|
||||
headers=constants.service_headers(),
|
||||
content_type="application/json",
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
@@ -0,0 +1,34 @@
|
||||
"""Integration: the musician Flask service enforces the shared key on its
|
||||
authenticated endpoints while leaving the open ones reachable.
|
||||
"""
|
||||
import conjurer_musician as m
|
||||
|
||||
|
||||
def _client(key="test-secret"):
|
||||
m.API_KEY = key
|
||||
return m.app.test_client()
|
||||
|
||||
|
||||
def test_clear_pr_pls_rejected_without_key():
|
||||
client = _client()
|
||||
assert client.get("/clear_pr_pls").status_code == 401
|
||||
|
||||
|
||||
def test_clear_pr_pls_accepted_with_key():
|
||||
client = _client()
|
||||
resp = client.get(
|
||||
"/clear_pr_pls", headers={"X-Conjurer-Api-Key": "test-secret"}
|
||||
)
|
||||
assert resp.status_code == 200
|
||||
|
||||
|
||||
def test_mp3_list_is_open():
|
||||
client = _client()
|
||||
resp = client.get("/mp3")
|
||||
assert resp.status_code == 200
|
||||
assert "music_file_list" in resp.get_json()
|
||||
|
||||
|
||||
def test_open_when_key_unset():
|
||||
client = _client(key=None)
|
||||
assert client.get("/clear_pr_pls").status_code == 200
|
||||
@@ -0,0 +1,63 @@
|
||||
"""Unit tests for the Conan Exiles bridge helpers (no Discord/RCON needed)."""
|
||||
import conanjurer_functions as cf
|
||||
|
||||
|
||||
def test_parse_players_basic():
|
||||
out = (
|
||||
"Idx | Char name | Player name | User ID\n"
|
||||
"0 | Conan | SteamGuy | 1\n"
|
||||
"1 | Khasar | OtherGuy | 2\n"
|
||||
"--- | --- | --- | ---"
|
||||
)
|
||||
assert cf.parse_players(out) == {"Conan", "Khasar"}
|
||||
|
||||
|
||||
def test_parse_players_empty_inputs():
|
||||
assert cf.parse_players("No players connected.") == set()
|
||||
assert cf.parse_players("") == set()
|
||||
|
||||
|
||||
def test_parse_line_login():
|
||||
event = cf.parse_line("SomeGuy joined the server")
|
||||
assert event is not None
|
||||
assert event.kind == "login"
|
||||
assert "SomeGuy" in event.text
|
||||
|
||||
|
||||
def test_parse_line_chat():
|
||||
event = cf.parse_line("Chat: Bob: hello there")
|
||||
assert event is not None
|
||||
assert event.kind == "chat"
|
||||
assert "Bob" in event.text and "hello" in event.text
|
||||
|
||||
|
||||
def test_parse_line_unrecognised_is_ignored():
|
||||
assert cf.parse_line("random server noise") is None
|
||||
|
||||
|
||||
def test_conanconfig_disabled_when_unconfigured():
|
||||
cfg = cf.ConanConfig("", 25575, "", "local", "", "", 22, "", "")
|
||||
assert cfg.rcon_enabled is False
|
||||
assert cfg.log_enabled is False
|
||||
|
||||
|
||||
def test_conanconfig_local_log_enabled():
|
||||
cfg = cf.ConanConfig("", 0, "", "local", "/tmp/conan.log", "", 22, "", "")
|
||||
assert cfg.log_enabled is True
|
||||
|
||||
|
||||
def test_conanconfig_rcon_enabled_requires_lib(monkeypatch):
|
||||
# Without aiomcrcon installed, rcon stays disabled even when host+pw are set.
|
||||
monkeypatch.setattr(cf, "_Rcon", None)
|
||||
cfg = cf.ConanConfig("1.2.3.4", 25575, "pw", "local", "", "", 22, "", "")
|
||||
assert cfg.rcon_enabled is False
|
||||
# With the lib present, it enables.
|
||||
monkeypatch.setattr(cf, "_Rcon", object)
|
||||
cfg2 = cf.ConanConfig("1.2.3.4", 25575, "pw", "local", "", "", 22, "", "")
|
||||
assert cfg2.rcon_enabled is True
|
||||
|
||||
|
||||
def test_conanconfig_sftp_log_requires_asyncssh(monkeypatch):
|
||||
monkeypatch.setattr(cf, "asyncssh", None)
|
||||
cfg = cf.ConanConfig("", 0, "", "sftp", "/log", "sftp.host", 22, "u", "p")
|
||||
assert cfg.log_enabled is False
|
||||
@@ -0,0 +1,35 @@
|
||||
"""Unit tests for constants helpers (defensive config / auth headers)."""
|
||||
import constants
|
||||
|
||||
|
||||
def test_service_headers_empty_when_no_key(monkeypatch):
|
||||
monkeypatch.setattr(constants, "API_SHARED_KEY", "")
|
||||
assert constants.service_headers() == {}
|
||||
|
||||
|
||||
def test_service_headers_with_key(monkeypatch):
|
||||
monkeypatch.setattr(constants, "API_SHARED_KEY", "s3cr3t")
|
||||
assert constants.service_headers() == {"X-Conjurer-Api-Key": "s3cr3t"}
|
||||
|
||||
|
||||
def test_load_json_missing_returns_fallback(tmp_path):
|
||||
missing = tmp_path / "nope.json"
|
||||
assert constants._load_json(str(missing), {"fallback": 1}) == {"fallback": 1}
|
||||
|
||||
|
||||
def test_load_json_valid(tmp_path):
|
||||
good = tmp_path / "ok.json"
|
||||
good.write_text('{"a": 2}', encoding="utf-8")
|
||||
assert constants._load_json(str(good), {}) == {"a": 2}
|
||||
|
||||
|
||||
def test_load_json_corrupt_returns_fallback(tmp_path):
|
||||
bad = tmp_path / "bad.json"
|
||||
bad.write_text("{ not valid json", encoding="utf-8")
|
||||
assert constants._load_json(str(bad), []) == []
|
||||
|
||||
|
||||
def test_conan_defaults_present():
|
||||
# Feature toggles default to "off" so the bridge stays dormant.
|
||||
assert constants.CONAN_JOIN_CHANNEL_ID == 0
|
||||
assert constants.CONAN_RCON_HOST == ""
|
||||
@@ -1,6 +1,5 @@
|
||||
import asyncio
|
||||
import logging
|
||||
import netrc
|
||||
import time
|
||||
import wave
|
||||
|
||||
@@ -9,11 +8,17 @@ import discord
|
||||
from discord.ext import commands, tasks, voice_recv
|
||||
from discord.opus import Decoder as OpusDecoder
|
||||
|
||||
# Replace with your API key
|
||||
NETRC_FILE = "/home/pi/.netrc"
|
||||
netrc_mod = netrc.netrc(NETRC_FILE)
|
||||
authTokens = netrc_mod.authenticators("assemblyai")
|
||||
aai.settings.api_key = authTokens[2]
|
||||
from constants import ASSEMBLYAI_API_KEY, TRANSCRIPTS_PATH
|
||||
|
||||
# Credentials come from constants (env ASSEMBLYAI_API_KEY, or the 'assemblyai'
|
||||
# machine in the netrc at CONJURER_NETRC_FILE). Raising here means the guarded
|
||||
# extension loader logs the reason and disables ONLY this cog.
|
||||
if not ASSEMBLYAI_API_KEY:
|
||||
raise RuntimeError(
|
||||
"AssemblyAI API key not configured (netrc machine 'assemblyai' or "
|
||||
"ASSEMBLYAI_API_KEY env) - voice recognition stays disabled"
|
||||
)
|
||||
aai.settings.api_key = ASSEMBLYAI_API_KEY
|
||||
|
||||
discord.opus._load_default()
|
||||
CHANNELS = OpusDecoder.CHANNELS
|
||||
@@ -23,7 +28,7 @@ SAMPLING_RATE = OpusDecoder.SAMPLING_RATE
|
||||
# rotate file after there is 0.5s between last received pcm for user.
|
||||
# delete messsages after user disconnect
|
||||
|
||||
LOCATION = "/home/pi/Conjurer/transcripts/"
|
||||
LOCATION = TRANSCRIPTS_PATH
|
||||
|
||||
|
||||
class CommunicationObject:
|
||||
|
||||
Reference in New Issue
Block a user