mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
docker: Proxmox deployment for bot/librarian/musician + code fixes
Containerises the three services (each intended for its own Proxmox VM)
and adds the code changes needed to run cleanly on Linux/Docker.
Code fixes:
- constants.py: CONJURER_DATA_DIR roots all writable bot state under one
mounted volume (per-variable overrides still win; native Pi unaffected)
- conjurer_librarian/search_bot.py + scrape_bot.py: the hardcoded Windows
DOI database path (C:\Database\chunks\) is now CONJURER_LIBRARIAN_DB_PATH,
with CONJURER_LIBRARIAN_MAXTHREADS / _CHUNK also env-overridable
Docker:
- docker/Dockerfile.{bot,librarian,musician} + compose.{bot,librarian,musician}.yaml
- docker/env/*.env.example (force-added; real *.env stays gitignored)
- docker/entrypoint.bot.sh seeds default JSON state into /data only when
absent, so preserved history is never overwritten
- .dockerignore
- docs/deployment/DOCKER_PROXMOX.md: step-by-step runbook incl. preserving
the existing command/conversation history and cross-VM auth
The bot image uses the vendored yt_dlp/spotify_dl forks (they win on
sys.path over the pip packages), dropping the old sed patching.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Michal Tuszowski
@@ -0,0 +1,17 @@
|
|||||||
|
# Keep build contexts lean. Component dirs (conjurer_librarian, conjurer_musician)
|
||||||
|
# are intentionally NOT ignored — their images copy them from this same context.
|
||||||
|
.git
|
||||||
|
.github
|
||||||
|
.trunk
|
||||||
|
.vscode
|
||||||
|
**/__pycache__/
|
||||||
|
*.pyc
|
||||||
|
*.pyo
|
||||||
|
*.log
|
||||||
|
*.mp3
|
||||||
|
docs/
|
||||||
|
tests/
|
||||||
|
conftest.py
|
||||||
|
pytest.ini
|
||||||
|
docker/env/*.env
|
||||||
|
secrets/
|
||||||
@@ -4,6 +4,7 @@ This module contains the code for the scrape bot.
|
|||||||
|
|
||||||
import json
|
import json
|
||||||
import logging
|
import logging
|
||||||
|
import os
|
||||||
import random
|
import random
|
||||||
import re
|
import re
|
||||||
import time
|
import time
|
||||||
@@ -15,9 +16,9 @@ from urllib.request import urlopen
|
|||||||
from requests import ConnectionError as RequestsConnectionError
|
from requests import ConnectionError as RequestsConnectionError
|
||||||
from requests import ConnectTimeout, Timeout
|
from requests import ConnectTimeout, Timeout
|
||||||
|
|
||||||
SCR_DATABASE_PATH = r"C:\\Database\\chunks\\"
|
SCR_DATABASE_PATH = os.getenv("CONJURER_LIBRARIAN_DB_PATH", r"C:\\Database\\chunks\\")
|
||||||
SCR_FILENAME = "40_chunk.txt"
|
SCR_FILENAME = os.getenv("CONJURER_LIBRARIAN_SCRAPE_CHUNK", "40_chunk.txt")
|
||||||
SCR_ENCODING = "utf-8"
|
SCR_ENCODING = os.getenv("CONJURER_ENCODING", "utf-8")
|
||||||
|
|
||||||
WORK_Q = Queue()
|
WORK_Q = Queue()
|
||||||
random.seed()
|
random.seed()
|
||||||
|
|||||||
@@ -19,22 +19,20 @@ Global Variables:
|
|||||||
"""
|
"""
|
||||||
|
|
||||||
# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
|
# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
|
||||||
|
import os
|
||||||
from queue import Empty, Queue
|
from queue import Empty, Queue
|
||||||
from threading import Thread
|
from threading import Thread
|
||||||
import time
|
import time
|
||||||
q = Queue()
|
q = Queue()
|
||||||
#TODO: Count number of lines in files and print to approximate on which part of the file search is
|
#TODO: Count number of lines in files and print to approximate on which part of the file search is
|
||||||
|
|
||||||
# DATA FOR TEST ONLY
|
# Deployment data is environment-overridable so the local DOI database can live
|
||||||
# MAXTHREADS = 5
|
# on a mounted volume (Docker/Linux) instead of the hardcoded Windows path.
|
||||||
# DATABASE_PATH = r'C:\\Database\\chunks_1\\'
|
MAXTHREADS = int(os.getenv("CONJURER_LIBRARIAN_MAXTHREADS", "41"))
|
||||||
|
DATABASE_PATH = os.getenv("CONJURER_LIBRARIAN_DB_PATH", r"C:\\Database\\chunks\\")
|
||||||
|
|
||||||
# DEPLOYMENT DATA
|
ENCODING = os.getenv("CONJURER_ENCODING", "utf-8")
|
||||||
MAXTHREADS = 41
|
CHUNK = os.getenv("CONJURER_LIBRARIAN_CHUNK", "_chunk.txt")
|
||||||
DATABASE_PATH = r"C:\\Database\\chunks\\"
|
|
||||||
|
|
||||||
ENCODING = "utf-8"
|
|
||||||
CHUNK = "_chunk.txt"
|
|
||||||
_sentinel = object()
|
_sentinel = object()
|
||||||
WORK_Q_SIZE = 35500000
|
WORK_Q_SIZE = 35500000
|
||||||
|
|
||||||
|
|||||||
@@ -158,6 +158,23 @@ else:
|
|||||||
# Values stay as plain strings so existing ``PATH + filename`` concatenation in
|
# Values stay as plain strings so existing ``PATH + filename`` concatenation in
|
||||||
# the command modules keeps working unchanged.
|
# the command modules keeps working unchanged.
|
||||||
|
|
||||||
|
# Container-friendly shortcut: point CONJURER_DATA_DIR at a single mounted
|
||||||
|
# volume and every writable data file/dir is rooted under it. The per-variable
|
||||||
|
# CONJURER_* overrides below still take precedence, so granular control remains
|
||||||
|
# possible and the native (Pi) deployment is unaffected when it is unset.
|
||||||
|
_DATA_DIR = os.getenv("CONJURER_DATA_DIR")
|
||||||
|
if _DATA_DIR:
|
||||||
|
LOGFILE = os.path.join(_DATA_DIR, "discord.log")
|
||||||
|
SETTINGS_FILE = os.path.join(_DATA_DIR, "settings.json")
|
||||||
|
MEMORY_FIVE_SIARA = os.path.join(_DATA_DIR, "pamiec.json")
|
||||||
|
MEMORY_FIVE_MUZYKA = os.path.join(_DATA_DIR, "pamiec_muzyki.json")
|
||||||
|
SYSTEM_GPT_SETTINGS = os.path.join(_DATA_DIR, "system_gpt_settings.json")
|
||||||
|
ACCIDENT_LOG = os.path.join(_DATA_DIR, "accident_log.json")
|
||||||
|
LOGSTORE = os.path.join(_DATA_DIR, "logs") + os.sep
|
||||||
|
GRAPHICS_PATH = os.path.join(_DATA_DIR, "Conjurer_graphics") + os.sep
|
||||||
|
MUSIC_FOLDER = os.path.join(_DATA_DIR, "music") + os.sep
|
||||||
|
DIR_PATH_SADOX = os.path.join(_DATA_DIR, "Fansadox") + os.sep
|
||||||
|
|
||||||
LOGFILE = os.getenv("CONJURER_LOG_FILE", LOGFILE)
|
LOGFILE = os.getenv("CONJURER_LOG_FILE", LOGFILE)
|
||||||
NETRC_FILE = os.getenv("CONJURER_NETRC_FILE", NETRC_FILE)
|
NETRC_FILE = os.getenv("CONJURER_NETRC_FILE", NETRC_FILE)
|
||||||
SETTINGS_FILE = os.getenv("CONJURER_SETTINGS_FILE", SETTINGS_FILE)
|
SETTINGS_FILE = os.getenv("CONJURER_SETTINGS_FILE", SETTINGS_FILE)
|
||||||
|
|||||||
@@ -0,0 +1,56 @@
|
|||||||
|
# Conjurer main Discord bot.
|
||||||
|
# Build from the repository root:
|
||||||
|
# docker build -f docker/Dockerfile.bot -t conjurer-bot .
|
||||||
|
FROM python:3.11-slim
|
||||||
|
|
||||||
|
# System dependencies:
|
||||||
|
# ffmpeg - audio download/convert (yt_dlp) and Discord voice
|
||||||
|
# libopus0 - Discord voice (PyNaCl / discord-ext-voice-recv)
|
||||||
|
# poppler-utils - pdf2image (librarian/latex previews)
|
||||||
|
# git - required by the git+https entry in requirements
|
||||||
|
# build-essential - native wheels (PyNaCl etc.)
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
ffmpeg \
|
||||||
|
libopus0 \
|
||||||
|
poppler-utils \
|
||||||
|
git \
|
||||||
|
build-essential \
|
||||||
|
curl \
|
||||||
|
ca-certificates \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
# Optional: Tectonic for the $latex command. Remove this layer if unused.
|
||||||
|
RUN curl -fsSL https://drop-sh.fullyjustified.net | sh \
|
||||||
|
&& mv tectonic /usr/local/bin/tectonic \
|
||||||
|
|| echo "tectonic not installed - the LaTeX feature will be disabled"
|
||||||
|
|
||||||
|
COPY requirements_bot.txt ./
|
||||||
|
RUN pip install --no-cache-dir --upgrade pip \
|
||||||
|
&& pip install --no-cache-dir -r requirements_bot.txt
|
||||||
|
|
||||||
|
# Vendored forks take import precedence over the pip packages of the same name,
|
||||||
|
# because /app (the script dir) is first on sys.path when running `python bot.py`.
|
||||||
|
COPY yt_dlp ./yt_dlp
|
||||||
|
COPY spotify_dl ./spotify_dl
|
||||||
|
|
||||||
|
# Bot sources + default config/asset templates. Runtime state (conversation
|
||||||
|
# history etc.) is read from the mounted /data volume via CONJURER_DATA_DIR,
|
||||||
|
# so these committed copies only act as first-run fallbacks.
|
||||||
|
COPY *.py ./
|
||||||
|
COPY settings.json system_gpt_settings.json accident_log.json pamiec.json pamiec_muzyki.json ./
|
||||||
|
COPY fuckery.jpg willowisp.png wod_beacon.jpg ./
|
||||||
|
COPY docker/entrypoint.bot.sh /usr/local/bin/entrypoint.sh
|
||||||
|
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||||
|
|
||||||
|
ENV PYTHONUNBUFFERED=1 \
|
||||||
|
CONJURER_DATA_DIR=/data \
|
||||||
|
CONJURER_DISCORD_HOST=0.0.0.0 \
|
||||||
|
CONJURER_DISCORD_PORT=5000
|
||||||
|
|
||||||
|
VOLUME ["/data"]
|
||||||
|
EXPOSE 5000
|
||||||
|
|
||||||
|
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||||
|
CMD ["python", "bot.py"]
|
||||||
@@ -0,0 +1,27 @@
|
|||||||
|
# Conjurer librarian (Crossref search + local DOI database lookup).
|
||||||
|
# Build from the repository root:
|
||||||
|
# docker build -f docker/Dockerfile.librarian -t conjurer-librarian .
|
||||||
|
FROM python:3.11-slim
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY conjurer_librarian/requirements_librarian.txt ./
|
||||||
|
RUN pip install --no-cache-dir --upgrade pip \
|
||||||
|
&& pip install --no-cache-dir -r requirements_librarian.txt requests
|
||||||
|
|
||||||
|
COPY conjurer_librarian/ ./
|
||||||
|
|
||||||
|
ENV PYTHONUNBUFFERED=1 \
|
||||||
|
CONJURER_LIBRARIAN_HOST=0.0.0.0 \
|
||||||
|
CONJURER_LIBRARIAN_PORT=5001 \
|
||||||
|
CONJURER_LIBRARIAN_DB_PATH=/doi/
|
||||||
|
|
||||||
|
# The local DOI chunk database (large) is mounted here.
|
||||||
|
VOLUME ["/doi"]
|
||||||
|
EXPOSE 5001
|
||||||
|
|
||||||
|
CMD ["python", "conjurer_librarian.py"]
|
||||||
@@ -0,0 +1,30 @@
|
|||||||
|
# Conjurer musician (Flask file/playlist service backing the radio).
|
||||||
|
# Build from the repository root:
|
||||||
|
# docker build -f docker/Dockerfile.musician -t conjurer-musician .
|
||||||
|
#
|
||||||
|
# NOTE: this containerises the musician *web service* only. The Liquidsoap
|
||||||
|
# radio (radio_conjurer.liq) and any Samba/NFS share tooling run separately.
|
||||||
|
FROM python:3.11-slim
|
||||||
|
|
||||||
|
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||||
|
build-essential \
|
||||||
|
&& rm -rf /var/lib/apt/lists/*
|
||||||
|
|
||||||
|
WORKDIR /app
|
||||||
|
|
||||||
|
COPY conjurer_musician/requirements_musician.txt ./
|
||||||
|
RUN pip install --no-cache-dir --upgrade pip \
|
||||||
|
&& pip install --no-cache-dir -r requirements_musician.txt
|
||||||
|
|
||||||
|
COPY conjurer_musician/ ./
|
||||||
|
|
||||||
|
ENV PYTHONUNBUFFERED=1 \
|
||||||
|
CONJURER_MUSICIAN_HOST=0.0.0.0 \
|
||||||
|
CONJURER_MUSICIAN_PORT=5000 \
|
||||||
|
CONJURER_MUSIC_FOLDER=/music
|
||||||
|
|
||||||
|
# The mp3 library is mounted here (read-only is fine for serving).
|
||||||
|
VOLUME ["/music"]
|
||||||
|
EXPOSE 5000
|
||||||
|
|
||||||
|
CMD ["python", "conjurer_musician.py"]
|
||||||
@@ -0,0 +1,22 @@
|
|||||||
|
# Main bot VM. Run from the repository root:
|
||||||
|
# cp docker/env/bot.env.example docker/env/bot.env # then edit
|
||||||
|
# docker compose -f docker/compose.bot.yaml up -d --build
|
||||||
|
services:
|
||||||
|
conjurer-bot:
|
||||||
|
build:
|
||||||
|
context: ..
|
||||||
|
dockerfile: docker/Dockerfile.bot
|
||||||
|
image: conjurer-bot:latest
|
||||||
|
container_name: conjurer-bot
|
||||||
|
restart: unless-stopped
|
||||||
|
env_file:
|
||||||
|
- env/bot.env
|
||||||
|
ports:
|
||||||
|
# Flask comm layer — musician/librarian POST results here (/prepped_tracks, /conjurer).
|
||||||
|
- "5000:5000"
|
||||||
|
volumes:
|
||||||
|
# Persistent state (conversation history, settings, logs). Populate this
|
||||||
|
# host dir with your existing pamiec.json etc. to preserve command history.
|
||||||
|
- /srv/conjurer/data:/data
|
||||||
|
# Tokens: a read-only netrc covers discord/openai/spotipy/youtube in one file.
|
||||||
|
- /srv/conjurer/secrets/.netrc:/secrets/.netrc:ro
|
||||||
@@ -0,0 +1,20 @@
|
|||||||
|
# Librarian VM. Run from the repository root:
|
||||||
|
# cp docker/env/librarian.env.example docker/env/librarian.env # then edit
|
||||||
|
# docker compose -f docker/compose.librarian.yaml up -d --build
|
||||||
|
services:
|
||||||
|
conjurer-librarian:
|
||||||
|
build:
|
||||||
|
context: ..
|
||||||
|
dockerfile: docker/Dockerfile.librarian
|
||||||
|
image: conjurer-librarian:latest
|
||||||
|
container_name: conjurer-librarian
|
||||||
|
restart: unless-stopped
|
||||||
|
env_file:
|
||||||
|
- env/librarian.env
|
||||||
|
ports:
|
||||||
|
- "5001:5001"
|
||||||
|
volumes:
|
||||||
|
# Local DOI chunk database (0_chunk.txt ... N_chunk.txt).
|
||||||
|
- /srv/librarian/doi:/doi:ro
|
||||||
|
# Optional: netrc holding Crossref credentials (or use CONJURER_CROSSREF_MAILTO).
|
||||||
|
- /srv/librarian/secrets/.netrc:/secrets/.netrc:ro
|
||||||
@@ -0,0 +1,21 @@
|
|||||||
|
# Musician VM (optional — you plan to adapt/implement this yourself).
|
||||||
|
# Run from the repository root:
|
||||||
|
# cp docker/env/musician.env.example docker/env/musician.env # then edit
|
||||||
|
# docker compose -f docker/compose.musician.yaml up -d --build
|
||||||
|
services:
|
||||||
|
conjurer-musician:
|
||||||
|
build:
|
||||||
|
context: ..
|
||||||
|
dockerfile: docker/Dockerfile.musician
|
||||||
|
image: conjurer-musician:latest
|
||||||
|
container_name: conjurer-musician
|
||||||
|
restart: unless-stopped
|
||||||
|
env_file:
|
||||||
|
- env/musician.env
|
||||||
|
ports:
|
||||||
|
- "5000:5000"
|
||||||
|
volumes:
|
||||||
|
# The mp3 library the service indexes and serves.
|
||||||
|
- /srv/musician/music:/music
|
||||||
|
# Runtime playlists/logs the service writes.
|
||||||
|
- /srv/musician/data:/data
|
||||||
Executable
+16
@@ -0,0 +1,16 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# Seed the data volume with the baked-in default JSON state on first run only.
|
||||||
|
# Existing files (e.g. your preserved pamiec.json history) are never overwritten.
|
||||||
|
set -e
|
||||||
|
|
||||||
|
DATA="${CONJURER_DATA_DIR:-/data}"
|
||||||
|
mkdir -p "$DATA"
|
||||||
|
|
||||||
|
for f in settings.json system_gpt_settings.json pamiec.json pamiec_muzyki.json accident_log.json; do
|
||||||
|
if [ ! -e "$DATA/$f" ] && [ -e "/app/$f" ]; then
|
||||||
|
cp "/app/$f" "$DATA/$f"
|
||||||
|
echo "entrypoint: seeded $f into $DATA"
|
||||||
|
fi
|
||||||
|
done
|
||||||
|
|
||||||
|
exec "$@"
|
||||||
Vendored
+38
@@ -0,0 +1,38 @@
|
|||||||
|
# Copy to docker/env/bot.env and fill in. Do NOT commit the real file.
|
||||||
|
|
||||||
|
# --- Secrets ------------------------------------------------------------
|
||||||
|
# Option A: mount a netrc (recommended — covers discord/openai/spotipy/youtube).
|
||||||
|
CONJURER_NETRC_FILE=/secrets/.netrc
|
||||||
|
# Option B: pass tokens directly (these take precedence over netrc).
|
||||||
|
# DISCORD_TOKEN=
|
||||||
|
# OPENAI_API_KEY=
|
||||||
|
# YOUTUBE_USERNAME=
|
||||||
|
# YOUTUBE_PASSWORD=
|
||||||
|
|
||||||
|
# --- Data ---------------------------------------------------------------
|
||||||
|
# Single mounted volume; all writable state is rooted here.
|
||||||
|
CONJURER_DATA_DIR=/data
|
||||||
|
|
||||||
|
# --- Flask comm layer (inbound from musician/librarian) -----------------
|
||||||
|
CONJURER_DISCORD_HOST=0.0.0.0
|
||||||
|
CONJURER_DISCORD_PORT=5000
|
||||||
|
|
||||||
|
# --- Internal service auth ----------------------------------------------
|
||||||
|
# Set the SAME value on bot + musician + librarian. Empty = auth disabled.
|
||||||
|
CONJURER_API_KEY=
|
||||||
|
|
||||||
|
# --- Where the bot reaches the other services (other Proxmox VMs) --------
|
||||||
|
CONJURER_FILE_SERVICE=http://MUSICIAN_VM_IP:5000
|
||||||
|
CONJURER_RADIO_HARBOR=http://MUSICIAN_VM_IP:54321
|
||||||
|
CONJURER_LIBRARIAN_SERVICE=http://LIBRARIAN_VM_IP:5001
|
||||||
|
|
||||||
|
# --- Conan Exiles bridge (optional; empty/0 = disabled) -----------------
|
||||||
|
# CONAN_GM_ROLE_ID=0
|
||||||
|
# CONAN_RCON_HOST=
|
||||||
|
# CONAN_RCON_PORT=25575
|
||||||
|
# CONAN_RCON_PASSWORD=
|
||||||
|
# CONAN_CHAT_CHANNEL_ID=0
|
||||||
|
# CONAN_EVENTS_CHANNEL_ID=0
|
||||||
|
# CONAN_JOIN_CHANNEL_ID=0
|
||||||
|
# CONAN_LOG_MODE=local
|
||||||
|
# CONAN_LOG_PATH=
|
||||||
Vendored
+21
@@ -0,0 +1,21 @@
|
|||||||
|
# Copy to docker/env/librarian.env and fill in.
|
||||||
|
|
||||||
|
CONJURER_LIBRARIAN_HOST=0.0.0.0
|
||||||
|
CONJURER_LIBRARIAN_PORT=5001
|
||||||
|
|
||||||
|
# Same shared secret as the bot (empty = auth disabled).
|
||||||
|
CONJURER_API_KEY=
|
||||||
|
|
||||||
|
# Where to POST search results back to (the bot's comm layer).
|
||||||
|
CONJURER_MAIN_BOT=http://BOT_VM_IP:5000
|
||||||
|
|
||||||
|
# Crossref polite-pool contact (or put credentials in netrc under "crossref").
|
||||||
|
CONJURER_CROSSREF_MAILTO=you@example.com
|
||||||
|
|
||||||
|
# Local DOI chunk database (mounted volume): expects 0_chunk.txt .. N_chunk.txt
|
||||||
|
CONJURER_LIBRARIAN_DB_PATH=/doi/
|
||||||
|
CONJURER_LIBRARIAN_MAXTHREADS=41
|
||||||
|
CONJURER_LIBRARIAN_CHUNK=_chunk.txt
|
||||||
|
|
||||||
|
# Optional netrc (for Crossref credentials)
|
||||||
|
CONJURER_NETRC_FILE=/secrets/.netrc
|
||||||
Vendored
+17
@@ -0,0 +1,17 @@
|
|||||||
|
# Copy to docker/env/musician.env and fill in.
|
||||||
|
|
||||||
|
CONJURER_MUSICIAN_HOST=0.0.0.0
|
||||||
|
CONJURER_MUSICIAN_PORT=5000
|
||||||
|
|
||||||
|
# Same shared secret as the bot (empty = auth disabled).
|
||||||
|
CONJURER_API_KEY=
|
||||||
|
|
||||||
|
# Where to POST "now playing" / prepped-track updates (the bot's comm layer).
|
||||||
|
CONJURER_MAIN_BOT=http://BOT_VM_IP:5000
|
||||||
|
|
||||||
|
# The mp3 library (mounted volume).
|
||||||
|
CONJURER_MUSIC_FOLDER=/music
|
||||||
|
|
||||||
|
# Runtime paths (mounted volume) — playlists/logs the service writes.
|
||||||
|
CONJURER_MUSICIAN_BASE=/data
|
||||||
|
CONJURER_LOGSTORE=/data/logs
|
||||||
@@ -0,0 +1,215 @@
|
|||||||
|
# Conjurer on Docker / Proxmox
|
||||||
|
|
||||||
|
Runbook for running Conjurer as Docker containers across Proxmox VMs:
|
||||||
|
|
||||||
|
| Component | VM | Container | Port | Image |
|
||||||
|
|-----------|----|-----------|------|-------|
|
||||||
|
| **Main bot** | VM-bot | `conjurer-bot` | 5000 (Flask comm) | `docker/Dockerfile.bot` |
|
||||||
|
| **Librarian** | VM-librarian | `conjurer-librarian` | 5001 | `docker/Dockerfile.librarian` |
|
||||||
|
| **Musician** | VM-musician | `conjurer-musician` | 5000 (+ radio) | `docker/Dockerfile.musician` |
|
||||||
|
|
||||||
|
The three talk to each other over HTTP on the Proxmox LAN. Direction of calls:
|
||||||
|
|
||||||
|
```
|
||||||
|
bot --(/mp3,/get_music,/add_to_priority,...)--> musician
|
||||||
|
bot --(/query)--------------------------------> librarian
|
||||||
|
musician --(/prepped_tracks)--------------------> bot
|
||||||
|
librarian --(/conjurer results)-----------------> bot
|
||||||
|
```
|
||||||
|
|
||||||
|
Everything is configured through `CONJURER_*` environment variables (see the
|
||||||
|
`docker/env/*.env.example` files). Nothing is hardcoded to a host path anymore.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 0. Prerequisites (per VM)
|
||||||
|
|
||||||
|
Create a small Linux VM in Proxmox (Debian 12 / Ubuntu 22.04+ is fine), then
|
||||||
|
install Docker:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo apt-get update && sudo apt-get install -y ca-certificates curl git
|
||||||
|
curl -fsSL https://get.docker.com | sh
|
||||||
|
sudo usermod -aG docker "$USER" # log out/in afterwards
|
||||||
|
```
|
||||||
|
|
||||||
|
Clone the repo on each VM (they build from it):
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo mkdir -p /opt && cd /opt
|
||||||
|
git clone https://github.com/migatu/conjurer.git
|
||||||
|
cd conjurer
|
||||||
|
```
|
||||||
|
|
||||||
|
> All `docker compose` commands below are run **from the repo root** (`/opt/conjurer`),
|
||||||
|
> because the compose files use `context: ..` relative to `docker/`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 1. Main bot (VM-bot)
|
||||||
|
|
||||||
|
### 1a. Prepare host directories
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo mkdir -p /srv/conjurer/data /srv/conjurer/secrets
|
||||||
|
```
|
||||||
|
|
||||||
|
### 1b. Preserve existing command history ⭐
|
||||||
|
|
||||||
|
The bot's conversation memory and settings live in JSON files. Copy them from
|
||||||
|
your current deployment (e.g. the Pi's `/home/pi/Conjurer/`) into the data
|
||||||
|
volume so the history carries over:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# run on the Pi, or scp the files across, then place them here:
|
||||||
|
sudo cp pamiec.json /srv/conjurer/data/ # AI conversation history
|
||||||
|
sudo cp pamiec_muzyki.json /srv/conjurer/data/ # music-DJ memory
|
||||||
|
sudo cp settings.json /srv/conjurer/data/ # word/cyclic reactions
|
||||||
|
sudo cp system_gpt_settings.json /srv/conjurer/data/
|
||||||
|
sudo cp accident_log.json /srv/conjurer/data/ # if present
|
||||||
|
```
|
||||||
|
|
||||||
|
If you skip this, the container starts with the (empty) template files baked
|
||||||
|
into the image and history begins fresh.
|
||||||
|
|
||||||
|
### 1c. Tokens
|
||||||
|
|
||||||
|
Drop your existing netrc (the one with `discord`, `openai`, `spotipy`,
|
||||||
|
`youtube` entries) into the secrets dir:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo cp ~/.netrc /srv/conjurer/secrets/.netrc
|
||||||
|
sudo chmod 600 /srv/conjurer/secrets/.netrc
|
||||||
|
```
|
||||||
|
|
||||||
|
(Alternatively skip netrc and set `DISCORD_TOKEN` / `OPENAI_API_KEY` in the env
|
||||||
|
file — those take precedence.)
|
||||||
|
|
||||||
|
### 1d. Configure and launch
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp docker/env/bot.env.example docker/env/bot.env
|
||||||
|
# edit docker/env/bot.env: set CONJURER_FILE_SERVICE / _LIBRARIAN_SERVICE to the
|
||||||
|
# other VMs' IPs, and CONJURER_API_KEY (same value on all three) if you want auth.
|
||||||
|
|
||||||
|
docker compose -f docker/compose.bot.yaml up -d --build
|
||||||
|
docker logs -f conjurer-bot
|
||||||
|
```
|
||||||
|
|
||||||
|
Look for `Loading … module done` for each cog, `Loading conanjurer commands
|
||||||
|
module done`, and `All systems: operational`.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 2. Librarian (VM-librarian)
|
||||||
|
|
||||||
|
### 2a. Mount the DOI database
|
||||||
|
|
||||||
|
The librarian checks keyword hits from Crossref against a local database of
|
||||||
|
DOI chunk files (`0_chunk.txt … N_chunk.txt`). Put that database on the VM and
|
||||||
|
point the volume at it:
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo mkdir -p /srv/librarian/doi /srv/librarian/secrets
|
||||||
|
# copy/replicate your chunk files into /srv/librarian/doi/
|
||||||
|
```
|
||||||
|
|
||||||
|
> The old Windows path `C:\Database\chunks\` is now `CONJURER_LIBRARIAN_DB_PATH`
|
||||||
|
> (defaults to `/doi/` in the container). `CONJURER_LIBRARIAN_MAXTHREADS` (41)
|
||||||
|
> and `CONJURER_LIBRARIAN_CHUNK` (`_chunk.txt`) are configurable too.
|
||||||
|
|
||||||
|
### 2b. Configure and launch
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cp docker/env/librarian.env.example docker/env/librarian.env
|
||||||
|
# edit: CONJURER_MAIN_BOT=http://BOT_VM_IP:5000, CONJURER_CROSSREF_MAILTO, CONJURER_API_KEY
|
||||||
|
|
||||||
|
docker compose -f docker/compose.librarian.yaml up -d --build
|
||||||
|
docker logs -f conjurer-librarian
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 3. Musician (VM-musician) — you will adapt this
|
||||||
|
|
||||||
|
The web service is container-ready (`docker/Dockerfile.musician` +
|
||||||
|
`compose.musician.yaml`). Note it containerises the **Flask file/playlist
|
||||||
|
service only**; the Liquidsoap radio (`radio_conjurer.liq`) and any
|
||||||
|
Samba/NFS share tooling are separate and typically stay on the host or a
|
||||||
|
dedicated setup.
|
||||||
|
|
||||||
|
```bash
|
||||||
|
sudo mkdir -p /srv/musician/music /srv/musician/data
|
||||||
|
cp docker/env/musician.env.example docker/env/musician.env
|
||||||
|
# edit: CONJURER_MAIN_BOT=http://BOT_VM_IP:5000, CONJURER_API_KEY, mount your mp3s at /music
|
||||||
|
|
||||||
|
docker compose -f docker/compose.musician.yaml up -d --build
|
||||||
|
```
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 4. Networking & auth
|
||||||
|
|
||||||
|
- Open the ports between VMs on the Proxmox LAN: **bot 5000**, **librarian 5001**,
|
||||||
|
**musician 5000** (+ radio harbor 54321 if used). A simple `ufw allow from
|
||||||
|
<lan-subnet>` per port is enough; do not expose them to the internet.
|
||||||
|
- **Auth:** set the same `CONJURER_API_KEY` in all three `*.env` files. Then
|
||||||
|
every internal call carries `X-Conjurer-Api-Key` and each service rejects
|
||||||
|
requests without it (HTTP 401). Leave it empty everywhere to disable auth
|
||||||
|
(fully backward compatible). ⚠️ Setting it on only one side breaks the link.
|
||||||
|
- The addresses point at each other by VM IP (or a DNS name). Set:
|
||||||
|
- bot: `CONJURER_FILE_SERVICE`, `CONJURER_RADIO_HARBOR`, `CONJURER_LIBRARIAN_SERVICE`
|
||||||
|
- librarian & musician: `CONJURER_MAIN_BOT`
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 5. Verify
|
||||||
|
|
||||||
|
```bash
|
||||||
|
# bot is up and reachable from another VM:
|
||||||
|
curl http://BOT_VM_IP:5000/conjurer # -> "ALIVE"
|
||||||
|
# librarian answers:
|
||||||
|
curl http://LIBRARIAN_VM_IP:5001/ -I # service reachable
|
||||||
|
docker ps # all containers "Up"
|
||||||
|
docker logs conjurer-bot --tail 50
|
||||||
|
```
|
||||||
|
|
||||||
|
In Discord, exercise a command that round-trips through a service (e.g. a music
|
||||||
|
search that hits the musician, or a librarian query) to confirm the wiring and
|
||||||
|
the API key.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 6. Updates
|
||||||
|
|
||||||
|
```bash
|
||||||
|
cd /opt/conjurer && git pull
|
||||||
|
docker compose -f docker/compose.bot.yaml up -d --build # rebuild + restart
|
||||||
|
```
|
||||||
|
|
||||||
|
Data in `/srv/.../data` and `/doi` / `/music` volumes survives rebuilds, so
|
||||||
|
history and databases persist across updates.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## 7. Rollback / coexistence
|
||||||
|
|
||||||
|
- The native (Raspberry Pi / systemd) deployment is unaffected — none of the
|
||||||
|
defaults changed; the container behaviour is opt-in via `CONJURER_DATA_DIR`
|
||||||
|
and the other env vars. You can run both during migration.
|
||||||
|
- To roll back a VM: `docker compose -f docker/compose.<svc>.yaml down` and
|
||||||
|
restart the previous deployment. The JSON state in `/srv/.../data` is plain
|
||||||
|
files you can copy back to the Pi if needed.
|
||||||
|
|
||||||
|
---
|
||||||
|
|
||||||
|
## Notes on the images
|
||||||
|
|
||||||
|
- **Bot** uses the vendored `yt_dlp/` and `spotify_dl/` forks (they win over the
|
||||||
|
pip packages because `/app` is first on `sys.path`), so your patches stay
|
||||||
|
active without the old `sed` hacks from `install_main_bot.sh`.
|
||||||
|
- **Tectonic** (LaTeX `$latex` command) is installed best-effort; if the build
|
||||||
|
step fails the bot still runs, just without LaTeX. Remove that layer from
|
||||||
|
`Dockerfile.bot` if you don't need it.
|
||||||
|
- Voice needs `ffmpeg` + `libopus0` (both in the image). No microphone/pyaudio
|
||||||
|
is required — voice is received over Discord and transcribed via AssemblyAI.
|
||||||
Reference in New Issue
Block a user