mirror of
https://github.com/migatu/conjurer.git
synced 2026-07-14 21:38:38 +00:00
docker: Proxmox deployment for bot/librarian/musician + code fixes
Containerises the three services (each intended for its own Proxmox VM)
and adds the code changes needed to run cleanly on Linux/Docker.
Code fixes:
- constants.py: CONJURER_DATA_DIR roots all writable bot state under one
mounted volume (per-variable overrides still win; native Pi unaffected)
- conjurer_librarian/search_bot.py + scrape_bot.py: the hardcoded Windows
DOI database path (C:\Database\chunks\) is now CONJURER_LIBRARIAN_DB_PATH,
with CONJURER_LIBRARIAN_MAXTHREADS / _CHUNK also env-overridable
Docker:
- docker/Dockerfile.{bot,librarian,musician} + compose.{bot,librarian,musician}.yaml
- docker/env/*.env.example (force-added; real *.env stays gitignored)
- docker/entrypoint.bot.sh seeds default JSON state into /data only when
absent, so preserved history is never overwritten
- .dockerignore
- docs/deployment/DOCKER_PROXMOX.md: step-by-step runbook incl. preserving
the existing command/conversation history and cross-VM auth
The bot image uses the vendored yt_dlp/spotify_dl forks (they win on
sys.path over the pip packages), dropping the old sed patching.
Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
This commit is contained in:
Michal Tuszowski
@@ -0,0 +1,56 @@
|
||||
# Conjurer main Discord bot.
|
||||
# Build from the repository root:
|
||||
# docker build -f docker/Dockerfile.bot -t conjurer-bot .
|
||||
FROM python:3.11-slim
|
||||
|
||||
# System dependencies:
|
||||
# ffmpeg - audio download/convert (yt_dlp) and Discord voice
|
||||
# libopus0 - Discord voice (PyNaCl / discord-ext-voice-recv)
|
||||
# poppler-utils - pdf2image (librarian/latex previews)
|
||||
# git - required by the git+https entry in requirements
|
||||
# build-essential - native wheels (PyNaCl etc.)
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
ffmpeg \
|
||||
libopus0 \
|
||||
poppler-utils \
|
||||
git \
|
||||
build-essential \
|
||||
curl \
|
||||
ca-certificates \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
# Optional: Tectonic for the $latex command. Remove this layer if unused.
|
||||
RUN curl -fsSL https://drop-sh.fullyjustified.net | sh \
|
||||
&& mv tectonic /usr/local/bin/tectonic \
|
||||
|| echo "tectonic not installed - the LaTeX feature will be disabled"
|
||||
|
||||
COPY requirements_bot.txt ./
|
||||
RUN pip install --no-cache-dir --upgrade pip \
|
||||
&& pip install --no-cache-dir -r requirements_bot.txt
|
||||
|
||||
# Vendored forks take import precedence over the pip packages of the same name,
|
||||
# because /app (the script dir) is first on sys.path when running `python bot.py`.
|
||||
COPY yt_dlp ./yt_dlp
|
||||
COPY spotify_dl ./spotify_dl
|
||||
|
||||
# Bot sources + default config/asset templates. Runtime state (conversation
|
||||
# history etc.) is read from the mounted /data volume via CONJURER_DATA_DIR,
|
||||
# so these committed copies only act as first-run fallbacks.
|
||||
COPY *.py ./
|
||||
COPY settings.json system_gpt_settings.json accident_log.json pamiec.json pamiec_muzyki.json ./
|
||||
COPY fuckery.jpg willowisp.png wod_beacon.jpg ./
|
||||
COPY docker/entrypoint.bot.sh /usr/local/bin/entrypoint.sh
|
||||
RUN chmod +x /usr/local/bin/entrypoint.sh
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
CONJURER_DATA_DIR=/data \
|
||||
CONJURER_DISCORD_HOST=0.0.0.0 \
|
||||
CONJURER_DISCORD_PORT=5000
|
||||
|
||||
VOLUME ["/data"]
|
||||
EXPOSE 5000
|
||||
|
||||
ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
|
||||
CMD ["python", "bot.py"]
|
||||
@@ -0,0 +1,27 @@
|
||||
# Conjurer librarian (Crossref search + local DOI database lookup).
|
||||
# Build from the repository root:
|
||||
# docker build -f docker/Dockerfile.librarian -t conjurer-librarian .
|
||||
FROM python:3.11-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY conjurer_librarian/requirements_librarian.txt ./
|
||||
RUN pip install --no-cache-dir --upgrade pip \
|
||||
&& pip install --no-cache-dir -r requirements_librarian.txt requests
|
||||
|
||||
COPY conjurer_librarian/ ./
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
CONJURER_LIBRARIAN_HOST=0.0.0.0 \
|
||||
CONJURER_LIBRARIAN_PORT=5001 \
|
||||
CONJURER_LIBRARIAN_DB_PATH=/doi/
|
||||
|
||||
# The local DOI chunk database (large) is mounted here.
|
||||
VOLUME ["/doi"]
|
||||
EXPOSE 5001
|
||||
|
||||
CMD ["python", "conjurer_librarian.py"]
|
||||
@@ -0,0 +1,30 @@
|
||||
# Conjurer musician (Flask file/playlist service backing the radio).
|
||||
# Build from the repository root:
|
||||
# docker build -f docker/Dockerfile.musician -t conjurer-musician .
|
||||
#
|
||||
# NOTE: this containerises the musician *web service* only. The Liquidsoap
|
||||
# radio (radio_conjurer.liq) and any Samba/NFS share tooling run separately.
|
||||
FROM python:3.11-slim
|
||||
|
||||
RUN apt-get update && apt-get install -y --no-install-recommends \
|
||||
build-essential \
|
||||
&& rm -rf /var/lib/apt/lists/*
|
||||
|
||||
WORKDIR /app
|
||||
|
||||
COPY conjurer_musician/requirements_musician.txt ./
|
||||
RUN pip install --no-cache-dir --upgrade pip \
|
||||
&& pip install --no-cache-dir -r requirements_musician.txt
|
||||
|
||||
COPY conjurer_musician/ ./
|
||||
|
||||
ENV PYTHONUNBUFFERED=1 \
|
||||
CONJURER_MUSICIAN_HOST=0.0.0.0 \
|
||||
CONJURER_MUSICIAN_PORT=5000 \
|
||||
CONJURER_MUSIC_FOLDER=/music
|
||||
|
||||
# The mp3 library is mounted here (read-only is fine for serving).
|
||||
VOLUME ["/music"]
|
||||
EXPOSE 5000
|
||||
|
||||
CMD ["python", "conjurer_musician.py"]
|
||||
@@ -0,0 +1,22 @@
|
||||
# Main bot VM. Run from the repository root:
|
||||
# cp docker/env/bot.env.example docker/env/bot.env # then edit
|
||||
# docker compose -f docker/compose.bot.yaml up -d --build
|
||||
services:
|
||||
conjurer-bot:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile.bot
|
||||
image: conjurer-bot:latest
|
||||
container_name: conjurer-bot
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- env/bot.env
|
||||
ports:
|
||||
# Flask comm layer — musician/librarian POST results here (/prepped_tracks, /conjurer).
|
||||
- "5000:5000"
|
||||
volumes:
|
||||
# Persistent state (conversation history, settings, logs). Populate this
|
||||
# host dir with your existing pamiec.json etc. to preserve command history.
|
||||
- /srv/conjurer/data:/data
|
||||
# Tokens: a read-only netrc covers discord/openai/spotipy/youtube in one file.
|
||||
- /srv/conjurer/secrets/.netrc:/secrets/.netrc:ro
|
||||
@@ -0,0 +1,20 @@
|
||||
# Librarian VM. Run from the repository root:
|
||||
# cp docker/env/librarian.env.example docker/env/librarian.env # then edit
|
||||
# docker compose -f docker/compose.librarian.yaml up -d --build
|
||||
services:
|
||||
conjurer-librarian:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile.librarian
|
||||
image: conjurer-librarian:latest
|
||||
container_name: conjurer-librarian
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- env/librarian.env
|
||||
ports:
|
||||
- "5001:5001"
|
||||
volumes:
|
||||
# Local DOI chunk database (0_chunk.txt ... N_chunk.txt).
|
||||
- /srv/librarian/doi:/doi:ro
|
||||
# Optional: netrc holding Crossref credentials (or use CONJURER_CROSSREF_MAILTO).
|
||||
- /srv/librarian/secrets/.netrc:/secrets/.netrc:ro
|
||||
@@ -0,0 +1,21 @@
|
||||
# Musician VM (optional — you plan to adapt/implement this yourself).
|
||||
# Run from the repository root:
|
||||
# cp docker/env/musician.env.example docker/env/musician.env # then edit
|
||||
# docker compose -f docker/compose.musician.yaml up -d --build
|
||||
services:
|
||||
conjurer-musician:
|
||||
build:
|
||||
context: ..
|
||||
dockerfile: docker/Dockerfile.musician
|
||||
image: conjurer-musician:latest
|
||||
container_name: conjurer-musician
|
||||
restart: unless-stopped
|
||||
env_file:
|
||||
- env/musician.env
|
||||
ports:
|
||||
- "5000:5000"
|
||||
volumes:
|
||||
# The mp3 library the service indexes and serves.
|
||||
- /srv/musician/music:/music
|
||||
# Runtime playlists/logs the service writes.
|
||||
- /srv/musician/data:/data
|
||||
Executable
+16
@@ -0,0 +1,16 @@
|
||||
#!/bin/sh
|
||||
# Seed the data volume with the baked-in default JSON state on first run only.
|
||||
# Existing files (e.g. your preserved pamiec.json history) are never overwritten.
|
||||
set -e
|
||||
|
||||
DATA="${CONJURER_DATA_DIR:-/data}"
|
||||
mkdir -p "$DATA"
|
||||
|
||||
for f in settings.json system_gpt_settings.json pamiec.json pamiec_muzyki.json accident_log.json; do
|
||||
if [ ! -e "$DATA/$f" ] && [ -e "/app/$f" ]; then
|
||||
cp "/app/$f" "$DATA/$f"
|
||||
echo "entrypoint: seeded $f into $DATA"
|
||||
fi
|
||||
done
|
||||
|
||||
exec "$@"
|
||||
Vendored
+38
@@ -0,0 +1,38 @@
|
||||
# Copy to docker/env/bot.env and fill in. Do NOT commit the real file.
|
||||
|
||||
# --- Secrets ------------------------------------------------------------
|
||||
# Option A: mount a netrc (recommended — covers discord/openai/spotipy/youtube).
|
||||
CONJURER_NETRC_FILE=/secrets/.netrc
|
||||
# Option B: pass tokens directly (these take precedence over netrc).
|
||||
# DISCORD_TOKEN=
|
||||
# OPENAI_API_KEY=
|
||||
# YOUTUBE_USERNAME=
|
||||
# YOUTUBE_PASSWORD=
|
||||
|
||||
# --- Data ---------------------------------------------------------------
|
||||
# Single mounted volume; all writable state is rooted here.
|
||||
CONJURER_DATA_DIR=/data
|
||||
|
||||
# --- Flask comm layer (inbound from musician/librarian) -----------------
|
||||
CONJURER_DISCORD_HOST=0.0.0.0
|
||||
CONJURER_DISCORD_PORT=5000
|
||||
|
||||
# --- Internal service auth ----------------------------------------------
|
||||
# Set the SAME value on bot + musician + librarian. Empty = auth disabled.
|
||||
CONJURER_API_KEY=
|
||||
|
||||
# --- Where the bot reaches the other services (other Proxmox VMs) --------
|
||||
CONJURER_FILE_SERVICE=http://MUSICIAN_VM_IP:5000
|
||||
CONJURER_RADIO_HARBOR=http://MUSICIAN_VM_IP:54321
|
||||
CONJURER_LIBRARIAN_SERVICE=http://LIBRARIAN_VM_IP:5001
|
||||
|
||||
# --- Conan Exiles bridge (optional; empty/0 = disabled) -----------------
|
||||
# CONAN_GM_ROLE_ID=0
|
||||
# CONAN_RCON_HOST=
|
||||
# CONAN_RCON_PORT=25575
|
||||
# CONAN_RCON_PASSWORD=
|
||||
# CONAN_CHAT_CHANNEL_ID=0
|
||||
# CONAN_EVENTS_CHANNEL_ID=0
|
||||
# CONAN_JOIN_CHANNEL_ID=0
|
||||
# CONAN_LOG_MODE=local
|
||||
# CONAN_LOG_PATH=
|
||||
Vendored
+21
@@ -0,0 +1,21 @@
|
||||
# Copy to docker/env/librarian.env and fill in.
|
||||
|
||||
CONJURER_LIBRARIAN_HOST=0.0.0.0
|
||||
CONJURER_LIBRARIAN_PORT=5001
|
||||
|
||||
# Same shared secret as the bot (empty = auth disabled).
|
||||
CONJURER_API_KEY=
|
||||
|
||||
# Where to POST search results back to (the bot's comm layer).
|
||||
CONJURER_MAIN_BOT=http://BOT_VM_IP:5000
|
||||
|
||||
# Crossref polite-pool contact (or put credentials in netrc under "crossref").
|
||||
CONJURER_CROSSREF_MAILTO=you@example.com
|
||||
|
||||
# Local DOI chunk database (mounted volume): expects 0_chunk.txt .. N_chunk.txt
|
||||
CONJURER_LIBRARIAN_DB_PATH=/doi/
|
||||
CONJURER_LIBRARIAN_MAXTHREADS=41
|
||||
CONJURER_LIBRARIAN_CHUNK=_chunk.txt
|
||||
|
||||
# Optional netrc (for Crossref credentials)
|
||||
CONJURER_NETRC_FILE=/secrets/.netrc
|
||||
Vendored
+17
@@ -0,0 +1,17 @@
|
||||
# Copy to docker/env/musician.env and fill in.
|
||||
|
||||
CONJURER_MUSICIAN_HOST=0.0.0.0
|
||||
CONJURER_MUSICIAN_PORT=5000
|
||||
|
||||
# Same shared secret as the bot (empty = auth disabled).
|
||||
CONJURER_API_KEY=
|
||||
|
||||
# Where to POST "now playing" / prepped-track updates (the bot's comm layer).
|
||||
CONJURER_MAIN_BOT=http://BOT_VM_IP:5000
|
||||
|
||||
# The mp3 library (mounted volume).
|
||||
CONJURER_MUSIC_FOLDER=/music
|
||||
|
||||
# Runtime paths (mounted volume) — playlists/logs the service writes.
|
||||
CONJURER_MUSICIAN_BASE=/data
|
||||
CONJURER_LOGSTORE=/data/logs
|
||||
Reference in New Issue
Block a user