Dockerization + ai review recomendations.

.dockerignore

@@ -0,0 +1,32 @@
+# Python
+__pycache__
+*.py[cod]
+*.pyo
+*.pyd
+*.so
+
+# VCS/CI
+.git
+.gitignore
+.gitattributes
+.github
+
+# Editor
+.vscode
+.idea
+
+# Python tooling
+.pytest_cache
+.mypy_cache
+.ruff_cache
+.tox
+dist
+build
+*.egg-info
+
+# Local assets
+logs
+music
+data
+*.env
+*.env.*

README.md

@@ -1,2 +1,23 @@
 # conjurer
 Discord.py bot for fun, sex and BDSM
+
+## Docker quick start
+
+Docker definitions live at the repository root and let you run the Discord bot
+(`thin_client.py`), the musician file service, and the librarian search worker as
+separate containers.
+
+1. **Prepare configuration**
+   - Edit the environment files under `docker/env/*.env` and replace the
+     placeholders (`replace-me`, `shared-secret`, etc.) with your real tokens.
+     Keep `CONJURER_API_KEY` identical across all services.
+2. **Create host directories** listed in `docker-compose.yml` (for example
+   `docker/volumes/bot/config`, `docker/volumes/musician/data`, …) and populate
+   them with the required JSON settings or media assets.
+3. **Launch the stack**
+   ```bash
+   docker compose up --build
+   ```
+
+Logs for each container are mounted under `docker/volumes/*` so you can diff the
+runtime JSON and log files in git if needed.

communication_subroutine.py

@@ -1,12 +1,14 @@
 import json
 import logging
+import os
 import re
 import threading
 import time
 from queue import Empty, Queue
+from typing import Optional
 from urllib import request as urequest
 
-from flask import Flask, jsonify, request
+from flask import Flask, abort, jsonify, request
 from waitress import serve
 
 HOST_ADDRESS = "192.168.1.191"
@@ -31,6 +33,13 @@ PREPPED_TRACKS = {
 }
 logger = logging.getLogger("discord")
 
+API_KEY = os.getenv("CONJURER_API_KEY")
+
+
+def _authorize_request() -> None:
+    if API_KEY and request.headers.get("X-Conjurer-Api-Key") != API_KEY:
+        abort(401)
+
 
 class QueryControl:
     """
@@ -53,6 +62,7 @@ class QueryControl:
 
 @app.route("/prepped_tracks", methods=["POST"])
 def log_radio_tracks():
+    _authorize_request()
     app.logger = logging.getLogger("discord")
 
     app.logger.info(request)
@@ -79,6 +89,7 @@ def log_radio_tracks():
 
 @app.route("/conjurer", methods=["POST"])
 def answer_external_command():
+    _authorize_request()
     """
     The function `answer_external_command` logs the request data, loads the data as JSON, logs the
     record, and then puts the record into an incoming queue before returning a success message.
@@ -129,7 +140,7 @@ def waitress_run():
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
 
-def scan_queue():
+def scan_queue(stop_event: Optional[threading.Event] = None):
     """
     The function `scan_queue` reads data from a queue, logs it, and appends it to another queue.
 
@@ -140,12 +151,18 @@ def scan_queue():
     """
     logger = logging.getLogger("discord")
     while True:
-        data = OUT_COMM_Q.get()
+        if stop_event and stop_event.is_set():
+            logger.info("scan_queue: stop requested")
+            break
+        try:
+            data = OUT_COMM_Q.get(timeout=1)
+        except Empty:
+            continue
         logger.info(data)
         awaiting_q.append(data)
 
 
-def scan_incoming():
+def scan_incoming(stop_event: Optional[threading.Event] = None):
     """
     The `scan_incoming` function continuously checks for incoming data, processes it, and logs when data
     is found.
@@ -157,6 +174,9 @@ def scan_incoming():
     """
     logger = logging.getLogger("discord")
     while True:
+        if stop_event and stop_event.is_set():
+            logger.info("scan_incoming: stop requested")
+            break
         try:
             answer = incoming_q.get(block=False)
             logger.info("DATA FOUND")
@@ -204,7 +224,7 @@ def id3(url: str) -> dict:
         return tagdata
 
 
-def comm_subroutine():
+def comm_subroutine(stop_event: Optional[threading.Event] = None):
     """
     The `comm_subroutine` function starts multiple threads to run different tasks concurrently.
 
@@ -217,14 +237,27 @@ def comm_subroutine():
     logger.info("Started comms")
     threads = []
     # threads.append(threading.Thread(target=flask_debug))
-    threads.append(threading.Thread(target=waitress_run))
-    threads.append(threading.Thread(target=scan_queue))
-    threads.append(threading.Thread(target=scan_incoming))
+    threads.append(
+        threading.Thread(target=waitress_run, daemon=True)
+    )
+    threads.append(
+        threading.Thread(target=scan_queue, kwargs={"stop_event": stop_event}, daemon=True)
+    )
+    threads.append(
+        threading.Thread(target=scan_incoming, kwargs={"stop_event": stop_event}, daemon=True)
+    )
 
     for worker in threads:
         worker.start()
-    for worker in threads:
-        worker.join()
+
+    try:
+        while any(thread.is_alive() for thread in threads):
+            if stop_event and stop_event.is_set():
+                break
+            time.sleep(0.5)
+    finally:
+        if stop_event:
+            stop_event.set()
 
 
 if __name__ == "__main__":

conjurer_librarian/conjurer_librarian.py

@@ -16,34 +16,52 @@ Functions:
 import asyncio
 import json
 import logging
-import netrc
+import os
 import threading
-import time
 from json.decoder import JSONDecodeError
 from logging import handlers
+from pathlib import Path
 from queue import Queue
+from typing import Dict, Optional
 
 import requests
 import scrape_bot
 import search_bot
-#import search_bot2 as search_bot
-from flask import Flask, jsonify, request
+# import search_bot2 as search_bot
+from flask import Flask, jsonify, request, abort
 from habanero import Crossref
 from waitress import serve
 
+try:
+    import netrc
+except ImportError:  # pragma: no cover
+    netrc = None
+
 # Constants
-NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
-HOST_ADDRESS = "192.168.1.192"
-PORT_ADDRESS = 5001
-MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
-SEND_RESULTS = "/conjurer"
-BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
 
-MAX_CR_RESULTS = 500
-#TEST PURPOSES ONLY!
-#MAX_CR_RESULTS = 5
 
-ENCODING = "utf-8"
+def _env(name: str, default: str) -> str:
+    return os.getenv(name, default)
+
+
+def _env_path(name: str, default: str) -> Path:
+    return Path(os.getenv(name, default)).expanduser().resolve()
+
+
+BASE_DIR = Path(
+    os.getenv("CONJURER_LIBRARIAN_BASE", str(Path(__file__).resolve().parent))
+)
+NETRC_FILE = _env_path("CONJURER_NETRC_FILE", str(Path.home() / ".netrc"))
+HOST_ADDRESS = _env("CONJURER_LIBRARIAN_HOST", "0.0.0.0")
+PORT_ADDRESS = int(_env("CONJURER_LIBRARIAN_PORT", "5001"))
+MAIN_BOT_ADDRESS = _env("CONJURER_MAIN_BOT", "http://127.0.0.1:5000")
+SEND_RESULTS = _env("CONJURER_LIBRARIAN_RESULTS_ENDPOINT", "/conjurer")
+MAX_CR_RESULTS = int(_env("CONJURER_LIBRARIAN_MAX_RESULTS", "500"))
+ENCODING = _env("CONJURER_ENCODING", "utf-8")
+API_KEY = os.getenv("CONJURER_API_KEY")
+LOGFILE_PATH = _env_path(
+    "CONJURER_LIBRARIAN_LOG", str(BASE_DIR / "librarian.log")
+)
 
 app = Flask(__name__)
 
@@ -51,6 +69,17 @@ librarian_queue = Queue()
 librarian_list = []
 
 
+def _service_headers() -> Dict[str, str]:
+    if API_KEY:
+        return {"X-Conjurer-Api-Key": API_KEY}
+    return {}
+
+
+def _authorize_request() -> None:
+    if API_KEY and request.headers.get("X-Conjurer-Api-Key") != API_KEY:
+        abort(401)
+
+
 # trunk-ignore(pylint/R0902)
 class Librarian(object):
     """
@@ -81,11 +110,24 @@ class Librarian(object):
         - search_result_from_cr: A dictionary to store the search results from Crossref.
         - done: A flag indicating if the search is done.
         """
-        netrc_mod = netrc.netrc(NETRC_FILE)
+        mailto_contact: Optional[str] = os.getenv("CONJURER_CROSSREF_MAILTO")
+        if netrc:
+            try:
+                netrc_mod = netrc.netrc(str(NETRC_FILE))
                 auth_tokens = netrc_mod.authenticators("crossref")
+                if auth_tokens:
+                    mailto_contact = auth_tokens[0]
+            except (FileNotFoundError, netrc.NetrcParseError):
+                logging.getLogger("conjurer_librarian").warning(
+                    "Crossref credentials missing in netrc %s", NETRC_FILE
+                )
+        if not mailto_contact:
+            raise RuntimeError(
+                "Crossref credentials not configured. Set CONJURER_CROSSREF_MAILTO or add to netrc."
+            )
         self.cr = Crossref(
-            mailto=auth_tokens[0],
-            ua_string=f"Conjurer project. mailto:{auth_tokens[0]}"
+            mailto=mailto_contact,
+            ua_string=f"Conjurer project. mailto:{mailto_contact}"
         )
         self.query = query
         self.uuid = str(uuid)
@@ -132,7 +174,7 @@ class Librarian(object):
                 self.fetched = len(cr_result["message"]["items"])
                 self.app.logger.info(self.total)
                 self.app.logger.info(self.fetched)
-                time.sleep(0.1)
+                await asyncio.sleep(0.1)
 
         else:
             cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True)
@@ -411,18 +453,20 @@ class BackgroundTaskSearch(threading.Thread):
                 requests.post,
                 f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
                 json=result,
+                headers=_service_headers(),
                 timeout=360,
             )
             self.app.logger.info("SENT")
             result = await coroutine
             self.app.logger.info(result.status_code)
             self.app.logger.info("SEND CONFIRMED")
-            time.sleep(1)
+            await asyncio.sleep(1)
 
 
 # ==================================SERVER ROUTES==========================================
 @app.route("/query", methods=["POST"])
 async def query_database():
+    _authorize_request()
     """
     Endpoint for querying the database.
 
@@ -455,6 +499,7 @@ async def query_database():
 
 @app.route("/get_partial_result", methods=["POST"])
 async def get_partial():
+    _authorize_request()
     """
     Retrieves the partial result for a given UUID.
 
@@ -478,9 +523,10 @@ async def get_partial():
 # =======================================MAIN===================================================
 if __name__ == "__main__":
     app.logger.setLevel(logging.DEBUG)
+    LOGFILE_PATH.parent.mkdir(parents=True, exist_ok=True)
     h1 = handlers.RotatingFileHandler(
-        filename="D:\\logs\\librarian.log",
-        encoding="utf-8",
+        filename=str(LOGFILE_PATH),
+        encoding=ENCODING,
         mode="a",
         maxBytes=6 * 1024 * 1024,
         backupCount=6,
@@ -488,20 +534,24 @@ if __name__ == "__main__":
 
     app.logger.addHandler(h1)
     threads = []
-    threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=waitress_run, daemon=True))
     # threads.append(threading.Thread(target=flask_debug))
     bgtask = BackgroundTaskSearch()
     bgtask.app = app
+    bgtask.daemon = True
     threads.append(bgtask)
-    threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,)))
+    threads.append(
+        threading.Thread(
+            target=scrape_bot.scraper, args=(app.logger,), daemon=True
+        )
+    )
     i = 0
-    for worker in threads:
     try:
+        for worker in threads:
             app.logger.info("App number: %s", i)
             i += 1
             worker.start()
-        except RuntimeError as e:
-            app.logger.error("Exploded")
-            print(str(e))
         for worker in threads:
             worker.join()
+    except KeyboardInterrupt:
+        app.logger.info("Shutdown requested - exiting librarian service")

conjurer_musician/conjurer_musician.py

@@ -11,17 +11,15 @@ import random
 import re
 import threading
 import time
-
-# from flask_autoindex import AutoIndex
 from datetime import datetime
 from logging import handlers
 from pathlib import Path
-from platform import uname
-from sys import platform
+from typing import Dict, List
 
 import requests
 from flask import (
     Flask,
+    abort,
     jsonify,
     redirect,
     render_template,
@@ -29,37 +27,99 @@ from flask import (
     send_from_directory,
 )
 from waitress import serve
+
 import media_search_functions
 
 
-MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
-MUSIC_TRACKER = "/prepped_tracks"
-HOST_ADDRESS = "192.168.1.15"
-PORT_ADDRESS = 5000
-if platform in ("linux", "linux2"):
-    SEPARATOR_FILE_PATH = "/"
-    if "microsoft-standard" in uname().release:
-        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
-        MUSIC_FOLDER = "/mnt/g/Muzyka/"
-        NETRC_FILE = "/home/mtuszowski/.netrc"
-        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
-        ENCODING = "utf-8"
-        RADIOLOG_PATH = "/home/pi/Conjurer/radio_log.log"
-        PERSISTENCE_PATH = "/home/pi/Conjurer/persistence.log"
+def _env(name: str, default: str) -> str:
+    return os.getenv(name, default)
 
-    else:
-        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
-        NETRC_FILE = "/home/pi/.netrc"
-        LOGSTORE = "/home/pi/MediaFolder/logs/"
-        ENCODING = "utf-8"
-        MUSIC_FOLDER = "/home/pi/MediaFolder/mp3/"
-        PRIORITY_FOLDER = "/home/pi/MediaFoldermp3/Magiczne i chuj/"
-        RADIOLOG_PATH = "/home/pi/Conjurer/radio_log.log"
-        PERSISTENCE_PATH = "/home/pi/Conjurer/persistence.log"
+
+def _env_path(name: str, default: str) -> Path:
+    value = os.getenv(name, default)
+    return Path(value).expanduser().resolve()
+
+
+API_KEY = os.getenv("CONJURER_API_KEY")
+MAIN_BOT_ADDRESS = _env("CONJURER_MAIN_BOT", "http://127.0.0.1:5000")
+MUSIC_TRACKER = _env("CONJURER_MUSIC_TRACKER_ENDPOINT", "/prepped_tracks")
+HOST_ADDRESS = _env("CONJURER_MUSICIAN_HOST", "0.0.0.0")
+PORT_ADDRESS = int(_env("CONJURER_MUSICIAN_PORT", "5000"))
+
+BASE_DIR = Path(
+    os.getenv("CONJURER_MUSICIAN_BASE", str(Path(__file__).resolve().parent))
+)
+LOGFILE = _env_path(
+    "CONJURER_MUSICIAN_LOG", str(BASE_DIR / "discord_mus_service.log")
+)
+LOGSTORE = _env_path("CONJURER_LOGSTORE", str(BASE_DIR / "logs"))
+MUSIC_FOLDER = _env_path(
+    "CONJURER_MUSIC_FOLDER", str(BASE_DIR / "music")
+)
+PRIORITY_FOLDER = _env_path(
+    "CONJURER_PRIORITY_FOLDER", str(MUSIC_FOLDER / "priority")
+)
+RADIOLOG_PATH = _env_path(
+    "CONJURER_RADIO_LOG", str(BASE_DIR / "radio_log.log")
+)
+PERSISTENCE_PATH = _env_path(
+    "CONJURER_PERSISTENCE_LOG", str(BASE_DIR / "persistence.log")
+)
+ALL_PLAYLIST_PATH = _env_path(
+    "CONJURER_ALL_PLAYLIST", str(BASE_DIR / "all_playlist.playlist")
+)
+HIT_PLAYLIST_PATH = _env_path(
+    "CONJURER_HIT_PLAYLIST", str(BASE_DIR / "hit.playlist")
+)
+REQUEST_PLAYLIST_PATH = _env_path(
+    "CONJURER_REQUEST_PLAYLIST", str(BASE_DIR / "request.playlist")
+)
+PRIORITY_PLAYLIST_PATH = _env_path(
+    "CONJURER_PRIORITY_PLAYLIST", str(BASE_DIR / "priority_queue.playlist")
+)
+STREAM_TEMPLATE = _env_path(
+    "CONJURER_STREAM_TEMPLATE", str(BASE_DIR / "stream.html")
+)
+
+ENCODING = _env("CONJURER_ENCODING", "utf-8")
+SEPARATOR_FILE_PATH = os.sep
+
+for playlist_path in (
+    ALL_PLAYLIST_PATH,
+    HIT_PLAYLIST_PATH,
+    REQUEST_PLAYLIST_PATH,
+    PRIORITY_PLAYLIST_PATH,
+):
+    playlist_path.parent.mkdir(parents=True, exist_ok=True)
 
 random.seed()
-music_file_list = []
-priority_list = []
+music_file_list: List[str] = []
+priority_list: List[str] = []
+
+
+def _build_headers() -> Dict[str, str]:
+    headers: Dict[str, str] = {}
+    if API_KEY:
+        headers["X-Conjurer-Api-Key"] = API_KEY
+    return headers
+
+
+def _authorize_request() -> None:
+    if API_KEY and request.headers.get("X-Conjurer-Api-Key") != API_KEY:
+        abort(401)
+
+
+def _post_to_bot(payload: List[str]) -> None:
+    response = requests.post(
+        f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}",
+        json=payload,
+        headers=_build_headers(),
+        timeout=60,
+    )
+    logger = logging.getLogger("conjurer_musician")
+    logger.info("SENT")
+    logger.info(response.status_code)
+    logger.info("SEND CONFIRMED")
 
 
 def rescan():
@@ -70,28 +130,29 @@ def rescan():
     logger = logging.getLogger("conjurer_musician")
     logger.info("Rescan triggered")
 
-    for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
+    music_file_list.clear()
+    priority_list.clear()
+
+    for mp3_item in MUSIC_FOLDER.glob("**/*.mp3"):
         temp_music_file = mp3_item.as_posix()
-        if platform == "win32":
+        if os.name == "nt":
             temp_music_file = temp_music_file.replace("/", "\\")
         music_file_list.append(temp_music_file)
 
-    for mp3_item in Path.glob(Path(PRIORITY_FOLDER), "**/*.mp3"):
+    for mp3_item in PRIORITY_FOLDER.glob("**/*.mp3"):
         temp_music_file = mp3_item.as_posix()
-        if platform == "win32":
+        if os.name == "nt":
             temp_music_file = temp_music_file.replace("/", "\\")
         priority_list.append(temp_music_file)
 
-    with open(
-        "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
-    ) as w_file:
+    with ALL_PLAYLIST_PATH.open("w", encoding=ENCODING) as w_file:
         try:
             for item in music_file_list:
                 w_file.write(item)
                 w_file.write("\n")
         except json.JSONDecodeError:
             pass
-    with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file:
+    with HIT_PLAYLIST_PATH.open("w", encoding=ENCODING) as w_file:
         try:
             for item in priority_list:
                 w_file.write(item)
@@ -116,71 +177,49 @@ def thread_rescan():
 def scan_tracks():
     # Set the filename and open the file
     logger = logging.getLogger("conjurer_musician")
+    with open(RADIOLOG_PATH, "r", encoding=ENCODING) as log_file:
+        log_file.seek(os.stat(RADIOLOG_PATH).st_size)
+        prev_size = os.stat(PERSISTENCE_PATH).st_size
 
-    file = open(RADIOLOG_PATH, "r")
-    # Find the size of the file and move to the end
-    st_results = os.stat(RADIOLOG_PATH)
-    st_size = st_results[6]
-    file.seek(st_size)
-    st_results1 = os.stat(PERSISTENCE_PATH)
-    prev_st_size1 = st_results[6]
-
-    while 1:
-
-        st_results1 = os.stat(PERSISTENCE_PATH)
-        st_size1 = st_results1[6]
-        if prev_st_size1 != st_size1:
-            while prev_st_size1 != st_size1:
-                prev_st_size1 = st_size1
-                st_results1 = os.stat(PERSISTENCE_PATH)
-                st_size1 = st_results1[6]
+        while True:
+            current_size = os.stat(PERSISTENCE_PATH).st_size
+            if prev_size != current_size:
+                while prev_size != current_size:
+                    prev_size = current_size
                     time.sleep(0.1)
-            file1 = open(PERSISTENCE_PATH, "r")
-            lines = file1.readlines()
-            result = ["next", lines[2]]
-            file1.close()
-            returned = requests.post(
-                f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}", json=result, timeout=360
-            )
-            logger.info("SENT")
-            logger.info(returned.status_code)
-            logger.info("SEND CONFIRMED")
+                    current_size = os.stat(PERSISTENCE_PATH).st_size
+                with open(PERSISTENCE_PATH, "r", encoding=ENCODING) as persistence:
+                    lines = persistence.readlines()
+                if len(lines) >= 3:
+                    _post_to_bot(["next", lines[2]])
 
-        where = file.tell()
-        line = file.readline()
+            position = log_file.tell()
+            line = log_file.readline()
             if not line:
                 time.sleep(1)
-            file.seek(where)
-        else:
-            if re.match(".*Prepared.*", line):
+                log_file.seek(position)
+                continue
+
+            if not re.match(r".*Prepared.*", line):
+                time.sleep(0.1)
+                continue
+
             result = None
-                if re.match(".*jingles.*", line):
-                    logger.info("jingles")
-                    logger.info(line)  # already has newline
+            if re.match(r".*jingles.*", line):
                 result = ["jingles", line]
-                elif re.match(".*priority.*", line):
-                    logger.info("priority")
-                    logger.info(line)  # already has newline
+            elif re.match(r".*priority.*", line):
                 result = ["priority", line]
-                elif re.match(".*hit.*", line):
-                    logger.info("hit")
-                    logger.info(line)  # already has newline
+            elif re.match(r".*hit.*", line):
                 result = ["hit", line]
-                elif re.match(".*all_playlist.*", line):
-                    logger.info("all")
-                    logger.info(line)  # already has newline
+            elif re.match(r".*all_playlist.*", line):
                 result = ["all", line]
-                elif re.match(".*request.*", line):
-                    logger.info("requests")
-                    logger.info(line)  # already has newline
+            elif re.match(r".*request.*", line):
                 result = ["requests", line]
+
             if result:
-                    returned = requests.post(
-                        f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}", json=result, timeout=360
-                    )
-                    logger.info("SENT")
-                    logger.info(returned.status_code)
-                    logger.info("SEND CONFIRMED")
+                logger.info("Forwarding radio log entry: %s", result[0])
+                _post_to_bot(result)
+
             time.sleep(0.1)
 
 
@@ -290,12 +329,8 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
             if search_weight[itr][0] == item_to_search:
                 return_list.append(search_weight[itr])
                 if not return_to_bot:
-                    with open(
-                        "/home/pi/Conjurer/priority_queue.playlist",
-                        "r+",
-                        encoding="utf-8",
-                    ) as s_file:
-                        s_file.write(search_weight[itr][1])
+                    with PRIORITY_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file:
+                        s_file.write(search_weight[itr][1] + "\n")
                 break
             itr += 1
     else:
@@ -336,6 +371,7 @@ def remove_characters(string, character):
 
 @app.route('/get_share_list', methods=['POST'])
 def get_share_list():
+    _authorize_request()
     data = request.get_json()
     entries = data.get('entries')
     keywords = data.get('keywords')
@@ -352,6 +388,7 @@ def get_share_list():
 
 @app.route('/get_share_links', methods=['POST'])
 def get_share_links():
+    _authorize_request()
     data = request.get_json()
     file_paths = data.get('file_paths')
     # Validate file_paths list
@@ -374,7 +411,7 @@ def stream_music():
     """
 
     # return send_from_directory("/tmp/hls", "stream.m3u8")
-    return render_template("/home/pi/Conjurer/stream.html")
+    return render_template(str(STREAM_TEMPLATE))
 
 
 @app.route("/<string:file_name>")
@@ -406,15 +443,14 @@ def stream_music_mp3():
 
 @app.route("/clear_pr_pls", methods=["GET"])
 def clear_pr_pls():
+    _authorize_request()
     """
     The function `clear_pr_pls` clears the contents of the priority queue playlist file.
 
     :return: A JSON response indicating the success of the operation.
     """
     app.logger.info("CLEARING PLAYLIST")
-    with open(
-        "/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
-    ) as cleared_pl:
+    with PRIORITY_PLAYLIST_PATH.open("w", encoding=ENCODING) as cleared_pl:
         cleared_pl.write("")
 
     return_data = jsonify(isError=False, message="Success", statusCode=200, data=[])
@@ -442,6 +478,7 @@ def update_music_list():
     received and added to the `music_file_list`.
     The status code returned is 200, indicating a successful response.
     """
+    _authorize_request()
     record = json.loads(request.data)
     app.logger.info(record["item"])
     music_file_list.append(record["item"])
@@ -463,6 +500,7 @@ def look_for_playlist():
              data that was received and added to the `music_file_list`. The status code returned is 200,
              indicating a successful response.
     """
+    _authorize_request()
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["lista_slow"])
@@ -481,13 +519,14 @@ def look_for_playlist():
 
 @app.route("/request_radio_file", methods=["POST"])
 def add_request():
+    _authorize_request()
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["lista_slow"])
     app.logger.info(record["UUID"])
     return_data = wyszukaj(record["lista_slow"], 0, app.logger, False)
 
-    with open("/home/pi/Conjurer/request.playlist", "a", encoding="utf-8") as s_file:
+    with REQUEST_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file:
         for item in return_data:
             s_file.write(item[1] + "\n")
     return_data = (
@@ -512,6 +551,7 @@ def create_priority_playlist():
         data that was received and added to the `music_file_list`.
         The status code returned is 200,indicating a successful response.
     """
+    _authorize_request()
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["lista_slow"])
@@ -521,7 +561,7 @@ def create_priority_playlist():
         record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
     )
     random.shuffle(return_data)
-    with open("/home/pi/Conjurer/request.playlist", "a", encoding="utf-8") as s_file:
+    with REQUEST_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file:
         for item in return_data:
             s_file.write(item[1] + "\n")
     return_data = (
@@ -546,6 +586,7 @@ def add_to_priority():
         data that was received and added to the `music_file_list`.
         The status code returned is 200,indicating a successful response.
     """
+    _authorize_request()
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["lista_slow"])
@@ -554,9 +595,7 @@ def add_to_priority():
     return_data = wyszukaj(
         record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
     )
-    with open(
-        "/home/pi/Conjurer/priority_queue.playlist", "a", encoding="utf-8"
-    ) as s_file:
+    with PRIORITY_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file:
         for item in return_data:
             s_file.write(item[1] + "\n")
     return_data = (
@@ -615,15 +654,19 @@ if __name__ == "__main__":
     logger.info("Started")
     threads = []
     # threads.append(threading.Thread(target=flask_debug))
-    threads.append(threading.Thread(target=waitress_run))
-    threads.append(threading.Thread(target=thread_rescan))
+    threads.append(threading.Thread(target=waitress_run, daemon=True))
+    threads.append(threading.Thread(target=thread_rescan, daemon=True))
 
     for worker in threads:
         worker.start()
 
     time.sleep(60)
-    threads.append(threading.Thread(target=scan_tracks))
-    threads[2].start()
+    track_thread = threading.Thread(target=scan_tracks, daemon=True)
+    track_thread.start()
 
+    try:
         for worker in threads:
             worker.join()
+        track_thread.join()
+    except KeyboardInterrupt:
+        logger.info("Shutdown requested - exiting musician service")

constants.py

@@ -1,13 +1,36 @@
-import json
-import netrc
-from datetime import datetime
-from platform import uname
-from sys import platform
-from typing import List, Optional, TypedDict
+"""Centralised configuration and runtime constants for Conjurer services.
 
-import openai
-import spotipy
-from spotipy.oauth2 import SpotifyClientCredentials
+This module used to perform heavy filesystem and credential reads at import
+time which made the project brittle on hosts that did not mirror the original
+paths.  The current implementation defers that work, reads configuration from
+environment variables (with sensible fallbacks), and protects optional
+dependencies so the main bot can start even when a secondary service is
+offline.
+"""
+
+import json
+import logging
+import os
+from datetime import datetime
+from pathlib import Path
+from typing import Dict, List, Optional, Tuple, TypedDict
+
+try:
+    import netrc
+except ImportError:  # pragma: no cover - standard on CPython
+    netrc = None
+
+try:
+    import openai
+except ImportError:  # pragma: no cover - optional at runtime
+    openai = None
+
+try:
+    import spotipy
+    from spotipy.oauth2 import SpotifyClientCredentials
+except ImportError:  # pragma: no cover - optional component
+    spotipy = None
+    SpotifyClientCredentials = None
 
 Music_Config = TypedDict(
     "Music_Config",
@@ -24,21 +47,43 @@ MASTER_TIMEOUT = datetime.now()
 INITIAL_TIME_WAIT = 500
 MUZYKA: Music_Config = {"ctx": None, "queue": [], "requester": []}
 
-LOGFILE = ""
-NETRC_FILE = ""
-MUSIC_FOLDER = ""
-MEMORY_FIVE_SIARA = ""
-MEMORY_FIVE_MUZYKA = ""
-SETTINGS_FILE = ""
-ENCODING = ""
-GRAPHICS_PATH = ""
+logger = logging.getLogger("discord")
+
+
+def _env_path(var_name: str, fallback: Path) -> Path:
+    value = os.getenv(var_name)
+    if value:
+        return Path(value).expanduser().resolve()
+    return fallback
+
+
+def _env(var_name: str, fallback: str) -> str:
+    return os.getenv(var_name, fallback)
+
+
+BASE_DIR = Path(os.getenv("CONJURER_BASE_DIR", Path(__file__).resolve().parent))
+
+LOGFILE = _env_path("CONJURER_LOG_FILE", BASE_DIR / "discord.log")
+NETRC_FILE = _env_path("CONJURER_NETRC_FILE", Path.home() / ".netrc")
+SETTINGS_FILE = _env_path("CONJURER_SETTINGS_FILE", BASE_DIR / "settings.json")
+MEMORY_FIVE_SIARA = _env_path("CONJURER_MEMORY_FILE", BASE_DIR / "pamiec.json")
+MEMORY_FIVE_MUZYKA = _env_path(
+    "CONJURER_MUSIC_MEMORY_FILE", BASE_DIR / "pamiec_muzyki.json"
+)
+GRAPHICS_PATH = _env_path(
+    "CONJURER_GRAPHICS_PATH", BASE_DIR / "Conjurer_graphics"
+)
+MUSIC_FOLDER = _env_path("CONJURER_MUSIC_FOLDER", BASE_DIR / "music")
+ENCODING = _env("CONJURER_ENCODING", "utf-8")
+
+API_SHARED_KEY = os.getenv("CONJURER_API_KEY", "")
 MUZYKA_MOJEGO_LUDU_HISTORIA = 1500
 MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE = 15
 MUZYKA_MOJEGO_LUDU_PLAJLISTA = 30
 
-FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000"
-RADIO_HARBOR_ADDRESS = "http://192.168.1.15:54321"
-SKIP_TRACK = "/skip"
+FILE_SERVICE_ADDRESS = _env("CONJURER_FILE_SERVICE", "http://127.0.0.1:5000")
+RADIO_HARBOR_ADDRESS = _env("CONJURER_RADIO_HARBOR", "http://127.0.0.1:54321")
+SKIP_TRACK = _env("CONJURER_SKIP_ENDPOINT", "/skip")
 
 GET_MP3 = "/mp3"
 SEND_MP3 = "/update_mp3"
@@ -48,98 +93,123 @@ CREATE_PRIO_PLAYLIST = "/create_priority_playlist"
 
 REQUEST_MUSIC = "/request_radio_file"
 CLEAR_PRIO = "/clear_pr_pls"
-LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001"
-SEND_QUERY = "/query"
+LIBRARIAN_SERVICE_ADDRESS = _env(
+    "CONJURER_LIBRARIAN_SERVICE", "http://127.0.0.1:5001"
+)
+SEND_QUERY = _env("CONJURER_LIBRARIAN_QUERY_ENDPOINT", "/query")
 TIME_BETWEEN_CALLS = 100000
 LAST_SPONTANEOUS_CALL = datetime.now()
-HOST_ADDRESS = "192.168.1.191"
-PORT_ADDRESS = 5000
+HOST_ADDRESS = _env("CONJURER_DISCORD_HOST", "0.0.0.0")
+PORT_ADDRESS = int(_env("CONJURER_DISCORD_PORT", "5000"))
 
 # *=========================================== Platform Specific Predefines
 
-if platform in ("linux", "linux2"):
-    SEPARATOR_FILE_PATH = "/"
-    if "microsoft-standard" in uname().release:
-        LOGFILE = "/home/mtuszowski/conjurer/discord.log"
-        MEMORY_FIVE_SIARA = "/home/mtuszowski/conjurer/pamiec.json"
-        SYSTEM_GPT_SETTINGS = "/home/mtuszowski/conjurer/system_gpt_settings.json"
-        MEMORY_FIVE_MUZYKA = "/home/mtuszowski/conjurer/pamiec_muzyki.json"
-        MUSIC_FOLDER = "/mnt/g/Muzyka/"
-        SETTINGS_FILE = "/home/mtuszowski/conjurer/settings.json"
-        NETRC_FILE = "/home/mtuszowski/.netrc"
-        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
-        ACCIDENT_LOG = "/home/mtuszowski/conjurer/accident_log.json"
-        ENCODING = "utf-8"
-        GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
-        DIR_PATH_SADOX = "/mnt/c/Users/mtusz/OneDrive/Dokumenty/Fansadox"
+SEPARATOR_FILE_PATH = _env("CONJURER_PATH_SEPARATOR", os.sep)
 
-    else:
-        LOGFILE = "/home/pi/Conjurer/discord.log"
-        MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
-        SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
-        MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
-        MUSIC_FOLDER = "/home/pi/MediaShare/mp3/"
-        SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
-        NETRC_FILE = "/home/pi/.netrc"
-        LOGSTORE = "/home/pi/MediaShara/logs/"
-        ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
-        ENCODING = "utf-8"
-        GRAPHICS_PATH = "/home/pi/MediaShare/Conjurer_graphics/"
-        DIR_PATH_SADOX = "/home/pi/MediaShare/Fansadox/"
-
-
-elif platform == "win32":
-    LOGFILE = "discord.log"
-    MEMORY_FIVE_SIARA = "pamiec.json"
-    SYSTEM_GPT_SETTINGS = "system_gpt_settings.json"
-    MEMORY_FIVE_MUZYKA = "pamiec_muzyki.json"
-    MUSIC_FOLDER = "G:\\Muzyka\\"
-    SETTINGS_FILE = "settings.json"
-    NETRC_FILE = "C:\\Users\\mtusz\\.netrc"
-    LOGSTORE = "C:\\Users\\mtusz\\OneDrive\\Pulpit\\Conjurer\\logs\\"
-    ACCIDENT_LOG = "accident_log.json"
-    ENCODING = "utf-8"
-    DIR_PATH_SADOX = "C:\\Users\\mtusz\\OneDrive\\Dokumenty\\Fansadox\\"
-    SEPARATOR_FILE_PATH = "\\"
-with open(SETTINGS_FILE, "r", encoding=ENCODING) as f_settings_file:
-    DATA = json.load(f_settings_file)
-REMOTE_HOST_NAME = "openai"
-netrc_mod = netrc.netrc(NETRC_FILE)
-authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME)
-openai.api_key = authTokens[2]
-OPENAICLIENT = openai.AsyncOpenAI(api_key=openai.api_key)
-REMOTE_HOST_NAME = "discord"
-authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME)
-TOKEN = authTokens[2]
-
-REMOTE_HOST_NAME = "spotipy"
-authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME)
-SPOTIFY_CTRL = spotipy.Spotify(
-    client_credentials_manager=SpotifyClientCredentials(
-        client_id=authTokens[0],
-        client_secret=authTokens[2],
-    )
+DIR_PATH_SADOX = _env_path(
+    "CONJURER_SADOX_DIR", BASE_DIR / "Fansadox"
 )
-REMOTE_HOST_NAME = "youtube"
-authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME)
-YOUTUBE_AUTH = [authTokens[0],authTokens[2]]
 
-WORD_REACTIONS = DATA["word_reactions"]
-CYCLIC_WORDS = DATA["cyclic_words"]
+SYSTEM_GPT_SETTINGS = _env_path(
+    "CONJURER_SYSTEM_GPT_SETTINGS", BASE_DIR / "system_gpt_settings.json"
+)
+
+LOGSTORE = _env_path("CONJURER_LOGSTORE", BASE_DIR / "logs")
+ACCIDENT_LOG = _env_path(
+    "CONJURER_ACCIDENT_LOG", BASE_DIR / "accident_log.json"
+)
+
+
+def _load_json(path: Path, fallback) -> object:
+    try:
+        with path.open("r", encoding=ENCODING) as handle:
+            return json.load(handle)
+    except FileNotFoundError:
+        logger.warning("Missing JSON file at %s – using fallback", path)
+        return fallback
+    except json.JSONDecodeError:
+        logger.warning("Corrupt JSON at %s – resetting to fallback", path)
+        return fallback
+
+
+DATA = _load_json(SETTINGS_FILE, {})
+WORD_REACTIONS = DATA.get("word_reactions", {})
+CYCLIC_WORDS = DATA.get("cyclic_words", {})
 for key in WORD_REACTIONS:
+    if isinstance(WORD_REACTIONS[key], list) and len(WORD_REACTIONS[key]) >= 3:
         WORD_REACTIONS[key][2] = datetime.now()
-with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as temp_memory_file:
-    # First we load existing data into a dict.
-    MESSAGE_TABLE = json.load(temp_memory_file)
 
-with open(SYSTEM_GPT_SETTINGS, "r+", encoding=ENCODING) as temp_settings_file:
-    # First we load existing data into a dict.
-    GPT_SETTINGS = json.load(temp_settings_file)
-with open(MEMORY_FIVE_MUZYKA, "r+", encoding=ENCODING) as temp_music_memory_file:
-    # First we load existing data into a dict.
-    MESSAGE_TABLE_MUZYKA = json.load(temp_music_memory_file)
-SPECJALNE_ZIEMNIACZKI = GPT_SETTINGS[1]
-ASSISTANTS = {}
+MESSAGE_TABLE = _load_json(MEMORY_FIVE_SIARA, {})
+GPT_SETTINGS = _load_json(SYSTEM_GPT_SETTINGS, {})
+MESSAGE_TABLE_MUZYKA = _load_json(MEMORY_FIVE_MUZYKA, {})
+
+SPECJALNE_ZIEMNIACZKI = GPT_SETTINGS[1] if isinstance(GPT_SETTINGS, list) else {}
+ASSISTANTS: Dict[str, Tuple[str, str, int, object]] = {}
+
+
+def _load_netrc_credentials(host: str) -> Optional[Tuple[str, str, str]]:
+    if netrc is None:
+        return None
+    try:
+        parsed = netrc.netrc(str(NETRC_FILE))
+    except FileNotFoundError:
+        logger.warning("netrc file %s not found", NETRC_FILE)
+        return None
+    except netrc.NetrcParseError:
+        logger.warning("netrc file %s is invalid", NETRC_FILE)
+        return None
+    return parsed.authenticators(host)
+
+
+def _resolve_token(host: str, env_var: str) -> Optional[str]:
+    env_value = os.getenv(env_var)
+    if env_value:
+        return env_value
+    creds = _load_netrc_credentials(host)
+    if creds:
+        return creds[2]
+    logger.warning("Token for %s not configured", host)
+    return None
+
+
+OPENAI_API_KEY = _resolve_token("openai", "OPENAI_API_KEY")
+if openai and OPENAI_API_KEY:
+    openai.api_key = OPENAI_API_KEY
+    OPENAICLIENT = openai.AsyncOpenAI(api_key=OPENAI_API_KEY)
+else:
+    OPENAICLIENT = None
+
+TOKEN = _resolve_token("discord", "DISCORD_TOKEN")
+
+if spotipy:
+    spotify_creds = _load_netrc_credentials("spotipy")
+    if spotify_creds and SpotifyClientCredentials:
+        SPOTIFY_CTRL = spotipy.Spotify(
+            client_credentials_manager=SpotifyClientCredentials(
+                client_id=spotify_creds[0],
+                client_secret=spotify_creds[2],
+            )
+        )
+    else:
+        SPOTIFY_CTRL = None
+else:
+    SPOTIFY_CTRL = None
+
+youtube_creds = _load_netrc_credentials("youtube")
+if youtube_creds:
+    YOUTUBE_AUTH = [youtube_creds[0], youtube_creds[2]]
+else:
+    YOUTUBE_AUTH = [
+            os.getenv("YOUTUBE_USERNAME", ""),
+            os.getenv("YOUTUBE_PASSWORD", ""),
+        ]
+
+
+def service_headers() -> Dict[str, str]:
+    """Shared header dict for internal HTTP calls."""
+    if API_SHARED_KEY:
+        return {"X-Conjurer-Api-Key": API_SHARED_KEY}
+    return {}
 
 
 LATEX_TEX_ENGINE = "tectonic"

docker-compose.yml

@@ -0,0 +1,45 @@
+version: "3.9"
+
+services:
+  bot:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.bot
+    container_name: conjurer-bot
+    env_file:
+      - docker/env/bot.env
+    volumes:
+      - ./docker/volumes/bot/config:/data/config
+      - ./docker/volumes/bot/logs:/data/logs
+    depends_on:
+      - musician
+      - librarian
+    restart: unless-stopped
+    ports:
+      - "5000:5000"
+
+  musician:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.musician
+    container_name: conjurer-musician
+    env_file:
+      - docker/env/musician.env
+    volumes:
+      - ./docker/volumes/musician/data:/data
+    restart: unless-stopped
+    ports:
+      - "5001:5000"
+
+  librarian:
+    build:
+      context: .
+      dockerfile: docker/Dockerfile.librarian
+    container_name: conjurer-librarian
+    env_file:
+      - docker/env/librarian.env
+    volumes:
+      - ./docker/volumes/librarian/data:/data
+    restart: unless-stopped
+    ports:
+      - "5002:5001"

docker/Dockerfile.bot

@@ -0,0 +1,23 @@
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements_bot.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
+
+COPY . /app
+
+RUN useradd --create-home appuser \
+    && chown -R appuser:appuser /app
+
+USER appuser
+
+CMD ["python", "thin_client.py"]

docker/Dockerfile.librarian

@@ -0,0 +1,24 @@
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        build-essential \
+        poppler-utils \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY conjurer_librarian/requirements_librarian.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
+
+COPY . /app
+
+RUN useradd --create-home appuser \
+    && chown -R appuser:appuser /app
+
+USER appuser
+
+CMD ["python", "-m", "conjurer_librarian.conjurer_librarian"]

docker/Dockerfile.musician

@@ -0,0 +1,23 @@
+FROM python:3.11-slim
+
+ENV PYTHONUNBUFFERED=1 \
+    PIP_NO_CACHE_DIR=1
+
+RUN apt-get update \
+    && apt-get install -y --no-install-recommends \
+        ffmpeg \
+    && rm -rf /var/lib/apt/lists/*
+
+WORKDIR /app
+
+COPY requirements_bot.txt /tmp/requirements.txt
+RUN pip install -r /tmp/requirements.txt
+
+COPY . /app
+
+RUN useradd --create-home appuser \
+    && chown -R appuser:appuser /app
+
+USER appuser
+
+CMD ["python", "-m", "conjurer_musician.conjurer_musician"]

docker/env/bot.env.example

@@ -0,0 +1,24 @@
+# Discord bot service configuration
+
+DISCORD_TOKEN=HACKME!
+OPENAI_API_KEY=HACKME!
+CONJURER_API_KEY=HACKME!
+
+# Internal service endpoints
+CONJURER_FILE_SERVICE=http://conjurer-musician:5000
+CONJURER_LIBRARIAN_SERVICE=http://conjurer-librarian:5001
+
+# Runtime paths mounted via docker-compose volumes
+CONJURER_BASE_DIR=/data/config
+CONJURER_SETTINGS_FILE=/data/config/settings.json
+CONJURER_MEMORY_FILE=/data/config/pamiec.json
+CONJURER_MUSIC_MEMORY_FILE=/data/config/pamiec_muzyki.json
+CONJURER_SYSTEM_GPT_SETTINGS=/data/config/system_gpt_settings.json
+CONJURER_GRAPHICS_PATH=/data/assets/graphics
+CONJURER_LOG_FILE=/data/logs/discord.log
+CONJURER_LOGSTORE=/data/logs
+
+# Optional external integrations
+CONJURER_NETRC_FILE=/data/secrets/.netrc
+YOUTUBE_USERNAME=HACKME!
+YOUTUBE_PASSWORD=HACKME!

docker/env/librarian.env.example

@@ -0,0 +1,14 @@
+# Librarian service configuration
+
+CONJURER_API_KEY=HACKME!
+
+CONJURER_MAIN_BOT=http://conjurer-bot:5000
+
+CONJURER_LIBRARIAN_HOST=0.0.0.0
+CONJURER_LIBRARIAN_PORT=5001
+CONJURER_LIBRARIAN_MAX_RESULTS=500
+
+CONJURER_CROSSREF_MAILTO=HACKME!
+CONJURER_LIBRARIAN_LOG=/data/logs/librarian.log
+
+CONJURER_NETRC_FILE=/data/secrets/.netrc

docker/env/musician.env.example

@@ -0,0 +1,20 @@
+# Musician service configuration
+
+CONJURER_API_KEY=HACKME!
+
+CONJURER_MAIN_BOT=http://conjurer-bot:5000
+CONJURER_MUSIC_TRACKER_ENDPOINT=/prepped_tracks
+
+CONJURER_MUSICIAN_HOST=0.0.0.0
+CONJURER_MUSICIAN_PORT=5000
+
+CONJURER_MUSIC_FOLDER=/data/music
+CONJURER_PRIORITY_FOLDER=/data/priority
+CONJURER_ALL_PLAYLIST=/data/playlists/all_playlist.playlist
+CONJURER_HIT_PLAYLIST=/data/playlists/hit.playlist
+CONJURER_REQUEST_PLAYLIST=/data/playlists/request.playlist
+CONJURER_PRIORITY_PLAYLIST=/data/playlists/priority_queue.playlist
+CONJURER_STREAM_TEMPLATE=/data/templates/stream.html
+
+CONJURER_RADIO_LOG=/data/logs/radio.log
+CONJURER_PERSISTENCE_LOG=/data/logs/persistence.log

docs/deployment/DEPLOYMENT_GUIDE.md

@@ -0,0 +1,163 @@
+# Conjurer Deployment Guide
+
+This document walks through deploying the Conjurer stack (Discord bot, music
+service, librarian service) on two Raspberry Pi 4s and one Windows host, first
+with plain Docker Compose, then with a future Kubernetes setup.
+
+## 1. Current Hardware Layout
+
+- **Windows PC**: Stores persistent data (JSON memories, configs, music
+  catalogue). Shares folders over the network (SMB) for the Pis.
+- **Raspberry Pi A** (“radio”): Runs the Liquidsoap/liq radio pipeline and the
+  musician service (Flask + file watcher).
+- **Raspberry Pi B** (“bot”): Runs the Discord bot, communication bridge, and
+  librarian Flask service.
+
+You can rebalance as follows:
+
+| Service             | Recommended Host | Notes |
+|---------------------|------------------|-------|
+| Discord bot + comms | Raspberry Pi B   | Needs outbound internet, moderate CPU |
+| Librarian service   | Raspberry Pi B   | CPU-heavy during Crossref queries; keep close to bot |
+| Musician service    | Raspberry Pi A   | Has direct disk access to music, same box as Liquidsoap |
+| Data storage        | Windows          | Expose via SMB; mount inside containers |
+
+## 2. Prepare Shared Storage on Windows
+
+1. Create directories, e.g. `C:\Conjurer\config`, `C:\Conjurer\logs`,
+   `C:\Conjurer\music`, `C:\Conjurer\playlists`, `C:\Conjurer\secrets`.
+2. Copy your existing JSON settings (`settings.json`, `pamiec.json`,
+   `pamiec_muzyki.json`, `system_gpt_settings.json`, etc.) into `config`.
+3. Create blank placeholder files if they do not exist yet.
+4. Share the root folder (`C:\Conjurer`) over SMB with read/write access for the
+   Pi user (create credentials if necessary).
+
+## 3. Configure Environment Files
+
+1. On your workstation, copy the example env files:
+   ```bash
+   cp docker/env/bot.env.example docker/env/bot.env
+   cp docker/env/musician.env.example docker/env/musician.env
+   cp docker/env/librarian.env.example docker/env/librarian.env
+   ```
+2. Edit each `docker/env/*.env` to replace `HACKME!` with real values:
+   - `DISCORD_TOKEN`, `OPENAI_API_KEY`, `CONJURER_API_KEY` (use the same value for
+     all services).
+   - For musician/librarian, adjust mounts to the SMB paths you will mount on the
+     Pis, e.g. `/mnt/conjurer/music`.
+   - Set `CONJURER_CROSSREF_MAILTO` to a real email as required by Crossref.
+3. If you rely on `.netrc`, copy it to `C:\Conjurer\secrets\.netrc` and set
+   `CONJURER_NETRC_FILE` accordingly.
+
+## 4. Install Docker on Raspberry Pis and Windows
+
+### Raspberry Pi
+```bash
+curl -fsSL https://get.docker.com | sh
+sudo usermod -aG docker $USER
+sudo reboot
+
+# Install docker compose plugin
+sudo apt-get install docker-compose-plugin
+```
+
+### Windows
+- Install **Docker Desktop**.
+- Enable WSL2 backend and expose the shared Windows folders to the containers
+  (Docker Desktop settings → Resources → File Sharing).
+
+## 5. Deploy Musician Service (Pi A)
+
+1. SSH into Raspberry Pi A.
+2. Mount the Windows SMB share:
+   ```bash
+   sudo mkdir -p /mnt/conjurer
+   sudo apt-get install cifs-utils
+   sudo mount -t cifs //WINDOWS_HOST/Conjurer /mnt/conjurer -o user=YOURUSER
+   ```
+   Add an entry to `/etc/fstab` for persistence.
+3. Copy the repo to the Pi or `git clone` it.
+4. On Pi A, create override compose file (optional) pointing volumes to
+   `/mnt/conjurer`.
+5. Start only the musician service:
+   ```bash
+   docker compose --profile musician up --build -d musician
+   ```
+   Alternatively, duplicate `docker-compose.yml`, strip other services, and run
+   `docker compose up -d`.
+
+## 6. Deploy Bot + Librarian (Pi B)
+
+1. Repeat SMB mount on Pi B (same mount path).
+2. Copy repo / pull latest changes.
+3. Create `.env` files with tokens (or copy from control machine).
+4. Start bot and librarian:
+   ```bash
+   docker compose up -d bot librarian
+   ```
+
+## 7. Optional: Run Supporting Liquidsoap Radio
+
+- Keep Liquidsoap on Pi A as-is, using the same music directories. Ensure the
+  musician container has read access to those directories (bind mount).
+
+## 8. Verifying
+
+1. `docker ps` on each Pi to confirm containers running.
+2. Inspect logs under the mounted logs directory (`/mnt/conjurer/logs`).
+3. Join Discord server; issue commands to confirm functionality.
+4. Hit health endpoints manually (e.g. `curl http://PIB:5000/conjurer`).
+
+## Rebalancing Suggestions
+
+- If librarian CPU spikes become an issue, move it to Pi A or another host.
+- If you add a dedicated NAS, mount the network share read-only for the musician
+  container and read/write for other services.
+
+## 9. Future Kubernetes Deployment (Outline)
+
+### Hardware Considerations
+
+- Minimum three nodes for HA: use the existing two Pis plus one additional Pi 4
+  (8 GB preferred). Use Windows PC as storage provider via NFS/SMB CSI driver or
+  as a data gateway.
+- Consider Pi clusters with USB SSDs for better I/O.
+
+### Cluster Setup Steps
+
+1. Install a lightweight Kubernetes distribution (e.g., k3s) on each Pi:
+   ```bash
+   curl -sfL https://get.k3s.io | sh -
+   # On additional nodes
+   curl -sfL https://get.k3s.io | K3S_URL=https://MASTER:6443 K3S_TOKEN=HACKME sh -
+   ```
+2. Install MetalLB for load balancer support on LAN.
+3. Configure persistent volumes using:
+   - `nfs-subdir-external-provisioner` pointing to Windows share (ensure Windows
+     host supports NFS or run an NFS gateway on another machine).
+   - Alternatively, attach individual USB drives to each Pi and use
+     `local-path-provisioner` for node-local storage.
+4. Create Kubernetes `Secret` objects for tokens (`DISCORD_TOKEN`, etc.).
+5. Define `Deployment` manifests for each service (bot, musician, librarian) and
+   associated `Services`.
+6. Expose Discord bot ports via `NodePort` or Ingress.
+7. Use `StatefulSet` if you need stable identity for the musician service (due to
+   local storage).
+
+### Optimisation Tips
+
+- Keep CPU-heavy librarian pods optionally on a beefier node; use
+  `nodeSelector`/`affinity` to pin workloads.
+- Consider splitting the persistent storage: music on Pi A (USB disk), logs and
+  configs on Pi B, backups on Windows.
+- For improved reliability, add at least one extra Pi for quorum and to host the
+  communication bridge if the bot node fails.
+
+## Summary Checklist
+
+1. Prepare Windows shares & tokens.
+2. Configure `docker/env/*.env` using `HACKME!` templates as reference.
+3. Install Docker on Pis, mount network shares.
+4. Launch musician on Pi A, bot + librarian on Pi B.
+5. Verify Discord functionality and API endpoints.
+6. Plan Kubernetes migration when ready (k3s + MetalLB + storage provisioner).

librarian_commands.py

@@ -15,7 +15,14 @@ from discord.ext import commands, tasks
 
 from ai_functions import handle_response
 from communication_subroutine import IN_COMM_Q, OUT_COMM_Q, QueryControl
-from constants import DIR_PATH_SADOX, LIBRARIAN_SERVICE_ADDRESS, SEND_QUERY
+from constants import (
+    DIR_PATH_SADOX,
+    LIBRARIAN_SERVICE_ADDRESS,
+    SEND_QUERY,
+    service_headers,
+)
+
+SERVICE_HEADERS = service_headers()
 
 
 class DataModule(commands.Cog):
@@ -165,6 +172,7 @@ class DataModule(commands.Cog):
             requests.post,
             f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
             json=json_query,
+            headers=SERVICE_HEADERS,
             timeout=360,
         )
         await ctx.send(
@@ -260,6 +268,7 @@ class DataModule(commands.Cog):
             requests.post,
             f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
             json=json_query,
+            headers=SERVICE_HEADERS,
             timeout=360,
         )
         await ctx.send(

music_functions.py

@@ -17,11 +17,16 @@ from constants import (
     SEND_MP3,
     SPOTIFY_CTRL,
     YOUTUBE_AUTH,
+    service_headers,
 )
 from spotify_dl import spotify
 from spotify_dl import youtube as youtube_download
 
 
+SERVICE_HEADERS = service_headers()
+MUSIC_ROOT = Path(MUSIC_FOLDER)
+
+
 
 class MusicFileList(object):
     """
@@ -43,11 +48,15 @@ class MusicFileList(object):
         """
         try:
             self.logger.info("Attempt to connect to file service")
-            response = requests.get(f"{FILE_SERVICE_ADDRESS}{GET_MP3}", timeout=360)
+            response = requests.get(
+                f"{FILE_SERVICE_ADDRESS}{GET_MP3}",
+                headers=SERVICE_HEADERS,
+                timeout=60,
+            )
             self.music_file_list = response.json()["music_file_list"]
             self.file_service_active = True
         except requests.exceptions.RequestException as e:
-            for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
+            for mp3_item in MUSIC_ROOT.glob("**/*.mp3"):
                 temp_music_file = mp3_item.as_posix()
                 if platform == "win32":
                     temp_music_file = temp_music_file.replace("/", "\\")
@@ -98,7 +107,12 @@ class MusicFileList(object):
         """
         self.music_file_list.append(item)
         post_data = {"item": str(item)}
-        requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360)
+        requests.post(
+            f"{FILE_SERVICE_ADDRESS}{SEND_MP3}",
+            json=post_data,
+            headers=SERVICE_HEADERS,
+            timeout=60,
+        )
 
 
 MUSIC_FILE_LIST = MusicFileList("discord")
@@ -142,7 +156,7 @@ async def get_file(ctx, source, link):
             url_data = {"urls": []}
             url_dict = {}
             url_dict["save_path"] = Path(
-                PurePath.joinpath(Path(MUSIC_FOLDER), Path(directory_name))
+                PurePath.joinpath(MUSIC_ROOT, Path(directory_name))
             )
             url_dict["save_path"].mkdir(parents=True, exist_ok=True)
             url_dict["songs"] = file_list
@@ -152,7 +166,7 @@ async def get_file(ctx, source, link):
             coro = asyncio.to_thread(
                 youtube_download.download_songs,
                 songs=url_data,
-                output_dir=MUSIC_FOLDER,
+                output_dir=str(MUSIC_ROOT),
                 format_str="bestaudio/best",
                 skip_mp3=False,
                 keep_playlist_order=False,
@@ -308,6 +322,7 @@ async def search_music(ctx, how_many=0, slowa_kluczowe=None):
         requests.post,
         f"{FILE_SERVICE_ADDRESS}{GET_PLAYLIST}",
         json=jrequest,
+        headers=SERVICE_HEADERS,
         timeout=360,
     )
     return_data = await coroutine

radio_commands.py

@@ -8,7 +8,18 @@ import uuid
 
 import asyncio
 from datetime import datetime
-from constants import RADIO_HARBOR_ADDRESS, SKIP_TRACK, FILE_SERVICE_ADDRESS, ADD_TO_PRIO_PLAYLIST, REQUEST_MUSIC, CREATE_PRIO_PLAYLIST, CLEAR_PRIO
+from constants import (
+    RADIO_HARBOR_ADDRESS,
+    SKIP_TRACK,
+    FILE_SERVICE_ADDRESS,
+    ADD_TO_PRIO_PLAYLIST,
+    REQUEST_MUSIC,
+    CREATE_PRIO_PLAYLIST,
+    CLEAR_PRIO,
+    service_headers,
+)
+
+SERVICE_HEADERS = service_headers()
 
 class RadioModule(commands.Cog):
     def __init__(self, bot, logger_name):
@@ -95,6 +106,7 @@ class RadioModule(commands.Cog):
                 requests.post,
                 f"{FILE_SERVICE_ADDRESS}{ADD_TO_PRIO_PLAYLIST}",
                 json=jrequest,
+                headers=SERVICE_HEADERS,
                 timeout=360,
             )
             result = await coroutine
@@ -130,6 +142,7 @@ class RadioModule(commands.Cog):
                 requests.post,
                 f"{FILE_SERVICE_ADDRESS}{REQUEST_MUSIC}",
                 json=jrequest,
+                headers=SERVICE_HEADERS,
                 timeout=360,
             )
             result = await coroutine
@@ -167,6 +180,7 @@ class RadioModule(commands.Cog):
                 requests.post,
                 f"{FILE_SERVICE_ADDRESS}{CREATE_PRIO_PLAYLIST}",
                 json=jrequest,
+                headers=SERVICE_HEADERS,
                 timeout=360,
             )
             result = await coroutine
@@ -194,6 +208,7 @@ class RadioModule(commands.Cog):
             coroutine = asyncio.to_thread(
                 requests.get,
                 f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}",
+                headers=SERVICE_HEADERS,
                 timeout=360,
             )
             result = await coroutine

thin_client.py

@@ -2,17 +2,19 @@
 # trunk-ignore-all(bandit/B311)
 # pylint: disable=line-too-long
 # pylint: disable=too-many-lines
-"""
-Module of a python bot named Conjurer - used to work on BDSM discord servers.
-"""
-import logging
+"""Discord entrypoint for the Conjurer bot.
 
-# *=========================================== Standard Library Imports
+The legacy bootstrap used threads and synchronous calls that made clean
+shutdowns difficult.  We now run everything from a single asyncio event loop
+with cooperative shutdown signals so the bot can stop gracefully.
+"""
+
+import asyncio
+import logging
 import random
 import threading
 from logging import handlers
 
-# *==============Imported libraries
 import discord
 from discord.ext import commands
 
@@ -82,22 +84,40 @@ async def on_ready():
     logger.info("All systems: operational")
 
 
-# *================================== Run
+async def _run_comm_subroutine(stop_event: threading.Event) -> None:
+    await asyncio.to_thread(comm_subroutine, stop_event)
+
+
+async def _run_bot(token: str, shutdown_event: asyncio.Event) -> None:
+    try:
+        await client.start(token, log_handler=None)
+    finally:
+        shutdown_event.set()
+
+
+async def main() -> None:
+    if not TOKEN:
+        logger.error("Discord token missing – set DISCORD_TOKEN or configure netrc")
+        return
+
+    shutdown_event = asyncio.Event()
+    comm_stop_event = threading.Event()
+
+    comm_task = asyncio.create_task(_run_comm_subroutine(comm_stop_event))
+    bot_task = asyncio.create_task(_run_bot(TOKEN, shutdown_event))
+
+    try:
+        await shutdown_event.wait()
+    except (KeyboardInterrupt, asyncio.CancelledError):
+        logger.info("Shutdown signal received")
+        comm_stop_event.set()
+        await client.close()
+    finally:
+        comm_stop_event.set()
+        if not client.is_closed():
+            await client.close()
+        await asyncio.gather(bot_task, comm_task, return_exceptions=True)
+
+
 if __name__ == "__main__":
-    logger.info("Starting discord bot")
-    threads = []
-    logger.info("Starting discord bot: Creating threads")
-    threads.append(threading.Thread(target=client.run, args=(TOKEN,),kwargs={"log_handler":None}))
-    threads.append(threading.Thread(target=comm_subroutine))
-    logger.info("Starting discord bot: Starting threads")
-    WRK_CNT = 0
-    for worker in threads:
-        WRK_CNT += 1
-        logger.info("Starting discord bot: Starting thread %s", WRK_CNT)
-        worker.start()
-    logger.info("Starting discord bot: Joining threads")
-    WRK_CNT = 0
-    for worker in threads:
-        WRK_CNT += 1
-        logger.info("Starting discord bot: Joining thread %s", WRK_CNT)
-        worker.join()
+    asyncio.run(main())