From f9ad6798330f11bc1a183005ae5687b8785a854b Mon Sep 17 00:00:00 2001 From: Polish Hammer Date: Wed, 29 Oct 2025 14:57:43 +0100 Subject: [PATCH] Dockerization + ai review recomendations. --- .dockerignore | 32 +++ README.md | 21 ++ communication_subroutine.py | 53 ++++- conjurer_librarian/conjurer_librarian.py | 112 ++++++--- conjurer_musician/conjurer_musician.py | 281 +++++++++++++---------- constants.py | 274 ++++++++++++++-------- docker-compose.yml | 45 ++++ docker/Dockerfile.bot | 23 ++ docker/Dockerfile.librarian | 24 ++ docker/Dockerfile.musician | 23 ++ docker/env/bot.env.example | 24 ++ docker/env/librarian.env.example | 14 ++ docker/env/musician.env.example | 20 ++ docs/deployment/DEPLOYMENT_GUIDE.md | 163 +++++++++++++ librarian_commands.py | 11 +- music_functions.py | 25 +- radio_commands.py | 17 +- thin_client.py | 68 ++++-- 18 files changed, 937 insertions(+), 293 deletions(-) create mode 100644 .dockerignore create mode 100644 docker-compose.yml create mode 100644 docker/Dockerfile.bot create mode 100644 docker/Dockerfile.librarian create mode 100644 docker/Dockerfile.musician create mode 100644 docker/env/bot.env.example create mode 100644 docker/env/librarian.env.example create mode 100644 docker/env/musician.env.example create mode 100644 docs/deployment/DEPLOYMENT_GUIDE.md diff --git a/.dockerignore b/.dockerignore new file mode 100644 index 0000000..d6dafb4 --- /dev/null +++ b/.dockerignore @@ -0,0 +1,32 @@ +# Python +__pycache__ +*.py[cod] +*.pyo +*.pyd +*.so + +# VCS/CI +.git +.gitignore +.gitattributes +.github + +# Editor +.vscode +.idea + +# Python tooling +.pytest_cache +.mypy_cache +.ruff_cache +.tox +dist +build +*.egg-info + +# Local assets +logs +music +data +*.env +*.env.* diff --git a/README.md b/README.md index a63ee26..020b74a 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,23 @@ # conjurer Discord.py bot for fun, sex and BDSM + +## Docker quick start + +Docker definitions live at the repository root and let you run the Discord bot +(`thin_client.py`), the musician file service, and the librarian search worker as +separate containers. + +1. **Prepare configuration** + - Edit the environment files under `docker/env/*.env` and replace the + placeholders (`replace-me`, `shared-secret`, etc.) with your real tokens. + Keep `CONJURER_API_KEY` identical across all services. +2. **Create host directories** listed in `docker-compose.yml` (for example + `docker/volumes/bot/config`, `docker/volumes/musician/data`, …) and populate + them with the required JSON settings or media assets. +3. **Launch the stack** + ```bash + docker compose up --build + ``` + +Logs for each container are mounted under `docker/volumes/*` so you can diff the +runtime JSON and log files in git if needed. diff --git a/communication_subroutine.py b/communication_subroutine.py index 41c8e17..7322aa0 100644 --- a/communication_subroutine.py +++ b/communication_subroutine.py @@ -1,12 +1,14 @@ import json import logging +import os import re import threading import time from queue import Empty, Queue +from typing import Optional from urllib import request as urequest -from flask import Flask, jsonify, request +from flask import Flask, abort, jsonify, request from waitress import serve HOST_ADDRESS = "192.168.1.191" @@ -31,6 +33,13 @@ PREPPED_TRACKS = { } logger = logging.getLogger("discord") +API_KEY = os.getenv("CONJURER_API_KEY") + + +def _authorize_request() -> None: + if API_KEY and request.headers.get("X-Conjurer-Api-Key") != API_KEY: + abort(401) + class QueryControl: """ @@ -53,6 +62,7 @@ class QueryControl: @app.route("/prepped_tracks", methods=["POST"]) def log_radio_tracks(): + _authorize_request() app.logger = logging.getLogger("discord") app.logger.info(request) @@ -79,6 +89,7 @@ def log_radio_tracks(): @app.route("/conjurer", methods=["POST"]) def answer_external_command(): + _authorize_request() """ The function `answer_external_command` logs the request data, loads the data as JSON, logs the record, and then puts the record into an incoming queue before returning a success message. @@ -129,7 +140,7 @@ def waitress_run(): serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS) -def scan_queue(): +def scan_queue(stop_event: Optional[threading.Event] = None): """ The function `scan_queue` reads data from a queue, logs it, and appends it to another queue. @@ -140,12 +151,18 @@ def scan_queue(): """ logger = logging.getLogger("discord") while True: - data = OUT_COMM_Q.get() + if stop_event and stop_event.is_set(): + logger.info("scan_queue: stop requested") + break + try: + data = OUT_COMM_Q.get(timeout=1) + except Empty: + continue logger.info(data) awaiting_q.append(data) -def scan_incoming(): +def scan_incoming(stop_event: Optional[threading.Event] = None): """ The `scan_incoming` function continuously checks for incoming data, processes it, and logs when data is found. @@ -157,6 +174,9 @@ def scan_incoming(): """ logger = logging.getLogger("discord") while True: + if stop_event and stop_event.is_set(): + logger.info("scan_incoming: stop requested") + break try: answer = incoming_q.get(block=False) logger.info("DATA FOUND") @@ -204,7 +224,7 @@ def id3(url: str) -> dict: return tagdata -def comm_subroutine(): +def comm_subroutine(stop_event: Optional[threading.Event] = None): """ The `comm_subroutine` function starts multiple threads to run different tasks concurrently. @@ -217,14 +237,27 @@ def comm_subroutine(): logger.info("Started comms") threads = [] # threads.append(threading.Thread(target=flask_debug)) - threads.append(threading.Thread(target=waitress_run)) - threads.append(threading.Thread(target=scan_queue)) - threads.append(threading.Thread(target=scan_incoming)) + threads.append( + threading.Thread(target=waitress_run, daemon=True) + ) + threads.append( + threading.Thread(target=scan_queue, kwargs={"stop_event": stop_event}, daemon=True) + ) + threads.append( + threading.Thread(target=scan_incoming, kwargs={"stop_event": stop_event}, daemon=True) + ) for worker in threads: worker.start() - for worker in threads: - worker.join() + + try: + while any(thread.is_alive() for thread in threads): + if stop_event and stop_event.is_set(): + break + time.sleep(0.5) + finally: + if stop_event: + stop_event.set() if __name__ == "__main__": diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index e7c140f..51ca627 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -16,34 +16,52 @@ Functions: import asyncio import json import logging -import netrc +import os import threading -import time from json.decoder import JSONDecodeError from logging import handlers +from pathlib import Path from queue import Queue +from typing import Dict, Optional import requests import scrape_bot import search_bot -#import search_bot2 as search_bot -from flask import Flask, jsonify, request +# import search_bot2 as search_bot +from flask import Flask, jsonify, request, abort from habanero import Crossref from waitress import serve +try: + import netrc +except ImportError: # pragma: no cover + netrc = None + # Constants -NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" -HOST_ADDRESS = "192.168.1.192" -PORT_ADDRESS = 5001 -MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" -SEND_RESULTS = "/conjurer" -BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147" -MAX_CR_RESULTS = 500 -#TEST PURPOSES ONLY! -#MAX_CR_RESULTS = 5 -ENCODING = "utf-8" +def _env(name: str, default: str) -> str: + return os.getenv(name, default) + + +def _env_path(name: str, default: str) -> Path: + return Path(os.getenv(name, default)).expanduser().resolve() + + +BASE_DIR = Path( + os.getenv("CONJURER_LIBRARIAN_BASE", str(Path(__file__).resolve().parent)) +) +NETRC_FILE = _env_path("CONJURER_NETRC_FILE", str(Path.home() / ".netrc")) +HOST_ADDRESS = _env("CONJURER_LIBRARIAN_HOST", "0.0.0.0") +PORT_ADDRESS = int(_env("CONJURER_LIBRARIAN_PORT", "5001")) +MAIN_BOT_ADDRESS = _env("CONJURER_MAIN_BOT", "http://127.0.0.1:5000") +SEND_RESULTS = _env("CONJURER_LIBRARIAN_RESULTS_ENDPOINT", "/conjurer") +MAX_CR_RESULTS = int(_env("CONJURER_LIBRARIAN_MAX_RESULTS", "500")) +ENCODING = _env("CONJURER_ENCODING", "utf-8") +API_KEY = os.getenv("CONJURER_API_KEY") +LOGFILE_PATH = _env_path( + "CONJURER_LIBRARIAN_LOG", str(BASE_DIR / "librarian.log") +) app = Flask(__name__) @@ -51,6 +69,17 @@ librarian_queue = Queue() librarian_list = [] +def _service_headers() -> Dict[str, str]: + if API_KEY: + return {"X-Conjurer-Api-Key": API_KEY} + return {} + + +def _authorize_request() -> None: + if API_KEY and request.headers.get("X-Conjurer-Api-Key") != API_KEY: + abort(401) + + # trunk-ignore(pylint/R0902) class Librarian(object): """ @@ -81,11 +110,24 @@ class Librarian(object): - search_result_from_cr: A dictionary to store the search results from Crossref. - done: A flag indicating if the search is done. """ - netrc_mod = netrc.netrc(NETRC_FILE) - auth_tokens = netrc_mod.authenticators("crossref") + mailto_contact: Optional[str] = os.getenv("CONJURER_CROSSREF_MAILTO") + if netrc: + try: + netrc_mod = netrc.netrc(str(NETRC_FILE)) + auth_tokens = netrc_mod.authenticators("crossref") + if auth_tokens: + mailto_contact = auth_tokens[0] + except (FileNotFoundError, netrc.NetrcParseError): + logging.getLogger("conjurer_librarian").warning( + "Crossref credentials missing in netrc %s", NETRC_FILE + ) + if not mailto_contact: + raise RuntimeError( + "Crossref credentials not configured. Set CONJURER_CROSSREF_MAILTO or add to netrc." + ) self.cr = Crossref( - mailto=auth_tokens[0], - ua_string=f"Conjurer project. mailto:{auth_tokens[0]}" + mailto=mailto_contact, + ua_string=f"Conjurer project. mailto:{mailto_contact}" ) self.query = query self.uuid = str(uuid) @@ -132,7 +174,7 @@ class Librarian(object): self.fetched = len(cr_result["message"]["items"]) self.app.logger.info(self.total) self.app.logger.info(self.fetched) - time.sleep(0.1) + await asyncio.sleep(0.1) else: cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True) @@ -411,18 +453,20 @@ class BackgroundTaskSearch(threading.Thread): requests.post, f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}", json=result, + headers=_service_headers(), timeout=360, ) self.app.logger.info("SENT") result = await coroutine self.app.logger.info(result.status_code) self.app.logger.info("SEND CONFIRMED") - time.sleep(1) + await asyncio.sleep(1) # ==================================SERVER ROUTES========================================== @app.route("/query", methods=["POST"]) async def query_database(): + _authorize_request() """ Endpoint for querying the database. @@ -455,6 +499,7 @@ async def query_database(): @app.route("/get_partial_result", methods=["POST"]) async def get_partial(): + _authorize_request() """ Retrieves the partial result for a given UUID. @@ -478,9 +523,10 @@ async def get_partial(): # =======================================MAIN=================================================== if __name__ == "__main__": app.logger.setLevel(logging.DEBUG) + LOGFILE_PATH.parent.mkdir(parents=True, exist_ok=True) h1 = handlers.RotatingFileHandler( - filename="D:\\logs\\librarian.log", - encoding="utf-8", + filename=str(LOGFILE_PATH), + encoding=ENCODING, mode="a", maxBytes=6 * 1024 * 1024, backupCount=6, @@ -488,20 +534,24 @@ if __name__ == "__main__": app.logger.addHandler(h1) threads = [] - threads.append(threading.Thread(target=waitress_run)) + threads.append(threading.Thread(target=waitress_run, daemon=True)) # threads.append(threading.Thread(target=flask_debug)) bgtask = BackgroundTaskSearch() bgtask.app = app + bgtask.daemon = True threads.append(bgtask) - threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,))) + threads.append( + threading.Thread( + target=scrape_bot.scraper, args=(app.logger,), daemon=True + ) + ) i = 0 - for worker in threads: - try: + try: + for worker in threads: app.logger.info("App number: %s", i) i += 1 worker.start() - except RuntimeError as e: - app.logger.error("Exploded") - print(str(e)) - for worker in threads: - worker.join() + for worker in threads: + worker.join() + except KeyboardInterrupt: + app.logger.info("Shutdown requested - exiting librarian service") diff --git a/conjurer_musician/conjurer_musician.py b/conjurer_musician/conjurer_musician.py index 548e90e..5823fd1 100644 --- a/conjurer_musician/conjurer_musician.py +++ b/conjurer_musician/conjurer_musician.py @@ -11,17 +11,15 @@ import random import re import threading import time - -# from flask_autoindex import AutoIndex from datetime import datetime from logging import handlers from pathlib import Path -from platform import uname -from sys import platform +from typing import Dict, List import requests from flask import ( Flask, + abort, jsonify, redirect, render_template, @@ -29,37 +27,99 @@ from flask import ( send_from_directory, ) from waitress import serve + import media_search_functions -MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" -MUSIC_TRACKER = "/prepped_tracks" -HOST_ADDRESS = "192.168.1.15" -PORT_ADDRESS = 5000 -if platform in ("linux", "linux2"): - SEPARATOR_FILE_PATH = "/" - if "microsoft-standard" in uname().release: - LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log" - MUSIC_FOLDER = "/mnt/g/Muzyka/" - NETRC_FILE = "/home/mtuszowski/.netrc" - LOGSTORE = "/home/mtuszowski/conjurer/logs/" - ENCODING = "utf-8" - RADIOLOG_PATH = "/home/pi/Conjurer/radio_log.log" - PERSISTENCE_PATH = "/home/pi/Conjurer/persistence.log" +def _env(name: str, default: str) -> str: + return os.getenv(name, default) - else: - LOGFILE = "/home/pi/Conjurer/discord_mus_service.log" - NETRC_FILE = "/home/pi/.netrc" - LOGSTORE = "/home/pi/MediaFolder/logs/" - ENCODING = "utf-8" - MUSIC_FOLDER = "/home/pi/MediaFolder/mp3/" - PRIORITY_FOLDER = "/home/pi/MediaFoldermp3/Magiczne i chuj/" - RADIOLOG_PATH = "/home/pi/Conjurer/radio_log.log" - PERSISTENCE_PATH = "/home/pi/Conjurer/persistence.log" + +def _env_path(name: str, default: str) -> Path: + value = os.getenv(name, default) + return Path(value).expanduser().resolve() + + +API_KEY = os.getenv("CONJURER_API_KEY") +MAIN_BOT_ADDRESS = _env("CONJURER_MAIN_BOT", "http://127.0.0.1:5000") +MUSIC_TRACKER = _env("CONJURER_MUSIC_TRACKER_ENDPOINT", "/prepped_tracks") +HOST_ADDRESS = _env("CONJURER_MUSICIAN_HOST", "0.0.0.0") +PORT_ADDRESS = int(_env("CONJURER_MUSICIAN_PORT", "5000")) + +BASE_DIR = Path( + os.getenv("CONJURER_MUSICIAN_BASE", str(Path(__file__).resolve().parent)) +) +LOGFILE = _env_path( + "CONJURER_MUSICIAN_LOG", str(BASE_DIR / "discord_mus_service.log") +) +LOGSTORE = _env_path("CONJURER_LOGSTORE", str(BASE_DIR / "logs")) +MUSIC_FOLDER = _env_path( + "CONJURER_MUSIC_FOLDER", str(BASE_DIR / "music") +) +PRIORITY_FOLDER = _env_path( + "CONJURER_PRIORITY_FOLDER", str(MUSIC_FOLDER / "priority") +) +RADIOLOG_PATH = _env_path( + "CONJURER_RADIO_LOG", str(BASE_DIR / "radio_log.log") +) +PERSISTENCE_PATH = _env_path( + "CONJURER_PERSISTENCE_LOG", str(BASE_DIR / "persistence.log") +) +ALL_PLAYLIST_PATH = _env_path( + "CONJURER_ALL_PLAYLIST", str(BASE_DIR / "all_playlist.playlist") +) +HIT_PLAYLIST_PATH = _env_path( + "CONJURER_HIT_PLAYLIST", str(BASE_DIR / "hit.playlist") +) +REQUEST_PLAYLIST_PATH = _env_path( + "CONJURER_REQUEST_PLAYLIST", str(BASE_DIR / "request.playlist") +) +PRIORITY_PLAYLIST_PATH = _env_path( + "CONJURER_PRIORITY_PLAYLIST", str(BASE_DIR / "priority_queue.playlist") +) +STREAM_TEMPLATE = _env_path( + "CONJURER_STREAM_TEMPLATE", str(BASE_DIR / "stream.html") +) + +ENCODING = _env("CONJURER_ENCODING", "utf-8") +SEPARATOR_FILE_PATH = os.sep + +for playlist_path in ( + ALL_PLAYLIST_PATH, + HIT_PLAYLIST_PATH, + REQUEST_PLAYLIST_PATH, + PRIORITY_PLAYLIST_PATH, +): + playlist_path.parent.mkdir(parents=True, exist_ok=True) random.seed() -music_file_list = [] -priority_list = [] +music_file_list: List[str] = [] +priority_list: List[str] = [] + + +def _build_headers() -> Dict[str, str]: + headers: Dict[str, str] = {} + if API_KEY: + headers["X-Conjurer-Api-Key"] = API_KEY + return headers + + +def _authorize_request() -> None: + if API_KEY and request.headers.get("X-Conjurer-Api-Key") != API_KEY: + abort(401) + + +def _post_to_bot(payload: List[str]) -> None: + response = requests.post( + f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}", + json=payload, + headers=_build_headers(), + timeout=60, + ) + logger = logging.getLogger("conjurer_musician") + logger.info("SENT") + logger.info(response.status_code) + logger.info("SEND CONFIRMED") def rescan(): @@ -70,28 +130,29 @@ def rescan(): logger = logging.getLogger("conjurer_musician") logger.info("Rescan triggered") - for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"): + music_file_list.clear() + priority_list.clear() + + for mp3_item in MUSIC_FOLDER.glob("**/*.mp3"): temp_music_file = mp3_item.as_posix() - if platform == "win32": + if os.name == "nt": temp_music_file = temp_music_file.replace("/", "\\") music_file_list.append(temp_music_file) - for mp3_item in Path.glob(Path(PRIORITY_FOLDER), "**/*.mp3"): + for mp3_item in PRIORITY_FOLDER.glob("**/*.mp3"): temp_music_file = mp3_item.as_posix() - if platform == "win32": + if os.name == "nt": temp_music_file = temp_music_file.replace("/", "\\") priority_list.append(temp_music_file) - with open( - "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8" - ) as w_file: + with ALL_PLAYLIST_PATH.open("w", encoding=ENCODING) as w_file: try: for item in music_file_list: w_file.write(item) w_file.write("\n") except json.JSONDecodeError: pass - with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file: + with HIT_PLAYLIST_PATH.open("w", encoding=ENCODING) as w_file: try: for item in priority_list: w_file.write(item) @@ -116,72 +177,50 @@ def thread_rescan(): def scan_tracks(): # Set the filename and open the file logger = logging.getLogger("conjurer_musician") + with open(RADIOLOG_PATH, "r", encoding=ENCODING) as log_file: + log_file.seek(os.stat(RADIOLOG_PATH).st_size) + prev_size = os.stat(PERSISTENCE_PATH).st_size - file = open(RADIOLOG_PATH, "r") - # Find the size of the file and move to the end - st_results = os.stat(RADIOLOG_PATH) - st_size = st_results[6] - file.seek(st_size) - st_results1 = os.stat(PERSISTENCE_PATH) - prev_st_size1 = st_results[6] + while True: + current_size = os.stat(PERSISTENCE_PATH).st_size + if prev_size != current_size: + while prev_size != current_size: + prev_size = current_size + time.sleep(0.1) + current_size = os.stat(PERSISTENCE_PATH).st_size + with open(PERSISTENCE_PATH, "r", encoding=ENCODING) as persistence: + lines = persistence.readlines() + if len(lines) >= 3: + _post_to_bot(["next", lines[2]]) - while 1: + position = log_file.tell() + line = log_file.readline() + if not line: + time.sleep(1) + log_file.seek(position) + continue - st_results1 = os.stat(PERSISTENCE_PATH) - st_size1 = st_results1[6] - if prev_st_size1 != st_size1: - while prev_st_size1 != st_size1: - prev_st_size1 = st_size1 - st_results1 = os.stat(PERSISTENCE_PATH) - st_size1 = st_results1[6] + if not re.match(r".*Prepared.*", line): time.sleep(0.1) - file1 = open(PERSISTENCE_PATH, "r") - lines = file1.readlines() - result = ["next", lines[2]] - file1.close() - returned = requests.post( - f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}", json=result, timeout=360 - ) - logger.info("SENT") - logger.info(returned.status_code) - logger.info("SEND CONFIRMED") + continue - where = file.tell() - line = file.readline() - if not line: - time.sleep(1) - file.seek(where) - else: - if re.match(".*Prepared.*", line): - result = None - if re.match(".*jingles.*", line): - logger.info("jingles") - logger.info(line) # already has newline - result = ["jingles", line] - elif re.match(".*priority.*", line): - logger.info("priority") - logger.info(line) # already has newline - result = ["priority", line] - elif re.match(".*hit.*", line): - logger.info("hit") - logger.info(line) # already has newline - result = ["hit", line] - elif re.match(".*all_playlist.*", line): - logger.info("all") - logger.info(line) # already has newline - result = ["all", line] - elif re.match(".*request.*", line): - logger.info("requests") - logger.info(line) # already has newline - result = ["requests", line] - if result: - returned = requests.post( - f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}", json=result, timeout=360 - ) - logger.info("SENT") - logger.info(returned.status_code) - logger.info("SEND CONFIRMED") - time.sleep(0.1) + result = None + if re.match(r".*jingles.*", line): + result = ["jingles", line] + elif re.match(r".*priority.*", line): + result = ["priority", line] + elif re.match(r".*hit.*", line): + result = ["hit", line] + elif re.match(r".*all_playlist.*", line): + result = ["all", line] + elif re.match(r".*request.*", line): + result = ["requests", line] + + if result: + logger.info("Forwarding radio log entry: %s", result[0]) + _post_to_bot(result) + + time.sleep(0.1) app = Flask(__name__) @@ -290,12 +329,8 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True): if search_weight[itr][0] == item_to_search: return_list.append(search_weight[itr]) if not return_to_bot: - with open( - "/home/pi/Conjurer/priority_queue.playlist", - "r+", - encoding="utf-8", - ) as s_file: - s_file.write(search_weight[itr][1]) + with PRIORITY_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file: + s_file.write(search_weight[itr][1] + "\n") break itr += 1 else: @@ -336,6 +371,7 @@ def remove_characters(string, character): @app.route('/get_share_list', methods=['POST']) def get_share_list(): + _authorize_request() data = request.get_json() entries = data.get('entries') keywords = data.get('keywords') @@ -352,6 +388,7 @@ def get_share_list(): @app.route('/get_share_links', methods=['POST']) def get_share_links(): + _authorize_request() data = request.get_json() file_paths = data.get('file_paths') # Validate file_paths list @@ -374,7 +411,7 @@ def stream_music(): """ # return send_from_directory("/tmp/hls", "stream.m3u8") - return render_template("/home/pi/Conjurer/stream.html") + return render_template(str(STREAM_TEMPLATE)) @app.route("/") @@ -406,15 +443,14 @@ def stream_music_mp3(): @app.route("/clear_pr_pls", methods=["GET"]) def clear_pr_pls(): + _authorize_request() """ The function `clear_pr_pls` clears the contents of the priority queue playlist file. :return: A JSON response indicating the success of the operation. """ app.logger.info("CLEARING PLAYLIST") - with open( - "/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8" - ) as cleared_pl: + with PRIORITY_PLAYLIST_PATH.open("w", encoding=ENCODING) as cleared_pl: cleared_pl.write("") return_data = jsonify(isError=False, message="Success", statusCode=200, data=[]) @@ -442,6 +478,7 @@ def update_music_list(): received and added to the `music_file_list`. The status code returned is 200, indicating a successful response. """ + _authorize_request() record = json.loads(request.data) app.logger.info(record["item"]) music_file_list.append(record["item"]) @@ -463,6 +500,7 @@ def look_for_playlist(): data that was received and added to the `music_file_list`. The status code returned is 200, indicating a successful response. """ + _authorize_request() record = json.loads(request.data) app.logger.info(record) app.logger.info(record["lista_slow"]) @@ -481,13 +519,14 @@ def look_for_playlist(): @app.route("/request_radio_file", methods=["POST"]) def add_request(): + _authorize_request() record = json.loads(request.data) app.logger.info(record) app.logger.info(record["lista_slow"]) app.logger.info(record["UUID"]) return_data = wyszukaj(record["lista_slow"], 0, app.logger, False) - with open("/home/pi/Conjurer/request.playlist", "a", encoding="utf-8") as s_file: + with REQUEST_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file: for item in return_data: s_file.write(item[1] + "\n") return_data = ( @@ -512,6 +551,7 @@ def create_priority_playlist(): data that was received and added to the `music_file_list`. The status code returned is 200,indicating a successful response. """ + _authorize_request() record = json.loads(request.data) app.logger.info(record) app.logger.info(record["lista_slow"]) @@ -521,7 +561,7 @@ def create_priority_playlist(): record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False ) random.shuffle(return_data) - with open("/home/pi/Conjurer/request.playlist", "a", encoding="utf-8") as s_file: + with REQUEST_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file: for item in return_data: s_file.write(item[1] + "\n") return_data = ( @@ -546,6 +586,7 @@ def add_to_priority(): data that was received and added to the `music_file_list`. The status code returned is 200,indicating a successful response. """ + _authorize_request() record = json.loads(request.data) app.logger.info(record) app.logger.info(record["lista_slow"]) @@ -554,9 +595,7 @@ def add_to_priority(): return_data = wyszukaj( record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False ) - with open( - "/home/pi/Conjurer/priority_queue.playlist", "a", encoding="utf-8" - ) as s_file: + with PRIORITY_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file: for item in return_data: s_file.write(item[1] + "\n") return_data = ( @@ -615,15 +654,19 @@ if __name__ == "__main__": logger.info("Started") threads = [] # threads.append(threading.Thread(target=flask_debug)) - threads.append(threading.Thread(target=waitress_run)) - threads.append(threading.Thread(target=thread_rescan)) + threads.append(threading.Thread(target=waitress_run, daemon=True)) + threads.append(threading.Thread(target=thread_rescan, daemon=True)) for worker in threads: worker.start() time.sleep(60) - threads.append(threading.Thread(target=scan_tracks)) - threads[2].start() + track_thread = threading.Thread(target=scan_tracks, daemon=True) + track_thread.start() - for worker in threads: - worker.join() + try: + for worker in threads: + worker.join() + track_thread.join() + except KeyboardInterrupt: + logger.info("Shutdown requested - exiting musician service") diff --git a/constants.py b/constants.py index 2185187..415716e 100644 --- a/constants.py +++ b/constants.py @@ -1,13 +1,36 @@ -import json -import netrc -from datetime import datetime -from platform import uname -from sys import platform -from typing import List, Optional, TypedDict +"""Centralised configuration and runtime constants for Conjurer services. -import openai -import spotipy -from spotipy.oauth2 import SpotifyClientCredentials +This module used to perform heavy filesystem and credential reads at import +time which made the project brittle on hosts that did not mirror the original +paths. The current implementation defers that work, reads configuration from +environment variables (with sensible fallbacks), and protects optional +dependencies so the main bot can start even when a secondary service is +offline. +""" + +import json +import logging +import os +from datetime import datetime +from pathlib import Path +from typing import Dict, List, Optional, Tuple, TypedDict + +try: + import netrc +except ImportError: # pragma: no cover - standard on CPython + netrc = None + +try: + import openai +except ImportError: # pragma: no cover - optional at runtime + openai = None + +try: + import spotipy + from spotipy.oauth2 import SpotifyClientCredentials +except ImportError: # pragma: no cover - optional component + spotipy = None + SpotifyClientCredentials = None Music_Config = TypedDict( "Music_Config", @@ -24,21 +47,43 @@ MASTER_TIMEOUT = datetime.now() INITIAL_TIME_WAIT = 500 MUZYKA: Music_Config = {"ctx": None, "queue": [], "requester": []} -LOGFILE = "" -NETRC_FILE = "" -MUSIC_FOLDER = "" -MEMORY_FIVE_SIARA = "" -MEMORY_FIVE_MUZYKA = "" -SETTINGS_FILE = "" -ENCODING = "" -GRAPHICS_PATH = "" +logger = logging.getLogger("discord") + + +def _env_path(var_name: str, fallback: Path) -> Path: + value = os.getenv(var_name) + if value: + return Path(value).expanduser().resolve() + return fallback + + +def _env(var_name: str, fallback: str) -> str: + return os.getenv(var_name, fallback) + + +BASE_DIR = Path(os.getenv("CONJURER_BASE_DIR", Path(__file__).resolve().parent)) + +LOGFILE = _env_path("CONJURER_LOG_FILE", BASE_DIR / "discord.log") +NETRC_FILE = _env_path("CONJURER_NETRC_FILE", Path.home() / ".netrc") +SETTINGS_FILE = _env_path("CONJURER_SETTINGS_FILE", BASE_DIR / "settings.json") +MEMORY_FIVE_SIARA = _env_path("CONJURER_MEMORY_FILE", BASE_DIR / "pamiec.json") +MEMORY_FIVE_MUZYKA = _env_path( + "CONJURER_MUSIC_MEMORY_FILE", BASE_DIR / "pamiec_muzyki.json" +) +GRAPHICS_PATH = _env_path( + "CONJURER_GRAPHICS_PATH", BASE_DIR / "Conjurer_graphics" +) +MUSIC_FOLDER = _env_path("CONJURER_MUSIC_FOLDER", BASE_DIR / "music") +ENCODING = _env("CONJURER_ENCODING", "utf-8") + +API_SHARED_KEY = os.getenv("CONJURER_API_KEY", "") MUZYKA_MOJEGO_LUDU_HISTORIA = 1500 MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE = 15 MUZYKA_MOJEGO_LUDU_PLAJLISTA = 30 -FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000" -RADIO_HARBOR_ADDRESS = "http://192.168.1.15:54321" -SKIP_TRACK = "/skip" +FILE_SERVICE_ADDRESS = _env("CONJURER_FILE_SERVICE", "http://127.0.0.1:5000") +RADIO_HARBOR_ADDRESS = _env("CONJURER_RADIO_HARBOR", "http://127.0.0.1:54321") +SKIP_TRACK = _env("CONJURER_SKIP_ENDPOINT", "/skip") GET_MP3 = "/mp3" SEND_MP3 = "/update_mp3" @@ -48,98 +93,123 @@ CREATE_PRIO_PLAYLIST = "/create_priority_playlist" REQUEST_MUSIC = "/request_radio_file" CLEAR_PRIO = "/clear_pr_pls" -LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001" -SEND_QUERY = "/query" +LIBRARIAN_SERVICE_ADDRESS = _env( + "CONJURER_LIBRARIAN_SERVICE", "http://127.0.0.1:5001" +) +SEND_QUERY = _env("CONJURER_LIBRARIAN_QUERY_ENDPOINT", "/query") TIME_BETWEEN_CALLS = 100000 LAST_SPONTANEOUS_CALL = datetime.now() -HOST_ADDRESS = "192.168.1.191" -PORT_ADDRESS = 5000 +HOST_ADDRESS = _env("CONJURER_DISCORD_HOST", "0.0.0.0") +PORT_ADDRESS = int(_env("CONJURER_DISCORD_PORT", "5000")) # *=========================================== Platform Specific Predefines -if platform in ("linux", "linux2"): - SEPARATOR_FILE_PATH = "/" - if "microsoft-standard" in uname().release: - LOGFILE = "/home/mtuszowski/conjurer/discord.log" - MEMORY_FIVE_SIARA = "/home/mtuszowski/conjurer/pamiec.json" - SYSTEM_GPT_SETTINGS = "/home/mtuszowski/conjurer/system_gpt_settings.json" - MEMORY_FIVE_MUZYKA = "/home/mtuszowski/conjurer/pamiec_muzyki.json" - MUSIC_FOLDER = "/mnt/g/Muzyka/" - SETTINGS_FILE = "/home/mtuszowski/conjurer/settings.json" - NETRC_FILE = "/home/mtuszowski/.netrc" - LOGSTORE = "/home/mtuszowski/conjurer/logs/" - ACCIDENT_LOG = "/home/mtuszowski/conjurer/accident_log.json" - ENCODING = "utf-8" - GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/" - DIR_PATH_SADOX = "/mnt/c/Users/mtusz/OneDrive/Dokumenty/Fansadox" +SEPARATOR_FILE_PATH = _env("CONJURER_PATH_SEPARATOR", os.sep) - else: - LOGFILE = "/home/pi/Conjurer/discord.log" - MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json" - SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json" - MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json" - MUSIC_FOLDER = "/home/pi/MediaShare/mp3/" - SETTINGS_FILE = "/home/pi/Conjurer/settings.json" - NETRC_FILE = "/home/pi/.netrc" - LOGSTORE = "/home/pi/MediaShara/logs/" - ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json" - ENCODING = "utf-8" - GRAPHICS_PATH = "/home/pi/MediaShare/Conjurer_graphics/" - DIR_PATH_SADOX = "/home/pi/MediaShare/Fansadox/" - - -elif platform == "win32": - LOGFILE = "discord.log" - MEMORY_FIVE_SIARA = "pamiec.json" - SYSTEM_GPT_SETTINGS = "system_gpt_settings.json" - MEMORY_FIVE_MUZYKA = "pamiec_muzyki.json" - MUSIC_FOLDER = "G:\\Muzyka\\" - SETTINGS_FILE = "settings.json" - NETRC_FILE = "C:\\Users\\mtusz\\.netrc" - LOGSTORE = "C:\\Users\\mtusz\\OneDrive\\Pulpit\\Conjurer\\logs\\" - ACCIDENT_LOG = "accident_log.json" - ENCODING = "utf-8" - DIR_PATH_SADOX = "C:\\Users\\mtusz\\OneDrive\\Dokumenty\\Fansadox\\" - SEPARATOR_FILE_PATH = "\\" -with open(SETTINGS_FILE, "r", encoding=ENCODING) as f_settings_file: - DATA = json.load(f_settings_file) -REMOTE_HOST_NAME = "openai" -netrc_mod = netrc.netrc(NETRC_FILE) -authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME) -openai.api_key = authTokens[2] -OPENAICLIENT = openai.AsyncOpenAI(api_key=openai.api_key) -REMOTE_HOST_NAME = "discord" -authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME) -TOKEN = authTokens[2] - -REMOTE_HOST_NAME = "spotipy" -authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME) -SPOTIFY_CTRL = spotipy.Spotify( - client_credentials_manager=SpotifyClientCredentials( - client_id=authTokens[0], - client_secret=authTokens[2], - ) +DIR_PATH_SADOX = _env_path( + "CONJURER_SADOX_DIR", BASE_DIR / "Fansadox" ) -REMOTE_HOST_NAME = "youtube" -authTokens = netrc_mod.authenticators(REMOTE_HOST_NAME) -YOUTUBE_AUTH = [authTokens[0],authTokens[2]] -WORD_REACTIONS = DATA["word_reactions"] -CYCLIC_WORDS = DATA["cyclic_words"] +SYSTEM_GPT_SETTINGS = _env_path( + "CONJURER_SYSTEM_GPT_SETTINGS", BASE_DIR / "system_gpt_settings.json" +) + +LOGSTORE = _env_path("CONJURER_LOGSTORE", BASE_DIR / "logs") +ACCIDENT_LOG = _env_path( + "CONJURER_ACCIDENT_LOG", BASE_DIR / "accident_log.json" +) + + +def _load_json(path: Path, fallback) -> object: + try: + with path.open("r", encoding=ENCODING) as handle: + return json.load(handle) + except FileNotFoundError: + logger.warning("Missing JSON file at %s – using fallback", path) + return fallback + except json.JSONDecodeError: + logger.warning("Corrupt JSON at %s – resetting to fallback", path) + return fallback + + +DATA = _load_json(SETTINGS_FILE, {}) +WORD_REACTIONS = DATA.get("word_reactions", {}) +CYCLIC_WORDS = DATA.get("cyclic_words", {}) for key in WORD_REACTIONS: - WORD_REACTIONS[key][2] = datetime.now() -with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as temp_memory_file: - # First we load existing data into a dict. - MESSAGE_TABLE = json.load(temp_memory_file) + if isinstance(WORD_REACTIONS[key], list) and len(WORD_REACTIONS[key]) >= 3: + WORD_REACTIONS[key][2] = datetime.now() -with open(SYSTEM_GPT_SETTINGS, "r+", encoding=ENCODING) as temp_settings_file: - # First we load existing data into a dict. - GPT_SETTINGS = json.load(temp_settings_file) -with open(MEMORY_FIVE_MUZYKA, "r+", encoding=ENCODING) as temp_music_memory_file: - # First we load existing data into a dict. - MESSAGE_TABLE_MUZYKA = json.load(temp_music_memory_file) -SPECJALNE_ZIEMNIACZKI = GPT_SETTINGS[1] -ASSISTANTS = {} +MESSAGE_TABLE = _load_json(MEMORY_FIVE_SIARA, {}) +GPT_SETTINGS = _load_json(SYSTEM_GPT_SETTINGS, {}) +MESSAGE_TABLE_MUZYKA = _load_json(MEMORY_FIVE_MUZYKA, {}) + +SPECJALNE_ZIEMNIACZKI = GPT_SETTINGS[1] if isinstance(GPT_SETTINGS, list) else {} +ASSISTANTS: Dict[str, Tuple[str, str, int, object]] = {} + + +def _load_netrc_credentials(host: str) -> Optional[Tuple[str, str, str]]: + if netrc is None: + return None + try: + parsed = netrc.netrc(str(NETRC_FILE)) + except FileNotFoundError: + logger.warning("netrc file %s not found", NETRC_FILE) + return None + except netrc.NetrcParseError: + logger.warning("netrc file %s is invalid", NETRC_FILE) + return None + return parsed.authenticators(host) + + +def _resolve_token(host: str, env_var: str) -> Optional[str]: + env_value = os.getenv(env_var) + if env_value: + return env_value + creds = _load_netrc_credentials(host) + if creds: + return creds[2] + logger.warning("Token for %s not configured", host) + return None + + +OPENAI_API_KEY = _resolve_token("openai", "OPENAI_API_KEY") +if openai and OPENAI_API_KEY: + openai.api_key = OPENAI_API_KEY + OPENAICLIENT = openai.AsyncOpenAI(api_key=OPENAI_API_KEY) +else: + OPENAICLIENT = None + +TOKEN = _resolve_token("discord", "DISCORD_TOKEN") + +if spotipy: + spotify_creds = _load_netrc_credentials("spotipy") + if spotify_creds and SpotifyClientCredentials: + SPOTIFY_CTRL = spotipy.Spotify( + client_credentials_manager=SpotifyClientCredentials( + client_id=spotify_creds[0], + client_secret=spotify_creds[2], + ) + ) + else: + SPOTIFY_CTRL = None +else: + SPOTIFY_CTRL = None + +youtube_creds = _load_netrc_credentials("youtube") +if youtube_creds: + YOUTUBE_AUTH = [youtube_creds[0], youtube_creds[2]] +else: + YOUTUBE_AUTH = [ + os.getenv("YOUTUBE_USERNAME", ""), + os.getenv("YOUTUBE_PASSWORD", ""), + ] + + +def service_headers() -> Dict[str, str]: + """Shared header dict for internal HTTP calls.""" + if API_SHARED_KEY: + return {"X-Conjurer-Api-Key": API_SHARED_KEY} + return {} LATEX_TEX_ENGINE = "tectonic" diff --git a/docker-compose.yml b/docker-compose.yml new file mode 100644 index 0000000..35d0c84 --- /dev/null +++ b/docker-compose.yml @@ -0,0 +1,45 @@ +version: "3.9" + +services: + bot: + build: + context: . + dockerfile: docker/Dockerfile.bot + container_name: conjurer-bot + env_file: + - docker/env/bot.env + volumes: + - ./docker/volumes/bot/config:/data/config + - ./docker/volumes/bot/logs:/data/logs + depends_on: + - musician + - librarian + restart: unless-stopped + ports: + - "5000:5000" + + musician: + build: + context: . + dockerfile: docker/Dockerfile.musician + container_name: conjurer-musician + env_file: + - docker/env/musician.env + volumes: + - ./docker/volumes/musician/data:/data + restart: unless-stopped + ports: + - "5001:5000" + + librarian: + build: + context: . + dockerfile: docker/Dockerfile.librarian + container_name: conjurer-librarian + env_file: + - docker/env/librarian.env + volumes: + - ./docker/volumes/librarian/data:/data + restart: unless-stopped + ports: + - "5002:5001" diff --git a/docker/Dockerfile.bot b/docker/Dockerfile.bot new file mode 100644 index 0000000..101d845 --- /dev/null +++ b/docker/Dockerfile.bot @@ -0,0 +1,23 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ffmpeg \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements_bot.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt + +COPY . /app + +RUN useradd --create-home appuser \ + && chown -R appuser:appuser /app + +USER appuser + +CMD ["python", "thin_client.py"] diff --git a/docker/Dockerfile.librarian b/docker/Dockerfile.librarian new file mode 100644 index 0000000..55b195a --- /dev/null +++ b/docker/Dockerfile.librarian @@ -0,0 +1,24 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + build-essential \ + poppler-utils \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY conjurer_librarian/requirements_librarian.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt + +COPY . /app + +RUN useradd --create-home appuser \ + && chown -R appuser:appuser /app + +USER appuser + +CMD ["python", "-m", "conjurer_librarian.conjurer_librarian"] diff --git a/docker/Dockerfile.musician b/docker/Dockerfile.musician new file mode 100644 index 0000000..45fe2d4 --- /dev/null +++ b/docker/Dockerfile.musician @@ -0,0 +1,23 @@ +FROM python:3.11-slim + +ENV PYTHONUNBUFFERED=1 \ + PIP_NO_CACHE_DIR=1 + +RUN apt-get update \ + && apt-get install -y --no-install-recommends \ + ffmpeg \ + && rm -rf /var/lib/apt/lists/* + +WORKDIR /app + +COPY requirements_bot.txt /tmp/requirements.txt +RUN pip install -r /tmp/requirements.txt + +COPY . /app + +RUN useradd --create-home appuser \ + && chown -R appuser:appuser /app + +USER appuser + +CMD ["python", "-m", "conjurer_musician.conjurer_musician"] diff --git a/docker/env/bot.env.example b/docker/env/bot.env.example new file mode 100644 index 0000000..070ac87 --- /dev/null +++ b/docker/env/bot.env.example @@ -0,0 +1,24 @@ +# Discord bot service configuration + +DISCORD_TOKEN=HACKME! +OPENAI_API_KEY=HACKME! +CONJURER_API_KEY=HACKME! + +# Internal service endpoints +CONJURER_FILE_SERVICE=http://conjurer-musician:5000 +CONJURER_LIBRARIAN_SERVICE=http://conjurer-librarian:5001 + +# Runtime paths mounted via docker-compose volumes +CONJURER_BASE_DIR=/data/config +CONJURER_SETTINGS_FILE=/data/config/settings.json +CONJURER_MEMORY_FILE=/data/config/pamiec.json +CONJURER_MUSIC_MEMORY_FILE=/data/config/pamiec_muzyki.json +CONJURER_SYSTEM_GPT_SETTINGS=/data/config/system_gpt_settings.json +CONJURER_GRAPHICS_PATH=/data/assets/graphics +CONJURER_LOG_FILE=/data/logs/discord.log +CONJURER_LOGSTORE=/data/logs + +# Optional external integrations +CONJURER_NETRC_FILE=/data/secrets/.netrc +YOUTUBE_USERNAME=HACKME! +YOUTUBE_PASSWORD=HACKME! diff --git a/docker/env/librarian.env.example b/docker/env/librarian.env.example new file mode 100644 index 0000000..d7ce48d --- /dev/null +++ b/docker/env/librarian.env.example @@ -0,0 +1,14 @@ +# Librarian service configuration + +CONJURER_API_KEY=HACKME! + +CONJURER_MAIN_BOT=http://conjurer-bot:5000 + +CONJURER_LIBRARIAN_HOST=0.0.0.0 +CONJURER_LIBRARIAN_PORT=5001 +CONJURER_LIBRARIAN_MAX_RESULTS=500 + +CONJURER_CROSSREF_MAILTO=HACKME! +CONJURER_LIBRARIAN_LOG=/data/logs/librarian.log + +CONJURER_NETRC_FILE=/data/secrets/.netrc diff --git a/docker/env/musician.env.example b/docker/env/musician.env.example new file mode 100644 index 0000000..4c4a028 --- /dev/null +++ b/docker/env/musician.env.example @@ -0,0 +1,20 @@ +# Musician service configuration + +CONJURER_API_KEY=HACKME! + +CONJURER_MAIN_BOT=http://conjurer-bot:5000 +CONJURER_MUSIC_TRACKER_ENDPOINT=/prepped_tracks + +CONJURER_MUSICIAN_HOST=0.0.0.0 +CONJURER_MUSICIAN_PORT=5000 + +CONJURER_MUSIC_FOLDER=/data/music +CONJURER_PRIORITY_FOLDER=/data/priority +CONJURER_ALL_PLAYLIST=/data/playlists/all_playlist.playlist +CONJURER_HIT_PLAYLIST=/data/playlists/hit.playlist +CONJURER_REQUEST_PLAYLIST=/data/playlists/request.playlist +CONJURER_PRIORITY_PLAYLIST=/data/playlists/priority_queue.playlist +CONJURER_STREAM_TEMPLATE=/data/templates/stream.html + +CONJURER_RADIO_LOG=/data/logs/radio.log +CONJURER_PERSISTENCE_LOG=/data/logs/persistence.log diff --git a/docs/deployment/DEPLOYMENT_GUIDE.md b/docs/deployment/DEPLOYMENT_GUIDE.md new file mode 100644 index 0000000..fb809f2 --- /dev/null +++ b/docs/deployment/DEPLOYMENT_GUIDE.md @@ -0,0 +1,163 @@ +# Conjurer Deployment Guide + +This document walks through deploying the Conjurer stack (Discord bot, music +service, librarian service) on two Raspberry Pi 4s and one Windows host, first +with plain Docker Compose, then with a future Kubernetes setup. + +## 1. Current Hardware Layout + +- **Windows PC**: Stores persistent data (JSON memories, configs, music + catalogue). Shares folders over the network (SMB) for the Pis. +- **Raspberry Pi A** (“radio”): Runs the Liquidsoap/liq radio pipeline and the + musician service (Flask + file watcher). +- **Raspberry Pi B** (“bot”): Runs the Discord bot, communication bridge, and + librarian Flask service. + +You can rebalance as follows: + +| Service | Recommended Host | Notes | +|---------------------|------------------|-------| +| Discord bot + comms | Raspberry Pi B | Needs outbound internet, moderate CPU | +| Librarian service | Raspberry Pi B | CPU-heavy during Crossref queries; keep close to bot | +| Musician service | Raspberry Pi A | Has direct disk access to music, same box as Liquidsoap | +| Data storage | Windows | Expose via SMB; mount inside containers | + +## 2. Prepare Shared Storage on Windows + +1. Create directories, e.g. `C:\Conjurer\config`, `C:\Conjurer\logs`, + `C:\Conjurer\music`, `C:\Conjurer\playlists`, `C:\Conjurer\secrets`. +2. Copy your existing JSON settings (`settings.json`, `pamiec.json`, + `pamiec_muzyki.json`, `system_gpt_settings.json`, etc.) into `config`. +3. Create blank placeholder files if they do not exist yet. +4. Share the root folder (`C:\Conjurer`) over SMB with read/write access for the + Pi user (create credentials if necessary). + +## 3. Configure Environment Files + +1. On your workstation, copy the example env files: + ```bash + cp docker/env/bot.env.example docker/env/bot.env + cp docker/env/musician.env.example docker/env/musician.env + cp docker/env/librarian.env.example docker/env/librarian.env + ``` +2. Edit each `docker/env/*.env` to replace `HACKME!` with real values: + - `DISCORD_TOKEN`, `OPENAI_API_KEY`, `CONJURER_API_KEY` (use the same value for + all services). + - For musician/librarian, adjust mounts to the SMB paths you will mount on the + Pis, e.g. `/mnt/conjurer/music`. + - Set `CONJURER_CROSSREF_MAILTO` to a real email as required by Crossref. +3. If you rely on `.netrc`, copy it to `C:\Conjurer\secrets\.netrc` and set + `CONJURER_NETRC_FILE` accordingly. + +## 4. Install Docker on Raspberry Pis and Windows + +### Raspberry Pi +```bash +curl -fsSL https://get.docker.com | sh +sudo usermod -aG docker $USER +sudo reboot + +# Install docker compose plugin +sudo apt-get install docker-compose-plugin +``` + +### Windows +- Install **Docker Desktop**. +- Enable WSL2 backend and expose the shared Windows folders to the containers + (Docker Desktop settings → Resources → File Sharing). + +## 5. Deploy Musician Service (Pi A) + +1. SSH into Raspberry Pi A. +2. Mount the Windows SMB share: + ```bash + sudo mkdir -p /mnt/conjurer + sudo apt-get install cifs-utils + sudo mount -t cifs //WINDOWS_HOST/Conjurer /mnt/conjurer -o user=YOURUSER + ``` + Add an entry to `/etc/fstab` for persistence. +3. Copy the repo to the Pi or `git clone` it. +4. On Pi A, create override compose file (optional) pointing volumes to + `/mnt/conjurer`. +5. Start only the musician service: + ```bash + docker compose --profile musician up --build -d musician + ``` + Alternatively, duplicate `docker-compose.yml`, strip other services, and run + `docker compose up -d`. + +## 6. Deploy Bot + Librarian (Pi B) + +1. Repeat SMB mount on Pi B (same mount path). +2. Copy repo / pull latest changes. +3. Create `.env` files with tokens (or copy from control machine). +4. Start bot and librarian: + ```bash + docker compose up -d bot librarian + ``` + +## 7. Optional: Run Supporting Liquidsoap Radio + +- Keep Liquidsoap on Pi A as-is, using the same music directories. Ensure the + musician container has read access to those directories (bind mount). + +## 8. Verifying + +1. `docker ps` on each Pi to confirm containers running. +2. Inspect logs under the mounted logs directory (`/mnt/conjurer/logs`). +3. Join Discord server; issue commands to confirm functionality. +4. Hit health endpoints manually (e.g. `curl http://PIB:5000/conjurer`). + +## Rebalancing Suggestions + +- If librarian CPU spikes become an issue, move it to Pi A or another host. +- If you add a dedicated NAS, mount the network share read-only for the musician + container and read/write for other services. + +## 9. Future Kubernetes Deployment (Outline) + +### Hardware Considerations + +- Minimum three nodes for HA: use the existing two Pis plus one additional Pi 4 + (8 GB preferred). Use Windows PC as storage provider via NFS/SMB CSI driver or + as a data gateway. +- Consider Pi clusters with USB SSDs for better I/O. + +### Cluster Setup Steps + +1. Install a lightweight Kubernetes distribution (e.g., k3s) on each Pi: + ```bash + curl -sfL https://get.k3s.io | sh - + # On additional nodes + curl -sfL https://get.k3s.io | K3S_URL=https://MASTER:6443 K3S_TOKEN=HACKME sh - + ``` +2. Install MetalLB for load balancer support on LAN. +3. Configure persistent volumes using: + - `nfs-subdir-external-provisioner` pointing to Windows share (ensure Windows + host supports NFS or run an NFS gateway on another machine). + - Alternatively, attach individual USB drives to each Pi and use + `local-path-provisioner` for node-local storage. +4. Create Kubernetes `Secret` objects for tokens (`DISCORD_TOKEN`, etc.). +5. Define `Deployment` manifests for each service (bot, musician, librarian) and + associated `Services`. +6. Expose Discord bot ports via `NodePort` or Ingress. +7. Use `StatefulSet` if you need stable identity for the musician service (due to + local storage). + +### Optimisation Tips + +- Keep CPU-heavy librarian pods optionally on a beefier node; use + `nodeSelector`/`affinity` to pin workloads. +- Consider splitting the persistent storage: music on Pi A (USB disk), logs and + configs on Pi B, backups on Windows. +- For improved reliability, add at least one extra Pi for quorum and to host the + communication bridge if the bot node fails. + +## Summary Checklist + +1. Prepare Windows shares & tokens. +2. Configure `docker/env/*.env` using `HACKME!` templates as reference. +3. Install Docker on Pis, mount network shares. +4. Launch musician on Pi A, bot + librarian on Pi B. +5. Verify Discord functionality and API endpoints. +6. Plan Kubernetes migration when ready (k3s + MetalLB + storage provisioner). diff --git a/librarian_commands.py b/librarian_commands.py index aa12f6b..95d3010 100644 --- a/librarian_commands.py +++ b/librarian_commands.py @@ -15,7 +15,14 @@ from discord.ext import commands, tasks from ai_functions import handle_response from communication_subroutine import IN_COMM_Q, OUT_COMM_Q, QueryControl -from constants import DIR_PATH_SADOX, LIBRARIAN_SERVICE_ADDRESS, SEND_QUERY +from constants import ( + DIR_PATH_SADOX, + LIBRARIAN_SERVICE_ADDRESS, + SEND_QUERY, + service_headers, +) + +SERVICE_HEADERS = service_headers() class DataModule(commands.Cog): @@ -165,6 +172,7 @@ class DataModule(commands.Cog): requests.post, f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}", json=json_query, + headers=SERVICE_HEADERS, timeout=360, ) await ctx.send( @@ -260,6 +268,7 @@ class DataModule(commands.Cog): requests.post, f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}", json=json_query, + headers=SERVICE_HEADERS, timeout=360, ) await ctx.send( diff --git a/music_functions.py b/music_functions.py index a6e2881..9523ae6 100644 --- a/music_functions.py +++ b/music_functions.py @@ -17,11 +17,16 @@ from constants import ( SEND_MP3, SPOTIFY_CTRL, YOUTUBE_AUTH, + service_headers, ) from spotify_dl import spotify from spotify_dl import youtube as youtube_download +SERVICE_HEADERS = service_headers() +MUSIC_ROOT = Path(MUSIC_FOLDER) + + class MusicFileList(object): """ @@ -43,11 +48,15 @@ class MusicFileList(object): """ try: self.logger.info("Attempt to connect to file service") - response = requests.get(f"{FILE_SERVICE_ADDRESS}{GET_MP3}", timeout=360) + response = requests.get( + f"{FILE_SERVICE_ADDRESS}{GET_MP3}", + headers=SERVICE_HEADERS, + timeout=60, + ) self.music_file_list = response.json()["music_file_list"] self.file_service_active = True except requests.exceptions.RequestException as e: - for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"): + for mp3_item in MUSIC_ROOT.glob("**/*.mp3"): temp_music_file = mp3_item.as_posix() if platform == "win32": temp_music_file = temp_music_file.replace("/", "\\") @@ -98,7 +107,12 @@ class MusicFileList(object): """ self.music_file_list.append(item) post_data = {"item": str(item)} - requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360) + requests.post( + f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", + json=post_data, + headers=SERVICE_HEADERS, + timeout=60, + ) MUSIC_FILE_LIST = MusicFileList("discord") @@ -142,7 +156,7 @@ async def get_file(ctx, source, link): url_data = {"urls": []} url_dict = {} url_dict["save_path"] = Path( - PurePath.joinpath(Path(MUSIC_FOLDER), Path(directory_name)) + PurePath.joinpath(MUSIC_ROOT, Path(directory_name)) ) url_dict["save_path"].mkdir(parents=True, exist_ok=True) url_dict["songs"] = file_list @@ -152,7 +166,7 @@ async def get_file(ctx, source, link): coro = asyncio.to_thread( youtube_download.download_songs, songs=url_data, - output_dir=MUSIC_FOLDER, + output_dir=str(MUSIC_ROOT), format_str="bestaudio/best", skip_mp3=False, keep_playlist_order=False, @@ -308,6 +322,7 @@ async def search_music(ctx, how_many=0, slowa_kluczowe=None): requests.post, f"{FILE_SERVICE_ADDRESS}{GET_PLAYLIST}", json=jrequest, + headers=SERVICE_HEADERS, timeout=360, ) return_data = await coroutine diff --git a/radio_commands.py b/radio_commands.py index 04d4d35..d32143e 100644 --- a/radio_commands.py +++ b/radio_commands.py @@ -8,7 +8,18 @@ import uuid import asyncio from datetime import datetime -from constants import RADIO_HARBOR_ADDRESS, SKIP_TRACK, FILE_SERVICE_ADDRESS, ADD_TO_PRIO_PLAYLIST, REQUEST_MUSIC, CREATE_PRIO_PLAYLIST, CLEAR_PRIO +from constants import ( + RADIO_HARBOR_ADDRESS, + SKIP_TRACK, + FILE_SERVICE_ADDRESS, + ADD_TO_PRIO_PLAYLIST, + REQUEST_MUSIC, + CREATE_PRIO_PLAYLIST, + CLEAR_PRIO, + service_headers, +) + +SERVICE_HEADERS = service_headers() class RadioModule(commands.Cog): def __init__(self, bot, logger_name): @@ -95,6 +106,7 @@ class RadioModule(commands.Cog): requests.post, f"{FILE_SERVICE_ADDRESS}{ADD_TO_PRIO_PLAYLIST}", json=jrequest, + headers=SERVICE_HEADERS, timeout=360, ) result = await coroutine @@ -130,6 +142,7 @@ class RadioModule(commands.Cog): requests.post, f"{FILE_SERVICE_ADDRESS}{REQUEST_MUSIC}", json=jrequest, + headers=SERVICE_HEADERS, timeout=360, ) result = await coroutine @@ -167,6 +180,7 @@ class RadioModule(commands.Cog): requests.post, f"{FILE_SERVICE_ADDRESS}{CREATE_PRIO_PLAYLIST}", json=jrequest, + headers=SERVICE_HEADERS, timeout=360, ) result = await coroutine @@ -194,6 +208,7 @@ class RadioModule(commands.Cog): coroutine = asyncio.to_thread( requests.get, f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}", + headers=SERVICE_HEADERS, timeout=360, ) result = await coroutine diff --git a/thin_client.py b/thin_client.py index 93b4a35..a96af33 100755 --- a/thin_client.py +++ b/thin_client.py @@ -2,17 +2,19 @@ # trunk-ignore-all(bandit/B311) # pylint: disable=line-too-long # pylint: disable=too-many-lines -""" -Module of a python bot named Conjurer - used to work on BDSM discord servers. -""" -import logging +"""Discord entrypoint for the Conjurer bot. -# *=========================================== Standard Library Imports +The legacy bootstrap used threads and synchronous calls that made clean +shutdowns difficult. We now run everything from a single asyncio event loop +with cooperative shutdown signals so the bot can stop gracefully. +""" + +import asyncio +import logging import random import threading from logging import handlers -# *==============Imported libraries import discord from discord.ext import commands @@ -82,22 +84,40 @@ async def on_ready(): logger.info("All systems: operational") -# *================================== Run +async def _run_comm_subroutine(stop_event: threading.Event) -> None: + await asyncio.to_thread(comm_subroutine, stop_event) + + +async def _run_bot(token: str, shutdown_event: asyncio.Event) -> None: + try: + await client.start(token, log_handler=None) + finally: + shutdown_event.set() + + +async def main() -> None: + if not TOKEN: + logger.error("Discord token missing – set DISCORD_TOKEN or configure netrc") + return + + shutdown_event = asyncio.Event() + comm_stop_event = threading.Event() + + comm_task = asyncio.create_task(_run_comm_subroutine(comm_stop_event)) + bot_task = asyncio.create_task(_run_bot(TOKEN, shutdown_event)) + + try: + await shutdown_event.wait() + except (KeyboardInterrupt, asyncio.CancelledError): + logger.info("Shutdown signal received") + comm_stop_event.set() + await client.close() + finally: + comm_stop_event.set() + if not client.is_closed(): + await client.close() + await asyncio.gather(bot_task, comm_task, return_exceptions=True) + + if __name__ == "__main__": - logger.info("Starting discord bot") - threads = [] - logger.info("Starting discord bot: Creating threads") - threads.append(threading.Thread(target=client.run, args=(TOKEN,),kwargs={"log_handler":None})) - threads.append(threading.Thread(target=comm_subroutine)) - logger.info("Starting discord bot: Starting threads") - WRK_CNT = 0 - for worker in threads: - WRK_CNT += 1 - logger.info("Starting discord bot: Starting thread %s", WRK_CNT) - worker.start() - logger.info("Starting discord bot: Joining threads") - WRK_CNT = 0 - for worker in threads: - WRK_CNT += 1 - logger.info("Starting discord bot: Joining thread %s", WRK_CNT) - worker.join() + asyncio.run(main())