diff --git a/bot.py b/bot.py index dd28937..d09afea 100755 --- a/bot.py +++ b/bot.py @@ -37,6 +37,7 @@ from constants import ( GET_MP3, LIBRARIAN_SERVICE_ADDRESS, LOGFILE, + RADIO_SERVICE_ADDRESS, TOKEN, ) @@ -96,10 +97,15 @@ CORE_EXTENSIONS = [ ] SERVICE_EXTENSION_GROUPS = { - # musician (file service): music download/search, radio control, file shares + # musician (file service): Discord music download/search, file shares "musician": { "health_url": f"{FILE_SERVICE_ADDRESS}{GET_MP3}", - "extensions": ["music_commands", "radio_commands", "file_search_commands"], + "extensions": ["music_commands", "file_search_commands"], + }, + # betoniarka (radio operator colocated with Liquidsoap): radio playlists + "radio": { + "health_url": f"{RADIO_SERVICE_ADDRESS}/ping", + "extensions": ["radio_commands"], }, # librarian: DOI / Crossref search "librarian": { diff --git a/conjurer_betoniarka/betoniarka.py b/conjurer_betoniarka/betoniarka.py new file mode 100644 index 0000000..41cb659 --- /dev/null +++ b/conjurer_betoniarka/betoniarka.py @@ -0,0 +1,353 @@ +"""Betoniarka - the radio operator service. + +Lives in the SAME container as Liquidsoap and runs as the SAME unprivileged +user ('radio'), which is the whole point: the process that writes the radio +playlists is colocated with the process that watches them, so there is no +cross-host permission juggling (root-owned network shares, failing chowns) +anymore. + +Responsibilities (extracted from conjurer_musician, which is now a pure +Discord music player): +- scan the local music library into all_playlist.playlist / hit.playlist +- serve the radio-management HTTP API the bot calls + (/add_to_priority, /create_priority_playlist, /request_radio_file, + /clear_pr_pls) plus /ping for health checks and /stream for the web page +- tail radio_log.log / persistence.log and forward "now playing" events to + the bot's /prepped_tracks +""" + +import json +import logging +import os +import random +import re +import threading +import time +from datetime import datetime +from pathlib import Path +from typing import Dict, List + +import requests +from flask import Flask, abort, jsonify, request, send_file +from waitress import serve + + +def _env(name: str, default: str) -> str: + return os.getenv(name, default) + + +API_KEY = os.getenv("CONJURER_API_KEY") +MAIN_BOT_ADDRESS = _env("CONJURER_MAIN_BOT", "http://127.0.0.1:5000") +MUSIC_TRACKER = _env("CONJURER_MUSIC_TRACKER_ENDPOINT", "/prepped_tracks") +HOST_ADDRESS = _env("BETONIARKA_HOST", "0.0.0.0") +PORT_ADDRESS = int(_env("BETONIARKA_PORT", "5005")) + +DATA_DIR = Path(_env("BETONIARKA_DATA", "/srv/betoniarka/data")) +MUSIC_FOLDER = Path(_env("BETONIARKA_MUSIC", "/srv/betoniarka/music")) +PRIORITY_FOLDER = Path(_env("BETONIARKA_PRIORITY_FOLDER", str(MUSIC_FOLDER / "priority"))) +STREAM_TEMPLATE = _env("BETONIARKA_STREAM_TEMPLATE", "/app/stream.html") +RESCAN_SECONDS = int(_env("BETONIARKA_RESCAN_SECONDS", str(24 * 60 * 60))) +# How many leading path tokens to ignore when keyword-matching +# (/srv/betoniarka/music/... -> '', 'srv', 'betoniarka', 'music'). +PATH_SKIP = int(_env("BETONIARKA_PATH_SKIP", "4")) + +ALL_PLAYLIST_PATH = DATA_DIR / "all_playlist.playlist" +HIT_PLAYLIST_PATH = DATA_DIR / "hit.playlist" +REQUEST_PLAYLIST_PATH = DATA_DIR / "request.playlist" +PRIORITY_PLAYLIST_PATH = DATA_DIR / "priority_queue.playlist" +RADIOLOG_PATH = DATA_DIR / "radio_log.log" +PERSISTENCE_PATH = DATA_DIR / "persistence.log" + +ENCODING = _env("CONJURER_ENCODING", "utf-8") + +logger = logging.getLogger("betoniarka") + +music_file_list: List[str] = [] +priority_list: List[str] = [] + +app = Flask(__name__) + + +def _build_headers() -> Dict[str, str]: + if API_KEY: + return {"X-Conjurer-Api-Key": API_KEY} + return {} + + +def _authorize_request() -> None: + if API_KEY and request.headers.get("X-Conjurer-Api-Key") != API_KEY: + abort(401) + + +def _post_to_bot(payload: List[str]) -> None: + try: + response = requests.post( + f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}", + json=payload, + headers=_build_headers(), + timeout=60, + ) + logger.info("Forwarded to bot (%s): %s", response.status_code, payload[0]) + except requests.exceptions.RequestException as exc: + logger.warning("Bot unreachable, dropping %s: %s", payload[0], exc) + + +# ---------------------------------------------------------------- library +def rescan(): + """Scan the local library into the playlists Liquidsoap watches. + + Paths written here are LOCAL container paths, the same ones Liquidsoap + resolves - no shared network filesystem involved. + """ + logger.info("Rescan triggered") + music_file_list.clear() + priority_list.clear() + + for mp3_item in MUSIC_FOLDER.glob("**/*.mp3"): + music_file_list.append(mp3_item.as_posix()) + for mp3_item in PRIORITY_FOLDER.glob("**/*.mp3"): + priority_list.append(mp3_item.as_posix()) + + with ALL_PLAYLIST_PATH.open("w", encoding=ENCODING) as w_file: + for item in music_file_list: + w_file.write(item + "\n") + with HIT_PLAYLIST_PATH.open("w", encoding=ENCODING) as w_file: + for item in priority_list: + w_file.write(item + "\n") + logger.info("Rescan done: %d tracks, %d hits", len(music_file_list), len(priority_list)) + + +def thread_rescan(): + while True: + time.sleep(RESCAN_SECONDS) + rescan() + + +# ---------------------------------------------------------------- search +def remove_characters(string, character): + return string.replace(character, "") + + +def max_weight(lista): + maximum_weight = 0 + for iterator in lista: + if iterator[0] > maximum_weight: + maximum_weight = iterator[0] + return maximum_weight + + +_CHAR_REMOVE = [ + ".", "^", "$", "*", "+", "?", "{", "}", "[", "]", + "\\", "/", "|", "(", ")", "!", ",", "-", ":", "mp3", +] + + +def wyszukaj(word_list, how_many, _logger=None, write_to=None): + """Keyword-score the library; optionally append hits to a playlist file. + + Ported unchanged from the musician (same scoring), minus the win32 + branches. ``write_to`` replaces the old ``return_to_bot`` flag: pass a + playlist Path to append the result, or None to just return it. + """ + fun_logger = _logger or logger + search_weight = [(0, "") for _ in range(len(music_file_list))] + + time_start = datetime.now() + skip_start = 2 if int(how_many) > 0 else 1 + for word in word_list[skip_start:]: + token_weight = len(word) + fun_logger.info("Słowo kluczowe: %s", word) + for itr, file in enumerate(music_file_list): + parts = file.split("/") + all_words = [] + for f_iter in parts: + for char in _CHAR_REMOVE: + f_iter = remove_characters(f_iter, char) + all_words.extend(f_iter.split()) + pingu = 1 + pattern_len = len(all_words) + matched_times = 1 + for itm in all_words[PATH_SKIP:]: + pingu += 1 + if re.match(".*" + word + ".*", itm, re.IGNORECASE): + temp_weight = ( + search_weight[itr][0] + + (token_weight + (pingu**1.5) / pattern_len) / matched_times + ) + search_weight[itr] = (temp_weight, music_file_list[itr]) + matched_times += 1 + + fun_logger.info("Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start) + best = max_weight(search_weight) + if best == 0: + return [] + + return_list = [] + if int(how_many) <= 0: + for weight, path in search_weight: + if weight == best: + return_list.append((weight, path)) + break + else: + search_weight.sort(key=lambda x: x[0], reverse=True) + return_list.extend(search_weight[: int(how_many)]) + + if write_to is not None: + with write_to.open("a", encoding=ENCODING) as s_file: + for item in return_list: + s_file.write(item[1] + "\n") + fun_logger.info("Done: %s", return_list) + return return_list + + +# ---------------------------------------------------------------- tailer +def scan_tracks(): + """Tail the radio logs and forward play events to the bot.""" + with open(RADIOLOG_PATH, "r", encoding=ENCODING) as log_file: + log_file.seek(os.stat(RADIOLOG_PATH).st_size) + prev_size = os.stat(PERSISTENCE_PATH).st_size + + while True: + current_size = os.stat(PERSISTENCE_PATH).st_size + if prev_size != current_size: + while prev_size != current_size: + prev_size = current_size + time.sleep(0.1) + current_size = os.stat(PERSISTENCE_PATH).st_size + with open(PERSISTENCE_PATH, "r", encoding=ENCODING) as persistence: + lines = persistence.readlines() + if len(lines) >= 3: + _post_to_bot(["next", lines[2]]) + + position = log_file.tell() + line = log_file.readline() + if not line: + time.sleep(1) + log_file.seek(position) + continue + + if not re.match(r".*Prepared.*", line): + time.sleep(0.1) + continue + + result = None + if re.match(r".*jingles.*", line): + result = ["jingles", line] + elif re.match(r".*priority.*", line): + result = ["priority", line] + elif re.match(r".*hit.*", line): + result = ["hit", line] + elif re.match(r".*all_playlist.*", line): + result = ["all", line] + elif re.match(r".*request.*", line): + result = ["requests", line] + + if result: + logger.info("Forwarding radio log entry: %s", result[0]) + _post_to_bot(result) + + time.sleep(0.1) + + +# ---------------------------------------------------------------- routes +@app.route("/ping", methods=["GET"]) +def ping(): + """Health check - the bot gates radio_commands on this answering.""" + return jsonify("ALIVE") + + +@app.route("/stream", methods=["GET"]) +def stream_page(): + return send_file(STREAM_TEMPLATE) + + +@app.route("/clear_pr_pls", methods=["GET"]) +def clear_pr_pls(): + _authorize_request() + app.logger.info("CLEARING PLAYLIST") + with PRIORITY_PLAYLIST_PATH.open("w", encoding=ENCODING) as cleared_pl: + cleared_pl.write("") + return jsonify(isError=False, message="Success", statusCode=200, data=[]), 200 + + +@app.route("/rescan", methods=["GET"]) +def manual_rescan(): + _authorize_request() + rescan() + return jsonify(isError=False, message="Success", statusCode=200, + data={"tracks": len(music_file_list)}), 200 + + +@app.route("/request_radio_file", methods=["POST"]) +def add_request(): + _authorize_request() + record = json.loads(request.data) + app.logger.info(record) + wyszukaj(record["lista_slow"], 0, app.logger, write_to=REQUEST_PLAYLIST_PATH) + return jsonify(isError=False, message="Success", statusCode=200, + data={"status": "OK"}), 200 + + +@app.route("/create_priority_playlist", methods=["POST"]) +def create_priority_playlist(): + _authorize_request() + record = json.loads(request.data) + app.logger.info(record) + return_data = wyszukaj( + record["lista_slow"], record["dlugosc_plejlisty"], app.logger, write_to=None + ) + random.shuffle(return_data) + # NOTE: appends to the REQUEST playlist - behaviour inherited verbatim + # from the musician implementation (the request queue picks it up). + with REQUEST_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file: + for item in return_data: + s_file.write(item[1] + "\n") + return jsonify(isError=False, message="Success", statusCode=200, + data={"status": "OK"}), 200 + + +@app.route("/add_to_priority", methods=["POST"]) +def add_to_priority(): + _authorize_request() + record = json.loads(request.data) + app.logger.info(record) + wyszukaj( + record["lista_slow"], record["dlugosc_plejlisty"], app.logger, + write_to=PRIORITY_PLAYLIST_PATH, + ) + return jsonify(isError=False, message="Success", statusCode=200, + data={"status": "OK"}), 200 + + +def waitress_run(): + serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS) + + +if __name__ == "__main__": + logger.setLevel(logging.DEBUG) + console = logging.StreamHandler() + console.setFormatter( + logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") + ) + logger.addHandler(console) + + rescan() + logger.info("Betoniarka started on %s:%s", HOST_ADDRESS, PORT_ADDRESS) + + threads = [ + threading.Thread(target=waitress_run, daemon=True), + threading.Thread(target=thread_rescan, daemon=True), + ] + for worker in threads: + worker.start() + + time.sleep(5) + track_thread = threading.Thread(target=scan_tracks, daemon=True) + track_thread.start() + + try: + for worker in threads: + worker.join() + track_thread.join() + except KeyboardInterrupt: + logger.info("Shutdown requested - exiting betoniarka") diff --git a/conjurer_musician/conjurer_musician.py b/conjurer_musician/conjurer_musician.py index 5823fd1..09e7c39 100644 --- a/conjurer_musician/conjurer_musician.py +++ b/conjurer_musician/conjurer_musician.py @@ -1,7 +1,12 @@ -""" -The provided Python script sets up a Flask web server to manage a list of music files, with -functions for rescanning the music folder, updating the music list, and serving the music list via -API endpoints. +"""Musician - the Discord music player service. + +Serves the music library index and keyword search the bot uses for Discord +playback (/mp3, /update_mp3, /get_music) plus the file-share endpoints. + +Radio playlist management and the radio-log tailer moved to +conjurer_betoniarka/betoniarka.py, which runs INSIDE the radio container as +the same user as Liquidsoap - the musician no longer writes any radio files, +so the old root-owned-network-share permission mess is gone. """ import json @@ -14,18 +19,9 @@ import time from datetime import datetime from logging import handlers from pathlib import Path -from typing import Dict, List +from typing import List -import requests -from flask import ( - Flask, - abort, - jsonify, - redirect, - render_template, - request, - send_from_directory, -) +from flask import Flask, abort, jsonify, request from waitress import serve import media_search_functions @@ -41,8 +37,6 @@ def _env_path(name: str, default: str) -> Path: API_KEY = os.getenv("CONJURER_API_KEY") -MAIN_BOT_ADDRESS = _env("CONJURER_MAIN_BOT", "http://127.0.0.1:5000") -MUSIC_TRACKER = _env("CONJURER_MUSIC_TRACKER_ENDPOINT", "/prepped_tracks") HOST_ADDRESS = _env("CONJURER_MUSICIAN_HOST", "0.0.0.0") PORT_ADDRESS = int(_env("CONJURER_MUSICIAN_PORT", "5000")) @@ -56,52 +50,12 @@ LOGSTORE = _env_path("CONJURER_LOGSTORE", str(BASE_DIR / "logs")) MUSIC_FOLDER = _env_path( "CONJURER_MUSIC_FOLDER", str(BASE_DIR / "music") ) -PRIORITY_FOLDER = _env_path( - "CONJURER_PRIORITY_FOLDER", str(MUSIC_FOLDER / "priority") -) -RADIOLOG_PATH = _env_path( - "CONJURER_RADIO_LOG", str(BASE_DIR / "radio_log.log") -) -PERSISTENCE_PATH = _env_path( - "CONJURER_PERSISTENCE_LOG", str(BASE_DIR / "persistence.log") -) -ALL_PLAYLIST_PATH = _env_path( - "CONJURER_ALL_PLAYLIST", str(BASE_DIR / "all_playlist.playlist") -) -HIT_PLAYLIST_PATH = _env_path( - "CONJURER_HIT_PLAYLIST", str(BASE_DIR / "hit.playlist") -) -REQUEST_PLAYLIST_PATH = _env_path( - "CONJURER_REQUEST_PLAYLIST", str(BASE_DIR / "request.playlist") -) -PRIORITY_PLAYLIST_PATH = _env_path( - "CONJURER_PRIORITY_PLAYLIST", str(BASE_DIR / "priority_queue.playlist") -) -STREAM_TEMPLATE = _env_path( - "CONJURER_STREAM_TEMPLATE", str(BASE_DIR / "stream.html") -) ENCODING = _env("CONJURER_ENCODING", "utf-8") SEPARATOR_FILE_PATH = os.sep -for playlist_path in ( - ALL_PLAYLIST_PATH, - HIT_PLAYLIST_PATH, - REQUEST_PLAYLIST_PATH, - PRIORITY_PLAYLIST_PATH, -): - playlist_path.parent.mkdir(parents=True, exist_ok=True) - random.seed() music_file_list: List[str] = [] -priority_list: List[str] = [] - - -def _build_headers() -> Dict[str, str]: - headers: Dict[str, str] = {} - if API_KEY: - headers["X-Conjurer-Api-Key"] = API_KEY - return headers def _authorize_request() -> None: @@ -109,29 +63,16 @@ def _authorize_request() -> None: abort(401) -def _post_to_bot(payload: List[str]) -> None: - response = requests.post( - f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}", - json=payload, - headers=_build_headers(), - timeout=60, - ) - logger = logging.getLogger("conjurer_musician") - logger.info("SENT") - logger.info(response.status_code) - logger.info("SEND CONFIRMED") - - def rescan(): - """ - The `rescan` function logs a message, scans for mp3 files in a specified folder, - and adds them to a list of music files. + """Refresh the in-memory library index used by /mp3 and /get_music. + + Radio playlist files are NOT written here anymore - that is the + betoniarka's job, colocated with Liquidsoap. """ logger = logging.getLogger("conjurer_musician") logger.info("Rescan triggered") music_file_list.clear() - priority_list.clear() for mp3_item in MUSIC_FOLDER.glob("**/*.mp3"): temp_music_file = mp3_item.as_posix() @@ -139,27 +80,6 @@ def rescan(): temp_music_file = temp_music_file.replace("/", "\\") music_file_list.append(temp_music_file) - for mp3_item in PRIORITY_FOLDER.glob("**/*.mp3"): - temp_music_file = mp3_item.as_posix() - if os.name == "nt": - temp_music_file = temp_music_file.replace("/", "\\") - priority_list.append(temp_music_file) - - with ALL_PLAYLIST_PATH.open("w", encoding=ENCODING) as w_file: - try: - for item in music_file_list: - w_file.write(item) - w_file.write("\n") - except json.JSONDecodeError: - pass - with HIT_PLAYLIST_PATH.open("w", encoding=ENCODING) as w_file: - try: - for item in priority_list: - w_file.write(item) - w_file.write("\n") - except json.JSONDecodeError: - pass - def thread_rescan(): """ @@ -174,55 +94,6 @@ def thread_rescan(): rescan() -def scan_tracks(): - # Set the filename and open the file - logger = logging.getLogger("conjurer_musician") - with open(RADIOLOG_PATH, "r", encoding=ENCODING) as log_file: - log_file.seek(os.stat(RADIOLOG_PATH).st_size) - prev_size = os.stat(PERSISTENCE_PATH).st_size - - while True: - current_size = os.stat(PERSISTENCE_PATH).st_size - if prev_size != current_size: - while prev_size != current_size: - prev_size = current_size - time.sleep(0.1) - current_size = os.stat(PERSISTENCE_PATH).st_size - with open(PERSISTENCE_PATH, "r", encoding=ENCODING) as persistence: - lines = persistence.readlines() - if len(lines) >= 3: - _post_to_bot(["next", lines[2]]) - - position = log_file.tell() - line = log_file.readline() - if not line: - time.sleep(1) - log_file.seek(position) - continue - - if not re.match(r".*Prepared.*", line): - time.sleep(0.1) - continue - - result = None - if re.match(r".*jingles.*", line): - result = ["jingles", line] - elif re.match(r".*priority.*", line): - result = ["priority", line] - elif re.match(r".*hit.*", line): - result = ["hit", line] - elif re.match(r".*all_playlist.*", line): - result = ["all", line] - elif re.match(r".*request.*", line): - result = ["requests", line] - - if result: - logger.info("Forwarding radio log entry: %s", result[0]) - _post_to_bot(result) - - time.sleep(0.1) - - app = Flask(__name__) # AutoIndex(app, browse_root="/") @@ -328,9 +199,6 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True): while not_found: if search_weight[itr][0] == item_to_search: return_list.append(search_weight[itr]) - if not return_to_bot: - with PRIORITY_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file: - s_file.write(search_weight[itr][1] + "\n") break itr += 1 else: @@ -401,62 +269,6 @@ def get_share_links(): -@app.route("/stream", methods=["GET"]) -def stream_music(): - """ - The function `stream_music` is a route handler for the "/stream" endpoint. - It returns a JSON response containing a key "music_file_list" with the value of the variable `music_file_list`. - - :return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`. - """ - - # return send_from_directory("/tmp/hls", "stream.m3u8") - return render_template(str(STREAM_TEMPLATE)) - - -@app.route("/") -def stream(file_name): - """ - Stream the specified file from the video directory. - - Args: - file_name (str): The name of the file to be streamed. - - Returns: - Response: The response containing the streamed file. - """ - # trunk-ignore(bandit/B108) - video_dir = "/tmp/hls" - return send_from_directory(video_dir, file_name) - - -@app.route("/stream_mp3", methods=["GET"]) -def stream_music_mp3(): - """ - The function `stream_music_mp3` is a route handler for the "/stream_mp3" endpoint. - It redirects the user to the URL "http://www.example.com" with a status code of 302. - - :return: A redirect response to "http://www.example.com" with a status code of 302. - """ - return redirect("http://www.example.com", code=302) - - -@app.route("/clear_pr_pls", methods=["GET"]) -def clear_pr_pls(): - _authorize_request() - """ - The function `clear_pr_pls` clears the contents of the priority queue playlist file. - - :return: A JSON response indicating the success of the operation. - """ - app.logger.info("CLEARING PLAYLIST") - with PRIORITY_PLAYLIST_PATH.open("w", encoding=ENCODING) as cleared_pl: - cleared_pl.write("") - - return_data = jsonify(isError=False, message="Success", statusCode=200, data=[]) - return return_data, 200 - - @app.route("/mp3", methods=["GET"]) def get_music_list(): """ @@ -517,96 +329,6 @@ def look_for_playlist(): return return_data -@app.route("/request_radio_file", methods=["POST"]) -def add_request(): - _authorize_request() - record = json.loads(request.data) - app.logger.info(record) - app.logger.info(record["lista_slow"]) - app.logger.info(record["UUID"]) - return_data = wyszukaj(record["lista_slow"], 0, app.logger, False) - - with REQUEST_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file: - for item in return_data: - s_file.write(item[1] + "\n") - return_data = ( - jsonify( - isError=False, message="Success", statusCode=200, data={"status": "OK"} - ), - 200, - ) - return return_data - - -@app.route("/create_priority_playlist", methods=["POST"]) -def create_priority_playlist(): - """ - The function `create_priority_playlist` receives a POST request with a JSON payload, logs the received - item, adds it to a music file list, and returns a success message along with the updated record. - - :return: A tuple containing a JSON response and a status code. - The JSON response includes keys `isError`, - `message`, `statusCode`, and `data`, with values - indicating the success of the operation and the - data that was received and added to the `music_file_list`. - The status code returned is 200,indicating a successful response. - """ - _authorize_request() - record = json.loads(request.data) - app.logger.info(record) - app.logger.info(record["lista_slow"]) - app.logger.info(record["UUID"]) - app.logger.info(record["dlugosc_plejlisty"]) - return_data = wyszukaj( - record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False - ) - random.shuffle(return_data) - with REQUEST_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file: - for item in return_data: - s_file.write(item[1] + "\n") - return_data = ( - jsonify( - isError=False, message="Success", statusCode=200, data={"status": "OK"} - ), - 200, - ) - return return_data - - -@app.route("/add_to_priority", methods=["POST"]) -def add_to_priority(): - """ - The function `add_to_priority` receives a POST request with a JSON payload, logs the received - item, adds it to a music file list, and returns a success message along with the updated record. - - :return: A tuple containing a JSON response and a status code. - The JSON response includes keys `isError`, - `message`, `statusCode`, and `data`, with values - indicating the success of the operation and the - data that was received and added to the `music_file_list`. - The status code returned is 200,indicating a successful response. - """ - _authorize_request() - record = json.loads(request.data) - app.logger.info(record) - app.logger.info(record["lista_slow"]) - app.logger.info(record["UUID"]) - app.logger.info(record["dlugosc_plejlisty"]) - return_data = wyszukaj( - record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False - ) - with PRIORITY_PLAYLIST_PATH.open("a", encoding=ENCODING) as s_file: - for item in return_data: - s_file.write(item[1] + "\n") - return_data = ( - jsonify( - isError=False, message="Success", statusCode=200, data={"status": "OK"} - ), - 200, - ) - return return_data - - def flask_debug(): """ The `flask_debug` function starts a Flask application in debug mode without using the reloader. @@ -660,13 +382,8 @@ if __name__ == "__main__": for worker in threads: worker.start() - time.sleep(60) - track_thread = threading.Thread(target=scan_tracks, daemon=True) - track_thread.start() - try: for worker in threads: worker.join() - track_thread.join() except KeyboardInterrupt: logger.info("Shutdown requested - exiting musician service") diff --git a/constants.py b/constants.py index bf95609..685ff89 100644 --- a/constants.py +++ b/constants.py @@ -198,6 +198,9 @@ TRANSCRIPTS_PATH = os.getenv( FILE_SERVICE_ADDRESS = os.getenv("CONJURER_FILE_SERVICE", "http://192.168.1.15:5000") RADIO_HARBOR_ADDRESS = os.getenv("CONJURER_RADIO_HARBOR", "http://192.168.1.15:54321") +# Betoniarka (radio-operator service colocated with Liquidsoap). Falls back to +# the musician address so deployments that have not split yet keep working. +RADIO_SERVICE_ADDRESS = os.getenv("CONJURER_RADIO_SERVICE", FILE_SERVICE_ADDRESS) SKIP_TRACK = os.getenv("CONJURER_SKIP_ENDPOINT", "/skip") LIBRARIAN_SERVICE_ADDRESS = os.getenv( "CONJURER_LIBRARIAN_SERVICE", "http://192.168.1.192:5001" diff --git a/docker/Dockerfile.radio b/docker/Dockerfile.radio index 70d71da..1e8fc26 100644 --- a/docker/Dockerfile.radio +++ b/docker/Dockerfile.radio @@ -45,6 +45,7 @@ RUN apt-get update && apt-get install -y --no-install-recommends \ ffmpeg \ pulseaudio pulseaudio-utils \ icecast2 jq \ + python3 python3-flask python3-waitress python3-requests \ && rm -rf /var/lib/apt/lists/* # Liquidsoap refuses to run as root (security exit), so it gets a dedicated @@ -88,6 +89,8 @@ COPY docker/pulse-system.pa /etc/pulse/system.pa WORKDIR /app COPY conjurer_musician/radio_conjurer.liq ./radio_conjurer.liq COPY conjurer_musician/script.params ./script.params +COPY conjurer_musician/stream.html ./stream.html +COPY conjurer_betoniarka/betoniarka.py ./betoniarka.py COPY docker/icecast.xml.tpl ./icecast.xml.tpl COPY docker/entrypoint.radio.sh /usr/local/bin/entrypoint.sh RUN chmod +x /usr/local/bin/entrypoint.sh @@ -98,9 +101,10 @@ RUN chmod +x /usr/local/bin/entrypoint.sh # /srv/betoniarka/secrets - icecast_credentials.json (provisioned at install) VOLUME ["/srv/betoniarka/data", "/srv/betoniarka/music", "/srv/betoniarka/secrets"] -# 8000 = icecast (listeners), 54321 = harbor /skip (the bot's RADIO_HARBOR), +# 8000 = icecast (listeners), 5005 = betoniarka HTTP API (the bot's +# CONJURER_RADIO_SERVICE), 54321 = harbor /skip (the bot's RADIO_HARBOR), # 9999 = interactive harbor. -EXPOSE 8000 54321 9999 +EXPOSE 8000 5005 54321 9999 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"] CMD ["liquidsoap", "/srv/betoniarka/data/radio_conjurer.liq"] diff --git a/docker/compose.radio.yaml b/docker/compose.radio.yaml index 1313cc0..d993ea8 100644 --- a/docker/compose.radio.yaml +++ b/docker/compose.radio.yaml @@ -23,8 +23,12 @@ services: # PULSE_SERVER: unix:/tmp/pulseaudio.socket # Hostname icecast reports in its status/YP pages: ICECAST_HOSTNAME: ${ICECAST_HOSTNAME:-localhost} + # Betoniarka (radio-operator API + log forwarder): + CONJURER_MAIN_BOT: ${CONJURER_MAIN_BOT:-http://127.0.0.1:5000} + CONJURER_API_KEY: ${CONJURER_API_KEY:-} ports: - "8000:8000" # icecast - listeners tune in here + - "5005:5005" # betoniarka API - the bot's CONJURER_RADIO_SERVICE - "54321:54321" # harbor /skip - the bot's CONJURER_RADIO_HARBOR - "9999:9999" # interactive harbor (keep LAN-only!) volumes: diff --git a/docker/entrypoint.radio.sh b/docker/entrypoint.radio.sh index 150db21..b2f8a05 100755 --- a/docker/entrypoint.radio.sh +++ b/docker/entrypoint.radio.sh @@ -91,13 +91,23 @@ esac # Liquidsoap refuses to run as root (init: security exit), so hand the data # volume to the dedicated 'radio' user and drop privileges for the main -# process. The script parses the icecast secret directly, so make that one -# file group-readable for 'radio' (kept 640, root-owned). -chown -R radio:radio "$DATA" +# process. chown is best-effort: on local volumes it always works (the +# supported layout); network filesystems with root-squash reject it, hence +# the warning instead of a fatal abort. +chown -R radio:radio "$DATA" 2>/dev/null \ + || echo "WARNING: chown of $DATA failed (network FS?) - keep this volume LOCAL to the radio VM" >&2 chgrp radio "$CREDS" 2>/dev/null && chmod 640 "$CREDS" || true if ! setpriv --reuid radio --regid radio --init-groups -- test -r "$MUSIC"; then echo "WARNING: music dir $MUSIC is not readable by the 'radio' user" >&2 fi +# Betoniarka: the radio-operator API + radio-log forwarder, running as the +# SAME user as liquidsoap on the SAME volume - this is what makes the old +# musician-writes-as-root-over-network-share permission mess go away. +BETONIARKA_DATA="$DATA" BETONIARKA_MUSIC="$MUSIC" \ + setpriv --reuid radio --regid radio --init-groups -- \ + python3 /app/betoniarka.py & +echo "betoniarka started (pid $!)" + cd "$DATA" exec setpriv --reuid radio --regid radio --init-groups -- "$@" diff --git a/docker/env/bot.env.example b/docker/env/bot.env.example index 000f6d9..b9cd93d 100644 --- a/docker/env/bot.env.example +++ b/docker/env/bot.env.example @@ -24,7 +24,10 @@ CONJURER_API_KEY= # --- Where the bot reaches the other services (other Proxmox VMs) -------- CONJURER_FILE_SERVICE=http://MUSICIAN_VM_IP:5000 -CONJURER_RADIO_HARBOR=http://MUSICIAN_VM_IP:54321 +# Betoniarka (radio-operator API, runs in the radio container): +CONJURER_RADIO_SERVICE=http://RADIO_VM_IP:5005 +# Liquidsoap harbor /skip (same radio container): +CONJURER_RADIO_HARBOR=http://RADIO_VM_IP:54321 CONJURER_LIBRARIAN_SERVICE=http://LIBRARIAN_VM_IP:5001 # --- Conan Exiles bridge (optional; empty/0 = disabled) ----------------- diff --git a/docs/deployment/DOCKER_PROXMOX.md b/docs/deployment/DOCKER_PROXMOX.md index c4aa653..96a91f0 100644 --- a/docs/deployment/DOCKER_PROXMOX.md +++ b/docs/deployment/DOCKER_PROXMOX.md @@ -270,7 +270,48 @@ created empty and a silent emergency-fallback mp3 is generated if the Wire-up: listeners tune to `http://RADIO_VM_IP:8000/mp3-stream`; the bot points at `CONJURER_RADIO_HARBOR=http://RADIO_VM_IP:54321` (harbor `/skip` -lives in the script itself). +lives in the script itself) and `CONJURER_RADIO_SERVICE=http://RADIO_VM_IP:5005` +(the betoniarka API below). + +### 3f. Betoniarka - the radio operator (and why the split) + +**Permissions post-mortem.** The pre-split layout had three actors fighting +over the same files: the musician wrote radio playlists **as root**, the +radio expected them **as user `radio`**, and both met on a **root-owned +network share** where `chown` fails by design (root squash / uid mapping). +Every component worked; the combination could not. + +**The fix is structural**: the process that *writes* the radio playlists now +lives in the same container as the process that *watches* them, running as +the same `radio` user on a **local** volume. No network share, no chown, no +uid mapping - the class of problem is gone, not patched. + +`conjurer_betoniarka/betoniarka.py` runs inside the radio container +(started by the entrypoint as user `radio`, port **5005**) and owns: +- the library scan → `all_playlist.playlist` / `hit.playlist` (local paths, + the same ones Liquidsoap resolves), on start + every 24h + `GET /rescan` +- the bot-facing radio API: `/add_to_priority`, `/create_priority_playlist`, + `/request_radio_file`, `/clear_pr_pls` (+ `GET /ping` for health gating, + `/stream` for the web page) +- tailing `radio_log.log`/`persistence.log` and forwarding play events to + the bot's `/prepped_tracks` (with the shared API key) + +The **musician** is now a pure Discord music player: `/mp3`, `/update_mp3`, +`/get_music` and the file-share endpoints. It no longer writes any radio +files and needs no shared partition with the radio VM. The music library +can still be replicated/mounted on both VMs (read-only on the radio side is +fine) - playlists reference the *radio VM's local* paths, generated locally. + +Bot wiring after the split (env on the bot): +``` +CONJURER_FILE_SERVICE=http://MUSICIAN_VM_IP:5000 # Discord music +CONJURER_RADIO_SERVICE=http://RADIO_VM_IP:5005 # betoniarka (radio cmds) +CONJURER_RADIO_HARBOR=http://RADIO_VM_IP:54321 # liquidsoap /skip +``` +`CONJURER_RADIO_SERVICE` defaults to `CONJURER_FILE_SERVICE`, so an +un-split deployment keeps working unchanged. The bot health-gates +`radio_commands` on betoniarka's `/ping` (separate from the musician group, +which now covers only `music_commands` + `file_search_commands`). **Privileges:** Liquidsoap refuses to run as root (`init: security exit`), so the main process runs as the dedicated **`radio`** user (member of diff --git a/radio_commands.py b/radio_commands.py index 609ab7c..6ce5428 100644 --- a/radio_commands.py +++ b/radio_commands.py @@ -8,7 +8,7 @@ import uuid import asyncio from datetime import datetime -from constants import RADIO_HARBOR_ADDRESS, SKIP_TRACK, FILE_SERVICE_ADDRESS, ADD_TO_PRIO_PLAYLIST, REQUEST_MUSIC, CREATE_PRIO_PLAYLIST, CLEAR_PRIO, service_headers +from constants import RADIO_HARBOR_ADDRESS, SKIP_TRACK, RADIO_SERVICE_ADDRESS, ADD_TO_PRIO_PLAYLIST, REQUEST_MUSIC, CREATE_PRIO_PLAYLIST, CLEAR_PRIO, service_headers SERVICE_HEADERS = service_headers() @@ -96,7 +96,7 @@ class RadioModule(commands.Cog): } coroutine = asyncio.to_thread( requests.post, - f"{FILE_SERVICE_ADDRESS}{ADD_TO_PRIO_PLAYLIST}", + f"{RADIO_SERVICE_ADDRESS}{ADD_TO_PRIO_PLAYLIST}", json=jrequest, headers=SERVICE_HEADERS, timeout=360, @@ -132,7 +132,7 @@ class RadioModule(commands.Cog): } coroutine = asyncio.to_thread( requests.post, - f"{FILE_SERVICE_ADDRESS}{REQUEST_MUSIC}", + f"{RADIO_SERVICE_ADDRESS}{REQUEST_MUSIC}", json=jrequest, headers=SERVICE_HEADERS, timeout=360, @@ -170,7 +170,7 @@ class RadioModule(commands.Cog): } coroutine = asyncio.to_thread( requests.post, - f"{FILE_SERVICE_ADDRESS}{CREATE_PRIO_PLAYLIST}", + f"{RADIO_SERVICE_ADDRESS}{CREATE_PRIO_PLAYLIST}", json=jrequest, headers=SERVICE_HEADERS, timeout=360, @@ -199,7 +199,7 @@ class RadioModule(commands.Cog): async with ctx.typing(): coroutine = asyncio.to_thread( requests.get, - f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}", + f"{RADIO_SERVICE_ADDRESS}{CLEAR_PRIO}", headers=SERVICE_HEADERS, timeout=360, )