diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index 9740009..e3595e4 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -1,37 +1,75 @@ -from logging import handlers -import re +""" +This module contains the implementation of the Librarian class and related functions for searching and refining queries. + +Classes: +- Librarian: Represents a librarian object that performs search and refinement operations on queries. + +Functions: +- flask_debug: Starts a Flask application in debug mode without using the reloader. +- waitress_run: Serves the Flask application using the Waitress WSGI server. +- BackgroundTaskSearch: Represents a background task for running the Librarian object asynchronously. +""" + +import asyncio +import json import logging +import netrc +import re import threading import time -from flask import Flask, jsonify, request -import netrc -import json -import asyncio from json.decoder import JSONDecodeError +from logging import handlers +from queue import Queue from urllib.request import urlopen + +import requests +import scrape_bot +import search_bot +from flask import Flask, jsonify, request from habanero import Crossref from waitress import serve -from queue import Queue -import search_bot -import scrape_bot -import requests -NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" +# Constants +NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" HOST_ADDRESS = "192.168.1.192" PORT_ADDRESS = 5001 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" SEND_RESULTS = "/conjurer" -BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147' +BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147" MAX_CR_RESULTS = 150000 ENCODING = "utf-8" + app = Flask(__name__) librarian_queue = Queue() librarian_list = [] -class Librarian(object): - def __init__(self, app, query, uuid) -> None: +class Librarian(object): + def __init__(self, _app, query, uuid) -> None: + """ + Initializes a Librarian object. + + Args: + - _app: The Flask application object. + - query: The query to be searched. + - uuid: The unique identifier for the search. + + Attributes: + - cr: The Crossref object for performing the search. + - query: The query to be searched. + - uuid: The unique identifier for the search. + - limit: The maximum number of search results to fetch. + - fetched: The number of search results fetched so far. + - hit: The number of search results that match the refinement criteria. + - total: The total number of search results. + - app: The Flask application object. + - live_results: A list to store live search results. + - final_result: A list to store the final refined search results. + - not_in_db: A list to store search results that are not in the local database. + - search_result_from_cr: A dictionary to store the search results from Crossref. + - done: A flag indicating if the search is done. + """ netrc_mod = netrc.netrc(NETRC_FILE) authTokens = netrc_mod.authenticators("crossref") self.cr = Crossref(mailto=authTokens[0]) @@ -41,7 +79,7 @@ class Librarian(object): self.fetched = 0 self.hit = 0 self.total = 0 - self.app = app + self.app = _app self.live_results = [] self.final_result = [] self.not_in_db = [] @@ -49,25 +87,37 @@ class Librarian(object): self.done = False async def search_crossref(self, query): + """ + Performs a search on Crossref for the given query. + + Args: + - query: The query to be searched. + + Returns: + - result: The search result from Crossref. + + Raises: + - None. + """ self.app.logger.info("STARTED SEARCH") - result = self.cr.works(query = query, limit = 1000) + result = self.cr.works(query=query, limit=1000) self.search_result_from_cr.update(result) - self.total = result['message']['total-results'] - self.fetched += len(result['message']['items']) - self.app.logger.info (self.total) + self.total = result["message"]["total-results"] + self.fetched += len(result["message"]["items"]) + self.app.logger.info(self.total) self.app.logger.info(self.fetched) while self.total > self.fetched and self.limit > self.fetched: - result = self.cr.works(query = query, limit = 1000) + result = self.cr.works(query=query, limit=1000) self.search_result_from_cr.update(result) - self.total = result['message']['total-results'] - self.fetched += len(result['message']['items']) - self.app.logger.info (self.total) + self.total = result["message"]["total-results"] + self.fetched += len(result["message"]["items"]) + self.app.logger.info(self.total) self.app.logger.info(self.fetched) time.sleep(0.1) self.app.logger.info("CROSSREF DONE") - #TODO: add to sql with UUID - #sql row - title, doi, author, uuid, last retrieved, is_available - + # TODO: add to sql with UUID + # sql row - title, doi, author, uuid, last retrieved, is_available + with open("rr_results.json", "r+", encoding="utf-8") as data_file: # First we load existing data into a dict. try: @@ -76,18 +126,34 @@ class Librarian(object): file_data = {} summarized_results = [] self.app.logger.info("REFINE: Removing all derived works from the list") - for item in self.search_result_from_cr['message']['items']: + for item in self.search_result_from_cr["message"]["items"]: container = "NO" if "container-title" in item: container = "YES" else: - summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container}) - result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}} + summarized_results.append( + { + "DOI": item["DOI"], + "title": item["title"] if "title" in item else None, + "type": item["type"] if "type" in item else None, + "container": container, + } + ) + result = { + self.uuid: { + "total_results": self.search_result_from_cr["message"][ + "total-results" + ], + "on_page": 1, + "summary": summarized_results, + "results": self.search_result_from_cr["message"]["items"], + } + } if file_data: file_data.update(result) else: file_data = result - + self.app.logger.info("REFINE: Dumping to file") data_file.truncate(0) data_file.seek(0) @@ -95,12 +161,38 @@ class Librarian(object): return result async def check_dois_in_local_db(self, doi): + """ + Checks if the given DOI exists in the local database. + + Args: + - doi: The DOI to be checked. + + Returns: + - result: The search result from the backend app. + + Raises: + - None. + """ self.app.logger.info("REFINE: Running search in the backend app") - coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger) + coro = asyncio.to_thread( + search_bot.search_for_doi, doi, self.live_results, self.app.logger + ) result = await coro return result async def check_if_exists_brute_force(self, page_url): + """ + Checks if the given page URL exists using brute force method. + + Args: + - page_url: The URL of the page to be checked. + + Returns: + - exists: A boolean indicating if the page exists. + + Raises: + - ValueError: If the URL does not start with 'http:' or 'https:'. + """ self.app.logger.error("REFINE: Brute force search!") if not page_url.startswith(("http:", "https:")): raise ValueError("URL must start with 'http:' or 'https:'") @@ -109,33 +201,68 @@ class Librarian(object): data = response.read() text = data.decode("utf-8") for line in text.splitlines(): - if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): + if re.match( + r".*Unfortunately, Sci-Hub doesn't have the requested document.*", + line, + ): return False - if m := re.match(r".*') + +@app.route("/") def stream(file_name): - video_dir = '/tmp/hls' + """ + Stream the specified file from the video directory. + + Args: + file_name (str): The name of the file to be streamed. + + Returns: + Response: The response containing the streamed file. + """ + video_dir = "/tmp/hls" return send_from_directory(video_dir, file_name) @app.route("/stream_mp3", methods=["GET"]) def stream_music_mp3(): """ - The function `get_music_list` returns a JSON object containing a list of music files. - :return: A JSON response containing a key "music_file_list" with the value of the variable - `music_file_list`. + The function `stream_music_mp3` is a route handler for the "/stream_mp3" endpoint. + It redirects the user to the URL "http://www.example.com" with a status code of 302. + + :return: A redirect response to "http://www.example.com" with a status code of 302. """ return redirect("http://www.example.com", code=302) @@ -278,19 +316,18 @@ def stream_music_mp3(): @app.route("/clear_pr_pls", methods=["GET"]) def clear_pr_pls(): """ - The function `get_music_list` returns a JSON object containing a list of music files. - :return: A JSON response containing a key "music_file_list" with the value of the variable - `music_file_list`. + The function `clear_pr_pls` clears the contents of the priority queue playlist file. + + :return: A JSON response indicating the success of the operation. """ with open( "/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8" - ) as s_file: + ) as s_file: s_file.truncate(0) return_data = jsonify(isError=False, message="Success", statusCode=200, data=[]) return return_data, 200 - @app.route("/mp3", methods=["GET"]) def get_music_list(): """ @@ -325,20 +362,22 @@ def update_music_list(): @app.route("/get_music", methods=["POST"]) def look_for_playlist(): """ - The function `update_music_list` receives a POST request with a JSON payload, logs the received + The function `look_for_playlist` receives a POST request with a JSON payload, logs the received item, adds it to a music file list, and returns a success message along with the updated record. - :return: The function `update_music_list` is returning a tuple containing a JSON response and a - status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`, - with values indicating the success of the operation and the data that was - received and added to the `music_file_list`. - The status code returned is 200, indicating a successful response. + + :return: A tuple containing a JSON response and a status code. The JSON response includes keys `isError`, + `message`, `statusCode`, and `data`, with values indicating the success of the operation and the + data that was received and added to the `music_file_list`. The status code returned is 200, + indicating a successful response. """ record = json.loads(request.data) app.logger.info(record) app.logger.info(record["lista_slow"]) app.logger.info(record["UUID"]) app.logger.info(record["dlugosc_plejlisty"]) - return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True) + return_data = wyszukaj( + record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True + ) return_data = ( jsonify(isError=False, message="Success", statusCode=200, data=return_data), @@ -346,23 +385,26 @@ def look_for_playlist(): ) return return_data + @app.route("/add_to_priority", methods=["POST"]) def add_to_priority(): """ - The function `update_music_list` receives a POST request with a JSON payload, logs the received + The function `add_to_priority` receives a POST request with a JSON payload, logs the received item, adds it to a music file list, and returns a success message along with the updated record. - :return: The function `update_music_list` is returning a tuple containing a JSON response and a - status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`, - with values indicating the success of the operation and the data that was - received and added to the `music_file_list`. - The status code returned is 200, indicating a successful response. + + :return: A tuple containing a JSON response and a status code. The JSON response includes keys `isError`, + `message`, `statusCode`, and `data`, with values indicating the success of the operation and the + data that was received and added to the `music_file_list`. The status code returned is 200, + indicating a successful response. """ record = json.loads(request.data) app.logger.info(record) app.logger.info(record["lista_slow"]) app.logger.info(record["UUID"]) app.logger.info(record["dlugosc_plejlisty"]) - return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False) + return_data = wyszukaj( + record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False + ) return_data = ( jsonify(isError=False, message="Success", statusCode=200, data=return_data), @@ -382,8 +424,18 @@ def flask_debug(): def waitress_run(): """ - The `waitress_run` function serves the `app` on host "0.0.0.0" + The `waitress_run` function serves the `app` on host HOST_ADDRESS and port 5000 using the Waitress WSGI server. + + This function starts the Waitress server and listens for incoming requests + on the specified host and port. It uses the `serve` function from the Waitress + library to handle the serving process. + + Parameters: + None + + Returns: + None """ serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS) @@ -407,7 +459,7 @@ if __name__ == "__main__": rescan() logger.info("Started") threads = [] - #threads.append(threading.Thread(target=flask_debug)) + # threads.append(threading.Thread(target=flask_debug)) threads.append(threading.Thread(target=waitress_run)) threads.append(threading.Thread(target=thread_rescan)) diff --git a/conjurer_musician/radio_conjurer.liq b/conjurer_musician/radio_conjurer.liq index a5afcd5..6ab2f70 100644 --- a/conjurer_musician/radio_conjurer.liq +++ b/conjurer_musician/radio_conjurer.liq @@ -1,48 +1,71 @@ +# FILEPATH: /home/mtuszowski/conjurer/conjurer_musician/radio_conjurer.liq + +# This script sets up a Liquidsoap radio stream with various features and configurations. + +# Load icecast credentials from a JSON file let json.parse credentials = file.contents("/home/pi/Conjurer/icecast_credentials.json") + +# Enable replaygain metadata processing enable_replaygain_metadata() +# Define playlists to be used in the stream s1 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/all_playlist.playlist")) s2 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/priority_queue.playlist")) +s3 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/hit.playlist")) -s = random(id="randomizer", weights=[7, 10], [s2, s1]) +# Randomly select a playlist with weights +s = random(id="randomizer", weights=[1, 3, 2], [s1, s2, s3]) +# Set up an interactive harbor for controlling the stream interactive.harbor(port = 9999) -f = interactive.float("f", description="Frequency", min=0., max=1000.,unit="Hz", 200.) -g = interactive.float("g", description="Gain", min=0., max=20.,unit="dB", 8.) + +# Set up interactive controls for bass boost +f = interactive.float("f", description="Frequency", min=0., max=1000., unit="Hz", 200.) +g = interactive.float("g", description="Gain", min=0., max=20., unit="dB", 8.) b = bass_boost(frequency=f, gain=g, s) s = add([s, b]) + +# Set up interactive control for main volume a = interactive.float("main_volume", min=0., max=20., 1.) s = compress.multiband.interactive(bands=3, s) -s = nrj(normalize (s)) +# Apply audio processing effects +s = nrj(normalize(s)) s = crossfade(fade_out=3., fade_in=3., duration=5., s) -s = amplify(a,s) +s = amplify(a, s) +# Skip blank sections in the stream s = blank.skip(max_blank=2., s) + +# Configure logging settings log_to_stdout = true log_to_file = true logpath = "/home/pi/Conjurer/radio_log.log" loglevel = 4 -set("log.stdout",log_to_stdout) -set("log.level",loglevel) +set("log.stdout", log_to_stdout) +set("log.level", loglevel) -# Enable logging to file. See variable: logpath -set("log.file",log_to_file) -set("log.file.path",logpath) +# Enable logging to file +set("log.file", log_to_file) +set("log.file.path", logpath) +# Set up emergency fallback track emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3") -radio = fallback(id="switcher",track_sensitive = false, [s, emergency]) +# Create a fallback stream with the main stream and emergency track +radio = fallback(id="switcher", track_sensitive=false, [s, emergency]) + +# Handle metadata events handle_metadata = fun (m) -> begin log.info(m["title"]) log.info(m["artist"]) log.info(m["filename"]) end radio.on_metadata(handle_metadata) + +# Enable persistent script parameters interactive.persistent("script.params") - -# Enable logging on Standard Output and set logging level. - -output.file.hls("/tmp/hls",[("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))],radio) -output.icecast(%mp3, host="retropie", port=8000,password=credentials.password, icy_metadata="true", mount="mp3-stream", radio) +# Configure output formats and destinations +output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio) +output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio) diff --git a/install_music_bot.sh b/install_music_bot.sh index 5a000dc..b708d18 100755 --- a/install_music_bot.sh +++ b/install_music_bot.sh @@ -8,9 +8,14 @@ cp /home/pi/conjurer/conjurer_musician/requirements_musician.txt /home/pi/Conjur cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/ cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/ touch /home/pi/Conjurer/discord_mus_service.log -touch /home/pi/Conjurer/all_playlist.txt + touch /home/pi/Conjurer/playlist.json +touch /home/pi/Conjurer/prio_playlist.json + +touch /home/pi/Conjurer/all_playlist.playlist touch /home/pi/Conjurer/priority_queue.playlist +touch /home/pi/Conjurer/hit.playlist + touch /home/pi/Conjurer/radio_log.log ./env/bin/python3 -m pip install --upgrade pip ./env/bin/python3 -m pip install -r requirements_musician.txt