From a8797df45c67f743609db01f1e2747912d6499b1 Mon Sep 17 00:00:00 2001 From: Polish Hammer Date: Thu, 30 Oct 2025 16:59:00 +0100 Subject: [PATCH] Tag: 1.5 MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Intermediate commits (oldest → newest): - Hit list queue for radio - fix - Documentation of conjurer_librarian - Reformatting of conjurer_librarian - Documentation of conjurer_musician - Formatting of conjurer_musician - Documentation of radio_conjurer.liq - Merge branch 'main' of https://github.com/migatu/conjurer - Removal from tracking - Fixing librarian result (#6) - Potential bugfix for some lazycoding in spotipy... Another one. fuck it - Music tunneling (#7) - Even more logging in spotify download - test requests - Fine tuning - Optimization bug fix #1 - Fix crossfade - Crossfadeing ctnd - Fix - Starting work on librarian headless mode --- .gitignore | 4 + bot.py | 298 ++++++---- cogs/music.py | 46 ++ communication_subroutine.py | 24 + conjurer_librarian/conjurer_librarian.py | 531 ++++++++++++------ .../install_librarian_service.sh | 40 +- conjurer_librarian/requirements_librarian.txt | 2 +- conjurer_librarian/scrape_bot.py | 107 ++-- conjurer_librarian/search_bot.py | 181 ++++-- conjurer_librarian/test_send.py | 45 ++ conjurer_musician/conjurer_musician.py | 192 ++++--- conjurer_musician/radio_conjurer.liq | 178 ++++-- conjurer_musician/requirements_musician.txt | 3 +- cr_results.json | 0 install_music_bot.sh | 10 +- not_in_db.json | 0 rr_results.json | 0 s_results.json | 0 settings.json | 7 + spotify_dl/spotify.py | 2 + 20 files changed, 1161 insertions(+), 509 deletions(-) create mode 100644 cogs/music.py create mode 100644 conjurer_librarian/test_send.py delete mode 100644 cr_results.json delete mode 100644 not_in_db.json delete mode 100644 rr_results.json delete mode 100644 s_results.json diff --git a/.gitignore b/.gitignore index 78099cb..15c6626 100644 --- a/.gitignore +++ b/.gitignore @@ -197,3 +197,7 @@ node_modules/prettier/esm/parser-postcss.mjs node_modules/prettier/esm/parser-typescript.mjs node_modules/prettier/esm/parser-yaml.mjs node_modules/prettier/esm/standalone.mjs +cr_results.json +not_in_db.json +rr_results.json +s_results.json diff --git a/bot.py b/bot.py index 3c6167c..9dadfad 100644 --- a/bot.py +++ b/bot.py @@ -44,7 +44,7 @@ import yt_dlp from spotify_dl import spotify from spotify_dl import youtube as youtube_download -from communication_subroutine import comm_subroutine, IN_COMM_Q, OUT_COMM_Q +from communication_subroutine import comm_subroutine, IN_COMM_Q, OUT_COMM_Q, QueryControl # *=========================== Classes definition @@ -62,22 +62,6 @@ Music_Config = TypedDict( ) -class QueryControl: - """ - This class `QueryControl` is used to manage queries with information about the author, UUID, - content, logger, context, and replies. - """ - def __init__(self, query_author, query_uuid, query_content, uplogger, ctx) -> None: - self.author = query_author - self.uuid = query_uuid - self.content = query_content - self.logger = uplogger - self.stop = False - self.ctx = ctx - self.logger.info( - f"Created Query control for {self.author}, {self.uuid}: {self.content}" - ) - self.replies = [] class MusicFileList(object): """ @@ -131,43 +115,6 @@ class MusicFileList(object): post_data = {"item": str(item)} requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360) -class DoSearchView(discord.ui.View): - def __init__(self, _uuid, result_list, *args, **kwargs) -> None: - super().__init__(*args, **kwargs) - self.uuid = _uuid - self.result_list = result_list - - @discord.ui.button( - label="Nastepna strona", style=discord.ButtonStyle.primary, row=0 - ) - async def next_callback(self, interaction, button): - # emoji = await ctx.guild.fetch_emoji(id) - # button.emoji=1102174413887111189 - logger.info("Pressed") - await interaction.response.edit_message( - content="You clicked the button!" - ) # Send a message when the button is clicked - - @discord.ui.button(label="Wyszukaj wiecej", style=discord.ButtonStyle.danger, row=0) - async def reset_callback(self, interaction, button): - # emoji = await ctx.guild.fetch_emoji(id) - # button.emoji=1102174413887111189 - logger.info("Pressed") - await interaction.response.edit_message( - content="You clicked the button!" - ) # Send a message when the button is clicked - - @discord.ui.button( - label="Poprzednia Strona", style=discord.ButtonStyle.primary, row=0 - ) - async def perv_callback(self, interaction, button): - # emoji = await ctx.guild.fetch_emoji(id) - # button.emoji=1102174413887111189 - logger.info("Pressed") - await interaction.response.edit_message( - content="You clicked the button!" - ) # Send a message when the button is clicked - # *=========================================== Predefines formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s") @@ -192,6 +139,7 @@ GET_MP3 = "/mp3" SEND_MP3 = "/update_mp3" GET_PLAYLIST = "/get_music" ADD_TO_PRIO_PLAYLIST = "/add_to_priority" +REQUEST_MUSIC = "/request_radio_file" CLEAR_PRIO = "/clear_pr_pls" LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001" SEND_QUERY = "/query" @@ -351,9 +299,6 @@ async def on_ready(): @client.event -# trunk-ignore(pylint/R0915) -# trunk-ignore(pylint/R0914) -# trunk-ignore(pylint/R0912) async def on_message(message): """ Handle incoming messages in a Discord server, perform various @@ -607,8 +552,6 @@ async def disconnect(ctx): await voice_client.disconnect() -# trunk-ignore(pylint/R0915) -# trunk-ignore(pylint/R0912) async def play(ctx, zamawial=None, arg=None): """ Play a music file, retrieve metadata about the file, and send a @@ -728,31 +671,41 @@ async def check_music(): await asyncio.sleep(2) @tasks.loop(seconds=3) -# trunk-ignore(pylint/R0915) async def check_data_q(): + """ + This function checks the data queue for any new entries and processes them accordingly. + """ try: - data = IN_COMM_Q.get(block=False) + fresh_data = IN_COMM_Q.get(block=False) entries = [] - if data.stop: - searcher = data.author - ctx = data.ctx - query = data.content + if fresh_data.stop: + searcher = fresh_data.author + query = fresh_data.content l_p = 1 - for doi in data.entries: + for doi in fresh_data.entries: logger.info(doi) - desc = data.entries[doi] + desc = fresh_data.entries[doi] title = desc['Title'][0] entries.append(f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n") l_p+=1 message = '*Z podłogi wysuwa się winda na książki*' - message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query}' + if fresh_data.ctx is not None: + ctx = fresh_data.ctx + message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query} nr {fresh_data.uuid}' + else: + ctx = client.get_channel(1062047571557744721) + message += f' Mam wynik dla zapytania {fresh_data.uuid} ale że Hammer coś grzebał nie pamiętam kto to chciał i co chciał' + + message += 'nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku ' if len(entries)<1: message += "niestety nie ma nic" + await ctx.send(message) elif len(entries)>=1 and len(entries)<5: message += " znajduje się coś:\n" for item in entries: message+=item + await ctx.send(message) else: message+= " znajduje się cholernie dużo:\n" for item in entries: @@ -761,20 +714,17 @@ async def check_data_q(): await ctx.send(message) message = "" - await ctx.send(message) #Kept for sentimental reasons #await ctx.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}") except Empty: pass @tasks.loop(seconds=120) -# trunk-ignore(pylint/R0915) async def check_self(): """ The function `check_data` periodically checks for conditions to send messages and manage log files in a Discord channel. """ - # trunk-ignore(codespell/misspelled) # logger.info("Heartbeat of cleanup proc") channel = client.get_channel(1062047367337095268) messages = [message async for message in channel.history(limit=1)] @@ -837,9 +787,7 @@ async def check_self(): await channel.send( "*Przeciąga się za barem* No dobra - porobione to można dalej pi... *Zauważa spojrzenie Hammera* eprzyć o głupotach z klientami... Szefie... *Bierze 'ukradkowy' łyk z piersiówki*" ) - # trunk-ignore(pylint/W0603) global LAST_SPONTANEOUS_CALL - # trunk-ignore(pylint/W0603) global TIME_BETWEEN_CALLS tdelta = datetime.now() - LAST_SPONTANEOUS_CALL tdelta = tdelta.total_seconds() @@ -875,7 +823,6 @@ async def get_random_cyclic_message(): mess_key = list(cyclic_words.keys())[messnum] return cyclic_words[mess_key][0] ai_check2 = random.randint(0, 10) - # trunk-ignore(pylint/W0603) global MESSAGE_TABLE if ai_check2 < 6: logger.info("Dykteryjka") @@ -909,10 +856,6 @@ async def get_random_cyclic_message(): return result -# trunk-ignore(pylint/R0912) -# trunk-ignore(pylint/R0913) -# trunk-ignore(pylint/R0914) -# trunk-ignore(pylint/R0915) async def handle_response( prompt, vykidailo, bartender, history, username, request_type ): @@ -1101,8 +1044,6 @@ async def handle_response( return result, MESSAGE_TABLE -# trunk-ignore(pylint/R0915) -# trunk-ignore(pylint/R0914) async def get_file(ctx, source, link): """ Take in three parameters: "ctx" (context), @@ -1124,11 +1065,15 @@ async def get_file(ctx, source, link): if source == "Spotify": dir_path = None item_type, item_id = spotify.parse_spotify_url(link) - if item_id: - coro = asyncio.to_thread(spotify.fetch_tracks, args=(spotify_ctrl, item_type, link)) - file_list = await coro - coro = asyncio.to_thread(spotify.get_item_name, args=(spotify_ctrl, item_type, item_id)) - directory_name = await coro + if item_type in ["album", "track", "playlist"] and item_id: + logger.info(item_type) + logger.info(item_id) + logger.info("Spotify link provided") + file_list = await asyncio.to_thread(spotify.fetch_tracks, spotify_ctrl, item_type, link) + logger.info(file_list) + directory_name = await asyncio.to_thread(spotify.get_item_name, spotify_ctrl, item_type, item_id) + logger.info(directory_name) + logger.info("Spotify scrape done") url_data = {"urls": []} url_dict = {} url_dict["save_path"] = Path( @@ -1138,6 +1083,7 @@ async def get_file(ctx, source, link): url_dict["songs"] = file_list url_data["urls"].append(url_dict.copy()) file_name_f = youtube_download.default_filename + logger.info("YT-DL") coro = asyncio.to_thread( youtube_download.download_songs, songs=url_data, @@ -1153,6 +1099,7 @@ async def get_file(ctx, source, link): proxy="", ) _ = await coro + logger.info("YT-DL done") dir_path = (url_dict["save_path"].resolve()).as_posix() if platform == "win32": dir_path = dir_path.replace("/", "\\") @@ -1323,9 +1270,6 @@ async def get_stats(ctx, history_limit): yield name, how_many -# trunk-ignore(pylint/R0915) -# trunk-ignore(pylint/R0912) -# trunk-ignore(pylint/R0914) async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None): """ Take in a context object and an optional integer parameter "how_many" and perform search @@ -1485,8 +1429,6 @@ async def dej_co_z_jutuba(ctx): description="Jeśli nie wiesz jak użyć tej komendy to nawet nie próbuj", guild=discord.Object(id=664789470779932693), ) -# trunk-ignore(pylint/R0914) -# trunk-ignore(pylint/R0912) async def update_banlist(ctx): """ Update a banlist in a Discord guild from preprovided list in channel. @@ -1638,6 +1580,40 @@ async def graj_muzyko(ctx): logger.info("Press play on tape completed") check_music.start() +@client.hybrid_command( + name="radio_hardkor", + description="Włącza radio Conjurer na kanale #nocna-zmiana.", + guild=discord.Object(id=664789470779932693), +) +async def radio_hardkor(ctx): + """ + Plays the radio hardkor stream in the voice channel. + + Args: + ctx: The context object representing the invocation context. + + Returns: + None + """ + logger.info("Press play on radio hardkor") + async with ctx.typing(): + global MUZYKA # pylint: disable=global-variable-not-assigned + MUZYKA["ctx"] = ctx + if not client.voice_clients: + await connect(ctx=ctx) + voice_client = client.voice_clients[0] + voice_client.play( + discord.PCMVolumeTransformer( + discord.FFmpegPCMAudio("http://95.175.16.246:666/mp3-stream"), volume=0.3 + ) + ) + logger.info("Press play on radio completed") + else: + logger.error("Already playing") + await asyncio.sleep(12) + check_music.start() + + @client.hybrid_command( name="cisza", @@ -1813,7 +1789,36 @@ async def dodaj_do_ulubionych(ctx): timeout=360, ) _ = await coroutine +@client.hybrid_command( + name="ja_chciol", + description="Dodaje do listy ulubionych w radiu", + guild=discord.Object(id=664789470779932693), +) +async def request_radio(ctx): + """ + Generate a playlist in queue. First word in this command shall be int defining length of the playlist. + Rest of the line are search terms. + :param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It + represents the context in which the command was invoked, including information such as the message, + the channel, the server, and the user who invoked the command. This parameter is required in all + Discord.py commands + """ + async with ctx.typing(): + logger.info("Zaczynam szukać timestamp %s", datetime.now()) + + word_list = ctx.message.content.split() + jrequest = { + "lista_slow": word_list, + "UUID": str(uuid.uuid4()), + } + coroutine = asyncio.to_thread( + requests.post, + f"{FILE_SERVICE_ADDRESS}{REQUEST_MUSIC}", + json=jrequest, + timeout=360, + ) + _ = await coroutine @client.hybrid_command( name="wyczysc_ulubione", description="Czysci liste ulubionych w radiu", @@ -1831,9 +1836,8 @@ async def wyczysc_ulubione(ctx): """ async with ctx.typing(): coroutine = asyncio.to_thread( - requests.post, + requests.get, f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}", - json={}, timeout=360, ) _ = await coroutine @@ -2058,8 +2062,6 @@ async def get_image_stable_diffusion(ctx): description="Gra muzyke wyszukana na bazie tematow konwersacji na kanale NZ", guild=discord.Object(id=664789470779932693), ) -# trunk-ignore(pylint/R0912) -# trunk-ignore(pylint/R0915) async def zagraj_muzyke_mego_ludu(ctx): """ Play completeley random playlist based on messaging history. @@ -2209,11 +2211,20 @@ async def parametry_muzyki_mego_ludu( guild=discord.Object(id=664789470779932693), ) async def wyszukaj_linki_do_dokumentow(ctx): + """ + The function `wyszukaj_linki_do_dokumentow` searches for links to documents in a crossref database and provides links to scihub. + + :param ctx: The `ctx` parameter is typically used in Discord.py, a Python library for creating Discord bots. + It represents the context of the command being executed, including information about the message, the server, + and the user who invoked the command. + """ query = ctx.message.content query_uuid = uuid.uuid4() #TODO: TESTING ONLY!! #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}') - json_query = {"UUID": str(query_uuid), "query": str(query)[30:], "page": 1} + ctx.message.content = ctx.message.content.replace("$wyszukaj_linki_do_dokumentow", "") + + json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": False} coroutine = asyncio.to_thread( requests.post, f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}", @@ -2222,8 +2233,8 @@ async def wyszukaj_linki_do_dokumentow(ctx): ) await ctx.send( "*Conjurer powoli notuje podane przez Ciebie. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" + - " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to te 3/4 stacji."+ - " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc te 12/8 stacji teraz będzie ciężko pracować" + " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to 3/4 stacji."+ + " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc 12/8 stacji teraz będzie ciężko pracować" ) query_response = await coroutine if not query_response.status_code == 200: @@ -2245,6 +2256,87 @@ async def wyszukaj_linki_do_dokumentow(ctx): await ctx.send(f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce."+ " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni.") +@client.hybrid_command( + name="glebokie_gardlo", + description="Przygotowuje drinka o nazwie głębokie gardło", + guild=discord.Object(id=664789470779932693), +) +async def wyszukaj_linki_do_dokumentow_deep(ctx): + """ + The function `wyszukaj_linki_do_dokumentow_deep` performs a deep search for links to documents in + a crossref database and provides links to scihub. + + :param ctx: The `ctx` parameter is typically used in Discord.py, a Python library for creating Discord bots. + It represents the context of the command being executed, including information about the message, the server, + and the user who invoked the command. + """ + # TODO: Implement deep search logic here + query = ctx.message.content.replace("$glebokie_gardlo", "") + allowed = False + for role in ctx.message.author.roles: + if role.name == "Bartender": + allowed = True + if role.name == "Scribe": + allowed = True + if role.name == "Thane": + allowed = True + if not allowed: + if ctx.message.author.nick: + username = ctx.message.author.nick + else: + username = ctx.message.author.name + vykidailo = False + bartender = False + prompt = "Przygotuj mi drinka o nazwie Głębokie Gardło, inspirowanego tym sławnym filmem oraz skandalem Watergate" + for role in ctx.message.author.roles: + if role.name == "Vykidailo": + vykidailo = True + if role.name == "Bartender": + bartender = True + global MESSAGE_TABLE # pylint: disable=global-statement + + result, MESSAGE_TABLE = await handle_response( + prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION" + ) + await ctx.send(result) + return + + query_uuid = uuid.uuid4() + #TODO: TESTING ONLY!! + #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}') + json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": True} + coroutine = asyncio.to_thread( + requests.post, + f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}", + json=json_query, + timeout=360, + ) + await ctx.send( + "*Conjurer mruga okiem i zamiast drinka wyjmuje dysk usb z terminala. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" + + " wrzuca ten dysk do środk* To teraz trza poczekać kilka godzin. TO będzie głębokie wyszukanie"+ + " Rozgląda się dookoła i stawia przed tobą mętnego drinka o zapachu mięty, gwiezdnego pyłu i oleju silnikowego" + ) + query_response = await coroutine + if not query_response.status_code == 200: + await ctx.send("*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."+ + " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało") + return + + query, query_uuid, queue_size = ( + query_response.json()["data"][0], + query_response.json()["data"][1], + query_response.json()["data"][2], + ) + if ctx.message.author.nick: + username = ctx.message.author.nick + else: + username = ctx.message.author.name + query_object = QueryControl(username, query_uuid, query, logger, ctx) + OUT_COMM_Q.put(query_object) + await ctx.send(f"Wypij wypi {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce niestety kolejka obowiazuje zawsze."+ + " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzin - ale mogą być też dni. Głębokie zabawy trwają dłużej. *Znowu mruga*") + + @client.hybrid_command( name="krecimy_pornola", @@ -2289,16 +2381,16 @@ if __name__ == "__main__": threads.append(threading.Thread(target=client.run, args=(TOKEN,))) threads.append(threading.Thread(target=comm_subroutine, args=(logger,))) logger.info("Starting discord bot: Starting threads") - i = 0 + WRK_CNT = 0 for worker in threads: - i+=1 - logger.info("Starting discord bot: Starting thread {i}") + WRK_CNT+=1 + logger.info("Starting discord bot: Starting thread %s", WRK_CNT) worker.start() logger.info("Starting discord bot: Joining threads") - i=0 + WRK_CNT=0 for worker in threads: - i+=1 - logger.info("Starting discord bot: Joining thread {i}") + WRK_CNT+=1 + logger.info("Starting discord bot: Joining thread %s", WRK_CNT) worker.join() # widok - generuje się 5 stron po 4 linki diff --git a/cogs/music.py b/cogs/music.py new file mode 100644 index 0000000..8d77915 --- /dev/null +++ b/cogs/music.py @@ -0,0 +1,46 @@ +import discord +from discord.ext import commands +from typing import Optional + +class Music(commands.Cog): + def __init__(self, bot): + self.bot = bot # This is so you can access Bot instance in your cog + +# You must have this function for `bot.load_extension` to call +def setup(bot): + bot.add_cog(Music(bot)) + +@commands.hybrid_command( + name="przytul", description="Przytul kogoś - daj mention po komendzie :)" +) +async def przytul(ctx, arg: Optional[discord.Member] = None): + """ + Generate a text about hugging mentioned user. + + :param ctx: ctx stands for "context" and is a required parameter in Discord.py commands. It + represents the context in which the command was invoked, including information such as the message, + the channel, the server, and the user who invoked the command + :param arg: arg is a parameter of the function "przytul" that expects a Discord member object. The + parameter is optional, meaning that if no member object is provided, it will default to None + :type arg: Optional[discord.Member] + """ + async with ctx.typing(): + nieprzytulac = False + for mention in ctx.message.mentions: + for role in mention.roles: + if role.name == "NIEPRZYTULAĆ!": + nieprzytulac = True + + if arg and nieprzytulac: + await ctx.send( + f"Żebym ja Ciebie nie przytulił {ctx.message.author.mention}" + ) + elif arg: + await ctx.send( + # trunk-ignore(codespell/misspelled) + f"Już dobrze.... Już dobrze... Ojej.. Biedactwo... :( *W ułamku sekundy {arg.mention} znajduje sie w duszącym uścisku. Żebra trzeszczą - kilka pęka. Pacnięcia po plecach grożą odbiciem nerek, a głaskanie po głowie powoduje wstrząs mózgu*" + ) + else: + await ctx.send( + "Kogo mam przytulić? *Wyłamuje kostki i przeciąga się - jego 200 kilowa sylwetka złożona z samych mięśni świadczy o tym że jest gotowy*" + ) diff --git a/communication_subroutine.py b/communication_subroutine.py index 4b53099..d9a648b 100644 --- a/communication_subroutine.py +++ b/communication_subroutine.py @@ -13,6 +13,22 @@ IN_COMM_Q = Queue() awaiting_q = [] incoming_q = Queue() app = Flask(__name__) +class QueryControl: + """ + This class `QueryControl` is used to manage queries with information about the author, UUID, + content, logger, context, and replies. + """ + def __init__(self, query_author, query_uuid, query_content, uplogger, ctx) -> None: + self.author = query_author + self.uuid = query_uuid + self.content = query_content + self.logger = uplogger + self.stop = False + self.ctx = ctx + self.logger.info( + f"Created Query control for {self.author}, {self.uuid}: {self.content}" + ) + self.replies = [] @app.route("/conjurer", methods=["POST"]) def answer_external_command(): @@ -86,8 +102,16 @@ def scan_incoming(_logger): try: answer = incoming_q.get(block=False) _logger.info("DATA FOUND") + record_stored = False for record in awaiting_q: if record.uuid in answer.keys(): + record_stored = True + record.stop = True + record.entries = answer[record.uuid] + IN_COMM_Q.put(record) + if not record_stored: + for key in answer.keys(): + record = QueryControl("Orphaned", key, "Orphan", _logger, None) record.stop = True record.entries = answer[record.uuid] IN_COMM_Q.put(record) diff --git a/conjurer_librarian/conjurer_librarian.py b/conjurer_librarian/conjurer_librarian.py index 9740009..dc4c522 100644 --- a/conjurer_librarian/conjurer_librarian.py +++ b/conjurer_librarian/conjurer_librarian.py @@ -1,243 +1,378 @@ -from logging import handlers -import re +""" +This module contains the implementation of the Librarian class and +related functions for searching and refining queries. + +Classes: +- Librarian: Represents a librarian object that performs search and + refinement operations on queries + +Functions: +- flask_debug: Starts a Flask application in debug mode without using the reloader. +- waitress_run: Serves the Flask application using the Waitress WSGI server. +- BackgroundTaskSearch: Represents a background task for running the + Librarian object asynchronously +""" + +import asyncio +import json import logging +import netrc import threading import time -from flask import Flask, jsonify, request -import netrc -import json -import asyncio from json.decoder import JSONDecodeError -from urllib.request import urlopen +from logging import handlers +from queue import Queue + +import requests +import scrape_bot +import search_bot +from flask import Flask, jsonify, request from habanero import Crossref from waitress import serve -from queue import Queue -import search_bot -import scrape_bot -import requests -NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" +# Constants +NETRC_FILE = r"C:\Users\Activcom.pl\.netrc" HOST_ADDRESS = "192.168.1.192" PORT_ADDRESS = 5001 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" SEND_RESULTS = "/conjurer" -BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147' -MAX_CR_RESULTS = 150000 +BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147" +MAX_CR_RESULTS = 7000 + ENCODING = "utf-8" + app = Flask(__name__) librarian_queue = Queue() librarian_list = [] -class Librarian(object): - def __init__(self, app, query, uuid) -> None: +# trunk-ignore(pylint/R0902) +class Librarian(object): + """ + Represents a librarian object that performs search and refinement operations on queries. + """ + + def __init__(self, _app, query, uuid, _deep_search) -> None: + """ + Initializes a Librarian object. + + Args: + - _app: The Flask application object. + - query: The query to be searched. + - uuid: The unique identifier for the search. + + Attributes: + - cr: The Crossref object for performing the search. + - query: The query to be searched. + - uuid: The unique identifier for the search. + - limit: The maximum number of search results to fetch. + - fetched: The number of search results fetched so far. + - hit: The number of search results that match the refinement criteria. + - total: The total number of search results. + - app: The Flask application object. + - live_results: A list to store live search results. + - final_result: A list to store the final refined search results. + - not_in_db: A list to store search results that are not in the local database. + - search_result_from_cr: A dictionary to store the search results from Crossref. + - done: A flag indicating if the search is done. + """ netrc_mod = netrc.netrc(NETRC_FILE) - authTokens = netrc_mod.authenticators("crossref") - self.cr = Crossref(mailto=authTokens[0]) + auth_tokens = netrc_mod.authenticators("crossref") + self.cr = Crossref( + mailto=auth_tokens[0], + ua_string=f"Conjurer project. mailto:{auth_tokens[0]}" + ) self.query = query self.uuid = str(uuid) self.limit = MAX_CR_RESULTS self.fetched = 0 self.hit = 0 self.total = 0 - self.app = app + self.app = _app self.live_results = [] - self.final_result = [] - self.not_in_db = [] + self.final_result = {} + self.not_in_db = {} self.search_result_from_cr = {} self.done = False + self.deep_search = _deep_search + + async def search_crossref(self, query, deep_search=False): + """ + Performs a search on Crossref for the given query. + + Args: + - query: The query to be searched. + + Returns: + - result: The search result from Crossref. + + Raises: + - None. + """ - async def search_crossref(self, query): self.app.logger.info("STARTED SEARCH") - result = self.cr.works(query = query, limit = 1000) - self.search_result_from_cr.update(result) - self.total = result['message']['total-results'] - self.fetched += len(result['message']['items']) - self.app.logger.info (self.total) - self.app.logger.info(self.fetched) - while self.total > self.fetched and self.limit > self.fetched: - result = self.cr.works(query = query, limit = 1000) - self.search_result_from_cr.update(result) - self.total = result['message']['total-results'] - self.fetched += len(result['message']['items']) - self.app.logger.info (self.total) - self.app.logger.info(self.fetched) - time.sleep(0.1) - self.app.logger.info("CROSSREF DONE") - #TODO: add to sql with UUID - #sql row - title, doi, author, uuid, last retrieved, is_available + if not deep_search: + cr_result = self.cr.works(query=query, limit=1000) + self.search_result_from_cr.update(cr_result) + self.total = cr_result["message"]["total-results"] + self.fetched += len(cr_result["message"]["items"]) + self.app.logger.info(self.total) + self.app.logger.info(self.fetched) + while self.total > self.fetched and self.limit > self.fetched: + tmp_result = self.cr.works(query=query, limit=1000, offset=self.fetched) + cr_result["message"]["items"].extend(tmp_result["message"]["items"]) + self.total = tmp_result["message"]["total-results"] + self.fetched = len(cr_result["message"]["items"]) + self.app.logger.info(self.total) + self.app.logger.info(self.fetched) + time.sleep(0.1) + + else: + cr_result = self.cr.works(query=query, cursor_max=10000, cursor='*', progress_bar = True) + result = cr_result[0] + for item in cr_result[1:]: + result["message"]["items"].extend(item["message"]["items"]) + self.total = item["message"]["total-results"] + self.fetched = len(result["message"]["items"]) + self.app.logger.info(self.total) + self.app.logger.info(self.fetched) + cr_result = result + self.app.logger.info("Total, fetched:") + self.app.logger.info(self.total) + self.app.logger.info(self.fetched) + self.search_result_from_cr.update(cr_result) + self.total = cr_result["message"]["total-results"] + self.app.logger.info("CROSSREF DONE") + + self.app.logger.info("CROSSREF DONE") + with open("cr_results.json", "r+", encoding="utf-8") as data_file: + # First we load existing data into a dict. + try: + file_data = json.load(data_file) + except JSONDecodeError: + file_data = {} + data_file.truncate(0) + data_file.seek(0) + tmp = {self.uuid : self.search_result_from_cr} + if file_data: + file_data.update(tmp) + else: + file_data = tmp + json.dump(file_data, data_file, indent=4) + return cr_result + + + async def refine_search(self, unrefined_result): + """ + Refines the search query based on the unrefined search result. + + Args: + - unrefined_result: The unrefined search result. + + Returns: + - refined_result: The refined search result. + + Raises: + - None. + """ + summarized_results = [] + self.app.logger.info("REFINE: Removing all derived works from the list") + for item in unrefined_result["message"]["items"]: + summarized_results.append( + { + "DOI": item["DOI"], + "title": item["title"] if "title" in item else None, + "type": item["type"] if "type" in item else None, + } + ) + + partial_result = { + self.uuid: { + "total_results": self.search_result_from_cr["message"][ + "total-results" + ], + "on_page": 1, + "summary": summarized_results, + "results": self.search_result_from_cr["message"]["items"], + } + } + self.app.logger.info("REFINE: Dumping to file") + temp = [] + refined_result = {} + for key in partial_result: + self.app.logger.info("KEY:") + self.app.logger.info(key) + + for item in partial_result[self.uuid]["summary"]: + if item["title"]: + temp.append(item) + + for item in temp: + refined_result[item["DOI"]]= item with open("rr_results.json", "r+", encoding="utf-8") as data_file: # First we load existing data into a dict. try: file_data = json.load(data_file) except JSONDecodeError: file_data = {} - summarized_results = [] - self.app.logger.info("REFINE: Removing all derived works from the list") - for item in self.search_result_from_cr['message']['items']: - container = "NO" - if "container-title" in item: - container = "YES" - else: - summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container}) - result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}} - if file_data: - file_data.update(result) - else: - file_data = result - - self.app.logger.info("REFINE: Dumping to file") data_file.truncate(0) data_file.seek(0) + tmp = {self.uuid: refined_result} + if file_data: + file_data.update(tmp) + else: + file_data = tmp json.dump(file_data, data_file, indent=4) - return result + return refined_result - async def check_dois_in_local_db(self, doi): - self.app.logger.info("REFINE: Running search in the backend app") - coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger) - result = await coro - return result + async def check_if_exists(self, refined_result): + """ + Checks if the given DOI exists. - async def check_if_exists_brute_force(self, page_url): - self.app.logger.error("REFINE: Brute force search!") - if not page_url.startswith(("http:", "https:")): - raise ValueError("URL must start with 'http:' or 'https:'") - # trunk-ignore(bandit/B310) - with urlopen(page_url) as response: - data = response.read() - text = data.decode("utf-8") - for line in text.splitlines(): - if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line): - return False - if m := re.match(r".* (doi, True) -def consumer(in_q, control_q, doi, live_results, logger): - sentinels = 0 + + +# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK +def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger): + """ + Consumes items from an input queue and checks if DOI exists in the live results list. + + Args: + in_q (Queue): Input queue. + control_q (Queue): Control queue. + doi (list): List of DOI to search for. + live_results (list): List to store the search results. + _logger: Logger object for logging. + """ + _logger.info("Consumer thread started:%s", no) for item in doi: - result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]}) + result_list.append({"DOI": item[0], "exists": False, "data": item[1]}) while True: done_check = True - data = in_q.get() - if data is _sentinel: - logger.info("Worker finished") - sentinels += 1 - else: - for item in result_list: - if item["DOI"] in data and not item["exists"]: - logger.info(data) - logger.info("HIT") - item["exists"] = True - live_results.append(item) - done_check = done_check and item["exists"] - if done_check: - control_q.put(_sentinel) - if sentinels >= MAXTHREADS: - logger.info("All workers finished") + try: + data = in_q.get(block=False) + if data is _sentinel: + _logger.info("Worker finished") + control_dict["sentinels"] += 1 + else: + for item in result_list: + if item["DOI"] in data and not item["exists"]: + _logger.info(data) + _logger.info("HIT") + item["exists"] = True + live_results.append(item) + done_check = done_check and item["exists"] + if done_check: + control_q.put(_sentinel) + except Empty: + pass + if control_dict["sentinels"] >= MAXTHREADS: + _logger.info("All workers finished") break -def search_for_doi(doi, live_results, logger): + +def search_for_doi(doi, live_results, _logger): + """ + Search for DOI in live_results using _logger for logging. + + Args: + doi (list): List of DOI to search for. + live_results (list): List to store the search results. + _logger: Logger object for logging. + """ + control_dict = {"sentinels":0} + result_list = [] threads = [] work_q = Queue(maxsize=WORK_Q_SIZE) control_q = Queue() - t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger)) - logger.info("Consumer thread created") - threads.append(t_cons) - for i in range(0,MAXTHREADS): - #TODO: TEST DATA - #filename = "test" + str(i) + CHUNk - #TODO: DEPLOYMENT DATA - filename = str(i) + CHUNk - logger.info(f"Creating worker thread no: {i}") - threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger))) + for i in range (0, (len(doi)//1000)+2): + t_cons = Thread( + target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, i, _logger) + ) + _logger.info("Consumer thread created") + threads.append(t_cons) + for i in range(0, MAXTHREADS): + # TEST DATA + # filename = "test" + str(i) + CHUNk + # DEPLOYMENT DATA + filename = str(i) + CHUNK + _logger.info(f"Creating worker thread no: {i}") + threads.append( + Thread(target=producer, args=(work_q, control_q, filename, _logger)) + ) for worker in threads: worker.start() for worker in threads: worker.join() return result_list + if __name__ == "__main__": import logging + logger = logging.getLogger() logger.setLevel("DEBUG") h1 = logging.StreamHandler() logger.addHandler(h1) logger.info("TEST RUN") live_result = [] - logger.info(search_for_doi([ - ("10.1002/9781118786352.wbieg0998.pub2","DATA"), - ("10.1002/j.2050-0416.2002.tb00563.x","DATA"), - ("10.1111/j.1365-2958.1994.tb00448.x","DATA"), - ("10.2165/00128415-200309690-00017","DATA"), - ("10.5772/48313","DATA"), - ("10.15803/ijnc.7.2_419","DATA"), - ("10.1051/0004-6361/201321596e","DATA"), - ("10.2307/40835941","DATA"), - ], live_result, logger)) + logger.info( + search_for_doi( + [ + ("10.1002/9781118786352.wbieg0998.pub2", "DATA"), + ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"), + ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"), + ("10.2165/00128415-200309690-00017", "DATA"), + ("10.5772/48313", "DATA"), + ("10.15803/ijnc.7.2_419", "DATA"), + ("10.1051/0004-6361/201321596e", "DATA"), + ("10.2307/40835941", "DATA"), + ], + live_result, + logger, + ) + ) logger.info(live_result) diff --git a/conjurer_librarian/test_send.py b/conjurer_librarian/test_send.py new file mode 100644 index 0000000..eea519f --- /dev/null +++ b/conjurer_librarian/test_send.py @@ -0,0 +1,45 @@ +import asyncio +import json +import logging +from json.decoder import JSONDecodeError +from logging import handlers +import requests + +MAIN_BOT_ADDRESS = "http://192.168.1.191:5000" +SEND_RESULTS = "/conjurer" + +async def send_results(): + logger = logging.getLogger() + logger.setLevel(logging.DEBUG) + h1 = handlers.RotatingFileHandler( + filename="D:\\logs\\librarian.log", + encoding="utf-8", + mode="a", + maxBytes=6 * 1024 * 1024, + backupCount=6, + ) + + logger.addHandler(h1) + + with open("s_results.json", "r+", encoding="utf-8") as s_file: + try: + database = json.load(s_file) + except JSONDecodeError: + pass + result = database["4bacbca0-ce3b-45c6-90a1-05c70b46b740"] + result = {"4bacbca0-ce3b-45c6-90a1-05c70b46b740":result} + + coroutine = asyncio.to_thread( + requests.post, + f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}", + json=result, + timeout=360, + ) + logger.info("SENT") + result = await coroutine + logger.info(result.status_code) + logger.info("SEND CONFIRMED") +async def main(): + await send_results() +if __name__ == "__main__": + asyncio.run(main()) \ No newline at end of file diff --git a/conjurer_musician/conjurer_musician.py b/conjurer_musician/conjurer_musician.py index cda0be2..5244f5e 100644 --- a/conjurer_musician/conjurer_musician.py +++ b/conjurer_musician/conjurer_musician.py @@ -35,34 +35,11 @@ if platform in ("linux", "linux2"): LOGSTORE = "/home/pi/RetroPie/logs/" ENCODING = "utf-8" MUSIC_FOLDER = "/home/pi/RetroPie/mp3/" + PRIORITY_FOLDER = "/home/pi/RetroPie/mp3/Magiczne i chuj/" music_file_list = [] - - -def create_playlist(): - """ - Reads a JSON file containing a playlist and writes the playlist items to a text file. - - The JSON file path is '/home/pi/Conjurer/playlist.json', - and the text file path is '/home/pi/Conjurer/all_playlist.playlist'. - - Raises: - JSONDecodeError: If the JSON file cannot be decoded. - - """ - with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file: - with open( - "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8" - ) as w_file: - try: - playlist = json.load(r_file) - for item in playlist: - w_file.write(item) - w_file.write("\n") - except json.JSONDecodeError: - pass - +priority_list = [] def rescan(): """ @@ -70,18 +47,35 @@ def rescan(): and adds them to a list of music files. """ logging.info("Rescan triggered") + for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"): temp_music_file = mp3_item.as_posix() if platform == "win32": temp_music_file = temp_music_file.replace("/", "\\") music_file_list.append(temp_music_file) - with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file: - database = music_file_list - # self.app.logger.info("DUMPING DATA") - s_file.truncate(0) - s_file.seek(0) - json.dump(database, s_file) - create_playlist() + + for mp3_item in Path.glob(Path(PRIORITY_FOLDER), "**/*.mp3"): + temp_music_file = mp3_item.as_posix() + if platform == "win32": + temp_music_file = temp_music_file.replace("/", "\\") + priority_list.append(temp_music_file) + + with open( + "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8" + ) as w_file: + try: + for item in music_file_list: + w_file.write(item) + w_file.write("\n") + except json.JSONDecodeError: + pass + with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file: + try: + for item in priority_list: + w_file.write(item) + w_file.write("\n") + except json.JSONDecodeError: + pass def thread_rescan(): @@ -202,7 +196,9 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True): return_list.append(search_weight[itr]) if not return_to_bot: with open( - "/home/pi/Conjurer/priority_queue.playlist", "r+", encoding="utf-8" + "/home/pi/Conjurer/priority_queue.playlist", + "r+", + encoding="utf-8", ) as s_file: s_file.write(search_weight[itr][1]) break @@ -211,13 +207,6 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True): fun_logger.info("Wiele plików do zagrania") search_weight.sort(key=lambda x: x[0], reverse=True) return_list.extend(search_weight[: int(how_many)]) - if not return_to_bot: - - with open( - "/home/pi/Conjurer/priority_queue.playlist", "a", encoding="utf-8" - ) as s_file: - for item in return_list: - s_file.write(item[1] + "\n") return return_list @@ -252,24 +241,36 @@ def remove_characters(string, character): @app.route("/stream", methods=["GET"]) def stream_music(): """ - The function `get_music_list` returns a JSON object containing a list of music files. - :return: A JSON response containing a key "music_file_list" with the value of the variable - `music_file_list`. + The function `stream_music` is a route handler for the "/stream" endpoint. + It returns a JSON response containing a key "music_file_list" with the value of the variable `music_file_list`. + + :return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`. """ return send_from_directory("/tmp/hls", "stream.m3u8") -@app.route('/') + +@app.route("/") def stream(file_name): - video_dir = '/tmp/hls' + """ + Stream the specified file from the video directory. + + Args: + file_name (str): The name of the file to be streamed. + + Returns: + Response: The response containing the streamed file. + """ + video_dir = "/tmp/hls" return send_from_directory(video_dir, file_name) @app.route("/stream_mp3", methods=["GET"]) def stream_music_mp3(): """ - The function `get_music_list` returns a JSON object containing a list of music files. - :return: A JSON response containing a key "music_file_list" with the value of the variable - `music_file_list`. + The function `stream_music_mp3` is a route handler for the "/stream_mp3" endpoint. + It redirects the user to the URL "http://www.example.com" with a status code of 302. + + :return: A redirect response to "http://www.example.com" with a status code of 302. """ return redirect("http://www.example.com", code=302) @@ -277,19 +278,18 @@ def stream_music_mp3(): @app.route("/clear_pr_pls", methods=["GET"]) def clear_pr_pls(): """ - The function `get_music_list` returns a JSON object containing a list of music files. - :return: A JSON response containing a key "music_file_list" with the value of the variable - `music_file_list`. + The function `clear_pr_pls` clears the contents of the priority queue playlist file. + + :return: A JSON response indicating the success of the operation. """ - with open( - "/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8" - ) as s_file: - s_file.truncate(0) + app.logger.info("CLEARING PLAYLIST") + with open('/home/pi/Conjurer/priority_queue.playlist', 'w', encoding='utf-8') as cleared_pl: + cleared_pl.write('') + return_data = jsonify(isError=False, message="Success", statusCode=200, data=[]) return return_data, 200 - @app.route("/mp3", methods=["GET"]) def get_music_list(): """ @@ -324,20 +324,22 @@ def update_music_list(): @app.route("/get_music", methods=["POST"]) def look_for_playlist(): """ - The function `update_music_list` receives a POST request with a JSON payload, logs the received + The function `look_for_playlist` receives a POST request with a JSON payload, logs the received item, adds it to a music file list, and returns a success message along with the updated record. - :return: The function `update_music_list` is returning a tuple containing a JSON response and a - status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`, - with values indicating the success of the operation and the data that was - received and added to the `music_file_list`. - The status code returned is 200, indicating a successful response. + + :return: A tuple containing a JSON response and a status code. The JSON response includes keys `isError`, + `message`, `statusCode`, and `data`, with values indicating the success of the operation and the + data that was received and added to the `music_file_list`. The status code returned is 200, + indicating a successful response. """ record = json.loads(request.data) app.logger.info(record) app.logger.info(record["lista_slow"]) app.logger.info(record["UUID"]) app.logger.info(record["dlugosc_plejlisty"]) - return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True) + return_data = wyszukaj( + record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True + ) return_data = ( jsonify(isError=False, message="Success", statusCode=200, data=return_data), @@ -345,26 +347,58 @@ def look_for_playlist(): ) return return_data + +@app.route("/request_radio_file", methods=["POST"]) +def add_request(): + record = json.loads(request.data) + app.logger.info(record) + app.logger.info(record["lista_slow"]) + app.logger.info(record["UUID"]) + return_data = wyszukaj( + record["lista_slow"], 0, app.logger, False + ) + + with open( + "/home/pi/Conjurer/request.playlist", "a", encoding="utf-8" + ) as s_file: + for item in return_data: + s_file.write(item[1] + "\n") + return_data = ( + jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}), + 200, + ) + return return_data + + + @app.route("/add_to_priority", methods=["POST"]) def add_to_priority(): """ - The function `update_music_list` receives a POST request with a JSON payload, logs the received + The function `add_to_priority` receives a POST request with a JSON payload, logs the received item, adds it to a music file list, and returns a success message along with the updated record. - :return: The function `update_music_list` is returning a tuple containing a JSON response and a - status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`, - with values indicating the success of the operation and the data that was - received and added to the `music_file_list`. - The status code returned is 200, indicating a successful response. + + :return: A tuple containing a JSON response and a status code. + The JSON response includes keys `isError`, + `message`, `statusCode`, and `data`, with values + indicating the success of the operation and the + data that was received and added to the `music_file_list`. + The status code returned is 200,indicating a successful response. """ record = json.loads(request.data) app.logger.info(record) app.logger.info(record["lista_slow"]) app.logger.info(record["UUID"]) app.logger.info(record["dlugosc_plejlisty"]) - return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False) - + return_data = wyszukaj( + record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False + ) + with open( + "/home/pi/Conjurer/priority_queue.playlist", "a", encoding="utf-8" + ) as s_file: + for item in return_data: + s_file.write(item[1] + "\n") return_data = ( - jsonify(isError=False, message="Success", statusCode=200, data=return_data), + jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}), 200, ) return return_data @@ -381,8 +415,18 @@ def flask_debug(): def waitress_run(): """ - The `waitress_run` function serves the `app` on host "0.0.0.0" + The `waitress_run` function serves the `app` on host HOST_ADDRESS and port 5000 using the Waitress WSGI server. + + This function starts the Waitress server and listens for incoming requests + on the specified host and port. It uses the `serve` function from the Waitress + library to handle the serving process. + + Parameters: + None + + Returns: + None """ serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS) @@ -406,7 +450,7 @@ if __name__ == "__main__": rescan() logger.info("Started") threads = [] - #threads.append(threading.Thread(target=flask_debug)) + # threads.append(threading.Thread(target=flask_debug)) threads.append(threading.Thread(target=waitress_run)) threads.append(threading.Thread(target=thread_rescan)) diff --git a/conjurer_musician/radio_conjurer.liq b/conjurer_musician/radio_conjurer.liq index a5afcd5..71bcced 100644 --- a/conjurer_musician/radio_conjurer.liq +++ b/conjurer_musician/radio_conjurer.liq @@ -1,48 +1,140 @@ +# FILEPATH: /home/mtuszowski/conjurer/conjurer_musician/radio_conjurer.liq + +# This script sets up a Liquidsoap radio stream with various features and configurations. + +# Load icecast credentials from a JSON file let json.parse credentials = file.contents("/home/pi/Conjurer/icecast_credentials.json") -enable_replaygain_metadata() -s1 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/all_playlist.playlist")) -s2 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/priority_queue.playlist")) - -s = random(id="randomizer", weights=[7, 10], [s2, s1]) - -interactive.harbor(port = 9999) -f = interactive.float("f", description="Frequency", min=0., max=1000.,unit="Hz", 200.) -g = interactive.float("g", description="Gain", min=0., max=20.,unit="dB", 8.) -b = bass_boost(frequency=f, gain=g, s) -s = add([s, b]) -a = interactive.float("main_volume", min=0., max=20., 1.) -s = compress.multiband.interactive(bands=3, s) - -s = nrj(normalize (s)) -s = crossfade(fade_out=3., fade_in=3., duration=5., s) -s = amplify(a,s) - -s = blank.skip(max_blank=2., s) -log_to_stdout = true -log_to_file = true -logpath = "/home/pi/Conjurer/radio_log.log" -loglevel = 4 -set("log.stdout",log_to_stdout) -set("log.level",loglevel) - -# Enable logging to file. See variable: logpath -set("log.file",log_to_file) -set("log.file.path",logpath) - -emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3") -radio = fallback(id="switcher",track_sensitive = false, [s, emergency]) - -handle_metadata = fun (m) -> begin - log.info(m["title"]) - log.info(m["artist"]) - log.info(m["filename"]) -end -radio.on_metadata(handle_metadata) -interactive.persistent("script.params") + # Enable replaygain metadata processing + enable_replaygain_metadata() + l = playlog(duration = 14400.0, persistency="/home/pi/Conjurer/persistence.log") + def check(r) + m = request.metadata(r) + if l.last(m) < 3600. then + log.info("Rejecting #{m['filename']} (played #{l.last(m)}s ago).") + false + else + l.add(m) + true + end + end -# Enable logging on Standard Output and set logging level. -output.file.hls("/tmp/hls",[("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))],radio) -output.icecast(%mp3, host="retropie", port=8000,password=credentials.password, icy_metadata="true", mount="mp3-stream", radio) + # Define playlists to be used in the stream + s1 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/all_playlist.playlist")) + s2 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/priority_queue.playlist")) + s3 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/hit.playlist")) + requests_queue = request.queue() + def queue_processing() + f1=file.read("/home/pi/Conjurer/request.playlist") + text = f1() + if text != "" then + requests_queue.push.uri(text) + file.remove("/home/pi/Conjurer/request.playlist") + f = file.open("/home/pi/Conjurer/request.playlist", create=true) + f.close() + log.info("done") + else + log.info("no requests") + end + end + + # Randomly select a playlist with weights + s = random(id="randomizer", weights=[1, 3, 2], [s1, s2, s3]) + #s = delay(duration=5., s) + #radio = add([delay(1800., jingles), music]) + #radio = fallback([delay(1800., jingles), music]) + #p = predicate.signal() + #radio = switch([(p, jingles), ({true}, music)]) + #thread.run(every=1200., {p.signal()}) +# def cmd(_) +# p.signal() +# "Jingle inserted!" +# end +# server.register("insert_jingle", cmd) + + # Set up an interactive harbor for controlling the stream + interactive.harbor(port = 9999) + # Set up interactive controls for bass boost + f = interactive.float("f", description="Frequency", min=0., max=1000., unit="Hz", 200.) + g = interactive.float("g", description="Gain", min=0., max=20., unit="dB", 8.) + b = bass_boost(frequency=f, gain=g, s) + s = add([s, b]) + # Set up interactive control for main volume + a = interactive.float("main_volume", min=0., max=20., 1.) + s = compress.multiband.interactive(bands=3, s) + + # Apply audio processing effects + s = nrj(normalize(s)) + s = amplify(a, s) + + # Skip blank sections in the stream + s = blank.skip(max_blank=2., s) + + # Configure logging settings + log_to_stdout = true + log_to_file = true + logpath = "/home/pi/Conjurer/radio_log.log" + loglevel = 4 + set("log.stdout", log_to_stdout) + set("log.level", loglevel) + + # Enable logging to file + set("log.file", log_to_file) + set("log.file.path", logpath) + + # Set up emergency fallback track + emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3") + + # Create a fallback stream with the main stream and emergency track + cross_s = crossfade(fade_out=3., fade_in=3., duration=5., fallback(id="switcher", track_sensitive=false, [requests_queue, s])) + radio = fallback(id="switcher2", track_sensitive=false, [cross_s, emergency]) + + thread.run(every=60., queue_processing) + + # Handle metadata events + handle_metadata = fun (m) -> begin + log.info(m["title"]) + log.info(m["artist"]) + log.info(m["filename"]) + end + radio.on_metadata(handle_metadata) + + # Enable persistent script parameters + interactive.persistent("script.params") + + # Configure output formats and destinations + output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio) + output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio) +# output.icecast(%opus, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="opus-stream", radio) +# output.icecast(%ogg, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="ogg-stream", radio) +# output.icecast(%aac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="aac-stream", radio) +# output.icecast(%flac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="flac-stream", radio) +# output.icecast(%vorbis, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="vorbis-stream", radio) +# output.icecast(%speex, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="speex-stream", radio) +# output.icecast(%wav, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="wav-stream", radio) +# output.icecast(%pcm, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="pcm-stream", radio) +# output.icecast(%raw, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="raw-stream", radio) +# output.icecast(%s16l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16l-stream", radio) +# output.icecast(%s16b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16b-stream", radio) +# output.icecast(%s24l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24l-stream", radio) +# output.icecast(%s24b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24b-stream", radio) +# output.icecast(%s32l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32l-stream", radio) +# output.icecast(%s32b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32b-stream", radio) +# output.icecast(%s64l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64l-stream", radio) +# output.icecast(%s64b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64b-stream", radio) +# output.icecast(%s128l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128l-stream", radio) +# output.icecast(%s128b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128b-stream", radio) +# output.icecast(%s256l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256l-stream", radio) +# output.icecast(%s256b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256b-stream", radio) +# output.icecast(%s512l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512l-stream", radio) +# output.icecast(%s512b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512b-stream", radio) +# output.icecast(%s1024l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024l-stream", radio) +# output.icecast(%s1024b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024b-stream", radio) +# output.icecast(%s2048l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048l-stream", radio) +# output.icecast(%s2048b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048b-stream", radio) +# output.icecast(%s4096l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096l-stream", radio) +# output.icecast(%s4096b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096b-stream", radio) +# output.icecast(%s8192l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192l-stream", radio) +# output.icecast(%s8192b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192b-stream", radio) diff --git a/conjurer_musician/requirements_musician.txt b/conjurer_musician/requirements_musician.txt index 8642e2b..e298c25 100644 --- a/conjurer_musician/requirements_musician.txt +++ b/conjurer_musician/requirements_musician.txt @@ -1,2 +1,3 @@ flask -waitress \ No newline at end of file +waitress +sshtunnel \ No newline at end of file diff --git a/cr_results.json b/cr_results.json deleted file mode 100644 index e69de29..0000000 diff --git a/install_music_bot.sh b/install_music_bot.sh index 5a000dc..210c6c5 100755 --- a/install_music_bot.sh +++ b/install_music_bot.sh @@ -8,9 +8,17 @@ cp /home/pi/conjurer/conjurer_musician/requirements_musician.txt /home/pi/Conjur cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/ cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/ touch /home/pi/Conjurer/discord_mus_service.log -touch /home/pi/Conjurer/all_playlist.txt + touch /home/pi/Conjurer/playlist.json +touch /home/pi/Conjurer/prio_playlist.json + +touch /home/pi/Conjurer/all_playlist.playlist touch /home/pi/Conjurer/priority_queue.playlist +touch /home/pi/Conjurer/hit.playlist +touch /home/pi/Conjurer/request.playlist +touch /home/pi/Conjurer/persistence.log +echo "[]" > /home/pi/Conjurer/persistence.log + touch /home/pi/Conjurer/radio_log.log ./env/bin/python3 -m pip install --upgrade pip ./env/bin/python3 -m pip install -r requirements_musician.txt diff --git a/not_in_db.json b/not_in_db.json deleted file mode 100644 index e69de29..0000000 diff --git a/rr_results.json b/rr_results.json deleted file mode 100644 index e69de29..0000000 diff --git a/s_results.json b/s_results.json deleted file mode 100644 index e69de29..0000000 diff --git a/settings.json b/settings.json index f423305..497a3c4 100644 --- a/settings.json +++ b/settings.json @@ -204,6 +204,13 @@ false, false ], + "same plusy\\b": [ + "Jak na cmentarzu Szefie. Jak na cmentarzu", + 15.0, + 0, + false, + false + ], "kld\\b": [ "KLD? Mieli tam burdel większy niż u Hammera. Ale o ile Hammer sprzątnął już nawet całkiem nieźle - to tam śmieci wyniosły się same. Uczcijmy ich minutą śmiechu.", 15.0, diff --git a/spotify_dl/spotify.py b/spotify_dl/spotify.py index a3511b5..7969a58 100644 --- a/spotify_dl/spotify.py +++ b/spotify_dl/spotify.py @@ -239,6 +239,8 @@ def get_item_name(sp, item_type, item_id): name = sp.album(album_id=item_id).get("name") elif item_type == "track": name = sp.track(track_id=item_id).get("name") + elif item_type == "artist_top_tracks": + name = sp.artist_top_tracks(artist_id=item_id).get("name") return sanitize(name)