CLock

,

.trunk/configs/.shellcheckrc

@@ -0,0 +1,7 @@
+enable=all
+source-path=SCRIPTDIR
+disable=SC2154
+
+# If you're having issues with shellcheck following source, disable the errors via:
+# disable=SC1090
+# disable=SC1091

.trunk/trunk.yaml

@@ -2,34 +2,38 @@
 # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
 version: 0.1
 cli:
-  version: 1.20.1
+  version: 1.22.2
 # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
 plugins:
   sources:
     - id: trunk
-      ref: v1.4.5
+      ref: v1.6.0
       uri: https://github.com/trunk-io/plugins
 # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
 runtimes:
   enabled:
+    - go@1.21.0
     - node@18.12.1
     - python@3.10.8
 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
 lint:
   enabled:
-    - actionlint@1.6.27
+    - shellcheck@0.10.0
-    - bandit@1.7.8
+    - shfmt@3.6.0
-    - black@24.3.0
+    - taplo@0.8.1
+    - actionlint@1.7.1
+    - bandit@1.7.9
+    - black@24.4.2
     - codespell
-    - checkov@3.2.48
+    - checkov@3.2.139
     - git-diff-check
     - isort@5.13.2
     - pylint
-    - markdownlint@0.39.0
+    - markdownlint@0.41.0
-    - prettier@3.2.5
+    - prettier@3.3.2
-    - ruff@0.3.4
+    - ruff@0.4.9
-    - trivy@0.50.1
+    - trivy@0.52.2
-    - trufflehog@3.71.0
+    - trufflehog@3.78.1
     - yamllint@1.35.1
 actions:
   disabled:

bot.py

@@ -16,6 +16,7 @@ import os
 import random
 import re
 import shutil
+import subprocess
 import sys
 import threading
 import uuid
@@ -41,11 +42,16 @@ from discord.ext import commands, tasks
 from spotipy.oauth2 import SpotifyClientCredentials
 
 import yt_dlp
+from communication_subroutine import (
+    IN_COMM_Q,
+    OUT_COMM_Q,
+    QueryControl,
+    comm_subroutine,
+    PREPPED_TRACKS,
+)
 from spotify_dl import spotify
 from spotify_dl import youtube as youtube_download
 
-from communication_subroutine import comm_subroutine, IN_COMM_Q, OUT_COMM_Q, QueryControl
-
 # *=========================== Classes definition
 
 # The above code is defining a TypedDict called `Music_Config` in Python. This TypedDict has three
@@ -62,7 +68,6 @@ Music_Config = TypedDict(
 )
 
 
-
 class MusicFileList(object):
     """
     The `MusicFileList` class manages a list of music files, with the ability to refresh the list from a
@@ -117,7 +122,6 @@ class MusicFileList(object):
 
 
 # *=========================================== Predefines
-formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 MASTER_TIMEOUT = datetime.now()
 INITIAL_TIME_WAIT = 500
 MUZYKA: Music_Config = {"ctx": None, "queue": [], "requester": []}
@@ -135,10 +139,15 @@ MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE = 15
 MUZYKA_MOJEGO_LUDU_PLAJLISTA = 30
 
 FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000"
+RADIO_HARBOR_ADDRESS = "http://192.168.1.15:54321"
+SKIP_TRACK = "/skip"
+
 GET_MP3 = "/mp3"
 SEND_MP3 = "/update_mp3"
 GET_PLAYLIST = "/get_music"
 ADD_TO_PRIO_PLAYLIST = "/add_to_priority"
+CREATE_PRIO_PLAYLIST = "/create_priority_playlist"
+
 REQUEST_MUSIC = "/request_radio_file"
 CLEAR_PRIO = "/clear_pr_pls"
 LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001"
@@ -233,6 +242,7 @@ handler = handlers.RotatingFileHandler(
     maxBytes=6 * 1024 * 1024,
     backupCount=6,
 )
+formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
 handler.setFormatter(formatter)
 logger.addHandler(handler)
 random.seed()
@@ -278,7 +288,6 @@ for key in word_reactions:
 logger.info("Done")
 
 
-
 # *=========================================== Create Client
 logger.info("Creating discord bot")
 client = commands.Bot(intents=intents, command_prefix="$")
@@ -292,8 +301,13 @@ logger.info("Done")
 async def on_ready():
     """Metoda wywoływana przy połączeniu do serwera."""
     logger.debug("%s has connected to Discord!", client.user)
+    await client.load_extension("voice_recognition")
+    logger.info("Cogs")
+    logger.info(client.cogs)
     await client.change_presence(activity=discord.Game(name="Axe Throwing Darts"))
     await client.tree.sync()
+    for com in client.commands:
+        logger.info(com.qualified_name)
     check_self.start()
     check_data_q.start()
 
@@ -398,7 +412,13 @@ async def on_message(message):
             result, MESSAGE_TABLE = await handle_response(
                 prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION"
             )
-        await message.reply(result)
+
+            if len(result) < 1500:
+                await message.reply(result)
+            else:
+                while len(result) > 1500:
+                    await message.reply(result[:1500])
+                    result = result[1500:]
     if "imaginuje sobie:" in message.content:
         async with channel.typing():
             logger.info("Poczatek procedury obrazkowej")
@@ -522,16 +542,20 @@ async def connect(ctx, arg=None):
     """
     if ctx and arg:
         logger.info("Ctx and arg defined for connect")
-    voice_channel = client.get_channel(1060349757349974066)
 
     if client.voice_clients:
-        voice_client = client.voice_clients[0]
+        logger.info("Already connected with other client")
-        if voice_client.is_connected():
+    else:
+        vc = None
+        if ctx.author.voice.channel:
+            vc = await ctx.author.voice.channel.connect()
+        else:
+            voice_channel = client.get_channel(1060349757349974066)
+            vc = await voice_channel.connect()
+        if not vc:
+            logger.error("Not possible to connect to voice")
             return
-    voice_channel_connection = await voice_channel.connect()
+        logger.info("Connected to voice")
-    if not voice_channel_connection:
-        logger.error("Not possible to connect to voice")
-    logger.info("Connecting to voice")
 
 
 async def disconnect(ctx):
@@ -631,7 +655,12 @@ async def play(ctx, zamawial=None, arg=None):
             query, vykidailo, bartender, message_table_muzyka, username, "MUSIC"
         )
         logger.debug("Obecna historia czatu: %s", message_table_muzyka)
-        await ctx.send(result)
+        if len(result) < 1500:
+            await ctx.send(result)
+        else:
+            while len(result) > 1500:
+                await ctx.send(result[:1500])
+                result = result[1500:]
 
 
 async def archive_channel(channel_no):
@@ -670,6 +699,7 @@ async def check_music():
             await play(MUZYKA["ctx"], MUZYKA["requester"])
             await asyncio.sleep(2)
 
+
 @tasks.loop(seconds=3)
 async def check_data_q():
     """
@@ -685,40 +715,42 @@ async def check_data_q():
             for doi in fresh_data.entries:
                 logger.info(doi)
                 desc = fresh_data.entries[doi]
-                title = desc['Title'][0]
+                title = desc["Title"][0]
-                entries.append(f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n")
+                entries.append(
-                l_p+=1
+                    f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n"
-            message = '*Z podłogi wysuwa się winda na książki*'
+                )
+                l_p += 1
+            message = "*Z podłogi wysuwa się winda na książki*"
             if fresh_data.ctx is not None:
                 ctx = fresh_data.ctx
-                message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query} nr {fresh_data.uuid}'
+                message += f" Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query} nr {fresh_data.uuid}"
             else:
                 ctx = client.get_channel(1062047571557744721)
-                message += f' Mam wynik dla zapytania {fresh_data.uuid} ale że Hammer coś grzebał nie pamiętam kto to chciał i co chciał'
+                message += f" Mam wynik dla zapytania {fresh_data.uuid} ale że Hammer coś grzebał nie pamiętam kto to chciał i co chciał"
 
-
+            message += "nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku "
-            message += 'nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku '
+            if len(entries) < 1:
-            if len(entries)<1:
                 message += "niestety nie ma nic"
                 await ctx.send(message)
-            elif len(entries)>=1 and len(entries)<5:
+            elif len(entries) >= 1 and len(entries) < 5:
                 message += " znajduje się coś:\n"
                 for item in entries:
-                    message+=item
+                    message += item
                 await ctx.send(message)
             else:
-                message+= " znajduje się cholernie dużo:\n"
+                message += " znajduje się cholernie dużo:\n"
                 for item in entries:
-                    message+=item
+                    message += item
                     if len(message) > 1500:
                         await ctx.send(message)
                         message = ""
 
-            #Kept for sentimental reasons
+            # Kept for sentimental reasons
-            #await ctx.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}")
+            # await ctx.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}")
     except Empty:
         pass
 
+
 @tasks.loop(seconds=120)
 async def check_self():
     """
@@ -929,17 +961,17 @@ async def handle_response(
         "Rozmiar zapytania przed dodaniem historii %s", chat_gpt_config_request_size
     )
     if request_type == "MUSIC":
-        algorithm = "gpt-3.5-turbo-0125"
+        algorithm = "gpt-4o"
         table = message_table_muzyka
         token_amount = 10700
     elif request_type == "RANDOM":
-        algorithm = "gpt-3.5-turbo-0125"
+        algorithm = "gpt-4o"
         table = MESSAGE_TABLE
         token_amount = 10700
     else:
         table = MESSAGE_TABLE
-        algorithm = "gpt-4"
+        algorithm = "gpt-4o"
-        token_amount = 7000
+        token_amount = 10700
 
     prompt_gpt_request_size = num_tokens_from_string(
         {"role": "user", "content": final_prompt}, "gpt-4"
@@ -1021,26 +1053,33 @@ async def handle_response(
     logger.info(result)
     temp = {"role": "assistant", "content": result}
     history.append(temp)
-    if request_type:
+    if request_type == "MUSIC":
-        with open(MEMORY_FIVE_MUZYKA, "r+", encoding=ENCODING) as file:
+        with open(MEMORY_FIVE_MUZYKA, "r+", encoding=ENCODING) as file_music_memory:
             # First we load existing data into a dict.
-            file_data = json.load(file)
+            file_data = json.load(file_music_memory)
             # Join new_data with file_data inside emp_details
             file_data.append(temp)
-            # Sets file's current position at offset.
+            file_music_memory.seek(0)
-            file.seek(0)
             # convert back to json.
-            json.dump(file_data, file, indent=4)
+            json.dump(file_data, file_music_memory, indent=4)
+    elif request_type == "RANDOM":
+        with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
+            # First we load existing data into a dict.
+            file_data = json.load(file_memory)
+            # Join new_data with file_data inside emp_details
+            file_data.append(temp)
+            file_memory.seek(0)
+            # convert back to json.
+            json.dump(file_data, file_memory, indent=4)
     else:
-        with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file:
+        with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
             # First we load existing data into a dict.
-            file_data = json.load(file)
+            file_data = json.load(file_memory)
             # Join new_data with file_data inside emp_details
             file_data.append(temp)
-            # Sets file's current position at offset.
+            file_memory.seek(0)
-            file.seek(0)
             # convert back to json.
-            json.dump(file_data, file, indent=4)
+            json.dump(file_data, file_memory, indent=4)
     return result, MESSAGE_TABLE
 
 
@@ -1069,9 +1108,13 @@ async def get_file(ctx, source, link):
             logger.info(item_type)
             logger.info(item_id)
             logger.info("Spotify link provided")
-            file_list = await asyncio.to_thread(spotify.fetch_tracks, spotify_ctrl, item_type, link)
+            file_list = await asyncio.to_thread(
+                spotify.fetch_tracks, spotify_ctrl, item_type, link
+            )
             logger.info(file_list)
-            directory_name = await asyncio.to_thread(spotify.get_item_name, spotify_ctrl, item_type, item_id)
+            directory_name = await asyncio.to_thread(
+                spotify.get_item_name, spotify_ctrl, item_type, item_id
+            )
             logger.info(directory_name)
             logger.info("Spotify scrape done")
             url_data = {"urls": []}
@@ -1339,6 +1382,53 @@ async def max_weight(lista):
 # *=========================================== Define Commands
 
 
+@client.hybrid_command(
+    name="skip_track",
+    description="Przeskocz kawałek w radiu",
+    guild=discord.Object(id=664789470779932693),
+)
+async def skip_track(ctx):
+    async with ctx.typing():
+        allowed = False
+        for role in ctx.author.roles:
+            if role.name in ("Thane", "Jarl"):
+                allowed = True
+        if not allowed:
+            await ctx.send("Łapy precz od radia")
+        else:
+            coroutine = asyncio.to_thread(
+                requests.get,
+                f"{RADIO_HARBOR_ADDRESS}{SKIP_TRACK}",
+                timeout=360,
+            )
+            result = await coroutine
+            logger.info("Done %s", result)
+            await ctx.send("Zrobione szefie!")
+
+
+@client.hybrid_command(
+    name="zrestartuj_radio",
+    description="Komenda ktora uruchamia radio ponownie jakby się zawiesiło",
+    guild=discord.Object(id=664789470779932693),
+)
+async def zrestartuj_radio(ctx):
+    async with ctx.typing():
+        allowed = False
+        for role in ctx.author.roles:
+            if role.name in ("Thane", "Jarl"):
+                allowed = True
+        if not allowed:
+            await ctx.send("Łapy precz od radia")
+        else:
+            retcode = subprocess.run(
+                "/home/pi/Conjurer/restart_radio.sh",
+                shell=False,
+                check=False,
+                capture_output=True,
+            )
+            logger.info("Wynik: %s", retcode)
+
+
 @client.hybrid_command(
     name="dej_co_ze_spotifaja",
     description="Podaj link do spotify - sciagnie i doda muzyke",
@@ -1580,6 +1670,7 @@ async def graj_muzyko(ctx):
         logger.info("Press play on tape completed")
         check_music.start()
 
+
 @client.hybrid_command(
     name="radio_hardkor",
     description="Włącza radio Conjurer na kanale #nocna-zmiana.",
@@ -1604,9 +1695,10 @@ async def radio_hardkor(ctx):
             voice_client = client.voice_clients[0]
             voice_client.play(
                 discord.PCMVolumeTransformer(
-                discord.FFmpegPCMAudio("http://95.175.16.246:666/mp3-stream"), volume=0.3
+                    discord.FFmpegPCMAudio("http://95.175.16.246:666/mp3-stream"),
-                )
+                    volume=0.3,
                 )
+            )
             logger.info("Press play on radio completed")
         else:
             logger.error("Already playing")
@@ -1614,7 +1706,6 @@ async def radio_hardkor(ctx):
         check_music.start()
 
 
-
 @client.hybrid_command(
     name="cisza",
     description="Wyłącza muzykę i czyści plejliste.",
@@ -1757,6 +1848,7 @@ async def zagraj_mi_kawalek(ctx):
                 "Obawiam się że nie mogę nic znaleźć u siebie. Spróbuj komendy '$dej_co_ze_spotifaja' jeśli masz link"
             )
 
+
 @client.hybrid_command(
     name="dodaj_do_ulubionych",
     description="Dodaje do listy ulubionych w radiu",
@@ -1788,7 +1880,11 @@ async def dodaj_do_ulubionych(ctx):
             json=jrequest,
             timeout=360,
         )
-        _ = await coroutine
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
+
+
 @client.hybrid_command(
     name="ja_chciol",
     description="Dodaje do listy ulubionych w radiu",
@@ -1818,7 +1914,47 @@ async def request_radio(ctx):
             json=jrequest,
             timeout=360,
         )
-        _ = await coroutine
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
+
+
+@client.hybrid_command(
+    name="stworz_audycje",
+    description="Dodaje do listy ulubionych w radiu",
+    guild=discord.Object(id=664789470779932693),
+)
+async def stworz_audycje(ctx):
+    """
+    Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
+    Rest of the line are search terms.
+
+    :param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
+    represents the context in which the command was invoked, including information such as the message,
+    the channel, the server, and the user who invoked the command. This parameter is required in all
+    Discord.py commands
+    """
+    async with ctx.typing():
+        logger.info("Zaczynam szukać timestamp %s", datetime.now())
+
+        dlugosc_playlisty = ctx.message.content.split()[1]
+        word_list = ctx.message.content.split()
+        jrequest = {
+            "lista_slow": word_list,
+            "dlugosc_plejlisty": dlugosc_playlisty,
+            "UUID": str(uuid.uuid4()),
+        }
+        coroutine = asyncio.to_thread(
+            requests.post,
+            f"{FILE_SERVICE_ADDRESS}{CREATE_PRIO_PLAYLIST}",
+            json=jrequest,
+            timeout=360,
+        )
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
+
+
 @client.hybrid_command(
     name="wyczysc_ulubione",
     description="Czysci liste ulubionych w radiu",
@@ -1840,7 +1976,9 @@ async def wyczysc_ulubione(ctx):
             f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}",
             timeout=360,
         )
-        _ = await coroutine
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
 
 
 @client.hybrid_command(
@@ -2220,11 +2358,18 @@ async def wyszukaj_linki_do_dokumentow(ctx):
     """
     query = ctx.message.content
     query_uuid = uuid.uuid4()
-    #TODO: TESTING ONLY!!
+    # TODO: TESTING ONLY!!
-    #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
+    # query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
-    ctx.message.content = ctx.message.content.replace("$wyszukaj_linki_do_dokumentow", "")
+    ctx.message.content = ctx.message.content.replace(
+        "$wyszukaj_linki_do_dokumentow", ""
+    )
 
-    json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": False}
+    json_query = {
+        "UUID": str(query_uuid),
+        "query": str(query),
+        "page": 1,
+        "deep_search": False,
+    }
     coroutine = asyncio.to_thread(
         requests.post,
         f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
@@ -2232,14 +2377,16 @@ async def wyszukaj_linki_do_dokumentow(ctx):
         timeout=360,
     )
     await ctx.send(
-        "*Conjurer powoli notuje podane przez Ciebie. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" +
+        "*Conjurer powoli notuje podane przez Ciebie. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i"
-        " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to 3/4 stacji."+
+        + " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to 3/4 stacji."
-        " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc 12/8 stacji teraz będzie ciężko pracować"
+        + " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc 12/8 stacji teraz będzie ciężko pracować"
     )
     query_response = await coroutine
     if not query_response.status_code == 200:
-        await ctx.send("*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."+
+        await ctx.send(
-        " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało")
+            "*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."
+            + " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało"
+        )
         return
 
     query, query_uuid, queue_size = (
@@ -2251,10 +2398,13 @@ async def wyszukaj_linki_do_dokumentow(ctx):
         username = ctx.message.author.nick
     else:
         username = ctx.message.author.name
-    query_object = QueryControl(username, query_uuid, query, logger, ctx)
+    query_object = QueryControl(username, query_uuid, query, ctx)
     OUT_COMM_Q.put(query_object)
-    await ctx.send(f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce."+
+    await ctx.send(
-    " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni.")
+        f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce."
+        + " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni."
+    )
+
 
 @client.hybrid_command(
     name="glebokie_gardlo",
@@ -2298,13 +2448,23 @@ async def wyszukaj_linki_do_dokumentow_deep(ctx):
         result, MESSAGE_TABLE = await handle_response(
             prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION"
         )
-        await ctx.send(result)
+        if len(result) < 1500:
+            await ctx.send(result)
+        else:
+            while len(result) > 1500:
+                await ctx.send(result[:1500])
+                result = result[1500:]
         return
 
     query_uuid = uuid.uuid4()
-    #TODO: TESTING ONLY!!
+    # TODO: TESTING ONLY!!
-    #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
+    # query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
-    json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": True}
+    json_query = {
+        "UUID": str(query_uuid),
+        "query": str(query),
+        "page": 1,
+        "deep_search": True,
+    }
     coroutine = asyncio.to_thread(
         requests.post,
         f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
@@ -2312,14 +2472,16 @@ async def wyszukaj_linki_do_dokumentow_deep(ctx):
         timeout=360,
     )
     await ctx.send(
-        "*Conjurer mruga okiem i zamiast drinka wyjmuje dysk usb z terminala. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" +
+        "*Conjurer mruga okiem i zamiast drinka wyjmuje dysk usb z terminala. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i"
-        " wrzuca ten dysk do środk* To teraz trza poczekać kilka godzin. TO będzie głębokie wyszukanie"+
+        + " wrzuca ten dysk do środk* To teraz trza poczekać kilka godzin. TO będzie głębokie wyszukanie"
-        " Rozgląda się dookoła i stawia przed tobą mętnego drinka o zapachu mięty, gwiezdnego pyłu i oleju silnikowego"
+        + " Rozgląda się dookoła i stawia przed tobą mętnego drinka o zapachu mięty, gwiezdnego pyłu i oleju silnikowego"
     )
     query_response = await coroutine
     if not query_response.status_code == 200:
-        await ctx.send("*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."+
+        await ctx.send(
-        " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało")
+            "*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."
+            + " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało"
+        )
         return
 
     query, query_uuid, queue_size = (
@@ -2331,11 +2493,12 @@ async def wyszukaj_linki_do_dokumentow_deep(ctx):
         username = ctx.message.author.nick
     else:
         username = ctx.message.author.name
-    query_object = QueryControl(username, query_uuid, query, logger, ctx)
+    query_object = QueryControl(username, query_uuid, query, ctx)
     OUT_COMM_Q.put(query_object)
-    await ctx.send(f"Wypij wypi {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce niestety kolejka obowiazuje zawsze."+
+    await ctx.send(
-    " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzin - ale mogą być też dni. Głębokie zabawy trwają dłużej. *Znowu mruga*")
+        f"Wypij wypi {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce niestety kolejka obowiazuje zawsze."
-
+        + " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzin - ale mogą być też dni. Głębokie zabawy trwają dłużej. *Znowu mruga*"
+    )
 
 
 @client.hybrid_command(
@@ -2367,11 +2530,29 @@ async def krecimy_pornola(ctx):
                     sciezka, files = await get_file(ctx, "Pornol", item_yt)
                     if files:
                         logger.info("Pornol udany")
-                        ctx.reply(f"Jest w tajnym archiwum pod adresem{sciezka})")
+                        await ctx.send(f"Jest w tajnym archiwum pod adresem{sciezka})")
     else:
-        ctx.reply("Jak ładnie szefa poprosisz.")
+        await ctx.reply("Jak ładnie szefa poprosisz.")
 
+@client.hybrid_command(
+    name="co_na_plejliscie_wariacie",
+    description="Wyswietl kawalki ktore sa na pocztku list radia",
+    guild=discord.Object(id=664789470779932693),
+)
+async def co_na_plejliscie_wariacie(ctx):
+    """
+    Download a music number from youtube.
 
+    :param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
+    represents the context in which the command was invoked, including information such as the message,
+    the channel, the server, and the user who invoked the command. It allows the command to interact
+    with the Discord API and
+    """
+    await ctx.send("Dej mnie chwilkę")
+    async with ctx.typing():
+        logger.info("plejlista")
+        logger.info(PREPPED_TRACKS)
+        await ctx.send(f"Didżej Hammer i jego sztuczna inteligencja (||he he||) prezentują: \n- obecnie grany jest {(PREPPED_TRACKS['now_playing']).rstrip()} (wg metadanych: {PREPPED_TRACKS['meta']}) \n- następny będzie{(PREPPED_TRACKS['next']).rstrip()} \n- na liście priorytetowej {(PREPPED_TRACKS['priority']).rstrip()} \n- na liście hitów obecnie czeka kawałek {(PREPPED_TRACKS['hit']).rstrip()} \n- na liście wszystkich kawałków {(PREPPED_TRACKS['all']).rstrip()} \n- najświeższym zamówieniem od słuchaczy jest {(PREPPED_TRACKS['requests']).rstrip()} \nZ kolei jingiel to: {(PREPPED_TRACKS['jingles']).rstrip()}")
 
 # *================================== Run
 if __name__ == "__main__":
@@ -2379,23 +2560,16 @@ if __name__ == "__main__":
     threads = []
     logger.info("Starting discord bot: Creating threads")
     threads.append(threading.Thread(target=client.run, args=(TOKEN,)))
-    threads.append(threading.Thread(target=comm_subroutine, args=(logger,)))
+    threads.append(threading.Thread(target=comm_subroutine))
     logger.info("Starting discord bot: Starting threads")
     WRK_CNT = 0
     for worker in threads:
-        WRK_CNT+=1
+        WRK_CNT += 1
         logger.info("Starting discord bot: Starting thread %s", WRK_CNT)
         worker.start()
     logger.info("Starting discord bot: Joining threads")
-    WRK_CNT=0
+    WRK_CNT = 0
     for worker in threads:
-        WRK_CNT+=1
+        WRK_CNT += 1
         logger.info("Starting discord bot: Joining thread %s", WRK_CNT)
         worker.join()
-
-# widok - generuje się 5 stron po 4 linki
-# odśwież - zamienić na "dalej" - zaciąga kolejne 20 rezultatów
-# po wyslaniu czekamy na odbior i dodajemy UUID oraz nick poszukiwacza do queue ktora jest sprawdzana
-# flask sobie chodzi i jak dostaje posta z wynikami to wyciaga z queue UUID i odpowiada że znalazła
-# po samym wyszukaniu dajemy info że "Zaczynam szukać ale pare godzin to potrwa."
-# OPCJONALNIE: Dosyłamy znalezione wyniki na bieżąco - wtedy wkładamy do queue dla UUID wyniki

communication_subroutine.py

@@ -2,27 +2,45 @@ import logging
 import threading
 import json
 import time
+import re
+from urllib import request as urequest
+
 from queue import Queue, Empty
 from flask import Flask, jsonify, request
 from waitress import serve
 
 HOST_ADDRESS = "192.168.1.191"
 PORT_ADDRESS = 5000
+ICECAST_ADDRESS = "http://192.168.1.15:8000"
 OUT_COMM_Q = Queue()
 IN_COMM_Q = Queue()
+SRCHTITLE = re.compile(br'StreamTitle=\\*(?P<title>[^;]*);').search
+
 awaiting_q = []
 incoming_q = Queue()
 app = Flask(__name__)
+PREPPED_TRACKS = {
+    "requests" : "",
+    "hit" : "",
+    "all": "",
+    "priority":"",
+    "jingles":"",
+    "now_playing":"",
+    "next": "",
+    "meta" : ""
+}
+logger = logging.getLogger("discord")
+
 class QueryControl:
     """
     This class `QueryControl` is used to manage queries with information about the author, UUID,
     content, logger, context, and replies.
     """
-    def __init__(self, query_author, query_uuid, query_content, uplogger, ctx) -> None:
+    def __init__(self, query_author, query_uuid, query_content, ctx) -> None:
         self.author = query_author
         self.uuid = query_uuid
         self.content = query_content
-        self.logger = uplogger
+        self.logger = logging.getLogger("discord")
         self.stop = False
         self.ctx = ctx
         self.logger.info(
@@ -30,6 +48,25 @@ class QueryControl:
         )
         self.replies = []
 
+@app.route("/prepped_tracks", methods=["POST"])
+def log_radio_tracks():
+    app.logger = logging.getLogger("discord")
+
+    app.logger.info(request)
+    record = json.loads(request.data)
+    app.logger.info(record)
+    if "next" in record[0]:
+        metadata = id3(ICECAST_ADDRESS)
+        PREPPED_TRACKS["now_playing"] = PREPPED_TRACKS["next"]
+        if metadata:
+            PREPPED_TRACKS["meta"] = metadata["name"] + " - " + metadata["title"] + "(" + metadata["genre"] + ")"
+        else:
+            PREPPED_TRACKS["meta"] = "Nie znaju"
+    PREPPED_TRACKS[record[0]] = record[1]
+    app.logger.info("DATA RECEIVED")
+    return jsonify("SUCCESS")
+
+
 @app.route("/conjurer", methods=["POST"])
 def answer_external_command():
     """
@@ -38,10 +75,11 @@ def answer_external_command():
     :return: The function `answer_external_command()` is returning a JSON response with the message
     "SUCCESS".
     """
-    app.logger.info(request)
+    logger = logging.getLogger("discord")
+    logger.info(request)
     record = json.loads(request.data)
-    app.logger.info(record)
+    logger.info(record)
-    app.logger.info("DATA RECEIVED")
+    logger.info("DATA RECEIVED")
     incoming_q.put(record)
     return jsonify("SUCCESS")
 
@@ -56,25 +94,29 @@ def check_alive():
     """
     return jsonify("ALIVE")
 
-def flask_debug(_logger):
+def flask_debug():
     """
     The `flask_debug` function starts a Flask application in debug mode without using the reloader.
     Do not use for production for fucks sake.
     """
-    _logger.info("Attempt debug")
+    logger = logging.getLogger("discord")
+
+    logger.info("Attempt debug")
     # trunk-ignore(bandit/B201)
     app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
 
-def waitress_run(_logger):
+def waitress_run():
     """
     The `waitress_run` function serves the `app` on host "0.0.0.0"
     and port 5000 using the Waitress WSGI server.
     """
-    _logger.info("Attempt waitress")
+    logger = logging.getLogger("discord")
+
+    logger.info("Attempt waitress")
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
-def scan_queue(_logger):
+def scan_queue():
     """
     The function `scan_queue` reads data from a queue, logs it, and appends it to another queue.
 
@@ -83,12 +125,13 @@ def scan_queue(_logger):
     and provide information for debugging purposes. In this code snippet, the `_logger` object is used
     to log the
     """
+    logger = logging.getLogger("discord")
     while True:
         data = OUT_COMM_Q.get()
-        _logger.info(data)
+        logger.info(data)
         awaiting_q.append(data)
 
-def scan_incoming(_logger):
+def scan_incoming():
     """
     The `scan_incoming` function continuously checks for incoming data, processes it, and logs when data
     is found.
@@ -98,10 +141,11 @@ def scan_incoming(_logger):
     debugging, monitoring, or tracking the flow of the program. In this case, the `_logger` is being
     used
     """
+    logger = logging.getLogger("discord")
     while True:
         try:
             answer = incoming_q.get(block=False)
-            _logger.info("DATA FOUND")
+            logger.info("DATA FOUND")
             record_stored = False
             for record in awaiting_q:
                 if record.uuid in answer.keys():
@@ -111,15 +155,37 @@ def scan_incoming(_logger):
                     IN_COMM_Q.put(record)
             if not record_stored:
                 for key in answer.keys():
-                    record = QueryControl("Orphaned", key, "Orphan", _logger, None)
+                    record = QueryControl("Orphaned", key, "Orphan", None)
                     record.stop = True
                     record.entries = answer[record.uuid]
                     IN_COMM_Q.put(record)
         except Empty:
             time.sleep(1)
 
+def get_stream_title(tag:bytes) -> str:
+    title = ''
+    if m := SRCHTITLE(tag):
+        #decode, strip, unescape and remove surrounding quotes (may not even be the same type of quote)
+        title = m.group('title').decode('utf-8').strip().replace('\\', '')[1:-1]
+    return title
 
-def comm_subroutine(logger):
+def id3(url:str) -> dict:
+    request = urequest.Request(url, headers={'Icy-MetaData': 1})
+
+    with urequest.urlopen(request) as resp:
+        metaint = int(resp.headers.get('icy-metaint', '-1'))
+        if metaint<0:
+            return False
+        resp.read(metaint) #this isn't seekable so, arbitrarily read to the point we want
+        tagdata = dict(
+            site_url = resp.headers.get('icy-url'  )        ,
+            name     = resp.headers.get('icy-name' ).title(),
+            genre    = resp.headers.get('icy-genre').title(),
+            title    = get_stream_title(resp.read(255))     )
+        return tagdata
+
+
+def comm_subroutine():
     """
     The `comm_subroutine` function starts multiple threads to run different tasks concurrently.
 
@@ -128,12 +194,13 @@ def comm_subroutine(logger):
     the provided code snippet, the logger is used to log messages at the "info" level
     """
     #logger.setLevel(logging.DEBUG)
-    logger.info("Started")
+    logger = logging.getLogger("discord")
+    logger.info("Started comms")
     threads = []
-    #threads.append(threading.Thread(target=flask_debug, args=(logger,)))
+    #threads.append(threading.Thread(target=flask_debug))
-    threads.append(threading.Thread(target=waitress_run, args=(logger,)))
+    threads.append(threading.Thread(target=waitress_run))
-    threads.append(threading.Thread(target=scan_queue,args=(logger,)))
+    threads.append(threading.Thread(target=scan_queue))
-    threads.append(threading.Thread(target=scan_incoming,args=(logger,)))
+    threads.append(threading.Thread(target=scan_incoming))
 
 
     for worker in threads:
@@ -142,5 +209,4 @@ def comm_subroutine(logger):
         worker.join()
 
 if __name__ == "__main__":
-    logging_client = logging.getLogger(__name__)
+    comm_subroutine()
-    comm_subroutine(logging_client)

conjurer_librarian/conjurer_librarian.py

@@ -26,6 +26,7 @@ from queue import Queue
 import requests
 import scrape_bot
 import search_bot
+#import search_bot2 as search_bot
 from flask import Flask, jsonify, request
 from habanero import Crossref
 from waitress import serve
@@ -37,7 +38,10 @@ PORT_ADDRESS = 5001
 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
 SEND_RESULTS = "/conjurer"
 BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
-MAX_CR_RESULTS = 7000
+
+MAX_CR_RESULTS = 1000
+#TEST PURPOSES ONLY!
+#MAX_CR_RESULTS = 5
 
 ENCODING = "utf-8"
 
@@ -114,14 +118,15 @@ class Librarian(object):
         self.app.logger.info("STARTED SEARCH")
         
         if not deep_search:
-            cr_result = self.cr.works(query=query, limit=1000)
+            query_limit = MAX_CR_RESULTS if MAX_CR_RESULTS < 1000 else 1000
+            cr_result = self.cr.works(query=query, limit=query_limit)
             self.search_result_from_cr.update(cr_result)
             self.total = cr_result["message"]["total-results"]
             self.fetched += len(cr_result["message"]["items"])
             self.app.logger.info(self.total)
             self.app.logger.info(self.fetched)
             while self.total > self.fetched and self.limit > self.fetched:
-                tmp_result = self.cr.works(query=query, limit=1000, offset=self.fetched)
+                tmp_result = self.cr.works(query=query, limit=query_limit, offset=self.fetched)
                 cr_result["message"]["items"].extend(tmp_result["message"]["items"])
                 self.total = tmp_result["message"]["total-results"]
                 self.fetched = len(cr_result["message"]["items"])
@@ -130,7 +135,7 @@ class Librarian(object):
                 time.sleep(0.1)
 
         else:
-            cr_result = self.cr.works(query=query, cursor_max=10000, cursor='*', progress_bar = True)
+            cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True)
             result = cr_result[0]
             for item in cr_result[1:]:
                 result["message"]["items"].extend(item["message"]["items"])

conjurer_librarian/count_lines.py

@@ -0,0 +1,8 @@
+
+FILEPTH = r"C:\Database\chunks\4_chunk.txt"
+
+with open(FILEPTH, "r", encoding="utf-8") as file:
+    tab = file.readlines()
+    print(len(tab))
+
+    #1842295

conjurer_librarian/scrape_bot.py

@@ -33,12 +33,14 @@ def load_ndb_to_q(logger):
             try:
                 ndb_database = json.load(ndb_file)
                 for _ in range (1,10):
-                    key = next(iter(ndb_database))
+                    try:
-                    item = ndb_database.pop(key)
+                        key = next(iter(ndb_database))
-                    logger.info(key)
+                        _ = ndb_database.pop(key)
-                    url = f"https://sci-hub.se/{key}"
+                        logger.info(key)
-                    WORK_Q.put([item, url, False])
+                        url = f"https://sci-hub.se/{key}"
-
+                        WORK_Q.put([key, url, False])
+                    except StopIteration:
+                        break
                 ndb_file.truncate(0)
                 ndb_file.seek(0)
                 json.dump(ndb_database, ndb_file, indent=4)

conjurer_librarian/search_bot.py

@@ -21,8 +21,9 @@ Global Variables:
 # TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
 from queue import Empty, Queue
 from threading import Thread
-
+import time
 q = Queue()
+#TODO: Count number of lines in files and print to approximate on which part of the file search is
 
 # DATA FOR TEST ONLY
 # MAXTHREADS = 5
@@ -49,9 +50,17 @@ def producer(out_q, control_q, filename, _logger):
         _logger: Logger object for logging.
     """
     with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
-        _logger.info(f"Worker {filename} ")
+        print(f"Worker {filename} ")
+        line_no = 0
         while True:
             line = operated_file.readline()
+            line_no += 1
+            print(f"\t \t \t \t \t \t W{filename}{line_no}\r", end="")
+
+            if not line:
+                print(f"EOF {filename}")
+                break
+
             if not line:
                 break
             out_q.put(line)
@@ -59,11 +68,12 @@ def producer(out_q, control_q, filename, _logger):
                 check = control_q.get(block=False)
             except Empty:
                 check = False
+
             if check is _sentinel:
-                _logger.info("TERM signal received")
+                print("TERM signal received")
                 control_q.put(check)
                 break
-        _logger.info(f"Worker finished: {filename}")
+        print(f"Worker finished: {filename}")
         out_q.put(_sentinel)
 
 
@@ -79,19 +89,25 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no,
         live_results (list): List to store the search results.
         _logger: Logger object for logging.
     """
-    _logger.info("Consumer thread started:%s", no)
+    print(f"Consumer thread started: {no} no")
-    for item in doi:
+    empty_counter = 0
-        result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
+    alive_no = 0
     while True:
         done_check = True
         try:
-            data = in_q.get(block=False)
+            data = in_q.get(block=True, timeout = 1)
             if data is _sentinel:
-                _logger.info("Worker finished")
                 control_dict["sentinels"] += 1
+                print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
+
             else:
+                empty_counter = 0
+                alive_no += 1
+                print(f"C{no}__{alive_no}\r", end="")
+
                 for item in result_list:
                     if item["DOI"] in data and not item["exists"]:
+                        print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
                         _logger.info(data)
                         _logger.info("HIT")
                         item["exists"] = True
@@ -100,9 +116,18 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no,
                 if done_check:
                     control_q.put(_sentinel)
         except Empty:
-            pass
+            empty_counter += 1
+            time.sleep(1)
+            print(f"Consumer {no} empty")
+            if empty_counter > 5:
+                print("Consumer %s empty lvl 2", no)
+                time.sleep(2)
+            elif empty_counter > 10:
+                print(f"Consumer thread finished {no}")
+                break
+
         if control_dict["sentinels"] >= MAXTHREADS:
-            _logger.info("All workers finished")
+            _logger.info(f"All workers finished {no}")
             break
 
 
@@ -121,6 +146,10 @@ def search_for_doi(doi, live_results, _logger):
     threads = []
     work_q = Queue(maxsize=WORK_Q_SIZE)
     control_q = Queue()
+
+    for item in doi:
+        result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
+
     for i in range (0, (len(doi)//1000)+2):
         t_cons = Thread(
             target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, i, _logger)

conjurer_librarian/search_bot2.py

@@ -0,0 +1,206 @@
+"""
+This module contains functions for searching for DOI (Digital Object Identifier)
+in a list of live results.
+It includes a producer-consumer pattern implementation using threads and queues.
+
+Functions:
+- producer: Reads lines from a file and puts them into an output queue.
+- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
+- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
+
+Global Variables:
+- MAXTHREADS: Maximum number of worker threads.
+- DATABASE_PATH: Path to the database files.
+- ENCODING: Encoding of the database files.
+- CHUNK: File name pattern for the database files.
+- _sentinel: Sentinel object used to signal termination.
+- result_list: List to store the search results.
+- WORK_Q_SIZE: Maximum size of the work queue.
+"""
+
+# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+from multiprocessing import Manager, Process, Queue
+from queue import Empty, Full
+import time
+
+# DATA FOR TEST ONLY
+#MAXTHREADS = 6
+#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+
+# DEPLOYMENT DATA
+MAXTHREADS = 41
+DATABASE_PATH = r"C:\\Database\\chunks\\"
+
+ENCODING = "utf-8"
+CHUNK = "_chunk.txt"
+_sentinel = None
+WORK_Q_SIZE = 355000
+
+
+def producer(out_q, control_q, filename, _logger):
+    """
+    Produces items from the output queue and puts them into the control queue.
+
+    Args:
+        out_q (Queue): Output queue.
+        control_q (Queue): Control queue.
+        filename (str): Name of the file.
+        _logger: Logger object for logging.
+    """
+    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
+        print(f"Worker {filename} ")
+        line_no = 0
+        while True:
+            line = operated_file.readline()
+            line_no += 1
+            print(f"\t \t \t W{filename}{line_no}\r", end="")
+
+            if not line:
+                print(f"EOF {filename}")
+                break
+            try:
+                out_q.put([line, line_no, filename], block = True, timeout = 60)
+            except Full:
+                print("queue fulll")
+            try:
+                check = control_q.get(block=False)
+            except Empty:
+                check = False
+            if check is None:
+                print(f"TERM signal received {filename}")
+                control_q.put(check)
+                control_q.close()
+                break
+        print(f"Worker finished: {filename}")
+        out_q.put(_sentinel)
+        out_q.close()
+
+
+
+# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
+    """
+    Consumes items from an input queue and checks if DOI exists in the live results list.
+
+    Args:
+        in_q (Queue): Input queue.
+        control_q (Queue): Control queue.
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    print(f"Consumer thread started: {no} no")
+    empty_counter = 0
+    alive_no = 0
+    while True:
+        done_check = True
+        try:
+            data = in_q.get(block=True, timeout = 1)
+            empty_counter = 0
+            if data is None:
+                control_dict["sentinels"] += 1
+                print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
+            else:
+                empty_counter = 0
+                if alive_no > 9:
+                    alive_no = 0
+                alive_no += 1
+                print(f"C{no}{alive_no}\r", end="")
+                for item in result_list:
+                    if item["DOI"] in data[0] and not item["exists"]:
+                        print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
+                        item["exists"] = True
+                        live_results.append(item)
+                    done_check = done_check and item["exists"]
+                if done_check:
+                    control_q.put(_sentinel)
+        except Empty:
+            empty_counter += 1
+            time.sleep(1)
+            print(f"Consumer {no} empty")
+            if empty_counter > 5:
+                print("Consumer %s empty lvl 2", no)
+                time.sleep(2)
+            elif empty_counter > 10:
+                print(f"Consumer thread finished {no}")
+                break
+        if control_dict["sentinels"] >= MAXTHREADS:
+            print(f"All workers finished killing {no}")
+            break
+    print("Consumer thread finished %s",no )
+    control_q.close()
+
+
+def search_for_doi(doi, live_results, _logger):
+    """
+    Search for DOI in live_results using _logger for logging.
+
+    Args:
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    with Manager() as manager:
+        control_dict = manager.dict()
+        control_dict["sentinels"] = 0
+        result_list = []
+        for item in doi:
+            result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
+        result_list_proxy = manager.list(result_list)
+
+        threads = []
+        work_q = Queue(maxsize=WORK_Q_SIZE)
+        control_q = Queue()
+        for i in range(0, MAXTHREADS):
+            # TEST DATA
+            #filename = "test" + str(i) + CHUNK
+            # DEPLOYMENT DATA
+            filename = str(i) + CHUNK
+            print(f"Creating worker thread no: {i}")
+            threads.append(
+                Process(target=producer, args=(work_q, control_q, filename, _logger))
+            )
+        for i in range (0, (len(doi)//1000)+5):
+            t_cons = Process(
+                target=consumer, args=(work_q, control_q, doi, live_results, result_list_proxy, control_dict, i, _logger)
+            )
+            print("Consumer thread created")
+            threads.append(t_cons)
+
+        for worker in threads:
+            worker.start()
+        for worker in threads:
+            worker.join()
+        result_list = list(result_list_proxy)
+        for worker in threads:
+            worker.close()
+        return result_list
+
+
+if __name__ == "__main__":
+    import logging
+
+    logger = logging.getLogger()
+    logger.setLevel("DEBUG")
+    h1 = logging.StreamHandler()
+    logger.addHandler(h1)
+    print("TEST RUN")
+    live_result = []
+    print(
+        search_for_doi(
+            [
+                ("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
+                ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
+                ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
+                ("10.2165/00128415-200309690-00017", "DATA"),
+                ("10.5772/48313", "DATA"),
+                ("10.15803/ijnc.7.2_419", "DATA"),
+                ("10.1051/0004-6361/201321596e", "DATA"),
+                ("10.2307/40835941", "DATA"),
+                ("NOT_ON_LIST", "DATA"),
+            ],
+            live_result,
+            logger,
+        )
+    )
+    print(live_result)

conjurer_musician/conjurer_musician.py

@@ -6,7 +6,9 @@ API endpoints.
 """
 import json
 import logging
+import os
 import re
+import requests
 import threading
 import time
 from datetime import datetime
@@ -14,10 +16,11 @@ from logging import handlers
 from pathlib import Path
 from platform import uname
 from sys import platform
-
+from flask import Flask, jsonify, redirect, request, send_from_directory, render_template
-from flask import Flask, jsonify, redirect, request, send_from_directory
 from waitress import serve
 
+MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
+MUSIC_TRACKER = "/prepped_tracks"
 HOST_ADDRESS = "192.168.1.15"
 PORT_ADDRESS = 5000
 if platform in ("linux", "linux2"):
@@ -28,6 +31,8 @@ if platform in ("linux", "linux2"):
         NETRC_FILE = "/home/mtuszowski/.netrc"
         LOGSTORE = "/home/mtuszowski/conjurer/logs/"
         ENCODING = "utf-8"
+        RADIOLOG_PATH = '/home/pi/Conjurer/radio_log.log'
+        PERSISTENCE_PATH = '/home/pi/Conjurer/persistence.log'
 
     else:
         LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
@@ -36,6 +41,8 @@ if platform in ("linux", "linux2"):
         ENCODING = "utf-8"
         MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
         PRIORITY_FOLDER = "/home/pi/RetroPie/mp3/Magiczne i chuj/"
+        RADIOLOG_PATH = '/home/pi/Conjurer/radio_log.log'
+        PERSISTENCE_PATH = '/home/pi/Conjurer/persistence.log'
 
 
 music_file_list = []
@@ -46,7 +53,8 @@ def rescan():
     The `rescan` function logs a message, scans for mp3 files in a specified folder,
     and adds them to a list of music files.
     """
-    logging.info("Rescan triggered")
+    logger = logging.getLogger("conjurer_musician")
+    logger.info("Rescan triggered")
 
     for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
         temp_music_file = mp3_item.as_posix()
@@ -83,13 +91,87 @@ def thread_rescan():
     The `thread_rescan` function periodically triggers a rescan operation after a specified time
     interval.
     """
-    logging.info("Starting filesystemupdater")
+    logger = logging.getLogger("conjurer_musician")
+    logger.info("Starting filesystemupdater")
     while True:
         time.sleep(60 * 60 * 24)
-        logging.info("Rescan triggered")
+        logger.info("Rescan triggered")
         rescan()
 
 
+def scan_tracks():
+    #Set the filename and open the file
+    logger = logging.getLogger("conjurer_musician")
+
+    file = open(RADIOLOG_PATH,'r')
+    #Find the size of the file and move to the end
+    st_results = os.stat(RADIOLOG_PATH)
+    st_size = st_results[6]
+    file.seek(st_size)
+    st_results1 = os.stat(PERSISTENCE_PATH)
+    prev_st_size1 = st_results[6]
+
+    while 1:
+
+        st_results1 = os.stat(PERSISTENCE_PATH)
+        st_size1 = st_results1[6]
+        if prev_st_size1 != st_size1:
+            while prev_st_size1 != st_size1:
+                prev_st_size1 = st_size1
+                st_results1 = os.stat(PERSISTENCE_PATH)
+                st_size1 = st_results1[6]
+                time.sleep(0.1)
+            file1 = open(PERSISTENCE_PATH, 'r')
+            lines = file1.readlines()
+            result = ["next", lines[2]]
+            file1.close()
+            returned = requests.post(
+                    f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}",
+                    json=result,
+                    timeout=360)
+            logger.info("SENT")
+            logger.info(returned.status_code)
+            logger.info("SEND CONFIRMED")
+
+        where = file.tell()
+        line = file.readline()
+        if not line:
+            time.sleep(1)
+            file.seek(where)
+        else:
+            if re.match(".*Prepared.*",line):
+                result = None
+                if re.match(".*jingles.*", line):
+                    logger.info("jingles")
+                    logger.info(line) # already has newline
+                    result = ["jingles", line]
+                elif re.match(".*priority.*", line):
+                    logger.info("priority")
+                    logger.info(line) # already has newline
+                    result = ["priority", line]
+                elif re.match(".*hit.*", line):
+                    logger.info("hit")
+                    logger.info(line) # already has newline
+                    result = ["hit", line]
+                elif re.match(".*all_playlist.*", line):
+                    logger.info("all")
+                    logger.info(line) # already has newline
+                    result = ["all", line]
+                elif re.match(".*request.*", line):
+                    logger.info("requests")
+                    logger.info(line) # already has newline
+                    result = ["requests", line]
+                if result:
+                    returned = requests.post(
+                    f"{MAIN_BOT_ADDRESS}{MUSIC_TRACKER}",
+                    json=result,
+                    timeout=360)
+                    logger.info("SENT")
+                    logger.info(returned.status_code)
+                    logger.info("SEND CONFIRMED")
+        time.sleep(0.1)
+
+
 app = Flask(__name__)
 
 
@@ -207,6 +289,7 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
         fun_logger.info("Wiele plików do zagrania")
         search_weight.sort(key=lambda x: x[0], reverse=True)
         return_list.extend(search_weight[: int(how_many)])
+    fun_logger.info("Done: %s", return_list)
     return return_list
 
 
@@ -246,8 +329,9 @@ def stream_music():
 
     :return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
     """
-    return send_from_directory("/tmp/hls", "stream.m3u8")
 
+    #return send_from_directory("/tmp/hls", "stream.m3u8")
+    return render_template("/home/pi/Conjurer/stream.html")
 
 @app.route("/<string:file_name>")
 def stream(file_name):
@@ -260,10 +344,10 @@ def stream(file_name):
     Returns:
         Response: The response containing the streamed file.
     """
+    # trunk-ignore(bandit/B108)
     video_dir = "/tmp/hls"
     return send_from_directory(video_dir, file_name)
 
-
 @app.route("/stream_mp3", methods=["GET"])
 def stream_music_mp3():
     """
@@ -369,6 +453,37 @@ def add_request():
     )
     return return_data
 
+@app.route("/create_priority_playlist", methods=["POST"])
+def create_priority_playlist():
+    """
+    The function `create_priority_playlist` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+
+    :return: A tuple containing a JSON response and a status code.
+        The JSON response includes keys `isError`,
+        `message`, `statusCode`, and `data`, with values
+        indicating the success of the operation and the
+        data that was received and added to the `music_file_list`.
+        The status code returned is 200,indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    app.logger.info(record["dlugosc_plejlisty"])
+    return_data = wyszukaj(
+        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
+    )
+    with open(
+        "/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
+    ) as s_file:
+        for item in return_data:
+            s_file.write(item[1] + "\n")
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
+        200,
+    )
+    return return_data
 
 
 @app.route("/add_to_priority", methods=["POST"])
@@ -457,5 +572,9 @@ if __name__ == "__main__":
     for worker in threads:
         worker.start()
 
+    time.sleep(60)
+    threads.append(threading.Thread(target=scan_tracks))
+    threads[2].start()
+
     for worker in threads:
         worker.join()

conjurer_musician/music_utils.py

@@ -0,0 +1,26 @@
+# This Python file uses the following encoding: utf-8
+"""This module contains utility scripts used by musi radiostation Conjurer"""
+import time
+import re
+
+NAME = "/home/pi/Conjurer/persistence.log"
+
+def print_top():
+    """
+    Prints the top 65 lines from a file specified by the NAME variable.
+    """
+    while True:
+        line = ""
+        buffer = ""
+        with open(NAME, "r", encoding="utf-8") as f:
+            for _ in range (45):
+                line = f.readline()
+                if re.match(".*mp3", line):
+                    buffer += line
+            print("=====================================")
+            print(buffer)
+            print("=====================================")
+        time.sleep(180)
+
+if __name__ == "__main__":
+    print_top()

conjurer_musician/pl.config

@@ -0,0 +1,14 @@
+$dodaj_do_ulubionych 100 nightwish
+$dodaj_do_ulubionych 100 amon amarth
+$dodaj_do_ulubionych 100 tactical sekt
+$dodaj_do_ulubionych 100 lacuna coil
+$dodaj_do_ulubionych 100 sefa
+$dodaj_do_ulubionych 100 frenchcore
+$dodaj_do_ulubionych 100 peacock
+$dodaj_do_ulubionych 100 vavamuffin
+$dodaj_do_ulubionych 100 abradab
+$dodaj_do_ulubionych 100 eluveitie
+$dodaj_do_ulubionych 100 youtube
+
+$dodaj_do_ulubionych 100 elyose
+$dodaj_do_ulubionych 1 artisans du chaos

conjurer_musician/radio_conjurer.liq

@@ -9,12 +9,13 @@ let json.parse credentials  = file.contents("/home/pi/Conjurer/icecast_credentia
 enable_replaygain_metadata()
 
 # Set up a playlog for tracking played tracks
-l = playlog(duration = 14400.0, persistency="/home/pi/Conjurer/persistence.log")
+
+l = playlog(duration = 72000.0, persistency="/home/pi/Conjurer/persistence.log")
 
 # Function to check if a track can be played based on its metadata
 def check(r)
     m = request.metadata(r)
-    if l.last(m) < 3600. then
+    if l.last(m) < 36000. then
         log.info("Rejecting #{m['filename']} (played #{l.last(m)}s ago).")
         false
     else
@@ -33,27 +34,25 @@ requests_queue = request.queue()
 
 # Function to process the request queue and add new requests
 def queue_processing()
-    f1=file.read("/home/pi/Conjurer/request.playlist")
+    text=file.lines("/home/pi/Conjurer/request.playlist")
-    text = f1()
+    if text != [] then
-    if text != "" then
+        list.iter(fun(item) -> requests_queue.push.uri(item), text)
-        requests_queue.push.uri(text)
         file.remove("/home/pi/Conjurer/request.playlist")
         f = file.open("/home/pi/Conjurer/request.playlist", create=true)
         f.close()
-        log.info("done")
-    else
-        log.info("no requests")
     end
 end
 
 # Randomly select a playlist with weights
-s4 = random(id="randomizer", weights=[1, 3, 2], [s1, s2, s3])
+s4 = random(id="randomizer", weights=[2, 3, 5], [s1, s2, s3])
 
 # Load jingles playlist
 jingles = (playlist(reload_mode="watch", "/home/pi/Conjurer/jingles.playlist"))
 
 # Create the main stream with random playlist and jingles
-s = rotate(id="randomizer", weights=[20, 1], [s4, jingles])
+s = rotate(id="randomizer", weights=[10, 1], [s4, jingles])
+
+s = crossfade(fade_out=2., fade_in=2., duration=4., fallback(id="switcher", track_sensitive=false, [requests_queue, s]))
 
 # Set up an interactive harbor for controlling the stream
 interactive.harbor(port = 9999)
@@ -65,21 +64,32 @@ b = bass_boost(frequency=f, gain=g, s)
 s = add([s, b])
 
 # Set up interactive control for main volume
-a = interactive.float("main_volume", min=0., max=20., 1.)
+a = interactive.float("main_volume", min=0., max=20., 0.4)
-s = compress.multiband.interactive(bands=3, s)
+s = compress.multiband.interactive(bands=7, s)
+
+mic_gain = interactive.float("mic_volume", min=0., max=20., 6.)
+# Apply audio processing effects
+tmic = buffer(input.pulseaudio()) # Microphone
+mic = amplify(mic_gain, tmic)
+mic = gate(threshold=-80., range=-120., mic)
+mic = compress(threshold=0., ratio=2.,mic)
+mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic)
+
+s = add([s, mic])
 
 # Apply audio processing effects
 s = nrj(normalize(s))
 s = amplify(a, s)
 
+
 # Skip blank sections in the stream
-s = blank.skip(max_blank=2., s)
+s = blank.skip(max_blank=10., s)
 
 # Configure logging settings
 log_to_stdout = true
 log_to_file = true
 logpath = "/home/pi/Conjurer/radio_log.log"
-loglevel = 4
+loglevel = 3
 set("log.stdout", log_to_stdout)
 set("log.level", loglevel)
 
@@ -89,10 +99,16 @@ set("log.file.path", logpath)
 
 # Set up emergency fallback track
 emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
+radio =  fallback(id="switcher2", track_sensitive=false, [s, emergency])
+# Set up an interactive control for skipping tracks
+p = interactive.bool("Skip track", false)
+
+def http_skip(~protocol, ~data, ~headers, uri)=
+    radio.skip()
+    http.response(code=200,data="Skipped")
+end
+harbor.http.register(port=54321,method="GET","/skip", http_skip)
 
-# Create a fallback stream with the main stream and emergency track
-cross_s = crossfade(fade_out=3., fade_in=3., duration=5., fallback(id="switcher", track_sensitive=false, [requests_queue, s]))
-radio =  fallback(id="switcher2", track_sensitive=false, [cross_s, emergency])
 
 # Function to process the request queue and skip the current track if requested
 def check_skip()
@@ -106,26 +122,19 @@ end
 # Run the queue processing function every 60 seconds
 thread.run(every=60., queue_processing)
 
-# Set up an interactive control for skipping tracks
-p = interactive.bool("r1", false)
 
 # Run the check_skip function every 5 seconds
-thread.run(every=5., check_skip)
+thread.run(every=15., check_skip)
 
-# Handle metadata events
-handle_metadata = fun (m) -> begin
-    log.info(m["title"])
-    log.info(m["artist"])
-    log.info(m["filename"])
-end
-radio.on_metadata(handle_metadata)
 
 # Enable persistent script parameters
 interactive.persistent("script.params")
 
 # Configure output formats and destinations
-output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio)
+
 output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)
+output.pulseaudio(radio)
+#output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio)
 # Uncomment the following lines to enable additional output formats
 # output.icecast(%opus, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="opus-stream", radio)
 # output.icecast(%ogg, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="ogg-stream", radio)

conjurer_musician/stream.html

@@ -0,0 +1,9 @@
+    <html>
+        <head>
+            <title>HTTP Live Streaming Example</title>
+        </head>
+        <body>
+            <video src="/tmp/hls/stream.m3u8" height="300" width="400">
+            </video>
+        </body>
+    </html>

install_main_bot.sh

@@ -1,4 +1,5 @@
 #!/bin/bash
+sudo apt-get install python3-dev
 cd /home/pi
 mdkir Conjurer
 cd Conjurer
@@ -9,6 +10,11 @@ source ./env/bin/activate
 ./env/bin/python3 -m pip install -r requirements_bot.txt
 sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
 sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
+sed -i '1i\import logging' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py
+
+sed -i '21i\         logger = logging.getLogger("discord")' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py
+sed -i '22i\         logger.info("Playlist")' ./env/lib/python3.11/site-packages/spotify_dl/spotify.py
+
 deactivate
 sudo cp ./conjurer.service /etc/systemd/system/
 sudo systemctl daemon-reload

install_music_bot.sh

@@ -32,3 +32,7 @@ sudo cp /home/pi/conjurer/conjurer_musician/radio_service.service /etc/systemd/s
 sudo systemctl daemon-reload
 sudo systemctl start radio_service.service
 sudo systemctl enable radio_service.service
+
+
+sed -i -e 's/os.rename/shutil.copy/g' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py
+sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.py

requirements_bot.txt

@@ -12,4 +12,8 @@ spotipy
 tiktoken
 PyNaCl
 flask
-waitress
+waitress
+clickupython
+assemblyai[extras]
+SpeechRecognition
+git+https://github.com/imayhaveborkedit/discord-ext-voice-recv

restart_radio.sh

@@ -0,0 +1,3 @@
+sshpass -p raspberry ssh pi@192.168.1.15 << 'ENDSSH'
+sudo reboot
+ENDSSH

sr_example.py

@@ -0,0 +1,229 @@
+# -*- coding: utf-8 -*-
+
+from __future__ import annotations
+
+import logging
+
+from discord.ext.voice_recv.sinks import AudioSink
+
+log = logging.getLogger(__name__)
+
+__all__ = [
+    'SpeechRecognitionSink',
+]
+
+try:
+    import discord.ext.speech_recognition as sr  # type: ignore
+except ImportError:
+
+    def SpeechRecognitionSink(**kwargs) -> AudioSink:
+        """A stub for when the SpeechRecognition module isn't found."""
+        raise RuntimeError('The SpeechRecognition module is required to use this sink.')
+
+else:
+    import time
+    import array
+    import asyncio
+    import audioop
+
+    from collections import defaultdict
+
+    from discord.ext.voice_recv.sinks  import SilencePacket
+
+    from typing import TYPE_CHECKING, TypedDict
+
+    if TYPE_CHECKING:
+        from concurrent.futures import Future as CFuture
+        from typing import Literal, Callable, Optional, Any, Final, Protocol, Awaitable, TypeVar
+
+        from discord import Member
+
+        from ..opus import VoiceData
+        from ..types import MemberOrUser as User
+
+        T = TypeVar('T')
+
+        SRRecognizerMethod = Literal[
+            'sphinx',
+            'google',
+            'google_cloud',
+            'wit',
+            'azure',
+            'bing',
+            'lex',
+            'houndify',
+            'amazon',
+            'assemblyai',
+            'ibm',
+            'tensorflow',
+            'whisper',
+            'vosk',
+        ]
+
+        class SRStopper(Protocol):
+            def __call__(self, wait: bool = True, /) -> None:
+                ...
+
+        SRProcessDataCB = Callable[[sr.Recognizer, sr.AudioData, User], Optional[str]]
+        SRTextCB = Callable[[User, str], Any]
+
+    class _StreamData(TypedDict):
+        stopper: Optional[SRStopper]
+        recognizer: sr.Recognizer
+        buffer: array.array[int]
+
+    class SpeechRecognitionSink(AudioSink):  # type: ignore
+        def __init__(
+            self,
+            *,
+            process_cb: Optional[SRProcessDataCB] = None,
+            text_cb: Optional[SRTextCB] = None,
+            default_recognizer: SRRecognizerMethod = 'assemblyai',
+            phrase_time_limit: int = 10,
+            ignore_silence_packets: bool = True,
+        ):
+            super().__init__(None)
+            self.process_cb: Optional[SRProcessDataCB] = process_cb
+            self.text_cb: Optional[SRTextCB] = text_cb
+            self.phrase_time_limmit: int = phrase_time_limit
+            self.ignore_silence_packets: bool = ignore_silence_packets
+
+            self.default_recognizer: SRRecognizerMethod = default_recognizer
+            self._stream_data: defaultdict[int, _StreamData] = defaultdict(
+                lambda: _StreamData(stopper=None, recognizer=sr.Recognizer(), buffer=array.array('B'))
+            )
+
+        def _await(self, coro: Awaitable[T]) -> CFuture[T]:
+            assert self.client is not None
+            return asyncio.run_coroutine_threadsafe(coro, self.client.loop)
+
+        def wants_opus(self) -> bool:
+            return False
+
+        def write(self, user: Optional[User], data: VoiceData) -> None:
+            if self.ignore_silence_packets and isinstance(data.packet, SilencePacket):
+                return
+
+            if user is None:
+                return
+
+            sdata = self._stream_data[user.id]
+            sdata['buffer'].extend(data.pcm)
+
+            if not sdata['stopper']:
+                sdata['stopper'] = sdata['recognizer'].listen_in_background(
+                    DiscordSRAudioSource(sdata['buffer']), self.background_listener(user), self.phrase_time_limmit
+                )
+
+        def background_listener(self, user: User):
+            process_cb = self.process_cb or self.get_default_process_callback()
+            text_cb = self.text_cb or self.get_default_text_callback()
+
+            def callback(_recognizer: sr.Recognizer, _audio: sr.AudioData):
+                output = process_cb(_recognizer, _audio, user)
+                if output is not None:
+                    text_cb(user, output)
+
+            return callback
+
+        def get_default_process_callback(self) -> SRProcessDataCB:
+            def cb(recognizer: sr.Recognizer, audio: sr.AudioData, user: Optional[User]) -> Optional[str]:
+                log.debug("Got %s, %s, %s", audio, audio.sample_rate, audio.sample_width)
+                text: Optional[str] = None
+                try:
+                    # they changed recognize_google to be optionally assigned at runtime...
+                    func = getattr(recognizer, 'recognize_' + self.default_recognizer, recognizer.recognize_google) # type: ignore
+                    text = func(audio)  # type: ignore
+                except sr.UnknownValueError:
+                    log.debug("Bad speech chunk")
+                    # self._debug_audio_chunk(audio)
+
+                return text
+
+            return cb
+
+        def get_default_text_callback(self) -> SRTextCB:
+            def cb(user: Optional[User], text: Optional[str]) -> Any:
+                log.info("%s said: %s", user.display_name if user else 'Someone', text)
+
+            return cb
+
+        @AudioSink.listener()
+        def on_voice_member_disconnect(self, member: Member, ssrc: Optional[int]) -> None:
+            self._drop(member.id)
+
+        def cleanup(self) -> None:
+            for user_id in tuple(self._stream_data.keys()):
+                self._drop(user_id)
+
+        def _drop(self, user_id: int) -> None:
+            data = self._stream_data.pop(user_id)
+
+            stopper = data.get('stopper')
+            if stopper:
+                stopper()
+
+            buffer = data.get('buffer')
+            if buffer:
+                # arrays don't have a clear function
+                del buffer[:]
+
+        def _debug_audio_chunk(self, audio: sr.AudioData, filename: str = 'sound.wav') -> None:
+            import io, wave, discord
+
+            with io.BytesIO() as b:
+                with wave.open(b, 'wb') as writer:
+                    writer.setframerate(48000)
+                    writer.setsampwidth(2)
+                    writer.setnchannels(2)
+                    writer.writeframes(audio.get_wav_data())
+
+                b.seek(0)
+                f = discord.File(b, filename)
+                self._await(self.voice_client.channel.send(file=f))  # type: ignore
+
+    class DiscordSRAudioSource(sr.AudioSource):
+        little_endian: Final[bool] = True
+        SAMPLE_RATE: Final[int] = 48_000
+        SAMPLE_WIDTH: Final[int] = 2
+        CHANNELS: Final[int] = 2
+        CHUNK: Final[int] = 960
+
+        def __init__(self, buffer: array.array[int]):
+            self.buffer = buffer
+            self._entered: bool = False
+
+        @property
+        def stream(self):
+            return self
+
+        def __enter__(self):
+            if self._entered:
+                log.warning('Already entered sr audio source')
+            self._entered = True
+            return self
+
+        def __exit__(self, *exc) -> None:
+            self._entered = False
+            if any(exc):
+                log.exception('Error closing sr audio source')
+
+        def read(self, size: int) -> bytes:
+            # TODO: make this timeout configurable
+            for _ in range(10):
+                if len(self.buffer) < size * self.CHANNELS:
+                    time.sleep(0.1)
+                else:
+                    break
+            else:
+                if len(self.buffer) == 0:
+                    return b''
+
+            chunksize = size * self.CHANNELS
+            audiochunk = self.buffer[:chunksize].tobytes()
+            del self.buffer[: min(chunksize, len(audiochunk))]
+            audiochunk = audioop.tomono(audiochunk, 2, 1, 1)
+            return audiochunk
+
+        def close(self) -> None:
+            self.buffer.clear()

stream_test.py

@@ -0,0 +1,62 @@
+# Start by making sure the `assemblyai` package is installed.
+# If not, you can install it by running the following command:
+# pip install -U assemblyai
+#
+# Then, make sure you have PyAudio installed: https://pypi.org/project/PyAudio/
+#
+# Note: Some macOS users might need to use `pip3` instead of `pip`.
+
+import pyaudio
+import assemblyai as aai
+
+aai.settings.api_key = "aa9962f0088a449a9c4ab2361e96cc08"
+
+def on_open(session_opened: aai.RealtimeSessionOpened):
+  "This function is called when the connection has been established."
+
+  print("Session ID:", session_opened.session_id)
+
+def on_data(transcript: aai.RealtimeTranscript):
+  "This function is called when a new transcript has been received."
+
+  if not transcript.text:
+    return
+
+  if isinstance(transcript, aai.RealtimeFinalTranscript):
+    print(transcript.text, end="\r\n")
+  else:
+    print(transcript.text, end="\r")
+
+def on_error(error: aai.RealtimeError):
+  "This function is called when the connection has been closed."
+
+  print("An error occured:", error)
+
+def on_close():
+  "This function is called when the connection has been closed."
+
+  print("Closing Session")
+
+transcriber = aai.RealtimeTranscriber(
+  on_data=on_data,
+  on_error=on_error,
+  sample_rate=44_100,
+  on_open=on_open, # optional
+  on_close=on_close, # optional
+)
+
+
+pa = pyaudio.PyAudio()
+for i in range(pa.get_device_count()):
+  print (pa.get_device_info_by_index(i))
+
+# Start the connection
+#transcriber.connect()
+
+# Open a microphone stream
+#microphone_stream = aai.extras.MicrophoneStream()
+
+# Press CTRL+C to abort
+#transcriber.stream(microphone_stream)
+
+#transcriber.close()

test_time.py

@@ -0,0 +1,7 @@
+import time
+first_time = time.time_ns()
+
+time.sleep(1)
+time_diff = time.time_ns() - first_time
+print(time_diff)
+#2000149433

utils/update_memory.py

@@ -0,0 +1,33 @@
+import json
+MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
+ENCODING = "utf-8"
+
+messages = [
+    {"role": "user", "content" : "Hammer: All systems operational"},
+    {"role": "assistant", "content": "Aye, Szefie! Wszystkie systemy działają sprawnie. Gotów do dalszej służby! Co podać?"},
+    {"role": "user", "content" : "Hammer: Podaj coś na orzeźwienie i przebudzenie. COś co rozgrzeje moje stare kości po ciężkiej pracy!"},
+    {"role": "assistant", "content": "Z przyjemnością Szefie! Dla Ciebie przygotuję Midzynarodówkę Interstelarną Wolf's Dragoon - wysoce energetyczny koktajl rozgrzewający i pobudzający jednocześnie."+
+                                    "Do przygotowania potrzebuję:"+
+                                    "\n"+
+                                    "60 ml mocnego rumu z Wolfa (lub innego, jeśli wolisz)"+
+                                    "30 ml soku z cytryny"+
+                                    "35 ml domowego syropu z mięty"+
+                                    "15 ml świeżo zmielonej kawy"+
+                                    "\n"+
+                                    "Mieszam wszystko w shakerze z lodem, a potem przelewam przez sitko do kieliszka koktajlowego. W górę serwuję z bigoskiem z miętą i zestem z cytryny. Rozgrzewa, orzeźwia i daje kopa na resztę nocy."+
+                                    "\n"+
+                                    "Co powiesz Szefie, podać Ci taki?"},
+    {"role": "user", "content" : "Hammer: Brzmi jak coś co zrobi robotę! Skal!"},
+]
+
+
+for temp in messages:
+
+    with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
+        # First we load existing data into a dict.
+        file_data = json.load(file_memory)
+        # Join new_data with file_data inside emp_details
+        file_data.append(temp)
+        file_memory.seek(0)
+        # convert back to json.
+        json.dump(file_data, file_memory, indent=4)

voice_example.py

@@ -0,0 +1,47 @@
+# -*- coding: utf-8 -*-
+
+import discord
+from discord.ext import commands, voice_recv
+
+discord.opus._load_default()
+
+bot = commands.Bot(command_prefix=commands.when_mentioned, intents=discord.Intents.all())
+
+class Testing(commands.Cog):
+    def __init__(self, bot):
+        self.bot = bot
+
+    @commands.command()
+    async def test(self, ctx):
+        def callback(user, data: voice_recv.VoiceData):
+            print(f"Got packet from {user}")
+
+            ## voice power level, how loud the user is speaking
+            # ext_data = packet.extension_data.get(voice_recv.ExtensionID.audio_power)
+            # value = int.from_bytes(ext_data, 'big')
+            # power = 127-(value & 127)
+            # print('#' * int(power * (79/128)))
+            ## instead of 79 you can use shutil.get_terminal_size().columns-1
+
+        vc = await ctx.author.voice.channel.connect(cls=voice_recv.VoiceRecvClient)
+        vc.listen(voice_recv.BasicSink(callback))
+
+    @commands.command()
+    async def stop(self, ctx):
+        await ctx.voice_client.disconnect()
+
+    @commands.command()
+    async def die(self, ctx):
+        ctx.voice_client.stop()
+        await ctx.bot.close()
+
+@bot.event
+async def on_ready():
+    print('Logged in as {0.id}/{0}'.format(bot.user))
+    print('------')
+    await bot.add_cog(Testing(bot))
+
+REMOTE_HOST_NAME = "discord"
+
+
+bot.run("NzUxNDQxODgzNDA2MDA4NDEw.Go--wb.Nr28Oo4eYAP1p2XFVHF5uIQfcPD7s_jO2NfMCQ")

voice_recognition.py

@@ -0,0 +1,233 @@
+import logging
+import wave
+import asyncio
+import assemblyai as aai
+import discord
+from discord.ext import commands, voice_recv, tasks
+from discord.opus import Decoder as OpusDecoder
+import time
+# Replace with your API key
+aai.settings.api_key = "aa9962f0088a449a9c4ab2361e96cc08"
+discord.opus._load_default()
+CHANNELS = OpusDecoder.CHANNELS
+SAMPLE_WIDTH = OpusDecoder.SAMPLE_SIZE // OpusDecoder.CHANNELS
+SAMPLING_RATE = OpusDecoder.SAMPLING_RATE
+
+#rotate file after there is 0.5s between last received pcm for user.
+#delete messsages after user disconnect
+
+logger = logging.getLogger("discord")
+location = "/home/pi/Conjurer/transcripts/"
+
+class CommunicationObject:
+    def __init__(self, msg_type, user, data):
+        self.type = msg_type
+        self.user = user
+        self.data = data
+    def __repr__(self):
+        return f"{self.type} : {self.user} : {self.data}"
+
+class WaveWriter:
+    def __init__(self, user_id, queue):
+        self.queue = queue
+        self.user = user_id
+        self.username = str(user_id.id)
+        #here we can add hashing function to make transcription files not possible to be connected with discord id
+        self.present_file_id = 0
+        self.file_name_past = None
+        self.file_name_present = location + self.username + "_" + str(self.present_file_id) + ".mp3"
+        self.transcript_file_present : wave.Wave_write = wave.open(
+            self.file_name_present, "wb"
+        )
+        self.transcript_file_present.setnchannels(CHANNELS)
+        self.transcript_file_present.setsampwidth(SAMPLE_WIDTH)
+        self.transcript_file_present.setframerate(SAMPLING_RATE)
+        self.file_name_future = location + self.username + "_" + str(self.present_file_id + 1) + ".mp3"
+        self.transcript_file_future : wave.Wave_write = wave.open(
+            self.file_name_future, "wb"
+        )
+        self.transcript_file_future.setnchannels(CHANNELS)
+        self.transcript_file_future.setsampwidth(SAMPLE_WIDTH)
+        self.transcript_file_future.setframerate(SAMPLING_RATE)
+
+    async def rotate(self):
+
+        self.transcript_file_present.close()
+        self.file_name_past = self.file_name_present
+        if self.present_file_id > 10:
+            self.present_file_id = 0
+        else:
+            self.present_file_id += 1
+        self.file_name_present = self.file_name_future
+        self.transcript_file_present = self.transcript_file_future
+        self.file_name_future = location + self.username + "_" + str(self.present_file_id + 1) + ".mp3"
+        self.transcript_file_future : wave.Wave_write = wave.open(
+            self.file_name_future, "wb"
+        )
+        self.transcript_file_future.setnchannels(CHANNELS)
+        self.transcript_file_future.setsampwidth(SAMPLE_WIDTH)
+        self.transcript_file_future.setframerate(SAMPLING_RATE)
+        operation = CommunicationObject(msg_type="send_file",user=self.user,data=[self.file_name_past])
+
+        await self.queue.put(operation)
+
+    def writeframes(self, pcmdata):
+        self.transcript_file_present.writeframes(pcmdata)
+
+    def cleanup(self):
+        logger.info("Cleanup for user %s", self.username)
+        self.transcript_file_present.close()
+        self.transcript_file_future.close()
+
+
+class SRBuffer(voice_recv.AudioSink):
+    """Endpoint AudioSink that generates a wav file.
+    Best used in conjunction with a silence generating sink. (TBD)
+    """
+    # on member join dodajemytypa do listy
+    # on member disconnect - dropujemy go
+    def __init__(self, queue):
+        super().__init__()
+        self.queue = queue
+        self.wavewriter = {}
+
+    def on_user_connect(self, username):
+        self.wavewriter[str(username.id)] = [WaveWriter(username, self.queue), time.time_ns(), time.time_ns(), False, False]
+
+    def on_user_disconnect(self,username):
+        self.wavewriter[str(username.id)].cleanup()
+        self.wavewriter.pop(str(username.id))
+
+    def wants_opus(self) -> bool:
+        return False
+
+    def write(self, user, data) -> None:
+        #logger.info("DAta write")
+        if user:
+            self.wavewriter[str(user.id)][0].writeframes(data.pcm)
+            self.wavewriter[str(user.id)][1] = time.time_ns() #time from last write
+            self.wavewriter[str(user.id)][3] = True #data written in last iter
+            self.wavewriter[str(user.id)][4] = True #data in buff
+            time.sleep(0.001)
+
+    def cleanup(self) -> None:
+        try:
+            logger.info("Cleanup for SRBuffer")
+            for item in self.wavewriter.values():
+                item[0].cleanup()
+        except Exception:
+            logger.warning("WaveSink got error closing file on cleanup", exc_info=True)
+
+
+class Transcriber(commands.Cog):
+    def __init__(self, bot):
+        self.bot = bot
+        self.threads = []
+        self.comm_queue = asyncio.Queue()
+        self.wsink = SRBuffer(self.comm_queue)
+        self.worker = None
+        self.mc = None
+        self.vc = None
+
+    @commands.hybrid_command(name="transcribe")
+    async def test(self, ctx):
+        if self.vc:
+            vc = self.vc #to juz powinien byc voice channel z funkcja conenct
+        else:
+            vc = None
+        self.mc = ctx.message.channel
+        if self.bot.voice_clients:
+            if  isinstance(self.bot.voice_clients[0], voice_recv.VoiceRecvClient):
+                logger.debug("Already transcribing")
+            else:
+                logger.debug("Already connected with other client")
+        else:
+            logger.debug("Connected")
+            vc = await ctx.author.voice.channel.connect(cls=voice_recv.VoiceRecvClient)
+            logger.info(self.bot.voice_clients)
+            self.check_data.start()
+            self.worker = asyncio.create_task(self.transcribe_output_queue())
+            vc.listen(self.wsink)
+
+
+    @tasks.loop(seconds=0.5)
+    async def check_data(self):
+        for item in self.wsink.wavewriter.values():
+            if not item[3]:
+                timediff_rotation = time.time_ns() - item[2]
+                timediff_write = time.time_ns() -item[1]
+                if  timediff_rotation > 13000149433 and timediff_write > 500014943 and item[4]:
+                    logger.info("File rotation time since last write %s %s", timediff_write, timediff_write/1e9)
+                    logger.info("File rotation time since last rotation %s %s",timediff_rotation, timediff_rotation/1e9)
+                    item[2] = time.time_ns()
+                    item[4] = False
+                    await item[0].rotate()
+            item[3] = False
+
+    @commands.Cog.listener()
+    async def on_voice_state_update(self, user, before, after):
+        if user == self.bot.user:
+            logger.debug("Ignoring self")
+            return
+
+        if before.channel is None:
+            logger.info("User %s connected to channel %s", user, after.channel.name)
+            self.wsink.on_user_connect(user)
+        elif after.channel is  None:
+            logger.info("User %s disconnected from channel %s", user, before.channel.name)
+            operation = CommunicationObject(msg_type="user_cleanup", user=user, data=None)
+            await self.comm_queue.put(operation)
+        else:
+            logger.debug("User VC status changed %s", user.id)
+            logger.debug("Before %s", before)
+            logger.debug("After %s", after)
+
+
+    @commands.command(name="stop_transcribe")
+    async def stop(self, ctx):
+        self.check_data.stop()
+        stop_token = CommunicationObject("STOP",None,None)
+
+        await self.comm_queue.put(stop_token)
+        await ctx.voice_client.disconnect()
+
+    async def transcribe_output_queue(self):
+        logger.info("Transcript uploader start")
+        config = aai.TranscriptionConfig(language_code="pl")
+        transcriber = aai.Transcriber()
+        logger.debug("Transcriber queue id: %s",id(self.comm_queue))
+
+        while True:
+            logger.debug("waiting for tasks")
+            item = await self.comm_queue.get()
+            logger.debug("Got %s", item)
+            if "STOP" in item.type:
+                logger.debug("Queue ended")
+                break
+            elif "send_file" in item.type:
+                logger.debug("Sending file for transcription")
+                transcript = None
+                for iter in item.data:
+                    try:
+                        coro = asyncio.to_thread(
+                            transcriber.transcribe,
+                            iter,
+                            config=config
+                        )
+                        transcript = await coro
+                    except Exception as e:
+                        logger.warn("Exceptiom occured %s", e)
+                    await self.mc.send(f"{item.user} : {transcript.text}")
+                    if transcript.error:
+                        logger.error(transcript.error)
+                        raise AssertionError
+            elif "user_cleanup" in item.type:
+                logger.info("User %s disconnected - cleanup action")
+            else:
+                logger.warn("Something went wrong with object %s", item)
+        logger.info("Transcript uploader stoped")
+
+async def setup(bot):
+    logger = logging.getLogger("discord")
+    await bot.add_cog(Transcriber(bot))
+    logger.info("Loading voice transcribed module done")