Change in weights

,

.gitignore

@@ -197,3 +197,7 @@ node_modules/prettier/esm/parser-postcss.mjs
 node_modules/prettier/esm/parser-typescript.mjs
 node_modules/prettier/esm/parser-yaml.mjs
 node_modules/prettier/esm/standalone.mjs
+cr_results.json
+not_in_db.json
+rr_results.json
+s_results.json

.trunk/configs/.shellcheckrc

@@ -0,0 +1,7 @@
+enable=all
+source-path=SCRIPTDIR
+disable=SC2154
+
+# If you're having issues with shellcheck following source, disable the errors via:
+# disable=SC1090
+# disable=SC1091

.trunk/trunk.yaml

@@ -2,34 +2,38 @@
 # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
 version: 0.1
 cli:
-  version: 1.20.1
+  version: 1.22.2
 # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
 plugins:
   sources:
     - id: trunk
-      ref: v1.4.5
+      ref: v1.6.0
       uri: https://github.com/trunk-io/plugins
 # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
 runtimes:
   enabled:
+    - go@1.21.0
     - node@18.12.1
     - python@3.10.8
 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
 lint:
   enabled:
-    - actionlint@1.6.27
+    - shellcheck@0.10.0
-    - bandit@1.7.8
+    - shfmt@3.6.0
-    - black@24.3.0
+    - taplo@0.8.1
+    - actionlint@1.7.1
+    - bandit@1.7.9
+    - black@24.4.2
     - codespell
-    - checkov@3.2.48
+    - checkov@3.2.139
     - git-diff-check
     - isort@5.13.2
     - pylint
-    - markdownlint@0.39.0
+    - markdownlint@0.41.0
-    - prettier@3.2.5
+    - prettier@3.3.2
-    - ruff@0.3.4
+    - ruff@0.4.9
-    - trivy@0.50.1
+    - trivy@0.52.2
-    - trufflehog@3.71.0
+    - trufflehog@3.78.1
     - yamllint@1.35.1
 actions:
   disabled:

bot.py

@@ -19,6 +19,7 @@ import shutil
 import sys
 import threading
 import uuid
+import subprocess
 from datetime import datetime
 from logging import handlers
 from pathlib import Path, PurePath
@@ -44,7 +45,7 @@ import yt_dlp
 from spotify_dl import spotify
 from spotify_dl import youtube as youtube_download
 
-from communication_subroutine import comm_subroutine, IN_COMM_Q, OUT_COMM_Q
+from communication_subroutine import comm_subroutine, IN_COMM_Q, OUT_COMM_Q, QueryControl
 
 # *=========================== Classes definition
 
@@ -62,22 +63,6 @@ Music_Config = TypedDict(
 )
 
 
-class QueryControl:
-    """
-    This class `QueryControl` is used to manage queries with information about the author, UUID,
-    content, logger, context, and replies.
-    """
-    def __init__(self, query_author, query_uuid, query_content, uplogger, ctx) -> None:
-        self.author = query_author
-        self.uuid = query_uuid
-        self.content = query_content
-        self.logger = uplogger
-        self.stop = False
-        self.ctx = ctx
-        self.logger.info(
-            f"Created Query control for {self.author}, {self.uuid}: {self.content}"
-        )
-        self.replies = []
 
 class MusicFileList(object):
     """
@@ -131,43 +116,6 @@ class MusicFileList(object):
         post_data = {"item": str(item)}
         requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360)
 
-class DoSearchView(discord.ui.View):
-    def __init__(self, _uuid, result_list, *args, **kwargs) -> None:
-        super().__init__(*args, **kwargs)
-        self.uuid = _uuid
-        self.result_list = result_list
-
-    @discord.ui.button(
-        label="Nastepna strona", style=discord.ButtonStyle.primary, row=0
-    )
-    async def next_callback(self, interaction, button):
-        # emoji = await ctx.guild.fetch_emoji(id)
-        # button.emoji=1102174413887111189
-        logger.info("Pressed")
-        await interaction.response.edit_message(
-            content="You clicked the button!"
-        )  # Send a message when the button is clicked
-
-    @discord.ui.button(label="Wyszukaj wiecej", style=discord.ButtonStyle.danger, row=0)
-    async def reset_callback(self, interaction, button):
-        # emoji = await ctx.guild.fetch_emoji(id)
-        # button.emoji=1102174413887111189
-        logger.info("Pressed")
-        await interaction.response.edit_message(
-            content="You clicked the button!"
-        )  # Send a message when the button is clicked
-
-    @discord.ui.button(
-        label="Poprzednia Strona", style=discord.ButtonStyle.primary, row=0
-    )
-    async def perv_callback(self, interaction, button):
-        # emoji = await ctx.guild.fetch_emoji(id)
-        # button.emoji=1102174413887111189
-        logger.info("Pressed")
-        await interaction.response.edit_message(
-            content="You clicked the button!"
-        )  # Send a message when the button is clicked
-
 
 # *=========================================== Predefines
 formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
@@ -188,10 +136,16 @@ MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE = 15
 MUZYKA_MOJEGO_LUDU_PLAJLISTA = 30
 
 FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000"
+RADIO_HARBOR_ADDRESS = "http://192.168.1.15:54321"
+SKIP_TRACK = "/skip"
+
 GET_MP3 = "/mp3"
 SEND_MP3 = "/update_mp3"
 GET_PLAYLIST = "/get_music"
 ADD_TO_PRIO_PLAYLIST = "/add_to_priority"
+CREATE_PRIO_PLAYLIST = "/create_priority_playlist"
+
+REQUEST_MUSIC = "/request_radio_file"
 CLEAR_PRIO = "/clear_pr_pls"
 LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001"
 SEND_QUERY = "/query"
@@ -351,9 +305,6 @@ async def on_ready():
 
 
 @client.event
-# trunk-ignore(pylint/R0915)
-# trunk-ignore(pylint/R0914)
-# trunk-ignore(pylint/R0912)
 async def on_message(message):
     """
     Handle incoming messages in a Discord server, perform various
@@ -607,8 +558,6 @@ async def disconnect(ctx):
             await voice_client.disconnect()
 
 
-# trunk-ignore(pylint/R0915)
-# trunk-ignore(pylint/R0912)
 async def play(ctx, zamawial=None, arg=None):
     """
     Play a music file, retrieve metadata about the file, and send a
@@ -728,31 +677,41 @@ async def check_music():
             await asyncio.sleep(2)
 
 @tasks.loop(seconds=3)
-# trunk-ignore(pylint/R0915)
 async def check_data_q():
+    """
+    This function checks the data queue for any new entries and processes them accordingly.
+    """
     try:
-        data = IN_COMM_Q.get(block=False)
+        fresh_data = IN_COMM_Q.get(block=False)
         entries = []
-        if data.stop:
+        if fresh_data.stop:
-            searcher = data.author
+            searcher = fresh_data.author
-            ctx = data.ctx
+            query = fresh_data.content
-            query = data.content
             l_p = 1
-            for doi in data.entries:
+            for doi in fresh_data.entries:
                 logger.info(doi)
-                desc = data.entries[doi]
+                desc = fresh_data.entries[doi]
                 title = desc['Title'][0]
                 entries.append(f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n")
                 l_p+=1
             message = '*Z podłogi wysuwa się winda na książki*'
-            message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query}'
+            if fresh_data.ctx is not None:
+                ctx = fresh_data.ctx
+                message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query} nr {fresh_data.uuid}'
+            else:
+                ctx = client.get_channel(1062047571557744721)
+                message += f' Mam wynik dla zapytania {fresh_data.uuid} ale że Hammer coś grzebał nie pamiętam kto to chciał i co chciał'
+
+
             message += 'nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku '
             if len(entries)<1:
                 message += "niestety nie ma nic"
+                await ctx.send(message)
             elif len(entries)>=1 and len(entries)<5:
                 message += " znajduje się coś:\n"
                 for item in entries:
                     message+=item
+                await ctx.send(message)
             else:
                 message+= " znajduje się cholernie dużo:\n"
                 for item in entries:
@@ -761,20 +720,17 @@ async def check_data_q():
                         await ctx.send(message)
                         message = ""
 
-            await ctx.send(message)
             #Kept for sentimental reasons
             #await ctx.send(f"O. A tak będzie wyglądało coś ciekawego w przyszłości: {data}")
     except Empty:
         pass
 
 @tasks.loop(seconds=120)
-# trunk-ignore(pylint/R0915)
 async def check_self():
     """
     The function `check_data` periodically checks for conditions to send messages and manage log files
     in a Discord channel.
     """
-    # trunk-ignore(codespell/misspelled)
     # logger.info("Heartbeat of cleanup proc")
     channel = client.get_channel(1062047367337095268)
     messages = [message async for message in channel.history(limit=1)]
@@ -837,9 +793,7 @@ async def check_self():
                 await channel.send(
                     "*Przeciąga się za barem* No dobra - porobione to można dalej pi... *Zauważa spojrzenie Hammera* eprzyć o głupotach z klientami... Szefie... *Bierze 'ukradkowy' łyk z piersiówki*"
                 )
-    # trunk-ignore(pylint/W0603)
     global LAST_SPONTANEOUS_CALL
-    # trunk-ignore(pylint/W0603)
     global TIME_BETWEEN_CALLS
     tdelta = datetime.now() - LAST_SPONTANEOUS_CALL
     tdelta = tdelta.total_seconds()
@@ -875,7 +829,6 @@ async def get_random_cyclic_message():
         mess_key = list(cyclic_words.keys())[messnum]
         return cyclic_words[mess_key][0]
     ai_check2 = random.randint(0, 10)
-    # trunk-ignore(pylint/W0603)
     global MESSAGE_TABLE
     if ai_check2 < 6:
         logger.info("Dykteryjka")
@@ -909,10 +862,6 @@ async def get_random_cyclic_message():
     return result
 
 
-# trunk-ignore(pylint/R0912)
-# trunk-ignore(pylint/R0913)
-# trunk-ignore(pylint/R0914)
-# trunk-ignore(pylint/R0915)
 async def handle_response(
     prompt, vykidailo, bartender, history, username, request_type
 ):
@@ -986,17 +935,17 @@ async def handle_response(
         "Rozmiar zapytania przed dodaniem historii %s", chat_gpt_config_request_size
     )
     if request_type == "MUSIC":
-        algorithm = "gpt-3.5-turbo-0125"
+        algorithm = "gpt-4o"
         table = message_table_muzyka
         token_amount = 10700
     elif request_type == "RANDOM":
-        algorithm = "gpt-3.5-turbo-0125"
+        algorithm = "gpt-4o"
         table = MESSAGE_TABLE
         token_amount = 10700
     else:
         table = MESSAGE_TABLE
-        algorithm = "gpt-4"
+        algorithm = "gpt-4o"
-        token_amount = 7000
+        token_amount = 10700
 
     prompt_gpt_request_size = num_tokens_from_string(
         {"role": "user", "content": final_prompt}, "gpt-4"
@@ -1078,31 +1027,36 @@ async def handle_response(
     logger.info(result)
     temp = {"role": "assistant", "content": result}
     history.append(temp)
-    if request_type:
+    if request_type == "MUSIC":
-        with open(MEMORY_FIVE_MUZYKA, "r+", encoding=ENCODING) as file:
+        with open(MEMORY_FIVE_MUZYKA, "r+", encoding=ENCODING) as file_music_memory:
             # First we load existing data into a dict.
-            file_data = json.load(file)
+            file_data = json.load(file_music_memory)
             # Join new_data with file_data inside emp_details
             file_data.append(temp)
-            # Sets file's current position at offset.
+            file_music_memory.seek(0)
-            file.seek(0)
             # convert back to json.
-            json.dump(file_data, file, indent=4)
+            json.dump(file_data, file_music_memory, indent=4)
+    elif request_type == "RANDOM":
+        with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
+            # First we load existing data into a dict.
+            file_data = json.load(file_memory)
+            # Join new_data with file_data inside emp_details
+            file_data.append(temp)
+            file_memory.seek(0)
+            # convert back to json.
+            json.dump(file_data, file_memory, indent=4)
     else:
-        with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file:
+        with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
             # First we load existing data into a dict.
-            file_data = json.load(file)
+            file_data = json.load(file_memory)
             # Join new_data with file_data inside emp_details
             file_data.append(temp)
-            # Sets file's current position at offset.
+            file_memory.seek(0)
-            file.seek(0)
             # convert back to json.
-            json.dump(file_data, file, indent=4)
+            json.dump(file_data, file_memory, indent=4)
     return result, MESSAGE_TABLE
 
 
-# trunk-ignore(pylint/R0915)
-# trunk-ignore(pylint/R0914)
 async def get_file(ctx, source, link):
     """
     Take in three parameters: "ctx" (context),
@@ -1124,11 +1078,15 @@ async def get_file(ctx, source, link):
     if source == "Spotify":
         dir_path = None
         item_type, item_id = spotify.parse_spotify_url(link)
-        if item_id:
+        if item_type in ["album", "track", "playlist"] and item_id:
-            coro = asyncio.to_thread(spotify.fetch_tracks, args=(spotify_ctrl, item_type, link))
+            logger.info(item_type)
-            file_list = await coro
+            logger.info(item_id)
-            coro = asyncio.to_thread(spotify.get_item_name, args=(spotify_ctrl, item_type, item_id))
+            logger.info("Spotify link provided")
-            directory_name = await coro
+            file_list = await asyncio.to_thread(spotify.fetch_tracks, spotify_ctrl, item_type, link)
+            logger.info(file_list)
+            directory_name = await asyncio.to_thread(spotify.get_item_name, spotify_ctrl, item_type, item_id)
+            logger.info(directory_name)
+            logger.info("Spotify scrape done")
             url_data = {"urls": []}
             url_dict = {}
             url_dict["save_path"] = Path(
@@ -1138,6 +1096,7 @@ async def get_file(ctx, source, link):
             url_dict["songs"] = file_list
             url_data["urls"].append(url_dict.copy())
             file_name_f = youtube_download.default_filename
+            logger.info("YT-DL")
             coro = asyncio.to_thread(
                 youtube_download.download_songs,
                 songs=url_data,
@@ -1153,6 +1112,7 @@ async def get_file(ctx, source, link):
                 proxy="",
             )
             _ = await coro
+            logger.info("YT-DL done")
             dir_path = (url_dict["save_path"].resolve()).as_posix()
             if platform == "win32":
                 dir_path = dir_path.replace("/", "\\")
@@ -1323,9 +1283,6 @@ async def get_stats(ctx, history_limit):
             yield name, how_many
 
 
-# trunk-ignore(pylint/R0915)
-# trunk-ignore(pylint/R0912)
-# trunk-ignore(pylint/R0914)
 async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None):
     """
     Take in a context object and an optional integer parameter "how_many" and perform search
@@ -1394,6 +1351,46 @@ async def max_weight(lista):
 
 # *=========================================== Define Commands
 
+@client.hybrid_command(
+    name="skip_track",
+    description="Przeskocz kawałek w radiu",
+    guild=discord.Object(id=664789470779932693),
+)
+async def skip_track(ctx):
+    async with ctx.typing():
+        allowed = False
+        for role in ctx.author.roles:
+            if role.name in ("Thane","Jarl"):
+                allowed = True
+        if not allowed:
+            await ctx.send("Łapy precz od radia")
+        else:
+            coroutine = asyncio.to_thread(
+            requests.get,
+            f"{RADIO_HARBOR_ADDRESS}{SKIP_TRACK}",
+            timeout=360,
+            )
+            result = await coroutine
+            logger.info("Done %s", result)
+            await ctx.send("Zrobione szefie!")
+
+
+@client.hybrid_command(
+    name="zrestartuj_radio",
+    description="Komenda ktora uruchamia radio ponownie jakby się zawiesiło",
+    guild=discord.Object(id=664789470779932693),
+)
+async def zrestartuj_radio(ctx):
+    async with ctx.typing():
+        allowed = False
+        for role in ctx.author.roles:
+            if role.name in ("Thane","Jarl"):
+                allowed = True
+        if not allowed:
+            await ctx.send("Łapy precz od radia")
+        else:
+            retcode = subprocess.run("/home/pi/Conjurer/restart_radio.sh", shell=False, check = False, capture_output = True)
+            logger.info("Wynik: %s",retcode)
 
 @client.hybrid_command(
     name="dej_co_ze_spotifaja",
@@ -1485,8 +1482,6 @@ async def dej_co_z_jutuba(ctx):
     description="Jeśli nie wiesz jak użyć tej komendy to nawet nie próbuj",
     guild=discord.Object(id=664789470779932693),
 )
-# trunk-ignore(pylint/R0914)
-# trunk-ignore(pylint/R0912)
 async def update_banlist(ctx):
     """
     Update a banlist in a Discord guild from preprovided list in channel.
@@ -1638,6 +1633,40 @@ async def graj_muzyko(ctx):
         logger.info("Press play on tape completed")
         check_music.start()
 
+@client.hybrid_command(
+    name="radio_hardkor",
+    description="Włącza radio Conjurer na kanale #nocna-zmiana.",
+    guild=discord.Object(id=664789470779932693),
+)
+async def radio_hardkor(ctx):
+    """
+    Plays the radio hardkor stream in the voice channel.
+
+    Args:
+        ctx: The context object representing the invocation context.
+
+    Returns:
+        None
+    """
+    logger.info("Press play on radio hardkor")
+    async with ctx.typing():
+        global MUZYKA  # pylint: disable=global-variable-not-assigned
+        MUZYKA["ctx"] = ctx
+        if not client.voice_clients:
+            await connect(ctx=ctx)
+            voice_client = client.voice_clients[0]
+            voice_client.play(
+                discord.PCMVolumeTransformer(
+                discord.FFmpegPCMAudio("http://95.175.16.246:666/mp3-stream"), volume=0.3
+                )
+                )
+            logger.info("Press play on radio completed")
+        else:
+            logger.error("Already playing")
+        await asyncio.sleep(12)
+        check_music.start()
+
+
 
 @client.hybrid_command(
     name="cisza",
@@ -1812,7 +1841,77 @@ async def dodaj_do_ulubionych(ctx):
             json=jrequest,
             timeout=360,
         )
-        _ = await coroutine
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
+@client.hybrid_command(
+    name="ja_chciol",
+    description="Dodaje do listy ulubionych w radiu",
+    guild=discord.Object(id=664789470779932693),
+)
+async def request_radio(ctx):
+    """
+    Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
+    Rest of the line are search terms.
+
+    :param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
+    represents the context in which the command was invoked, including information such as the message,
+    the channel, the server, and the user who invoked the command. This parameter is required in all
+    Discord.py commands
+    """
+    async with ctx.typing():
+        logger.info("Zaczynam szukać timestamp %s", datetime.now())
+
+        word_list = ctx.message.content.split()
+        jrequest = {
+            "lista_slow": word_list,
+            "UUID": str(uuid.uuid4()),
+        }
+        coroutine = asyncio.to_thread(
+            requests.post,
+            f"{FILE_SERVICE_ADDRESS}{REQUEST_MUSIC}",
+            json=jrequest,
+            timeout=360,
+        )
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
+
+@client.hybrid_command(
+    name="stworz_audycje",
+    description="Dodaje do listy ulubionych w radiu",
+    guild=discord.Object(id=664789470779932693),
+)
+async def stworz_audycje(ctx):
+    """
+    Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
+    Rest of the line are search terms.
+
+    :param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
+    represents the context in which the command was invoked, including information such as the message,
+    the channel, the server, and the user who invoked the command. This parameter is required in all
+    Discord.py commands
+    """
+    async with ctx.typing():
+        logger.info("Zaczynam szukać timestamp %s", datetime.now())
+
+        dlugosc_playlisty = ctx.message.content.split()[1]
+        word_list = ctx.message.content.split()
+        jrequest = {
+            "lista_slow": word_list,
+            "dlugosc_plejlisty": dlugosc_playlisty,
+            "UUID": str(uuid.uuid4()),
+        }
+        coroutine = asyncio.to_thread(
+            requests.post,
+            f"{FILE_SERVICE_ADDRESS}{CREATE_PRIO_PLAYLIST}",
+            json=jrequest,
+            timeout=360,
+        )
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
+
 
 @client.hybrid_command(
     name="wyczysc_ulubione",
@@ -1831,12 +1930,13 @@ async def wyczysc_ulubione(ctx):
     """
     async with ctx.typing():
         coroutine = asyncio.to_thread(
-            requests.post,
+            requests.get,
             f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}",
-            json={},
             timeout=360,
         )
-        _ = await coroutine
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
 
 
 @client.hybrid_command(
@@ -2058,8 +2158,6 @@ async def get_image_stable_diffusion(ctx):
     description="Gra muzyke wyszukana na bazie tematow konwersacji na kanale NZ",
     guild=discord.Object(id=664789470779932693),
 )
-# trunk-ignore(pylint/R0912)
-# trunk-ignore(pylint/R0915)
 async def zagraj_muzyke_mego_ludu(ctx):
     """
     Play completeley random playlist based on messaging history.
@@ -2209,11 +2307,20 @@ async def parametry_muzyki_mego_ludu(
     guild=discord.Object(id=664789470779932693),
 )
 async def wyszukaj_linki_do_dokumentow(ctx):
+    """
+    The function `wyszukaj_linki_do_dokumentow` searches for links to documents in a crossref database and provides links to scihub.
+
+    :param ctx: The `ctx` parameter is typically used in Discord.py, a Python library for creating Discord bots.
+    It represents the context of the command being executed, including information about the message, the server,
+    and the user who invoked the command.
+    """
     query = ctx.message.content
     query_uuid = uuid.uuid4()
     #TODO: TESTING ONLY!!
     #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
-    json_query = {"UUID": str(query_uuid), "query": str(query)[30:], "page": 1}
+    ctx.message.content = ctx.message.content.replace("$wyszukaj_linki_do_dokumentow", "")
+
+    json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": False}
     coroutine = asyncio.to_thread(
         requests.post,
         f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
@@ -2222,8 +2329,8 @@ async def wyszukaj_linki_do_dokumentow(ctx):
     )
     await ctx.send(
         "*Conjurer powoli notuje podane przez Ciebie. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" +
-        " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to te 3/4 stacji."+
+        " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to 3/4 stacji."+
-        " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc te 12/8 stacji teraz będzie ciężko pracować"
+        " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc 12/8 stacji teraz będzie ciężko pracować"
     )
     query_response = await coroutine
     if not query_response.status_code == 200:
@@ -2245,6 +2352,87 @@ async def wyszukaj_linki_do_dokumentow(ctx):
     await ctx.send(f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce."+
     " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni.")
 
+@client.hybrid_command(
+    name="glebokie_gardlo",
+    description="Przygotowuje drinka o nazwie głębokie gardło",
+    guild=discord.Object(id=664789470779932693),
+)
+async def wyszukaj_linki_do_dokumentow_deep(ctx):
+    """
+    The function `wyszukaj_linki_do_dokumentow_deep` performs a deep search for links to documents in
+    a crossref database and provides links to scihub.
+
+    :param ctx: The `ctx` parameter is typically used in Discord.py, a Python library for creating Discord bots.
+    It represents the context of the command being executed, including information about the message, the server,
+    and the user who invoked the command.
+    """
+    # TODO: Implement deep search logic here
+    query = ctx.message.content.replace("$glebokie_gardlo", "")
+    allowed = False
+    for role in ctx.message.author.roles:
+        if role.name == "Bartender":
+            allowed = True
+        if role.name == "Scribe":
+            allowed = True
+        if role.name == "Thane":
+            allowed = True
+    if not allowed:
+        if ctx.message.author.nick:
+            username = ctx.message.author.nick
+        else:
+            username = ctx.message.author.name
+        vykidailo = False
+        bartender = False
+        prompt = "Przygotuj mi drinka o nazwie Głębokie Gardło, inspirowanego tym sławnym filmem oraz skandalem Watergate"
+        for role in ctx.message.author.roles:
+            if role.name == "Vykidailo":
+                vykidailo = True
+            if role.name == "Bartender":
+                bartender = True
+        global MESSAGE_TABLE  # pylint: disable=global-statement
+
+        result, MESSAGE_TABLE = await handle_response(
+            prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION"
+        )
+        await ctx.send(result)
+        return
+
+    query_uuid = uuid.uuid4()
+    #TODO: TESTING ONLY!!
+    #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
+    json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": True}
+    coroutine = asyncio.to_thread(
+        requests.post,
+        f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
+        json=json_query,
+        timeout=360,
+    )
+    await ctx.send(
+        "*Conjurer mruga okiem i zamiast drinka wyjmuje dysk usb z terminala. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" +
+        " wrzuca ten dysk do środk* To teraz trza poczekać kilka godzin. TO będzie głębokie wyszukanie"+
+        " Rozgląda się dookoła i stawia przed tobą mętnego drinka o zapachu mięty, gwiezdnego pyłu i oleju silnikowego"
+    )
+    query_response = await coroutine
+    if not query_response.status_code == 200:
+        await ctx.send("*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."+
+        " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało")
+        return
+
+    query, query_uuid, queue_size = (
+        query_response.json()["data"][0],
+        query_response.json()["data"][1],
+        query_response.json()["data"][2],
+    )
+    if ctx.message.author.nick:
+        username = ctx.message.author.nick
+    else:
+        username = ctx.message.author.name
+    query_object = QueryControl(username, query_uuid, query, logger, ctx)
+    OUT_COMM_Q.put(query_object)
+    await ctx.send(f"Wypij wypi {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce niestety kolejka obowiazuje zawsze."+
+    " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzin - ale mogą być też dni. Głębokie zabawy trwają dłużej. *Znowu mruga*")
+
+
 
 @client.hybrid_command(
     name="krecimy_pornola",
@@ -2275,7 +2463,7 @@ async def krecimy_pornola(ctx):
                     sciezka, files = await get_file(ctx, "Pornol", item_yt)
                     if files:
                         logger.info("Pornol udany")
-                        ctx.reply(f"Jest w tajnym archiwum pod adresem{sciezka})")
+                        ctx.send(f"Jest w tajnym archiwum pod adresem{sciezka})")
     else:
         ctx.reply("Jak ładnie szefa poprosisz.")
 
@@ -2289,21 +2477,14 @@ if __name__ == "__main__":
     threads.append(threading.Thread(target=client.run, args=(TOKEN,)))
     threads.append(threading.Thread(target=comm_subroutine, args=(logger,)))
     logger.info("Starting discord bot: Starting threads")
-    i = 0
+    WRK_CNT = 0
     for worker in threads:
-        i+=1
+        WRK_CNT+=1
-        logger.info("Starting discord bot: Starting thread {i}")
+        logger.info("Starting discord bot: Starting thread %s", WRK_CNT)
         worker.start()
     logger.info("Starting discord bot: Joining threads")
-    i=0
+    WRK_CNT=0
     for worker in threads:
-        i+=1
+        WRK_CNT+=1
-        logger.info("Starting discord bot: Joining thread {i}")
+        logger.info("Starting discord bot: Joining thread %s", WRK_CNT)
         worker.join()
-
-# widok - generuje się 5 stron po 4 linki
-# odśwież - zamienić na "dalej" - zaciąga kolejne 20 rezultatów
-# po wyslaniu czekamy na odbior i dodajemy UUID oraz nick poszukiwacza do queue ktora jest sprawdzana
-# flask sobie chodzi i jak dostaje posta z wynikami to wyciaga z queue UUID i odpowiada że znalazła
-# po samym wyszukaniu dajemy info że "Zaczynam szukać ale pare godzin to potrwa."
-# OPCJONALNIE: Dosyłamy znalezione wyniki na bieżąco - wtedy wkładamy do queue dla UUID wyniki

cogs/music.py

@@ -0,0 +1,46 @@
+import discord
+from discord.ext import commands
+from typing import  Optional
+
+class Music(commands.Cog):
+    def __init__(self, bot):
+        self.bot = bot # This is so you can access Bot instance in your cog
+
+# You must have this function for `bot.load_extension` to call
+def setup(bot):
+    bot.add_cog(Music(bot))
+
+@commands.hybrid_command(
+    name="przytul", description="Przytul kogoś - daj mention po komendzie :)"
+)
+async def przytul(ctx, arg: Optional[discord.Member] = None):
+    """
+    Generate a text about hugging mentioned user.
+
+    :param ctx: ctx stands for "context" and is a required parameter in Discord.py commands. It
+    represents the context in which the command was invoked, including information such as the message,
+    the channel, the server, and the user who invoked the command
+    :param arg: arg is a parameter of the function "przytul" that expects a Discord member object. The
+    parameter is optional, meaning that if no member object is provided, it will default to None
+    :type arg: Optional[discord.Member]
+    """
+    async with ctx.typing():
+        nieprzytulac = False
+        for mention in ctx.message.mentions:
+            for role in mention.roles:
+                if role.name == "NIEPRZYTULAĆ!":
+                    nieprzytulac = True
+
+        if arg and nieprzytulac:
+            await ctx.send(
+                f"Żebym ja Ciebie nie przytulił {ctx.message.author.mention}"
+            )
+        elif arg:
+            await ctx.send(
+                # trunk-ignore(codespell/misspelled)
+                f"Już dobrze.... Już dobrze... Ojej.. Biedactwo... :( *W ułamku sekundy {arg.mention} znajduje sie w duszącym uścisku. Żebra trzeszczą - kilka pęka. Pacnięcia po plecach grożą odbiciem nerek, a głaskanie po głowie powoduje wstrząs mózgu*"
+            )
+        else:
+            await ctx.send(
+                "Kogo mam przytulić? *Wyłamuje kostki i przeciąga się - jego 200 kilowa sylwetka złożona z samych mięśni świadczy o tym że jest gotowy*"
+            )

communication_subroutine.py

@@ -13,6 +13,22 @@ IN_COMM_Q = Queue()
 awaiting_q = []
 incoming_q = Queue()
 app = Flask(__name__)
+class QueryControl:
+    """
+    This class `QueryControl` is used to manage queries with information about the author, UUID,
+    content, logger, context, and replies.
+    """
+    def __init__(self, query_author, query_uuid, query_content, uplogger, ctx) -> None:
+        self.author = query_author
+        self.uuid = query_uuid
+        self.content = query_content
+        self.logger = uplogger
+        self.stop = False
+        self.ctx = ctx
+        self.logger.info(
+            f"Created Query control for {self.author}, {self.uuid}: {self.content}"
+        )
+        self.replies = []
 
 @app.route("/conjurer", methods=["POST"])
 def answer_external_command():
@@ -86,8 +102,16 @@ def scan_incoming(_logger):
         try:
             answer = incoming_q.get(block=False)
             _logger.info("DATA FOUND")
+            record_stored = False
             for record in awaiting_q:
                 if record.uuid in answer.keys():
+                    record_stored = True
+                    record.stop = True
+                    record.entries = answer[record.uuid]
+                    IN_COMM_Q.put(record)
+            if not record_stored:
+                for key in answer.keys():
+                    record = QueryControl("Orphaned", key, "Orphan", _logger, None)
                     record.stop = True
                     record.entries = answer[record.uuid]
                     IN_COMM_Q.put(record)

conjurer_librarian/conjurer_librarian.py

@@ -1,243 +1,383 @@
-import re
+"""
+This module contains the implementation of the Librarian class and
+related functions for searching and refining queries.
+
+Classes:
+-   Librarian: Represents a librarian object that performs search and
+    refinement operations on queries
+
+Functions:
+-   flask_debug: Starts a Flask application in debug mode without using the reloader.
+-   waitress_run: Serves the Flask application using the Waitress WSGI server.
+-   BackgroundTaskSearch: Represents a background task for running the
+    Librarian object asynchronously
+"""
+
+import asyncio
+import json
 import logging
+import netrc
 import threading
 import time
-from flask import Flask, jsonify, request
-import netrc
-import json
-import asyncio
 from json.decoder import JSONDecodeError
-from urllib.request import urlopen
+from logging import handlers
+from queue import Queue
+
+import requests
+import scrape_bot
+import search_bot
+#import search_bot2 as search_bot
+from flask import Flask, jsonify, request
 from habanero import Crossref
 from waitress import serve
-from queue import Queue
-import search_bot
-import scrape_bot
-import requests
-NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
 
+# Constants
+NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
 HOST_ADDRESS = "192.168.1.192"
 PORT_ADDRESS = 5001
 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
 SEND_RESULTS = "/conjurer"
-BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
+BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
-MAX_CR_RESULTS = 150000
+
+MAX_CR_RESULTS = 1000
+#TEST PURPOSES ONLY!
+#MAX_CR_RESULTS = 5
+
 ENCODING = "utf-8"
+
 app = Flask(__name__)
 
 librarian_queue = Queue()
 librarian_list = []
 
-class Librarian(object):
 
-    def __init__(self, app, query, uuid) -> None:
+# trunk-ignore(pylint/R0902)
+class Librarian(object):
+    """
+    Represents a librarian object that performs search and refinement operations on queries.
+    """
+
+    def __init__(self, _app, query, uuid, _deep_search) -> None:
+        """
+        Initializes a Librarian object.
+
+        Args:
+        - _app: The Flask application object.
+        - query: The query to be searched.
+        - uuid: The unique identifier for the search.
+
+        Attributes:
+        - cr: The Crossref object for performing the search.
+        - query: The query to be searched.
+        - uuid: The unique identifier for the search.
+        - limit: The maximum number of search results to fetch.
+        - fetched: The number of search results fetched so far.
+        - hit: The number of search results that match the refinement criteria.
+        - total: The total number of search results.
+        - app: The Flask application object.
+        - live_results: A list to store live search results.
+        - final_result: A list to store the final refined search results.
+        - not_in_db: A list to store search results that are not in the local database.
+        - search_result_from_cr: A dictionary to store the search results from Crossref.
+        - done: A flag indicating if the search is done.
+        """
         netrc_mod = netrc.netrc(NETRC_FILE)
-        authTokens = netrc_mod.authenticators("crossref")
+        auth_tokens = netrc_mod.authenticators("crossref")
-        self.cr = Crossref(mailto=authTokens[0])
+        self.cr = Crossref(
+            mailto=auth_tokens[0],
+            ua_string=f"Conjurer project. mailto:{auth_tokens[0]}"
+        )
         self.query = query
         self.uuid = str(uuid)
         self.limit = MAX_CR_RESULTS
         self.fetched = 0
         self.hit = 0
         self.total = 0
-        self.app = app
+        self.app = _app
-        self.logger = app.logger
         self.live_results = []
-        self.final_result = []
+        self.final_result = {}
-        self.not_in_db = []
+        self.not_in_db = {}
         self.search_result_from_cr = {}
         self.done = False
+        self.deep_search = _deep_search
+
+    async def search_crossref(self, query, deep_search=False):
+        """
+        Performs a search on Crossref for the given query.
+
+        Args:
+        - query: The query to be searched.
+
+        Returns:
+        - result: The search result from Crossref.
+
+        Raises:
+        - None.
+        """
 
-    async def search_crossref(self, query):
         self.app.logger.info("STARTED SEARCH")
-        result = self.cr.works(query = query, limit = 1000)
-        self.search_result_from_cr.update(result)
-        self.total = result['message']['total-results']
-        self.fetched += len(result['message']['items'])
-        self.logger.info (self.total)
-        self.logger.info(self.fetched)
-        while self.total > self.fetched and self.limit > self.fetched:
-            result = self.cr.works(query = query, limit = 1000)
-            self.search_result_from_cr.update(result)
-            self.total = result['message']['total-results']
-            self.fetched += len(result['message']['items'])
-            self.logger.info (self.total)
-            self.logger.info(self.fetched)
-            time.sleep(0.1)
-        self.app.logger.info("CROSSREF DONE")
-        #TODO: add to sql with UUID
-        #sql row - title, doi, author, uuid, last retrieved, is_available
         
+        if not deep_search:
+            query_limit = MAX_CR_RESULTS if MAX_CR_RESULTS < 1000 else 1000
+            cr_result = self.cr.works(query=query, limit=query_limit)
+            self.search_result_from_cr.update(cr_result)
+            self.total = cr_result["message"]["total-results"]
+            self.fetched += len(cr_result["message"]["items"])
+            self.app.logger.info(self.total)
+            self.app.logger.info(self.fetched)
+            while self.total > self.fetched and self.limit > self.fetched:
+                tmp_result = self.cr.works(query=query, limit=query_limit, offset=self.fetched)
+                cr_result["message"]["items"].extend(tmp_result["message"]["items"])
+                self.total = tmp_result["message"]["total-results"]
+                self.fetched = len(cr_result["message"]["items"])
+                self.app.logger.info(self.total)
+                self.app.logger.info(self.fetched)
+                time.sleep(0.1)
+
+        else:
+            cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True)
+            result = cr_result[0]
+            for item in cr_result[1:]:
+                result["message"]["items"].extend(item["message"]["items"])
+                self.total = item["message"]["total-results"]
+                self.fetched = len(result["message"]["items"])
+                self.app.logger.info(self.total)
+                self.app.logger.info(self.fetched)
+            cr_result = result
+            self.app.logger.info("Total, fetched:")
+            self.app.logger.info(self.total)
+            self.app.logger.info(self.fetched)
+            self.search_result_from_cr.update(cr_result)
+            self.total = cr_result["message"]["total-results"]
+            self.app.logger.info("CROSSREF DONE")
+
+            self.app.logger.info("CROSSREF DONE")
+        with open("cr_results.json", "r+", encoding="utf-8") as data_file:
+            # First we load existing data into a dict.
+            try:
+                file_data = json.load(data_file)
+            except JSONDecodeError:
+                file_data = {}
+            data_file.truncate(0)
+            data_file.seek(0)
+            tmp = {self.uuid : self.search_result_from_cr}
+            if file_data:
+                file_data.update(tmp)
+            else:
+                file_data = tmp
+            json.dump(file_data, data_file, indent=4)
+        return cr_result
+
+
+    async def refine_search(self, unrefined_result):
+        """
+        Refines the search query based on the unrefined search result.
+
+        Args:
+        - unrefined_result: The unrefined search result.
+
+        Returns:
+        - refined_result: The refined search result.
+
+        Raises:
+        - None.
+        """
+        summarized_results = []
+        self.app.logger.info("REFINE: Removing all derived works from the list")
+        for item in unrefined_result["message"]["items"]:
+            summarized_results.append(
+                {
+                    "DOI": item["DOI"],
+                    "title": item["title"] if "title" in item else None,
+                    "type": item["type"] if "type" in item else None,
+                }
+            )
+
+        partial_result = {
+            self.uuid: {
+                "total_results": self.search_result_from_cr["message"][
+                    "total-results"
+                ],
+                "on_page": 1,
+                "summary": summarized_results,
+                "results": self.search_result_from_cr["message"]["items"],
+            }
+        }
+        self.app.logger.info("REFINE: Dumping to file")
+        temp = []
+        refined_result = {}
+        for key in partial_result:
+            self.app.logger.info("KEY:")
+            self.app.logger.info(key)
+
+        for item in partial_result[self.uuid]["summary"]:
+            if item["title"]:
+                temp.append(item)
+
+        for item in temp:
+            refined_result[item["DOI"]]=  item
         with open("rr_results.json", "r+", encoding="utf-8") as data_file:
             # First we load existing data into a dict.
             try:
                 file_data = json.load(data_file)
             except JSONDecodeError:
                 file_data = {}
-            summarized_results = []
-            self.app.logger.info("REFINE: Removing all derived works from the list")
-            for item in self.search_result_from_cr['message']['items']:
-                container = "NO"
-                if "container-title" in item:
-                    container = "YES"
-                else:
-                    summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
-            result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
-            if file_data:
-                file_data.update(result)
-            else:
-                file_data = result
-            
-            self.app.logger.info("REFINE: Dumping to file")
             data_file.truncate(0)
             data_file.seek(0)
+            tmp = {self.uuid: refined_result}
+            if file_data:
+                file_data.update(tmp)
+            else:
+                file_data = tmp
             json.dump(file_data, data_file, indent=4)
-            return result
+        return refined_result
 
-    async def check_dois_in_local_db(self, doi):
+    async def check_if_exists(self, refined_result):
-        self.app.logger.info("REFINE: Running search in the backend app")
+        """
-        coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
+        Checks if the given DOI exists.
-        result = await coro
-        return result
 
-    async def check_if_exists_brute_force(self, page_url):
+        Args:
-        self.app.logger.error("REFINE: Brute force search!")
+        - doi: The DOI to be checked.
-        if not page_url.startswith(("http:", "https:")):
+        - brute_force: A flag indicating if brute force method should be used.
-            raise ValueError("URL must start with 'http:' or 'https:'")
-        # trunk-ignore(bandit/B310)
-        with urlopen(page_url) as response:
-            data = response.read()
-            text = data.decode("utf-8")
-            for line in text.splitlines():
-                if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
-                    return False
-                if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
-                    direct_download_link = "https:" + str(m.group(1))
-                    self.app.logger.info (direct_download_link)
-                    return True
-        return False
 
-    async def check_if_exists(self, doi, brute_force = False):
+        Returns:
+        - result: The search result.
+
+        Raises:
+        - None.
+        """
         result = {}
-        result = await self.check_dois_in_local_db(doi)
+        self.app.logger.info("REFINE: Running search in the backend app")
-        if brute_force and not result:
+        dois = []
-            for item in doi:
+        for item, value in refined_result.items():
-                item_link = "https://sci-hub.se/" + item[0]
+            dois.append([item, value])
-                tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
+        coro = asyncio.to_thread(
-                result.update(tmp)
+            search_bot.search_for_doi, dois, self.live_results, self.app.logger
-        return result
+        )
-
+        result = await coro
-    async def refine_query(self, unrefined_reqult, brute_force = False):
+        result_list = []
-        temp = []
+        result_no_db = []
-        refined_result = []
+        for item in result:
-        for key in unrefined_reqult:
-            self.logger.info("KEY:")
-            self.logger.info(key)
-
-        for item in unrefined_reqult[self.uuid]["summary"]:
-            if item["container"] == "NO" and item["title"]:                
-                refined_result.append(item)
-
-        for item in refined_result:
-            temp.append([item["DOI"], item])
-
-        refined_result = await self.check_if_exists(temp)
-
-        for item in refined_result:
             if item["exists"]:
-                self.final_result.append(item["data"])
+                result_list.append(item)
             else:
-                self.not_in_db.append(item["data"])
+                result_no_db.append(item)
-        self.hit = len(self.final_result)
+        self.hit = len(result)
-        return self.final_result
+        return result_list, result_no_db
-    #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
+
-    async def answer_query(self):
+
-        #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
+    async def answer_query(self, deep_search=False):
-        #Search sql for UUID
+        """
-        #SQL SELECT * FROM RESULTS WHERE UUID = uuid
+        Answers the search query.
-        self.logger.info(f"Search started {self.uuid}")
+
-        database = None
+        Args:
-        query = self.query
+        - None.
-        with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
+
-            #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
+        Returns:
-            #ale tymczasowo plik
+        - result: The search result.
-            try:
+
-                database =  json.load(cr_file)
+        Raises:
-            except JSONDecodeError:
+        - None.
-                pass
+        """
-            #TODO: Zwracać i final_result i not_in_db. Not in db_do pliku - i do ręcznego przeglądania potem.
+        self.app.logger.info(f"Search started {self.uuid}")
-            if database:
+        cr_result = await self.search_crossref(query=self.query, deep_search=deep_search)
-                for item in database.keys():
+        refined_result = await self.refine_search(cr_result)
-                    if item in self.uuid:
+        answer, negative_answer = await self.check_if_exists(refined_result)
-                        self.result = await self.refine_query(database)
+
-                        temp = {}
+        self.app.logger.info("Returning result")
-                        for item in self.result:
+        self.app.logger.info(answer)
-                            temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
+        self.app.logger.info(negative_answer)
-                        self.result = temp
+
-                        temp2 = {}
+        for item in answer:
-                        for item in self.not_in_db:
+            self.final_result[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
-                            temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
+        for item in negative_answer:
-                        self.not_in_db = temp2
+            self.not_in_db[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
-                        return self.result
+        self.app.logger.info("Returning result case2")
-                answer = await self.search_crossref(query=query)
+        self.app.logger.info(self.final_result)
-                database.update(answer)
+        return self.final_result
-                cr_file.truncate(0)
+
-                cr_file.seek(0)
+# ============================= FLASK INTERNALS===============================
-                json.dump(database,cr_file)
+
-                self.result = await self.refine_query(answer)
-            else:
-                answer = await self.search_crossref(query=query)
-                database = answer
-                cr_file.truncate(0)
-                cr_file.seek(0)
-                json.dump(database,cr_file)
-                self.result = await self.refine_query(answer)
-                self.app.logger.info("Refined")
-                self.done = True
-            temp = {}
-            for item in self.result:
-                temp[item['DOI']] = {"Title" : item['title'], "type": item['type']}
-            self.result = temp
-            temp2 = {}
-            for item in self.not_in_db:
-                temp2[item['DOI']] = {"Title" : item['title'], "type": item['type']}
-                self.not_in_db = temp2
-            return self.result
 
-#============================= FLASK INTERNALS===============================
 def flask_debug():
     """
-    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Starts a Flask application in debug mode without using the reloader.
-    Do not use for production for fucks sake.
+
+    Args:
+    - None.
+
+    Returns:
+    - None.
+
+    Raises:
+    - None.
     """
     # trunk-ignore(bandit/B201)
     app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
+
 def waitress_run():
     """
-    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    Serves the Flask application using the Waitress WSGI server.
-    and port 5000 using the Waitress WSGI server.
+
+    Args:
+    - None.
+
+    Returns:
+    - None.
+
+    Raises:
+    - None.
     """
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
+
 class BackgroundTaskSearch(threading.Thread):
+    """
+    A background task for searching and saving results to files.
+
+    This class extends the `threading.Thread` class and is responsible for running
+    the search task in the background. It retrieves queries from a queue, performs
+    the search, and saves the results to files.
+
+    Attributes:
+        app (App): The application instance.
+    """
+
     def run(self):
-        loop = asyncio.new_event_loop()  # loop = asyncio.get_event_loop()
+        """
+        Run the background task.
+
+        This method is called when the thread is started. It creates a new event loop,
+        runs the `_run` method, and closes the event loop.
+        """
+        loop = asyncio.new_event_loop()
         loop.run_until_complete(self._run())
         loop.close()
 
     async def _run(self):
+        """
+        Perform the search task.
+
+        This method is an asynchronous coroutine that runs in a loop. It retrieves a
+        librarian from the queue, answers the query, and saves the results to files.
+        It also sends the results to a remote server.
+
+        The search task continues running indefinitely until the thread is stopped.
+        """
         while True:
             database = None
             ndb_database = None
             librarian = librarian_queue.get()
             self.app.logger.info("STARTED")
-            result = await librarian.answer_query()
+            result = await librarian.answer_query(librarian.deep_search)
             result = {librarian.uuid: result}
             self.app.logger.info("Saving to file")
+
+            # Save results to "not_in_db.json" file
             with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
                 try:
-                    ndb_database =  json.load(ndb_file)
+                    ndb_database = json.load(ndb_file)
                 except JSONDecodeError:
                     pass
                 if ndb_database:
@@ -248,9 +388,10 @@ class BackgroundTaskSearch(threading.Thread):
                 ndb_file.seek(0)
                 json.dump(ndb_database, ndb_file)
 
+            # Save results to "s_results.json" file
             with open("s_results.json", "r+", encoding="utf-8") as s_file:
                 try:
-                    database =  json.load(s_file)
+                    database = json.load(s_file)
                 except JSONDecodeError:
                     pass
                 if database:
@@ -267,26 +408,41 @@ class BackgroundTaskSearch(threading.Thread):
 
             self.app.logger.info(result)
             coroutine = asyncio.to_thread(
-            requests.post,
+                requests.post,
-            f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
+                f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
-            json=result,
+                json=result,
-            timeout=360,
+                timeout=360,
-    )       
+            )
             self.app.logger.info("SENT")
             result = await coroutine
             self.app.logger.info(result.status_code)
             self.app.logger.info("SEND CONFIRMED")
             time.sleep(1)
-            
+
-#==================================SERVER ROUTES==========================================
+
+# ==================================SERVER ROUTES==========================================
 @app.route("/query", methods=["POST"])
 async def query_database():
+    """
+    Endpoint for querying the database.
+
+    This function receives a POST request containing a JSON payload with a query and a UUID.
+    It creates a Librarian object with the query and UUID,
+    and adds it to the librarian_queue and librarian_list.
+    Finally, it returns a JSON response indicating the success
+    of the operation, along with the query, UUID,
+    and the current size of the librarian_queue.
+
+    Returns:
+        tuple: A tuple containing a JSON response and a status code.
+    """
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["query"])
     app.logger.info(record["UUID"])
     uuid = record["UUID"]
-    cl = Librarian(app, record["query"], uuid)
+    deep_search = record["deep_search"]
+    cl = Librarian(app, record["query"], uuid, deep_search)
     librarian_queue.put(cl)
     librarian_list.append(cl)
     answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
@@ -296,8 +452,15 @@ async def query_database():
     )
     return return_data
 
+
 @app.route("/get_partial_result", methods=["POST"])
 async def get_partial():
+    """
+    Retrieves the partial result for a given UUID.
+
+    Returns:
+        A JSON response containing the partial result.
+    """
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["UUID"])
@@ -311,25 +474,34 @@ async def get_partial():
     )
     return return_data
 
-#=======================================MAIN===================================================
+
+# =======================================MAIN===================================================
 if __name__ == "__main__":
-    logger = app.logger
+    app.logger.setLevel(logging.DEBUG)
-    logger.setLevel(logging.DEBUG)
+    h1 = handlers.RotatingFileHandler(
+        filename="D:\\logs\\librarian.log",
+        encoding="utf-8",
+        mode="a",
+        maxBytes=6 * 1024 * 1024,
+        backupCount=6,
+    )
+
+    app.logger.addHandler(h1)
     threads = []
     threads.append(threading.Thread(target=waitress_run))
-    #threads.append(threading.Thread(target=flask_debug))
+    # threads.append(threading.Thread(target=flask_debug))
     bgtask = BackgroundTaskSearch()
     bgtask.app = app
     threads.append(bgtask)
-    threads.append(threading.Thread(target = scrape_bot.scraper, args=(logger,)))
+    threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,)))
     i = 0
     for worker in threads:
         try:
-            logger.info(f"App number: {i}")
+            app.logger.info("App number: %s", i)
-            i+=1
+            i += 1
             worker.start()
         except RuntimeError as e:
-            logger.error("Exploded")
+            app.logger.error("Exploded")
             print(str(e))
     for worker in threads:
         worker.join()

conjurer_librarian/count_lines.py

@@ -0,0 +1,8 @@
+
+FILEPTH = r"C:\Database\chunks\4_chunk.txt"
+
+with open(FILEPTH, "r", encoding="utf-8") as file:
+    tab = file.readlines()
+    print(len(tab))
+
+    #1842295

conjurer_librarian/install_librarian_service.sh

@@ -1,11 +1,33 @@
 #!/bin/bash
-python3 -m venv /home/pi/Conjurer_librarian/env
+# This script sets up a virtual environment for the Conjurer Librarian service and installs the required dependencies.
-cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
-cp  /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
-cd /home/pi/Conjurer_librarian
-source /home/pi/Conjurer_librarian/env/bin/activate
-./env/bin/python3 -m pip install --upgrade pip
-./env/bin/python3 -m pip install -r requirements_librarian.txt
-deactivate
 
-cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
+
+# Create a virtual environment at /home/pi/Conjurer_librarian/env
+python3 -m venv /home/pi/Conjurer_librarian/env
+
+# Copy the Conjurer Librarian Python script to the appropriate directory
+cp /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
+
+# Copy the requirements file to the appropriate directory
+cp /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
+
+# Change directory to /home/pi/Conjurer_librarian
+cd /home/pi/Conjurer_librarian
+
+touch /home/pi/Conjurer/rr_results.json
+touch /home/pi/Conjurer/cr_results.json
+touch /home/pi/Conjurer/s_results.json
+touch /home/pi/Conjurer/rr_results.json
+touch /home/pi/Conjurer/not_in_db.json
+
+# Activate the virtual environment
+source /home/pi/Conjurer_librarian/env/bin/activate
+
+# Upgrade pip
+./env/bin/python3 -m pip install --upgrade pip
+
+# Install the required dependencies
+./env/bin/python3 -m pip install -r requirements_librarian.txt
+
+# Deactivate the virtual environment
+deactivate

conjurer_librarian/requirements_librarian.txt

@@ -1,3 +1,3 @@
 habanero
 waitress
-flask
+flask[async]

conjurer_librarian/scrape_bot.py

@@ -1,46 +1,69 @@
+"""
+This module contains the code for the scrape bot.
+"""
+
 import json
-from json.decoder import JSONDecodeError
+import logging
-from urllib.request import urlopen
+import random
-from requests import Timeout,ConnectTimeout, ConnectionError
-from queue import Queue
 import re
 import time
-import random
+from json import JSONDecodeError
-import logging
+from queue import Queue
 from threading import Thread
-SCR_DATABASE_PATH = r'C:\\Database\\chunks\\'
+from urllib.request import urlopen
+
+from requests import ConnectionError as RequestsConnectionError
+from requests import ConnectTimeout, Timeout
+
+SCR_DATABASE_PATH = r"C:\\Database\\chunks\\"
 SCR_FILENAME = "40_chunk.txt"
 SCR_ENCODING = "utf-8"
 
 WORK_Q = Queue()
 random.seed()
+
+
 def load_ndb_to_q(logger):
+    """
+    This function loads items from the not_in_db.json file into the work queue.
+    """
     logger.info("Loader started")
-    while True:    
+    while True:
         with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
             try:
-                ndb_database =  json.load(ndb_file)
+                ndb_database = json.load(ndb_file)
-                for item in ndb_database.keys():
+                for _ in range (1,10):
-                    logger.info(item)
+                    try:
-                    url = f"https://sci-hub.se/{item}"
+                        key = next(iter(ndb_database))
-                    WORK_Q.put([item, url, False])
+                        _ = ndb_database.pop(key)
+                        logger.info(key)
+                        url = f"https://sci-hub.se/{key}"
+                        WORK_Q.put([key, url, False])
+                    except StopIteration:
+                        break
                 ndb_file.truncate(0)
+                ndb_file.seek(0)
+                json.dump(ndb_database, ndb_file, indent=4)
             except JSONDecodeError:
-                time.sleep(60*60*2)
+                time.sleep(60 * 60 * 3)
-                pass
+        time.sleep(60*60*3)
-        time.sleep(180)
+
 
 def check_if_exists_brute_force(logger):
+    """
+    This function checks if a document exists using brute force search.
+    """
+    # Function code here
     while True:
         logger.info("Scraper tick")
         item = WORK_Q.get()
-        page_url =item[1] 
+        page_url = item[1]
         logger.error("REFINE: Brute force search!")
         if not page_url.startswith(("http:", "https:")):
             raise ValueError("URL must start with 'http:' or 'https:'")
         blocked = True
         try:
-        # trunk-ignore(bandit/B310)
+            # trunk-ignore(bandit/B310)
             with urlopen(page_url) as response:
                 data = response.read()
                 text = data.decode("utf-8")
@@ -50,44 +73,64 @@ def check_if_exists_brute_force(logger):
                     blocked = False
                 else:
                     for line in text.splitlines():
-                        if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
+                        if re.match(
+                            r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
+                            line,
+                        ):
                             blocked = False
                             logger.info("Not found")
                             item[2] = False
-                        if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
+                        if m := re.match(
+                            r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
+                        ):
                             blocked = False
                             direct_download_link = "https:" + str(m.group(1))
-                            logger.info (direct_download_link)
+                            logger.info(direct_download_link)
-                            with open(SCR_DATABASE_PATH + SCR_FILENAME, "a", encoding=SCR_ENCODING) as operated_file:
+                            with open(
+                                SCR_DATABASE_PATH + SCR_FILENAME,
+                                "a",
+                                encoding=SCR_ENCODING,
+                            ) as operated_file:
                                 operated_file.write("\n")
                                 operated_file.write(item[0])
                                 item[2] = True
-        except (Timeout,ConnectTimeout, ConnectionRefusedError, ConnectionError):
+        except (
+            Timeout,
+            ConnectTimeout,
+            ConnectionRefusedError,
+            ConnectionError,
+            RequestsConnectionError,
+        ):
             pass
         if blocked:
             logger.info(item)
             logger.info(text)
             logger.error("Got blocked. Fuck.")
-            time.sleep(60*60*72)
+            time.sleep(60 * 60 * 72)
         # trunk-ignore(bandit/B311)
-        rand = random.randint(1,100)
+        rand = random.randint(1, 60)
-        logger.info(f"Sleeping for {3*rand} minutes")
+        logger.info(f"Sleeping for {2*rand} minutes")
-        time.sleep(180*rand)
+        time.sleep(120 * rand)
 
-def scraper(logger = None):
+
+def scraper(logger=None):
+    """
+    This function is responsible for scraping data.
+    """
     if not logger:
         logger = logging.getLogger()
         logger.setLevel("DEBUG")
         h1 = logging.StreamHandler()
         logger.addHandler(h1)
         logger.info("No logger set. Test run")
-  
+
-    loader = Thread(target = load_ndb_to_q, args = (logger,))
+    loader = Thread(target=load_ndb_to_q, args=(logger,))
-    scraper = Thread(target = check_if_exists_brute_force, args = (logger,))
+    _scraper = Thread(target=check_if_exists_brute_force, args=(logger,))
     loader.start()
-    scraper.start()
+    _scraper.start()
     loader.join()
-    scraper.join()
+    _scraper.join()
+
 
 if __name__ == "__main__":
-    scraper()
+    scraper()

conjurer_librarian/search_bot.py

@@ -1,28 +1,66 @@
-#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+"""
+This module contains functions for searching for DOI (Digital Object Identifier)
+in a list of live results.
+It includes a producer-consumer pattern implementation using threads and queues.
 
-from queue import Queue, Empty
+Functions:
+- producer: Reads lines from a file and puts them into an output queue.
+- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
+- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
+
+Global Variables:
+- MAXTHREADS: Maximum number of worker threads.
+- DATABASE_PATH: Path to the database files.
+- ENCODING: Encoding of the database files.
+- CHUNK: File name pattern for the database files.
+- _sentinel: Sentinel object used to signal termination.
+- result_list: List to store the search results.
+- WORK_Q_SIZE: Maximum size of the work queue.
+"""
+
+# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+from queue import Empty, Queue
 from threading import Thread
+import time
 q = Queue()
+#TODO: Count number of lines in files and print to approximate on which part of the file search is
 
-#TODO: DATA FOR TEST ONLY
+# DATA FOR TEST ONLY
-#MAXTHREADS = 5
+# MAXTHREADS = 5
-#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+# DATABASE_PATH = r'C:\\Database\\chunks_1\\'
 
-#TODO: DEPLOYMENT DATA
+# DEPLOYMENT DATA
 MAXTHREADS = 41
-DATABASE_PATH = r'C:\\Database\\chunks\\'
+DATABASE_PATH = r"C:\\Database\\chunks\\"
 
 ENCODING = "utf-8"
-CHUNk = "_chunk.txt"
+CHUNK = "_chunk.txt"
 _sentinel = object()
-result_list = []
+WORK_Q_SIZE = 35500000
-WORK_Q_SIZE = 15500000
 
-def producer(out_q, control_q, filename, logger):
+
+def producer(out_q, control_q, filename, _logger):
+    """
+    Produces items from the output queue and puts them into the control queue.
+
+    Args:
+        out_q (Queue): Output queue.
+        control_q (Queue): Control queue.
+        filename (str): Name of the file.
+        _logger: Logger object for logging.
+    """
     with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
-        logger.info(f"Worker {filename} ")
+        print(f"Worker {filename} ")
+        line_no = 0
         while True:
             line = operated_file.readline()
+            line_no += 1
+            print(f"\t \t \t \t \t \t W{filename}{line_no}\r", end="")
+
+            if not line:
+                print(f"EOF {filename}")
+                break
+
             if not line:
                 break
             out_q.put(line)
@@ -30,77 +68,133 @@ def producer(out_q, control_q, filename, logger):
                 check = control_q.get(block=False)
             except Empty:
                 check = False
-                pass
+
             if check is _sentinel:
-                logger.info("TERM signal received")
+                print("TERM signal received")
                 control_q.put(check)
                 break
-        logger.info(f"Worker finished: {filename}")
+        print(f"Worker finished: {filename}")
         out_q.put(_sentinel)
- 
+
-#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+
-#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
+# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
-def consumer(in_q, control_q, doi, live_results, logger):
+def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
-    sentinels = 0
+    """
-    for item in doi:
+    Consumes items from an input queue and checks if DOI exists in the live results list.
-        result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
+
+    Args:
+        in_q (Queue): Input queue.
+        control_q (Queue): Control queue.
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    print(f"Consumer thread started: {no} no")
+    empty_counter = 0
+    alive_no = 0
     while True:
         done_check = True
-        data = in_q.get()
+        try:
-        if data is _sentinel:
+            data = in_q.get(block=True, timeout = 1)
-            logger.info("Worker finished")
+            if data is _sentinel:
-            sentinels += 1
+                control_dict["sentinels"] += 1
-        else:
+                print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
-            for item in result_list:
+
-                if item["DOI"] in data and not item["exists"]:
+            else:
-                    logger.info(data)
+                empty_counter = 0
-                    logger.info("HIT")
+                alive_no += 1
-                    item["exists"] = True
+                print(f"C{no}__{alive_no}\r", end="")
-                    live_results.append(item)
+
-                done_check = done_check and item["exists"]
+                for item in result_list:
-            if done_check:
+                    if item["DOI"] in data and not item["exists"]:
-                control_q.put(_sentinel)
+                        print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
-        if sentinels >= MAXTHREADS:
+                        _logger.info(data)
-            logger.info("All workers finished")
+                        _logger.info("HIT")
+                        item["exists"] = True
+                        live_results.append(item)
+                    done_check = done_check and item["exists"]
+                if done_check:
+                    control_q.put(_sentinel)
+        except Empty:
+            empty_counter += 1
+            time.sleep(1)
+            print(f"Consumer {no} empty")
+            if empty_counter > 5:
+                print("Consumer %s empty lvl 2", no)
+                time.sleep(2)
+            elif empty_counter > 10:
+                print(f"Consumer thread finished {no}")
+                break
+
+        if control_dict["sentinels"] >= MAXTHREADS:
+            _logger.info(f"All workers finished {no}")
             break
 
 
-def search_for_doi(doi, live_results, logger):
+
+def search_for_doi(doi, live_results, _logger):
+    """
+    Search for DOI in live_results using _logger for logging.
+
+    Args:
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    control_dict = {"sentinels":0}
+    result_list = []
     threads = []
     work_q = Queue(maxsize=WORK_Q_SIZE)
     control_q = Queue()
-    t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
+
-    logger.info("Consumer thread created")
+    for item in doi:
-    threads.append(t_cons)
+        result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
-    for i in range(0,MAXTHREADS):
+
-        #TODO: TEST DATA
+    for i in range (0, (len(doi)//1000)+2):
-        #filename = "test" + str(i) + CHUNk
+        t_cons = Thread(
-        #TODO: DEPLOYMENT DATA
+            target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, i, _logger)
-        filename = str(i) + CHUNk
+        )
-        logger.info(f"Creating worker thread no: {i}")
+        _logger.info("Consumer thread created")
-        threads.append(Thread(target = producer, args = (work_q, control_q, filename,logger)))
+        threads.append(t_cons)
+    for i in range(0, MAXTHREADS):
+        # TEST DATA
+        # filename = "test" + str(i) + CHUNk
+        # DEPLOYMENT DATA
+        filename = str(i) + CHUNK
+        _logger.info(f"Creating worker thread no: {i}")
+        threads.append(
+            Thread(target=producer, args=(work_q, control_q, filename, _logger))
+        )
     for worker in threads:
         worker.start()
     for worker in threads:
         worker.join()
     return result_list
 
+
 if __name__ == "__main__":
     import logging
+
     logger = logging.getLogger()
     logger.setLevel("DEBUG")
     h1 = logging.StreamHandler()
     logger.addHandler(h1)
     logger.info("TEST RUN")
     live_result = []
-    logger.info(search_for_doi([
+    logger.info(
-        ("10.1002/9781118786352.wbieg0998.pub2","DATA"),
+        search_for_doi(
-        ("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
+            [
-        ("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
+                ("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
-        ("10.2165/00128415-200309690-00017","DATA"),
+                ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
-        ("10.5772/48313","DATA"),
+                ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
-        ("10.15803/ijnc.7.2_419","DATA"),
+                ("10.2165/00128415-200309690-00017", "DATA"),
-        ("10.1051/0004-6361/201321596e","DATA"),
+                ("10.5772/48313", "DATA"),
-        ("10.2307/40835941","DATA"),
+                ("10.15803/ijnc.7.2_419", "DATA"),
-        ], live_result, logger))
+                ("10.1051/0004-6361/201321596e", "DATA"),
+                ("10.2307/40835941", "DATA"),
+            ],
+            live_result,
+            logger,
+        )
+    )
     logger.info(live_result)

conjurer_librarian/search_bot2.py

@@ -0,0 +1,206 @@
+"""
+This module contains functions for searching for DOI (Digital Object Identifier)
+in a list of live results.
+It includes a producer-consumer pattern implementation using threads and queues.
+
+Functions:
+- producer: Reads lines from a file and puts them into an output queue.
+- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
+- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
+
+Global Variables:
+- MAXTHREADS: Maximum number of worker threads.
+- DATABASE_PATH: Path to the database files.
+- ENCODING: Encoding of the database files.
+- CHUNK: File name pattern for the database files.
+- _sentinel: Sentinel object used to signal termination.
+- result_list: List to store the search results.
+- WORK_Q_SIZE: Maximum size of the work queue.
+"""
+
+# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+from multiprocessing import Manager, Process, Queue
+from queue import Empty, Full
+import time
+
+# DATA FOR TEST ONLY
+#MAXTHREADS = 6
+#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+
+# DEPLOYMENT DATA
+MAXTHREADS = 41
+DATABASE_PATH = r"C:\\Database\\chunks\\"
+
+ENCODING = "utf-8"
+CHUNK = "_chunk.txt"
+_sentinel = None
+WORK_Q_SIZE = 355000
+
+
+def producer(out_q, control_q, filename, _logger):
+    """
+    Produces items from the output queue and puts them into the control queue.
+
+    Args:
+        out_q (Queue): Output queue.
+        control_q (Queue): Control queue.
+        filename (str): Name of the file.
+        _logger: Logger object for logging.
+    """
+    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
+        print(f"Worker {filename} ")
+        line_no = 0
+        while True:
+            line = operated_file.readline()
+            line_no += 1
+            print(f"\t \t \t W{filename}{line_no}\r", end="")
+
+            if not line:
+                print(f"EOF {filename}")
+                break
+            try:
+                out_q.put([line, line_no, filename], block = True, timeout = 60)
+            except Full:
+                print("queue fulll")
+            try:
+                check = control_q.get(block=False)
+            except Empty:
+                check = False
+            if check is None:
+                print(f"TERM signal received {filename}")
+                control_q.put(check)
+                control_q.close()
+                break
+        print(f"Worker finished: {filename}")
+        out_q.put(_sentinel)
+        out_q.close()
+
+
+
+# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
+    """
+    Consumes items from an input queue and checks if DOI exists in the live results list.
+
+    Args:
+        in_q (Queue): Input queue.
+        control_q (Queue): Control queue.
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    print(f"Consumer thread started: {no} no")
+    empty_counter = 0
+    alive_no = 0
+    while True:
+        done_check = True
+        try:
+            data = in_q.get(block=True, timeout = 1)
+            empty_counter = 0
+            if data is None:
+                control_dict["sentinels"] += 1
+                print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
+            else:
+                empty_counter = 0
+                if alive_no > 9:
+                    alive_no = 0
+                alive_no += 1
+                print(f"C{no}{alive_no}\r", end="")
+                for item in result_list:
+                    if item["DOI"] in data[0] and not item["exists"]:
+                        print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
+                        item["exists"] = True
+                        live_results.append(item)
+                    done_check = done_check and item["exists"]
+                if done_check:
+                    control_q.put(_sentinel)
+        except Empty:
+            empty_counter += 1
+            time.sleep(1)
+            print(f"Consumer {no} empty")
+            if empty_counter > 5:
+                print("Consumer %s empty lvl 2", no)
+                time.sleep(2)
+            elif empty_counter > 10:
+                print(f"Consumer thread finished {no}")
+                break
+        if control_dict["sentinels"] >= MAXTHREADS:
+            print(f"All workers finished killing {no}")
+            break
+    print("Consumer thread finished %s",no )
+    control_q.close()
+
+
+def search_for_doi(doi, live_results, _logger):
+    """
+    Search for DOI in live_results using _logger for logging.
+
+    Args:
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    with Manager() as manager:
+        control_dict = manager.dict()
+        control_dict["sentinels"] = 0
+        result_list = []
+        for item in doi:
+            result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
+        result_list_proxy = manager.list(result_list)
+
+        threads = []
+        work_q = Queue(maxsize=WORK_Q_SIZE)
+        control_q = Queue()
+        for i in range(0, MAXTHREADS):
+            # TEST DATA
+            #filename = "test" + str(i) + CHUNK
+            # DEPLOYMENT DATA
+            filename = str(i) + CHUNK
+            print(f"Creating worker thread no: {i}")
+            threads.append(
+                Process(target=producer, args=(work_q, control_q, filename, _logger))
+            )
+        for i in range (0, (len(doi)//1000)+5):
+            t_cons = Process(
+                target=consumer, args=(work_q, control_q, doi, live_results, result_list_proxy, control_dict, i, _logger)
+            )
+            print("Consumer thread created")
+            threads.append(t_cons)
+
+        for worker in threads:
+            worker.start()
+        for worker in threads:
+            worker.join()
+        result_list = list(result_list_proxy)
+        for worker in threads:
+            worker.close()
+        return result_list
+
+
+if __name__ == "__main__":
+    import logging
+
+    logger = logging.getLogger()
+    logger.setLevel("DEBUG")
+    h1 = logging.StreamHandler()
+    logger.addHandler(h1)
+    print("TEST RUN")
+    live_result = []
+    print(
+        search_for_doi(
+            [
+                ("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
+                ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
+                ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
+                ("10.2165/00128415-200309690-00017", "DATA"),
+                ("10.5772/48313", "DATA"),
+                ("10.15803/ijnc.7.2_419", "DATA"),
+                ("10.1051/0004-6361/201321596e", "DATA"),
+                ("10.2307/40835941", "DATA"),
+                ("NOT_ON_LIST", "DATA"),
+            ],
+            live_result,
+            logger,
+        )
+    )
+    print(live_result)

conjurer_librarian/test_send.py

@@ -0,0 +1,45 @@
+import asyncio
+import json
+import logging
+from json.decoder import JSONDecodeError
+from logging import handlers
+import requests
+
+MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
+SEND_RESULTS = "/conjurer"
+
+async def send_results():
+    logger = logging.getLogger()
+    logger.setLevel(logging.DEBUG)
+    h1 = handlers.RotatingFileHandler(
+        filename="D:\\logs\\librarian.log",
+        encoding="utf-8",
+        mode="a",
+        maxBytes=6 * 1024 * 1024,
+        backupCount=6,
+    )
+
+    logger.addHandler(h1)
+
+    with open("s_results.json", "r+", encoding="utf-8") as s_file:
+        try:
+            database = json.load(s_file)
+        except JSONDecodeError:
+            pass
+    result = database["4bacbca0-ce3b-45c6-90a1-05c70b46b740"]
+    result = {"4bacbca0-ce3b-45c6-90a1-05c70b46b740":result}
+
+    coroutine = asyncio.to_thread(
+        requests.post,
+        f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
+        json=result,
+        timeout=360,
+    )
+    logger.info("SENT")
+    result = await coroutine
+    logger.info(result.status_code)
+    logger.info("SEND CONFIRMED")
+async def main():
+    await send_results()
+if __name__ == "__main__":
+    asyncio.run(main())

conjurer_musician/conjurer_musician.py

@@ -14,8 +14,7 @@ from logging import handlers
 from pathlib import Path
 from platform import uname
 from sys import platform
-
+from flask import Flask, jsonify, redirect, request, send_from_directory, render_template
-from flask import Flask, jsonify, redirect, request, send_from_directory
 from waitress import serve
 
 HOST_ADDRESS = "192.168.1.15"
@@ -35,34 +34,11 @@ if platform in ("linux", "linux2"):
         LOGSTORE = "/home/pi/RetroPie/logs/"
         ENCODING = "utf-8"
         MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
+        PRIORITY_FOLDER = "/home/pi/RetroPie/mp3/Magiczne i chuj/"
 
 
 music_file_list = []
-
+priority_list = []
-
-def create_playlist():
-    """
-    Reads a JSON file containing a playlist and writes the playlist items to a text file.
-
-    The JSON file path is '/home/pi/Conjurer/playlist.json',
-    and the text file path is '/home/pi/Conjurer/all_playlist.playlist'.
-
-    Raises:
-        JSONDecodeError: If the JSON file cannot be decoded.
-
-    """
-    with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
-        with open(
-            "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
-        ) as w_file:
-            try:
-                playlist = json.load(r_file)
-                for item in playlist:
-                    w_file.write(item)
-                    w_file.write("\n")
-            except json.JSONDecodeError:
-                pass
-
 
 def rescan():
     """
@@ -70,18 +46,35 @@ def rescan():
     and adds them to a list of music files.
     """
     logging.info("Rescan triggered")
+
     for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
         temp_music_file = mp3_item.as_posix()
         if platform == "win32":
             temp_music_file = temp_music_file.replace("/", "\\")
         music_file_list.append(temp_music_file)
-    with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
+
-        database = music_file_list
+    for mp3_item in Path.glob(Path(PRIORITY_FOLDER), "**/*.mp3"):
-        # self.app.logger.info("DUMPING DATA")
+        temp_music_file = mp3_item.as_posix()
-        s_file.truncate(0)
+        if platform == "win32":
-        s_file.seek(0)
+            temp_music_file = temp_music_file.replace("/", "\\")
-        json.dump(database, s_file)
+        priority_list.append(temp_music_file)
-    create_playlist()
+
+    with open(
+            "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
+        ) as w_file:
+        try:
+            for item in music_file_list:
+                w_file.write(item)
+                w_file.write("\n")
+        except json.JSONDecodeError:
+            pass
+    with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file:
+        try:
+            for item in priority_list:
+                w_file.write(item)
+                w_file.write("\n")
+        except json.JSONDecodeError:
+            pass
 
 
 def thread_rescan():
@@ -202,7 +195,9 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
                 return_list.append(search_weight[itr])
                 if not return_to_bot:
                     with open(
-                        "/home/pi/Conjurer/priority_queue.playlist", "r+", encoding="utf-8"
+                        "/home/pi/Conjurer/priority_queue.playlist",
+                        "r+",
+                        encoding="utf-8",
                     ) as s_file:
                         s_file.write(search_weight[itr][1])
                 break
@@ -211,14 +206,7 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
         fun_logger.info("Wiele plików do zagrania")
         search_weight.sort(key=lambda x: x[0], reverse=True)
         return_list.extend(search_weight[: int(how_many)])
-        if not return_to_bot:
+    fun_logger.info("Done: %s", return_list)
-
-            with open(
-               "/home/pi/Conjurer/priority_queue.playlist", "r+", encoding="utf-8"
-            ) as s_file:
-                s_file.truncate(0)
-                for item in return_list:
-                    s_file.write(item[1] + "\n")
     return return_list
 
 
@@ -253,24 +241,37 @@ def remove_characters(string, character):
 @app.route("/stream", methods=["GET"])
 def stream_music():
     """
-    The function `get_music_list` returns a JSON object containing a list of music files.
+    The function `stream_music` is a route handler for the "/stream" endpoint.
-    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    It returns a JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
-    `music_file_list`.
+
+    :return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
     """
-    return send_from_directory("/tmp/hls", "stream.m3u8")
 
-@app.route('/<string:file_name>')
+    #return send_from_directory("/tmp/hls", "stream.m3u8")
+    return render_template("/home/pi/Conjurer/stream.html")
+
+@app.route("/<string:file_name>")
 def stream(file_name):
-    video_dir = '/tmp/hls'
+    """
-    return send_from_directory(video_dir, file_name)
+    Stream the specified file from the video directory.
 
+    Args:
+        file_name (str): The name of the file to be streamed.
+
+    Returns:
+        Response: The response containing the streamed file.
+    """
+    # trunk-ignore(bandit/B108)
+    video_dir = "/tmp/hls"
+    return send_from_directory(video_dir, file_name)
 
 @app.route("/stream_mp3", methods=["GET"])
 def stream_music_mp3():
     """
-    The function `get_music_list` returns a JSON object containing a list of music files.
+    The function `stream_music_mp3` is a route handler for the "/stream_mp3" endpoint.
-    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    It redirects the user to the URL "http://www.example.com" with a status code of 302.
-    `music_file_list`.
+
+    :return: A redirect response to "http://www.example.com" with a status code of 302.
     """
     return redirect("http://www.example.com", code=302)
 
@@ -278,19 +279,18 @@ def stream_music_mp3():
 @app.route("/clear_pr_pls", methods=["GET"])
 def clear_pr_pls():
     """
-    The function `get_music_list` returns a JSON object containing a list of music files.
+    The function `clear_pr_pls` clears the contents of the priority queue playlist file.
-    :return: A JSON response containing a key "music_file_list" with the value of the variable
+
-    `music_file_list`.
+    :return: A JSON response indicating the success of the operation.
     """
-    with open(
+    app.logger.info("CLEARING PLAYLIST")
-        "/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
+    with open('/home/pi/Conjurer/priority_queue.playlist', 'w', encoding='utf-8') as cleared_pl:
-        ) as s_file:
+        cleared_pl.write('')
-        s_file.truncate(0)
+
     return_data = jsonify(isError=False, message="Success", statusCode=200, data=[])
     return return_data, 200
 
 
-
 @app.route("/mp3", methods=["GET"])
 def get_music_list():
     """
@@ -325,20 +325,22 @@ def update_music_list():
 @app.route("/get_music", methods=["POST"])
 def look_for_playlist():
     """
-    The function `update_music_list` receives a POST request with a JSON payload, logs the received
+    The function `look_for_playlist` receives a POST request with a JSON payload, logs the received
     item, adds it to a music file list, and returns a success message along with the updated record.
-    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+
-    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    :return: A tuple containing a JSON response and a status code. The JSON response includes keys `isError`,
-    with values indicating the success of the operation and the data that was
+             `message`, `statusCode`, and `data`, with values indicating the success of the operation and the
-    received and added to the `music_file_list`.
+             data that was received and added to the `music_file_list`. The status code returned is 200,
-    The status code returned is 200, indicating a successful response.
+             indicating a successful response.
     """
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["lista_slow"])
     app.logger.info(record["UUID"])
     app.logger.info(record["dlugosc_plejlisty"])
-    return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True)
+    return_data = wyszukaj(
+        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True
+    )
 
     return_data = (
         jsonify(isError=False, message="Success", statusCode=200, data=return_data),
@@ -346,26 +348,89 @@ def look_for_playlist():
     )
     return return_data
 
-@app.route("/add_to_priority", methods=["POST"])
+
-def add_to_priority():
+@app.route("/request_radio_file", methods=["POST"])
+def add_request():
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    return_data = wyszukaj(
+        record["lista_slow"], 0, app.logger, False
+    )
+
+    with open(
+        "/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
+    ) as s_file:
+        for item in return_data:
+            s_file.write(item[1] + "\n")
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
+        200,
+    )
+    return return_data
+
+@app.route("/create_priority_playlist", methods=["POST"])
+def create_priority_playlist():
     """
-    The function `update_music_list` receives a POST request with a JSON payload, logs the received
+    The function `create_priority_playlist` receives a POST request with a JSON payload, logs the received
     item, adds it to a music file list, and returns a success message along with the updated record.
-    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+
-    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    :return: A tuple containing a JSON response and a status code.
-    with values indicating the success of the operation and the data that was
+        The JSON response includes keys `isError`,
-    received and added to the `music_file_list`.
+        `message`, `statusCode`, and `data`, with values
-    The status code returned is 200, indicating a successful response.
+        indicating the success of the operation and the
+        data that was received and added to the `music_file_list`.
+        The status code returned is 200,indicating a successful response.
     """
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["lista_slow"])
     app.logger.info(record["UUID"])
     app.logger.info(record["dlugosc_plejlisty"])
-    return_data = wyszukaj(record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False)
+    return_data = wyszukaj(
-
+        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
+    )
+    with open(
+        "/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
+    ) as s_file:
+        for item in return_data:
+            s_file.write(item[1] + "\n")
     return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data=return_data),
+        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
+        200,
+    )
+    return return_data
+
+
+@app.route("/add_to_priority", methods=["POST"])
+def add_to_priority():
+    """
+    The function `add_to_priority` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+
+    :return: A tuple containing a JSON response and a status code.
+        The JSON response includes keys `isError`,
+        `message`, `statusCode`, and `data`, with values
+        indicating the success of the operation and the
+        data that was received and added to the `music_file_list`.
+        The status code returned is 200,indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    app.logger.info(record["dlugosc_plejlisty"])
+    return_data = wyszukaj(
+        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
+    )
+    with open(
+        "/home/pi/Conjurer/priority_queue.playlist", "a", encoding="utf-8"
+    ) as s_file:
+        for item in return_data:
+            s_file.write(item[1] + "\n")
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
         200,
     )
     return return_data
@@ -382,8 +447,18 @@ def flask_debug():
 
 def waitress_run():
     """
-    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    The `waitress_run` function serves the `app` on host HOST_ADDRESS
     and port 5000 using the Waitress WSGI server.
+
+    This function starts the Waitress server and listens for incoming requests
+    on the specified host and port. It uses the `serve` function from the Waitress
+    library to handle the serving process.
+
+    Parameters:
+        None
+
+    Returns:
+        None
     """
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
@@ -407,7 +482,7 @@ if __name__ == "__main__":
     rescan()
     logger.info("Started")
     threads = []
-    #threads.append(threading.Thread(target=flask_debug))
+    # threads.append(threading.Thread(target=flask_debug))
     threads.append(threading.Thread(target=waitress_run))
     threads.append(threading.Thread(target=thread_rescan))
 

conjurer_musician/music_utils.py

@@ -0,0 +1,26 @@
+# This Python file uses the following encoding: utf-8
+"""This module contains utility scripts used by musi radiostation Conjurer"""
+import time
+import re
+
+NAME = "/home/pi/Conjurer/persistence.log"
+
+def print_top():
+    """
+    Prints the top 65 lines from a file specified by the NAME variable.
+    """
+    while True:
+        line = ""
+        buffer = ""
+        with open(NAME, "r", encoding="utf-8") as f:
+            for _ in range (45):
+                line = f.readline()
+                if re.match(".*mp3", line):
+                    buffer += line
+            print("=====================================")
+            print(buffer)
+            print("=====================================")
+        time.sleep(180)
+
+if __name__ == "__main__":
+    print_top()

conjurer_musician/pl.config

@@ -0,0 +1,14 @@
+$dodaj_do_ulubionych 100 nightwish
+$dodaj_do_ulubionych 100 amon amarth
+$dodaj_do_ulubionych 100 tactical sekt
+$dodaj_do_ulubionych 100 lacuna coil
+$dodaj_do_ulubionych 100 sefa
+$dodaj_do_ulubionych 100 frenchcore
+$dodaj_do_ulubionych 100 peacock
+$dodaj_do_ulubionych 100 vavamuffin
+$dodaj_do_ulubionych 100 abradab
+$dodaj_do_ulubionych 100 eluveitie
+$dodaj_do_ulubionych 100 youtube
+
+$dodaj_do_ulubionych 100 elyose
+$dodaj_do_ulubionych 1 artisans du chaos

conjurer_musician/radio_conjurer.liq

@@ -1,48 +1,167 @@
+# FILEPATH: /home/mtuszowski/conjurer/conjurer_musician/radio_conjurer.liq
+
+# This script sets up a Liquidsoap radio stream with various features and configurations.
+
+# Load icecast credentials from a JSON file
 let json.parse credentials  = file.contents("/home/pi/Conjurer/icecast_credentials.json")
+
+# Enable replaygain metadata processing
 enable_replaygain_metadata()
 
-s1 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/all_playlist.playlist"))
+# Set up a playlog for tracking played tracks
-s2 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/priority_queue.playlist"))
 
-s = random(id="randomizer", weights=[7000, 100], [s2, s1])
+l = playlog(duration = 72000.0, persistency="/home/pi/Conjurer/persistence.log")
 
+# Function to check if a track can be played based on its metadata
+def check(r)
+    m = request.metadata(r)
+    if l.last(m) < 36000. then
+        log.info("Rejecting #{m['filename']} (played #{l.last(m)}s ago).")
+        false
+    else
+        l.add(m)
+        true
+    end
+end
+
+# Define playlists to be used in the stream
+s1 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/all_playlist.playlist"))
+s2 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/priority_queue.playlist"))
+s3 = replaygain(playlist(reload_mode="watch", check_next=check, "/home/pi/Conjurer/hit.playlist"))
+
+# Create a request queue for user-generated requests
+requests_queue = request.queue()
+
+# Function to process the request queue and add new requests
+def queue_processing()
+    text=file.lines("/home/pi/Conjurer/request.playlist")
+    if text != [] then
+        list.iter(fun(item) -> requests_queue.push.uri(item), text)
+        file.remove("/home/pi/Conjurer/request.playlist")
+        f = file.open("/home/pi/Conjurer/request.playlist", create=true)
+        f.close()
+    end
+end
+
+# Randomly select a playlist with weights
+s4 = random(id="randomizer", weights=[2, 3, 5], [s1, s2, s3])
+
+# Load jingles playlist
+jingles = (playlist(reload_mode="watch", "/home/pi/Conjurer/jingles.playlist"))
+
+# Create the main stream with random playlist and jingles
+s = rotate(id="randomizer", weights=[10, 1], [s4, jingles])
+
+s = crossfade(fade_out=2., fade_in=2., duration=4., fallback(id="switcher", track_sensitive=false, [requests_queue, s]))
+
+# Set up an interactive harbor for controlling the stream
 interactive.harbor(port = 9999)
-f = interactive.float("f", description="Frequency", min=0., max=1000.,unit="Hz", 200.)
+
-g = interactive.float("g", description="Gain", min=0., max=20.,unit="dB", 8.)
+# Set up interactive controls for bass boost
+f = interactive.float("f", description="Frequency", min=0., max=1000., unit="Hz", 200.)
+g = interactive.float("g", description="Gain", min=0., max=20., unit="dB", 8.)
 b = bass_boost(frequency=f, gain=g, s)
 s = add([s, b])
+
+# Set up interactive control for main volume
 a = interactive.float("main_volume", min=0., max=20., 1.)
-s = compress.multiband.interactive(bands=3, s)
+s = compress.multiband.interactive(bands=7, s)
 
-s = nrj(normalize (s))
+# Apply audio processing effects
-s = crossfade(fade_out=3., fade_in=3., duration=5., s)
+s = nrj(normalize(s))
-s = amplify(a,s)
+s = amplify(a, s)
 
-s = blank.skip(max_blank=2., s)
+mic = buffer(input.pulseaudio()) # Microphone
+mic = gate(threshold=-30., range=-80., mic)
+mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic)
 
-emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
+s = add([s, mic])
-radio = fallback(id="switcher",track_sensitive = false, [s, emergency]) 
+#w = interactive.float("wetness", min=0., max=1., 1.)
+#s = dry_wet(w,s,s2)
 
-handle_metadata = fun (m) -> begin
-    print(m["title"])
-    print(m["artist"])
-    print(m["filename"])
-end
-radio.on_metadata(handle_metadata)
-interactive.persistent("script.params")
 
+# Skip blank sections in the stream
+s = blank.skip(max_blank=10., s)
+
+# Configure logging settings
 log_to_stdout = true
 log_to_file = true
 logpath = "/home/pi/Conjurer/radio_log.log"
 loglevel = 3
+set("log.stdout", log_to_stdout)
+set("log.level", loglevel)
 
-# Enable logging on Standard Output and set logging level.
+# Enable logging to file
-set("log.stdout",log_to_stdout)
+set("log.file", log_to_file)
-set("log.level",loglevel)
+set("log.file.path", logpath)
 
-# Enable logging to file. See variable: logpath
+# Set up emergency fallback track
-set("log.file",log_to_file)
+emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
-set("log.file.path",logpath)
+radio =  fallback(id="switcher2", track_sensitive=false, [s, emergency])
+# Set up an interactive control for skipping tracks
+p = interactive.bool("Skip track", false)
 
-output.file.hls("/tmp/hls",[("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))],radio)
+def http_skip(~protocol, ~data, ~headers, uri)=
-output.icecast(%mp3, host="retropie", port=8000,password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)
+    radio.skip()
+    http.response(code=200,data="Skipped")
+end
+harbor.http.register(port=54321,method="GET","/skip", http_skip)
+
+
+# Function to process the request queue and skip the current track if requested
+def check_skip()
+    if p() then
+        log.info("Skipping current track.")
+        radio.skip()
+        p.set(false)
+    end
+end
+
+# Run the queue processing function every 60 seconds
+thread.run(every=60., queue_processing)
+
+
+# Run the check_skip function every 5 seconds
+thread.run(every=15., check_skip)
+
+
+# Enable persistent script parameters
+interactive.persistent("script.params")
+
+# Configure output formats and destinations
+
+output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)
+output.pulseaudio(radio)
+#output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio)
+# Uncomment the following lines to enable additional output formats
+# output.icecast(%opus, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="opus-stream", radio)
+# output.icecast(%ogg, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="ogg-stream", radio)
+# output.icecast(%aac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="aac-stream", radio)
+# output.icecast(%flac, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="flac-stream", radio)
+# output.icecast(%vorbis, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="vorbis-stream", radio)
+# output.icecast(%speex, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="speex-stream", radio)
+# output.icecast(%wav, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="wav-stream", radio)
+# output.icecast(%pcm, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="pcm-stream", radio)
+# output.icecast(%raw, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="raw-stream", radio)
+# output.icecast(%s16l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16l-stream", radio)
+# output.icecast(%s16b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s16b-stream", radio)
+# output.icecast(%s24l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24l-stream", radio)
+# output.icecast(%s24b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s24b-stream", radio)
+# output.icecast(%s32l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32l-stream", radio)
+# output.icecast(%s32b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s32b-stream", radio)
+# output.icecast(%s64l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64l-stream", radio)
+# output.icecast(%s64b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s64b-stream", radio)
+# output.icecast(%s128l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128l-stream", radio)
+# output.icecast(%s128b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s128b-stream", radio)
+# output.icecast(%s256l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256l-stream", radio)
+# output.icecast(%s256b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s256b-stream", radio)
+# output.icecast(%s512l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512l-stream", radio)
+# output.icecast(%s512b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s512b-stream", radio)
+# output.icecast(%s1024l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024l-stream", radio)
+# output.icecast(%s1024b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s1024b-stream", radio)
+# output.icecast(%s2048l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048l-stream", radio)
+# output.icecast(%s2048b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s2048b-stream", radio)
+# output.icecast(%s4096l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096l-stream", radio)
+# output.icecast(%s4096b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s4096b-stream", radio)
+# output.icecast(%s8192l, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192l-stream", radio)
+# output.icecast(%s8192b, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="s8192b-stream", radio)

conjurer_musician/requirements_musician.txt

@@ -1,2 +1,3 @@
 flask
-waitress
+waitress
+sshtunnel

conjurer_musician/stream.html

@@ -0,0 +1,9 @@
+    <html>
+        <head>
+            <title>HTTP Live Streaming Example</title>
+        </head>
+        <body>
+            <video src="/tmp/hls/stream.m3u8" height="300" width="400">
+            </video>
+        </body>
+    </html>

install_music_bot.sh

@@ -8,9 +8,17 @@ cp /home/pi/conjurer/conjurer_musician/requirements_musician.txt /home/pi/Conjur
 cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/
 cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/
 touch /home/pi/Conjurer/discord_mus_service.log
-touch /home/pi/Conjurer/all_playlist.txt
+
 touch /home/pi/Conjurer/playlist.json
+touch /home/pi/Conjurer/prio_playlist.json
+
+touch /home/pi/Conjurer/all_playlist.playlist
 touch /home/pi/Conjurer/priority_queue.playlist
+touch /home/pi/Conjurer/hit.playlist
+touch /home/pi/Conjurer/request.playlist
+touch /home/pi/Conjurer/persistence.log
+echo "[]" > /home/pi/Conjurer/persistence.log
+
 touch /home/pi/Conjurer/radio_log.log
 ./env/bin/python3 -m pip install --upgrade pip
 ./env/bin/python3 -m pip install -r requirements_musician.txt

requirements_bot.txt

@@ -12,4 +12,5 @@ spotipy
 tiktoken
 PyNaCl
 flask
-waitress
+waitress
+clickupython

restart_radio.sh

@@ -0,0 +1,3 @@
+sshpass -p raspberry ssh pi@192.168.1.15 << 'ENDSSH'
+sudo reboot
+ENDSSH

settings.json

@@ -204,6 +204,13 @@
             false,
             false
         ],
+        "same plusy\\b": [
+            "Jak na cmentarzu Szefie. Jak na cmentarzu",
+            15.0,
+            0,
+            false,
+            false
+        ],
         "kld\\b": [
             "KLD? Mieli tam burdel większy niż u Hammera. Ale o ile Hammer sprzątnął już nawet całkiem nieźle - to tam śmieci wyniosły się same. Uczcijmy ich minutą śmiechu.",
             15.0,

spotify_dl/spotify.py

@@ -239,6 +239,8 @@ def get_item_name(sp, item_type, item_id):
         name = sp.album(album_id=item_id).get("name")
     elif item_type == "track":
         name = sp.track(track_id=item_id).get("name")
+    elif item_type == "artist_top_tracks":
+        name = sp.artist_top_tracks(artist_id=item_id).get("name")
     return sanitize(name)
 
 

utils/update_memory.py

@@ -0,0 +1,33 @@
+import json
+MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
+ENCODING = "utf-8"
+
+messages = [
+    {"role": "user", "content" : "Hammer: All systems operational"},
+    {"role": "assistant", "content": "Aye, Szefie! Wszystkie systemy działają sprawnie. Gotów do dalszej służby! Co podać?"},
+    {"role": "user", "content" : "Hammer: Podaj coś na orzeźwienie i przebudzenie. COś co rozgrzeje moje stare kości po ciężkiej pracy!"},
+    {"role": "assistant", "content": "Z przyjemnością Szefie! Dla Ciebie przygotuję Midzynarodówkę Interstelarną Wolf's Dragoon - wysoce energetyczny koktajl rozgrzewający i pobudzający jednocześnie."+
+                                    "Do przygotowania potrzebuję:"+
+                                    "\n"+
+                                    "60 ml mocnego rumu z Wolfa (lub innego, jeśli wolisz)"+
+                                    "30 ml soku z cytryny"+
+                                    "35 ml domowego syropu z mięty"+
+                                    "15 ml świeżo zmielonej kawy"+
+                                    "\n"+
+                                    "Mieszam wszystko w shakerze z lodem, a potem przelewam przez sitko do kieliszka koktajlowego. W górę serwuję z bigoskiem z miętą i zestem z cytryny. Rozgrzewa, orzeźwia i daje kopa na resztę nocy."+
+                                    "\n"+
+                                    "Co powiesz Szefie, podać Ci taki?"},
+    {"role": "user", "content" : "Hammer: Brzmi jak coś co zrobi robotę! Skal!"},
+]
+
+
+for temp in messages:
+
+    with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
+        # First we load existing data into a dict.
+        file_data = json.load(file_memory)
+        # Join new_data with file_data inside emp_details
+        file_data.append(temp)
+        file_memory.seek(0)
+        # convert back to json.
+        json.dump(file_data, file_memory, indent=4)