Small fix

,

.gitignore

@@ -201,3 +201,4 @@ cr_results.json
 not_in_db.json
 rr_results.json
 s_results.json
+*.bak

.trunk/configs/.shellcheckrc

@@ -0,0 +1,7 @@
+enable=all
+source-path=SCRIPTDIR
+disable=SC2154
+
+# If you're having issues with shellcheck following source, disable the errors via:
+# disable=SC1090
+# disable=SC1091

.trunk/trunk.yaml

@@ -2,34 +2,38 @@
 # To learn more about the format of this file, see https://docs.trunk.io/reference/trunk-yaml
 version: 0.1
 cli:
-  version: 1.20.1
+  version: 1.22.2
 # Trunk provides extensibility via plugins. (https://docs.trunk.io/plugins)
 plugins:
   sources:
     - id: trunk
-      ref: v1.4.5
+      ref: v1.6.0
       uri: https://github.com/trunk-io/plugins
 # Many linters and tools depend on runtimes - configure them here. (https://docs.trunk.io/runtimes)
 runtimes:
   enabled:
+    - go@1.21.0
     - node@18.12.1
     - python@3.10.8
 # This is the section where you manage your linters. (https://docs.trunk.io/check/configuration)
 lint:
   enabled:
-    - actionlint@1.6.27
+    - shellcheck@0.10.0
-    - bandit@1.7.8
+    - shfmt@3.6.0
-    - black@24.3.0
+    - taplo@0.8.1
+    - actionlint@1.7.1
+    - bandit@1.7.9
+    - black@24.4.2
     - codespell
-    - checkov@3.2.48
+    - checkov@3.2.139
     - git-diff-check
     - isort@5.13.2
     - pylint
-    - markdownlint@0.39.0
+    - markdownlint@0.41.0
-    - prettier@3.2.5
+    - prettier@3.3.2
-    - ruff@0.3.4
+    - ruff@0.4.9
-    - trivy@0.50.1
+    - trivy@0.52.2
-    - trufflehog@3.71.0
+    - trufflehog@3.78.1
     - yamllint@1.35.1
 actions:
   disabled:

bot.py

@@ -19,6 +19,7 @@ import shutil
 import sys
 import threading
 import uuid
+import subprocess
 from datetime import datetime
 from logging import handlers
 from pathlib import Path, PurePath
@@ -135,10 +136,15 @@ MUZYKA_MOJEGO_LUDU_SLOWA_KLUCZOWE = 15
 MUZYKA_MOJEGO_LUDU_PLAJLISTA = 30
 
 FILE_SERVICE_ADDRESS = "http://192.168.1.15:5000"
+RADIO_HARBOR_ADDRESS = "http://192.168.1.15:54321"
+SKIP_TRACK = "/skip"
+
 GET_MP3 = "/mp3"
 SEND_MP3 = "/update_mp3"
 GET_PLAYLIST = "/get_music"
 ADD_TO_PRIO_PLAYLIST = "/add_to_priority"
+CREATE_PRIO_PLAYLIST = "/create_priority_playlist"
+
 REQUEST_MUSIC = "/request_radio_file"
 CLEAR_PRIO = "/clear_pr_pls"
 LIBRARIAN_SERVICE_ADDRESS = "http://192.168.1.192:5001"
@@ -929,17 +935,17 @@ async def handle_response(
         "Rozmiar zapytania przed dodaniem historii %s", chat_gpt_config_request_size
     )
     if request_type == "MUSIC":
-        algorithm = "gpt-3.5-turbo-0125"
+        algorithm = "gpt-4o"
         table = message_table_muzyka
         token_amount = 10700
     elif request_type == "RANDOM":
-        algorithm = "gpt-3.5-turbo-0125"
+        algorithm = "gpt-4o"
         table = MESSAGE_TABLE
         token_amount = 10700
     else:
         table = MESSAGE_TABLE
-        algorithm = "gpt-4"
+        algorithm = "gpt-4o"
-        token_amount = 7000
+        token_amount = 10700
 
     prompt_gpt_request_size = num_tokens_from_string(
         {"role": "user", "content": final_prompt}, "gpt-4"
@@ -1021,26 +1027,33 @@ async def handle_response(
     logger.info(result)
     temp = {"role": "assistant", "content": result}
     history.append(temp)
-    if request_type:
+    if request_type == "MUSIC":
-        with open(MEMORY_FIVE_MUZYKA, "r+", encoding=ENCODING) as file:
+        with open(MEMORY_FIVE_MUZYKA, "r+", encoding=ENCODING) as file_music_memory:
             # First we load existing data into a dict.
-            file_data = json.load(file)
+            file_data = json.load(file_music_memory)
             # Join new_data with file_data inside emp_details
             file_data.append(temp)
-            # Sets file's current position at offset.
+            file_music_memory.seek(0)
-            file.seek(0)
             # convert back to json.
-            json.dump(file_data, file, indent=4)
+            json.dump(file_data, file_music_memory, indent=4)
+    elif request_type == "RANDOM":
+        with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
+            # First we load existing data into a dict.
+            file_data = json.load(file_memory)
+            # Join new_data with file_data inside emp_details
+            file_data.append(temp)
+            file_memory.seek(0)
+            # convert back to json.
+            json.dump(file_data, file_memory, indent=4)
     else:
-        with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file:
+        with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
             # First we load existing data into a dict.
-            file_data = json.load(file)
+            file_data = json.load(file_memory)
             # Join new_data with file_data inside emp_details
             file_data.append(temp)
-            # Sets file's current position at offset.
+            file_memory.seek(0)
-            file.seek(0)
             # convert back to json.
-            json.dump(file_data, file, indent=4)
+            json.dump(file_data, file_memory, indent=4)
     return result, MESSAGE_TABLE
 
 
@@ -1338,6 +1351,46 @@ async def max_weight(lista):
 
 # *=========================================== Define Commands
 
+@client.hybrid_command(
+    name="skip_track",
+    description="Przeskocz kawałek w radiu",
+    guild=discord.Object(id=664789470779932693),
+)
+async def skip_track(ctx):
+    async with ctx.typing():
+        allowed = False
+        for role in ctx.author.roles:
+            if role.name in ("Thane","Jarl"):
+                allowed = True
+        if not allowed:
+            await ctx.send("Łapy precz od radia")
+        else:
+            coroutine = asyncio.to_thread(
+            requests.get,
+            f"{RADIO_HARBOR_ADDRESS}{SKIP_TRACK}",
+            timeout=360,
+            )
+            result = await coroutine
+            logger.info("Done %s", result)
+            await ctx.send("Zrobione szefie!")
+
+
+@client.hybrid_command(
+    name="zrestartuj_radio",
+    description="Komenda ktora uruchamia radio ponownie jakby się zawiesiło",
+    guild=discord.Object(id=664789470779932693),
+)
+async def zrestartuj_radio(ctx):
+    async with ctx.typing():
+        allowed = False
+        for role in ctx.author.roles:
+            if role.name in ("Thane","Jarl"):
+                allowed = True
+        if not allowed:
+            await ctx.send("Łapy precz od radia")
+        else:
+            retcode = subprocess.run("/home/pi/Conjurer/restart_radio.sh", shell=False, check = False, capture_output = True)
+            logger.info("Wynik: %s",retcode)
 
 @client.hybrid_command(
     name="dej_co_ze_spotifaja",
@@ -1788,7 +1841,9 @@ async def dodaj_do_ulubionych(ctx):
             json=jrequest,
             timeout=360,
         )
-        _ = await coroutine
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
 @client.hybrid_command(
     name="ja_chciol",
     description="Dodaje do listy ulubionych w radiu",
@@ -1818,7 +1873,46 @@ async def request_radio(ctx):
             json=jrequest,
             timeout=360,
         )
-        _ = await coroutine
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
+
+@client.hybrid_command(
+    name="stworz_audycje",
+    description="Dodaje do listy ulubionych w radiu",
+    guild=discord.Object(id=664789470779932693),
+)
+async def stworz_audycje(ctx):
+    """
+    Generate a playlist in queue. First word in this command shall be int defining length of the playlist.
+    Rest of the line are search terms.
+
+    :param ctx: ctx stands for "context" and is a parameter commonly used in Discord.py commands. It
+    represents the context in which the command was invoked, including information such as the message,
+    the channel, the server, and the user who invoked the command. This parameter is required in all
+    Discord.py commands
+    """
+    async with ctx.typing():
+        logger.info("Zaczynam szukać timestamp %s", datetime.now())
+
+        dlugosc_playlisty = ctx.message.content.split()[1]
+        word_list = ctx.message.content.split()
+        jrequest = {
+            "lista_slow": word_list,
+            "dlugosc_plejlisty": dlugosc_playlisty,
+            "UUID": str(uuid.uuid4()),
+        }
+        coroutine = asyncio.to_thread(
+            requests.post,
+            f"{FILE_SERVICE_ADDRESS}{CREATE_PRIO_PLAYLIST}",
+            json=jrequest,
+            timeout=360,
+        )
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
+
+
 @client.hybrid_command(
     name="wyczysc_ulubione",
     description="Czysci liste ulubionych w radiu",
@@ -1840,7 +1934,9 @@ async def wyczysc_ulubione(ctx):
             f"{FILE_SERVICE_ADDRESS}{CLEAR_PRIO}",
             timeout=360,
         )
-        _ = await coroutine
+        result = await coroutine
+        logger.info("Done %s", result)
+        await ctx.send("Zrobione szefie!")
 
 
 @client.hybrid_command(
@@ -2367,7 +2463,7 @@ async def krecimy_pornola(ctx):
                     sciezka, files = await get_file(ctx, "Pornol", item_yt)
                     if files:
                         logger.info("Pornol udany")
-                        ctx.reply(f"Jest w tajnym archiwum pod adresem{sciezka})")
+                        ctx.send(f"Jest w tajnym archiwum pod adresem{sciezka})")
     else:
         ctx.reply("Jak ładnie szefa poprosisz.")
 
@@ -2392,10 +2488,3 @@ if __name__ == "__main__":
         WRK_CNT+=1
         logger.info("Starting discord bot: Joining thread %s", WRK_CNT)
         worker.join()
-
-# widok - generuje się 5 stron po 4 linki
-# odśwież - zamienić na "dalej" - zaciąga kolejne 20 rezultatów
-# po wyslaniu czekamy na odbior i dodajemy UUID oraz nick poszukiwacza do queue ktora jest sprawdzana
-# flask sobie chodzi i jak dostaje posta z wynikami to wyciaga z queue UUID i odpowiada że znalazła
-# po samym wyszukaniu dajemy info że "Zaczynam szukać ale pare godzin to potrwa."
-# OPCJONALNIE: Dosyłamy znalezione wyniki na bieżąco - wtedy wkładamy do queue dla UUID wyniki

conjurer_librarian/conjurer_librarian.py

@@ -26,6 +26,7 @@ from queue import Queue
 import requests
 import scrape_bot
 import search_bot
+#import search_bot2 as search_bot
 from flask import Flask, jsonify, request
 from habanero import Crossref
 from waitress import serve
@@ -37,7 +38,10 @@ PORT_ADDRESS = 5001
 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
 SEND_RESULTS = "/conjurer"
 BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
-MAX_CR_RESULTS = 7000
+
+MAX_CR_RESULTS = 500
+#TEST PURPOSES ONLY!
+#MAX_CR_RESULTS = 5
 
 ENCODING = "utf-8"
 
@@ -114,14 +118,15 @@ class Librarian(object):
         self.app.logger.info("STARTED SEARCH")
         
         if not deep_search:
-            cr_result = self.cr.works(query=query, limit=1000)
+            query_limit = MAX_CR_RESULTS if MAX_CR_RESULTS < 1000 else 1000
+            cr_result = self.cr.works(query=query, limit=query_limit)
             self.search_result_from_cr.update(cr_result)
             self.total = cr_result["message"]["total-results"]
             self.fetched += len(cr_result["message"]["items"])
             self.app.logger.info(self.total)
             self.app.logger.info(self.fetched)
             while self.total > self.fetched and self.limit > self.fetched:
-                tmp_result = self.cr.works(query=query, limit=1000, offset=self.fetched)
+                tmp_result = self.cr.works(query=query, limit=query_limit, offset=self.fetched)
                 cr_result["message"]["items"].extend(tmp_result["message"]["items"])
                 self.total = tmp_result["message"]["total-results"]
                 self.fetched = len(cr_result["message"]["items"])
@@ -130,7 +135,7 @@ class Librarian(object):
                 time.sleep(0.1)
 
         else:
-            cr_result = self.cr.works(query=query, cursor_max=10000, cursor='*', progress_bar = True)
+            cr_result = self.cr.works(query=query, cursor_max=15000, cursor='*', progress_bar = True)
             result = cr_result[0]
             for item in cr_result[1:]:
                 result["message"]["items"].extend(item["message"]["items"])

conjurer_librarian/count_lines.py

@@ -0,0 +1,8 @@
+
+FILEPTH = r"C:\Database\chunks\4_chunk.txt"
+
+with open(FILEPTH, "r", encoding="utf-8") as file:
+    tab = file.readlines()
+    print(len(tab))
+
+    #1842295

conjurer_librarian/scrape_bot.py

@@ -33,12 +33,14 @@ def load_ndb_to_q(logger):
             try:
                 ndb_database = json.load(ndb_file)
                 for _ in range (1,10):
-                    key = next(iter(ndb_database))
+                    try:
-                    item = ndb_database.pop(key)
+                        key = next(iter(ndb_database))
-                    logger.info(key)
+                        _ = ndb_database.pop(key)
-                    url = f"https://sci-hub.se/{key}"
+                        logger.info(key)
-                    WORK_Q.put([item, url, False])
+                        url = f"https://sci-hub.se/{key}"
-
+                        WORK_Q.put([key, url, False])
+                    except StopIteration:
+                        break
                 ndb_file.truncate(0)
                 ndb_file.seek(0)
                 json.dump(ndb_database, ndb_file, indent=4)
@@ -102,9 +104,8 @@ def check_if_exists_brute_force(logger):
             pass
         if blocked:
             logger.info(item)
-            logger.info(text)
             logger.error("Got blocked. Fuck.")
-            time.sleep(60 * 60 * 72)
+            time.sleep(60 * 60)
         # trunk-ignore(bandit/B311)
         rand = random.randint(1, 60)
         logger.info(f"Sleeping for {2*rand} minutes")

conjurer_librarian/search_bot.py

@@ -21,8 +21,9 @@ Global Variables:
 # TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
 from queue import Empty, Queue
 from threading import Thread
-
+import time
 q = Queue()
+#TODO: Count number of lines in files and print to approximate on which part of the file search is
 
 # DATA FOR TEST ONLY
 # MAXTHREADS = 5
@@ -49,9 +50,17 @@ def producer(out_q, control_q, filename, _logger):
         _logger: Logger object for logging.
     """
     with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
-        _logger.info(f"Worker {filename} ")
+        print(f"Worker {filename} ")
+        line_no = 0
         while True:
             line = operated_file.readline()
+            line_no += 1
+            print(f"\t \t \t \t \t \t W{filename}{line_no}\r", end="")
+
+            if not line:
+                print(f"EOF {filename}")
+                break
+
             if not line:
                 break
             out_q.put(line)
@@ -59,11 +68,12 @@ def producer(out_q, control_q, filename, _logger):
                 check = control_q.get(block=False)
             except Empty:
                 check = False
+
             if check is _sentinel:
-                _logger.info("TERM signal received")
+                print("TERM signal received")
                 control_q.put(check)
                 break
-        _logger.info(f"Worker finished: {filename}")
+        print(f"Worker finished: {filename}")
         out_q.put(_sentinel)
 
 
@@ -79,19 +89,25 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no,
         live_results (list): List to store the search results.
         _logger: Logger object for logging.
     """
-    _logger.info("Consumer thread started:%s", no)
+    print(f"Consumer thread started: {no} no")
-    for item in doi:
+    empty_counter = 0
-        result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
+    alive_no = 0
     while True:
         done_check = True
         try:
-            data = in_q.get(block=False)
+            data = in_q.get(block=True, timeout = 1)
             if data is _sentinel:
-                _logger.info("Worker finished")
                 control_dict["sentinels"] += 1
+                print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
+
             else:
+                empty_counter = 0
+                alive_no += 1
+                print(f"C{no}__{alive_no}\r", end="")
+
                 for item in result_list:
                     if item["DOI"] in data and not item["exists"]:
+                        print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
                         _logger.info(data)
                         _logger.info("HIT")
                         item["exists"] = True
@@ -100,9 +116,18 @@ def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no,
                 if done_check:
                     control_q.put(_sentinel)
         except Empty:
-            pass
+            empty_counter += 1
+            time.sleep(1)
+            print(f"Consumer {no} empty")
+            if empty_counter > 5:
+                print("Consumer %s empty lvl 2", no)
+                time.sleep(2)
+            elif empty_counter > 10:
+                print(f"Consumer thread finished {no}")
+                break
+
         if control_dict["sentinels"] >= MAXTHREADS:
-            _logger.info("All workers finished")
+            _logger.info(f"All workers finished {no}")
             break
 
 
@@ -121,6 +146,10 @@ def search_for_doi(doi, live_results, _logger):
     threads = []
     work_q = Queue(maxsize=WORK_Q_SIZE)
     control_q = Queue()
+
+    for item in doi:
+        result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
+
     for i in range (0, (len(doi)//1000)+2):
         t_cons = Thread(
             target=consumer, args=(work_q, control_q, doi, live_results, result_list, control_dict, i, _logger)

conjurer_librarian/search_bot2.py

@@ -0,0 +1,206 @@
+"""
+This module contains functions for searching for DOI (Digital Object Identifier)
+in a list of live results.
+It includes a producer-consumer pattern implementation using threads and queues.
+
+Functions:
+- producer: Reads lines from a file and puts them into an output queue.
+- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
+- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
+
+Global Variables:
+- MAXTHREADS: Maximum number of worker threads.
+- DATABASE_PATH: Path to the database files.
+- ENCODING: Encoding of the database files.
+- CHUNK: File name pattern for the database files.
+- _sentinel: Sentinel object used to signal termination.
+- result_list: List to store the search results.
+- WORK_Q_SIZE: Maximum size of the work queue.
+"""
+
+# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
+from multiprocessing import Manager, Process, Queue
+from queue import Empty, Full
+import time
+
+# DATA FOR TEST ONLY
+#MAXTHREADS = 6
+#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+
+# DEPLOYMENT DATA
+MAXTHREADS = 41
+DATABASE_PATH = r"C:\\Database\\chunks\\"
+
+ENCODING = "utf-8"
+CHUNK = "_chunk.txt"
+_sentinel = None
+WORK_Q_SIZE = 355000
+
+
+def producer(out_q, control_q, filename, _logger):
+    """
+    Produces items from the output queue and puts them into the control queue.
+
+    Args:
+        out_q (Queue): Output queue.
+        control_q (Queue): Control queue.
+        filename (str): Name of the file.
+        _logger: Logger object for logging.
+    """
+    with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
+        print(f"Worker {filename} ")
+        line_no = 0
+        while True:
+            line = operated_file.readline()
+            line_no += 1
+            print(f"\t \t \t W{filename}{line_no}\r", end="")
+
+            if not line:
+                print(f"EOF {filename}")
+                break
+            try:
+                out_q.put([line, line_no, filename], block = True, timeout = 60)
+            except Full:
+                print("queue fulll")
+            try:
+                check = control_q.get(block=False)
+            except Empty:
+                check = False
+            if check is None:
+                print(f"TERM signal received {filename}")
+                control_q.put(check)
+                control_q.close()
+                break
+        print(f"Worker finished: {filename}")
+        out_q.put(_sentinel)
+        out_q.close()
+
+
+
+# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+def consumer(in_q, control_q, doi, live_results, result_list, control_dict, no, _logger):
+    """
+    Consumes items from an input queue and checks if DOI exists in the live results list.
+
+    Args:
+        in_q (Queue): Input queue.
+        control_q (Queue): Control queue.
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    print(f"Consumer thread started: {no} no")
+    empty_counter = 0
+    alive_no = 0
+    while True:
+        done_check = True
+        try:
+            data = in_q.get(block=True, timeout = 1)
+            empty_counter = 0
+            if data is None:
+                control_dict["sentinels"] += 1
+                print(f"Workers finished: {control_dict['sentinels']} reported by consumer {no}")
+            else:
+                empty_counter = 0
+                if alive_no > 9:
+                    alive_no = 0
+                alive_no += 1
+                print(f"C{no}{alive_no}\r", end="")
+                for item in result_list:
+                    if item["DOI"] in data[0] and not item["exists"]:
+                        print(f"HIT in {no} content {data[0]} line {data[1]} file {data[2]} {item["exists"]}")
+                        item["exists"] = True
+                        live_results.append(item)
+                    done_check = done_check and item["exists"]
+                if done_check:
+                    control_q.put(_sentinel)
+        except Empty:
+            empty_counter += 1
+            time.sleep(1)
+            print(f"Consumer {no} empty")
+            if empty_counter > 5:
+                print("Consumer %s empty lvl 2", no)
+                time.sleep(2)
+            elif empty_counter > 10:
+                print(f"Consumer thread finished {no}")
+                break
+        if control_dict["sentinels"] >= MAXTHREADS:
+            print(f"All workers finished killing {no}")
+            break
+    print("Consumer thread finished %s",no )
+    control_q.close()
+
+
+def search_for_doi(doi, live_results, _logger):
+    """
+    Search for DOI in live_results using _logger for logging.
+
+    Args:
+        doi (list): List of DOI to search for.
+        live_results (list): List to store the search results.
+        _logger: Logger object for logging.
+    """
+    with Manager() as manager:
+        control_dict = manager.dict()
+        control_dict["sentinels"] = 0
+        result_list = []
+        for item in doi:
+            result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
+        result_list_proxy = manager.list(result_list)
+
+        threads = []
+        work_q = Queue(maxsize=WORK_Q_SIZE)
+        control_q = Queue()
+        for i in range(0, MAXTHREADS):
+            # TEST DATA
+            #filename = "test" + str(i) + CHUNK
+            # DEPLOYMENT DATA
+            filename = str(i) + CHUNK
+            print(f"Creating worker thread no: {i}")
+            threads.append(
+                Process(target=producer, args=(work_q, control_q, filename, _logger))
+            )
+        for i in range (0, (len(doi)//1000)+5):
+            t_cons = Process(
+                target=consumer, args=(work_q, control_q, doi, live_results, result_list_proxy, control_dict, i, _logger)
+            )
+            print("Consumer thread created")
+            threads.append(t_cons)
+
+        for worker in threads:
+            worker.start()
+        for worker in threads:
+            worker.join()
+        result_list = list(result_list_proxy)
+        for worker in threads:
+            worker.close()
+        return result_list
+
+
+if __name__ == "__main__":
+    import logging
+
+    logger = logging.getLogger()
+    logger.setLevel("DEBUG")
+    h1 = logging.StreamHandler()
+    logger.addHandler(h1)
+    print("TEST RUN")
+    live_result = []
+    print(
+        search_for_doi(
+            [
+                ("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
+                ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
+                ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
+                ("10.2165/00128415-200309690-00017", "DATA"),
+                ("10.5772/48313", "DATA"),
+                ("10.15803/ijnc.7.2_419", "DATA"),
+                ("10.1051/0004-6361/201321596e", "DATA"),
+                ("10.2307/40835941", "DATA"),
+                ("NOT_ON_LIST", "DATA"),
+            ],
+            live_result,
+            logger,
+        )
+    )
+    print(live_result)

conjurer_musician/conjurer_musician.py

@@ -14,8 +14,7 @@ from logging import handlers
 from pathlib import Path
 from platform import uname
 from sys import platform
-
+from flask import Flask, jsonify, redirect, request, send_from_directory, render_template
-from flask import Flask, jsonify, redirect, request, send_from_directory
 from waitress import serve
 
 HOST_ADDRESS = "192.168.1.15"
@@ -207,6 +206,7 @@ def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
         fun_logger.info("Wiele plików do zagrania")
         search_weight.sort(key=lambda x: x[0], reverse=True)
         return_list.extend(search_weight[: int(how_many)])
+    fun_logger.info("Done: %s", return_list)
     return return_list
 
 
@@ -246,8 +246,9 @@ def stream_music():
 
     :return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
     """
-    return send_from_directory("/tmp/hls", "stream.m3u8")
 
+    #return send_from_directory("/tmp/hls", "stream.m3u8")
+    return render_template("/home/pi/Conjurer/stream.html")
 
 @app.route("/<string:file_name>")
 def stream(file_name):
@@ -260,10 +261,10 @@ def stream(file_name):
     Returns:
         Response: The response containing the streamed file.
     """
+    # trunk-ignore(bandit/B108)
     video_dir = "/tmp/hls"
     return send_from_directory(video_dir, file_name)
 
-
 @app.route("/stream_mp3", methods=["GET"])
 def stream_music_mp3():
     """
@@ -369,6 +370,37 @@ def add_request():
     )
     return return_data
 
+@app.route("/create_priority_playlist", methods=["POST"])
+def create_priority_playlist():
+    """
+    The function `create_priority_playlist` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+
+    :return: A tuple containing a JSON response and a status code.
+        The JSON response includes keys `isError`,
+        `message`, `statusCode`, and `data`, with values
+        indicating the success of the operation and the
+        data that was received and added to the `music_file_list`.
+        The status code returned is 200,indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    app.logger.info(record["dlugosc_plejlisty"])
+    return_data = wyszukaj(
+        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
+    )
+    with open(
+        "/home/pi/Conjurer/request.playlist", "a", encoding="utf-8"
+    ) as s_file:
+        for item in return_data:
+            s_file.write(item[1] + "\n")
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data={"status": "OK"}),
+        200,
+    )
+    return return_data
 
 
 @app.route("/add_to_priority", methods=["POST"])

conjurer_musician/music_utils.py

@@ -0,0 +1,26 @@
+# This Python file uses the following encoding: utf-8
+"""This module contains utility scripts used by musi radiostation Conjurer"""
+import time
+import re
+
+NAME = "/home/pi/Conjurer/persistence.log"
+
+def print_top():
+    """
+    Prints the top 65 lines from a file specified by the NAME variable.
+    """
+    while True:
+        line = ""
+        buffer = ""
+        with open(NAME, "r", encoding="utf-8") as f:
+            for _ in range (45):
+                line = f.readline()
+                if re.match(".*mp3", line):
+                    buffer += line
+            print("=====================================")
+            print(buffer)
+            print("=====================================")
+        time.sleep(180)
+
+if __name__ == "__main__":
+    print_top()

conjurer_musician/pl.config

@@ -0,0 +1,14 @@
+$dodaj_do_ulubionych 100 nightwish
+$dodaj_do_ulubionych 100 amon amarth
+$dodaj_do_ulubionych 100 tactical sekt
+$dodaj_do_ulubionych 100 lacuna coil
+$dodaj_do_ulubionych 100 sefa
+$dodaj_do_ulubionych 100 frenchcore
+$dodaj_do_ulubionych 100 peacock
+$dodaj_do_ulubionych 100 vavamuffin
+$dodaj_do_ulubionych 100 abradab
+$dodaj_do_ulubionych 100 eluveitie
+$dodaj_do_ulubionych 100 youtube
+
+$dodaj_do_ulubionych 100 elyose
+$dodaj_do_ulubionych 1 artisans du chaos

conjurer_musician/radio_conjurer.liq

@@ -9,12 +9,13 @@ let json.parse credentials  = file.contents("/home/pi/Conjurer/icecast_credentia
 enable_replaygain_metadata()
 
 # Set up a playlog for tracking played tracks
-l = playlog(duration = 14400.0, persistency="/home/pi/Conjurer/persistence.log")
+
+l = playlog(duration = 72000.0, persistency="/home/pi/Conjurer/persistence.log")
 
 # Function to check if a track can be played based on its metadata
 def check(r)
     m = request.metadata(r)
-    if l.last(m) < 3600. then
+    if l.last(m) < 36000. then
         log.info("Rejecting #{m['filename']} (played #{l.last(m)}s ago).")
         false
     else
@@ -33,27 +34,25 @@ requests_queue = request.queue()
 
 # Function to process the request queue and add new requests
 def queue_processing()
-    f1=file.read("/home/pi/Conjurer/request.playlist")
+    text=file.lines("/home/pi/Conjurer/request.playlist")
-    text = f1()
+    if text != [] then
-    if text != "" then
+        list.iter(fun(item) -> requests_queue.push.uri(item), text)
-        requests_queue.push.uri(text)
         file.remove("/home/pi/Conjurer/request.playlist")
         f = file.open("/home/pi/Conjurer/request.playlist", create=true)
         f.close()
-        log.info("done")
-    else
-        log.info("no requests")
     end
 end
 
 # Randomly select a playlist with weights
-s4 = random(id="randomizer", weights=[1, 3, 2], [s1, s2, s3])
+s4 = random(id="randomizer", weights=[2, 3, 5], [s1, s2, s3])
 
 # Load jingles playlist
 jingles = (playlist(reload_mode="watch", "/home/pi/Conjurer/jingles.playlist"))
 
 # Create the main stream with random playlist and jingles
-s = rotate(id="randomizer", weights=[20, 1], [s4, jingles])
+s = rotate(id="randomizer", weights=[10, 1], [s4, jingles])
+
+s = crossfade(fade_out=2., fade_in=2., duration=4., fallback(id="switcher", track_sensitive=false, [requests_queue, s]))
 
 # Set up an interactive harbor for controlling the stream
 interactive.harbor(port = 9999)
@@ -66,20 +65,29 @@ s = add([s, b])
 
 # Set up interactive control for main volume
 a = interactive.float("main_volume", min=0., max=20., 1.)
-s = compress.multiband.interactive(bands=3, s)
+s = compress.multiband.interactive(bands=7, s)
 
 # Apply audio processing effects
 s = nrj(normalize(s))
 s = amplify(a, s)
 
+mic = buffer(input.pulseaudio()) # Microphone
+mic = gate(threshold=-30., range=-80., mic)
+mic = blank.strip(max_blank=10., min_noise=.1, threshold=-20., mic)
+
+s = add([s, mic])
+#w = interactive.float("wetness", min=0., max=1., 1.)
+#s = dry_wet(w,s,s2)
+
+
 # Skip blank sections in the stream
-s = blank.skip(max_blank=2., s)
+s = blank.skip(max_blank=10., s)
 
 # Configure logging settings
 log_to_stdout = true
 log_to_file = true
 logpath = "/home/pi/Conjurer/radio_log.log"
-loglevel = 4
+loglevel = 3
 set("log.stdout", log_to_stdout)
 set("log.level", loglevel)
 
@@ -89,10 +97,16 @@ set("log.file.path", logpath)
 
 # Set up emergency fallback track
 emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
+radio =  fallback(id="switcher2", track_sensitive=false, [s, emergency])
+# Set up an interactive control for skipping tracks
+p = interactive.bool("Skip track", false)
+
+def http_skip(~protocol, ~data, ~headers, uri)=
+    radio.skip()
+    http.response(code=200,data="Skipped")
+end
+harbor.http.register(port=54321,method="GET","/skip", http_skip)
 
-# Create a fallback stream with the main stream and emergency track
-cross_s = crossfade(fade_out=3., fade_in=3., duration=5., fallback(id="switcher", track_sensitive=false, [requests_queue, s]))
-radio =  fallback(id="switcher2", track_sensitive=false, [cross_s, emergency])
 
 # Function to process the request queue and skip the current track if requested
 def check_skip()
@@ -106,26 +120,19 @@ end
 # Run the queue processing function every 60 seconds
 thread.run(every=60., queue_processing)
 
-# Set up an interactive control for skipping tracks
-p = interactive.bool("r1", false)
 
 # Run the check_skip function every 5 seconds
-thread.run(every=5., check_skip)
+thread.run(every=15., check_skip)
 
-# Handle metadata events
-handle_metadata = fun (m) -> begin
-    log.info(m["title"])
-    log.info(m["artist"])
-    log.info(m["filename"])
-end
-radio.on_metadata(handle_metadata)
 
 # Enable persistent script parameters
 interactive.persistent("script.params")
 
 # Configure output formats and destinations
-output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio)
+
 output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)
+output.pulseaudio(radio)
+#output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio)
 # Uncomment the following lines to enable additional output formats
 # output.icecast(%opus, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="opus-stream", radio)
 # output.icecast(%ogg, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="ogg-stream", radio)

conjurer_musician/stream.html

@@ -0,0 +1,9 @@
+    <html>
+        <head>
+            <title>HTTP Live Streaming Example</title>
+        </head>
+        <body>
+            <video src="/tmp/hls/stream.m3u8" height="300" width="400">
+            </video>
+        </body>
+    </html>

requirements_bot.txt

@@ -13,3 +13,4 @@ tiktoken
 PyNaCl
 flask
 waitress
+clickupython

restart_radio.sh

@@ -0,0 +1,3 @@
+sshpass -p raspberry ssh pi@192.168.1.15 << 'ENDSSH'
+sudo reboot
+ENDSSH

utils/update_memory.py

@@ -0,0 +1,33 @@
+import json
+MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
+ENCODING = "utf-8"
+
+messages = [
+    {"role": "user", "content" : "Hammer: All systems operational"},
+    {"role": "assistant", "content": "Aye, Szefie! Wszystkie systemy działają sprawnie. Gotów do dalszej służby! Co podać?"},
+    {"role": "user", "content" : "Hammer: Podaj coś na orzeźwienie i przebudzenie. COś co rozgrzeje moje stare kości po ciężkiej pracy!"},
+    {"role": "assistant", "content": "Z przyjemnością Szefie! Dla Ciebie przygotuję Midzynarodówkę Interstelarną Wolf's Dragoon - wysoce energetyczny koktajl rozgrzewający i pobudzający jednocześnie."+
+                                    "Do przygotowania potrzebuję:"+
+                                    "\n"+
+                                    "60 ml mocnego rumu z Wolfa (lub innego, jeśli wolisz)"+
+                                    "30 ml soku z cytryny"+
+                                    "35 ml domowego syropu z mięty"+
+                                    "15 ml świeżo zmielonej kawy"+
+                                    "\n"+
+                                    "Mieszam wszystko w shakerze z lodem, a potem przelewam przez sitko do kieliszka koktajlowego. W górę serwuję z bigoskiem z miętą i zestem z cytryny. Rozgrzewa, orzeźwia i daje kopa na resztę nocy."+
+                                    "\n"+
+                                    "Co powiesz Szefie, podać Ci taki?"},
+    {"role": "user", "content" : "Hammer: Brzmi jak coś co zrobi robotę! Skal!"},
+]
+
+
+for temp in messages:
+
+    with open(MEMORY_FIVE_SIARA, "r+", encoding=ENCODING) as file_memory:
+        # First we load existing data into a dict.
+        file_data = json.load(file_memory)
+        # Join new_data with file_data inside emp_details
+        file_data.append(temp)
+        file_memory.seek(0)
+        # convert back to json.
+        json.dump(file_data, file_memory, indent=4)