fx

aa

aa

aaa

aa

Chyba fix

aa

aa

aa

tst

test

FX

Maybe this ?

Fix

last fix ?

dix

.gitignore

@@ -197,7 +197,3 @@ node_modules/prettier/esm/parser-postcss.mjs
 node_modules/prettier/esm/parser-typescript.mjs
 node_modules/prettier/esm/parser-yaml.mjs
 node_modules/prettier/esm/standalone.mjs
-cr_results.json
-not_in_db.json
-rr_results.json
-s_results.json

bot.py

@@ -131,6 +131,43 @@ class MusicFileList(object):
         post_data = {"item": str(item)}
         requests.post(f"{FILE_SERVICE_ADDRESS}{SEND_MP3}", json=post_data, timeout=360)
 
+class DoSearchView(discord.ui.View):
+    def __init__(self, _uuid, result_list, *args, **kwargs) -> None:
+        super().__init__(*args, **kwargs)
+        self.uuid = _uuid
+        self.result_list = result_list
+
+    @discord.ui.button(
+        label="Nastepna strona", style=discord.ButtonStyle.primary, row=0
+    )
+    async def next_callback(self, interaction, button):
+        # emoji = await ctx.guild.fetch_emoji(id)
+        # button.emoji=1102174413887111189
+        logger.info("Pressed")
+        await interaction.response.edit_message(
+            content="You clicked the button!"
+        )  # Send a message when the button is clicked
+
+    @discord.ui.button(label="Wyszukaj wiecej", style=discord.ButtonStyle.danger, row=0)
+    async def reset_callback(self, interaction, button):
+        # emoji = await ctx.guild.fetch_emoji(id)
+        # button.emoji=1102174413887111189
+        logger.info("Pressed")
+        await interaction.response.edit_message(
+            content="You clicked the button!"
+        )  # Send a message when the button is clicked
+
+    @discord.ui.button(
+        label="Poprzednia Strona", style=discord.ButtonStyle.primary, row=0
+    )
+    async def perv_callback(self, interaction, button):
+        # emoji = await ctx.guild.fetch_emoji(id)
+        # button.emoji=1102174413887111189
+        logger.info("Pressed")
+        await interaction.response.edit_message(
+            content="You clicked the button!"
+        )  # Send a message when the button is clicked
+
 
 # *=========================================== Predefines
 formatter = logging.Formatter("%(asctime)s - %(name)s - %(levelname)s - %(message)s")
@@ -314,6 +351,9 @@ async def on_ready():
 
 
 @client.event
+# trunk-ignore(pylint/R0915)
+# trunk-ignore(pylint/R0914)
+# trunk-ignore(pylint/R0912)
 async def on_message(message):
     """
     Handle incoming messages in a Discord server, perform various
@@ -567,6 +607,8 @@ async def disconnect(ctx):
             await voice_client.disconnect()
 
 
+# trunk-ignore(pylint/R0915)
+# trunk-ignore(pylint/R0912)
 async def play(ctx, zamawial=None, arg=None):
     """
     Play a music file, retrieve metadata about the file, and send a
@@ -686,26 +728,24 @@ async def check_music():
             await asyncio.sleep(2)
 
 @tasks.loop(seconds=3)
+# trunk-ignore(pylint/R0915)
 async def check_data_q():
-    """
-    This function checks the data queue for any new entries and processes them accordingly.
-    """
     try:
-        fresh_data = IN_COMM_Q.get(block=False)
+        data = IN_COMM_Q.get(block=False)
         entries = []
-        if fresh_data.stop:
-            searcher = fresh_data.author
-            ctx = fresh_data.ctx
-            query = fresh_data.content
+        if data.stop:
+            searcher = data.author
+            ctx = data.ctx
+            query = data.content
             l_p = 1
-            for doi in fresh_data.entries:
+            for doi in data.entries:
                 logger.info(doi)
-                desc = fresh_data.entries[doi]
+                desc = data.entries[doi]
                 title = desc['Title'][0]
                 entries.append(f"{l_p}. {title} pod linkiem https://www.sci-hub.se/{doi} i jest to {desc['type']}\n")
                 l_p+=1
             message = '*Z podłogi wysuwa się winda na książki*'
-            message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query} nr {fresh_data.uuid}'
+            message += f' Dzielny poszukiwaczu @{searcher} w odpowiedzi na twoje zapytanie {query}'
             message += 'nasza biblioteka służy Ci odpowiedzią. Przy dźwięku fanfar winda się otwiera a w środku '
             if len(entries)<1:
                 message += "niestety nie ma nic"
@@ -728,11 +768,13 @@ async def check_data_q():
         pass
 
 @tasks.loop(seconds=120)
+# trunk-ignore(pylint/R0915)
 async def check_self():
     """
     The function `check_data` periodically checks for conditions to send messages and manage log files
     in a Discord channel.
     """
+    # trunk-ignore(codespell/misspelled)
     # logger.info("Heartbeat of cleanup proc")
     channel = client.get_channel(1062047367337095268)
     messages = [message async for message in channel.history(limit=1)]
@@ -795,7 +837,9 @@ async def check_self():
                 await channel.send(
                     "*Przeciąga się za barem* No dobra - porobione to można dalej pi... *Zauważa spojrzenie Hammera* eprzyć o głupotach z klientami... Szefie... *Bierze 'ukradkowy' łyk z piersiówki*"
                 )
+    # trunk-ignore(pylint/W0603)
     global LAST_SPONTANEOUS_CALL
+    # trunk-ignore(pylint/W0603)
     global TIME_BETWEEN_CALLS
     tdelta = datetime.now() - LAST_SPONTANEOUS_CALL
     tdelta = tdelta.total_seconds()
@@ -831,6 +875,7 @@ async def get_random_cyclic_message():
         mess_key = list(cyclic_words.keys())[messnum]
         return cyclic_words[mess_key][0]
     ai_check2 = random.randint(0, 10)
+    # trunk-ignore(pylint/W0603)
     global MESSAGE_TABLE
     if ai_check2 < 6:
         logger.info("Dykteryjka")
@@ -864,6 +909,10 @@ async def get_random_cyclic_message():
     return result
 
 
+# trunk-ignore(pylint/R0912)
+# trunk-ignore(pylint/R0913)
+# trunk-ignore(pylint/R0914)
+# trunk-ignore(pylint/R0915)
 async def handle_response(
     prompt, vykidailo, bartender, history, username, request_type
 ):
@@ -1052,6 +1101,8 @@ async def handle_response(
     return result, MESSAGE_TABLE
 
 
+# trunk-ignore(pylint/R0915)
+# trunk-ignore(pylint/R0914)
 async def get_file(ctx, source, link):
     """
     Take in three parameters: "ctx" (context),
@@ -1272,6 +1323,9 @@ async def get_stats(ctx, history_limit):
             yield name, how_many
 
 
+# trunk-ignore(pylint/R0915)
+# trunk-ignore(pylint/R0912)
+# trunk-ignore(pylint/R0914)
 async def wyszukaj(ctx, how_many=0, slowa_kluczowe=None):
     """
     Take in a context object and an optional integer parameter "how_many" and perform search
@@ -1431,6 +1485,8 @@ async def dej_co_z_jutuba(ctx):
     description="Jeśli nie wiesz jak użyć tej komendy to nawet nie próbuj",
     guild=discord.Object(id=664789470779932693),
 )
+# trunk-ignore(pylint/R0914)
+# trunk-ignore(pylint/R0912)
 async def update_banlist(ctx):
     """
     Update a banlist in a Discord guild from preprovided list in channel.
@@ -2002,6 +2058,8 @@ async def get_image_stable_diffusion(ctx):
     description="Gra muzyke wyszukana na bazie tematow konwersacji na kanale NZ",
     guild=discord.Object(id=664789470779932693),
 )
+# trunk-ignore(pylint/R0912)
+# trunk-ignore(pylint/R0915)
 async def zagraj_muzyke_mego_ludu(ctx):
     """
     Play completeley random playlist based on messaging history.
@@ -2151,20 +2209,11 @@ async def parametry_muzyki_mego_ludu(
     guild=discord.Object(id=664789470779932693),
 )
 async def wyszukaj_linki_do_dokumentow(ctx):
-    """
-    The function `wyszukaj_linki_do_dokumentow` searches for links to documents in a crossref database and provides links to scihub.
-
-    :param ctx: The `ctx` parameter is typically used in Discord.py, a Python library for creating Discord bots.
-    It represents the context of the command being executed, including information about the message, the server,
-    and the user who invoked the command.
-    """
     query = ctx.message.content
     query_uuid = uuid.uuid4()
     #TODO: TESTING ONLY!!
     #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
-    ctx.message.content = ctx.message.content.replace("$wyszukaj_linki_do_dokumentow", "")
-
-    json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": False}
+    json_query = {"UUID": str(query_uuid), "query": str(query)[30:], "page": 1}
     coroutine = asyncio.to_thread(
         requests.post,
         f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
@@ -2173,8 +2222,8 @@ async def wyszukaj_linki_do_dokumentow(ctx):
     )
     await ctx.send(
         "*Conjurer powoli notuje podane przez Ciebie. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" +
-        " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to 3/4 stacji."+
-        " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc 12/8 stacji teraz będzie ciężko pracować"
+        " wrzuca twój liścik do środka* To teraz trza poczekać kilka godzin. Biblioteka to te 3/4 stacji."+
+        " A przecież pół stacji to browar, motelik dla zabaw cielesnych i ten Bar. Więc te 12/8 stacji teraz będzie ciężko pracować"
     )
     query_response = await coroutine
     if not query_response.status_code == 200:
@@ -2196,87 +2245,6 @@ async def wyszukaj_linki_do_dokumentow(ctx):
     await ctx.send(f"No dobra poszło. Zapamiętaj proszę {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce."+
     " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzi - ale mogą być też dni.")
 
-@client.hybrid_command(
-    name="glebokie_gardlo",
-    description="Przygotowuje drinka o nazwie głębokie gardło",
-    guild=discord.Object(id=664789470779932693),
-)
-async def wyszukaj_linki_do_dokumentow_deep(ctx):
-    """
-    The function `wyszukaj_linki_do_dokumentow_deep` performs a deep search for links to documents in
-    a crossref database and provides links to scihub.
-
-    :param ctx: The `ctx` parameter is typically used in Discord.py, a Python library for creating Discord bots.
-    It represents the context of the command being executed, including information about the message, the server,
-    and the user who invoked the command.
-    """
-    # TODO: Implement deep search logic here
-    query = ctx.message.content.replace("$glebokie_gardlo", "")
-    allowed = False
-    for role in ctx.message.author.roles:
-        if role.name == "Bartender":
-            allowed = True
-        if role.name == "Scribe":
-            allowed = True
-        if role.name == "Thane":
-            allowed = True
-    if not allowed:
-        if ctx.message.author.nick:
-            username = ctx.message.author.nick
-        else:
-            username = ctx.message.author.name
-        vykidailo = False
-        bartender = False
-        prompt = "Przygotuj mi drinka o nazwie Głębokie Gardło, inspirowanego tym sławnym filmem oraz skandalem Watergate"
-        for role in ctx.message.author.roles:
-            if role.name == "Vykidailo":
-                vykidailo = True
-            if role.name == "Bartender":
-                bartender = True
-        global MESSAGE_TABLE  # pylint: disable=global-statement
-
-        result, MESSAGE_TABLE = await handle_response(
-            prompt, vykidailo, bartender, MESSAGE_TABLE, username, "CONVERSATION"
-        )
-        await ctx.send(result)
-        return
-
-    query_uuid = uuid.uuid4()
-    #TODO: TESTING ONLY!!
-    #query_uuid = uuid.UUID('{12345678-1234-5678-1234-567812345678}')
-    json_query = {"UUID": str(query_uuid), "query": str(query), "page": 1, "deep_search": True}
-    coroutine = asyncio.to_thread(
-        requests.post,
-        f"{LIBRARIAN_SERVICE_ADDRESS}{SEND_QUERY}",
-        json=json_query,
-        timeout=360,
-    )
-    await ctx.send(
-        "*Conjurer mruga okiem i zamiast drinka wyjmuje dysk usb z terminala. Podchodzi do sprytnie ukrytej przy barze rury od poczty pneumatycznej i" +
-        " wrzuca ten dysk do środk* To teraz trza poczekać kilka godzin. TO będzie głębokie wyszukanie"+
-        " Rozgląda się dookoła i stawia przed tobą mętnego drinka o zapachu mięty, gwiezdnego pyłu i oleju silnikowego"
-    )
-    query_response = await coroutine
-    if not query_response.status_code == 200:
-        await ctx.send("*Z rury wydobywa się dym. Conjurer łapie pierwszą ciecz pod ręką i pryska na rurę. Następuje drobna eksplozja i wszystko zaczyna się palić."+
-        " Conjurer jest skonfundowany..* Wołaj szefa - mam nieodparte wrażenie że się coś wyjebało")
-        return
-
-    query, query_uuid, queue_size = (
-        query_response.json()["data"][0],
-        query_response.json()["data"][1],
-        query_response.json()["data"][2],
-    )
-    if ctx.message.author.nick:
-        username = ctx.message.author.nick
-    else:
-        username = ctx.message.author.name
-    query_object = QueryControl(username, query_uuid, query, logger, ctx)
-    OUT_COMM_Q.put(query_object)
-    await ctx.send(f"Wypij wypi {query_uuid} - to identyfikator twojego zapytania. Jesteś {queue_size} w kolejce niestety kolejka obowiazuje zawsze."+
-    " Zapytania obsługuje algorytm zasilany czterema chomikami zapierdalającymi w kołowrotku - więc wyniki najwcześniej za kilka godzin - ale mogą być też dni. Głębokie zabawy trwają dłużej. *Znowu mruga*")
-
-
 
 @client.hybrid_command(
     name="krecimy_pornola",
@@ -2321,16 +2289,16 @@ if __name__ == "__main__":
     threads.append(threading.Thread(target=client.run, args=(TOKEN,)))
     threads.append(threading.Thread(target=comm_subroutine, args=(logger,)))
     logger.info("Starting discord bot: Starting threads")
-    WRK_CNT = 0
+    i = 0
     for worker in threads:
-        WRK_CNT+=1
-        logger.info("Starting discord bot: Starting thread {WRK_CNT}")
+        i+=1
+        logger.info("Starting discord bot: Starting thread {i}")
         worker.start()
     logger.info("Starting discord bot: Joining threads")
-    WRK_CNT=0
+    i=0
     for worker in threads:
-        WRK_CNT+=1
-        logger.info("Starting discord bot: Joining thread {WRK_CNT}")
+        i+=1
+        logger.info("Starting discord bot: Joining thread {i}")
         worker.join()
 
 # widok - generuje się 5 stron po 4 linki

conjurer_librarian/conjurer_librarian.py

@@ -1,398 +1,231 @@
-"""
-This module contains the implementation of the Librarian class and
-related functions for searching and refining queries.
-
-Classes:
--   Librarian: Represents a librarian object that performs search and
-    refinement operations on queries
-
-Functions:
--   flask_debug: Starts a Flask application in debug mode without using the reloader.
--   waitress_run: Serves the Flask application using the Waitress WSGI server.
--   BackgroundTaskSearch: Represents a background task for running the
-    Librarian object asynchronously
-"""
-
-import asyncio
-import json
+import re
 import logging
-import netrc
 import threading
 import time
-from json.decoder import JSONDecodeError
-from logging import handlers
-from queue import Queue
-
-import requests
-import scrape_bot
-import search_bot
 from flask import Flask, jsonify, request
+import netrc
+import json
+import asyncio
+from json.decoder import JSONDecodeError
+from urllib.request import urlopen
 from habanero import Crossref
 from waitress import serve
-
-# Constants
+from queue import Queue
+import search_bot
+import requests
+from typing import Callable
+from uuid import UUID
 NETRC_FILE = r"C:\Users\Activcom.pl\.netrc"
+
 HOST_ADDRESS = "192.168.1.192"
 PORT_ADDRESS = 5001
 MAIN_BOT_ADDRESS = "http://192.168.1.191:5000"
 SEND_RESULTS = "/conjurer"
-BDSM_UUID_TEST = "96b7f85a-1142-4908-8986-62a2ea25a147"
-MAX_CR_RESULTS = 7000
+BDSM_UUID_TEST = '96b7f85a-1142-4908-8986-62a2ea25a147'
 
 ENCODING = "utf-8"
-
 app = Flask(__name__)
 
 librarian_queue = Queue()
 librarian_list = []
 
-
-# trunk-ignore(pylint/R0902)
 class Librarian(object):
-    """
-    Represents a librarian object that performs search and refinement operations on queries.
-    """
 
-    def __init__(self, _app, query, uuid, _deep_search) -> None:
-        """
-        Initializes a Librarian object.
-
-        Args:
-        - _app: The Flask application object.
-        - query: The query to be searched.
-        - uuid: The unique identifier for the search.
-
-        Attributes:
-        - cr: The Crossref object for performing the search.
-        - query: The query to be searched.
-        - uuid: The unique identifier for the search.
-        - limit: The maximum number of search results to fetch.
-        - fetched: The number of search results fetched so far.
-        - hit: The number of search results that match the refinement criteria.
-        - total: The total number of search results.
-        - app: The Flask application object.
-        - live_results: A list to store live search results.
-        - final_result: A list to store the final refined search results.
-        - not_in_db: A list to store search results that are not in the local database.
-        - search_result_from_cr: A dictionary to store the search results from Crossref.
-        - done: A flag indicating if the search is done.
-        """
+    def __init__(self, app, query, uuid) -> None:
         netrc_mod = netrc.netrc(NETRC_FILE)
-        auth_tokens = netrc_mod.authenticators("crossref")
-        self.cr = Crossref(
-            mailto=auth_tokens[0],
-            ua_string=f"Conjurer project. mailto:{auth_tokens[0]}"
-        )
+        authTokens = netrc_mod.authenticators("crossref")
+        self.cr = Crossref(mailto=authTokens[0])
         self.query = query
         self.uuid = str(uuid)
-        self.limit = MAX_CR_RESULTS
+        self.limit = 3000
         self.fetched = 0
         self.hit = 0
         self.total = 0
-        self.app = _app
+        self.app = app
+        self.logger = app.logger
         self.live_results = []
-        self.final_result = {}
-        self.not_in_db = {}
         self.search_result_from_cr = {}
         self.done = False
-        self.deep_search = _deep_search
-
-    async def search_crossref(self, query, deep_search=False):
-        """
-        Performs a search on Crossref for the given query.
-
-        Args:
-        - query: The query to be searched.
-
-        Returns:
-        - result: The search result from Crossref.
-
-        Raises:
-        - None.
-        """
 
+    async def search_crossref(self, query):
         self.app.logger.info("STARTED SEARCH")
+        result = self.cr.works(query = query, limit = 1000)
+        self.search_result_from_cr.update(result)
+        self.total = result['message']['total-results']
+        self.fetched += len(result['message']['items'])
+        self.logger.info (self.total)
+        self.logger.info(self.fetched)
+        while self.total > self.fetched and self.limit > self.fetched:
+            result = self.cr.works(query = query, limit = 1000)
+            self.search_result_from_cr.update(result)
+            self.total = result['message']['total-results']
+            self.fetched += len(result['message']['items'])
+            self.logger.info (self.total)
+            self.logger.info(self.fetched)
+
+            time.sleep(1)
+        self.app.logger.info("CROSSREF DONE")
+        #TODO: add to sql with UUID
+        #sql row - title, doi, author, uuid, last retrieved, is_available
         
-        if not deep_search:
-            cr_result = self.cr.works(query=query, limit=1000)
-            self.search_result_from_cr.update(cr_result)
-            self.total = cr_result["message"]["total-results"]
-            self.fetched += len(cr_result["message"]["items"])
-            self.app.logger.info(self.total)
-            self.app.logger.info(self.fetched)
-            while self.total > self.fetched and self.limit > self.fetched:
-                tmp_result = self.cr.works(query=query, limit=1000, offset=self.fetched)
-                cr_result["message"]["items"].extend(tmp_result["message"]["items"])
-                self.total = tmp_result["message"]["total-results"]
-                self.fetched = len(cr_result["message"]["items"])
-                self.app.logger.info(self.total)
-                self.app.logger.info(self.fetched)
-                time.sleep(0.1)
-
-        else:
-            cr_result = self.cr.works(query=query, cursor_max=10000, cursor='*', progress_bar = True)
-            result = cr_result[0]
-            for item in cr_result[1:]:
-                result["message"]["items"].extend(item["message"]["items"])
-                self.total = item["message"]["total-results"]
-                self.fetched = len(result["message"]["items"])
-                self.app.logger.info(self.total)
-                self.app.logger.info(self.fetched)
-            cr_result = result
-            self.app.logger.info("Total, fetched:")
-            self.app.logger.info(self.total)
-            self.app.logger.info(self.fetched)
-            self.search_result_from_cr.update(cr_result)
-            self.total = cr_result["message"]["total-results"]
-            self.app.logger.info("CROSSREF DONE")
-
-            self.app.logger.info("CROSSREF DONE")
-        with open("cr_results.json", "r+", encoding="utf-8") as data_file:
-            # First we load existing data into a dict.
-            try:
-                file_data = json.load(data_file)
-            except JSONDecodeError:
-                file_data = {}
-            data_file.truncate(0)
-            data_file.seek(0)
-            tmp = {self.uuid : self.search_result_from_cr}
-            if file_data:
-                file_data.update(tmp)
-            else:
-                file_data = tmp
-            json.dump(file_data, data_file, indent=4)
-        return cr_result
-
-
-    async def refine_search(self, unrefined_result):
-        """
-        Refines the search query based on the unrefined search result.
-
-        Args:
-        - unrefined_result: The unrefined search result.
-
-        Returns:
-        - refined_result: The refined search result.
-
-        Raises:
-        - None.
-        """
-        summarized_results = []
-        self.app.logger.info("REFINE: Removing all derived works from the list")
-        for item in unrefined_result["message"]["items"]:
-            summarized_results.append(
-                {
-                    "DOI": item["DOI"],
-                    "title": item["title"] if "title" in item else None,
-                    "type": item["type"] if "type" in item else None,
-                }
-            )
-
-        partial_result = {
-            self.uuid: {
-                "total_results": self.search_result_from_cr["message"][
-                    "total-results"
-                ],
-                "on_page": 1,
-                "summary": summarized_results,
-                "results": self.search_result_from_cr["message"]["items"],
-            }
-        }
-        self.app.logger.info("REFINE: Dumping to file")
-        temp = []
-        refined_result = {}
-        for key in partial_result:
-            self.app.logger.info("KEY:")
-            self.app.logger.info(key)
-
-        for item in partial_result[self.uuid]["summary"]:
-            if item["title"]:
-                temp.append(item)
-
-        for item in temp:
-            refined_result[item["DOI"]]=  item
         with open("rr_results.json", "r+", encoding="utf-8") as data_file:
             # First we load existing data into a dict.
             try:
                 file_data = json.load(data_file)
             except JSONDecodeError:
                 file_data = {}
+            summarized_results = []
+            self.app.logger.info("REFINE: Removing all derived works from the list")
+            for item in self.search_result_from_cr['message']['items']:
+                container = "NO"
+                if "container-title" in item:
+                    container = "YES"
+                else:
+                    summarized_results.append({"DOI" : item['DOI'], "title": item['title'] if 'title' in item else None,"type": item['type'] if 'type' in item else None, "container": container})
+            result = {self.uuid : {"total_results": self.search_result_from_cr['message']['total-results'], "on_page" : 1, "summary":summarized_results, "results":self.search_result_from_cr['message']['items']}}
+            if file_data:
+                file_data.update(result)
+            else:
+                file_data = result
+            
+            self.app.logger.info("REFINE: Dumping to file")
             data_file.truncate(0)
             data_file.seek(0)
-            tmp = {self.uuid: refined_result}
-            if file_data:
-                file_data.update(tmp)
-            else:
-                file_data = tmp
             json.dump(file_data, data_file, indent=4)
-        return refined_result
+            return result
 
-    async def check_if_exists(self, refined_result):
-        """
-        Checks if the given DOI exists.
-
-        Args:
-        - doi: The DOI to be checked.
-        - brute_force: A flag indicating if brute force method should be used.
-
-        Returns:
-        - result: The search result.
-
-        Raises:
-        - None.
-        """
-        result = {}
+    async def check_dois_in_local_db(self, doi):
         self.app.logger.info("REFINE: Running search in the backend app")
-        dois = []
-        for item, value in refined_result.items():
-            dois.append([item, value])
-        coro = asyncio.to_thread(
-            search_bot.search_for_doi, dois, self.live_results, self.app.logger
-        )
+        coro = asyncio.to_thread(search_bot.search_for_doi, doi, self.live_results, self.app.logger)
         result = await coro
-        result_list = []
-        result_no_db = []
-        for item in result:
+        return result
+
+    async def check_if_exists_brute_force(self, page_url):
+        self.app.logger.error("REFINE: Brute force search!")
+        if not page_url.startswith(("http:", "https:")):
+            raise ValueError("URL must start with 'http:' or 'https:'")
+        # trunk-ignore(bandit/B310)
+        with urlopen(page_url) as response:
+            data = response.read()
+            text = data.decode("utf-8")
+            for line in text.splitlines():
+                if re.match(r".*Unfortunately, Sci-Hub doesn't have the requested document.*", line):
+                    return False
+                if m := re.match(r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)",line):
+                    direct_download_link = "https:" + str(m.group(1))
+                    self.app.logger.info (direct_download_link)
+                    return True
+        return False
+
+    async def check_if_exists(self, doi, brute_force = False):
+        result = {}
+        result = await self.check_dois_in_local_db(doi)
+        if brute_force and not result:
+            for item in doi:
+                item_link = "https://sci-hub.se/" + item[0]
+                tmp = {"DOI" : item["DOI"], "exists" : await self.check_if_exists_brute_force(item_link), "data": item}
+                result.update(tmp)
+        return result
+
+    async def refine_query(self, unrefined_reqult, brute_force = False):
+        temp = []
+        refined_result = []
+        for key in unrefined_reqult:
+            self.logger.info("KEY:")
+            self.logger.info(key)
+
+        for item in unrefined_reqult[self.uuid]["summary"]:
+            if item["container"] == "NO" and item["title"]:                
+                refined_result.append(item)
+
+        for item in refined_result:
+            temp.append([item["DOI"], item])
+
+        refined_result = await self.check_if_exists(temp)
+        temp = []
+        for item in refined_result:
             if item["exists"]:
-                result_list.append(item)
+                temp.append(item["data"])
+        refined_result = temp
+        self.hit = len(refined_result)
+        return refined_result
+    #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku
+    async def answer_query(self):
+        #TODO: DOłożyć sprawdzenie czy w rafinowanym pliku już nie mamy częściowego wyniku tutaj bo zanim w ogóle otworzymy nierafinowany
+        #Search sql for UUID
+        #SQL SELECT * FROM RESULTS WHERE UUID = uuid
+        self.logger.info(f"Search started {self.uuid}")
+        uuid_found = False
+        database = None
+        query = self.query
+        with open("cr_results.json", "r+", encoding="utf-8") as cr_file:
+            #tu logika komunikacji z SQL i cała logika związana z wyszukaniem
+            #ale tymczasowo plik
+            try:
+                database =  json.load(cr_file)
+            except JSONDecodeError:
+                pass
+
+            if database:
+                for item in database.keys():
+                    if item in self.uuid:
+                        self.result = await self.refine_query(database)
+                        return self.result
+                answer = await self.search_crossref(query=query)
+                database.update(answer)
+                cr_file.truncate(0)
+                cr_file.seek(0)
+                json.dump(database,cr_file)
+                self.result = await self.refine_query(answer)
+                return self.result
             else:
-                result_no_db.append(item)
-        self.hit = len(result)
-        return result_list, result_no_db
-
-
-    async def answer_query(self, deep_search=False):
-        """
-        Answers the search query.
-
-        Args:
-        - None.
-
-        Returns:
-        - result: The search result.
-
-        Raises:
-        - None.
-        """
-        self.app.logger.info(f"Search started {self.uuid}")
-        cr_result = await self.search_crossref(query=self.query, deep_search=deep_search)
-        refined_result = await self.refine_search(cr_result)
-        answer, negative_answer = await self.check_if_exists(refined_result)
-
-        self.app.logger.info("Returning result")
-        self.app.logger.info(answer)
-        self.app.logger.info(negative_answer)
-
-        for item in answer:
-            self.final_result[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
-        for item in negative_answer:
-            self.not_in_db[item["DOI"]] = {"Title": item["data"]["title"], "type": item["data"]["type"]}
-        self.app.logger.info("Returning result case2")
-        self.app.logger.info(self.final_result)
-        return self.final_result
-
-# ============================= FLASK INTERNALS===============================
-
+                answer = await self.search_crossref(query=query)
+                database = answer
+                cr_file.truncate(0)
+                cr_file.seek(0)
+                json.dump(database,cr_file)
+                self.result = await self.refine_query(answer)
+                self.app.logger.info("Refined")
+                self.done = True
+                return self.result
 
+#============================= FLASK INTERNALS===============================
 def flask_debug():
     """
-    Starts a Flask application in debug mode without using the reloader.
-
-    Args:
-    - None.
-
-    Returns:
-    - None.
-
-    Raises:
-    - None.
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
     """
     # trunk-ignore(bandit/B201)
     app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
-
 def waitress_run():
     """
-    Serves the Flask application using the Waitress WSGI server.
-
-    Args:
-    - None.
-
-    Returns:
-    - None.
-
-    Raises:
-    - None.
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
     """
     serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
 
-
 class BackgroundTaskSearch(threading.Thread):
-    """
-    A background task for searching and saving results to files.
-
-    This class extends the `threading.Thread` class and is responsible for running
-    the search task in the background. It retrieves queries from a queue, performs
-    the search, and saves the results to files.
-
-    Attributes:
-        app (App): The application instance.
-    """
-
     def run(self):
-        """
-        Run the background task.
-
-        This method is called when the thread is started. It creates a new event loop,
-        runs the `_run` method, and closes the event loop.
-        """
-        loop = asyncio.new_event_loop()
+        loop = asyncio.new_event_loop()  # loop = asyncio.get_event_loop()
         loop.run_until_complete(self._run())
         loop.close()
 
     async def _run(self):
-        """
-        Perform the search task.
-
-        This method is an asynchronous coroutine that runs in a loop. It retrieves a
-        librarian from the queue, answers the query, and saves the results to files.
-        It also sends the results to a remote server.
-
-        The search task continues running indefinitely until the thread is stopped.
-        """
         while True:
             database = None
-            ndb_database = None
             librarian = librarian_queue.get()
             self.app.logger.info("STARTED")
-            result = await librarian.answer_query(librarian.deep_search)
+            result = await librarian.answer_query()
             result = {librarian.uuid: result}
             self.app.logger.info("Saving to file")
-
-            # Save results to "not_in_db.json" file
-            with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
-                try:
-                    ndb_database = json.load(ndb_file)
-                except JSONDecodeError:
-                    pass
-                if ndb_database:
-                    ndb_database.update(librarian.not_in_db)
-                else:
-                    ndb_database = librarian.not_in_db
-                ndb_file.truncate(0)
-                ndb_file.seek(0)
-                json.dump(ndb_database, ndb_file)
-
-            # Save results to "s_results.json" file
             with open("s_results.json", "r+", encoding="utf-8") as s_file:
                 try:
-                    database = json.load(s_file)
+                    database =  json.load(s_file)
                 except JSONDecodeError:
                     pass
                 if database:
-                    self.app.logger.info(database)
-                    self.app.logger.info(result)
-                    database.update(result)
+                    database.extend(result)
                 else:
                     database = result
                 self.app.logger.info("DUMPING DATA")
@@ -403,41 +236,28 @@ class BackgroundTaskSearch(threading.Thread):
 
             self.app.logger.info(result)
             coroutine = asyncio.to_thread(
-                requests.post,
-                f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
-                json=result,
-                timeout=360,
-            )
+            requests.post,
+            f"{MAIN_BOT_ADDRESS}{SEND_RESULTS}",
+            json=result,
+            timeout=360,
+    )       
             self.app.logger.info("SENT")
-            result = await coroutine
-            self.app.logger.info(result.status_code)
+            _ = await coroutine
             self.app.logger.info("SEND CONFIRMED")
             time.sleep(1)
-
-
-# ==================================SERVER ROUTES==========================================
+            
+#==================================SERVER ROUTES==========================================
 @app.route("/query", methods=["POST"])
 async def query_database():
-    """
-    Endpoint for querying the database.
-
-    This function receives a POST request containing a JSON payload with a query and a UUID.
-    It creates a Librarian object with the query and UUID,
-    and adds it to the librarian_queue and librarian_list.
-    Finally, it returns a JSON response indicating the success
-    of the operation, along with the query, UUID,
-    and the current size of the librarian_queue.
-
-    Returns:
-        tuple: A tuple containing a JSON response and a status code.
-    """
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["query"])
     app.logger.info(record["UUID"])
     uuid = record["UUID"]
-    deep_search = record["deep_search"]
-    cl = Librarian(app, record["query"], uuid, deep_search)
+    #TODO: TESTING ONLY======================
+    uuid = UUID('{12345678-1234-5678-1234-567812345678}')
+
+    cl = Librarian(app, record["query"], uuid)
     librarian_queue.put(cl)
     librarian_list.append(cl)
     answer_data = (record["query"], record["UUID"], librarian_queue.qsize())
@@ -447,15 +267,8 @@ async def query_database():
     )
     return return_data
 
-
 @app.route("/get_partial_result", methods=["POST"])
 async def get_partial():
-    """
-    Retrieves the partial result for a given UUID.
-
-    Returns:
-        A JSON response containing the partial result.
-    """
     record = json.loads(request.data)
     app.logger.info(record)
     app.logger.info(record["UUID"])
@@ -469,34 +282,24 @@ async def get_partial():
     )
     return return_data
 
-
-# =======================================MAIN===================================================
+#=======================================MAIN===================================================
 if __name__ == "__main__":
-    app.logger.setLevel(logging.DEBUG)
-    h1 = handlers.RotatingFileHandler(
-        filename="D:\\logs\\librarian.log",
-        encoding="utf-8",
-        mode="a",
-        maxBytes=6 * 1024 * 1024,
-        backupCount=6,
-    )
-
-    app.logger.addHandler(h1)
+    logger = logging.getLogger('conjurer_musician')
+    logger.setLevel(logging.DEBUG)
     threads = []
-    threads.append(threading.Thread(target=waitress_run))
-    # threads.append(threading.Thread(target=flask_debug))
+    #threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=flask_debug))
     bgtask = BackgroundTaskSearch()
     bgtask.app = app
     threads.append(bgtask)
-    threads.append(threading.Thread(target=scrape_bot.scraper, args=(app.logger,)))
     i = 0
     for worker in threads:
         try:
-            app.logger.info("App number: %s", i)
-            i += 1
+            logger.info(f"App number: {i}")
+            i+=1
             worker.start()
         except RuntimeError as e:
-            app.logger.error("Exploded")
+            logger.error("Exploded")
             print(str(e))
     for worker in threads:
         worker.join()

conjurer_librarian/install_librarian_service.sh

@@ -1,33 +1,11 @@
 #!/bin/bash
-# This script sets up a virtual environment for the Conjurer Librarian service and installs the required dependencies.
-
-
-# Create a virtual environment at /home/pi/Conjurer_librarian/env
 python3 -m venv /home/pi/Conjurer_librarian/env
-
-# Copy the Conjurer Librarian Python script to the appropriate directory
-cp /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
-
-# Copy the requirements file to the appropriate directory
-cp /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
-
-# Change directory to /home/pi/Conjurer_librarian
+cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/
+cp  /home/pi/conjurer/conjurer_librarian/requirements_librarian.txt ./Conjurer_librarian/
 cd /home/pi/Conjurer_librarian
-
-touch /home/pi/Conjurer/rr_results.json
-touch /home/pi/Conjurer/cr_results.json
-touch /home/pi/Conjurer/s_results.json
-touch /home/pi/Conjurer/rr_results.json
-touch /home/pi/Conjurer/not_in_db.json
-
-# Activate the virtual environment
 source /home/pi/Conjurer_librarian/env/bin/activate
-
-# Upgrade pip
 ./env/bin/python3 -m pip install --upgrade pip
-
-# Install the required dependencies
 ./env/bin/python3 -m pip install -r requirements_librarian.txt
-
-# Deactivate the virtual environment
 deactivate
+
+cp  /home/pi/conjurer/conjurer_librarian/conjurer_librarian.py ./Conjurer_librarian/

conjurer_librarian/run_server.bat

@@ -1 +0,0 @@
-c:\conjurer\env\Scripts\activate.bat && c:\conjurer\conjurer_librarian\conjurer_librarian.py

conjurer_librarian/scrape_bot.py

@@ -1,134 +0,0 @@
-"""
-This module contains the code for the scrape bot.
-"""
-
-import json
-import logging
-import random
-import re
-import time
-from json import JSONDecodeError
-from queue import Queue
-from threading import Thread
-from urllib.request import urlopen
-
-from requests import ConnectionError as RequestsConnectionError
-from requests import ConnectTimeout, Timeout
-
-SCR_DATABASE_PATH = r"C:\\Database\\chunks\\"
-SCR_FILENAME = "40_chunk.txt"
-SCR_ENCODING = "utf-8"
-
-WORK_Q = Queue()
-random.seed()
-
-
-def load_ndb_to_q(logger):
-    """
-    This function loads items from the not_in_db.json file into the work queue.
-    """
-    logger.info("Loader started")
-    while True:
-        with open("not_in_db.json", "r+", encoding="utf-8") as ndb_file:
-            try:
-                ndb_database = json.load(ndb_file)
-                for _ in range (1,10):
-                    key = next(iter(ndb_database))
-                    item = ndb_database.pop(key)
-                    logger.info(key)
-                    url = f"https://sci-hub.se/{key}"
-                    WORK_Q.put([item, url, False])
-
-                ndb_file.truncate(0)
-                ndb_file.seek(0)
-                json.dump(ndb_database, ndb_file, indent=4)
-            except JSONDecodeError:
-                time.sleep(60 * 60 * 3)
-        time.sleep(60*60*3)
-
-
-def check_if_exists_brute_force(logger):
-    """
-    This function checks if a document exists using brute force search.
-    """
-    # Function code here
-    while True:
-        logger.info("Scraper tick")
-        item = WORK_Q.get()
-        page_url = item[1]
-        logger.error("REFINE: Brute force search!")
-        if not page_url.startswith(("http:", "https:")):
-            raise ValueError("URL must start with 'http:' or 'https:'")
-        blocked = True
-        try:
-            # trunk-ignore(bandit/B310)
-            with urlopen(page_url) as response:
-                data = response.read()
-                text = data.decode("utf-8")
-                if item[2]:
-                    logger.info("Already found")
-                    time.sleep(180)
-                    blocked = False
-                else:
-                    for line in text.splitlines():
-                        if re.match(
-                            r".*Unfortunately, Sci-Hub doesn't have the requested document.*",
-                            line,
-                        ):
-                            blocked = False
-                            logger.info("Not found")
-                            item[2] = False
-                        if m := re.match(
-                            r".*<embed type=\"application.pdf\"\s*src=\"(.*\.pdf)", line
-                        ):
-                            blocked = False
-                            direct_download_link = "https:" + str(m.group(1))
-                            logger.info(direct_download_link)
-                            with open(
-                                SCR_DATABASE_PATH + SCR_FILENAME,
-                                "a",
-                                encoding=SCR_ENCODING,
-                            ) as operated_file:
-                                operated_file.write("\n")
-                                operated_file.write(item[0])
-                                item[2] = True
-        except (
-            Timeout,
-            ConnectTimeout,
-            ConnectionRefusedError,
-            ConnectionError,
-            RequestsConnectionError,
-        ):
-            pass
-        if blocked:
-            logger.info(item)
-            logger.info(text)
-            logger.error("Got blocked. Fuck.")
-            time.sleep(60 * 60 * 72)
-        # trunk-ignore(bandit/B311)
-        rand = random.randint(1, 60)
-        logger.info(f"Sleeping for {2*rand} minutes")
-        time.sleep(120 * rand)
-
-
-def scraper(logger=None):
-    """
-    This function is responsible for scraping data.
-    """
-    if not logger:
-        logger = logging.getLogger()
-        logger.setLevel("DEBUG")
-        h1 = logging.StreamHandler()
-        logger.addHandler(h1)
-        logger.info("No logger set. Test run")
-
-    loader = Thread(target=load_ndb_to_q, args=(logger,))
-    _scraper = Thread(target=check_if_exists_brute_force, args=(logger,))
-    loader.start()
-    _scraper.start()
-    loader.join()
-    _scraper.join()
-
-
-if __name__ == "__main__":
-    scraper()

conjurer_librarian/search_bot.py

@@ -1,135 +1,82 @@
-"""
-This module contains functions for searching for DOI (Digital Object Identifier)
-in a list of live results.
-It includes a producer-consumer pattern implementation using threads and queues.
-
-Functions:
-- producer: Reads lines from a file and puts them into an output queue.
-- consumer: Consumes items from an input queue and checks if DOI exists in the live results list.
-- search_for_doi: Searches for DOI in live results using the producer-consumer pattern.
-
-Global Variables:
-- MAXTHREADS: Maximum number of worker threads.
-- DATABASE_PATH: Path to the database files.
-- ENCODING: Encoding of the database files.
-- CHUNK: File name pattern for the database files.
-- _sentinel: Sentinel object used to signal termination.
-- result_list: List to store the search results.
-- WORK_Q_SIZE: Maximum size of the work queue.
-"""
-
-# TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
-from queue import Empty, Queue
+from queue import Queue, Empty
 from threading import Thread
-
 q = Queue()
 
-# DATA FOR TEST ONLY
-# MAXTHREADS = 5
-# DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+#TODO: TO w klasie jako parametry wybierane czy robimy pelen run czy testowy
+#MAXTHREADS = 3
+MAXTHREADS = 1
 
-# DEPLOYMENT DATA
-MAXTHREADS = 41
-DATABASE_PATH = r"C:\\Database\\chunks\\"
+#DATABASE_PATH = r'C:\\Database\\chunks_1\\'
+DATABASE_PATH = r'C:\\Database\\chunks\\'
 
 ENCODING = "utf-8"
-CHUNK = "_chunk.txt"
+CHUNk = "_chunk.txt"
 _sentinel = object()
-WORK_Q_SIZE = 15500000
+result_list = []
 
+#TODO: Wpiemdolić to wszystko w klasę z loggerem przysłanym z góry
 
-def producer(out_q, control_q, filename, _logger):
-    """
-    Produces items from the output queue and puts them into the control queue.
-
-    Args:
-        out_q (Queue): Output queue.
-        control_q (Queue): Control queue.
-        filename (str): Name of the file.
-        _logger: Logger object for logging.
-    """
+def producer(out_q, control_q, filename):
     with open(DATABASE_PATH + filename, "r+", encoding=ENCODING) as operated_file:
-        _logger.info(f"Worker {filename} ")
-        while True:
-            line = operated_file.readline()
-            if not line:
-                break
+        lines = operated_file.readlines()
+        for line in lines:
             out_q.put(line)
             try:
                 check = control_q.get(block=False)
             except Empty:
                 check = False
+                pass
             if check is _sentinel:
-                _logger.info("TERM signal received")
+                print("TERM signal received")
                 control_q.put(check)
                 break
-        _logger.info(f"Worker finished: {filename}")
-        out_q.put(_sentinel)
-
-
-# IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
-def consumer(in_q, control_q, doi, live_results, result_list, _logger, ):
-    """
-    Consumes items from an input queue and checks if DOI exists in the live results list.
-
-    Args:
-        in_q (Queue): Input queue.
-        control_q (Queue): Control queue.
-        doi (list): List of DOI to search for.
-        live_results (list): List to store the search results.
-        _logger: Logger object for logging.
-    """
+    print(f"Worker finished: {filename}")
+    out_q.put(_sentinel)
+ 
+#IMPORTANT!!! ONLY ONE CONSUMER THREAD AS WE ARE NOT PUTTING SENTINELS BACK
+#TODO: Change doi to tuple list and only put term when all doi are found => (doi, True)
+def consumer(in_q, control_q, doi, live_results, logger):
     sentinels = 0
     for item in doi:
-        result_list.append({"DOI": item[0], "exists": False, "data": item[1]})
+        result_list.append({"DOI" : item[0], "exists" : False, "data" : item[1]})
     while True:
+        #if in_q.empty():
+        #    print("Queue empty!")
+        #if in_q.full():
+        #    print("Queue full!")
         done_check = True
         data = in_q.get()
         if data is _sentinel:
-            _logger.info("Worker finished")
+            logger.info("Worker finished")
             sentinels += 1
         else:
             for item in result_list:
-                if item["DOI"] in data and not item["exists"]:
-                    _logger.info(data)
-                    _logger.info("HIT")
+                if item["DOI"] in data:
+                    logger.info(data)
+                    logger.info("HIT")
                     item["exists"] = True
                     live_results.append(item)
                 done_check = done_check and item["exists"]
             if done_check:
                 control_q.put(_sentinel)
         if sentinels >= MAXTHREADS:
-            _logger.info("All workers finished")
+            logger.info("All workers finished")
             break
 
 
-def search_for_doi(doi, live_results, _logger):
-    """
-    Search for DOI in live_results using _logger for logging.
-
-    Args:
-        doi (list): List of DOI to search for.
-        live_results (list): List to store the search results.
-        _logger: Logger object for logging.
-    """
-    result_list = []
+def search_for_doi(doi, live_results, logger):
     threads = []
-    work_q = Queue(maxsize=WORK_Q_SIZE)
+    work_q = Queue(maxsize=200000)
     control_q = Queue()
-    t_cons = Thread(
-        target=consumer, args=(work_q, control_q, doi, live_results, result_list, _logger)
-    )
-    _logger.info("Consumer thread created")
+    t_cons = Thread(target = consumer, args = (work_q, control_q, doi, live_results, logger))
+    logger.info("Consumer thread created")
     threads.append(t_cons)
-    for i in range(0, MAXTHREADS):
-        # TEST DATA
-        # filename = "test" + str(i) + CHUNk
-        # DEPLOYMENT DATA
-        filename = str(i) + CHUNK
-        _logger.info(f"Creating worker thread no: {i}")
-        threads.append(
-            Thread(target=producer, args=(work_q, control_q, filename, _logger))
-        )
+    for i in range(0,MAXTHREADS):
+        #filename = "test" + str(i) + CHUNk
+        filename = "40" + CHUNk
+        #filename = str(i) + CHUNk
+        logger.info(f"Creating worker thread no: {i}")
+        threads.append(Thread(target = producer, args = (work_q, control_q, filename)))
     for worker in threads:
         worker.start()
     for worker in threads:
@@ -137,29 +84,23 @@ def search_for_doi(doi, live_results, _logger):
     return result_list
 
 
+
 if __name__ == "__main__":
     import logging
-
     logger = logging.getLogger()
     logger.setLevel("DEBUG")
     h1 = logging.StreamHandler()
     logger.addHandler(h1)
     logger.info("TEST RUN")
     live_result = []
-    logger.info(
-        search_for_doi(
-            [
-                ("10.1002/9781118786352.wbieg0998.pub2", "DATA"),
-                ("10.1002/j.2050-0416.2002.tb00563.x", "DATA"),
-                ("10.1111/j.1365-2958.1994.tb00448.x", "DATA"),
-                ("10.2165/00128415-200309690-00017", "DATA"),
-                ("10.5772/48313", "DATA"),
-                ("10.15803/ijnc.7.2_419", "DATA"),
-                ("10.1051/0004-6361/201321596e", "DATA"),
-                ("10.2307/40835941", "DATA"),
-            ],
-            live_result,
-            logger,
-        )
-    )
+    logger.info(search_for_doi([
+        ("10.1002/9781118786352.wbieg0998.pub2","DATA"),
+        ("10.1002/j.2050-0416.2002.tb00563.x","DATA"),
+        ("10.1111/j.1365-2958.1994.tb00448.x","DATA"),
+        ("10.2165/00128415-200309690-00017","DATA"),
+        ("10.5772/48313","DATA"),
+        ("10.15803/ijnc.7.2_419","DATA"),
+        ("10.1051/0004-6361/201321596e","DATA"),
+        ("10.2307/40835941","DATA"),
+        ], live_result, logger))
     logger.info(live_result)

conjurer_musician/conjurer_musician.py

@@ -1,470 +0,0 @@
-# This Python file uses the following encoding: utf-8
-"""
-The provided Python script sets up a Flask web server to manage a list of music files, with
-functions for rescanning the music folder, updating the music list, and serving the music list via
-API endpoints.
-"""
-import json
-import logging
-import re
-import threading
-import time
-from datetime import datetime
-from logging import handlers
-from pathlib import Path
-from platform import uname
-from sys import platform
-
-from flask import Flask, jsonify, redirect, request, send_from_directory
-from waitress import serve
-
-HOST_ADDRESS = "192.168.1.15"
-PORT_ADDRESS = 5000
-if platform in ("linux", "linux2"):
-    SEPARATOR_FILE_PATH = "/"
-    if "microsoft-standard" in uname().release:
-        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
-        MUSIC_FOLDER = "/mnt/g/Muzyka/"
-        NETRC_FILE = "/home/mtuszowski/.netrc"
-        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
-        ENCODING = "utf-8"
-
-    else:
-        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
-        NETRC_FILE = "/home/pi/.netrc"
-        LOGSTORE = "/home/pi/RetroPie/logs/"
-        ENCODING = "utf-8"
-        MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
-        PRIORITY_FOLDER = "/home/pi/RetroPie/mp3/Magiczne i chuj/"
-
-
-music_file_list = []
-priority_list = []
-
-
-def create_playlist():
-    """
-    Reads a JSON file containing a playlist and writes the playlist items to a text file.
-
-    The JSON file path is '/home/pi/Conjurer/playlist.json',
-    and the text file path is '/home/pi/Conjurer/all_playlist.playlist'.
-
-    Raises:
-        JSONDecodeError: If the JSON file cannot be decoded.
-
-    """
-    with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as r_file:
-        with open(
-            "/home/pi/Conjurer/all_playlist.playlist", "w", encoding="utf-8"
-        ) as w_file:
-            try:
-                playlist = json.load(r_file)
-                for item in playlist:
-                    w_file.write(item)
-                    w_file.write("\n")
-            except json.JSONDecodeError:
-                pass
-
-    with open("/home/pi/Conjurer/prio_playlist.json", "r+", encoding="utf-8") as r_file:
-        with open("/home/pi/Conjurer/hit.playlist", "w", encoding="utf-8") as w_file:
-            try:
-                playlist = json.load(r_file)
-                for item in playlist:
-                    w_file.write(item)
-                    w_file.write("\n")
-            except json.JSONDecodeError:
-                pass
-
-
-def rescan():
-    """
-    The `rescan` function logs a message, scans for mp3 files in a specified folder,
-    and adds them to a list of music files.
-    """
-    logging.info("Rescan triggered")
-
-    for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
-        temp_music_file = mp3_item.as_posix()
-        if platform == "win32":
-            temp_music_file = temp_music_file.replace("/", "\\")
-        music_file_list.append(temp_music_file)
-
-    for mp3_item in Path.glob(Path(PRIORITY_FOLDER), "**/*.mp3"):
-        temp_music_file = mp3_item.as_posix()
-        if platform == "win32":
-            temp_music_file = temp_music_file.replace("/", "\\")
-        priority_list.append(temp_music_file)
-
-    with open("/home/pi/Conjurer/playlist.json", "r+", encoding="utf-8") as s_file:
-        database = music_file_list
-        # self.app.logger.info("DUMPING DATA")
-        s_file.truncate(0)
-        s_file.seek(0)
-        json.dump(database, s_file)
-    with open("/home/pi/Conjurer/prio_playlist.json", "r+", encoding="utf-8") as s_file:
-        # self.app.logger.info("DUMPING DATA")
-        s_file.truncate(0)
-        s_file.seek(0)
-        json.dump(priority_list, s_file)
-    create_playlist()
-
-
-def thread_rescan():
-    """
-    The `thread_rescan` function periodically triggers a rescan operation after a specified time
-    interval.
-    """
-    logging.info("Starting filesystemupdater")
-    while True:
-        time.sleep(60 * 60 * 24)
-        logging.info("Rescan triggered")
-        rescan()
-
-
-app = Flask(__name__)
-
-
-# TODO: Odpalić wyszukiwarki w wątkach i dopiero po wszystkim zsumować wyszukiwanie.
-def wyszukaj(word_list, how_many, _logger=None, return_to_bot=True):
-    """
-    Searches for music files based on a list of keywords and returns a list of matching files.
-
-    Args:
-        word_list (list): A list of keywords to search for.
-        how_many (int): The maximum number of matching files to return.
-
-    Returns:
-        list: A list of matching files, sorted by their search weight.
-
-    """
-    if _logger is None:
-        fun_logger = logging.getLogger()
-    else:
-        fun_logger = _logger
-    return_list = []
-    search_weight = []
-
-    for _ in range(len(music_file_list)):
-        search_weight.append((0, ""))
-
-    time_start = datetime.now()
-    if int(how_many) > 0:
-        skip_start = 2
-    else:
-        skip_start = 1
-    for word in word_list[skip_start:]:
-        token_weight = len(word)
-        fun_logger.info("Słowo kluczowe: %s", word)
-        itr = 0
-        for file in music_file_list:
-            file = file.split(SEPARATOR_FILE_PATH)
-
-            char_remove = [
-                ".",
-                "^",
-                "$",
-                "*",
-                "+",
-                "?",
-                "{",
-                "}",
-                "[",
-                "]",
-                "\\",
-                "/",
-                "|",
-                "(",
-                ")",
-                "!",
-                ",",
-                "-",
-                ":",
-                "mp3",
-            ]
-            all_words = []
-            for f_iter in file:
-                for char in char_remove:
-                    f_iter = remove_characters(f_iter, char)
-                tmp = f_iter.split()
-                all_words.extend(tmp)
-            pingu = 1
-            pattern_len = len(all_words)
-            if platform == "win32":
-                skip = 2
-            else:
-                skip = 4
-            matched_times = 1
-            # TODO: To do wątków albo for asynchroniczny
-            for itm in all_words[skip:]:
-                pingu += 1
-                pattern = ".*" + word + ".*"
-                if re.match(pattern, itm, re.IGNORECASE):
-                    temp_weight = (
-                        search_weight[itr][0]
-                        + (token_weight + (pingu**1.5) / pattern_len) / matched_times
-                    )
-                    search_weight[itr] = (
-                        temp_weight,
-                        music_file_list[itr],
-                    )
-                    matched_times += 1
-            itr += 1
-
-    fun_logger.info(
-        "Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start
-    )
-    time_start = datetime.now()
-    item_to_search = max_weight(search_weight)
-    itr = 0
-    if item_to_search == 0:
-        return []
-    not_found = True
-    return_list = []
-    if int(how_many) <= 0:
-        fun_logger.info("Jeden plik do zagrania")
-        while not_found:
-            if search_weight[itr][0] == item_to_search:
-                return_list.append(search_weight[itr])
-                if not return_to_bot:
-                    with open(
-                        "/home/pi/Conjurer/priority_queue.playlist",
-                        "r+",
-                        encoding="utf-8",
-                    ) as s_file:
-                        s_file.write(search_weight[itr][1])
-                break
-            itr += 1
-    else:
-        fun_logger.info("Wiele plików do zagrania")
-        search_weight.sort(key=lambda x: x[0], reverse=True)
-        return_list.extend(search_weight[: int(how_many)])
-        if not return_to_bot:
-
-            with open(
-                "/home/pi/Conjurer/priority_queue.playlist", "a", encoding="utf-8"
-            ) as s_file:
-                for item in return_list:
-                    s_file.write(item[1] + "\n")
-    return return_list
-
-
-def max_weight(lista):
-    """
-    Take a list of weights and return the maximum weight.
-
-    :param lista: It seems like the parameter `lista` is a list of items, possibly representing weights.
-    The function name `max_weight` suggests that the function is intended to find the maximum weight
-    from the list. However, without more context or information about the problem, it's difficult to say
-    for sure what the
-    """
-    maximum_weight = 0
-    for iterator in lista:
-        if iterator[0] > maximum_weight:
-            maximum_weight = iterator[0]
-    return maximum_weight
-
-
-def remove_characters(string, character):
-    """
-    The `remove_characters` function removes all occurrences of a specified character from a given
-    string.
-
-    :param string: The string parameter is the input string from which characters will be removed
-    :param character: The character parameter is the character that you want to remove from the string
-    :return: a new string where all occurrences of the specified character have been removed.
-    """
-    return string.replace(character, "")
-
-
-@app.route("/stream", methods=["GET"])
-def stream_music():
-    """
-    The function `stream_music` is a route handler for the "/stream" endpoint.
-    It returns a JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
-
-    :return: A JSON response containing a key "music_file_list" with the value of the variable `music_file_list`.
-    """
-    return send_from_directory("/tmp/hls", "stream.m3u8")
-
-
-@app.route("/<string:file_name>")
-def stream(file_name):
-    """
-    Stream the specified file from the video directory.
-
-    Args:
-        file_name (str): The name of the file to be streamed.
-
-    Returns:
-        Response: The response containing the streamed file.
-    """
-    video_dir = "/tmp/hls"
-    return send_from_directory(video_dir, file_name)
-
-
-@app.route("/stream_mp3", methods=["GET"])
-def stream_music_mp3():
-    """
-    The function `stream_music_mp3` is a route handler for the "/stream_mp3" endpoint.
-    It redirects the user to the URL "http://www.example.com" with a status code of 302.
-
-    :return: A redirect response to "http://www.example.com" with a status code of 302.
-    """
-    return redirect("http://www.example.com", code=302)
-
-
-@app.route("/clear_pr_pls", methods=["GET"])
-def clear_pr_pls():
-    """
-    The function `clear_pr_pls` clears the contents of the priority queue playlist file.
-
-    :return: A JSON response indicating the success of the operation.
-    """
-    with open(
-        "/home/pi/Conjurer/priority_queue.playlist", "w", encoding="utf-8"
-    ) as s_file:
-        s_file.truncate(0)
-    return_data = jsonify(isError=False, message="Success", statusCode=200, data=[])
-    return return_data, 200
-
-
-@app.route("/mp3", methods=["GET"])
-def get_music_list():
-    """
-    The function `get_music_list` returns a JSON object containing a list of music files.
-    :return: A JSON response containing a key "music_file_list" with the value of the variable
-    `music_file_list`.
-    """
-    return jsonify({"music_file_list": music_file_list})
-
-
-@app.route("/update_mp3", methods=["POST"])
-def update_music_list():
-    """
-    The function `update_music_list` receives a POST request with a JSON payload, logs the received
-    item, adds it to a music file list, and returns a success message along with the updated record.
-    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
-    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
-    with values indicating the success of the operation and the data that was
-    received and added to the `music_file_list`.
-    The status code returned is 200, indicating a successful response.
-    """
-    record = json.loads(request.data)
-    app.logger.info(record["item"])
-    music_file_list.append(record["item"])
-    return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data=record),
-        200,
-    )
-    return return_data
-
-
-@app.route("/get_music", methods=["POST"])
-def look_for_playlist():
-    """
-    The function `look_for_playlist` receives a POST request with a JSON payload, logs the received
-    item, adds it to a music file list, and returns a success message along with the updated record.
-
-    :return: A tuple containing a JSON response and a status code. The JSON response includes keys `isError`,
-             `message`, `statusCode`, and `data`, with values indicating the success of the operation and the
-             data that was received and added to the `music_file_list`. The status code returned is 200,
-             indicating a successful response.
-    """
-    record = json.loads(request.data)
-    app.logger.info(record)
-    app.logger.info(record["lista_slow"])
-    app.logger.info(record["UUID"])
-    app.logger.info(record["dlugosc_plejlisty"])
-    return_data = wyszukaj(
-        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, True
-    )
-
-    return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data=return_data),
-        200,
-    )
-    return return_data
-
-
-@app.route("/add_to_priority", methods=["POST"])
-def add_to_priority():
-    """
-    The function `add_to_priority` receives a POST request with a JSON payload, logs the received
-    item, adds it to a music file list, and returns a success message along with the updated record.
-
-    :return: A tuple containing a JSON response and a status code. The JSON response includes keys `isError`,
-             `message`, `statusCode`, and `data`, with values indicating the success of the operation and the
-             data that was received and added to the `music_file_list`. The status code returned is 200,
-             indicating a successful response.
-    """
-    record = json.loads(request.data)
-    app.logger.info(record)
-    app.logger.info(record["lista_slow"])
-    app.logger.info(record["UUID"])
-    app.logger.info(record["dlugosc_plejlisty"])
-    return_data = wyszukaj(
-        record["lista_slow"], record["dlugosc_plejlisty"], app.logger, False
-    )
-
-    return_data = (
-        jsonify(isError=False, message="Success", statusCode=200, data=return_data),
-        200,
-    )
-    return return_data
-
-
-def flask_debug():
-    """
-    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
-    Do not use for production for fucks sake.
-    """
-    # trunk-ignore(bandit/B201)
-    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
-
-
-def waitress_run():
-    """
-    The `waitress_run` function serves the `app` on host HOST_ADDRESS
-    and port 5000 using the Waitress WSGI server.
-
-    This function starts the Waitress server and listens for incoming requests
-    on the specified host and port. It uses the `serve` function from the Waitress
-    library to handle the serving process.
-
-    Parameters:
-        None
-
-    Returns:
-        None
-    """
-    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
-
-
-if __name__ == "__main__":
-    logger = logging.getLogger("conjurer_musician")
-    logger.setLevel(logging.DEBUG)
-    formatter = logging.Formatter(
-        "%(asctime)s - %(name)s - %(levelname)s - %(message)s"
-    )
-    handler = handlers.RotatingFileHandler(
-        filename=LOGFILE,
-        encoding=ENCODING,
-        mode="a",
-        maxBytes=6 * 1024 * 1024,
-        backupCount=6,
-    )
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-
-    rescan()
-    logger.info("Started")
-    threads = []
-    # threads.append(threading.Thread(target=flask_debug))
-    threads.append(threading.Thread(target=waitress_run))
-    threads.append(threading.Thread(target=thread_rescan))
-
-    for worker in threads:
-        worker.start()
-
-    for worker in threads:
-        worker.join()

conjurer_musician/conjurer_musician.service

@@ -1,11 +0,0 @@
-[Unit]
-Description=Conjurer_musician
-After=multi-user.target
-
-[Service]
-Type=simple
-ExecStart=/home/pi/Conjurer/env/bin/python3 /home/pi/Conjurer/conjurer_musician.py
-Restart=on-abort
-
-[Install]
-WantedBy=multi-user.target

conjurer_musician/radio_conjurer.liq

@@ -1,71 +0,0 @@
-# FILEPATH: /home/mtuszowski/conjurer/conjurer_musician/radio_conjurer.liq
-
-# This script sets up a Liquidsoap radio stream with various features and configurations.
-
-# Load icecast credentials from a JSON file
-let json.parse credentials  = file.contents("/home/pi/Conjurer/icecast_credentials.json")
-
-# Enable replaygain metadata processing
-enable_replaygain_metadata()
-
-# Define playlists to be used in the stream
-s1 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/all_playlist.playlist"))
-s2 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/priority_queue.playlist"))
-s3 = replaygain(playlist(reload_mode="watch", "/home/pi/Conjurer/hit.playlist"))
-
-# Randomly select a playlist with weights
-s = random(id="randomizer", weights=[1, 3, 2], [s1, s2, s3])
-
-# Set up an interactive harbor for controlling the stream
-interactive.harbor(port = 9999)
-
-# Set up interactive controls for bass boost
-f = interactive.float("f", description="Frequency", min=0., max=1000., unit="Hz", 200.)
-g = interactive.float("g", description="Gain", min=0., max=20., unit="dB", 8.)
-b = bass_boost(frequency=f, gain=g, s)
-s = add([s, b])
-
-# Set up interactive control for main volume
-a = interactive.float("main_volume", min=0., max=20., 1.)
-s = compress.multiband.interactive(bands=3, s)
-
-# Apply audio processing effects
-s = nrj(normalize(s))
-s = crossfade(fade_out=3., fade_in=3., duration=5., s)
-s = amplify(a, s)
-
-# Skip blank sections in the stream
-s = blank.skip(max_blank=2., s)
-
-# Configure logging settings
-log_to_stdout = true
-log_to_file = true
-logpath = "/home/pi/Conjurer/radio_log.log"
-loglevel = 4
-set("log.stdout", log_to_stdout)
-set("log.level", loglevel)
-
-# Enable logging to file
-set("log.file", log_to_file)
-set("log.file.path", logpath)
-
-# Set up emergency fallback track
-emergency = single("/home/pi/RetroPie/mp3/Youtube/Dr. Peacock - Trip to Ireland [GvrvQTUbUcA].mp3")
-
-# Create a fallback stream with the main stream and emergency track
-radio = fallback(id="switcher", track_sensitive=false, [s, emergency])
-
-# Handle metadata events
-handle_metadata = fun (m) -> begin
-    log.info(m["title"])
-    log.info(m["artist"])
-    log.info(m["filename"])
-end
-radio.on_metadata(handle_metadata)
-
-# Enable persistent script parameters
-interactive.persistent("script.params")
-
-# Configure output formats and destinations
-output.file.hls("/tmp/hls", [("mp3-low", %mp3(bitrate=96)), ("mp3-hi", %mp3(bitrate=160))], radio)
-output.icecast(%mp3, host="retropie", port=8000, password=credentials.password, icy_metadata="true", mount="mp3-stream", radio)

conjurer_musician/radio_service.service

@@ -1,14 +0,0 @@
-[Unit]
-Description=radio_conjurer Liquidsoap daemon
-After=network.target
-Documentation=http://liquidsoap.fm/
-
-[Service]
-User=pi
-WorkingDirectory=/home/pi/Conjurer/
-ExecStart=/home/pi/.opam/4.13.0/bin/liquidsoap /home/pi/Conjurer/radio_conjurer.liq
-Restart=always
-
-[Install]
-WantedBy=multi-user.target
-Alias=radio_conjurer-liquidsoap.service

deploy_music.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-cd ./conjurer
-git pull
-cd ..
-cp ./conjurer/conjurer_musician/conjurer_musician.py ./Conjurer
-sudo systemctl restart conjurer_musician.service

deploy_radio.sh

@@ -1,6 +0,0 @@
-#!/bin/bash
-cd ./conjurer
-git pull
-cd ..
-cp ./conjurer/conjurer_musician/radio_conjurer.liq ./Conjurer
-sudo systemctl restart radio_service.service

file_webservice/conjurer_musician.py

@@ -0,0 +1,276 @@
+"""
+The provided Python script sets up a Flask web server to manage a list of music files, with
+functions for rescanning the music folder, updating the music list, and serving the music list via
+API endpoints.
+"""
+import json
+import re
+from pathlib import Path
+from platform import uname
+import threading
+import time
+from datetime import datetime
+from sys import platform
+import logging
+from flask import Flask, jsonify, request
+from waitress import serve
+
+
+HOST_ADDRESS = "192.168.1.15"
+PORT_ADDRESS = 5000
+if platform in ("linux", "linux2"):
+    SEPARATOR_FILE_PATH = "/"
+    if "microsoft-standard" in uname().release:
+        LOGFILE = "/home/mtuszowski/conjurer/discord_mus_service.log"
+        MUSIC_FOLDER = "/mnt/g/Muzyka/"
+        NETRC_FILE = "/home/mtuszowski/.netrc"
+        LOGSTORE = "/home/mtuszowski/conjurer/logs/"
+        ENCODING = "utf-8"
+        GRAPHICS_PATH = "/home/mtuszowski/conjurer/Conjurer_graphics/"
+
+    else:
+        LOGFILE = "/home/pi/Conjurer/discord_mus_service.log"
+        MEMORY_FIVE_SIARA = "/home/pi/Conjurer/pamiec.json"
+        SYSTEM_GPT_SETTINGS = "/home/pi/Conjurer/system_gpt_settings.json"
+        MEMORY_FIVE_MUZYKA = "/home/pi/Conjurer/pamiec_muzyki.json"
+        MUSIC_FOLDER = "/home/pi/RetroPie/mp3/"
+        SETTINGS_FILE = "/home/pi/Conjurer/settings.json"
+        NETRC_FILE = "/home/pi/.netrc"
+        LOGSTORE = "/home/pi/RetroPie/logs/"
+        ACCIDENT_LOG = "/home/pi/Conjurer/accident_log.json"
+        ENCODING = "utf-8"
+        GRAPHICS_PATH = "/home/pi/RetroPie/Conjurer_graphics/"
+        DIR_PATH_SADOX = "/home/pi/RetroPie/Fansadox/"
+
+
+music_file_list = []
+def rescan():
+    """
+    The `rescan` function logs a message, scans for mp3 files in a specified folder,
+    and adds them to a list of music files.
+    """
+    logging.info("Rescan triggered")
+    for mp3_item in Path.glob(Path(MUSIC_FOLDER), "**/*.mp3"):
+        temp_music_file = mp3_item.as_posix()
+        if platform == "win32":
+            temp_music_file = temp_music_file.replace("/", "\\")
+        music_file_list.append(temp_music_file)
+
+def thread_rescan():
+    """
+    The `thread_rescan` function periodically triggers a rescan operation after a specified time
+    interval.
+    """
+    logging.info("Starting filesystemupdater")
+    while True:
+        time.sleep(60*60*24)
+        logging.info("Rescan triggered")
+        rescan()
+
+app = Flask(__name__)
+
+def wyszukaj(word_list, how_many):
+    logger = logging.getLogger()
+    return_list = []
+    search_weight = []
+
+    for _ in range(len(music_file_list)):
+        search_weight.append((0, ""))
+
+    time_start = datetime.now()
+    if int(how_many) > 0:
+        skip_start = 2
+    else:
+        skip_start = 1
+    for word in word_list[skip_start:]:
+        token_weight = len(word)
+        logger.info("Słowo kluczowe: %s", word)
+        itr = 0
+        for file in music_file_list:
+            file = file.split(SEPARATOR_FILE_PATH)
+
+            char_remove = [
+                ".",
+                "^",
+                "$",
+                "*",
+                "+",
+                "?",
+                "{",
+                "}",
+                "[",
+                "]",
+                "\\",
+                "/",
+                "|",
+                "(",
+                ")",
+                "!",
+                ",",
+                "-",
+                ":",
+                "mp3",
+            ]
+            all_words = []
+            for f_iter in file:
+                for char in char_remove:
+                    f_iter = remove_characters(f_iter, char)
+                tmp = f_iter.split()
+                all_words.extend(tmp)
+            pingu = 1
+            pattern_len = len(all_words)
+            if platform == "win32":
+                skip = 2
+            else:
+                skip = 4
+            matched_times = 1
+            for itm in all_words[skip:]:
+                pingu += 1
+                pattern = ".*" + word + ".*"
+                if re.match(pattern, itm, re.IGNORECASE):
+                    temp_weight = (
+                        search_weight[itr][0]
+                        + (token_weight + (pingu**1.5) / pattern_len) / matched_times
+                    )
+                    search_weight[itr] = (
+                        temp_weight,
+                        music_file_list[itr],
+                    )
+                    matched_times += 1
+            itr += 1
+
+    logger.info("Stworzylem tablice wag zajęło mi to %s", datetime.now() - time_start)
+    time_start = datetime.now()
+    item_to_search = max_weight(search_weight)
+    itr = 0
+    if item_to_search == 0:
+        return None
+    not_found = True
+    return_list = []
+    if int(how_many) <= 0:
+        logger.info("Jeden plik do zagrania")
+        while not_found:
+            if search_weight[itr][0] == item_to_search:
+                return_list.append(search_weight[itr])
+                return return_list
+            itr += 1
+    else:
+        logger.info("Wiele plików do zagrania")
+        search_weight.sort(key=lambda x: x[0], reverse=True)
+        return_list.extend(search_weight[: int(how_many)])
+        return return_list
+
+def max_weight(lista):
+    """
+    Take a list of weights and return the maximum weight.
+
+    :param lista: It seems like the parameter `lista` is a list of items, possibly representing weights.
+    The function name `max_weight` suggests that the function is intended to find the maximum weight
+    from the list. However, without more context or information about the problem, it's difficult to say
+    for sure what the
+    """
+    maximum_weight = 0
+    for iterator in lista:
+        if iterator[0] > maximum_weight:
+            maximum_weight = iterator[0]
+    return maximum_weight
+
+
+def remove_characters(string, character):
+    """
+    The `remove_characters` function removes all occurrences of a specified character from a given
+    string.
+
+    :param string: The string parameter is the input string from which characters will be removed
+    :param character: The character parameter is the character that you want to remove from the string
+    :return: a new string where all occurrences of the specified character have been removed.
+    """
+    return string.replace(character, "")
+
+
+@app.route("/mp3", methods=["GET"])
+def get_music_list():
+    """
+    The function `get_music_list` returns a JSON object containing a list of music files.
+    :return: A JSON response containing a key "music_file_list" with the value of the variable
+    `music_file_list`.
+    """
+    return jsonify({"music_file_list": music_file_list})
+
+
+@app.route("/update_mp3", methods=["POST"])
+def update_music_list():
+    """
+    The function `update_music_list` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    with values indicating the success of the operation and the data that was
+    received and added to the `music_file_list`.
+    The status code returned is 200, indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record["item"])
+    music_file_list.append(record["item"])
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=record),
+        200,
+    )
+    return return_data
+
+@app.route("/get_music", methods=["POST"])
+def look_for_playlist():
+    """
+    The function `update_music_list` receives a POST request with a JSON payload, logs the received
+    item, adds it to a music file list, and returns a success message along with the updated record.
+    :return: The function `update_music_list` is returning a tuple containing a JSON response and a
+    status code. The JSON response includes keys `isError`, `message`, `statusCode`, and `data`,
+    with values indicating the success of the operation and the data that was
+    received and added to the `music_file_list`.
+    The status code returned is 200, indicating a successful response.
+    """
+    record = json.loads(request.data)
+    app.logger.info(record)
+    app.logger.info(record["lista_slow"])
+    app.logger.info(record["UUID"])
+    app.logger.info(record["dlugosc_plejlisty"])
+    return_data = wyszukaj(record["lista_slow"],record["dlugosc_plejlisty"])
+
+    return_data = (
+        jsonify(isError=False, message="Success", statusCode=200, data=return_data),
+        200,
+    )
+    return return_data
+
+
+def flask_debug():
+    """
+    The `flask_debug` function starts a Flask application in debug mode without using the reloader.
+    Do not use for production for fucks sake.
+    """
+    # trunk-ignore(bandit/B201)
+    app.run(debug=True, use_reloader=False, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+def waitress_run():
+    """
+    The `waitress_run` function serves the `app` on host "0.0.0.0"
+    and port 5000 using the Waitress WSGI server.
+    """
+    serve(app, host=HOST_ADDRESS, port=PORT_ADDRESS)
+
+if __name__ == "__main__":
+    logger = logging.getLogger('conjurer_musician')
+    logger.setLevel(logging.DEBUG)
+    rescan()
+    logger.info("Started")
+    threads = []
+    threads.append(threading.Thread(target=waitress_run))
+    threads.append(threading.Thread(target=thread_rescan))
+
+    for worker in threads:
+        worker.start()
+
+    for worker in threads:
+        worker.join()
+
+#TODO: Inne pliki - sadox, dokumenty naukowe itp.

file_webservice/conjurer_musician.service

@@ -0,0 +1,11 @@
+[Unit]
+Description=Conjurer_musician
+After=multi-user.target
+
+[Service]
+Type=simple
+ExecStart=/home/pi/Conjurer-service/conjurer/file_webservice/bin/python3 /home/pi/Conjurer-service/conjurer/file_webservice/conjurer_musician.py
+Restart=on-abort
+
+[Install]
+WantedBy=multi-user.target

file_webservice/install_musician_service.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+python3 -m venv /home/pi/Conjurer-service/conjurer/file_webservice
+source ./bin/activate
+./bin/python3 -m pip install --upgrade pip
+./bin/python3 -m pip install -r requirements_musician.txt
+deactivate
+
+sudo cp ./conjurer_musician.service /etc/systemd/system/
+sudo systemctl daemon-reload
+sudo systemctl start conjurer_musician.service
+sudo systemctl enable conjurer_musician.service

install_main_bot.sh

@@ -1,9 +1,5 @@
 #!/bin/bash
-cd /home/pi
-mdkir Conjurer
-cd Conjurer
 python3 -m venv /home/pi/Conjurer/env
-cp /home/pi/conjurer/requirements_bot.txt /home/pi/Conjurer/
 source ./env/bin/activate
 ./env/bin/python3 -m pip install --upgrade pip
 ./env/bin/python3 -m pip install -r requirements_bot.txt
@@ -12,5 +8,5 @@ sed -i '1i\import shutil' ./env/lib/python3.11/site-packages/spotify_dl/youtube.
 deactivate
 sudo cp ./conjurer.service /etc/systemd/system/
 sudo systemctl daemon-reload
-sudo systemctl start conjurer.service
-sudo systemctl enable conjurer.service
+sudo systemctl start conjurer_musician.service
+sudo systemctl enable conjurer_musician.service

install_music_bot.sh

@@ -1,31 +0,0 @@
-#!/bin/bash
-cd /home/pi
-mkdir Conjurer
-cd Conjurer
-python3 -m venv /home/pi/Conjurer/env
-source /home/pi/Conjurer/env/bin/activate
-cp /home/pi/conjurer/conjurer_musician/requirements_musician.txt /home/pi/Conjurer/
-cp /home/pi/conjurer/conjurer_musician/conjurer_musician.py /home/pi/Conjurer/
-cp /home/pi/conjurer/conjurer_musician/radio_conjurer.liq /home/pi/Conjurer/
-touch /home/pi/Conjurer/discord_mus_service.log
-
-touch /home/pi/Conjurer/playlist.json
-touch /home/pi/Conjurer/prio_playlist.json
-
-touch /home/pi/Conjurer/all_playlist.playlist
-touch /home/pi/Conjurer/priority_queue.playlist
-touch /home/pi/Conjurer/hit.playlist
-
-touch /home/pi/Conjurer/radio_log.log
-./env/bin/python3 -m pip install --upgrade pip
-./env/bin/python3 -m pip install -r requirements_musician.txt
-deactivate
-sudo cp /home/pi/conjurer/conjurer_musician/conjurer_musician.service /etc/systemd/system/
-sudo systemctl daemon-reload
-sudo systemctl start conjurer_musician.service
-sudo systemctl enable conjurer_musician.service
-
-sudo cp /home/pi/conjurer/conjurer_musician/radio_service.service /etc/systemd/system/
-sudo systemctl daemon-reload
-sudo systemctl start radio_service.service
-sudo systemctl enable radio_service.service

s_results.json

@@ -0,0 +1 @@
+{"12345678-1234-5678-1234-567812345678": [{"DOI": "10.1057/9781137435026", "title": ["Queer BDSM Intimacies"], "type": "book", "container": "NO"}]}